From b8f8ebe2e6489c1994140d6cfff8ac31c8333264 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 10 Sep 2025 20:27:41 +0200 Subject: [PATCH 1/4] nfc(C++ modules) run tests with two phases compilation disabled for clang --- tests/projects/c++/modules/test_base.lua | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/projects/c++/modules/test_base.lua b/tests/projects/c++/modules/test_base.lua index 3571ad1085..25181fe702 100644 --- a/tests/projects/c++/modules/test_base.lua +++ b/tests/projects/c++/modules/test_base.lua @@ -81,8 +81,10 @@ function build_tests(toolchain_name, opt) return end + local two_phases = (opt.two_phases == nil or opt.two_phases == true) local policies = "--policies=build.c++.modules.std:" .. (opt.stdmodule and "y" or "n") policies = policies .. ",build.c++.modules.fallbackscanner:" .. (opt.fallbackscanner and "y" or "n") + policies = policies .. ",build.c++.modules.two_phases:" .. (two_phases and "y" or "n") local platform = " " if opt.platform then @@ -92,10 +94,8 @@ function build_tests(toolchain_name, opt) local runtimes = " " if opt.runtimes then runtimes = " --runtimes=" .. opt.runtimes .. " " - print("running with config: (toolchain: %s, compiler: %s, version: %s, runtimes: %s)", toolchain_name, compiler, version, opt.runtimes) - else - print("running with config: (toolchain: %s, compiler: %s, version: %s)", toolchain_name, compiler, version) end + print("running with config: (toolchain: %s, compiler: %s, version: %s, runtimes: %s, stdmodule: %s, fallback scanner: %s, two phases: %s)", toolchain_name, compiler, version, opt.runtimes or "default", opt.stdmodule or false, opt.fallbackscanner or false, two_phases) local flags = "" if opt.flags then @@ -124,21 +124,25 @@ function run_tests(clang_options, gcc_options, msvc_options) if clang_options then build_tests("llvm", clang_options) build_tests("clang", clang_options) + build_tests("clang", table.join(clang_options, {two_phases = false})) if not clang_options.disable_clang_cl then local clang_cl_options = table.clone(clang_options) clang_cl_options.compiler = "clang-cl" clang_cl_options.version = CLANG_CL_MIN_VER build_tests("clang-cl", clang_cl_options) + build_tests("clang-cl", table.join(clang_options, {two_phases = false})) end if not clang_options.stdmodule then build_tests("llvm", clang_libcpp_options) build_tests("clang", clang_libcpp_options) + build_tests("clang", table.join(clang_libcpp_options, {two_phases = false})) else wprint("std modules tests skipped for Windows clang libc++ as it's not currently supported officially") end end if msvc_options then build_tests("msvc", msvc_options) + build_tests("msvc", table.join(msvc_options, {two_phases = false})) end elseif is_subhost("macosx") then if clang_options then @@ -161,6 +165,7 @@ function run_tests(clang_options, gcc_options, msvc_options) end build_tests("llvm", clang_options) build_tests("clang", clang_options) + build_tests("clang", table.join(clang_options, {two_phases = false})) end elseif is_subhost("msys") then if clang_options then @@ -168,22 +173,28 @@ function run_tests(clang_options, gcc_options, msvc_options) clang_libcpp_options.platform = "mingw" build_tests("llvm", clang_options) build_tests("clang", clang_options) + build_tests("clang", table.join(clang_options, {two_phases = false})) build_tests("llvm", clang_libcpp_options) build_tests("clang", clang_libcpp_options) + build_tests("clang", table.join(clang_libcpp_options, {two_phases = false})) end if gcc_options then gcc_options.platform = "mingw" build_tests("gcc", gcc_options) + build_tests("gcc", table.join(gcc_options, {two_phases = false})) end elseif is_host("linux") then if clang_options then build_tests("llvm", clang_options) build_tests("clang", clang_options) + build_tests("clang", table.join(clang_options, {two_phases = false})) build_tests("llvm", clang_libcpp_options) build_tests("clang", clang_libcpp_options) + build_tests("clang", table.join(clang_libcpp_options, {two_phases = false})) end if gcc_options then build_tests("gcc", gcc_options) + build_tests("gcc", table.join(gcc_options, {two_phases = false})) end end end From 750f981b289ffc6fff43a342be73623bc23eba22 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 10 Sep 2025 21:37:49 +0200 Subject: [PATCH 2/4] nfc(C++ modules) optimize two phases build for clang by using pcm file to compile objectfile instead of module sourcefile --- xmake/rules/c++/modules/clang/builder.lua | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/xmake/rules/c++/modules/clang/builder.lua b/xmake/rules/c++/modules/clang/builder.lua index a43c8aebe1..042148910c 100644 --- a/xmake/rules/c++/modules/clang/builder.lua +++ b/xmake/rules/c++/modules/clang/builder.lua @@ -34,6 +34,7 @@ import(".builder", {inherit = true}) function _make_modulebuildflags(target, module, opt) assert(not module.headerunit) local flags + local two_phases = target:policy("build.c++.modules.two_phases") if opt.bmi then local module_outputflag = support.get_moduleoutputflag(target) @@ -50,7 +51,10 @@ function _make_modulebuildflags(target, module, opt) end table.insert(flags, module_outputflag .. module.bmifile) else - flags = {"-x", "c++"} + flags = {} + if not two_phases or not module.bmifile then + flags = {"-x", "c++"} + end local std = (module.name == "std" or module.name == "std.compat") if std then table.join2(flags, {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier", "-Wno-deprecated-declarations"}) @@ -127,10 +131,13 @@ function _compile(target, flags, module, opt) opt = opt or {} local sourcefile = module.sourcefile + if not opt.bmi and opt.objectfile and module.bmifile then + sourcefile = module.bmifile + end local outputfile = ((opt.bmi and not opt.objectfile) or opt.headerunit) and module.bmifile or module.objectfile local dryrun = option.get("dry-run") local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"}) + local compflags = compinst:compflags({sourcefile = module.sourcefile, target = target, sourcekind = "cxx"}) flags = table.join(compflags or {}, flags or {}) -- trace local cmd @@ -152,7 +159,7 @@ function _batchcmds_compile(batchcmds, target, flags, module, opt) local sourcefile = module.sourcefile local outputfile = (opt.bmi and not opt.objectfile) and module.bmifile or module.objectfile local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"}) + local compflags = compinst:compflags({sourcefile = module.sourcefile, target = target, sourcekind = "cxx"}) flags = table.join("-c", compflags or {}, flags or {}, {"-o", outputfile, sourcefile}) -- trace From 37b7101e21ce6a6dbfe29339db736bfe150a4069 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 10 Sep 2025 22:20:38 +0200 Subject: [PATCH 3/4] feat(C++ modules) add support of reduced bmi for clang --- xmake/rules/c++/modules/clang/builder.lua | 36 ++++++++++++++++++++--- xmake/rules/c++/modules/clang/support.lua | 14 +++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/xmake/rules/c++/modules/clang/builder.lua b/xmake/rules/c++/modules/clang/builder.lua index 042148910c..684a6a0303 100644 --- a/xmake/rules/c++/modules/clang/builder.lua +++ b/xmake/rules/c++/modules/clang/builder.lua @@ -31,14 +31,32 @@ import("support") import(".mapper") import(".builder", {inherit = true}) +function _get_bmifile(target, module) + local has_reduced_bmi = support.get_modulesreducedbmiflag(target) + local has_two_phases = target:policy("build.c++.modules.two_phases") + -- disabled with two phases currently, LLVM currently have a bug which prevent to emit reduced bmi when using two phase compilation + -- will be enabled after the fix + local add_reduced_flag = not has_two_phases and has_reduced_bmi + local bmifile = module.bmifile + + if has_two_phases and add_reduced_flag then + bmifile = path.join(path.directory(module.bmifile), "reduced." .. path.filename(module.bmifile)) + end + + return bmifile, add_reduced_flag +end + function _make_modulebuildflags(target, module, opt) assert(not module.headerunit) + + local modules_reduced_bmi_flag = support.get_modulesreducedbmiflag(target) + local has_two_phases = target:policy("build.c++.modules.two_phases") local flags - local two_phases = target:policy("build.c++.modules.two_phases") if opt.bmi then local module_outputflag = support.get_moduleoutputflag(target) flags = {"-x", "c++-module"} + if not opt.objectfile then table.insert(flags, "--precompile") if target:has_tool("cxx", "clang_cl") then @@ -49,10 +67,18 @@ function _make_modulebuildflags(target, module, opt) if std then table.join2(flags, {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier", "-Wno-deprecated-declarations"}) end - table.insert(flags, module_outputflag .. module.bmifile) + + local bmifile, add_reduced_flag = _get_bmifile(target, module) + if add_reduced_flag then + table.insert(flags, modules_reduced_bmi_flag) + end + + if not has_two_phases or add_reduced_flag then + table.insert(flags, module_outputflag .. bmifile) + end else flags = {} - if not two_phases or not module.bmifile then + if not has_two_phases or not module.bmifile then flags = {"-x", "c++"} end local std = (module.name == "std" or module.name == "std.compat") @@ -200,7 +226,8 @@ function _get_requiresflags(target, module) local dep_module = mapper.get(target, required) assert(dep_module, "module dependency %s required for %s not found", required, name) - local mapflag = dep_module.headerunit and modulefileflag .. dep_module.bmifile or format("%s%s=%s", modulefileflag, required, dep_module.bmifile) + local dep_bmifile, _ = dep.headerunit and dep_module.bmifile or _get_bmifile(target, dep_module) + local mapflag = dep_module.headerunit and modulefileflag .. dep_bmifile or format("%s%s=%s", modulefileflag, required, dep_bmifile) table.insert(requiresflags, mapflag) -- append deps @@ -219,6 +246,7 @@ end function _append_requires_flags(target, module) local cxxflags = {} local requiresflags = _get_requiresflags(target, module) + local has_two_phases = target:policy("build.c++.modules.two_phases") local hide_dependencies = target:policy("build.c++.modules.hide_dependencies") if #requiresflags> 0 then for _, flag in ipairs(requiresflags) do diff --git a/xmake/rules/c++/modules/clang/support.lua b/xmake/rules/c++/modules/clang/support.lua index 07f39e35dc..c1684a1b9a 100644 --- a/xmake/rules/c++/modules/clang/support.lua +++ b/xmake/rules/c++/modules/clang/support.lua @@ -347,6 +347,20 @@ function get_moduleheaderflag(target) return moduleheaderflag or nil end +function get_modulesreducedbmiflag(target) + local modulesreducedbmiflag = _g.modulesreducedbmiflag + if modulesreducedbmiflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodules-reduced-bmi", "cxxflags", {flagskey = "clang_modules_reduced_bmi"}) then + modulesreducedbmiflag = "-fmodules-reduced-bmi" + elseif compinst:has_flags("-fexperimental-modules-reduced-bmi", "cxxflags", {flagskey = "clang_modules_reduced_bmi"}) then + modulesreducedbmiflag = "-fexperimental-modules-reduced-bmi" + end + _g.modulesreducedbmiflag = modulesreducedbmiflag or false + end + return modulesreducedbmiflag or nil +end + function has_clangscandepssupport(target) local support_clangscandeps = _g.support_clangscandeps if support_clangscandeps == nil then From c71d77375c51440a4a5a743709e4945c00859990 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 10 Sep 2025 22:22:04 +0200 Subject: [PATCH 4/4] feat(C++ modules) add build.c++.modules.non_cascading_changes policy to enable bmi hash comparison as incremental build optimisation for clang --- xmake/core/project/policy.lua | 2 ++ xmake/rules/c++/modules/builder.lua | 19 +++++++------- xmake/rules/c++/modules/clang/builder.lua | 32 ++++++++++++++++++++++- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua index 6949a6b94a..967f9062e3 100644 --- a/xmake/core/project/policy.lua +++ b/xmake/core/project/policy.lua @@ -72,6 +72,8 @@ function policy.policies() ["build.rpath"] = {description = "Enable build rpath.", default = true, type = "boolean"}, -- Enable C++ modules for C++ building, even if no .mpp is involved in the compilation ["build.c++.modules"] = {description = "Enable C++ modules for C++ building.", type = "boolean"}, + -- Enable non cascading changes (experimental) + ["build.c++.modules.non_cascading_changes"] = {description = "Enable non cascading changes when supported (experimental).", default = false, type = "boolean"}, -- Hide C++ required files to reduce noise (may reduce build performance) ["build.c++.modules.hide_dependencies"] = {description = "Hide dependencies from the commandline when build C++ modules.", default = false, type = "boolean"}, -- Enable two phase compilation for C++ modules if supported by the compiler diff --git a/xmake/rules/c++/modules/builder.lua b/xmake/rules/c++/modules/builder.lua index b84d5d23e6..a1c93a35f1 100644 --- a/xmake/rules/c++/modules/builder.lua +++ b/xmake/rules/c++/modules/builder.lua @@ -180,6 +180,15 @@ function should_build(target, module) local old_dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {}) old_dependinfo.files = {module.sourcefile} + -- need build this object? + local dryrun = option.get("dry-run") + if dryrun or depend.is_changed(old_dependinfo, dependinfo) then + depend.save(dependinfo, dependfile) + memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true) + profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt") + return true + end + -- force rebuild a module if any of its module dependency is rebuilt for dep_name, dep_module in table.orderpairs(module.deps) do local mapped_dep = mapper.get(target, dep_module.headerunit and dep_name .. dep_module.key or dep_name) @@ -188,18 +197,10 @@ function should_build(target, module) depend.save(dependinfo, dependfile) memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true) profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt") - return true + return true, true end end - -- need build this object? - local dryrun = option.get("dry-run") - if dryrun or depend.is_changed(old_dependinfo, dependinfo) then - depend.save(dependinfo, dependfile) - memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true) - profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt") - return true - end memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, false) profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt") return false diff --git a/xmake/rules/c++/modules/clang/builder.lua b/xmake/rules/c++/modules/clang/builder.lua index 684a6a0303..60f2186e55 100644 --- a/xmake/rules/c++/modules/clang/builder.lua +++ b/xmake/rules/c++/modules/clang/builder.lua @@ -20,6 +20,7 @@ -- imports import("core.base.json") +import("core.base.bytes") import("core.base.option") import("core.base.semver") import("utils.progress") @@ -46,6 +47,19 @@ function _get_bmifile(target, module) return bmifile, add_reduced_flag end +function _update_bmihash(target, module) + local localcache = support.localcache() + + local bmifile = _get_bmifile(target, module) + local bmihash = hash.xxhash128(bytes(io.readfile(bmifile))) + local old_bmihash = localcache:get2(bmifile, "hash") + + if not old_bmihash or bmihash ~= old_bmihash then + localcache:set2(bmifile, "hash", bmihash) + support.memcache():set2(bmifile, "updated", true) + end +end + function _make_modulebuildflags(target, module, opt) assert(not module.headerunit) @@ -285,11 +299,23 @@ end function make_module_job(target, module, opt) local dryrun = option.get("dry-run") + local enable_hash_comparison = target:policy("build.c++.modules.non_cascading_changes") - local build = should_build(target, module) + local build, because_of_dependencies = should_build(target, module) local bmi = opt and opt.bmi local objectfile = opt and opt.objectfile + if build and enable_hash_comparison and because_of_dependencies then + build = false + for dep_name, dep_module in table.orderpairs(module.deps) do + local mapped_dep = mapper.get(target, dep_module.headerunit and dep_name .. dep_module.key or dep_name) + if support.memcache():get2(_get_bmifile(target, dep_module), "updated") then + build = true + break + end + end + end + if build then if not dryrun then local objectdir = path.directory(module.objectfile) @@ -315,6 +341,10 @@ function make_module_job(target, module, opt) os.tryrm(module.objectfile) -- force rebuild for .cpp files end end + + if enable_hash_comparison and bmi then + _update_bmihash(target, module) + end end end