feat(C++ modules) Improve incremental build clang

tests/projects/c++/modules/test_base.lua

@@ -81,8 +81,10 @@ function build_tests(toolchain_name, opt)
         return
     end
 
+    local two_phases = (opt.two_phases == nil or opt.two_phases == true)
     local policies = "--policies=build.c++.modules.std:" .. (opt.stdmodule and "y" or "n")
     policies = policies .. ",build.c++.modules.fallbackscanner:" .. (opt.fallbackscanner and "y" or "n")
+    policies = policies .. ",build.c++.modules.two_phases:" .. (two_phases and "y" or "n")
 
     local platform = " "
     if opt.platform then
@@ -92,10 +94,8 @@ function build_tests(toolchain_name, opt)
     local runtimes = " "
     if opt.runtimes then
         runtimes = " --runtimes=" .. opt.runtimes .. " "
-        print("running with config: (toolchain: %s, compiler: %s, version: %s, runtimes: %s)", toolchain_name, compiler, version, opt.runtimes)
-    else
-        print("running with config: (toolchain: %s, compiler: %s, version: %s)", toolchain_name, compiler, version)
     end
+    print("running with config: (toolchain: %s, compiler: %s, version: %s, runtimes: %s, stdmodule: %s, fallback scanner: %s, two phases: %s)", toolchain_name, compiler, version, opt.runtimes or "default", opt.stdmodule or false, opt.fallbackscanner or false, two_phases)
 
     local flags = ""
     if opt.flags then
@@ -124,21 +124,25 @@ function run_tests(clang_options, gcc_options, msvc_options)
         if clang_options then
             build_tests("llvm", clang_options)
             build_tests("clang", clang_options)
+            build_tests("clang", table.join(clang_options, {two_phases = false}))
             if not clang_options.disable_clang_cl then
                 local clang_cl_options = table.clone(clang_options)
                 clang_cl_options.compiler = "clang-cl"
                 clang_cl_options.version = CLANG_CL_MIN_VER
                 build_tests("clang-cl", clang_cl_options)
+                build_tests("clang-cl", table.join(clang_options, {two_phases = false}))
             end
             if not clang_options.stdmodule then
                 build_tests("llvm", clang_libcpp_options)
                 build_tests("clang", clang_libcpp_options)
+                build_tests("clang", table.join(clang_libcpp_options, {two_phases = false}))
             else
                 wprint("std modules tests skipped for Windows clang libc++ as it's not currently supported officially")
             end
         end
         if msvc_options then
             build_tests("msvc", msvc_options)
+            build_tests("msvc", table.join(msvc_options, {two_phases = false}))
         end
     elseif is_subhost("macosx") then
         if clang_options then
@@ -161,29 +165,36 @@ function run_tests(clang_options, gcc_options, msvc_options)
             end
             build_tests("llvm", clang_options)
             build_tests("clang", clang_options)
+            build_tests("clang", table.join(clang_options, {two_phases = false}))
         end
     elseif is_subhost("msys") then
         if clang_options then
             clang_options.platform = "mingw"
             clang_libcpp_options.platform = "mingw"
             build_tests("llvm", clang_options)
             build_tests("clang", clang_options)
+            build_tests("clang", table.join(clang_options, {two_phases = false}))
             build_tests("llvm", clang_libcpp_options)
             build_tests("clang", clang_libcpp_options)
+            build_tests("clang", table.join(clang_libcpp_options, {two_phases = false}))
         end
         if gcc_options then
             gcc_options.platform = "mingw"
             build_tests("gcc", gcc_options)
+            build_tests("gcc", table.join(gcc_options, {two_phases = false}))
         end
     elseif is_host("linux") then
         if clang_options then
             build_tests("llvm", clang_options)
             build_tests("clang", clang_options)
+            build_tests("clang", table.join(clang_options, {two_phases = false}))
             build_tests("llvm", clang_libcpp_options)
             build_tests("clang", clang_libcpp_options)
+            build_tests("clang", table.join(clang_libcpp_options, {two_phases = false}))
         end
         if gcc_options then
             build_tests("gcc", gcc_options)
+            build_tests("gcc", table.join(gcc_options, {two_phases = false}))
         end
     end
 end

xmake/core/project/policy.lua

@@ -72,6 +72,8 @@ function policy.policies()
             ["build.rpath"]                       = {description = "Enable build rpath.", default = true, type = "boolean"},
             -- Enable C++ modules for C++ building, even if no .mpp is involved in the compilation
             ["build.c++.modules"]                 = {description = "Enable C++ modules for C++ building.", type = "boolean"},
+            -- Enable non cascading changes (experimental)
+            ["build.c++.modules.non_cascading_changes"] = {description = "Enable non cascading changes when supported (experimental).", default = false, type = "boolean"},
             -- Hide C++ required files to reduce noise (may reduce build performance)
             ["build.c++.modules.hide_dependencies"] = {description = "Hide dependencies from the commandline when build C++ modules.", default = false, type = "boolean"},
             -- Enable two phase compilation for C++ modules if supported by the compiler

xmake/rules/c++/modules/builder.lua

@@ -180,6 +180,15 @@ function should_build(target, module)
         local old_dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {})
         old_dependinfo.files = {module.sourcefile}
 
+        -- need build this object?
+        local dryrun = option.get("dry-run")
+        if dryrun or depend.is_changed(old_dependinfo, dependinfo) then
+            depend.save(dependinfo, dependfile)
+            memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true)
+            profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt")
+            return true
+        end
+
         -- force rebuild a module if any of its module dependency is rebuilt
         for dep_name, dep_module in table.orderpairs(module.deps) do
             local mapped_dep = mapper.get(target, dep_module.headerunit and dep_name .. dep_module.key or dep_name)
@@ -188,18 +197,10 @@ function should_build(target, module)
                 depend.save(dependinfo, dependfile)
                 memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true)
                 profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt")
-                return true
+                return true, true
             end
         end
 
-        -- need build this object?
-        local dryrun = option.get("dry-run")
-        if dryrun or depend.is_changed(old_dependinfo, dependinfo) then
-            depend.save(dependinfo, dependfile)
-            memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true)
-            profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt")
-            return true
-        end
         memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, false)
         profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt")
         return false

xmake/rules/c++/modules/clang/builder.lua

@@ -20,6 +20,7 @@
 
 -- imports
 import("core.base.json")
+import("core.base.bytes")
 import("core.base.option")
 import("core.base.semver")
 import("utils.progress")
@@ -31,13 +32,45 @@ import("support")
 import(".mapper")
 import(".builder", {inherit = true})
 
+function _get_bmifile(target, module)
+    local has_reduced_bmi = support.get_modulesreducedbmiflag(target)
+    local has_two_phases = target:policy("build.c++.modules.two_phases")
+    -- disabled with two phases currently, LLVM currently have a bug which prevent to emit reduced bmi when using two phase compilation
+    -- will be enabled after the fix
+    local add_reduced_flag = not has_two_phases and has_reduced_bmi
+    local bmifile = module.bmifile
+
+    if has_two_phases and add_reduced_flag then
+        bmifile = path.join(path.directory(module.bmifile), "reduced." .. path.filename(module.bmifile))
+    end
+
+    return bmifile, add_reduced_flag
+end
+
+function _update_bmihash(target, module)
+    local localcache = support.localcache()
+
+    local bmifile = _get_bmifile(target, module)
+    local bmihash = hash.xxhash128(bytes(io.readfile(bmifile)))
+    local old_bmihash = localcache:get2(bmifile, "hash")
+
+    if not old_bmihash or bmihash ~= old_bmihash then
+        localcache:set2(bmifile, "hash", bmihash)
+        support.memcache():set2(bmifile, "updated", true)
+    end
+end
+
 function _make_modulebuildflags(target, module, opt)
     assert(not module.headerunit)
+
+    local modules_reduced_bmi_flag = support.get_modulesreducedbmiflag(target)
+    local has_two_phases = target:policy("build.c++.modules.two_phases")
     local flags
     if opt.bmi then
         local module_outputflag = support.get_moduleoutputflag(target)
 
         flags = {"-x", "c++-module"}
+
         if not opt.objectfile then
             table.insert(flags, "--precompile")
             if target:has_tool("cxx", "clang_cl") then
@@ -48,9 +81,20 @@ function _make_modulebuildflags(target, module, opt)
         if std then
             table.join2(flags, {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier", "-Wno-deprecated-declarations"})
         end
-        table.insert(flags, module_outputflag .. module.bmifile)
+
+        local bmifile, add_reduced_flag = _get_bmifile(target, module)
+        if add_reduced_flag then
+            table.insert(flags, modules_reduced_bmi_flag)
+        end
+
+        if not has_two_phases or add_reduced_flag  then
+            table.insert(flags, module_outputflag .. bmifile)
+        end
     else
-        flags = {"-x", "c++"}
+        flags = {}
+        if not has_two_phases or not module.bmifile then
+            flags = {"-x", "c++"}
+        end
         local std = (module.name == "std" or module.name == "std.compat")
         if std then
             table.join2(flags, {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier", "-Wno-deprecated-declarations"})
@@ -127,10 +171,13 @@ function _compile(target, flags, module, opt)
 
     opt = opt or {}
     local sourcefile = module.sourcefile
+    if not opt.bmi and opt.objectfile and module.bmifile then
+        sourcefile = module.bmifile
+    end
     local outputfile = ((opt.bmi and not opt.objectfile) or opt.headerunit) and module.bmifile or module.objectfile
     local dryrun = option.get("dry-run")
     local compinst = target:compiler("cxx")
-    local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"})
+    local compflags = compinst:compflags({sourcefile = module.sourcefile, target = target, sourcekind = "cxx"})
     flags = table.join(compflags or {}, flags or {})
     -- trace
     local cmd
@@ -152,7 +199,7 @@ function _batchcmds_compile(batchcmds, target, flags, module, opt)
     local sourcefile = module.sourcefile
     local outputfile = (opt.bmi and not opt.objectfile) and module.bmifile or module.objectfile
     local compinst = target:compiler("cxx")
-    local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"})
+    local compflags = compinst:compflags({sourcefile = module.sourcefile, target = target, sourcekind = "cxx"})
     flags = table.join("-c", compflags or {}, flags or {}, {"-o", outputfile, sourcefile})
 
     -- trace
@@ -193,7 +240,8 @@ function _get_requiresflags(target, module)
             local dep_module = mapper.get(target, required)
             assert(dep_module, "module dependency %s required for %s not found", required, name)
 
-            local mapflag = dep_module.headerunit and modulefileflag .. dep_module.bmifile or format("%s%s=%s", modulefileflag, required, dep_module.bmifile)
+            local dep_bmifile, _ = dep.headerunit and dep_module.bmifile or _get_bmifile(target, dep_module)
+            local mapflag = dep_module.headerunit and modulefileflag .. dep_bmifile or format("%s%s=%s", modulefileflag, required, dep_bmifile)
             table.insert(requiresflags, mapflag)
 
             -- append deps
@@ -212,6 +260,7 @@ end
 function _append_requires_flags(target, module)
     local cxxflags = {}
     local requiresflags = _get_requiresflags(target, module)
+    local has_two_phases = target:policy("build.c++.modules.two_phases")
     local hide_dependencies = target:policy("build.c++.modules.hide_dependencies")
     if #requiresflags> 0 then
         for _, flag in ipairs(requiresflags) do
@@ -250,11 +299,23 @@ end
 function make_module_job(target, module, opt)
 
     local dryrun = option.get("dry-run")
+    local enable_hash_comparison = target:policy("build.c++.modules.non_cascading_changes")
 
-    local build = should_build(target, module)
+    local build, because_of_dependencies = should_build(target, module)
     local bmi = opt and opt.bmi
     local objectfile = opt and opt.objectfile
 
+    if build and enable_hash_comparison and because_of_dependencies then
+        build = false
+        for dep_name, dep_module in table.orderpairs(module.deps) do
+            local mapped_dep = mapper.get(target, dep_module.headerunit and dep_name .. dep_module.key or dep_name)
+            if support.memcache():get2(_get_bmifile(target, dep_module), "updated") then
+                build = true
+                break
+            end
+        end
+    end
+
     if build then
         if not dryrun then
             local objectdir = path.directory(module.objectfile)
@@ -280,6 +341,10 @@ function make_module_job(target, module, opt)
                 os.tryrm(module.objectfile) -- force rebuild for .cpp files
             end
         end
+
+        if enable_hash_comparison and bmi then
+            _update_bmihash(target, module)
+        end
     end
 end
 

xmake/rules/c++/modules/clang/support.lua

@@ -347,6 +347,20 @@ function get_moduleheaderflag(target)
     return moduleheaderflag or nil
 end
 
+function get_modulesreducedbmiflag(target)
+    local modulesreducedbmiflag = _g.modulesreducedbmiflag
+    if modulesreducedbmiflag == nil then
+        local compinst = target:compiler("cxx")
+        if compinst:has_flags("-fmodules-reduced-bmi", "cxxflags", {flagskey = "clang_modules_reduced_bmi"}) then
+            modulesreducedbmiflag = "-fmodules-reduced-bmi"
+        elseif compinst:has_flags("-fexperimental-modules-reduced-bmi", "cxxflags", {flagskey = "clang_modules_reduced_bmi"}) then
+            modulesreducedbmiflag = "-fexperimental-modules-reduced-bmi"
+        end
+        _g.modulesreducedbmiflag = modulesreducedbmiflag or false
+    end
+    return modulesreducedbmiflag or nil
+end
+
 function has_clangscandepssupport(target)
     local support_clangscandeps = _g.support_clangscandeps
     if support_clangscandeps == nil then