Spack 1 update

.gitlab/tests/non_shared.yml

@@ -17,7 +17,7 @@ workflow:
       - HOST: lassen
         ARCHCONFIG: llnl-sierra
         SCHEDULER_PARAMETERS: -nnodes 1 -W 40 -q pci
-        BENCHMARK: [amg2023, kripke]
+        BENCHMARK: [amg2023]
         VARIANT: [+cuda, +openmp]
 run_tests_nonshared:
   resource_group: $HOST

.gitlab/utils/run-experiment.sh

@@ -4,6 +4,20 @@ set -e
 # Activate Virtual Environment
 . /usr/workspace/benchpark-dev/benchpark-venv/$SYS_TYPE/bin/activate
 
+
+if [ "$HOST" == "lassen" ]; then
+    echo "./bin/benchpark system init --dest=${HOST} ${ARCHCONFIG} $SYSTEM_ARGS"
+else
+    echo "./bin/benchpark system init --dest=${HOST} ${ARCHCONFIG} cluster=$HOST $SYSTEM_ARGS"
+fi
+echo "./bin/benchpark experiment init --dest=${BENCHMARK} --system=${HOST} ${BENCHMARK} ${VARIANT}"
+echo "./bin/benchpark setup ${BENCHMARK} wkp/"
+echo ". wkp/setup.sh"
+echo "cd ./wkp/${BENCHMARK}/${HOST}/workspace/"
+echo "ramble --disable-logger --workspace-dir . workspace setup"
+echo "ramble --disable-logger --workspace-dir . on --executor '{execute_experiment}' --where '{n_nodes} == 1'"
+echo "ramble --disable-logger --workspace-dir . workspace analyze --format json yaml text"
+
 # Initialize System
 if [ "$HOST" == "lassen" ]; then
     ./bin/benchpark system init --dest=${HOST} ${ARCHCONFIG} $SYSTEM_ARGS

checkout-versions.yaml

@@ -4,4 +4,5 @@
 # SPDX-License-Identifier: Apache-2.0
 versions:
   ramble: 6b4ecac  # develop on 5/06/2025 (newer than 0.6.0 release)
-  spack: 7e4b8aa  # develop on 2/22/2025 (pre-breaking changes on 1.0)
+  spack: 7eeef1f
+  spack-packages: a1941c9

experiments/amg2023/experiment.py

@@ -115,3 +115,4 @@ def compute_applications_section(self):
 
     def compute_package_section(self):
         self.add_package_spec(self.name, [f"amg2023{self.determine_version()}"])
+        self.add_package_spec("hypre", ["hypre+lapack"])

experiments/kripke/experiment.py

@@ -156,5 +156,5 @@ def compute_package_section(self):
             else "~single_memory"
         )
         self.add_package_spec(
-            self.name, [f"kripke{self.determine_version()} {single_memory} "]
+            self.name, [f"kripke{self.determine_version()} {single_memory} +mpi"]
         )

experiments/laghos/experiment.py

@@ -75,3 +75,4 @@ def compute_applications_section(self):
 
     def compute_package_section(self):
         self.add_package_spec(self.name, [f"laghos{self.determine_version()} +metis"])
+        self.add_package_spec("hypre", ["hypre@2.32.0: +lapack"])

experiments/raja-perf/experiment.py

@@ -91,4 +91,4 @@ def compute_applications_section(self):
             self.add_experiment_variable("n_ranks", "{n_resources}", True)
 
     def compute_package_section(self):
-        self.add_package_spec(self.name, [f"raja-perf{self.determine_version()}"])
+        self.add_package_spec(self.name, [f"raja-perf{self.determine_version()} +mpi"])

lib/benchpark/cmd/setup.py

@@ -173,7 +173,17 @@ def include_fn(fname):
     if pkg_manager == "spack":
         spack, first_time_spack = per_workspace_setup.spack_first_time_setup()
         if first_time_spack:
-            spack("repo", "add", "--scope=site", f"{source_dir}/repo")
+            site_repos = (
+                per_workspace_setup.spack_location / "etc" / "spack" / "repos.yaml"
+            )
+            with open(site_repos, "w") as f:
+                f.write(
+                    f"""\
+repos::
+  benchpark: {source_dir}/repo
+  builtin: {per_workspace_setup.pkgs_location}/repos/spack_repo/builtin/
+"""
+                )
 
         pkg_str = f"""\
 . {per_workspace_setup.spack_location}/share/spack/setup-env.sh

lib/benchpark/runtime.py

@@ -68,29 +68,43 @@ def __init__(self, dest, upstream=None):
         self.dest = pathlib.Path(dest)
         self.upstream = upstream
 
-        self.ramble_location, self.spack_location = (
+        self.ramble_location, self.spack_location, self.pkgs_location = (
             self.dest / "ramble",
-            self.dest / "spack",
+            self.dest / "spk",
+            self.dest / "spack-packages",
         )
 
         # Read pinned versions of ramble and spack
         with open(benchpark.paths.checkout_versions, "r") as yaml_file:
             data = yaml.safe_load(yaml_file)["versions"]
-            self.ramble_commit, self.spack_commit = data["ramble"], data["spack"]
+            self.ramble_commit, self.spack_commit, self.pkgs_commit = (
+                data["ramble"],
+                data["spack"],
+                data["spack-packages"],
+            )
 
         # Read remote urls for ramble and spack
         with open(benchpark.paths.remote_urls, "r") as yaml_file:
             data = yaml.safe_load(yaml_file)["urls"]
-            remote_ramble_url, remote_spack_url = data["ramble"], data["spack"]
+            remote_ramble_url, remote_spack_url, remote_pkgs_url = (
+                data["ramble"],
+                data["spack"],
+                data["spack-packages"],
+            )
 
         # If this does not have an upstream, then we will be cloning from the URLs indicated in remote-urls.yaml
         if self.upstream is None:
-            self.ramble_url, self.spack_url = remote_ramble_url, remote_spack_url
+            self.ramble_url, self.spack_url, self.pkgs_url = (
+                remote_ramble_url,
+                remote_spack_url,
+                remote_pkgs_url,
+            )
         else:
             # Clone from local "upstream" repository
-            self.ramble_url, self.spack_url = (
+            self.ramble_url, self.spack_url, self.pkgs_url = (
                 self.upstream.ramble_location,
                 self.upstream.spack_location,
+                self.upstream.pkgs_location,
             )
 
     def _check_and_update_bootstrap(self, desired_commit, location):
@@ -117,6 +131,11 @@ def bootstrap(self):
         if internals not in sys.path:
             sys.path.insert(1, internals)
 
+        if not self.pkgs_location.exists():
+            self._install_packages()
+        else:
+            self._check_and_update_bootstrap(self.pkgs_commit, self.pkgs_location)
+
         # Spack does not go in sys.path, but we will manually access modules from it
         # The reason for this oddity is that spack modules will compete with the internal
         # spack modules from ramble
@@ -143,6 +162,15 @@ def _install_spack(self):
         )
         debug_print(f"Done cloning Spack ({self.spack_location})")
 
+    def _install_packages(self):
+        print(f"Cloning packages to {self.pkgs_location}")
+        git_clone_commit(
+            self.pkgs_url,
+            self.pkgs_commit,
+            self.pkgs_location,
+        )
+        debug_print(f"Done cloning spack-packages ({self.pkgs_location})")
+
     def _ramble(self):
         first_time = False
         if not self.ramble_location.exists():
@@ -151,10 +179,13 @@ def _ramble(self):
         return Command(self.ramble_location / "bin" / "ramble", env={}), first_time
 
     def _spack(self):
+        if not self.pkgs_location.exists():
+            self._install_packages()
+
         env = {"SPACK_DISABLE_LOCAL_CONFIG": "1"}
         spack = Command(self.spack_location / "bin" / "spack", env)
         spack_cache_location = self.spack_location / "misc-cache"
-        bootstrap_cache_location = self.spack_location / "b-c"
+        bootstrap_cache_location = self.dest / "sbc"
         first_time = False
         if not self.spack_location.exists():
             first_time = True

lib/benchpark/system.py

@@ -203,20 +203,22 @@ def compute_software_section(self):
 
     def compute_dict(self):
         # This can be overridden by any subclass that needs more flexibility
-        compilers = self.compute_compilers_section()
+        pkg_cfg = self.compute_packages_section() or {}
+        compiler_cfg = self.compute_compilers_section()
+        if compiler_cfg:
+            pkg_cfg = merge_dicts(pkg_cfg, compiler_cfg)
+        # "'packages:':" syntax is required to enforce spack to use benchpark-defined
+        # compilers instead of external compilers defined by spack compiler search (from ramble).
+        if pkg_cfg:
+            pkg_cfg = {"packages:": pkg_cfg["packages"]}
+        else:
+            pkg_cfg = {"packages": {}}
         return {
             "system_id": self.compute_system_id(),
             "variables": self.compute_variables_section(),
             "software": self.compute_software_section(),
             "auxiliary_software_files": {
-                "compilers": (
-                    # "'compilers:':" syntax is required to enforce spack to use benchpark-defined
-                    # compilers instead of external compilers defined by spack compiler search (from ramble).
-                    {"compilers:": compilers["compilers"]}
-                    if compilers
-                    else None
-                ),
-                "packages": self.compute_packages_section(),
+                "packages": pkg_cfg,
             },
         }
 
@@ -233,3 +235,81 @@ def _write_key_file(destdir, key, sys_dict):
                     _write_key_file(destdir + "/" + key, k, system_dict[key])
             else:
                 _write_key_file(destdir, key, system_dict)
+
+
+def merge_dicts(*dicts):
+    current = {}
+    for d in dicts:
+        if not d:
+            continue
+        current = _merge_dicts(current, d)
+    return current
+
+
+def _merge_dicts(d1, d2):
+    result = dict(d1)
+    for k, v2 in d2.items():
+        if k in result:
+            v1 = result[k]
+            if all(isinstance(x, dict) for x in (v1, v2)):
+                result[k] = _merge_dicts(v1, v2)
+            elif all(isinstance(x, list) for x in (v1, v2)):
+                result[k] = v1 + v2
+            else:
+                raise ValueError(f"{k} merge mismatch: {v1}/{v2}")
+        else:
+            result[k] = v2
+    return result
+
+
+def hybrid_compiler_requirements(c_cmp, f_cmp):
+    return {
+        "packages": {
+            "all": {
+                "require": [
+                    {
+                        "spec": rf"%[virtuals=c] {c_cmp}",
+                        "when": r"%c",
+                    },
+                    {"spec": rf"%[virtuals=cxx] {c_cmp}", "when": r"%cxx"},
+                    {"spec": rf"^[virtuals=fortran] {f_cmp}", "when": r"^fortran"},
+                ]
+            }
+        }
+    }
+
+
+def compiler_section_for(name, entries):
+    return {"packages": {name: {"externals": entries, "buildable": False}}}
+
+
+def compiler_def(
+    spec,
+    prefix,
+    exes,
+    env=None,
+    extra_rpaths=None,
+    modules=None,
+    flags=None,
+    compilers_use_relative_paths=False,
+):
+    lang_map = {}
+    for lang, exe in exes.items():
+        if os.path.isabs(exe) or compilers_use_relative_paths:
+            lang_map[lang] = exe
+        else:
+            lang_map[lang] = os.path.join(prefix, "bin", exe)
+    entry = {
+        "spec": spec,
+        "prefix": prefix,
+        "extra_attributes": {"compilers": lang_map},
+    }
+    if env:
+        entry["extra_attributes"]["environment"] = env
+    if extra_rpaths:
+        entry["extra_attributes"]["extra_rpaths"] = extra_rpaths
+    if modules:
+        entry["modules"] = modules
+    if flags:
+        entry["extra_attributes"]["flags"] = flags
+    return entry

remote-urls.yaml

@@ -5,3 +5,4 @@
 urls:
   ramble: https://github.com/GoogleCloudPlatform/ramble.git
   spack: https://github.com/spack/spack.git
+  spack-packages: https://github.com/spack/spack-packages.git

repo/ad/package.py

@@ -19,6 +19,10 @@ class Ad(CMakePackage):
 
     version("main", branch="main")
 
+    depends_on("c", type="build")
+    depends_on("cxx", type="build")
+    depends_on("fortran", type="build")
+
     depends_on("enzyme")
 
     def cmake_args(self):

repo/affinity/package.py

@@ -20,6 +20,9 @@ class Affinity(CMakePackage, CudaPackage, ROCmPackage):
     variant("mpi", default=False, description="Build MPI support")
     variant("rocm", default=False, description="Build ROCm Support")
 
+    depends_on("c", type="build")
+    depends_on("cxx", type="build")
+
     depends_on("mpi", when="+mpi")
     depends_on("hip", when="+rocm")
     depends_on("cuda", when="+cuda")

repo/amg2023/package.py

@@ -26,6 +26,9 @@ class Amg2023(CMakePackage, CudaPackage, ROCmPackage):
     variant("openmp", default=False, description="Enable OpenMP support")
     variant("caliper", default=False, description="Enable Caliper monitoring")
 
+    depends_on("c", type="build")
+    depends_on("cxx", type="build")
+
     depends_on("mpi", when="+mpi")
     depends_on("hypre+mpi", when="+mpi")
     requires("+mpi", when="^hypre+mpi")
@@ -36,14 +39,12 @@ class Amg2023(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("hypre+mixedint~fortran")
 
     depends_on("hypre+cuda", when="+cuda")
-    depends_on("hypre+cublas", when="+cuda")
     depends_on("hypre+openmp", when="+openmp")
     requires("+cuda", when="^hypre+cuda")
     for arch in ("none", "50", "60", "70", "80", "90"):
         depends_on(f"hypre cuda_arch={arch}", when=f"cuda_arch={arch}")
 
     depends_on("hypre+rocm", when="+rocm")
-    depends_on("hypre+rocblas", when="+rocm")
     requires("+rocm", when="^hypre+rocm")
     for target in ("none", "gfx803", "gfx900", "gfx906", "gfx908", "gfx90a", "gfx942"):
         depends_on(f"hypre amdgpu_target={target}", when=f"amdgpu_target={target}")

repo/babelstream/package.py

@@ -87,6 +87,11 @@ class Babelstream(CMakePackage, CudaPackage, ROCmPackage):
         msg="RAJA implementation requires architecture to be specified by dir=",
     )
 
+    # Confirmed c++ and Fortran
+    depends_on("c", type="build")
+    depends_on("cxx", type="build")
+    depends_on("fortran", type="build")
+
     # Thrust Conflict
     # conflicts("~cuda", when="+thrust", msg="Thrust requires +cuda variant")
     depends_on("thrust", when="+thrust")

repo/branson/package.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from spack.package import *
-from spack.pkg.builtin.boost import Boost
+from spack_repo.builtin.packages.boost.package import Boost
 
 
 class Branson(CMakePackage):
@@ -30,6 +30,10 @@ class Branson(CMakePackage):
     )
     version("0.81", sha256="493f720904791f06b49ff48c17a681532c6a4d9fa59636522cf3f9700e77efe4")
     version("0.8", sha256="85ffee110f89be00c37798700508b66b0d15de1d98c54328b6d02a9eb2cf1cb8")
+
+    depends_on("c", type="build")
+    depends_on("cxx", type="build")
+
     #depends_on("mpi")
     depends_on("mpi@2:")
 

repo/caliper/package.py

@@ -109,6 +109,10 @@ class Caliper(CachedCMakePackage, CudaPackage, ROCmPackage):
     variant("tools", default=True, description="Enable tools")
     variant("python", default=False, when="@v2.12:", description="Build Python bindings")
 
+    depends_on("c", type="build")
+    depends_on("cxx", type="build")
+    depends_on("fortran", type="build")
+
     depends_on("adiak@0.1:0", when="@2.2:2.10 +adiak")
     depends_on("adiak@0.4:0", when="@2.11: +adiak")
 

repo/camp/package.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from spack.package import *
-from spack.pkg.builtin.camp import Camp as BuiltinCamp
+from spack_repo.builtin.packages.camp.package import Camp as BuiltinCamp
 
 
 class Camp(BuiltinCamp):

repo/chai/package.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from spack.package import *
-from spack.pkg.builtin.chai import Chai as BuiltinChai
+from spack_repo.builtin.packages.chai.package import Chai as BuiltinChai
 
 
 class Chai(BuiltinChai):