SNL sparta

experiments/sparta-snl/experiment.py

@@ -0,0 +1,121 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from benchpark.directives import variant, maintainers
+from benchpark.experiment import Experiment
+from benchpark.mpi import MpiOnlyExperiment
+from benchpark.openmp import OpenMPExperiment
+from benchpark.cuda import CudaExperiment
+from benchpark.rocm import ROCmExperiment
+
+
+class SpartaSnl(
+    Experiment,
+    MpiOnlyExperiment,
+    OpenMPExperiment,
+    CudaExperiment,
+    ROCmExperiment,
+):
+    variant(
+        "workload",
+        default="cylinder",
+        values=("cylinder",),
+        description="workloads",
+    )
+
+    variant(
+        "version",
+        default="master",
+        values=("master",),
+        description="app version",
+    )
+
+    variant(
+        "fft",
+        default="fftw3",
+        description="Which FFT library to use",
+        values=("fftw3", "mkl", "kiss"),
+        multi=False,
+    )
+
+    variant(
+        "fft_kokkos",
+        default="fftw3",
+        description="FFT library for Kokkos package",
+        values=("kiss", "fftw3", "mkl", "hipfft", "cufft"),
+        multi=False,
+    )
+
+    # variant(
+    #     "gpu-aware-mpi",
+    #     default=True,
+    #     values=(True, False),
+    #     when=("+cuda" or "+rocm"),
+    #     description="Enable GPU-aware MPI",
+    # )
+
+    maintainers("rfhaque")
+
+    def compute_applications_section(self):
+        if self.spec.satisfies("exec_mode=test"):
+            L = 1
+            ppc = 32
+            stats = 10
+            run = 100
+        else:
+            L = 2  # will increase problem size by 4X
+            ppc = 64
+            stats = 10
+            run = 100
+
+        self.add_experiment_variable("L", L, True)
+        self.add_experiment_variable("ppc", ppc, True)
+        self.add_experiment_variable("stats", stats, True)
+        self.add_experiment_variable("run", run, True)
+
+        kokkos_mode = ""
+        if self.spec.satisfies("+openmp"):
+            kokkos_mode += "t {n_threads_per_proc}"
+        if self.spec.satisfies("+rocm") or self.spec.satisfies("+cuda"):
+            kokkos_mode += " g {n_gpus}"
+
+        self.add_experiment_variable(
+            "sparta_flags",
+            f"-k on {kokkos_mode} -sf kk",
+            False,
+        )
+
+        self.add_experiment_variable("n_resources", 1, False)
+
+        if self.spec.satisfies("+openmp"):
+            self.add_experiment_variable("n_threads_per_proc", 16, True)
+
+        if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"):
+            self.add_experiment_variable("n_gpus", "{n_resources}", True)
+        else:
+            self.add_experiment_variable("n_ranks", "{n_resources}", True)
+
+        if self.spec.satisfies("workload=cylinder"):
+            self.add_experiment_variable("in", "in.cylinder", False)
+
+        self.set_required_variables(
+            process_problem_size="{L}*{ppc}*1000",  # 1000 is a placeholder multiplier
+            total_problem_size="{process_problem_size}*{n_resources}",
+        )
+
+    def compute_package_section(self):
+        fft = self.spec.variants["fft"][0]
+        fft_kokkos = self.spec.variants["fft_kokkos"][0]
+        if self.spec.satisfies("+cuda"):
+            fft_kokkos = "cufft"
+        if self.spec.satisfies("+rocm"):
+            fft_kokkos = "hipfft"
+
+        self.add_package_spec(
+            self.name,
+            [
+                f"sparta-snl{self.determine_version()} +mpi+kokkos fft_kokkos={fft_kokkos} fft={fft} "
+            ],
+        )

repo/sparta-snl/application.py

@@ -0,0 +1,72 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import sys
+
+from ramble.appkit import *
+
+
+class SpartaSnl(ExecutableApplication):
+    """Define SNL Sparta application"""
+
+    name = "sparta-snl"
+
+    maintainers("rfhaque")
+
+    executable('sparta', 'spa_ -in {in} {sparta_flags}', use_mpi=True)
+
+    executable(
+        "copy",
+        template=["cp {problem_dir}/* {experiment_run_dir}/"],
+        use_mpi=False,
+    )
+    executable(
+        "set-size",
+        template=[
+            "sed -i -e 's/ppc equal .*/ppc equal {ppc}/g' -i {in}",
+            "sed -i -e 's/L equal .*/L equal {L}./g' -i {in}",
+            "sed -i -e 's/collide_modify .*/collide_modify      {collide_modify}/g' -i {in}",
+        ],
+        use_mpi=False,
+    )
+    executable(
+        "set-timesteps",
+        template=[
+            "sed 's/stats.*[0-9]\+/stats               {stats}/g' -i {in}",
+            "sed 's/run.*[0-9]\+/run                 {run}/g' -i {in}",
+        ],
+        use_mpi=False,
+    )
+
+    workload('cylinder', executables=["copy", "set-size", "set-timesteps", "sparta"])
+
+    workload_variable('problem_dir', default='{sparta-snl}/examples/cylinder',
+            description='problem dir',
+            workloads=['cylinder'])
+
+    workload_variable('in', default='in.cylinder',
+            description='input file',
+            workloads=['cylinder'])
+
+    workload_variable('L', default='1',
+            description='length scale factor',
+            workloads=['cylinder'])
+
+    workload_variable('ppc', default='32',
+            description='particles per cell',
+            workloads=['cylinder'])
+
+    workload_variable('collide_modify', default='vremax 100 yes vibrate no rotate smooth nearcp yes 10',
+            description='collisions config',
+            workloads=['cylinder'])
+
+    workload_variable('stats', default='10',
+            description='FOM interval',
+            workloads=['cylinder'])
+
+    workload_variable('run', default='100',
+            description='number of intervals',
+            workloads=['cylinder'])

repo/sparta-snl/package.py

@@ -0,0 +1,143 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+
+from spack.package import *
+
+
+class SpartaSnl(CMakePackage, CudaPackage, ROCmPackage):
+
+    homepage = "https://sparta.github.io/"
+    git = "https://github.com/sparta/sparta.git"
+
+    version("master", branch="master")
+    version("20Jan2025", tag="20Jan2025")
+
+    variant("mpi", default=True, description="Build with mpi")
+    variant("openmp", default=False, description="Enable OpenMP support")
+    variant("jpeg", default=False, description="Build with jpeg support")
+    variant("png", default=False, description="Build with png support")
+
+    variant(
+        "kokkos",
+        default=True,
+        description="Build with KOKKOS"
+    )
+
+    variant(
+        "fft",
+        default="fftw3",
+        description="Which FFT library to use",
+        values=("fftw3", "mkl", "kiss"),
+        multi=False,
+    )
+
+    variant(
+        "fft_kokkos",
+        default="fftw3",
+        when="+kokkos",
+        description="FFT library for Kokkos package",
+        values=("kiss", "fftw3", "mkl", "hipfft", "cufft"),
+        multi=False,
+    )
+
+    depends_on("mpi", when="+mpi")
+
+    depends_on("kokkos", when="+kokkos")
+    depends_on("kokkos+openmp cxxstd=17", when="+openmp")
+    depends_on("kokkos+rocm", when="+rocm")
+    depends_on("kokkos+wrapper+cuda cxxstd=17", when="+cuda")
+
+    depends_on("jpeg", when="+jpeg")
+    depends_on("libpng", when="+png")
+
+    depends_on("fftw-api@3", when="fft=fftw3")
+    depends_on("mkl", when="fft=mkl")
+
+    depends_on("hipfft", when="+kokkos+rocm fft_kokkos=hipfft")
+    depends_on("fftw-api@3", when="+kokkos fft_kokkos=fftw3")
+    depends_on("mkl", when="+kokkos fft_kokkos=mkl")
+    depends_on("kokkos+shared@3.1:", when="+kokkos")
+
+    for arch in CudaPackage.cuda_arch_values:
+        depends_on("kokkos+cuda cuda_arch=%s" % arch, when="+kokkos+cuda cuda_arch=%s" % arch)
+
+    for arch in ROCmPackage.amdgpu_targets:
+        depends_on(
+            "kokkos+rocm amdgpu_target=%s" % arch, when="+kokkos+rocm amdgpu_target=%s" % arch
+        )
+
+    conflicts("+rocm", when="+cuda")
+    conflicts("+cuda", when="+rocm")
+
+    def setup_run_environment(self, env):
+        if self.compiler.extra_rpaths:
+            for rpath in self.compiler.extra_rpaths:
+                env.prepend_path("LD_LIBRARY_PATH", rpath)
+
+    def setup_build_environment(self, env):
+        if "+cuda" in self.spec:
+            env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler")
+
+    root_cmakelists_dir = "cmake"
+
+    def flag_handler(self, name, flags):
+        wrapper_flags = []
+        build_system_flags = []
+
+        if self.spec.satisfies("+mpi+cuda") or self.spec.satisfies("+mpi+rocm"):
+            if self.spec.satisfies("^[virtuals=mpi] cray-mpich"):
+                gtl_lib = self.spec["cray-mpich"].package.gtl_lib
+                build_system_flags.extend(gtl_lib.get(name) or [])
+            # hipcc is not wrapped, we need to pass the flags via the build
+            # system.
+            build_system_flags.extend(flags)
+        else:
+            wrapper_flags.extend(flags)
+
+        return (wrapper_flags, [], build_system_flags)
+
+    def cmake_args(self):
+        spec = self.spec
+
+        args = [
+            self.define("SPARTA_LIST_PKGS", False),
+            self.define("SPARTA_LIST_TPLS", False),
+            self.define("SPARTA_ENABLE_TESTING", False),
+            self.define("SPARTA_ENABLE_PARAVIEW_TESTING", False),
+            self.define("CMAKE_CXX_STANDARD", 17),
+        ]
+
+        if spec.satisfies("+mpi"):
+            args.append(self.define_from_variant("BUILD_MPI", "mpi"))
+            args.append(self.define("PKG_MPI_STUBS", False))
+            args.append(f"-DMPI_ROOT={spec['mpi'].prefix}")
+            args.append(f"-DMPI_CXX_LINK_FLAGS='{spec['mpi'].libs.ld_flags}'")
+            args.append(f"-DMPI_C_COMPILER={spec['mpi'].mpicc}")
+            args.append(f"-DMPI_CXX_COMPILER={spec['mpi'].mpicxx}")
+
+        if spec.satisfies("+jpeg"):
+            args.append(self.define_from_variant("BUILD_JPEG", "jpeg"))
+            args.append(f"-DJPEG_ROOT='{spec['jpeg'].prefix}'")
+        if spec.satisfies("+png"):
+            args.append(self.define_from_variant("BUILD_PNG", "png"))
+            args.append(f"-DPNG_ROOT='{spec['libpng'].prefix}'")
+
+        args.append(self.define("PKG_FFT", True))
+        args.append(f"-DFFT={spec.variants['fft'].value.upper()}")
+
+        if spec.satisfies("+kokkos"):
+            args.append(self.define("PKG_KOKKOS", True))
+            args.append(self.define("USE_EXTERNAL_KOKKOS", True))
+            args.append(f"-DFFT_KOKKOS={spec.variants['fft_kokkos'].value.upper()}")
+        else:
+            args.append(self.define("PKG_KOKKOS", False))
+
+        return args
+
+    def install(self, spec, prefix):
+        super().install(spec, prefix)
+        install_tree("examples", prefix.examples)