Merge branch 'develop' into 809-self-standard-hwloc-configs

Author: Amr Akmal Moustafa Abouelmagd

.github/actions/dynamic-dry-run/action.yml

@@ -0,0 +1,21 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+#!/bin/bash
+
+benchmark_spec="$1"
+system_spec="$2"
+
+timestamp=$(date +%s)
+benchmark="b-$timestamp"
+system="s-$timestamp"
+./bin/benchpark system init --dest=$system $system_spec
+./bin/benchpark experiment init --dest=$benchmark --system=$system $benchmark_spec
+./bin/benchpark setup ./$benchmark ./$system workspace/
+. workspace/setup.sh
+ramble \
+    --workspace-dir "workspace/$benchmark/$system/workspace" \
+    --disable-logger \
+    workspace setup --dry-run

.github/utils/dryrun.sh

@@ -0,0 +1,185 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import subprocess
+import time
+import sys
+import argparse
+import os
+
+import benchpark.paths
+
+sys.path.append(str(benchpark.paths.benchpark_home) + "/spack/lib/spack")
+from lib.benchpark.accounting import benchpark_experiments  # noqa: E402
+
+DEFAULT_SYSTEM = "llnl-cluster cluster=dane"
+# Skip experiments
+SKIP_EXPR = [
+    # System not enough cores/node
+    "gromacs+openmp aws-pcluster instance_type=c6g.xlarge",
+    "gromacs+openmp aws-pcluster instance_type=c4.xlarge",
+    "gromacs+openmp generic-x86",
+    "stream aws-pcluster instance_type=c6g.xlarge",
+    "stream aws-pcluster instance_type=c4.xlarge",
+    "stream cscs-daint",
+    "stream generic-x86",
+    # Broken URL's in application.py going to cause dryrun failure
+    "genesis",
+]
+
+
+def run_subprocess_cmd(cmd_list, decode=False):
+    try:
+        result = subprocess.run(cmd_list, capture_output=True, check=True)
+        return result.stdout.decode("utf-8") if decode else result
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError(
+            f"Command: {' '.join(cmd_list)}\nOutput: {e.stdout}\nError: {e.stderr}"
+        )
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--test",
+        choices=[
+            "mpi",
+            "cuda",
+            "rocm",
+            "openmp",
+            "strong",
+            "weak",
+            "throughput",
+            "modifiers",
+        ],
+        help="Only run tests of this type",
+    )
+    parser.add_argument(
+        "--dryrun", action="store_true", help="Dry runs this script for testing."
+    )
+    args = parser.parse_args()
+
+    expr_str = run_subprocess_cmd(
+        ["./bin/benchpark", "list", "experiments", "--no-title"], decode=True
+    )
+    experiments = [e for e in expr_str.replace(" ", "").replace("\t", "").split("\n") if e != ""]
+
+    mpi_only_expr = set()
+    cuda_expr = []
+    rocm_expr = []
+    openmp_expr = []
+    strong_expr = []
+    weak_expr = []
+    throughput_expr = []
+
+    for e in experiments:
+        if "scaling" in e:
+            e = e.replace("scaling=", " scaling=")
+        if "+strong" in e or "+weak" in e or "+throughput" in e:
+            e = e.replace(e, e + "~single_node")
+        elif "+" not in e and "=" not in e:
+            mpi_only_expr.add(e)
+
+        if "cuda" in e:
+            cuda_expr.append(e)
+        elif "rocm" in e:
+            rocm_expr.append(e)
+        elif "openmp" in e:
+            openmp_expr.append(e)
+        elif "strong" in e:
+            strong_expr.append(e)
+        elif "weak" in e:
+            weak_expr.append(e)
+        elif "throughput" in e:
+            throughput_expr.append(e)
+
+    str_dict = {}
+    for pmodel in ["mpi", "cuda", "rocm", "openmp"]:
+        cmd = ["./bin/benchpark", "list", "systems", "--no-title"]
+        if pmodel != "mpi":
+            cmd += ["-p", pmodel]
+        output = run_subprocess_cmd(cmd, decode=True)
+        str_dict[pmodel] = [
+            i
+            for i in output.replace(" " * 4, "").replace("\t", "").split("\n")
+            if i != ""
+        ]
+
+    mods_str = run_subprocess_cmd(
+        ["./bin/benchpark", "list", "modifiers", "--no-title"], decode=True
+    )
+    nmods = [
+        i
+        for i in mods_str.replace(" " * 4, "").replace("\t", "").split("\n")
+        if i != "" and i not in ["allocation", "caliper"]
+    ]
+
+    caliper_exp = [
+        e.replace("+caliper", " caliper=time") for e in benchpark_experiments(exclude_variants=[]) if "+caliper" in e and e.split("+")[0] in mpi_only_expr
+    ]
+    modifiers_expr = caliper_exp + [e + " " + m + "=on" for e in mpi_only_expr for m in nmods]
+
+    exprs_to_sys = [
+        ("mpi", mpi_only_expr, str_dict["mpi"]),
+        ("cuda", cuda_expr, str_dict["cuda"]),
+        ("rocm", rocm_expr, str_dict["rocm"]),
+        ("openmp", openmp_expr, str_dict["openmp"]),
+        ("strong", strong_expr, str_dict["mpi"]),
+        ("weak", weak_expr, str_dict["mpi"]),
+        ("throughput", throughput_expr, str_dict["mpi"]),
+        ("modifiers", modifiers_expr, [DEFAULT_SYSTEM]),
+    ]
+
+    if args.test:
+        exprs_to_sys = [tup for tup in exprs_to_sys if tup[0] == args.test]
+
+    total_tests = sum(
+        len(expr_spec_list) * len(sys_spec_list)
+        for _, expr_spec_list, sys_spec_list in exprs_to_sys
+    )
+    print(f"Total tests to run: {total_tests}")
+
+    start = time.time()
+    errors = {}
+    fail_tests = 0
+    ran_tests = 0
+    skip_tests = 0
+    for _, expr_spec_list, sys_spec_list in exprs_to_sys:
+        for espec in expr_spec_list:
+            for sspec in sys_spec_list:
+                expr = f"{espec} {sspec}"
+                if expr in SKIP_EXPR:
+                    skip_tests += 1
+                    print(f'Skipping "{expr}"')
+                    continue
+                ran_tests += 1
+                print(f'Running "{expr}"')
+                if args.dryrun:
+                    continue
+                try:
+                    subprocess.run(
+                        ["bash", ".github/utils/dryrun.sh", espec, sspec],
+                        env={**os.environ},
+                        capture_output=True,
+                        check=True
+                    )
+                except subprocess.CalledProcessError as e:
+                    errors[f"{espec} {sspec}"] = e.stderr.decode()
+                    fail_tests += 1
+    end = time.time()
+
+    for i, (key, value) in enumerate(errors.items()):
+        print("=" * 100)
+        print(str(i + 1) + ". " + key)
+        print(value)
+
+    print(f"Elapsed: {(end - start) / 60:.2f} minutes")
+    print(f"{ran_tests - fail_tests} Passing. {fail_tests} Failing. {skip_tests} Skipped.")
+
+    sys.exit(1 if fail_tests > 0 else 0)
+
+
+if __name__ == "__main__":
+    main()

.github/utils/dryruns.py

@@ -127,6 +127,11 @@ jobs:
     if: ${{ needs.changes.outputs.run == 'true' }}
     needs: changes
     uses: ./.github/workflows/run.yml
+    # 'workflow_call' actions do not get access to secrets
+    # normally. So, we add this (plus the corresponding)
+    # settings in coverage.yml to pass the token on
+    secrets:
+      BENCHPARK_CODECOV_TOKEN: ${{ secrets.BENCHPARK_CODECOV_TOKEN }}
   coverage:
     if: ${{ needs.changes.outputs.coverage == 'true' }}
     needs: changes

.github/workflows/ci.yml

@@ -18,7 +18,6 @@ jobs:
           ./bin/benchpark unit-test --cov=./ --cov-branch --cov-report=xml --durations=20 -ra
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
-        if: true
         with:
           token: ${{ secrets.BENCHPARK_CODECOV_TOKEN }}
           directory: ./coverage/reports