From 86a4c491f85404689b32ae83f339dd950d70b155 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 12 Mar 2025 16:30:43 -0700 Subject: [PATCH 01/21] Update radiuss-spack-config commit --- .ci-scripts/radiuss-spack-configs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-scripts/radiuss-spack-configs b/.ci-scripts/radiuss-spack-configs index bea38964..8d24e6dc 160000 --- a/.ci-scripts/radiuss-spack-configs +++ b/.ci-scripts/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit bea389645c6878e3e2118a2107b47d9b7fa154de +Subproject commit 8d24e6dcef097ebafbf834d9e46e71e6b19e18a4 From b75c0164807174ebdd7fdbbe68164722e734943f Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 17 Mar 2025 15:59:54 -0600 Subject: [PATCH 02/21] Remove unneeded link command --- src/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4cf17f31..7436e76d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -25,9 +25,6 @@ find_package(PkgConfig REQUIRED) pkg_search_module(PETSC REQUIRED IMPORTED_TARGET PETSc) target_link_libraries(quandary_lib PUBLIC PkgConfig::PETSC) -# Other libraries -target_link_libraries(quandary_lib PUBLIC m blas lapack) - # Put executable in root directory set_target_properties( quandary From dbc5c2e622a0f394b52134a37d060e4cf4c7e0ef Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 17 Mar 2025 17:10:59 -0600 Subject: [PATCH 03/21] Either load BLT from CMakeCache or submodule --- CMakeLists.txt | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b5bfbf36..c66b0267 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,29 @@ set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") set(ENABLE_MPI ON CACHE BOOL "") -include(blt/SetupBLT.cmake) +################################ +# BLT +################################ +if (NOT BLT_LOADED) + if (DEFINED BLT_SOURCE_DIR) + if (NOT EXISTS ${BLT_SOURCE_DIR}/SetupBLT.cmake) + message(FATAL_ERROR "Given BLT_SOURCE_DIR does not contain SetupBLT.cmake") + endif() + else () + set (BLT_SOURCE_DIR ${PROJECT_SOURCE_DIR}/blt CACHE PATH "") + + if (NOT EXISTS ${BLT_SOURCE_DIR}/SetupBLT.cmake) + message(FATAL_ERROR + "The BLT git submodule is not present. " + "Either run the following two commands in your git repository: \n" + " git submodule init\n" + " git submodule update\n" + "Or add -DBLT_SOURCE_DIR=/path/to/blt to your CMake command." ) + endif () + endif () + + message(STATUS "BLT Source Directory: ${BLT_SOURCE_DIR}") + include(${BLT_SOURCE_DIR}/SetupBLT.cmake) +endif() add_subdirectory(src) From 219233fd539df84b38422b56aa4a3f27e91bcc7d Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 17 Mar 2025 17:16:54 -0600 Subject: [PATCH 04/21] Update radiuss-spack-configs quandary spack package --- .ci-scripts/radiuss-spack-configs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-scripts/radiuss-spack-configs b/.ci-scripts/radiuss-spack-configs index 8d24e6dc..231631a1 160000 --- a/.ci-scripts/radiuss-spack-configs +++ b/.ci-scripts/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 8d24e6dcef097ebafbf834d9e46e71e6b19e18a4 +Subproject commit 231631a104a22d0832321c029734861d7de1be44 From fad0d7008e12346bc42333c28eb2d8499f6c9554 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 17 Mar 2025 18:10:33 -0600 Subject: [PATCH 05/21] Remove CI shared job workaround --- .gitlab/jobs/lassen.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.gitlab/jobs/lassen.yml b/.gitlab/jobs/lassen.yml index 6fc2951a..487ea802 100644 --- a/.gitlab/jobs/lassen.yml +++ b/.gitlab/jobs/lassen.yml @@ -13,13 +13,3 @@ ######################## # Overridden shared jobs ######################## -# We duplicate the shared jobs description and add necessary changes for RAJA. -# We keep ${PROJECT__VARIANTS} and ${PROJECT__DEPS} So that -# the comparison with the original job is easier. - -gcc_11_2_1_cuda_11_8_0: - variables: - # Force petsc to use same compiler - SPEC: "${PROJECT_LASSEN_VARIANTS} +cuda %gcc@=11.2.1 ^cuda@11.8.0+allow-unsupported-compilers ${PROJECT_LASSEN_DEPS} ^petsc%gcc@=11.2.1" - MODULE_LIST: "cuda/11.8.0" - extends: [.job_on_lassen] From c4bc6cb5c584ce4dc28d1973a7a04be6bad34189 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 17 Mar 2025 18:58:10 -0600 Subject: [PATCH 06/21] Update radiuss-spack-configs netlib-lapack --- .ci-scripts/radiuss-spack-configs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-scripts/radiuss-spack-configs b/.ci-scripts/radiuss-spack-configs index 231631a1..599e44f6 160000 --- a/.ci-scripts/radiuss-spack-configs +++ b/.ci-scripts/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 231631a104a22d0832321c029734861d7de1be44 +Subproject commit 599e44f64c7f64d5f36f9a344d635679a55dc5a8 From 81d5a5433f1230109af5af4174760a9910075be2 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 18 Mar 2025 14:58:32 -0600 Subject: [PATCH 07/21] Force use of working version of netlib-lapack in CI --- .gitlab/custom-jobs-and-variables.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index 91307cec..58cdd68c 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -39,7 +39,7 @@ variables: # Project specific variants for lassen PROJECT_LASSEN_VARIANTS: "+test" # Project specific deps for lassen - PROJECT_LASSEN_DEPS: "" + PROJECT_LASSEN_DEPS: "^netlib-lapack@3.9.0" # Configuration shared by build and test jobs specific to this project. # Not all configuration can be shared. Here projects can fine tune the From 0fba338dc6a4790d986c1ef86bcf11195a9b4c08 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 19 Mar 2025 14:57:24 -0600 Subject: [PATCH 08/21] Update CI script and tests to account for the case that mpiexec is flux run --- .ci-scripts/build_and_test.sh | 4 ++-- regression_tests/regression_test.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.ci-scripts/build_and_test.sh b/.ci-scripts/build_and_test.sh index efe29fac..c126592d 100755 --- a/.ci-scripts/build_and_test.sh +++ b/.ci-scripts/build_and_test.sh @@ -261,8 +261,8 @@ then timed_message "Run regression tests" - mpi_exe=`grep 'MPIEXEC_EXECUTABLE' ${hostconfig_path} | cut -d'"' -f2` - pytest -v -s regression_tests --mpi-exec=${mpi_exe} + mpi_exe=$(grep 'MPIEXEC_EXECUTABLE' "${hostconfig_path}" | cut -d'"' -f2 | sed 's/;/ /g') + pytest -v -s regression_tests --mpi-exec="${mpi_exe}" timed_message "Quandary tests completed" fi diff --git a/regression_tests/regression_test.py b/regression_tests/regression_test.py index d56946a6..88934012 100644 --- a/regression_tests/regression_test.py +++ b/regression_tests/regression_test.py @@ -53,7 +53,8 @@ def test_eval(test_case: Case, request): def run_test(simulation_dir, number_of_processes, config_file, files_to_compare, exact, mpi_exec, mpi_opt): os.chdir(simulation_dir) - command = [mpi_exec, "-n", str(number_of_processes)] + command = mpi_exec.split() + command.extend(["-n", str(number_of_processes)]) if mpi_opt: command.extend([mpi_opt]) command.extend([QUANDARY_PATH, config_file]) From 85dd85e5b5063d035b3b929c0aa7317983c10324 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 17 Mar 2025 17:16:21 -0600 Subject: [PATCH 09/21] Require mpi on ruby to use the same compiler as quandary --- .gitlab/jobs/ruby.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab/jobs/ruby.yml b/.gitlab/jobs/ruby.yml index 103f716a..246446fb 100644 --- a/.gitlab/jobs/ruby.yml +++ b/.gitlab/jobs/ruby.yml @@ -17,3 +17,9 @@ oneapi_2023_2_1: variables: ON_RUBY: "OFF" extends: .job_on_ruby + +# Force mvapich2 to be built with gcc +gcc_10_3_1: + variables: + SPEC: "${PROJECT_RUBY_VARIANTS} %gcc@=10.3.1 ${PROJECT_RUBY_DEPS} ^mvapich2%gcc@=10.3.1" + extends: .job_on_ruby From 913af977b7816359ebb43c337953e2f106ec0538 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 21 Mar 2025 14:37:58 -0600 Subject: [PATCH 10/21] Add linux github workflow job --- .github/workflows/{test-macos.yml => test.yml} | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) rename .github/workflows/{test-macos.yml => test.yml} (68%) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test.yml similarity index 68% rename from .github/workflows/test-macos.yml rename to .github/workflows/test.yml index 170a2e34..000c05c5 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Build and Test on MacOS +name: Build and Test on: pull_request: @@ -6,7 +6,10 @@ on: jobs: build-and-test: - runs-on: macos-latest + strategy: + matrix: + os: [macos-latest, ubuntu-24.04] + runs-on: ${{ matrix.os }} permissions: packages: write @@ -21,7 +24,7 @@ jobs: - name: Configure Spack run: | - spack compiler find + spack -e .spack_env mirror set binary_mirror --unsigned spack mirror add --type binary --unsigned --oci-username GITHUB_USER --oci-password-variable GITHUB_TOKEN local-buildcache oci://ghcr.io/tdrwenski/spack-buildcache - name: Install @@ -40,5 +43,10 @@ jobs: env: GITHUB_USER: ${{ github.actor }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: spack -e .spack_env buildcache push --update-index local-buildcache + run: | + if [[ "${{ matrix.os }}" == "ubuntu-24.04" ]]; then + spack -e .spack_env buildcache push --base-image ubuntu:24.04 --update-index local-buildcache + else + spack -e .spack_env buildcache push --update-index local-buildcache + fi if: ${{ !cancelled() }} From 4f0bb8935acd045028da9cec983a61164da6cf76 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 27 Mar 2025 10:02:57 -0600 Subject: [PATCH 11/21] Never use quandary from buildcache --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 000c05c5..94a9e6b2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,7 @@ jobs: - name: Install run: | - spack -e .spack_env install + spack -e .spack_env install --use-buildcache package:never - name: Run regression tests shell: spack-sh {0} From a9084ad7808eb646357709755ff414a24695e27e Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 27 Mar 2025 11:20:32 -0600 Subject: [PATCH 12/21] Also run tests on main --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 94a9e6b2..bae06715 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,6 +3,8 @@ name: Build and Test on: pull_request: branches: ['main'] + push: + branches: ['main'] jobs: build-and-test: From a35b3f07bac80bf4db4ad867c48826374461d015 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 27 Mar 2025 11:20:58 -0600 Subject: [PATCH 13/21] Force push quandary to cache --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bae06715..ee50689d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,6 +47,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | if [[ "${{ matrix.os }}" == "ubuntu-24.04" ]]; then + # Force push quandary package + spack -e .spack_env buildcache push -f --base-image ubuntu:24.04 --update-index --only package local-buildcache spack -e .spack_env buildcache push --base-image ubuntu:24.04 --update-index local-buildcache else spack -e .spack_env buildcache push --update-index local-buildcache From 9a63c1e835824a63356c9409b3860146212c467c Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 27 Mar 2025 11:21:52 -0600 Subject: [PATCH 14/21] Remove unnecessary cp since quandary is always rebuilt --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ee50689d..55c561b8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,6 @@ jobs: shell: spack-sh {0} run: | spack env activate .spack_env - cp $(which quandary) . python -m pip install -e . --prefer-binary pytest -v -s regression_tests --mpi-opt="--oversubscribe" From cf175664007c6685c4369e9aa2143766983226b3 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 28 Mar 2025 16:50:55 -0600 Subject: [PATCH 15/21] Explicitly ask for openmpi so that mpich does not get used (avoid rpath issue) --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 55c561b8..ef48ceae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,7 +30,10 @@ jobs: spack mirror add --type binary --unsigned --oci-username GITHUB_USER --oci-password-variable GITHUB_TOKEN local-buildcache oci://ghcr.io/tdrwenski/spack-buildcache - name: Install + # explicitly use openmpi and mpich cases rpath problems in binary run: | + spack -e .spack_env rm quandary + spack -e .spack_env add quandary@develop+test^openmpi spack -e .spack_env install --use-buildcache package:never - name: Run regression tests From 6683ed6b7b60c5287f956c4dc98c8ceaf649ee46 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 1 Apr 2025 18:06:39 -0600 Subject: [PATCH 16/21] Add Docker platform to spack buildcache images --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef48ceae..4b364719 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,6 +47,7 @@ jobs: env: GITHUB_USER: ${{ github.actor }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKER_DEFAULT_PLATFORM: linux/amd64 run: | if [[ "${{ matrix.os }}" == "ubuntu-24.04" ]]; then # Force push quandary package From b604da478df9633db1513211f1a8182bbc5092f9 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 1 Apr 2025 18:08:22 -0600 Subject: [PATCH 17/21] Force push linux buildcache packages to ensure dockerized quandary has dependencies --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4b364719..4e205106 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,8 +51,7 @@ jobs: run: | if [[ "${{ matrix.os }}" == "ubuntu-24.04" ]]; then # Force push quandary package - spack -e .spack_env buildcache push -f --base-image ubuntu:24.04 --update-index --only package local-buildcache - spack -e .spack_env buildcache push --base-image ubuntu:24.04 --update-index local-buildcache + spack -e .spack_env buildcache push -f --base-image ubuntu:24.04 --update-index local-buildcache else spack -e .spack_env buildcache push --update-index local-buildcache fi From f5a094bd1a657deb60114679ae58e80ef52ab25e Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 1 Apr 2025 18:12:40 -0600 Subject: [PATCH 18/21] Use LLNL ghcr for spack buildcache --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4e205106..0ad723a1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,7 +27,7 @@ jobs: - name: Configure Spack run: | spack -e .spack_env mirror set binary_mirror --unsigned - spack mirror add --type binary --unsigned --oci-username GITHUB_USER --oci-password-variable GITHUB_TOKEN local-buildcache oci://ghcr.io/tdrwenski/spack-buildcache + spack mirror add --type binary --unsigned --oci-username GITHUB_USER --oci-password-variable GITHUB_TOKEN local-buildcache oci://ghcr.io/LLNL/quandary-spack-buildcache - name: Install # explicitly use openmpi and mpich cases rpath problems in binary From 35769e5f1473d3b8448847ccc0644a5bb423735a Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 1 Apr 2025 18:14:40 -0600 Subject: [PATCH 19/21] Fix comment --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0ad723a1..18827426 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,7 +30,7 @@ jobs: spack mirror add --type binary --unsigned --oci-username GITHUB_USER --oci-password-variable GITHUB_TOKEN local-buildcache oci://ghcr.io/LLNL/quandary-spack-buildcache - name: Install - # explicitly use openmpi and mpich cases rpath problems in binary + # explicitly use openmpi, since mpich causes rpath problems in binary run: | spack -e .spack_env rm quandary spack -e .spack_env add quandary@develop+test^openmpi From 6e6e6aa3ef5417a6578b156097901b83a37d51ae Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 2 Apr 2025 09:17:24 -0600 Subject: [PATCH 20/21] Update radiuss-spack-configs version --- .ci-scripts/radiuss-spack-configs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-scripts/radiuss-spack-configs b/.ci-scripts/radiuss-spack-configs index 599e44f6..aabd0d39 160000 --- a/.ci-scripts/radiuss-spack-configs +++ b/.ci-scripts/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 599e44f64c7f64d5f36f9a344d635679a55dc5a8 +Subproject commit aabd0d39c4003a0be01239913212d54ac321a58b From 175e13e0220a4eefa8f77023664a10af4e93e69f Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 2 Apr 2025 13:13:23 -0600 Subject: [PATCH 21/21] Fix workflow buildcache push to only force push quandary and not other dependencies --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 18827426..d26e1c16 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,7 +51,8 @@ jobs: run: | if [[ "${{ matrix.os }}" == "ubuntu-24.04" ]]; then # Force push quandary package - spack -e .spack_env buildcache push -f --base-image ubuntu:24.04 --update-index local-buildcache + spack -e .spack_env buildcache push -f --base-image ubuntu:24.04 --update-index local-buildcache quandary + spack -e .spack_env buildcache push --base-image ubuntu:24.04 --update-index local-buildcache else spack -e .spack_env buildcache push --update-index local-buildcache fi