CI and build fixes

.ci-scripts/build_and_test.sh

@@ -261,8 +261,8 @@ then
 
     timed_message "Run regression tests"
 
-    mpi_exe=`grep 'MPIEXEC_EXECUTABLE' ${hostconfig_path} | cut -d'"' -f2`
-    pytest -v -s regression_tests --mpi-exec=${mpi_exe}
+    mpi_exe=$(grep 'MPIEXEC_EXECUTABLE' "${hostconfig_path}" | cut -d'"' -f2 | sed 's/;/ /g')
+    pytest -v -s regression_tests --mpi-exec="${mpi_exe}"
 
     timed_message "Quandary tests completed"
 fi

.github/workflows/test.yml

@@ -0,0 +1,59 @@
+name: Build and Test
+
+on:
+  pull_request:
+    branches: ['main']
+  push:
+    branches: ['main']
+
+jobs:
+  build-and-test:
+    strategy:
+      matrix:
+        os: [macos-latest, ubuntu-24.04]
+    runs-on: ${{ matrix.os }}
+    permissions:
+      packages: write
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Set up Spack
+      uses: spack/setup-spack@v2
+
+    - name: Configure Spack
+      run: |
+        spack -e .spack_env mirror set binary_mirror --unsigned
+        spack mirror add --type binary --unsigned --oci-username GITHUB_USER --oci-password-variable GITHUB_TOKEN local-buildcache oci://ghcr.io/LLNL/quandary-spack-buildcache
+
+    - name: Install
+      # explicitly use openmpi, since mpich causes rpath problems in binary
+      run: |
+        spack -e .spack_env rm quandary
+        spack -e .spack_env add quandary@develop+test^openmpi
+        spack -e .spack_env install --use-buildcache package:never
+
+    - name: Run regression tests
+      shell: spack-sh {0}
+      run: |
+        spack env activate .spack_env
+        python -m pip install -e . --prefer-binary
+        pytest -v -s regression_tests --mpi-opt="--oversubscribe"
+
+    - name: Push packages and update index
+      env:
+        GITHUB_USER: ${{ github.actor }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        DOCKER_DEFAULT_PLATFORM: linux/amd64
+      run: |
+        if [[ "${{ matrix.os }}" == "ubuntu-24.04" ]]; then
+          # Force push quandary package
+          spack -e .spack_env buildcache push -f --base-image ubuntu:24.04 --update-index local-buildcache quandary
+          spack -e .spack_env buildcache push --base-image ubuntu:24.04 --update-index local-buildcache
+        else
+          spack -e .spack_env buildcache push --update-index local-buildcache
+        fi
+      if: ${{ !cancelled() }}

.gitlab/custom-jobs-and-variables.yml

@@ -39,7 +39,7 @@ variables:
 # Project specific variants for lassen
   PROJECT_LASSEN_VARIANTS: "+test"
 # Project specific deps for lassen
-  PROJECT_LASSEN_DEPS: ""
+  PROJECT_LASSEN_DEPS: "^netlib-lapack@3.9.0"
 
 # Configuration shared by build and test jobs specific to this project.
 # Not all configuration can be shared. Here projects can fine tune the

.gitlab/jobs/lassen.yml

@@ -13,13 +13,3 @@
 ########################
 # Overridden shared jobs
 ########################
-# We duplicate the shared jobs description and add necessary changes for RAJA.
-# We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
-# the comparison with the original job is easier.
-
-gcc_11_2_1_cuda_11_8_0:
-  variables:
-    # Force petsc to use same compiler
-    SPEC: "${PROJECT_LASSEN_VARIANTS} +cuda %gcc@=11.2.1 ^cuda@11.8.0+allow-unsupported-compilers ${PROJECT_LASSEN_DEPS} ^petsc%gcc@=11.2.1"
-    MODULE_LIST: "cuda/11.8.0"
-  extends: [.job_on_lassen]

.gitlab/jobs/ruby.yml

@@ -17,3 +17,9 @@ oneapi_2023_2_1:
   variables:
     ON_RUBY: "OFF"
   extends: .job_on_ruby
+
+# Force mvapich2 to be built with gcc
+gcc_10_3_1:
+  variables:
+    SPEC: "${PROJECT_RUBY_VARIANTS} %gcc@=10.3.1 ${PROJECT_RUBY_DEPS} ^mvapich2%gcc@=10.3.1"
+  extends: .job_on_ruby

CMakeLists.txt

@@ -11,6 +11,29 @@ set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
 set(ENABLE_MPI ON CACHE BOOL "")
 
-include(blt/SetupBLT.cmake)
+################################
+# BLT
+################################
+if (NOT BLT_LOADED)
+  if (DEFINED BLT_SOURCE_DIR)
+    if (NOT EXISTS ${BLT_SOURCE_DIR}/SetupBLT.cmake)
+        message(FATAL_ERROR "Given BLT_SOURCE_DIR does not contain SetupBLT.cmake")
+    endif()
+  else ()
+    set (BLT_SOURCE_DIR ${PROJECT_SOURCE_DIR}/blt CACHE PATH "")
+
+    if (NOT EXISTS ${BLT_SOURCE_DIR}/SetupBLT.cmake)
+      message(FATAL_ERROR
+          "The BLT git submodule is not present. "
+          "Either run the following two commands in your git repository: \n"
+          "    git submodule init\n"
+          "    git submodule update\n"
+          "Or add -DBLT_SOURCE_DIR=/path/to/blt to your CMake command." )
+    endif ()
+  endif ()
+
+  message(STATUS "BLT Source Directory: ${BLT_SOURCE_DIR}")
+  include(${BLT_SOURCE_DIR}/SetupBLT.cmake)
+endif()
 
 add_subdirectory(src)

regression_tests/regression_test.py

@@ -53,7 +53,8 @@ def test_eval(test_case: Case, request):
 def run_test(simulation_dir, number_of_processes, config_file, files_to_compare, exact, mpi_exec, mpi_opt):
     os.chdir(simulation_dir)
 
-    command = [mpi_exec, "-n", str(number_of_processes)]
+    command = mpi_exec.split()
+    command.extend(["-n", str(number_of_processes)])
     if mpi_opt:
         command.extend([mpi_opt])
     command.extend([QUANDARY_PATH, config_file])

src/CMakeLists.txt

@@ -25,9 +25,6 @@ find_package(PkgConfig REQUIRED)
 pkg_search_module(PETSC REQUIRED IMPORTED_TARGET PETSc)
 target_link_libraries(quandary_lib PUBLIC PkgConfig::PETSC)
 
-# Other libraries
-target_link_libraries(quandary_lib PUBLIC m blas lapack)
-
 # Put executable in root directory
 set_target_properties(
     quandary