Add gitlab job for performance tests and github benchmark workflow for visualization

Author: tdrwenski

.ci-scripts/build_and_test.sh

@@ -29,6 +29,7 @@ use_dev_shm=${USE_DEV_SHM:-true}
 spack_debug=${SPACK_DEBUG:-false}
 debug_mode=${DEBUG_MODE:-false}
 push_to_registry=${PUSH_TO_REGISTRY:-true}
+performance_tests=${PERFORMANCE_TESTS:-false}
 
 # REGISTRY_TOKEN allows you to provide your own personal access token to the CI
 # registry. Be sure to set the token with at least read access to the registry.
@@ -223,7 +224,7 @@ then
 fi
 
 # Test
-if [[ "${option}" != "--build-only" ]]
+if [[ "${option}" != "--build-only" && "${performance_tests}" != "true" ]]
 then
 
     if [[ ! -d ${build_dir} ]]
@@ -267,6 +268,27 @@ then
     timed_message "Quandary tests completed"
 fi
 
+# Performance tests
+if [[ "${option}" != "--build-only" && "${performance_tests}" == "true" ]]
+then
+
+    cd ${project_dir}
+
+    timed_message "Performance tests for Quandary"
+
+    timed_message "Install python test dependencies"
+
+    eval `${spack_cmd} env activate ${spack_env_path} --sh`
+    python -m pip install -e . --prefer-binary
+
+    timed_message "Run performance tests"
+
+    mpi_exe=$(grep 'MPIEXEC_EXECUTABLE' "${hostconfig_path}" | cut -d'"' -f2 | sed 's/;/ /g')
+    pytest -v -s performance_tests --mpi-exec="${mpi_exe}" --benchmark-json=benchmark_results.json
+
+    timed_message "Quandary performance tests completed"
+fi
+
 cd ${project_dir}
 
 timed_message "Build and test completed"

.github/workflows/benchmark.yml

@@ -0,0 +1,33 @@
+name: Process Benchmark
+
+on:
+  workflow_dispatch:
+    inputs:
+      benchmark_data:
+        description: 'Base64 encoded benchmark data'
+        required: true
+  pull_request:  # TODO remove this later
+    branches:
+      - main
+
+jobs:
+  store-and-visualize:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Decode benchmark data
+        run: echo "${{ github.event.inputs.benchmark_data }}" | base64 -d > benchmark.json
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'pytest'
+          output-file-path: benchmark.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: dev/bench
+          comment-on-alert: true
+          alert-threshold: '120%'
+          max-items-in-chart: 100

.gitlab/custom-jobs-and-variables.yml

@@ -49,6 +49,24 @@ variables:
     reports:
       junit: junit.xml
 
+.performance_job:
+  artifacts:
+    paths:
+      - benchmark_results.json
+  after_script:
+    - |
+      if [ -f "benchmark_results.json" ]; then
+        echo "Sending benchmark results to GitHub..."
+        BENCHMARK_DATA=$(base64 -w 0 benchmark_results.json)
+        curl -X POST \
+        -H "Authorization: token $GITHUB_TOKEN" \
+        -H "Accept: application/vnd.github.v3+json" \
+        https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/actions/workflows/benchmark.yml/dispatches \
+        -d "{\"ref\":\"main\",\"inputs\":{\"benchmark_data\":\"$BENCHMARK_DATA\"}}"
+      else
+        echo "benchmark_results.json not found, skipping GitHub API integration."
+      fi
+
 .reproducer_vars:
   script:
     - |

.gitlab/jobs/ruby.yml

@@ -23,3 +23,14 @@ gcc_10_3_1:
   variables:
     SPEC: "${PROJECT_RUBY_VARIANTS} %gcc@=10.3.1 ${PROJECT_RUBY_DEPS} ^mvapich2%gcc@=10.3.1"
   extends: .job_on_ruby
+
+############
+# Extra jobs
+############
+performance_tests_clang_14_0_6:
+  variables:
+    SPEC: "${PROJECT_RUBY_VARIANTS} %clang@=14.0.6 ${PROJECT_RUBY_DEPS}"
+    PERFORMANCE_TESTS: "true"
+  extends:
+    - .job_on_ruby
+    - .performance_job