CDash Integration (#730)

* Enable CDash for Gitlab test results

* Update comment

---------

Co-authored-by: Michael McKinsey <michaelmckinsey1@gmail.com>

Author: slabasan

.gitignore

@@ -5,3 +5,4 @@ __pycache__
 _build
 spack.lock
 .spack-env
+.env

.gitlab/ci/test.yml

@@ -4,24 +4,35 @@
       - HOST: tioga
         ARCHCONFIG: llnl-elcapitan
         SCHEDULER_PARAMETERS: -N 1 -t 20m
-        BENCHMARK: [amg2023, kripke, saxpy]
-        VARIANT: [rocm]
+        BENCHMARK: [amg2023, kripke, laghos]
+        VARIANT: [+rocm]
+        DASHBOARD_NAME: "Tioga HPECray-zen3-MI250X-Slingshot [benchpark system init --dest=tioga-system llnl-elcapitan cluster=tioga]"
       - HOST: dane
         ARCHCONFIG: llnl-cluster
         SCHEDULER_PARAMETERS: -N 1 -t 00:20:00
-        BENCHMARK: [amg2023, kripke, saxpy]
-        VARIANT: [openmp]
+        BENCHMARK: [hpl, hpcg, amg2023, kripke]
+        VARIANT: [+openmp]
+        DASHBOARD_NAME: "Dane DELL-sapphirerapids-OmniPath [benchpark system init --dest=dane-system llnl-cluster cluster=dane]"
       - HOST: ruby
         ARCHCONFIG: llnl-cluster
         SCHEDULER_PARAMETERS: -N 1 -t 00:20:00
-        BENCHMARK: [amg2023, kripke, saxpy]
-        VARIANT: [openmp]
+        BENCHMARK: [hpl, hpcg, amg2023, kripke]
+        VARIANT: [+openmp]
+        DASHBOARD_NAME: "Ruby Supermicro-icelake-OmniPath [benchpark system init --dest=ruby-system llnl-cluster cluster=ruby]"
+      - HOST: ruby
+        ARCHCONFIG: llnl-cluster
+        SCHEDULER_PARAMETERS: -N 1 -t 00:20:00
+        BENCHMARK: [laghos]
+        VARIANT: [""]
+        DASHBOARD_NAME: "Ruby Supermicro-icelake-OmniPath [benchpark system init --dest=ruby-system llnl-cluster cluster=ruby]"
 .push_status: &push_status
+  # Max job name length 140 characters
   - |
+    TRUNCATED_JOB_NAME=$(echo "${CI_JOB_NAME}" | cut -c1-140)
     curl -X POST --url "https://api.github.com/repos/llnl/${CI_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \
          --header 'Content-Type: application/json' \
          --header "authorization: Bearer ${GITHUB_TOKEN}" \
-         --data "{ \"state\": \"${pipeline_status}\", \"target_url\": \"${CI_JOB_URL}\", \"description\": \"${CI_JOB_NAME}\", \"context\": \"ci/gitlab/${CI_JOB_NAME}\" }"
+         --data "{ \"state\": \"${pipeline_status}\", \"target_url\": \"${CI_JOB_URL}\", \"description\": \"${TRUNCATED_JOB_NAME}\", \"context\": \"ci/gitlab/${CI_JOB_NAME}\" }"
 .report_status: &report_status
   before_script:
     - export pipeline_status="pending"
@@ -30,9 +41,18 @@
     - |
       if [[ "$CI_JOB_STATUS" == "failed" ]]; then
         export pipeline_status="failure"
+        export EXIT_CODE=false
       else
         export pipeline_status="$CI_JOB_STATUS"
+        export EXIT_CODE=true
       fi
+      ctest -S CTestGitlab.cmake \
+        -DHOST="${HOST}" \
+        -DDASHBOARD_NAME="${DASHBOARD_NAME}" \
+        -DBUILD_NAME="${HOST}/${BENCHMARK}${VARIANT}" \
+        -DSITE="$(hostname)" \
+        -DTEST_TYPE="${TEST_TYPE}" \
+
     - *push_status
 workflow:
   auto_cancel:
@@ -65,7 +85,7 @@ test_run:
       fi
       ./bin/benchpark system init --dest=${HOST}-system ${ARCHCONFIG} cluster=$HOST $EXTRA_ARGS
     # Initialize Experiment
-    - ./bin/benchpark experiment init --dest=${BENCHMARK}-benchmark ${BENCHMARK}+${VARIANT}
+    - ./bin/benchpark experiment init --dest=${BENCHMARK}-benchmark ${BENCHMARK}${VARIANT}
     # Build Workspace
     - ./bin/benchpark setup ${BENCHMARK}-benchmark ${HOST}-system workspace/
     # Setup Ramble & Spack

CMakeLists.txt

@@ -0,0 +1,8 @@
+cmake_minimum_required(VERSION 3.10)
+project(PythonTestProject LANGUAGES NONE)
+ENABLE_TESTING()
+INCLUDE(CTest)
+find_package(Python3 REQUIRED)
+
+add_test(NAME Gitlab
+        COMMAND ${CMAKE_COMMAND} -E $ENV{EXIT_CODE})

CTestConfig.cmake

@@ -0,0 +1,23 @@
+## This file should be placed in the root directory of your project.
+## Then modify the CMakeLists.txt file in the root directory of your
+## project to incorporate the testing dashboard.
+##
+## The following are required to submit to the CDash dashboard:
+#ENABLE_TESTING()
+#INCLUDE(CTest)
+
+set(CTEST_PROJECT_NAME benchpark)
+set(CTEST_NIGHTLY_START_TIME 01:00:00 UTC)
+
+if(CMAKE_VERSION VERSION_GREATER 3.14)
+  set(CTEST_SUBMIT_URL https://my.cdash.org/submit.php?project=Benchpark)
+else()
+  set(CTEST_DROP_METHOD "https")
+  set(CTEST_DROP_SITE "my.cdash.org")
+  set(CTEST_DROP_LOCATION "/submit.php?project=Benchpark")
+endif()
+
+set(CTEST_DROP_SITE_CDASH TRUE)
+
+# CDash token expected in ENV
+set(_auth_token $ENV{CDASH_AUTH_TOKEN})

CTestGitlab.cmake

@@ -0,0 +1,27 @@
+set(CTEST_SOURCE_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}")
+set(CTEST_BINARY_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}")
+set(CTEST_OUTPUT_ON_FAILURE ON)
+set(CTEST_CMAKE_GENERATOR "Unix Makefiles")
+
+set(CTEST_UPDATE_COMMAND "git")
+set(CTEST_UPDATE_VERSION_ONLY 1)
+
+set(CTEST_SITE ${SITE})
+set(CTEST_BUILD_NAME ${BUILD_NAME})
+
+# Separate dashboards for Nightly testing and PRs
+if ("${TEST_TYPE}" STREQUAL "Nightly")
+    ctest_start("Nightly" GROUP "Nightly [benchpark-develop]")
+else()
+    ctest_start("Continuous" GROUP "${DASHBOARD_NAME}")
+endif()
+
+ctest_update()
+ctest_configure()
+ctest_test(INCLUDE "Gitlab")
+
+# Submit results
+ctest_submit(
+    PARTS Update Test # 'Configure' and 'Build' not uploaded
+    HTTPHEADER "Authorization: Bearer ${_auth_token}"
+)