diff --git a/.gitignore b/.gitignore index a104b6781..067f6d170 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__ _build spack.lock .spack-env +.env diff --git a/.gitlab/ci/test.yml b/.gitlab/ci/test.yml index aa4e8a102..ee6a73019 100644 --- a/.gitlab/ci/test.yml +++ b/.gitlab/ci/test.yml @@ -4,24 +4,35 @@ - HOST: tioga ARCHCONFIG: llnl-elcapitan SCHEDULER_PARAMETERS: -N 1 -t 20m - BENCHMARK: [amg2023, kripke, saxpy] - VARIANT: [rocm] + BENCHMARK: [amg2023, kripke, laghos] + VARIANT: [+rocm] + DASHBOARD_NAME: "Tioga HPECray-zen3-MI250X-Slingshot [benchpark system init --dest=tioga-system llnl-elcapitan cluster=tioga]" - HOST: dane ARCHCONFIG: llnl-cluster SCHEDULER_PARAMETERS: -N 1 -t 00:20:00 - BENCHMARK: [amg2023, kripke, saxpy] - VARIANT: [openmp] + BENCHMARK: [hpl, hpcg, amg2023, kripke] + VARIANT: [+openmp] + DASHBOARD_NAME: "Dane DELL-sapphirerapids-OmniPath [benchpark system init --dest=dane-system llnl-cluster cluster=dane]" - HOST: ruby ARCHCONFIG: llnl-cluster SCHEDULER_PARAMETERS: -N 1 -t 00:20:00 - BENCHMARK: [amg2023, kripke, saxpy] - VARIANT: [openmp] + BENCHMARK: [hpl, hpcg, amg2023, kripke] + VARIANT: [+openmp] + DASHBOARD_NAME: "Ruby Supermicro-icelake-OmniPath [benchpark system init --dest=ruby-system llnl-cluster cluster=ruby]" + - HOST: ruby + ARCHCONFIG: llnl-cluster + SCHEDULER_PARAMETERS: -N 1 -t 00:20:00 + BENCHMARK: [laghos] + VARIANT: [""] + DASHBOARD_NAME: "Ruby Supermicro-icelake-OmniPath [benchpark system init --dest=ruby-system llnl-cluster cluster=ruby]" .push_status: &push_status + # Max job name length 140 characters - | + TRUNCATED_JOB_NAME=$(echo "${CI_JOB_NAME}" | cut -c1-140) curl -X POST --url "https://api.github.com/repos/llnl/${CI_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \ --header 'Content-Type: application/json' \ --header "authorization: Bearer ${GITHUB_TOKEN}" \ - --data "{ \"state\": \"${pipeline_status}\", \"target_url\": \"${CI_JOB_URL}\", \"description\": \"${CI_JOB_NAME}\", \"context\": \"ci/gitlab/${CI_JOB_NAME}\" }" + --data "{ \"state\": \"${pipeline_status}\", \"target_url\": \"${CI_JOB_URL}\", \"description\": \"${TRUNCATED_JOB_NAME}\", \"context\": \"ci/gitlab/${CI_JOB_NAME}\" }" .report_status: &report_status before_script: - export pipeline_status="pending" @@ -30,9 +41,18 @@ - | if [[ "$CI_JOB_STATUS" == "failed" ]]; then export pipeline_status="failure" + export EXIT_CODE=false else export pipeline_status="$CI_JOB_STATUS" + export EXIT_CODE=true fi + ctest -S CTestGitlab.cmake \ + -DHOST="${HOST}" \ + -DDASHBOARD_NAME="${DASHBOARD_NAME}" \ + -DBUILD_NAME="${HOST}/${BENCHMARK}${VARIANT}" \ + -DSITE="$(hostname)" \ + -DTEST_TYPE="${TEST_TYPE}" \ + - *push_status workflow: auto_cancel: @@ -65,7 +85,7 @@ test_run: fi ./bin/benchpark system init --dest=${HOST}-system ${ARCHCONFIG} cluster=$HOST $EXTRA_ARGS # Initialize Experiment - - ./bin/benchpark experiment init --dest=${BENCHMARK}-benchmark ${BENCHMARK}+${VARIANT} + - ./bin/benchpark experiment init --dest=${BENCHMARK}-benchmark ${BENCHMARK}${VARIANT} # Build Workspace - ./bin/benchpark setup ${BENCHMARK}-benchmark ${HOST}-system workspace/ # Setup Ramble & Spack diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..a7da425d6 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required(VERSION 3.10) +project(PythonTestProject LANGUAGES NONE) +ENABLE_TESTING() +INCLUDE(CTest) +find_package(Python3 REQUIRED) + +add_test(NAME Gitlab + COMMAND ${CMAKE_COMMAND} -E $ENV{EXIT_CODE}) diff --git a/CTestConfig.cmake b/CTestConfig.cmake new file mode 100644 index 000000000..5bce0fa6f --- /dev/null +++ b/CTestConfig.cmake @@ -0,0 +1,23 @@ +## This file should be placed in the root directory of your project. +## Then modify the CMakeLists.txt file in the root directory of your +## project to incorporate the testing dashboard. +## +## The following are required to submit to the CDash dashboard: +#ENABLE_TESTING() +#INCLUDE(CTest) + +set(CTEST_PROJECT_NAME benchpark) +set(CTEST_NIGHTLY_START_TIME 01:00:00 UTC) + +if(CMAKE_VERSION VERSION_GREATER 3.14) + set(CTEST_SUBMIT_URL https://my.cdash.org/submit.php?project=Benchpark) +else() + set(CTEST_DROP_METHOD "https") + set(CTEST_DROP_SITE "my.cdash.org") + set(CTEST_DROP_LOCATION "/submit.php?project=Benchpark") +endif() + +set(CTEST_DROP_SITE_CDASH TRUE) + +# CDash token expected in ENV +set(_auth_token $ENV{CDASH_AUTH_TOKEN}) diff --git a/CTestGitlab.cmake b/CTestGitlab.cmake new file mode 100644 index 000000000..0698b28e2 --- /dev/null +++ b/CTestGitlab.cmake @@ -0,0 +1,27 @@ +set(CTEST_SOURCE_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}") +set(CTEST_BINARY_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}") +set(CTEST_OUTPUT_ON_FAILURE ON) +set(CTEST_CMAKE_GENERATOR "Unix Makefiles") + +set(CTEST_UPDATE_COMMAND "git") +set(CTEST_UPDATE_VERSION_ONLY 1) + +set(CTEST_SITE ${SITE}) +set(CTEST_BUILD_NAME ${BUILD_NAME}) + +# Separate dashboards for Nightly testing and PRs +if ("${TEST_TYPE}" STREQUAL "Nightly") + ctest_start("Nightly" GROUP "Nightly [benchpark-develop]") +else() + ctest_start("Continuous" GROUP "${DASHBOARD_NAME}") +endif() + +ctest_update() +ctest_configure() +ctest_test(INCLUDE "Gitlab") + +# Submit results +ctest_submit( + PARTS Update Test # 'Configure' and 'Build' not uploaded + HTTPHEADER "Authorization: Bearer ${_auth_token}" +)