Update shared ci, spack configs and specs in the CI

.gitlab-ci.yml

@@ -47,6 +47,7 @@ variables:
 
 # High level stages
 stages:
+  - machine-checks
   - build-and-test
   - multi-project #TODO unused so far
   - radiuss-spack-testing #TODO unused so far
@@ -58,7 +59,7 @@ stages:
     include:
       - local: '.gitlab/custom-jobs-and-variables.yml'
       - project: 'radiuss/radiuss-shared-ci'
-        ref: v2023.03.0rc
+        ref: v2023.03.1
         file: '${CI_MACHINE}-build-and-test.yml'
       - local: '.gitlab/${CI_MACHINE}-build-and-test-extra.yml'
     strategy: depend
@@ -68,7 +69,7 @@ stages:
 include:
   # [Optional] checks preliminary to running the actual CI test
   #- project: 'radiuss/radiuss-shared-ci'
-  #  ref: v2023.03.0rc
+  #  ref: v2023.03.1
   #  file: 'preliminary-ignore-draft-pr.yml'
   # pipelines subscribed by the project
   - local: .gitlab/subscribed-pipelines.yml

.gitlab/corona-build-and-test-extra.yml

@@ -25,8 +25,8 @@
 # This job intentionally tests our umpire package.py because although this job does not
 # explicitly have the ~tools, the package.py should still disable tools from being built.
 ###
-rocmcc_5_4_1_hip_openmp:
+rocmcc_5_4_1_hip_openmp_device_alloc:
   variables:
-    SPEC: "~shared +fortran +openmp +rocm +device_alloc tests=basic amdgpu_target=gfx906 %rocmcc@5.4.1 ^hip@5.4.1 ^blt@develop"
+    SPEC: "~shared +fortran +openmp +rocm +device_alloc tests=basic amdgpu_target=gfx906 %rocmcc@5.4.1 ^hip@5.4.1"
   extends: .build_and_test_on_corona
 

.gitlab/custom-jobs-and-variables.yml

@@ -18,36 +18,36 @@ variables:
 # Arguments for job level allocation
   RUBY_BUILD_AND_TEST_JOB_ALLOC: "--overlap --reservation=ci --qos=ci_ruby --time=10 --nodes=1"
 # Project specific variants for ruby
-  PROJECT_RUBY_VARIANTS: "~shared +fortran ~device_alloc +tools tests=basic "
+  PROJECT_RUBY_VARIANTS: "~shared +fortran +tools tests=basic "
 # Project specific deps for ruby
   PROJECT_RUBY_DEPS: ""
 
 # Corona
 # Arguments for top level allocation
   CORONA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=15m --nodes=1"
 # Arguments for job level allocation
-  CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=10m --nodes=1"
+  CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=10m --nodes=1 --begin-time=+5s"
 # Project specific variants for corona
-  PROJECT_CORONA_VARIANTS: "~shared +fortran ~device_alloc tests=basic "
+  PROJECT_CORONA_VARIANTS: "~shared +fortran +device_alloc tests=basic "
 # Project specific deps for corona
-  PROJECT_CORONA_DEPS: "^blt@develop"
+  PROJECT_CORONA_DEPS: ""
 
 # Tioga
 # Arguments for top level allocation
   TIOGA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=15m --nodes=1"
 # Arguments for job level allocation
-  TIOGA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=10m --nodes=1"
+  TIOGA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=10m --nodes=1 --begin-time=+5s"
 # Project specific variants for tioga
-  PROJECT_TIOGA_VARIANTS: "~shared +fortran ~device_alloc tests=basic "
+  PROJECT_TIOGA_VARIANTS: "~shared +fortran +device_alloc tests=basic "
 # Project specific deps for tioga
-  PROJECT_TIOGA_DEPS: "^blt@develop"
+  PROJECT_TIOGA_DEPS: ""
 
 # Lassen and Butte use a different job scheduler (spectrum lsf) that does not
 # allow pre-allocation the same way slurm does.
 # Arguments for job level allocation
   LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 18"
 # Project specific variants for lassen
-  PROJECT_LASSEN_VARIANTS: "~shared +fortran ~device_alloc +tools tests=basic "
+  PROJECT_LASSEN_VARIANTS: "~shared +fortran +tools tests=basic "
 # Project specific deps for lassen
   PROJECT_LASSEN_DEPS: ""
 

.gitlab/lassen-build-and-test-extra.yml

@@ -12,13 +12,6 @@
 # We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
 # the comparison with the original job is easier.
 
-# Spec has test disabled in shared CI, conflicting with default test setting in Umpire
-# TODO: This is likely an issue in RADIUSS Shared CI: why disable tests ?.
-xl_16_1_1_12_gcc_8_3_1_cuda_11_0_2:
-  variables:
-    SPEC: "~shared +fortran ~device_alloc +tools +cuda tests=none %xl@16.1.1.12 cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ^cuda@11.0.2 ${PROJECT_LASSEN_DEPS}"
-  extends: .build_and_test_on_lassen
-
 
 ############
 # Extra jobs
@@ -27,30 +20,30 @@ xl_16_1_1_12_gcc_8_3_1_cuda_11_0_2:
 # ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is no reason not to fully
 # describe the spec here.
 
-ibm_clang_10_0_1_mpi_shmem:
+ibm_clang_14_0_5_mpi_shmem:
   variables:
-    SPEC: "~shared ~device_alloc +tools tests=basic %clang@ibm.10.0.1 +ipc_shmem +mpi ^spectrum-mpi"
+    SPEC: "~shared +tools tests=basic +ipc_shmem +mpi %clang@14.0.5.ibm.gcc.8.3.1 ^spectrum-mpi"
   extends: .build_and_test_on_lassen
 
-ibm_clang_10_0_1_mpi:
+ibm_clang_14_0_5_mpi:
   variables:
-    SPEC: "~shared +fortran ~device_alloc +tools tests=basic %clang@ibm.10.0.1 +mpi ^spectrum-mpi"
+    SPEC: "~shared +fortran +tools +mpi tests=basic %clang@14.0.5.ibm.gcc.8.3.1 ^spectrum-mpi"
   extends: .build_and_test_on_lassen
 
-ibm_clang_10_0_1_cuda_10_1_168_mpi_shmem:
+ibm_clang_14_0_5_gcc_8_3_1_cuda_11_7_0_mpi_shmem:
   variables:
-    SPEC: "~shared ~device_alloc +tools tests=basic %clang@ibm.10.0.1 +cuda +ipc_shmem +mpi cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ^spectrum-mpi ^cuda@10.1.168"
+    SPEC: "~shared +tools +cuda +ipc_shmem +mpi tests=basic %clang@14.0.5.ibm.gcc.8.3.1 ^spectrum-mpi ^cuda@11.7.0+allow-unsupported-compilers"
+    MODULE_LIST: "cuda/11.7.0"
   extends: .build_and_test_on_lassen
 
-# Extras.
-clang_9_0_0_libcpp:
+clang_12_0_1_libcpp:
   variables:
-    SPEC: "~shared ~device_alloc +tools tests=basic %clang@9.0.0+libcpp cxxflags==\"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0\" cflags==\"-DGTEST_HAS_CXXABI_H_=0\""
+    SPEC: "~shared +tools tests=basic %clang@12.0.1 cflags==\"-DGTEST_HAS_CXXABI_H_=0\" cxxflags==\"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0\""
   extends: .build_and_test_on_lassen
 
-clang_9_0_0_memleak:
+clang_12_0_1_gcc_8_3_1_memleak:
   variables:
-    SPEC: "~shared +asan +sanitizer_tests +tools ~device_alloc tests=basic %clang@9.0.0 cxxflags==-fsanitize=address"
+    SPEC: "~shared +asan +sanitizer_tests +tools tests=basic %clang@12.0.1.gcc.8.3.1 cxxflags==-fsanitize=address"
     ASAN_OPTIONS: "detect_leaks=1"
   extends: .build_and_test_on_lassen
 
@@ -62,38 +55,41 @@ clang_9_0_0_memleak:
 
 gcc_8_3_1_dev_benchmarks:
   variables:
-    SPEC: "~shared +dev_benchmarks +tools ~device_alloc %gcc@8.3.1 build_type=Release"
+    SPEC: "~shared +dev_benchmarks +tools build_type=Release %gcc@8.3.1"
   extends: .build_and_test_on_lassen
 
-xl_default_omp_target:
+xl_2022_08_19_default_omp_target:
   variables:
-    SPEC: "~shared ~device_alloc +tools tests=basic %xl@default+openmp+openmp_target"
+    SPEC: "~shared +tools +openmp +openmp_target tests=basic %xl@16.1.1.12"
   allow_failure: true
   extends: .build_and_test_on_lassen
 
 gcc_8_3_1_numa:
   variables:
-    SPEC: "~shared +fortran +numa +tools ~device_alloc tests=basic %gcc@8.3.1"
+    SPEC: "~shared +fortran +numa +tools tests=basic %gcc@8.3.1"
   extends: .build_and_test_on_lassen
 
-gcc_8_3_1_cuda_10_1_168:
+gcc_8_3_1_cuda_11_7_0:
   variables:
-    SPEC: "+fortran +cuda +tools +backtrace ~device_alloc tests=basic %gcc@8.3.1 ^cuda@10.1.168"
+    SPEC: "+fortran +cuda +tools +backtrace tests=basic %gcc@8.3.1 ^cuda@11.7.0+allow-unsupported-compilers"
+    MODULE_LIST: "cuda/11.7.0"
   extends: .build_and_test_on_lassen
 
 # Develop/main builds against specific tpl versions.
 
 gcc_8_3_1_tpls:
   variables:
-    SPEC: "~shared +fortran ~device_alloc +tools tests=basic %gcc@8.3.1 ^blt@develop"
+    SPEC: "~shared +fortran +tools tests=basic %gcc@8.3.1"
   extends: .build_and_test_on_lassen
 
-ibm_clang_9_0_0_gcc_8_3_1_cuda_10_1_168_tpls:
+ibm_clang_14_0_5_gcc_8_3_1_cuda_11_7_0_tpls:
   variables:
-    SPEC: "~shared +fortran +cuda ~device_alloc +tools tests=basic %clang@ibm.9.0.0 cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ^cuda@10.1.168 ^blt@develop"
+    SPEC: "~shared +fortran +cuda +tools tests=basic %clang@14.0.5.ibm.gcc.8.3.1 ^cuda@11.7.0+allow-unsupported-compilers"
+    MODULE_LIST: "cuda/11.7.0"
   extends: .build_and_test_on_lassen
 
-xl_16_1_1_12_gcc_8_3_1_cuda_10_1_168_tpls:
+xl_2022_08_19_gcc_8_3_1_cuda_11_7_tpls:
   variables:
-    SPEC: "~shared +fortran +cuda ~device_alloc +tools tests=basic %xl@16.1.1.12 cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-7.3.1\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-7.3.1\" ^cuda@10.1.168 ^blt@develop"
+    SPEC: "~shared +fortran +cuda +tools tests=basic %xl@16.1.1.12.gcc.8.3.1 ^cuda@11.7.0+allow-unsupported-compilers"
+    MODULE_LIST: "cuda/11.7.0"
   extends: .build_and_test_on_lassen

.gitlab/ruby-build-and-test-extra.yml

@@ -21,29 +21,39 @@
 # ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is not reason not to fully
 # describe the spec here.
 
-gcc_8_1_0_numa:
+gcc_10_3_1_numa:
   variables:
-    SPEC: "~shared +fortran +numa ~device_alloc +tools tests=basic %gcc@8.1.0"
+    SPEC: "~shared +fortran +numa +tools tests=basic %gcc@10.3.1"
   extends: .build_and_test_on_ruby
 
-clang_10_0_1_sqlite_experimental:
+clang_14_0_6_gcc_10_3_1_sqlite_experimental:
   variables:
-    SPEC: "~shared +sqlite_experimental ~device_alloc +tools tests=basic %clang@10.0.1"
+    SPEC: "~shared +sqlite_experimental +tools tests=basic %clang@14.0.6.gcc.10.3.1"
   extends: .build_and_test_on_ruby
 
 # Develop builds against specific tpl version.
-pgi_20_1_gcc_local_8_3_1_tpls:
+clang_14_0_6_gcc_10_3_1_tpls:
   variables:
-    SPEC: "~shared +fortran ~device_alloc +tools tests=basic %pgi@20.1 cxxflags==\"-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" cflags==\"-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" fflags==\"-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" ^blt@develop"
+    SPEC: "~shared +fortran +tools tests=basic %clang@14.0.6.gcc.10.3.1"
   extends: .build_and_test_on_ruby
 
-clang_10_0_1_tpls:
+gcc_10_3_1_tpls:
   variables:
-    SPEC: "~shared +fortran ~device_alloc +tools tests=basic %clang@10.0.1 ^blt@develop"
+    SPEC: "~shared +fortran +tools tests=basic %gcc@10.3.1"
   extends: .build_and_test_on_ruby
 
-gcc_8_1_0_tpls:
-  variables:
-    SPEC: "~shared +fortran ~device_alloc +tools tests=basic %gcc@8.1.0 ^blt@develop"
-  extends: .build_and_test_on_ruby
+# Oneapi is not available on ruby@toss4 (rhel8)
+## We deactivate this job as it is known to fail with Umpire: needs gcc toolchain.
+#intel_oneapi_2022_2:
+#  variables:
+#    ON_RUBY: "OFF"
+#    SPEC: "${PROJECT_RUBY_VARIANTS} %oneapi@2022.2 ${PROJECT_RUBY_DEPS}"
+#  extends: .build_and_test_on_ruby
+#
+#intel_oneapi_2022_2_gcc_8_3_1:
+#  variables:
+#    SPEC: "~shared +fortran +tools tests=basic %oneapi@2022.2.gcc.8.3.1"
+#  extends: .build_and_test_on_ruby
+#  allow_failure: true
+
 

.gitlab/subscribed-pipelines.yml

@@ -5,27 +5,79 @@
 # SPDX-License-Identifier: (MIT)
 ###############################################################################
 
-# Trigger a build-and-test pipeline for ruby, corona and lassen
+# The template job to test whether a machine is up.
+# Expects CI_MACHINE defined to machine name.
+.machine-check:
+  stage: machine-checks
+  tags: [shell, oslic]
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - |
+      if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]]
+      then
+        echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m"
+        curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \
+             --header 'Content-Type: application/json' \
+             --header "authorization: Bearer ${GITHUB_TOKEN}" \
+             --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }"
+        exit 1
+      fi
+
+###
+# Trigger a build-and-test pipeline for a machine.
+# Comment the jobs for machines you don’t need.
+###
+
+# RUBY
+ruby-up-check:
+  variables:
+    CI_MACHINE: "ruby"
+  extends: [.machine-check]
+
 ruby-build-and-test:
   variables:
     CI_MACHINE: "ruby"
+  needs: [ruby-up-check]
   extends: [.build-and-test]
 
+# CORONA
+corona-up-check:
+  variables:
+    CI_MACHINE: "corona"
+  extends: [.machine-check]
+
 corona-build-and-test:
   variables:
     CI_MACHINE: "corona"
+  needs: [corona-up-check]
   extends: [.build-and-test]
 
+# TIOGA
+tioga-up-check:
+  variables:
+    CI_MACHINE: "tioga"
+  extends: [.machine-check]
+
 tioga-build-and-test:
   variables:
     CI_MACHINE: "tioga"
+  needs: [tioga-up-check]
   extends: [.build-and-test]
 
+# LASSEN
+lassen-up-check:
+  variables:
+    CI_MACHINE: "lassen"
+  extends: [.machine-check]
+
 lassen-build-and-test:
   variables:
     CI_MACHINE: "lassen"
+  needs: [lassen-up-check]
   extends: [.build-and-test]
 
+
 ## If testing develop branch, trigger CHAI pipeline with this version of Umpire.
 ## TODO: Once Spack allows to clone a specific commit on demand, then point to
 ## the exact commit. This will prevent from sticking to a branch (here develop).

.gitlab/tioga-build-and-test-extra.yml

@@ -12,7 +12,11 @@
 # We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
 # the comparison with the original job is easier.
 
-# No overridden jobs so far.
+# We override the cce job because we can’t use +device-alloc with it
+cce_15_0_1:
+  variables:
+    SPEC: "~shared +fortran tests=basic %cce@15.0.1"
+  extends: .build_and_test_on_tioga
 
 ############
 # Extra jobs
@@ -25,8 +29,8 @@
 # This job intentionally tests our umpire package.py because although this job does not
 # explicitly have the ~tools, the package.py should still disable tools from being built.
 ###
-rocmcc_5_4_1_hip_openmp:
+rocmcc_5_4_3_hip_openmp_device_alloc:
   variables:
-    SPEC: "~shared +fortran +openmp +rocm +device_alloc tests=basic amdgpu_target=gfx90a %rocmcc@5.4.1 ^hip@5.4.1 ^blt@develop"
+    SPEC: "~shared +fortran +openmp +rocm +device_alloc tests=basic amdgpu_target=gfx90a %rocmcc@5.4.3 ^hip@5.4.3"
   extends: .build_and_test_on_tioga
 

.uberenv_config.json

@@ -4,7 +4,7 @@
 "package_final_phase" : "initconfig",
 "package_source_dir" : "../..",
 "spack_url": "https://github.com/spack/spack.git",
-"spack_branch": "v0.19.0",
+"spack_branch": "e4s-23.02",
 "spack_activate" : {},
 "spack_configs_path": "scripts/radiuss-spack-configs",
 "spack_packages_path": "scripts/radiuss-spack-configs/packages",

scripts/gitlab/build_and_test.sh

@@ -24,8 +24,17 @@ project_dir="$(pwd)"
 build_root=${BUILD_ROOT:-""}
 hostconfig=${HOST_CONFIG:-""}
 spec=${SPEC:-""}
+module_list=${MODULE_LIST:-""}
 job_unique_id=${CI_JOB_ID:-""}
 
+if [[ -n ${module_list} ]]
+then
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    echo "~~~~~ Modules to load: ${module_list}"
+    echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+    module load ${module_list}
+fi
+
 prefix=""
 
 if [[ -d /dev/shm ]]

tests/tools/sanitizers/sanitizer_test_runner.py

@@ -49,7 +49,7 @@ def run_sanitizer_test(strategy, kind):
     output = test_program.stdout
     error = test_program.stderr
 
-    expected_string = '{} of size 8'.format(kind.upper())
+    expected_string = '{} of size 8'.format(kind.upper()).encode()
     check_output(error, expected_string)