diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ac47634a0..ad2e00585 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -47,6 +47,7 @@ variables: # High level stages stages: + - machine-checks - build-and-test - multi-project #TODO unused so far - radiuss-spack-testing #TODO unused so far @@ -58,7 +59,7 @@ stages: include: - local: '.gitlab/custom-jobs-and-variables.yml' - project: 'radiuss/radiuss-shared-ci' - ref: v2023.03.0rc + ref: v2023.03.1 file: '${CI_MACHINE}-build-and-test.yml' - local: '.gitlab/${CI_MACHINE}-build-and-test-extra.yml' strategy: depend @@ -68,7 +69,7 @@ stages: include: # [Optional] checks preliminary to running the actual CI test #- project: 'radiuss/radiuss-shared-ci' - # ref: v2023.03.0rc + # ref: v2023.03.1 # file: 'preliminary-ignore-draft-pr.yml' # pipelines subscribed by the project - local: .gitlab/subscribed-pipelines.yml diff --git a/.gitlab/corona-build-and-test-extra.yml b/.gitlab/corona-build-and-test-extra.yml index 6d0d4ea83..540191479 100644 --- a/.gitlab/corona-build-and-test-extra.yml +++ b/.gitlab/corona-build-and-test-extra.yml @@ -25,8 +25,8 @@ # This job intentionally tests our umpire package.py because although this job does not # explicitly have the ~tools, the package.py should still disable tools from being built. ### -rocmcc_5_4_1_hip_openmp: +rocmcc_5_4_1_hip_openmp_device_alloc: variables: - SPEC: "~shared +fortran +openmp +rocm +device_alloc tests=basic amdgpu_target=gfx906 %rocmcc@5.4.1 ^hip@5.4.1 ^blt@develop" + SPEC: "~shared +fortran +openmp +rocm +device_alloc tests=basic amdgpu_target=gfx906 %rocmcc@5.4.1 ^hip@5.4.1" extends: .build_and_test_on_corona diff --git a/.gitlab/custom-jobs-and-variables.yml b/.gitlab/custom-jobs-and-variables.yml index 628c0516a..24b3ba706 100644 --- a/.gitlab/custom-jobs-and-variables.yml +++ b/.gitlab/custom-jobs-and-variables.yml @@ -18,7 +18,7 @@ variables: # Arguments for job level allocation RUBY_BUILD_AND_TEST_JOB_ALLOC: "--overlap --reservation=ci --qos=ci_ruby --time=10 --nodes=1" # Project specific variants for ruby - PROJECT_RUBY_VARIANTS: "~shared +fortran ~device_alloc +tools tests=basic " + PROJECT_RUBY_VARIANTS: "~shared +fortran +tools tests=basic " # Project specific deps for ruby PROJECT_RUBY_DEPS: "" @@ -26,28 +26,28 @@ variables: # Arguments for top level allocation CORONA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=15m --nodes=1" # Arguments for job level allocation - CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=10m --nodes=1" + CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=10m --nodes=1 --begin-time=+5s" # Project specific variants for corona - PROJECT_CORONA_VARIANTS: "~shared +fortran ~device_alloc tests=basic " + PROJECT_CORONA_VARIANTS: "~shared +fortran +device_alloc tests=basic " # Project specific deps for corona - PROJECT_CORONA_DEPS: "^blt@develop" + PROJECT_CORONA_DEPS: "" # Tioga # Arguments for top level allocation TIOGA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=15m --nodes=1" # Arguments for job level allocation - TIOGA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=10m --nodes=1" + TIOGA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=10m --nodes=1 --begin-time=+5s" # Project specific variants for tioga - PROJECT_TIOGA_VARIANTS: "~shared +fortran ~device_alloc tests=basic " + PROJECT_TIOGA_VARIANTS: "~shared +fortran +device_alloc tests=basic " # Project specific deps for tioga - PROJECT_TIOGA_DEPS: "^blt@develop" + PROJECT_TIOGA_DEPS: "" # Lassen and Butte use a different job scheduler (spectrum lsf) that does not # allow pre-allocation the same way slurm does. # Arguments for job level allocation LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 18" # Project specific variants for lassen - PROJECT_LASSEN_VARIANTS: "~shared +fortran ~device_alloc +tools tests=basic " + PROJECT_LASSEN_VARIANTS: "~shared +fortran +tools tests=basic " # Project specific deps for lassen PROJECT_LASSEN_DEPS: "" diff --git a/.gitlab/lassen-build-and-test-extra.yml b/.gitlab/lassen-build-and-test-extra.yml index a865e8f1c..b66113b44 100644 --- a/.gitlab/lassen-build-and-test-extra.yml +++ b/.gitlab/lassen-build-and-test-extra.yml @@ -12,13 +12,6 @@ # We keep ${PROJECT__VARIANTS} and ${PROJECT__DEPS} So that # the comparison with the original job is easier. -# Spec has test disabled in shared CI, conflicting with default test setting in Umpire -# TODO: This is likely an issue in RADIUSS Shared CI: why disable tests ?. -xl_16_1_1_12_gcc_8_3_1_cuda_11_0_2: - variables: - SPEC: "~shared +fortran ~device_alloc +tools +cuda tests=none %xl@16.1.1.12 cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ^cuda@11.0.2 ${PROJECT_LASSEN_DEPS}" - extends: .build_and_test_on_lassen - ############ # Extra jobs @@ -27,30 +20,30 @@ xl_16_1_1_12_gcc_8_3_1_cuda_11_0_2: # ${PROJECT__DEPS} in the extra jobs. There is no reason not to fully # describe the spec here. -ibm_clang_10_0_1_mpi_shmem: +ibm_clang_14_0_5_mpi_shmem: variables: - SPEC: "~shared ~device_alloc +tools tests=basic %clang@ibm.10.0.1 +ipc_shmem +mpi ^spectrum-mpi" + SPEC: "~shared +tools tests=basic +ipc_shmem +mpi %clang@14.0.5.ibm.gcc.8.3.1 ^spectrum-mpi" extends: .build_and_test_on_lassen -ibm_clang_10_0_1_mpi: +ibm_clang_14_0_5_mpi: variables: - SPEC: "~shared +fortran ~device_alloc +tools tests=basic %clang@ibm.10.0.1 +mpi ^spectrum-mpi" + SPEC: "~shared +fortran +tools +mpi tests=basic %clang@14.0.5.ibm.gcc.8.3.1 ^spectrum-mpi" extends: .build_and_test_on_lassen -ibm_clang_10_0_1_cuda_10_1_168_mpi_shmem: +ibm_clang_14_0_5_gcc_8_3_1_cuda_11_7_0_mpi_shmem: variables: - SPEC: "~shared ~device_alloc +tools tests=basic %clang@ibm.10.0.1 +cuda +ipc_shmem +mpi cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ^spectrum-mpi ^cuda@10.1.168" + SPEC: "~shared +tools +cuda +ipc_shmem +mpi tests=basic %clang@14.0.5.ibm.gcc.8.3.1 ^spectrum-mpi ^cuda@11.7.0+allow-unsupported-compilers" + MODULE_LIST: "cuda/11.7.0" extends: .build_and_test_on_lassen -# Extras. -clang_9_0_0_libcpp: +clang_12_0_1_libcpp: variables: - SPEC: "~shared ~device_alloc +tools tests=basic %clang@9.0.0+libcpp cxxflags==\"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0\" cflags==\"-DGTEST_HAS_CXXABI_H_=0\"" + SPEC: "~shared +tools tests=basic %clang@12.0.1 cflags==\"-DGTEST_HAS_CXXABI_H_=0\" cxxflags==\"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0\"" extends: .build_and_test_on_lassen -clang_9_0_0_memleak: +clang_12_0_1_gcc_8_3_1_memleak: variables: - SPEC: "~shared +asan +sanitizer_tests +tools ~device_alloc tests=basic %clang@9.0.0 cxxflags==-fsanitize=address" + SPEC: "~shared +asan +sanitizer_tests +tools tests=basic %clang@12.0.1.gcc.8.3.1 cxxflags==-fsanitize=address" ASAN_OPTIONS: "detect_leaks=1" extends: .build_and_test_on_lassen @@ -62,38 +55,41 @@ clang_9_0_0_memleak: gcc_8_3_1_dev_benchmarks: variables: - SPEC: "~shared +dev_benchmarks +tools ~device_alloc %gcc@8.3.1 build_type=Release" + SPEC: "~shared +dev_benchmarks +tools build_type=Release %gcc@8.3.1" extends: .build_and_test_on_lassen -xl_default_omp_target: +xl_2022_08_19_default_omp_target: variables: - SPEC: "~shared ~device_alloc +tools tests=basic %xl@default+openmp+openmp_target" + SPEC: "~shared +tools +openmp +openmp_target tests=basic %xl@16.1.1.12" allow_failure: true extends: .build_and_test_on_lassen gcc_8_3_1_numa: variables: - SPEC: "~shared +fortran +numa +tools ~device_alloc tests=basic %gcc@8.3.1" + SPEC: "~shared +fortran +numa +tools tests=basic %gcc@8.3.1" extends: .build_and_test_on_lassen -gcc_8_3_1_cuda_10_1_168: +gcc_8_3_1_cuda_11_7_0: variables: - SPEC: "+fortran +cuda +tools +backtrace ~device_alloc tests=basic %gcc@8.3.1 ^cuda@10.1.168" + SPEC: "+fortran +cuda +tools +backtrace tests=basic %gcc@8.3.1 ^cuda@11.7.0+allow-unsupported-compilers" + MODULE_LIST: "cuda/11.7.0" extends: .build_and_test_on_lassen # Develop/main builds against specific tpl versions. gcc_8_3_1_tpls: variables: - SPEC: "~shared +fortran ~device_alloc +tools tests=basic %gcc@8.3.1 ^blt@develop" + SPEC: "~shared +fortran +tools tests=basic %gcc@8.3.1" extends: .build_and_test_on_lassen -ibm_clang_9_0_0_gcc_8_3_1_cuda_10_1_168_tpls: +ibm_clang_14_0_5_gcc_8_3_1_cuda_11_7_0_tpls: variables: - SPEC: "~shared +fortran +cuda ~device_alloc +tools tests=basic %clang@ibm.9.0.0 cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ^cuda@10.1.168 ^blt@develop" + SPEC: "~shared +fortran +cuda +tools tests=basic %clang@14.0.5.ibm.gcc.8.3.1 ^cuda@11.7.0+allow-unsupported-compilers" + MODULE_LIST: "cuda/11.7.0" extends: .build_and_test_on_lassen -xl_16_1_1_12_gcc_8_3_1_cuda_10_1_168_tpls: +xl_2022_08_19_gcc_8_3_1_cuda_11_7_tpls: variables: - SPEC: "~shared +fortran +cuda ~device_alloc +tools tests=basic %xl@16.1.1.12 cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-7.3.1\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-7.3.1\" ^cuda@10.1.168 ^blt@develop" + SPEC: "~shared +fortran +cuda +tools tests=basic %xl@16.1.1.12.gcc.8.3.1 ^cuda@11.7.0+allow-unsupported-compilers" + MODULE_LIST: "cuda/11.7.0" extends: .build_and_test_on_lassen diff --git a/.gitlab/ruby-build-and-test-extra.yml b/.gitlab/ruby-build-and-test-extra.yml index fc509c342..2dddb04c9 100644 --- a/.gitlab/ruby-build-and-test-extra.yml +++ b/.gitlab/ruby-build-and-test-extra.yml @@ -21,29 +21,39 @@ # ${PROJECT__DEPS} in the extra jobs. There is not reason not to fully # describe the spec here. -gcc_8_1_0_numa: +gcc_10_3_1_numa: variables: - SPEC: "~shared +fortran +numa ~device_alloc +tools tests=basic %gcc@8.1.0" + SPEC: "~shared +fortran +numa +tools tests=basic %gcc@10.3.1" extends: .build_and_test_on_ruby -clang_10_0_1_sqlite_experimental: +clang_14_0_6_gcc_10_3_1_sqlite_experimental: variables: - SPEC: "~shared +sqlite_experimental ~device_alloc +tools tests=basic %clang@10.0.1" + SPEC: "~shared +sqlite_experimental +tools tests=basic %clang@14.0.6.gcc.10.3.1" extends: .build_and_test_on_ruby # Develop builds against specific tpl version. -pgi_20_1_gcc_local_8_3_1_tpls: +clang_14_0_6_gcc_10_3_1_tpls: variables: - SPEC: "~shared +fortran ~device_alloc +tools tests=basic %pgi@20.1 cxxflags==\"-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" cflags==\"-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" fflags==\"-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" ^blt@develop" + SPEC: "~shared +fortran +tools tests=basic %clang@14.0.6.gcc.10.3.1" extends: .build_and_test_on_ruby -clang_10_0_1_tpls: +gcc_10_3_1_tpls: variables: - SPEC: "~shared +fortran ~device_alloc +tools tests=basic %clang@10.0.1 ^blt@develop" + SPEC: "~shared +fortran +tools tests=basic %gcc@10.3.1" extends: .build_and_test_on_ruby -gcc_8_1_0_tpls: - variables: - SPEC: "~shared +fortran ~device_alloc +tools tests=basic %gcc@8.1.0 ^blt@develop" - extends: .build_and_test_on_ruby +# Oneapi is not available on ruby@toss4 (rhel8) +## We deactivate this job as it is known to fail with Umpire: needs gcc toolchain. +#intel_oneapi_2022_2: +# variables: +# ON_RUBY: "OFF" +# SPEC: "${PROJECT_RUBY_VARIANTS} %oneapi@2022.2 ${PROJECT_RUBY_DEPS}" +# extends: .build_and_test_on_ruby +# +#intel_oneapi_2022_2_gcc_8_3_1: +# variables: +# SPEC: "~shared +fortran +tools tests=basic %oneapi@2022.2.gcc.8.3.1" +# extends: .build_and_test_on_ruby +# allow_failure: true + diff --git a/.gitlab/subscribed-pipelines.yml b/.gitlab/subscribed-pipelines.yml index ca4107ed4..e5ee45a9e 100644 --- a/.gitlab/subscribed-pipelines.yml +++ b/.gitlab/subscribed-pipelines.yml @@ -5,27 +5,79 @@ # SPDX-License-Identifier: (MIT) ############################################################################### -# Trigger a build-and-test pipeline for ruby, corona and lassen +# The template job to test whether a machine is up. +# Expects CI_MACHINE defined to machine name. +.machine-check: + stage: machine-checks + tags: [shell, oslic] + variables: + GIT_STRATEGY: none + script: + - | + if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]] + then + echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m" + curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \ + --header 'Content-Type: application/json' \ + --header "authorization: Bearer ${GITHUB_TOKEN}" \ + --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }" + exit 1 + fi + +### +# Trigger a build-and-test pipeline for a machine. +# Comment the jobs for machines you don’t need. +### + +# RUBY +ruby-up-check: + variables: + CI_MACHINE: "ruby" + extends: [.machine-check] + ruby-build-and-test: variables: CI_MACHINE: "ruby" + needs: [ruby-up-check] extends: [.build-and-test] +# CORONA +corona-up-check: + variables: + CI_MACHINE: "corona" + extends: [.machine-check] + corona-build-and-test: variables: CI_MACHINE: "corona" + needs: [corona-up-check] extends: [.build-and-test] +# TIOGA +tioga-up-check: + variables: + CI_MACHINE: "tioga" + extends: [.machine-check] + tioga-build-and-test: variables: CI_MACHINE: "tioga" + needs: [tioga-up-check] extends: [.build-and-test] +# LASSEN +lassen-up-check: + variables: + CI_MACHINE: "lassen" + extends: [.machine-check] + lassen-build-and-test: variables: CI_MACHINE: "lassen" + needs: [lassen-up-check] extends: [.build-and-test] + ## If testing develop branch, trigger CHAI pipeline with this version of Umpire. ## TODO: Once Spack allows to clone a specific commit on demand, then point to ## the exact commit. This will prevent from sticking to a branch (here develop). diff --git a/.gitlab/tioga-build-and-test-extra.yml b/.gitlab/tioga-build-and-test-extra.yml index a4be5184f..fa743156c 100644 --- a/.gitlab/tioga-build-and-test-extra.yml +++ b/.gitlab/tioga-build-and-test-extra.yml @@ -12,7 +12,11 @@ # We keep ${PROJECT__VARIANTS} and ${PROJECT__DEPS} So that # the comparison with the original job is easier. -# No overridden jobs so far. +# We override the cce job because we can’t use +device-alloc with it +cce_15_0_1: + variables: + SPEC: "~shared +fortran tests=basic %cce@15.0.1" + extends: .build_and_test_on_tioga ############ # Extra jobs @@ -25,8 +29,8 @@ # This job intentionally tests our umpire package.py because although this job does not # explicitly have the ~tools, the package.py should still disable tools from being built. ### -rocmcc_5_4_1_hip_openmp: +rocmcc_5_4_3_hip_openmp_device_alloc: variables: - SPEC: "~shared +fortran +openmp +rocm +device_alloc tests=basic amdgpu_target=gfx90a %rocmcc@5.4.1 ^hip@5.4.1 ^blt@develop" + SPEC: "~shared +fortran +openmp +rocm +device_alloc tests=basic amdgpu_target=gfx90a %rocmcc@5.4.3 ^hip@5.4.3" extends: .build_and_test_on_tioga diff --git a/.uberenv_config.json b/.uberenv_config.json index ba68cfe38..062096cd1 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -4,7 +4,7 @@ "package_final_phase" : "initconfig", "package_source_dir" : "../..", "spack_url": "https://github.com/spack/spack.git", -"spack_branch": "v0.19.0", +"spack_branch": "e4s-23.02", "spack_activate" : {}, "spack_configs_path": "scripts/radiuss-spack-configs", "spack_packages_path": "scripts/radiuss-spack-configs/packages", diff --git a/blt b/blt index 84fe0d15d..09f7a4be4 160000 --- a/blt +++ b/blt @@ -1 +1 @@ -Subproject commit 84fe0d15d841a79d923e97a9105b86b59147bfed +Subproject commit 09f7a4be48886e1175885e10ed6af1225ae12a90 diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index 7e3f14e1b..b5c702a0a 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -24,8 +24,17 @@ project_dir="$(pwd)" build_root=${BUILD_ROOT:-""} hostconfig=${HOST_CONFIG:-""} spec=${SPEC:-""} +module_list=${MODULE_LIST:-""} job_unique_id=${CI_JOB_ID:-""} +if [[ -n ${module_list} ]] +then + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "~~~~~ Modules to load: ${module_list}" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + module load ${module_list} +fi + prefix="" if [[ -d /dev/shm ]] diff --git a/scripts/radiuss-spack-configs b/scripts/radiuss-spack-configs index d674c30ba..f6626f790 160000 --- a/scripts/radiuss-spack-configs +++ b/scripts/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit d674c30babee357e5c98bb396d6b7978ba3c8c64 +Subproject commit f6626f790b07fbd399f238aa9792a11ca78dfa97 diff --git a/tests/tools/sanitizers/sanitizer_test_runner.py b/tests/tools/sanitizers/sanitizer_test_runner.py index 88ee36421..544f74693 100644 --- a/tests/tools/sanitizers/sanitizer_test_runner.py +++ b/tests/tools/sanitizers/sanitizer_test_runner.py @@ -49,7 +49,7 @@ def run_sanitizer_test(strategy, kind): output = test_program.stdout error = test_program.stderr - expected_string = '{} of size 8'.format(kind.upper()) + expected_string = '{} of size 8'.format(kind.upper()).encode() check_output(error, expected_string)