inherit safety from charged tracks #397
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: 2026 CERN | |
| # SPDX-License-Identifier: Apache-2.0 | |
| name: Self-Hosted PR CI (EL9) | |
| on: | |
| issue_comment: | |
| types: [created] | |
| permissions: | |
| contents: read | |
| issues: read | |
| pull-requests: read | |
| statuses: write | |
| jobs: | |
| prepare: | |
| name: Prepare | |
| if: github.event.issue.pull_request != null | |
| runs-on: ubuntu-latest | |
| outputs: | |
| should_run: ${{ steps.prepare.outputs.should_run }} | |
| pr_number: ${{ steps.prepare.outputs.pr_number }} | |
| pr_sha: ${{ steps.prepare.outputs.pr_sha }} | |
| pr_repo: ${{ steps.prepare.outputs.pr_repo }} | |
| pr_ref: ${{ steps.prepare.outputs.pr_ref }} | |
| command: ${{ steps.prepare.outputs.command }} | |
| steps: | |
| - name: Validate command and resolve PR | |
| id: prepare | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| const body = (context.payload.comment.body || '').trim(); | |
| const assoc = context.payload.comment.author_association || 'NONE'; | |
| const trusted = ['OWNER', 'MEMBER'].includes(assoc); | |
| core.setOutput('command', body); | |
| core.setOutput('should_run', 'false'); | |
| const supportedCommands = new Set(['/run-test', '/run-test-full']); | |
| if (!supportedCommands.has(body)) { | |
| core.notice(`Ignoring comment '${body}'`); | |
| return; | |
| } | |
| if (!trusted) { | |
| core.notice(`Ignoring ${body} from untrusted role ${assoc}`); | |
| return; | |
| } | |
| const prNumber = context.payload.issue.number; | |
| const { data: pr } = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: prNumber, | |
| }); | |
| core.info(`Resolved PR #${pr.number} -> ${pr.head.repo.full_name}@${pr.head.sha}`); | |
| core.setOutput('should_run', 'true'); | |
| core.setOutput('pr_number', String(pr.number)); | |
| core.setOutput('pr_sha', pr.head.sha); | |
| core.setOutput('pr_repo', pr.head.repo.full_name); | |
| core.setOutput('pr_ref', pr.head.ref); | |
| - name: Set pending PR contexts | |
| if: steps.prepare.outputs.should_run == 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_SHA: ${{ steps.prepare.outputs.pr_sha }} | |
| TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| COMMAND: ${{ steps.prepare.outputs.command }} | |
| run: | | |
| set -eo pipefail | |
| post_status() { | |
| local state=$1 | |
| local context=$2 | |
| local description=$3 | |
| local payload | |
| local response_file | |
| local http_code | |
| local curl_status | |
| local attempt | |
| local delay=5 | |
| payload=$(jq -n \ | |
| --arg state "${state}" \ | |
| --arg context "${context}" \ | |
| --arg description "${description}" \ | |
| --arg target_url "${TARGET_URL}" \ | |
| '{state: $state, context: $context, description: $description, target_url: $target_url}') | |
| response_file=$(mktemp) | |
| for attempt in 1 2 3 4 5; do | |
| http_code=$(curl --silent --show-error \ | |
| --output "${response_file}" \ | |
| --write-out "%{http_code}" \ | |
| --connect-timeout 10 \ | |
| --max-time 60 \ | |
| -H "Authorization: Bearer ${GH_TOKEN}" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Content-Type: application/json" \ | |
| -X POST \ | |
| -d "${payload}" \ | |
| "https://api.github.com/repos/${GITHUB_REPOSITORY}/statuses/${PR_SHA}") && curl_status=0 || curl_status=$? | |
| if [ "${curl_status}" -eq 0 ] && [ "${http_code}" -ge 200 ] && [ "${http_code}" -lt 300 ]; then | |
| rm -f "${response_file}" | |
| return 0 | |
| fi | |
| if [ "${curl_status}" -eq 0 ] && [ "${http_code}" -ge 400 ] && [ "${http_code}" -lt 500 ]; then | |
| cat "${response_file}" >&2 | |
| rm -f "${response_file}" | |
| return 1 | |
| fi | |
| if [ "${attempt}" -lt 5 ]; then | |
| echo "Posting status '${context}' failed (attempt ${attempt}/5, curl=${curl_status}, http=${http_code}); retrying in ${delay}s" >&2 | |
| sleep "${delay}" | |
| delay=$((delay * 2)) | |
| fi | |
| done | |
| cat "${response_file}" >&2 | |
| rm -f "${response_file}" | |
| return 1 | |
| } | |
| full_description="Self-hosted EL9 CI queued" | |
| validation_description="Waiting for drift result in EL9" | |
| if [ "${COMMAND}" = "/run-test-full" ]; then | |
| full_description="Self-hosted EL9 CI with full validation and drift smoke queued" | |
| validation_description="Full validation requested in EL9" | |
| fi | |
| post_status pending "AdePT / Full CI (EL9)" "${full_description}" | |
| post_status pending "AdePT / Drift + Unit (EL9)" "Building PR/reference and running drift in EL9" | |
| post_status pending "AdePT / Validation (EL9)" "${validation_description}" | |
| full-ci: | |
| name: Full CI (EL9) | |
| needs: prepare | |
| if: needs.prepare.outputs.should_run == 'true' | |
| concurrency: | |
| group: self-hosted-pr-ci-${{ needs.prepare.outputs.pr_number || github.event.issue.number }} | |
| cancel-in-progress: true | |
| runs-on: | |
| - self-hosted | |
| - linux | |
| - x64 | |
| - gpu | |
| - benchmark | |
| - adept | |
| timeout-minutes: 480 | |
| env: | |
| RESULTS_FILE: ${{ github.workspace }}/ci-results.env | |
| steps: | |
| - name: Checkout PR head | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: ${{ needs.prepare.outputs.pr_repo }} | |
| ref: ${{ needs.prepare.outputs.pr_sha }} | |
| persist-credentials: false | |
| fetch-depth: 0 | |
| submodules: recursive | |
| - name: Fetch upstream master reference | |
| run: | | |
| set -eo pipefail | |
| if git remote get-url upstream >/dev/null 2>&1; then | |
| git remote set-url upstream "https://github.com/${GITHUB_REPOSITORY}.git" | |
| else | |
| git remote add upstream "https://github.com/${GITHUB_REPOSITORY}.git" | |
| fi | |
| git fetch --no-tags upstream +refs/heads/master:refs/remotes/upstream/master | |
| - name: Show runner basics | |
| run: | | |
| set -eo pipefail | |
| cat /etc/os-release | sed -n '1,6p' | |
| echo "PR #${{ needs.prepare.outputs.pr_number }}" | |
| echo "Repository: ${{ needs.prepare.outputs.pr_repo }}" | |
| echo "SHA: ${{ needs.prepare.outputs.pr_sha }}" | |
| echo "hostname=$(hostname)" | |
| echo "user=$(whoami)" | |
| echo "pwd=$(pwd)" | |
| id | |
| - name: Pre-touch required CVMFS repos on host | |
| run: | | |
| set -eo pipefail | |
| ls /cvmfs/sft.cern.ch/lcg/views/devAdePT/latest/x86_64-el9-gcc13-opt/setup.sh | |
| ls /cvmfs/geant4.cern.ch/share/data/G4ENSDFSTATE3.0/ENSDFSTATE.dat | |
| - name: Check GPU visibility | |
| run: | | |
| set -eo pipefail | |
| nvidia-smi | |
| - name: Check Docker basics | |
| run: | | |
| set -eo pipefail | |
| docker --version | |
| - name: Run self-hosted PR CI in Alma 9 container | |
| id: run_ci | |
| continue-on-error: true | |
| env: | |
| ALWAYS_RUN_VALIDATION: ${{ needs.prepare.outputs.command == '/run-test-full' && '1' || '0' }} | |
| RUN_DRIFT_SMOKE: ${{ needs.prepare.outputs.command == '/run-test-full' && '1' || '0' }} | |
| run: | | |
| set -eo pipefail | |
| rm -f "${RESULTS_FILE}" | |
| ci_container="adept-self-hosted-pr-ci-${{ needs.prepare.outputs.pr_number }}" | |
| cleanup_container() { | |
| docker rm -f "${ci_container}" >/dev/null 2>&1 || true | |
| } | |
| cleanup_container | |
| trap cleanup_container EXIT | |
| trap 'cleanup_container; exit 130' INT TERM | |
| docker_group_args=() | |
| for group_name in video render; do | |
| if getent group "${group_name}" >/dev/null 2>&1; then | |
| docker_group_args+=(--group-add "$(getent group "${group_name}" | cut -d: -f3)") | |
| fi | |
| done | |
| docker run --rm --pull always \ | |
| --name "${ci_container}" \ | |
| --label adept.workflow=self-hosted-pr-ci \ | |
| --label adept.pr="${{ needs.prepare.outputs.pr_number }}" \ | |
| --gpus all \ | |
| --user "$(id -u):$(id -g)" \ | |
| "${docker_group_args[@]}" \ | |
| --security-opt seccomp=unconfined \ | |
| -v /build:/build \ | |
| -v /ec/conf:/ec/conf \ | |
| -v /cvmfs:/cvmfs:ro,rslave \ | |
| -v "${GITHUB_WORKSPACE}:/work/AdePT" \ | |
| -e HOME=/tmp/adeptci-home \ | |
| -e ADEPT_PR_CI_ALWAYS_RUN_VALIDATION="${ALWAYS_RUN_VALIDATION}" \ | |
| -e ADEPT_PR_CI_RUN_DRIFT_SMOKE="${RUN_DRIFT_SMOKE}" \ | |
| -e ADEPT_VALIDATION_DEFAULT_NUM_THREADS=32 \ | |
| -e ADEPT_VALIDATION_DEFAULT_NUM_TRACKSLOTS=6 \ | |
| -e ADEPT_VALIDATION_DEFAULT_NUM_HITSLOTS=30 \ | |
| -e ADEPT_VALIDATION_DEFAULT_CPU_CAPACITY_FACTOR=5.0 \ | |
| -e ADEPT_VALIDATION_REGIONS_NUM_THREADS=32 \ | |
| -e ADEPT_VALIDATION_REGIONS_NUM_TRACKSLOTS=6 \ | |
| -e ADEPT_VALIDATION_REGIONS_NUM_HITSLOTS=30 \ | |
| -e ADEPT_VALIDATION_REGIONS_CPU_CAPACITY_FACTOR=5.0 \ | |
| -e ADEPT_VALIDATION_WDT_NUM_THREADS=32 \ | |
| -e ADEPT_VALIDATION_WDT_NUM_TRACKSLOTS=8 \ | |
| -e ADEPT_VALIDATION_WDT_NUM_HITSLOTS=28 \ | |
| -e ADEPT_VALIDATION_WDT_CPU_CAPACITY_FACTOR=5.0 \ | |
| -w /work/AdePT \ | |
| gitlab-registry.cern.ch/sft/docker/alma9 \ | |
| bash -lc ' | |
| set -eo pipefail | |
| mkdir -p "${HOME}" | |
| git config --global --add safe.directory /work/AdePT | |
| pr_ci_args=( | |
| ./test/run_pr_ci.sh | |
| --build-root "/tmp/adept-self-hosted-ci/pr-${{ needs.prepare.outputs.pr_number }}" | |
| --results-file "/work/AdePT/ci-results.env" | |
| --master-ref upstream/master | |
| --no-fetch-master | |
| --cuda-arch 89 | |
| --jobs auto | |
| --ctest-timeout-sec 900 | |
| ) | |
| if [[ "${ADEPT_PR_CI_ALWAYS_RUN_VALIDATION:-0}" == "1" ]]; then | |
| pr_ci_args+=(--always-run-validation) | |
| fi | |
| if [[ "${ADEPT_PR_CI_RUN_DRIFT_SMOKE:-0}" == "1" ]]; then | |
| pr_ci_args+=(--run-drift-smoke) | |
| fi | |
| "${pr_ci_args[@]}" | |
| ' | |
| - name: Publish PR contexts | |
| if: always() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_SHA: ${{ needs.prepare.outputs.pr_sha }} | |
| TARGET_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| COMMAND: ${{ needs.prepare.outputs.command }} | |
| run: | | |
| set -eo pipefail | |
| if [ -r "${RESULTS_FILE}" ]; then | |
| # shellcheck disable=SC1091 | |
| source "${RESULTS_FILE}" | |
| fi | |
| : "${DRIFT_STATUS:=1}" | |
| : "${UNIT_STATUS:=1}" | |
| : "${VALIDATION_RAN:=1}" | |
| if [ "${COMMAND}" = "/run-test-full" ]; then | |
| : "${VALIDATION_FORCED:=1}" | |
| else | |
| : "${VALIDATION_FORCED:=0}" | |
| fi | |
| : "${VALIDATION_STATUS:=1}" | |
| if [ "${COMMAND}" = "/run-test-full" ]; then | |
| : "${DRIFT_SMOKE_RAN:=1}" | |
| : "${DRIFT_SMOKE_STATUS:=1}" | |
| else | |
| : "${DRIFT_SMOKE_RAN:=0}" | |
| : "${DRIFT_SMOKE_STATUS:=0}" | |
| fi | |
| : "${FULL_CI_STATUS:=1}" | |
| if [ "${UNIT_STATUS}" -ne 0 ]; then | |
| drift_state="failure" | |
| drift_description="Unit tests failed" | |
| elif [ "${DRIFT_STATUS}" -eq 0 ]; then | |
| drift_state="success" | |
| drift_description="Drift matched master; unit tests passed" | |
| else | |
| drift_state="failure" | |
| drift_description="Physics drift differs from master" | |
| fi | |
| if [ "${VALIDATION_RAN}" -eq 0 ] && [ "${DRIFT_STATUS}" -eq 0 ]; then | |
| validation_state="success" | |
| validation_description="Skipped: drift matched master" | |
| elif [ "${VALIDATION_STATUS}" -eq 0 ] && [ "${VALIDATION_FORCED}" -eq 1 ] && [ "${DRIFT_STATUS}" -eq 0 ]; then | |
| validation_state="success" | |
| validation_description="Full validation passed" | |
| elif [ "${VALIDATION_STATUS}" -eq 0 ]; then | |
| validation_state="success" | |
| validation_description="Validation tests passed" | |
| else | |
| validation_state="failure" | |
| validation_description="Validation tests failed" | |
| fi | |
| if [ "${FULL_CI_STATUS}" -eq 0 ] && [ "${DRIFT_SMOKE_RAN}" -eq 1 ]; then | |
| full_state="success" | |
| full_description="Drift smoke and validation passed" | |
| elif [ "${FULL_CI_STATUS}" -eq 0 ] && [ "${VALIDATION_RAN}" -eq 0 ]; then | |
| full_state="success" | |
| full_description="Drift matched master; validation skipped" | |
| elif [ "${FULL_CI_STATUS}" -eq 0 ] && [ "${DRIFT_STATUS}" -eq 0 ]; then | |
| full_state="success" | |
| full_description="Drift matched master; validation passed" | |
| elif [ "${FULL_CI_STATUS}" -eq 0 ]; then | |
| full_state="success" | |
| full_description="Validation passed after drift mismatch" | |
| elif [ "${UNIT_STATUS}" -ne 0 ]; then | |
| full_state="failure" | |
| full_description="Unit tests failed" | |
| elif [ "${DRIFT_SMOKE_RAN}" -eq 1 ] && [ "${DRIFT_SMOKE_STATUS}" -ne 0 ]; then | |
| full_state="failure" | |
| full_description="Drift smoke failed" | |
| elif [ "${VALIDATION_RAN}" -eq 1 ] && [ "${VALIDATION_STATUS}" -ne 0 ]; then | |
| full_state="failure" | |
| full_description="Validation tests failed" | |
| else | |
| full_state="failure" | |
| full_description="Self-hosted CI failed" | |
| fi | |
| post_status() { | |
| local state=$1 | |
| local context=$2 | |
| local description=$3 | |
| local payload | |
| local response_file | |
| local http_code | |
| local curl_status | |
| local attempt | |
| local delay=5 | |
| payload=$(jq -n \ | |
| --arg state "${state}" \ | |
| --arg context "${context}" \ | |
| --arg description "${description}" \ | |
| --arg target_url "${TARGET_URL}" \ | |
| '{state: $state, context: $context, description: $description, target_url: $target_url}') | |
| response_file=$(mktemp) | |
| for attempt in 1 2 3 4 5; do | |
| http_code=$(curl --silent --show-error \ | |
| --output "${response_file}" \ | |
| --write-out "%{http_code}" \ | |
| --connect-timeout 10 \ | |
| --max-time 60 \ | |
| -H "Authorization: Bearer ${GH_TOKEN}" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Content-Type: application/json" \ | |
| -X POST \ | |
| -d "${payload}" \ | |
| "https://api.github.com/repos/${GITHUB_REPOSITORY}/statuses/${PR_SHA}") && curl_status=0 || curl_status=$? | |
| if [ "${curl_status}" -eq 0 ] && [ "${http_code}" -ge 200 ] && [ "${http_code}" -lt 300 ]; then | |
| rm -f "${response_file}" | |
| return 0 | |
| fi | |
| if [ "${curl_status}" -eq 0 ] && [ "${http_code}" -ge 400 ] && [ "${http_code}" -lt 500 ]; then | |
| cat "${response_file}" >&2 | |
| rm -f "${response_file}" | |
| return 1 | |
| fi | |
| if [ "${attempt}" -lt 5 ]; then | |
| echo "Posting status '${context}' failed (attempt ${attempt}/5, curl=${curl_status}, http=${http_code}); retrying in ${delay}s" >&2 | |
| sleep "${delay}" | |
| delay=$((delay * 2)) | |
| fi | |
| done | |
| cat "${response_file}" >&2 | |
| rm -f "${response_file}" | |
| return 1 | |
| } | |
| post_status "${drift_state}" "AdePT / Drift + Unit (EL9)" "${drift_description}" | |
| post_status "${validation_state}" "AdePT / Validation (EL9)" "${validation_description}" | |
| post_status "${full_state}" "AdePT / Full CI (EL9)" "${full_description}" | |
| - name: Write summary | |
| if: always() | |
| run: | | |
| if [ -r "${RESULTS_FILE}" ]; then | |
| # shellcheck disable=SC1091 | |
| source "${RESULTS_FILE}" | |
| fi | |
| { | |
| echo "### Self-hosted PR CI" | |
| echo | |
| echo "- Command: \`${{ needs.prepare.outputs.command }}\`" | |
| echo "- PR: #${{ needs.prepare.outputs.pr_number }}" | |
| echo "- Runner: \`$(hostname)\`" | |
| echo "- User: \`$(whoami)\`" | |
| echo "- Workflow step outcome: \`${{ steps.run_ci.outcome }}\`" | |
| echo "- Drift status: \`${DRIFT_STATUS:-missing}\`" | |
| echo "- Drift smoke ran: \`${DRIFT_SMOKE_RAN:-missing}\`" | |
| echo "- Drift smoke status: \`${DRIFT_SMOKE_STATUS:-missing}\`" | |
| echo "- Unit status: \`${UNIT_STATUS:-missing}\`" | |
| echo "- Validation ran: \`${VALIDATION_RAN:-missing}\`" | |
| echo "- Validation forced: \`${VALIDATION_FORCED:-missing}\`" | |
| echo "- Validation status: \`${VALIDATION_STATUS:-missing}\`" | |
| echo "- Full CI status: \`${FULL_CI_STATUS:-missing}\`" | |
| echo "- Build root: \`/tmp/adept-self-hosted-ci/pr-${{ needs.prepare.outputs.pr_number }}\`" | |
| } >> "${GITHUB_STEP_SUMMARY}" | |
| - name: Fail workflow if full CI failed | |
| if: always() | |
| run: | | |
| if [ -r "${RESULTS_FILE}" ]; then | |
| # shellcheck disable=SC1091 | |
| source "${RESULTS_FILE}" | |
| else | |
| FULL_CI_STATUS=1 | |
| fi | |
| test "${FULL_CI_STATUS}" -eq 0 |