kind-testing.yml

name: kind-testing

on:
  schedule:
    - cron: '0 0 * * 1-5' # Runs at midnight UTC (1 AM CET) Monday-Friday
  workflow_call:
    inputs:
      image_tag:
        description: 'Pre-built image tag to use (skips docker-build if provided)'
        required: false
        type: string
    secrets:
      ZWS_BOT_TOKEN:
        required: true
      HARBOR_URL:
        required: true
      HARBOR_READ_LOGIN:
        required: true
      HARBOR_READ_TOKEN:
        required: true
      BLOCKCHAIN_ACTIONS_TOKEN:
        required: true
      AWS_ACCESS_KEY_S3_USER:
        required: true
      AWS_SECRET_KEY_S3_USER:
        required: true
      CGR_USERNAME:
        required: true
      CGR_PASSWORD:
        required: true

permissions: {}

# Controls concurrent workflow runs:
# - Groups runs by git ref
# - Cancels in-progress runs for non-main/release branches
concurrency:
  group: kind-testing-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  ############################################################################
  # Docker Build Job
  # Only runs when called without a pre-built image_tag (e.g., scheduled runs)
  ############################################################################
  docker-build:
    if: inputs.image_tag == ''
    name: kind-testing/docker-build
    permissions:
      actions: read # Read workflow run information
      contents: write # Checkout repository code
      id-token: write # OIDC authentication
      pull-requests: read # read pull requests information
      packages: write # Publish Docker images
      attestations: write # Create build attestations
    uses: ./.github/workflows/docker-build.yml
    secrets:
      BLOCKCHAIN_ACTIONS_TOKEN: ${{ secrets.BLOCKCHAIN_ACTIONS_TOKEN }}
      AWS_ACCESS_KEY_S3_USER: ${{ secrets.AWS_ACCESS_KEY_S3_USER }}
      AWS_SECRET_KEY_S3_USER: ${{ secrets.AWS_SECRET_KEY_S3_USER }}
      CGR_USERNAME: ${{ secrets.CGR_USERNAME }}
      CGR_PASSWORD: ${{ secrets.CGR_PASSWORD }}

  ############################################################################
  # Kind Testing Job
  # Main job that sets up a Kubernetes cluster using Kind and runs the tests
  ############################################################################
  kind-testing:
    name: kind-testing (${{ matrix.name }})
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: cargo-check
            cargo-check: true
            runner-label: "runs-on=${{ github.run_id }}/runner=16cpu-linux-x64/spot=false/volume=200gb/extras=s3-cache"
          - name: threshold
            deployment-type: threshold
            num-parties: 4
            runner-label: "runs-on=${{ github.run_id }}/runner=64cpu-linux-x64/spot=false/volume=200gb/extras=s3-cache"
            enable-tls: true
          - name: centralized
            deployment-type: centralized
            num-parties: 1
            runner-label: "runs-on=${{ github.run_id }}/runner=64cpu-linux-x64/spot=false/volume=200gb/extras=s3-cache"
    needs:
      - docker-build
    if: |
      always() && !cancelled() &&
      (inputs.image_tag != '' || needs.docker-build.result == 'success')
    runs-on: ${{ matrix.runner-label }}

    steps:
      - name: Setup Runs-on
        uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
        with:
          metrics: cpu,network,memory,disk,io

      - name: Checkout Project
        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          submodules: true
          token: ${{ secrets.BLOCKCHAIN_ACTIONS_TOKEN }}
          persist-credentials: true

      - name: Get Rust version
        env:
          GH_WORKSPACE: ${{ github.workspace }}
        run: |
          version="$(grep 'channel' "$GH_WORKSPACE/rust-toolchain.toml" | awk -F' = ' '{print $2}' | tr -d '"')"
          echo "RUST_IMAGE_VERSION=$version" >> "$GITHUB_ENV"

      # ==========================================================================
      # LFS files are needed for backward compatibility tests
      # Install git LFS before checkout does not seem possible on our self-hosted runners, so we
      # need to do it right after and then manually pull LFS files
      - name: setup gitlfs from binary
        env:
          GIT_LFS_VERSION: '3.6.1'
        run: |
          wget "https://github.com/git-lfs/git-lfs/releases/download/v${GIT_LFS_VERSION}/git-lfs-linux-amd64-v${GIT_LFS_VERSION}.tar.gz"
          tar -xzf "git-lfs-linux-amd64-v${GIT_LFS_VERSION}.tar.gz"
          sudo ./git-lfs-"${GIT_LFS_VERSION}"/install.sh
          git lfs pull

      # ==========================================================================
      # Docker Buildx Setup
      # Configures Docker Buildx for building multi-platform images
      #
      - name: Set up Docker Buildx
        if: matrix.cargo-check == false
        uses: ./.github/actions/docker-setup-buildx
        with:
          platforms: linux/amd64 # Target platform for builds
          buildkitd-flags: --debug # Enable debug logging for BuildKit

      - name: Install kubectl
        if: matrix.cargo-check == false
        run: |
          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl.sha256"
          check_sum=$(echo "$(cat kubectl.sha256)  kubectl" | sha256sum --check)
          if [ "$check_sum" != "kubectl: OK" ]; then
            echo "kubectl checksum does not match"
            exit 1
          fi
          mkdir -p "$HOME/.local/bin"
          install -m 0755 kubectl "$HOME/.local/bin/kubectl"
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
          export PATH="$HOME/.local/bin:$PATH"
          kubectl version --client

      - name: Install KIND
        if: matrix.cargo-check == false
        run: |
          # For AMD64 / x86_64
          [ "$(uname -m)" = "x86_64" ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.30.0/kind-linux-amd64
          # For ARM64
          [ "$(uname -m)" = "aarch64" ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.30.0/kind-linux-arm64
          chmod +x ./kind
          mv ./kind /usr/local/bin/kind

      - name: Install Helm
        if: matrix.cargo-check == false
        run: |
          curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
          chmod 700 get_helm.sh
          ./get_helm.sh

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_S3_USER }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY_S3_USER }}
          aws-region: eu-west-3

      # Needed to be able to pull some docker images for the simulator test
      - name: Login to harbor Container Registry
        if: matrix.cargo-check == false
        uses: ./.github/actions/docker-login
        with:
          registry: ${{ secrets.HARBOR_URL }}
          username: ${{ secrets.HARBOR_READ_LOGIN }}
          password: ${{ secrets.HARBOR_READ_TOKEN }}

      # ==========================================================================
      # Rust toolchain setup
      #
      - name: Set up Rust
        uses: ./.github/actions/setup-rust-and-protoc
        with:
          rust-toolchain: ${{ env.RUST_IMAGE_VERSION }}
          rust-components: rustfmt, clippy, llvm-tools-preview, llvm-tools
          rust-cache: "false"
          setup-protoc: "false"

      # ==========================================================================
      # Setup cache for dockerfile and inject in docker
      #
      - name: Setup Cache
        uses: runs-on/cache@a5f51d6f3fece787d03b7b4e981c82538a0654ed # latest commit on `v4` branch at this time
        id: cache
        env:
          RUNS_ON_S3_BUCKET_CACHE: gh-actions-cache-eu-west-3
          RUNS_ON_AWS_REGION: eu-west-3
        with:
          path: |
            /home/ubuntu/.cargo/bin/
            /home/ubuntu/.cargo/registry/index/
            /home/ubuntu/.cargo/registry/cache/
            /home/ubuntu/.cargo/git/db/
            ${{ github.workspace }}/target
          key: |
            ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

      - name: Install Protoc
        uses: ./.github/actions/setup-rust-and-protoc
        with:
          setup-rust: "false"

      - name: Setup cargo nextest
        if: matrix.cargo-check == false
        run: |
          cargo install cargo-nextest@0.9.133 --locked --force
          # Verify the installed version
          cargo nextest --version

      - name: Setup cargo dylint
        if: matrix.cargo-check == true
        run: |
          cargo install cargo-dylint dylint-link --force

      - name: clippy and rustfmt versions
        if: matrix.cargo-check == true
        run: |
          cargo clippy -V && cargo fmt --version

      - name: Formatting
        if: matrix.cargo-check == true
        run: cargo fmt --all -- --check

      - name: Linting All features
        if: matrix.cargo-check == true
        env:
          PACKAGE_NAME: 'kms-core-client'
        run: cargo clippy --all-targets --all-features --package "${PACKAGE_NAME}" -- -D warnings

      - name: Linting dylint
        if: matrix.cargo-check == true
        run: cargo dylint --all

      # ==========================================================================
      # Test Execution
      # Runs the Kubernetes integration tests using the configured environment
      #
      - name: Setup Kind Cluster in threshold mode with 4 parties
        if: matrix.cargo-check == false
        id: setup
        env:
          DEPLOYMENT_TYPE: ${{ matrix.deployment-type }} # Deployment type: threshold or centralized
          GITHUB_TOKEN: ${{ secrets.ZWS_BOT_TOKEN }}
          HARBOR_READ_LOGIN: ${{ secrets.HARBOR_READ_LOGIN }}
          HARBOR_READ_TOKEN: ${{ secrets.HARBOR_READ_TOKEN }}
          KMS_CORE_IMAGE_TAG: ${{ inputs.image_tag || needs.docker-build.outputs.image_tag }}
          KMS_CORE_CLIENT_IMAGE_TAG: ${{ inputs.image_tag || needs.docker-build.outputs.image_tag }}
          NAMESPACE: kms-test-${{ matrix.deployment-type }}
          NUM_PARTIES: ${{ matrix.num-parties }} # Number of parties
          ENABLE_TLS: ${{ matrix.enable-tls || 'false' }} # Enable TLS for threshold mode
        run: |
          # Start KMS setup and wait for completion
          # Uses environment variables: NAMESPACE, KMS_CORE_IMAGE_TAG, KMS_CORE_CLIENT_IMAGE_TAG,
          # DEPLOYMENT_TYPE, NUM_PARTIES, ENABLE_TLS
          chmod +x ci/scripts/manage_lifecycle.sh
          chmod +x ci/scripts/deploy.sh
          ./ci/scripts/manage_lifecycle.sh start

      - name: Run Kubernetes Tests
        if: matrix.cargo-check == false
        id: tests
        env:
          DEPLOYMENT_TYPE: ${{ matrix.deployment-type }} # Deployment type: threshold or centralized
          NAMESPACE: kms-test-${{ matrix.deployment-type }}
          NUM_PARTIES: ${{ matrix.num-parties }} # Number of parties
          NEXTEST_PROFILE: ${{ github.event_name == 'schedule' && 'ci-nightly' || 'ci' }}
        run: |
          #============================================================
          # Here we run the tests and capture exit code
          #============================================================
          #
          set +e
          cargo nextest run --test 'kubernetes_test_'"${DEPLOYMENT_TYPE}"'*' --features kind_tests --profile "${NEXTEST_PROFILE}" --no-fail-fast
          echo "TEST_EXIT_CODE=$?" >> "${GITHUB_ENV}"
          set -e

          # Collect logs after tests complete
          chmod +x ci/scripts/deploy.sh
          ./ci/scripts/deploy.sh \
            --target kind-ci \
            --namespace "${NAMESPACE}" \
            --deployment-type "${DEPLOYMENT_TYPE}" \
            --collect-logs

      - name: Copy Test Results
        if: always() && !cancelled()
        env:
          NEXTEST_PROFILE: ${{ github.event_name == 'schedule' && 'ci-nightly' || 'ci' }}
        run: |
          NEXTEST_OUT="${GITHUB_WORKSPACE}/target/nextest/${NEXTEST_PROFILE}"
          if find "${NEXTEST_OUT}" -mindepth 1 -type f -print -quit 2>/dev/null | grep -q .; then
            echo "Files found in ${NEXTEST_OUT}"
            ls -R "${NEXTEST_OUT}"
            echo "TIMESTAMP=$(date +"%Y%m%d%H%M%S%3N")" >> "$GITHUB_ENV"
            cp -Lpr "${NEXTEST_OUT}"/junit.xml /tmp/junit.xml
            echo "EXIT_CODE=0" >> "$GITHUB_ENV"
          else
            echo "No junit test files found in ${NEXTEST_OUT}"
            echo "EXIT_CODE=1" >> "$GITHUB_ENV"
            exit 0
          fi
        shell: bash

      - name: Upload Test Results
        if: always() && !cancelled() && env.EXIT_CODE == 0
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: junit-test-report-${{ env.TIMESTAMP }}
          path: /tmp/junit.xml

      - name: Cleanup Kind Setup and Collect Logs
        if: always() && matrix.cargo-check == false
        env:
          DEPLOYMENT_TYPE: ${{ matrix.deployment-type }}
          NAMESPACE: kms-test-${{ matrix.deployment-type }}
          NUM_PARTIES: ${{ matrix.num-parties }}
        run: |
          ###################################################################
          # Collect logs before teardown (especially on failures)
          ###################################################################
          chmod +x ci/scripts/deploy.sh
          ./ci/scripts/deploy.sh \
            --target kind-ci \
            --namespace "${NAMESPACE}" \
            --deployment-type "${DEPLOYMENT_TYPE}" \
            --collect-logs

          # Stop setup script and cleanup resources
          # This will collect logs from pods based on DEPLOYMENT_TYPE and NUM_PARTIES
          SETUP_PID=$(cat .setup_pid 2>/dev/null || echo "")
          TAIL_PID=$(cat .tail_pid 2>/dev/null || echo "")
          chmod +x ci/scripts/manage_lifecycle.sh
          ./ci/scripts/manage_lifecycle.sh stop "${SETUP_PID}" "${TAIL_PID}"

      - name: Detect kms-core logs
        if: always() && matrix.cargo-check == false
        id: detect-kms-logs
        run: |
          shopt -s nullglob
          files=(logs/*.log logs/port-forward/*.log setup_kms.log)
          if [ ${#files[@]} -gt 0 ]; then
            echo "found=true" >> "$GITHUB_OUTPUT"
            echo "Found ${#files[@]} log files for upload."
          else
            echo "found=false" >> "$GITHUB_OUTPUT"
            echo "No kms-core logs found; skipping artifact upload."
          fi
        shell: bash

      - name: Upload kms-core logs
        if: always() && matrix.cargo-check == false && steps.detect-kms-logs.outputs.found == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: kms-core-${{ matrix.deployment-type }}-logs
          path: |
            logs/*.log
            logs/port-forward/*.log
            setup_kms.log
          retention-days: 7 # Keep logs for 7 days
          if-no-files-found: error

      - name: Check Test Results
        if: always() && matrix.cargo-check == false
        run: |
          # Fail the workflow if tests failed
          if [ "${TEST_EXIT_CODE:-0}" -ne 0 ]; then
            echo "Tests failed with exit code ${TEST_EXIT_CODE}"
            exit "${TEST_EXIT_CODE}"
          fi
          echo "Tests passed successfully"

  clean-up-build:
    name: kind-testing/clean-up-build
    if: |
      always() &&
      github.event_name != 'schedule' &&
      !contains(github.event.pull_request.labels.*.name, 'docker')
    needs:
      - docker-build
      - kind-testing
    runs-on: ubuntu-latest
    permissions:
      packages: write # Required to delete GitHub packages/container registry
    steps:
      # TODO: remove once GitHub runner images ship the CVE-2026-31431 kernel fix
      - name: Workaround CVE-2026-31431 (copy.fail)
        run: |
          echo "install algif_aead /bin/false" | sudo tee /etc/modprobe.d/disable-algif-aead.conf
          if lsmod | grep -q algif_aead; then
            sudo rmmod algif_aead || echo "WARNING: rmmod failed - module may be in use"
          elif modinfo algif_aead 2>/dev/null | grep -q builtin; then
            echo "WARNING: algif_aead built-in - modprobe.d blacklist has no effect"
          fi

      - name: Clean up build
        env:
          GH_TOKEN: ${{ github.token }}
          IMAGE_TAG: ${{ inputs.image_tag || needs.docker-build.outputs.image_tag }}
        run: |
          delete_package_versions_by_tag() {
            local package_name="$1"
            local tag="$2"

            # URL-encode package name (slashes -> %2F)
            local encoded_name
            encoded_name="$(jq -rn --arg v "$package_name" '$v|@uri')"

            local package_path="/orgs/zama-ai/packages/container/${encoded_name}"

            # Check package exists/readable
            if ! gh api -H "Accept: application/vnd.github+json" "$package_path" >/dev/null 2>&1; then
              echo "Package not readable/found: ${package_name}"
              return 0
            fi

            gh api --paginate -H "Accept: application/vnd.github+json" "${package_path}/versions?per_page=100" \
            | jq -r --arg tag "$tag" '
                .[]
                | [(.metadata.container.tags[]? // empty | select(startswith($tag)))] as $matched
                | select(($matched | length) > 0)
                | [.id, ($matched | join(","))] | @tsv
              ' \
            | while IFS=$'\t' read -r id matched_tags; do
                [ -z "$id" ] && continue
                echo "Deleting ${package_name} version_id=${id} tags=${matched_tags}"
                gh api --method DELETE -H "Accept: application/vnd.github+json" "${package_path}/versions/${id}"
              done
          }
          if [ -n "${IMAGE_TAG}" ]; then
            delete_package_versions_by_tag "kms/core-service-enclave" "${IMAGE_TAG}"
            delete_package_versions_by_tag "kms/core-service" "${IMAGE_TAG}"
            delete_package_versions_by_tag "kms/core-client" "${IMAGE_TAG}"
          fi

  # Test reporting job that runs after all tests complete
  # Only runs on pull requests to generate test reports
  test-reporter:
    name: kms-testing/test-reporter
    if: always() && !cancelled()
    needs:
      - kind-testing
    uses: ./.github/workflows/test-reporter.yml
    permissions:
      checks: write # Required to create GitHub checks for test results
      packages: read # Required to read GitHub packages/container registry
      issues: write # Required to create comments on issues
      pull-requests: write # Required to create comments on pull requests
      actions: read # Required to read workflow run information and download artifacts
      contents: read # Required to checkout repository code
    secrets:
      BLOCKCHAIN_ACTIONS_TOKEN: ${{ secrets.BLOCKCHAIN_ACTIONS_TOKEN }}