E2E Nightly #95
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Nightly | |
| on: | |
| schedule: | |
| # Run at 3:00 AM UTC every day | |
| - cron: "0 3 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| k8s-version: | |
| description: Kubernetes version to test | |
| required: true | |
| type: choice | |
| default: all | |
| options: | |
| - all | |
| - v1.32 | |
| - v1.33 | |
| - v1.34 | |
| - v1.35 | |
| suite: | |
| description: Test suite to run | |
| required: true | |
| type: choice | |
| default: all | |
| options: | |
| - all | |
| - chainsaw | |
| - go-e2e | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: e2e-nightly-${{ github.ref }}-${{ github.event_name }}-${{ inputs.k8s-version || 'all' }}-${{ inputs.suite || 'all' }} | |
| cancel-in-progress: true | |
| env: | |
| K3D_VERSION: "v5.8.3" | |
| CERT_MANAGER_VERSION: "v1.17.2" | |
| PROMETHEUS_IMAGE: "quay.io/prometheus/prometheus:v3.4.1" | |
| PROMETHEUS_CHART_VERSION: "27.22.0" | |
| STRESS_NG_IMAGE: "ghcr.io/alexei-led/stress-ng:0.20.01" | |
| CPU_BURN_IMAGE: "docker.io/library/busybox:1.37" | |
| jobs: | |
| prepare-matrix: | |
| name: Select nightly matrix | |
| runs-on: ${{ vars.RUNNER || 'ubuntu-latest' }} | |
| timeout-minutes: 5 | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| suite: ${{ steps.set-matrix.outputs.suite }} | |
| steps: | |
| - name: Select version and suite | |
| id: set-matrix | |
| shell: bash -Eeuo pipefail {0} | |
| run: | | |
| selected_version="${{ inputs.k8s-version || 'all' }}" | |
| selected_suite="${{ inputs.suite || 'all' }}" | |
| case "$selected_version" in | |
| v1.32) | |
| matrix='{"include":[{"k8s-version":"v1.32","k3s-image":"v1.32.13-k3s1"}]}' | |
| ;; | |
| v1.33) | |
| matrix='{"include":[{"k8s-version":"v1.33","k3s-image":"v1.33.11-k3s1"}]}' | |
| ;; | |
| v1.34) | |
| matrix='{"include":[{"k8s-version":"v1.34","k3s-image":"v1.34.7-k3s1"}]}' | |
| ;; | |
| v1.35) | |
| matrix='{"include":[{"k8s-version":"v1.35","k3s-image":"v1.35.4-k3s1"}]}' | |
| ;; | |
| all) | |
| matrix='{"include":[{"k8s-version":"v1.32","k3s-image":"v1.32.13-k3s1"},{"k8s-version":"v1.33","k3s-image":"v1.33.11-k3s1"},{"k8s-version":"v1.34","k3s-image":"v1.34.7-k3s1"},{"k8s-version":"v1.35","k3s-image":"v1.35.4-k3s1"}]}' | |
| ;; | |
| *) | |
| echo "Unknown k8s-version input: $selected_version" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| echo "matrix=$matrix" >> "$GITHUB_OUTPUT" | |
| echo "suite=$selected_suite" >> "$GITHUB_OUTPUT" | |
| test-e2e: | |
| name: E2E (K8s ${{ matrix.k8s-version }}) | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| needs: prepare-matrix | |
| runs-on: ${{ vars.RUNNER || 'ubuntu-latest' }} | |
| timeout-minutes: 75 | |
| env: | |
| SELECTED_SUITE: ${{ needs.prepare-matrix.outputs.suite }} | |
| CLUSTER_NAME: e2e-${{ matrix.k8s-version }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| KUBECONFIG: ${{ github.workspace }}/.kubeconfig-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.k8s-version }}-${{ needs.prepare-matrix.outputs.suite }} | |
| strategy: | |
| max-parallel: 4 | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.prepare-matrix.outputs.matrix) }} | |
| steps: | |
| - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - uses: ./.github/actions/setup-clean-docker-config | |
| - name: Verify Docker and buildx | |
| shell: bash -Eeuo pipefail {0} | |
| run: | | |
| echo "Docker version: $(docker version --format '{{.Client.Version}}')" | |
| echo "Docker config: ${DOCKER_CONFIG:-~/.docker}" | |
| if ! docker buildx version >/dev/null 2>&1; then | |
| echo "::error::Docker buildx is not available. Check that cli-plugins are accessible from DOCKER_CONFIG." | |
| docker buildx version || true | |
| ls -la "${DOCKER_CONFIG:-$HOME/.docker}/cli-plugins/" 2>/dev/null || echo "No cli-plugins directory found" | |
| exit 1 | |
| fi | |
| echo "buildx: $(docker buildx version)" | |
| - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 | |
| with: | |
| go-version-file: go.mod | |
| cache: true | |
| - name: Cache E2E container images | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: | | |
| /tmp/cert-manager-controller.tar | |
| /tmp/cert-manager-webhook.tar | |
| /tmp/cert-manager-cainjector.tar | |
| /tmp/cert-manager.yaml | |
| /tmp/prometheus.tar | |
| /tmp/stress-ng.tar | |
| /tmp/busybox.tar | |
| /tmp/k3s.tar | |
| key: e2e-images-${{ runner.os }}-cm${{ env.CERT_MANAGER_VERSION }}-prom${{ env.PROMETHEUS_CHART_VERSION }}-stress0.20.01-busybox1.37-k3s${{ matrix.k3s-image }} | |
| - uses: ./.github/actions/install-binary-tool | |
| with: | |
| name: k3d | |
| version: ${{ env.K3D_VERSION }} | |
| install-command: | | |
| curl -s https://raw.githubusercontent.com/k3d-io/k3d/${{ env.K3D_VERSION }}/install.sh | K3D_INSTALL_DIR="$TOOL_DIR" USE_SUDO=false bash | |
| - name: Cleanup stale k3d clusters and Docker resources | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| # Delete any k3d cluster that has been running for more than 1 hour. | |
| # Stale clusters from crashed/cancelled jobs waste Docker resources | |
| # and cause container name conflicts during creation. | |
| for cluster in $(k3d cluster list -o json 2>/dev/null | jq -r '.[].name // empty'); do | |
| # Get the server container start time; skip if we cannot determine age. | |
| started=$(docker inspect --format '{{.State.StartedAt}}' "k3d-${cluster}-server-0" 2>/dev/null) || continue | |
| started_epoch=$(date -d "$started" +%s 2>/dev/null) || continue | |
| age=$(( $(date +%s) - started_epoch )) | |
| if (( age > 3600 )); then | |
| echo "Deleting stale k3d cluster '$cluster' (age: ${age}s)" | |
| k3d cluster delete "$cluster" || true | |
| fi | |
| done | |
| docker builder prune -f || true | |
| docker system prune -f || true | |
| - name: Pre-pull container images | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| # Pull images with docker (parallel layer downloads) and save as | |
| # tarballs for k3d import. Skip images already restored from cache. | |
| # Accepts an optional 3rd argument: a fallback image reference to try | |
| # when the primary registry is unreachable (e.g. quay.io outage). | |
| # If the fallback succeeds, the image is retagged to the primary name | |
| # so downstream manifests (cert-manager.yaml) resolve correctly. | |
| pull_and_save() { | |
| local image="$1" tarball="$2" fallback="${3:-}" | |
| [[ -f "$tarball" ]] && { echo "Cached: $tarball"; return 0; } | |
| for attempt in 1 2 3; do | |
| if docker pull --platform linux/amd64 "$image"; then | |
| docker save "$image" -o "$tarball" | |
| return 0 | |
| fi | |
| echo "::warning::docker pull attempt $attempt for $image failed, retrying in 10s..." | |
| sleep 10 | |
| done | |
| if [[ -n "$fallback" ]]; then | |
| echo "::warning::Primary registry failed for $image, trying fallback: $fallback" | |
| for attempt in 1 2 3; do | |
| if docker pull --platform linux/amd64 "$fallback"; then | |
| docker tag "$fallback" "$image" | |
| docker save "$image" -o "$tarball" | |
| return 0 | |
| fi | |
| echo "::warning::fallback pull attempt $attempt for $fallback failed, retrying in 10s..." | |
| sleep 10 | |
| done | |
| fi | |
| echo "::error::Failed to pull $image after all attempts" | |
| return 1 | |
| } | |
| CM_VER="${{ env.CERT_MANAGER_VERSION }}" | |
| OWNER="attune-io" | |
| pull_and_save "quay.io/jetstack/cert-manager-controller:${CM_VER}" /tmp/cert-manager-controller.tar "ghcr.io/${OWNER}/mirrors/cert-manager-controller:${CM_VER}" | |
| pull_and_save "quay.io/jetstack/cert-manager-webhook:${CM_VER}" /tmp/cert-manager-webhook.tar "ghcr.io/${OWNER}/mirrors/cert-manager-webhook:${CM_VER}" | |
| pull_and_save "quay.io/jetstack/cert-manager-cainjector:${CM_VER}" /tmp/cert-manager-cainjector.tar "ghcr.io/${OWNER}/mirrors/cert-manager-cainjector:${CM_VER}" | |
| PROM_TAG="${{ env.PROMETHEUS_IMAGE }}" | |
| PROM_TAG="${PROM_TAG##*:}" | |
| pull_and_save "${{ env.PROMETHEUS_IMAGE }}" /tmp/prometheus.tar "docker.io/prom/prometheus:${PROM_TAG}" | |
| pull_and_save "${{ env.STRESS_NG_IMAGE }}" /tmp/stress-ng.tar | |
| pull_and_save "${{ env.CPU_BURN_IMAGE }}" /tmp/busybox.tar | |
| pull_and_save "rancher/k3s:${{ matrix.k3s-image }}" /tmp/k3s.tar | |
| - name: Create k3d cluster | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| rm -f "$KUBECONFIG" | |
| # Load the pre-pulled k3s image into Docker so k3d uses it | |
| # without pulling from Docker Hub (avoids rate-limit/timeout failures). | |
| docker load -i /tmp/k3s.tar | |
| # Retry cluster creation up to 3 times. k3d can fail transiently | |
| # when Docker containers vanish mid-creation due to daemon | |
| # contention with concurrent CI runs. | |
| # | |
| # K8s 1.32 feature gate: InPlacePodVerticalScaling is alpha and off | |
| # by default. We enable it via a k3s config file mounted into the | |
| # server container, NOT via --k3s-arg CLI flags. | |
| # | |
| # Why config file instead of --k3s-arg: | |
| # k3s processes --kube-apiserver-arg flags through GetArgs() in | |
| # pkg/util/args.go. CLI args have a timing race where GetArgs() | |
| # may process extras before defaults are populated (issue #296). | |
| # The config file is loaded during server bootstrap, reducing | |
| # but not eliminating the race (issue #325 showed += still fails | |
| # intermittently even via config file). | |
| # | |
| # We use = (set) not += (append) to avoid the GetArgs() timing | |
| # race entirely. The = syntax sets the feature gate directly | |
| # without depending on any existing value being populated first. | |
| # This is safe for K8s 1.32: k3s's default apiserver feature | |
| # gates at this version are all GA (locked on/off, not present | |
| # in --feature-gates), so there are no alpha/beta defaults to | |
| # preserve. | |
| EXTRA_K3D_ARGS=() | |
| if [[ "${{ matrix.k8s-version }}" == "v1.32" ]]; then | |
| K3S_CONFIG="/tmp/k3s-v132-config.yaml" | |
| cat > "$K3S_CONFIG" <<'K3SEOF' | |
| kube-apiserver-arg: | |
| - "feature-gates=InPlacePodVerticalScaling=true" | |
| kube-controller-manager-arg: | |
| - "feature-gates=InPlacePodVerticalScaling=true" | |
| kube-scheduler-arg: | |
| - "feature-gates=InPlacePodVerticalScaling=true" | |
| kubelet-arg: | |
| - "feature-gates=InPlacePodVerticalScaling=true" | |
| K3SEOF | |
| EXTRA_K3D_ARGS=(--volume "${K3S_CONFIG}:/etc/rancher/k3s/config.yaml@server:*") | |
| fi | |
| for attempt in 1 2 3; do | |
| if ! k3d cluster create "$CLUSTER_NAME" \ | |
| --image rancher/k3s:${{ matrix.k3s-image }} \ | |
| --k3s-arg "--disable=traefik,servicelb@server:*" \ | |
| "${EXTRA_K3D_ARGS[@]}" \ | |
| --wait --timeout 120s \ | |
| --kubeconfig-update-default=false \ | |
| --kubeconfig-switch-context=false; then | |
| echo "::warning::k3d cluster create attempt $attempt failed, retrying in 10s..." | |
| k3d cluster delete "$CLUSTER_NAME" 2>/dev/null || true | |
| sleep 10 | |
| if (( attempt == 3 )); then | |
| echo "::error::k3d cluster creation failed after 3 attempts" | |
| exit 1 | |
| fi | |
| continue | |
| fi | |
| break | |
| done | |
| k3d kubeconfig merge "$CLUSTER_NAME" --output "$KUBECONFIG" --overwrite | |
| - name: Wait for node Ready | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: kubectl wait --for=condition=Ready nodes --all --timeout=360s | |
| - name: Verify resize subresource (K8s 1.32) | |
| if: matrix.k8s-version == 'v1.32' | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| # InPlacePodVerticalScaling is alpha on 1.32. k3s has an intermittent | |
| # embedded-apiserver race (k3s-io/k3s#14286, tracked in attune #352) | |
| # where the gate is present in process args but pods/resize is never | |
| # registered. Recreating the cluster once recovers in practice. | |
| verify_resize() { | |
| local attempt | |
| for attempt in $(seq 1 24); do | |
| if kubectl get --raw /api/v1 | jq -e '.resources[] | select(.name == "pods/resize")' >/dev/null 2>&1; then | |
| echo "pods/resize subresource is registered" | |
| return 0 | |
| fi | |
| echo "Waiting for pods/resize subresource (attempt ${attempt}/24)..." | |
| sleep 5 | |
| done | |
| return 1 | |
| } | |
| recreate_v132_cluster() { | |
| echo "Recreating k3d cluster ${CLUSTER_NAME} for feature-gate race recovery..." | |
| k3d cluster delete "$CLUSTER_NAME" 2>/dev/null || true | |
| K3S_CONFIG="/tmp/k3s-v132-config.yaml" | |
| cat > "$K3S_CONFIG" <<'K3SEOF' | |
| kube-apiserver-arg: | |
| - "feature-gates=InPlacePodVerticalScaling=true" | |
| kube-controller-manager-arg: | |
| - "feature-gates=InPlacePodVerticalScaling=true" | |
| kube-scheduler-arg: | |
| - "feature-gates=InPlacePodVerticalScaling=true" | |
| kubelet-arg: | |
| - "feature-gates=InPlacePodVerticalScaling=true" | |
| K3SEOF | |
| k3d cluster create "$CLUSTER_NAME" \ | |
| --image "rancher/k3s:${{ matrix.k3s-image }}" \ | |
| --k3s-arg "--disable=traefik,servicelb@server:*" \ | |
| --volume "${K3S_CONFIG}:/etc/rancher/k3s/config.yaml@server:*" \ | |
| --wait --timeout 120s \ | |
| --kubeconfig-update-default=false \ | |
| --kubeconfig-switch-context=false | |
| k3d kubeconfig merge "$CLUSTER_NAME" --output "$KUBECONFIG" --overwrite | |
| kubectl wait --for=condition=Ready nodes --all --timeout=360s | |
| } | |
| echo "Verifying InPlacePodVerticalScaling feature gate is active..." | |
| echo "k3s config inside container:" | |
| docker exec "k3d-${CLUSTER_NAME}-server-0" cat /etc/rancher/k3s/config.yaml || true | |
| if verify_resize; then | |
| exit 0 | |
| fi | |
| echo "::warning::pods/resize missing after first cluster (known k3s race, #352). Recreating once..." | |
| recreate_v132_cluster | |
| echo "k3s config inside recreated container:" | |
| docker exec "k3d-${CLUSTER_NAME}-server-0" cat /etc/rancher/k3s/config.yaml || true | |
| if verify_resize; then | |
| exit 0 | |
| fi | |
| echo "::error::pods/resize subresource not found after recreate. The InPlacePodVerticalScaling feature gate did not take effect." | |
| echo "Diagnostic: k3s config file:" | |
| docker exec "k3d-${CLUSTER_NAME}-server-0" cat /etc/rancher/k3s/config.yaml || true | |
| echo "Diagnostic: k3s process (apiserver runs embedded):" | |
| docker exec "k3d-${CLUSTER_NAME}-server-0" sh -c "ps aux | grep k3s" || true | |
| echo "Diagnostic: API resources containing 'resize':" | |
| kubectl get --raw /api/v1 | jq '[.resources[] | select(.name | contains("resize"))]' || true | |
| echo "Diagnostic: k3s logs (last 100 lines):" | |
| docker logs "k3d-${CLUSTER_NAME}-server-0" --tail 100 2>&1 | grep -iE 'feature|gate|resize|error|fatal|panic' || true | |
| exit 1 | |
| - name: Load cert-manager images into cluster | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| k3d image import \ | |
| /tmp/cert-manager-controller.tar \ | |
| /tmp/cert-manager-webhook.tar \ | |
| /tmp/cert-manager-cainjector.tar \ | |
| -c "$CLUSTER_NAME" | |
| - name: Download cert-manager manifest | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| if [[ -f /tmp/cert-manager.yaml ]]; then | |
| echo "Cached: /tmp/cert-manager.yaml" | |
| else | |
| for attempt in 1 2 3; do | |
| if curl -fsSL -o /tmp/cert-manager.yaml \ | |
| "https://github.com/cert-manager/cert-manager/releases/download/${{ env.CERT_MANAGER_VERSION }}/cert-manager.yaml"; then | |
| break | |
| fi | |
| echo "::warning::cert-manager.yaml download attempt $attempt failed, retrying in 10s..." | |
| sleep 10 | |
| if (( attempt == 3 )); then | |
| echo "::error::Failed to download cert-manager.yaml after 3 attempts" | |
| exit 1 | |
| fi | |
| done | |
| fi | |
| - name: Install cert-manager | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| kubectl apply -f /tmp/cert-manager.yaml | |
| kubectl wait --for=condition=Available deployment/cert-manager -n cert-manager --timeout=120s | |
| kubectl wait --for=condition=Available deployment/cert-manager-webhook -n cert-manager --timeout=120s | |
| kubectl wait --for=condition=Available deployment/cert-manager-cainjector -n cert-manager --timeout=120s | |
| - name: Load Prometheus and test images into cluster | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: k3d image import /tmp/prometheus.tar /tmp/stress-ng.tar /tmp/busybox.tar -c "$CLUSTER_NAME" | |
| - name: Install Prometheus | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| helm repo add prometheus-community https://prometheus-community.github.io/helm-charts | |
| helm repo update | |
| # Pre-pull the chart tarball with retry to avoid transient CDN failures | |
| # (e.g. 504 Gateway Timeout from GitHub-hosted chart assets). | |
| for attempt in 1 2 3; do | |
| if helm pull prometheus-community/prometheus \ | |
| --version ${{ env.PROMETHEUS_CHART_VERSION }} \ | |
| --destination /tmp; then | |
| break | |
| fi | |
| echo "::warning::helm pull attempt $attempt failed, retrying in 10s..." | |
| sleep 10 | |
| if (( attempt == 3 )); then | |
| echo "::error::Failed to pull Prometheus chart after 3 attempts" | |
| exit 1 | |
| fi | |
| done | |
| helm install prometheus /tmp/prometheus-${{ env.PROMETHEUS_CHART_VERSION }}.tgz \ | |
| --namespace monitoring --create-namespace \ | |
| --set server.image.repository=quay.io/prometheus/prometheus \ | |
| --set server.image.tag=v3.4.1 \ | |
| --set server.persistentVolume.enabled=false \ | |
| --set alertmanager.enabled=false \ | |
| --set prometheus-pushgateway.enabled=false \ | |
| --set server.global.scrape_interval=15s \ | |
| --wait --timeout 5m | |
| - name: Wait for cAdvisor metrics in Prometheus | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| echo "Waiting for Prometheus to scrape cAdvisor metrics..." | |
| PROM_POD=$(kubectl get pods -n monitoring -l app.kubernetes.io/name=prometheus,app.kubernetes.io/component=server -o name | head -1) | |
| for i in $(seq 1 30); do | |
| result=$(kubectl exec -n monitoring "$PROM_POD" -- \ | |
| wget -qO- 'http://localhost:9090/api/v1/query?query=container_cpu_usage_seconds_total' 2>/dev/null || true) | |
| if echo "$result" | grep -q '"result":\[{'; then | |
| echo "cAdvisor metrics available after ${i}x5s" | |
| break | |
| fi | |
| if [ "$i" -eq 30 ]; then | |
| echo "WARNING: cAdvisor metrics not found after 150s, proceeding anyway" | |
| fi | |
| sleep 5 | |
| done | |
| - uses: ./.github/actions/install-go-tool | |
| with: | |
| name: ko | |
| version: v0.18.0 | |
| package: github.com/google/ko | |
| - name: Build and load operator image | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| # Use ko to build the image as an OCI tarball. This bypasses the | |
| # Docker daemon entirely, avoiding containerd storage races. | |
| VERSION=e2e COMMIT=$(git rev-parse --short HEAD) DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) \ | |
| KO_DOCKER_REPO=attune ko build ./cmd/manager/ \ | |
| --bare --tags=e2e --platform=linux/$(go env GOARCH) \ | |
| --tarball=/tmp/attune-e2e.tar --push=false | |
| k3d image import /tmp/attune-e2e.tar -c "$CLUSTER_NAME" | |
| - name: Verify operator image inside cluster | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| # k3d image import can report success while the underlying ctr | |
| # import fails silently (known issue). Verify the image is | |
| # actually available inside the node and retry if not. | |
| for attempt in 1 2 3; do | |
| if docker exec "k3d-${CLUSTER_NAME}-server-0" \ | |
| crictl images --output json 2>/dev/null \ | |
| | jq -e '.images[].repoTags[] | select(test("attune.*e2e"))' >/dev/null 2>&1; then | |
| echo "Image attune:e2e verified inside cluster" | |
| break | |
| fi | |
| echo "::warning::attune:e2e not found inside cluster (attempt $attempt), re-importing..." | |
| sleep 5 | |
| k3d image import /tmp/attune-e2e.tar -c "$CLUSTER_NAME" | |
| if (( attempt == 3 )); then | |
| echo "::error::attune:e2e image not available after 3 import attempts" | |
| docker exec "k3d-${CLUSTER_NAME}-server-0" crictl images || true | |
| exit 1 | |
| fi | |
| done | |
| - name: Install operator via Helm | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| helm install attune ./charts/attune \ | |
| --namespace attune-system --create-namespace \ | |
| --set image.repository=attune \ | |
| --set image.tag=e2e \ | |
| --set image.pullPolicy=Never \ | |
| --set webhooks.enabled=true \ | |
| --set metrics.enabled=true \ | |
| --set leaderElection.enabled=false \ | |
| --set maxConcurrentReconciles=4 \ | |
| --set logging.level=1 \ | |
| --set resources.limits.memory=512Mi \ | |
| --set resources.requests.memory=256Mi \ | |
| --wait --timeout 3m | |
| - name: Install Chainsaw | |
| if: inputs.suite != 'go-e2e' | |
| uses: ./.github/actions/install-binary-tool | |
| with: | |
| name: chainsaw | |
| version: v0.2.15 | |
| install-command: | | |
| ARCH=$(uname -m) | |
| case "$ARCH" in | |
| x86_64) ARCH=amd64 ;; | |
| aarch64) ARCH=arm64 ;; | |
| esac | |
| URL="https://github.com/kyverno/chainsaw/releases/download/v0.2.15/chainsaw_linux_${ARCH}.tar.gz" | |
| for attempt in 1 2 3; do | |
| if curl -fsSL "$URL" -o chainsaw.tar.gz; then | |
| tar -xf chainsaw.tar.gz -C "$TOOL_DIR" chainsaw | |
| rm chainsaw.tar.gz | |
| chmod +x "$TOOL_DIR/chainsaw" | |
| exit 0 | |
| fi | |
| echo "::warning::Chainsaw download attempt $attempt failed, retrying in 10s..." | |
| sleep 10 | |
| done | |
| echo "::error::Failed to download Chainsaw after 3 attempts" | |
| exit 1 | |
| - name: Run Chainsaw suite | |
| if: inputs.suite != 'go-e2e' | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| mkdir -p test-results | |
| chainsaw test test/e2e/ --config .chainsaw.yaml 2>&1 | tee test-results/chainsaw-${{ matrix.k8s-version }}.log | |
| - name: Run Go E2E tests | |
| if: inputs.suite != 'chainsaw' | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| mkdir -p test-results | |
| go test -tags=e2e ./test/e2e-go/... -race -count=1 -timeout=15m -v 2>&1 | tee test-results/go-e2e-${{ matrix.k8s-version }}.log | |
| - name: Collect debug info on failure | |
| if: failure() | |
| shell: bash -Eeuo pipefail {0} | |
| run: | | |
| mkdir -p test-results | |
| { | |
| echo "=== cluster name ===" | |
| echo "$CLUSTER_NAME" | |
| echo "=== kubeconfig path ===" | |
| echo "$KUBECONFIG" | |
| echo "=== kubeconfig server ===" | |
| kubectl config view --minify --raw -o jsonpath='{.clusters[0].cluster.server}' || true | |
| printf '\n' | |
| echo "=== cert-manager pods ===" | |
| kubectl get pods -n cert-manager || true | |
| echo "=== operator logs ===" | |
| kubectl logs -n attune-system -l app.kubernetes.io/name=attune --tail=300 || true | |
| echo "=== pod status ===" | |
| kubectl get pods -A || true | |
| echo "=== pod descriptions ===" | |
| kubectl describe pods -A || true | |
| echo "=== events (attune-system) ===" | |
| kubectl get events -n attune-system --sort-by='.lastTimestamp' || true | |
| echo "=== events (all namespaces, last 100) ===" | |
| kubectl get events -A --sort-by='.lastTimestamp' | tail -100 || true | |
| } | tee test-results/debug-${{ matrix.k8s-version }}.log | |
| - name: Upload test artifacts | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| with: | |
| name: e2e-nightly-${{ matrix.k8s-version }}-${{ env.SELECTED_SUITE }} | |
| path: test-results/ | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Cleanup k3d cluster | |
| if: always() | |
| shell: bash -Eeuo pipefail {0} | |
| run: | | |
| k3d cluster delete "$CLUSTER_NAME" 2>/dev/null || true | |
| rm -f "$KUBECONFIG" | |
| fuzz: | |
| name: Fuzz Tests | |
| runs-on: ${{ vars.RUNNER || 'ubuntu-latest' }} | |
| timeout-minutes: 20 | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| steps: | |
| - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 | |
| with: | |
| go-version-file: go.mod | |
| cache: true | |
| - name: Run fuzz tests (coverage-guided, 30s per target) | |
| shell: bash -Eeuo pipefail -x {0} | |
| run: | | |
| # Time-based limits let Go's coverage-guided engine focus on | |
| # inputs that explore new code paths. 30s per target achieves | |
| # full branch coverage faster than 1M random iterations. | |
| go test ./internal/recommendation/... -run='^$' -fuzz=FuzzPercentileEstimator -fuzztime=30s | |
| go test ./internal/recommendation/... -run='^$' -fuzz=FuzzRecommendationEngine -fuzztime=30s | |
| go test ./internal/webhook/... -run='^$' -fuzz=FuzzValidateFloatFields -fuzztime=30s | |
| report: | |
| name: Nightly Results | |
| runs-on: ${{ vars.RUNNER || 'ubuntu-latest' }} | |
| timeout-minutes: 5 | |
| permissions: | |
| issues: write | |
| needs: [test-e2e, fuzz] | |
| if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Summary | |
| shell: bash -Eeuo pipefail {0} | |
| run: | | |
| e2e_result="${{ needs.test-e2e.result }}" | |
| fuzz_result="${{ needs.fuzz.result }}" | |
| failed=0 | |
| if [[ "$e2e_result" != "success" && "$e2e_result" != "skipped" ]]; then | |
| echo "::error::E2E nightly failed (result: $e2e_result)" | |
| failed=1 | |
| fi | |
| if [[ "$fuzz_result" != "success" && "$fuzz_result" != "skipped" ]]; then | |
| echo "::error::Fuzz nightly failed (result: $fuzz_result)" | |
| failed=1 | |
| fi | |
| if (( failed )); then | |
| exit 1 | |
| fi | |
| echo "All nightly tests passed (E2E: $e2e_result, Fuzz: $fuzz_result)" | |
| - name: Create issue on failure | |
| if: failure() && github.event_name == 'schedule' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| shell: bash -Eeuo pipefail {0} | |
| run: | | |
| # Only create if no open issue with this label already exists. | |
| existing=$(gh issue list --label e2e-nightly-failure --state open --json number --jq 'length') | |
| if [[ "$existing" -gt 0 ]]; then | |
| echo "Open nightly failure issue already exists, skipping creation" | |
| exit 0 | |
| fi | |
| body=$(cat <<EOF | |
| The [nightly run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) failed. | |
| - E2E result: \`${{ needs.test-e2e.result }}\` | |
| - Fuzz result: \`${{ needs.fuzz.result }}\` | |
| Check the workflow run for details. | |
| EOF | |
| ) | |
| gh issue create \ | |
| --title "Nightly failed on $(date -u +%Y-%m-%d)" \ | |
| --label e2e-nightly-failure \ | |
| --body "$body" |