feat: remediate cves in gosu #91
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Slurm Cluster | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| workflow_dispatch: # Allow manual trigger | |
| jobs: | |
| build-gpu: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Read config from .env.example | |
| id: config | |
| run: | | |
| echo "slurm_version=$(grep '^SLURM_VERSION=' .env.example | cut -d= -f2)" >> "$GITHUB_OUTPUT" | |
| echo "builder_base=$(grep '#BUILDER_BASE=.*nvidia' .env.example | sed 's/^#//' | cut -d= -f2)" >> "$GITHUB_OUTPUT" | |
| echo "runtime_base=$(grep '#RUNTIME_BASE=.*nvidia' .env.example | sed 's/^#//' | cut -d= -f2)" >> "$GITHUB_OUTPUT" | |
| - name: Build GPU image | |
| run: | | |
| docker build \ | |
| --build-arg SLURM_VERSION=${{ steps.config.outputs.slurm_version }} \ | |
| --build-arg GPU_ENABLE=true \ | |
| --build-arg BUILDER_BASE=${{ steps.config.outputs.builder_base }} \ | |
| --build-arg RUNTIME_BASE=${{ steps.config.outputs.runtime_base }} \ | |
| . | |
| test-cluster: | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false # Allow all matrix jobs to complete independently | |
| matrix: | |
| # All versions on amd64, latest only on arm64 | |
| slurm_version: ['25.11.2', '25.05.6'] | |
| runner: ['ubuntu-latest'] | |
| monitoring: ['disabled', 'enabled'] | |
| include: | |
| - runner: ubuntu-latest | |
| arch: amd64 | |
| - slurm_version: '25.11.2' | |
| runner: ubuntu-24.04-arm | |
| arch: arm64 | |
| monitoring: disabled | |
| - slurm_version: '25.11.2' | |
| runner: ubuntu-24.04-arm | |
| arch: arm64 | |
| monitoring: enabled | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Configure cluster - Slurm ${{ matrix.slurm_version }} (${{ matrix.arch }}) - Monitoring ${{ matrix.monitoring }} | |
| run: | | |
| echo "SLURM_VERSION=${{ matrix.slurm_version }}" > .env | |
| if [ "${{ matrix.monitoring }}" == "enabled" ]; then | |
| echo "ELASTICSEARCH_HOST=http://elasticsearch:9200" >> .env | |
| echo "✓ Monitoring ENABLED" | |
| else | |
| echo "✓ Monitoring DISABLED" | |
| fi | |
| cat .env | |
| - name: Build cluster | |
| run: make build | |
| - name: Start cluster | |
| run: | | |
| make up | |
| echo "Waiting for services to start..." | |
| sleep 15 | |
| - name: Wait for MySQL to be healthy | |
| run: | | |
| timeout 60 bash -c 'until docker compose ps mysql | grep -q "healthy"; do | |
| echo "Waiting for MySQL..." | |
| sleep 2 | |
| done' | |
| echo "MySQL is healthy" | |
| - name: Wait for slurmdbd to be healthy | |
| run: | | |
| timeout 60 bash -c 'until docker compose ps slurmdbd | grep -q "healthy"; do | |
| echo "Waiting for slurmdbd..." | |
| sleep 2 | |
| done' | |
| echo "slurmdbd is healthy" | |
| - name: Wait for slurmctld to be healthy | |
| run: | | |
| timeout 60 bash -c 'until docker compose ps slurmctld | grep -q "healthy"; do | |
| echo "Waiting for slurmctld..." | |
| sleep 2 | |
| done' | |
| echo "slurmctld is healthy" | |
| - name: Wait for slurmrestd to be healthy | |
| run: | | |
| timeout 60 bash -c 'until docker compose ps slurmrestd | grep -q "healthy"; do | |
| echo "Waiting for slurmrestd..." | |
| sleep 2 | |
| done' | |
| echo "slurmrestd is healthy" | |
| - name: Wait for compute nodes to be healthy | |
| run: | | |
| timeout 90 bash -c 'until docker compose ps cpu-worker 2>/dev/null | grep -q "healthy"; do | |
| echo "Waiting for cpu-workers..." | |
| sleep 2 | |
| done' | |
| echo "CPU workers are healthy" | |
| echo "Waiting for cluster to auto-register with slurmdbd..." | |
| sleep 10 | |
| - name: Wait for Elasticsearch to be healthy | |
| if: matrix.monitoring == 'enabled' | |
| run: | | |
| timeout 90 bash -c 'until docker compose ps elasticsearch | grep -q "healthy"; do | |
| echo "Waiting for Elasticsearch..." | |
| sleep 2 | |
| done' | |
| echo "Elasticsearch is healthy" | |
| - name: Wait for Kibana to be healthy | |
| if: matrix.monitoring == 'enabled' | |
| run: | | |
| timeout 120 bash -c 'until docker compose ps kibana | grep -q "healthy"; do | |
| echo "Waiting for Kibana..." | |
| sleep 2 | |
| done' | |
| echo "Kibana is healthy" | |
| - name: Show cluster status | |
| run: make status | |
| - name: Show monitoring status | |
| if: matrix.monitoring == 'enabled' | |
| run: | | |
| echo "=== Elasticsearch Health ===" | |
| docker exec elasticsearch curl -s "http://localhost:9200/_cluster/health?pretty" | |
| echo "" | |
| echo "=== Kibana Status ===" | |
| docker exec kibana curl -s "http://localhost:5601/api/status" | head -20 | |
| - name: Run test suite | |
| run: make test | |
| - name: Run monitoring test suite | |
| if: matrix.monitoring == 'enabled' | |
| run: make test-monitoring | |
| - name: Show logs on failure | |
| if: failure() | |
| run: make logs | |
| - name: Cleanup | |
| if: always() | |
| run: make clean |