refactor: multi-version Slurm support and improved developer experience #1

Workflow file for this run

.github/workflows/test-scaling.yml at c4aeaaa

	name: Test Dynamic Worker Scaling

	on:
	push:
	branches: [ main ]
	pull_request:
	branches: [ main ]
	workflow_dispatch: # Allow manual trigger

	jobs:
	test-scaling:
	runs-on: ubuntu-latest
	timeout-minutes: 45

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3

	- name: Build Docker images
	run: docker compose build

	- name: Start cluster with 2 nodes (baseline)
	run: \|
	docker compose up -d --scale node=2
	echo "Waiting for services to start..."
	sleep 15

	- name: Wait for core services to be healthy
	run: \|
	echo "=== Waiting for MySQL ==="
	timeout 60 bash -c 'until docker compose ps mysql \| grep -q "healthy"; do
	echo "Waiting for MySQL..."
	sleep 2
	done'

	echo "=== Waiting for slurmdbd ==="
	timeout 60 bash -c 'until docker compose ps slurmdbd \| grep -q "healthy"; do
	echo "Waiting for slurmdbd..."
	sleep 2
	done'

	echo "=== Waiting for slurmctld ==="
	timeout 60 bash -c 'until docker compose ps slurmctld \| grep -q "healthy"; do
	echo "Waiting for slurmctld..."
	sleep 2
	done'

	echo "=== Waiting for slurmrestd ==="
	timeout 60 bash -c 'until docker compose ps slurmrestd \| grep -q "healthy"; do
	echo "Waiting for slurmrestd..."
	sleep 2
	done'

	echo "All core services are healthy"

	- name: Wait for initial 2 worker nodes to be healthy
	run: \|
	timeout 90 bash -c 'until [ "$(docker compose ps node --format json 2>/dev/null \| jq -s \"[.[] \| select(.Health == \\\"healthy\\\")] \| length\")" -eq 2 ]; do
	HEALTHY=$(docker compose ps node --format json 2>/dev/null \| jq -s "[.[] \| select(.Health == \"healthy\")] \| length")
	echo "Waiting for workers... ($HEALTHY/2 healthy)"
	sleep 2
	done'
	echo "All 2 initial worker nodes are healthy"

	- name: Register cluster
	run: \|
	./register_cluster.sh
	sleep 5

	- name: Verify baseline (2 nodes)
	run: \|
	echo "=== Container Status ==="
	docker compose ps
	echo ""
	echo "=== Slurm Node Info ==="
	docker exec slurmctld sinfo -N
	echo ""
	NODE_COUNT=$(docker exec slurmctld sinfo -N -h \| wc -l)
	if [ "$NODE_COUNT" -ne 2 ]; then
	echo "ERROR: Expected 2 nodes, found $NODE_COUNT"
	exit 1
	fi
	echo "✓ Verified 2 nodes are registered"

	- name: Run baseline test suite
	run: chmod +x test_cluster.sh && ./test_cluster.sh

	- name: Submit baseline test job
	run: \|
	JOB_ID=$(docker exec slurmctld bash -c "cd /data && sbatch --wrap='hostname' 2>&1" \| grep -oP 'Submitted batch job \K\d+')
	echo "Submitted job $JOB_ID"
	sleep 10
	docker exec slurmctld sacct -j $JOB_ID --format=JobID,State,ExitCode
	echo "✓ Baseline job completed"

	- name: Scale UP to 5 nodes
	run: \|
	echo "=== Scaling from 2 to 5 nodes ==="
	chmod +x scale_workers.sh
	./scale_workers.sh up 5

	- name: Verify 5 nodes are healthy and recognized
	run: \|
	echo "=== Verifying 5 nodes ==="
	CONTAINER_COUNT=$(docker compose ps node --format json 2>/dev/null \| jq -s '[.[] \| select(.Health == "healthy")] \| length')
	NODE_COUNT=$(docker exec slurmctld sinfo -N -h \| wc -l)

	echo "Healthy containers: $CONTAINER_COUNT"
	echo "Slurm registered nodes: $NODE_COUNT"

	docker exec slurmctld sinfo

	if [ "$CONTAINER_COUNT" -ne 5 ]; then
	echo "ERROR: Expected 5 healthy containers, found $CONTAINER_COUNT"
	exit 1
	fi

	if [ "$NODE_COUNT" -ne 5 ]; then
	echo "ERROR: Expected 5 Slurm nodes, found $NODE_COUNT"
	exit 1
	fi

	echo "✓ All 5 nodes healthy and registered"

	- name: Submit multi-node job (5 nodes)
	run: \|
	echo "=== Testing multi-node job ==="
	JOB_OUTPUT=$(docker exec slurmctld bash -c "srun -N 3 hostname" 2>&1 \|\| echo "FAILED")
	echo "Job output: $JOB_OUTPUT"

	if echo "$JOB_OUTPUT" \| grep -q "slurm-node"; then
	echo "✓ Multi-node job executed successfully"
	else
	echo "ERROR: Multi-node job failed"
	exit 1
	fi

	- name: Scale DOWN to 3 nodes
	run: \|
	echo "=== Scaling from 5 to 3 nodes ==="
	./scale_workers.sh up 3

	- name: Verify 3 nodes after scale down
	run: \|
	echo "=== Verifying 3 nodes ==="
	CONTAINER_COUNT=$(docker compose ps node --format json 2>/dev/null \| jq -s '[.[] \| select(.Health == "healthy")] \| length')
	NODE_COUNT=$(docker exec slurmctld sinfo -N -h \| wc -l)

	echo "Healthy containers: $CONTAINER_COUNT"
	echo "Slurm registered nodes: $NODE_COUNT"

	if [ "$CONTAINER_COUNT" -ne 3 ]; then
	echo "ERROR: Expected 3 healthy containers, found $CONTAINER_COUNT"
	exit 1
	fi

	if [ "$NODE_COUNT" -ne 3 ]; then
	echo "ERROR: Expected 3 Slurm nodes, found $NODE_COUNT"
	exit 1
	fi

	echo "✓ Scaled down to 3 nodes successfully"

	- name: Test cluster functionality after scale down
	run: \|
	JOB_ID=$(docker exec slurmctld bash -c "cd /data && sbatch --wrap='echo SCALE_DOWN_TEST' 2>&1" \| grep -oP 'Submitted batch job \K\d+')
	echo "Submitted test job $JOB_ID after scale down"
	sleep 10
	docker exec slurmctld sacct -j $JOB_ID --format=JobID,State,ExitCode
	echo "✓ Cluster functional after scale down"

	- name: Scale DOWN to minimum (1 node)
	run: \|
	echo "=== Scaling to minimum (1 node) ==="
	./scale_workers.sh up 1

	- name: Verify single node
	run: \|
	echo "=== Verifying 1 node ==="
	CONTAINER_COUNT=$(docker compose ps node --format json 2>/dev/null \| jq -s '[.[] \| select(.Health == "healthy")] \| length')
	NODE_COUNT=$(docker exec slurmctld sinfo -N -h \| wc -l)

	docker exec slurmctld sinfo

	if [ "$CONTAINER_COUNT" -ne 1 ]; then
	echo "ERROR: Expected 1 healthy container, found $CONTAINER_COUNT"
	exit 1
	fi

	if [ "$NODE_COUNT" -ne 1 ]; then
	echo "ERROR: Expected 1 Slurm node, found $NODE_COUNT"
	exit 1
	fi

	echo "✓ Single node configuration working"

	- name: Test single node job
	run: \|
	JOB_ID=$(docker exec slurmctld bash -c "cd /data && sbatch --wrap='echo SINGLE_NODE_TEST' 2>&1" \| grep -oP 'Submitted batch job \K\d+')
	echo "Submitted job $JOB_ID on single node"
	sleep 10
	docker exec slurmctld sacct -j $JOB_ID --format=JobID,State,ExitCode
	echo "✓ Single node job completed"

	- name: Scale UP again to 4 nodes
	run: \|
	echo "=== Scaling back up to 4 nodes ==="
	./scale_workers.sh up 4

	- name: Verify final scale to 4 nodes
	run: \|
	echo "=== Verifying 4 nodes ==="
	CONTAINER_COUNT=$(docker compose ps node --format json 2>/dev/null \| jq -s '[.[] \| select(.Health == "healthy")] \| length')
	NODE_COUNT=$(docker exec slurmctld sinfo -N -h \| wc -l)

	docker exec slurmctld sinfo

	if [ "$CONTAINER_COUNT" -ne 4 ]; then
	echo "ERROR: Expected 4 healthy containers, found $CONTAINER_COUNT"
	exit 1
	fi

	if [ "$NODE_COUNT" -ne 4 ]; then
	echo "ERROR: Expected 4 Slurm nodes, found $NODE_COUNT"
	exit 1
	fi

	echo "✓ Scaled back up to 4 nodes successfully"

	- name: Final cluster test
	run: \|
	echo "=== Final cluster functionality test ==="
	docker exec slurmctld scontrol show nodes

	# Submit a final multi-node job
	JOB_OUTPUT=$(docker exec slurmctld bash -c "srun -N 2 hostname" 2>&1 \|\| echo "FAILED")
	if echo "$JOB_OUTPUT" \| grep -q "slurm-node"; then
	echo "✓ Final multi-node job successful"
	else
	echo "ERROR: Final job failed"
	exit 1
	fi

	- name: Show scaling test summary
	run: \|
	echo "=== Scaling Test Summary ==="
	echo "✓ Started with 2 nodes"
	echo "✓ Scaled up to 5 nodes"
	echo "✓ Scaled down to 3 nodes"
	echo "✓ Scaled down to 1 node (minimum)"
	echo "✓ Scaled back up to 4 nodes"
	echo "✓ All jobs executed successfully"
	echo "✓ Dynamic scaling tests PASSED"

	- name: Show logs on failure
	if: failure()
	run: \|
	echo "=== Docker Compose Logs ==="
	docker compose logs --tail=100
	echo ""
	echo "=== Container Status ==="
	docker compose ps
	echo ""
	echo "=== Slurm Node Status ==="
	docker exec slurmctld sinfo \|\| true
	docker exec slurmctld scontrol show nodes \|\| true
	echo ""
	echo "=== Lock Directory ==="
	docker exec slurm-node-1 ls -la /data/.slurm_node_locks/ \|\| true

	- name: Cleanup
	if: always()
	run: docker compose down -v

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

refactor: multi-version Slurm support and improved developer experience #1

Workflow file

refactor: multi-version Slurm support and improved developer experience #1

Uh oh!

Workflow file for this run