ci: run tests under cargo-nextest to fix the load-dependent deadlock (closes #533)

.config/nextest.toml

@@ -0,0 +1,40 @@
+# cargo-nextest configuration (#533). nextest runs each test in its OWN process, which
+# (a) isolates shared-global-state / port-contention deadlocks that hung the old in-process
+# `cargo test --workspace` under parallel load, and (b) enforces a PER-TEST timeout so a stuck
+# test is killed and reported instead of hanging the whole CI job for hours.
+[profile.default]
+# Timeout budget. Most tests are sub-second; the heaviest are the Raft DST seed sweeps, which run
+# ~15-35s each once the simulation crates are opt-level 3 (see [profile.dev.package.*] in the root
+# Cargo.toml; unoptimized they were 100-260s and starved CI, which is what looked like a hang). 90s
+# is a generous ceiling for those under CI parallel load; `terminate-after = 3` SIGKILLs a test still
+# running after 3 * 90s = 270s and fails the run with a clear "TMT" line naming it -- turning any
+# genuine unbounded hang into a fast, diagnosable failure instead of a multi-hour runner burn.
+slow-timeout = { period = "90s", terminate-after = 3 }
+# A flaky test may be retried a bounded number of times (does NOT mask a hard failure: a test that
+# fails every attempt still fails). Keeps a genuinely flaky env hiccup from red-flagging a PR while
+# a real regression still blocks.
+retries = 2
+
+[profile.ci]
+# CI inherits default timeouts; fail-fast off so one failure does not hide others.
+fail-fast = false
+slow-timeout = { period = "90s", terminate-after = 3 }
+retries = 2
+final-status-level = "flaky"
+
+# io_uring tests contend on a LIMITED kernel resource (io_uring rings + registered buffers). Running
+# them concurrently with the rest of the workspace under load is the diagnosed cause of the
+# intermittent CI hang: the `fixed_datapath` ring tests (a 6-test module, matching the observed
+# "6-test binary hangs" signature) block on a ring operation that never completes under contention.
+# Pin every io_uring test to a single-slot group so they run SERIALLY, one ring lifecycle at a time.
+# The per-test timeout above remains the backstop; this removes the contention that triggered it.
+[test-groups]
+io-uring = { max-threads = 1 }
+
+[[profile.default.overrides]]
+filter = 'package(=ironcache-runtime) and (test(/io_uring/) or test(/fixed_datapath/) or test(/uring/))'
+test-group = 'io-uring'
+
+[[profile.ci.overrides]]
+filter = 'package(=ironcache-runtime) and (test(/io_uring/) or test(/fixed_datapath/) or test(/uring/))'
+test-group = 'io-uring'

.github/workflows/rust.yml

@@ -4,7 +4,8 @@
 # job short-circuits the whole workflow while the repo is docs-only (no root
 # Cargo.toml), so the workflow is inert until the first crate lands and active
 # thereafter. Gates: fmt, clippy (pedantic, -D warnings), test (ubuntu; Linux is
-# the only supported deployment OS), msrv (1.85), musl static build, and the
+# the only supported deployment OS; run under cargo-nextest for process-per-test
+# isolation + per-test timeouts, #533), msrv (1.85), musl static build, and the
 # invariant lints (INVARIANTS.md).
 name: rust
 
@@ -76,6 +77,10 @@ jobs:
     needs: guard
     if: needs.guard.outputs.has_cargo == 'true'
     runs-on: ${{ matrix.os }}
+    # Belt-and-suspenders wall cap (#533): nextest's per-test timeout catches a hung test, and this
+    # bounds the WHOLE job so a pathological run can never burn a 6-hour runner (the previous
+    # `cargo test --workspace` deadlocked under parallel load with no bound).
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
@@ -89,7 +94,16 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
-      - run: cargo test --workspace --all-features
+      # cargo-nextest (#533): runs each test in its OWN process, isolating the shared-state /
+      # port-contention deadlock that intermittently hung the old in-process `cargo test --workspace`
+      # under parallel load, and enforcing a per-test timeout (see .config/nextest.toml) so a stuck
+      # test is killed + named instead of hanging the job. Prebuilt binary, no compile.
+      - uses: taiki-e/install-action@nextest
+      - name: nextest run (workspace, all features)
+        run: cargo nextest run --profile ci --workspace --all-features
+      # nextest does NOT run doctests; run them separately so doc examples stay gated.
+      - name: doctests
+        run: cargo test --doc --workspace --all-features
 
   msrv:
     name: msrv (1.85)
@@ -134,6 +148,7 @@ jobs:
     name: io_uring datapath (Linux, default-off feature)
     needs: guard
     if: needs.guard.outputs.has_cargo == 'true'
+    timeout-minutes: 30
     # The OPTIONAL Linux io_uring backend (PROD-10 / #28, docs/design/IOURING_DATAPATH.md) is
     # default-OFF and Linux-only; this ubuntu job is the dedicated gate that BUILDS + TESTS it
     # with `--features io_uring` (the default jobs above prove the byte-unchanged default build).
@@ -149,18 +164,19 @@ jobs:
         with:
           components: clippy
       - uses: Swatinem/rust-cache@v2
+      - uses: taiki-e/install-action@nextest
       # Clippy (pedantic, -D warnings) over the io_uring feature so the Linux-only path is held to
       # the SAME lint bar as the default build (the macOS dev box cannot lint this cfg).
       - name: clippy --features io_uring
         run: cargo clippy -p ironcache-runtime -p ironcache --features io_uring --all-targets -- -D warnings
       # Build the binary with the io_uring datapath compiled in.
       - name: build --features io_uring
         run: cargo build -p ironcache-runtime -p ironcache --features io_uring
-      # Run the runtime crate's tests with io_uring on: the existing suite plus the io_uring
-      # `Runtime` round-trip (accept/recv/send over a real ring) prove the owned-buffer append
-      # semantics map correctly onto tokio-uring.
+      # Run the runtime crate's tests with io_uring on (under nextest, per #533): the existing suite
+      # plus the io_uring `Runtime` round-trip (accept/recv/send over a real ring) prove the
+      # owned-buffer append semantics map correctly onto tokio-uring.
       - name: test --features io_uring
-        run: cargo test -p ironcache-runtime --features io_uring
+        run: cargo nextest run --profile ci -p ironcache-runtime --features io_uring
 
   invariant-lints:
     name: invariant lints (INVARIANTS.md)

Cargo.toml

@@ -176,3 +176,18 @@ lto = "thin"
 codegen-units = 1
 panic = "abort"
 strip = "symbols"
+
+# DETERMINISTIC-SIMULATION SPEED (#533). The Raft DST seed sweeps run millions of simulated ticks;
+# unoptimized (debug/test-profile) they take 100-260s EACH, which starved CI under parallel load and
+# looked like a hang. Optimize ONLY the simulation crates in dev/test builds -- opt-level 3 gives the
+# sim loop a ~10-20x speedup (seconds, not minutes) while every other crate stays debug (fast compile,
+# debug-assertions + overflow-checks preserved, so the sims still exercise the checked build). This is
+# the standard DST posture (compile the harness optimized, keep asserts on).
+[profile.dev.package.ironcache-raft]
+opt-level = 3
+[profile.dev.package.ironcache-sim]
+opt-level = 3
+[profile.dev.package.ironcache-cluster]
+opt-level = 3
+[profile.dev.package.ironcache-store]
+opt-level = 3