Skip to content

Commit 74ce9df

Browse files
committed
test(cluster): gate test execution on stable Raft leader election
Add `NodeHandle::metadata_group_leader()` which reads the observed metadata-group leader id from the node's local Raft state, returning `0` while an election is still in progress. Wire a leader-stability barrier into `ClusterHarness::spawn_three()` that polls until every node reports the same non-zero leader id before returning control to the test. This closes the race where CPU pressure delayed the initial Raft heartbeats past topology convergence, producing `not leader (leader hint: None)` errors on the first DDL or descriptor-lease call issued by the test. Also add the `pgwire_gateway_migration` binary to the nextest cluster test group so it runs serialised with the other 3-node integration tests and does not compete for threads.
1 parent 19c3929 commit 74ce9df

File tree

3 files changed

+53
-0
lines changed

3 files changed

+53
-0
lines changed

.config/nextest.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ binary(/cluster/)
4545
| binary(descriptor_versioning_cross_node)
4646
| binary(prepared_cache_invalidation)
4747
| binary(sql_cluster_cross_node_dml)
48+
| binary(pgwire_gateway_migration)
4849
'''
4950
test-group = 'cluster'
5051
threads-required = 'num-test-threads'

nodedb/tests/common/cluster_harness/cluster.rs

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,39 @@ impl TestCluster {
8686
)
8787
.await;
8888

89+
// CRITICAL: wait for the metadata Raft group to elect a leader
90+
// and for every node's local view to agree on the same leader id.
91+
//
92+
// Topology convergence + rolling-upgrade exit only guarantees
93+
// membership and wire version are agreed; they say nothing about
94+
// election state. Under heavy host load (e.g. running this test
95+
// immediately after another full-suite cluster test exits and
96+
// the unit-test pool ramps back up), the initial Raft heartbeat
97+
// window can be missed and the first `acquire`/`propose` issued
98+
// by the test races a re-election — surfacing as
99+
// `raft error: not leader (leader hint: None)` from a
100+
// descriptor-lease or DDL call.
101+
//
102+
// Waiting until every node reports the same non-zero leader id
103+
// closes the window deterministically. Symmetric to the
104+
// rolling-upgrade wait above: no retries, no flakes, no
105+
// wasted CI minutes on cleanup of a doomed cluster bringup.
106+
wait_for(
107+
"metadata group has stable leader visible on every node",
108+
Duration::from_secs(10),
109+
Duration::from_millis(20),
110+
|| {
111+
let leaders: Vec<u64> = cluster
112+
.nodes
113+
.iter()
114+
.map(|n| n.metadata_group_leader())
115+
.collect();
116+
let first = leaders[0];
117+
first != 0 && leaders.iter().all(|&l| l == first)
118+
},
119+
)
120+
.await;
121+
89122
Ok(cluster)
90123
}
91124

nodedb/tests/common/cluster_harness/node.rs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,25 @@ impl TestClusterNode {
320320
.unwrap_or(0)
321321
}
322322

323+
/// Observed metadata-group leader id from this node's local Raft
324+
/// state, or `0` if no leader is known yet (election in progress).
325+
/// Polled by the cluster harness `spawn_three()` to gate test
326+
/// execution on a stable leader — otherwise tests racing the first
327+
/// election see `not leader (leader hint: None)` errors when CPU
328+
/// pressure delays the initial heartbeats past topology convergence.
329+
pub fn metadata_group_leader(&self) -> u64 {
330+
let Some(observer) = self.shared.cluster_observer.get() else {
331+
return 0;
332+
};
333+
observer
334+
.group_status
335+
.group_statuses()
336+
.into_iter()
337+
.find(|g| g.group_id == nodedb_cluster::METADATA_GROUP_ID)
338+
.map(|g| g.leader_id)
339+
.unwrap_or(0)
340+
}
341+
323342
/// Number of active collections visible on this node (read through
324343
/// the local `SystemCatalog` redb — populated by the
325344
/// `MetadataCommitApplier` on every node via

0 commit comments

Comments
 (0)