Collect peer metrics as quickly as possible to avoid skewing results

Author: marta-lokhova

src/FSLibrary/MinBlockTimeTest.fs

@@ -195,14 +195,27 @@ let private withOverlayOnlyMode (formation: StellarFormation) (coreSets: CoreSet
         LogInfo "Disabling overlay-only mode"
         toggleOverlayOnlyMode formation coreSets
 
-let private readLedgerAgePercentiles (peer: Peer) : float * float =
+let private readLedgerAgePercentiles (peer: Peer) : Peer * float * float =
     let h = peer.GetMetrics().LedgerAgeClosedHistogram
-    float h.``75``, float h.``99``
+    peer, float h.``75``, float h.``99``
+
+let private collectLedgerAgePercentiles
+    (formation: StellarFormation)
+    (coreSets: CoreSet list)
+    : (Peer * float * float) list =
+    formation.NetworkCfg.PeersInSets(List.toArray coreSets)
+    |> List.map (fun peer -> async { return readLedgerAgePercentiles peer })
+    |> Async.Parallel
+    |> Async.RunSynchronously
+    |> Array.toList
 
 // Returns true iff every peer's ledger.age.closed-histogram satisfies:
 //   P75 in [0.80*T, 1.20*T)
 //   P99 <= 2*T
 //
+// Snapshot all peers' metrics before evaluating any peer. Core keeps closing
+// ledgers after loadgen exits, so serial fetch-and-check skews later peers.
+//
 // FIXME: the P75 tolerance is temporarily widened to +/-20% because
 // stellar-core currently has perf regressions that prevent the intended
 // +/-5% band from being achievable. Tighten this back to 0.95/1.05 (or
@@ -213,22 +226,20 @@ let private checkLedgerAgeSLA (formation: StellarFormation) (coreSets: CoreSet l
     let tHi = tf * 1.20
     let p99Max = tf * 2.0
     let mutable ok = true
+    let percentiles = collectLedgerAgePercentiles formation coreSets
 
-    formation.NetworkCfg.EachPeerInSets
-        (List.toArray coreSets)
-        (fun peer ->
-            let p75, p99 = readLedgerAgePercentiles peer
-            let peerOk = p75 >= tLo && p75 < tHi && p99 <= p99Max
-
-            LogInfo
-                "peer=%s T=%dms p75=%.0f p99=%.0f -> %s"
-                peer.ShortName.StringName
-                targetMs
-                p75
-                p99
-                (if peerOk then "PASS" else "FAIL")
-
-            if not peerOk then ok <- false)
+    for peer, p75, p99 in percentiles do
+        let peerOk = p75 >= tLo && p75 < tHi && p99 <= p99Max
+
+        LogInfo
+            "peer=%s T=%dms p75=%.0f p99=%.0f -> %s"
+            peer.ShortName.StringName
+            targetMs
+            p75
+            p99
+            (if peerOk then "PASS" else "FAIL")
+
+        if not peerOk then ok <- false
 
     ok
 

src/FSLibrary/StellarCorePeer.fs

@@ -27,7 +27,10 @@ type NetworkCfg with
             for i in 0 .. (coreSet.CurrentCount - 1) do
                 f (self.GetPeer coreSet i)
 
+    member self.PeersInSets(coreSetArray: CoreSet array) : Peer list =
+        [ for coreSet in coreSetArray do
+              for i in 0 .. (coreSet.CurrentCount - 1) do
+                  self.GetPeer coreSet i ]
+
     member self.EachPeerInSets (coreSetArray: CoreSet array) (f: Peer -> unit) : unit =
-        for coreSet in coreSetArray do
-            for i in 0 .. (coreSet.CurrentCount - 1) do
-                f (self.GetPeer coreSet i)
+        self.PeersInSets coreSetArray |> List.iter f

src/FSLibrary/StellarStatefulSets.fs

@@ -401,7 +401,11 @@ type StellarFormation with
             (fun peer -> peer.UpgradeNetworkSetting contractKey upgradeTime)
 
     member self.clearMetrics(coreSets: CoreSet list) =
-        self.NetworkCfg.EachPeerInSets(coreSets |> List.toArray) (fun peer -> peer.ClearMetrics())
+        self.NetworkCfg.PeersInSets(coreSets |> List.toArray)
+        |> List.map (fun peer -> async { peer.ClearMetrics() })
+        |> Async.Parallel
+        |> Async.RunSynchronously
+        |> ignore
 
     // This is similar to RunLoadgen but runs a 1/N fractional portion of a
     // given LoadGen on node 0 of each of N CoreSets.