@@ -20,7 +20,13 @@ will time out expensively after holding resources.
2020The status has hysteresis to avoid flapping at the boundary: it becomes
2121` busy ` when the sampled run queue reaches `high_watermark x
2222schedulers_online` and returns to ` normal` only when it falls to
23- ` low_watermark x schedulers_online ` . Thresholds are expressed as factors
23+ ` low_watermark x schedulers_online ` . A crossing must also hold for three
24+ consecutive samples before it is committed. Hysteresis alone does not
25+ cover the dominant case on a quiet node: an instantaneous run queue
26+ spikes whenever a wave of periodic timers wakes together, and such a
27+ spike clears within one sample, so a single-sample commit would refuse
28+ admission — and log a state change — while nothing is actually
29+ saturated. Thresholds are expressed as factors
2430of the online scheduler count so a configuration is portable across
2531machine sizes: a run queue of N x schedulers means roughly N runnable
2632processes ahead of any newly runnable one on every scheduler.
@@ -49,12 +55,19 @@ Configuration (`bondy_regulator` application environment, set via the
4955-define (DEFAULT_HIGH_WATERMARK , 8 ).
5056-define (DEFAULT_LOW_WATERMARK , 4 ).
5157-define (DEFAULT_SAMPLE_INTERVAL_MS , 100 ).
58+ % % Consecutive samples a crossing must hold before the status changes. An
59+ % % instantaneous run queue spikes whenever a wave of periodic timers wakes
60+ % % together, which an idle node does routinely; committing on one sample
61+ % % turns that into a refused HELLO and a pair of log lines.
62+ -define (DWELL_SAMPLES , 3 ).
5263
5364-record (state , {
5465 ref :: atomics :atomics_ref (),
5566 high :: pos_integer (),
5667 low :: non_neg_integer (),
57- interval_ms :: pos_integer ()
68+ interval_ms :: pos_integer (),
69+ % % Consecutive samples the pending (not yet committed) status has held.
70+ dwell = 0 :: non_neg_integer ()
5871}).
5972
6073% % API
@@ -63,6 +76,13 @@ Configuration (`bondy_regulator` application environment, set via the
6376-export ([start_link /0 ]).
6477-export ([status /0 ]).
6578
79+ -ifdef (TEST ).
80+ % % Exposed so the dwell window can be pinned directly: driving it through
81+ % % the sampler would mean manufacturing a real run-queue spike shorter
82+ % % than the sampling period, which is not reproducible.
83+ -export ([step /3 ]).
84+ -endif .
85+
6686% % GEN_SERVER CALLBACKS
6787-export ([code_change /3 ]).
6888-export ([handle_call /3 ]).
@@ -167,35 +187,41 @@ handle_cast(Event, State) ->
167187 }),
168188 {noreply , State }.
169189
170- handle_info (sample , State ) ->
171- # state {ref = Ref , high = High , low = Low } = State ,
190+ handle_info (sample , State0 ) ->
191+ # state {ref = Ref , high = High , low = Low , dwell = Dwell0 } = State0 ,
172192 RunQueue = erlang :statistics (total_run_queue_lengths_all ),
173193 ok = atomics :put (Ref , ? RUN_QUEUE_SLOT , RunQueue ),
174194
175195 Status = atomics :get (Ref , ? STATUS_SLOT ),
176-
177- case transition (Status , RunQueue , High , Low ) of
178- Status ->
179- ok ;
180- 1 ->
181- ok = atomics :put (Ref , ? STATUS_SLOT , 1 ),
182- ? LOG_NOTICE (#{
183- description =>
184- " Node entered the busy state: admission gates will "
185- " refuse new work until the run queue drains below "
186- " the low watermark."
187- run_queue => RunQueue ,
188- high_watermark => High ,
189- low_watermark => Low
190- });
191- 0 ->
192- ok = atomics :put (Ref , ? STATUS_SLOT , 0 ),
193- ? LOG_NOTICE (#{
194- description => " Node returned to the normal state."
195- run_queue => RunQueue ,
196- low_watermark => Low
197- })
198- end ,
196+ Pending = transition (Status , RunQueue , High , Low ),
197+
198+ State =
199+ case step (Status , Pending , Dwell0 ) of
200+ {hold , Dwell } ->
201+ State0 # state {dwell = Dwell };
202+ {commit , 1 } ->
203+ ok = atomics :put (Ref , ? STATUS_SLOT , 1 ),
204+ ? LOG_NOTICE (#{
205+ description =>
206+ " Node entered the busy state: admission gates will "
207+ " refuse new work until the run queue drains below "
208+ " the low watermark."
209+ run_queue => RunQueue ,
210+ high_watermark => High ,
211+ low_watermark => Low ,
212+ dwell_samples => ? DWELL_SAMPLES
213+ }),
214+ State0 # state {dwell = 0 };
215+ {commit , 0 } ->
216+ ok = atomics :put (Ref , ? STATUS_SLOT , 0 ),
217+ ? LOG_NOTICE (#{
218+ description => " Node returned to the normal state."
219+ run_queue => RunQueue ,
220+ low_watermark => Low ,
221+ dwell_samples => ? DWELL_SAMPLES
222+ }),
223+ State0 # state {dwell = 0 }
224+ end ,
199225
200226 {noreply , schedule_sample (State )};
201227handle_info (Info , State ) ->
@@ -217,6 +243,18 @@ code_change(_OldVsn, State, _Extra) ->
217243% % PRIVATE
218244% % =============================================================================
219245
246+ % % @private
247+ % % The dwell step. A crossing is committed only once it has held for
248+ % % `?DWELL_SAMPLES` consecutive samples; a return to the committed side
249+ % % voids any partial dwell, so a spike shorter than the dwell window
250+ % % never changes the status.
251+ step (Status , Status , _Dwell ) ->
252+ {hold , 0 };
253+ step (_Status , Pending , Dwell ) when Dwell + 1 >= ? DWELL_SAMPLES ->
254+ {commit , Pending };
255+ step (_Status , _Pending , Dwell ) ->
256+ {hold , Dwell + 1 }.
257+
220258% % @private
221259% % The hysteresis step: `1` (busy) at or above the high watermark, `0`
222260% % (normal) at or below the low watermark, unchanged in between.
0 commit comments