fix(client): prevent stagger gate from accumulating wakeup latency

This commit is contained in:
Nils Andreas Svee committed 2026-09-14 15:33:56 +07:00
1 parent b9c2d024d9
commit c426884fd1
2 files changed
+97 -1

No files matched your search

+25 -1
View File
@@ -1613,6 +1613,30 @@ impl ManagedClientTask {
self.send_gate = self.send_gate.filter(|gate| *gate > now);
}
/// The next stagger gate after a send was accepted at `accepted_at`.
///
/// The gate advances on its **own** cadence rather than being re-anchored
/// on every observed accept time. Anchoring on `accepted_at` folds each
/// send's wakeup latency into the next slot and then into every slot after
/// it, so the gate — and with it the whole group's send cadence — drifts
/// later without bound while the per-target schedules stay on their
/// absolute grid. The drift is not hypothetical: it shows up as a
/// monotonically growing `EchoSent::timer_error` and, once it exceeds one
/// interval, as probe slots the schedule then skips.
///
/// `accepted_at` is still the anchor in the two cases where continuing the
/// old cadence would be wrong: there is no previous gate, or the pacer has
/// fallen a full slot behind it. Re-anchoring there is what keeps a gate
/// stranded in the past from releasing a burst of catch-up sends once the
/// group starts moving again.
fn next_stagger_gate(&self, spacing: Duration, accepted_at: Instant) -> Option<Instant> {
let base = match self.send_gate {
Some(gate) if gate <= accepted_at && accepted_at.duration_since(gate) < spacing => gate,
_ => accepted_at,
};
base.checked_add(spacing)
}
fn record_stagger_acceptance(
&mut self,
result: SendResult,
@@ -1626,7 +1650,7 @@ impl ManagedClientTask {
return;
};
self.last_stagger_send = Some(accepted_at);
self.send_gate = accepted_at.checked_add(spacing);
self.send_gate = self.next_stagger_gate(spacing, accepted_at);
#[cfg(test)]
if let Some(observations) = &self.stagger_observations {
observations.lock().unwrap().push((_active, spacing));
+72
View File
@@ -1895,6 +1895,78 @@ fn stagger_gate_tracks_active_membership() {
second.finish();
}
/// The stagger gate must not fold each send's observed wakeup latency into
/// every later slot.
///
/// The gate is the binding constraint on when the *group* may send next (see
/// `next_deadline`), while each target's own schedule stays on an absolute
/// grid anchored at its open. Re-anchoring the gate on every observed accept
/// time therefore made the whole group's cadence drift later without bound:
/// `EchoSent::timer_error` grew monotonically, and once the drift exceeded one
/// interval `skip_missed_probe_slots_at` started dropping probe slots
/// outright. Measured before this was fixed, a single-target 20 s run at a
/// 50 ms interval sent 381 probes instead of 400.
///
/// This asserts the invariant directly rather than through a real-time run,
/// because reproducing accumulation behaviorally needs seconds of wall clock
/// and a jitter assumption. Two properties, both durable:
///
/// - while the pacer keeps up, N accepted sends advance the gate by exactly
/// N spacings from the first anchor, whatever lateness each send is
/// observed with;
/// - a gate the pacer has fallen a full spacing behind re-anchors on the
/// accept time instead, so a stranded gate cannot release a burst of
/// catch-up sends.
#[test]
fn the_stagger_gate_advances_by_one_spacing_per_send_without_accumulating_lateness() {
// No socket is involved: the gate is driven directly and the task is never
// polled, so an unreachable address is all a valid target set needs here.
let (mut task, _) = ManagedClient::task(
config(ManagedPacing::Staggered),
vec![target("only", "127.0.0.1:9".parse().unwrap())],
)
.unwrap();
let spacing = Duration::from_millis(50);
let start = Instant::now();
let lateness = Duration::from_millis(3);
task.record_stagger_acceptance(
SendResult::Ready { accepted: true },
Some((1, spacing)),
start,
);
assert_eq!(task.send_gate, Some(start + spacing), "first send anchors");
// Every later send is observed `lateness` after the gate that released it.
// The gate must still advance one spacing per send from `start`, not
// `spacing + lateness`.
for slot in 1..=8_u32 {
let released_at = start + spacing * slot;
task.record_stagger_acceptance(
SendResult::Ready { accepted: true },
Some((1, spacing)),
released_at + lateness,
);
assert_eq!(
task.send_gate,
Some(start + spacing * (slot + 1)),
"slot {slot} accumulated observed lateness into the gate"
);
}
// A full spacing behind is no longer the same cadence, so the gate
// re-anchors rather than staying stranded in the past.
let stalled_gate = task.send_gate.expect("the gate is set");
let resumed_at = stalled_gate + spacing;
task.record_stagger_acceptance(
SendResult::Ready { accepted: true },
Some((1, spacing)),
resumed_at,
);
assert_eq!(task.send_gate, Some(resumed_at + spacing));
}
#[test]
fn timeout_discovery_inspects_at_most_one_budget_of_targets() {
let target_count = TIMEOUT_WORK_BUDGET + 1;