fix(managed): bound timeout work per poll #55
No files matched your search
@@ -591,6 +591,8 @@ pub struct ManagedClientTask {
|
|||||||
stagger_observations: Option<StaggerObservations>,
|
stagger_observations: Option<StaggerObservations>,
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
drain_test_hook: DrainTestHook,
|
drain_test_hook: DrainTestHook,
|
||||||
|
#[cfg(test)]
|
||||||
|
timeout_inspections: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -1730,6 +1732,10 @@ impl ManagedClientTask {
|
|||||||
for _ in 0..TIMEOUT_WORK_BUDGET {
|
for _ in 0..TIMEOUT_WORK_BUDGET {
|
||||||
let index = self.timeout_cursor;
|
let index = self.timeout_cursor;
|
||||||
self.timeout_cursor = (self.timeout_cursor + 1) % target_len;
|
self.timeout_cursor = (self.timeout_cursor + 1) % target_len;
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
self.timeout_inspections += 1;
|
||||||
|
}
|
||||||
if self.target_has_due_timeout(index, now) {
|
if self.target_has_due_timeout(index, now) {
|
||||||
let step = self.poll_target_timeout(index, now);
|
let step = self.poll_target_timeout(index, now);
|
||||||
more_due |= step.more_due;
|
more_due |= step.more_due;
|
||||||
@@ -2263,6 +2269,8 @@ fn build_task(
|
|||||||
stagger_observations: None,
|
stagger_observations: None,
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
drain_test_hook: DrainTestHook::default(),
|
drain_test_hook: DrainTestHook::default(),
|
||||||
|
#[cfg(test)]
|
||||||
|
timeout_inspections: 0,
|
||||||
};
|
};
|
||||||
let handle = ManagedClientHandle {
|
let handle = ManagedClientHandle {
|
||||||
stop,
|
stop,
|
||||||
|
|||||||
@@ -1738,6 +1738,77 @@ fn stagger_gate_tracks_active_membership() {
|
|||||||
second.finish();
|
second.finish();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn timeout_discovery_inspects_at_most_one_budget_of_targets() {
|
||||||
|
let target_count = TIMEOUT_WORK_BUDGET + 1;
|
||||||
|
let mut managed = config(ManagedPacing::Burst);
|
||||||
|
managed.max_live_target_generations = target_count;
|
||||||
|
let targets = (0..target_count)
|
||||||
|
.map(|index| ManagedTargetConfig::new(format!("target-{index}"), "127.0.0.1:9"))
|
||||||
|
.collect();
|
||||||
|
let (mut task, _) = ManagedClient::task(managed, targets).unwrap();
|
||||||
|
|
||||||
|
task.timeout_inspections = 0;
|
||||||
|
assert!(!task.poll_timeout_pass(Instant::now()));
|
||||||
|
|
||||||
|
assert_eq!(task.timeout_inspections, TIMEOUT_WORK_BUDGET);
|
||||||
|
assert_eq!(task.timeout_cursor, TIMEOUT_WORK_BUDGET);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn timeout_discovery_reaches_later_target_after_budget_slice() {
|
||||||
|
let server = start_server(ServerBehavior::Echo, None);
|
||||||
|
let target_count = TIMEOUT_WORK_BUDGET + 1;
|
||||||
|
let late_index = target_count - 1;
|
||||||
|
let mut managed = config(ManagedPacing::Burst);
|
||||||
|
managed.completion = ManagedCompletionPolicy::ExplicitStop;
|
||||||
|
managed.client.duration = None;
|
||||||
|
managed.max_live_target_generations = target_count;
|
||||||
|
let targets = (0..target_count)
|
||||||
|
.map(|index| {
|
||||||
|
if index == late_index {
|
||||||
|
target("late", server.addr)
|
||||||
|
} else {
|
||||||
|
ManagedTargetConfig::new(format!("target-{index}"), "127.0.0.1:9")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let (mut task, _) = ManagedClient::task(managed, targets).unwrap();
|
||||||
|
for target in &mut task.targets[..late_index] {
|
||||||
|
target.state = TargetState::Terminal;
|
||||||
|
}
|
||||||
|
let observations = Arc::new(Mutex::new(Vec::new()));
|
||||||
|
task.event_observations = Some(Arc::clone(&observations));
|
||||||
|
let runtime = runtime();
|
||||||
|
open_target_for_timeout_test(&runtime, &mut task, late_index);
|
||||||
|
let now = Instant::now();
|
||||||
|
replace_pending_for_timeout_test(
|
||||||
|
&mut task,
|
||||||
|
late_index,
|
||||||
|
timeout_probe(0, now - Duration::from_secs(1), now),
|
||||||
|
);
|
||||||
|
|
||||||
|
task.timeout_inspections = 0;
|
||||||
|
assert!(!task.poll_timeout_pass(now));
|
||||||
|
assert_eq!(task.timeout_inspections, TIMEOUT_WORK_BUDGET);
|
||||||
|
assert_eq!(task.timeout_cursor, late_index);
|
||||||
|
assert_eq!(timeout_losses_for(&observations, "late"), 0);
|
||||||
|
|
||||||
|
task.timeout_inspections = 0;
|
||||||
|
assert!(!task.poll_timeout_pass(now));
|
||||||
|
assert_eq!(task.timeout_inspections, TIMEOUT_WORK_BUDGET);
|
||||||
|
assert_eq!(timeout_losses_for(&observations, "late"), 1);
|
||||||
|
|
||||||
|
let TargetState::Active { mut client } =
|
||||||
|
mem::replace(&mut task.targets[late_index].state, TargetState::Terminal)
|
||||||
|
else {
|
||||||
|
panic!("later timeout target did not remain active");
|
||||||
|
};
|
||||||
|
runtime.block_on(client.close()).unwrap();
|
||||||
|
drop(task);
|
||||||
|
server.finish();
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn timeout_work_is_bounded_per_task_poll_and_eventually_drains() {
|
fn timeout_work_is_bounded_per_task_poll_and_eventually_drains() {
|
||||||
let server = start_server(ServerBehavior::Echo, None);
|
let server = start_server(ServerBehavior::Echo, None);
|
||||||
@@ -1864,6 +1935,101 @@ fn timeout_work_round_robins_simultaneous_target_backlogs() {
|
|||||||
second.finish();
|
second.finish();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn budget_deferred_timeout_blocks_send_until_timeout_processing() {
|
||||||
|
let reply_gate = Arc::new(PacketGate::default());
|
||||||
|
let server = start_server_with_gates(
|
||||||
|
ServerBehavior::Echo,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
Some(Arc::clone(&reply_gate)),
|
||||||
|
);
|
||||||
|
let target_count = TIMEOUT_WORK_BUDGET + 1;
|
||||||
|
let late_index = target_count - 1;
|
||||||
|
let mut managed = config(ManagedPacing::Burst);
|
||||||
|
managed.completion = ManagedCompletionPolicy::ExplicitStop;
|
||||||
|
managed.client.duration = None;
|
||||||
|
managed.client.interval = Duration::from_nanos(1);
|
||||||
|
managed.client.probe_timeout = Duration::from_secs(1);
|
||||||
|
managed.client.max_pending_probes = 1;
|
||||||
|
managed.max_live_target_generations = target_count;
|
||||||
|
let targets = (0..target_count)
|
||||||
|
.map(|index| {
|
||||||
|
if index == late_index {
|
||||||
|
target("late", server.addr)
|
||||||
|
} else {
|
||||||
|
ManagedTargetConfig::new(format!("target-{index}"), "127.0.0.1:9")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let (mut task, _) = ManagedClient::task(managed, targets).unwrap();
|
||||||
|
for target in &mut task.targets[..late_index] {
|
||||||
|
target.state = TargetState::Terminal;
|
||||||
|
}
|
||||||
|
let observations = Arc::new(Mutex::new(Vec::new()));
|
||||||
|
task.event_observations = Some(Arc::clone(&observations));
|
||||||
|
let runtime = runtime();
|
||||||
|
open_target_for_timeout_test(&runtime, &mut task, late_index);
|
||||||
|
task.targets[late_index].desired = true;
|
||||||
|
let sent = match &mut task.targets[late_index].state {
|
||||||
|
TargetState::Active { client } => runtime.block_on(client.send_probe()).unwrap(),
|
||||||
|
_ => panic!("deferred-send target did not open"),
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
sent.as_slice(),
|
||||||
|
[ClientEvent::EchoSent { seq: 0, .. }]
|
||||||
|
));
|
||||||
|
wait_flag(&server.probe_seen);
|
||||||
|
|
||||||
|
let overdue = Instant::now();
|
||||||
|
replace_pending_for_timeout_test(
|
||||||
|
&mut task,
|
||||||
|
late_index,
|
||||||
|
timeout_probe(0, overdue - Duration::from_secs(1), overdue),
|
||||||
|
);
|
||||||
|
task.state = DriverState::Running;
|
||||||
|
task.lifecycle = ManagedLifecycle::Running;
|
||||||
|
task.timeout_cursor = 0;
|
||||||
|
task.send_cursor = late_index;
|
||||||
|
|
||||||
|
let mut task = Box::pin(task);
|
||||||
|
poll_task_once(&runtime, &mut task);
|
||||||
|
assert_eq!(task.timeout_inspections, TIMEOUT_WORK_BUDGET);
|
||||||
|
assert_eq!(timeout_losses_for(&observations, "late"), 0);
|
||||||
|
assert!(matches!(
|
||||||
|
&task.targets[late_index].state,
|
||||||
|
TargetState::Active { client }
|
||||||
|
if client.packets_sent() == 1
|
||||||
|
&& client.next_probe_timeout_deadline() == Some(overdue)
|
||||||
|
));
|
||||||
|
assert!(!task.targets[late_index].send_waiting);
|
||||||
|
|
||||||
|
poll_task_once(&runtime, &mut task);
|
||||||
|
assert_eq!(timeout_losses_for(&observations, "late"), 1);
|
||||||
|
assert!(matches!(
|
||||||
|
&task.targets[late_index].state,
|
||||||
|
TargetState::Active { client }
|
||||||
|
if client.next_probe_timeout_deadline().is_none()
|
||||||
|
));
|
||||||
|
|
||||||
|
poll_task_once(&runtime, &mut task);
|
||||||
|
assert!(matches!(
|
||||||
|
&task.targets[late_index].state,
|
||||||
|
TargetState::Active { client } if client.packets_sent() == 2
|
||||||
|
));
|
||||||
|
|
||||||
|
reply_gate.release();
|
||||||
|
let TargetState::Active { mut client } =
|
||||||
|
mem::replace(&mut task.targets[late_index].state, TargetState::Terminal)
|
||||||
|
else {
|
||||||
|
panic!("deferred-send target did not remain active");
|
||||||
|
};
|
||||||
|
runtime.block_on(client.close()).unwrap();
|
||||||
|
drop(task);
|
||||||
|
server.finish();
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn partial_timeout_batch_blocks_active_receive_until_loss_commits() {
|
fn partial_timeout_batch_blocks_active_receive_until_loss_commits() {
|
||||||
let reply_gate = Arc::new(PacketGate::default());
|
let reply_gate = Arc::new(PacketGate::default());
|
||||||
|
|||||||
Reference in new issue
Block a user