fix(managed): bound timeout work per poll #55
No files matched your search
@@ -425,8 +425,26 @@ fn finite_multi_target_peer_close_is_accepted_as_completion() {
|
||||
#[test]
|
||||
fn continuous_all_peer_closed_targets_exit_nonzero() {
|
||||
let params = test_params(None, Duration::from_millis(10));
|
||||
let a = start_peer_close_server(params.clone());
|
||||
let b = start_peer_close_server(params);
|
||||
let (a_seen_tx, a_seen_rx) = mpsc::channel();
|
||||
let (a_release_tx, a_release_rx) = mpsc::channel();
|
||||
let (b_seen_tx, b_seen_rx) = mpsc::channel();
|
||||
let (b_release_tx, b_release_rx) = mpsc::channel();
|
||||
let a = start_synchronized_peer_close_server(params.clone(), a_seen_tx, a_release_rx);
|
||||
let b = start_synchronized_peer_close_server(params, b_seen_tx, b_release_rx);
|
||||
let coordinator = thread::spawn(move || {
|
||||
a_seen_rx
|
||||
.recv_timeout(Duration::from_secs(2))
|
||||
.expect("timed out waiting for target A's first probe");
|
||||
b_seen_rx
|
||||
.recv_timeout(Duration::from_secs(2))
|
||||
.expect("timed out waiting for target B's first probe");
|
||||
a_release_tx
|
||||
.send(())
|
||||
.expect("target A stopped waiting for peer-close release");
|
||||
b_release_tx
|
||||
.send(())
|
||||
.expect("target B stopped waiting for peer-close release");
|
||||
});
|
||||
let mut command = Command::new(env!("CARGO_BIN_EXE_irtt-cli"));
|
||||
command.args([
|
||||
"--duration",
|
||||
@@ -446,6 +464,7 @@ fn continuous_all_peer_closed_targets_exit_nonzero() {
|
||||
]);
|
||||
|
||||
let output = run_with_timeout(command, Duration::from_secs(3));
|
||||
coordinator.join().unwrap();
|
||||
a.join();
|
||||
b.join();
|
||||
|
||||
@@ -860,6 +879,55 @@ fn start_peer_close_server(params: Params) -> FakeServer {
|
||||
start_gated_peer_close_server(params, Vec::new())
|
||||
}
|
||||
|
||||
fn start_synchronized_peer_close_server(
|
||||
params: Params,
|
||||
first_probe_seen_tx: mpsc::Sender<()>,
|
||||
release_peer_close_rx: mpsc::Receiver<()>,
|
||||
) -> FakeServer {
|
||||
let socket = UdpSocket::bind("127.0.0.1:0").unwrap();
|
||||
socket
|
||||
.set_read_timeout(Some(Duration::from_secs(2)))
|
||||
.unwrap();
|
||||
let addr = socket.local_addr().unwrap();
|
||||
let done = thread::spawn(move || {
|
||||
let (_, peer) = recv_request(&socket);
|
||||
socket
|
||||
.send_to(&open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms), peer)
|
||||
.unwrap();
|
||||
let (packet, peer) = recv_request(&socket);
|
||||
let flags = packet[3];
|
||||
assert_eq!(
|
||||
flags, 0,
|
||||
"target closed before synchronized first peer-close probe"
|
||||
);
|
||||
assert_eq!(
|
||||
packet.len(),
|
||||
echo_packet_len(false, ¶ms).unwrap(),
|
||||
"expected a complete echo probe before synchronized peer-close reply"
|
||||
);
|
||||
let seq = u32::from_le_bytes(packet[12..16].try_into().unwrap());
|
||||
first_probe_seen_tx
|
||||
.send(())
|
||||
.expect("peer-close coordinator stopped waiting for first probe");
|
||||
release_peer_close_rx
|
||||
.recv_timeout(Duration::from_secs(2))
|
||||
.expect("timed out waiting to release synchronized peer-close reply");
|
||||
socket
|
||||
.send_to(
|
||||
&echo_reply_packet(
|
||||
TOKEN,
|
||||
seq,
|
||||
¶ms,
|
||||
&TimestampFields::default(),
|
||||
FLAG_REPLY | FLAG_CLOSE,
|
||||
),
|
||||
peer,
|
||||
)
|
||||
.unwrap();
|
||||
});
|
||||
FakeServer { addr, done }
|
||||
}
|
||||
|
||||
fn start_gated_peer_close_server(
|
||||
params: Params,
|
||||
release_gates: Vec<mpsc::Receiver<()>>,
|
||||
@@ -875,10 +943,19 @@ fn start_gated_peer_close_server(
|
||||
.send_to(&open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms), peer)
|
||||
.unwrap();
|
||||
let (packet, peer) = recv_request(&socket);
|
||||
let flags = packet[3];
|
||||
if flags & FLAG_CLOSE != 0 {
|
||||
return;
|
||||
}
|
||||
for gate in release_gates {
|
||||
gate.recv_timeout(Duration::from_secs(2))
|
||||
.expect("timed out waiting to release peer-close reply");
|
||||
}
|
||||
assert_eq!(
|
||||
packet.len(),
|
||||
echo_packet_len(false, ¶ms).unwrap(),
|
||||
"expected a complete echo probe before peer-close reply"
|
||||
);
|
||||
let seq = u32::from_le_bytes(packet[12..16].try_into().unwrap());
|
||||
socket
|
||||
.send_to(
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::{
|
||||
receive::try_recv_tokio_datagram,
|
||||
session::machine::{
|
||||
recv_buffer_size, OpenDatagramDisposition, PreparedOpenAcceptance, PreparedOpenRequest,
|
||||
PreparedProbe, SessionMachine, MAX_OPEN_PACKET_SIZE,
|
||||
PreparedProbe, SessionMachine, TimeoutBatch, MAX_OPEN_PACKET_SIZE,
|
||||
},
|
||||
socket::{connect_tokio_udp_socket, resolve_remote_tokio, validate_open_timeouts},
|
||||
socket_options::{apply_dscp_to_tokio_socket, clear_dscp_on_tokio_socket},
|
||||
@@ -300,6 +300,14 @@ impl AsyncClient {
|
||||
self.machine.poll_timeouts_at(now)
|
||||
}
|
||||
|
||||
pub(crate) fn poll_timeouts_bounded_at(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
limit: usize,
|
||||
) -> Result<TimeoutBatch, ClientError> {
|
||||
self.machine.poll_timeouts_bounded_at(now, limit)
|
||||
}
|
||||
|
||||
/// Send the retained close packet and commit local close exactly once.
|
||||
///
|
||||
/// DSCP is cleared only after write readiness and immediately before the
|
||||
@@ -765,6 +773,19 @@ impl AsyncClient {
|
||||
self.socket.poll_recv_ready(cx)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn remove_pending_for_test(
|
||||
&mut self,
|
||||
wire_seq: u32,
|
||||
) -> Option<crate::probe::PendingProbe> {
|
||||
self.machine.remove_pending_for_test(wire_seq)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn replace_pending_for_test(&mut self, probe: crate::probe::PendingProbe) {
|
||||
self.machine.replace_pending_for_test(probe);
|
||||
}
|
||||
|
||||
pub(crate) fn skip_missed_probe_slots_at(&mut self, now: Instant) -> Result<(), ClientError> {
|
||||
let Some(schedule) = self.schedule.as_mut() else {
|
||||
return Ok(());
|
||||
|
||||
@@ -16,7 +16,10 @@ use irtt_proto::{
|
||||
use tokio::runtime::{Builder, Runtime};
|
||||
|
||||
use super::*;
|
||||
use crate::{socket_options::tokio_socket_traffic_class, Client, RunMode, SocketConfig};
|
||||
use crate::{
|
||||
probe::PendingProbe, socket_options::tokio_socket_traffic_class, Client, ClientTimestamp,
|
||||
RunMode, SocketConfig,
|
||||
};
|
||||
|
||||
const TOKEN: u64 = 0x1234_5678_90ab_cdef;
|
||||
|
||||
@@ -708,6 +711,35 @@ fn probe_false_readiness_resamples_all_timestamps() {
|
||||
assert_eq!(server.finish().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_timeout_polling_remains_exhaustive() {
|
||||
let server = start_open_server(None, 0);
|
||||
runtime().block_on(async {
|
||||
let mut client = opened_client(server.addr, 0).await;
|
||||
let now = Instant::now();
|
||||
for seq in 0..3 {
|
||||
client.replace_pending_for_test(PendingProbe {
|
||||
wire_seq: seq,
|
||||
sent_at: ClientTimestamp {
|
||||
mono: now - Duration::from_secs(2),
|
||||
wall: SystemTime::UNIX_EPOCH,
|
||||
},
|
||||
timeout_at: now - Duration::from_secs(1),
|
||||
});
|
||||
}
|
||||
|
||||
assert!(matches!(
|
||||
client.poll_timeouts_at(now).unwrap().as_slice(),
|
||||
[
|
||||
ClientEvent::EchoLoss { seq: 0, .. },
|
||||
ClientEvent::EchoLoss { seq: 1, .. },
|
||||
ClientEvent::EchoLoss { seq: 2, .. },
|
||||
]
|
||||
));
|
||||
});
|
||||
assert_eq!(server.finish().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn successful_short_probe_commits_before_length_error() {
|
||||
let server = start_open_server(None, 0);
|
||||
|
||||
@@ -37,6 +37,7 @@ use super::{
|
||||
|
||||
const TARGET_WORK_BUDGET: usize = 128;
|
||||
const COMMAND_WORK_BUDGET: usize = 32;
|
||||
const TIMEOUT_WORK_BUDGET: usize = 128;
|
||||
const POST_DEADLINE_RECEIVE_BUDGET: usize = TARGET_WORK_BUDGET;
|
||||
// Tokio 1.53 broadcast asserts this exact bound before rounding capacity to a
|
||||
// power of two; Tokio does not expose it as a public constant.
|
||||
@@ -415,6 +416,12 @@ struct TargetRuntime {
|
||||
state: TargetState,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct TimeoutBacklogEntry {
|
||||
index: usize,
|
||||
generation: u64,
|
||||
}
|
||||
|
||||
struct PlannedTarget {
|
||||
target: ManagedTargetConfig,
|
||||
client_config: ClientConfig,
|
||||
@@ -573,6 +580,8 @@ pub struct ManagedClientTask {
|
||||
wake: Option<WakeFuture>,
|
||||
timer: Option<Pin<Box<Sleep>>>,
|
||||
cursor: usize,
|
||||
timeout_cursor: usize,
|
||||
timeout_backlog: VecDeque<TimeoutBacklogEntry>,
|
||||
scan_remaining: usize,
|
||||
send_cursor: usize,
|
||||
burst_remaining: usize,
|
||||
@@ -589,6 +598,10 @@ pub struct ManagedClientTask {
|
||||
stagger_observations: Option<StaggerObservations>,
|
||||
#[cfg(test)]
|
||||
drain_test_hook: DrainTestHook,
|
||||
#[cfg(test)]
|
||||
timeout_inspections: usize,
|
||||
#[cfg(test)]
|
||||
deadline_inspections: usize,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1046,9 +1059,11 @@ impl ManagedClientTask {
|
||||
let len = self.targets.len();
|
||||
if len == 0 {
|
||||
self.cursor = 0;
|
||||
self.timeout_cursor = 0;
|
||||
self.send_cursor = 0;
|
||||
} else {
|
||||
self.cursor %= len;
|
||||
self.timeout_cursor %= len;
|
||||
self.send_cursor %= len;
|
||||
}
|
||||
self.scan_remaining = self.scan_remaining.min(len);
|
||||
@@ -1205,19 +1220,8 @@ impl ManagedClientTask {
|
||||
.next_probe_timeout_deadline()
|
||||
.is_some_and(|deadline| deadline <= now)
|
||||
{
|
||||
match client.poll_timeouts_at(now) {
|
||||
Ok(events) => self.publish_client_events(index, events),
|
||||
Err(error) => {
|
||||
return self.begin_open_session_failure(
|
||||
index,
|
||||
client,
|
||||
ManagedTargetFailurePhase::Timing,
|
||||
error,
|
||||
now,
|
||||
OpenSessionFailureCleanup::Close,
|
||||
);
|
||||
}
|
||||
}
|
||||
self.targets[index].state = TargetState::Active { client };
|
||||
return true;
|
||||
}
|
||||
let received = match client.poll_recv(cx) {
|
||||
Poll::Pending => false,
|
||||
@@ -1277,35 +1281,26 @@ impl ManagedClientTask {
|
||||
return false;
|
||||
}
|
||||
|
||||
if client
|
||||
.next_probe_timeout_deadline()
|
||||
.is_some_and(|timeout| timeout <= now)
|
||||
{
|
||||
self.targets[index].state = TargetState::Draining {
|
||||
client,
|
||||
drain_started_at,
|
||||
deadline,
|
||||
primary_end,
|
||||
cleanup_failure,
|
||||
post_deadline_receives_remaining,
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
let injected_receive_failure = self.take_drain_receive_failure();
|
||||
#[cfg(not(test))]
|
||||
let injected_receive_failure = None;
|
||||
let mut retained_state_changed = false;
|
||||
if client
|
||||
.next_probe_timeout_deadline()
|
||||
.is_some_and(|timeout| timeout <= now)
|
||||
{
|
||||
match client.poll_timeouts_at(now) {
|
||||
Ok(events) => {
|
||||
self.publish_client_events(index, events);
|
||||
retained_state_changed = true;
|
||||
}
|
||||
Err(error) => {
|
||||
self.targets[index].counters.packets_sent = client.packets_sent();
|
||||
cleanup_failure.get_or_insert_with(|| {
|
||||
classify_client_error(ManagedTargetFailurePhase::Timing, &error)
|
||||
});
|
||||
return self.begin_close(
|
||||
index,
|
||||
client,
|
||||
primary_end,
|
||||
cleanup_failure,
|
||||
now,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let received = match client.poll_recv(cx) {
|
||||
Poll::Pending => false,
|
||||
Poll::Ready(Ok(events)) => {
|
||||
@@ -1595,6 +1590,14 @@ impl ManagedClientTask {
|
||||
self.targets[index].state = state;
|
||||
return SendResult::NotAttempted;
|
||||
};
|
||||
if client
|
||||
.next_probe_timeout_deadline()
|
||||
.is_some_and(|deadline| deadline <= now)
|
||||
{
|
||||
self.targets[index].send_waiting = false;
|
||||
self.targets[index].state = TargetState::Active { client };
|
||||
return SendResult::NotAttempted;
|
||||
}
|
||||
if client
|
||||
.next_send_deadline()
|
||||
.is_none_or(|deadline| deadline > now)
|
||||
@@ -1728,13 +1731,176 @@ impl ManagedClientTask {
|
||||
immediate || self.scan_remaining > 0
|
||||
}
|
||||
|
||||
fn poll_timeout_pass(&mut self, now: Instant) -> bool {
|
||||
let target_len = self.targets.len();
|
||||
if target_len == 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let mut transitions_remaining = TIMEOUT_WORK_BUDGET;
|
||||
for _ in 0..target_len.min(TIMEOUT_WORK_BUDGET) {
|
||||
if transitions_remaining == 0 {
|
||||
break;
|
||||
}
|
||||
let index = self.timeout_cursor;
|
||||
self.timeout_cursor = (self.timeout_cursor + 1) % target_len;
|
||||
#[cfg(test)]
|
||||
{
|
||||
self.timeout_inspections += 1;
|
||||
}
|
||||
if self.target_has_due_timeout(index, now) {
|
||||
let generation = self.targets[index].instance.generation;
|
||||
let step = self.poll_target_timeout(index, now);
|
||||
transitions_remaining -= 1;
|
||||
if step.more_due {
|
||||
self.enqueue_timeout_backlog(index, generation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
while transitions_remaining > 0 {
|
||||
let Some(entry) = self.timeout_backlog.pop_front() else {
|
||||
break;
|
||||
};
|
||||
let valid = self
|
||||
.targets
|
||||
.get(entry.index)
|
||||
.is_some_and(|target| target.instance.generation == entry.generation)
|
||||
&& self.target_has_due_timeout(entry.index, now);
|
||||
if !valid {
|
||||
continue;
|
||||
}
|
||||
|
||||
let step = self.poll_target_timeout(entry.index, now);
|
||||
transitions_remaining -= 1;
|
||||
if step.more_due {
|
||||
self.enqueue_timeout_backlog(entry.index, entry.generation);
|
||||
}
|
||||
}
|
||||
|
||||
!self.timeout_backlog.is_empty()
|
||||
}
|
||||
|
||||
fn enqueue_timeout_backlog(&mut self, index: usize, generation: u64) {
|
||||
if self.timeout_backlog.len() == TIMEOUT_WORK_BUDGET
|
||||
|| self
|
||||
.timeout_backlog
|
||||
.iter()
|
||||
.any(|entry| entry.index == index && entry.generation == generation)
|
||||
{
|
||||
return;
|
||||
}
|
||||
self.timeout_backlog
|
||||
.push_back(TimeoutBacklogEntry { index, generation });
|
||||
}
|
||||
|
||||
fn target_has_due_timeout(&self, index: usize, now: Instant) -> bool {
|
||||
match &self.targets[index].state {
|
||||
TargetState::Active { client } => {
|
||||
self.state != DriverState::Stopping
|
||||
&& self.effective_retirement(index).is_none()
|
||||
&& client
|
||||
.next_probe_timeout_deadline()
|
||||
.is_some_and(|deadline| deadline <= now)
|
||||
}
|
||||
TargetState::Draining {
|
||||
client,
|
||||
deadline: _deadline,
|
||||
..
|
||||
} => {
|
||||
#[cfg(test)]
|
||||
let defer_work = self.drain_test_hook.defer_work_until_deadline && now < *_deadline;
|
||||
#[cfg(not(test))]
|
||||
let defer_work = false;
|
||||
!defer_work
|
||||
&& client
|
||||
.next_probe_timeout_deadline()
|
||||
.is_some_and(|timeout| timeout <= now)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_target_timeout(&mut self, index: usize, now: Instant) -> TimeoutStep {
|
||||
let state = mem::replace(&mut self.targets[index].state, TargetState::Terminal);
|
||||
match state {
|
||||
TargetState::Active { mut client } => match client.poll_timeouts_bounded_at(now, 1) {
|
||||
Ok(batch) => {
|
||||
self.publish_client_events(index, batch.events);
|
||||
self.targets[index].state = TargetState::Active { client };
|
||||
TimeoutStep {
|
||||
more_due: batch.more_due,
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
self.begin_open_session_failure(
|
||||
index,
|
||||
client,
|
||||
ManagedTargetFailurePhase::Timing,
|
||||
error,
|
||||
now,
|
||||
OpenSessionFailureCleanup::Close,
|
||||
);
|
||||
TimeoutStep::NONE
|
||||
}
|
||||
},
|
||||
TargetState::Draining {
|
||||
mut client,
|
||||
drain_started_at,
|
||||
deadline,
|
||||
primary_end,
|
||||
mut cleanup_failure,
|
||||
post_deadline_receives_remaining,
|
||||
} => match client.poll_timeouts_bounded_at(now, 1) {
|
||||
Ok(batch) => {
|
||||
self.publish_client_events(index, batch.events);
|
||||
let deadline = match self.drain_deadline(&client, drain_started_at) {
|
||||
Some(candidate) => deadline.min(candidate),
|
||||
None => {
|
||||
cleanup_failure.get_or_insert_with(duration_overflow_failure);
|
||||
self.begin_close(index, client, primary_end, cleanup_failure, now);
|
||||
return TimeoutStep::NONE;
|
||||
}
|
||||
};
|
||||
self.targets[index].state = TargetState::Draining {
|
||||
client,
|
||||
drain_started_at,
|
||||
deadline,
|
||||
primary_end,
|
||||
cleanup_failure,
|
||||
post_deadline_receives_remaining,
|
||||
};
|
||||
TimeoutStep {
|
||||
more_due: batch.more_due,
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
self.targets[index].counters.packets_sent = client.packets_sent();
|
||||
|
|
||||
cleanup_failure.get_or_insert_with(|| {
|
||||
classify_client_error(ManagedTargetFailurePhase::Timing, &error)
|
||||
});
|
||||
self.begin_close(index, client, primary_end, cleanup_failure, now);
|
||||
TimeoutStep::NONE
|
||||
}
|
||||
},
|
||||
state => {
|
||||
self.targets[index].state = state;
|
||||
TimeoutStep::NONE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn all_targets_terminal(&self) -> bool {
|
||||
self.targets
|
||||
.iter()
|
||||
.all(|target| matches!(target.state, TargetState::Terminal))
|
||||
}
|
||||
|
||||
fn next_deadline(&self) -> Option<Instant> {
|
||||
fn next_deadline(&mut self) -> Option<Instant> {
|
||||
#[cfg(test)]
|
||||
{
|
||||
self.deadline_inspections += self.targets.len();
|
||||
}
|
||||
let mut non_send_deadline = None;
|
||||
let mut send_deadline = None;
|
||||
for target in &self.targets {
|
||||
@@ -1888,6 +2054,15 @@ enum SendResult {
|
||||
Failed { accepted: bool },
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct TimeoutStep {
|
||||
more_due: bool,
|
||||
}
|
||||
|
||||
impl TimeoutStep {
|
||||
const NONE: Self = Self { more_due: false };
|
||||
}
|
||||
|
||||
impl SendResult {
|
||||
fn accepted(self) -> bool {
|
||||
matches!(
|
||||
@@ -1922,6 +2097,7 @@ impl Future for ManagedClientTask {
|
||||
this.observe_stop();
|
||||
}
|
||||
let now = Instant::now();
|
||||
immediate |= this.poll_timeout_pass(now);
|
||||
immediate |= this.poll_target_pass(cx, now);
|
||||
this.prune_undesired_terminal();
|
||||
if this.state == DriverState::Running {
|
||||
@@ -1962,7 +2138,9 @@ impl Future for ManagedClientTask {
|
||||
}
|
||||
|
||||
immediate |= this.register_stop_wake(cx);
|
||||
if !immediate {
|
||||
immediate |= this.register_timer(cx);
|
||||
}
|
||||
if immediate {
|
||||
cx.waker().wake_by_ref();
|
||||
}
|
||||
@@ -2129,6 +2307,8 @@ fn build_task(
|
||||
wake: Some(arm_wake(wake_receiver)),
|
||||
timer: None,
|
||||
cursor: 0,
|
||||
timeout_cursor: 0,
|
||||
timeout_backlog: VecDeque::with_capacity(TIMEOUT_WORK_BUDGET),
|
||||
scan_remaining: 0,
|
||||
send_cursor: 0,
|
||||
burst_remaining: 0,
|
||||
@@ -2145,6 +2325,10 @@ fn build_task(
|
||||
stagger_observations: None,
|
||||
#[cfg(test)]
|
||||
drain_test_hook: DrainTestHook::default(),
|
||||
#[cfg(test)]
|
||||
timeout_inspections: 0,
|
||||
#[cfg(test)]
|
||||
deadline_inspections: 0,
|
||||
};
|
||||
let handle = ManagedClientHandle {
|
||||
stop,
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
+307
-19
@@ -157,6 +157,12 @@ pub(crate) struct ProbeCommit {
|
||||
next_packets_sent: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct TimeoutBatch {
|
||||
pub events: Vec<ClientEvent>,
|
||||
pub more_due: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) struct ProbeSent {
|
||||
pub(crate) seq: u32,
|
||||
@@ -408,11 +414,21 @@ impl SessionMachine {
|
||||
&mut self,
|
||||
now: Instant,
|
||||
) -> Result<Vec<ClientEvent>, ClientError> {
|
||||
let batch = self.poll_timeouts_bounded_at(now, usize::MAX)?;
|
||||
debug_assert!(!batch.more_due);
|
||||
Ok(batch.events)
|
||||
}
|
||||
|
||||
pub(crate) fn poll_timeouts_bounded_at(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
limit: usize,
|
||||
) -> Result<TimeoutBatch, ClientError> {
|
||||
let session = self.open_session_mut()?;
|
||||
|
||||
let expired = session.pending.drain_expired(now);
|
||||
let mut events = Vec::with_capacity(expired.len());
|
||||
for probe in expired {
|
||||
let expired = session.pending.drain_expired_bounded(now, limit);
|
||||
let mut events = Vec::with_capacity(expired.probes.len());
|
||||
for probe in expired.probes {
|
||||
events.push(ClientEvent::EchoLoss {
|
||||
seq: probe.wire_seq,
|
||||
sent_at: probe.sent_at,
|
||||
@@ -421,7 +437,10 @@ impl SessionMachine {
|
||||
session.timed_out.insert(probe);
|
||||
}
|
||||
|
||||
Ok(events)
|
||||
Ok(TimeoutBatch {
|
||||
events,
|
||||
more_due: expired.more_due,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_close(&self) -> Result<PreparedClose<'_>, ClientError> {
|
||||
@@ -1019,6 +1038,25 @@ impl SessionMachine {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "tokio"))]
|
||||
impl SessionMachine {
|
||||
pub(crate) fn remove_pending_for_test(&mut self, wire_seq: u32) -> Option<PendingProbe> {
|
||||
self.open_session_mut()
|
||||
.expect("test pending probes require an open session")
|
||||
.pending
|
||||
.remove(wire_seq)
|
||||
}
|
||||
|
||||
pub(crate) fn replace_pending_for_test(&mut self, probe: PendingProbe) {
|
||||
let session = self
|
||||
.open_session_mut()
|
||||
.expect("test pending probes require an open session");
|
||||
session.pending.remove(probe.wire_seq);
|
||||
session.pending.preflight_insert(probe.wire_seq).unwrap();
|
||||
session.pending.commit_insert(probe);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -1803,6 +1841,34 @@ mod tests {
|
||||
assert_eq!(active(&machine).pending.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exhaustive_timeout_polling_returns_every_due_loss() {
|
||||
let mut machine = open_machine(4, Duration::from_secs(1));
|
||||
let now = Instant::now();
|
||||
for seq in [2, 1, 0] {
|
||||
let sent_at = timestamp(now - Duration::from_secs(u64::from(seq) + 1));
|
||||
let session = active_mut(&mut machine);
|
||||
session.pending.preflight_insert(seq).unwrap();
|
||||
session.pending.commit_insert(PendingProbe {
|
||||
wire_seq: seq,
|
||||
sent_at,
|
||||
timeout_at: sent_at.mono + Duration::from_secs(1),
|
||||
});
|
||||
}
|
||||
|
||||
let events = machine.poll_timeouts_at(now).unwrap();
|
||||
assert!(matches!(
|
||||
events.as_slice(),
|
||||
[
|
||||
ClientEvent::EchoLoss { seq: 2, .. },
|
||||
ClientEvent::EchoLoss { seq: 1, .. },
|
||||
ClientEvent::EchoLoss { seq: 0, .. },
|
||||
]
|
||||
));
|
||||
assert_eq!(active(&machine).pending.len(), 0);
|
||||
assert_eq!(active(&machine).timed_out.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wrapping_sequence_from_max_to_zero_remains_valid() {
|
||||
let mut machine = open_machine(2, Duration::from_secs(1));
|
||||
|
||||
Reference in new issue
Block a user
When
targets.len() > TIMEOUT_WORK_BUDGETand one sparsely located target has multiple overdue probes, advancingtimeout_cursorunconditionally after processing only one probe postpones that target's next expiration until a complete target sweep.more_duemerely triggers another poll; each intervening poll also callsnext_deadline, which scans every target, so a sparse backlog can requireO(N / TIMEOUT_WORK_BUDGET)polls and repeatedO(N)scans per loss, severely delaying timeout events and drain completion. Preserve or requeue indexes whoseTimeoutStep::more_dueis true while still round-robining among backlogged targets. Fresh evidence beyond the earlier timeout-discovery comment is that the current fix still advances the cursor despite the returned backlog signal.Useful? React with 👍 / 👎.