fix(client): separate probe admission and send timing
This commit is contained in:
6 files changed
+433
-122
No files matched your search
@@ -14,7 +14,6 @@ pub(crate) struct ProbeSchedule {
|
||||
pub(crate) struct ScheduleCommit {
|
||||
pub(crate) scheduled_at: Instant,
|
||||
pub(crate) next_send_at: Option<Instant>,
|
||||
pub(crate) timer_error: Duration,
|
||||
}
|
||||
|
||||
impl ProbeSchedule {
|
||||
@@ -64,7 +63,6 @@ impl ProbeSchedule {
|
||||
|
||||
pub(crate) fn preflight_caller_commit(
|
||||
&self,
|
||||
sent_at: Instant,
|
||||
next_packets_sent: u64,
|
||||
) -> Result<ScheduleCommit, ClientError> {
|
||||
let scheduled_at = self
|
||||
@@ -73,17 +71,17 @@ impl ProbeSchedule {
|
||||
let interval_ns =
|
||||
u64::try_from(self.interval.as_nanos()).map_err(|_| ClientError::DurationOverflow)?;
|
||||
let next_send_at = next_probe_deadline(self.start_at, interval_ns, next_packets_sent)?;
|
||||
Ok(self.finish_commit(scheduled_at, next_send_at, sent_at))
|
||||
Ok(self.finish_commit(scheduled_at, next_send_at))
|
||||
}
|
||||
|
||||
pub(crate) fn preflight_managed_commit(
|
||||
&self,
|
||||
scheduled_at: Instant,
|
||||
sent_at: Instant,
|
||||
permission_at: Instant,
|
||||
) -> Result<ScheduleCommit, ClientError> {
|
||||
let (scheduled_at, next_send_at, _) =
|
||||
advance_cadence(scheduled_at, self.interval, sent_at)?;
|
||||
Ok(self.finish_commit(scheduled_at, next_send_at, sent_at))
|
||||
advance_cadence(scheduled_at, self.interval, permission_at)?;
|
||||
Ok(self.finish_commit(scheduled_at, next_send_at))
|
||||
}
|
||||
|
||||
pub(crate) fn commit(&mut self, commit: ScheduleCommit) {
|
||||
@@ -94,12 +92,7 @@ impl ProbeSchedule {
|
||||
self.next_send_at.is_none()
|
||||
}
|
||||
|
||||
fn finish_commit(
|
||||
&self,
|
||||
scheduled_at: Instant,
|
||||
next_send_at: Instant,
|
||||
sent_at: Instant,
|
||||
) -> ScheduleCommit {
|
||||
fn finish_commit(&self, scheduled_at: Instant, next_send_at: Instant) -> ScheduleCommit {
|
||||
ScheduleCommit {
|
||||
scheduled_at,
|
||||
next_send_at: if self.end_at.is_some_and(|end| next_send_at >= end) {
|
||||
@@ -107,7 +100,6 @@ impl ProbeSchedule {
|
||||
} else {
|
||||
Some(next_send_at)
|
||||
},
|
||||
timer_error: instant_abs_diff(sent_at, scheduled_at),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -173,7 +165,7 @@ fn duration_from_nanos(nanos: u128) -> Result<Duration, ClientError> {
|
||||
Ok(Duration::new(seconds, subsec_nanos))
|
||||
}
|
||||
|
||||
fn instant_abs_diff(left: Instant, right: Instant) -> Duration {
|
||||
pub(crate) fn instant_abs_diff(left: Instant, right: Instant) -> Duration {
|
||||
left.checked_duration_since(right)
|
||||
.or_else(|| right.checked_duration_since(left))
|
||||
.unwrap_or(Duration::ZERO)
|
||||
@@ -209,13 +201,11 @@ mod tests {
|
||||
fn caller_preflight_keeps_absolute_cadence_without_mutation() {
|
||||
let start = Instant::now();
|
||||
let schedule = ProbeSchedule::new(start, &negotiated(10_000_000, 0)).unwrap();
|
||||
let sent_at = start + Duration::from_millis(4);
|
||||
|
||||
let commit = schedule.preflight_caller_commit(sent_at, 3).unwrap();
|
||||
let commit = schedule.preflight_caller_commit(3).unwrap();
|
||||
|
||||
assert_eq!(commit.scheduled_at, start);
|
||||
assert_eq!(commit.next_send_at, Some(start + Duration::from_millis(30)));
|
||||
assert_eq!(commit.timer_error, Duration::from_millis(4));
|
||||
assert_eq!(schedule.next_send_deadline(), Some(start));
|
||||
}
|
||||
|
||||
@@ -224,7 +214,7 @@ mod tests {
|
||||
let start = Instant::now();
|
||||
let mut schedule = ProbeSchedule::new(start, &negotiated(10_000_000, 0)).unwrap();
|
||||
|
||||
let commit = schedule.preflight_caller_commit(start, 1).unwrap();
|
||||
let commit = schedule.preflight_caller_commit(1).unwrap();
|
||||
assert_eq!(schedule.next_send_deadline(), Some(start));
|
||||
|
||||
schedule.commit(commit);
|
||||
@@ -239,16 +229,14 @@ mod tests {
|
||||
let start = Instant::now();
|
||||
let mut schedule = ProbeSchedule::new(start, &negotiated(10_000_000, 20_000_000)).unwrap();
|
||||
|
||||
let first = schedule.preflight_caller_commit(start, 1).unwrap();
|
||||
let first = schedule.preflight_caller_commit(1).unwrap();
|
||||
schedule.commit(first);
|
||||
assert_eq!(
|
||||
schedule.next_send_deadline(),
|
||||
Some(start + Duration::from_millis(10))
|
||||
);
|
||||
|
||||
let second = schedule
|
||||
.preflight_caller_commit(start + Duration::from_millis(10), 2)
|
||||
.unwrap();
|
||||
let second = schedule.preflight_caller_commit(2).unwrap();
|
||||
schedule.commit(second);
|
||||
|
||||
assert!(schedule.is_finished());
|
||||
@@ -276,7 +264,6 @@ mod tests {
|
||||
|
||||
assert_eq!(commit.scheduled_at, start + Duration::from_millis(40));
|
||||
assert_eq!(commit.next_send_at, Some(start + Duration::from_millis(50)));
|
||||
assert_eq!(commit.timer_error, Duration::from_millis(5));
|
||||
assert_eq!(schedule.next_send_deadline(), Some(start));
|
||||
}
|
||||
|
||||
@@ -304,9 +291,20 @@ mod tests {
|
||||
let schedule = ProbeSchedule::new(start, &negotiated(2, 0)).unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
schedule.preflight_caller_commit(start, u64::MAX),
|
||||
schedule.preflight_caller_commit(u64::MAX),
|
||||
Err(ClientError::DurationOverflow)
|
||||
));
|
||||
assert_eq!(schedule.next_send_deadline(), Some(start));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timer_error_is_adapter_owned() {
|
||||
let scheduled_at = Instant::now();
|
||||
let sent_at = scheduled_at + Duration::from_millis(7);
|
||||
|
||||
assert_eq!(
|
||||
instant_abs_diff(sent_at, scheduled_at),
|
||||
Duration::from_millis(7)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,14 @@
|
||||
use super::*;
|
||||
|
||||
fn probe_timestamps(permission_at: Instant, sent_at: ClientTimestamp) -> ProbeSendTimestamps {
|
||||
ProbeSendTimestamps {
|
||||
permission_at,
|
||||
sent_at,
|
||||
send_call_start: sent_at.mono,
|
||||
send_finished_at: sent_at.mono,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn send_probe_fails_before_open() {
|
||||
let server = start_fake_server(|_socket, _tx| {});
|
||||
@@ -103,7 +112,40 @@ fn blocking_failed_probe_send_preserves_machine_and_schedule() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocking_probe_uses_one_timestamp_for_permission_and_schedule_commit() {
|
||||
fn blocking_timeout_finalization_overflow_does_not_send_or_commit() {
|
||||
let params = default_params();
|
||||
let server = silent_open_server(params);
|
||||
let config = ClientConfig {
|
||||
probe_timeout: Duration::MAX,
|
||||
..default_test_config(server.addr)
|
||||
};
|
||||
let mut client = Client::connect(config).unwrap();
|
||||
assert_open_started(client.open().unwrap());
|
||||
|
||||
let initial_deadline = client.next_send_deadline().unwrap();
|
||||
let sent_at = ClientTimestamp {
|
||||
mono: Instant::now(),
|
||||
wall: SystemTime::now(),
|
||||
};
|
||||
|
||||
assert!(matches!(
|
||||
client.send_probe_at(probe_timestamps(initial_deadline, sent_at)),
|
||||
Err(ClientError::DurationOverflow)
|
||||
));
|
||||
assert_eq!(client.runtime.packets_sent(), 0);
|
||||
assert_eq!(client.next_send_deadline(), Some(initial_deadline));
|
||||
thread::sleep(Duration::from_millis(30));
|
||||
assert!(server
|
||||
.rx
|
||||
.try_iter()
|
||||
.all(|packet| packet.len() >= 4 && packet[3] & FLAG_OPEN != 0));
|
||||
|
||||
client.close().unwrap();
|
||||
server.join();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocking_probe_separates_permission_committed_send_and_send_call_timing() {
|
||||
let duration = Duration::from_secs(1);
|
||||
let interval = Duration::from_millis(100);
|
||||
let params = Params {
|
||||
@@ -133,26 +175,83 @@ fn blocking_probe_uses_one_timestamp_for_permission_and_schedule_commit() {
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
let permission_at = opened_mono + Duration::from_millis(500);
|
||||
let sent_at = ClientTimestamp {
|
||||
wall: SystemTime::now(),
|
||||
mono: opened_mono + Duration::from_millis(500),
|
||||
mono: opened_mono + Duration::from_millis(525),
|
||||
};
|
||||
let timestamps = ProbeSendTimestamps {
|
||||
permission_at,
|
||||
sent_at,
|
||||
send_call_start: opened_mono + Duration::from_millis(526),
|
||||
send_finished_at: opened_mono + Duration::from_millis(533),
|
||||
};
|
||||
|
||||
let events = client.send_probe_at(sent_at).unwrap();
|
||||
let events = client.send_probe_at(timestamps).unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
events.as_slice(),
|
||||
[ClientEvent::EchoSent {
|
||||
scheduled_at,
|
||||
sent_at: event_sent_at,
|
||||
send_call,
|
||||
timer_error,
|
||||
..
|
||||
}] if *scheduled_at == opened_mono
|
||||
&& *event_sent_at == sent_at
|
||||
&& *timer_error == Duration::from_millis(500)
|
||||
&& *send_call == Duration::from_millis(7)
|
||||
&& *timer_error == Duration::from_millis(525)
|
||||
));
|
||||
assert_eq!(client.next_send_deadline(), Some(opened_mono + interval));
|
||||
assert_eq!(client.runtime.packets_sent(), 1);
|
||||
let timeout_at = sent_at.mono + client.probe_timeout();
|
||||
assert!(client
|
||||
.poll_timeouts_at(timeout_at - Duration::from_nanos(1))
|
||||
.unwrap()
|
||||
.is_empty());
|
||||
assert!(matches!(
|
||||
client.poll_timeouts_at(timeout_at).unwrap().as_slice(),
|
||||
[ClientEvent::EchoLoss {
|
||||
sent_at: loss_sent_at,
|
||||
..
|
||||
}] if *loss_sent_at == sent_at
|
||||
));
|
||||
|
||||
client.close().unwrap();
|
||||
server.join();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocking_send_call_excludes_wrapped_history_cleanup() {
|
||||
let params = default_params();
|
||||
let server = silent_open_server(params);
|
||||
let mut client = Client::connect(default_test_config(server.addr)).unwrap();
|
||||
assert_open_started(client.open().unwrap());
|
||||
|
||||
let scheduled_at = client.next_send_deadline().unwrap();
|
||||
let sent_at = ClientTimestamp {
|
||||
wall: SystemTime::now(),
|
||||
mono: scheduled_at + Duration::from_millis(5),
|
||||
};
|
||||
client.runtime.seed_wrapped_probe_history_for_test(sent_at);
|
||||
let timestamps = ProbeSendTimestamps {
|
||||
permission_at: scheduled_at,
|
||||
sent_at,
|
||||
send_call_start: scheduled_at + Duration::from_millis(6),
|
||||
send_finished_at: scheduled_at + Duration::from_millis(13),
|
||||
};
|
||||
|
||||
let events = client.send_probe_at(timestamps).unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
events.as_slice(),
|
||||
[ClientEvent::EchoSent {
|
||||
seq: 0,
|
||||
send_call,
|
||||
..
|
||||
}] if *send_call == Duration::from_millis(7)
|
||||
));
|
||||
assert!(!client.runtime.has_timed_out_metadata());
|
||||
|
||||
client.close().unwrap();
|
||||
server.join();
|
||||
@@ -189,19 +288,25 @@ fn send_probe_respects_finite_duration_exclusive_end() {
|
||||
mono: start,
|
||||
wall: SystemTime::now(),
|
||||
};
|
||||
assert!(client.send_probe_at(now0).is_ok());
|
||||
assert!(client
|
||||
.send_probe_at(probe_timestamps(now0.mono, now0))
|
||||
.is_ok());
|
||||
|
||||
let now1 = ClientTimestamp {
|
||||
mono: start + interval,
|
||||
wall: SystemTime::now(),
|
||||
};
|
||||
assert!(client.send_probe_at(now1).is_ok());
|
||||
assert!(client
|
||||
.send_probe_at(probe_timestamps(now1.mono, now1))
|
||||
.is_ok());
|
||||
|
||||
let now2 = ClientTimestamp {
|
||||
mono: start + Duration::from_secs(1),
|
||||
wall: SystemTime::now(),
|
||||
};
|
||||
let events = client.send_probe_at(now2).unwrap();
|
||||
let events = client
|
||||
.send_probe_at(probe_timestamps(now2.mono, now2))
|
||||
.unwrap();
|
||||
assert!(events.is_empty());
|
||||
assert!(client.next_send_deadline().is_none());
|
||||
|
||||
@@ -231,12 +336,19 @@ fn managed_probe_skips_missed_schedule_slots() {
|
||||
assert_open_started(client.open().unwrap());
|
||||
|
||||
let first_deadline = client.next_send_deadline().unwrap();
|
||||
let permission_at = first_deadline + Duration::from_millis(45);
|
||||
let delayed_send = ClientTimestamp {
|
||||
mono: first_deadline + Duration::from_millis(45),
|
||||
mono: first_deadline + Duration::from_millis(48),
|
||||
wall: SystemTime::now(),
|
||||
};
|
||||
let timestamps = ProbeSendTimestamps {
|
||||
permission_at,
|
||||
sent_at: delayed_send,
|
||||
send_call_start: first_deadline + Duration::from_millis(49),
|
||||
send_finished_at: first_deadline + Duration::from_millis(55),
|
||||
};
|
||||
let events = client
|
||||
.send_managed_probe_at(first_deadline, delayed_send)
|
||||
.send_managed_probe_at(first_deadline, timestamps)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(events.len(), 1);
|
||||
@@ -246,9 +358,13 @@ fn managed_probe_skips_missed_schedule_slots() {
|
||||
seq: 0,
|
||||
scheduled_at,
|
||||
sent_at,
|
||||
send_call,
|
||||
timer_error,
|
||||
..
|
||||
} if scheduled_at == first_deadline + Duration::from_millis(40)
|
||||
&& sent_at == delayed_send
|
||||
&& send_call == Duration::from_millis(6)
|
||||
&& timer_error == Duration::from_millis(8)
|
||||
));
|
||||
assert_eq!(
|
||||
client.next_send_deadline(),
|
||||
@@ -287,15 +403,19 @@ fn managed_probe_skip_preserves_finite_run_end() {
|
||||
wall: SystemTime::now(),
|
||||
};
|
||||
let first = client
|
||||
.send_managed_probe_at(first_deadline, delayed_send)
|
||||
.send_managed_probe_at(
|
||||
first_deadline,
|
||||
probe_timestamps(delayed_send.mono, delayed_send),
|
||||
)
|
||||
.unwrap();
|
||||
let second_sent_at = ClientTimestamp {
|
||||
mono: first_deadline + Duration::from_millis(46),
|
||||
wall: SystemTime::now(),
|
||||
};
|
||||
let second = client
|
||||
.send_managed_probe_at(
|
||||
first_deadline + duration,
|
||||
ClientTimestamp {
|
||||
mono: first_deadline + Duration::from_millis(46),
|
||||
wall: SystemTime::now(),
|
||||
},
|
||||
probe_timestamps(second_sent_at.mono, second_sent_at),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
|
||||
Reference in new issue
Block a user