From 04e1f11f3a63e1ffad6f3e98aa51da2e5eab2260 Mon Sep 17 00:00:00 2001 From: Nils Andreas Svee Date: Tue, 27 Oct 2026 01:26:20 +0700 Subject: [PATCH] feat(client): make reconnect exponential backoff configurable --- crates/irtt-app/src/cmd/client/args.rs | 19 +++++++++++++++- crates/irtt-app/src/cmd/client/run.rs | 4 ++++ crates/irtt-client/src/managed/model.rs | 13 +++++++++++ .../irtt-client/src/managed/task/control.rs | 22 ++++++++++++++++++- crates/irtt-client/src/managed/task/target.rs | 19 +++++++++++----- crates/irtt-client/src/managed/task/update.rs | 2 +- docs/irtt-client.md | 4 +++- 7 files changed, 73 insertions(+), 10 deletions(-) diff --git a/crates/irtt-app/src/cmd/client/args.rs b/crates/irtt-app/src/cmd/client/args.rs index 40690ee..87ff4fa 100644 --- a/crates/irtt-app/src/cmd/client/args.rs +++ b/crates/irtt-app/src/cmd/client/args.rs @@ -72,7 +72,7 @@ pub struct ClientArgs { #[arg( long, conflicts_with = "no_test", - long_help = "Retry failed initial opens and closed, failed, or silent sessions independently per target. Requires --continuous or --duration 0. Unsuccessful connections back off from 250ms to 10s. Use --reconnect-timeout to configure silence detection and --no-stats when only measurement events are needed." + long_help = "Retry failed initial opens and closed, failed, or silent sessions independently per target. Requires --continuous or --duration 0. Unsuccessful connections use exponential backoff (defaults: 250ms initial, 10s maximum), configurable with --reconnect-initial-delay and --reconnect-max-delay. Use --reconnect-timeout to configure silence detection and --no-stats when only measurement events are needed." )] pub reconnect: bool, @@ -80,6 +80,14 @@ pub struct ClientArgs { #[arg(long, default_value = "10s", value_parser = crate::shared::client::parse_duration, requires = "reconnect")] pub reconnect_timeout: Duration, + /// Initial retry delay, restored after a valid reply. + #[arg(long, default_value = "250ms", value_parser = crate::shared::client::parse_duration, requires = "reconnect")] + pub reconnect_initial_delay: Duration, + + /// Maximum retry delay (must be at least twice the initial delay). + #[arg(long, default_value = "10s", value_parser = crate::shared::client::parse_duration, requires = "reconnect")] + pub reconnect_max_delay: Duration, + #[command(flatten)] pub common: CommonClientArgs, @@ -132,6 +140,15 @@ impl ClientArgs { if self.reconnect && !self.is_continuous() { return Err("--reconnect requires --continuous or --duration 0".to_owned()); } + if self + .reconnect_initial_delay + .checked_mul(2) + .is_none_or(|minimum| self.reconnect_max_delay < minimum) + { + return Err( + "--reconnect-max-delay must be at least twice --reconnect-initial-delay".to_owned(), + ); + } if self.targets_stdin && !self.is_continuous() { return Err("--targets-stdin requires --continuous or --duration 0".to_owned()); } diff --git a/crates/irtt-app/src/cmd/client/run.rs b/crates/irtt-app/src/cmd/client/run.rs index 9dcb760..f5b160f 100644 --- a/crates/irtt-app/src/cmd/client/run.rs +++ b/crates/irtt-app/src/cmd/client/run.rs @@ -78,6 +78,8 @@ pub async fn run_stream( { let mut config = setup.managed_config(); config.reconnect_timeout = args.reconnect.then_some(args.reconnect_timeout); + config.reconnect_initial_delay = args.reconnect_initial_delay; + config.reconnect_max_delay = args.reconnect_max_delay; config }, setup.managed_targets(), @@ -482,6 +484,8 @@ async fn run_stdin_stream( { let mut config = setup.managed_config(); config.reconnect_timeout = args.reconnect.then_some(args.reconnect_timeout); + config.reconnect_initial_delay = args.reconnect_initial_delay; + config.reconnect_max_delay = args.reconnect_max_delay; config }, setup.managed_targets(), diff --git a/crates/irtt-client/src/managed/model.rs b/crates/irtt-client/src/managed/model.rs index d83f8e1..45af4f2 100644 --- a/crates/irtt-client/src/managed/model.rs +++ b/crates/irtt-client/src/managed/model.rs @@ -104,6 +104,12 @@ pub struct ManagedClientConfig { /// Retry failed/closed sessions and detect silence while sending. Only valid /// for continuous measurement runs. None (the default) disables reconnection. pub reconnect_timeout: Option, + /// First retry delay, also restored after a valid reply (default: 250 ms). + /// Must be positive and at most half of `reconnect_max_delay`. + pub reconnect_initial_delay: Duration, + /// Cap for per-target exponential backoff with a fixed multiplier of two + /// (default: 10 seconds). Must be schedulable from the current instant. + pub reconnect_max_delay: Duration, } impl Default for ManagedClientConfig { @@ -118,6 +124,8 @@ impl Default for ManagedClientConfig { max_live_target_generations: DEFAULT_MANAGED_MAX_LIVE_TARGET_GENERATIONS, final_drain: DEFAULT_MANAGED_FINAL_DRAIN, reconnect_timeout: None, + reconnect_initial_delay: Duration::from_millis(250), + reconnect_max_delay: Duration::from_secs(10), } } } @@ -370,6 +378,11 @@ pub struct ManagedTargetFailure { pub enum ManagedConfigError { #[error("reconnection requires continuous measurement mode and a positive, schedulable silence timeout")] InvalidReconnectPolicy, + #[error("reconnection delays must be positive and schedulable, with maximum delay at least twice the initial delay (initial: {initial:?}, maximum: {maximum:?})")] + InvalidReconnectDelays { + initial: Duration, + maximum: Duration, + }, #[error("FinishWhenQuiescent requires at least one initial target")] EmptyInitialTargets, #[error("duplicate managed target id {id}")] diff --git a/crates/irtt-client/src/managed/task/control.rs b/crates/irtt-client/src/managed/task/control.rs index 102ff96..f8a4cca 100644 --- a/crates/irtt-client/src/managed/task/control.rs +++ b/crates/irtt-client/src/managed/task/control.rs @@ -367,6 +367,26 @@ fn build_task( maximum: tokio::sync::Semaphore::MAX_PERMITS, }); } + if config.reconnect_initial_delay.is_zero() + || config.reconnect_max_delay.is_zero() + || config + .reconnect_initial_delay + .checked_mul(2) + .is_none_or(|minimum| config.reconnect_max_delay < minimum) + // Tokio represents timer deadlines in u64 milliseconds, reserving the + // top two values for state. Leave room for its millisecond rounding. + || config.reconnect_max_delay.as_millis() >= u128::from(u64::MAX - 2) + || config + .reconnect_max_delay + .checked_add(std::time::Duration::from_millis(1)) + .and_then(|delay| Instant::now().checked_add(delay)) + .is_none() + { + return Err(ManagedConfigError::InvalidReconnectDelays { + initial: config.reconnect_initial_delay, + maximum: config.reconnect_max_delay, + }); + } if let Some(timeout) = config.reconnect_timeout { if timeout.is_zero() || Instant::now().checked_add(timeout).is_none() @@ -427,7 +447,7 @@ fn build_task( remote: None, counters: TargetCounters::default(), sent_before_session: 0, - retry_delay: std::time::Duration::from_millis(250), + retry_delay: config.reconnect_initial_delay, silence_since: None, last_reply_at: None, phase: Some(TargetPhase::Pending { client_config }), diff --git a/crates/irtt-client/src/managed/task/target.rs b/crates/irtt-client/src/managed/task/target.rs index 4467d3c..6938f09 100644 --- a/crates/irtt-client/src/managed/task/target.rs +++ b/crates/irtt-client/src/managed/task/target.rs @@ -329,7 +329,7 @@ impl ManagedClientTask { // A healthy sender with a long interval can remain idle. self.targets[index].silence_since = None; self.targets[index].last_reply_at = Some(received_at.mono); - self.targets[index].retry_delay = Duration::from_millis(250); + self.targets[index].retry_delay = self.config.reconnect_initial_delay; } _ => {} } @@ -344,7 +344,7 @@ impl ManagedClientTask { fn finish_target( &mut self, index: usize, - end_reason: ManagedTargetEndReason, + mut end_reason: ManagedTargetEndReason, cleanup_failure: Option, ) { if self.can_reconnect(index) @@ -359,10 +359,17 @@ impl ManagedClientTask { target.sent_before_session = target.counters.packets_sent; target.silence_since = None; target.last_reply_at = None; - let deadline = Instant::now() + target.retry_delay; - target.retry_delay = (target.retry_delay * 2).min(Duration::from_secs(10)); - self.install_target_phase(index, TargetPhase::Retrying { deadline }); - return; + if let Some(deadline) = Instant::now().checked_add(target.retry_delay) { + target.retry_delay = target + .retry_delay + .checked_mul(2) + .unwrap_or(self.config.reconnect_max_delay) + .min(self.config.reconnect_max_delay); + self.install_target_phase(index, TargetPhase::Retrying { deadline }); + return; + } + // A delay validated at construction may no longer fit after a long run. + end_reason = ManagedTargetEndReason::Failed(duration_overflow_failure()); } let outcome = Arc::new(self.targets[index].outcome(end_reason, cleanup_failure)); // Finalize history before the phase publisher installs Terminal, updates diff --git a/crates/irtt-client/src/managed/task/update.rs b/crates/irtt-client/src/managed/task/update.rs index d172a7a..24b0717 100644 --- a/crates/irtt-client/src/managed/task/update.rs +++ b/crates/irtt-client/src/managed/task/update.rs @@ -142,7 +142,7 @@ impl ManagedClientTask { remote: None, counters: TargetCounters::default(), sent_before_session: 0, - retry_delay: std::time::Duration::from_millis(250), + retry_delay: self.config.reconnect_initial_delay, silence_since: None, last_reply_at: None, phase: Some(TargetPhase::Pending { diff --git a/docs/irtt-client.md b/docs/irtt-client.md index d968c81..5f483c9 100644 --- a/docs/irtt-client.md +++ b/docs/irtt-client.md @@ -237,7 +237,9 @@ warnings are disabled, and IPDV columns retain their names with absent values (`-` in tables, empty CSV/TSV fields, and JSON `null`). Measurement events and protocol-level received statistics remain available. -`--reconnect` enables automatic recovery for individual targets in continuous mode (`--continuous` or `--duration 0`). Failed connections, closed sessions, and unresponsive servers are retried without interrupting other targets. Retries use exponential backoff from 250 ms up to 10 seconds. +`--reconnect` enables automatic recovery for individual targets in continuous mode (`--continuous` or `--duration 0`). Failed connections, closed sessions, and unresponsive servers are retried without interrupting other targets. Retries double the delay after each failed attempt, from 250 ms up to 10 seconds by default. A valid reply resets the delay to the initial value; each target backs off independently. + +`--reconnect-initial-delay DURATION` (default: `250ms`) and `--reconnect-max-delay DURATION` (default: `10s`) configure these delays. Both require `--reconnect`, must be positive and schedulable, and the maximum must be at least twice the initial delay. For example, `--reconnect-initial-delay 1s --reconnect-max-delay 5s` retries after `1s`, `2s`, `4s`, `5s`, then `5s` thereafter. `--reconnect-timeout DURATION` controls how long a session can go without valid replies before reconnection is attempted (default: `10s`). The timeout applies only while actively sending probes and should account for the probe interval and expected packet loss.