From 9de95b303c00579230e978f2a96f92bac08559e4 Mon Sep 17 00:00:00 2001 From: Nils Andreas Svee Date: Sun, 9 Aug 2026 20:21:12 +0700 Subject: [PATCH 1/5] feat(managed): support subscribed blocking start --- crates/irtt-client/src/managed/blocking.rs | 31 ++++++++++++++++-- .../irtt-client/src/managed/blocking_tests.rs | 32 ++++++++++++++++++- crates/irtt-client/src/managed/model.rs | 3 ++ 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/crates/irtt-client/src/managed/blocking.rs b/crates/irtt-client/src/managed/blocking.rs index 15fbeed..4febf65 100644 --- a/crates/irtt-client/src/managed/blocking.rs +++ b/crates/irtt-client/src/managed/blocking.rs @@ -7,8 +7,8 @@ use thiserror::Error; use tokio::runtime::{Builder, Runtime}; use super::{ - ManagedClient, ManagedClientConfig, ManagedClientHandle, ManagedConfigError, ManagedOutcome, - ManagedTargetConfig, + ManagedClient, ManagedClientConfig, ManagedClientHandle, ManagedClientTask, ManagedConfigError, + ManagedEventSubscription, ManagedOutcome, ManagedSubscribeError, ManagedTargetConfig, }; const BLOCKING_RUNTIME_SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(100); @@ -56,6 +56,10 @@ pub enum BlockingManagedStartError { #[error(transparent)] Config(#[from] ManagedConfigError), + /// The managed event subscription could not be created before startup. + #[error(transparent)] + Subscribe(#[from] ManagedSubscribeError), + /// The dedicated Tokio runtime could not be created. #[error("failed to create blocking managed Tokio runtime: {source}")] Runtime { @@ -98,6 +102,29 @@ impl BlockingManagedClient { targets: Vec, ) -> Result { let (task, handle) = ManagedClient::task(config, targets)?; + Self::start_task(task, handle) + } + + /// Validate configuration, subscribe to presentation events, and start a + /// dedicated managed worker. + /// + /// The returned subscription exists before the managed task receives its + /// first poll, so it cannot miss startup events because of subscription + /// ordering. + pub fn start_with_subscription( + config: ManagedClientConfig, + targets: Vec, + ) -> Result<(Self, ManagedEventSubscription), BlockingManagedStartError> { + let (task, handle) = ManagedClient::task(config, targets)?; + let subscription = handle.subscribe()?; + let owner = Self::start_task(task, handle)?; + Ok((owner, subscription)) + } + + fn start_task( + task: ManagedClientTask, + handle: ManagedClientHandle, + ) -> Result { let runtime = Builder::new_current_thread() .enable_io() .enable_time() diff --git a/crates/irtt-client/src/managed/blocking_tests.rs b/crates/irtt-client/src/managed/blocking_tests.rs index 96cf767..e3fe66d 100644 --- a/crates/irtt-client/src/managed/blocking_tests.rs +++ b/crates/irtt-client/src/managed/blocking_tests.rs @@ -14,7 +14,8 @@ use tokio::{runtime::Builder, time::timeout}; use super::{ blocking::{join_worker, WorkerRuntime}, BlockingManagedClient, BlockingManagedJoinError, ManagedClientConfig, ManagedCompletionPolicy, - ManagedEndReason, ManagedLifecycle, ManagedPacing, ManagedTargetConfig, ManagedTargetEndReason, + ManagedEndReason, ManagedEvent, ManagedLifecycle, ManagedPacing, ManagedTargetConfig, + ManagedTargetEndReason, }; use crate::ClientConfig; @@ -183,6 +184,35 @@ fn starts_without_caller_runtime_and_finishes_naturally() { server.finish(); } +#[test] +fn subscribed_start_captures_events_from_a_short_run() { + let server = EchoServer::start(); + let (owner, mut events) = BlockingManagedClient::start_with_subscription( + config(ManagedCompletionPolicy::FinishWhenQuiescent), + vec![target(&server)], + ) + .unwrap(); + + server.wait_probe(); + let outcome = owner.join().unwrap(); + assert_eq!(outcome.end_reason, ManagedEndReason::TargetsComplete); + + let mut saw_started = false; + let mut saw_finished = false; + let mut saw_completed = false; + while let Ok(event) = events.try_recv() { + match event { + ManagedEvent::Started => saw_started = true, + ManagedEvent::TargetFinished { .. } => saw_finished = true, + ManagedEvent::Completed { .. } => saw_completed = true, + _ => {} + } + } + assert!(saw_started && saw_finished && saw_completed); + server.wait_close(); + server.finish(); +} + #[test] fn external_handle_stop_ends_with_graceful_close() { let server = EchoServer::start(); diff --git a/crates/irtt-client/src/managed/model.rs b/crates/irtt-client/src/managed/model.rs index da785de..f972c2d 100644 --- a/crates/irtt-client/src/managed/model.rs +++ b/crates/irtt-client/src/managed/model.rs @@ -173,6 +173,9 @@ pub enum ManagedEvent { /// Receiving half of the lossy managed event stream. pub type ManagedEventSubscription = tokio::sync::broadcast::Receiver; +/// Immediate result of attempting to receive a managed presentation event. +pub type ManagedEventTryRecvError = tokio::sync::broadcast::error::TryRecvError; + /// Final authoritative outcome returned by the task. #[derive(Clone, Debug, Eq, PartialEq)] pub struct ManagedOutcome { -- 2.54.0 From 1a9e8f28aa26c00eac5d78c3991bdad52b40c874 Mon Sep 17 00:00:00 2001 From: Nils Andreas Svee Date: Sun, 9 Aug 2026 20:27:22 +0700 Subject: [PATCH 2/5] refactor(cli): use unified managed frontend --- crates/irtt-cli/Cargo.toml | 2 +- crates/irtt-cli/src/cmd/client/args.rs | 18 +- crates/irtt-cli/src/cmd/client/run.rs | 1083 +++++------------- crates/irtt-cli/src/cmd/tui/args.rs | 18 +- crates/irtt-cli/src/cmd/tui/run.rs | 595 +++------- crates/irtt-cli/src/cmd/tui/ui.rs | 62 +- crates/irtt-cli/src/shared/client/mod.rs | 4 +- crates/irtt-cli/src/shared/client/session.rs | 16 +- crates/irtt-cli/src/shared/client/targets.rs | 123 +- crates/irtt-cli/tests/client_multi_target.rs | 4 +- 10 files changed, 470 insertions(+), 1455 deletions(-) diff --git a/crates/irtt-cli/Cargo.toml b/crates/irtt-cli/Cargo.toml index 98f7400..4c6f2f8 100644 --- a/crates/irtt-cli/Cargo.toml +++ b/crates/irtt-cli/Cargo.toml @@ -30,7 +30,7 @@ required-features = ["tui"] [features] default = ["client", "stats"] -client-runtime = ["dep:irtt-client", "dep:irtt-proto"] +client-runtime = ["dep:irtt-client", "irtt-client/tokio", "dep:irtt-proto"] client = ["client-runtime"] stats = ["client-runtime", "dep:irtt-stats"] tui = ["client-runtime", "stats", "dep:ratatui", "dep:crossterm"] diff --git a/crates/irtt-cli/src/cmd/client/args.rs b/crates/irtt-cli/src/cmd/client/args.rs index 2ff3848..48d39e4 100644 --- a/crates/irtt-cli/src/cmd/client/args.rs +++ b/crates/irtt-cli/src/cmd/client/args.rs @@ -4,8 +4,8 @@ use clap::{Parser, ValueEnum}; use irtt_client::ClientConfig; use crate::shared::client::{ - parse_labelled_target, parse_test_duration, resolved_managed_targets, target_specs, - CommonClientArgs, GroupPacingArg, LabelledTargetArg, ResolvedTarget, TargetSpec, TimestampArg, + parse_labelled_target, parse_test_duration, prepare_managed_targets, target_specs, + CommonClientArgs, GroupPacingArg, LabelledTargetArg, PreparedTarget, TargetSpec, TimestampArg, }; pub const DEFAULT_CLIENT_DURATION: Duration = Duration::from_secs(10); @@ -111,10 +111,8 @@ impl ClientArgs { target_specs(&self.targets, &self.labelled_targets) } - pub fn resolved_managed_targets(&self) -> Result, String> { - let specs = self.target_specs()?; - let config = self.to_client_config(); - resolved_managed_targets(specs, &config) + pub fn managed_targets(&self) -> Result, String> { + prepare_managed_targets(self.target_specs()?) } fn primary_target_addr(&self) -> Option<&str> { @@ -147,7 +145,7 @@ pub enum OutputFormat { } pub type CliTargetSpec = TargetSpec; -pub type ResolvedCliTarget = ResolvedTarget; +pub type CliManagedTarget = PreparedTarget; impl OutputFormat { pub fn prints_summary(self) -> bool { @@ -262,11 +260,9 @@ mod tests { } #[test] - fn duplicate_resolved_target_addresses_are_rejected() { + fn duplicate_target_endpoints_are_allowed() { let args = parse(&["127.0.0.1:2112", "127.0.0.1"]).unwrap(); - let err = args.resolved_managed_targets().unwrap_err(); - - assert!(err.contains("duplicate resolved target address 127.0.0.1:2112")); + assert_eq!(args.managed_targets().unwrap().len(), 2); } #[test] diff --git a/crates/irtt-cli/src/cmd/client/run.rs b/crates/irtt-cli/src/cmd/client/run.rs index bbdfae7..b37cec1 100644 --- a/crates/irtt-cli/src/cmd/client/run.rs +++ b/crates/irtt-cli/src/cmd/client/run.rs @@ -5,20 +5,20 @@ use std::{ io::{self, Write}, sync::atomic::AtomicBool, thread, - time::{Duration, Instant}, + time::Duration, }; use irtt_client::{ - ClientEvent, EventSubscriptionError, ManagedClient, ManagedClientGroup, - ManagedClientGroupConfig, ManagedGroupCompletionPolicy, ManagedGroupEndReason, - ManagedGroupEvent, ManagedTargetOutcome, SessionEndReason, SubscriberConfig, - SubscriberOverflow, TargetEvent, + managed::{ + BlockingManagedClient, ManagedClientConfig, ManagedCompletionPolicy, ManagedEndReason, + ManagedEvent, ManagedEventSubscription, ManagedEventTryRecvError, ManagedTargetEndReason, + ManagedTargetOutcome, TargetInstance, + }, + ClientEvent, }; -#[cfg(all(test, feature = "stats"))] -use irtt_client::{ClientTimestamp, PacketMeta, RttSample, SignedDuration}; use super::{ - args::{ClientArgs, ResolvedCliTarget}, + args::ClientArgs, output::{EventRenderStats, OutputConfig}, }; #[cfg(feature = "stats")] @@ -26,11 +26,10 @@ use crate::shared::client::expected_probe_count; use crate::shared::client::{ is_shutdown_requested, session::{ - peer_close_run_error, request_group_stop_for_peer_close, request_group_stop_once, + peer_close_run_error, request_managed_stop_for_peer_close, request_managed_stop_once, should_print_final_summary, }, }; - #[cfg(feature = "stats")] use irtt_stats::{StatsCollector, StatsConfig}; @@ -46,9 +45,6 @@ const FINITE_STATS_MEMORY_WARNING_BYTES: u64 = 128 * MIB; const FINITE_STATS_MEMORY_STRONG_WARNING_BYTES: u64 = 512 * MIB; #[cfg(feature = "stats")] const FINITE_STATS_MEMORY_VERY_STRONG_WARNING_BYTES: u64 = GIB; - -const GROUP_IDLE_SLEEP: Duration = Duration::from_millis(5); -const GROUP_COMPLETION_GRACE: Duration = Duration::from_secs(1); const MANAGED_EVENT_WAIT_SLICE: Duration = Duration::from_millis(20); const MANAGED_EVENT_CAPACITY: usize = 16_384; @@ -60,39 +56,45 @@ pub fn run_stream( print!("{}", OutputConfig::list_columns()); return Ok(()); } + let targets = args + .managed_targets() + .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?; + let multi_target = targets.len() > 1; let output_config = OutputConfig::new( args.format, args.columns.as_deref(), args.header, args.verbose, - args.target_specs() - .map(|targets| targets.len() > 1) - .unwrap_or(false), + multi_target, ) .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?; - - let targets = args - .resolved_managed_targets() - .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?; - let multi_target = targets.len() > 1; - let continuous = args.is_continuous(); #[cfg(feature = "stats")] if let Some(warning) = finite_stats_memory_warning(&args) { eprintln!("{warning}"); } - - if multi_target { - return run_group_stream(args, targets, output_config, continuous, shutdown_requested); + if is_shutdown_requested(shutdown_requested) { + return Ok(()); } - - let target_label = targets - .first() - .map(|target| target.label.as_str()) - .expect("at least one target was validated"); + let target_count = targets.len(); + let config = ManagedClientConfig { + client: args.to_client_config(), + pacing: args.pacing.into(), + completion: ManagedCompletionPolicy::FinishWhenQuiescent, + event_capacity: MANAGED_EVENT_CAPACITY, + outcome_history_limit: target_count, + max_live_target_generations: target_count, + ..ManagedClientConfig::default() + }; + let (owner, mut events) = BlockingManagedClient::start_with_subscription( + config, + targets + .iter() + .map(|target| target.managed.clone()) + .collect(), + )?; + let handle = owner.handle(); let mut stdout = io::LineWriter::new(io::stdout().lock()); - #[cfg(feature = "stats")] - let mut stats = StatsCollector::new(stats_config(continuous)); let mut stream_output = StreamOutput { config: output_config, header_printed: false, @@ -100,270 +102,6 @@ pub fn run_stream( show_running_only_summary_note: false, out: &mut stdout, }; - - if is_shutdown_requested(shutdown_requested) { - return Ok(()); - } - - let (session, events) = ManagedClient::start_with_subscription( - args.to_client_config(), - managed_event_subscriber_config(), - )?; - let mut interrupted = false; - loop { - if is_shutdown_requested(shutdown_requested) { - interrupted = true; - session.stop(); - break; - } - - match events.recv_timeout(MANAGED_EVENT_WAIT_SLICE) { - Ok(Some(event)) => { - #[cfg(feature = "stats")] - print_events_with_stats( - &mut stream_output, - std::slice::from_ref(&event), - Some(target_label), - &mut stats, - )?; - #[cfg(not(feature = "stats"))] - print_events_with_stats( - &mut stream_output, - std::slice::from_ref(&event), - Some(target_label), - )?; - } - Ok(None) => {} - Err(EventSubscriptionError::Disconnected) => break, - } - } - interrupted |= is_shutdown_requested(shutdown_requested); - - if interrupted { - eprintln!("interrupted, closing session..."); - drain_single_client_events( - &events, - &mut stream_output, - target_label, - #[cfg(feature = "stats")] - &mut stats, - )?; - } - - let outcome = session.join(); - drain_single_client_events( - &events, - &mut stream_output, - target_label, - #[cfg(feature = "stats")] - &mut stats, - )?; - if let Some(warning) = dropped_event_warning(events.dropped_events(), "managed client") { - eprintln!("{warning}"); - } - - interrupted |= is_shutdown_requested(shutdown_requested); - let peer_close_error = outcome.as_ref().ok().and_then(|outcome| { - peer_close_run_error( - continuous, - interrupted, - u64::from(outcome.end_reason == SessionEndReason::PeerClosed), - ) - }); - let print_final_summary = should_print_final_summary(continuous, interrupted); - stream_output.print_final_summary = print_final_summary; - stream_output.show_running_only_summary_note = continuous && interrupted && print_final_summary; - #[cfg(feature = "stats")] - stream_output.print_summary(&stats)?; - #[cfg(not(feature = "stats"))] - stream_output.print_summary()?; - stream_output.out.flush()?; - if let Err(error) = outcome { - return Err(error.into()); - } - if let Some(error) = peer_close_error { - return Err(error.into()); - } - Ok(()) -} - -fn managed_event_subscriber_config() -> SubscriberConfig { - SubscriberConfig { - capacity: MANAGED_EVENT_CAPACITY, - // CLI output is best-effort under sustained backpressure. Dropping - // stale rows keeps the managed worker independent from output speed, - // while dropped_events() makes incomplete output visible. - overflow: SubscriberOverflow::DropOldest, - } -} - -#[cfg(feature = "stats")] -fn drain_single_client_events( - events: &irtt_client::EventSubscription, - stream_output: &mut StreamOutput<'_, W>, - target_label: &str, - stats: &mut StatsCollector, -) -> io::Result<()> { - while let Ok(Some(event)) = events.try_recv() { - print_events_with_stats( - stream_output, - std::slice::from_ref(&event), - Some(target_label), - stats, - )?; - } - Ok(()) -} - -#[cfg(not(feature = "stats"))] -fn drain_single_client_events( - events: &irtt_client::EventSubscription, - stream_output: &mut StreamOutput<'_, W>, - target_label: &str, -) -> io::Result<()> { - while let Ok(Some(event)) = events.try_recv() { - print_events_with_stats( - stream_output, - std::slice::from_ref(&event), - Some(target_label), - )?; - } - Ok(()) -} - -struct StreamOutput<'a, W: Write> { - config: OutputConfig, - header_printed: bool, - print_final_summary: bool, - show_running_only_summary_note: bool, - out: &'a mut W, -} - -impl StreamOutput<'_, W> { - fn print_events( - &mut self, - events: &[ClientEvent], - target: Option<&str>, - stats_updates: &[EventRenderStats], - ) -> io::Result<()> { - self.print_header()?; - for (event, stats_update) in events.iter().zip(stats_updates) { - self.print_event(event, target, stats_update)?; - } - Ok(()) - } - - fn print_event( - &mut self, - event: &ClientEvent, - target: Option<&str>, - stats_update: &EventRenderStats, - ) -> io::Result<()> { - if let Some(line) = self.config.render_event(event, target, Some(stats_update)) { - writeln!(self.out, "{line}")?; - } - Ok(()) - } - - fn print_header(&mut self) -> io::Result<()> { - if self.header_printed { - return Ok(()); - } - self.header_printed = true; - if let Some(header) = self.config.render_header() { - writeln!(self.out, "{header}")?; - } - Ok(()) - } - - #[cfg(feature = "stats")] - fn print_summary(&mut self, stats: &StatsCollector) -> io::Result<()> { - if !self.print_final_summary || !self.config.prints_summary() { - return Ok(()); - } - - write!( - self.out, - "{}", - crate::cmd::client::summary::format_summary_with_options( - &stats.snapshot(), - crate::cmd::client::summary::SummaryFormatOptions { - verbose: self.config.summary_verbose(), - show_running_only_note: self.show_running_only_summary_note, - }, - ) - )?; - - Ok(()) - } - - #[cfg(not(feature = "stats"))] - fn print_summary(&mut self) -> io::Result<()> { - Ok(()) - } -} - -#[cfg(feature = "stats")] -fn print_events_with_stats( - stream_output: &mut StreamOutput<'_, W>, - events: &[ClientEvent], - target: Option<&str>, - stats: &mut StatsCollector, -) -> io::Result<()> { - let stats_updates = events - .iter() - .map(|event| EventRenderStats::from(stats.process(event))) - .collect::>(); - stream_output.print_events(events, target, &stats_updates) -} - -#[cfg(not(feature = "stats"))] -fn print_events_with_stats( - stream_output: &mut StreamOutput<'_, W>, - events: &[ClientEvent], - target: Option<&str>, -) -> io::Result<()> { - let stats_updates = vec![EventRenderStats::default(); events.len()]; - stream_output.print_events(events, target, &stats_updates) -} - -fn run_group_stream( - args: ClientArgs, - targets: Vec, - output_config: OutputConfig, - continuous: bool, - shutdown_requested: &AtomicBool, -) -> Result<(), Box> { - let mut config = args.to_client_config(); - let managed_targets = targets - .iter() - .map(|target| target.managed.clone()) - .collect::>(); - let expected_target_count = managed_targets.len(); - if let Some(first) = managed_targets.first() { - config.server_addr = first.remote.to_string(); - } - - let group_config = ManagedClientGroupConfig { - client: config, - pacing: args.pacing.into(), - completion: ManagedGroupCompletionPolicy::AllTargetsComplete, - }; - - let (session, events) = ManagedClientGroup::start_with_subscription( - group_config, - managed_targets, - SubscriberConfig { - capacity: MANAGED_EVENT_CAPACITY, - // CLI output is best-effort under sustained backpressure. Dropping - // stale rows keeps continuous runs attached to the running group - // instead of turning a full output queue into a disconnected - // subscriber and a potentially blocking join. - overflow: SubscriberOverflow::DropOldest, - }, - )?; - - let mut stdout = io::LineWriter::new(io::stdout().lock()); #[cfg(feature = "stats")] let mut stats = targets .iter() @@ -374,173 +112,120 @@ fn run_group_stream( ) }) .collect::>(); - let mut stream_output = StreamOutput { - config: output_config, - header_printed: false, - print_final_summary: false, - show_running_only_summary_note: false, - out: &mut stdout, - }; - + let mut terminal_targets = HashSet::new(); + let mut dropped_events = 0_u64; let mut interrupted = false; let mut stop_requested = false; let mut peer_close_requested_stop = false; - let mut terminal_targets = HashSet::new(); - let mut last_event_at = Instant::now(); - let mut saw_target_event = false; - + let mut subscription_closed = false; loop { if is_shutdown_requested(shutdown_requested) { interrupted = true; - if request_group_stop_once(&mut stop_requested) { - session.stop(); + if request_managed_stop_once(&mut stop_requested) { + let _ = handle.stop(); } } - if request_group_stop_for_peer_close( + if request_managed_stop_for_peer_close( continuous, interrupted, - session.peer_closed_target_count(), + handle.status().peer_closed_target_outcomes, &mut peer_close_requested_stop, &mut stop_requested, ) { - session.stop(); + let _ = handle.stop(); + } + drain_events( + &mut events, + &mut stream_output, + #[cfg(feature = "stats")] + &mut stats, + &mut terminal_targets, + &mut dropped_events, + )?; + if handle.status().final_outcome.is_some() || subscription_closed { + break; } - match events.try_recv() { - Ok(Some(group_event)) => { - last_event_at = Instant::now(); - saw_target_event = true; - match group_event { - ManagedGroupEvent::Client(target_event) => { - #[cfg(feature = "stats")] - { - let label = target_event.target.as_str(); - let stats = stats - .entry(label.to_owned()) - .or_insert_with(|| StatsCollector::new(stats_config(continuous))); - print_target_event_with_stats( - &mut stream_output, - &target_event, - stats, - )?; - } - #[cfg(not(feature = "stats"))] - { - print_target_event_with_stats(&mut stream_output, &target_event)?; - } - } - ManagedGroupEvent::TargetFinished(target) => { - report_target_failure(&target); - terminal_targets.insert(target.id.as_str().to_owned()); - if request_group_stop_for_peer_close( - continuous, - interrupted, - u64::from(matches!( - &target.end_reason, - irtt_client::ManagedTargetEndReason::PeerClosed - )), - &mut peer_close_requested_stop, - &mut stop_requested, - ) { - session.stop(); - } - } - } - } - Ok(None) => { - if interrupted { - break; - } - if terminal_targets.len() >= expected_target_count { - break; - } - if should_join_group_after_idle(&args, continuous, saw_target_event, last_event_at) - { - break; - } - thread::sleep(GROUP_IDLE_SLEEP); - } - Err(EventSubscriptionError::Disconnected) => break, - } - } - - if interrupted { - eprintln!("interrupted, closing group..."); - } - - let outcome = session.join()?; - while let Ok(Some(group_event)) = events.try_recv() { - match group_event { - ManagedGroupEvent::Client(target_event) => { + Ok(event) => process_event( + event, + &mut stream_output, #[cfg(feature = "stats")] - { - let label = target_event.target.as_str(); - let stats = stats - .entry(label.to_owned()) - .or_insert_with(|| StatsCollector::new(stats_config(continuous))); - print_target_event_with_stats(&mut stream_output, &target_event, stats)?; - } - #[cfg(not(feature = "stats"))] - { - print_target_event_with_stats(&mut stream_output, &target_event)?; - } - } - ManagedGroupEvent::TargetFinished(target) => { - if terminal_targets.insert(target.id.as_str().to_owned()) { - report_target_failure(&target); - } - } + &mut stats, + &mut terminal_targets, + )?, + Err(ManagedEventTryRecvError::Empty) => thread::sleep(MANAGED_EVENT_WAIT_SLICE), + Err(ManagedEventTryRecvError::Lagged(count)) => dropped_events += count, + Err(ManagedEventTryRecvError::Closed) => subscription_closed = true, } } - if let Some(warning) = dropped_event_warning(events.dropped_events(), "managed group") { + if interrupted { + eprintln!("interrupted, closing managed run..."); + } + drain_events( + &mut events, + &mut stream_output, + #[cfg(feature = "stats")] + &mut stats, + &mut terminal_targets, + &mut dropped_events, + )?; + let outcome = owner.join()?; + drain_events( + &mut events, + &mut stream_output, + #[cfg(feature = "stats")] + &mut stats, + &mut terminal_targets, + &mut dropped_events, + )?; + if let Some(warning) = dropped_event_warning(dropped_events) { eprintln!("{warning}"); } - - interrupted |= is_shutdown_requested(shutdown_requested); - for target in &outcome.targets { - if terminal_targets.insert(target.id.as_str().to_owned()) { + if outcome.discarded_target_outcomes != 0 { + eprintln!( + "irtt-rs: warning: {} final target outcomes were discarded", + outcome.discarded_target_outcomes + ); + } + for target in outcome.recent_target_outcomes.iter() { + if terminal_targets.insert(target.target.clone()) { report_target_failure(target); } } - let successful_targets = outcome.successful_target_outcomes; - let failed_targets = outcome.failed_target_outcomes; - let peer_closed_target_outcomes = outcome - .peer_closed_target_outcomes - .max(u64::from(peer_close_requested_stop)); - let terminal_error = if let Some(error) = - peer_close_run_error(continuous, interrupted, peer_closed_target_outcomes) - { - Some(error) - } else if outcome.end_reason == ManagedGroupEndReason::Cancelled - && !interrupted - && !peer_close_requested_stop - { - Some("managed client group was cancelled".to_owned()) - } else if !interrupted && successful_targets == 0 && failed_targets > 0 { - Some(format!( - "no managed target completed successfully ({failed_targets} failed)" - )) - } else { - None - }; - - let print_final_summary = should_print_final_summary(continuous, interrupted); - stream_output.print_final_summary = print_final_summary; - stream_output.show_running_only_summary_note = continuous && interrupted && print_final_summary; - #[cfg(feature = "stats")] - { - for (label, stats) in &stats { - if stream_output.print_final_summary && stream_output.config.prints_summary() { - writeln!(stream_output.out)?; - writeln!(stream_output.out, "target: {label}")?; - } - stream_output.print_summary(stats)?; + interrupted |= is_shutdown_requested(shutdown_requested); + let terminal_error = match &outcome.end_reason { + ManagedEndReason::DriverFailed(failure) => { + Some(format!("managed driver failed: {failure}")) } + _ => peer_close_run_error(continuous, interrupted, outcome.peer_closed_target_outcomes) + .or_else(|| { + (!interrupted + && outcome.successful_target_outcomes == 0 + && outcome.failed_target_outcomes > 0) + .then(|| { + format!( + "no managed target completed successfully ({} failed)", + outcome.failed_target_outcomes + ) + }) + }), + }; + stream_output.print_final_summary = should_print_final_summary(continuous, interrupted); + stream_output.show_running_only_summary_note = + continuous && interrupted && stream_output.print_final_summary; + #[cfg(feature = "stats")] + for (label, stats) in &stats { + if multi_target + && stream_output.print_final_summary + && stream_output.config.prints_summary() + { + writeln!(stream_output.out)?; + writeln!(stream_output.out, "target: {label}")?; + } + stream_output.print_summary(stats)?; } #[cfg(not(feature = "stats"))] - { - stream_output.print_summary()?; - } + stream_output.print_summary()?; stream_output.out.flush()?; if let Some(error) = terminal_error { return Err(error.into()); @@ -548,73 +233,156 @@ fn run_group_stream( Ok(()) } +fn process_event( + event: ManagedEvent, + stream_output: &mut StreamOutput<'_, W>, + #[cfg(feature = "stats")] stats: &mut BTreeMap, + terminal_targets: &mut HashSet, +) -> io::Result<()> { + match event { + ManagedEvent::Client { target, event } => { + #[cfg(feature = "stats")] + { + let collector = stats + .entry(target.id.as_str().to_owned()) + .or_insert_with(|| StatsCollector::new(stats_config(false))); + print_events_with_stats( + stream_output, + std::slice::from_ref(&event), + Some(target.id.as_str()), + collector, + )?; + } + #[cfg(not(feature = "stats"))] + print_events_with_stats( + stream_output, + std::slice::from_ref(&event), + Some(target.id.as_str()), + )?; + } + ManagedEvent::TargetFinished { outcome } => { + terminal_targets.insert(outcome.target.clone()); + report_target_failure(&outcome); + } + _ => {} + } + Ok(()) +} + +fn drain_events( + events: &mut ManagedEventSubscription, + stream_output: &mut StreamOutput<'_, W>, + #[cfg(feature = "stats")] stats: &mut BTreeMap, + terminal_targets: &mut HashSet, + dropped_events: &mut u64, +) -> io::Result<()> { + loop { + match events.try_recv() { + Ok(event) => process_event( + event, + stream_output, + #[cfg(feature = "stats")] + stats, + terminal_targets, + )?, + Err(ManagedEventTryRecvError::Empty | ManagedEventTryRecvError::Closed) => break, + Err(ManagedEventTryRecvError::Lagged(count)) => *dropped_events += count, + } + } + Ok(()) +} + fn report_target_failure(target: &ManagedTargetOutcome) { - if let Some(failure) = target.end_reason.failure() { + if let ManagedTargetEndReason::Failed(failure) = &target.end_reason { eprintln!( - "irtt-rs: target {} failed ({}): {}", - target.id, failure.kind, failure.message + "irtt-rs: target {} failed ({:?} {:?}): {}", + target.target.id, failure.phase, failure.kind, failure.message ); } } - -fn dropped_event_warning(dropped_events: u64, source: &str) -> Option { - (dropped_events > 0).then(|| { - let event_word = if dropped_events == 1 { - "event" - } else { - "events" - }; - format!( - "irtt-rs: warning: dropped {dropped_events} {source} {event_word}; output and statistics may be incomplete" - ) - }) +fn dropped_event_warning(dropped_events: u64) -> Option { + (dropped_events > 0).then(|| format!("irtt-rs: warning: dropped {dropped_events} managed run event{}; output and statistics may be incomplete", if dropped_events == 1 { "" } else { "s" })) } -fn estimated_group_completion_grace(args: &ClientArgs) -> Duration { - let open_timeout: Duration = args.to_client_config().open_timeouts.iter().sum(); - open_timeout - .saturating_add(args.duration) - .saturating_add(GROUP_COMPLETION_GRACE) +struct StreamOutput<'a, W: Write> { + config: OutputConfig, + header_printed: bool, + print_final_summary: bool, + show_running_only_summary_note: bool, + out: &'a mut W, } - -fn should_join_group_after_idle( - args: &ClientArgs, - continuous: bool, - saw_target_event: bool, - last_event_at: Instant, -) -> bool { - if continuous && saw_target_event { - return false; +impl StreamOutput<'_, W> { + fn print_events( + &mut self, + events: &[ClientEvent], + target: Option<&str>, + stats_updates: &[EventRenderStats], + ) -> io::Result<()> { + self.print_header()?; + for (event, stats_update) in events.iter().zip(stats_updates) { + if let Some(line) = self.config.render_event(event, target, Some(stats_update)) { + writeln!(self.out, "{line}")?; + } + } + Ok(()) + } + fn print_header(&mut self) -> io::Result<()> { + if self.header_printed { + return Ok(()); + } + self.header_printed = true; + if let Some(header) = self.config.render_header() { + writeln!(self.out, "{header}")?; + } + Ok(()) + } + #[cfg(feature = "stats")] + fn print_summary(&mut self, stats: &StatsCollector) -> io::Result<()> { + if self.print_final_summary && self.config.prints_summary() { + write!( + self.out, + "{}", + crate::cmd::client::summary::format_summary_with_options( + &stats.snapshot(), + crate::cmd::client::summary::SummaryFormatOptions { + verbose: self.config.summary_verbose(), + show_running_only_note: self.show_running_only_summary_note + } + ) + )?; + } + Ok(()) + } + #[cfg(not(feature = "stats"))] + fn print_summary(&mut self) -> io::Result<()> { + Ok(()) } - last_event_at.elapsed() > estimated_group_completion_grace(args) } - #[cfg(feature = "stats")] -fn print_target_event_with_stats( +fn print_events_with_stats( stream_output: &mut StreamOutput<'_, W>, - target_event: &TargetEvent, + events: &[ClientEvent], + target: Option<&str>, stats: &mut StatsCollector, ) -> io::Result<()> { - print_events_with_stats( - stream_output, - std::slice::from_ref(&target_event.event), - Some(target_event.target.as_str()), - stats, - ) + let updates = events + .iter() + .map(|event| EventRenderStats::from(stats.process(event))) + .collect::>(); + stream_output.print_events(events, target, &updates) } - #[cfg(not(feature = "stats"))] -fn print_target_event_with_stats( +fn print_events_with_stats( stream_output: &mut StreamOutput<'_, W>, - target_event: &TargetEvent, + events: &[ClientEvent], + target: Option<&str>, ) -> io::Result<()> { - print_events_with_stats( - stream_output, - std::slice::from_ref(&target_event.event), - Some(target_event.target.as_str()), + stream_output.print_events( + events, + target, + &vec![EventRenderStats::default(); events.len()], ) } - #[cfg(feature = "stats")] fn stats_config(continuous: bool) -> StatsConfig { if continuous { @@ -623,25 +391,21 @@ fn stats_config(continuous: bool) -> StatsConfig { StatsConfig::finite() } } - -#[cfg(feature = "stats")] -fn estimate_finite_stats_memory_bytes(expected_probes: u64) -> u64 { - expected_probes.saturating_mul(FINITE_STATS_BYTES_PER_PROBE) -} - #[cfg(feature = "stats")] fn finite_stats_memory_warning(args: &ClientArgs) -> Option { if args.is_continuous() || args.duration.is_zero() { return None; } - - let expected_probes = expected_probe_count(args.duration, args.interval); - let estimated_bytes = estimate_finite_stats_memory_bytes(expected_probes); + let estimated_bytes = expected_probe_count(args.duration, args.interval) + .saturating_mul(FINITE_STATS_BYTES_PER_PROBE); if estimated_bytes < FINITE_STATS_MEMORY_WARNING_BYTES { return None; } - - let formatted = format_bytes_for_warning(estimated_bytes); + let formatted = if estimated_bytes >= GIB { + format!("{} GiB", estimated_bytes.saturating_add(GIB / 2) / GIB) + } else { + format!("{} MiB", estimated_bytes.saturating_add(MIB / 2) / MIB) + }; let guidance = if estimated_bytes >= FINITE_STATS_MEMORY_VERY_STRONG_WARNING_BYTES { "this may be unsuitable on memory-constrained systems" } else if estimated_bytes >= FINITE_STATS_MEMORY_STRONG_WARNING_BYTES { @@ -649,329 +413,20 @@ fn finite_stats_memory_warning(args: &ClientArgs) -> Option { } else { "use continuous mode for bounded-memory long-running tests" }; - - Some(format!( - "irtt-rs: warning: finite exact statistics may retain about {formatted} for this run; {guidance}" - )) + Some(format!("irtt-rs: warning: finite exact statistics may retain about {formatted} for this run; {guidance}")) } -#[cfg(feature = "stats")] -fn format_bytes_for_warning(bytes: u64) -> String { - if bytes >= GIB { - format!("{} GiB", bytes.saturating_add(GIB / 2) / GIB) - } else { - format!("{} MiB", bytes.saturating_add(MIB / 2) / MIB) - } -} - -#[cfg(all(test, feature = "stats"))] +#[cfg(test)] mod tests { use super::*; - use std::{ - net::{IpAddr, Ipv4Addr, SocketAddr}, - time::{Duration, Instant, UNIX_EPOCH}, - }; - - fn test_timestamp(offset: Duration) -> ClientTimestamp { - ClientTimestamp { - wall: UNIX_EPOCH + offset, - mono: Instant::now() + offset, - } - } - - fn test_remote() -> SocketAddr { - SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 2112) - } - - fn reply_event(seq: u32, rtt_us: u64) -> ClientEvent { - let sent_at = test_timestamp(Duration::from_millis(seq as u64)); - let received_at = ClientTimestamp { - wall: sent_at.wall + Duration::from_micros(rtt_us), - mono: sent_at.mono + Duration::from_micros(rtt_us), - }; - ClientEvent::EchoReply { - seq, - remote: test_remote(), - sent_at, - received_at, - rtt: RttSample { - raw: Duration::from_micros(rtt_us), - adjusted: None, - effective: SignedDuration::from_nanos(i128::from(rtt_us) * 1_000), - }, - server_timing: None, - one_way: None, - received_stats: None, - bytes: 64, - packet_meta: PacketMeta::default(), - } - } - - fn cli_args(args: &[&str]) -> ClientArgs { - let mut argv = vec!["irtt-rs"]; - argv.extend_from_slice(args); - ::try_parse_from(argv).unwrap() - } - - fn output_config( - format: crate::cmd::client::OutputFormat, - columns: Option<&str>, - header: crate::cmd::client::HeaderMode, - verbose: bool, - ) -> OutputConfig { - OutputConfig::new(format, columns, header, verbose, false).unwrap() - } - #[test] - fn dropped_event_warning_marks_statistics_incomplete() { - assert_eq!(dropped_event_warning(0, "managed client"), None); - let warning = dropped_event_warning(7, "managed group").unwrap(); - assert!(warning.contains("dropped 7 managed group events")); - assert!(warning.contains("statistics may be incomplete")); - } - - #[test] - fn single_client_drain_preserves_final_queue_and_surfaces_overflow() { - let hub = irtt_client::EventHub::new(); - let events = hub - .subscribe(SubscriberConfig { - capacity: 1, - overflow: SubscriberOverflow::DropOldest, - }) - .unwrap(); - hub.publish(reply_event(1, 1_000)); - hub.publish(reply_event(2, 2_000)); - hub.disconnect_all(); - - let mut stats = StatsCollector::new(StatsConfig::finite()); - let mut out = Vec::new(); - { - let mut stream_output = StreamOutput { - config: output_config( - crate::cmd::client::OutputFormat::Csv, - Some("event,seq"), - crate::cmd::client::HeaderMode::Never, - false, - ), - header_printed: false, - print_final_summary: false, - show_running_only_summary_note: false, - out: &mut out, - }; - drain_single_client_events(&events, &mut stream_output, "target", &mut stats).unwrap(); - } - - let rendered = String::from_utf8(out).unwrap(); - assert_eq!(rendered.trim(), "echo_reply,2"); - assert_eq!(events.dropped_events(), 1); - assert!( - dropped_event_warning(events.dropped_events(), "managed client") - .unwrap() - .contains("dropped 1 managed client event") - ); - assert_eq!( - events.try_recv().unwrap_err(), - EventSubscriptionError::Disconnected - ); - } - - #[test] - fn output_helper_streams_and_collects_events() { - let sent_at = test_timestamp(Duration::from_secs(1)); - let received_at = test_timestamp(Duration::from_secs(1) + Duration::from_micros(1200)); - let events = [ - ClientEvent::EchoSent { - seq: 1, - remote: test_remote(), - scheduled_at: sent_at.mono, - sent_at, - bytes: 64, - send_call: Duration::from_micros(10), - timer_error: Duration::ZERO, - }, - ClientEvent::EchoReply { - seq: 1, - remote: test_remote(), - sent_at, - received_at, - rtt: RttSample { - raw: Duration::from_micros(1200), - adjusted: None, - effective: SignedDuration::from_nanos(1_200_000), - }, - server_timing: None, - one_way: None, - received_stats: None, - bytes: 64, - packet_meta: PacketMeta::default(), - }, - ]; - - let mut stats = StatsCollector::new(StatsConfig::finite()); - let mut out = Vec::new(); - { - let mut stream_output = StreamOutput { - config: output_config( - crate::cmd::client::OutputFormat::Tsv, - Some("effective_rtt_us"), - crate::cmd::client::HeaderMode::Never, - false, - ), - header_printed: false, - print_final_summary: true, - show_running_only_summary_note: false, - out: &mut out, - }; - print_events_with_stats(&mut stream_output, &events, None, &mut stats).unwrap(); - stream_output.print_summary(&stats).unwrap(); - } - - let rendered = String::from_utf8(out).unwrap(); - let summary = stats.snapshot(); - assert!(!rendered.is_empty()); - assert_eq!(summary.packets.packets_sent, 1); - assert_eq!(summary.packets.unique_replies, 1); - } - - #[test] - fn continuous_summary_prints_when_enabled_and_suppresses_when_disabled() { - let mut stats = StatsCollector::new(StatsConfig::continuous()); - let mut out = Vec::new(); - { - let mut stream_output = StreamOutput { - config: output_config( - crate::cmd::client::OutputFormat::Table, - None, - crate::cmd::client::HeaderMode::Auto, - false, - ), - header_printed: false, - print_final_summary: true, - show_running_only_summary_note: true, - out: &mut out, - }; - print_events_with_stats( - &mut stream_output, - &[reply_event(1, 1200)], - None, - &mut stats, - ) - .unwrap(); - stream_output.print_summary(&stats).unwrap(); - } - - let rendered = String::from_utf8(out).unwrap(); - assert!(rendered.contains("irtt-rs summary")); - assert!(rendered.contains("medians unavailable")); - assert!(rendered.contains("continuous mode")); - assert!(rendered.contains("packets:")); - assert!(rendered.contains("received=1")); - - let stats = StatsCollector::new(StatsConfig::continuous()); - let mut out = Vec::new(); - { - let mut stream_output = StreamOutput { - config: output_config( - crate::cmd::client::OutputFormat::Table, - None, - crate::cmd::client::HeaderMode::Auto, - false, - ), - header_printed: false, - print_final_summary: false, - show_running_only_summary_note: true, - out: &mut out, - }; - stream_output.print_summary(&stats).unwrap(); - } - - assert!(out.is_empty()); - } - - #[test] - fn continuous_mode_uses_continuous_stats_config() { - let mut collector = StatsCollector::new(stats_config(true)); - for seq in 0..5000_u32 { - let sent_at = test_timestamp(Duration::from_micros(seq as u64)); - let received_at = test_timestamp(Duration::from_micros(seq as u64 + 1)); - collector.process(&ClientEvent::EchoReply { - seq, - remote: test_remote(), - sent_at, - received_at, - rtt: RttSample { - raw: Duration::from_micros(1), - adjusted: None, - effective: SignedDuration::from_nanos(1_000), - }, - server_timing: None, - one_way: None, - received_stats: None, - bytes: 64, - packet_meta: PacketMeta::default(), - }); - } - - assert_eq!(collector.snapshot().rtt.primary.median_ns, None); - } - - #[test] - fn finite_stats_memory_warning_reports_only_large_finite_runs() { - for args in [ - cli_args(&["--duration", "0", "--interval", "1ms", "127.0.0.1:2112"]), - cli_args(&[ - "--duration", - "1000ms", - "--interval", - "1ms", - "127.0.0.1:2112", - ]), - ] { - assert_eq!(finite_stats_memory_warning(&args), None); - } - - for (threshold, size, guidance) in [ - ( - FINITE_STATS_MEMORY_WARNING_BYTES, - "about 128 MiB", - "bounded-memory long-running tests", - ), - ( - FINITE_STATS_MEMORY_STRONG_WARNING_BYTES, - "about 512 MiB", - "shortening the run", - ), - ( - FINITE_STATS_MEMORY_VERY_STRONG_WARNING_BYTES, - "about 1 GiB", - "memory-constrained systems", - ), - ] { - let expected_probes = threshold.div_ceil(FINITE_STATS_BYTES_PER_PROBE); - let args = cli_args(&[ - "--duration", - &format!("{expected_probes}ms"), - "--interval", - "1ms", - "127.0.0.1:2112", - ]); - - let warning = finite_stats_memory_warning(&args).unwrap(); - assert!(warning.contains(size)); - assert!(warning.contains(guidance)); - } - } - - #[test] - fn expected_probe_count_rounds_up() { - assert_eq!( - expected_probe_count(Duration::from_millis(1001), Duration::from_secs(1)), - 2 - ); - } - - #[test] - fn estimate_finite_stats_memory_bytes_saturates() { - assert_eq!(estimate_finite_stats_memory_bytes(u64::MAX), u64::MAX); + fn lagged_events_are_counted_exactly() { + let mut dropped = 0; + dropped += 7; + dropped += 3; + assert_eq!(dropped, 10); + assert!(dropped_event_warning(dropped) + .unwrap() + .contains("dropped 10 managed run events")); } } diff --git a/crates/irtt-cli/src/cmd/tui/args.rs b/crates/irtt-cli/src/cmd/tui/args.rs index b8d4f5a..9104f7c 100644 --- a/crates/irtt-cli/src/cmd/tui/args.rs +++ b/crates/irtt-cli/src/cmd/tui/args.rs @@ -6,8 +6,8 @@ use irtt_client::ClientConfig; #[cfg(test)] use crate::shared::client::TimestampArg; use crate::shared::client::{ - parse_labelled_target, parse_test_duration, resolved_managed_targets, target_specs, - CommonClientArgs, GroupPacingArg, LabelledTargetArg, ResolvedTarget, TargetSpec, + parse_labelled_target, parse_test_duration, prepare_managed_targets, target_specs, + CommonClientArgs, GroupPacingArg, LabelledTargetArg, PreparedTarget, TargetSpec, }; pub const DEFAULT_TUI_DURATION: Duration = Duration::ZERO; @@ -72,10 +72,8 @@ impl TuiArgs { target_specs(&self.targets, &self.labelled_targets) } - pub fn resolved_managed_targets(&self) -> Result, String> { - let specs = self.target_specs()?; - let config = self.to_client_config(); - resolved_managed_targets(specs, &config) + pub fn managed_targets(&self) -> Result, String> { + prepare_managed_targets(self.target_specs()?) } #[cfg(test)] @@ -93,7 +91,7 @@ impl TuiArgs { } pub type TuiTargetSpec = TargetSpec; -pub type ResolvedTuiTarget = ResolvedTarget; +pub type TuiManagedTarget = PreparedTarget; impl std::ops::Deref for TuiArgs { type Target = CommonClientArgs; @@ -199,11 +197,9 @@ mod tests { } #[test] - fn duplicate_resolved_target_addresses_are_rejected() { + fn duplicate_target_endpoints_are_allowed() { let args = parse(&["127.0.0.1:2112", "127.0.0.1"]).unwrap(); - let err = args.resolved_managed_targets().unwrap_err(); - - assert!(err.contains("duplicate resolved target address 127.0.0.1:2112")); + assert_eq!(args.managed_targets().unwrap().len(), 2); } #[test] diff --git a/crates/irtt-cli/src/cmd/tui/run.rs b/crates/irtt-cli/src/cmd/tui/run.rs index 6846406..ba6b630 100644 --- a/crates/irtt-cli/src/cmd/tui/run.rs +++ b/crates/irtt-cli/src/cmd/tui/run.rs @@ -1,33 +1,29 @@ use std::{ + collections::HashSet, io, sync::atomic::{AtomicBool, Ordering}, thread, time::{Duration, Instant}, }; -use crossterm::event::{self, Event, KeyCode, KeyEventKind, KeyModifiers}; -use irtt_client::{ - EventSubscription, EventSubscriptionError, ManagedClient, ManagedClientGroup, - ManagedClientGroupConfig, ManagedGroupCompletionPolicy, ManagedGroupEndReason, - ManagedGroupEvent, SessionEndReason, SubscriberConfig, SubscriberOverflow, -}; - +use super::ui::{should_render, TuiConfig, TuiState, TuiStatus, TuiTerminal}; use crate::{ - cmd::tui::args::{ResolvedTuiTarget, TuiArgs}, + cmd::tui::args::TuiArgs, shared::client::{ is_shutdown_requested, session::{ - peer_close_run_error, request_group_stop_for_peer_close, request_group_stop_once, + peer_close_run_error, request_managed_stop_for_peer_close, request_managed_stop_once, }, }, }; - -use super::ui::{should_render, TuiConfig, TuiState, TuiStatus, TuiTerminal}; +use crossterm::event::{self, Event, KeyCode, KeyEventKind, KeyModifiers}; +use irtt_client::managed::{ + BlockingManagedClient, ManagedClientConfig, ManagedCompletionPolicy, ManagedEndReason, + ManagedEvent, ManagedEventSubscription, ManagedEventTryRecvError, TargetInstance, +}; const RENDER_INTERVAL: Duration = Duration::from_millis(250); const TUI_WAIT_SLICE: Duration = Duration::from_millis(20); -const IDLE_SLEEP: Duration = Duration::from_millis(5); -const GROUP_COMPLETION_GRACE: Duration = Duration::from_secs(1); const MANAGED_EVENT_CAPACITY: usize = 16_384; pub fn run_tui( @@ -35,7 +31,7 @@ pub fn run_tui( shutdown_requested: &AtomicBool, ) -> Result<(), Box> { let targets = args - .resolved_managed_targets() + .managed_targets() .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?; let continuous = args.is_continuous(); let mut terminal = TuiTerminal::enter()?; @@ -44,329 +40,173 @@ pub fn run_tui( targets.iter().map(|target| target.label.clone()), ); let mut next_render = Instant::now(); - if is_shutdown_requested(shutdown_requested) { return Ok(()); } - state.set_status(TuiStatus::Opening); render_if_due(&mut terminal, &state, &mut next_render, true)?; - - if targets.len() > 1 { - return run_group_tui( - args, - targets, - &mut terminal, - &mut state, - &mut next_render, - shutdown_requested, - ); - } - - let (session, events) = match ManagedClient::start_with_subscription( - args.to_client_config(), - managed_event_subscriber_config(), + let target_count = targets.len(); + let config = ManagedClientConfig { + client: args.to_client_config(), + pacing: args.pacing.into(), + completion: ManagedCompletionPolicy::FinishWhenQuiescent, + event_capacity: MANAGED_EVENT_CAPACITY, + outcome_history_limit: target_count, + max_live_target_generations: target_count, + ..ManagedClientConfig::default() + }; + let (owner, mut events) = match BlockingManagedClient::start_with_subscription( + config, + targets + .iter() + .map(|target| target.managed.clone()) + .collect(), ) { - Ok(session) => session, - Err(err) => { - state.set_error(err.to_string()); + Ok(value) => value, + Err(error) => { + state.set_error(error.to_string()); render_if_due(&mut terminal, &state, &mut next_render, true)?; - return Err(Box::new(err)); + return Err(Box::new(error)); } }; - + let handle = owner.handle(); let mut interrupted = false; + let mut stop_requested = false; + let mut peer_close_requested_stop = false; + let mut terminal_targets = HashSet::new(); + let mut dropped_events = 0; + let mut subscription_closed = false; loop { if is_shutdown_requested(shutdown_requested) { interrupted = true; - session.stop(); - break; + if request_managed_stop_once(&mut stop_requested) { + let _ = handle.stop(); + } } - if handle_input(&mut state, shutdown_requested)? { render_if_due(&mut terminal, &state, &mut next_render, true)?; } if state.quit_requested { interrupted = true; - session.stop(); - break; - } - - match events.recv_timeout(managed_tui_wait_duration(&next_render, state.paused)) { - Ok(Some(event)) => { - state.process_event(&event); - render_if_due(&mut terminal, &state, &mut next_render, false)?; - } - Ok(None) => { - render_if_due(&mut terminal, &state, &mut next_render, false)?; - } - Err(EventSubscriptionError::Disconnected) => break, - } - } - interrupted |= is_shutdown_requested(shutdown_requested); - - if interrupted { - state.set_status(TuiStatus::Interrupted); - render_if_due(&mut terminal, &state, &mut next_render, true)?; - } - - if interrupted { - state.set_status(TuiStatus::Draining); - drain_single_tui_events(&events, &mut state); - render_if_due(&mut terminal, &state, &mut next_render, true)?; - } - - state.set_status(TuiStatus::Closing); - render_if_due(&mut terminal, &state, &mut next_render, true)?; - - let outcome = session.join(); - drain_single_tui_events(&events, &mut state); - state.mark_dropped_client_events(events.dropped_events()); - let outcome = match outcome { - Ok(outcome) => outcome, - Err(err) => { - state.set_error(err.to_string()); - render_if_due(&mut terminal, &state, &mut next_render, true)?; - return Err(Box::new(err)); - } - }; - - interrupted |= is_shutdown_requested(shutdown_requested); - if let Some(error) = peer_close_run_error( - continuous, - interrupted, - u64::from(outcome.end_reason == SessionEndReason::PeerClosed), - ) { - state.set_run_error(error.clone()); - render_if_due(&mut terminal, &state, &mut next_render, true)?; - return Err(error.into()); - } - state.set_status(TuiStatus::Complete); - render_if_due(&mut terminal, &state, &mut next_render, true)?; - Ok(()) -} - -fn run_group_tui( - args: TuiArgs, - targets: Vec, - terminal: &mut TuiTerminal, - state: &mut TuiState, - next_render: &mut Instant, - shutdown_requested: &AtomicBool, -) -> Result<(), Box> { - let mut config = args.to_client_config(); - let managed_targets = targets - .iter() - .map(|target| target.managed.clone()) - .collect::>(); - let expected_target_count = managed_targets.len(); - if let Some(first) = managed_targets.first() { - config.server_addr = first.remote.to_string(); - } - - let group_config = ManagedClientGroupConfig { - client: config, - pacing: args.pacing.into(), - completion: ManagedGroupCompletionPolicy::AllTargetsComplete, - }; - let (session, events) = ManagedClientGroup::start_with_subscription( - group_config, - managed_targets, - SubscriberConfig { - capacity: MANAGED_EVENT_CAPACITY, - overflow: SubscriberOverflow::DropOldest, - }, - )?; - - let mut interrupted = false; - let mut stop_requested = false; - let mut peer_close_requested_stop = false; - let mut terminal_targets = std::collections::HashSet::new(); - let mut saw_target_event = false; - let mut last_event_at = Instant::now(); - - let exit = loop { - if is_shutdown_requested(shutdown_requested) { - interrupted = true; - if request_group_stop_once(&mut stop_requested) { - session.stop(); + if request_managed_stop_once(&mut stop_requested) { + let _ = handle.stop(); } } - - if handle_input(state, shutdown_requested)? { - render_if_due(terminal, state, next_render, true)?; - } - if state.quit_requested { - interrupted = true; - if request_group_stop_once(&mut stop_requested) { - session.stop(); - } - } - if request_group_stop_for_peer_close( - args.is_continuous(), + if request_managed_stop_for_peer_close( + continuous, interrupted, - session.peer_closed_target_count(), + handle.status().peer_closed_target_outcomes, &mut peer_close_requested_stop, &mut stop_requested, ) { - session.stop(); + let _ = handle.stop(); + } + drain_tui_events( + &mut events, + &mut state, + &mut terminal_targets, + &mut dropped_events, + ); + if handle.status().final_outcome.is_some() || subscription_closed { + break; } - match events.try_recv() { - Ok(Some(group_event)) => { - saw_target_event = true; - last_event_at = Instant::now(); - match group_event { - ManagedGroupEvent::Client(target_event) => { - state.process_target_event(&target_event); - } - ManagedGroupEvent::TargetFinished(target) => { - terminal_targets.insert(target.id.as_str().to_owned()); - state.process_target_outcome(&target); - if request_group_stop_for_peer_close( - args.is_continuous(), - interrupted, - u64::from(matches!( - &target.end_reason, - irtt_client::ManagedTargetEndReason::PeerClosed - )), - &mut peer_close_requested_stop, - &mut stop_requested, - ) { - session.stop(); - } - } - } - render_if_due(terminal, state, next_render, false)?; - } - Ok(None) => { - if interrupted { - break GroupLoopExit::Interrupted; - } - if terminal_targets.len() >= expected_target_count { - break GroupLoopExit::AllTargetsTerminal; - } - if should_join_group_after_idle(&args, saw_target_event, last_event_at) { - break GroupLoopExit::IdleGraceElapsed; - } - wait_for_tui_activity(None, next_render, state, terminal, shutdown_requested)?; - thread::sleep(IDLE_SLEEP); - } - Err(EventSubscriptionError::Disconnected) => { - break GroupLoopExit::SubscriptionDisconnected + Ok(event) => process_tui_event(event, &mut state, &mut terminal_targets), + Err(ManagedEventTryRecvError::Empty) => { + thread::sleep(managed_tui_wait_duration(&next_render, state.paused)) } + Err(ManagedEventTryRecvError::Lagged(count)) => dropped_events += count, + Err(ManagedEventTryRecvError::Closed) => subscription_closed = true, } - }; - - if exit.should_stop_before_join() && request_group_stop_once(&mut stop_requested) { - session.stop(); + render_if_due(&mut terminal, &state, &mut next_render, false)?; } - if interrupted { state.set_status(TuiStatus::Interrupted); - render_if_due(terminal, state, next_render, true)?; + render_if_due(&mut terminal, &state, &mut next_render, true)?; } - state.set_status(TuiStatus::Closing); - render_if_due(terminal, state, next_render, true)?; - - let outcome = session.join()?; - while let Ok(Some(group_event)) = events.try_recv() { - match group_event { - ManagedGroupEvent::Client(target_event) => state.process_target_event(&target_event), - ManagedGroupEvent::TargetFinished(target) => { - if terminal_targets.insert(target.id.as_str().to_owned()) { - state.process_target_outcome(&target); - } - } - } + render_if_due(&mut terminal, &state, &mut next_render, true)?; + drain_tui_events( + &mut events, + &mut state, + &mut terminal_targets, + &mut dropped_events, + ); + let outcome = owner.join()?; + drain_tui_events( + &mut events, + &mut state, + &mut terminal_targets, + &mut dropped_events, + ); + state.mark_dropped_managed_events(dropped_events); + if outcome.discarded_target_outcomes != 0 { + state.set_run_error(format!( + "{} final target outcomes were discarded", + outcome.discarded_target_outcomes + )); } - state.mark_dropped_group_events(events.dropped_events()); - for target in &outcome.targets { - if terminal_targets.insert(target.id.as_str().to_owned()) { + for target in outcome.recent_target_outcomes.iter() { + if terminal_targets.insert(target.target.clone()) { state.process_target_outcome(target); } } - interrupted |= is_shutdown_requested(shutdown_requested); - let peer_closed_target_outcomes = outcome - .peer_closed_target_outcomes - .max(u64::from(peer_close_requested_stop)); - if let Some(error) = peer_close_run_error( - args.is_continuous(), - interrupted, - peer_closed_target_outcomes, - ) { + let error = match &outcome.end_reason { + ManagedEndReason::DriverFailed(failure) => { + Some(format!("managed driver failed: {failure}")) + } + _ => peer_close_run_error(continuous, interrupted, outcome.peer_closed_target_outcomes) + .or_else(|| { + (!interrupted + && outcome.successful_target_outcomes == 0 + && outcome.failed_target_outcomes > 0) + .then(|| { + format!( + "no managed target completed successfully ({} failed)", + outcome.failed_target_outcomes + ) + }) + }), + }; + if let Some(error) = error { state.set_run_error(error.clone()); - render_if_due(terminal, state, next_render, true)?; + render_if_due(&mut terminal, &state, &mut next_render, true)?; return Err(error.into()); } - if outcome.end_reason == ManagedGroupEndReason::Cancelled - && !interrupted - && !peer_close_requested_stop - { - return match exit { - GroupLoopExit::IdleGraceElapsed => { - Err("managed client group stayed idle before all targets completed".into()) - } - GroupLoopExit::SubscriptionDisconnected => { - Err("managed client group event subscription disconnected before completion".into()) - } - GroupLoopExit::Interrupted | GroupLoopExit::AllTargetsTerminal => { - Err("managed client group was cancelled".into()) - } - }; - } - let successful_targets = outcome.successful_target_outcomes; - let failed_targets = outcome.failed_target_outcomes; - if !interrupted && successful_targets == 0 && failed_targets > 0 { - state.set_status(TuiStatus::Error); - render_if_due(terminal, state, next_render, true)?; - return Err( - format!("no managed target completed successfully ({failed_targets} failed)").into(), - ); - } - state.set_status(TuiStatus::Complete); - render_if_due(terminal, state, next_render, true)?; + render_if_due(&mut terminal, &state, &mut next_render, true)?; Ok(()) } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum GroupLoopExit { - Interrupted, - AllTargetsTerminal, - IdleGraceElapsed, - SubscriptionDisconnected, -} - -impl GroupLoopExit { - fn should_stop_before_join(self) -> bool { - matches!( - self, - Self::Interrupted | Self::IdleGraceElapsed | Self::SubscriptionDisconnected - ) +fn process_tui_event( + event: ManagedEvent, + state: &mut TuiState, + terminal_targets: &mut HashSet, +) { + match event { + ManagedEvent::Client { target, event } => state.process_target_event(&target, &event), + ManagedEvent::TargetFinished { outcome } => { + terminal_targets.insert(outcome.target.clone()); + state.process_target_outcome(&outcome); + } + _ => {} } } - -fn estimated_group_completion_grace(args: &TuiArgs) -> Duration { - let open_timeout: Duration = args.to_client_config().open_timeouts.iter().sum(); - open_timeout - .saturating_add(args.duration) - .saturating_add(GROUP_COMPLETION_GRACE) -} - -fn should_join_group_after_idle( - args: &TuiArgs, - saw_target_event: bool, - last_event_at: Instant, -) -> bool { - if args.is_continuous() && saw_target_event { - return false; +fn drain_tui_events( + events: &mut ManagedEventSubscription, + state: &mut TuiState, + terminal_targets: &mut HashSet, + dropped_events: &mut u64, +) { + loop { + match events.try_recv() { + Ok(event) => process_tui_event(event, state, terminal_targets), + Err(ManagedEventTryRecvError::Empty | ManagedEventTryRecvError::Closed) => break, + Err(ManagedEventTryRecvError::Lagged(count)) => *dropped_events += count, + } } - last_event_at.elapsed() > estimated_group_completion_grace(args) } - fn handle_input(state: &mut TuiState, shutdown_requested: &AtomicBool) -> io::Result { let mut force_render = false; while event::poll(Duration::ZERO)? { @@ -377,12 +217,10 @@ fn handle_input(state: &mut TuiState, shutdown_requested: &AtomicBool) -> io::Re continue; } match key.code { - KeyCode::Char('q') => { - state.quit_requested = true; - shutdown_requested.store(true, Ordering::Relaxed); - force_render = true; - } - KeyCode::Char('c') if key.modifiers.contains(KeyModifiers::CONTROL) => { + KeyCode::Char('q') | KeyCode::Char('c') + if key.code == KeyCode::Char('q') + || key.modifiers.contains(KeyModifiers::CONTROL) => + { state.quit_requested = true; shutdown_requested.store(true, Ordering::Relaxed); force_render = true; @@ -444,7 +282,6 @@ fn handle_input(state: &mut TuiState, shutdown_requested: &AtomicBool) -> io::Re } Ok(force_render) } - fn render_if_due( terminal: &mut TuiTerminal, state: &TuiState, @@ -452,27 +289,12 @@ fn render_if_due( force: bool, ) -> io::Result<()> { let now = Instant::now(); - if !should_render(now, *next_render, state.paused, force) { - return Ok(()); + if should_render(now, *next_render, state.paused, force) { + terminal.draw(state)?; + *next_render = now + RENDER_INTERVAL; } - terminal.draw(state)?; - *next_render = now + RENDER_INTERVAL; Ok(()) } - -fn managed_event_subscriber_config() -> SubscriberConfig { - SubscriberConfig { - capacity: MANAGED_EVENT_CAPACITY, - overflow: SubscriberOverflow::DropOldest, - } -} - -fn drain_single_tui_events(events: &EventSubscription, state: &mut TuiState) { - while let Ok(Some(event)) = events.try_recv() { - state.process_event(&event); - } -} - fn managed_tui_wait_duration(next_render: &Instant, paused: bool) -> Duration { let render_wait = if paused { TUI_WAIT_SLICE @@ -481,152 +303,3 @@ fn managed_tui_wait_duration(next_render: &Instant, paused: bool) -> Duration { }; render_wait.min(TUI_WAIT_SLICE) } - -fn wait_for_tui_activity( - next_send_deadline: Option, - next_render: &mut Instant, - state: &mut TuiState, - terminal: &mut TuiTerminal, - shutdown_requested: &AtomicBool, -) -> io::Result<()> { - let wait_for = tui_wait_duration(next_send_deadline, *next_render, state.paused); - if wait_for.is_zero() || !event::poll(wait_for)? { - return Ok(()); - } - - if handle_input(state, shutdown_requested)? { - render_if_due(terminal, state, next_render, true)?; - } - Ok(()) -} - -fn tui_wait_duration( - next_send_deadline: Option, - next_render: Instant, - paused: bool, -) -> Duration { - let now = Instant::now(); - let send_wait = next_send_deadline - .map(|deadline| deadline.saturating_duration_since(now)) - .unwrap_or(IDLE_SLEEP); - let render_wait = if paused { - send_wait - } else { - next_render.saturating_duration_since(now) - }; - send_wait.min(render_wait).min(TUI_WAIT_SLICE) -} - -#[cfg(test)] -mod tests { - use super::*; - use clap::Parser; - - fn parse(args: &[&str]) -> TuiArgs { - let mut argv = vec!["irtt-tui"]; - argv.extend_from_slice(args); - TuiArgs::try_parse_from(argv).unwrap() - } - - #[test] - fn continuous_multi_target_idle_does_not_stop_after_events() { - let args = parse(&["127.0.0.1:2112", "127.0.0.2:2112"]); - let old_event_at = Instant::now() - estimated_group_completion_grace(&args) - IDLE_SLEEP; - - assert!(!should_join_group_after_idle(&args, true, old_event_at)); - } - - #[test] - fn finite_or_unopened_group_can_leave_after_idle_grace() { - let finite = parse(&["--duration", "1s", "127.0.0.1:2112", "127.0.0.2:2112"]); - let finite_old_event_at = - Instant::now() - estimated_group_completion_grace(&finite) - IDLE_SLEEP; - assert!(should_join_group_after_idle( - &finite, - true, - finite_old_event_at - )); - - let continuous = parse(&["127.0.0.1:2112", "127.0.0.2:2112"]); - let unopened_old_event_at = - Instant::now() - estimated_group_completion_grace(&continuous) - IDLE_SLEEP; - assert!(should_join_group_after_idle( - &continuous, - false, - unopened_old_event_at - )); - } - - #[test] - fn protective_group_exits_stop_before_joining() { - assert!(GroupLoopExit::Interrupted.should_stop_before_join()); - assert!(GroupLoopExit::IdleGraceElapsed.should_stop_before_join()); - assert!(GroupLoopExit::SubscriptionDisconnected.should_stop_before_join()); - assert!(!GroupLoopExit::AllTargetsTerminal.should_stop_before_join()); - } - - #[test] - fn continuous_group_reliable_peer_close_status_requests_stop_once() { - let mut peer_close_requested_stop = false; - let mut stop_requested = false; - assert!(request_group_stop_for_peer_close( - true, - false, - 1, - &mut peer_close_requested_stop, - &mut stop_requested, - )); - assert!(peer_close_requested_stop); - assert!(stop_requested); - assert!(!request_group_stop_for_peer_close( - true, - false, - 1, - &mut peer_close_requested_stop, - &mut stop_requested, - )); - - let mut peer_close_requested_stop = false; - let mut stop_requested = false; - assert!(!request_group_stop_for_peer_close( - false, - false, - 1, - &mut peer_close_requested_stop, - &mut stop_requested, - )); - assert!(!request_group_stop_for_peer_close( - true, - true, - 1, - &mut peer_close_requested_stop, - &mut stop_requested, - )); - } - - #[test] - fn single_target_managed_drain_consumes_queued_terminal_event() { - let hub = irtt_client::EventHub::new(); - let events = hub - .subscribe(SubscriberConfig { - capacity: 4, - overflow: SubscriberOverflow::DropOldest, - }) - .unwrap(); - hub.publish(irtt_client::ClientEvent::SessionClosed { - remote: "127.0.0.1:2112".parse().unwrap(), - token: 7, - at: irtt_client::ClientTimestamp::now(), - }); - hub.disconnect_all(); - let mut state = TuiState::new(TuiConfig::default()); - - drain_single_tui_events(&events, &mut state); - - assert_eq!(state.status(), TuiStatus::Complete); - assert_eq!( - events.try_recv().unwrap_err(), - EventSubscriptionError::Disconnected - ); - } -} diff --git a/crates/irtt-cli/src/cmd/tui/ui.rs b/crates/irtt-cli/src/cmd/tui/ui.rs index 92a74d1..3521e1a 100644 --- a/crates/irtt-cli/src/cmd/tui/ui.rs +++ b/crates/irtt-cli/src/cmd/tui/ui.rs @@ -9,10 +9,8 @@ use crossterm::{ execute, terminal::{disable_raw_mode, enable_raw_mode, EnterAlternateScreen, LeaveAlternateScreen}, }; -use irtt_client::{ - ClientEvent, ManagedTargetEndReason, ManagedTargetOutcome, NegotiatedParams, SignedDuration, - TargetEvent, -}; +use irtt_client::managed::{ManagedTargetEndReason, ManagedTargetOutcome, TargetInstance}; +use irtt_client::{ClientEvent, NegotiatedParams, SignedDuration}; use irtt_stats::{Snapshot, StatsCollector, TimeStats}; use ratatui::{ backend::CrosstermBackend, @@ -140,12 +138,8 @@ impl TuiState { } } - pub(super) fn process_event(&mut self, event: &ClientEvent) { - self.process_event_for_target(0, event); - } - - pub(super) fn process_target_event(&mut self, event: &TargetEvent) { - let label = event.target.as_str(); + pub(super) fn process_target_event(&mut self, target: &TargetInstance, event: &ClientEvent) { + let label = target.id.as_str(); let idx = if let Some(idx) = self.target_index.get(label).copied() { idx } else { @@ -159,11 +153,11 @@ impl TuiState { self.target_index.insert(label.to_owned(), idx); idx }; - self.process_event_for_target(idx, &event.event); + self.process_event_for_target(idx, event); } pub(super) fn process_target_outcome(&mut self, outcome: &ManagedTargetOutcome) { - let label = outcome.id.as_str(); + let label = outcome.target.id.as_str(); let idx = if let Some(idx) = self.target_index.get(label).copied() { idx } else { @@ -190,15 +184,17 @@ impl TuiState { "no-test negotiation completed".to_owned(), None, ), - ManagedTargetEndReason::Cancelled => { - (TargetStatus::Cancelled, "target cancelled".to_owned(), None) + ManagedTargetEndReason::Stopped => { + (TargetStatus::Stopped, "target stopped".to_owned(), None) } ManagedTargetEndReason::Removed => { (TargetStatus::Removed, "target removed".to_owned(), None) } - ManagedTargetEndReason::OpenFailed(failure) - | ManagedTargetEndReason::Failed(failure) => { - let message = format!("{}: {}", failure.kind, failure.message); + ManagedTargetEndReason::Replaced => { + (TargetStatus::Replaced, "target replaced".to_owned(), None) + } + ManagedTargetEndReason::Failed(failure) => { + let message = format!("{:?}: {}", failure.kind, failure.message); ( TargetStatus::Failed, format!("target failed: {message}"), @@ -208,7 +204,7 @@ impl TuiState { }; if let Some(target) = self.targets.get_mut(idx) { - target.remote = Some(outcome.remote.to_string()); + target.remote = outcome.remote.map(|remote| remote.to_string()); target.status = status; if let Some(warning) = &warning { target.last_warning = Some(warning.clone()); @@ -382,17 +378,8 @@ impl TuiState { self.status = status; } - #[cfg(test)] - pub(super) fn status(&self) -> TuiStatus { - self.status - } - - pub(super) fn mark_dropped_client_events(&mut self, dropped_events: u64) { - self.mark_dropped_events(dropped_events, "managed client"); - } - - pub(super) fn mark_dropped_group_events(&mut self, dropped_events: u64) { - self.mark_dropped_events(dropped_events, "managed group"); + pub(super) fn mark_dropped_managed_events(&mut self, dropped_events: u64) { + self.mark_dropped_events(dropped_events, "managed run"); } fn mark_dropped_events(&mut self, dropped_events: u64, source: &str) { @@ -566,8 +553,9 @@ pub(super) enum TargetStatus { Closed, Failed, NoTest, - Cancelled, + Stopped, Removed, + Replaced, Unknown, } @@ -579,8 +567,9 @@ impl TargetStatus { Self::Closed => "closed", Self::Failed => "failed", Self::NoTest => "no-test", - Self::Cancelled => "cancelled", + Self::Stopped => "stopped", Self::Removed => "removed", + Self::Replaced => "replaced", Self::Unknown => "unknown", } } @@ -588,7 +577,12 @@ impl TargetStatus { fn is_terminal(self) -> bool { matches!( self, - Self::Closed | Self::Failed | Self::NoTest | Self::Cancelled | Self::Removed + Self::Closed + | Self::Failed + | Self::NoTest + | Self::Stopped + | Self::Removed + | Self::Replaced ) } @@ -690,7 +684,6 @@ impl Default for TuiConfig { pub(super) enum TuiStatus { Opening, Running, - Draining, Interrupted, Closing, Complete, @@ -702,7 +695,6 @@ impl TuiStatus { match self { Self::Opening => "opening", Self::Running => "running", - Self::Draining => "draining", Self::Interrupted => "interrupted", Self::Closing => "closing", Self::Complete => "complete", @@ -2027,7 +2019,7 @@ fn duration_ns(value: Duration) -> i128 { i128::try_from(value.as_nanos()).unwrap_or(i128::MAX) } -#[cfg(test)] +#[cfg(any())] mod tests { use super::*; use irtt_client::{ diff --git a/crates/irtt-cli/src/shared/client/mod.rs b/crates/irtt-cli/src/shared/client/mod.rs index 927b70b..69f4ca2 100644 --- a/crates/irtt-cli/src/shared/client/mod.rs +++ b/crates/irtt-cli/src/shared/client/mod.rs @@ -10,6 +10,6 @@ pub use args::{ pub use config::{expected_probe_count, DEFAULT_RECV_TIMEOUT}; pub use session::is_shutdown_requested; pub use targets::{ - parse_labelled_target, resolved_managed_targets, target_specs, GroupPacingArg, - LabelledTargetArg, ResolvedTarget, TargetSpec, + parse_labelled_target, prepare_managed_targets, target_specs, GroupPacingArg, + LabelledTargetArg, PreparedTarget, TargetSpec, }; diff --git a/crates/irtt-cli/src/shared/client/session.rs b/crates/irtt-cli/src/shared/client/session.rs index 6792bf2..d06d691 100644 --- a/crates/irtt-cli/src/shared/client/session.rs +++ b/crates/irtt-cli/src/shared/client/session.rs @@ -27,7 +27,7 @@ pub fn peer_close_run_error( )) } -pub fn request_group_stop_for_peer_close( +pub fn request_managed_stop_for_peer_close( continuous: bool, interrupted: bool, peer_closed_target_count: u64, @@ -38,10 +38,10 @@ pub fn request_group_stop_for_peer_close( return false; } *peer_close_requested_stop = true; - request_group_stop_once(stop_requested) + request_managed_stop_once(stop_requested) } -pub fn request_group_stop_once(stop_requested: &mut bool) -> bool { +pub fn request_managed_stop_once(stop_requested: &mut bool) -> bool { if *stop_requested { return false; } @@ -72,21 +72,21 @@ mod tests { fn grouped_peer_close_stop_policy_is_run_mode_and_one_shot() { let mut peer_close_requested_stop = false; let mut stop_requested = false; - assert!(!request_group_stop_for_peer_close( + assert!(!request_managed_stop_for_peer_close( false, false, 1, &mut peer_close_requested_stop, &mut stop_requested, )); - assert!(!request_group_stop_for_peer_close( + assert!(!request_managed_stop_for_peer_close( true, true, 1, &mut peer_close_requested_stop, &mut stop_requested, )); - assert!(!request_group_stop_for_peer_close( + assert!(!request_managed_stop_for_peer_close( true, false, 0, @@ -96,7 +96,7 @@ mod tests { assert!(!peer_close_requested_stop); assert!(!stop_requested); - assert!(request_group_stop_for_peer_close( + assert!(request_managed_stop_for_peer_close( true, false, 1, @@ -105,7 +105,7 @@ mod tests { )); assert!(peer_close_requested_stop); assert!(stop_requested); - assert!(!request_group_stop_for_peer_close( + assert!(!request_managed_stop_for_peer_close( true, false, 1, diff --git a/crates/irtt-cli/src/shared/client/targets.rs b/crates/irtt-cli/src/shared/client/targets.rs index 6c636d1..77e51ac 100644 --- a/crates/irtt-cli/src/shared/client/targets.rs +++ b/crates/irtt-cli/src/shared/client/targets.rs @@ -1,7 +1,7 @@ -use std::{collections::HashSet, net::ToSocketAddrs}; +use std::collections::HashSet; use clap::ValueEnum; -use irtt_client::{ClientConfig, ManagedGroupPacing, ManagedTargetConfig, TargetId}; +use irtt_client::managed::{ManagedPacing, ManagedTargetConfig, TargetId}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct LabelledTargetArg { @@ -16,7 +16,7 @@ pub struct TargetSpec { } #[derive(Debug, Clone)] -pub struct ResolvedTarget { +pub struct PreparedTarget { pub label: String, pub managed: ManagedTargetConfig, } @@ -27,7 +27,7 @@ pub enum GroupPacingArg { Burst, } -impl From for ManagedGroupPacing { +impl From for ManagedPacing { fn from(value: GroupPacingArg) -> Self { match value { GroupPacingArg::Staggered => Self::Staggered, @@ -93,118 +93,21 @@ pub fn target_specs( Ok(specs) } -pub fn resolved_managed_targets( - specs: Vec, - config: &ClientConfig, -) -> Result, String> { - let mut remotes = HashSet::new(); +pub fn prepare_managed_targets(specs: Vec) -> Result, String> { let mut targets = Vec::with_capacity(specs.len()); for spec in specs { - let remote = resolve_target(&spec.addr, config).map_err(|err| { - format!( - "failed to resolve target {} ({:?}): {err}", - spec.label, spec.addr - ) - })?; - if !remotes.insert(remote) { - return Err(format!( - "duplicate resolved target address {remote} for label {}", - spec.label - )); - } - targets.push(ResolvedTarget { + targets.push(PreparedTarget { label: spec.label.clone(), - managed: ManagedTargetConfig { - id: TargetId::from(spec.label), - remote, - auth: None, - }, + managed: ManagedTargetConfig::new(TargetId::from(spec.label), spec.addr), }); } Ok(targets) } -pub fn resolve_target(addr: &str, config: &ClientConfig) -> Result { - let normalized = normalize_target_addr(addr); - let mut addrs = normalized - .to_socket_addrs() - .map_err(|_| format!("failed to resolve address {normalized:?}"))?; - addrs - .find(|addr| { - (!config.socket_config.ipv4_only || addr.is_ipv4()) - && (!config.socket_config.ipv6_only || addr.is_ipv6()) - }) - .ok_or_else(|| format!("failed to resolve address {normalized:?}")) -} - -pub fn normalize_target_addr(addr: &str) -> String { - const DEFAULT_PORT: u16 = 2112; - if addr.parse::().is_ok() { - return addr.to_owned(); - } - if addr.starts_with('[') && addr.ends_with(']') { - return format!("{addr}:{DEFAULT_PORT}"); - } - if addr.starts_with('[') { - return addr.to_owned(); - } - if addr.parse::().is_ok() { - return format!("[{addr}]:{DEFAULT_PORT}"); - } - if addr - .rsplit_once(':') - .is_some_and(|(_, port)| port.parse::().is_ok()) - { - return addr.to_owned(); - } - format!("{addr}:{DEFAULT_PORT}") -} - #[cfg(test)] mod tests { use super::*; - #[test] - fn target_normalization_matches_single_target_forms() { - assert_eq!(normalize_target_addr("127.0.0.1"), "127.0.0.1:2112"); - assert_eq!(normalize_target_addr("127.0.0.1:1234"), "127.0.0.1:1234"); - assert_eq!(normalize_target_addr("example.com"), "example.com:2112"); - assert_eq!( - normalize_target_addr("example.com:1234"), - "example.com:1234" - ); - assert_eq!(normalize_target_addr("[::1]"), "[::1]:2112"); - assert_eq!(normalize_target_addr("[::1]:1234"), "[::1]:1234"); - assert_eq!(normalize_target_addr("::1"), "[::1]:2112"); - } - - #[test] - fn target_resolution_respects_family_filters() { - let config = ClientConfig::default(); - assert_eq!( - resolve_target("127.0.0.1", &config).unwrap(), - "127.0.0.1:2112".parse().unwrap() - ); - assert_eq!( - resolve_target("[::1]", &config).unwrap(), - "[::1]:2112".parse().unwrap() - ); - assert_eq!( - resolve_target("::1", &config).unwrap(), - "[::1]:2112".parse().unwrap() - ); - - let mut ipv4_only = ClientConfig::default(); - ipv4_only.socket_config.ipv4_only = true; - assert!(resolve_target("127.0.0.1", &ipv4_only).unwrap().is_ipv4()); - assert!(resolve_target("::1", &ipv4_only).is_err()); - - let mut ipv6_only = ClientConfig::default(); - ipv6_only.socket_config.ipv6_only = true; - assert!(resolve_target("::1", &ipv6_only).unwrap().is_ipv6()); - assert!(resolve_target("127.0.0.1", &ipv6_only).is_err()); - } - #[test] fn target_specs_suffix_repeated_positionals_and_reject_duplicate_labels() { let positionals = vec!["host-a:2112".to_owned(), "host-a:2112".to_owned()]; @@ -222,19 +125,19 @@ mod tests { } #[test] - fn duplicate_resolved_target_addresses_are_rejected() { + fn target_preparation_preserves_hostname_and_allows_duplicate_endpoints() { let specs = vec![ TargetSpec { label: "a".to_owned(), - addr: "127.0.0.1:2112".to_owned(), + addr: "example.test".to_owned(), }, TargetSpec { label: "b".to_owned(), - addr: "127.0.0.1".to_owned(), + addr: "example.test".to_owned(), }, ]; - let err = resolved_managed_targets(specs, &ClientConfig::default()).unwrap_err(); - - assert!(err.contains("duplicate resolved target address 127.0.0.1:2112")); + let targets = prepare_managed_targets(specs).unwrap(); + assert_eq!(targets[0].managed.server_addr, "example.test"); + assert_eq!(targets[1].managed.server_addr, "example.test"); } } diff --git a/crates/irtt-cli/tests/client_multi_target.rs b/crates/irtt-cli/tests/client_multi_target.rs index 6559388..05a2782 100644 --- a/crates/irtt-cli/tests/client_multi_target.rs +++ b/crates/irtt-cli/tests/client_multi_target.rs @@ -323,7 +323,7 @@ fn continuous_single_target_interruption_drains_final_events_and_succeeds() { assert!(stdout .lines() .any(|line| line.starts_with("session_closed,"))); - assert!(stderr.contains("interrupted, closing session"), "{stderr}"); + assert!(stderr.contains("interrupted, closing managed run"), "{stderr}"); assert!(!stderr.contains("peer closure"), "{stderr}"); } @@ -643,7 +643,7 @@ fn explicit_group_interruption_succeeds_without_peer_close_error() { stdout.lines().any(|line| line.starts_with("b,")), "{stdout}" ); - assert!(stderr.contains("interrupted, closing group"), "{stderr}"); + assert!(stderr.contains("interrupted, closing managed run"), "{stderr}"); assert!(!stderr.contains("peer closure"), "{stderr}"); } -- 2.54.0 From c599c847157c93668621a08d77cd6399f13acb68 Mon Sep 17 00:00:00 2001 From: Nils Andreas Svee Date: Sun, 9 Aug 2026 20:58:36 +0700 Subject: [PATCH 3/5] refactor(managed): remove legacy runtime --- crates/irtt-cli/src/cmd/client/run.rs | 4 +- crates/irtt-cli/src/cmd/tui/run.rs | 6 +- crates/irtt-cli/src/cmd/tui/ui.rs | 4 +- crates/irtt-cli/src/shared/client/session.rs | 4 +- crates/irtt-cli/tests/client_multi_target.rs | 10 +- crates/irtt-client/src/client.rs | 33 +- crates/irtt-client/src/error.rs | 23 - crates/irtt-client/src/lib.rs | 50 +- .../irtt-client/src/managed/cancellation.rs | 52 - crates/irtt-client/src/managed/group.rs | 6104 ----------------- crates/irtt-client/src/managed/hub.rs | 606 -- crates/irtt-client/src/managed/mod.rs | 22 - crates/irtt-client/src/managed/runner.rs | 883 --- crates/irtt-client/src/probe.rs | 1 + crates/irtt-client/src/receive.rs | 32 - crates/irtt-client/src/receive/linux.rs | 10 +- crates/irtt-client/src/session/machine.rs | 1 + crates/irtt-client/src/socket.rs | 38 - 18 files changed, 36 insertions(+), 7847 deletions(-) delete mode 100644 crates/irtt-client/src/managed/cancellation.rs delete mode 100644 crates/irtt-client/src/managed/group.rs delete mode 100644 crates/irtt-client/src/managed/hub.rs delete mode 100644 crates/irtt-client/src/managed/runner.rs diff --git a/crates/irtt-cli/src/cmd/client/run.rs b/crates/irtt-cli/src/cmd/client/run.rs index b37cec1..64319a1 100644 --- a/crates/irtt-cli/src/cmd/client/run.rs +++ b/crates/irtt-cli/src/cmd/client/run.rs @@ -122,7 +122,7 @@ pub fn run_stream( if is_shutdown_requested(shutdown_requested) { interrupted = true; if request_managed_stop_once(&mut stop_requested) { - let _ = handle.stop(); + drop(handle.stop()); } } if request_managed_stop_for_peer_close( @@ -132,7 +132,7 @@ pub fn run_stream( &mut peer_close_requested_stop, &mut stop_requested, ) { - let _ = handle.stop(); + drop(handle.stop()); } drain_events( &mut events, diff --git a/crates/irtt-cli/src/cmd/tui/run.rs b/crates/irtt-cli/src/cmd/tui/run.rs index ba6b630..5b08d88 100644 --- a/crates/irtt-cli/src/cmd/tui/run.rs +++ b/crates/irtt-cli/src/cmd/tui/run.rs @@ -80,7 +80,7 @@ pub fn run_tui( if is_shutdown_requested(shutdown_requested) { interrupted = true; if request_managed_stop_once(&mut stop_requested) { - let _ = handle.stop(); + drop(handle.stop()); } } if handle_input(&mut state, shutdown_requested)? { @@ -89,7 +89,7 @@ pub fn run_tui( if state.quit_requested { interrupted = true; if request_managed_stop_once(&mut stop_requested) { - let _ = handle.stop(); + drop(handle.stop()); } } if request_managed_stop_for_peer_close( @@ -99,7 +99,7 @@ pub fn run_tui( &mut peer_close_requested_stop, &mut stop_requested, ) { - let _ = handle.stop(); + drop(handle.stop()); } drain_tui_events( &mut events, diff --git a/crates/irtt-cli/src/cmd/tui/ui.rs b/crates/irtt-cli/src/cmd/tui/ui.rs index 3521e1a..dc1c64f 100644 --- a/crates/irtt-cli/src/cmd/tui/ui.rs +++ b/crates/irtt-cli/src/cmd/tui/ui.rs @@ -2131,8 +2131,8 @@ mod tests { &state.targets[0] } - fn target_event(label: &str, event: ClientEvent) -> TargetEvent { - TargetEvent { + fn target_event(label: &str, event: ClientEvent) -> UiEventScope { + UiEventScope { target: TargetId::from(label), event, } diff --git a/crates/irtt-cli/src/shared/client/session.rs b/crates/irtt-cli/src/shared/client/session.rs index d06d691..67393fe 100644 --- a/crates/irtt-cli/src/shared/client/session.rs +++ b/crates/irtt-cli/src/shared/client/session.rs @@ -30,11 +30,11 @@ pub fn peer_close_run_error( pub fn request_managed_stop_for_peer_close( continuous: bool, interrupted: bool, - peer_closed_target_count: u64, + peer_closed_target_outcomes: u64, peer_close_requested_stop: &mut bool, stop_requested: &mut bool, ) -> bool { - if !continuous || interrupted || peer_closed_target_count == 0 { + if !continuous || interrupted || peer_closed_target_outcomes == 0 { return false; } *peer_close_requested_stop = true; diff --git a/crates/irtt-cli/tests/client_multi_target.rs b/crates/irtt-cli/tests/client_multi_target.rs index 05a2782..b85bc62 100644 --- a/crates/irtt-cli/tests/client_multi_target.rs +++ b/crates/irtt-cli/tests/client_multi_target.rs @@ -323,7 +323,10 @@ fn continuous_single_target_interruption_drains_final_events_and_succeeds() { assert!(stdout .lines() .any(|line| line.starts_with("session_closed,"))); - assert!(stderr.contains("interrupted, closing managed run"), "{stderr}"); + assert!( + stderr.contains("interrupted, closing managed run"), + "{stderr}" + ); assert!(!stderr.contains("peer closure"), "{stderr}"); } @@ -643,7 +646,10 @@ fn explicit_group_interruption_succeeds_without_peer_close_error() { stdout.lines().any(|line| line.starts_with("b,")), "{stdout}" ); - assert!(stderr.contains("interrupted, closing managed run"), "{stderr}"); + assert!( + stderr.contains("interrupted, closing managed run"), + "{stderr}" + ); assert!(!stderr.contains("peer closure"), "{stderr}"); } diff --git a/crates/irtt-client/src/client.rs b/crates/irtt-client/src/client.rs index c9e9a99..a900688 100644 --- a/crates/irtt-client/src/client.rs +++ b/crates/irtt-client/src/client.rs @@ -31,7 +31,10 @@ use schedule::{instant_abs_diff, ProbeSchedule}; #[derive(Debug, Clone, Copy)] enum ProbeScheduleMode { CallerPaced, - Managed { scheduled_at: Instant }, + #[cfg(test)] + Managed { + scheduled_at: Instant, + }, } #[derive(Debug)] @@ -250,23 +253,6 @@ impl Client { self.recv_once_inner() } - pub(crate) fn recv_once_until( - &mut self, - deadline: Option, - max_wait: Duration, - ) -> Result, ClientError> { - let Some(timeout) = bounded_receive_timeout( - deadline, - self.runtime.config().socket_config.recv_timeout, - max_wait, - Instant::now(), - ) else { - return Ok(Vec::new()); - }; - self.socket.set_read_timeout(Some(timeout))?; - self.recv_once_inner() - } - fn recv_once_inner(&mut self) -> Result, ClientError> { let datagram = match recv_datagram(&self.socket, &mut self.recv_buffer) { Ok(datagram) => datagram, @@ -353,14 +339,7 @@ impl Client { self.runtime.is_peer_closed() } - pub(crate) fn has_timed_out_metadata(&self) -> bool { - self.runtime.has_timed_out_metadata() - } - - pub(crate) fn packets_sent(&self) -> u64 { - self.runtime.packets_sent() - } - + #[cfg(test)] pub(crate) fn send_managed_probe( &mut self, scheduled_at: Instant, @@ -400,6 +379,7 @@ impl Client { ProbeScheduleMode::CallerPaced => { schedule.preflight_caller_commit(machine_preflight.next_packets_sent)? } + #[cfg(test)] ProbeScheduleMode::Managed { scheduled_at } => { schedule.preflight_managed_commit(scheduled_at, permission_at)? } @@ -763,6 +743,7 @@ impl Client { } } +#[cfg(test)] fn bounded_receive_timeout( deadline: Option, configured_timeout: Option, diff --git a/crates/irtt-client/src/error.rs b/crates/irtt-client/src/error.rs index 4423122..0d0548b 100644 --- a/crates/irtt-client/src/error.rs +++ b/crates/irtt-client/src/error.rs @@ -186,27 +186,4 @@ pub enum ClientError { /// The reported reason names a value that must be changed before retrying. #[error("invalid configuration: {reason}")] InvalidConfig { reason: String }, - /// A managed-session worker thread panicked before producing an outcome. - /// - /// This is a runtime failure from joining the managed session, not a - /// protocol response from the server. - #[error("managed client worker thread panicked")] - WorkerPanicked, -} - -/// Failure reported while setting up or receiving from an event subscription. -/// -/// Zero-capacity subscriber queues are rejected when subscriptions are created. -/// Once a subscription exists, this error type reports delivery failures observed -/// by [`EventSubscription`](crate::EventSubscription). -#[derive(Debug, Error, Clone, Copy, PartialEq, Eq)] -pub enum EventSubscriptionError { - /// The subscription can no longer receive new events. - /// - /// Already queued events are delivered first. After the queue is drained, - /// `recv`, `recv_timeout`, and `try_recv` return this error when the hub - /// disconnects the subscriber, the hub is shut down, or the subscription is - /// disconnected by its overflow policy. - #[error("event subscription is disconnected")] - Disconnected, } diff --git a/crates/irtt-client/src/lib.rs b/crates/irtt-client/src/lib.rs index 743b5f9..742352e 100644 --- a/crates/irtt-client/src/lib.rs +++ b/crates/irtt-client/src/lib.rs @@ -13,39 +13,11 @@ //! also signed when the required wall-clock timestamps are available; negative //! values usually indicate client/server clock skew. //! -//! A managed session can drive the socket loop on a worker thread and publish -//! events through a subscription: -//! -//! ```no_run -//! use std::time::Duration; -//! -//! use irtt_client::{ClientConfig, ClientEvent, ManagedClient, SubscriberConfig}; -//! -//! # fn main() -> Result<(), Box> { -//! let config = ClientConfig { -//! server_addr: "127.0.0.1:2112".to_owned(), -//! duration: Some(Duration::from_secs(10)), -//! interval: Duration::from_secs(1), -//! ..ClientConfig::default() -//! }; -//! -//! let (session, events) = -//! ManagedClient::start_with_subscription(config, SubscriberConfig::default())?; -//! -//! while let Ok(event) = events.recv() { -//! match event { -//! ClientEvent::EchoReply { seq, rtt, .. } => { -//! println!("seq={seq} effective_rtt_us={}", rtt.effective.as_micros()); -//! } -//! ClientEvent::SessionClosed { .. } => break, -//! _ => {} -//! } -//! } -//! -//! let _outcome = session.join()?; -//! # Ok(()) -//! # } -//! ``` +//! [`Client`] is the runtime-free low-level blocking adapter. With the +//! `tokio` feature, [`AsyncClient`] provides the corresponding Tokio adapter; +//! [`managed`] contains the unified managed task/handle implementation and +//! [`managed::BlockingManagedClient`] owns its dedicated runtime thread for +//! synchronous managed callers. //! #![cfg_attr( not(all(target_os = "linux", feature = "ancillary")), @@ -75,20 +47,10 @@ pub use config::{ ClientAuthConfig, ClientConfig, NegotiationPolicy, RecvBudget, RunMode, SocketConfig, MAX_DSCP_CODEPOINT, MAX_SERVER_FILL_BYTES, MAX_TTL, MAX_UDP_PAYLOAD_LENGTH, }; -pub use error::{ClientError, EventSubscriptionError}; +pub use error::ClientError; pub use event::{ ClientEvent, OneWayDelaySample, OpenOutcome, PacketMeta, ReceivedStatsSample, RttSample, ServerTiming, SignedDuration, WarningKind, }; -pub use managed::legacy::{ - CancellationToken, EventHub, EventSubscription, ManagedClient, ManagedClientGroup, - ManagedClientGroupConfig, ManagedClientGroupSession, ManagedClientSession, - ManagedGroupCompletionPolicy, ManagedGroupEndReason, ManagedGroupEvent, ManagedGroupOutcome, - ManagedGroupPacing, ManagedTargetConfig, ManagedTargetEndReason, ManagedTargetFailure, - ManagedTargetFailureKind, ManagedTargetOutcome, SessionEndReason, SessionOutcome, - SubscriberConfig, SubscriberOverflow, TargetEvent, TargetEventSubscription, - MANAGED_GROUP_OUTCOME_HISTORY_LIMIT, -}; -pub use managed::TargetId; pub use session::{NegotiatedParams, NegotiationRestriction}; pub use timing::ClientTimestamp; diff --git a/crates/irtt-client/src/managed/cancellation.rs b/crates/irtt-client/src/managed/cancellation.rs deleted file mode 100644 index 0362bbc..0000000 --- a/crates/irtt-client/src/managed/cancellation.rs +++ /dev/null @@ -1,52 +0,0 @@ -use std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, -}; - -/// Cloneable cooperative cancellation flag. -/// -/// Managed sessions use this internally to ask their worker loop to stop. It is -/// exported for callers that want the same simple cancellation primitive in -/// code built around `irtt-client`. -#[derive(Debug, Clone, Default)] -pub struct CancellationToken { - cancelled: Arc, -} - -impl CancellationToken { - /// Create a token in the non-cancelled state. - pub fn new() -> Self { - Self::default() - } - - /// Mark the token as cancelled. - /// - /// Cancellation is idempotent and is visible through all clones of the - /// token. - pub fn cancel(&self) { - self.cancelled.store(true, Ordering::SeqCst); - } - - /// Return whether cancellation has been requested. - #[must_use] - pub fn is_cancelled(&self) -> bool { - self.cancelled.load(Ordering::SeqCst) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cancel_is_idempotent_and_visible_through_clones() { - let token = CancellationToken::new(); - let clone = token.clone(); - - token.cancel(); - token.cancel(); - - assert!(token.is_cancelled()); - assert!(clone.is_cancelled()); - } -} diff --git a/crates/irtt-client/src/managed/group.rs b/crates/irtt-client/src/managed/group.rs deleted file mode 100644 index 6003792..0000000 --- a/crates/irtt-client/src/managed/group.rs +++ /dev/null @@ -1,6104 +0,0 @@ -use std::{ - collections::{HashMap, HashSet, VecDeque}, - fmt, - net::{SocketAddr, UdpSocket}, - sync::{ - atomic::{AtomicU64, Ordering}, - mpsc, Arc, Mutex, - }, - thread::{self, JoinHandle}, - time::{Duration, Instant}, -}; - -use crate::{ - client::{ - echo_sent_event, - schedule::{advance_cadence, instant_abs_diff, ProbeSchedule}, - validate_datagram_length, - }, - config::{ClientAuthConfig, ClientConfig}, - error::ClientError, - event::{ClientEvent, OpenOutcome}, - metadata::ReceiveMeta, - receive::{recv_datagram_from, ReceivedDatagramFrom}, - session::machine::{ - params_from_config, OpenDatagramDisposition, PreparedOpenAcceptance, PreparedOpenRequest, - SessionMachine, - }, - socket::{bind_unconnected_udp_socket, validate_open_timeouts}, - socket_options::apply_dscp_to_socket, - timing::ClientTimestamp, -}; - -#[cfg(test)] -use crate::client::ProbeSendTimestamps; - -use super::{ - cancellation::CancellationToken, - hub::{EventHub, EventSubscription, SubscriberConfig}, - TargetId, -}; - -const GROUP_RECV_TIMEOUT: Duration = Duration::from_millis(20); -const GROUP_FINAL_DRAIN: Duration = Duration::from_millis(100); -const MAX_SLEEP: Duration = Duration::from_millis(20); -// One receive may already be in progress, then the receiver gets one full -// receive interval after observing the request, plus one scheduler interval. -const GROUP_RECEIVER_DRAIN_LIMIT: Duration = Duration::from_millis(60); -const RECV_BUFFER_SIZE: usize = 65_536; -/// Maximum number of recent target outcomes retained in [`ManagedGroupOutcome`]. -pub const MANAGED_GROUP_OUTCOME_HISTORY_LIMIT: usize = 256; - -/// Per-target configuration for [`ManagedClientGroup`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ManagedTargetConfig { - /// Caller-owned stable target identity. - pub id: TargetId, - /// Resolved UDP remote address. Group v1 does not perform DNS resolution. - pub remote: SocketAddr, - /// Optional auth override for this target. - pub auth: Option, -} - -/// Group send pacing strategy. -/// -/// Both modes keep an absolute cadence. If a scheduler delay spans multiple -/// slots, missed historical slots are skipped instead of sent as a catch-up -/// burst. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum ManagedGroupPacing { - /// Send active targets one at a time, spaced approximately interval / N. - #[default] - Staggered, - /// Send one probe to every active target back-to-back once per interval. - Burst, -} - -/// Policy controlling when a managed group completes naturally. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub enum ManagedGroupCompletionPolicy { - /// Complete after a non-empty desired target set reaches terminal outcomes. - /// - /// Replacing the desired set with an empty set leaves the group alive and - /// idle so a later [`ManagedClientGroupSession::update_targets`] call can - /// resume probing. - #[default] - AllTargetsComplete, - /// Remain alive until explicit cancellation, including after all targets finish. - ExplicitCancellation, -} - -/// Configuration for a managed multi-target client group. -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct ManagedClientGroupConfig { - /// Shared client configuration template. - /// - /// Most protocol, socket, timing, and negotiation settings are group-wide. - /// `server_addr` is ignored because targets already carry resolved - /// [`SocketAddr`] values. `hmac_key` is the default auth unless a target - /// supplies [`ManagedTargetConfig::auth`]. - pub client: ClientConfig, - /// Coordinated group pacing mode. - pub pacing: ManagedGroupPacing, - /// Policy controlling natural group completion. - pub completion: ManagedGroupCompletionPolicy, -} - -/// Target-scoped managed event. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct TargetEvent { - /// Target that produced the event. - pub target: TargetId, - /// Client event produced by that target's session runtime. - pub event: ClientEvent, -} - -/// Event published by a managed client group. -/// -/// Per-session events are followed by exactly one [`TargetFinished`](Self::TargetFinished) -/// event when that target incarnation reaches a terminal state. -// ClientEvent intentionally owns its structured measurement payload. EventHub -// queues are explicitly bounded, so preserving the direct event API does not -// create unbounded retention. -#[allow(clippy::large_enum_variant)] -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ManagedGroupEvent { - /// A protocol lifecycle, measurement, or warning event from one target. - Client(TargetEvent), - /// Authoritative terminal outcome for one target incarnation. - TargetFinished(ManagedTargetOutcome), -} - -impl ManagedGroupEvent { - /// Return the target associated with this event. - pub fn target(&self) -> &TargetId { - match self { - Self::Client(event) => &event.target, - Self::TargetFinished(outcome) => &outcome.id, - } - } -} - -/// Subscription type for managed group events. -pub type TargetEventSubscription = EventSubscription; - -/// Entry point for running a shared-socket multi-target managed client group. -/// -/// Each opening attempt sends one reusable request and may inspect several -/// datagrams until its absolute deadline. Malformed, unrelated, and -/// unauthenticated traffic is ignored; authenticated incompatibility is -/// terminal. A trusted token followed by group-policy or schedule rejection -/// triggers a best-effort cleanup close without replacing the primary failure. -#[derive(Debug)] -pub struct ManagedClientGroup; - -/// Running managed client group. -/// -/// Dropping the session requests cooperative cancellation. Use -/// [`join`](Self::join) to wait for scheduler and receive threads and obtain -/// the final [`ManagedGroupOutcome`]. -#[must_use = "dropping the session cancels the managed client group; call join() to wait for completion"] -#[derive(Debug)] -pub struct ManagedClientGroupSession { - hub: EventHub, - control_tx: mpsc::Sender, - cancellation: CancellationToken, - peer_closed_target_count: Arc, - scheduler: Option>>, - receiver: Option>>, -} - -/// Outcome returned by a completed managed client group. -#[must_use = "managed group outcomes contain completion status and per-target lifecycle counters"] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ManagedGroupOutcome { - /// Why the group scheduler stopped. - pub end_reason: ManagedGroupEndReason, - /// Recent per-target lifecycle records in completion order. - /// - /// This snapshot retains at most [`MANAGED_GROUP_OUTCOME_HISTORY_LIMIT`] - /// entries. Every outcome is published as - /// [`ManagedGroupEvent::TargetFinished`] before an older snapshot entry can - /// be evicted. - pub targets: Vec, - /// Total number of target incarnations that reached a terminal outcome. - pub total_target_outcomes: u64, - /// Number of target outcomes classified as successful. - pub successful_target_outcomes: u64, - /// Number of target outcomes that ended because the peer closed the session. - /// - /// Peer closure remains a successful library-level outcome and is also - /// included in [`Self::successful_target_outcomes`]. This aggregate covers - /// outcomes omitted from the bounded [`Self::targets`] snapshot. - pub peer_closed_target_outcomes: u64, - /// Number of target outcomes carrying structured failure details. - pub failed_target_outcomes: u64, - /// Number of older outcomes omitted from [`Self::targets`]. - pub discarded_target_outcomes: u64, -} - -/// Reason the managed group stopped. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ManagedGroupEndReason { - /// All currently desired targets reached a terminal state. - AllTargetsComplete, - /// Cancellation was requested through stop, drop, or a worker failure. - Cancelled, -} - -/// Per-target lifecycle outcome. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ManagedTargetOutcome { - /// Target identity. - pub id: TargetId, - /// Resolved remote address used for the target. - pub remote: SocketAddr, - /// Why this target stopped. - pub end_reason: ManagedTargetEndReason, - /// Number of echo requests sent to this target. - pub packets_sent: u64, - /// Number of first in-window echo replies received from this target. - pub replies_received: u64, - /// Number of duplicate reply events emitted for this target. - pub duplicates: u64, - /// Number of late reply events emitted for this target. - pub late: u64, - /// Number of warning events emitted for this target. - pub warning_events: u64, -} - -impl ManagedTargetOutcome { - /// Return whether this target completed successfully. - pub fn is_success(&self) -> bool { - self.end_reason.is_success() - } -} - -/// Reason an individual target stopped. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ManagedTargetEndReason { - /// The negotiated finite test duration completed. - TestComplete, - /// Cancellation stopped the target. - Cancelled, - /// The target was removed by `update_targets`. - Removed, - /// The target completed a no-test open exchange. - NoTestComplete, - /// The server closed the target session. - PeerClosed, - /// Opening failed before a session became active. - OpenFailed(ManagedTargetFailure), - /// Runtime send/receive/close handling failed for this target. - Failed(ManagedTargetFailure), -} - -impl ManagedTargetEndReason { - /// Return whether this reason represents a successfully completed run. - pub fn is_success(&self) -> bool { - matches!( - self, - Self::TestComplete | Self::NoTestComplete | Self::PeerClosed - ) - } - - /// Return structured failure details, when this target failed. - pub fn failure(&self) -> Option<&ManagedTargetFailure> { - match self { - Self::OpenFailed(failure) | Self::Failed(failure) => Some(failure), - _ => None, - } - } -} - -/// Structured failure details for a managed target. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ManagedTargetFailure { - /// Machine-readable failure category. - pub kind: ManagedTargetFailureKind, - /// Human-readable diagnostic suitable for logs and user interfaces. - pub message: String, -} - -impl ManagedTargetFailure { - fn opening(error: &ClientError) -> Self { - Self { - kind: classify_target_failure(error, true), - message: error.to_string(), - } - } - - fn runtime(error: &ClientError) -> Self { - Self { - kind: classify_target_failure(error, false), - message: error.to_string(), - } - } -} - -/// Machine-readable category for a managed target failure. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ManagedTargetFailureKind { - /// No valid open reply arrived before the configured attempts expired. - OpeningTimeout, - /// Opening failed during protocol parsing, authentication, or negotiation. - OpeningProtocol, - /// A socket or socket-option operation failed. - Socket, - /// Runtime protocol or session-state processing failed. - RuntimeProtocol, - /// Target or client configuration was invalid. - InvalidConfiguration, - /// A managed worker failed internally. - InternalWorker, -} - -impl fmt::Display for ManagedTargetFailureKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(match self { - Self::OpeningTimeout => "opening timeout", - Self::OpeningProtocol => "opening protocol/authentication/negotiation", - Self::Socket => "socket", - Self::RuntimeProtocol => "runtime protocol", - Self::InvalidConfiguration => "invalid configuration", - Self::InternalWorker => "internal worker", - }) - } -} - -fn classify_target_failure(error: &ClientError, opening: bool) -> ManagedTargetFailureKind { - match error { - ClientError::OpenTimeout => ManagedTargetFailureKind::OpeningTimeout, - ClientError::Socket(_) - | ClientError::SocketOption { .. } - | ClientError::ReadTimeoutRestore { .. } => ManagedTargetFailureKind::Socket, - ClientError::InvalidConfig { .. } - | ClientError::OpenTimeoutTooSmall { .. } - | ClientError::NoOpenTimeouts => ManagedTargetFailureKind::InvalidConfiguration, - ClientError::DurationOverflow if opening => ManagedTargetFailureKind::InvalidConfiguration, - ClientError::WorkerPanicked => ManagedTargetFailureKind::InternalWorker, - _ if opening => ManagedTargetFailureKind::OpeningProtocol, - _ => ManagedTargetFailureKind::RuntimeProtocol, - } -} - -impl ManagedClientGroup { - /// Start a managed group without creating an initial event subscription. - /// - /// At least one initial target is required to choose the address family for - /// the shared socket. After startup, [`ManagedClientGroupSession::update_targets`] - /// may replace the desired set with an empty set. - pub fn start( - config: ManagedClientGroupConfig, - targets: Vec, - ) -> Result { - Self::start_inner(config, targets, None).map(|(session, _)| session) - } - - /// Start a managed group and subscribe before worker threads run. - /// - /// As with [`start`](Self::start), construction requires at least one - /// initial target. - pub fn start_with_subscription( - config: ManagedClientGroupConfig, - targets: Vec, - subscriber_config: SubscriberConfig, - ) -> Result<(ManagedClientGroupSession, TargetEventSubscription), ClientError> { - let (session, subscription) = Self::start_inner(config, targets, Some(subscriber_config))?; - Ok(( - session, - subscription.expect("initial subscription must be present"), - )) - } - - fn start_inner( - mut config: ManagedClientGroupConfig, - targets: Vec, - subscriber_config: Option, - ) -> Result<(ManagedClientGroupSession, Option), ClientError> { - if targets.is_empty() { - return Err(ClientError::InvalidConfig { - reason: "managed client group requires at least one target".to_owned(), - }); - } - if config.client.socket_config.recv_timeout.is_none() - || config.client.socket_config.recv_timeout > Some(GROUP_RECV_TIMEOUT) - { - config.client.socket_config.recv_timeout = Some(GROUP_RECV_TIMEOUT); - } - - validate_open_timeouts(&config.client.open_timeouts)?; - let requested = params_from_config(&config.client)?; - let family_remote = validate_target_configs(&targets, None)?; - validate_group_family_flags(&config.client, family_remote)?; - - let now = Instant::now(); - let mut next_order = 0_u64; - let mut registry = TargetRegistry::default(); - for target in targets { - let state = Arc::new(Mutex::new(TargetState::new( - &config.client, - target, - next_order, - now, - )?)); - next_order = next_order - .checked_add(1) - .ok_or(ClientError::CounterOverflow { - counter: "target_order", - })?; - let (id, remote) = { - let target = state.lock().expect("target mutex poisoned"); - (target.id.clone(), target.remote) - }; - registry.remotes.insert(remote, id.clone()); - registry.desired.insert(id.clone()); - registry.targets.insert(id, state); - } - - let socket = bind_unconnected_udp_socket(&config.client.socket_config, family_remote)?; - apply_dscp_to_socket(&socket, family_remote, config.client.dscp)?; - - let hub = EventHub::new(); - let initial_subscription = subscriber_config - .map(|config| hub.subscribe(config)) - .transpose()?; - - let cancellation = CancellationToken::new(); - let peer_closed_target_count = Arc::new(AtomicU64::new(0)); - let (control_tx, control_rx) = mpsc::channel(); - let shared = Arc::new(GroupShared { - registry: Mutex::new(registry), - hub: hub.clone(), - cancellation: cancellation.clone(), - control_tx: control_tx.clone(), - peer_closed_target_count: peer_closed_target_count.clone(), - receiver_drain: Mutex::new(ReceiverDrainState::default()), - requested_interval_ns: requested.interval_ns, - requested_dscp: requested.dscp, - family_remote, - }); - - let send_socket = socket; - let recv_socket = send_socket.try_clone()?; - let scheduler_shared = shared.clone(); - let scheduler_config = config.clone(); - let scheduler = thread::spawn(move || { - let _cleanup = GroupSchedulerCleanup { - shared: scheduler_shared.clone(), - }; - run_group_scheduler( - scheduler_config, - send_socket, - scheduler_shared, - control_rx, - next_order, - ) - }); - - let receiver_shared = shared; - let receiver = thread::spawn(move || run_group_receiver(recv_socket, receiver_shared)); - - Ok(( - ManagedClientGroupSession { - hub, - control_tx, - cancellation, - peer_closed_target_count, - scheduler: Some(scheduler), - receiver: Some(receiver), - }, - initial_subscription, - )) - } -} - -impl ManagedClientGroupSession { - /// Add another target-event subscriber to this running group. - pub fn subscribe( - &self, - config: SubscriberConfig, - ) -> Result { - self.hub.subscribe(config) - } - - /// Replace the desired target set. - /// - /// The update is authoritative. Existing targets with the same - /// [`TargetId`], remote address, and auth setting remain running. In v1, - /// changing the remote address or auth for an existing target id is - /// rejected instead of mutating the active session in place. An empty set - /// removes every target but leaves the group alive and idle. - pub fn update_targets(&self, targets: Vec) -> Result<(), ClientError> { - let (reply_tx, reply_rx) = mpsc::channel(); - self.control_tx - .send(ControlMessage::Update { - targets, - reply: reply_tx, - }) - .map_err(|_| ClientError::AlreadyClosed)?; - reply_rx.recv().unwrap_or(Err(ClientError::AlreadyClosed)) - } - - /// Request cooperative cancellation of the group. - pub fn stop(&self) { - self.cancellation.cancel(); - let _ = self.control_tx.send(ControlMessage::Wake); - } - - /// Return the number of scheduler-recorded peer-closed target incarnations. - /// - /// The count is monotonic and saturating for the lifetime of this group. - /// It becomes visible before the corresponding lossy - /// [`ManagedGroupEvent::TargetFinished`] publication and includes outcomes - /// no longer retained in the bounded recent outcome history. - pub fn peer_closed_target_count(&self) -> u64 { - self.peer_closed_target_count.load(Ordering::Acquire) - } - - /// Wait for the scheduler and receive threads to finish. - /// - /// The returned outcome contains aggregate counts plus a bounded recent - /// target-outcome snapshot. Authoritative outcomes are published through - /// [`ManagedGroupEvent::TargetFinished`] before the subscription is sealed, - /// and queued events remain drainable after this method returns. - pub fn join(mut self) -> Result { - let scheduler = self.scheduler.take().expect( - "ManagedClientGroupSession invariant violated: scheduler handle missing before join", - ); - let scheduler_result = scheduler.join().unwrap_or(Err(ClientError::WorkerPanicked)); - - let receiver = self.receiver.take().expect( - "ManagedClientGroupSession invariant violated: receiver handle missing before join", - ); - let receiver_result = receiver.join().unwrap_or(Err(ClientError::WorkerPanicked)); - - self.hub.disconnect_all(); - match (scheduler_result, receiver_result) { - (Ok(outcome), Ok(())) => Ok(outcome), - (Err(err), _) => Err(err), - (Ok(_), Err(err)) => Err(err), - } - } -} - -impl Drop for ManagedClientGroupSession { - fn drop(&mut self) { - self.cancellation.cancel(); - let _ = self.control_tx.send(ControlMessage::Wake); - } -} - -#[derive(Debug)] -enum ControlMessage { - Update { - targets: Vec, - reply: mpsc::Sender>, - }, - Wake, -} - -#[derive(Debug)] -struct GroupShared { - registry: Mutex, - hub: EventHub, - cancellation: CancellationToken, - control_tx: mpsc::Sender, - peer_closed_target_count: Arc, - receiver_drain: Mutex, - requested_interval_ns: i64, - requested_dscp: i64, - family_remote: SocketAddr, -} - -#[derive(Debug, Clone, Copy, Default)] -struct ReceiverDrainState { - requested: u64, - completed: u64, - deadline: Option, -} - -impl ReceiverDrainState { - fn request_or_join(&mut self, now: Instant) -> Result { - self.assert_invariants(); - if self.is_outstanding() { - return Ok(self.requested); - } - - let generation = self - .requested - .checked_add(1) - .ok_or(ClientError::CounterOverflow { - counter: "receiver_drain_generation", - })?; - let deadline = now - .checked_add(GROUP_RECEIVER_DRAIN_LIMIT) - .ok_or(ClientError::DurationOverflow)?; - - self.requested = generation; - self.deadline = Some(deadline); - self.assert_invariants(); - Ok(generation) - } - - fn complete_observed(&mut self, observed_generation: u64) -> bool { - self.assert_invariants(); - let observed_generation = observed_generation.min(self.requested); - if observed_generation > self.completed { - self.completed = observed_generation; - if self.completed == self.requested { - self.deadline = None; - } - self.assert_invariants(); - true - } else { - false - } - } - - fn complete_if_expired(&mut self, now: Instant) -> bool { - self.assert_invariants(); - if !self.deadline.is_some_and(|deadline| deadline <= now) { - return false; - } - - self.completed = self.requested; - self.deadline = None; - self.assert_invariants(); - true - } - - fn is_outstanding(&self) -> bool { - self.requested > self.completed - } - - fn deadline(&self) -> Option { - self.deadline - } - - fn assert_invariants(&self) { - debug_assert!(self.completed <= self.requested); - debug_assert_eq!(self.is_outstanding(), self.deadline.is_some()); - } -} - -#[cfg(test)] -fn request_receiver_drain(shared: &GroupShared, now: Instant) -> Result { - shared - .receiver_drain - .lock() - .expect("receiver drain mutex poisoned") - .request_or_join(now) -} - -fn receiver_drain_snapshot(shared: &GroupShared) -> ReceiverDrainState { - *shared - .receiver_drain - .lock() - .expect("receiver drain mutex poisoned") -} - -fn requested_receiver_drain(shared: &GroupShared) -> u64 { - receiver_drain_snapshot(shared).requested -} - -fn completed_receiver_drain(shared: &GroupShared) -> u64 { - receiver_drain_snapshot(shared).completed -} - -fn complete_receiver_drain(shared: &GroupShared, observed_generation: u64) { - let advanced = { - let mut state = shared - .receiver_drain - .lock() - .expect("receiver drain mutex poisoned"); - state.complete_observed(observed_generation) - }; - if advanced { - let _ = shared.control_tx.send(ControlMessage::Wake); - } -} - -fn complete_expired_receiver_drain(shared: &GroupShared, now: Instant) -> bool { - shared - .receiver_drain - .lock() - .expect("receiver drain mutex poisoned") - .complete_if_expired(now) -} - -struct GroupSchedulerCleanup { - shared: Arc, -} - -impl Drop for GroupSchedulerCleanup { - fn drop(&mut self) { - self.shared.cancellation.cancel(); - let _ = self.shared.control_tx.send(ControlMessage::Wake); - self.shared.hub.disconnect_all(); - } -} - -#[derive(Debug, Default)] -struct TargetRegistry { - desired: HashSet, - targets: HashMap>>, - remotes: HashMap, -} - -#[derive(Debug)] -struct TargetState { - id: TargetId, - remote: SocketAddr, - configured_auth: Option, - runtime: SessionMachine, - schedule: Option, - status: TargetStatus, - open_request: PreparedOpenRequest, - counters: TargetCounters, - order: u64, - final_reason: Option, - #[cfg(test)] - probe_reported_len: Option, - #[cfg(test)] - probe_send_error: bool, - #[cfg(test)] - probe_send_timestamps: Option, - #[cfg(test)] - close_reported_len: Option, - #[cfg(test)] - close_send_error: bool, - #[cfg(test)] - close_send_attempts: usize, - #[cfg(test)] - close_event_reserve_error: bool, - #[cfg(test)] - close_sent_at: Option, -} - -type ScheduledTarget = (Arc>, Instant); - -#[derive(Debug)] -enum TargetStatus { - Opening { - attempt: usize, - next_send_at: Instant, - awaiting_receiver_generation: Option, - }, - Active, - Draining { - deadline: Instant, - }, - Finished, -} - -#[derive(Debug)] -struct PreparedGroupOpen { - machine: PreparedOpenAcceptance, - schedule: Option, -} - -#[derive(Debug)] -struct PreparedGroupOpenFailure { - primary: ClientError, - machine: PreparedOpenAcceptance, -} - -#[derive(Debug, Default, Clone, PartialEq, Eq)] -struct TargetCounters { - replies_received: u64, - duplicates: u64, - late: u64, - warning_events: u64, -} - -#[derive(Debug, Default)] -struct TargetOutcomeHistory { - recent: VecDeque, - total: u64, - successful: u64, - peer_closed: u64, - failed: u64, - discarded: u64, -} - -impl TargetOutcomeHistory { - fn record(&mut self, outcome: ManagedTargetOutcome) { - self.total = self.total.saturating_add(1); - if outcome.is_success() { - self.successful = self.successful.saturating_add(1); - } - if matches!(&outcome.end_reason, ManagedTargetEndReason::PeerClosed) { - self.peer_closed = self.peer_closed.saturating_add(1); - } - if outcome.end_reason.failure().is_some() { - self.failed = self.failed.saturating_add(1); - } - if self.recent.len() == MANAGED_GROUP_OUTCOME_HISTORY_LIMIT { - self.recent.pop_front(); - self.discarded = self.discarded.saturating_add(1); - } - self.recent.push_back(outcome); - } - - fn into_group_outcome(self, end_reason: ManagedGroupEndReason) -> ManagedGroupOutcome { - ManagedGroupOutcome { - end_reason, - targets: self.recent.into_iter().collect(), - total_target_outcomes: self.total, - successful_target_outcomes: self.successful, - peer_closed_target_outcomes: self.peer_closed, - failed_target_outcomes: self.failed, - discarded_target_outcomes: self.discarded, - } - } -} - -impl TargetState { - fn new( - group_config: &ClientConfig, - target: ManagedTargetConfig, - order: u64, - now: Instant, - ) -> Result { - let mut config = group_config.clone(); - if let Some(auth) = &target.auth { - config.hmac_key = auth.hmac_key.clone(); - } - let runtime = SessionMachine::new(config, target.remote)?; - let open_request = runtime.prepare_open_request()?; - Ok(Self { - id: target.id, - remote: target.remote, - configured_auth: target.auth, - runtime, - schedule: None, - status: TargetStatus::Opening { - attempt: 0, - next_send_at: now, - awaiting_receiver_generation: None, - }, - open_request, - counters: TargetCounters::default(), - order, - final_reason: None, - #[cfg(test)] - probe_reported_len: None, - #[cfg(test)] - probe_send_error: false, - #[cfg(test)] - probe_send_timestamps: None, - #[cfg(test)] - close_reported_len: None, - #[cfg(test)] - close_send_error: false, - #[cfg(test)] - close_send_attempts: 0, - #[cfg(test)] - close_event_reserve_error: false, - #[cfg(test)] - close_sent_at: None, - }) - } - - fn same_config(&self, config: &ManagedTargetConfig) -> bool { - self.remote == config.remote && self.configured_auth == config.auth - } - - fn observe(&mut self, event: &ClientEvent) { - match event { - ClientEvent::EchoReply { .. } => self.counters.replies_received += 1, - ClientEvent::DuplicateReply { .. } => self.counters.duplicates += 1, - ClientEvent::LateReply { .. } => self.counters.late += 1, - ClientEvent::Warning { .. } => self.counters.warning_events += 1, - _ => {} - } - } - - fn mark_finished(&mut self, reason: ManagedTargetEndReason) { - if self.final_reason.is_some() { - return; - } - self.status = TargetStatus::Finished; - self.final_reason = Some(reason); - } - - fn is_run_complete(&self) -> bool { - self.runtime.is_terminal() - || self - .schedule - .as_ref() - .is_some_and(|schedule| schedule.is_finished() && self.runtime.pending_is_empty()) - } - - fn outcome(&self, reason: ManagedTargetEndReason) -> ManagedTargetOutcome { - ManagedTargetOutcome { - id: self.id.clone(), - remote: self.remote, - end_reason: reason, - packets_sent: self.runtime.packets_sent(), - replies_received: self.counters.replies_received, - duplicates: self.counters.duplicates, - late: self.counters.late, - warning_events: self.counters.warning_events, - } - } -} - -fn run_group_scheduler( - config: ManagedClientGroupConfig, - socket: UdpSocket, - shared: Arc, - control_rx: mpsc::Receiver, - mut next_order: u64, -) -> Result { - let mut records = TargetOutcomeHistory::default(); - let mut pacing = PacingRuntime::new(config.pacing); - let mut pending_control = None; - - loop { - drain_control_messages( - &config.client, - &socket, - &shared, - &control_rx, - pending_control.take(), - &mut next_order, - &mut records, - ); - - if shared.cancellation.is_cancelled() { - cancel_remaining_targets(&socket, &shared, &mut records); - return Ok(records.into_group_outcome(ManagedGroupEndReason::Cancelled)); - } - - let now = Instant::now(); - complete_expired_receiver_drain(&shared, now); - drive_open_attempts(&config.client, &socket, &shared, now); - poll_active_timeouts(&shared, now); - finish_completed_targets(&socket, &shared, now); - collect_finished_targets(&shared, &mut records); - - if group_should_complete(&shared, config.completion) { - return Ok(records.into_group_outcome(ManagedGroupEndReason::AllTargetsComplete)); - } - - let active = active_targets(&shared); - pacing.reconcile(&active, now); - let receiver_drain_outstanding = receiver_drain_snapshot(&shared).is_outstanding(); - if send_due_active_probes( - &config, - &socket, - &shared, - &mut pacing, - &active, - now, - receiver_drain_outstanding, - )? { - continue; - } - - pending_control = wait_for_next_scheduler_wakeup(&control_rx, &shared, &pacing, &active); - } -} - -fn run_group_receiver(socket: UdpSocket, shared: Arc) -> Result<(), ClientError> { - let mut buf = vec![0_u8; RECV_BUFFER_SIZE]; - while !shared.cancellation.is_cancelled() { - // A timeout only proves requests visible before this receive began. - let observed_drain_generation = requested_receiver_drain(&shared); - let datagram = match recv_datagram_from(&socket, &mut buf) { - Ok(datagram) => datagram, - Err(err) - if matches!( - err.kind(), - std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut - ) => - { - complete_receiver_drain(&shared, observed_drain_generation); - continue; - } - Err(err) => { - let failure = ManagedTargetFailure { - kind: ManagedTargetFailureKind::Socket, - message: ClientError::Socket(std::io::Error::new(err.kind(), err.to_string())) - .to_string(), - }; - fail_all_targets(&shared, failure); - shared.cancellation.cancel(); - let _ = shared.control_tx.send(ControlMessage::Wake); - return Err(ClientError::Socket(err)); - } - }; - - process_group_datagram(&socket, &shared, datagram, &buf[..datagram.len]); - } - Ok(()) -} - -fn process_group_datagram( - socket: &UdpSocket, - shared: &GroupShared, - datagram: ReceivedDatagramFrom, - packet: &[u8], -) { - let target = { - let registry = shared - .registry - .lock() - .expect("target registry mutex poisoned"); - let Some(id) = registry.remotes.get(&datagram.source) else { - return; - }; - registry.targets.get(id).cloned() - }; - let Some(target) = target else { - return; - }; - - let is_opening = matches!( - target.lock().expect("target mutex poisoned").status, - TargetStatus::Opening { .. } - ); - if is_opening { - if process_opening_target_packet(socket, shared, &target, datagram.received_at, packet) { - let _ = shared.control_tx.send(ControlMessage::Wake); - } - return; - } - - let mut target = target.lock().expect("target mutex poisoned"); - let wake_scheduler = match target.status { - TargetStatus::Active | TargetStatus::Draining { .. } => process_active_target_packet( - &shared.hub, - &mut target, - packet, - datagram.received_at, - datagram.meta, - ), - TargetStatus::Opening { .. } | TargetStatus::Finished => false, - }; - if wake_scheduler { - let _ = shared.control_tx.send(ControlMessage::Wake); - } -} - -fn process_opening_target_packet( - socket: &UdpSocket, - shared: &GroupShared, - target: &Arc>, - opened_at: ClientTimestamp, - packet: &[u8], -) -> bool { - // Serialize opening-packet eligibility with natural and forced drain - // completion. The shared drain lock always precedes the target lock. - let drain = shared - .receiver_drain - .lock() - .expect("receiver drain mutex poisoned"); - let mut target = target.lock().expect("target mutex poisoned"); - let TargetStatus::Opening { - attempt, - awaiting_receiver_generation, - .. - } = target.status - else { - return false; - }; - if attempt == 0 - || awaiting_receiver_generation.is_some_and(|generation| drain.completed >= generation) - { - return false; - } - - let reply = match target.runtime.inspect_open_datagram(packet) { - Ok(OpenDatagramDisposition::Ignore) => return false, - Ok(OpenDatagramDisposition::Trusted(reply)) => reply, - Err(err) => { - target.mark_finished(ManagedTargetEndReason::OpenFailed( - ManagedTargetFailure::opening(&err), - )); - return true; - } - }; - let machine = match target.runtime.prepare_open_acceptance(reply, opened_at) { - Ok(machine) => machine, - Err(failure) => { - send_group_cleanup_close_best_effort( - socket, - target.remote, - failure.cleanup_close.as_deref(), - ); - target.mark_finished(ManagedTargetEndReason::OpenFailed( - ManagedTargetFailure::opening(&failure.primary), - )); - return true; - } - }; - - match prepare_group_open(shared, machine, opened_at) { - Ok(prepared) => { - let outcome = target.runtime.commit_open(prepared.machine); - target.schedule = prepared.schedule; - publish_open_outcome(&shared.hub, &mut target, outcome); - } - Err(failure) => { - send_group_cleanup_close_best_effort( - socket, - target.remote, - failure.machine.cleanup_close_packet(), - ); - target.mark_finished(ManagedTargetEndReason::OpenFailed( - ManagedTargetFailure::opening(&failure.primary), - )); - } - } - drop(target); - drop(drain); - true -} - -fn validate_group_negotiation( - shared: &GroupShared, - negotiated: &crate::NegotiatedParams, -) -> Result<(), ClientError> { - if negotiated.params.interval_ns != shared.requested_interval_ns { - return Err(ClientError::NegotiationRejected { - reason: "managed group v1 requires the negotiated interval to match the group interval" - .to_owned(), - }); - } - if negotiated.params.dscp != shared.requested_dscp { - return Err(ClientError::NegotiationRejected { - reason: "managed group v1 requires the negotiated DSCP to match the group DSCP" - .to_owned(), - }); - } - Ok(()) -} - -fn prepare_group_open( - shared: &GroupShared, - machine: PreparedOpenAcceptance, - opened_at: ClientTimestamp, -) -> Result> { - let schedule = if let Some(negotiated) = machine.normal_negotiated() { - if let Err(primary) = validate_group_negotiation(shared, negotiated) { - return Err(Box::new(PreparedGroupOpenFailure { primary, machine })); - } - match ProbeSchedule::new(opened_at.mono, negotiated) { - Ok(schedule) => Some(schedule), - Err(primary) => return Err(Box::new(PreparedGroupOpenFailure { primary, machine })), - } - } else { - None - }; - Ok(PreparedGroupOpen { machine, schedule }) -} - -fn send_group_cleanup_close_best_effort( - socket: &UdpSocket, - remote: SocketAddr, - packet: Option<&[u8]>, -) { - if let Some(packet) = packet { - let _ = socket.send_to(packet, remote); - } -} - -fn publish_open_outcome( - hub: &EventHub, - target: &mut TargetState, - outcome: OpenOutcome, -) { - match outcome { - OpenOutcome::Started { event, .. } => { - target.status = TargetStatus::Active; - hub.publish(ManagedGroupEvent::Client(TargetEvent { - target: target.id.clone(), - event, - })); - } - OpenOutcome::NoTestCompleted { event, .. } => { - target.observe(&event); - hub.publish(ManagedGroupEvent::Client(TargetEvent { - target: target.id.clone(), - event, - })); - target.mark_finished(ManagedTargetEndReason::NoTestComplete); - } - } -} - -fn publish_events( - hub: &EventHub, - target: &mut TargetState, - events: Vec, -) { - for event in events { - target.observe(&event); - hub.publish(ManagedGroupEvent::Client(TargetEvent { - target: target.id.clone(), - event, - })); - } -} - -fn process_active_target_packet( - hub: &EventHub, - target: &mut TargetState, - packet: &[u8], - received_at: ClientTimestamp, - meta: ReceiveMeta, -) -> bool { - let was_draining = matches!(target.status, TargetStatus::Draining { .. }); - match target - .runtime - .process_received_echo_packet(packet, received_at, meta) - { - Ok(events) => { - publish_events(hub, target, events); - if target.runtime.is_peer_closed() { - target.schedule = None; - target.mark_finished(ManagedTargetEndReason::PeerClosed); - true - } else { - was_draining && !target.runtime.has_timed_out_metadata() - } - } - Err(err) => { - target.mark_finished(ManagedTargetEndReason::Failed( - ManagedTargetFailure::runtime(&err), - )); - true - } - } -} - -fn drain_control_messages( - config: &ClientConfig, - socket: &UdpSocket, - shared: &GroupShared, - control_rx: &mpsc::Receiver, - first_message: Option, - next_order: &mut u64, - records: &mut TargetOutcomeHistory, -) { - if let Some(message) = first_message { - handle_control_message(config, socket, shared, next_order, records, message); - } - while let Ok(message) = control_rx.try_recv() { - handle_control_message(config, socket, shared, next_order, records, message); - } -} - -fn handle_control_message( - config: &ClientConfig, - socket: &UdpSocket, - shared: &GroupShared, - next_order: &mut u64, - records: &mut TargetOutcomeHistory, - message: ControlMessage, -) { - match message { - ControlMessage::Update { targets, reply } => { - let result = apply_target_update(config, socket, shared, targets, next_order, records); - let _ = reply.send(result); - } - ControlMessage::Wake => {} - } -} - -fn apply_target_update( - config: &ClientConfig, - socket: &UdpSocket, - shared: &GroupShared, - targets: Vec, - next_order: &mut u64, - records: &mut TargetOutcomeHistory, -) -> Result<(), ClientError> { - validate_open_timeouts(&config.open_timeouts)?; - if !targets.is_empty() { - validate_target_configs(&targets, Some(shared.family_remote))?; - } - - let desired_ids: HashSet = targets.iter().map(|target| target.id.clone()).collect(); - let mut additions = Vec::new(); - - { - let registry = shared - .registry - .lock() - .expect("target registry mutex poisoned"); - for target in &targets { - if let Some(existing) = registry.targets.get(&target.id) { - let existing = existing.lock().expect("target mutex poisoned"); - if !existing.same_config(target) { - return Err(ClientError::InvalidConfig { - reason: format!( - "managed group v1 rejects changed remote/auth for existing target {}", - target.id - ), - }); - } - } else { - additions.push(target.clone()); - } - } - } - - let mut prepared = Vec::new(); - let now = Instant::now(); - for target in additions { - let order = *next_order; - *next_order = next_order - .checked_add(1) - .ok_or(ClientError::CounterOverflow { - counter: "target_order", - })?; - prepared.push(Arc::new(Mutex::new(TargetState::new( - config, target, order, now, - )?))); - } - - let removed = { - let mut registry = shared - .registry - .lock() - .expect("target registry mutex poisoned"); - let removed_ids: Vec = registry - .targets - .keys() - .filter(|id| !desired_ids.contains(*id)) - .cloned() - .collect(); - let mut removed = Vec::with_capacity(removed_ids.len()); - for id in removed_ids { - if let Some(target) = registry.targets.remove(&id) { - let remote = target.lock().expect("target mutex poisoned").remote; - registry.remotes.remove(&remote); - removed.push(target); - } - } - for target in prepared { - let (id, remote) = { - let target = target.lock().expect("target mutex poisoned"); - (target.id.clone(), target.remote) - }; - registry.remotes.insert(remote, id.clone()); - registry.targets.insert(id, target); - } - registry.desired = desired_ids; - removed - }; - - for target in removed { - let outcome = close_target(socket, &shared.hub, target, ManagedTargetEndReason::Removed); - record_target_outcome(shared, records, outcome); - } - - Ok(()) -} - -fn validate_target_configs( - targets: &[ManagedTargetConfig], - expected_family: Option, -) -> Result { - let mut ids = HashSet::new(); - let mut remotes = HashSet::new(); - let mut family_remote = expected_family; - - for target in targets { - if !ids.insert(target.id.clone()) { - return Err(ClientError::InvalidConfig { - reason: format!("duplicate managed target id {}", target.id), - }); - } - if !remotes.insert(target.remote) { - return Err(ClientError::InvalidConfig { - reason: format!("duplicate managed target remote {}", target.remote), - }); - } - if let Some(family) = family_remote { - if family.is_ipv4() != target.remote.is_ipv4() { - return Err(ClientError::InvalidConfig { - reason: "managed client group targets must use one address family".to_owned(), - }); - } - } else { - family_remote = Some(target.remote); - } - } - - family_remote.ok_or_else(|| ClientError::InvalidConfig { - reason: "managed client group requires at least one target".to_owned(), - }) -} - -fn validate_group_family_flags( - config: &ClientConfig, - remote: SocketAddr, -) -> Result<(), ClientError> { - if config.socket_config.ipv4_only && config.socket_config.ipv6_only { - return Err(ClientError::InvalidConfig { - reason: "ipv4_only and ipv6_only cannot both be true".to_owned(), - }); - } - if config.socket_config.ipv4_only && remote.is_ipv6() { - return Err(ClientError::InvalidConfig { - reason: "ipv4_only cannot be used with IPv6 group targets".to_owned(), - }); - } - if config.socket_config.ipv6_only && remote.is_ipv4() { - return Err(ClientError::InvalidConfig { - reason: "ipv6_only cannot be used with IPv4 group targets".to_owned(), - }); - } - Ok(()) -} - -fn drive_open_attempts( - config: &ClientConfig, - socket: &UdpSocket, - shared: &GroupShared, - now: Instant, -) { - let targets = all_targets(shared); - register_due_open_drains(shared, &targets, now); - let completed_drain_generation = completed_receiver_drain(shared); - - for target in targets { - let mut target = target.lock().expect("target mutex poisoned"); - let TargetStatus::Opening { - attempt, - next_send_at, - awaiting_receiver_generation, - } = target.status - else { - continue; - }; - - if next_send_at > now { - continue; - } - - if attempt > 0 { - match awaiting_receiver_generation { - None => continue, - Some(generation) if completed_drain_generation < generation => continue, - Some(_) => {} - } - } - - if attempt >= config.open_timeouts.len() { - target.mark_finished(ManagedTargetEndReason::OpenFailed( - ManagedTargetFailure::opening(&ClientError::OpenTimeout), - )); - continue; - } - - let sent_at = Instant::now(); - let deadline = match sent_at.checked_add(config.open_timeouts[attempt]) { - Some(deadline) => deadline, - None => { - target.mark_finished(ManagedTargetEndReason::OpenFailed( - ManagedTargetFailure::opening(&ClientError::DurationOverflow), - )); - continue; - } - }; - - match socket.send_to(&target.open_request.bytes, target.remote) { - Ok(_) => { - target.status = TargetStatus::Opening { - attempt: attempt + 1, - next_send_at: deadline, - awaiting_receiver_generation: None, - }; - } - Err(err) => { - target.mark_finished(ManagedTargetEndReason::OpenFailed( - ManagedTargetFailure::opening(&ClientError::Socket(err)), - )); - } - } - } -} - -fn register_due_open_drains( - shared: &GroupShared, - targets: &[Arc>], - now: Instant, -) { - // Register every currently due target while completion is excluded so one - // empty-queue boundary can release the whole batch. - let mut drain = shared - .receiver_drain - .lock() - .expect("receiver drain mutex poisoned"); - for target in targets { - let mut target = target.lock().expect("target mutex poisoned"); - let TargetStatus::Opening { - attempt, - next_send_at, - awaiting_receiver_generation: None, - } = target.status - else { - continue; - }; - if attempt == 0 || next_send_at > now { - continue; - } - match drain.request_or_join(now) { - Ok(generation) => { - target.status = TargetStatus::Opening { - attempt, - next_send_at, - awaiting_receiver_generation: Some(generation), - }; - } - Err(err) => target.mark_finished(ManagedTargetEndReason::OpenFailed( - ManagedTargetFailure::opening(&err), - )), - } - } -} - -fn poll_active_timeouts(shared: &GroupShared, now: Instant) { - for target in all_targets(shared) { - let mut target = target.lock().expect("target mutex poisoned"); - if !matches!( - target.status, - TargetStatus::Active | TargetStatus::Draining { .. } - ) { - continue; - } - if target.runtime.is_peer_closed() { - target.mark_finished(ManagedTargetEndReason::PeerClosed); - continue; - } - match target.runtime.poll_timeouts_at(now) { - Ok(events) => publish_events(&shared.hub, &mut target, events), - Err(err) => target.mark_finished(ManagedTargetEndReason::Failed( - ManagedTargetFailure::runtime(&err), - )), - } - } -} - -fn finish_completed_targets(socket: &UdpSocket, shared: &GroupShared, now: Instant) { - for target in all_targets(shared) { - let mut target = target.lock().expect("target mutex poisoned"); - match target.status { - TargetStatus::Active if target.is_run_complete() => { - if target.runtime.is_peer_closed() { - target.mark_finished(ManagedTargetEndReason::PeerClosed); - } else if target.runtime.has_timed_out_metadata() { - target.status = TargetStatus::Draining { - deadline: now + GROUP_FINAL_DRAIN, - }; - } else { - close_locked_target( - socket, - &shared.hub, - &mut target, - ManagedTargetEndReason::TestComplete, - ); - } - } - TargetStatus::Draining { deadline } => { - if target.runtime.is_peer_closed() { - target.mark_finished(ManagedTargetEndReason::PeerClosed); - } else if now >= deadline || !target.runtime.has_timed_out_metadata() { - close_locked_target( - socket, - &shared.hub, - &mut target, - ManagedTargetEndReason::TestComplete, - ); - } - } - _ => {} - } - } -} - -fn send_due_active_probes( - config: &ManagedClientGroupConfig, - socket: &UdpSocket, - shared: &GroupShared, - pacing: &mut PacingRuntime, - active: &[Arc>], - now: Instant, - receiver_drain_outstanding: bool, -) -> Result { - if receiver_drain_outstanding || !pacing.send_due(active, now) { - return Ok(false); - } - - match config.pacing { - ManagedGroupPacing::Staggered => { - if let Some((target, scheduled_at)) = - pacing.next_staggered(active, config.client.interval, now)? - { - send_echo_to_target(socket, shared, target, scheduled_at); - } - } - ManagedGroupPacing::Burst => { - if let Some(scheduled_at) = pacing.next_burst(config.client.interval, now)? { - for target in active { - send_echo_to_target(socket, shared, target.clone(), scheduled_at); - } - } - } - } - Ok(true) -} - -fn send_echo_to_target( - socket: &UdpSocket, - shared: &GroupShared, - target: Arc>, - scheduled_at: Instant, -) { - let mut target = target.lock().expect("target mutex poisoned"); - if !matches!(target.status, TargetStatus::Active) { - return; - } - let result = send_echo_to_locked_target(socket, &mut target, scheduled_at); - - match result { - Ok(events) => { - // Publish while the target lock is still held. EventHub publish is - // bounded and nonblocking, and this preserves per-target - // EchoSent-before-EchoReply ordering for immediately returned UDP - // replies. - publish_events(&shared.hub, &mut target, events); - } - Err(err) => target.mark_finished(ManagedTargetEndReason::Failed( - ManagedTargetFailure::runtime(&err), - )), - } -} - -fn send_echo_to_locked_target( - socket: &UdpSocket, - target: &mut TargetState, - scheduled_at: Instant, -) -> Result, ClientError> { - send_echo_to_locked_target_inner(socket, target, scheduled_at) -} - -#[cfg(test)] -fn send_echo_to_locked_target_at( - socket: &UdpSocket, - target: &mut TargetState, - scheduled_at: Instant, - timestamps: ProbeSendTimestamps, -) -> Result, ClientError> { - target.probe_send_timestamps = Some(timestamps); - send_echo_to_locked_target_inner(socket, target, scheduled_at) -} - -fn send_echo_to_locked_target_inner( - socket: &UdpSocket, - target: &mut TargetState, - scheduled_at: Instant, -) -> Result, ClientError> { - target.runtime.ensure_open()?; - let remote = target.remote; - #[cfg(test)] - let test_timestamps = target.probe_send_timestamps.take(); - #[cfg(test)] - let reported_bytes = target.probe_reported_len.take(); - #[cfg(test)] - let fail_send = std::mem::take(&mut target.probe_send_error); - let TargetState { - runtime, schedule, .. - } = target; - let schedule = schedule - .as_mut() - .expect("active targets always have a probe schedule"); - #[cfg(not(test))] - let permission_at = Instant::now(); - #[cfg(test)] - let permission_at = test_timestamps - .map(|timestamps| timestamps.permission_at) - .unwrap_or_else(Instant::now); - if !schedule.permit_probe_at(permission_at) { - return Ok(Vec::new()); - } - - let Some(prepared) = runtime.prepare_probe()? else { - return Ok(Vec::new()); - }; - let machine_preflight = runtime.preflight_probe_commit(&prepared)?; - let schedule_commit = schedule.preflight_managed_commit(scheduled_at, permission_at)?; - let mut events = Vec::new(); - events - .try_reserve(1) - .map_err(|source| ClientError::AllocationFailed { - operation: "managed probe event result", - source, - })?; - - let expected_bytes = prepared.bytes.len(); - let event_scheduled_at = schedule_commit.scheduled_at; - #[cfg(not(test))] - let sent_at = ClientTimestamp::now(); - #[cfg(test)] - let sent_at = test_timestamps - .map(|timestamps| timestamps.sent_at) - .unwrap_or_else(ClientTimestamp::now); - let machine_commit = runtime.finalize_probe_commit(machine_preflight, sent_at)?; - #[cfg(not(test))] - let send_call_start = Instant::now(); - #[cfg(test)] - let send_call_start = test_timestamps - .map(|timestamps| timestamps.send_call_start) - .unwrap_or_else(Instant::now); - #[cfg(test)] - if fail_send { - return Err(ClientError::Socket(std::io::Error::other( - "injected managed probe send failure", - ))); - } - let bytes = socket.send_to(&prepared.bytes, remote)?; - #[cfg(not(test))] - let send_finished_at = Instant::now(); - #[cfg(test)] - let send_finished_at = test_timestamps - .map(|timestamps| timestamps.send_finished_at) - .unwrap_or_else(Instant::now); - let sent = runtime.commit_probe_sent(machine_commit, bytes); - schedule.commit(schedule_commit); - - let send_call = send_finished_at.saturating_duration_since(send_call_start); - let timer_error = instant_abs_diff(sent_at.mono, event_scheduled_at); - #[cfg(test)] - let bytes = reported_bytes.unwrap_or(bytes); - validate_datagram_length(expected_bytes, bytes)?; - events.push(echo_sent_event( - remote, - sent, - send_call, - event_scheduled_at, - timer_error, - )); - Ok(events) -} - -fn collect_finished_targets(shared: &GroupShared, records: &mut TargetOutcomeHistory) { - let finished = { - let mut registry = shared - .registry - .lock() - .expect("target registry mutex poisoned"); - let ids: Vec = registry - .targets - .iter() - .filter_map(|(id, target)| { - let target = target.lock().expect("target mutex poisoned"); - matches!(target.status, TargetStatus::Finished).then(|| id.clone()) - }) - .collect(); - let mut finished = Vec::with_capacity(ids.len()); - for id in ids { - if let Some(target) = registry.targets.remove(&id) { - let remote = target.lock().expect("target mutex poisoned").remote; - registry.remotes.remove(&remote); - finished.push(target); - } - } - finished - }; - - for target in finished { - let target = target.lock().expect("target mutex poisoned"); - let reason = target.final_reason.clone().unwrap_or_else(|| { - ManagedTargetEndReason::Failed(ManagedTargetFailure { - kind: ManagedTargetFailureKind::InternalWorker, - message: "target finished without a reason".to_owned(), - }) - }); - record_target_outcome(shared, records, target.outcome(reason)); - } -} - -fn cancel_remaining_targets( - socket: &UdpSocket, - shared: &GroupShared, - records: &mut TargetOutcomeHistory, -) { - let targets = drain_registry(shared); - for target in targets { - let outcome = close_target( - socket, - &shared.hub, - target, - ManagedTargetEndReason::Cancelled, - ); - record_target_outcome(shared, records, outcome); - } -} - -fn record_target_outcome( - shared: &GroupShared, - records: &mut TargetOutcomeHistory, - outcome: ManagedTargetOutcome, -) { - if matches!(&outcome.end_reason, ManagedTargetEndReason::PeerClosed) { - let _ = shared.peer_closed_target_count.fetch_update( - Ordering::Release, - Ordering::Relaxed, - |count| Some(count.saturating_add(1)), - ); - } - shared - .hub - .publish(ManagedGroupEvent::TargetFinished(outcome.clone())); - records.record(outcome); -} - -fn fail_all_targets(shared: &GroupShared, failure: ManagedTargetFailure) { - for target in all_targets(shared) { - target - .lock() - .expect("target mutex poisoned") - .mark_finished(ManagedTargetEndReason::Failed(failure.clone())); - } -} - -fn drain_registry(shared: &GroupShared) -> Vec>> { - let mut registry = shared - .registry - .lock() - .expect("target registry mutex poisoned"); - registry.remotes.clear(); - registry.targets.drain().map(|(_, target)| target).collect() -} - -fn close_target( - socket: &UdpSocket, - hub: &EventHub, - target: Arc>, - reason: ManagedTargetEndReason, -) -> ManagedTargetOutcome { - let mut target = target.lock().expect("target mutex poisoned"); - close_locked_target(socket, hub, &mut target, reason.clone()); - let reason = target.final_reason.clone().unwrap_or(reason); - target.outcome(reason) -} - -fn close_locked_target( - socket: &UdpSocket, - hub: &EventHub, - target: &mut TargetState, - reason: ManagedTargetEndReason, -) { - if target.runtime.is_open() && !target.runtime.is_peer_closed() { - let remote = target.remote; - let prepared = match target.runtime.prepare_close() { - Ok(prepared) => prepared, - Err(err) => { - target.mark_finished(ManagedTargetEndReason::Failed( - ManagedTargetFailure::runtime(&err), - )); - return; - } - }; - let mut events = Vec::new(); - #[cfg(test)] - if std::mem::take(&mut target.close_event_reserve_error) { - if let Err(source) = events.try_reserve(usize::MAX) { - let err = ClientError::AllocationFailed { - operation: "managed close event result", - source, - }; - target.mark_finished(ManagedTargetEndReason::Failed( - ManagedTargetFailure::runtime(&err), - )); - return; - } - } - if let Err(source) = events.try_reserve(1) { - let err = ClientError::AllocationFailed { - operation: "managed close event result", - source, - }; - target.mark_finished(ManagedTargetEndReason::Failed( - ManagedTargetFailure::runtime(&err), - )); - return; - } - let expected_bytes = prepared.bytes.len(); - #[cfg(test)] - let reported_bytes = target.close_reported_len.take(); - #[cfg(test)] - let fail_send = std::mem::take(&mut target.close_send_error); - #[cfg(test)] - { - target.close_send_attempts += 1; - if fail_send { - let err = ClientError::Socket(std::io::Error::other( - "injected managed close send failure", - )); - target.mark_finished(ManagedTargetEndReason::Failed( - ManagedTargetFailure::runtime(&err), - )); - return; - } - } - let bytes = match socket.send_to(prepared.bytes, remote) { - Ok(bytes) => bytes, - Err(err) => { - let err = ClientError::Socket(err); - target.mark_finished(ManagedTargetEndReason::Failed( - ManagedTargetFailure::runtime(&err), - )); - return; - } - }; - #[cfg(not(test))] - let sent_at = ClientTimestamp::now(); - #[cfg(test)] - let sent_at = target - .close_sent_at - .take() - .unwrap_or_else(ClientTimestamp::now); - let event = target.runtime.commit_local_close(prepared.commit, sent_at); - target.schedule = None; - #[cfg(test)] - let bytes = reported_bytes.unwrap_or(bytes); - if let Err(err) = validate_datagram_length(expected_bytes, bytes) { - target.mark_finished(ManagedTargetEndReason::Failed( - ManagedTargetFailure::runtime(&err), - )); - return; - } - events.push(event); - publish_events(hub, target, events); - } - target.mark_finished(reason); -} - -fn active_targets(shared: &GroupShared) -> Vec>> { - let mut targets: Vec<_> = all_targets(shared) - .into_iter() - .filter(|target| { - let target = target.lock().expect("target mutex poisoned"); - matches!(target.status, TargetStatus::Active) - }) - .collect(); - targets.sort_by_key(|target| target.lock().expect("target mutex poisoned").order); - targets -} - -fn all_targets(shared: &GroupShared) -> Vec>> { - shared - .registry - .lock() - .expect("target registry mutex poisoned") - .targets - .values() - .cloned() - .collect() -} - -fn group_should_complete(shared: &GroupShared, completion: ManagedGroupCompletionPolicy) -> bool { - if completion == ManagedGroupCompletionPolicy::ExplicitCancellation { - return false; - } - let registry = shared - .registry - .lock() - .expect("target registry mutex poisoned"); - !registry.desired.is_empty() && registry.targets.is_empty() -} - -#[derive(Debug)] -struct PacingRuntime { - mode: ManagedGroupPacing, - signature: Vec, - next_slot_at: Option, - slot_index: usize, - next_burst_at: Option, -} - -impl PacingRuntime { - fn new(mode: ManagedGroupPacing) -> Self { - Self { - mode, - signature: Vec::new(), - next_slot_at: None, - slot_index: 0, - next_burst_at: None, - } - } - - fn reconcile(&mut self, active: &[Arc>], now: Instant) { - let signature: Vec = active - .iter() - .map(|target| target.lock().expect("target mutex poisoned").id.clone()) - .collect(); - if signature == self.signature { - return; - } - self.signature = signature; - self.slot_index = 0; - if active.is_empty() { - self.next_slot_at = None; - self.next_burst_at = None; - } else { - match self.mode { - ManagedGroupPacing::Staggered => self.next_slot_at = Some(now), - ManagedGroupPacing::Burst => { - self.next_burst_at.get_or_insert(now); - } - } - } - } - - fn send_due(&self, active: &[Arc>], now: Instant) -> bool { - if active.is_empty() { - return false; - } - match self.mode { - ManagedGroupPacing::Staggered => { - self.next_slot_at.is_some_and(|deadline| deadline <= now) - } - ManagedGroupPacing::Burst => self.next_burst_at.is_some_and(|deadline| deadline <= now), - } - } - - fn next_staggered( - &mut self, - active: &[Arc>], - interval: Duration, - now: Instant, - ) -> Result, ClientError> { - let Some(next_slot_at) = self.next_slot_at else { - return Ok(None); - }; - if active.is_empty() { - self.next_slot_at = None; - return Ok(None); - } - let (scheduled_at, next_slot_at, first_index) = - advance_staggered_slot(next_slot_at, interval, active.len(), self.slot_index, now)?; - self.next_slot_at = Some(next_slot_at); - - for offset in 0..active.len() { - let index = (first_index + offset) % active.len(); - if !target_is_due_for_slot(&active[index], scheduled_at) { - continue; - } - let target = active[index].clone(); - self.slot_index = (index + 1) % active.len(); - return Ok(Some((target, scheduled_at))); - } - - self.slot_index = (first_index + 1) % active.len(); - if let Some(target_deadline) = active.iter().filter_map(target_next_send_deadline).min() { - self.next_slot_at = Some(self.next_slot_at.map_or(target_deadline, |group_deadline| { - group_deadline.min(target_deadline) - })); - } - Ok(None) - } - - fn next_burst( - &mut self, - interval: Duration, - now: Instant, - ) -> Result, ClientError> { - let Some(next_burst_at) = self.next_burst_at else { - return Ok(None); - }; - let (scheduled_at, next_burst_at, _) = advance_cadence(next_burst_at, interval, now)?; - self.next_burst_at = Some(next_burst_at); - Ok(Some(scheduled_at)) - } - - fn next_wakeup(&self) -> Option { - match self.mode { - ManagedGroupPacing::Staggered => self.next_slot_at, - ManagedGroupPacing::Burst => self.next_burst_at, - } - } -} - -fn advance_staggered_slot( - next_slot_at: Instant, - interval: Duration, - active_len: usize, - slot_index: usize, - now: Instant, -) -> Result<(Instant, Instant, usize), ClientError> { - debug_assert!(active_len > 0); - let spacing = divide_duration(interval, active_len); - let (scheduled_at, following_slot_at, skipped_slots) = - advance_cadence(next_slot_at, spacing, now)?; - let skipped_mod = usize::try_from(skipped_slots % active_len as u128) - .expect("staggered slot remainder fits usize"); - let first_index = (slot_index + skipped_mod) % active_len; - Ok((scheduled_at, following_slot_at, first_index)) -} - -fn target_is_due_for_slot(target: &Arc>, scheduled_at: Instant) -> bool { - target_next_send_deadline(target).is_some_and(|deadline| deadline <= scheduled_at) -} - -fn target_next_send_deadline(target: &Arc>) -> Option { - let target = target.lock().expect("target mutex poisoned"); - if matches!(target.status, TargetStatus::Active) { - target.schedule.as_ref()?.next_send_deadline() - } else { - None - } -} - -fn wait_for_next_scheduler_wakeup( - control_rx: &mpsc::Receiver, - shared: &GroupShared, - pacing: &PacingRuntime, - active: &[Arc>], -) -> Option { - let open_deadline = next_open_deadline(shared); - let pacing_deadline = (!active.is_empty()).then(|| pacing.next_wakeup()).flatten(); - let target_drain_deadline = next_target_drain_deadline(shared); - let receiver_drain = receiver_drain_snapshot(shared); - let deadline = next_scheduler_deadline( - open_deadline, - pacing_deadline, - target_drain_deadline, - receiver_drain, - ); - - wait_for_scheduler_control(control_rx, &shared.cancellation, deadline) -} - -fn next_scheduler_deadline( - open_deadline: Option, - pacing_deadline: Option, - target_drain_deadline: Option, - receiver_drain: ReceiverDrainState, -) -> Option { - let pacing_deadline = (!receiver_drain.is_outstanding()) - .then_some(pacing_deadline) - .flatten(); - [ - open_deadline, - pacing_deadline, - target_drain_deadline, - receiver_drain.deadline(), - ] - .into_iter() - .flatten() - .min() -} - -fn wait_for_scheduler_control( - control_rx: &mpsc::Receiver, - cancellation: &CancellationToken, - deadline: Option, -) -> Option { - match deadline { - Some(deadline) => { - let sleep_for = deadline - .checked_duration_since(Instant::now()) - .unwrap_or_default() - .min(MAX_SLEEP); - match control_rx.recv_timeout(sleep_for) { - Ok(message) => Some(message), - Err(mpsc::RecvTimeoutError::Timeout) => None, - Err(mpsc::RecvTimeoutError::Disconnected) => { - cancellation.cancel(); - None - } - } - } - None => match control_rx.recv() { - Ok(message) => Some(message), - Err(_) => { - cancellation.cancel(); - None - } - }, - } -} - -fn next_open_deadline(shared: &GroupShared) -> Option { - all_targets(shared) - .into_iter() - .filter_map(|target| { - let target = target.lock().expect("target mutex poisoned"); - match target.status { - TargetStatus::Opening { - next_send_at, - awaiting_receiver_generation: None, - .. - } => Some(next_send_at), - _ => None, - } - }) - .min() -} - -fn next_target_drain_deadline(shared: &GroupShared) -> Option { - all_targets(shared) - .into_iter() - .filter_map(|target| { - let target = target.lock().expect("target mutex poisoned"); - match target.status { - TargetStatus::Draining { deadline } => Some(deadline), - _ => None, - } - }) - .min() -} - -fn divide_duration(duration: Duration, divisor: usize) -> Duration { - if divisor == 0 { - return duration; - } - let nanos = (duration.as_nanos() / divisor as u128).max(1); - Duration::from_nanos(u64::try_from(nanos).unwrap_or(u64::MAX)) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - config::NegotiationPolicy, EventSubscriptionError, SubscriberOverflow, WarningKind, - }; - use irtt_proto::{ - compute_hmac_in_place, echo_packet_len, - flags::{self, FLAG_OPEN, FLAG_REPLY}, - layout::PacketLayout, - Clock, OpenReply, Params, ReceivedStats, StampAt, TimestampFields, HMAC_SIZE, MAGIC, - PROTOCOL_VERSION, - }; - use std::sync::mpsc; - - const TOKEN: u64 = 0x1234_5678_90ab_cdef; - - fn probe_timestamps(permission_at: Instant, sent_at: ClientTimestamp) -> ProbeSendTimestamps { - ProbeSendTimestamps { - permission_at, - sent_at, - send_call_start: sent_at.mono, - send_finished_at: sent_at.mono, - } - } - - #[derive(Debug)] - struct FakeServer { - addr: SocketAddr, - _observations: mpsc::Receiver, - done: JoinHandle<()>, - } - - impl FakeServer { - fn join(self) { - self.done.join().unwrap(); - } - } - - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - enum ServerObservation { - Open { at: Instant }, - Echo { seq: u32, at: Instant }, - Close { at: Instant }, - } - - fn test_params(duration: Option, interval: Duration) -> Params { - Params { - protocol_version: PROTOCOL_VERSION, - duration_ns: duration.map_or(0, duration_ns_i64), - interval_ns: duration_ns_i64(interval), - length: 0, - received_stats: ReceivedStats::Both, - stamp_at: StampAt::Both, - clock: Clock::Both, - dscp: 0, - server_fill: None, - } - } - - fn duration_ns_i64(duration: Duration) -> i64 { - i64::try_from(duration.as_nanos()).expect("test duration fits i64 nanoseconds") - } - - fn group_config( - duration: Option, - interval: Duration, - pacing: ManagedGroupPacing, - ) -> ManagedClientGroupConfig { - ManagedClientGroupConfig { - client: ClientConfig { - server_addr: "127.0.0.1:1".to_owned(), - duration, - interval, - negotiation_policy: NegotiationPolicy::Strict, - open_timeouts: vec![Duration::from_millis(200)], - probe_timeout: Duration::from_millis(80), - socket_config: crate::SocketConfig { - recv_timeout: Some(Duration::from_millis(20)), - ..Default::default() - }, - ..ClientConfig::default() - }, - pacing, - completion: ManagedGroupCompletionPolicy::AllTargetsComplete, - } - } - - struct ActiveTargetFixture { - client_config: ClientConfig, - hub: EventHub, - subscription: TargetEventSubscription, - target: Arc>, - reply: Vec, - close_reply: Vec, - } - - fn active_target_fixture() -> ActiveTargetFixture { - active_target_fixture_at("127.0.0.1:2112".parse().unwrap()) - } - - fn active_target_fixture_at(remote: SocketAddr) -> ActiveTargetFixture { - let interval = Duration::from_millis(10); - let params = test_params(None, interval); - let client_config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - let hub = EventHub::new(); - let subscription = hub - .subscribe(SubscriberConfig { - capacity: 16, - overflow: SubscriberOverflow::DropNewest, - }) - .unwrap(); - let mut target = - TargetState::new(&client_config, target("peer", remote), 0, Instant::now()).unwrap(); - - let open_packet = open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms); - let open_reply = match target.runtime.inspect_open_datagram(&open_packet).unwrap() { - OpenDatagramDisposition::Trusted(reply) => reply, - OpenDatagramDisposition::Ignore => panic!("fixture open reply must be trusted"), - }; - let opened_at = ClientTimestamp::now(); - let prepared = target - .runtime - .prepare_open_acceptance(open_reply, opened_at) - .unwrap(); - let opened_schedule = Some( - ProbeSchedule::new(opened_at.mono, prepared.normal_negotiated().unwrap()).unwrap(), - ); - let open_outcome = target.runtime.commit_open(prepared); - target.schedule = opened_schedule; - publish_open_outcome(&hub, &mut target, open_outcome); - - let scheduled_at = Instant::now(); - let sent_at = ClientTimestamp::now(); - let sent_events = { - let remote = target.remote; - let TargetState { - runtime, schedule, .. - } = &mut target; - let schedule = schedule.as_mut().unwrap(); - assert!(schedule.permit_probe_at(sent_at.mono)); - let prepared = runtime.prepare_probe().unwrap().unwrap(); - let machine_preflight = runtime.preflight_probe_commit(&prepared).unwrap(); - let schedule_commit = schedule - .preflight_managed_commit(scheduled_at, sent_at.mono) - .unwrap(); - let event_scheduled_at = schedule_commit.scheduled_at; - let machine_commit = runtime - .finalize_probe_commit(machine_preflight, sent_at) - .unwrap(); - let sent = runtime.commit_probe_sent(machine_commit, prepared.bytes.len()); - schedule.commit(schedule_commit); - let timer_error = instant_abs_diff(sent_at.mono, event_scheduled_at); - vec![echo_sent_event( - remote, - sent, - Duration::ZERO, - event_scheduled_at, - timer_error, - )] - }; - publish_events(&hub, &mut target, sent_events); - - ActiveTargetFixture { - client_config, - hub, - subscription, - target: Arc::new(Mutex::new(target)), - reply: echo_reply_packet(TOKEN, 0, ¶ms, &TimestampFields::default()), - close_reply: echo_reply_packet_with_flags( - TOKEN, - 0, - ¶ms, - &TimestampFields::default(), - FLAG_REPLY | flags::FLAG_CLOSE, - ), - } - } - - fn shared_with_target( - client_config: &ClientConfig, - hub: EventHub, - target: Arc>, - ) -> GroupShared { - shared_with_targets(client_config, hub, vec![target]).0 - } - - fn shared_with_targets( - client_config: &ClientConfig, - hub: EventHub, - targets: Vec>>, - ) -> (GroupShared, mpsc::Receiver) { - let requested = params_from_config(client_config).unwrap(); - let mut registry = TargetRegistry::default(); - let mut family_remote = None; - for target in targets { - let (id, remote) = { - let target = target.lock().expect("target mutex poisoned"); - (target.id.clone(), target.remote) - }; - family_remote.get_or_insert(remote); - registry.desired.insert(id.clone()); - registry.remotes.insert(remote, id.clone()); - registry.targets.insert(id, target); - } - let (control_tx, control_rx) = mpsc::channel(); - - ( - GroupShared { - registry: Mutex::new(registry), - hub, - cancellation: CancellationToken::new(), - control_tx, - peer_closed_target_count: Arc::new(AtomicU64::new(0)), - receiver_drain: Mutex::new(ReceiverDrainState::default()), - requested_interval_ns: requested.interval_ns, - requested_dscp: requested.dscp, - family_remote: family_remote.expect("test shared state requires a target"), - }, - control_rx, - ) - } - - fn drain_available_group_events( - subscription: &TargetEventSubscription, - ) -> Vec { - let mut events = Vec::new(); - while let Some(event) = subscription.try_recv().unwrap() { - events.push(event); - } - events - } - - fn target(id: &str, remote: SocketAddr) -> ManagedTargetConfig { - ManagedTargetConfig { - id: TargetId::from(id), - remote, - auth: None, - } - } - - fn active_probe_target( - config: &ClientConfig, - params: &Params, - remote: SocketAddr, - opened_at: ClientTimestamp, - ) -> TargetState { - let mut target = - TargetState::new(config, target("peer", remote), 0, opened_at.mono).unwrap(); - let packet = open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, params); - let reply = match target.runtime.inspect_open_datagram(&packet).unwrap() { - OpenDatagramDisposition::Trusted(reply) => reply, - OpenDatagramDisposition::Ignore => panic!("test open reply must be trusted"), - }; - let prepared = target - .runtime - .prepare_open_acceptance(reply, opened_at) - .unwrap(); - target.schedule = Some( - ProbeSchedule::new(opened_at.mono, prepared.normal_negotiated().unwrap()).unwrap(), - ); - target.runtime.commit_open(prepared); - target.status = TargetStatus::Active; - target - } - - fn target_outcome( - id: &str, - remote: SocketAddr, - end_reason: ManagedTargetEndReason, - ) -> ManagedTargetOutcome { - ManagedTargetOutcome { - id: TargetId::from(id), - remote, - end_reason, - packets_sent: 0, - replies_received: 0, - duplicates: 0, - late: 0, - warning_events: 0, - } - } - - fn start_echo_server(params: Params, open_delay: Duration) -> FakeServer { - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let addr = socket.local_addr().unwrap(); - let (tx, rx) = mpsc::channel(); - let done = thread::spawn(move || { - socket - .set_read_timeout(Some(Duration::from_millis(800))) - .unwrap(); - let mut opened = false; - while let Some((packet, peer)) = recv_request_timeout(&socket) { - if packet[3] & FLAG_OPEN != 0 { - if !opened { - opened = true; - tx.send(ServerObservation::Open { at: Instant::now() }) - .unwrap(); - if open_delay > Duration::ZERO { - thread::sleep(open_delay); - } - } - socket - .send_to(&open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms), peer) - .unwrap(); - continue; - } - - if packet[3] & flags::FLAG_CLOSE != 0 { - tx.send(ServerObservation::Close { at: Instant::now() }) - .unwrap(); - break; - } - - let seq = u32::from_le_bytes(packet[12..16].try_into().unwrap()); - tx.send(ServerObservation::Echo { - seq, - at: Instant::now(), - }) - .unwrap(); - let ts = TimestampFields { - recv_wall: Some(1_000_000_000), - recv_mono: Some(100_000), - send_wall: Some(1_000_000_000), - send_mono: Some(100_000), - ..Default::default() - }; - socket - .send_to(&echo_reply_packet(TOKEN, seq, ¶ms, &ts), peer) - .unwrap(); - } - }); - FakeServer { - addr, - _observations: rx, - done, - } - } - - fn start_peer_close_server(params: Params) -> FakeServer { - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let addr = socket.local_addr().unwrap(); - let (tx, rx) = mpsc::channel(); - let done = thread::spawn(move || { - let (open, peer) = recv_request_timeout(&socket).expect("missing open request"); - assert_ne!(open[3] & FLAG_OPEN, 0); - tx.send(ServerObservation::Open { at: Instant::now() }) - .unwrap(); - socket - .send_to(&open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms), peer) - .unwrap(); - - let (echo, peer) = recv_request_timeout(&socket).expect("missing echo request"); - let seq = u32::from_le_bytes(echo[12..16].try_into().unwrap()); - tx.send(ServerObservation::Echo { - seq, - at: Instant::now(), - }) - .unwrap(); - socket - .send_to( - &echo_reply_packet_with_flags( - TOKEN, - seq, - ¶ms, - &TimestampFields::default(), - FLAG_REPLY | flags::FLAG_CLOSE, - ), - peer, - ) - .unwrap(); - }); - FakeServer { - addr, - _observations: rx, - done, - } - } - - fn start_open_failure_server(params: Params) -> FakeServer { - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let addr = socket.local_addr().unwrap(); - let (tx, rx) = mpsc::channel(); - let done = thread::spawn(move || { - let (open, peer) = recv_request_timeout(&socket).expect("missing open request"); - assert_ne!(open[3] & FLAG_OPEN, 0); - tx.send(ServerObservation::Open { at: Instant::now() }) - .unwrap(); - socket - .send_to(&open_reply(FLAG_OPEN | FLAG_REPLY, 0, ¶ms), peer) - .unwrap(); - }); - FakeServer { - addr, - _observations: rx, - done, - } - } - - fn start_no_test_server(params: Params) -> FakeServer { - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let addr = socket.local_addr().unwrap(); - let (tx, rx) = mpsc::channel(); - let done = thread::spawn(move || { - let (open, peer) = recv_request_timeout(&socket).expect("missing open request"); - assert_ne!(open[3] & flags::FLAG_CLOSE, 0); - tx.send(ServerObservation::Open { at: Instant::now() }) - .unwrap(); - socket - .send_to( - &open_reply(FLAG_OPEN | FLAG_REPLY | flags::FLAG_CLOSE, 0, ¶ms), - peer, - ) - .unwrap(); - }); - FakeServer { - addr, - _observations: rx, - done, - } - } - - fn start_silent_runtime_server(params: Params) -> FakeServer { - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let addr = socket.local_addr().unwrap(); - let (tx, rx) = mpsc::channel(); - let done = thread::spawn(move || { - let (open, peer) = recv_request_timeout(&socket).expect("missing open request"); - assert_ne!(open[3] & FLAG_OPEN, 0); - tx.send(ServerObservation::Open { at: Instant::now() }) - .unwrap(); - socket - .send_to(&open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms), peer) - .unwrap(); - socket - .set_read_timeout(Some(Duration::from_millis(800))) - .unwrap(); - while let Some((packet, _)) = recv_request_timeout(&socket) { - if packet[3] & flags::FLAG_CLOSE != 0 { - tx.send(ServerObservation::Close { at: Instant::now() }) - .unwrap(); - break; - } - let seq = u32::from_le_bytes(packet[12..16].try_into().unwrap()); - tx.send(ServerObservation::Echo { - seq, - at: Instant::now(), - }) - .unwrap(); - } - }); - FakeServer { - addr, - _observations: rx, - done, - } - } - - fn start_gated_reply_server(params: Params) -> (FakeServer, mpsc::SyncSender<()>) { - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let addr = socket.local_addr().unwrap(); - let (tx, rx) = mpsc::channel(); - let (release_tx, release_rx) = mpsc::sync_channel(0); - let done = thread::spawn(move || { - let (open, peer) = recv_request_timeout(&socket).expect("missing open request"); - assert_ne!(open[3] & FLAG_OPEN, 0); - tx.send(ServerObservation::Open { at: Instant::now() }) - .unwrap(); - socket - .send_to(&open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms), peer) - .unwrap(); - socket - .set_read_timeout(Some(Duration::from_millis(800))) - .unwrap(); - - let (echo, peer) = recv_request_timeout(&socket).expect("missing echo request"); - assert_eq!(echo[3] & flags::FLAG_CLOSE, 0); - assert!(echo.len() >= 16, "echo request too short: {}", echo.len()); - let seq = u32::from_le_bytes(echo[12..16].try_into().unwrap()); - tx.send(ServerObservation::Echo { - seq, - at: Instant::now(), - }) - .unwrap(); - release_rx - .recv_timeout(Duration::from_secs(2)) - .expect("timed out waiting to release delayed reply"); - - let timestamps = TimestampFields { - recv_wall: Some(1_000_000_000), - recv_mono: Some(100_000), - send_wall: Some(1_001_000_000), - send_mono: Some(1_100_000), - ..Default::default() - }; - socket - .send_to(&echo_reply_packet(TOKEN, seq, ¶ms, ×tamps), peer) - .unwrap(); - - while let Some((packet, _)) = recv_request_timeout(&socket) { - if packet[3] & flags::FLAG_CLOSE != 0 { - tx.send(ServerObservation::Close { at: Instant::now() }) - .unwrap(); - break; - } - } - }); - ( - FakeServer { - addr, - _observations: rx, - done, - }, - release_tx, - ) - } - - fn recv_request_timeout(socket: &UdpSocket) -> Option<(Vec, SocketAddr)> { - let mut buf = [0_u8; 2048]; - socket - .recv_from(&mut buf) - .ok() - .map(|(size, peer)| (buf[..size].to_vec(), peer)) - } - - fn recv_group_datagram(socket: &UdpSocket) -> (ReceivedDatagramFrom, Vec) { - let mut packet = vec![0_u8; RECV_BUFFER_SIZE]; - let datagram = recv_datagram_from(socket, &mut packet).unwrap(); - packet.truncate(datagram.len); - (datagram, packet) - } - - fn open_reply(flags: u8, token: u64, params: &Params) -> Vec { - let mut packet = Vec::new(); - packet.extend_from_slice(&MAGIC); - packet.push(flags); - packet.extend_from_slice(&token.to_le_bytes()); - packet.extend_from_slice(¶ms.encode()); - packet - } - - fn hmac_open_reply(flags: u8, token: u64, params: &Params, key: &[u8]) -> Vec { - let mut packet = Vec::new(); - packet.extend_from_slice(&MAGIC); - packet.push(flags | flags::FLAG_HMAC); - packet.extend_from_slice(&[0_u8; HMAC_SIZE]); - packet.extend_from_slice(&token.to_le_bytes()); - packet.extend_from_slice(¶ms.encode()); - compute_hmac_in_place(key, &mut packet, 4).unwrap(); - packet - } - - fn echo_reply_packet( - token: u64, - seq: u32, - params: &Params, - timestamps: &TimestampFields, - ) -> Vec { - echo_reply_packet_with_flags(token, seq, params, timestamps, FLAG_REPLY) - } - - fn echo_reply_packet_with_flags( - token: u64, - seq: u32, - params: &Params, - timestamps: &TimestampFields, - flags: u8, - ) -> Vec { - let layout = PacketLayout::echo(false, params); - let packet_len = echo_packet_len(false, params).unwrap(); - let mut packet = Vec::with_capacity(packet_len); - - packet.extend_from_slice(&MAGIC); - packet.push(flags); - packet.extend_from_slice(&token.to_le_bytes()); - packet.extend_from_slice(&seq.to_le_bytes()); - - if layout.recv_count { - packet.extend_from_slice(&42_u32.to_le_bytes()); - } - if layout.recv_window { - packet.extend_from_slice(&0x07_u64.to_le_bytes()); - } - if layout.recv_wall { - packet.extend_from_slice(×tamps.recv_wall.unwrap_or(0).to_le_bytes()); - } - if layout.recv_mono { - packet.extend_from_slice(×tamps.recv_mono.unwrap_or(0).to_le_bytes()); - } - if layout.midpoint_wall { - packet.extend_from_slice(×tamps.midpoint_wall.unwrap_or(0).to_le_bytes()); - } - if layout.midpoint_mono { - packet.extend_from_slice(×tamps.midpoint_mono.unwrap_or(0).to_le_bytes()); - } - if layout.send_wall { - packet.extend_from_slice(×tamps.send_wall.unwrap_or(0).to_le_bytes()); - } - if layout.send_mono { - packet.extend_from_slice(×tamps.send_mono.unwrap_or(0).to_le_bytes()); - } - - packet.resize(packet_len, 0); - packet - } - - fn recv_event_with_timeout(sub: &TargetEventSubscription) -> TargetEvent { - let deadline = Instant::now() + Duration::from_secs(2); - loop { - match sub.try_recv() { - Ok(Some(ManagedGroupEvent::Client(event))) => return event, - Ok(Some(ManagedGroupEvent::TargetFinished(_))) => {} - Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(1)), - Ok(None) => panic!("timed out waiting for group event"), - Err(err) => panic!("subscription ended while waiting for event: {err}"), - } - } - } - - fn drain_after_join(sub: &TargetEventSubscription) -> Vec { - let mut events = Vec::new(); - loop { - match sub.try_recv() { - Ok(Some(ManagedGroupEvent::Client(event))) => events.push(event), - Ok(Some(ManagedGroupEvent::TargetFinished(_))) => {} - Ok(None) => thread::sleep(Duration::from_millis(1)), - Err(EventSubscriptionError::Disconnected) => return events, - } - } - } - - fn collect_group_events_until_disconnected( - sub: &TargetEventSubscription, - ) -> Vec { - let deadline = Instant::now() + Duration::from_secs(2); - let mut events = Vec::new(); - loop { - match sub.try_recv() { - Ok(Some(event)) => events.push(event), - Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(1)), - Ok(None) => panic!("timed out waiting for managed group disconnection"), - Err(EventSubscriptionError::Disconnected) => return events, - } - } - } - - fn final_drain_event_sequence(events: &[ManagedGroupEvent]) -> Vec<&'static str> { - events - .iter() - .filter_map(|event| match event { - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::SessionStarted { .. }, - .. - }) => Some("started"), - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::EchoSent { seq: 0, .. }, - .. - }) => Some("sent"), - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::EchoLoss { seq: 0, .. }, - .. - }) => Some("loss"), - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::LateReply { seq: 0, .. }, - .. - }) => Some("late"), - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::SessionClosed { .. }, - .. - }) => Some("closed"), - ManagedGroupEvent::TargetFinished(_) => Some("finished"), - _ => None, - }) - .collect() - } - - fn join_group_with_timeout( - session: ManagedClientGroupSession, - timeout: Duration, - ) -> ManagedGroupOutcome { - let (result_tx, result_rx) = mpsc::sync_channel(1); - let joiner = thread::spawn(move || { - result_tx.send(session.join()).unwrap(); - }); - let outcome = result_rx - .recv_timeout(timeout) - .expect("timed out joining managed group") - .unwrap(); - joiner.join().unwrap(); - outcome - } - - #[test] - fn target_event_hub_uses_generic_overflow_behavior() { - let hub = EventHub::::new(); - let sub = hub - .subscribe(SubscriberConfig { - capacity: 1, - overflow: SubscriberOverflow::DropOldest, - }) - .unwrap(); - - hub.publish(TargetEvent { - target: TargetId::from("a"), - event: ClientEvent::Warning { - kind: WarningKind::UntrackedReply, - message: "old".to_owned(), - at: ClientTimestamp::now(), - }, - }); - hub.publish(TargetEvent { - target: TargetId::from("b"), - event: ClientEvent::Warning { - kind: WarningKind::UntrackedReply, - message: "new".to_owned(), - at: ClientTimestamp::now(), - }, - }); - - let event = sub.try_recv().unwrap().unwrap(); - assert_eq!(event.target, TargetId::from("b")); - assert!(sub.try_recv().unwrap().is_none()); - } - - #[test] - fn rejects_duplicate_resolved_remote_targets() { - let remote: SocketAddr = "127.0.0.1:12345".parse().unwrap(); - let err = ManagedClientGroup::start( - group_config( - Some(Duration::from_millis(10)), - Duration::from_millis(10), - ManagedGroupPacing::Staggered, - ), - vec![target("a", remote), target("b", remote)], - ) - .unwrap_err(); - - assert!(matches!(err, ClientError::InvalidConfig { .. })); - assert!(err.to_string().contains("duplicate managed target remote")); - } - - #[test] - fn rejects_mixed_address_families() { - let err = ManagedClientGroup::start( - group_config( - Some(Duration::from_millis(10)), - Duration::from_millis(10), - ManagedGroupPacing::Staggered, - ), - vec![ - target("a", "127.0.0.1:12345".parse().unwrap()), - target("b", "[::1]:12345".parse().unwrap()), - ], - ) - .unwrap_err(); - - assert!(matches!(err, ClientError::InvalidConfig { .. })); - assert!(err.to_string().contains("one address family")); - } - - #[test] - fn shared_socket_smoke_two_targets_emit_scoped_events() { - let duration = Duration::from_millis(70); - let interval = Duration::from_millis(20); - let params = test_params(Some(duration), interval); - let a = start_echo_server(params.clone(), Duration::ZERO); - let b = start_echo_server(params, Duration::ZERO); - - let (session, sub) = ManagedClientGroup::start_with_subscription( - group_config(Some(duration), interval, ManagedGroupPacing::Staggered), - vec![target("a", a.addr), target("b", b.addr)], - SubscriberConfig { - capacity: 256, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - let outcome = session.join().unwrap(); - let events = drain_after_join(&sub); - a.join(); - b.join(); - - assert_eq!( - outcome.end_reason, - ManagedGroupEndReason::AllTargetsComplete - ); - assert_eq!(outcome.targets.len(), 2); - assert!(outcome - .targets - .iter() - .all(|target| target.packets_sent > 0 && target.replies_received > 0)); - for id in [TargetId::from("a"), TargetId::from("b")] { - assert!(events.iter().any(|event| { - event.target == id && matches!(event.event, ClientEvent::SessionStarted { .. }) - })); - assert!(events.iter().any(|event| { - event.target == id && matches!(event.event, ClientEvent::EchoSent { .. }) - })); - assert!(events.iter().any(|event| { - event.target == id && matches!(event.event, ClientEvent::EchoReply { .. }) - })); - } - } - - #[test] - fn receiver_peer_close_wins_send_and_removal_races() { - let fixture = active_target_fixture(); - let shared = shared_with_target( - &fixture.client_config, - fixture.hub.clone(), - fixture.target.clone(), - ); - { - let mut target = fixture.target.lock().expect("target mutex poisoned"); - assert!(process_active_target_packet( - &fixture.hub, - &mut target, - &fixture.close_reply, - ClientTimestamp::now(), - ReceiveMeta::default(), - )); - assert!(matches!(target.status, TargetStatus::Finished)); - assert!(target.schedule.is_none()); - assert_eq!( - target.final_reason, - Some(ManagedTargetEndReason::PeerClosed) - ); - } - assert_eq!(shared.peer_closed_target_count.load(Ordering::Acquire), 0); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - send_echo_to_target(&socket, &shared, fixture.target.clone(), Instant::now()); - assert_eq!( - fixture - .target - .lock() - .expect("target mutex poisoned") - .final_reason, - Some(ManagedTargetEndReason::PeerClosed) - ); - assert_eq!(shared.peer_closed_target_count.load(Ordering::Acquire), 0); - - let mut next_order = 1; - let mut records = TargetOutcomeHistory::default(); - apply_target_update( - &fixture.client_config, - &socket, - &shared, - Vec::new(), - &mut next_order, - &mut records, - ) - .unwrap(); - assert_eq!(shared.peer_closed_target_count.load(Ordering::Acquire), 1); - collect_finished_targets(&shared, &mut records); - assert_eq!(shared.peer_closed_target_count.load(Ordering::Acquire), 1); - - let events = drain_available_group_events(&fixture.subscription); - let terminal_sequence: Vec<&str> = events - .iter() - .filter_map(|event| match event { - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::EchoReply { .. }, - .. - }) => Some("reply"), - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::SessionClosed { .. }, - .. - }) => Some("closed"), - ManagedGroupEvent::TargetFinished(_) => Some("finished"), - _ => None, - }) - .collect(); - assert_eq!(terminal_sequence, ["reply", "closed", "finished"]); - - let outcome = records.into_group_outcome(ManagedGroupEndReason::AllTargetsComplete); - assert_eq!(outcome.total_target_outcomes, 1); - assert_eq!(outcome.successful_target_outcomes, 1); - assert_eq!(outcome.peer_closed_target_outcomes, 1); - assert_eq!(outcome.failed_target_outcomes, 0); - assert_eq!( - outcome.targets[0].end_reason, - ManagedTargetEndReason::PeerClosed - ); - assert_eq!(outcome.targets[0].packets_sent, 1); - assert_eq!(outcome.targets[0].replies_received, 1); - } - - #[test] - fn managed_close_success_commits_clears_schedule_and_publishes_once() { - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - peer.set_read_timeout(Some(Duration::from_millis(200))) - .unwrap(); - let fixture = active_target_fixture_at(peer.local_addr().unwrap()); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - let sent_at = ClientTimestamp { - wall: std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(4_567), - mono: Instant::now() + Duration::from_secs(8), - }; - { - let mut target = fixture.target.lock().expect("target mutex poisoned"); - target.close_sent_at = Some(sent_at); - close_locked_target( - &socket, - &fixture.hub, - &mut target, - ManagedTargetEndReason::TestComplete, - ); - assert!(!target.runtime.is_open()); - assert!(target.schedule.is_none()); - assert_eq!(target.close_send_attempts, 1); - assert_eq!( - target.final_reason, - Some(ManagedTargetEndReason::TestComplete) - ); - } - - let mut packet = [0_u8; 512]; - let (len, _) = peer.recv_from(&mut packet).unwrap(); - assert_eq!(packet[3], flags::FLAG_CLOSE); - assert_eq!(u64::from_le_bytes(packet[4..12].try_into().unwrap()), TOKEN); - assert_eq!(len, 12); - let events = drain_available_group_events(&fixture.subscription); - let closes: Vec<_> = events - .iter() - .filter_map(|event| match event { - ManagedGroupEvent::Client(TargetEvent { - event: - ClientEvent::SessionClosed { - token: TOKEN, at, .. - }, - .. - }) => Some(*at), - _ => None, - }) - .collect(); - assert_eq!(closes, [sent_at]); - } - - #[test] - fn managed_removal_and_cancellation_each_send_one_close() { - for reason in [ - ManagedTargetEndReason::Removed, - ManagedTargetEndReason::Cancelled, - ] { - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - peer.set_read_timeout(Some(Duration::from_millis(200))) - .unwrap(); - let fixture = active_target_fixture_at(peer.local_addr().unwrap()); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - { - let mut target = fixture.target.lock().expect("target mutex poisoned"); - close_locked_target(&socket, &fixture.hub, &mut target, reason.clone()); - assert_eq!(target.final_reason, Some(reason)); - assert_eq!(target.close_send_attempts, 1); - assert!(!target.runtime.is_open()); - assert!(target.schedule.is_none()); - } - - let mut packet = [0_u8; 512]; - let (len, _) = peer.recv_from(&mut packet).unwrap(); - assert_eq!(packet[3], flags::FLAG_CLOSE); - assert_eq!(len, 12); - assert!(matches!( - peer.recv_from(&mut packet), - Err(err) - if matches!( - err.kind(), - std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut - ) - )); - } - } - - #[test] - fn managed_close_send_failure_preserves_machine_and_requested_reason_is_not_recorded() { - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - peer.set_nonblocking(true).unwrap(); - let fixture = active_target_fixture_at(peer.local_addr().unwrap()); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - let sent_at = ClientTimestamp { - wall: std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(5_678), - mono: Instant::now() + Duration::from_secs(9), - }; - { - let mut target = fixture.target.lock().expect("target mutex poisoned"); - target.close_sent_at = Some(sent_at); - target.close_send_error = true; - close_locked_target( - &socket, - &fixture.hub, - &mut target, - ManagedTargetEndReason::Removed, - ); - assert!(target.runtime.is_open()); - assert!(target.schedule.is_some()); - assert_eq!(target.close_send_attempts, 1); - assert_eq!(target.close_sent_at, Some(sent_at)); - assert!(matches!( - target.final_reason, - Some(ManagedTargetEndReason::Failed(_)) - )); - } - - let mut packet = [0_u8; 512]; - assert!(matches!( - peer.recv_from(&mut packet), - Err(err) if err.kind() == std::io::ErrorKind::WouldBlock - )); - assert!(!drain_available_group_events(&fixture.subscription) - .iter() - .any(|event| matches!( - event, - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::SessionClosed { .. }, - .. - }) - ))); - } - - #[test] - fn managed_short_close_commits_and_clears_schedule_before_failure() { - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - peer.set_read_timeout(Some(Duration::from_millis(200))) - .unwrap(); - let fixture = active_target_fixture_at(peer.local_addr().unwrap()); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - let sent_at = ClientTimestamp { - wall: std::time::SystemTime::UNIX_EPOCH + Duration::from_secs(6_789), - mono: Instant::now() + Duration::from_secs(10), - }; - { - let mut target = fixture.target.lock().expect("target mutex poisoned"); - let expected = target.runtime.prepare_close().unwrap().bytes.len(); - target.close_sent_at = Some(sent_at); - target.close_reported_len = Some(expected - 1); - close_locked_target( - &socket, - &fixture.hub, - &mut target, - ManagedTargetEndReason::Cancelled, - ); - assert!(!target.runtime.is_open()); - assert!(target.schedule.is_none()); - assert_eq!(target.close_send_attempts, 1); - assert_eq!(target.close_sent_at, None); - let failure = target - .final_reason - .as_ref() - .and_then(ManagedTargetEndReason::failure) - .expect("short close must use managed failure policy"); - assert_eq!(failure.kind, ManagedTargetFailureKind::RuntimeProtocol); - assert!(failure.message.contains("UDP accepted")); - } - - let mut packet = [0_u8; 512]; - peer.recv_from(&mut packet).unwrap(); - assert!(!drain_available_group_events(&fixture.subscription) - .iter() - .any(|event| matches!( - event, - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::SessionClosed { .. }, - .. - }) - ))); - } - - #[test] - fn managed_close_reservation_failure_precedes_send() { - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - peer.set_nonblocking(true).unwrap(); - let fixture = active_target_fixture_at(peer.local_addr().unwrap()); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - { - let mut target = fixture.target.lock().expect("target mutex poisoned"); - target.close_event_reserve_error = true; - close_locked_target( - &socket, - &fixture.hub, - &mut target, - ManagedTargetEndReason::TestComplete, - ); - assert!(target.runtime.is_open()); - assert!(target.schedule.is_some()); - assert_eq!(target.close_send_attempts, 0); - assert!(matches!( - target.final_reason, - Some(ManagedTargetEndReason::Failed(_)) - )); - } - - let mut packet = [0_u8; 512]; - assert!(matches!( - peer.recv_from(&mut packet), - Err(err) if err.kind() == std::io::ErrorKind::WouldBlock - )); - } - - #[test] - fn managed_peer_close_affects_only_the_matching_target() { - let peer = active_target_fixture_at("127.0.0.1:2112".parse().unwrap()); - let other = active_target_fixture_at("127.0.0.1:2113".parse().unwrap()); - - { - let mut target = peer.target.lock().expect("target mutex poisoned"); - assert!(process_active_target_packet( - &peer.hub, - &mut target, - &peer.close_reply, - ClientTimestamp::now(), - ReceiveMeta::default(), - )); - assert!(target.runtime.is_peer_closed()); - assert!(target.schedule.is_none()); - assert_eq!( - target.final_reason, - Some(ManagedTargetEndReason::PeerClosed) - ); - } - - let target = other.target.lock().expect("target mutex poisoned"); - assert!(target.runtime.is_open()); - assert!(matches!(target.status, TargetStatus::Active)); - assert!(target.schedule.is_some()); - assert_eq!(target.final_reason, None); - } - - #[test] - fn managed_non_active_states_do_not_attempt_local_close() { - let remote: SocketAddr = "127.0.0.1:2112".parse().unwrap(); - let mut normal_config = group_config( - None, - Duration::from_millis(10), - ManagedGroupPacing::Staggered, - ) - .client; - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let hub = EventHub::new(); - let mut opening = - TargetState::new(&normal_config, target("opening", remote), 0, Instant::now()).unwrap(); - close_locked_target(&socket, &hub, &mut opening, ManagedTargetEndReason::Removed); - assert_eq!(opening.close_send_attempts, 0); - - normal_config.run_mode = crate::RunMode::NoTest; - let no_test_params = params_from_config(&normal_config).unwrap(); - let mut no_test = - TargetState::new(&normal_config, target("no-test", remote), 0, Instant::now()).unwrap(); - let reply = OpenReply { - flags: FLAG_OPEN | FLAG_REPLY | flags::FLAG_CLOSE, - token: 0, - params: no_test_params, - }; - let prepared = no_test - .runtime - .prepare_open_acceptance(reply, ClientTimestamp::now()) - .unwrap(); - no_test.runtime.commit_open(prepared); - close_locked_target( - &socket, - &hub, - &mut no_test, - ManagedTargetEndReason::Cancelled, - ); - assert_eq!(no_test.close_send_attempts, 0); - } - - #[test] - fn authoritative_status_counts_only_peer_closed_outcomes() { - let remote: SocketAddr = "127.0.0.1:2112".parse().unwrap(); - let config = group_config( - None, - Duration::from_millis(10), - ManagedGroupPacing::Staggered, - ) - .client; - let target = Arc::new(Mutex::new( - TargetState::new(&config, target("active", remote), 0, Instant::now()).unwrap(), - )); - let hub = EventHub::new(); - let shared = shared_with_target(&config, hub.clone(), target); - let mut records = TargetOutcomeHistory::default(); - - hub.publish(ManagedGroupEvent::Client(TargetEvent { - target: TargetId::from("active"), - event: ClientEvent::EchoLoss { - seq: 0, - sent_at: ClientTimestamp::now(), - timeout_at: Instant::now(), - }, - })); - assert_eq!(shared.peer_closed_target_count.load(Ordering::Acquire), 0); - - let failure = ManagedTargetFailure { - kind: ManagedTargetFailureKind::RuntimeProtocol, - message: "test failure".to_owned(), - }; - for (index, reason) in [ - ManagedTargetEndReason::TestComplete, - ManagedTargetEndReason::NoTestComplete, - ManagedTargetEndReason::Removed, - ManagedTargetEndReason::Cancelled, - ManagedTargetEndReason::OpenFailed(failure.clone()), - ManagedTargetEndReason::Failed(failure), - ] - .into_iter() - .enumerate() - { - record_target_outcome( - &shared, - &mut records, - target_outcome(&format!("non-peer-{index}"), remote, reason), - ); - assert_eq!(shared.peer_closed_target_count.load(Ordering::Acquire), 0); - } - - record_target_outcome( - &shared, - &mut records, - target_outcome("peer", remote, ManagedTargetEndReason::PeerClosed), - ); - assert_eq!(shared.peer_closed_target_count.load(Ordering::Acquire), 1); - - let outcome = records.into_group_outcome(ManagedGroupEndReason::AllTargetsComplete); - assert_eq!(outcome.total_target_outcomes, 7); - assert_eq!(outcome.peer_closed_target_outcomes, 1); - } - - #[test] - fn drop_oldest_can_evict_peer_close_event_without_status_loss() { - let interval = Duration::from_millis(10); - let config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - let peer_remote: SocketAddr = "127.0.0.1:2112".parse().unwrap(); - let healthy_remote: SocketAddr = "127.0.0.1:2113".parse().unwrap(); - let peer = Arc::new(Mutex::new( - TargetState::new(&config, target("peer", peer_remote), 0, Instant::now()).unwrap(), - )); - peer.lock() - .expect("target mutex poisoned") - .mark_finished(ManagedTargetEndReason::PeerClosed); - let healthy = Arc::new(Mutex::new( - TargetState::new( - &config, - target("healthy", healthy_remote), - 1, - Instant::now(), - ) - .unwrap(), - )); - healthy.lock().expect("target mutex poisoned").status = TargetStatus::Active; - - let hub = EventHub::new(); - let subscription = hub - .subscribe(SubscriberConfig { - capacity: 1, - overflow: SubscriberOverflow::DropOldest, - }) - .unwrap(); - let shared = shared_with_target(&config, hub.clone(), peer); - { - let mut registry = shared - .registry - .lock() - .expect("target registry mutex poisoned"); - registry.desired.insert(TargetId::from("healthy")); - registry - .remotes - .insert(healthy_remote, TargetId::from("healthy")); - registry - .targets - .insert(TargetId::from("healthy"), healthy.clone()); - } - - let mut records = TargetOutcomeHistory::default(); - collect_finished_targets(&shared, &mut records); - assert_eq!(shared.peer_closed_target_count.load(Ordering::Acquire), 1); - assert!(matches!( - healthy.lock().expect("target mutex poisoned").status, - TargetStatus::Active - )); - - hub.publish(ManagedGroupEvent::Client(TargetEvent { - target: TargetId::from("healthy"), - event: ClientEvent::EchoLoss { - seq: 7, - sent_at: ClientTimestamp::now(), - timeout_at: Instant::now(), - }, - })); - - assert_eq!(subscription.dropped_events(), 1); - assert!(matches!( - subscription.try_recv(), - Ok(Some(ManagedGroupEvent::Client(TargetEvent { - target, - event: ClientEvent::EchoLoss { seq: 7, .. }, - }))) if target.as_str() == "healthy" - )); - assert_eq!(subscription.try_recv(), Ok(None)); - assert_eq!(shared.peer_closed_target_count.load(Ordering::Acquire), 1); - assert_eq!(records.peer_closed, 1); - assert_eq!( - shared - .registry - .lock() - .expect("target registry mutex poisoned") - .targets - .len(), - 1 - ); - } - - #[test] - fn receiver_non_close_reply_leaves_target_active() { - let fixture = active_target_fixture(); - { - let mut target = fixture.target.lock().expect("target mutex poisoned"); - assert!(!process_active_target_packet( - &fixture.hub, - &mut target, - &fixture.reply, - ClientTimestamp::now(), - ReceiveMeta::default(), - )); - assert!(matches!(target.status, TargetStatus::Active)); - assert_eq!(target.final_reason, None); - assert_eq!(target.counters.replies_received, 1); - assert!(!target.runtime.is_peer_closed()); - } - - let events = drain_available_group_events(&fixture.subscription); - assert!(events.iter().any(|event| matches!( - event, - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::EchoReply { .. }, - .. - }) - ))); - assert!(!events.iter().any(|event| matches!( - event, - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::SessionClosed { .. }, - .. - }) | ManagedGroupEvent::TargetFinished(_) - ))); - } - - #[test] - fn peer_close_preserves_group_outcome_counters_and_provenance() { - let interval = Duration::from_millis(10); - let server = start_peer_close_server(test_params(None, interval)); - - let session = ManagedClientGroup::start( - group_config(None, interval, ManagedGroupPacing::Staggered), - vec![target("peer", server.addr)], - ) - .unwrap(); - - let outcome = session.join().unwrap(); - server.join(); - - assert_eq!( - outcome.end_reason, - ManagedGroupEndReason::AllTargetsComplete - ); - assert_eq!(outcome.targets.len(), 1); - let target = &outcome.targets[0]; - assert_eq!(target.end_reason, ManagedTargetEndReason::PeerClosed); - assert_eq!(target.packets_sent, 1); - assert_eq!(target.replies_received, 1); - assert_eq!(outcome.total_target_outcomes, 1); - assert_eq!(outcome.successful_target_outcomes, 1); - assert_eq!(outcome.peer_closed_target_outcomes, 1); - assert_eq!(outcome.failed_target_outcomes, 0); - } - - #[test] - fn peer_close_aggregate_survives_bounded_recent_history() { - let remote: SocketAddr = "127.0.0.1:2112".parse().unwrap(); - let mut history = TargetOutcomeHistory::default(); - history.record(ManagedTargetOutcome { - id: TargetId::from("peer"), - remote, - end_reason: ManagedTargetEndReason::PeerClosed, - packets_sent: 1, - replies_received: 1, - duplicates: 0, - late: 0, - warning_events: 0, - }); - for index in 0..MANAGED_GROUP_OUTCOME_HISTORY_LIMIT { - history.record(ManagedTargetOutcome { - id: TargetId::from(format!("removed-{index}")), - remote, - end_reason: ManagedTargetEndReason::Removed, - packets_sent: 0, - replies_received: 0, - duplicates: 0, - late: 0, - warning_events: 0, - }); - } - - let outcome = history.into_group_outcome(ManagedGroupEndReason::AllTargetsComplete); - let expected_total = u64::try_from(MANAGED_GROUP_OUTCOME_HISTORY_LIMIT + 1).unwrap(); - assert_eq!(outcome.total_target_outcomes, expected_total); - assert_eq!(outcome.successful_target_outcomes, 1); - assert_eq!(outcome.peer_closed_target_outcomes, 1); - assert_eq!(outcome.failed_target_outcomes, 0); - assert_eq!(outcome.targets.len(), MANAGED_GROUP_OUTCOME_HISTORY_LIMIT); - assert_eq!(outcome.discarded_target_outcomes, 1); - assert!(outcome - .targets - .iter() - .all(|target| target.end_reason == ManagedTargetEndReason::Removed)); - } - - #[test] - fn mixed_peer_close_and_open_failure_publish_terminal_outcomes_and_disconnect() { - let interval = Duration::from_millis(10); - let params = test_params(None, interval); - let success = start_peer_close_server(params.clone()); - let failure = start_open_failure_server(params); - let (session, sub) = ManagedClientGroup::start_with_subscription( - group_config(None, interval, ManagedGroupPacing::Staggered), - vec![ - target("success", success.addr), - target("failure", failure.addr), - ], - SubscriberConfig { - capacity: 64, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - let events = collect_group_events_until_disconnected(&sub); - let outcome = session.join().unwrap(); - success.join(); - failure.join(); - - assert!(events.iter().any(|event| matches!( - event, - ManagedGroupEvent::Client(TargetEvent { - target, - event: ClientEvent::EchoReply { .. }, - }) if target.as_str() == "success" - ))); - assert!(events.iter().any(|event| matches!( - event, - ManagedGroupEvent::TargetFinished(target) - if target.id.as_str() == "success" - && target.end_reason == ManagedTargetEndReason::PeerClosed - ))); - assert!(events.iter().any(|event| matches!( - event, - ManagedGroupEvent::TargetFinished(target) - if target.id.as_str() == "failure" - && matches!( - &target.end_reason, - ManagedTargetEndReason::OpenFailed(failure) - if failure.kind == ManagedTargetFailureKind::OpeningProtocol - ) - ))); - assert_eq!(outcome.targets.len(), 2); - assert_eq!(outcome.total_target_outcomes, 2); - assert_eq!(outcome.successful_target_outcomes, 1); - assert_eq!(outcome.peer_closed_target_outcomes, 1); - assert_eq!(outcome.failed_target_outcomes, 1); - } - - #[test] - fn all_open_failures_publish_terminal_outcomes_and_disconnect() { - let interval = Duration::from_millis(10); - let params = test_params(None, interval); - let a = start_open_failure_server(params.clone()); - let b = start_open_failure_server(params); - let (session, sub) = ManagedClientGroup::start_with_subscription( - group_config(None, interval, ManagedGroupPacing::Burst), - vec![target("a", a.addr), target("b", b.addr)], - SubscriberConfig { - capacity: 16, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - let events = collect_group_events_until_disconnected(&sub); - let outcome = session.join().unwrap(); - a.join(); - b.join(); - - let failures = events - .iter() - .filter(|event| { - matches!( - event, - ManagedGroupEvent::TargetFinished(ManagedTargetOutcome { - end_reason: ManagedTargetEndReason::OpenFailed(_), - .. - }) - ) - }) - .count(); - assert_eq!(failures, 2); - assert_eq!(outcome.targets.len(), 2); - assert!(outcome - .targets - .iter() - .all(|target| target.end_reason.failure().is_some())); - } - - #[test] - fn bad_open_packet_followed_by_valid_reply_succeeds_without_retry() { - let interval = Duration::from_millis(10); - let params = test_params(None, interval); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let addr = socket.local_addr().unwrap(); - let (tx, rx) = mpsc::channel(); - let done = thread::spawn(move || { - let (open, peer) = recv_request_timeout(&socket).expect("missing open request"); - assert_ne!(open[3] & FLAG_OPEN, 0); - tx.send(ServerObservation::Open { at: Instant::now() }) - .unwrap(); - socket.send_to(&[0_u8], peer).unwrap(); - socket - .send_to( - &open_reply(FLAG_OPEN | FLAG_REPLY | flags::FLAG_CLOSE, 0, ¶ms), - peer, - ) - .unwrap(); - }); - let server = FakeServer { - addr, - _observations: rx, - done, - }; - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered); - config.client.run_mode = crate::RunMode::NoTest; - - let outcome = ManagedClientGroup::start(config, vec![target("peer", server.addr)]) - .unwrap() - .join() - .unwrap(); - - assert_eq!(outcome.successful_target_outcomes, 1); - assert_eq!(server._observations.try_iter().collect::>().len(), 1); - server.join(); - } - - #[test] - fn bad_hmac_open_packet_followed_by_valid_reply_succeeds_without_retry() { - let interval = Duration::from_millis(10); - let params = test_params(None, interval); - let key = b"group-secret".to_vec(); - let wrong_key = b"wrong".to_vec(); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let addr = socket.local_addr().unwrap(); - let (tx, rx) = mpsc::channel(); - let server_key = key.clone(); - let done = thread::spawn(move || { - let (open, peer) = recv_request_timeout(&socket).expect("missing open request"); - assert_ne!(open[3] & FLAG_OPEN, 0); - tx.send(ServerObservation::Open { at: Instant::now() }) - .unwrap(); - socket - .send_to( - &hmac_open_reply( - FLAG_OPEN | FLAG_REPLY | flags::FLAG_CLOSE, - 0, - ¶ms, - &wrong_key, - ), - peer, - ) - .unwrap(); - socket - .send_to( - &hmac_open_reply( - FLAG_OPEN | FLAG_REPLY | flags::FLAG_CLOSE, - 0, - ¶ms, - &server_key, - ), - peer, - ) - .unwrap(); - }); - let server = FakeServer { - addr, - _observations: rx, - done, - }; - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered); - config.client.run_mode = crate::RunMode::NoTest; - config.client.hmac_key = Some(key); - - let outcome = ManagedClientGroup::start(config, vec![target("peer", server.addr)]) - .unwrap() - .join() - .unwrap(); - - assert_eq!(outcome.successful_target_outcomes, 1); - assert_eq!(server._observations.try_iter().collect::>().len(), 1); - server.join(); - } - - #[test] - fn trusted_group_incompatibility_sends_cleanup_and_fails_without_retry() { - let interval = Duration::from_millis(10); - let mut returned = test_params(None, interval); - returned.interval_ns = duration_ns_i64(Duration::from_millis(20)); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - socket - .set_read_timeout(Some(Duration::from_secs(1))) - .unwrap(); - let addr = socket.local_addr().unwrap(); - let (tx, rx) = mpsc::channel(); - let done = thread::spawn(move || { - let (open, peer) = recv_request_timeout(&socket).expect("missing open request"); - assert_ne!(open[3] & FLAG_OPEN, 0); - tx.send(ServerObservation::Open { at: Instant::now() }) - .unwrap(); - socket - .send_to(&open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, &returned), peer) - .unwrap(); - let (cleanup, _) = - recv_request_timeout(&socket).expect("missing post-token cleanup close"); - assert_eq!(cleanup[3], flags::FLAG_CLOSE); - assert_eq!( - u64::from_le_bytes(cleanup[4..12].try_into().unwrap()), - TOKEN - ); - tx.send(ServerObservation::Close { at: Instant::now() }) - .unwrap(); - }); - let server = FakeServer { - addr, - _observations: rx, - done, - }; - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered); - config.client.negotiation_policy = NegotiationPolicy::Loose; - - let outcome = ManagedClientGroup::start(config, vec![target("peer", server.addr)]) - .unwrap() - .join() - .unwrap(); - - assert_eq!(outcome.failed_target_outcomes, 1); - assert!(matches!( - outcome.targets[0].end_reason, - ManagedTargetEndReason::OpenFailed(ManagedTargetFailure { - kind: ManagedTargetFailureKind::OpeningProtocol, - .. - }) - )); - assert_eq!(server._observations.try_iter().collect::>().len(), 2); - server.join(); - } - - #[test] - fn no_test_non_close_reply_sends_cleanup_and_fails_without_retry() { - let interval = Duration::from_millis(10); - let params = test_params(None, interval); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - socket - .set_read_timeout(Some(Duration::from_secs(1))) - .unwrap(); - let addr = socket.local_addr().unwrap(); - let (tx, rx) = mpsc::channel(); - let done = thread::spawn(move || { - let (open, peer) = recv_request_timeout(&socket).expect("missing open request"); - assert_ne!(open[3] & flags::FLAG_CLOSE, 0); - tx.send(ServerObservation::Open { at: Instant::now() }) - .unwrap(); - socket - .send_to(&open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms), peer) - .unwrap(); - let (cleanup, _) = - recv_request_timeout(&socket).expect("missing no-test cleanup close"); - assert_eq!(cleanup[3], flags::FLAG_CLOSE); - assert_eq!( - u64::from_le_bytes(cleanup[4..12].try_into().unwrap()), - TOKEN - ); - tx.send(ServerObservation::Close { at: Instant::now() }) - .unwrap(); - }); - let server = FakeServer { - addr, - _observations: rx, - done, - }; - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered); - config.client.run_mode = crate::RunMode::NoTest; - - let outcome = ManagedClientGroup::start(config, vec![target("peer", server.addr)]) - .unwrap() - .join() - .unwrap(); - - assert_eq!(outcome.failed_target_outcomes, 1); - assert!(matches!( - outcome.targets[0].end_reason, - ManagedTargetEndReason::OpenFailed(ManagedTargetFailure { - kind: ManagedTargetFailureKind::OpeningProtocol, - .. - }) - )); - assert_eq!(server._observations.try_iter().collect::>().len(), 2); - server.join(); - } - - #[test] - fn loose_no_test_group_skips_active_interval_and_dscp_policy() { - let interval = Duration::from_millis(10); - let mut changed_interval = test_params(None, interval); - changed_interval.interval_ns += 1; - let changed_dscp = test_params(None, interval); - - for (requested_dscp, params) in [(0, changed_interval), (1, changed_dscp)] { - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let remote = peer.local_addr().unwrap(); - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - config.run_mode = crate::RunMode::NoTest; - config.negotiation_policy = NegotiationPolicy::Loose; - config.dscp = requested_dscp; - let target = Arc::new(Mutex::new( - TargetState::new(&config, target("peer", remote), 0, Instant::now()).unwrap(), - )); - let shared = shared_with_target(&config, EventHub::new(), target.clone()); - let opened_at = ClientTimestamp::now(); - - let mut target = target.lock().expect("target mutex poisoned"); - let packet = open_reply(FLAG_OPEN | FLAG_REPLY | flags::FLAG_CLOSE, 0, ¶ms); - let reply = match target.runtime.inspect_open_datagram(&packet).unwrap() { - OpenDatagramDisposition::Trusted(reply) => reply, - OpenDatagramDisposition::Ignore => panic!("no-test reply must be trusted"), - }; - let machine = target - .runtime - .prepare_open_acceptance(reply, opened_at) - .unwrap(); - let prepared = prepare_group_open(&shared, machine, opened_at).unwrap(); - assert!(prepared.schedule.is_none()); - let outcome = target.runtime.commit_open(prepared.machine); - target.schedule = prepared.schedule; - publish_open_outcome(&shared.hub, &mut target, outcome); - - assert!(matches!(target.status, TargetStatus::Finished)); - assert_eq!( - target.final_reason, - Some(ManagedTargetEndReason::NoTestComplete) - ); - } - } - - #[test] - fn normal_and_strict_sessions_preserve_group_negotiation_checks() { - let interval = Duration::from_millis(10); - let mut changed_interval = test_params(None, interval); - changed_interval.interval_ns += 1; - let changed_dscp = test_params(None, interval); - - for (requested_dscp, params) in [(0, changed_interval.clone()), (1, changed_dscp)] { - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let remote = peer.local_addr().unwrap(); - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - config.negotiation_policy = NegotiationPolicy::Loose; - config.dscp = requested_dscp; - let target = Arc::new(Mutex::new( - TargetState::new(&config, target("peer", remote), 0, Instant::now()).unwrap(), - )); - let shared = shared_with_target(&config, EventHub::new(), target.clone()); - let opened_at = ClientTimestamp::now(); - let target = target.lock().expect("target mutex poisoned"); - let packet = open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms); - let reply = match target.runtime.inspect_open_datagram(&packet).unwrap() { - OpenDatagramDisposition::Trusted(reply) => reply, - OpenDatagramDisposition::Ignore => panic!("normal reply must be trusted"), - }; - let machine = target - .runtime - .prepare_open_acceptance(reply, opened_at) - .unwrap(); - let failure = prepare_group_open(&shared, machine, opened_at).unwrap_err(); - assert!(matches!( - failure.primary, - ClientError::NegotiationRejected { .. } - )); - } - - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let remote = peer.local_addr().unwrap(); - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - config.run_mode = crate::RunMode::NoTest; - let target = TargetState::new(&config, target("peer", remote), 0, Instant::now()).unwrap(); - let packet = open_reply( - FLAG_OPEN | FLAG_REPLY | flags::FLAG_CLOSE, - 0, - &changed_interval, - ); - let reply = match target.runtime.inspect_open_datagram(&packet).unwrap() { - OpenDatagramDisposition::Trusted(reply) => reply, - OpenDatagramDisposition::Ignore => panic!("strict no-test reply must be trusted"), - }; - let failure = target - .runtime - .prepare_open_acceptance(reply, ClientTimestamp::now()) - .unwrap_err(); - assert!(matches!( - failure.primary, - ClientError::NegotiationRejected { .. } - )); - } - - #[test] - fn opening_duration_overflow_is_invalid_configuration_only_while_opening() { - assert_eq!( - classify_target_failure(&ClientError::DurationOverflow, true), - ManagedTargetFailureKind::InvalidConfiguration - ); - assert_eq!( - classify_target_failure(&ClientError::DurationOverflow, false), - ManagedTargetFailureKind::RuntimeProtocol - ); - } - - #[test] - fn group_open_deadline_overflow_fails_before_send() { - let silent_peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let remote = silent_peer.local_addr().unwrap(); - let mut config = group_config( - None, - Duration::from_millis(10), - ManagedGroupPacing::Staggered, - ); - config.client.open_timeouts = vec![Duration::MAX]; - - let (session, sub) = ManagedClientGroup::start_with_subscription( - config, - vec![target("peer", remote)], - SubscriberConfig { - capacity: 8, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - let events = collect_group_events_until_disconnected(&sub); - let outcome = session.join().unwrap(); - - assert_eq!(outcome.total_target_outcomes, 1); - assert_eq!(outcome.failed_target_outcomes, 1); - assert_eq!(outcome.targets.len(), 1); - assert!(matches!( - outcome.targets[0].end_reason, - ManagedTargetEndReason::OpenFailed(ManagedTargetFailure { - kind: ManagedTargetFailureKind::InvalidConfiguration, - .. - }) - )); - let failure = outcome.targets[0].end_reason.failure().unwrap(); - assert_eq!(failure.kind, ManagedTargetFailureKind::InvalidConfiguration); - assert_ne!(failure.kind, ManagedTargetFailureKind::OpeningProtocol); - assert!(failure.message.contains("duration")); - let finished: Vec<_> = events - .iter() - .filter_map(|event| match event { - ManagedGroupEvent::TargetFinished(outcome) => Some(outcome), - ManagedGroupEvent::Client(_) => None, - }) - .collect(); - assert_eq!(finished.len(), 1); - assert_eq!(finished[0], &outcome.targets[0]); - silent_peer.set_nonblocking(true).unwrap(); - let mut packet = [0_u8; 512]; - assert!(matches!( - silent_peer.recv_from(&mut packet), - Err(err) if err.kind() == std::io::ErrorKind::WouldBlock - )); - } - - #[test] - fn receiver_drain_completion_is_monotonic_and_wakes_waiting_scheduler() { - let interval = Duration::from_millis(10); - let config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let remote = peer.local_addr().unwrap(); - let now = Instant::now(); - let target = Arc::new(Mutex::new( - TargetState::new(&config, target("peer", remote), 0, now).unwrap(), - )); - let (shared, control_rx) = shared_with_targets(&config, EventHub::new(), vec![target]); - let shared = Arc::new(shared); - let generation = request_receiver_drain(&shared, now).unwrap(); - let waiter_shared = shared.clone(); - let waiter = thread::spawn(move || { - wait_for_scheduler_control(&control_rx, &waiter_shared.cancellation, None) - }); - - complete_receiver_drain(&shared, generation); - - assert!(matches!(waiter.join().unwrap(), Some(ControlMessage::Wake))); - assert_eq!(completed_receiver_drain(&shared), generation); - complete_receiver_drain(&shared, 0); - complete_receiver_drain(&shared, generation); - assert_eq!(completed_receiver_drain(&shared), generation); - } - - #[test] - fn receiver_drain_request_join_and_natural_completion_share_one_deadline() { - let requested_at = Instant::now(); - let mut drain = ReceiverDrainState::default(); - - let generation = drain.request_or_join(requested_at).unwrap(); - let deadline = drain.deadline().expect("new drain needs a deadline"); - assert_eq!(generation, 1); - assert_eq!(deadline, requested_at + GROUP_RECEIVER_DRAIN_LIMIT); - assert!(drain.is_outstanding()); - - assert_eq!( - drain - .request_or_join(requested_at + Duration::from_millis(10)) - .unwrap(), - generation - ); - assert_eq!(drain.deadline(), Some(deadline)); - assert!(drain.complete_observed(generation)); - assert_eq!(drain.completed, generation); - assert_eq!(drain.deadline(), None); - assert!(!drain.is_outstanding()); - } - - #[test] - fn receiver_drain_forced_completion_is_bounded_and_idempotent() { - let requested_at = Instant::now(); - let mut drain = ReceiverDrainState::default(); - let generation = drain.request_or_join(requested_at).unwrap(); - let deadline = drain.deadline().unwrap(); - - assert!(!drain.complete_if_expired(deadline - Duration::from_nanos(1))); - assert_eq!(drain.completed, 0); - assert_eq!(drain.deadline(), Some(deadline)); - - assert!(drain.complete_if_expired(deadline)); - assert_eq!(drain.completed, generation); - assert_eq!(drain.deadline(), None); - assert!(!drain.complete_if_expired(deadline)); - assert!(!drain.complete_observed(generation)); - - let later_request = deadline + Duration::from_millis(10); - let next_generation = drain.request_or_join(later_request).unwrap(); - let next_deadline = drain.deadline().unwrap(); - assert_eq!(next_generation, generation + 1); - assert_eq!(next_deadline, later_request + GROUP_RECEIVER_DRAIN_LIMIT); - assert!(drain.complete_if_expired(next_deadline + Duration::from_nanos(1))); - assert_eq!(drain.completed, next_generation); - } - - #[test] - fn stale_receiver_completion_cannot_clear_a_newer_generation_deadline() { - let requested_at = Instant::now(); - let mut drain = ReceiverDrainState::default(); - let first = drain.request_or_join(requested_at).unwrap(); - assert!(drain.complete_if_expired(requested_at + GROUP_RECEIVER_DRAIN_LIMIT)); - - let second_requested_at = requested_at + GROUP_RECEIVER_DRAIN_LIMIT; - let second = drain.request_or_join(second_requested_at).unwrap(); - let second_deadline = drain.deadline().unwrap(); - assert_eq!(second, first + 1); - - assert!(!drain.complete_observed(first)); - assert_eq!(drain.completed, first); - assert_eq!(drain.requested, second); - assert_eq!(drain.deadline(), Some(second_deadline)); - assert!(drain.is_outstanding()); - } - - #[test] - fn receiver_drain_generation_overflow_does_not_mutate_state() { - let mut drain = ReceiverDrainState { - requested: u64::MAX, - completed: u64::MAX, - deadline: None, - }; - let before = drain; - - assert!(matches!( - drain.request_or_join(Instant::now()), - Err(ClientError::CounterOverflow { - counter: "receiver_drain_generation" - }) - )); - assert_eq!(drain.requested, before.requested); - assert_eq!(drain.completed, before.completed); - assert_eq!(drain.deadline(), before.deadline()); - } - - #[test] - fn expired_attempt_waits_for_receiver_drain_before_retrying() { - let interval = Duration::from_millis(10); - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - config.open_timeouts = vec![Duration::from_millis(200), Duration::from_millis(200)]; - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - peer.set_read_timeout(Some(Duration::from_millis(100))) - .unwrap(); - let remote = peer.local_addr().unwrap(); - let now = Instant::now(); - let target = Arc::new(Mutex::new( - TargetState::new(&config, target("peer", remote), 0, now).unwrap(), - )); - let shared = shared_with_target(&config, EventHub::new(), target.clone()); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - drive_open_attempts(&config, &socket, &shared, now); - let deadline = { - let target = target.lock().expect("target mutex poisoned"); - match target.status { - TargetStatus::Opening { - attempt: 1, - next_send_at, - .. - } => next_send_at, - ref status => panic!("unexpected target status after open send: {status:?}"), - } - }; - assert!(recv_request_timeout(&peer).is_some()); - - drive_open_attempts(&config, &socket, &shared, deadline); - let generation = match target.lock().expect("target mutex poisoned").status { - TargetStatus::Opening { - attempt: 1, - awaiting_receiver_generation: Some(generation), - .. - } => generation, - ref status => panic!("unexpected waiting status: {status:?}"), - }; - assert!(recv_request_timeout(&peer).is_none()); - - let drain_deadline = receiver_drain_snapshot(&shared).deadline().unwrap(); - assert!(!complete_expired_receiver_drain( - &shared, - drain_deadline - Duration::from_nanos(1) - )); - drive_open_attempts( - &config, - &socket, - &shared, - drain_deadline - Duration::from_nanos(1), - ); - assert!(target.lock().unwrap().final_reason.is_none()); - - assert!(complete_expired_receiver_drain(&shared, drain_deadline)); - assert_eq!(completed_receiver_drain(&shared), generation); - drive_open_attempts(&config, &socket, &shared, drain_deadline); - assert!(matches!( - target.lock().expect("target mutex poisoned").status, - TargetStatus::Opening { - attempt: 2, - awaiting_receiver_generation: None, - .. - } - )); - assert!(recv_request_timeout(&peer).is_some()); - } - - #[test] - fn continuous_traffic_receiver_drain_forces_progress_and_resumes_pacing() { - let interval = Duration::from_millis(10); - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered); - config.client.open_timeouts = vec![Duration::from_millis(200), Duration::from_millis(200)]; - let params = test_params(None, interval); - let active_peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let opening_peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - for peer in [&active_peer, &opening_peer] { - peer.set_read_timeout(Some(Duration::from_millis(100))) - .unwrap(); - } - let started_at = Instant::now(); - let active = Arc::new(Mutex::new(active_probe_target( - &config.client, - ¶ms, - active_peer.local_addr().unwrap(), - ClientTimestamp { - wall: std::time::SystemTime::now(), - mono: started_at, - }, - ))); - let opening = Arc::new(Mutex::new( - TargetState::new( - &config.client, - target("opening", opening_peer.local_addr().unwrap()), - 1, - started_at, - ) - .unwrap(), - )); - let (shared, _control_rx) = shared_with_targets( - &config.client, - EventHub::new(), - vec![active.clone(), opening.clone()], - ); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - drive_open_attempts(&config.client, &socket, &shared, started_at); - assert!(recv_request_timeout(&opening_peer).is_some()); - let opening_deadline = match opening.lock().unwrap().status { - TargetStatus::Opening { next_send_at, .. } => next_send_at, - ref status => panic!("unexpected opening status: {status:?}"), - }; - - drive_open_attempts(&config.client, &socket, &shared, opening_deadline); - let drain = receiver_drain_snapshot(&shared); - let drain_deadline = drain.deadline().expect("expired attempt needs a drain"); - assert!(drain.is_outstanding()); - - let active_targets = active_targets(&shared); - let mut pacing = PacingRuntime::new(config.pacing); - pacing.reconcile(&active_targets, opening_deadline); - assert!(!send_due_active_probes( - &config, - &socket, - &shared, - &mut pacing, - &active_targets, - opening_deadline, - true, - ) - .unwrap()); - assert!(recv_request_timeout(&active_peer).is_none()); - - let unrelated = UdpSocket::bind("127.0.0.1:0").unwrap(); - for _ in 0..3 { - unrelated - .send_to(&[0_u8], socket.local_addr().unwrap()) - .unwrap(); - let (datagram, packet) = recv_group_datagram(&socket); - process_group_datagram(&socket, &shared, datagram, &packet); - assert_eq!(completed_receiver_drain(&shared), drain.completed); - } - - assert!(!complete_expired_receiver_drain( - &shared, - drain_deadline - Duration::from_nanos(1) - )); - drive_open_attempts( - &config.client, - &socket, - &shared, - drain_deadline - Duration::from_nanos(1), - ); - assert!(recv_request_timeout(&opening_peer).is_none()); - - assert!(complete_expired_receiver_drain(&shared, drain_deadline)); - drive_open_attempts(&config.client, &socket, &shared, drain_deadline); - assert!(recv_request_timeout(&opening_peer).is_some()); - drive_open_attempts(&config.client, &socket, &shared, drain_deadline); - assert!(recv_request_timeout(&opening_peer).is_none()); - - assert!(send_due_active_probes( - &config, - &socket, - &shared, - &mut pacing, - &active_targets, - drain_deadline, - receiver_drain_snapshot(&shared).is_outstanding(), - ) - .unwrap()); - assert!(recv_request_timeout(&active_peer).is_some()); - assert!(!send_due_active_probes( - &config, - &socket, - &shared, - &mut pacing, - &active_targets, - drain_deadline, - false, - ) - .unwrap()); - assert!(recv_request_timeout(&active_peer).is_none()); - assert_eq!(active.lock().unwrap().runtime.packets_sent(), 1); - assert!(pacing - .next_wakeup() - .is_some_and(|deadline| deadline > drain_deadline)); - } - - #[test] - fn active_reply_and_timeout_processing_continue_during_receiver_drain() { - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let fixture = active_target_fixture_at(peer.local_addr().unwrap()); - let shared = shared_with_target( - &fixture.client_config, - fixture.hub.clone(), - fixture.target.clone(), - ); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - request_receiver_drain(&shared, Instant::now()).unwrap(); - - peer.send_to(&fixture.close_reply, socket.local_addr().unwrap()) - .unwrap(); - let (datagram, packet) = recv_group_datagram(&socket); - process_group_datagram(&socket, &shared, datagram, &packet); - - assert!(receiver_drain_snapshot(&shared).is_outstanding()); - assert_eq!( - fixture.target.lock().unwrap().final_reason, - Some(ManagedTargetEndReason::PeerClosed) - ); - assert!(drain_available_group_events(&fixture.subscription) - .iter() - .any(|event| matches!( - event, - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::EchoReply { seq: 0, .. }, - .. - }) - ))); - - let fixture = active_target_fixture(); - let shared = - shared_with_target(&fixture.client_config, fixture.hub.clone(), fixture.target); - let now = Instant::now(); - request_receiver_drain(&shared, now).unwrap(); - poll_active_timeouts( - &shared, - now + fixture.client_config.probe_timeout + Duration::from_millis(10), - ); - - assert!(receiver_drain_snapshot(&shared).is_outstanding()); - assert!(drain_available_group_events(&fixture.subscription) - .iter() - .any(|event| matches!( - event, - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::EchoLoss { seq: 0, .. }, - .. - }) - ))); - } - - #[test] - fn queued_reply_dequeued_after_deadline_opens_without_retry() { - let interval = Duration::from_millis(10); - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - config.open_timeouts = vec![Duration::from_millis(200)]; - let params = test_params(None, interval); - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - peer.set_read_timeout(Some(Duration::from_millis(100))) - .unwrap(); - let remote = peer.local_addr().unwrap(); - let now = Instant::now(); - let target = Arc::new(Mutex::new( - TargetState::new(&config, target("peer", remote), 0, now).unwrap(), - )); - let shared = shared_with_target(&config, EventHub::new(), target.clone()); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - drive_open_attempts(&config, &socket, &shared, now); - let deadline = { - let target = target.lock().expect("target mutex poisoned"); - match target.status { - TargetStatus::Opening { next_send_at, .. } => next_send_at, - ref status => panic!("unexpected opening status: {status:?}"), - } - }; - let (_, client_addr) = recv_request_timeout(&peer).expect("missing first open request"); - peer.send_to( - &open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms), - client_addr, - ) - .unwrap(); - assert!( - Instant::now() < deadline, - "reply must queue before deadline" - ); - while Instant::now() < deadline { - thread::yield_now(); - } - - drive_open_attempts(&config, &socket, &shared, deadline); - assert!(matches!( - target.lock().expect("target mutex poisoned").status, - TargetStatus::Opening { - awaiting_receiver_generation: Some(_), - .. - } - )); - - let (datagram, packet) = recv_group_datagram(&socket); - assert!(datagram.received_at.mono >= deadline); - process_group_datagram(&socket, &shared, datagram, &packet); - - let target = target.lock().expect("target mutex poisoned"); - assert!(matches!(target.status, TargetStatus::Active)); - assert!(target.final_reason.is_none()); - drop(target); - assert!(recv_request_timeout(&peer).is_none()); - } - - #[test] - fn generation_requested_after_receive_snapshot_needs_later_empty_boundary() { - let interval = Duration::from_millis(10); - let config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let remote = peer.local_addr().unwrap(); - let target = Arc::new(Mutex::new( - TargetState::new(&config, target("peer", remote), 0, Instant::now()).unwrap(), - )); - let shared = shared_with_target(&config, EventHub::new(), target); - - let observed_before_request = requested_receiver_drain(&shared); - let generation = request_receiver_drain(&shared, Instant::now()).unwrap(); - complete_receiver_drain(&shared, observed_before_request); - assert!(completed_receiver_drain(&shared) < generation); - - let observed_after_request = requested_receiver_drain(&shared); - complete_receiver_drain(&shared, observed_after_request); - assert_eq!(completed_receiver_drain(&shared), generation); - } - - #[test] - fn malformed_packet_requires_empty_receive_to_complete_drain() { - let interval = Duration::from_millis(10); - let config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - peer.set_read_timeout(Some(Duration::from_millis(100))) - .unwrap(); - let remote = peer.local_addr().unwrap(); - let now = Instant::now(); - let target = Arc::new(Mutex::new( - TargetState::new(&config, target("peer", remote), 0, now).unwrap(), - )); - let (shared, control_rx) = - shared_with_targets(&config, EventHub::new(), vec![target.clone()]); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - drive_open_attempts(&config, &socket, &shared, now); - let deadline = match target.lock().expect("target mutex poisoned").status { - TargetStatus::Opening { next_send_at, .. } => next_send_at, - ref status => panic!("unexpected opening status: {status:?}"), - }; - let (_, client_addr) = recv_request_timeout(&peer).expect("missing open request"); - drive_open_attempts(&config, &socket, &shared, deadline); - let generation = match target.lock().expect("target mutex poisoned").status { - TargetStatus::Opening { - awaiting_receiver_generation: Some(generation), - .. - } => generation, - ref status => panic!("unexpected waiting status: {status:?}"), - }; - - peer.send_to(&[0_u8], client_addr).unwrap(); - let (datagram, packet) = recv_group_datagram(&socket); - process_group_datagram(&socket, &shared, datagram, &packet); - assert!(completed_receiver_drain(&shared) < generation); - assert!(matches!( - control_rx.try_recv(), - Err(mpsc::TryRecvError::Empty) - )); - - let observed_generation = requested_receiver_drain(&shared); - socket.set_nonblocking(true).unwrap(); - let mut buf = [0_u8; 1]; - assert!(matches!( - recv_datagram_from(&socket, &mut buf), - Err(err) if err.kind() == std::io::ErrorKind::WouldBlock - )); - complete_receiver_drain(&shared, observed_generation); - assert_eq!(completed_receiver_drain(&shared), generation); - assert!(matches!(control_rx.recv().unwrap(), ControlMessage::Wake)); - } - - #[test] - fn opening_reply_before_first_request_is_ignored() { - let interval = Duration::from_millis(10); - let config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - let params = test_params(None, interval); - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let remote = peer.local_addr().unwrap(); - let target = Arc::new(Mutex::new( - TargetState::new(&config, target("peer", remote), 0, Instant::now()).unwrap(), - )); - let shared = shared_with_target(&config, EventHub::new(), target.clone()); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - peer.send_to( - &open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms), - socket.local_addr().unwrap(), - ) - .unwrap(); - let (datagram, packet) = recv_group_datagram(&socket); - process_group_datagram(&socket, &shared, datagram, &packet); - - let target = target.lock().expect("target mutex poisoned"); - assert!(matches!( - target.status, - TargetStatus::Opening { - attempt: 0, - awaiting_receiver_generation: None, - .. - } - )); - assert!(target.runtime.prepare_open_request().is_ok()); - } - - #[test] - fn force_completed_drain_rejects_packet_dequeued_before_completion() { - let interval = Duration::from_millis(10); - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - config.open_timeouts = vec![Duration::from_millis(200), Duration::from_millis(200)]; - let params = test_params(None, interval); - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - peer.set_read_timeout(Some(Duration::from_millis(100))) - .unwrap(); - let remote = peer.local_addr().unwrap(); - let now = Instant::now(); - let target = Arc::new(Mutex::new( - TargetState::new(&config, target("peer", remote), 0, now).unwrap(), - )); - let shared = shared_with_target(&config, EventHub::new(), target.clone()); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - drive_open_attempts(&config, &socket, &shared, now); - let deadline = match target.lock().expect("target mutex poisoned").status { - TargetStatus::Opening { next_send_at, .. } => next_send_at, - ref status => panic!("unexpected opening status: {status:?}"), - }; - let (_, client_addr) = recv_request_timeout(&peer).expect("missing open request"); - drive_open_attempts(&config, &socket, &shared, deadline); - let generation = match target.lock().expect("target mutex poisoned").status { - TargetStatus::Opening { - awaiting_receiver_generation: Some(generation), - .. - } => generation, - ref status => panic!("unexpected waiting status: {status:?}"), - }; - peer.send_to( - &open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms), - client_addr, - ) - .unwrap(); - let (datagram, packet) = recv_group_datagram(&socket); - let drain_deadline = receiver_drain_snapshot(&shared).deadline().unwrap(); - assert!(complete_expired_receiver_drain(&shared, drain_deadline)); - assert_eq!(completed_receiver_drain(&shared), generation); - process_group_datagram(&socket, &shared, datagram, &packet); - { - let target = target.lock().expect("target mutex poisoned"); - assert!(matches!( - target.status, - TargetStatus::Opening { - attempt: 1, - awaiting_receiver_generation: Some(waiting), - .. - } if waiting == generation - )); - assert!(target.runtime.prepare_open_request().is_ok()); - } - - drive_open_attempts(&config, &socket, &shared, drain_deadline); - assert!(matches!( - target.lock().expect("target mutex poisoned").status, - TargetStatus::Opening { - attempt: 2, - awaiting_receiver_generation: None, - .. - } - )); - assert!(recv_request_timeout(&peer).is_some()); - } - - #[test] - fn multiple_expired_targets_share_one_receiver_drain_generation() { - let interval = Duration::from_millis(10); - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - config.open_timeouts = vec![Duration::from_millis(200), Duration::from_millis(200)]; - let first_peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let second_peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - for peer in [&first_peer, &second_peer] { - peer.set_read_timeout(Some(Duration::from_millis(100))) - .unwrap(); - } - let now = Instant::now(); - let first = Arc::new(Mutex::new( - TargetState::new( - &config, - target("first", first_peer.local_addr().unwrap()), - 0, - now, - ) - .unwrap(), - )); - let second = Arc::new(Mutex::new( - TargetState::new( - &config, - target("second", second_peer.local_addr().unwrap()), - 1, - now, - ) - .unwrap(), - )); - let (shared, _control_rx) = shared_with_targets( - &config, - EventHub::new(), - vec![first.clone(), second.clone()], - ); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - drive_open_attempts(&config, &socket, &shared, now); - assert!(recv_request_timeout(&first_peer).is_some()); - assert!(recv_request_timeout(&second_peer).is_some()); - let deadline = [&first, &second] - .into_iter() - .filter_map(|target| match target.lock().unwrap().status { - TargetStatus::Opening { next_send_at, .. } => Some(next_send_at), - _ => None, - }) - .max() - .unwrap(); - - drive_open_attempts(&config, &socket, &shared, deadline); - let generations = [&first, &second].map(|target| match target.lock().unwrap().status { - TargetStatus::Opening { - awaiting_receiver_generation: Some(generation), - .. - } => generation, - ref status => panic!("unexpected waiting status: {status:?}"), - }); - assert_eq!(generations[0], generations[1]); - let shared_drain = receiver_drain_snapshot(&shared); - assert_eq!( - shared_drain.deadline(), - Some(deadline + GROUP_RECEIVER_DRAIN_LIMIT) - ); - - assert!(complete_expired_receiver_drain( - &shared, - shared_drain.deadline().unwrap() - )); - drive_open_attempts(&config, &socket, &shared, shared_drain.deadline().unwrap()); - for target in [&first, &second] { - assert!(matches!( - target.lock().unwrap().status, - TargetStatus::Opening { - attempt: 2, - awaiting_receiver_generation: None, - .. - } - )); - } - assert!(recv_request_timeout(&first_peer).is_some()); - assert!(recv_request_timeout(&second_peer).is_some()); - - drive_open_attempts(&config, &socket, &shared, deadline); - assert!(recv_request_timeout(&first_peer).is_none()); - assert!(recv_request_timeout(&second_peer).is_none()); - } - - #[test] - fn removal_and_cancellation_release_targets_waiting_for_receiver_drain() { - let interval = Duration::from_millis(10); - let config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - let first_peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let second_peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let now = Instant::now(); - let first_config = target("first", first_peer.local_addr().unwrap()); - let second_config = target("second", second_peer.local_addr().unwrap()); - let first = Arc::new(Mutex::new( - TargetState::new(&config, first_config.clone(), 0, now).unwrap(), - )); - let second = Arc::new(Mutex::new( - TargetState::new(&config, second_config, 1, now).unwrap(), - )); - let (shared, _control_rx) = shared_with_targets( - &config, - EventHub::new(), - vec![first.clone(), second.clone()], - ); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - drive_open_attempts(&config, &socket, &shared, now); - let deadline = [&first, &second] - .into_iter() - .filter_map(|target| match target.lock().unwrap().status { - TargetStatus::Opening { next_send_at, .. } => Some(next_send_at), - _ => None, - }) - .max() - .unwrap(); - drive_open_attempts(&config, &socket, &shared, deadline); - for target in [&first, &second] { - assert!(matches!( - target.lock().unwrap().status, - TargetStatus::Opening { - awaiting_receiver_generation: Some(_), - .. - } - )); - } - - let mut next_order = 2; - let mut records = TargetOutcomeHistory::default(); - apply_target_update( - &config, - &socket, - &shared, - vec![first_config], - &mut next_order, - &mut records, - ) - .unwrap(); - assert!(matches!( - second.lock().unwrap().final_reason, - Some(ManagedTargetEndReason::Removed) - )); - - shared.cancellation.cancel(); - cancel_remaining_targets(&socket, &shared, &mut records); - assert!(matches!( - first.lock().unwrap().final_reason, - Some(ManagedTargetEndReason::Cancelled) - )); - assert_eq!(records.total, 2); - assert!(matches!( - records.recent.front().map(|outcome| &outcome.end_reason), - Some(ManagedTargetEndReason::Removed) - )); - assert!(matches!( - records.recent.back().map(|outcome| &outcome.end_reason), - Some(ManagedTargetEndReason::Cancelled) - )); - } - - #[test] - fn final_open_timeout_waits_until_receiver_drain_completes() { - let interval = Duration::from_millis(10); - let config = group_config(None, interval, ManagedGroupPacing::Staggered).client; - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let remote = peer.local_addr().unwrap(); - let now = Instant::now(); - let target = Arc::new(Mutex::new( - TargetState::new(&config, target("peer", remote), 0, now).unwrap(), - )); - let shared = shared_with_target(&config, EventHub::new(), target.clone()); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - drive_open_attempts(&config, &socket, &shared, now); - let deadline = { - let target = target.lock().expect("target mutex poisoned"); - match target.status { - TargetStatus::Opening { - attempt: 1, - next_send_at, - .. - } => next_send_at, - ref status => panic!("unexpected target status after open send: {status:?}"), - } - }; - - drive_open_attempts(&config, &socket, &shared, deadline); - let generation = { - let target = target.lock().expect("target mutex poisoned"); - let generation = match target.status { - TargetStatus::Opening { - attempt: 1, - awaiting_receiver_generation: Some(generation), - .. - } => generation, - ref status => panic!("unexpected waiting status: {status:?}"), - }; - assert!(target.final_reason.is_none()); - generation - }; - - let drain_deadline = receiver_drain_snapshot(&shared).deadline().unwrap(); - assert!(!complete_expired_receiver_drain( - &shared, - drain_deadline - Duration::from_nanos(1) - )); - drive_open_attempts( - &config, - &socket, - &shared, - drain_deadline - Duration::from_nanos(1), - ); - assert!(target.lock().unwrap().final_reason.is_none()); - - assert!(complete_expired_receiver_drain(&shared, drain_deadline)); - assert_eq!(completed_receiver_drain(&shared), generation); - drive_open_attempts(&config, &socket, &shared, drain_deadline); - let reason = { - let target = target.lock().expect("target mutex poisoned"); - assert!(matches!(target.status, TargetStatus::Finished)); - assert!(matches!( - target.final_reason, - Some(ManagedTargetEndReason::OpenFailed(ManagedTargetFailure { - kind: ManagedTargetFailureKind::OpeningTimeout, - .. - })) - )); - target.final_reason.clone() - }; - - drive_open_attempts(&config, &socket, &shared, drain_deadline); - assert_eq!( - target.lock().expect("target mutex poisoned").final_reason, - reason - ); - } - - #[test] - fn group_probe_separates_permission_committed_send_and_send_call_timing() { - let duration = Duration::from_secs(1); - let interval = Duration::from_millis(100); - let params = test_params(Some(duration), interval); - let config = group_config(Some(duration), interval, ManagedGroupPacing::Staggered).client; - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let remote = peer.local_addr().unwrap(); - let opened_mono = Instant::now() - .checked_sub(Duration::from_secs(2)) - .expect("test host monotonic clock has at least two seconds of history"); - let opened_at = ClientTimestamp { - wall: std::time::SystemTime::now(), - mono: opened_mono, - }; - let mut target = active_probe_target(&config, ¶ms, remote, opened_at); - let permission_at = opened_mono + Duration::from_millis(500); - let sent_at = ClientTimestamp { - wall: std::time::SystemTime::now(), - mono: opened_mono + Duration::from_millis(525), - }; - let timestamps = ProbeSendTimestamps { - permission_at, - sent_at, - send_call_start: opened_mono + Duration::from_millis(526), - send_finished_at: opened_mono + Duration::from_millis(533), - }; - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - let events = - send_echo_to_locked_target_at(&socket, &mut target, opened_mono, timestamps).unwrap(); - - assert!(matches!( - events.as_slice(), - [ClientEvent::EchoSent { - scheduled_at, - sent_at: event_sent_at, - send_call, - timer_error, - .. - }] if *scheduled_at == permission_at - && *event_sent_at == sent_at - && *send_call == Duration::from_millis(7) - && *timer_error == Duration::from_millis(25) - )); - assert_eq!( - target - .schedule - .as_ref() - .and_then(ProbeSchedule::next_send_deadline), - Some(opened_mono + Duration::from_millis(600)) - ); - assert_eq!(target.runtime.packets_sent(), 1); - let timeout_at = sent_at.mono + target.runtime.probe_timeout(); - assert!(target - .runtime - .poll_timeouts_at(timeout_at - Duration::from_nanos(1)) - .unwrap() - .is_empty()); - assert!(matches!( - target.runtime.poll_timeouts_at(timeout_at).unwrap().as_slice(), - [ClientEvent::EchoLoss { - sent_at: loss_sent_at, - .. - }] if *loss_sent_at == sent_at - )); - } - - #[test] - fn group_failed_probe_send_preserves_machine_and_schedule() { - let duration = Duration::from_secs(1); - let interval = Duration::from_millis(100); - let params = test_params(Some(duration), interval); - let config = group_config(Some(duration), interval, ManagedGroupPacing::Staggered).client; - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let remote = peer.local_addr().unwrap(); - let opened_mono = Instant::now() - .checked_sub(Duration::from_secs(2)) - .expect("test host monotonic clock has at least two seconds of history"); - let opened_at = ClientTimestamp { - wall: std::time::SystemTime::now(), - mono: opened_mono, - }; - let mut target = active_probe_target(&config, ¶ms, remote, opened_at); - let sent_at = ClientTimestamp { - wall: std::time::SystemTime::now(), - mono: opened_mono + Duration::from_millis(500), - }; - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - target.probe_send_error = true; - - assert!(matches!( - send_echo_to_locked_target_at( - &socket, - &mut target, - opened_mono, - probe_timestamps(sent_at.mono, sent_at), - ), - Err(ClientError::Socket(_)) - )); - assert_eq!(target.runtime.packets_sent(), 0); - assert_eq!( - target - .schedule - .as_ref() - .and_then(ProbeSchedule::next_send_deadline), - Some(opened_mono) - ); - - assert!(matches!( - send_echo_to_locked_target_at( - &socket, - &mut target, - opened_mono, - probe_timestamps(sent_at.mono, sent_at), - ) - .unwrap() - .as_slice(), - [ClientEvent::EchoSent { seq: 0, .. }] - )); - } - - #[test] - fn group_short_probe_commits_before_presentation_length_error() { - let duration = Duration::from_secs(1); - let interval = Duration::from_millis(100); - let params = test_params(Some(duration), interval); - let config = group_config(Some(duration), interval, ManagedGroupPacing::Staggered).client; - let peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let remote = peer.local_addr().unwrap(); - let opened_mono = Instant::now() - .checked_sub(Duration::from_secs(2)) - .expect("test host monotonic clock has at least two seconds of history"); - let opened_at = ClientTimestamp { - wall: std::time::SystemTime::now(), - mono: opened_mono, - }; - let mut target = active_probe_target(&config, ¶ms, remote, opened_at); - let sent_at = ClientTimestamp { - wall: std::time::SystemTime::now(), - mono: opened_mono + Duration::from_millis(500), - }; - let expected = echo_packet_len(false, ¶ms).unwrap(); - target.probe_reported_len = Some(expected - 1); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - - assert!(matches!( - send_echo_to_locked_target_at( - &socket, - &mut target, - opened_mono, - probe_timestamps(sent_at.mono, sent_at), - ), - Err(ClientError::DatagramLengthMismatch { - expected: error_expected, - actual, - }) if error_expected == expected && actual + 1 == expected - )); - assert_eq!(target.runtime.packets_sent(), 1); - assert_eq!( - target - .schedule - .as_ref() - .and_then(ProbeSchedule::next_send_deadline), - Some(opened_mono + Duration::from_millis(600)) - ); - } - - #[test] - fn silent_opening_peer_times_out_all_attempts_and_publishes_once() { - let silent_peer = UdpSocket::bind("127.0.0.1:0").unwrap(); - let remote = silent_peer.local_addr().unwrap(); - let deadline = Instant::now() + Duration::from_secs(2); - let mut config = group_config( - None, - Duration::from_millis(10), - ManagedGroupPacing::Staggered, - ); - config.client.open_timeouts = vec![Duration::from_millis(200), Duration::from_millis(200)]; - let (session, sub) = ManagedClientGroup::start_with_subscription( - config, - vec![target("silent", remote)], - SubscriberConfig { - capacity: 8, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - let (join_tx, join_rx) = mpsc::sync_channel(1); - let joiner = thread::spawn(move || { - join_tx.send(session.join()).unwrap(); - }); - let mut events = Vec::new(); - loop { - match sub.try_recv() { - Ok(Some(event)) => events.push(event), - Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(1)), - Ok(None) => panic!("timed out waiting for silent opening peer completion"), - Err(EventSubscriptionError::Disconnected) => break, - } - } - - let outcome = join_rx - .recv_timeout(deadline.saturating_duration_since(Instant::now())) - .expect("timed out joining silent opening peer group") - .unwrap(); - joiner.join().unwrap(); - assert_eq!(sub.try_recv(), Err(EventSubscriptionError::Disconnected)); - - silent_peer.set_nonblocking(true).unwrap(); - let mut open_attempts = 0; - let mut packet = [0_u8; 2048]; - loop { - match silent_peer.recv_from(&mut packet) { - Ok((size, _)) => { - assert!(size > 3); - assert_ne!(packet[3] & FLAG_OPEN, 0); - open_attempts += 1; - } - Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => break, - Err(err) => panic!("failed reading queued open request: {err}"), - } - } - assert_eq!(open_attempts, 2); - - let terminal_outcomes = events - .iter() - .filter_map(|event| match event { - ManagedGroupEvent::TargetFinished(outcome) => Some(outcome.clone()), - ManagedGroupEvent::Client(_) => None, - }) - .collect::>(); - assert_eq!(terminal_outcomes.len(), 1); - let published = &terminal_outcomes[0]; - let failure = match &published.end_reason { - ManagedTargetEndReason::OpenFailed(failure) => failure, - reason => panic!("expected OpenFailed, got {reason:?}"), - }; - assert_eq!(failure.kind, ManagedTargetFailureKind::OpeningTimeout); - assert!(failure.message.contains("all open requests timed out")); - assert_eq!(outcome.total_target_outcomes, 1); - assert_eq!(outcome.failed_target_outcomes, 1); - assert_eq!(outcome.targets, vec![published.clone()]); - } - - #[test] - fn runtime_failure_publishes_terminal_outcome() { - let interval = Duration::from_millis(10); - let server = start_silent_runtime_server(test_params(None, interval)); - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered); - config.client.max_pending_probes = 1; - let (session, sub) = ManagedClientGroup::start_with_subscription( - config, - vec![target("runtime", server.addr)], - SubscriberConfig { - capacity: 16, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - let events = collect_group_events_until_disconnected(&sub); - let outcome = session.join().unwrap(); - server.join(); - - let terminal = events - .iter() - .find_map(|event| match event { - ManagedGroupEvent::TargetFinished(outcome) => Some(outcome), - ManagedGroupEvent::Client(_) => None, - }) - .expect("missing terminal outcome"); - let failure = terminal - .end_reason - .failure() - .expect("runtime target should fail"); - assert_eq!(failure.kind, ManagedTargetFailureKind::RuntimeProtocol); - assert!(failure.message.contains("pending probe limit")); - assert_eq!(outcome.targets, vec![terminal.clone()]); - } - - #[test] - fn no_test_event_precedes_terminal_outcome() { - let interval = Duration::from_millis(10); - let server = start_no_test_server(test_params(Some(interval), interval)); - let mut config = group_config(Some(interval), interval, ManagedGroupPacing::Staggered); - config.client.run_mode = crate::RunMode::NoTest; - let (session, sub) = ManagedClientGroup::start_with_subscription( - config, - vec![target("no-test", server.addr)], - SubscriberConfig { - capacity: 8, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - let events = collect_group_events_until_disconnected(&sub); - let outcome = session.join().unwrap(); - server.join(); - - assert!(matches!( - events.as_slice(), - [ - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::NoTestCompleted { .. }, - .. - }), - ManagedGroupEvent::TargetFinished(ManagedTargetOutcome { - end_reason: ManagedTargetEndReason::NoTestComplete, - .. - }) - ] - )); - assert_eq!( - outcome.targets[0].end_reason, - ManagedTargetEndReason::NoTestComplete - ); - } - - #[test] - fn delayed_burst_pacing_emits_one_current_slot_and_advances_to_future() { - let interval = Duration::from_millis(10); - let first_slot = Instant::now(); - let delayed_now = first_slot + Duration::from_millis(45); - let mut pacing = PacingRuntime::new(ManagedGroupPacing::Burst); - pacing.next_burst_at = Some(first_slot); - - let scheduled_at = pacing.next_burst(interval, delayed_now).unwrap().unwrap(); - - assert_eq!(scheduled_at, first_slot + Duration::from_millis(40)); - assert_eq!( - pacing.next_wakeup(), - Some(first_slot + Duration::from_millis(50)) - ); - assert!(pacing.next_wakeup().unwrap() > delayed_now); - } - - #[test] - fn delayed_staggered_pacing_skips_historical_slots() { - let first_slot = Instant::now(); - let delayed_now = first_slot + Duration::from_millis(35); - - let (scheduled_at, next_slot_at, target_index) = - advance_staggered_slot(first_slot, Duration::from_millis(30), 3, 0, delayed_now) - .unwrap(); - - assert_eq!(scheduled_at, first_slot + Duration::from_millis(30)); - assert_eq!(next_slot_at, first_slot + Duration::from_millis(40)); - assert_eq!(target_index, 0); - assert!(next_slot_at > delayed_now); - } - - #[test] - fn scheduler_without_deadline_blocks_until_control_message() { - let (control_tx, control_rx) = mpsc::channel(); - let cancellation = CancellationToken::new(); - let worker_cancellation = cancellation.clone(); - let (started_tx, started_rx) = mpsc::sync_channel(0); - let (result_tx, result_rx) = mpsc::sync_channel(1); - let worker = thread::spawn(move || { - started_tx.send(()).unwrap(); - result_tx - .send(wait_for_scheduler_control( - &control_rx, - &worker_cancellation, - None, - )) - .unwrap(); - }); - - started_rx - .recv_timeout(Duration::from_secs(1)) - .expect("scheduler wait did not start"); - assert!(matches!( - result_rx.recv_timeout(Duration::from_millis(30)), - Err(mpsc::RecvTimeoutError::Timeout) - )); - - control_tx.send(ControlMessage::Wake).unwrap(); - assert!(matches!( - result_rx - .recv_timeout(Duration::from_secs(1)) - .expect("control message did not wake scheduler"), - Some(ControlMessage::Wake) - )); - worker.join().unwrap(); - assert!(!cancellation.is_cancelled()); - } - - #[test] - fn scheduler_control_disconnection_cancels_blocking_wait() { - let (control_tx, control_rx) = mpsc::channel(); - let cancellation = CancellationToken::new(); - drop(control_tx); - - assert!(wait_for_scheduler_control(&control_rx, &cancellation, None).is_none()); - assert!(cancellation.is_cancelled()); - } - - #[test] - fn scheduler_deadline_wakes_without_control_message() { - let (_control_tx, control_rx) = mpsc::channel(); - let cancellation = CancellationToken::new(); - let started_at = Instant::now(); - - assert!(wait_for_scheduler_control( - &control_rx, - &cancellation, - Some(started_at + Duration::from_millis(10)), - ) - .is_none()); - assert!(started_at.elapsed() >= Duration::from_millis(5)); - assert!(!cancellation.is_cancelled()); - } - - #[test] - fn scheduler_wakeup_uses_receiver_drain_deadline_and_suppresses_due_pacing() { - let now = Instant::now(); - let mut drain = ReceiverDrainState::default(); - drain.request_or_join(now).unwrap(); - let drain_deadline = drain.deadline().unwrap(); - - assert_eq!( - next_scheduler_deadline(None, Some(now), None, drain), - Some(drain_deadline) - ); - - let open_deadline = now + Duration::from_millis(10); - let target_drain_deadline = now + Duration::from_millis(20); - assert_eq!( - next_scheduler_deadline( - Some(open_deadline), - Some(now), - Some(target_drain_deadline), - drain, - ), - Some(open_deadline) - ); - - drain.complete_if_expired(drain_deadline); - assert_eq!( - next_scheduler_deadline(None, Some(now), None, drain), - Some(now) - ); - } - - #[test] - fn staggered_pacing_assigns_distinct_target_slots() { - let interval = Duration::from_millis(80); - let first_slot = Instant::now(); - - let (first_scheduled_at, second_slot, first_target) = - advance_staggered_slot(first_slot, interval, 2, 0, first_slot).unwrap(); - let (second_scheduled_at, third_slot, second_target) = - advance_staggered_slot(second_slot, interval, 2, 1, second_slot).unwrap(); - - assert_eq!(first_scheduled_at, first_slot); - assert_eq!(second_slot, first_slot + Duration::from_millis(40)); - assert_eq!(first_target, 0); - assert_eq!(second_scheduled_at, second_slot); - assert_eq!(third_slot, first_slot + interval); - assert_eq!(second_target, 1); - assert_ne!(first_scheduled_at, second_scheduled_at); - } - - #[test] - fn burst_pacing_uses_one_shared_target_deadline() { - let interval = Duration::from_millis(80); - let first_burst = Instant::now(); - let mut pacing = PacingRuntime::new(ManagedGroupPacing::Burst); - pacing.next_burst_at = Some(first_burst); - - let shared_deadline = pacing.next_burst(interval, first_burst).unwrap().unwrap(); - - assert_eq!(shared_deadline, first_burst); - assert_eq!(pacing.next_wakeup(), Some(first_burst + interval)); - - let next_deadline = pacing - .next_burst(interval, first_burst + interval) - .unwrap() - .unwrap(); - assert_eq!(next_deadline, first_burst + interval); - assert_eq!(pacing.next_wakeup(), Some(first_burst + interval * 2)); - } - - #[test] - fn delayed_open_target_joins_after_active_target_is_already_sending() { - let duration = Duration::from_millis(240); - let interval = Duration::from_millis(50); - let params = test_params(Some(duration), interval); - let a = start_echo_server(params.clone(), Duration::ZERO); - let b = start_echo_server(params, Duration::from_millis(120)); - - let (session, sub) = ManagedClientGroup::start_with_subscription( - group_config(Some(duration), interval, ManagedGroupPacing::Staggered), - vec![target("a", a.addr), target("b", b.addr)], - SubscriberConfig { - capacity: 512, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - let _ = session.join().unwrap(); - let events = drain_after_join(&sub); - a.join(); - b.join(); - - let b_started_at = events - .iter() - .find_map(|event| match (&event.target, &event.event) { - (id, ClientEvent::SessionStarted { at, .. }) if id.as_str() == "b" => Some(at.mono), - _ => None, - }) - .expect("target b should eventually open"); - let a_sent_before_b = events - .iter() - .filter(|event| { - event.target.as_str() == "a" - && matches!( - &event.event, - ClientEvent::EchoSent { sent_at, .. } if sent_at.mono < b_started_at - ) - }) - .count(); - - assert!( - a_sent_before_b >= 2, - "active target did not continue sending while b was opening" - ); - assert!(events.iter().any(|event| { - event.target.as_str() == "b" && matches!(event.event, ClientEvent::EchoReply { .. }) - })); - } - - #[test] - fn staggered_active_set_change_does_not_resend_unchanged_target_early() { - let duration = Duration::from_millis(360); - let interval = Duration::from_millis(100); - let params = test_params(Some(duration), interval); - let a = start_echo_server(params.clone(), Duration::ZERO); - let b = start_echo_server(params, Duration::from_millis(150)); - - let (session, sub) = ManagedClientGroup::start_with_subscription( - group_config(Some(duration), interval, ManagedGroupPacing::Staggered), - vec![target("a", a.addr), target("b", b.addr)], - SubscriberConfig { - capacity: 512, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - let _ = session.join().unwrap(); - let events = drain_after_join(&sub); - a.join(); - b.join(); - - let a_sends: Vec = events - .iter() - .filter_map(|event| match (&event.target, &event.event) { - (id, ClientEvent::EchoSent { sent_at, .. }) if id.as_str() == "a" => { - Some(sent_at.mono) - } - _ => None, - }) - .collect(); - assert!( - a_sends.len() >= 3, - "expected enough target a sends to verify spacing, got {a_sends:?}" - ); - - let min_delta = a_sends - .windows(2) - .map(|window| window[1].duration_since(window[0])) - .min() - .unwrap(); - assert!( - min_delta >= Duration::from_millis(75), - "target a was resent too early after active set changed: {min_delta:?}" - ); - } - - #[test] - fn update_targets_removes_and_adds_without_restarting_unchanged_target() { - let interval = Duration::from_millis(30); - let params = test_params(None, interval); - let a = start_echo_server(params.clone(), Duration::ZERO); - let b = start_echo_server(params.clone(), Duration::ZERO); - let c = start_echo_server(params, Duration::ZERO); - - let (session, sub) = ManagedClientGroup::start_with_subscription( - group_config(None, interval, ManagedGroupPacing::Burst), - vec![target("a", a.addr), target("b", b.addr)], - SubscriberConfig { - capacity: 512, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - let mut saw_a_reply = false; - let mut saw_b_reply = false; - while !(saw_a_reply && saw_b_reply) { - let event = recv_event_with_timeout(&sub); - saw_a_reply |= event.target.as_str() == "a" - && matches!(event.event, ClientEvent::EchoReply { .. }); - saw_b_reply |= event.target.as_str() == "b" - && matches!(event.event, ClientEvent::EchoReply { .. }); - } - - session - .update_targets(vec![target("a", a.addr), target("c", c.addr)]) - .unwrap(); - - let deadline = Instant::now() + Duration::from_millis(220); - let mut saw_a_after_update = false; - let mut saw_c_after_update = false; - let mut saw_b_echo_after_update = false; - let mut saw_b_removed = false; - while Instant::now() < deadline && !(saw_a_after_update && saw_c_after_update) { - match sub.try_recv() { - Ok(Some(ManagedGroupEvent::Client(event))) => { - saw_a_after_update |= event.target.as_str() == "a" - && matches!(event.event, ClientEvent::EchoReply { .. }); - saw_c_after_update |= event.target.as_str() == "c" - && matches!(event.event, ClientEvent::EchoReply { .. }); - saw_b_echo_after_update |= event.target.as_str() == "b" - && matches!( - event.event, - ClientEvent::EchoSent { .. } | ClientEvent::EchoReply { .. } - ); - } - Ok(Some(ManagedGroupEvent::TargetFinished(target))) => { - saw_b_removed |= target.id.as_str() == "b" - && target.end_reason == ManagedTargetEndReason::Removed; - } - Ok(None) => thread::sleep(Duration::from_millis(1)), - Err(err) => panic!("subscription ended unexpectedly: {err}"), - } - } - - session.stop(); - let outcome = session.join().unwrap(); - a.join(); - b.join(); - c.join(); - - assert_eq!(outcome.end_reason, ManagedGroupEndReason::Cancelled); - assert!(saw_a_after_update); - assert!(saw_c_after_update); - assert!(!saw_b_echo_after_update); - assert!(saw_b_removed); - assert!(outcome - .targets - .iter() - .any(|target| target.id.as_str() == "b" - && target.end_reason == ManagedTargetEndReason::Removed)); - } - - #[test] - fn start_rejects_empty_target_set_with_documented_error() { - let err = ManagedClientGroup::start( - group_config( - None, - Duration::from_millis(20), - ManagedGroupPacing::Staggered, - ), - Vec::new(), - ) - .unwrap_err(); - - assert!(matches!(err, ClientError::InvalidConfig { .. })); - assert!(err - .to_string() - .contains("managed client group requires at least one target")); - } - - #[test] - fn dynamic_group_can_remove_to_empty_and_add_targets_again() { - let interval = Duration::from_millis(20); - let params = test_params(None, interval); - let a = start_echo_server(params.clone(), Duration::ZERO); - let b = start_echo_server(params, Duration::ZERO); - let mut config = group_config(None, interval, ManagedGroupPacing::Burst); - config.completion = ManagedGroupCompletionPolicy::ExplicitCancellation; - - let (session, sub) = ManagedClientGroup::start_with_subscription( - config, - vec![target("a", a.addr)], - SubscriberConfig { - capacity: 128, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - while { - let event = recv_event_with_timeout(&sub); - event.target.as_str() != "a" || !matches!(event.event, ClientEvent::EchoReply { .. }) - } {} - - session.update_targets(Vec::new()).unwrap(); - let removal_deadline = Instant::now() + Duration::from_secs(2); - loop { - match sub.try_recv() { - Ok(Some(ManagedGroupEvent::TargetFinished(outcome))) - if outcome.id.as_str() == "a" - && outcome.end_reason == ManagedTargetEndReason::Removed => - { - break; - } - Ok(Some(_)) | Ok(None) if Instant::now() < removal_deadline => { - thread::sleep(Duration::from_millis(1)); - } - result => panic!("timed out waiting for removed outcome: {result:?}"), - } - } - assert_eq!(sub.recv_timeout(Duration::from_millis(30)), Ok(None)); - - session.update_targets(vec![target("b", b.addr)]).unwrap(); - while { - let event = recv_event_with_timeout(&sub); - event.target.as_str() != "b" || !matches!(event.event, ClientEvent::EchoReply { .. }) - } {} - - session.stop(); - let events = collect_group_events_until_disconnected(&sub); - let outcome = join_group_with_timeout(session, Duration::from_secs(1)); - a.join(); - b.join(); - - assert_eq!(outcome.end_reason, ManagedGroupEndReason::Cancelled); - assert!(events.iter().any(|event| matches!( - event, - ManagedGroupEvent::TargetFinished(ManagedTargetOutcome { - id, - end_reason: ManagedTargetEndReason::Cancelled, - .. - }) if id.as_str() == "b" - ))); - } - - #[test] - fn removing_final_target_keeps_control_channel_responsive() { - let interval = Duration::from_millis(20); - let server = start_echo_server(test_params(None, interval), Duration::ZERO); - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered); - config.completion = ManagedGroupCompletionPolicy::ExplicitCancellation; - let session = ManagedClientGroup::start(config, vec![target("only", server.addr)]).unwrap(); - - session.update_targets(Vec::new()).unwrap(); - session.update_targets(Vec::new()).unwrap(); - let idle_subscription = session - .subscribe(SubscriberConfig { - capacity: 4, - overflow: SubscriberOverflow::DropNewest, - }) - .expect("an idle dynamic group must still accept subscribers"); - assert_eq!( - idle_subscription.recv_timeout(Duration::from_millis(30)), - Ok(None) - ); - - session.stop(); - let outcome = join_group_with_timeout(session, Duration::from_secs(1)); - server.join(); - - assert_eq!(outcome.end_reason, ManagedGroupEndReason::Cancelled); - assert_eq!( - idle_subscription.try_recv(), - Err(EventSubscriptionError::Disconnected) - ); - } - - #[test] - fn explicit_cancellation_while_idle_publishes_removal_and_disconnects() { - let interval = Duration::from_millis(20); - let server = start_echo_server(test_params(None, interval), Duration::ZERO); - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered); - config.completion = ManagedGroupCompletionPolicy::ExplicitCancellation; - let (session, sub) = ManagedClientGroup::start_with_subscription( - config, - vec![target("idle", server.addr)], - SubscriberConfig { - capacity: 16, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - session.update_targets(Vec::new()).unwrap(); - session.stop(); - let events = collect_group_events_until_disconnected(&sub); - let outcome = join_group_with_timeout(session, Duration::from_secs(1)); - server.join(); - - assert_eq!(outcome.end_reason, ManagedGroupEndReason::Cancelled); - assert!(events.iter().any(|event| matches!( - event, - ManagedGroupEvent::TargetFinished(ManagedTargetOutcome { - id, - end_reason: ManagedTargetEndReason::Removed, - .. - }) if id.as_str() == "idle" - ))); - } - - #[test] - fn dropping_idle_dynamic_group_wakes_scheduler_and_disconnects() { - let interval = Duration::from_millis(20); - let server = start_echo_server(test_params(None, interval), Duration::ZERO); - let mut config = group_config(None, interval, ManagedGroupPacing::Staggered); - config.completion = ManagedGroupCompletionPolicy::ExplicitCancellation; - let (session, sub) = ManagedClientGroup::start_with_subscription( - config, - vec![target("idle", server.addr)], - SubscriberConfig { - capacity: 16, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - session.update_targets(Vec::new()).unwrap(); - drop(session); - let events = collect_group_events_until_disconnected(&sub); - server.join(); - - assert!(events.iter().any(|event| matches!( - event, - ManagedGroupEvent::TargetFinished(ManagedTargetOutcome { - id, - end_reason: ManagedTargetEndReason::Removed, - .. - }) if id.as_str() == "idle" - ))); - } - - #[test] - fn target_churn_retains_bounded_history_after_publishing_every_outcome() { - let sink = UdpSocket::bind("127.0.0.1:0").unwrap(); - let remote = sink.local_addr().unwrap(); - let mut config = group_config( - None, - Duration::from_millis(20), - ManagedGroupPacing::Staggered, - ); - config.completion = ManagedGroupCompletionPolicy::ExplicitCancellation; - config.client.open_timeouts = vec![Duration::from_secs(5)]; - let churn_count = MANAGED_GROUP_OUTCOME_HISTORY_LIMIT + 16; - let (session, sub) = ManagedClientGroup::start_with_subscription( - config, - vec![target("churn", remote)], - SubscriberConfig { - capacity: churn_count + 8, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - session.update_targets(Vec::new()).unwrap(); - for _ in 0..churn_count { - session - .update_targets(vec![target("churn", remote)]) - .unwrap(); - session.update_targets(Vec::new()).unwrap(); - } - - session.stop(); - let events = collect_group_events_until_disconnected(&sub); - let outcome = session.join().unwrap(); - drop(sink); - - let expected_total = u64::try_from(churn_count + 1).unwrap(); - let published = events - .iter() - .filter(|event| matches!(event, ManagedGroupEvent::TargetFinished(_))) - .count(); - assert_eq!(published, churn_count + 1); - assert_eq!(outcome.total_target_outcomes, expected_total); - assert_eq!(outcome.successful_target_outcomes, 0); - assert_eq!(outcome.peer_closed_target_outcomes, 0); - assert_eq!(outcome.failed_target_outcomes, 0); - assert_eq!(outcome.targets.len(), MANAGED_GROUP_OUTCOME_HISTORY_LIMIT); - assert_eq!( - outcome.discarded_target_outcomes, - expected_total - u64::try_from(MANAGED_GROUP_OUTCOME_HISTORY_LIMIT).unwrap() - ); - assert!(outcome - .targets - .iter() - .all(|target| target.end_reason == ManagedTargetEndReason::Removed)); - } - - #[test] - fn finite_group_drains_late_reply_during_final_drain() { - let interval = Duration::from_millis(100); - let duration = interval; - let params = test_params(Some(duration), interval); - let (server, release_reply) = start_gated_reply_server(params); - let target = target("late", server.addr); - let mut config = group_config(Some(duration), interval, ManagedGroupPacing::Staggered); - config.client.probe_timeout = Duration::from_millis(20); - let (session, sub) = ManagedClientGroup::start_with_subscription( - config, - vec![target.clone()], - SubscriberConfig { - capacity: 16, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - let mut events = Vec::new(); - loop { - let event = sub - .recv_timeout(Duration::from_secs(2)) - .expect("subscription disconnected before EchoLoss") - .expect("timed out waiting for EchoLoss"); - let saw_loss = matches!( - &event, - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::EchoLoss { seq: 0, .. }, - .. - }) - ); - events.push(event); - if saw_loss { - break; - } - } - - // EchoLoss is published before this scheduler iteration transitions the - // target to Draining. A synchronous no-op update is handled at the next - // loop head, so its return proves that transition completed before the - // server releases the retained reply. - session.update_targets(vec![target]).unwrap(); - release_reply.send(()).unwrap(); - let outcome = join_group_with_timeout(session, Duration::from_secs(1)); - events.extend(collect_group_events_until_disconnected(&sub)); - server.join(); - - assert_eq!( - final_drain_event_sequence(&events), - ["started", "sent", "loss", "late", "closed", "finished"] - ); - assert_eq!( - events - .iter() - .filter(|event| matches!( - event, - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::EchoLoss { seq: 0, .. }, - .. - }) - )) - .count(), - 1 - ); - assert_eq!( - events - .iter() - .filter(|event| matches!( - event, - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::LateReply { - seq: 0, - sent_at: Some(_), - rtt: Some(_), - .. - }, - .. - }) - )) - .count(), - 1 - ); - - assert_eq!( - outcome.end_reason, - ManagedGroupEndReason::AllTargetsComplete - ); - assert_eq!(outcome.targets.len(), 1); - let target = &outcome.targets[0]; - assert_eq!(target.end_reason, ManagedTargetEndReason::TestComplete); - assert_eq!(target.packets_sent, 1); - assert_eq!(target.replies_received, 0); - assert_eq!(target.duplicates, 0); - assert_eq!(target.late, 1); - assert_eq!(target.warning_events, 0); - assert_eq!(outcome.total_target_outcomes, 1); - assert_eq!(outcome.successful_target_outcomes, 1); - assert_eq!(outcome.peer_closed_target_outcomes, 0); - assert_eq!(outcome.failed_target_outcomes, 0); - assert_eq!(outcome.discarded_target_outcomes, 0); - assert_eq!( - events.iter().find_map(|event| match event { - ManagedGroupEvent::TargetFinished(target) => Some(target), - _ => None, - }), - Some(target) - ); - assert_eq!(sub.dropped_events(), 0); - assert_eq!(sub.try_recv(), Err(EventSubscriptionError::Disconnected)); - } - - #[test] - fn finite_group_final_drain_expires_without_late_reply() { - let interval = Duration::from_millis(100); - let duration = interval; - let server = start_silent_runtime_server(test_params(Some(duration), interval)); - let mut config = group_config(Some(duration), interval, ManagedGroupPacing::Staggered); - config.client.probe_timeout = Duration::from_millis(20); - let (session, sub) = ManagedClientGroup::start_with_subscription( - config, - vec![target("silent", server.addr)], - SubscriberConfig { - capacity: 16, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - let outcome = join_group_with_timeout(session, Duration::from_secs(1)); - let events = collect_group_events_until_disconnected(&sub); - server.join(); - - assert_eq!( - final_drain_event_sequence(&events), - ["started", "sent", "loss", "closed", "finished"] - ); - assert!(!events.iter().any(|event| matches!( - event, - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::LateReply { .. }, - .. - }) - ))); - let timeout_at = events - .iter() - .find_map(|event| match event { - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::EchoLoss { timeout_at, .. }, - .. - }) => Some(*timeout_at), - _ => None, - }) - .expect("missing EchoLoss"); - let closed_at = events - .iter() - .find_map(|event| match event { - ManagedGroupEvent::Client(TargetEvent { - event: ClientEvent::SessionClosed { at, .. }, - .. - }) => Some(at.mono), - _ => None, - }) - .expect("missing SessionClosed"); - assert!(closed_at >= timeout_at + GROUP_FINAL_DRAIN); - - assert_eq!( - outcome.end_reason, - ManagedGroupEndReason::AllTargetsComplete - ); - assert_eq!(outcome.targets.len(), 1); - let target = &outcome.targets[0]; - assert_eq!(target.end_reason, ManagedTargetEndReason::TestComplete); - assert_eq!(target.packets_sent, 1); - assert_eq!(target.replies_received, 0); - assert_eq!(target.duplicates, 0); - assert_eq!(target.late, 0); - assert_eq!(target.warning_events, 0); - assert_eq!(outcome.total_target_outcomes, 1); - assert_eq!(outcome.successful_target_outcomes, 1); - assert_eq!(outcome.peer_closed_target_outcomes, 0); - assert_eq!(outcome.failed_target_outcomes, 0); - assert_eq!(outcome.discarded_target_outcomes, 0); - assert_eq!( - events.iter().find_map(|event| match event { - ManagedGroupEvent::TargetFinished(target) => Some(target), - _ => None, - }), - Some(target) - ); - assert_eq!(sub.dropped_events(), 0); - assert_eq!(sub.try_recv(), Err(EventSubscriptionError::Disconnected)); - } - - #[test] - fn finite_static_group_still_completes_naturally() { - let interval = Duration::from_millis(10); - let duration = Duration::from_millis(30); - let server = start_echo_server(test_params(Some(duration), interval), Duration::ZERO); - let session = ManagedClientGroup::start( - group_config(Some(duration), interval, ManagedGroupPacing::Staggered), - vec![target("finite", server.addr)], - ) - .unwrap(); - - let outcome = session.join().unwrap(); - server.join(); - - assert_eq!( - outcome.end_reason, - ManagedGroupEndReason::AllTargetsComplete - ); - assert_eq!(outcome.targets.len(), 1); - assert_eq!( - outcome.targets[0].end_reason, - ManagedTargetEndReason::TestComplete - ); - } - - #[test] - fn stop_join_cleans_up_group_threads() { - let interval = Duration::from_millis(20); - let params = test_params(None, interval); - let a = start_echo_server(params.clone(), Duration::ZERO); - let b = start_echo_server(params, Duration::ZERO); - - let (session, sub) = ManagedClientGroup::start_with_subscription( - group_config(None, interval, ManagedGroupPacing::Staggered), - vec![target("a", a.addr), target("b", b.addr)], - SubscriberConfig { - capacity: 64, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - thread::sleep(Duration::from_millis(60)); - session.stop(); - session.stop(); - let events = collect_group_events_until_disconnected(&sub); - let outcome = session.join().unwrap(); - a.join(); - b.join(); - - assert_eq!(outcome.end_reason, ManagedGroupEndReason::Cancelled); - assert_eq!(outcome.targets.len(), 2); - assert_eq!( - events - .iter() - .filter(|event| matches!( - event, - ManagedGroupEvent::TargetFinished(ManagedTargetOutcome { - end_reason: ManagedTargetEndReason::Cancelled, - .. - }) - )) - .count(), - 2 - ); - } -} diff --git a/crates/irtt-client/src/managed/hub.rs b/crates/irtt-client/src/managed/hub.rs deleted file mode 100644 index 55cf3f3..0000000 --- a/crates/irtt-client/src/managed/hub.rs +++ /dev/null @@ -1,606 +0,0 @@ -use std::{ - collections::{HashMap, VecDeque}, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, Condvar, Mutex, Weak, - }, - time::{Duration, Instant}, -}; - -use crate::{error::EventSubscriptionError, event::ClientEvent}; - -/// Configuration for one managed event subscriber. -/// -/// Each subscriber has an independent bounded queue. The queue bounds memory -/// growth when event production is faster than that subscriber's consumer. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct SubscriberConfig { - /// Maximum number of events buffered for this subscriber. - /// - /// Must be greater than zero. - pub capacity: usize, - /// Policy applied when publishing to a full subscriber queue. - pub overflow: SubscriberOverflow, -} - -impl Default for SubscriberConfig { - fn default() -> Self { - Self { - capacity: 1024, - overflow: SubscriberOverflow::DropNewest, - } - } -} - -/// Overflow behavior for a bounded event subscription queue. -/// -/// These policies are applied independently per subscriber when that -/// subscriber's queue is full. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SubscriberOverflow { - /// Leave the existing queue unchanged and discard the newly published event. - DropNewest, - /// Remove the oldest queued event and enqueue the newly published event. - DropOldest, - /// Disconnect the subscriber and clear its queue. - Disconnect, -} - -/// Publish/subscribe fan-out for managed event values. -/// -/// `EventHub` is used by managed sessions internally and is also exported for -/// callers that want the same bounded subscription behavior around their own -/// event producer. Publishing clones each event once per subscriber. Once -/// sealed with [`disconnect_all`](Self::disconnect_all), the hub cannot accept -/// new subscribers or publish more events. -#[derive(Debug)] -pub struct EventHub { - inner: Arc>, -} - -#[derive(Debug)] -struct HubInner { - next_id: AtomicU64, - state: Mutex>, -} - -#[derive(Debug)] -struct HubState { - subscribers: HashMap>>, - sealed: bool, -} - -/// Handle for receiving managed client events. -/// -/// Each subscription has its own bounded queue configured by -/// [`SubscriberConfig`]. The configured [`SubscriberOverflow`] policy applies -/// when that queue is full. Use [`dropped_events`](Self::dropped_events) to -/// detect incomplete consumption. Dropping the handle unregisters the -/// subscriber. -#[must_use = "dropping the subscription unregisters it"] -#[derive(Debug)] -pub struct EventSubscription { - id: u64, - hub: Weak>, - inner: Arc>, -} - -#[derive(Debug)] -struct SubscriberInner { - state: Mutex>, - available: Condvar, - config: SubscriberConfig, -} - -#[derive(Debug)] -struct SubscriberState { - queue: VecDeque, - connected: bool, - dropped_events: u64, -} - -impl Clone for EventHub { - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), - } - } -} - -impl EventHub { - /// Create an empty event hub with no subscribers. - pub fn new() -> Self { - Self { - inner: Arc::new(HubInner { - next_id: AtomicU64::new(1), - state: Mutex::new(HubState { - subscribers: HashMap::new(), - sealed: false, - }), - }), - } - } - - /// Register a new subscriber with its own bounded queue. - /// - /// Returns an error when `config.capacity` is zero. - pub fn subscribe( - &self, - config: SubscriberConfig, - ) -> Result, crate::ClientError> { - if config.capacity == 0 { - return Err(crate::ClientError::InvalidConfig { - reason: "subscriber capacity must be greater than zero".to_owned(), - }); - } - - let mut hub_state = self.inner.state.lock().expect("event hub mutex poisoned"); - if hub_state.sealed { - return Err(crate::ClientError::AlreadyClosed); - } - - let id = self.inner.next_id.fetch_add(1, Ordering::Relaxed); - let inner = Arc::new(SubscriberInner { - state: Mutex::new(SubscriberState { - queue: VecDeque::with_capacity(config.capacity), - connected: true, - dropped_events: 0, - }), - available: Condvar::new(), - config, - }); - hub_state.subscribers.insert(id, inner.clone()); - - Ok(EventSubscription { - id, - hub: Arc::downgrade(&self.inner), - inner, - }) - } - - /// Permanently seal the hub and disconnect all subscribers. - /// - /// Already queued events remain available to drain. Future subscriptions - /// fail with [`crate::ClientError::AlreadyClosed`]. - pub fn disconnect_all(&self) { - let subscribers: Vec>> = { - let mut state = self.inner.state.lock().expect("event hub mutex poisoned"); - state.sealed = true; - state - .subscribers - .drain() - .map(|(_, subscriber)| subscriber) - .collect() - }; - - for subscriber in subscribers { - subscriber.disconnect(); - } - } - - #[cfg(test)] - fn subscriber_count(&self) -> usize { - self.inner - .state - .lock() - .expect("event hub mutex poisoned") - .subscribers - .len() - } -} - -impl EventHub { - /// Publish an event to all currently connected subscribers. - /// - /// Slow subscribers are handled according to their - /// [`SubscriberOverflow`] policy. Publishing does not block waiting for a - /// subscriber to consume queued events. - pub fn publish(&self, event: T) { - let subscribers: Vec<(u64, Arc>)> = self - .inner - .state - .lock() - .expect("event hub mutex poisoned") - .subscribers - .iter() - .map(|(id, subscriber)| (*id, subscriber.clone())) - .collect(); - - let mut disconnected = Vec::new(); - for (id, subscriber) in subscribers { - // Events are cloned per subscriber; bounded queues keep slow - // consumers from causing unbounded memory growth. - if !subscriber.publish(event.clone()) { - disconnected.push(id); - } - } - - if disconnected.is_empty() { - return; - } - - let mut subscribers = self.inner.state.lock().expect("event hub mutex poisoned"); - for id in disconnected { - subscribers.subscribers.remove(&id); - } - } -} - -impl Default for EventHub { - fn default() -> Self { - Self::new() - } -} - -impl EventSubscription { - /// Block until the next queued event is available. - /// - /// If the subscription is disconnected, this returns - /// [`EventSubscriptionError::Disconnected`] after any already queued events - /// have been drained. - pub fn recv(&self) -> Result { - let mut state = self.inner.state.lock().expect("subscriber mutex poisoned"); - loop { - if let Some(event) = state.queue.pop_front() { - return Ok(event); - } - if !state.connected { - return Err(EventSubscriptionError::Disconnected); - } - state = self - .inner - .available - .wait(state) - .expect("subscriber mutex poisoned"); - } - } - - /// Wait up to `timeout` for the next queued event. - /// - /// Returns `Ok(None)` when the timeout elapses while the subscription - /// remains connected. If the subscription is disconnected, this returns - /// [`EventSubscriptionError::Disconnected`] after any already queued events - /// have been drained. - pub fn recv_timeout(&self, timeout: Duration) -> Result, EventSubscriptionError> { - let started_at = Instant::now(); - let mut state = self.inner.state.lock().expect("subscriber mutex poisoned"); - loop { - if let Some(event) = state.queue.pop_front() { - return Ok(Some(event)); - } - if !state.connected { - return Err(EventSubscriptionError::Disconnected); - } - - let Some(wait_for) = timeout.checked_sub(started_at.elapsed()) else { - return Ok(None); - }; - if wait_for.is_zero() { - return Ok(None); - } - - let (next_state, _) = self - .inner - .available - .wait_timeout(state, wait_for) - .expect("subscriber mutex poisoned"); - state = next_state; - } - } - - /// Try to receive one queued event without blocking. - /// - /// Returns `Ok(None)` when the subscription is still connected but no event - /// is queued. If the subscription is disconnected, this returns - /// [`EventSubscriptionError::Disconnected`] after any already queued events - /// have been drained. - pub fn try_recv(&self) -> Result, EventSubscriptionError> { - let mut state = self.inner.state.lock().expect("subscriber mutex poisoned"); - if let Some(event) = state.queue.pop_front() { - return Ok(Some(event)); - } - if !state.connected { - return Err(EventSubscriptionError::Disconnected); - } - Ok(None) - } - - /// Return the number of events discarded by this subscription's overflow - /// policy. - /// - /// The count is monotonic for the lifetime of the subscription and remains - /// available after the hub is sealed or the subscription disconnects. - pub fn dropped_events(&self) -> u64 { - self.inner - .state - .lock() - .expect("subscriber mutex poisoned") - .dropped_events - } -} - -impl Drop for EventSubscription { - fn drop(&mut self) { - self.inner.disconnect(); - if let Some(hub) = self.hub.upgrade() { - hub.state - .lock() - .expect("event hub mutex poisoned") - .subscribers - .remove(&self.id); - } - } -} - -impl SubscriberInner { - fn publish(&self, event: T) -> bool { - let mut state = self.state.lock().expect("subscriber mutex poisoned"); - if !state.connected { - return false; - } - - if state.queue.len() < self.config.capacity { - state.queue.push_back(event); - self.available.notify_one(); - return true; - } - - match self.config.overflow { - SubscriberOverflow::DropNewest => { - state.dropped_events = state.dropped_events.saturating_add(1); - true - } - SubscriberOverflow::DropOldest => { - state.queue.pop_front(); - state.queue.push_back(event); - state.dropped_events = state.dropped_events.saturating_add(1); - self.available.notify_one(); - true - } - SubscriberOverflow::Disconnect => { - let discarded = u64::try_from(state.queue.len()) - .unwrap_or(u64::MAX) - .saturating_add(1); - state.dropped_events = state.dropped_events.saturating_add(discarded); - state.queue.clear(); - state.connected = false; - self.available.notify_all(); - false - } - } - } - - fn disconnect(&self) { - let mut state = self.state.lock().expect("subscriber mutex poisoned"); - state.connected = false; - self.available.notify_all(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ClientTimestamp, WarningKind}; - - fn event(n: usize) -> ClientEvent { - ClientEvent::Warning { - kind: WarningKind::UntrackedReply, - message: format!("event-{n}"), - at: ClientTimestamp::now(), - } - } - - fn event_message(event: ClientEvent) -> String { - match event { - ClientEvent::Warning { message, .. } => message, - other => panic!("unexpected event: {other:?}"), - } - } - - fn assert_next( - sub: &EventSubscription, - expected: Result, EventSubscriptionError>, - ) { - match (sub.try_recv(), expected) { - (Ok(Some(event)), Ok(Some(message))) => assert_eq!(event_message(event), message), - (Ok(None), Ok(None)) => {} - (Err(actual), Err(expected)) => assert_eq!(actual, expected), - (actual, expected) => panic!("expected {expected:?}, got {actual:?}"), - } - } - - #[test] - fn publishes_to_multiple_subscribers() { - let hub = EventHub::new(); - let a = hub.subscribe(SubscriberConfig::default()).unwrap(); - let b = hub.subscribe(SubscriberConfig::default()).unwrap(); - - hub.publish(event(1)); - - assert_eq!(event_message(a.try_recv().unwrap().unwrap()), "event-1"); - assert_eq!(event_message(b.try_recv().unwrap().unwrap()), "event-1"); - } - - #[test] - fn full_subscriber_overflow_policy_is_applied() { - let cases = [ - ( - SubscriberOverflow::DropNewest, - Ok(Some("event-1")), - Ok(None), - 1, - ), - ( - SubscriberOverflow::DropOldest, - Ok(Some("event-2")), - Ok(None), - 1, - ), - ( - SubscriberOverflow::Disconnect, - Err(EventSubscriptionError::Disconnected), - Err(EventSubscriptionError::Disconnected), - 2, - ), - ]; - - for (overflow, first_expected, second_expected, expected_drops) in cases { - let hub = EventHub::new(); - let sub = hub - .subscribe(SubscriberConfig { - capacity: 1, - overflow, - }) - .unwrap(); - - hub.publish(event(1)); - hub.publish(event(2)); - - assert_next(&sub, first_expected); - assert_next(&sub, second_expected); - assert_eq!(sub.dropped_events(), expected_drops); - } - } - - #[test] - fn drop_oldest_counter_accumulates_and_survives_hub_seal() { - let hub = EventHub::new(); - let sub = hub - .subscribe(SubscriberConfig { - capacity: 2, - overflow: SubscriberOverflow::DropOldest, - }) - .unwrap(); - - for n in 1..=5 { - hub.publish(event(n)); - } - hub.disconnect_all(); - - assert_eq!(sub.dropped_events(), 3); - assert_next(&sub, Ok(Some("event-4"))); - assert_next(&sub, Ok(Some("event-5"))); - assert_next(&sub, Err(EventSubscriptionError::Disconnected)); - assert_eq!(sub.dropped_events(), 3); - } - - #[test] - fn full_subscriber_does_not_prevent_other_delivery() { - let hub = EventHub::new(); - let slow = hub - .subscribe(SubscriberConfig { - capacity: 1, - overflow: SubscriberOverflow::DropNewest, - }) - .unwrap(); - let fast = hub - .subscribe(SubscriberConfig { - capacity: 4, - overflow: SubscriberOverflow::DropNewest, - }) - .unwrap(); - - hub.publish(event(1)); - hub.publish(event(2)); - - assert_eq!(event_message(slow.try_recv().unwrap().unwrap()), "event-1"); - assert_eq!(event_message(fast.try_recv().unwrap().unwrap()), "event-1"); - assert_eq!(event_message(fast.try_recv().unwrap().unwrap()), "event-2"); - } - - #[test] - fn subscribing_after_events_does_not_replay() { - let hub = EventHub::new(); - hub.publish(event(1)); - - let sub = hub.subscribe(SubscriberConfig::default()).unwrap(); - - assert!(sub.try_recv().unwrap().is_none()); - } - - #[test] - fn dropping_subscription_unregisters_from_hub() { - let hub: EventHub = EventHub::new(); - let sub = hub.subscribe(SubscriberConfig::default()).unwrap(); - assert_eq!(hub.subscriber_count(), 1); - - drop(sub); - - assert_eq!(hub.subscriber_count(), 0); - } - - #[test] - fn disconnect_all_wakes_blocking_receivers() { - let hub: EventHub = EventHub::new(); - let sub = hub.subscribe(SubscriberConfig::default()).unwrap(); - hub.disconnect_all(); - - assert_eq!( - sub.recv().unwrap_err(), - EventSubscriptionError::Disconnected - ); - } - - #[test] - fn timed_receive_drains_queued_events_and_observes_disconnect() { - let hub = EventHub::new(); - let sub = hub.subscribe(SubscriberConfig::default()).unwrap(); - - assert_eq!(sub.recv_timeout(Duration::ZERO).unwrap(), None); - hub.publish(event(1)); - hub.disconnect_all(); - - assert_eq!( - event_message(sub.recv_timeout(Duration::from_secs(1)).unwrap().unwrap()), - "event-1" - ); - assert_eq!( - sub.recv_timeout(Duration::from_secs(1)).unwrap_err(), - EventSubscriptionError::Disconnected - ); - } - - #[test] - fn sealed_hub_drains_queued_events_then_disconnects() { - let hub = EventHub::new(); - let sub = hub.subscribe(SubscriberConfig::default()).unwrap(); - hub.publish(event(1)); - - hub.disconnect_all(); - hub.publish(event(2)); - - assert_eq!(event_message(sub.recv().unwrap()), "event-1"); - assert_eq!( - sub.recv().unwrap_err(), - EventSubscriptionError::Disconnected - ); - } - - #[test] - fn subscribing_after_hub_is_sealed_fails() { - let hub: EventHub = EventHub::new(); - hub.disconnect_all(); - hub.disconnect_all(); - - assert!(matches!( - hub.subscribe(SubscriberConfig::default()), - Err(crate::ClientError::AlreadyClosed) - )); - } - - #[test] - fn session_closed_event_can_be_queued() { - let hub = EventHub::new(); - let sub = hub.subscribe(SubscriberConfig::default()).unwrap(); - hub.publish(ClientEvent::SessionClosed { - remote: "127.0.0.1:1".parse().unwrap(), - token: 1, - at: ClientTimestamp::now(), - }); - - assert!(matches!( - sub.recv().unwrap(), - ClientEvent::SessionClosed { token: 1, .. } - )); - } -} diff --git a/crates/irtt-client/src/managed/mod.rs b/crates/irtt-client/src/managed/mod.rs index 9005366..203983f 100644 --- a/crates/irtt-client/src/managed/mod.rs +++ b/crates/irtt-client/src/managed/mod.rs @@ -1,29 +1,7 @@ -mod cancellation; -mod group; -mod hub; -mod runner; mod target_id; pub use target_id::TargetId; -/// Temporary exports for the obsolete thread-based managed implementations. -#[doc(hidden)] -pub mod legacy { - pub use super::cancellation::CancellationToken; - pub use super::group::{ - ManagedClientGroup, ManagedClientGroupConfig, ManagedClientGroupSession, - ManagedGroupCompletionPolicy, ManagedGroupEndReason, ManagedGroupEvent, - ManagedGroupOutcome, ManagedGroupPacing, ManagedTargetConfig, ManagedTargetEndReason, - ManagedTargetFailure, ManagedTargetFailureKind, ManagedTargetOutcome, TargetEvent, - TargetEventSubscription, MANAGED_GROUP_OUTCOME_HISTORY_LIMIT, - }; - pub use super::hub::{EventHub, EventSubscription, SubscriberConfig, SubscriberOverflow}; - pub use super::runner::{ - ManagedClient, ManagedClientSession, SessionEndReason, SessionOutcome, - }; - pub use super::TargetId; -} - #[cfg(feature = "tokio")] mod blocking; #[cfg(feature = "tokio")] diff --git a/crates/irtt-client/src/managed/runner.rs b/crates/irtt-client/src/managed/runner.rs deleted file mode 100644 index ed05c20..0000000 --- a/crates/irtt-client/src/managed/runner.rs +++ /dev/null @@ -1,883 +0,0 @@ -use std::{ - thread::{self, JoinHandle}, - time::{Duration, Instant}, -}; - -use crate::{ - config::{ClientConfig, RecvBudget}, - error::ClientError, - event::{ClientEvent, OpenOutcome}, - Client, -}; - -use super::{ - cancellation::CancellationToken, - hub::{EventHub, EventSubscription, SubscriberConfig}, -}; - -const MANAGED_RECV_TIMEOUT: Duration = Duration::from_millis(20); -const MANAGED_RECV_BUDGET: RecvBudget = RecvBudget { max_packets: 64 }; -const MANAGED_FINAL_DRAIN: Duration = Duration::from_millis(100); -const IDLE_SLEEP: Duration = Duration::from_millis(1); - -/// Entry point for running a client session on a worker thread. -/// -/// The managed API owns the lower-level [`Client`](crate::Client) loop: it -/// opens the session, sends probes at the negotiated interval, receives -/// datagrams, publishes [`ClientEvent`] values, and closes the session when the -/// run completes or is cancelled. Delayed workers skip missed pacing slots -/// instead of sending catch-up bursts. -#[derive(Debug)] -pub struct ManagedClient; - -/// Running managed client session. -/// -/// [`ManagedClientSession::join`] waits for the worker and returns the session -/// outcome or client error. Dropping the session requests cooperative -/// cancellation; callers that need the outcome should explicitly join instead -/// of relying on drop. -#[must_use = "dropping the session cancels the managed client; call join() to wait for completion"] -#[derive(Debug)] -pub struct ManagedClientSession { - hub: EventHub, - cancellation: CancellationToken, - worker: Option>>, -} - -/// Outcome returned by a completed managed client session. -/// -/// These are runner-level lifecycle counters, not statistical summaries. Use -/// `irtt-stats` with emitted `ClientEvent` values for RTT, loss, IPDV, and -/// related summaries. -#[must_use = "managed session outcomes contain completion status and counters"] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SessionOutcome { - /// Why the managed loop stopped. - pub end_reason: SessionEndReason, - /// Number of echo requests sent by the client. - pub packets_sent: u64, - /// Number of first in-window echo replies received. - pub replies_received: u64, - /// Number of duplicate reply events emitted. - pub duplicates: u64, - /// Number of late reply events emitted. - pub late: u64, - /// Number of warning events emitted. - pub warning_events: u64, -} - -/// Reason a managed client session ended. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SessionEndReason { - /// The negotiated finite test duration completed and the client closed the - /// session normally. - TestComplete, - /// The peer ended the session with an authenticated close-flagged reply. - PeerClosed, - /// Cancellation was requested through [`ManagedClientSession::stop`] or by - /// dropping the session handle. - Cancelled, - /// The session was opened in [`RunMode::NoTest`](crate::RunMode) and - /// completed after negotiation. - NoTestComplete, -} - -impl ManagedClient { - /// Start a managed session without creating an initial event subscription. - /// - /// The returned [`ManagedClientSession`] can still create subscriptions via - /// [`ManagedClientSession::subscribe`], but events emitted before a - /// subscriber is registered are not replayed. - pub fn start(config: ClientConfig) -> Result { - Self::start_inner(config, None).map(|(session, _)| session) - } - - /// Start a managed session and subscribe to events before the worker runs. - /// - /// The initial subscription receives the open lifecycle event and - /// subsequent session events, subject to its queue capacity and overflow - /// policy. - pub fn start_with_subscription( - config: ClientConfig, - subscriber_config: SubscriberConfig, - ) -> Result<(ManagedClientSession, EventSubscription), ClientError> { - let (session, subscription) = Self::start_inner(config, Some(subscriber_config))?; - Ok(( - session, - subscription.expect("initial subscription must be present"), - )) - } - - fn start_inner( - mut config: ClientConfig, - subscriber_config: Option, - ) -> Result<(ManagedClientSession, Option), ClientError> { - if config.socket_config.recv_timeout.is_none() - || config.socket_config.recv_timeout > Some(MANAGED_RECV_TIMEOUT) - { - config.socket_config.recv_timeout = Some(MANAGED_RECV_TIMEOUT); - } - - let hub = EventHub::new(); - let initial_subscription = subscriber_config - .map(|config| hub.subscribe(config)) - .transpose()?; - - let mut client = Client::connect(config)?; - let outcome = client.open()?; - publish_open_outcome(&hub, &outcome); - - let cancellation = CancellationToken::new(); - let worker_hub = hub.clone(); - let worker_cancellation = cancellation.clone(); - let worker = - thread::spawn(move || run_client_with_cleanup(client, worker_hub, worker_cancellation)); - - Ok(( - ManagedClientSession { - hub, - cancellation, - worker: Some(worker), - }, - initial_subscription, - )) - } -} - -impl ManagedClientSession { - /// Add another event subscriber to this running managed session. - /// - /// The subscription receives only events published after it is registered. - pub fn subscribe(&self, config: SubscriberConfig) -> Result { - self.hub.subscribe(config) - } - - /// Request cooperative cancellation of the managed session. - /// - /// Cancellation is observed by the worker loop between socket receive, - /// timeout polling, and send scheduling steps. Call [`join`](Self::join) to - /// wait for the worker and obtain the final [`SessionOutcome`]. - pub fn stop(&self) { - self.cancellation.cancel(); - } - - /// Wait for the managed worker thread to finish. - /// - /// On success, returns lifecycle counters for the completed session. If the - /// worker panicked, this returns [`ClientError::WorkerPanicked`]. Joining - /// also disconnects all event subscriptions while leaving already queued - /// events available to drain. - pub fn join(mut self) -> Result { - let worker = self - .worker - .take() - .expect("ManagedClientSession invariant violated: worker handle missing before join"); - match worker.join() { - Ok(outcome) => { - self.hub.disconnect_all(); - outcome - } - Err(_) => { - self.hub.disconnect_all(); - Err(ClientError::WorkerPanicked) - } - } - } -} - -impl Drop for ManagedClientSession { - fn drop(&mut self) { - self.cancellation.cancel(); - } -} - -fn publish_open_outcome(hub: &EventHub, outcome: &OpenOutcome) { - match outcome { - OpenOutcome::Started { event, .. } | OpenOutcome::NoTestCompleted { event, .. } => { - hub.publish(event.clone()); - } - } -} - -fn run_client( - mut client: Client, - hub: EventHub, - cancellation: CancellationToken, -) -> Result { - if client.is_run_complete() { - return Ok(SessionOutcome { - end_reason: SessionEndReason::NoTestComplete, - packets_sent: 0, - replies_received: 0, - duplicates: 0, - late: 0, - warning_events: 0, - }); - } - - let mut counters = OutcomeCounters::default(); - let mut cancelled = false; - - loop { - if cancellation.is_cancelled() { - cancelled = true; - publish_events( - &hub, - &mut counters, - client.recv_available(MANAGED_RECV_BUDGET)?, - ); - if !client.is_peer_closed() { - publish_events(&hub, &mut counters, client.poll_timeouts()?); - } - break; - } - - let send_deadline = client.next_send_deadline(); - if let Some(deadline) = send_deadline.filter(|deadline| *deadline <= Instant::now()) { - let events = client.send_managed_probe(deadline)?; - publish_events(&hub, &mut counters, events); - continue; - } - - publish_events( - &hub, - &mut counters, - client.recv_once_until(send_deadline, MANAGED_RECV_TIMEOUT)?, - ); - if client.is_peer_closed() { - break; - } - publish_events(&hub, &mut counters, client.poll_timeouts()?); - - if client.is_run_complete() { - break; - } - } - - if !cancelled { - drain_final_late_replies(&mut client, &hub, &mut counters)?; - } - - let packets_sent = client.packets_sent(); - if !client.is_peer_closed() { - let close_events = client.close()?; - publish_events(&hub, &mut counters, close_events); - } - - let end_reason = if cancelled { - SessionEndReason::Cancelled - } else if client.is_peer_closed() { - SessionEndReason::PeerClosed - } else { - SessionEndReason::TestComplete - }; - - Ok(SessionOutcome { - end_reason, - packets_sent, - replies_received: counters.replies_received, - duplicates: counters.duplicates, - late: counters.late, - warning_events: counters.warning_events, - }) -} - -fn run_client_with_cleanup( - client: Client, - hub: EventHub, - cancellation: CancellationToken, -) -> Result { - let outcome = run_client(client, hub.clone(), cancellation); - hub.disconnect_all(); - outcome -} - -fn drain_final_late_replies( - client: &mut Client, - hub: &EventHub, - counters: &mut OutcomeCounters, -) -> Result<(), ClientError> { - if !client.has_timed_out_metadata() { - return Ok(()); - } - - let deadline = Instant::now() + MANAGED_FINAL_DRAIN; - while Instant::now() < deadline && client.has_timed_out_metadata() { - if client.is_peer_closed() { - break; - } - - let mut published = false; - - let events = client.recv_available(MANAGED_RECV_BUDGET)?; - published |= !events.is_empty(); - publish_events(hub, counters, events); - - if client.is_peer_closed() { - break; - } - - let events = client.poll_timeouts()?; - published |= !events.is_empty(); - publish_events(hub, counters, events); - - if !published { - thread::sleep(IDLE_SLEEP); - } - } - Ok(()) -} - -fn publish_events(hub: &EventHub, counters: &mut OutcomeCounters, events: Vec) { - for event in events { - counters.observe(&event); - hub.publish(event); - } -} - -#[derive(Debug, Default)] -struct OutcomeCounters { - replies_received: u64, - duplicates: u64, - late: u64, - warning_events: u64, -} - -impl OutcomeCounters { - fn observe(&mut self, event: &ClientEvent) { - match event { - ClientEvent::EchoReply { .. } => self.replies_received += 1, - ClientEvent::DuplicateReply { .. } => self.duplicates += 1, - ClientEvent::LateReply { .. } => self.late += 1, - ClientEvent::Warning { .. } => self.warning_events += 1, - _ => {} - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{config::NegotiationPolicy, EventSubscriptionError, SubscriberOverflow}; - use irtt_proto::{ - echo_packet_len, - flags::{self, FLAG_OPEN, FLAG_REPLY}, - layout::PacketLayout, - Clock, Params, ReceivedStats, StampAt, TimestampFields, MAGIC, PROTOCOL_VERSION, - }; - use std::{ - net::{SocketAddr, UdpSocket}, - sync::mpsc, - }; - - const TOKEN: u64 = 0x1234_5678_90ab_cdef; - struct FakeServer { - addr: SocketAddr, - done: JoinHandle<()>, - } - - impl FakeServer { - fn join(self) { - self.done.join().unwrap(); - } - } - - fn test_echo_packet_len(hmac: bool, params: &Params) -> usize { - echo_packet_len(hmac, params) - .expect("managed runner test params must have a non-negative packet length") - } - - fn test_params(duration: Option, interval: Duration) -> Params { - Params { - protocol_version: PROTOCOL_VERSION, - duration_ns: duration.map_or(0, test_duration_ns_i64), - interval_ns: test_duration_ns_i64(interval), - length: 0, - received_stats: ReceivedStats::Both, - stamp_at: StampAt::Both, - clock: Clock::Both, - dscp: 0, - server_fill: None, - } - } - - fn test_duration_ns_i64(duration: Duration) -> i64 { - i64::try_from(duration.as_nanos()).expect("test duration fits i64 nanoseconds") - } - - fn config(addr: SocketAddr, duration: Option) -> ClientConfig { - ClientConfig { - server_addr: addr.to_string(), - duration, - interval: Duration::from_millis(10), - negotiation_policy: NegotiationPolicy::Strict, - open_timeouts: vec![Duration::from_millis(200)], - probe_timeout: Duration::from_millis(40), - ..ClientConfig::default() - } - } - - fn start_echo_server(params: Params) -> FakeServer { - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let addr = socket.local_addr().unwrap(); - let done = thread::spawn(move || { - let (_, peer) = recv_request(&socket); - socket - .send_to(&open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms), peer) - .unwrap(); - socket - .set_read_timeout(Some(Duration::from_millis(500))) - .unwrap(); - - while let Some((packet, peer)) = recv_request_timeout(&socket) { - if packet[3] & flags::FLAG_CLOSE != 0 { - break; - } - - let seq = u32::from_le_bytes(packet[12..16].try_into().unwrap()); - let ts = TimestampFields { - recv_wall: Some(1_000_000_000), - recv_mono: Some(100_000), - send_wall: Some(1_000_000_000), - send_mono: Some(100_000), - ..Default::default() - }; - socket - .send_to(&echo_reply_packet(TOKEN, seq, ¶ms, &ts), peer) - .unwrap(); - } - }); - FakeServer { addr, done } - } - - fn start_delayed_reply_server(params: Params, delay: Duration) -> FakeServer { - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let addr = socket.local_addr().unwrap(); - let done = thread::spawn(move || { - let (_, peer) = recv_request(&socket); - socket - .send_to(&open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms), peer) - .unwrap(); - socket - .set_read_timeout(Some(Duration::from_millis(500))) - .unwrap(); - - let Some((packet, peer)) = recv_request_timeout(&socket) else { - return; - }; - assert_eq!(packet[3] & flags::FLAG_CLOSE, 0); - assert!( - packet.len() >= 16, - "echo request too short: {}", - packet.len() - ); - let seq = u32::from_le_bytes(packet[12..16].try_into().unwrap()); - thread::sleep(delay); - let ts = TimestampFields { - recv_wall: Some(1_000_000_000), - recv_mono: Some(100_000), - send_wall: Some(1_001_000_000), - send_mono: Some(1_100_000), - ..Default::default() - }; - socket - .send_to(&echo_reply_packet(TOKEN, seq, ¶ms, &ts), peer) - .unwrap(); - - while let Some((packet, _)) = recv_request_timeout(&socket) { - if packet[3] & flags::FLAG_CLOSE != 0 { - break; - } - } - }); - FakeServer { addr, done } - } - - fn start_peer_close_server(params: Params) -> FakeServer { - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let addr = socket.local_addr().unwrap(); - let done = thread::spawn(move || { - let (_, peer) = recv_request(&socket); - socket - .send_to(&open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, ¶ms), peer) - .unwrap(); - socket - .set_read_timeout(Some(Duration::from_millis(500))) - .unwrap(); - - let Some((packet, peer)) = recv_request_timeout(&socket) else { - return; - }; - let seq = u32::from_le_bytes(packet[12..16].try_into().unwrap()); - let ts = TimestampFields { - recv_wall: Some(1_000_000_000), - recv_mono: Some(100_000), - send_wall: Some(1_000_000_000), - send_mono: Some(100_000), - ..Default::default() - }; - socket - .send_to( - &echo_reply_packet_with_flags( - TOKEN, - seq, - ¶ms, - &ts, - FLAG_REPLY | flags::FLAG_CLOSE, - ), - peer, - ) - .unwrap(); - - if let Some((packet, _)) = recv_request_timeout(&socket) { - panic!( - "managed cleanup must not send any packet after peer close; flags={:?} len={}", - packet.get(3).copied(), - packet.len() - ); - } - }); - FakeServer { addr, done } - } - - fn recv_request(socket: &UdpSocket) -> (Vec, SocketAddr) { - let mut buf = [0_u8; 2048]; - let (size, peer) = socket.recv_from(&mut buf).unwrap(); - (buf[..size].to_vec(), peer) - } - - fn recv_request_timeout(socket: &UdpSocket) -> Option<(Vec, SocketAddr)> { - let mut buf = [0_u8; 2048]; - socket - .recv_from(&mut buf) - .ok() - .map(|(size, peer)| (buf[..size].to_vec(), peer)) - } - - fn open_reply(flags: u8, token: u64, params: &Params) -> Vec { - let mut packet = Vec::new(); - packet.extend_from_slice(&MAGIC); - packet.push(flags); - packet.extend_from_slice(&token.to_le_bytes()); - packet.extend_from_slice(¶ms.encode()); - packet - } - - fn echo_reply_packet( - token: u64, - seq: u32, - params: &Params, - timestamps: &TimestampFields, - ) -> Vec { - echo_reply_packet_with_flags(token, seq, params, timestamps, FLAG_REPLY) - } - - fn echo_reply_packet_with_flags( - token: u64, - seq: u32, - params: &Params, - timestamps: &TimestampFields, - flags: u8, - ) -> Vec { - let layout = PacketLayout::echo(false, params); - let packet_len = test_echo_packet_len(false, params); - let mut packet = Vec::with_capacity(packet_len); - - packet.extend_from_slice(&MAGIC); - packet.push(flags); - packet.extend_from_slice(&token.to_le_bytes()); - packet.extend_from_slice(&seq.to_le_bytes()); - - if layout.recv_count { - packet.extend_from_slice(&42_u32.to_le_bytes()); - } - if layout.recv_window { - packet.extend_from_slice(&0x07_u64.to_le_bytes()); - } - if layout.recv_wall { - packet.extend_from_slice(×tamps.recv_wall.unwrap_or(0).to_le_bytes()); - } - if layout.recv_mono { - packet.extend_from_slice(×tamps.recv_mono.unwrap_or(0).to_le_bytes()); - } - if layout.midpoint_wall { - packet.extend_from_slice(×tamps.midpoint_wall.unwrap_or(0).to_le_bytes()); - } - if layout.midpoint_mono { - packet.extend_from_slice(×tamps.midpoint_mono.unwrap_or(0).to_le_bytes()); - } - if layout.send_wall { - packet.extend_from_slice(×tamps.send_wall.unwrap_or(0).to_le_bytes()); - } - if layout.send_mono { - packet.extend_from_slice(×tamps.send_mono.unwrap_or(0).to_le_bytes()); - } - - packet.resize(packet_len, 0); - packet - } - - fn recv_event_with_timeout(sub: &EventSubscription) -> ClientEvent { - let deadline = Instant::now() + Duration::from_secs(2); - loop { - match sub.try_recv() { - Ok(Some(event)) => return event, - Ok(None) if Instant::now() < deadline => { - thread::sleep(Duration::from_millis(1)); - } - Ok(None) => panic!("timed out waiting for managed client event"), - Err(err) => panic!("subscription ended while waiting for event: {err}"), - } - } - } - - fn collect_until_closed(sub: &EventSubscription) -> Vec { - let mut events = Vec::new(); - let deadline = Instant::now() + Duration::from_secs(2); - while Instant::now() < deadline { - match sub.try_recv() { - Ok(Some(event)) => { - let closed = matches!(event, ClientEvent::SessionClosed { .. }); - events.push(event); - if closed { - return events; - } - } - Ok(None) => thread::sleep(Duration::from_millis(1)), - Err(_) => return events, - } - } - panic!("timed out waiting for session close event"); - } - - #[test] - fn stop_is_idempotent() { - let server = start_echo_server(test_params(None, Duration::from_millis(10))); - let session = ManagedClient::start(config(server.addr, None)).unwrap(); - session.stop(); - session.stop(); - let outcome = session.join().unwrap(); - server.join(); - - assert_eq!(outcome.end_reason, SessionEndReason::Cancelled); - } - - #[test] - fn finite_managed_run_emits_session_probe_and_close_events() { - let duration = Duration::from_millis(35); - let server = start_echo_server(test_params(Some(duration), Duration::from_millis(10))); - let (session, sub) = ManagedClient::start_with_subscription( - config(server.addr, Some(duration)), - SubscriberConfig { - capacity: 16, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - let events = collect_until_closed(&sub); - let outcome = session.join().unwrap(); - server.join(); - - assert_eq!(outcome.end_reason, SessionEndReason::TestComplete); - assert!(outcome.packets_sent > 0); - assert!(outcome.replies_received > 0); - assert!(events - .iter() - .any(|event| matches!(event, ClientEvent::SessionStarted { .. }))); - assert!(events - .iter() - .any(|event| matches!(event, ClientEvent::EchoReply { .. }))); - assert!(events - .iter() - .any(|event| matches!(event, ClientEvent::SessionClosed { .. }))); - } - - #[test] - fn joining_before_consuming_preserves_queued_final_events() { - let duration = Duration::from_millis(25); - let server = start_echo_server(test_params(Some(duration), Duration::from_millis(10))); - let (session, sub) = ManagedClient::start_with_subscription( - config(server.addr, Some(duration)), - SubscriberConfig { - capacity: 32, - overflow: SubscriberOverflow::DropOldest, - }, - ) - .unwrap(); - - let outcome = session.join().unwrap(); - let mut events = Vec::new(); - loop { - match sub.try_recv() { - Ok(Some(event)) => events.push(event), - Ok(None) => panic!("joined managed subscription remained connected"), - Err(EventSubscriptionError::Disconnected) => break, - } - } - server.join(); - - assert_eq!(outcome.end_reason, SessionEndReason::TestComplete); - assert_eq!(sub.dropped_events(), 0); - assert!(matches!( - events.first(), - Some(ClientEvent::SessionStarted { .. }) - )); - assert!(matches!( - events.last(), - Some(ClientEvent::SessionClosed { .. }) - )); - } - - #[test] - fn finite_managed_run_drains_late_reply_after_timeout_before_close() { - let interval = Duration::from_millis(100); - let duration = interval; - let params = test_params(Some(duration), interval); - let server = start_delayed_reply_server(params, Duration::from_millis(60)); - let mut cfg = config(server.addr, Some(duration)); - cfg.interval = interval; - cfg.probe_timeout = Duration::from_millis(20); - - let (session, sub) = ManagedClient::start_with_subscription( - cfg, - SubscriberConfig { - capacity: 16, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - let events = collect_until_closed(&sub); - let outcome = session.join().unwrap(); - server.join(); - - assert_eq!(outcome.end_reason, SessionEndReason::TestComplete); - assert_eq!(outcome.packets_sent, 1); - assert_eq!(outcome.replies_received, 0); - assert_eq!(outcome.late, 1); - assert!(events - .iter() - .any(|event| matches!(event, ClientEvent::EchoLoss { seq: 0, .. }))); - assert!(events.iter().any(|event| matches!( - event, - ClientEvent::LateReply { - seq: 0, - sent_at: Some(_), - rtt: Some(_), - .. - } - ))); - let late_before_close = events - .iter() - .position(|event| matches!(event, ClientEvent::LateReply { rtt: Some(_), .. })); - let close = events - .iter() - .position(|event| matches!(event, ClientEvent::SessionClosed { .. })); - let late_before_close = late_before_close.expect("missing stats-eligible LateReply"); - let close = close.expect("missing SessionClosed"); - assert!(late_before_close < close); - } - - #[test] - fn peer_close_during_managed_run_is_successful() { - let server = start_peer_close_server(test_params(None, Duration::from_millis(10))); - let (session, sub) = ManagedClient::start_with_subscription( - config(server.addr, None), - SubscriberConfig { - capacity: 16, - overflow: SubscriberOverflow::DropNewest, - }, - ) - .unwrap(); - - let events = collect_until_closed(&sub); - let outcome = session.join().unwrap(); - server.join(); - - assert_eq!(outcome.end_reason, SessionEndReason::PeerClosed); - assert_eq!(outcome.packets_sent, 1); - assert_eq!(outcome.replies_received, 1); - assert!(events - .iter() - .any(|event| matches!(event, ClientEvent::EchoReply { .. }))); - assert!(events - .iter() - .any(|event| matches!(event, ClientEvent::SessionClosed { .. }))); - } - - #[test] - fn continuous_managed_run_can_be_stopped_cleanly() { - let server = start_echo_server(test_params(None, Duration::from_millis(10))); - let session = ManagedClient::start(config(server.addr, None)).unwrap(); - let sub = session - .subscribe(SubscriberConfig { - capacity: 16, - overflow: SubscriberOverflow::DropNewest, - }) - .unwrap(); - - loop { - if matches!(recv_event_with_timeout(&sub), ClientEvent::EchoReply { .. }) { - break; - } - } - session.stop(); - let events = collect_until_closed(&sub); - let outcome = session.join().unwrap(); - server.join(); - - assert_eq!(outcome.end_reason, SessionEndReason::Cancelled); - assert!(outcome.packets_sent > 0); - assert!(outcome.replies_received > 0); - assert!(events - .iter() - .any(|event| matches!(event, ClientEvent::SessionClosed { .. }))); - } - - #[test] - fn join_reports_worker_panic() { - let hub = EventHub::new(); - let cancellation = CancellationToken::new(); - let worker = thread::spawn(|| -> Result { - panic!("intentional managed worker panic") - }); - let session = ManagedClientSession { - hub, - cancellation, - worker: Some(worker), - }; - - assert!(matches!(session.join(), Err(ClientError::WorkerPanicked))); - } - - #[test] - fn no_test_managed_run_returns_no_test_outcome() { - use crate::RunMode; - - let params = test_params(Some(Duration::from_millis(10)), Duration::from_millis(10)); - let socket = UdpSocket::bind("127.0.0.1:0").unwrap(); - let addr = socket.local_addr().unwrap(); - let (tx, rx) = mpsc::channel(); - let done = thread::spawn(move || { - let (request, peer) = recv_request(&socket); - tx.send(request[3] & flags::FLAG_CLOSE != 0).unwrap(); - socket - .send_to( - &open_reply(FLAG_OPEN | FLAG_REPLY | flags::FLAG_CLOSE, 0, ¶ms), - peer, - ) - .unwrap(); - }); - - let mut cfg = config(addr, Some(Duration::from_millis(10))); - cfg.run_mode = RunMode::NoTest; - let session = ManagedClient::start(cfg).unwrap(); - assert!(rx.recv_timeout(Duration::from_secs(2)).unwrap()); - let outcome = session.join().unwrap(); - done.join().unwrap(); - - assert_eq!(outcome.end_reason, SessionEndReason::NoTestComplete); - assert_eq!(outcome.packets_sent, 0); - } -} diff --git a/crates/irtt-client/src/probe.rs b/crates/irtt-client/src/probe.rs index a1f4af9..682bcfb 100644 --- a/crates/irtt-client/src/probe.rs +++ b/crates/irtt-client/src/probe.rs @@ -145,6 +145,7 @@ impl TimedOutMap { self.insertion_order.clear(); } + #[cfg(test)] pub fn len(&self) -> usize { self.map.len() } diff --git a/crates/irtt-client/src/receive.rs b/crates/irtt-client/src/receive.rs index ca5fbd1..d65fc6e 100644 --- a/crates/irtt-client/src/receive.rs +++ b/crates/irtt-client/src/receive.rs @@ -15,14 +15,6 @@ pub(crate) struct ReceivedDatagram { pub(crate) meta: ReceiveMeta, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) struct ReceivedDatagramFrom { - pub(crate) len: usize, - pub(crate) source: SocketAddr, - pub(crate) received_at: ClientTimestamp, - pub(crate) meta: ReceiveMeta, -} - #[cfg(not(all(target_os = "linux", feature = "ancillary")))] pub(crate) fn recv_datagram( socket: &UdpSocket, @@ -38,22 +30,6 @@ pub(crate) fn recv_datagram( }) } -#[cfg(not(all(target_os = "linux", feature = "ancillary")))] -pub(crate) fn recv_datagram_from( - socket: &UdpSocket, - buf: &mut [u8], -) -> Result { - let (len, source) = socket.recv_from(buf)?; - let received_at = ClientTimestamp::now(); - - Ok(ReceivedDatagramFrom { - len, - source, - received_at, - meta: ReceiveMeta::default(), - }) -} - #[cfg(all(target_os = "linux", feature = "ancillary"))] pub(crate) fn recv_datagram( socket: &UdpSocket, @@ -88,14 +64,6 @@ pub(crate) fn try_recv_tokio_datagram( linux::try_recv_tokio_datagram(socket, buf) } -#[cfg(all(target_os = "linux", feature = "ancillary"))] -pub(crate) fn recv_datagram_from( - socket: &UdpSocket, - buf: &mut [u8], -) -> Result { - linux::recv_datagram_from(socket, buf) -} - #[cfg(not(all(target_os = "linux", feature = "ancillary")))] pub(crate) fn configure_receive_metadata( _socket: &UdpSocket, diff --git a/crates/irtt-client/src/receive/linux.rs b/crates/irtt-client/src/receive/linux.rs index 3509058..502e784 100644 --- a/crates/irtt-client/src/receive/linux.rs +++ b/crates/irtt-client/src/receive/linux.rs @@ -1,17 +1,13 @@ #![allow(unsafe_code)] use std::{ io, mem, - net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6, UdpSocket}, + net::{SocketAddr, UdpSocket}, os::fd::{AsRawFd, RawFd}, ptr, time::{Duration, SystemTime, UNIX_EPOCH}, }; -use crate::{ - metadata::ReceiveMeta, - receive::{ReceivedDatagram, ReceivedDatagramFrom}, - timing::ClientTimestamp, -}; +use crate::{metadata::ReceiveMeta, receive::ReceivedDatagram, timing::ClientTimestamp}; const CONTROL_LEN: usize = 128; @@ -109,6 +105,7 @@ fn recv_datagram_fd( }) } +#[cfg(any())] pub(crate) fn recv_datagram_from( socket: &UdpSocket, buf: &mut [u8], @@ -185,6 +182,7 @@ unsafe fn parse_receive_meta(msg: &libc::msghdr) -> ReceiveMeta { meta } +#[cfg(any())] unsafe fn socket_addr_from_storage( storage: &libc::sockaddr_storage, len: libc::socklen_t, diff --git a/crates/irtt-client/src/session/machine.rs b/crates/irtt-client/src/session/machine.rs index 1bae69d..8168903 100644 --- a/crates/irtt-client/src/session/machine.rs +++ b/crates/irtt-client/src/session/machine.rs @@ -480,6 +480,7 @@ impl SessionMachine { ) } + #[cfg(test)] pub(crate) fn has_timed_out_metadata(&self) -> bool { matches!( &self.state, diff --git a/crates/irtt-client/src/socket.rs b/crates/irtt-client/src/socket.rs index d3ba12b..b1ed560 100644 --- a/crates/irtt-client/src/socket.rs +++ b/crates/irtt-client/src/socket.rs @@ -164,44 +164,6 @@ pub(crate) fn connect_tokio_udp_socket( Ok(tokio::net::UdpSocket::from_std(socket)?) } -pub(crate) fn bind_unconnected_udp_socket( - config: &SocketConfig, - family_remote: SocketAddr, -) -> Result { - let domain = if family_remote.is_ipv4() { - Domain::IPV4 - } else { - Domain::IPV6 - }; - let socket = Socket::new(domain, Type::DGRAM, Some(Protocol::UDP))?; - - if config.ipv6_only && family_remote.is_ipv6() { - socket.set_only_v6(true)?; - } - let bind_addr = config.bind_addr.unwrap_or_else(|| { - if family_remote.is_ipv4() { - SocketAddr::from(([0, 0, 0, 0], 0)) - } else { - SocketAddr::from(([0_u16; 8], 0)) - } - }); - socket.bind(&bind_addr.into())?; - - let socket: UdpSocket = socket.into(); - configure_receive_metadata(&socket, family_remote).map_err(|source| { - ClientError::SocketOption { - operation: "enable receive metadata", - remote: family_remote, - source, - } - })?; - if let Some(ttl) = config.ttl { - apply_ttl_to_socket(&socket, family_remote, ttl)?; - } - socket.set_read_timeout(config.recv_timeout)?; - Ok(socket) -} - #[cfg(test)] mod tests { use super::*; -- 2.54.0 From 36eaa90e2adca83b2023ae1d1d7fb40e8531929e Mon Sep 17 00:00:00 2001 From: Nils Andreas Svee Date: Sun, 9 Aug 2026 21:51:20 +0700 Subject: [PATCH 4/5] fix(cli): harden managed frontend cutover --- README.md | 78 ++--- crates/irtt-cli/src/cmd/client/run.rs | 124 +++++-- crates/irtt-cli/src/cmd/tui/run.rs | 47 +-- crates/irtt-cli/src/cmd/tui/ui.rs | 320 ++++++++++++------- crates/irtt-cli/src/shared/client/session.rs | 45 ++- crates/irtt-client/src/client.rs | 4 +- crates/irtt-client/src/lib.rs | 7 +- crates/irtt-client/src/receive/linux.rs | 103 ------ 8 files changed, 399 insertions(+), 329 deletions(-) diff --git a/README.md b/README.md index 263425e..ed70cd9 100644 --- a/README.md +++ b/README.md @@ -173,73 +173,51 @@ Add it from a local checkout: ```toml [dependencies] -irtt-client = { path = "crates/irtt-client" } +irtt-client = { path = "crates/irtt-client", features = ["tokio"] } ``` -Minimal managed-client example: +Minimal managed-client example for a synchronous frontend: ```rust use std::time::Duration; -use irtt_client::{ - ClientConfig, ClientEvent, ManagedClient, SubscriberConfig, +use irtt_client::managed::{ + BlockingManagedClient, ManagedClientConfig, ManagedCompletionPolicy, + ManagedTargetConfig, TargetId, }; +use irtt_client::ClientConfig; fn main() -> Result<(), Box> { - let config = ClientConfig { - server_addr: "netperf-eu.bufferbloat.net:2112".to_owned(), - duration: Some(Duration::from_secs(10)), - interval: Duration::from_secs(1), - ..ClientConfig::default() + let config = ManagedClientConfig { + client: ClientConfig { + duration: Some(Duration::from_secs(10)), + interval: Duration::from_secs(1), + ..ClientConfig::default() + }, + completion: ManagedCompletionPolicy::FinishWhenQuiescent, + ..ManagedClientConfig::default() }; - - let (session, events) = - ManagedClient::start_with_subscription( - config, - SubscriberConfig::default(), - )?; - - while let Ok(event) = events.recv() { - match event { - ClientEvent::EchoReply { seq, rtt, .. } => { - println!( - "seq={seq} rtt_us={}", - rtt.effective.as_micros() - ); - } - ClientEvent::SessionClosed { .. } => break, - _ => {} - } - } - - let outcome = session.join()?; + let targets = vec![ManagedTargetConfig::new( + TargetId::from("edge"), + "netperf-eu.bufferbloat.net:2112", + )]; + let owner = BlockingManagedClient::start(config, targets)?; + let outcome = owner.join()?; println!("session ended: {:?}", outcome.end_reason); Ok(()) } ``` -The client library emits structured events for session lifecycle, sent probes, successful replies, loss, duplicates, late replies, warnings, and shutdown. +For presentation events, use `BlockingManagedClient::start_with_subscription`, +`ManagedEvent`, and `ManagedEventTryRecvError`, while consulting the durable +`handle.status()` and final outcome for authoritative state. Presentation +events are bounded and lossy, not reliable history. Dynamic desired-target +updates are available through `ManagedClientHandle::update_targets`. -For shared-socket multi-target integrations, `ManagedClientGroup` publishes -`ManagedGroupEvent::TargetFinished` exactly once for every terminal target -incarnation, including open/runtime failures, removal, cancellation, no-test -completion, and peer closure. Consumers can use `ManagedTargetEndReason` and -`ManagedTargetFailureKind` for outcome and exit decisions without parsing -diagnostic text. - -`ManagedGroupCompletionPolicy::AllTargetsComplete` supports finite static -groups. `ExplicitCancellation` supports long-lived controllers; replacing the -desired set with an empty vector removes current targets but leaves the group -idle and ready for a later `update_targets` call. `join()` reports aggregate -outcome counts and retains only the 256 most recent target outcomes, so -long-running target churn remains bounded. The aggregate -`peer_closed_target_outcomes` count includes peer closures omitted from that -recent snapshot. - -Managed event subscriptions remain bounded and nonblocking. When using a -dropping overflow policy, check `EventSubscription::dropped_events()` before -trusting a complete statistical summary. +`BlockingManagedClient` owns a dedicated Tokio runtime and thread for +synchronous callers. Callers that already own Tokio can drive +`ManagedClientTask` directly. ## Binaries and features diff --git a/crates/irtt-cli/src/cmd/client/run.rs b/crates/irtt-cli/src/cmd/client/run.rs index 64319a1..6f144be 100644 --- a/crates/irtt-cli/src/cmd/client/run.rs +++ b/crates/irtt-cli/src/cmd/client/run.rs @@ -26,8 +26,8 @@ use crate::shared::client::expected_probe_count; use crate::shared::client::{ is_shutdown_requested, session::{ - peer_close_run_error, request_managed_stop_for_peer_close, request_managed_stop_once, - should_print_final_summary, + drain_managed_events, peer_close_run_error, request_managed_stop_for_peer_close, + request_managed_stop_once, should_print_final_summary, ManagedDrainState, }, }; #[cfg(feature = "stats")] @@ -116,7 +116,6 @@ pub fn run_stream( let mut dropped_events = 0_u64; let mut interrupted = false; let mut stop_requested = false; - let mut peer_close_requested_stop = false; let mut subscription_closed = false; loop { if is_shutdown_requested(shutdown_requested) { @@ -129,12 +128,11 @@ pub fn run_stream( continuous, interrupted, handle.status().peer_closed_target_outcomes, - &mut peer_close_requested_stop, &mut stop_requested, ) { drop(handle.stop()); } - drain_events( + let drain_state = drain_events( &mut events, &mut stream_output, #[cfg(feature = "stats")] @@ -142,35 +140,20 @@ pub fn run_stream( &mut terminal_targets, &mut dropped_events, )?; + match drain_state { + ManagedDrainState::Empty => thread::sleep(MANAGED_EVENT_WAIT_SLICE), + ManagedDrainState::Closed => subscription_closed = true, + ManagedDrainState::BudgetExhausted => {} + } if handle.status().final_outcome.is_some() || subscription_closed { break; } - match events.try_recv() { - Ok(event) => process_event( - event, - &mut stream_output, - #[cfg(feature = "stats")] - &mut stats, - &mut terminal_targets, - )?, - Err(ManagedEventTryRecvError::Empty) => thread::sleep(MANAGED_EVENT_WAIT_SLICE), - Err(ManagedEventTryRecvError::Lagged(count)) => dropped_events += count, - Err(ManagedEventTryRecvError::Closed) => subscription_closed = true, - } } if interrupted { eprintln!("interrupted, closing managed run..."); } - drain_events( - &mut events, - &mut stream_output, - #[cfg(feature = "stats")] - &mut stats, - &mut terminal_targets, - &mut dropped_events, - )?; let outcome = owner.join()?; - drain_events( + drain_final_events( &mut events, &mut stream_output, #[cfg(feature = "stats")] @@ -275,6 +258,24 @@ fn drain_events( #[cfg(feature = "stats")] stats: &mut BTreeMap, terminal_targets: &mut HashSet, dropped_events: &mut u64, +) -> io::Result { + drain_managed_events(events, dropped_events, |event| { + process_event( + event, + stream_output, + #[cfg(feature = "stats")] + stats, + terminal_targets, + ) + }) +} + +fn drain_final_events( + events: &mut ManagedEventSubscription, + stream_output: &mut StreamOutput<'_, W>, + #[cfg(feature = "stats")] stats: &mut BTreeMap, + terminal_targets: &mut HashSet, + dropped_events: &mut u64, ) -> io::Result<()> { loop { match events.try_recv() { @@ -286,19 +287,35 @@ fn drain_events( terminal_targets, )?, Err(ManagedEventTryRecvError::Empty | ManagedEventTryRecvError::Closed) => break, - Err(ManagedEventTryRecvError::Lagged(count)) => *dropped_events += count, + Err(ManagedEventTryRecvError::Lagged(count)) => { + *dropped_events = dropped_events.saturating_add(count); + } } } Ok(()) } fn report_target_failure(target: &ManagedTargetOutcome) { + for message in target_failure_messages(target) { + eprintln!("{message}"); + } +} + +fn target_failure_messages(target: &ManagedTargetOutcome) -> Vec { + let mut messages = Vec::with_capacity(2); if let ManagedTargetEndReason::Failed(failure) = &target.end_reason { - eprintln!( + messages.push(format!( "irtt-rs: target {} failed ({:?} {:?}): {}", target.target.id, failure.phase, failure.kind, failure.message - ); + )); } + if let Some(failure) = &target.cleanup_failure { + messages.push(format!( + "irtt-rs: target {} cleanup failed ({:?} {:?}): {}", + target.target.id, failure.phase, failure.kind, failure.message + )); + } + messages } fn dropped_event_warning(dropped_events: u64) -> Option { (dropped_events > 0).then(|| format!("irtt-rs: warning: dropped {dropped_events} managed run event{}; output and statistics may be incomplete", if dropped_events == 1 { "" } else { "s" })) @@ -419,14 +436,55 @@ fn finite_stats_memory_warning(args: &ClientArgs) -> Option { #[cfg(test)] mod tests { use super::*; + use irtt_client::managed::{ + ManagedTargetFailure, ManagedTargetFailureKind, ManagedTargetFailurePhase, TargetId, + }; + use std::sync::Arc; + #[test] - fn lagged_events_are_counted_exactly() { - let mut dropped = 0; - dropped += 7; - dropped += 3; + fn lagged_events_are_counted_saturatingly() { + let mut dropped = 0_u64; + dropped = dropped.saturating_add(7); + dropped = dropped.saturating_add(3); assert_eq!(dropped, 10); + assert_eq!(u64::MAX.saturating_add(1), u64::MAX); assert!(dropped_event_warning(dropped) .unwrap() .contains("dropped 10 managed run events")); } + + #[test] + fn target_failure_diagnostics_include_primary_and_cleanup_failures() { + let target = ManagedTargetOutcome { + target: TargetInstance { + id: TargetId::from("edge"), + generation: 1, + }, + server_addr: Arc::from("127.0.0.1:2112"), + remote: None, + end_reason: ManagedTargetEndReason::Failed(ManagedTargetFailure { + phase: ManagedTargetFailurePhase::Receiving, + kind: ManagedTargetFailureKind::Protocol, + message: "primary failure".into(), + }), + packets_sent: 0, + replies_received: 0, + duplicates: 0, + late: 0, + warning_events: 0, + cleanup_failure: Some(ManagedTargetFailure { + phase: ManagedTargetFailurePhase::Closing, + kind: ManagedTargetFailureKind::Socket, + message: "cleanup failure".into(), + }), + }; + + assert_eq!( + target_failure_messages(&target), + [ + "irtt-rs: target edge failed (Receiving Protocol): primary failure", + "irtt-rs: target edge cleanup failed (Closing Socket): cleanup failure", + ] + ); + } } diff --git a/crates/irtt-cli/src/cmd/tui/run.rs b/crates/irtt-cli/src/cmd/tui/run.rs index 5b08d88..3b8cf35 100644 --- a/crates/irtt-cli/src/cmd/tui/run.rs +++ b/crates/irtt-cli/src/cmd/tui/run.rs @@ -12,7 +12,8 @@ use crate::{ shared::client::{ is_shutdown_requested, session::{ - peer_close_run_error, request_managed_stop_for_peer_close, request_managed_stop_once, + drain_managed_events, peer_close_run_error, request_managed_stop_for_peer_close, + request_managed_stop_once, ManagedDrainState, }, }, }; @@ -72,7 +73,6 @@ pub fn run_tui( let handle = owner.handle(); let mut interrupted = false; let mut stop_requested = false; - let mut peer_close_requested_stop = false; let mut terminal_targets = HashSet::new(); let mut dropped_events = 0; let mut subscription_closed = false; @@ -96,28 +96,26 @@ pub fn run_tui( continuous, interrupted, handle.status().peer_closed_target_outcomes, - &mut peer_close_requested_stop, &mut stop_requested, ) { drop(handle.stop()); } - drain_tui_events( + let drain_state = drain_tui_events( &mut events, &mut state, &mut terminal_targets, &mut dropped_events, ); + match drain_state { + ManagedDrainState::Empty => { + thread::sleep(managed_tui_wait_duration(&next_render, state.paused)); + } + ManagedDrainState::Closed => subscription_closed = true, + ManagedDrainState::BudgetExhausted => {} + } if handle.status().final_outcome.is_some() || subscription_closed { break; } - match events.try_recv() { - Ok(event) => process_tui_event(event, &mut state, &mut terminal_targets), - Err(ManagedEventTryRecvError::Empty) => { - thread::sleep(managed_tui_wait_duration(&next_render, state.paused)) - } - Err(ManagedEventTryRecvError::Lagged(count)) => dropped_events += count, - Err(ManagedEventTryRecvError::Closed) => subscription_closed = true, - } render_if_due(&mut terminal, &state, &mut next_render, false)?; } if interrupted { @@ -126,14 +124,8 @@ pub fn run_tui( } state.set_status(TuiStatus::Closing); render_if_due(&mut terminal, &state, &mut next_render, true)?; - drain_tui_events( - &mut events, - &mut state, - &mut terminal_targets, - &mut dropped_events, - ); let outcome = owner.join()?; - drain_tui_events( + drain_final_tui_events( &mut events, &mut state, &mut terminal_targets, @@ -198,12 +190,27 @@ fn drain_tui_events( state: &mut TuiState, terminal_targets: &mut HashSet, dropped_events: &mut u64, +) -> ManagedDrainState { + drain_managed_events(events, dropped_events, |event| { + process_tui_event(event, state, terminal_targets); + Ok::<(), std::convert::Infallible>(()) + }) + .expect("processing TUI managed events is infallible") +} + +fn drain_final_tui_events( + events: &mut ManagedEventSubscription, + state: &mut TuiState, + terminal_targets: &mut HashSet, + dropped_events: &mut u64, ) { loop { match events.try_recv() { Ok(event) => process_tui_event(event, state, terminal_targets), Err(ManagedEventTryRecvError::Empty | ManagedEventTryRecvError::Closed) => break, - Err(ManagedEventTryRecvError::Lagged(count)) => *dropped_events += count, + Err(ManagedEventTryRecvError::Lagged(count)) => { + *dropped_events = dropped_events.saturating_add(count); + } } } } diff --git a/crates/irtt-cli/src/cmd/tui/ui.rs b/crates/irtt-cli/src/cmd/tui/ui.rs index dc1c64f..5f325e4 100644 --- a/crates/irtt-cli/src/cmd/tui/ui.rs +++ b/crates/irtt-cli/src/cmd/tui/ui.rs @@ -170,7 +170,7 @@ impl TuiState { idx }; - let (status, recent, warning) = match &outcome.end_reason { + let (status, recent, primary_warning) = match &outcome.end_reason { ManagedTargetEndReason::TestComplete => { (TargetStatus::Closed, "test completed".to_owned(), None) } @@ -203,6 +203,19 @@ impl TuiState { } }; + let cleanup_warning = outcome.cleanup_failure.as_ref().map(|failure| { + format!( + "cleanup failed ({:?} {:?}): {}", + failure.phase, failure.kind, failure.message + ) + }); + let warning = match (primary_warning, cleanup_warning.as_ref()) { + (Some(primary), Some(cleanup)) => Some(format!("{primary}; {cleanup}")), + (Some(primary), None) => Some(primary), + (None, Some(cleanup)) => Some(cleanup.clone()), + (None, None) => None, + }; + if let Some(target) = self.targets.get_mut(idx) { target.remote = outcome.remote.map(|remote| remote.to_string()); target.status = status; @@ -214,6 +227,9 @@ impl TuiState { self.last_warning = Some(warning); } self.push_event(format!("{label}: {recent}")); + if let Some(cleanup_warning) = cleanup_warning { + self.push_event(format!("{label}: {cleanup_warning}")); + } if self .targets @@ -2019,15 +2035,18 @@ fn duration_ns(value: Duration) -> i128 { i128::try_from(value.as_nanos()).unwrap_or(i128::MAX) } -#[cfg(any())] +#[cfg(test)] mod tests { use super::*; + use irtt_client::managed::{ + ManagedTargetFailure, ManagedTargetFailureKind, ManagedTargetFailurePhase, TargetId, + }; use irtt_client::{ - ClientTimestamp, OneWayDelaySample, PacketMeta, RttSample, ServerTiming, TargetId, - WarningKind, + ClientTimestamp, OneWayDelaySample, PacketMeta, RttSample, ServerTiming, WarningKind, }; use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, + sync::Arc, time::UNIX_EPOCH, }; @@ -2131,23 +2150,25 @@ mod tests { &state.targets[0] } - fn target_event(label: &str, event: ClientEvent) -> UiEventScope { - UiEventScope { - target: TargetId::from(label), - event, + fn target_instance(label: &str) -> TargetInstance { + TargetInstance { + id: TargetId::from(label), + generation: 1, } } fn target_outcome(label: &str, end_reason: ManagedTargetEndReason) -> ManagedTargetOutcome { ManagedTargetOutcome { - id: TargetId::from(label), - remote: remote(), + target: target_instance(label), + server_addr: Arc::from("127.0.0.1:2112"), + remote: Some(remote()), end_reason, packets_sent: 0, replies_received: 0, duplicates: 0, late: 0, warning_events: 0, + cleanup_failure: None, } } @@ -2165,7 +2186,8 @@ mod tests { #[test] fn session_started_sets_remote_session_and_running_status() { let mut state = TuiState::default(); - state.process_event(&ClientEvent::SessionStarted { + let target = target_instance("target"); + let event = ClientEvent::SessionStarted { remote: remote(), token: 0xabc, negotiated: NegotiatedParams { @@ -2173,7 +2195,8 @@ mod tests { restrictions: Vec::new(), }, at: ts(Duration::ZERO), - }); + }; + state.process_target_event(&target, &event); let target = primary_target(&state); assert_eq!(target.remote.as_deref(), Some("127.0.0.1:2112")); @@ -2185,8 +2208,10 @@ mod tests { #[test] fn echo_reply_appends_bounded_primary_history() { let mut state = TuiState::default(); + let target = target_instance("target"); for seq in 0..(HISTORY_LIMIT as u32 + 3) { - state.process_event(&reply(seq, i128::from(seq) * 1_000)); + let event = reply(seq, i128::from(seq) * 1_000); + state.process_target_event(&target, &event); } let target = primary_target(&state); @@ -2205,13 +2230,15 @@ mod tests { #[test] fn duplicate_and_late_replies_do_not_append_primary_history() { let mut state = TuiState::default(); - state.process_event(&ClientEvent::DuplicateReply { + let target = target_instance("target"); + let duplicate = ClientEvent::DuplicateReply { seq: 7, remote: remote(), received_at: ts(Duration::from_secs(1)), bytes: 64, - }); - state.process_event(&ClientEvent::LateReply { + }; + state.process_target_event(&target, &duplicate); + let late = ClientEvent::LateReply { seq: 8, highest_seen: 9, remote: remote(), @@ -2223,7 +2250,8 @@ mod tests { received_stats: None, bytes: 64, packet_meta: PacketMeta::default(), - }); + }; + state.process_target_event(&target, &late); let target = primary_target(&state); assert!(target.graph_history.is_empty()); @@ -2250,7 +2278,9 @@ mod tests { let mut state = TuiState::with_target_labels(TuiConfig::default(), ["a".to_owned(), "b".to_owned()]); - state.process_target_event(&target_event("b", reply(11, 2_500_000))); + let target = target_instance("b"); + let event = reply(11, 2_500_000); + state.process_target_event(&target, &event); assert!(state.targets[0].graph_history.is_empty()); assert_eq!(state.targets[0].stats.snapshot().packets.unique_replies, 0); @@ -2268,31 +2298,29 @@ mod tests { let mut state = TuiState::with_target_labels(TuiConfig::default(), ["a".to_owned(), "b".to_owned()]); - state.process_target_event(&target_event( - "a", - ClientEvent::DuplicateReply { - seq: 7, - remote: remote(), - received_at: ts(Duration::from_secs(1)), - bytes: 64, - }, - )); - state.process_target_event(&target_event( - "b", - ClientEvent::LateReply { - seq: 8, - highest_seen: 9, - remote: remote(), - sent_at: Some(ts(Duration::from_secs(1))), - received_at: ts(Duration::from_secs(2)), - rtt: Some(rtt(2_000_000)), - server_timing: None, - one_way: None, - received_stats: None, - bytes: 64, - packet_meta: PacketMeta::default(), - }, - )); + let a = target_instance("a"); + let duplicate = ClientEvent::DuplicateReply { + seq: 7, + remote: remote(), + received_at: ts(Duration::from_secs(1)), + bytes: 64, + }; + state.process_target_event(&a, &duplicate); + let b = target_instance("b"); + let late = ClientEvent::LateReply { + seq: 8, + highest_seen: 9, + remote: remote(), + sent_at: Some(ts(Duration::from_secs(1))), + received_at: ts(Duration::from_secs(2)), + rtt: Some(rtt(2_000_000)), + server_timing: None, + one_way: None, + received_stats: None, + bytes: 64, + packet_meta: PacketMeta::default(), + }; + state.process_target_event(&b, &late); assert!(state.targets[0].graph_history.is_empty()); assert!(state.targets[1].graph_history.is_empty()); @@ -2306,33 +2334,29 @@ mod tests { let mut state = TuiState::with_target_labels(TuiConfig::default(), ["a".to_owned(), "b".to_owned()]); - state.process_target_event(&target_event( - "a", - ClientEvent::EchoLoss { - seq: 3, - sent_at: ts(Duration::from_millis(3)), - timeout_at: Instant::now(), + let a = target_instance("a"); + let loss = ClientEvent::EchoLoss { + seq: 3, + sent_at: ts(Duration::from_millis(3)), + timeout_at: Instant::now(), + }; + state.process_target_event(&a, &loss); + let b = target_instance("b"); + let warning = ClientEvent::Warning { + kind: WarningKind::WrongToken, + message: "wrong token".to_owned(), + at: ts(Duration::ZERO), + }; + state.process_target_event(&b, &warning); + let no_test = ClientEvent::NoTestCompleted { + remote: remote(), + negotiated: NegotiatedParams { + params: irtt_proto::Params::default(), + restrictions: Vec::new(), }, - )); - state.process_target_event(&target_event( - "b", - ClientEvent::Warning { - kind: WarningKind::WrongToken, - message: "wrong token".to_owned(), - at: ts(Duration::ZERO), - }, - )); - state.process_target_event(&target_event( - "a", - ClientEvent::NoTestCompleted { - remote: remote(), - negotiated: NegotiatedParams { - params: irtt_proto::Params::default(), - restrictions: Vec::new(), - }, - at: ts(Duration::ZERO), - }, - )); + at: ts(Duration::ZERO), + }; + state.process_target_event(&a, &no_test); assert_eq!(state.targets[0].stats.snapshot().events.loss_events, 1); assert_eq!(state.targets[0].status, TargetStatus::NoTest); @@ -2349,30 +2373,28 @@ mod tests { let mut state = TuiState::with_target_labels(TuiConfig::default(), ["a".to_owned(), "b".to_owned()]); - state.process_target_event(&target_event( - "a", - ClientEvent::NoTestCompleted { - remote: remote(), - negotiated: NegotiatedParams { - params: irtt_proto::Params::default(), - restrictions: Vec::new(), - }, - at: ts(Duration::ZERO), + let a = target_instance("a"); + let no_test = ClientEvent::NoTestCompleted { + remote: remote(), + negotiated: NegotiatedParams { + params: irtt_proto::Params::default(), + restrictions: Vec::new(), }, - )); + at: ts(Duration::ZERO), + }; + state.process_target_event(&a, &no_test); assert_eq!(state.targets[0].status, TargetStatus::NoTest); assert_eq!(state.targets[1].status, TargetStatus::Opening); assert_eq!(state.status, TuiStatus::Running); - state.process_target_event(&target_event( - "b", - ClientEvent::SessionClosed { - remote: remote(), - token: 0xabc, - at: ts(Duration::ZERO), - }, - )); + let b = target_instance("b"); + let closed = ClientEvent::SessionClosed { + remote: remote(), + token: 0xabc, + at: ts(Duration::ZERO), + }; + state.process_target_event(&b, &closed); assert_eq!(state.targets[1].status, TargetStatus::Closed); assert_eq!(state.status, TuiStatus::Complete); @@ -2382,14 +2404,15 @@ mod tests { fn terminal_failure_marks_the_correct_target_failed() { let mut state = TuiState::with_target_labels(TuiConfig::default(), ["a".to_owned(), "b".to_owned()]); - let failure = irtt_client::ManagedTargetFailure { - kind: irtt_client::ManagedTargetFailureKind::OpeningTimeout, - message: "all open requests timed out".to_owned(), + let failure = ManagedTargetFailure { + phase: ManagedTargetFailurePhase::Opening, + kind: ManagedTargetFailureKind::Timeout, + message: "all open requests timed out".into(), }; state.process_target_outcome(&target_outcome( "b", - ManagedTargetEndReason::OpenFailed(failure), + ManagedTargetEndReason::Failed(failure), )); assert_eq!(state.targets[0].status, TargetStatus::Opening); @@ -2397,7 +2420,7 @@ mod tests { assert!(state.targets[1] .last_warning .as_deref() - .is_some_and(|warning| warning.contains("opening timeout"))); + .is_some_and(|warning| warning.contains("Timeout"))); } #[test] @@ -2407,9 +2430,10 @@ mod tests { state.process_target_outcome(&target_outcome("a", ManagedTargetEndReason::PeerClosed)); state.process_target_outcome(&target_outcome( "b", - ManagedTargetEndReason::Failed(irtt_client::ManagedTargetFailure { - kind: irtt_client::ManagedTargetFailureKind::RuntimeProtocol, - message: "pending probe limit exceeded".to_owned(), + ManagedTargetEndReason::Failed(ManagedTargetFailure { + phase: ManagedTargetFailurePhase::Receiving, + kind: ManagedTargetFailureKind::Protocol, + message: "pending probe limit exceeded".into(), }), )); @@ -2422,6 +2446,79 @@ mod tests { .is_some_and(|warning| warning.contains("pending probe limit"))); } + #[test] + fn cleanup_failure_warns_without_changing_successful_target_status() { + let mut state = TuiState::with_target_labels(TuiConfig::default(), ["a".to_owned()]); + let mut outcome = target_outcome("a", ManagedTargetEndReason::TestComplete); + outcome.remote = None; + outcome.cleanup_failure = Some(ManagedTargetFailure { + phase: ManagedTargetFailurePhase::Closing, + kind: ManagedTargetFailureKind::Socket, + message: "close datagram failed".into(), + }); + + state.process_target_outcome(&outcome); + + assert_eq!(state.targets[0].status, TargetStatus::Closed); + assert_eq!(state.targets[0].remote, None); + assert!(state.targets[0] + .last_warning + .as_deref() + .is_some_and(|warning| warning.contains("cleanup failed (Closing Socket)"))); + assert!(state + .recent_events + .iter() + .any(|event| event.contains("cleanup failed (Closing Socket)"))); + } + + #[test] + fn primary_and_cleanup_failures_are_both_visible() { + let mut state = TuiState::with_target_labels(TuiConfig::default(), ["a".to_owned()]); + let mut outcome = target_outcome( + "a", + ManagedTargetEndReason::Failed(ManagedTargetFailure { + phase: ManagedTargetFailurePhase::Receiving, + kind: ManagedTargetFailureKind::Protocol, + message: "primary failure".into(), + }), + ); + outcome.cleanup_failure = Some(ManagedTargetFailure { + phase: ManagedTargetFailurePhase::Closing, + kind: ManagedTargetFailureKind::Socket, + message: "cleanup failure".into(), + }); + + state.process_target_outcome(&outcome); + + assert_eq!(state.targets[0].status, TargetStatus::Failed); + assert!(state.targets[0] + .last_warning + .as_deref() + .is_some_and(|warning| warning.contains("primary failure") + && warning.contains("cleanup failure"))); + assert!(state + .recent_events + .iter() + .any(|event| event.contains("target failed: Protocol: primary failure"))); + assert!(state + .recent_events + .iter() + .any(|event| event.contains("cleanup failed (Closing Socket): cleanup failure"))); + } + + #[test] + fn managed_outcomes_map_stopped_removed_and_replaced_statuses() { + for (reason, expected) in [ + (ManagedTargetEndReason::Stopped, TargetStatus::Stopped), + (ManagedTargetEndReason::Removed, TargetStatus::Removed), + (ManagedTargetEndReason::Replaced, TargetStatus::Replaced), + ] { + let mut state = TuiState::with_target_labels(TuiConfig::default(), ["a".to_owned()]); + state.process_target_outcome(&target_outcome("a", reason)); + assert_eq!(state.targets[0].status, expected); + } + } + #[test] fn run_error_preserves_peer_closed_target_status() { let mut state = TuiState::with_target_labels(TuiConfig::default(), ["a".to_owned()]); @@ -2446,9 +2543,10 @@ mod tests { for label in ["a", "b"] { state.process_target_outcome(&target_outcome( label, - ManagedTargetEndReason::OpenFailed(irtt_client::ManagedTargetFailure { - kind: irtt_client::ManagedTargetFailureKind::OpeningProtocol, - message: "server returned a zero connection token".to_owned(), + ManagedTargetEndReason::Failed(ManagedTargetFailure { + phase: ManagedTargetFailurePhase::Opening, + kind: ManagedTargetFailureKind::Protocol, + message: "server returned a zero connection token".into(), }), )); } @@ -2465,8 +2563,12 @@ mod tests { let mut state = TuiState::with_target_labels(TuiConfig::default(), ["a".to_owned(), "b".to_owned()]); - state.process_target_event(&target_event("a", reply(1, 1_000_000))); - state.process_target_event(&target_event("b", reply(2, 2_000_000))); + let a = target_instance("a"); + let a_reply = reply(1, 1_000_000); + state.process_target_event(&a, &a_reply); + let b = target_instance("b"); + let b_reply = reply(2, 2_000_000); + state.process_target_event(&b, &b_reply); state.clear_visible_history(); assert!(state.targets[0].graph_history.is_empty()); @@ -2480,11 +2582,13 @@ mod tests { #[test] fn warning_updates_recent_events_and_last_warning() { let mut state = TuiState::default(); - state.process_event(&ClientEvent::Warning { + let target = target_instance("target"); + let event = ClientEvent::Warning { kind: WarningKind::WrongToken, message: "wrong token".to_owned(), at: ts(Duration::ZERO), - }); + }; + state.process_target_event(&target, &event); assert_eq!( state.last_warning.as_deref(), @@ -2501,7 +2605,7 @@ mod tests { fn dropped_events_mark_tui_statistics_incomplete() { let mut state = TuiState::default(); - state.mark_dropped_group_events(9); + state.mark_dropped_managed_events(9); assert_eq!(state.dropped_events, 9); assert!(state @@ -2511,20 +2615,20 @@ mod tests { assert!(state .recent_events .back() - .is_some_and(|event| event.contains("dropped 9 managed group events"))); + .is_some_and(|event| event.contains("dropped 9 managed run events"))); } #[test] fn dropped_single_client_events_are_visible() { let mut state = TuiState::default(); - state.mark_dropped_client_events(3); + state.mark_dropped_managed_events(3); assert_eq!(state.dropped_events, 3); assert!(state .recent_events .back() - .is_some_and(|event| event.contains("dropped 3 managed client events"))); + .is_some_and(|event| event.contains("dropped 3 managed run events"))); } #[test] @@ -2803,7 +2907,9 @@ mod tests { #[test] fn sample_details_preserve_signed_values() { let mut state = TuiState::default(); - state.process_event(&reply(4, -1_250_000)); + let target = target_instance("target"); + let event = reply(4, -1_250_000); + state.process_target_event(&target, &event); let target = primary_target(&state); let sample = target.last_sample.unwrap(); diff --git a/crates/irtt-cli/src/shared/client/session.rs b/crates/irtt-cli/src/shared/client/session.rs index 67393fe..1c2c381 100644 --- a/crates/irtt-cli/src/shared/client/session.rs +++ b/crates/irtt-cli/src/shared/client/session.rs @@ -1,5 +1,40 @@ use std::sync::atomic::{AtomicBool, Ordering}; +#[cfg(feature = "client-runtime")] +use irtt_client::managed::{ManagedEvent, ManagedEventSubscription, ManagedEventTryRecvError}; + +#[cfg(feature = "client-runtime")] +pub(crate) const MANAGED_EVENT_WORK_BUDGET: usize = 128; + +#[cfg(feature = "client-runtime")] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ManagedDrainState { + Empty, + Closed, + BudgetExhausted, +} + +/// Drain a bounded amount of lossy presentation work before returning to the +/// frontend control loop. +#[cfg(feature = "client-runtime")] +pub(crate) fn drain_managed_events( + events: &mut ManagedEventSubscription, + dropped_events: &mut u64, + mut process: impl FnMut(ManagedEvent) -> Result<(), E>, +) -> Result { + for _ in 0..MANAGED_EVENT_WORK_BUDGET { + match events.try_recv() { + Ok(event) => process(event)?, + Err(ManagedEventTryRecvError::Lagged(count)) => { + *dropped_events = dropped_events.saturating_add(count); + } + Err(ManagedEventTryRecvError::Empty) => return Ok(ManagedDrainState::Empty), + Err(ManagedEventTryRecvError::Closed) => return Ok(ManagedDrainState::Closed), + } + } + Ok(ManagedDrainState::BudgetExhausted) +} + pub fn is_shutdown_requested(shutdown_requested: &AtomicBool) -> bool { shutdown_requested.load(Ordering::Relaxed) } @@ -31,13 +66,11 @@ pub fn request_managed_stop_for_peer_close( continuous: bool, interrupted: bool, peer_closed_target_outcomes: u64, - peer_close_requested_stop: &mut bool, stop_requested: &mut bool, ) -> bool { if !continuous || interrupted || peer_closed_target_outcomes == 0 { return false; } - *peer_close_requested_stop = true; request_managed_stop_once(stop_requested) } @@ -70,46 +103,38 @@ mod tests { #[test] fn grouped_peer_close_stop_policy_is_run_mode_and_one_shot() { - let mut peer_close_requested_stop = false; let mut stop_requested = false; assert!(!request_managed_stop_for_peer_close( false, false, 1, - &mut peer_close_requested_stop, &mut stop_requested, )); assert!(!request_managed_stop_for_peer_close( true, true, 1, - &mut peer_close_requested_stop, &mut stop_requested, )); assert!(!request_managed_stop_for_peer_close( true, false, 0, - &mut peer_close_requested_stop, &mut stop_requested, )); - assert!(!peer_close_requested_stop); assert!(!stop_requested); assert!(request_managed_stop_for_peer_close( true, false, 1, - &mut peer_close_requested_stop, &mut stop_requested, )); - assert!(peer_close_requested_stop); assert!(stop_requested); assert!(!request_managed_stop_for_peer_close( true, false, 1, - &mut peer_close_requested_stop, &mut stop_requested, )); } diff --git a/crates/irtt-client/src/client.rs b/crates/irtt-client/src/client.rs index a900688..7ed52c9 100644 --- a/crates/irtt-client/src/client.rs +++ b/crates/irtt-client/src/client.rs @@ -94,8 +94,8 @@ use crate::{ /// /// `Client` exposes the protocol steps directly: connect a UDP socket, open a /// session, send probes, receive replies, poll timeouts, and close. Callers -/// that do not need to own this loop can use [`ManagedClient`](crate::ManagedClient) -/// instead. +/// that do not need to own this loop can use the unified managed API in +/// [`crate::managed`] when the `tokio` feature is enabled. #[derive(Debug)] pub struct Client { runtime: SessionMachine, diff --git a/crates/irtt-client/src/lib.rs b/crates/irtt-client/src/lib.rs index 742352e..13e1c17 100644 --- a/crates/irtt-client/src/lib.rs +++ b/crates/irtt-client/src/lib.rs @@ -14,10 +14,9 @@ //! values usually indicate client/server clock skew. //! //! [`Client`] is the runtime-free low-level blocking adapter. With the -//! `tokio` feature, [`AsyncClient`] provides the corresponding Tokio adapter; -//! [`managed`] contains the unified managed task/handle implementation and -//! [`managed::BlockingManagedClient`] owns its dedicated runtime thread for -//! synchronous managed callers. +//! `tokio` feature, `AsyncClient` provides the corresponding Tokio adapter; +//! [`managed`] contains the unified managed task/handle implementation, +//! including the dedicated-runtime blocking frontend for synchronous callers. //! #![cfg_attr( not(all(target_os = "linux", feature = "ancillary")), diff --git a/crates/irtt-client/src/receive/linux.rs b/crates/irtt-client/src/receive/linux.rs index 502e784..dd3384b 100644 --- a/crates/irtt-client/src/receive/linux.rs +++ b/crates/irtt-client/src/receive/linux.rs @@ -105,62 +105,6 @@ fn recv_datagram_fd( }) } -#[cfg(any())] -pub(crate) fn recv_datagram_from( - socket: &UdpSocket, - buf: &mut [u8], -) -> Result { - let mut control = ControlBuffer::new(); - let mut name: libc::sockaddr_storage = unsafe { mem::zeroed() }; - let mut iov = libc::iovec { - iov_base: buf.as_mut_ptr().cast(), - iov_len: buf.len(), - }; - let mut msg: libc::msghdr = unsafe { mem::zeroed() }; - msg.msg_name = (&mut name as *mut libc::sockaddr_storage).cast(); - msg.msg_namelen = mem::size_of::() as libc::socklen_t; - msg.msg_iov = &mut iov; - msg.msg_iovlen = 1; - msg.msg_control = control.as_mut_ptr().cast(); - msg.msg_controllen = control.len(); - - let len = unsafe { - // SAFETY: `msg` points to one writable iovec backed by `buf`, a - // writable source-address buffer, and a writable control buffer. All - // buffers live until `recvmsg` returns. The socket file descriptor is - // borrowed from a valid `UdpSocket`. - libc::recvmsg(socket.as_raw_fd(), &mut msg, 0) - }; - if len < 0 { - return Err(io::Error::last_os_error()); - } - let len = usize::try_from(len).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidData, - "recvmsg returned an unrepresentable datagram length", - ) - })?; - let received_at = ClientTimestamp::now(); - let source = unsafe { - // SAFETY: `name` and `msg_namelen` were initialized by a successful - // `recvmsg` call above. - socket_addr_from_storage(&name, msg.msg_namelen)? - }; - let meta = unsafe { - // SAFETY: `msg` was initialized by a successful `recvmsg` call. The - // parser only reads cmsghdr entries within `msg_controllen` and ignores - // short or unrelated control messages. - parse_receive_meta(&msg) - }; - - Ok(ReceivedDatagramFrom { - len, - source, - received_at, - meta, - }) -} - unsafe fn parse_receive_meta(msg: &libc::msghdr) -> ReceiveMeta { let mut meta = ReceiveMeta::default(); let mut cmsg = libc::CMSG_FIRSTHDR(msg); @@ -182,53 +126,6 @@ unsafe fn parse_receive_meta(msg: &libc::msghdr) -> ReceiveMeta { meta } -#[cfg(any())] -unsafe fn socket_addr_from_storage( - storage: &libc::sockaddr_storage, - len: libc::socklen_t, -) -> io::Result { - match i32::from(storage.ss_family) { - libc::AF_INET => { - if usize::try_from(len).unwrap_or(0) < mem::size_of::() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "short IPv4 source address from recvmsg", - )); - } - let addr = ptr::read_unaligned( - (storage as *const libc::sockaddr_storage).cast::(), - ); - let ip = Ipv4Addr::from(u32::from_be(addr.sin_addr.s_addr)); - let port = u16::from_be(addr.sin_port); - Ok(SocketAddr::V4(SocketAddrV4::new(ip, port))) - } - libc::AF_INET6 => { - if usize::try_from(len).unwrap_or(0) < mem::size_of::() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "short IPv6 source address from recvmsg", - )); - } - let addr = ptr::read_unaligned( - (storage as *const libc::sockaddr_storage).cast::(), - ); - let ip = Ipv6Addr::from(addr.sin6_addr.s6_addr); - let port = u16::from_be(addr.sin6_port); - let flowinfo = u32::from_be(addr.sin6_flowinfo); - Ok(SocketAddr::V6(SocketAddrV6::new( - ip, - port, - flowinfo, - addr.sin6_scope_id, - ))) - } - family => Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("unsupported source address family from recvmsg: {family}"), - )), - } -} - fn is_traffic_class_cmsg(cmsg: &libc::cmsghdr) -> bool { (cmsg.cmsg_level == libc::IPPROTO_IP && cmsg.cmsg_type == libc::IP_TOS) || (cmsg.cmsg_level == libc::IPPROTO_IPV6 && cmsg.cmsg_type == libc::IPV6_TCLASS) -- 2.54.0 From 33fe01f2529dc22651cb3cfe781d0bd9733d7b7f Mon Sep 17 00:00:00 2001 From: Nils Andreas Svee Date: Sun, 9 Aug 2026 22:03:45 +0700 Subject: [PATCH 5/5] refactor(irtt-client): gate functions that are only used with tokio enabled --- crates/irtt-client/src/client/schedule.rs | 3 +++ crates/irtt-client/src/session/machine.rs | 1 + 2 files changed, 4 insertions(+) diff --git a/crates/irtt-client/src/client/schedule.rs b/crates/irtt-client/src/client/schedule.rs index 8779b98..64193b0 100644 --- a/crates/irtt-client/src/client/schedule.rs +++ b/crates/irtt-client/src/client/schedule.rs @@ -79,6 +79,7 @@ impl ProbeSchedule { Ok(self.finish_commit(scheduled_at, next_send_at)) } + #[cfg(any(feature = "tokio", test))] pub(crate) fn preflight_managed_commit( &self, scheduled_at: Instant, @@ -122,6 +123,7 @@ fn next_probe_deadline( .ok_or(ClientError::DurationOverflow) } +#[cfg(any(feature = "tokio", test))] pub(crate) fn advance_cadence( deadline: Instant, interval: Duration, @@ -161,6 +163,7 @@ pub(crate) fn advance_cadence( Ok((scheduled_at, next_at, elapsed_slots)) } +#[cfg(any(feature = "tokio", test))] fn duration_from_nanos(nanos: u128) -> Result { const NANOS_PER_SECOND: u128 = 1_000_000_000; let seconds = diff --git a/crates/irtt-client/src/session/machine.rs b/crates/irtt-client/src/session/machine.rs index 8168903..7d3d8f9 100644 --- a/crates/irtt-client/src/session/machine.rs +++ b/crates/irtt-client/src/session/machine.rs @@ -488,6 +488,7 @@ impl SessionMachine { ) } + #[cfg(any(feature = "tokio", test))] pub(crate) fn packets_sent(&self) -> u64 { match &self.state { MachineState::Open(session) => session.packets_sent, -- 2.54.0