fix(managed): publish terminal outcomes for failed targets

Extend the managed group stream with authoritative per-target terminal outcomes and structured failure classifications. Final client events are published first, followed by exactly one TargetFinished outcome for success, peer close, no-test, opening/runtime failure, removal, or cancellation.

Scheduler-owned cleanup now stops the receiver and seals subscriptions on natural completion, eliminating the event-consumer/join circular wait while preserving queued drain. TargetEventSubscription now yields ManagedGroupEvent, so consumers must handle Client and TargetFinished variants.

Strict-timeout regressions cover mixed success/failure, all-open failure, runtime failure, no-test ordering, removal, cancellation, and disconnection before join.
This commit is contained in:
Nils Andreas Svee committed 2026-07-28 00:07:18 +07:00
1 parent 3e0047e505
commit 97aac829ec
5 files changed
+526 -110

No files matched your search

+19 -16
View File
@@ -10,7 +10,7 @@ use std::{
use irtt_client::{ use irtt_client::{
ClientEvent, EventSubscriptionError, ManagedClientGroup, ManagedClientGroupConfig, ClientEvent, EventSubscriptionError, ManagedClientGroup, ManagedClientGroupConfig,
ManagedGroupEndReason, SubscriberConfig, SubscriberOverflow, TargetEvent, ManagedGroupEndReason, ManagedGroupEvent, SubscriberConfig, SubscriberOverflow, TargetEvent,
}; };
#[cfg(all(test, feature = "stats"))] #[cfg(all(test, feature = "stats"))]
use irtt_client::{ClientTimestamp, PacketMeta, RttSample, SignedDuration}; use irtt_client::{ClientTimestamp, PacketMeta, RttSample, SignedDuration};
@@ -324,25 +324,33 @@ fn run_group_stream(
} }
match events.try_recv() { match events.try_recv() {
Ok(Some(target_event)) => { Ok(Some(group_event)) => {
last_event_at = Instant::now(); last_event_at = Instant::now();
saw_target_event = true; saw_target_event = true;
let label = target_event.target.as_str(); match group_event {
if is_terminal_target_event(&target_event.event) { ManagedGroupEvent::Client(target_event) => {
terminal_targets.insert(label.to_owned());
}
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
{ {
let label = target_event.target.as_str();
let stats = stats let stats = stats
.entry(label.to_owned()) .entry(label.to_owned())
.or_insert_with(|| StatsCollector::new(stats_config(continuous))); .or_insert_with(|| StatsCollector::new(stats_config(continuous)));
print_target_event_with_stats(&mut stream_output, &target_event, stats)?; print_target_event_with_stats(
&mut stream_output,
&target_event,
stats,
)?;
} }
#[cfg(not(feature = "stats"))] #[cfg(not(feature = "stats"))]
{ {
print_target_event_with_stats(&mut stream_output, &target_event)?; print_target_event_with_stats(&mut stream_output, &target_event)?;
} }
} }
ManagedGroupEvent::TargetFinished(target) => {
terminal_targets.insert(target.id.as_str().to_owned());
}
}
}
Ok(None) => { Ok(None) => {
if interrupted { if interrupted {
break; break;
@@ -365,10 +373,11 @@ fn run_group_stream(
} }
let outcome = session.join()?; let outcome = session.join()?;
while let Ok(Some(target_event)) = events.try_recv() { while let Ok(Some(group_event)) = events.try_recv() {
let label = target_event.target.as_str(); if let ManagedGroupEvent::Client(target_event) = group_event {
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
{ {
let label = target_event.target.as_str();
let stats = stats let stats = stats
.entry(label.to_owned()) .entry(label.to_owned())
.or_insert_with(|| StatsCollector::new(stats_config(continuous))); .or_insert_with(|| StatsCollector::new(stats_config(continuous)));
@@ -379,6 +388,7 @@ fn run_group_stream(
print_target_event_with_stats(&mut stream_output, &target_event)?; print_target_event_with_stats(&mut stream_output, &target_event)?;
} }
} }
}
if outcome.end_reason == ManagedGroupEndReason::Cancelled && !interrupted { if outcome.end_reason == ManagedGroupEndReason::Cancelled && !interrupted {
return Err("managed client group was cancelled".into()); return Err("managed client group was cancelled".into());
@@ -405,13 +415,6 @@ fn run_group_stream(
Ok(()) Ok(())
} }
fn is_terminal_target_event(event: &ClientEvent) -> bool {
matches!(
event,
ClientEvent::SessionClosed { .. } | ClientEvent::NoTestCompleted { .. }
)
}
fn estimated_group_completion_grace(args: &ClientArgs) -> Duration { fn estimated_group_completion_grace(args: &ClientArgs) -> Duration {
let open_timeout: Duration = args.to_client_config().open_timeouts.iter().sum(); let open_timeout: Duration = args.to_client_config().open_timeouts.iter().sum();
open_timeout open_timeout
+13 -14
View File
@@ -7,8 +7,8 @@ use std::{
use crossterm::event::{self, Event, KeyCode, KeyEventKind, KeyModifiers}; use crossterm::event::{self, Event, KeyCode, KeyEventKind, KeyModifiers};
use irtt_client::{ use irtt_client::{
ClientEvent, EventSubscriptionError, ManagedClientGroup, ManagedClientGroupConfig, EventSubscriptionError, ManagedClientGroup, ManagedClientGroupConfig, ManagedGroupEndReason,
ManagedGroupEndReason, SubscriberConfig, SubscriberOverflow, ManagedGroupEvent, SubscriberConfig, SubscriberOverflow,
}; };
use crate::{ use crate::{
@@ -184,13 +184,17 @@ fn run_group_tui(
} }
match events.try_recv() { match events.try_recv() {
Ok(Some(target_event)) => { Ok(Some(group_event)) => {
saw_target_event = true; saw_target_event = true;
last_event_at = Instant::now(); last_event_at = Instant::now();
if is_terminal_target_event(&target_event.event) { match group_event {
terminal_targets.insert(target_event.target.as_str().to_owned()); ManagedGroupEvent::Client(target_event) => {
}
state.process_target_event(&target_event); state.process_target_event(&target_event);
}
ManagedGroupEvent::TargetFinished(target) => {
terminal_targets.insert(target.id.as_str().to_owned());
}
}
render_if_due(terminal, state, next_render, false)?; render_if_due(terminal, state, next_render, false)?;
} }
Ok(None) => { Ok(None) => {
@@ -225,9 +229,11 @@ fn run_group_tui(
render_if_due(terminal, state, next_render, true)?; render_if_due(terminal, state, next_render, true)?;
let outcome = session.join()?; let outcome = session.join()?;
while let Ok(Some(target_event)) = events.try_recv() { while let Ok(Some(group_event)) = events.try_recv() {
if let ManagedGroupEvent::Client(target_event) = group_event {
state.process_target_event(&target_event); state.process_target_event(&target_event);
} }
}
if outcome.end_reason == ManagedGroupEndReason::Cancelled && !interrupted { if outcome.end_reason == ManagedGroupEndReason::Cancelled && !interrupted {
return match exit { return match exit {
@@ -265,13 +271,6 @@ impl GroupLoopExit {
} }
} }
fn is_terminal_target_event(event: &ClientEvent) -> bool {
matches!(
event,
ClientEvent::SessionClosed { .. } | ClientEvent::NoTestCompleted { .. }
)
}
fn estimated_group_completion_grace(args: &TuiArgs) -> Duration { fn estimated_group_completion_grace(args: &TuiArgs) -> Duration {
let open_timeout: Duration = args.to_client_config().open_timeouts.iter().sum(); let open_timeout: Duration = args.to_client_config().open_timeouts.iter().sum();
open_timeout open_timeout
+4 -3
View File
@@ -78,9 +78,10 @@ pub use event::{
pub use managed::{ pub use managed::{
CancellationToken, EventHub, EventSubscription, ManagedClient, ManagedClientGroup, CancellationToken, EventHub, EventSubscription, ManagedClient, ManagedClientGroup,
ManagedClientGroupConfig, ManagedClientGroupSession, ManagedClientSession, ManagedClientGroupConfig, ManagedClientGroupSession, ManagedClientSession,
ManagedGroupEndReason, ManagedGroupOutcome, ManagedGroupPacing, ManagedTargetConfig, ManagedGroupEndReason, ManagedGroupEvent, ManagedGroupOutcome, ManagedGroupPacing,
ManagedTargetEndReason, ManagedTargetOutcome, SessionEndReason, SessionOutcome, ManagedTargetConfig, ManagedTargetEndReason, ManagedTargetFailure, ManagedTargetFailureKind,
SubscriberConfig, SubscriberOverflow, TargetEvent, TargetEventSubscription, TargetId, ManagedTargetOutcome, SessionEndReason, SessionOutcome, SubscriberConfig, SubscriberOverflow,
TargetEvent, TargetEventSubscription, TargetId,
}; };
pub use session::{NegotiatedParams, NegotiationRestriction}; pub use session::{NegotiatedParams, NegotiationRestriction};
pub use timing::ClientTimestamp; pub use timing::ClientTimestamp;
+487 -75
View File
@@ -127,8 +127,30 @@ pub struct TargetEvent {
pub event: ClientEvent, pub event: ClientEvent,
} }
/// Subscription type for target-scoped group events. /// Event published by a managed client group.
pub type TargetEventSubscription = EventSubscription<TargetEvent>; ///
/// Per-session events are followed by exactly one [`TargetFinished`](Self::TargetFinished)
/// event when that target incarnation reaches a terminal state.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ManagedGroupEvent {
/// A protocol lifecycle, measurement, or warning event from one target.
Client(TargetEvent),
/// Authoritative terminal outcome for one target incarnation.
TargetFinished(ManagedTargetOutcome),
}
impl ManagedGroupEvent {
/// Return the target associated with this event.
pub fn target(&self) -> &TargetId {
match self {
Self::Client(event) => &event.target,
Self::TargetFinished(outcome) => &outcome.id,
}
}
}
/// Subscription type for managed group events.
pub type TargetEventSubscription = EventSubscription<ManagedGroupEvent>;
/// Entry point for running a shared-socket multi-target managed client group. /// Entry point for running a shared-socket multi-target managed client group.
#[derive(Debug)] #[derive(Debug)]
@@ -142,7 +164,7 @@ pub struct ManagedClientGroup;
#[must_use = "dropping the session cancels the managed client group; call join() to wait for completion"] #[must_use = "dropping the session cancels the managed client group; call join() to wait for completion"]
#[derive(Debug)] #[derive(Debug)]
pub struct ManagedClientGroupSession { pub struct ManagedClientGroupSession {
hub: EventHub<TargetEvent>, hub: EventHub<ManagedGroupEvent>,
control_tx: mpsc::Sender<ControlMessage>, control_tx: mpsc::Sender<ControlMessage>,
cancellation: CancellationToken, cancellation: CancellationToken,
scheduler: Option<JoinHandle<Result<ManagedGroupOutcome, ClientError>>>, scheduler: Option<JoinHandle<Result<ManagedGroupOutcome, ClientError>>>,
@@ -202,10 +224,85 @@ pub enum ManagedTargetEndReason {
NoTestComplete, NoTestComplete,
/// The server closed the target session. /// The server closed the target session.
PeerClosed, PeerClosed,
/// Opening failed before a `ClientEvent` lifecycle event existed. /// Opening failed before a session became active.
OpenFailed { message: String }, OpenFailed(ManagedTargetFailure),
/// Runtime send/receive/close handling failed for this target. /// Runtime send/receive/close handling failed for this target.
Failed { message: String }, Failed(ManagedTargetFailure),
}
impl ManagedTargetEndReason {
/// Return whether this reason represents a successfully completed run.
pub fn is_success(&self) -> bool {
matches!(
self,
Self::TestComplete | Self::NoTestComplete | Self::PeerClosed
)
}
/// Return structured failure details, when this target failed.
pub fn failure(&self) -> Option<&ManagedTargetFailure> {
match self {
Self::OpenFailed(failure) | Self::Failed(failure) => Some(failure),
_ => None,
}
}
}
/// Structured failure details for a managed target.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ManagedTargetFailure {
/// Machine-readable failure category.
pub kind: ManagedTargetFailureKind,
/// Human-readable diagnostic suitable for logs and user interfaces.
pub message: String,
}
impl ManagedTargetFailure {
fn opening(error: &ClientError) -> Self {
Self {
kind: classify_target_failure(error, true),
message: error.to_string(),
}
}
fn runtime(error: &ClientError) -> Self {
Self {
kind: classify_target_failure(error, false),
message: error.to_string(),
}
}
}
/// Machine-readable category for a managed target failure.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ManagedTargetFailureKind {
/// No valid open reply arrived before the configured attempts expired.
OpeningTimeout,
/// Opening failed during protocol parsing, authentication, or negotiation.
OpeningProtocol,
/// A socket or socket-option operation failed.
Socket,
/// Runtime protocol or session-state processing failed.
RuntimeProtocol,
/// Target or client configuration was invalid.
InvalidConfiguration,
/// A managed worker failed internally.
InternalWorker,
}
fn classify_target_failure(error: &ClientError, opening: bool) -> ManagedTargetFailureKind {
match error {
ClientError::OpenTimeout => ManagedTargetFailureKind::OpeningTimeout,
ClientError::Socket(_)
| ClientError::SocketOption { .. }
| ClientError::ReadTimeoutRestore { .. } => ManagedTargetFailureKind::Socket,
ClientError::InvalidConfig { .. }
| ClientError::OpenTimeoutTooSmall { .. }
| ClientError::NoOpenTimeouts => ManagedTargetFailureKind::InvalidConfiguration,
ClientError::WorkerPanicked => ManagedTargetFailureKind::InternalWorker,
_ if opening => ManagedTargetFailureKind::OpeningProtocol,
_ => ManagedTargetFailureKind::RuntimeProtocol,
}
} }
impl ManagedClientGroup { impl ManagedClientGroup {
@@ -299,6 +396,9 @@ impl ManagedClientGroup {
let scheduler_shared = shared.clone(); let scheduler_shared = shared.clone();
let scheduler_config = config.clone(); let scheduler_config = config.clone();
let scheduler = thread::spawn(move || { let scheduler = thread::spawn(move || {
let _cleanup = GroupSchedulerCleanup {
shared: scheduler_shared.clone(),
};
run_group_scheduler( run_group_scheduler(
scheduler_config, scheduler_config,
send_socket, send_socket,
@@ -361,28 +461,12 @@ impl ManagedClientGroupSession {
let scheduler = self.scheduler.take().expect( let scheduler = self.scheduler.take().expect(
"ManagedClientGroupSession invariant violated: scheduler handle missing before join", "ManagedClientGroupSession invariant violated: scheduler handle missing before join",
); );
let scheduler_result = match scheduler.join() { let scheduler_result = scheduler.join().unwrap_or(Err(ClientError::WorkerPanicked));
Ok(outcome) => outcome,
Err(_) => {
self.cancellation.cancel();
self.hub.disconnect_all();
return Err(ClientError::WorkerPanicked);
}
};
self.cancellation.cancel();
let _ = self.control_tx.send(ControlMessage::Wake);
let receiver = self.receiver.take().expect( let receiver = self.receiver.take().expect(
"ManagedClientGroupSession invariant violated: receiver handle missing before join", "ManagedClientGroupSession invariant violated: receiver handle missing before join",
); );
let receiver_result = match receiver.join() { let receiver_result = receiver.join().unwrap_or(Err(ClientError::WorkerPanicked));
Ok(outcome) => outcome,
Err(_) => {
self.hub.disconnect_all();
return Err(ClientError::WorkerPanicked);
}
};
self.hub.disconnect_all(); self.hub.disconnect_all();
match (scheduler_result, receiver_result) { match (scheduler_result, receiver_result) {
@@ -412,7 +496,7 @@ enum ControlMessage {
#[derive(Debug)] #[derive(Debug)]
struct GroupShared { struct GroupShared {
registry: Mutex<TargetRegistry>, registry: Mutex<TargetRegistry>,
hub: EventHub<TargetEvent>, hub: EventHub<ManagedGroupEvent>,
cancellation: CancellationToken, cancellation: CancellationToken,
control_tx: mpsc::Sender<ControlMessage>, control_tx: mpsc::Sender<ControlMessage>,
requested_interval_ns: i64, requested_interval_ns: i64,
@@ -420,6 +504,18 @@ struct GroupShared {
family_remote: SocketAddr, family_remote: SocketAddr,
} }
struct GroupSchedulerCleanup {
shared: Arc<GroupShared>,
}
impl Drop for GroupSchedulerCleanup {
fn drop(&mut self) {
self.shared.cancellation.cancel();
let _ = self.shared.control_tx.send(ControlMessage::Wake);
self.shared.hub.disconnect_all();
}
}
#[derive(Debug, Default)] #[derive(Debug, Default)]
struct TargetRegistry { struct TargetRegistry {
targets: HashMap<TargetId, Arc<Mutex<TargetState>>>, targets: HashMap<TargetId, Arc<Mutex<TargetState>>>,
@@ -504,6 +600,9 @@ impl TargetState {
} }
fn mark_finished(&mut self, reason: ManagedTargetEndReason) { fn mark_finished(&mut self, reason: ManagedTargetEndReason) {
if self.final_reason.is_some() {
return;
}
self.status = TargetStatus::Finished; self.status = TargetStatus::Finished;
self.final_reason = Some(reason); self.final_reason = Some(reason);
} }
@@ -605,6 +704,12 @@ fn run_group_receiver(socket: UdpSocket, shared: Arc<GroupShared>) -> Result<(),
continue; continue;
} }
Err(err) => { Err(err) => {
let failure = ManagedTargetFailure {
kind: ManagedTargetFailureKind::Socket,
message: ClientError::Socket(std::io::Error::new(err.kind(), err.to_string()))
.to_string(),
};
fail_all_targets(&shared, failure);
shared.cancellation.cancel(); shared.cancellation.cancel();
let _ = shared.control_tx.send(ControlMessage::Wake); let _ = shared.control_tx.send(ControlMessage::Wake);
return Err(ClientError::Socket(err)); return Err(ClientError::Socket(err));
@@ -645,9 +750,9 @@ fn run_group_receiver(socket: UdpSocket, shared: Arc<GroupShared>) -> Result<(),
wake_scheduler = true; wake_scheduler = true;
} }
Err(err) => { Err(err) => {
target.mark_finished(ManagedTargetEndReason::OpenFailed { target.mark_finished(ManagedTargetEndReason::OpenFailed(
message: err.to_string(), ManagedTargetFailure::opening(&err),
}); ));
wake_scheduler = true; wake_scheduler = true;
} }
} }
@@ -665,9 +770,9 @@ fn run_group_receiver(socket: UdpSocket, shared: Arc<GroupShared>) -> Result<(),
} }
} }
Err(err) => { Err(err) => {
target.mark_finished(ManagedTargetEndReason::Failed { target.mark_finished(ManagedTargetEndReason::Failed(
message: err.to_string(), ManagedTargetFailure::runtime(&err),
}); ));
wake_scheduler = true; wake_scheduler = true;
} }
} }
@@ -703,36 +808,40 @@ fn validate_group_negotiation(
} }
fn publish_open_outcome( fn publish_open_outcome(
hub: &EventHub<TargetEvent>, hub: &EventHub<ManagedGroupEvent>,
target: &mut TargetState, target: &mut TargetState,
outcome: OpenOutcome, outcome: OpenOutcome,
) { ) {
match outcome { match outcome {
OpenOutcome::Started { event, .. } => { OpenOutcome::Started { event, .. } => {
target.status = TargetStatus::Active; target.status = TargetStatus::Active;
hub.publish(TargetEvent { hub.publish(ManagedGroupEvent::Client(TargetEvent {
target: target.id.clone(), target: target.id.clone(),
event, event,
}); }));
} }
OpenOutcome::NoTestCompleted { event, .. } => { OpenOutcome::NoTestCompleted { event, .. } => {
target.observe(&event); target.observe(&event);
hub.publish(TargetEvent { hub.publish(ManagedGroupEvent::Client(TargetEvent {
target: target.id.clone(), target: target.id.clone(),
event, event,
}); }));
target.mark_finished(ManagedTargetEndReason::NoTestComplete); target.mark_finished(ManagedTargetEndReason::NoTestComplete);
} }
} }
} }
fn publish_events(hub: &EventHub<TargetEvent>, target: &mut TargetState, events: Vec<ClientEvent>) { fn publish_events(
hub: &EventHub<ManagedGroupEvent>,
target: &mut TargetState,
events: Vec<ClientEvent>,
) {
for event in events { for event in events {
target.observe(&event); target.observe(&event);
hub.publish(TargetEvent { hub.publish(ManagedGroupEvent::Client(TargetEvent {
target: target.id.clone(), target: target.id.clone(),
event, event,
}); }));
} }
} }
@@ -855,12 +964,8 @@ fn apply_target_update(
}; };
for target in removed { for target in removed {
records.push(close_target( let outcome = close_target(socket, &shared.hub, target, ManagedTargetEndReason::Removed);
socket, record_target_outcome(shared, records, outcome);
&shared.hub,
target,
ManagedTargetEndReason::Removed,
));
} }
Ok(()) Ok(())
@@ -944,9 +1049,9 @@ fn drive_open_attempts(
} }
if attempt >= config.open_timeouts.len() { if attempt >= config.open_timeouts.len() {
target.mark_finished(ManagedTargetEndReason::OpenFailed { target.mark_finished(ManagedTargetEndReason::OpenFailed(
message: ClientError::OpenTimeout.to_string(), ManagedTargetFailure::opening(&ClientError::OpenTimeout),
}); ));
continue; continue;
} }
@@ -958,9 +1063,9 @@ fn drive_open_attempts(
}; };
} }
Err(err) => { Err(err) => {
target.mark_finished(ManagedTargetEndReason::Failed { target.mark_finished(ManagedTargetEndReason::OpenFailed(
message: ClientError::Socket(err).to_string(), ManagedTargetFailure::opening(&ClientError::Socket(err)),
}); ));
} }
} }
} }
@@ -981,9 +1086,9 @@ fn poll_active_timeouts(shared: &GroupShared, now: Instant) {
} }
match target.runtime.poll_timeouts_at(now) { match target.runtime.poll_timeouts_at(now) {
Ok(events) => publish_events(&shared.hub, &mut target, events), Ok(events) => publish_events(&shared.hub, &mut target, events),
Err(err) => target.mark_finished(ManagedTargetEndReason::Failed { Err(err) => target.mark_finished(ManagedTargetEndReason::Failed(
message: err.to_string(), ManagedTargetFailure::runtime(&err),
}), )),
} }
} }
} }
@@ -1058,9 +1163,9 @@ fn send_echo_to_target(
// replies. // replies.
publish_events(&shared.hub, &mut target, events); publish_events(&shared.hub, &mut target, events);
} }
Err(err) => target.mark_finished(ManagedTargetEndReason::Failed { Err(err) => target.mark_finished(ManagedTargetEndReason::Failed(
message: err.to_string(), ManagedTargetFailure::runtime(&err),
}), )),
} }
} }
@@ -1091,14 +1196,13 @@ fn collect_finished_targets(shared: &GroupShared, records: &mut Vec<ManagedTarge
for target in finished { for target in finished {
let target = target.lock().expect("target mutex poisoned"); let target = target.lock().expect("target mutex poisoned");
let reason = let reason = target.final_reason.clone().unwrap_or_else(|| {
target ManagedTargetEndReason::Failed(ManagedTargetFailure {
.final_reason kind: ManagedTargetFailureKind::InternalWorker,
.clone()
.unwrap_or_else(|| ManagedTargetEndReason::Failed {
message: "target finished without a reason".to_owned(), message: "target finished without a reason".to_owned(),
})
}); });
records.push(target.outcome(reason)); record_target_outcome(shared, records, target.outcome(reason));
} }
} }
@@ -1109,12 +1213,13 @@ fn cancel_remaining_targets(
) { ) {
let targets = drain_registry(shared); let targets = drain_registry(shared);
for target in targets { for target in targets {
records.push(close_target( let outcome = close_target(
socket, socket,
&shared.hub, &shared.hub,
target, target,
ManagedTargetEndReason::Cancelled, ManagedTargetEndReason::Cancelled,
)); );
record_target_outcome(shared, records, outcome);
} }
} }
@@ -1126,7 +1231,28 @@ fn remove_all_targets(
) { ) {
let targets = drain_registry(shared); let targets = drain_registry(shared);
for target in targets { for target in targets {
records.push(close_target(socket, &shared.hub, target, reason.clone())); let outcome = close_target(socket, &shared.hub, target, reason.clone());
record_target_outcome(shared, records, outcome);
}
}
fn record_target_outcome(
shared: &GroupShared,
records: &mut Vec<ManagedTargetOutcome>,
outcome: ManagedTargetOutcome,
) {
shared
.hub
.publish(ManagedGroupEvent::TargetFinished(outcome.clone()));
records.push(outcome);
}
fn fail_all_targets(shared: &GroupShared, failure: ManagedTargetFailure) {
for target in all_targets(shared) {
target
.lock()
.expect("target mutex poisoned")
.mark_finished(ManagedTargetEndReason::Failed(failure.clone()));
} }
} }
@@ -1141,7 +1267,7 @@ fn drain_registry(shared: &GroupShared) -> Vec<Arc<Mutex<TargetState>>> {
fn close_target( fn close_target(
socket: &UdpSocket, socket: &UdpSocket,
hub: &EventHub<TargetEvent>, hub: &EventHub<ManagedGroupEvent>,
target: Arc<Mutex<TargetState>>, target: Arc<Mutex<TargetState>>,
reason: ManagedTargetEndReason, reason: ManagedTargetEndReason,
) -> ManagedTargetOutcome { ) -> ManagedTargetOutcome {
@@ -1153,7 +1279,7 @@ fn close_target(
fn close_locked_target( fn close_locked_target(
socket: &UdpSocket, socket: &UdpSocket,
hub: &EventHub<TargetEvent>, hub: &EventHub<ManagedGroupEvent>,
target: &mut TargetState, target: &mut TargetState,
reason: ManagedTargetEndReason, reason: ManagedTargetEndReason,
) { ) {
@@ -1165,9 +1291,9 @@ fn close_locked_target(
}) { }) {
Ok(events) => publish_events(hub, target, events), Ok(events) => publish_events(hub, target, events),
Err(err) => { Err(err) => {
target.mark_finished(ManagedTargetEndReason::Failed { target.mark_finished(ManagedTargetEndReason::Failed(
message: err.to_string(), ManagedTargetFailure::runtime(&err),
}); ));
return; return;
} }
} }
@@ -1545,6 +1671,85 @@ mod tests {
} }
} }
fn start_open_failure_server(params: Params) -> FakeServer {
let socket = UdpSocket::bind("127.0.0.1:0").unwrap();
let addr = socket.local_addr().unwrap();
let (tx, rx) = mpsc::channel();
let done = thread::spawn(move || {
let (open, peer) = recv_request_timeout(&socket).expect("missing open request");
assert_ne!(open[3] & FLAG_OPEN, 0);
tx.send(ServerObservation::Open { at: Instant::now() })
.unwrap();
socket
.send_to(&open_reply(FLAG_OPEN | FLAG_REPLY, 0, &params), peer)
.unwrap();
});
FakeServer {
addr,
_observations: rx,
done,
}
}
fn start_no_test_server(params: Params) -> FakeServer {
let socket = UdpSocket::bind("127.0.0.1:0").unwrap();
let addr = socket.local_addr().unwrap();
let (tx, rx) = mpsc::channel();
let done = thread::spawn(move || {
let (open, peer) = recv_request_timeout(&socket).expect("missing open request");
assert_ne!(open[3] & flags::FLAG_CLOSE, 0);
tx.send(ServerObservation::Open { at: Instant::now() })
.unwrap();
socket
.send_to(
&open_reply(FLAG_OPEN | FLAG_REPLY | flags::FLAG_CLOSE, 0, &params),
peer,
)
.unwrap();
});
FakeServer {
addr,
_observations: rx,
done,
}
}
fn start_silent_runtime_server(params: Params) -> FakeServer {
let socket = UdpSocket::bind("127.0.0.1:0").unwrap();
let addr = socket.local_addr().unwrap();
let (tx, rx) = mpsc::channel();
let done = thread::spawn(move || {
let (open, peer) = recv_request_timeout(&socket).expect("missing open request");
assert_ne!(open[3] & FLAG_OPEN, 0);
tx.send(ServerObservation::Open { at: Instant::now() })
.unwrap();
socket
.send_to(&open_reply(FLAG_OPEN | FLAG_REPLY, TOKEN, &params), peer)
.unwrap();
socket
.set_read_timeout(Some(Duration::from_millis(800)))
.unwrap();
while let Some((packet, _)) = recv_request_timeout(&socket) {
if packet[3] & flags::FLAG_CLOSE != 0 {
tx.send(ServerObservation::Close { at: Instant::now() })
.unwrap();
break;
}
let seq = u32::from_le_bytes(packet[12..16].try_into().unwrap());
tx.send(ServerObservation::Echo {
seq,
at: Instant::now(),
})
.unwrap();
}
});
FakeServer {
addr,
_observations: rx,
done,
}
}
fn recv_request_timeout(socket: &UdpSocket) -> Option<(Vec<u8>, SocketAddr)> { fn recv_request_timeout(socket: &UdpSocket) -> Option<(Vec<u8>, SocketAddr)> {
let mut buf = [0_u8; 2048]; let mut buf = [0_u8; 2048];
socket socket
@@ -1620,7 +1825,8 @@ mod tests {
let deadline = Instant::now() + Duration::from_secs(2); let deadline = Instant::now() + Duration::from_secs(2);
loop { loop {
match sub.try_recv() { match sub.try_recv() {
Ok(Some(event)) => return event, Ok(Some(ManagedGroupEvent::Client(event))) => return event,
Ok(Some(ManagedGroupEvent::TargetFinished(_))) => {}
Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(1)), Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(1)),
Ok(None) => panic!("timed out waiting for group event"), Ok(None) => panic!("timed out waiting for group event"),
Err(err) => panic!("subscription ended while waiting for event: {err}"), Err(err) => panic!("subscription ended while waiting for event: {err}"),
@@ -1632,13 +1838,29 @@ mod tests {
let mut events = Vec::new(); let mut events = Vec::new();
loop { loop {
match sub.try_recv() { match sub.try_recv() {
Ok(Some(event)) => events.push(event), Ok(Some(ManagedGroupEvent::Client(event))) => events.push(event),
Ok(Some(ManagedGroupEvent::TargetFinished(_))) => {}
Ok(None) => thread::sleep(Duration::from_millis(1)), Ok(None) => thread::sleep(Duration::from_millis(1)),
Err(EventSubscriptionError::Disconnected) => return events, Err(EventSubscriptionError::Disconnected) => return events,
} }
} }
} }
fn collect_group_events_until_disconnected(
sub: &TargetEventSubscription,
) -> Vec<ManagedGroupEvent> {
let deadline = Instant::now() + Duration::from_secs(2);
let mut events = Vec::new();
loop {
match sub.try_recv() {
Ok(Some(event)) => events.push(event),
Ok(None) if Instant::now() < deadline => thread::sleep(Duration::from_millis(1)),
Ok(None) => panic!("timed out waiting for managed group disconnection"),
Err(EventSubscriptionError::Disconnected) => return events,
}
}
}
fn sent_events_after_start(events: &[TargetEvent]) -> Vec<(&TargetId, Instant)> { fn sent_events_after_start(events: &[TargetEvent]) -> Vec<(&TargetId, Instant)> {
let both_started_at = events let both_started_at = events
.iter() .iter()
@@ -1800,6 +2022,172 @@ mod tests {
assert_eq!(target.replies_received, 1); assert_eq!(target.replies_received, 1);
} }
#[test]
fn mixed_peer_close_and_open_failure_publish_terminal_outcomes_and_disconnect() {
let interval = Duration::from_millis(10);
let params = test_params(None, interval);
let success = start_peer_close_server(params.clone());
let failure = start_open_failure_server(params);
let (session, sub) = ManagedClientGroup::start_with_subscription(
group_config(None, interval, ManagedGroupPacing::Staggered),
vec![
target("success", success.addr),
target("failure", failure.addr),
],
SubscriberConfig {
capacity: 64,
overflow: SubscriberOverflow::DropNewest,
},
)
.unwrap();
let events = collect_group_events_until_disconnected(&sub);
let outcome = session.join().unwrap();
success.join();
failure.join();
assert!(events.iter().any(|event| matches!(
event,
ManagedGroupEvent::Client(TargetEvent {
target,
event: ClientEvent::EchoReply { .. },
}) if target.as_str() == "success"
)));
assert!(events.iter().any(|event| matches!(
event,
ManagedGroupEvent::TargetFinished(target)
if target.id.as_str() == "success"
&& target.end_reason == ManagedTargetEndReason::PeerClosed
)));
assert!(events.iter().any(|event| matches!(
event,
ManagedGroupEvent::TargetFinished(target)
if target.id.as_str() == "failure"
&& matches!(
&target.end_reason,
ManagedTargetEndReason::OpenFailed(failure)
if failure.kind == ManagedTargetFailureKind::OpeningProtocol
)
)));
assert_eq!(outcome.targets.len(), 2);
}
#[test]
fn all_open_failures_publish_terminal_outcomes_and_disconnect() {
let interval = Duration::from_millis(10);
let params = test_params(None, interval);
let a = start_open_failure_server(params.clone());
let b = start_open_failure_server(params);
let (session, sub) = ManagedClientGroup::start_with_subscription(
group_config(None, interval, ManagedGroupPacing::Burst),
vec![target("a", a.addr), target("b", b.addr)],
SubscriberConfig {
capacity: 16,
overflow: SubscriberOverflow::DropNewest,
},
)
.unwrap();
let events = collect_group_events_until_disconnected(&sub);
let outcome = session.join().unwrap();
a.join();
b.join();
let failures = events
.iter()
.filter(|event| {
matches!(
event,
ManagedGroupEvent::TargetFinished(ManagedTargetOutcome {
end_reason: ManagedTargetEndReason::OpenFailed(_),
..
})
)
})
.count();
assert_eq!(failures, 2);
assert_eq!(outcome.targets.len(), 2);
assert!(outcome
.targets
.iter()
.all(|target| target.end_reason.failure().is_some()));
}
#[test]
fn runtime_failure_publishes_terminal_outcome() {
let interval = Duration::from_millis(10);
let server = start_silent_runtime_server(test_params(None, interval));
let mut config = group_config(None, interval, ManagedGroupPacing::Staggered);
config.client.max_pending_probes = 1;
let (session, sub) = ManagedClientGroup::start_with_subscription(
config,
vec![target("runtime", server.addr)],
SubscriberConfig {
capacity: 16,
overflow: SubscriberOverflow::DropNewest,
},
)
.unwrap();
let events = collect_group_events_until_disconnected(&sub);
let outcome = session.join().unwrap();
server.join();
let terminal = events
.iter()
.find_map(|event| match event {
ManagedGroupEvent::TargetFinished(outcome) => Some(outcome),
ManagedGroupEvent::Client(_) => None,
})
.expect("missing terminal outcome");
let failure = terminal
.end_reason
.failure()
.expect("runtime target should fail");
assert_eq!(failure.kind, ManagedTargetFailureKind::RuntimeProtocol);
assert!(failure.message.contains("pending probe limit"));
assert_eq!(outcome.targets, vec![terminal.clone()]);
}
#[test]
fn no_test_event_precedes_terminal_outcome() {
let interval = Duration::from_millis(10);
let server = start_no_test_server(test_params(Some(interval), interval));
let mut config = group_config(Some(interval), interval, ManagedGroupPacing::Staggered);
config.client.run_mode = crate::RunMode::NoTest;
let (session, sub) = ManagedClientGroup::start_with_subscription(
config,
vec![target("no-test", server.addr)],
SubscriberConfig {
capacity: 8,
overflow: SubscriberOverflow::DropNewest,
},
)
.unwrap();
let events = collect_group_events_until_disconnected(&sub);
let outcome = session.join().unwrap();
server.join();
assert!(matches!(
events.as_slice(),
[
ManagedGroupEvent::Client(TargetEvent {
event: ClientEvent::NoTestCompleted { .. },
..
}),
ManagedGroupEvent::TargetFinished(ManagedTargetOutcome {
end_reason: ManagedTargetEndReason::NoTestComplete,
..
})
]
));
assert_eq!(
outcome.targets[0].end_reason,
ManagedTargetEndReason::NoTestComplete
);
}
#[test] #[test]
fn staggered_pacing_does_not_burst_active_targets() { fn staggered_pacing_does_not_burst_active_targets() {
let duration = Duration::from_millis(170); let duration = Duration::from_millis(170);
@@ -2004,9 +2392,10 @@ mod tests {
let mut saw_a_after_update = false; let mut saw_a_after_update = false;
let mut saw_c_after_update = false; let mut saw_c_after_update = false;
let mut saw_b_echo_after_update = false; let mut saw_b_echo_after_update = false;
let mut saw_b_removed = false;
while Instant::now() < deadline && !(saw_a_after_update && saw_c_after_update) { while Instant::now() < deadline && !(saw_a_after_update && saw_c_after_update) {
match sub.try_recv() { match sub.try_recv() {
Ok(Some(event)) => { Ok(Some(ManagedGroupEvent::Client(event))) => {
saw_a_after_update |= event.target.as_str() == "a" saw_a_after_update |= event.target.as_str() == "a"
&& matches!(event.event, ClientEvent::EchoReply { .. }); && matches!(event.event, ClientEvent::EchoReply { .. });
saw_c_after_update |= event.target.as_str() == "c" saw_c_after_update |= event.target.as_str() == "c"
@@ -2017,6 +2406,10 @@ mod tests {
ClientEvent::EchoSent { .. } | ClientEvent::EchoReply { .. } ClientEvent::EchoSent { .. } | ClientEvent::EchoReply { .. }
); );
} }
Ok(Some(ManagedGroupEvent::TargetFinished(target))) => {
saw_b_removed |= target.id.as_str() == "b"
&& target.end_reason == ManagedTargetEndReason::Removed;
}
Ok(None) => thread::sleep(Duration::from_millis(1)), Ok(None) => thread::sleep(Duration::from_millis(1)),
Err(err) => panic!("subscription ended unexpectedly: {err}"), Err(err) => panic!("subscription ended unexpectedly: {err}"),
} }
@@ -2032,6 +2425,7 @@ mod tests {
assert!(saw_a_after_update); assert!(saw_a_after_update);
assert!(saw_c_after_update); assert!(saw_c_after_update);
assert!(!saw_b_echo_after_update); assert!(!saw_b_echo_after_update);
assert!(saw_b_removed);
assert!(outcome assert!(outcome
.targets .targets
.iter() .iter()
@@ -2046,20 +2440,38 @@ mod tests {
let a = start_echo_server(params.clone(), Duration::ZERO); let a = start_echo_server(params.clone(), Duration::ZERO);
let b = start_echo_server(params, Duration::ZERO); let b = start_echo_server(params, Duration::ZERO);
let session = ManagedClientGroup::start( let (session, sub) = ManagedClientGroup::start_with_subscription(
group_config(None, interval, ManagedGroupPacing::Staggered), group_config(None, interval, ManagedGroupPacing::Staggered),
vec![target("a", a.addr), target("b", b.addr)], vec![target("a", a.addr), target("b", b.addr)],
SubscriberConfig {
capacity: 64,
overflow: SubscriberOverflow::DropNewest,
},
) )
.unwrap(); .unwrap();
thread::sleep(Duration::from_millis(60)); thread::sleep(Duration::from_millis(60));
session.stop(); session.stop();
session.stop(); session.stop();
let events = collect_group_events_until_disconnected(&sub);
let outcome = session.join().unwrap(); let outcome = session.join().unwrap();
a.join(); a.join();
b.join(); b.join();
assert_eq!(outcome.end_reason, ManagedGroupEndReason::Cancelled); assert_eq!(outcome.end_reason, ManagedGroupEndReason::Cancelled);
assert_eq!(outcome.targets.len(), 2); assert_eq!(outcome.targets.len(), 2);
assert_eq!(
events
.iter()
.filter(|event| matches!(
event,
ManagedGroupEvent::TargetFinished(ManagedTargetOutcome {
end_reason: ManagedTargetEndReason::Cancelled,
..
})
))
.count(),
2
);
} }
} }
+3 -2
View File
@@ -6,8 +6,9 @@ mod runner;
pub use cancellation::CancellationToken; pub use cancellation::CancellationToken;
pub use group::{ pub use group::{
ManagedClientGroup, ManagedClientGroupConfig, ManagedClientGroupSession, ManagedGroupEndReason, ManagedClientGroup, ManagedClientGroupConfig, ManagedClientGroupSession, ManagedGroupEndReason,
ManagedGroupOutcome, ManagedGroupPacing, ManagedTargetConfig, ManagedTargetEndReason, ManagedGroupEvent, ManagedGroupOutcome, ManagedGroupPacing, ManagedTargetConfig,
ManagedTargetOutcome, TargetEvent, TargetEventSubscription, TargetId, ManagedTargetEndReason, ManagedTargetFailure, ManagedTargetFailureKind, ManagedTargetOutcome,
TargetEvent, TargetEventSubscription, TargetId,
}; };
pub use hub::{EventHub, EventSubscription, SubscriberConfig, SubscriberOverflow}; pub use hub::{EventHub, EventSubscription, SubscriberConfig, SubscriberOverflow};
pub use runner::{ManagedClient, ManagedClientSession, SessionEndReason, SessionOutcome}; pub use runner::{ManagedClient, ManagedClientSession, SessionEndReason, SessionOutcome};