refactor(client): split managed task into coherent internal modules

Group control and construction, target lifecycle, target-set transactions,
send pacing and commitment, and bounded timeout processing into private
children of the existing task owner. Keep durable status, outcome history,
top-level polling, sealing, and abandonment in task.rs.

Preserve phase-owned resources, stop/update synchronization, publication
ordering, pacing, and timeout fairness. Move the existing private gate and
timeout tests with their implementation without adding coverage or hooks.
This commit is contained in:
Nils Andreas Svee committed 2026-10-05 00:24:15 +07:00
1 parent b9cc3e2779
commit e7de8bdb30
6 files changed
+2220 -2074

No files matched your search

@@ -0,0 +1,780 @@
//! Target-generation state, phase transitions, accounting, and completion.
use std::{
future::Future,
pin::Pin,
sync::Arc,
task::{Context, Poll},
time::Instant,
};
use crate::{
async_client::{AsyncClient, AsyncOpenState},
managed::schedule::ProbeSchedule,
managed::{
classify_client_error, ManagedEvent, ManagedTargetConfig, ManagedTargetEndReason,
ManagedTargetFailure, ManagedTargetFailureKind, ManagedTargetFailurePhase,
ManagedTargetLifecycle, ManagedTargetOutcome, TargetInstance,
},
ClientConfig, ClientError, ClientEvent, OpenOutcome,
};
use super::{ManagedClientTask, TARGET_WORK_BUDGET};
const POST_DEADLINE_RECEIVE_BUDGET: usize = TARGET_WORK_BUDGET;
type ConnectFuture =
Pin<Box<dyn Future<Output = Result<AsyncClient, ClientError>> + Send + 'static>>;
pub(super) enum TargetPhase {
Pending {
client_config: ClientConfig,
},
Connecting {
future: ConnectFuture,
},
Opening {
client: AsyncClient,
open: Box<AsyncOpenState>,
},
Active {
client: AsyncClient,
schedule: ProbeSchedule,
send_waiting: bool,
},
Draining {
client: AsyncClient,
drain_started_at: Instant,
deadline: Instant,
primary_end: ManagedTargetEndReason,
cleanup_failure: Option<ManagedTargetFailure>,
post_deadline_receives_remaining: usize,
},
Closing {
client: AsyncClient,
deadline: Instant,
primary_end: ManagedTargetEndReason,
cleanup_failure: Option<ManagedTargetFailure>,
},
/// Terminal state always owns its finalized authoritative outcome.
Terminal {
outcome: Arc<ManagedTargetOutcome>,
},
}
impl TargetPhase {
pub(super) fn lifecycle(&self) -> ManagedTargetLifecycle {
match self {
Self::Pending { .. } => ManagedTargetLifecycle::Pending,
Self::Connecting { .. } => ManagedTargetLifecycle::Connecting,
Self::Opening { .. } => ManagedTargetLifecycle::Opening,
Self::Active { .. } => ManagedTargetLifecycle::Active,
Self::Draining { .. } => ManagedTargetLifecycle::Draining,
Self::Closing { .. } => ManagedTargetLifecycle::Closing,
Self::Terminal { .. } => ManagedTargetLifecycle::Terminal,
}
}
pub(super) fn client(&self) -> &AsyncClient {
match self {
Self::Opening { client, .. }
| Self::Active { client, .. }
| Self::Draining { client, .. }
| Self::Closing { client, .. } => client,
_ => panic!("managed target has no client in this phase"),
}
}
fn into_client(self) -> AsyncClient {
match self {
Self::Opening { client, .. }
| Self::Active { client, .. }
| Self::Draining { client, .. }
| Self::Closing { client, .. } => client,
_ => panic!("managed target has no client in this phase"),
}
}
}
#[derive(Default)]
pub(super) struct TargetCounters {
packets_sent: u64,
/// Unique echo replies that arrived in order and before their probe timed
/// out.
///
/// A reply that arrives behind an already-received sequence, or after its
/// own probe timed out, is counted by `late` and deliberately not here.
/// These counters are durable, unlike the lossy managed presentation
/// events.
replies_received: u64,
duplicates: u64,
late: u64,
warning_events: u64,
}
impl TargetCounters {
/// Account for one client event.
///
/// Events that carry no durable target-local count are ignored. Probe
/// sends are not counted here: `packets_sent` is authoritative in the
/// underlying client and is synchronized through
/// [`TargetRuntime::sync_packets_sent`].
fn observe(&mut self, event: &ClientEvent) {
match event {
ClientEvent::EchoReply { .. } => {
self.replies_received = self.replies_received.saturating_add(1);
}
ClientEvent::DuplicateReply { .. } => {
self.duplicates = self.duplicates.saturating_add(1);
}
ClientEvent::LateReply { .. } => {
self.late = self.late.saturating_add(1);
}
ClientEvent::Warning { .. } => {
self.warning_events = self.warning_events.saturating_add(1);
}
ClientEvent::EchoSent { .. }
| ClientEvent::EchoLoss { .. }
| ClientEvent::SessionStarted(_)
| ClientEvent::NoTestCompleted(_)
| ClientEvent::SessionClosed { .. } => {}
}
}
}
#[derive(Clone, Copy)]
pub(super) enum RetirementReason {
Removed,
Replaced,
}
impl From<RetirementReason> for ManagedTargetEndReason {
fn from(reason: RetirementReason) -> Self {
match reason {
RetirementReason::Removed => Self::Removed,
RetirementReason::Replaced => Self::Replaced,
}
}
}
pub(super) enum TargetMembership {
Desired,
Withdrawn(RetirementReason),
}
impl TargetMembership {
pub(super) fn is_desired(&self) -> bool {
matches!(self, Self::Desired)
}
fn retirement_reason(&self) -> Option<RetirementReason> {
match self {
Self::Desired => None,
Self::Withdrawn(reason) => Some(*reason),
}
}
}
pub(super) struct TargetRuntime {
pub(super) instance: TargetInstance,
pub(super) config: ManagedTargetConfig,
pub(super) membership: TargetMembership,
pub(super) server_addr: Arc<str>,
pub(super) remote: Option<std::net::SocketAddr>,
pub(super) counters: TargetCounters,
// None only while a synchronous operation owns the phase locally.
pub(super) phase: Option<TargetPhase>,
}
impl TargetRuntime {
pub(super) fn phase(&self) -> &TargetPhase {
self.phase
.as_ref()
.expect("managed target phase is extracted")
}
pub(super) fn phase_mut(&mut self) -> &mut TargetPhase {
self.phase
.as_mut()
.expect("managed target phase is extracted")
}
pub(super) fn take_phase(&mut self) -> TargetPhase {
self.phase
.take()
.expect("managed target phase is extracted")
}
pub(super) fn restore_phase(&mut self, phase: TargetPhase) {
assert!(
self.phase.is_none(),
"managed target phase is already installed"
);
self.phase = Some(phase);
}
pub(super) fn is_paced_active(&self) -> bool {
let phase = self.phase();
self.membership.is_desired() && matches!(phase, TargetPhase::Active { .. })
}
/// Adopt the underlying client's authoritative sent count.
///
/// The client owns `packets_sent`; this runtime only mirrors it so a
/// terminal outcome can report it after the client is gone.
pub(super) fn sync_packets_sent(&mut self, client: &AsyncClient) {
self.counters.packets_sent = client.packets_sent();
}
pub(super) fn sync_live_packets_sent(&mut self) {
self.counters.packets_sent = self.phase().client().packets_sent();
}
/// Build this target's durable outcome.
pub(super) fn outcome(
&self,
end_reason: ManagedTargetEndReason,
cleanup_failure: Option<ManagedTargetFailure>,
) -> ManagedTargetOutcome {
ManagedTargetOutcome {
target: self.instance.clone(),
server_addr: Arc::clone(&self.server_addr),
remote: self.remote,
end_reason,
packets_sent: self.counters.packets_sent,
replies_received: self.counters.replies_received,
duplicates: self.counters.duplicates,
late: self.counters.late,
warning_events: self.counters.warning_events,
cleanup_failure,
}
}
}
#[derive(Clone, Copy)]
pub(super) enum OpenSessionFailureCleanup {
Drain,
Close,
}
impl ManagedClientTask {
pub(super) fn install_target_phase(&mut self, index: usize, phase: TargetPhase) {
let previous = self.targets[index].phase().lifecycle();
self.targets[index].phase = Some(phase);
self.publish_target_phase(index, previous);
}
fn publish_target_phase(&mut self, index: usize, previous: ManagedTargetLifecycle) {
let lifecycle = self.targets[index].phase().lifecycle();
let desired = self.targets[index].membership.is_desired();
let was_paced_active = desired && previous == ManagedTargetLifecycle::Active;
let paced_active = self.targets[index].is_paced_active();
let now = Instant::now();
match (was_paced_active, paced_active) {
(false, true) => self.stagger_target_added(now),
(true, false) => self.stagger_target_removed(now),
_ => {}
}
self.replace_status();
self.publish_event(ManagedEvent::TargetStateChanged {
target: self.targets[index].instance.clone(),
lifecycle,
});
}
pub(super) fn publish_client_events(&mut self, index: usize, events: Vec<ClientEvent>) {
for event in events {
self.targets[index].counters.observe(&event);
self.publish_event(ManagedEvent::Client {
target: self.targets[index].instance.clone(),
event,
});
}
}
fn finish_target(
&mut self,
index: usize,
end_reason: ManagedTargetEndReason,
cleanup_failure: Option<ManagedTargetFailure>,
) {
let outcome = Arc::new(self.targets[index].outcome(end_reason, cleanup_failure));
// Finalize history before the phase publisher installs Terminal, updates
// pacing and publishes one complete status followed by the lifecycle event.
self.history.record(outcome.as_ref().clone());
self.install_target_phase(
index,
TargetPhase::Terminal {
outcome: Arc::clone(&outcome),
},
);
self.publish_event(ManagedEvent::TargetFinished { outcome });
}
fn fail_target(&mut self, index: usize, phase: ManagedTargetFailurePhase, error: ClientError) {
let failure = classify_client_error(phase, &error);
self.finish_target(index, ManagedTargetEndReason::Failed(failure), None);
}
pub(super) fn begin_open_session_failure(
&mut self,
index: usize,
phase: ManagedTargetFailurePhase,
error: ClientError,
now: Instant,
cleanup: OpenSessionFailureCleanup,
) -> bool {
let primary_end = ManagedTargetEndReason::Failed(classify_client_error(phase, &error));
let (TargetPhase::Opening { client, .. } | TargetPhase::Active { client, .. }) =
self.targets[index].phase_mut()
else {
unreachable!("open-session failure requires an opening or active target")
};
client.discard_prepared_probe();
self.targets[index].sync_live_packets_sent();
match cleanup {
OpenSessionFailureCleanup::Drain => self.begin_drain(index, primary_end, now),
OpenSessionFailureCleanup::Close => self.begin_close(index, primary_end, None, now),
}
}
pub(super) fn effective_retirement(&self, index: usize) -> Option<ManagedTargetEndReason> {
self.targets[index]
.membership
.retirement_reason()
.map(Into::into)
}
fn start_connecting(&mut self, index: usize) {
let TargetPhase::Pending { client_config } = self.targets[index].take_phase() else {
unreachable!("connecting starts from a pending target")
};
let endpoint = Arc::clone(&self.targets[index].server_addr);
let future = Box::pin(async move { AsyncClient::connect(endpoint, client_config).await });
self.targets[index].restore_phase(TargetPhase::Connecting { future });
self.publish_target_phase(index, ManagedTargetLifecycle::Pending);
}
pub(super) fn poll_target(&mut self, index: usize, cx: &mut Context<'_>, now: Instant) -> bool {
let retirement = self.effective_retirement(index);
let stopping = self.phase.is_stopping() || retirement.is_some();
match self.targets[index].phase_mut() {
TargetPhase::Pending { .. } => {
if stopping {
self.finish_target(
index,
retirement
.clone()
.unwrap_or(ManagedTargetEndReason::Stopped),
None,
);
false
} else {
self.start_connecting(index);
true
}
}
TargetPhase::Connecting { future } => {
if stopping {
self.finish_target(
index,
retirement
.clone()
.unwrap_or(ManagedTargetEndReason::Stopped),
None,
);
return false;
}
match future.as_mut().poll(cx) {
Poll::Pending => false,
Poll::Ready(Ok(client)) => {
self.targets[index].remote = Some(client.remote_addr());
self.install_target_phase(
index,
TargetPhase::Opening {
client,
open: Box::new(AsyncOpenState::new()),
},
);
true
}
Poll::Ready(Err(error)) => {
self.fail_target(index, ManagedTargetFailurePhase::Connecting, error);
false
}
}
}
TargetPhase::Opening { client, open } => {
if stopping {
let retirement = retirement
.clone()
.unwrap_or(ManagedTargetEndReason::Stopped);
if !open.has_in_flight_work() {
self.targets[index].sync_live_packets_sent();
self.finish_target(index, retirement, None);
return false;
}
open.request_stop_after_current_attempt();
}
match client.poll_open(open, cx) {
Poll::Pending => false,
Poll::Ready(Ok(OpenOutcome::Started(started))) => {
let schedule = match ProbeSchedule::new(
started.at.mono,
&started.negotiation.accepted,
) {
Ok(schedule) => schedule,
Err(error) => {
return self.begin_open_session_failure(
index,
ManagedTargetFailurePhase::Timing,
error,
now,
OpenSessionFailureCleanup::Close,
)
}
};
self.publish_client_events(
index,
vec![ClientEvent::SessionStarted(started)],
);
if stopping {
self.targets[index].sync_live_packets_sent();
self.begin_drain(
index,
retirement
.clone()
.unwrap_or(ManagedTargetEndReason::Stopped),
now,
)
} else {
let previous = self.targets[index].take_phase();
self.targets[index].restore_phase(TargetPhase::Active {
client: previous.into_client(),
schedule,
send_waiting: false,
});
self.publish_target_phase(index, ManagedTargetLifecycle::Opening);
true
}
}
Poll::Ready(Ok(OpenOutcome::NoTestCompleted(completed))) => {
self.publish_client_events(
index,
vec![ClientEvent::NoTestCompleted(completed)],
);
let end_reason = if stopping {
retirement
.clone()
.unwrap_or(ManagedTargetEndReason::Stopped)
} else {
ManagedTargetEndReason::NoTestComplete
};
self.finish_target(index, end_reason, None);
false
}
Poll::Ready(Err(error)) => {
if stopping {
let TargetPhase::Opening { open, .. } = self.targets[index].phase()
else {
unreachable!()
};
let cleanup_failure =
(!open.stopped_after_current_attempt()).then(|| {
classify_client_error(
ManagedTargetFailurePhase::Opening,
&error,
)
});
self.finish_target(
index,
retirement
.clone()
.unwrap_or(ManagedTargetEndReason::Stopped),
cleanup_failure,
);
} else {
self.fail_target(index, ManagedTargetFailurePhase::Opening, error);
}
false
}
}
}
TargetPhase::Active { client, .. } => {
if stopping {
client.discard_prepared_probe();
self.targets[index].sync_live_packets_sent();
return self.begin_drain(
index,
retirement
.clone()
.unwrap_or(ManagedTargetEndReason::Stopped),
now,
);
}
if client
.next_probe_timeout_deadline()
.is_some_and(|deadline| deadline <= now)
{
return true;
}
let received = match client.poll_recv(cx) {
Poll::Pending => false,
Poll::Ready(Ok(events)) => {
self.publish_client_events(index, events);
true
}
Poll::Ready(Err(error)) => {
return self.begin_open_session_failure(
index,
ManagedTargetFailurePhase::Receiving,
error,
now,
OpenSessionFailureCleanup::Close,
);
}
};
if self.targets[index].phase().client().is_peer_closed() {
self.targets[index].sync_live_packets_sent();
self.finish_target(index, ManagedTargetEndReason::PeerClosed, None);
return false;
}
let TargetPhase::Active {
client, schedule, ..
} = self.targets[index].phase_mut()
else {
unreachable!()
};
schedule.permit_probe_at(now);
if schedule.is_finished() && !client.has_pending_probes() {
self.targets[index].sync_live_packets_sent();
return self.begin_drain(index, ManagedTargetEndReason::TestComplete, now);
}
received
}
TargetPhase::Draining { client, .. } => {
if client
.next_probe_timeout_deadline()
.is_some_and(|timeout| timeout <= now)
{
return true;
}
let mut retained_state_changed = false;
let received = match client.poll_recv(cx) {
Poll::Pending => false,
Poll::Ready(Ok(events)) => {
self.publish_client_events(index, events);
retained_state_changed = true;
true
}
Poll::Ready(Err(error)) => {
self.targets[index].sync_live_packets_sent();
let TargetPhase::Draining {
primary_end,
cleanup_failure,
..
} = self.targets[index].phase_mut()
else {
unreachable!()
};
cleanup_failure.get_or_insert_with(|| {
classify_client_error(ManagedTargetFailurePhase::Receiving, &error)
});
let primary_end = primary_end.clone();
let cleanup_failure = cleanup_failure.clone();
return self.begin_close(index, primary_end, cleanup_failure, now);
}
};
if self.targets[index].phase().client().is_peer_closed() {
self.targets[index].sync_live_packets_sent();
let TargetPhase::Draining {
cleanup_failure, ..
} = self.targets[index].phase()
else {
unreachable!()
};
self.finish_target(
index,
ManagedTargetEndReason::PeerClosed,
cleanup_failure.clone(),
);
return false;
}
if retained_state_changed {
let TargetPhase::Draining {
client,
drain_started_at,
..
} = self.targets[index].phase()
else {
unreachable!()
};
let candidate = self.drain_deadline(client, *drain_started_at);
let TargetPhase::Draining {
deadline,
primary_end,
cleanup_failure,
..
} = self.targets[index].phase_mut()
else {
unreachable!()
};
match candidate {
Some(candidate) => *deadline = (*deadline).min(candidate),
None => {
cleanup_failure.get_or_insert_with(duration_overflow_failure);
let primary_end = primary_end.clone();
let cleanup_failure = cleanup_failure.clone();
return self.begin_close(index, primary_end, cleanup_failure, now);
}
}
}
let TargetPhase::Draining {
deadline,
primary_end,
cleanup_failure,
post_deadline_receives_remaining,
..
} = self.targets[index].phase_mut()
else {
unreachable!()
};
if now >= *deadline {
if received && *post_deadline_receives_remaining > 1 {
*post_deadline_receives_remaining -= 1;
true
} else {
let primary_end = primary_end.clone();
let cleanup_failure = cleanup_failure.clone();
self.begin_close(index, primary_end, cleanup_failure, now)
}
} else {
received
}
}
TargetPhase::Closing { client, .. } => {
let result = client.poll_close(cx);
let TargetPhase::Closing {
deadline,
primary_end,
cleanup_failure,
..
} = self.targets[index].phase()
else {
unreachable!()
};
let deadline = *deadline;
let primary_end = primary_end.clone();
let cleanup_failure = cleanup_failure.clone();
match result {
Poll::Pending => {
if now >= deadline {
self.targets[index].sync_live_packets_sent();
self.finish_target(
index,
primary_end,
cleanup_failure.or_else(|| Some(close_timeout_failure())),
);
}
false
}
Poll::Ready(Ok(events)) => {
self.targets[index].sync_live_packets_sent();
self.publish_client_events(index, events);
self.finish_target(index, primary_end, cleanup_failure);
false
}
Poll::Ready(Err(error)) => {
self.targets[index].sync_live_packets_sent();
let cleanup =
classify_client_error(ManagedTargetFailurePhase::Closing, &error);
self.finish_target(index, primary_end, cleanup_failure.or(Some(cleanup)));
false
}
}
}
TargetPhase::Terminal { .. } => false,
}
}
pub(super) fn begin_drain(
&mut self,
index: usize,
primary_end: ManagedTargetEndReason,
_now: Instant,
) -> bool {
let drain_started_at = Instant::now();
let Some(deadline) =
self.drain_deadline(self.targets[index].phase().client(), drain_started_at)
else {
return self.begin_close(
index,
primary_end,
Some(duration_overflow_failure()),
drain_started_at,
);
};
let previous = self.targets[index].take_phase();
let lifecycle = previous.lifecycle();
self.targets[index].restore_phase(TargetPhase::Draining {
client: previous.into_client(),
drain_started_at,
deadline,
primary_end,
cleanup_failure: None,
post_deadline_receives_remaining: POST_DEADLINE_RECEIVE_BUDGET,
});
self.publish_target_phase(index, lifecycle);
true
}
pub(super) fn drain_deadline(
&self,
client: &AsyncClient,
drain_started_at: Instant,
) -> Option<Instant> {
client
.latest_probe_timeout_deadline()
.unwrap_or(drain_started_at)
.max(drain_started_at)
.checked_add(self.config.final_drain)
}
pub(super) fn begin_close(
&mut self,
index: usize,
primary_end: ManagedTargetEndReason,
mut cleanup_failure: Option<ManagedTargetFailure>,
now: Instant,
) -> bool {
let deadline = now.checked_add(self.config.final_drain).unwrap_or_else(|| {
cleanup_failure.get_or_insert_with(duration_overflow_failure);
now
});
let previous = self.targets[index].take_phase();
let lifecycle = previous.lifecycle();
self.targets[index].restore_phase(TargetPhase::Closing {
client: previous.into_client(),
deadline,
primary_end,
cleanup_failure,
});
self.publish_target_phase(index, lifecycle);
true
}
}
pub(super) fn duration_overflow_failure() -> ManagedTargetFailure {
classify_client_error(
ManagedTargetFailurePhase::Timing,
&ClientError::DurationOverflow,
)
}
fn close_timeout_failure() -> ManagedTargetFailure {
ManagedTargetFailure {
phase: ManagedTargetFailurePhase::Closing,
kind: ManagedTargetFailureKind::Timeout,
message: Arc::from("best-effort close did not become writable before its deadline"),
}
}