fix(client): preserve partial summaries on graceful interruption

This commit is contained in:
Nils Andreas Svee committed 2026-10-28 11:10:18 +07:00
1 parent c8a911297d
commit 5b5f5b2609
1 file changed
+87 -22
+87 -22
View File
@@ -3,6 +3,7 @@ use std::{
io::{self, BufRead, Write}, io::{self, BufRead, Write},
sync::Arc, sync::Arc,
thread, thread,
time::Duration,
}; };
use irtt_client::{ use irtt_client::{
@@ -35,18 +36,53 @@ use irtt_client::managed::ManagedStatus;
use std::collections::{BTreeMap, BTreeSet}; use std::collections::{BTreeMap, BTreeSet};
const MAX_STDIN_RECORD_BYTES: usize = 64 * 1024; const MAX_STDIN_RECORD_BYTES: usize = 64 * 1024;
const INTERRUPTED_OUTPUT_GRACE: Duration = Duration::from_millis(250);
// The frontend only renders into memory. This thread is the sole stdout owner; // The frontend only renders into memory. This thread is the sole stdout owner;
// dropping its JoinHandle deliberately detaches a writer blocked in a pipe. // dropping its JoinHandle deliberately detaches a writer blocked in a pipe.
struct StdoutWriter { struct StdoutWriter {
sender: tokio::sync::mpsc::Sender<Vec<u8>>, sender: tokio::sync::mpsc::Sender<Vec<u8>>,
shutdown: tokio::sync::watch::Receiver<bool>,
grace_deadline: Option<tokio::time::Instant>,
} }
impl StdoutWriter { impl StdoutWriter {
// Measurement output stops waiting for capacity as soon as interrupted.
async fn send(&self, buffer: &mut Vec<u8>) -> io::Result<()> { async fn send(&self, buffer: &mut Vec<u8>) -> io::Result<()> {
let mut shutdown = self.shutdown.clone();
tokio::select! {
biased;
_ = shutdown.wait_for(|requested| *requested) => {
buffer.clear();
Ok(())
}
result = self.enqueue(buffer) => result,
}
}
// Summaries and writer completion share one grace period, regardless of
// the number of targets. Normal output never has a delivery deadline.
async fn send_summary(&mut self, buffer: &mut Vec<u8>) -> io::Result<()> {
let mut shutdown = self.shutdown.clone();
tokio::select! {
biased;
_ = shutdown.wait_for(|requested| *requested) => {
let deadline = *self.grace_deadline.get_or_insert_with(|| {
tokio::time::Instant::now() + INTERRUPTED_OUTPUT_GRACE
});
match tokio::time::timeout_at(deadline, self.enqueue(buffer)).await {
Ok(result) => result,
Err(_) => { buffer.clear(); Ok(()) }
}
}
result = self.enqueue(buffer) => result,
}
}
async fn enqueue(&self, buffer: &mut Vec<u8>) -> io::Result<()> {
if !buffer.is_empty() { if !buffer.is_empty() {
// Reserve before taking the buffer: cancellation never discards an // Take ownership only once capacity is available. Normal operation
// already accepted chunk, and normal operation preserves ordering. // preserves ordering without dropping rendered chunks.
let permit = self let permit = self
.sender .sender
.reserve() .reserve()
@@ -80,25 +116,36 @@ pub async fn run_stream(
})(); })();
let _ = completed.send(result); let _ = completed.send(result);
})?; })?;
let output = StdoutWriter { sender }; let mut output = StdoutWriter {
sender,
shutdown: shutdown.clone(),
grace_deadline: None,
};
let run_shutdown = shutdown.clone(); let run_shutdown = shutdown.clone();
let result = tokio::select! { let result = tokio::select! {
biased; biased;
_ = shutdown.wait_for(|requested| *requested) => return Ok(()),
result = &mut completion => { result = &mut completion => {
// Dropping the run future stops and joins its ManagedWorker first. // Dropping the run future stops and joins its ManagedWorker first.
writer.join().map_err(|_| io::Error::other("stdout writer thread panicked"))?; writer.join().map_err(|_| io::Error::other("stdout writer thread panicked"))?;
result.map_err(|_| io::Error::other("stdout writer completed without a result"))??; result.map_err(|_| io::Error::other("stdout writer completed without a result"))??;
return Err(io::Error::other("stdout writer stopped unexpectedly").into()); return Err(io::Error::other("stdout writer stopped unexpectedly").into());
} }
result = run_stream_output(args, run_shutdown, &output) => result, result = run_stream_output(args, run_shutdown, &mut output) => result,
}; };
let grace_deadline = output.grace_deadline;
drop(output); drop(output);
// The same cancellation guard applies after the managed worker has ended, // An interrupt during normal flushing also starts the bounded grace period.
// while queued events and summaries are still being written.
let written = tokio::select! { let written = tokio::select! {
biased; biased;
_ = shutdown.wait_for(|requested| *requested) => return result, _ = shutdown.wait_for(|requested| *requested) => {
let deadline = grace_deadline.unwrap_or_else(|| {
tokio::time::Instant::now() + INTERRUPTED_OUTPUT_GRACE
});
match tokio::time::timeout_at(deadline, &mut completion).await {
Ok(written) => written,
Err(_) => return result,
}
}
result = &mut completion => result, result = &mut completion => result,
}; };
writer writer
@@ -111,7 +158,7 @@ pub async fn run_stream(
async fn run_stream_output( async fn run_stream_output(
args: ClientArgs, args: ClientArgs,
mut shutdown: tokio::sync::watch::Receiver<bool>, mut shutdown: tokio::sync::watch::Receiver<bool>,
output: &StdoutWriter, output: &mut StdoutWriter,
) -> Result<(), Box<dyn std::error::Error>> { ) -> Result<(), Box<dyn std::error::Error>> {
let setup = tokio::select! { let setup = tokio::select! {
result = args.prepare() => result.map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?, result = args.prepare() => result.map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err))?,
@@ -202,14 +249,15 @@ async fn run_stream_output(
) { ) {
drop(handle.stop()); drop(handle.stop());
} }
if snapshot.final_outcome.is_some() { if interrupted || snapshot.final_outcome.is_some() {
break; break;
} }
tokio::select! { tokio::select! {
_ = shutdown.changed(), if !interrupted => {} _ = shutdown.changed() => {}
changed = status.changed() => { if changed.is_err() { break; } } changed = status.changed() => { if changed.is_err() { break; } }
event = events.recv() => match event { event = events.recv() => match event {
Ok(event) => { Ok(event) => {
if *shutdown.borrow() { continue; }
process_event(event, &mut stream_output, #[cfg(feature = "stats")] &mut stats, &mut terminal_targets)?; process_event(event, &mut stream_output, #[cfg(feature = "stats")] &mut stats, &mut terminal_targets)?;
output.send(stream_output.out).await?; output.send(stream_output.out).await?;
} }
@@ -224,7 +272,7 @@ async fn run_stream_output(
eprintln!("interrupted, closing managed run..."); eprintln!("interrupted, closing managed run...");
} }
let outcome = owner.join().await?; let outcome = owner.join().await?;
loop { while !*shutdown.borrow() {
match events.try_recv() { match events.try_recv() {
Ok(event) => { Ok(event) => {
process_event( process_event(
@@ -292,10 +340,10 @@ async fn run_stream_output(
writeln!(stream_output.out, "target: {}", target.label)?; writeln!(stream_output.out, "target: {}", target.label)?;
} }
stream_output.print_summary(target_stats)?; stream_output.print_summary(target_stats)?;
output.send(stream_output.out).await?; output.send_summary(stream_output.out).await?;
} }
} }
output.send(stream_output.out).await?; output.send_summary(stream_output.out).await?;
if let Some(error) = terminal_error { if let Some(error) = terminal_error {
return Err(error.into()); return Err(error.into());
} }
@@ -557,7 +605,7 @@ async fn run_stdin_stream(
output_config: OutputConfig, output_config: OutputConfig,
mut shutdown: tokio::sync::watch::Receiver<bool>, mut shutdown: tokio::sync::watch::Receiver<bool>,
args: &ClientArgs, args: &ClientArgs,
output: &StdoutWriter, output: &mut StdoutWriter,
) -> Result<(), Box<dyn std::error::Error>> { ) -> Result<(), Box<dyn std::error::Error>> {
if *shutdown.borrow() { if *shutdown.borrow() {
return Ok(()); return Ok(());
@@ -610,10 +658,19 @@ async fn run_stdin_stream(
let mut summary_targets = None; let mut summary_targets = None;
loop { loop {
let snapshot = status.borrow_and_update().clone(); let snapshot = status.borrow_and_update().clone();
if stop.is_none() {
if *shutdown.borrow() { if *shutdown.borrow() {
stop = Some(StdinStop::Interrupted); stop = Some(StdinStop::Interrupted);
} else if stdin_changed || stdin.has_changed().unwrap_or(true) { #[cfg(feature = "stats")]
{
summary_targets = Some(stats.keys().cloned().collect());
}
if request_managed_stop_once(&mut stop_requested) {
drop(handle.stop());
}
break;
}
if stop.is_none() {
if stdin_changed || stdin.has_changed().unwrap_or(true) {
stdin_changed = false; stdin_changed = false;
match stdin.borrow_and_update().clone() { match stdin.borrow_and_update().clone() {
StdinUpdate::Targets(specs) => { StdinUpdate::Targets(specs) => {
@@ -677,7 +734,7 @@ async fn run_stdin_stream(
break; break;
} }
tokio::select! { tokio::select! {
_ = shutdown.changed(), if stop.is_none() => {} _ = shutdown.changed() => {}
_ = stdin.changed(), if stop.is_none() => { stdin_changed = true; } _ = stdin.changed(), if stop.is_none() => { stdin_changed = true; }
changed = status.changed() => { if changed.is_err() { break; } } changed = status.changed() => { if changed.is_err() { break; } }
result = async { preparing.as_mut().expect("guarded target preparation").await }, if preparing.is_some() && stop.is_none() => { result = async { preparing.as_mut().expect("guarded target preparation").await }, if preparing.is_some() && stop.is_none() => {
@@ -727,10 +784,11 @@ async fn run_stdin_stream(
} }
event = events.recv() => match event { event = events.recv() => match event {
Ok(event) => { Ok(event) => {
if *shutdown.borrow() { continue; }
process_stdin_event(event, &mut stream_output, #[cfg(feature = "stats")] &mut stats, &mut terminal_targets)?; process_stdin_event(event, &mut stream_output, #[cfg(feature = "stats")] &mut stats, &mut terminal_targets)?;
output.send(stream_output.out).await?;
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
reconcile_stdin_stats(&mut stats, &status.borrow(), summary_targets.as_ref()); reconcile_stdin_stats(&mut stats, &status.borrow(), summary_targets.as_ref());
output.send(stream_output.out).await?;
} }
Err(tokio::sync::broadcast::error::RecvError::Lagged(count)) => { Err(tokio::sync::broadcast::error::RecvError::Lagged(count)) => {
dropped_events = dropped_events.saturating_add(count); dropped_events = dropped_events.saturating_add(count);
@@ -741,7 +799,7 @@ async fn run_stdin_stream(
} }
let outcome = owner.join().await?; let outcome = owner.join().await?;
loop { while !*shutdown.borrow() {
match events.try_recv() { match events.try_recv() {
Ok(event) => { Ok(event) => {
process_stdin_event( process_stdin_event(
@@ -774,6 +832,13 @@ async fn run_stdin_stream(
); );
} }
if *shutdown.borrow() && !matches!(stop, Some(StdinStop::Interrupted)) {
stop = Some(StdinStop::Interrupted);
#[cfg(feature = "stats")]
{
summary_targets = Some(stats.keys().cloned().collect());
}
}
#[cfg(feature = "stats")] #[cfg(feature = "stats")]
{ {
let print_summary = let print_summary =
@@ -795,11 +860,11 @@ async fn run_stdin_stream(
)?; )?;
} }
stream_output.print_summary(stats)?; stream_output.print_summary(stats)?;
output.send(stream_output.out).await?; output.send_summary(stream_output.out).await?;
} }
} }
} }
output.send(stream_output.out).await?; output.send_summary(stream_output.out).await?;
match stop { match stop {
Some(StdinStop::Fatal(error)) => Err(error.into()), Some(StdinStop::Fatal(error)) => Err(error.into()),