test(client): add integration tests for compatibility with real IRTT backends

Introduce a mechanism to run tests against a live irtt binary to verify real-world protocol interoperability, including HMAC authentication and various timestamp modes.

crates/irtt-client/tests/hmac.rs

@@ -9,9 +9,10 @@ use irtt_client::{
 use irtt_proto::{echo_packet_len, ProtoError, TimestampFields};
 
 use support::{
-    config_for_params, default_params, params_for_modes, run_one_probe_with_config,
+    config_for_params, default_params, params_for_modes, require_real_backend,
-    standard_timestamps, start_bad_hmac_echo_reply_server, start_hmac_close_server,
+    run_one_probe_with_config, standard_timestamps, start_bad_hmac_echo_reply_server,
-    start_hmac_required_open_drop_server, ServerObservation, TOKEN,
+    start_hmac_close_server, start_hmac_required_open_drop_server, RealIrtServer,
+    ServerObservation, TOKEN,
 };
 
 #[test]
@@ -294,3 +295,64 @@ fn assert_no_warning_or_error(events: &[ClientEvent]) {
 fn assert_no_duplicate_late_or_warning(events: &[ClientEvent]) {
     assert_no_warning_or_error(events);
 }
+
+#[test]
+fn real_hmac_correct_key_succeeds() {
+    if !require_real_backend() {
+        return;
+    }
+    let key = b"compat-secret".to_vec();
+    let server = RealIrtServer::start(Some(&key)).unwrap();
+    let params = default_params();
+    let mut config = config_for_params(server.addr(), &params);
+    config.hmac_key = Some(key);
+
+    let mut client = Client::connect(config).unwrap();
+    let outcome = client.open(ClientTimestamp::now()).unwrap();
+    assert!(matches!(outcome, OpenOutcome::Started { .. }));
+
+    let sent = client.send_probe().unwrap();
+    assert_eq!(sent.len(), 1);
+
+    let events = client.recv_once().unwrap();
+    assert_eq!(events.len(), 1);
+    assert!(matches!(events[0], ClientEvent::EchoReply { .. }));
+
+    client.close(ClientTimestamp::now()).unwrap();
+}
+
+#[test]
+fn real_hmac_wrong_key_fails() {
+    if !require_real_backend() {
+        return;
+    }
+    let server_key = b"compat-secret".to_vec();
+    let server = RealIrtServer::start(Some(&server_key)).unwrap();
+    let mut config = config_for_params(server.addr(), &default_params());
+    config.open_timeouts = vec![Duration::from_millis(200)];
+    config.hmac_key = Some(b"wrong-secret".to_vec());
+
+    let mut client = Client::connect(config).unwrap();
+    assert!(matches!(
+        client.open(ClientTimestamp::now()),
+        Err(ClientError::OpenTimeout)
+    ));
+}
+
+#[test]
+fn real_hmac_missing_key_fails() {
+    if !require_real_backend() {
+        return;
+    }
+    let server_key = b"compat-secret".to_vec();
+    let server = RealIrtServer::start(Some(&server_key)).unwrap();
+    let mut config = config_for_params(server.addr(), &default_params());
+    config.open_timeouts = vec![Duration::from_millis(200)];
+    config.hmac_key = None;
+
+    let mut client = Client::connect(config).unwrap();
+    assert!(matches!(
+        client.open(ClientTimestamp::now()),
+        Err(ClientError::OpenTimeout)
+    ));
+}

crates/irtt-client/tests/protocol_options.rs

@@ -1,5 +1,6 @@
 mod support;
 
+use irtt_client::ClientTimestamp;
 use std::time::Duration;
 
 use irtt_client::{
@@ -9,9 +10,9 @@ use irtt_client::{
 use irtt_proto::{echo_packet_len, Clock, Params, ReceivedStats, StampAt, TimestampFields};
 
 use support::{
-    config_for_params, default_params, params_for_modes, run_one_probe, run_one_probe_with_config,
+    config_for_params, default_params, params_for_modes, require_real_backend, run_one_probe,
-    server_fill, standard_timestamps, start_open_server, OneProbeRun, ServerObservation,
+    run_one_probe_with_config, server_fill, standard_timestamps, start_open_server, OneProbeRun,
-    RECV_COUNT, RECV_WINDOW, TOKEN,
+    RealIrtServer, ServerObservation, RECV_COUNT, RECV_WINDOW, TOKEN,
 };
 
 struct ReplyView<'a> {
@@ -371,3 +372,148 @@ fn assert_one_way_presence(
     assert_eq!(sample.client_to_server.is_some(), client_to_server);
     assert_eq!(sample.server_to_client.is_some(), server_to_client);
 }
+
+#[test]
+fn real_basic_open_echo_close() {
+    if !require_real_backend() {
+        return;
+    }
+    let server = RealIrtServer::start(None).unwrap();
+    let params = default_params();
+    let mut client = Client::connect(config_for_params(server.addr(), &params)).unwrap();
+
+    let outcome = client.open(ClientTimestamp::now()).unwrap();
+    assert!(matches!(outcome, irtt_client::OpenOutcome::Started { .. }));
+
+    let sent = client.send_probe().unwrap();
+    assert_eq!(sent.len(), 1);
+
+    let events = client.recv_once().unwrap();
+    assert_eq!(events.len(), 1);
+    assert!(matches!(events[0], ClientEvent::EchoReply { .. }));
+
+    client.close(ClientTimestamp::now()).unwrap();
+}
+
+#[test]
+fn real_received_stats_smoke() {
+    if !require_real_backend() {
+        return;
+    }
+    for mode in [
+        ReceivedStats::None,
+        ReceivedStats::Count,
+        ReceivedStats::Window,
+        ReceivedStats::Both,
+    ] {
+        let server = RealIrtServer::start(None).unwrap();
+        let params = params_for_modes(mode, StampAt::None, Clock::Both);
+        let mut client = Client::connect(config_for_params(server.addr(), &params)).unwrap();
+
+        client.open(ClientTimestamp::now()).unwrap();
+        client.send_probe().unwrap();
+        let events = client.recv_once().unwrap();
+        assert_eq!(events.len(), 1);
+        assert!(matches!(events[0], ClientEvent::EchoReply { .. }));
+
+        client.close(ClientTimestamp::now()).unwrap();
+    }
+}
+
+#[test]
+fn real_timestamp_smoke() {
+    if !require_real_backend() {
+        return;
+    }
+    for mode in [
+        StampAt::None,
+        StampAt::Send,
+        StampAt::Receive,
+        StampAt::Both,
+        StampAt::Midpoint,
+    ] {
+        let server = RealIrtServer::start(None).unwrap();
+        let params = params_for_modes(ReceivedStats::None, mode, Clock::Both);
+        let mut client = Client::connect(config_for_params(server.addr(), &params)).unwrap();
+
+        client.open(ClientTimestamp::now()).unwrap();
+        client.send_probe().unwrap();
+        let events = client.recv_once().unwrap();
+        assert_eq!(events.len(), 1);
+
+        match &events[0] {
+            ClientEvent::EchoReply {
+                server_timing,
+                one_way,
+                ..
+            } => match mode {
+                StampAt::None => {
+                    assert!(server_timing.is_none());
+                    assert!(one_way.is_none());
+                }
+                StampAt::Send => {
+                    let st = server_timing.as_ref().unwrap();
+                    assert!(st.send_wall_ns.is_some() || st.send_mono_ns.is_some());
+                }
+                StampAt::Receive => {
+                    let st = server_timing.as_ref().unwrap();
+                    assert!(st.receive_wall_ns.is_some() || st.receive_mono_ns.is_some());
+                }
+                StampAt::Both => {
+                    let st = server_timing.as_ref().unwrap();
+                    assert!(st.receive_wall_ns.is_some() || st.receive_mono_ns.is_some());
+                    assert!(st.send_wall_ns.is_some() || st.send_mono_ns.is_some());
+                }
+                StampAt::Midpoint => {
+                    let st = server_timing.as_ref().unwrap();
+                    assert!(st.midpoint_wall_ns.is_some() || st.midpoint_mono_ns.is_some());
+                }
+            },
+            other => panic!("expected EchoReply, got {other:?}"),
+        }
+
+        client.close(ClientTimestamp::now()).unwrap();
+    }
+}
+
+#[test]
+fn real_clock_smoke() {
+    if !require_real_backend() {
+        return;
+    }
+    for clock in [Clock::Wall, Clock::Monotonic, Clock::Both] {
+        let server = RealIrtServer::start(None).unwrap();
+        let params = params_for_modes(ReceivedStats::None, StampAt::Both, clock);
+        let mut client = Client::connect(config_for_params(server.addr(), &params)).unwrap();
+
+        client.open(ClientTimestamp::now()).unwrap();
+        client.send_probe().unwrap();
+        let events = client.recv_once().unwrap();
+        assert_eq!(events.len(), 1);
+
+        match &events[0] {
+            ClientEvent::EchoReply { server_timing, .. } => {
+                let st = server_timing.as_ref().unwrap();
+                match clock {
+                    Clock::Wall => {
+                        assert!(st.receive_wall_ns.is_some());
+                        assert!(st.send_wall_ns.is_some());
+                    }
+                    Clock::Monotonic => {
+                        assert!(st.receive_mono_ns.is_some());
+                        assert!(st.send_mono_ns.is_some());
+                    }
+                    Clock::Both => {
+                        assert!(st.receive_wall_ns.is_some());
+                        assert!(st.receive_mono_ns.is_some());
+                        assert!(st.send_wall_ns.is_some());
+                        assert!(st.send_mono_ns.is_some());
+                    }
+                }
+            }
+            other => panic!("expected EchoReply, got {other:?}"),
+        }
+
+        client.close(ClientTimestamp::now()).unwrap();
+    }
+}

crates/irtt-client/tests/support/mod.rs

@@ -1,7 +1,10 @@
 #![allow(dead_code)]
 
+mod real_irtt;
+
 use std::{
     net::{SocketAddr, UdpSocket},
+    process::Command,
     sync::mpsc,
     thread::{self, JoinHandle},
     time::{Duration, SystemTime},
@@ -15,11 +18,27 @@ use irtt_proto::{
     ReceivedStats, ServerFill, StampAt, TimestampFields, HMAC_SIZE, MAGIC, PROTOCOL_VERSION,
 };
 
+pub use real_irtt::RealIrtServer;
+
 const HMAC_OFFSET: usize = 4;
 pub const TOKEN: u64 = 0x1234_5678_90ab_cdef;
 pub const RECV_COUNT: u32 = 42;
 pub const RECV_WINDOW: u64 = 0x07;
 
+pub fn require_real_backend() -> bool {
+    match std::env::var("IRTT_TEST_BACKEND").as_deref() {
+        Ok("real") => {
+            let irtt_bin = std::env::var("IRTT_BIN").unwrap_or_else(|_| "irtt".to_string());
+            match Command::new(&irtt_bin).arg("version").output() {
+                Ok(output) if output.status.success() => true,
+                _ => panic!("IRTT_TEST_BACKEND=real but irtt binary not found at '{irtt_bin}'"),
+            }
+        }
+        Ok("fake") | Err(_) => false,
+        Ok(other) => panic!("unknown IRTT_TEST_BACKEND value: {other}"),
+    }
+}
+
 pub struct FakeServer {
     pub addr: SocketAddr,
     rx: mpsc::Receiver<ServerObservation>,

crates/irtt-client/tests/support/real_irtt.rs

@@ -0,0 +1,115 @@
+#![allow(dead_code)]
+
+use std::io::{BufRead, BufReader};
+use std::net::{SocketAddr, UdpSocket};
+use std::process::{Child, Command, Stdio};
+use std::sync::mpsc;
+use std::thread;
+use std::time::{Duration, Instant};
+
+pub struct RealIrtServer {
+    addr: SocketAddr,
+    child: Child,
+}
+
+impl RealIrtServer {
+    pub fn start(hmac_key: Option<&[u8]>) -> Result<Self, String> {
+        let port = Self::find_free_port()?;
+        let addr = SocketAddr::from(([127, 0, 0, 1], port));
+        let bind = format!("127.0.0.1:{}", port);
+
+        let irtt_bin = std::env::var("IRTT_BIN").unwrap_or_else(|_| "irtt".to_string());
+
+        let mut cmd = Command::new(&irtt_bin);
+        cmd.arg("server")
+            .arg("-b")
+            .arg(&bind)
+            .arg("--tstamp=dual")
+            .stderr(Stdio::piped())
+            .stdout(Stdio::piped());
+
+        if let Some(key) = hmac_key {
+            let hex_key: String = key.iter().map(|b| format!("{:02x}", b)).collect();
+            cmd.arg(format!("--hmac=0x{}", hex_key));
+        }
+
+        let mut child = cmd
+            .spawn()
+            .map_err(|e| format!("failed to spawn irtt server: {}", e))?;
+
+        let stdout = child.stdout.take().ok_or("failed to capture stdout")?;
+        let (tx, rx) = mpsc::channel();
+        thread::spawn(move || {
+            let reader = BufReader::new(stdout);
+            for line in reader.lines() {
+                match line {
+                    Ok(l) => {
+                        if tx.send(l).is_err() {
+                            break;
+                        }
+                    }
+                    Err(_) => break,
+                }
+            }
+        });
+
+        let timeout = Duration::from_secs(5);
+        let start = Instant::now();
+        let mut stdout_lines = Vec::new();
+        loop {
+            let remaining = timeout
+                .checked_sub(start.elapsed())
+                .unwrap_or(Duration::ZERO);
+            match rx.recv_timeout(remaining) {
+                Ok(line) => {
+                    stdout_lines.push(line.clone());
+                    if line.contains("[ListenerStart]") {
+                        break;
+                    }
+                }
+                Err(mpsc::RecvTimeoutError::Timeout) => {
+                    child.kill().ok();
+                    child.wait().ok();
+                    return Err(format!(
+                        "timeout waiting for irtt server to start\nstdout:\n{}",
+                        stdout_lines.join("\n")
+                    ));
+                }
+                Err(mpsc::RecvTimeoutError::Disconnected) => {
+                    child.kill().ok();
+                    child.wait().ok();
+                    return Err(format!(
+                        "irtt server exited unexpectedly\nstdout:\n{}",
+                        stdout_lines.join("\n")
+                    ));
+                }
+            }
+        }
+
+        drop(rx);
+
+        Ok(Self { addr, child })
+    }
+
+    pub fn addr(&self) -> SocketAddr {
+        self.addr
+    }
+
+    fn find_free_port() -> Result<u16, String> {
+        let socket = UdpSocket::bind("127.0.0.1:0")
+            .map_err(|e| format!("failed to find free port: {}", e))?;
+        let port = socket
+            .local_addr()
+            .map_err(|e| format!("failed to get local addr: {}", e))?
+            .port();
+        drop(socket);
+        Ok(port)
+    }
+}
+
+impl Drop for RealIrtServer {
+    fn drop(&mut self) {
+        self.child.kill().ok();
+        self.child.wait().ok();
+    }
+}