diff --git a/Cargo.lock b/Cargo.lock index 2313376..fb939f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,15 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -123,6 +132,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "config" version = "0.15.27" @@ -135,6 +150,21 @@ dependencies = [ "winnow", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -156,6 +186,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctutils" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +dependencies = [ + "cmov", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "ctutils", +] + [[package]] name = "either" version = "1.19.0" @@ -242,6 +293,24 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest", +] + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "icmp-socket2" version = "0.6.4" @@ -265,6 +334,31 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "irtt-client" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a59c24704bd94f19b84b2b0190aa9163efede499f8cf4eb02922443cfee08339" +dependencies = [ + "irtt-proto", + "nix", + "socket2", + "thiserror", + "tokio", +] + +[[package]] +name = "irtt-proto" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56ff307983b8f6477724e08cf726c5e24975b9affeb853d0830d91c60026ab2" +dependencies = [ + "hmac", + "md-5", + "subtle", + "thiserror", +] + [[package]] name = "itertools" version = "0.13.0" @@ -337,6 +431,16 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.3" @@ -679,6 +783,7 @@ dependencies = [ "fastrand", "flume", "icmp-socket2", + "irtt-client", "libc", "log", "netlink-bindings", @@ -697,6 +802,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85ec932792e62b2d3936163eacbd2e09984453bcd114f665c1e1694e36ff59c9" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.119" @@ -836,6 +947,12 @@ dependencies = [ "toml", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "uci-derive" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 777f1c5..b4df466 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ csv = "1.4.0" fastrand = "2" flume = { version = "0.12.0", default-features = false, features = ["async", "eventual-fairness", "select"] } icmp-socket2 = { version = "0.6.4", features = ["tokio"] } +irtt-client = { version = "0.6.0", optional = true, features = ["ancillary", "tokio"] } libc = "0.2" rustix = { version = "1.1", features = ["fs", "system", "thread", "time"] } serde = { version = "1", features = ["derive"] } @@ -44,7 +45,8 @@ netlink-socket2 = { version = "0.3.5" } rust-uci = { git = "https://github.com/Lochnair/rust-uci.git", rev = "6835397f4f4ba8b69e75e727105186865c357417", optional = true, default-features = false } [features] -default = ["toml", "uci-vendored"] +default = ["irtt", "toml", "uci-vendored"] +irtt = ["dep:irtt-client"] toml = ["config/toml"] uci = ["dep:rust-uci"] uci-vendored = ["uci", "rust-uci/vendored"] diff --git a/src/main.rs b/src/main.rs index 560cab9..f252695 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,6 +22,7 @@ mod util; use crate::baseliner::Baseliner; use crate::controller::Controller; use crate::measurement::icmp::IcmpEngine; +use crate::measurement::irtt::IrttEngine; use crate::metrics::{Metric, Metrics, MetricsSender}; use crate::platform::{TrafficControlBackend, traffic_control_backend, warn_platform_limitations}; use crate::ratecontroller::Ratecontroller; @@ -236,6 +237,7 @@ fn run(settings: &Settings) -> anyhow::Result<()> { let (control_snapshot_tx, control_snapshot_rx) = flume::unbounded(); let (error_tx, error_rx) = flume::unbounded::(); let (icmp_binding_tx, icmp_binding_rx) = flume::unbounded(); + let (irtt_binding_tx, irtt_binding_rx) = flume::unbounded(); let (measurement_tx, measurement_rx) = flume::unbounded(); let (reflector_tx, reflector_rx) = flume::unbounded(); let (reselect_tx, reselect_rx) = flume::bounded(1); @@ -247,17 +249,22 @@ fn run(settings: &Settings) -> anyhow::Result<()> { }; let controller = Controller::new(reflector_rx, icmp_binding_tx); - let engine = { + let icmp_engine = { let _guard = rt.enter(); IcmpEngine::new( start_time, &settings.network, - measurement_tx, + measurement_tx.clone(), icmp_binding_rx, )? }; + let (_irtt_task, irtt_engine) = { + let _guard = rt.enter(); + IrttEngine::new(start_time, measurement_tx, irtt_binding_rx)? + }; - spawn_task(&rt, &error_tx, engine.run()); + spawn_task(&rt, &error_tx, icmp_engine.run()); + spawn_task(&rt, &error_tx, irtt_engine.run()); let dropped = Arc::new(AtomicU32::new(0)); diff --git a/src/measurement/irtt.rs b/src/measurement/irtt.rs new file mode 100644 index 0000000..77c7f8d --- /dev/null +++ b/src/measurement/irtt.rs @@ -0,0 +1,343 @@ +use std::{collections::HashMap, net::SocketAddr, time::Instant}; + +use flume::{Receiver, Sender}; +use irtt_client::{ + ClientConfig, ClientEvent, + managed::{ + ManagedClient, ManagedClientConfig, ManagedClientHandle, ManagedClientTask, + ManagedCommandAcknowledgement, ManagedCommandApplyError, ManagedCommandError, + ManagedCompletionPolicy, ManagedConfigError, ManagedEndReason, ManagedEvent, + ManagedLifecycle, ManagedSubscribeError, ManagedTargetConfig, ManagedTargetEndReason, + TargetId, + }, +}; +use log::{debug, trace, warn}; +use thiserror::Error; +use tokio::sync::broadcast::error::RecvError; + +use crate::measurement::model::{ + MeasurementEvent, MeasurementLoss, MeasurementObservation, MeasurementStream, + MeasurementStreamId, OneWayClock, OneWayLatency, ProbeSchedule, SignedDuration, +}; + +#[derive(Debug, Error)] +pub enum IrttError { + #[error("failed to build IRTT config")] + InvalidConfig(#[from] ManagedConfigError), + + #[error("event channel closed")] + EventChannelClosed, + + #[error("binding channel closed")] + BindingChannelClosed, + + #[error("error while reciving on channel")] + ChannelRecvError(#[from] RecvError), + + #[error("unable to subscribe to irtt events")] + SubscriptionError(#[from] ManagedSubscribeError), + + #[error("failed to submit IRTT target update")] + CommandError(#[from] ManagedCommandError), + + #[error("failed to apply IRTT target update")] + CommandApplyError(#[from] ManagedCommandApplyError), + + #[error("IRTT managed client failed: {0:?}")] + ManagedFailure(ManagedEndReason), + + #[error("IRTT managed task was abandoned")] + ManagedAbandoned, +} + +pub struct IrttBinding { + stream: MeasurementStream, + schedule: ProbeSchedule, + port: u16, +} + +impl IrttBinding { + pub fn new(stream: MeasurementStream, schedule: ProbeSchedule, port: u16) -> Self { + Self { + stream, + schedule, + port, + } + } +} + +pub struct IrttEngine { + client: ManagedClientHandle, + + bindings: HashMap, + + origin: Instant, + + binding_rx: Receiver>, + observation_tx: Sender, +} + +impl IrttEngine { + pub fn new( + origin: Instant, + observation_tx: Sender, + binding_rx: Receiver>, + ) -> Result<(ManagedClientTask, Self), IrttError> { + let config = ManagedClientConfig { + client: ClientConfig { + request: irtt_client::SessionRequest { + duration: None, + ..Default::default() + }, + ..Default::default() + }, + completion: ManagedCompletionPolicy::ExplicitStop, + ..Default::default() + }; + let (task, handle) = ManagedClient::task(config, Vec::new())?; + + Ok(( + task, + Self { + client: handle, + bindings: HashMap::new(), + origin, + binding_rx, + observation_tx, + }, + )) + } + + async fn set_bindings(&mut self, bindings: Vec) -> Result<(), IrttError> { + let targets = bindings + .iter() + .map(|binding| { + ManagedTargetConfig::new( + TargetId::new(binding.stream.id.0.to_string()), + SocketAddr::new(binding.stream.peer, binding.port).to_string(), + ) + }) + .collect(); + + self.client.update_targets(targets)?.await?; + + self.bindings = bindings + .into_iter() + .map(|binding| (binding.stream.id, binding)) + .collect(); + + Ok(()) + } + + pub async fn run(mut self) -> Result<(), IrttError> { + let mut events = self.client.subscribe()?; + + loop { + tokio::select! { + bindings = self.binding_rx.recv_async() => { + let bindings = bindings + .map_err(|_| IrttError::BindingChannelClosed)?; + + self.set_bindings(bindings).await?; + } + + event = events.recv() => { + let event = match event { + Ok(event) => event, + + Err(RecvError::Lagged(n)) => { + warn!("IRTT event receiver dropped {n} events"); + continue; + } + + Err(RecvError::Closed) => { + let status = self.client.status(); + + match status.lifecycle { + ManagedLifecycle::Completed => return Ok(()), + + ManagedLifecycle::Failed => { + if let Some(outcome) = &status.final_outcome { + return Err(IrttError::ManagedFailure( + outcome.end_reason.clone() + )); + } + + return Err(IrttError::ManagedAbandoned); + } + + _ => return Err(IrttError::ManagedAbandoned), + } + } + }; + + match event { + ManagedEvent::Started => { + debug!("IRTT managed client started"); + } + + ManagedEvent::TargetStateChanged { + target, + lifecycle, + } => { + debug!("IRTT target {target:?}: {lifecycle:?}"); + } + + ManagedEvent::Client { target, event } => { + let stream = target.id.as_str() + .parse::() + .ok() + .and_then(|id| { + self.bindings.get(&MeasurementStreamId(id)) + }) + .map(|binding| binding.stream); + + match event { + ClientEvent::SessionStarted(session) => { + debug!( + "IRTT session started for {}: {}", + target.id, + session.remote + ); + } + + ClientEvent::NoTestCompleted(outcome) => { + debug!( + "IRTT no-test completed for {}: {}", + target.id, + outcome.remote + ); + } + + ClientEvent::SessionClosed { .. } => { + debug!("IRTT session closed for {}", target.id); + } + + ClientEvent::EchoSent { seq, .. } => { + trace!("IRTT {} sent probe {seq}", target.id); + } + + ClientEvent::EchoReply { + sent_at, + received_at, + one_way, + .. + } => { + let Some(stream) = stream else { + continue; + }; + + let observation = MeasurementObservation { + stream, + started_at: sent_at.mono, + observed_at: received_at.mono, + one_way: one_way.and_then(map_one_way), + }; + + self.observation_tx + .send_async(MeasurementEvent::Observation(observation)) + .await + .map_err(|_| IrttError::EventChannelClosed)?; + } + + ClientEvent::EchoLoss { + sent_at, + timeout_at, + .. + } => { + let Some(stream) = stream else { + continue; + }; + + let loss = MeasurementLoss { + stream, + started_at: sent_at.mono, + detected_at: timeout_at, + }; + + self.observation_tx + .send_async(MeasurementEvent::Loss(loss)) + .await + .map_err(|_| IrttError::EventChannelClosed)?; + } + + ClientEvent::DuplicateReply { seq, .. } => { + trace!( + "IRTT {} duplicate reply {seq}", + target.id + ); + } + + ClientEvent::LateReply { seq, .. } => { + trace!( + "IRTT {} late reply {seq}", + target.id + ); + } + + ClientEvent::Warning { kind, message, .. } => { + debug!( + "IRTT {} warning ({kind:?}): {message}", + target.id + ); + } + } + } + + ManagedEvent::TargetFinished { outcome } => { + match &outcome.end_reason { + ManagedTargetEndReason::Failed(failure) => { + warn!( + "IRTT target {} failed: {failure:?}", + outcome.target.id + ); + } + + reason => { + debug!( + "IRTT target {} finished: {reason:?}", + outcome.target.id + ); + } + } + } + + ManagedEvent::Stopping => { + debug!("IRTT managed client stopping"); + } + + ManagedEvent::Completed { outcome } => { + debug!( + "IRTT managed client completed: {:?}", + outcome.end_reason + ); + return Ok(()); + } + + ManagedEvent::Failed { outcome } => { + return Err(IrttError::ManagedFailure( + outcome.end_reason.clone() + )); + } + + ManagedEvent::Abandoned => { + return Err(IrttError::ManagedAbandoned); + } + } + } + } + } + } +} + +fn map_one_way(sample: irtt_client::OneWayDelaySample) -> Option { + let uplink = i64::try_from(sample.client_to_server?.as_nanos()).ok()?; + + let downlink = i64::try_from(sample.server_to_client?.as_nanos()).ok()?; + + Some(OneWayLatency { + uplink: SignedDuration::from_nanos(uplink), + downlink: SignedDuration::from_nanos(downlink), + clock: OneWayClock::Uncalibrated, + }) +} diff --git a/src/measurement/mod.rs b/src/measurement/mod.rs index 0ee60f2..836dd96 100644 --- a/src/measurement/mod.rs +++ b/src/measurement/mod.rs @@ -1,2 +1,4 @@ pub(crate) mod icmp; +#[cfg(feature = "irtt")] +pub(crate) mod irtt; pub(crate) mod model;