diff --git a/AGENTS.md b/AGENTS.md index 69fda30..c2c54de 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,7 +57,7 @@ Tokio must remain optional for `irtt-client`; default builds must remain runtime ### Server architecture (`irtt-server`) -`irtt-server` is a first-class reusable server crate. It contains the deterministic OPEN/ECHO/session core — packet admission, authentication policy, open negotiation, the bounded session table, open reply construction, echo processing with its per-session receive state and timestamps, per-session rate limiting, session lifetime with idle expiry, the maximum-duration server-initiated close, and client-initiated close — plus a reusable Tokio UDP `Server` that runs one sequential `ServerCore` per listener with caller-controlled shutdown and scheduled idle maintenance. Logical expiry remains exact on authenticated, structurally valid requests, while the runtime also reclaims expired sessions once per second when no traffic arrives. Each reply carries the raw traffic class it must be sent with, which the runtime applies to the listener before every send. The full server fill policy and wildcard-bind packet-info handling remain unimplemented; see `crates/irtt-server/AGENTS.md`. The rest of this section describes the implemented boundary and the parts not yet written. +`irtt-server` is a first-class reusable server crate. It contains the deterministic OPEN/ECHO/session core — packet admission, authentication policy, open negotiation, the bounded session table, open reply construction, echo processing with its per-session receive state and timestamps, per-session rate limiting, session lifetime with idle expiry, the maximum-duration server-initiated close, and client-initiated close — plus a reusable Tokio UDP `Server` that runs one sequential `ServerCore` per listener with caller-controlled shutdown and scheduled idle maintenance. Logical expiry remains exact on authenticated, structurally valid requests, while the runtime also reclaims expired sessions once per second when no traffic arrives. Each reply carries the raw traffic class it must be sent with, which the runtime applies to the listener before every send. A wildcard listener additionally recovers each request's local destination from packet metadata and sends that request's reply from it, on Linux, macOS and FreeBSD; a wildcard bind on a target without that path is refused at construction rather than served from a routing-table source address. Explicit-address listeners are unaffected everywhere. The full server fill policy remains unimplemented; see `crates/irtt-server/AGENTS.md`. The rest of this section describes the implemented boundary and the parts not yet written. - It shares wire semantics with the client through `irtt-proto` and follows the same clean-room boundary. - It must **not** depend on `irtt-client` merely to reuse protocol behavior. diff --git a/Cargo.lock b/Cargo.lock index 1862351..033c7d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -735,6 +735,7 @@ dependencies = [ "getrandom 0.4.3", "irtt-client", "irtt-proto", + "nix 0.31.3", "socket2", "thiserror 2.0.20", "tokio", diff --git a/README.md b/README.md index 36ab907..70b8ee5 100644 --- a/README.md +++ b/README.md @@ -180,11 +180,14 @@ The same applet is reachable through the dispatcher: irtt-rs server --bind 192.0.2.10:2112 ``` -An explicit interface address is preferred. A wildcard bind such as -`--bind 0.0.0.0:2112` is accepted, but reply source-address selection is then -left to the kernel: per-packet destination-address handling for multi-homed -hosts is not implemented. One process serves exactly one address, so run a -second process to serve a second address or family. +An explicit interface address works on every supported system. A wildcard bind +such as `--bind 0.0.0.0:2112` reads each request's local destination from packet +metadata and sends that request's reply from the same address, so a client on a +multi-homed host is answered from the endpoint it contacted. That is implemented +on Linux, macOS and FreeBSD; elsewhere a wildcard bind is refused rather than +served from a routing-table source address a client would discard. One process +serves exactly one address, so run a second process to serve a second address or +family. The bound endpoint is printed on startup, which also resolves a port of `0`. `Ctrl-C` stops the server gracefully. @@ -304,7 +307,7 @@ The client, event stream, machine-readable output, multi-target execution, local The server library and the server applet are implemented: open negotiation, session state, echo processing, per-session rate limiting, idle expiry, the -maximum-duration close, HMAC authentication, and the negotiated per-session -reply traffic class, which is applied on sockets that support it. The full -server fill policy, per-packet source-address selection for wildcard binds on -multi-homed hosts, and multiple listeners in one process are not implemented. +maximum-duration close, HMAC authentication, the negotiated per-session reply +traffic class, which is applied on sockets that support it, and wildcard +reply-source selection on Linux, macOS and FreeBSD. The full server fill policy +and multiple listeners in one process are not implemented. diff --git a/crates/irtt-cli/src/cmd/server/args.rs b/crates/irtt-cli/src/cmd/server/args.rs index 4963101..ec21582 100644 --- a/crates/irtt-cli/src/cmd/server/args.rs +++ b/crates/irtt-cli/src/cmd/server/args.rs @@ -17,7 +17,7 @@ pub struct ServerArgs { #[arg( long, value_name = "ADDR", - long_help = "Local UDP address to bind, as ADDR:PORT, for example 127.0.0.1:2112 or [::1]:2112. Host names are not resolved.\n\nAn explicit interface address is preferred. A wildcard bind such as 0.0.0.0:2112 is allowed, but reply source-address selection is then left to the kernel: per-packet destination-address handling for multi-homed hosts is not implemented." + long_help = "Local UDP address to bind, as ADDR:PORT, for example 127.0.0.1:2112 or [::1]:2112. Host names are not resolved.\n\nA wildcard bind such as 0.0.0.0:2112 answers each request from the address it was sent to, using per-packet destination metadata. That is supported on Linux, macOS and FreeBSD; on other systems a wildcard bind is refused and an explicit interface address is required." )] pub bind: SocketAddr, diff --git a/crates/irtt-server/AGENTS.md b/crates/irtt-server/AGENTS.md index c78aea1..1f9b178 100644 --- a/crates/irtt-server/AGENTS.md +++ b/crates/irtt-server/AGENTS.md @@ -53,14 +53,62 @@ material does not establish a semantic, state the ambiguity instead of guessing. the listener. Core state is not rolled back after a transport send failure. - Runtime maintenance calls the core's expiry hook. It must not inspect session internals or grow a second expiry rule. -- Explicit-address binds are the source-address-safe supported path. A wildcard - bind on a multi-homed host remains subject to kernel source-address selection - until packet-info handling is implemented. - Do not add a blocking server, an alternate-runtime variant, a transport trait, a runtime abstraction layer, or a mirrored client-style API family. There is no product requirement for any of them, and `ServerCore` being public is not a promise of runtime independence. +## Reply source address + +A reply must leave from the address its request was sent to, or a client reading +from a connected socket discards it. An explicit-address listener satisfies that +by construction and keeps the plain `recv_from`/`send_to` path — do not route it +through ancillary machinery merely because the helper exists. + +A wildcard listener does not, so it recovers each request's local destination +and sends that reply from it. The supported targets are Linux, macOS and +FreeBSD; a wildcard bind anywhere else fails at construction. Correct refusal is +the point: a listener that starts and then answers from a routing-table source +address fails invisibly, as loss. + +- **Destination metadata is runtime transport state.** It never enters + `ServerCore`, `Session` or `OutboundDatagram`. The core knows peers and + sessions; which of the host's addresses a request arrived on is not session + policy, and adding it there would make the core platform-shaped. +- **A wildcard listener must never process a datagram whose local destination it + could not recover.** Missing destination metadata, `MSG_CTRUNC` and `MSG_TRUNC` + all drop the datagram *before* the core, so no receive, rate or lifetime state + moves for a request that cannot be answered correctly. There is no fall back + to kernel source selection: the listener promised correct sources when it + accepted the bind. +- Wildcard-ness is decided once, from the bound address, and never inferred from + a peer. +- Source selection is **per-packet send metadata**; the traffic class stays the + existing socket-wide setting applied before every send. Two transport + mechanisms, deliberately: the marking policy is already implemented and tested + across a wider target matrix, and sequential send ownership is what keeps it + race-free. Do not migrate it into control messages merely because `sendmsg` is + now in reach. +- `recvmsg` and `sendmsg` integrate through Tokio readiness — `readable()` / + `writable()` with `try_io` — following the client's ancillary receive path. + No blocking call, no `spawn_blocking`, no dedicated receive thread, and still + one socket and one task. +- The crate forbids unsafe code and that is not negotiable for this feature. Use + `nix`'s safe `recvmsg`/`sendmsg`, `ControlMessage`, `ControlMessageOwned` and + `sockopt` wrappers; no raw syscall, no `CMSG_*` walking, no hand-written ABI + layer. A target that cannot be served this way is refused, not excepted. +- The receive and send structures are not interchangeable, and the BSD IPv4 + options are not Linux's under other names. Build each direction's control + message explicitly rather than echoing a received one back. +- **Pin the source address, not the route.** What the protocol requires is the + address a reply leaves from; which interface carries it is the routing table's + business. Naming the arrival interface on a send would lose replies on a host + whose path back to the peer leaves by another one. The single exception is a + link-local source, which is not unique across interfaces and must name its + own — every other source, IPv4 included, goes out with no interface set. +- A send failure is per-packet loss, exactly as an ordinary `send_to` failure + is. It never terminates the listener and never rolls back core state. + ## Rejection is silence Malformed, unauthenticated, unsupported and policy-refused input is discarded @@ -210,7 +258,20 @@ session policy lives. Whether an unappliable zero is nevertheless safe is the runtime's decision, made from what this server itself applied. - The option itself comes from `socket2`'s safe API, chosen by the listener's - bound address family rather than the peer's. The crate forbids unsafe code; + bound address family rather than the peer's. **On Linux an IPv4-mapped bound + address is an IPv4 listener**, whatever shape it wears: the socket is + `AF_INET6`, but the kernel binds it to the IPv4 side and `IPV6_TCLASS` does not + reach the packets it emits — it leaves their TOS at zero and the negotiated + marking is lost, while `IP_TOS` on that same socket marks them. macOS rejects + `IP_TOS` on an `AF_INET6` socket outright and marks nothing either way, so it + keeps the IPv6 option: reading a mapped bind as IPv4 there would turn an + unmarked reply into a *dropped* one, because an unappliable marking stops the + send. Verify a platform before extending that list. +- A genuinely dual-stack `[::]` listener still emits **unmarked** IPv4-mapped + replies, and this is known. It is not fixed by choosing the option per reply, + for the macOS reason above, and it predates wildcard source selection. Leave it + until it is worth its own slice, and do not "fix" it by weakening the rule that + an unappliable marking stops the send. The crate forbids unsafe code; there is no raw `setsockopt` and no `sendmsg`/`cmsg` machinery. Keep the target lists identical to the pinned `socket2` version's own gates — check its source rather than copying another crate's older matrix. diff --git a/crates/irtt-server/Cargo.toml b/crates/irtt-server/Cargo.toml index 78b364e..e4cec03 100644 --- a/crates/irtt-server/Cargo.toml +++ b/crates/irtt-server/Cargo.toml @@ -18,5 +18,11 @@ socket2 = { version = "0.6.5", features = ["all"] } thiserror = "2.0.20" tokio = { version = "1.53.1", default-features = false, features = ["rt", "net", "time", "sync", "macros"] } +# Wildcard listeners recover each request's local destination address and send +# the reply from it. Only these targets have a safe destination-metadata and +# source-selection path here; elsewhere a wildcard bind is refused. +[target.'cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd"))'.dependencies] +nix = { version = "0.31.3", default-features = false, features = ["net", "uio"] } + [dev-dependencies] irtt-client = { workspace = true, features = ["ancillary", "tokio"] } diff --git a/crates/irtt-server/src/lib.rs b/crates/irtt-server/src/lib.rs index 5d5a1b3..7b46657 100644 --- a/crates/irtt-server/src/lib.rs +++ b/crates/irtt-server/src/lib.rs @@ -39,14 +39,29 @@ //! not carried an echo request are explicitly not compatibility targets: an //! `irtt-rs` session ages from the moment it is opened. //! +//! # Reply source address +//! +//! A reply leaves from the address its request was sent to. An +//! explicit-address listener gets that from the bind and uses ordinary +//! receive/send. A wildcard listener (`0.0.0.0` or `[::]`) instead reads each +//! request's local destination from packet metadata and sends that request's +//! reply from it, so a client on the host's second address is answered from the +//! endpoint it contacted rather than from whichever one the routing table would +//! have chosen. +//! +//! That path is implemented for Linux, macOS and FreeBSD. On other targets a +//! wildcard bind is refused by [`Server::from_socket`] — and so by +//! [`Server::bind`] — rather than served incorrectly; explicit-address +//! listeners work everywhere they did before. +//! //! # Current scope //! //! Open handling and negotiation, session creation, normal echo processing, //! per-session rate limiting, idle expiry, the maximum-duration -//! server-initiated close, the negotiated per-session reply traffic class and -//! Tokio UDP orchestration. The full server fill policy and wildcard-bind -//! destination-address metadata remain separate slices. Echo payloads are -//! zero-filled. +//! server-initiated close, the negotiated per-session reply traffic class, +//! wildcard reply-source selection and Tokio UDP orchestration. The full server +//! fill policy and running several listeners in one process remain separate +//! slices. Echo payloads are zero-filled. #![forbid(unsafe_code)] mod clock; @@ -56,6 +71,7 @@ mod error; mod negotiate; mod runtime; mod session; +mod socket_io; mod socket_options; mod token; diff --git a/crates/irtt-server/src/runtime.rs b/crates/irtt-server/src/runtime.rs index fbb5805..7c6e68f 100644 --- a/crates/irtt-server/src/runtime.rs +++ b/crates/irtt-server/src/runtime.rs @@ -1,12 +1,21 @@ //! Tokio UDP orchestration around [`ServerCore`](crate::ServerCore). -use std::{future::Future, io, net::SocketAddr, time::Duration}; +use std::{ + future::Future, + io, + net::{IpAddr, SocketAddr}, + time::Duration, +}; use socket2::SockRef; use thiserror::Error; use tokio::{net::UdpSocket, time::MissedTickBehavior}; -use crate::{socket_options::set_reply_traffic_class, ServerConfig, ServerCore, ServerError}; +use crate::{ + socket_io, + socket_options::{marks_with_ipv4_option, set_reply_traffic_class}, + ServerConfig, ServerCore, ServerError, +}; /// Fixed transport receive capacity. /// @@ -30,20 +39,34 @@ const MAINTENANCE_INTERVAL: Duration = Duration::from_secs(1); /// marking. Adding a concurrent sender would break that and require per-packet /// control messages instead. /// -/// Bind an explicit local address when reply source-address identity matters. -/// Wildcard binds rely on the kernel's source-address choice; destination -/// packet metadata and per-packet source selection are not yet implemented for -/// multi-homed hosts. +/// # Reply source address +/// +/// A reply must leave from the exact address the request was sent to. An +/// explicit-address listener gets that from the bind: it can send from nothing +/// else. A wildcard listener (`0.0.0.0` or `[::]`) cannot, because the routing +/// table would choose the source, so it asks the kernel for each request's +/// local destination and sends that request's reply from it. +/// +/// That per-packet path exists on Linux, macOS and FreeBSD. Elsewhere a +/// wildcard bind is **refused** at construction — see +/// [`ServerRuntimeError::WildcardSourceSelectionUnsupported`] — rather than +/// served by a listener whose replies a client on a second address would +/// silently discard. Explicit-address listeners are unaffected everywhere. #[derive(Debug)] pub struct Server { socket: UdpSocket, core: ServerCore, recv_buffer: Vec, /// The listener's own address family, which selects the IPv4 TOS or IPv6 - /// Traffic Class option. `None` only if the bound address could not be - /// queried at construction, in which case the send path asks again rather - /// than guessing a family. - listener_is_ipv4: Option, + /// Traffic Class option. + listener_is_ipv4: bool, + /// Whether this listener is wildcard-bound and must therefore recover each + /// request's local destination and reply from it. + /// + /// Decided once, from the bound address. It is never inferred later from a + /// peer: a listener that accepted a wildcard bind owes correct reply + /// sources to every request, not to the ones that look multi-homed. + select_reply_source: bool, /// Whether this server has ever successfully applied a nonzero traffic /// class to the socket. /// @@ -57,24 +80,55 @@ pub struct Server { impl Server { /// Binds one UDP listener at `addr` and creates its independent session /// namespace. + /// + /// A wildcard `addr` additionally configures reply source selection, and + /// fails where that is unavailable; see [`Server::from_socket`]. pub async fn bind(addr: SocketAddr, config: ServerConfig) -> Result { let socket = UdpSocket::bind(addr) .await .map_err(|source| ServerRuntimeError::Bind { addr, source })?; - Ok(Self::from_socket(socket, config)) + Self::from_socket(socket, config) } /// Wraps an already prepared Tokio UDP socket in a server. - #[must_use] - pub fn from_socket(socket: UdpSocket, config: ServerConfig) -> Self { - let listener_is_ipv4 = socket.local_addr().ok().map(|addr| addr.is_ipv4()); - Self { + /// + /// # Errors + /// + /// Construction is fallible because a wildcard listener has setup to do + /// before it may serve anything. It fails when the socket's bound address + /// cannot be queried — which is what says whether this is a wildcard + /// listener at all — and, for a wildcard socket, when this target has no + /// reply source selection or the kernel refuses to configure it. + /// + /// Failing here is the point. A wildcard listener that cannot recover a + /// request's local destination would start, run, and answer clients on a + /// second local address from an endpoint they never contacted, which they + /// discard as though the network had dropped it. + pub fn from_socket( + socket: UdpSocket, + config: ServerConfig, + ) -> Result { + let addr = socket + .local_addr() + .map_err(|source| ServerRuntimeError::LocalAddr { source })?; + let select_reply_source = is_wildcard(addr); + + if select_reply_source { + if !socket_io::SUPPORTED { + return Err(ServerRuntimeError::WildcardSourceSelectionUnsupported { addr }); + } + socket_io::configure_destination_metadata(&socket, addr.is_ipv4()) + .map_err(|source| ServerRuntimeError::SourceSelectionSetup { addr, source })?; + } + + Ok(Self { socket, core: ServerCore::new(config), recv_buffer: vec![0; RECEIVE_BUFFER_LEN], - listener_is_ipv4, + listener_is_ipv4: marks_with_ipv4_option(addr), + select_reply_source, socket_is_marked: false, - } + }) } /// Applies one reply's raw traffic class to the listener socket, and @@ -104,11 +158,11 @@ impl Server { } fn apply_reply_traffic_class(&self, traffic_class: u8) -> io::Result<()> { - let is_ipv4 = match self.listener_is_ipv4 { - Some(is_ipv4) => is_ipv4, - None => self.socket.local_addr()?.is_ipv4(), - }; - set_reply_traffic_class(SockRef::from(&self.socket), is_ipv4, traffic_class) + set_reply_traffic_class( + SockRef::from(&self.socket), + self.listener_is_ipv4, + traffic_class, + ) } /// Returns the local endpoint selected for the listener. @@ -133,6 +187,12 @@ impl Server { /// *unmarked* reply on a socket this server has never marked is the one /// exception, so a host that does not support the option at all — some /// Windows builds do not support `IP_TOS` — still runs a working server. + /// + /// A wildcard listener drops a request whose local destination did not + /// arrive with it, before the core sees it. There is no fall back to a + /// routing-table source: the listener promised a correct reply source when + /// it accepted the bind, and a request it cannot answer correctly must not + /// move a session's receive, rate or lifetime state either. pub async fn run(&mut self, shutdown: F) -> Result<(), ServerRuntimeError> where F: Future, @@ -146,12 +206,18 @@ impl Server { tokio::select! { _ = &mut shutdown => return Ok(()), _ = maintenance.tick() => self.core.maintain(), - received = self.socket.recv_from(&mut self.recv_buffer) => { - let (len, peer) = match received { - Ok(received) => received, + received = socket_io::receive( + &self.socket, + &mut self.recv_buffer, + self.select_reply_source, + ) => { + let received = match received { + Ok(Some(received)) => received, + Ok(None) => continue, Err(source) if source.kind() == io::ErrorKind::Interrupted => continue, Err(source) => return Err(ServerRuntimeError::Receive { source }), }; + let (len, peer) = (received.len, received.peer); if len == RECEIVE_BUFFER_LEN { continue; } @@ -160,7 +226,12 @@ impl Server { if !self.prepare_reply_traffic_class(reply.traffic_class()) { continue; } - let send = self.socket.send_to(reply.bytes(), peer); + let send = socket_io::send( + &self.socket, + reply.bytes(), + peer, + received.reply_source, + ); tokio::pin!(send); let sent = loop { tokio::select! { @@ -179,6 +250,30 @@ impl Server { } } +/// Whether a bound address names no particular local address, and so owes every +/// reply an explicitly selected source. +/// +/// `0.0.0.0` and `[::]` are the obvious forms. The third is the IPv4-mapped +/// unspecified address, `[::ffff:0.0.0.0]`: Linux accepts it as an IPv4 wildcard +/// bind and reports it back verbatim from `getsockname`, so asking +/// `is_unspecified` alone would take a working wildcard listener for an explicit +/// one and answer its requests from whatever source the routing table picked. +/// (macOS normalizes that bind to `[::]` and never reaches the second test.) +/// +/// A mapped address that names an actual host address — `[::ffff:127.0.0.1]` — +/// is explicit, and stays on the plain path. +fn is_wildcard(addr: SocketAddr) -> bool { + match addr.ip() { + IpAddr::V4(address) => address.is_unspecified(), + IpAddr::V6(address) => { + address.is_unspecified() + || address + .to_ipv4_mapped() + .is_some_and(|mapped| mapped.is_unspecified()) + } + } +} + /// Whether a reply may still be sent after its traffic class could not be /// applied. /// @@ -210,6 +305,26 @@ pub enum ServerRuntimeError { #[source] source: io::Error, }, + /// A wildcard listener was requested on a target with no safe per-packet + /// reply source selection. + /// + /// The listener would answer from whichever local address the routing table + /// chose, which a client that contacted a different one discards. + #[error( + "wildcard listener {addr} cannot select its reply source address on this target: \ + bind an explicit local address instead" + )] + WildcardSourceSelectionUnsupported { addr: SocketAddr }, + /// Configuring destination-address metadata for a wildcard listener failed. + #[error( + "could not configure reply source-address selection for wildcard listener {addr}: \ + {source}; bind an explicit local address instead" + )] + SourceSelectionSetup { + addr: SocketAddr, + #[source] + source: io::Error, + }, /// Receiving from the listener failed irrecoverably. #[error("UDP receive failed: {source}")] Receive { @@ -253,6 +368,31 @@ mod tests { .unwrap(); } + /// Construction is where a wildcard listener's promise is made or refused. + /// + /// Both outcomes are asserted from the same code, because which one a target + /// gets is exactly the thing under test: a target with a source-selection + /// path configures it and serves; one without refuses the bind rather than + /// starting a listener whose replies may come from an address no client + /// contacted. An explicit bind is unaffected either way. + #[tokio::test(flavor = "current_thread")] + async fn from_socket_settles_wildcard_source_selection_before_serving_anything() { + let explicit = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + assert!(Server::from_socket(explicit, ServerConfig::default()).is_ok()); + + let wildcard = UdpSocket::bind("0.0.0.0:0").await.unwrap(); + let wildcard = Server::from_socket(wildcard, ServerConfig::default()); + if socket_io::SUPPORTED { + let server = wildcard.expect("a supported target configures source selection"); + assert!(server.select_reply_source); + } else { + assert!(matches!( + wildcard, + Err(ServerRuntimeError::WildcardSourceSelectionUnsupported { .. }) + )); + } + } + #[tokio::test(flavor = "current_thread")] async fn ipv4_open_echo_close_crosses_a_real_udp_socket() { exercise_udp_path(SocketAddr::from((Ipv4Addr::LOCALHOST, 0))).await; @@ -370,6 +510,32 @@ mod tests { ServerConfig::default().with_min_send_interval(Duration::ZERO) } + /// Which bound addresses owe their replies a selected source. + /// + /// The IPv4-mapped unspecified address is the one that is not obvious, and + /// it is not hypothetical: Linux accepts `[::ffff:0.0.0.0]` as an IPv4 + /// wildcard bind and reports it back unchanged, so missing it would leave a + /// working wildcard listener on the plain send path. A mapped address + /// naming a real host address is explicit and must stay there. + #[test] + fn a_bound_address_naming_no_particular_local_address_is_a_wildcard() { + for (addr, wildcard) in [ + ("0.0.0.0:2112", true), + ("[::]:2112", true), + ("[::ffff:0.0.0.0]:2112", true), + ("127.0.0.1:2112", false), + ("192.0.2.10:2112", false), + ("[::1]:2112", false), + ("[::ffff:127.0.0.1]:2112", false), + ] { + assert_eq!( + is_wildcard(addr.parse().unwrap()), + wildcard, + "{addr} classified wrongly" + ); + } + } + /// The rule that decides what an unappliable traffic class means, which no /// normal interface can reach: it needs a host that refuses the socket /// option, and the hosts these tests run on do not. diff --git a/crates/irtt-server/src/socket_io.rs b/crates/irtt-server/src/socket_io.rs new file mode 100644 index 0000000..912801a --- /dev/null +++ b/crates/irtt-server/src/socket_io.rs @@ -0,0 +1,157 @@ +//! Receiving a request's local destination and replying from it. +//! +//! A reply must leave from the exact address and port the request was addressed +//! to, because clients read from connected UDP sockets and discard anything +//! arriving from another endpoint. A listener bound to an explicit address +//! satisfies that for free: every datagram it sends carries the address it is +//! bound to, so ordinary `recv_from`/`send_to` is all it needs. +//! +//! A wildcard listener does not. On a multi-homed host the routing table +//! chooses the source address, so a request to one of the host's addresses can +//! be answered from another, and the client drops the reply. Such a listener +//! therefore asks the kernel for each request's destination address as +//! ancillary data and hands it back as the source of that request's reply. +//! +//! Which mechanism carries that is per-platform, and only [`SUPPORTED`] targets +//! have one reachable through a safe API. Everything here goes through `nix`'s +//! safe `recvmsg`/`sendmsg` wrappers; the crate forbids unsafe code, so there is +//! no raw syscall, no `CMSG_*` walking and no hand-written ABI layer. + +use std::{io, net::SocketAddr}; + +use tokio::net::UdpSocket; + +#[cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd"))] +mod ancillary; + +/// Whether this target can pin a wildcard listener's reply source address. +/// +/// A wildcard bind is refused where this is false, rather than served by a +/// listener whose replies may leave from an address the client never contacted. +pub(crate) const SUPPORTED: bool = cfg!(any( + target_os = "linux", + target_os = "macos", + target_os = "freebsd" +)); + +/// Where one reply must leave from. +/// +/// This is transport state belonging to a single datagram. It never reaches +/// [`ServerCore`](crate::ServerCore), which knows peers and sessions and has no +/// business knowing which of the host's addresses a request arrived on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ReplySource { + /// Whatever the socket is bound to: an explicit-address listener already + /// sends from the one address it can. + Bound, + /// The IPv4 address this request was addressed to. + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd"))] + V4 { address: std::net::Ipv4Addr }, + /// The IPv6 address this request was addressed to, and the interface it + /// arrived on. The interface is part of the identity of a *scoped* address, + /// so it is carried rather than rediscovered — and it is used on the send + /// only where the source address is one, so that a reply is never forced + /// out of an interface the route to the peer does not use. + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd"))] + V6 { + address: std::net::Ipv6Addr, + interface_index: u32, + }, +} + +/// One received datagram and what answering it requires. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ReceivedDatagram { + pub(crate) len: usize, + pub(crate) peer: SocketAddr, + pub(crate) reply_source: ReplySource, +} + +/// Asks the kernel to report each datagram's local destination address. +/// +/// Called once, at construction, and only for a wildcard listener: an +/// explicit-address listener needs no ancillary metadata, and adding it would +/// widen the platform surface of the common path for nothing. +#[cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd"))] +pub(crate) fn configure_destination_metadata(socket: &UdpSocket, is_ipv4: bool) -> io::Result<()> { + ancillary::configure_destination_metadata(socket, is_ipv4) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "freebsd")))] +pub(crate) fn configure_destination_metadata( + _socket: &UdpSocket, + _is_ipv4: bool, +) -> io::Result<()> { + Err(unsupported()) +} + +/// Receives one datagram, reporting where its reply must come from. +/// +/// `select_reply_source` is the listener's own wildcard-ness, decided once at +/// construction. It is never inferred from a peer address. +/// +/// `Ok(None)` is a datagram the runtime must drop without letting it reach the +/// core: it was truncated, its ancillary metadata was truncated, or a wildcard +/// listener could not recover its local destination. Answering it correctly is +/// impossible, so it must not advance any session's state either. +/// +/// Cancel-safe: nothing is consumed until a datagram is fully received. +pub(crate) async fn receive( + socket: &UdpSocket, + buffer: &mut [u8], + select_reply_source: bool, +) -> io::Result> { + if select_reply_source { + return receive_with_destination(socket, buffer).await; + } + + let (len, peer) = socket.recv_from(buffer).await?; + Ok(Some(ReceivedDatagram { + len, + peer, + reply_source: ReplySource::Bound, + })) +} + +#[cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd"))] +async fn receive_with_destination( + socket: &UdpSocket, + buffer: &mut [u8], +) -> io::Result> { + ancillary::receive_with_destination(socket, buffer).await +} + +/// Unreachable in practice: a wildcard bind is refused at construction here, so +/// no listener on this target ever asks for source selection. +#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "freebsd")))] +async fn receive_with_destination( + _socket: &UdpSocket, + _buffer: &mut [u8], +) -> io::Result> { + Err(unsupported()) +} + +/// Sends one reply to `peer` from `reply_source`. +/// +/// A short send is reported as such and is the caller's to treat as the loss of +/// that one reply, exactly as an ordinary `send_to` short send is. +pub(crate) async fn send( + socket: &UdpSocket, + bytes: &[u8], + peer: SocketAddr, + reply_source: ReplySource, +) -> io::Result { + match reply_source { + ReplySource::Bound => socket.send_to(bytes, peer).await, + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd"))] + selected => ancillary::send_from_source(socket, bytes, peer, selected).await, + } +} + +#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "freebsd")))] +fn unsupported() -> io::Error { + io::Error::new( + io::ErrorKind::Unsupported, + "reply source-address selection is unsupported on this target", + ) +} diff --git a/crates/irtt-server/src/socket_io/ancillary.rs b/crates/irtt-server/src/socket_io/ancillary.rs new file mode 100644 index 0000000..ec8ae57 --- /dev/null +++ b/crates/irtt-server/src/socket_io/ancillary.rs @@ -0,0 +1,376 @@ +//! Destination-address ancillary data on Linux, macOS and FreeBSD. +//! +//! Three platforms, two mechanisms per address family, all reached through +//! `nix`'s safe wrappers: +//! +//! | | receive | send | +//! |---|---|---| +//! | Linux IPv4 | `IP_PKTINFO` | `IP_PKTINFO` | +//! | Linux IPv6 | `IPV6_RECVPKTINFO` | `IPV6_PKTINFO` | +//! | macOS IPv4 | `IP_RECVDSTADDR` | `IP_PKTINFO` | +//! | macOS IPv6 | `IPV6_RECVPKTINFO` | `IPV6_PKTINFO` | +//! | FreeBSD IPv4 | `IP_RECVDSTADDR` | `IP_SENDSRCADDR` | +//! | FreeBSD IPv6 | `IPV6_RECVPKTINFO` | `IPV6_PKTINFO` | +//! +//! The BSD IPv4 options are not Linux's under another name, and the receive +//! structure is not the send structure: `in_pktinfo` carries three fields on +//! receive of which only two mean anything on send. Each direction is built +//! explicitly for that reason. +//! +//! Both directions run nonblocking through Tokio readiness, following the same +//! `readable()`/`try_io` shape the client's ancillary receive path uses. One +//! socket, one task, no blocking call and no spawned thread. + +use std::{ + io::{self, IoSlice, IoSliceMut}, + net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV6}, + os::fd::AsRawFd, +}; + +use nix::{ + libc, + sys::socket::{ + recvmsg, sendmsg, setsockopt, sockopt, ControlMessage, ControlMessageOwned, MsgFlags, + RecvMsg, SockaddrStorage, + }, +}; +use tokio::{io::Interest, net::UdpSocket}; + +use super::{ReceivedDatagram, ReplySource}; + +/// Room for the destination control message. The largest of them is an +/// `in6_pktinfo`, and this leaves headroom for a second message the kernel may +/// add, so `MSG_CTRUNC` reflects a real problem rather than a tight buffer. +const CONTROL_LEN: usize = 128; + +pub(super) fn configure_destination_metadata(socket: &UdpSocket, is_ipv4: bool) -> io::Result<()> { + if is_ipv4 { + configure_ipv4_destination(socket) + } else { + // A dual-stack wildcard socket reports IPv4-mapped destinations through + // this option too, so the IPv6 listener needs only this one. + setsockopt(socket, sockopt::Ipv6RecvPacketInfo, &true).map_err(io::Error::from) + } +} + +#[cfg(target_os = "linux")] +fn configure_ipv4_destination(socket: &UdpSocket) -> io::Result<()> { + setsockopt(socket, sockopt::Ipv4PacketInfo, &true).map_err(io::Error::from) +} + +#[cfg(any(target_os = "macos", target_os = "freebsd"))] +fn configure_ipv4_destination(socket: &UdpSocket) -> io::Result<()> { + setsockopt(socket, sockopt::Ipv4RecvDstAddr, &true).map_err(io::Error::from) +} + +pub(super) async fn receive_with_destination( + socket: &UdpSocket, + buffer: &mut [u8], +) -> io::Result> { + loop { + socket.readable().await?; + match socket.try_io(Interest::READABLE, || receive_once(socket, buffer)) { + Err(error) if error.kind() == io::ErrorKind::WouldBlock => continue, + result => return result, + } + } +} + +fn receive_once(socket: &UdpSocket, buffer: &mut [u8]) -> io::Result> { + let mut control = ControlBuffer::new(); + let mut iov = [IoSliceMut::new(buffer)]; + let message = recvmsg::( + socket.as_raw_fd(), + &mut iov, + Some(control.as_mut_slice()), + MsgFlags::MSG_DONTWAIT, + )?; + + Ok(received_datagram(&message)) +} + +/// Turns one received message into a datagram the runtime may serve, or `None` +/// for one it must drop. +/// +/// A wildcard listener promised to answer from the address the request was sent +/// to. If the payload was truncated, if the control buffer was truncated, or if +/// no destination arrived with the datagram, that promise cannot be kept for +/// this request — so it is dropped here, before the core sees it and before any +/// session's receive, rate or lifetime state could move. +fn received_datagram(message: &RecvMsg<'_, '_, SockaddrStorage>) -> Option { + if message + .flags + .intersects(MsgFlags::MSG_TRUNC | MsgFlags::MSG_CTRUNC) + { + return None; + } + + Some(ReceivedDatagram { + len: message.bytes, + peer: peer_endpoint(message.address.as_ref()?)?, + reply_source: reply_source(message)?, + }) +} + +/// The request's full source endpoint, keeping the IPv6 zone the core compares +/// endpoints by. +fn peer_endpoint(address: &SockaddrStorage) -> Option { + if let Some(peer) = address.as_sockaddr_in() { + return Some(SocketAddr::from(*peer)); + } + address + .as_sockaddr_in6() + .map(|peer| SocketAddr::from(*peer)) +} + +fn reply_source(message: &RecvMsg<'_, '_, SockaddrStorage>) -> Option { + for control_message in message.cmsgs().ok()? { + match control_message { + // Linux reports the header destination in `ipi_addr`; `ipi_spec_dst` + // is the route's local address, which is the same for the unicast + // traffic a server sees and is not what the client addressed. + #[cfg(target_os = "linux")] + ControlMessageOwned::Ipv4PacketInfo(info) => { + return ipv4_reply_source(ipv4_addr(info.ipi_addr)); + } + #[cfg(any(target_os = "macos", target_os = "freebsd"))] + ControlMessageOwned::Ipv4RecvDstAddr(address) => { + return ipv4_reply_source(ipv4_addr(address)); + } + ControlMessageOwned::Ipv6PacketInfo(info) => { + let address = Ipv6Addr::from(info.ipi6_addr.s6_addr); + if address.is_unspecified() { + return None; + } + return Some(ReplySource::V6 { + address, + interface_index: info.ipi6_ifindex, + }); + } + _ => {} + } + } + None +} + +/// An unspecified destination is no destination: nothing can be sent from +/// `0.0.0.0`, so the datagram is dropped rather than answered from a source the +/// kernel would pick instead. +fn ipv4_reply_source(address: Ipv4Addr) -> Option { + (!address.is_unspecified()).then_some(ReplySource::V4 { address }) +} + +fn ipv4_addr(address: libc::in_addr) -> Ipv4Addr { + Ipv4Addr::from(address.s_addr.to_ne_bytes()) +} + +fn in_addr(address: Ipv4Addr) -> libc::in_addr { + libc::in_addr { + s_addr: u32::from_ne_bytes(address.octets()), + } +} + +pub(super) async fn send_from_source( + socket: &UdpSocket, + bytes: &[u8], + peer: SocketAddr, + reply_source: ReplySource, +) -> io::Result { + loop { + socket.writable().await?; + match socket.try_io(Interest::WRITABLE, || { + send_once(socket, bytes, peer, reply_source) + }) { + Err(error) if error.kind() == io::ErrorKind::WouldBlock => continue, + result => return result, + } + } +} + +fn send_once( + socket: &UdpSocket, + bytes: &[u8], + peer: SocketAddr, + reply_source: ReplySource, +) -> io::Result { + let destination = SockaddrStorage::from(peer); + let iov = [IoSlice::new(bytes)]; + + let sent = match reply_source { + ReplySource::Bound => sendmsg( + socket.as_raw_fd(), + &iov, + &[], + MsgFlags::MSG_DONTWAIT, + Some(&destination), + )?, + ReplySource::V4 { address } => send_from_ipv4(socket, &iov, &destination, address)?, + ReplySource::V6 { + address, + interface_index, + } => { + // For a send, `in6_pktinfo` names the source address and, where it + // is needed, the interface to leave by. + let info = libc::in6_pktinfo { + ipi6_addr: libc::in6_addr { + s6_addr: address.octets(), + }, + ipi6_ifindex: egress_interface_index(address, interface_index), + }; + // The destination has to be stated in the same family the control + // message is: an IPv6 source with an `AF_INET` destination is a + // rejected send, not a mixed-family one. The two can disagree — + // macOS reports an IPv4 peer for a listener bound to the IPv4-mapped + // wildcard while still delivering IPv6 packet info for it — and the + // peer identity the core sees stays exactly as it was received. + let destination = SockaddrStorage::from(as_ipv6_endpoint(peer)); + sendmsg( + socket.as_raw_fd(), + &iov, + &[ControlMessage::Ipv6PacketInfo(&info)], + MsgFlags::MSG_DONTWAIT, + Some(&destination), + )? + } + }; + + Ok(sent) +} + +/// The interface a reply must leave by, which is usually none of our business. +/// +/// A link-local source address is not unique across a host's interfaces, so a +/// reply carrying one has to name the interface it belongs to. Every other +/// source — global, unique-local, loopback, IPv4-mapped — identifies itself +/// without help. +/// +/// Pinning the arrival interface for those would only add a way for a reply to +/// fail: on a host whose route back to the peer leaves by a different interface +/// than the request arrived on, forcing the ingress interface can make the send +/// fail outright, losing a reply whose source address was perfectly valid. What +/// the protocol requires is the source *address*; the route to the peer is the +/// routing table's business. The IPv4 path makes the same choice for the same +/// reason. +fn egress_interface_index(address: Ipv6Addr, interface_index: u32) -> u32 { + if address.is_unicast_link_local() { + interface_index + } else { + 0 + } +} + +/// The same endpoint expressed in IPv6, mapping an IPv4 peer. +/// +/// A datagram sent to an IPv4-mapped destination from an `AF_INET6` socket goes +/// out as ordinary IPv4 to that address; this changes how the endpoint is +/// spelled for one `sendmsg`, not what reaches the client. +fn as_ipv6_endpoint(peer: SocketAddr) -> SocketAddrV6 { + match peer { + SocketAddr::V6(peer) => peer, + SocketAddr::V4(peer) => SocketAddrV6::new(peer.ip().to_ipv6_mapped(), peer.port(), 0, 0), + } +} + +/// `IP_PKTINFO` on a send reads only `ipi_spec_dst`, the source address, and +/// `ipi_ifindex`. `ipi_addr` is a receive-only field and is left zero. +/// +/// The interface is left unset deliberately. What the protocol requires is the +/// source *address*; pinning the arrival interface as well would add a way for +/// a reply to fail on a host whose return path is not the one the request came +/// in on, and buy nothing. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn send_from_ipv4( + socket: &UdpSocket, + iov: &[IoSlice<'_>], + destination: &SockaddrStorage, + address: Ipv4Addr, +) -> io::Result { + let info = libc::in_pktinfo { + ipi_ifindex: 0, + ipi_spec_dst: in_addr(address), + ipi_addr: in_addr(Ipv4Addr::UNSPECIFIED), + }; + + sendmsg( + socket.as_raw_fd(), + iov, + &[ControlMessage::Ipv4PacketInfo(&info)], + MsgFlags::MSG_DONTWAIT, + Some(destination), + ) + .map_err(io::Error::from) +} + +/// FreeBSD has no send-side `IP_PKTINFO`; `IP_SENDSRCADDR` carries the source +/// address on its own. +#[cfg(target_os = "freebsd")] +fn send_from_ipv4( + socket: &UdpSocket, + iov: &[IoSlice<'_>], + destination: &SockaddrStorage, + address: Ipv4Addr, +) -> io::Result { + let source = in_addr(address); + + sendmsg( + socket.as_raw_fd(), + iov, + &[ControlMessage::Ipv4SendSrcAddr(&source)], + MsgFlags::MSG_DONTWAIT, + Some(destination), + ) + .map_err(io::Error::from) +} + +/// Per-receive control buffer. Control data must satisfy `cmsghdr` alignment, +/// which 8-byte alignment covers on these targets. It lives on the stack for +/// the duration of one receive; nothing is allocated per datagram. +#[repr(align(8))] +struct ControlBuffer([u8; CONTROL_LEN]); + +impl ControlBuffer { + fn new() -> Self { + Self([0; CONTROL_LEN]) + } + + fn as_mut_slice(&mut self) -> &mut [u8] { + &mut self.0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Which reply sources need their interface named, which no normal + /// interface can show: it takes a host whose route back to a peer leaves by + /// a different interface than the request arrived on, and a test host has + /// one loopback. + #[test] + fn only_a_link_local_source_pins_the_egress_interface() { + const ARRIVED_ON: u32 = 7; + + for (address, expected, why) in [ + ( + "fe80::1", + ARRIVED_ON, + "a link-local source needs its link named", + ), + ( + "fe80::7858:baff:fe91:183d", + ARRIVED_ON, + "and so does any other one", + ), + ("2001:db8::1", 0, "a global source identifies itself"), + ("fd00::1", 0, "so does a unique-local one"), + ("::1", 0, "and loopback"), + ("::ffff:127.0.0.2", 0, "and an IPv4-mapped destination"), + ] { + assert_eq!( + egress_interface_index(address.parse().unwrap(), ARRIVED_ON), + expected, + "{address}: {why}" + ); + } + } +} diff --git a/crates/irtt-server/src/socket_options.rs b/crates/irtt-server/src/socket_options.rs index 8709181..3a40232 100644 --- a/crates/irtt-server/src/socket_options.rs +++ b/crates/irtt-server/src/socket_options.rs @@ -10,10 +10,40 @@ //! there is no raw `setsockopt` or per-packet control message anywhere in the //! server. -use std::io; +use std::{ + io, + net::{IpAddr, SocketAddr}, +}; use socket2::SockRef; +/// Whether this listener's replies are marked with the IPv4 TOS option rather +/// than the IPv6 Traffic Class one. +/// +/// The listener's own bound family answers this, not the peer's — one socket, +/// one answer. The exception is an IPv4-mapped bound address, where the socket +/// is `AF_INET6` but every packet it emits is IPv4, and the platforms disagree +/// about it: +/// +/// - **Linux** binds such a socket to the IPv4 side. `IPV6_TCLASS` does not +/// reach the packets it sends — their TOS stays zero and a negotiated marking +/// is silently lost — while `IP_TOS` on that same socket marks them. So a +/// mapped bind is read as the IPv4 listener it is. +/// - **macOS** rejects `IP_TOS` on an `AF_INET6` socket with `EINVAL` whatever +/// it is bound to, and marks nothing through either option. Reading a mapped +/// bind as IPv4 there would turn an unmarked reply into a *dropped* one, since +/// an unappliable marking stops the send — worse than the marking simply not +/// arriving. It keeps the IPv6 option and its existing behavior. (The mapped +/// *wildcard* never reaches this at all: macOS normalizes that bind to `[::]`.) +/// - **FreeBSD** refuses a mapped bind under its default `IPV6_V6ONLY`, so the +/// case does not arise. +pub(crate) fn marks_with_ipv4_option(addr: SocketAddr) -> bool { + match addr.ip() { + IpAddr::V4(_) => true, + IpAddr::V6(address) => cfg!(target_os = "linux") && address.to_ipv4_mapped().is_some(), + } +} + /// Applies `traffic_class` to a socket bound in the given family. /// /// `is_ipv4` is the *listener's* family, not the peer's. @@ -119,3 +149,36 @@ fn unsupported_traffic_class(_traffic_class: u32, option: &'static str) -> io::R format!("{option} are unsupported on this target"), )) } + +#[cfg(test)] +mod tests { + use super::*; + + /// Which listeners mark with the IPv4 option. + /// + /// The mapped forms are the point: such a socket is `AF_INET6` but sends + /// IPv4, and `IPV6_TCLASS` does not reach those packets. Linux marks them + /// through `IP_TOS`; the platforms that would reject that keep the IPv6 + /// option, because an unappliable marking stops the send and a dropped + /// reply is worse than an unmarked one. + #[test] + fn a_mapped_listener_marks_with_the_ipv4_option_where_that_option_works() { + let mapped = cfg!(target_os = "linux"); + + for (addr, is_ipv4) in [ + ("0.0.0.0:2112", true), + ("127.0.0.1:2112", true), + ("[::ffff:0.0.0.0]:2112", mapped), + ("[::ffff:127.0.0.1]:2112", mapped), + ("[::]:2112", false), + ("[::1]:2112", false), + ("[2001:db8::1]:2112", false), + ] { + assert_eq!( + marks_with_ipv4_option(addr.parse().unwrap()), + is_ipv4, + "{addr} would mark with the wrong option" + ); + } + } +} diff --git a/crates/irtt-server/tests/wildcard_source.rs b/crates/irtt-server/tests/wildcard_source.rs new file mode 100644 index 0000000..b772750 --- /dev/null +++ b/crates/irtt-server/tests/wildcard_source.rs @@ -0,0 +1,263 @@ +//! A wildcard listener answers from the address the request was sent to. +//! +//! Every test here drives a real [`AsyncClient`], which reads from a *connected* +//! UDP socket. That connection is the assertion: the kernel discards any +//! datagram whose source is not the endpoint the client sent to, so a reply +//! that left from another of the host's addresses is indistinguishable from no +//! reply at all. An open, an echo and a close completing therefore prove the +//! source address, without the test having to inspect it. +//! +//! Compiled on the targets that implement wildcard source selection. Elsewhere +//! a wildcard bind is refused outright, which the runtime's own tests cover. + +#![cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd"))] + +use std::{ + io, + net::{Ipv4Addr, SocketAddr, UdpSocket}, + time::Duration, +}; + +use irtt_client::{AsyncClient, ClientConfig, ClientEvent, OpenOutcome, PacketMeta, SocketConfig}; +use irtt_proto::{Clock, ReceivedStats, StampAt}; +use irtt_server::{Server, ServerConfig, ServerRuntimeError}; +use tokio::{sync::oneshot, task::JoinHandle, time::timeout}; + +/// EF, as the client's configuration takes it, and the raw TOS byte it occupies +/// the upper six bits of. +const EF_DSCP: u8 = 46; +const EF_TRAFFIC_CLASS: u8 = 0xb8; + +#[tokio::test(flavor = "current_thread")] +async fn a_wildcard_ipv4_listener_replies_from_the_requested_address() { + timeout(Duration::from_secs(5), async { + let server = Server::bind("0.0.0.0:0".parse().unwrap(), unthrottled()) + .await + .expect("a wildcard IPv4 listener must be supported on this target"); + let port = server.local_addr().unwrap().port(); + // A second loopback address makes this a genuine multi-homed case: the + // listener is bound to none of the host's addresses in particular, and + // the client contacts one the routing table would not have picked on + // its own. Hosts carrying only 127.0.0.1 — macOS, unless an alias is + // configured — still exercise the whole ancillary path, one address + // wide. + let target = SocketAddr::from((secondary_ipv4_loopback(), port)); + let running = spawn(server); + + let mut client = connected_client(target).await; + assert_started(&mut client, target).await; + + let meta = echo_once(&mut client).await; + // Source selection is per-packet ancillary data and the traffic class + // is a socket-wide setting applied before the send. Both act on the + // same reply, so this is where they have to agree. + if cfg!(target_os = "linux") { + assert_eq!( + meta.traffic_class, + Some(EF_TRAFFIC_CLASS), + "a source-selected reply still carries its session's marking" + ); + assert_eq!(meta.dscp, Some(EF_DSCP)); + assert_eq!(meta.ecn, Some(0)); + } + + assert!(matches!( + client.close().await.unwrap().as_slice(), + [ClientEvent::SessionClosed { .. }] + )); + running.stop().await; + }) + .await + .expect("the wildcard IPv4 source-selection test exceeded its bounded runtime"); +} + +#[tokio::test(flavor = "current_thread")] +async fn an_ipv4_mapped_wildcard_listener_replies_from_the_requested_address() { + // `[::ffff:0.0.0.0]` is a wildcard that does not look like one. Linux + // accepts it as an IPv4 wildcard bind and `getsockname` reports it back + // unchanged, so a listener that only recognized `0.0.0.0` and `[::]` would + // take this for an explicit address and answer from a routing-selected + // source. macOS normalizes the bind to `[::]`, where this is simply the + // IPv6 wildcard case again. + timeout(Duration::from_secs(5), async { + let server = match Server::bind("[::ffff:0.0.0.0]:0".parse().unwrap(), unthrottled()).await + { + Ok(server) => server, + // FreeBSD refuses a mapped bind under its default `IPV6_V6ONLY`. + Err(ServerRuntimeError::Bind { source, .. }) if is_family_unavailable(&source) => { + eprintln!("skipping IPv4-mapped wildcard test: the bind is unavailable here"); + return; + } + Err(error) => panic!("unexpected IPv4-mapped wildcard bind failure: {error}"), + }; + let port = server.local_addr().unwrap().port(); + let target = SocketAddr::from((secondary_ipv4_loopback(), port)); + let running = spawn(server); + + let mut client = connected_client(target).await; + assert_started(&mut client, target).await; + let meta = echo_once(&mut client).await; + // A mapped listener is `AF_INET6` but emits IPv4, and the IPv6 Traffic + // Class option does not reach those packets — it leaves their TOS at + // zero. Only the IPv4 option marks them, so this asserts the listener + // was read as the IPv4 one it is. + if cfg!(target_os = "linux") { + assert_eq!( + meta.traffic_class, + Some(EF_TRAFFIC_CLASS), + "a mapped listener's reply still carries its session's marking" + ); + assert_eq!(meta.dscp, Some(EF_DSCP)); + } + assert!(matches!( + client.close().await.unwrap().as_slice(), + [ClientEvent::SessionClosed { .. }] + )); + + running.stop().await; + }) + .await + .expect("the IPv4-mapped wildcard test exceeded its bounded runtime"); +} + +#[tokio::test(flavor = "current_thread")] +async fn a_wildcard_ipv6_listener_replies_from_the_requested_address() { + timeout(Duration::from_secs(5), async { + if !ipv6_loopback_available() { + eprintln!("skipping wildcard IPv6 test: IPv6 loopback unavailable"); + return; + } + let server = match Server::bind("[::]:0".parse().unwrap(), unthrottled()).await { + Ok(server) => server, + Err(ServerRuntimeError::Bind { source, .. }) if is_family_unavailable(&source) => { + eprintln!("skipping wildcard IPv6 test: IPv6 unavailable"); + return; + } + Err(error) => panic!("unexpected wildcard IPv6 bind failure: {error}"), + }; + let port = server.local_addr().unwrap().port(); + let target: SocketAddr = format!("[::1]:{port}").parse().unwrap(); + let running = spawn(server); + + // Loopback is one address, so this does not prove multi-homed IPv6. It + // does exercise the parts that differ from IPv4: the received packet + // info, the interface index carried with it, and the source-selected + // IPv6 send. + let mut client = connected_client(target).await; + assert_started(&mut client, target).await; + echo_once(&mut client).await; + assert!(matches!( + client.close().await.unwrap().as_slice(), + [ClientEvent::SessionClosed { .. }] + )); + + running.stop().await; + }) + .await + .expect("the wildcard IPv6 source-selection test exceeded its bounded runtime"); +} + +/// A local IPv4 address that is not the primary loopback one, where the host +/// has one. +/// +/// Bindability is the test: an address this process can bind is one the host +/// accepts datagrams for. `127.0.0.0/8` supplies a second local destination on +/// hosts that carry the whole range, without touching interface configuration. +fn secondary_ipv4_loopback() -> Ipv4Addr { + let candidate = Ipv4Addr::new(127, 0, 0, 2); + if UdpSocket::bind((candidate, 0)).is_ok() { + return candidate; + } + eprintln!("{candidate} is not a local address on this host; using 127.0.0.1 instead"); + Ipv4Addr::LOCALHOST +} + +fn ipv6_loopback_available() -> bool { + match UdpSocket::bind("[::1]:0") { + Ok(_) => true, + Err(error) if is_family_unavailable(&error) => false, + Err(error) => panic!("unexpected IPv6 loopback bind failure: {error}"), + } +} + +fn is_family_unavailable(error: &io::Error) -> bool { + matches!( + error.kind(), + io::ErrorKind::AddrNotAvailable | io::ErrorKind::Unsupported + ) +} + +fn unthrottled() -> ServerConfig { + ServerConfig::default().with_min_send_interval(Duration::ZERO) +} + +async fn connected_client(server_addr: SocketAddr) -> AsyncClient { + let config = ClientConfig { + server_addr: server_addr.to_string(), + duration: Some(Duration::from_secs(30)), + interval: Duration::from_millis(10), + length: 64, + dscp: EF_DSCP, + received_stats: ReceivedStats::Both, + stamp_at: StampAt::Both, + clock: Clock::Both, + open_timeouts: vec![Duration::from_millis(500)], + socket_config: SocketConfig { + recv_timeout: Some(Duration::from_millis(500)), + ..SocketConfig::default() + }, + probe_timeout: Duration::from_millis(500), + ..ClientConfig::default() + }; + AsyncClient::connect(config).await.unwrap() +} + +async fn assert_started(client: &mut AsyncClient, server_addr: SocketAddr) { + match client.open().await.unwrap() { + OpenOutcome::Started { remote, token, .. } => { + assert_eq!(remote, server_addr); + assert_ne!(token, 0); + } + OpenOutcome::NoTestCompleted { .. } => panic!("a normal client unexpectedly ran no-test"), + } +} + +/// Sends one echo and returns its reply's receive metadata. +async fn echo_once(client: &mut AsyncClient) -> PacketMeta { + client.send_probe().await.unwrap(); + loop { + for event in client.recv().await.unwrap() { + if let ClientEvent::EchoReply { + seq, packet_meta, .. + } = event + { + assert_eq!(seq, 0); + return packet_meta; + } + } + } +} + +struct Running { + shutdown: oneshot::Sender<()>, + task: JoinHandle>, +} + +impl Running { + async fn stop(self) { + self.shutdown.send(()).unwrap(); + self.task.await.unwrap().unwrap(); + } +} + +fn spawn(mut server: Server) -> Running { + let (shutdown, shutdown_rx) = oneshot::channel(); + let task = tokio::spawn(async move { + server + .run(async { + let _ = shutdown_rx.await; + }) + .await + }); + Running { shutdown, task } +}