From 2882beb2c9a61f29886982c26427d3a32cace58c Mon Sep 17 00:00:00 2001 From: Nils Andreas Svee Date: Tue, 15 Sep 2026 12:31:26 +0700 Subject: [PATCH] docs(client): clarify TX timestamp correlation --- crates/irtt-client/AGENTS.md | 41 +++++++++---------- crates/irtt-client/src/receive/linux.rs | 17 ++++---- .../src/receive/linux/error_queue.rs | 28 +++++-------- crates/irtt-client/src/session/machine.rs | 10 ++--- 4 files changed, 43 insertions(+), 53 deletions(-) diff --git a/crates/irtt-client/AGENTS.md b/crates/irtt-client/AGENTS.md index f025376..abc0c7a 100644 --- a/crates/irtt-client/AGENTS.md +++ b/crates/irtt-client/AGENTS.md @@ -26,30 +26,27 @@ configuration (`try_enable_tx_timestamping` restores it explicitly) and timing falls back to userspace `sent_at` as it always has. TX capability failure is never a public error and is never logged from the library. -### Correlation invariant: kernel ID == wire sequence +### Best-effort kernel-ID correlation -The client uses the kernel's automatic `OPT_ID` directly as the probe's wire -sequence number — no separate correlation map. This is safe because: +The client uses the kernel's automatic `OPT_ID` as the probe's wire-sequence +correlation ID — no separate correlation map. On the normal successful-send +path, both counters start at zero after Open and advance together. This is +not a permanent identity: a kernel ID can theoretically be consumed while a +send is being prepared and then survive a subsequent send failure, leaving +kernel IDs one or more steps ahead of `next_wire_seq`. Do not restore an +assumption that failed or nonblocking sends cannot cause that desynchrony. -- **A failed/nonblocking send never consumes an ID.** Linux increments the - per-socket ID counter (`sk->sk_tskey`) speculatively while building a - datagram and rolls the increment back (`atomic_dec`) on every error path - out of `__ip_append_data`/`__ip6_append_data` - (`net/ipv4/ip_output.c`, `net/ipv6/ip6_output.c`), guarded by a local - `hold_tskey` flag. A `WouldBlock` `try_send` therefore can never leave a - gap in, or otherwise advance, the ID the kernel will assign to the next - *successful* submission. (`include/net/sock.h`'s `_sock_tx_timestamp` - performs the same increment for the non-corked path.) Client-side, the - existing send transaction only calls `commit_probe_sent` — which is the - only thing that advances `next_wire_seq` — after a successful socket send; - a `WouldBlock` result never reaches it. -- **The two counters start together and advance together.** `wire_seq` - starts at 0 on every successful Open (`SessionMachine::commit_open` builds - a fresh `ActiveSession` with `next_wire_seq: 0`) and advances by exactly 1 - only via `commit_probe_sent`, which only runs after a confirmed successful - send — the same condition under which the kernel's counter advances by 1. - TX timestamping is enabled only after that same successful Open and before - any probe is sent, so both counters' zero points line up exactly. +The effect is confined to the optional timing enhancement. A kernel timestamp +whose ID cannot be matched to a pending or retained timed-out probe is +discarded, as is any implausible timestamp; `sent_at` remains the ordinary +userspace fallback. This is safe because: + +- **Normal successful sends track together.** `wire_seq` starts at 0 on every + successful Open (`SessionMachine::commit_open` builds a fresh + `ActiveSession` with `next_wire_seq: 0`) and advances by exactly 1 only via + `commit_probe_sent`, after a confirmed successful socket send. TX + timestamping is enabled after that same successful Open and before any probe + is sent, so the normal zero points line up. - **One socket, one session, no reopen.** `SessionMachine`'s state machine only reaches `Open` from `Connected`, and `commit_open`/`commit_local_close` never route back to `Connected`. A `Client`/`AsyncClient` cannot reopen a diff --git a/crates/irtt-client/src/receive/linux.rs b/crates/irtt-client/src/receive/linux.rs index cdefb6b..893e116 100644 --- a/crates/irtt-client/src/receive/linux.rs +++ b/crates/irtt-client/src/receive/linux.rs @@ -12,14 +12,15 @@ //! After a successful Open, the client adapter best-effort upgrades the //! socket from [`RX_TIMESTAMPING_FLAGS`] to [`error_queue::TX_TIMESTAMPING_FLAGS`] //! via [`try_enable_tx_timestamping`]. When that succeeds, the automatic -//! `SOF_TIMESTAMPING_OPT_ID` counter the kernel assigns to each successfully -//! submitted datagram is used directly as the probe's wire sequence number -//! (see the client crate's `AGENTS.md` for the correlation invariant this -//! relies on). [`drain_tx_timestamps`] performs a small bounded, nonblocking -//! read of `MSG_ERRQUEUE` so the adapter can opportunistically collect those -//! timestamps without ever waiting for one. The observed kernel TX wall time -//! is retained as metadata only; it is not consumed by any measurement in -//! this change. +//! `SOF_TIMESTAMPING_OPT_ID` counter normally tracks the probe wire sequence +//! after successful sends, so it is used as a best-effort correlation ID. +//! A kernel ID can theoretically be consumed by a send that later fails, +//! desynchronizing that correlation. [`drain_tx_timestamps`] discards +//! unmatched or implausible records; the userspace `sent_at` timestamp remains +//! the fallback. It performs a small bounded, nonblocking read of +//! `MSG_ERRQUEUE` so the adapter can opportunistically collect timestamps +//! without ever waiting for one. The observed kernel TX wall time is retained +//! as metadata only; it is not consumed by any measurement in this change. use std::{ io::{self, IoSliceMut}, diff --git a/crates/irtt-client/src/receive/linux/error_queue.rs b/crates/irtt-client/src/receive/linux/error_queue.rs index c224919..8d16e65 100644 --- a/crates/irtt-client/src/receive/linux/error_queue.rs +++ b/crates/irtt-client/src/receive/linux/error_queue.rs @@ -624,22 +624,14 @@ mod tests { )); } - // NOTE: a live test proving that a failed nonblocking `sendmsg` does not - // consume an automatic OPT_ID is not included here. Forcing a reliable, - // non-flaky `WouldBlock` on a connected loopback UDP socket (e.g. via - // socket-buffer saturation) has no deterministic OS-level mechanism - // available in this environment, and the repository's testing policy - // explicitly forbids faking coverage with sleep/saturation-based - // flakiness. That safety is instead established from Linux kernel - // source: `_sock_tx_timestamp` (include/net/sock.h) and - // `__ip_append_data`/`__ip6_append_data` (net/ipv4/ip_output.c, - // net/ipv6/ip6_output.c) increment `sk->sk_tskey` speculatively before - // a datagram is fully built and explicitly `atomic_dec` it back out on - // every error path (tracked via a local `hold_tskey` flag), so a failed - // send can never leave a gap in — or otherwise advance — the ID space - // the kernel hands out to successfully submitted datagrams. See the - // client crate's `AGENTS.md` for the full citation. The client-side - // half of this invariant (a `WouldBlock` `try_send` never advances the - // local wire-sequence counter used as the correlation ID) is covered by - // a deterministic test in the adapter module that injects `WouldBlock`. + // NOTE: There is no live test for whether a failed nonblocking `sendmsg` + // consumes an automatic OPT_ID. Forcing a reliable, non-flaky `WouldBlock` + // on a connected loopback UDP socket (for example through socket-buffer + // saturation) has no deterministic OS-level mechanism in this environment, + // and the repository's testing policy forbids sleep/saturation-based + // flakiness. The correlation is intentionally best effort: a kernel ID can + // theoretically be consumed by a send that later fails. The deterministic + // adapter test covers only the local half: an injected `WouldBlock` does + // not advance the local wire-sequence counter. Any later unmatched kernel + // timestamp is discarded and the userspace `sent_at` remains the fallback. } diff --git a/crates/irtt-client/src/session/machine.rs b/crates/irtt-client/src/session/machine.rs index e80fc09..3c925f5 100644 --- a/crates/irtt-client/src/session/machine.rs +++ b/crates/irtt-client/src/session/machine.rs @@ -427,11 +427,11 @@ impl SessionMachine { /// automatic `SOF_TIMESTAMPING_OPT_ID` the kernel assigned when the /// datagram was submitted. /// - /// `wire_seq` doubles as that correlation ID: both counters start at - /// zero for a session's first timestamped send and advance by exactly - /// one only on a confirmed successful submission, so they stay - /// identical for the life of one open session (see the client crate's - /// `AGENTS.md` for the full invariant). + /// `wire_seq` normally matches that correlation ID after successful + /// sends. This is best effort rather than an invariant: a kernel ID can + /// theoretically be consumed by a send that later fails, leaving the + /// counters desynchronized. An unmatched or implausible ID is discarded; + /// the probe's userspace `sent_at` remains the fallback. /// /// Updates a still-pending or already-timed-out probe in place. Never /// resurrects a completed, evicted, or unknown probe, and never touches