// SPDX-FileCopyrightText: 2022-Present Charles Corrigan mailto:chas-iot@runegate.org (github @chas-iot) // SPDX-FileCopyrightText: 2022-Present Daniel Lakeland mailto:dlakelan@street-artists.org (github @dlakelan) // SPDX-FileCopyrightText: 2022-Present Mark Baker mailto:mark@vpost.net (github @Fail-Safe) // SPDX-FileCopyrightText: 2022-Present Nils Andreas Svee mailto:contact@lochnair.net (github @Lochnair) // // SPDX-License-Identifier: MPL-2.0 extern crate core; mod baseliner; mod log; mod measurement; mod metrics; mod pinger; mod pinger_icmp; mod pinger_icmp_ts; mod platform; mod ratecontroller; mod reflector_selector; mod settings; mod time; mod util; use crate::baseliner::Baseliner; use crate::measurement::icmp::{IcmpBinding, IcmpEngine}; use crate::measurement::model::{ MeasurementSource, MeasurementStream, MeasurementStreamId, ProbeSchedule, }; use crate::metrics::{Metric, Metrics, MetricsSender}; use crate::pinger::{InFlightProbeCache, PingListener, PingSender}; use crate::pinger_icmp::{PingerICMPEchoListener, PingerICMPEchoSender}; use crate::pinger_icmp_ts::{PingerICMPTimestampListener, PingerICMPTimestampSender}; use crate::platform::{TrafficControlBackend, traffic_control_backend, warn_platform_limitations}; use crate::ratecontroller::Ratecontroller; use crate::reflector_selector::ReflectorSelector; use crate::settings::MeasurementType; use crate::settings::Settings; use ::log::{info, warn}; use flume::RecvTimeoutError; use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr}; use std::str::FromStr; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::{Arc, Mutex, RwLock}; use std::thread::sleep; use std::time::{Duration, Instant}; use std::{process, thread}; struct ReflectorSetup { peers: Arc>>, pool: Vec, reselection_enabled: bool, active_count: usize, } type PingListenerBox = Box; type PingSenderBox = Box; pub static SHUTDOWN: AtomicBool = AtomicBool::new(false); extern "C" fn signal_handler(_: libc::c_int) { SHUTDOWN.store(true, Ordering::Relaxed); } const VERSION: &str = env!("CARGO_PKG_VERSION"); const RESELECTION_CANDIDATE_BURST: usize = 20; const INFLIGHT_CAPACITY_DUPLICATE_FACTOR: usize = 2; const INFLIGHT_CAPACITY_MIN: usize = 256; fn compute_inflight_probe_capacity( settings: &Settings, active_reflector_count: usize, reselection_enabled: bool, ) -> usize { let tick_interval_ms = (settings.advanced_settings.tick_interval * 1000.0).max(1.0); let expected_path_delay_ms = (settings.advanced_settings.download_delay_ms + settings.advanced_settings.upload_delay_ms) .max(10.0); // Keep enough room for severe queueing events while still adapting to configured delay budgets. let max_rtt_ms = (expected_path_delay_ms * 20.0).clamp(1000.0, 10000.0); let burst_reflector_count = if reselection_enabled { active_reflector_count + RESELECTION_CANDIDATE_BURST } else { active_reflector_count }; let probes_per_reflector = (max_rtt_ms / tick_interval_ms).ceil() as usize; (burst_reflector_count .saturating_mul(probes_per_reflector) .saturating_mul(INFLIGHT_CAPACITY_DUPLICATE_FACTOR)) .max(INFLIGHT_CAPACITY_MIN) } fn create_pinger(measurement_type: MeasurementType) -> (PingListenerBox, PingSenderBox) { match measurement_type { MeasurementType::Icmp => ( Box::new(PingerICMPEchoListener {}), Box::new(PingerICMPEchoSender {}), ), MeasurementType::IcmpTimestamps => ( Box::new(PingerICMPTimestampListener {}), Box::new(PingerICMPTimestampSender {}), ), } } fn initialize_shaper( settings: &Settings, traffic_control: &mut T, ) -> anyhow::Result<(T::Handle, T::Handle)> { initialize_shaper_with_settle_time(settings, traffic_control, Duration::from_secs(2)) } fn initialize_shaper_with_settle_time( settings: &Settings, traffic_control: &mut T, settle_time: Duration, ) -> anyhow::Result<(T::Handle, T::Handle)> { let down = traffic_control.find_shaper(&settings.network.download_interface)?; let up = traffic_control.find_shaper(&settings.network.upload_interface)?; info!( "Requesting minimum shaper rates (D/U): {} / {}", settings.network.download_min_kbits(), settings.network.upload_min_kbits(), ); traffic_control.set_rate( &down, settings.network.download_min_kbits() as u64, settings.advanced_settings.dry_run, )?; traffic_control.set_rate( &up, settings.network.upload_min_kbits() as u64, settings.advanced_settings.dry_run, )?; info!( "Sleeping for {} seconds to give the shaper a chance to control existing bloat", settle_time.as_secs_f64(), ); sleep(settle_time); Ok((down, up)) } fn install_signal_handlers() { unsafe { libc::signal( libc::SIGINT, signal_handler as *const () as libc::sighandler_t, ); libc::signal( libc::SIGTERM, signal_handler as *const () as libc::sighandler_t, ); } } fn probe_identifier() -> u16 { // The ICMP identifier is only two bytes. (process::id() & 0xffff) as u16 } fn restore_shaper( settings: &Settings, traffic_control: &mut T, down: &T::Handle, up: &T::Handle, ) { info!( "Requesting restoration to base shaper rates (D/U): {} / {}", settings.network.download_base_kbits, settings.network.upload_base_kbits, ); if let Err(error) = traffic_control.set_rate( down, settings.network.download_base_kbits as u64, settings.advanced_settings.dry_run, ) { warn!("Failed to restore download shaper rate: {error}"); } if let Err(error) = traffic_control.set_rate( up, settings.network.upload_base_kbits as u64, settings.advanced_settings.dry_run, ) { warn!("Failed to restore upload shaper rate: {error}"); } } fn setup_reflectors(settings: &Settings) -> anyhow::Result { let configured = settings.load_reflectors()?; let configured_count = configured.len(); let default_reflectors = [ IpAddr::from_str("9.9.9.9")?, IpAddr::from_str("8.238.120.14")?, IpAddr::from_str("74.82.42.42")?, IpAddr::from_str("194.242.2.2")?, IpAddr::from_str("208.67.222.222")?, IpAddr::from_str("94.140.14.14")?, ]; let reselection_enabled = configured_count > settings.advanced_settings.num_reflectors as usize; let pool = if reselection_enabled { configured } else { Vec::new() }; Ok(ReflectorSetup { peers: Arc::new(RwLock::new(default_reflectors.to_vec())), pool, reselection_enabled, active_count: default_reflectors .len() .max(settings.advanced_settings.num_reflectors as usize), }) } fn wait_for_exit(error_rx: &flume::Receiver) -> anyhow::Result<()> { loop { match error_rx.recv_timeout(Duration::from_secs(1)) { Ok(error) => { return Err(anyhow::anyhow!("thread exited with error: {error}")); } Err(RecvTimeoutError::Disconnected) => { return Ok(()); } Err(RecvTimeoutError::Timeout) => { if SHUTDOWN.load(Ordering::Relaxed) { info!("Received shutdown signal"); return Ok(()); } } } } } fn run(settings: &Settings) -> anyhow::Result<()> { if settings.advanced_settings.dry_run { info!("*** MONITORING MODE ACTIVE — qdisc rates will NOT be changed ***"); } let start_time = Instant::now(); let probe_id = probe_identifier(); let (ie_txt, _) = flume::unbounded(); let (binding_tx, binding_rx) = flume::unbounded(); let engine = IcmpEngine::new(start_time, &settings.network, ie_txt, binding_rx)?; binding_tx.send(vec![IcmpBinding::new( MeasurementStream { id: MeasurementStreamId(0), peer: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), source: MeasurementSource::IcmpEcho, }, ProbeSchedule { period: Duration::from_secs(1), offset: Duration::from_secs(0), }, )])?; let rt = tokio::runtime::Runtime::new().unwrap(); rt.spawn(engine.run()); let ReflectorSetup { peers: reflector_peers, pool: reflector_pool, reselection_enabled, active_count: active_reflector_count, } = setup_reflectors(settings)?; let (baseliner_stats_tx, baseliner_stats_rx) = flume::unbounded(); let (control_snapshot_tx, control_snapshot_rx) = flume::unbounded(); let (selection_snapshot_tx, selection_snapshot_rx) = if reselection_enabled { let (tx, rx) = flume::unbounded(); (Some(tx), Some(rx)) } else { (None, None) }; let (error_tx, error_rx) = flume::unbounded::(); let (reselect_tx, reselect_rx) = flume::bounded(1); let dropped = Arc::new(AtomicU32::new(0)); let (metrics_tx, metrics_thread_handle) = if settings.observability.enabled { let (tx, rx) = flume::bounded(1000); let metrics = Metrics { settings: settings.clone(), metrics_rx: rx, metrics_dropped: Arc::clone(&dropped), }; let err_tx = error_tx.clone(); let handle = thread::Builder::new() .name("metrics".to_string()) .spawn(move || { if let Err(error) = metrics.run() { let _ = err_tx.send(error); } })?; (Some(tx), Some(handle)) } else { (None, None) }; let make_sender = |enabled: bool| -> MetricsSender { metrics_tx .as_ref() .filter(|_| enabled) .map(|tx| MetricsSender::new(tx.clone(), Arc::clone(&dropped))) .unwrap_or_else(MetricsSender::disabled) }; let ping_metrics = make_sender(settings.observability.export_ping_metrics); let baseline_metrics = make_sender(settings.observability.export_baseline_metrics); let event_metrics = make_sender(settings.observability.export_events); let rate_metrics = make_sender(settings.observability.export_rate_metrics); event_metrics.send(Metric::Event { name: "starting", reason: "", reflector: None, tags: if settings.advanced_settings.dry_run { &[("dry_run", "true")] } else { &[] }, }); let (mut ping_listener, mut ping_sender) = create_pinger(settings.advanced_settings.measurement_type); let inflight_probe_capacity = compute_inflight_probe_capacity(settings, active_reflector_count, reselection_enabled); info!( "In-flight probe cache capacity: {} (active_reflectors={}, reselection_enabled={}, tick_interval_s={})", inflight_probe_capacity, active_reflector_count, reselection_enabled, settings.advanced_settings.tick_interval ); let inflight: InFlightProbeCache = Arc::new(Mutex::new(HashMap::with_capacity(inflight_probe_capacity))); let baseliner = Baseliner { settings: settings.clone(), reselect_trigger: reselect_tx.clone(), start_time, stats_rx: baseliner_stats_rx, control_tx: control_snapshot_tx, selection_tx: selection_snapshot_tx, baseline_metrics, event_metrics: event_metrics.clone(), }; let mut main_traffic_control = traffic_control_backend(); let (down_shaper, up_shaper) = initialize_shaper(settings, &mut main_traffic_control)?; let err_tx = error_tx.clone(); let listener_peers = Arc::clone(&reflector_peers); let listener_inflight = Arc::clone(&inflight); let measurement_type = settings.advanced_settings.measurement_type; thread::Builder::new() .name("receiver".to_string()) .spawn(move || { if let Err(error) = ping_listener.listen( probe_id, measurement_type, listener_peers, listener_inflight, baseliner_stats_tx, ping_metrics, ) { let _ = err_tx.send(error); } })?; let err_tx = error_tx.clone(); thread::Builder::new() .name("baseliner".to_string()) .spawn(move || { if let Err(error) = baseliner.run() { let _ = err_tx.send(error); } })?; let err_tx = error_tx.clone(); let sender_peers = Arc::clone(&reflector_peers); let sender_inflight = Arc::clone(&inflight); let measurement_type = settings.advanced_settings.measurement_type; let tick_interval = settings.advanced_settings.tick_interval; thread::Builder::new() .name("sender".to_string()) .spawn(move || { if let Err(error) = ping_sender.send( probe_id, measurement_type, sender_peers, sender_inflight, tick_interval, ) { let _ = err_tx.send(error); } })?; let main_event_metrics = event_metrics.clone(); if reselection_enabled { let reflector_selector = ReflectorSelector { settings: settings.clone(), snapshot_rx: selection_snapshot_rx .expect("reselection snapshot receiver must exist when reselection is enabled"), reflector_peers_lock: Arc::clone(&reflector_peers), reflector_pool, trigger_channel: reselect_rx, metrics: event_metrics, }; let err_tx = error_tx.clone(); thread::Builder::new() .name("reselection".to_string()) .spawn(move || { if let Err(error) = reflector_selector.run() { let _ = err_tx.send(error); } })?; } // Give the baseliner time to collect initial samples before adjusting rates. sleep(Duration::from_secs(10)); let mut ratecontroller = Ratecontroller::new( settings.clone(), control_snapshot_rx, reflector_peers, reselect_tx, rate_metrics, )?; let err_tx = error_tx.clone(); thread::Builder::new() .name("ratecontroller".to_string()) .spawn(move || { if let Err(error) = ratecontroller.run() { let _ = err_tx.send(error); } })?; // Drop the original sender so the channel disconnects if all workers exit cleanly. drop(error_tx); let result = wait_for_exit(&error_rx); // Make the error path request shutdown too, rather than leaving the other workers running. SHUTDOWN.store(true, Ordering::Relaxed); let stopping_reason = if result.is_err() { "error" } else { "signal" }; main_event_metrics.send(Metric::Event { name: "stopping", reason: stopping_reason, reflector: None, tags: &[], }); // Drop all MetricsSender instances and the raw tx so the metrics channel // disconnects once all worker threads also drop their copies. drop(main_event_metrics); drop(metrics_tx); if let Some(handle) = metrics_thread_handle { let _ = handle.join(); } restore_shaper( settings, &mut main_traffic_control, &down_shaper, &up_shaper, ); result } fn main() -> anyhow::Result<()> { println!("Starting sqm-autorate-rust version {}", VERSION); install_signal_handlers(); let settings = Settings::load()?; settings.validate()?; log::init(settings.output.log_level)?; warn_platform_limitations(); run(&settings) }