//! Metric selection, descriptive statistics, and display units. use anyhow::{bail, ensure, Result}; use crate::benchmark::measurement::{Measurement, Measurements}; use crate::quantity::statistics::{max, mean, median, min, standard_deviation}; use crate::quantity::{byte, ratio, second, Ratio}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Metric { TimeWallClock, TimeCpu, TimeUser, TimeSystem, MemoryPeakResident, CpuCycles, Instructions, CacheReferences, CacheMisses, BranchMisses, } impl Metric { pub const ALL: [Self; 10] = [ Self::TimeWallClock, Self::TimeCpu, Self::TimeUser, Self::TimeSystem, Self::MemoryPeakResident, Self::CpuCycles, Self::Instructions, Self::CacheReferences, Self::CacheMisses, Self::BranchMisses, ]; pub fn name(self) -> &'static str { match self { Self::TimeWallClock => "time_wall_clock", Self::TimeCpu => "time_cpu", Self::TimeUser => "time_user", Self::TimeSystem => "time_system", Self::MemoryPeakResident => "memory_peak_resident", Self::CpuCycles => "cpu_cycles", Self::Instructions => "instructions", Self::CacheReferences => "cache_references", Self::CacheMisses => "cache_misses", Self::BranchMisses => "branch_misses", } } pub fn label(self) -> &'static str { match self { Self::TimeWallClock => "Wall Time", Self::TimeCpu => "CPU time", Self::TimeUser => "User time", Self::TimeSystem => "System time", Self::MemoryPeakResident => "Memory", Self::CpuCycles => "CPU cycles", Self::Instructions => "Instructions", Self::CacheReferences => "Cache references", Self::CacheMisses => "Cache misses", Self::BranchMisses => "Branch misses", } } /// Extract a value in canonical units: seconds, bytes, or counts. pub fn value(self, measurement: &Measurement) -> Option { let counters = &measurement.hardware_counters; match self { Self::TimeWallClock => Some(measurement.time_wall_clock.get::()), Self::TimeCpu => Some(measurement.time_cpu.get::()), Self::TimeUser => Some(measurement.time_user.get::()), Self::TimeSystem => Some(measurement.time_system.get::()), Self::MemoryPeakResident => measurement.memory_peak_resident.map(|v| v.get::()), Self::CpuCycles => counters.cpu_cycles.map(|v| v as f64), Self::Instructions => counters.instructions.map(|v| v as f64), Self::CacheReferences => counters.cache_references.map(|v| v as f64), Self::CacheMisses => counters.cache_misses.map(|v| v as f64), Self::BranchMisses => counters.branch_misses.map(|v| v as f64), } } pub fn is_time(self) -> bool { matches!( self, Self::TimeWallClock | Self::TimeCpu | Self::TimeUser | Self::TimeSystem ) } pub fn is_counter(self) -> bool { !self.is_time() && self != Self::MemoryPeakResident } /// Reject known unsupported configurations before any commands execute. /// Availability that depends on permissions or hardware is checked after collection. pub fn ensure_available(self, shell: bool) -> Result<()> { if self.is_counter() { ensure!( !shell, "Metric '{}' is unavailable when a shell is enabled", self.name() ); ensure!( cfg!(any( target_os = "macos", all( target_os = "linux", target_pointer_width = "64", any( target_arch = "aarch64", target_arch = "x86_64", target_arch = "riscv64", target_arch = "powerpc64", target_arch = "loongarch64" ) ) )), "Metric '{}' is unavailable on this platform", self.name() ); ensure!( !cfg!(target_os = "macos") || matches!(self, Self::CpuCycles | Self::Instructions), "Metric '{}' is unavailable on macOS", self.name() ); } ensure!( !cfg!(windows) || self != Self::MemoryPeakResident, "Metric '{}' is unavailable on Windows", self.name() ); Ok(()) } pub fn base_unit(self) -> Unit { if self.is_time() { Unit::new("s", 1.0) } else if self.is_counter() { Unit::new("count", 1.0) } else { Unit::new("B", 1.0) } } fn units(self) -> &'static [Unit] { if self.is_time() { TIME_UNITS } else if self.is_counter() { COUNT_UNITS } else { MEMORY_UNITS } } /// Require complete coverage so different metrics describe the same runs. pub fn summarize(self, measurements: &Measurements) -> Option { let values = measurements .measurements .iter() .map(|measurement| self.value(measurement)) .collect::>>()?; if values.is_empty() { return None; } Some(Stats { count: values.len(), mean: mean(values.iter().copied()), stddev: (values.len() > 1).then(|| { standard_deviation(values.iter().map(|&v| Ratio::new::(v))).get::() }), median: median(values.iter().copied()), min: min(values.iter().copied()), max: max(values.iter().copied()), }) } } #[derive(Debug, Clone, Copy, PartialEq)] pub struct MetricSelection { pub metric: Metric, pub unit: Option, } impl Default for MetricSelection { fn default() -> Self { Self { metric: Metric::TimeWallClock, unit: None, } } } impl MetricSelection { /// Parse an ordered, nonempty list of `METRIC[:UNIT]` selections. pub fn parse_list(input: &str) -> Result> { let mut selections = Vec::new(); for item in input.split(',') { let (name, unit_name) = match item.split_once(':') { Some((name, unit)) => (name, Some(unit)), None => (item, None), }; let metric = match Metric::ALL.iter().copied().find(|m| m.name() == name) { Some(metric) => metric, None => bail!( "Unknown metric '{name}'. Expected one of: {}", Metric::ALL.map(Metric::name).join(", ") ), }; if selections .iter() .any(|selection: &Self| selection.metric == metric) { bail!("Metric '{name}' was selected more than once"); } let unit = match unit_name { None => None, Some(unit_name) => { // Accept an ASCII spelling for microseconds as well. let unit_name = if unit_name == "us" { "µs" } else { unit_name }; match metric.units().iter().find(|unit| unit.symbol == unit_name) { Some(unit) => Some(*unit), None => bail!( "Unit '{unit_name}' is not valid for metric '{name}'. Expected one of: {}", metric.units().iter().map(|u| u.symbol).collect::>().join(", ") ), } } }; selections.push(Self { metric, unit }); } Ok(selections) } pub fn display_unit(self, mean: f64) -> Unit { if let Some(unit) = self.unit { return unit; } if mean == 0.0 { return self.metric.base_unit(); } let units = if self.metric == Metric::MemoryPeakResident { BINARY_MEMORY_UNITS } else { self.metric.units() }; units .iter() .rev() .find(|unit| mean.abs() >= unit.scale) .copied() .unwrap_or(units[0]) } /// CSV never automatically scales values. pub fn csv_unit(self) -> Unit { self.unit.unwrap_or_else(|| self.metric.base_unit()) } } #[derive(Debug, Clone, Copy, PartialEq)] pub struct Unit { pub symbol: &'static str, /// The number of canonical units in this unit (e.g. 1e9 for a billion). pub scale: f64, } impl Unit { const fn new(symbol: &'static str, scale: f64) -> Self { Self { symbol, scale } } pub fn format(self, value: f64) -> String { let value = self.format_value(value); if self.symbol == "count" { value } else { format!("{value} {}", self.symbol) } } pub fn format_value(self, value: f64) -> String { let precision = if self.symbol == "s" { 3 } else { 1 }; format!("{:.precision$}", value / self.scale) } } const TIME_UNITS: &[Unit] = &[ Unit::new("ns", 1e-9), Unit::new("µs", 1e-6), Unit::new("ms", 1e-3), Unit::new("s", 1.0), Unit::new("min", 60.0), Unit::new("h", 3600.0), ]; const BINARY_MEMORY_UNITS: &[Unit] = &[ Unit::new("B", 1.0), Unit::new("KiB", 1024.0), Unit::new("MiB", 1_048_576.0), Unit::new("GiB", 1_073_741_824.0), Unit::new("TiB", 1_099_511_627_776.0), ]; const MEMORY_UNITS: &[Unit] = &[ Unit::new("B", 1.0), Unit::new("kB", 1e3), Unit::new("MB", 1e6), Unit::new("GB", 1e9), Unit::new("TB", 1e12), Unit::new("KiB", 1024.0), Unit::new("MiB", 1_048_576.0), Unit::new("GiB", 1_073_741_824.0), Unit::new("TiB", 1_099_511_627_776.0), ]; const COUNT_UNITS: &[Unit] = &[ Unit::new("count", 1.0), Unit::new("k", 1e3), Unit::new("M", 1e6), Unit::new("B", 1e9), ]; #[derive(Debug, Clone, Copy)] pub struct Stats { pub count: usize, pub mean: f64, pub stddev: Option, pub median: f64, pub min: f64, pub max: f64, } impl Stats { pub fn change_from(&self, reference: &Self) -> Option { (reference.mean != 0.0).then(|| 100.0 * (self.mean / reference.mean - 1.0)) } pub fn format_change_from(&self, reference: &Self) -> String { let Some(change) = self.change_from(reference) else { return "N/A".to_owned(); }; let mut formatted = format!("{change:+.1}%"); if formatted == "+0.0%" || formatted == "-0.0%" { formatted = "0.0%".to_owned(); } formatted } pub fn format_factor_from(&self, reference: &Self, metric: Metric) -> String { if reference.mean > 0.0 && (self.mean >= 1.3 * reference.mean || self.mean <= 0.7 * reference.mean) { let decrease = self.mean < reference.mean; let factor = if decrease { reference.mean / self.mean } else { self.mean / reference.mean }; let direction = match (metric.is_time(), decrease) { (true, true) => "faster", (true, false) => "slower", (false, true) => "less", (false, false) => "more", }; let factor = if factor.is_infinite() { "∞".to_owned() } else { format!("{factor:.1}") }; format!("({factor}x {direction})") } else { String::new() } } } #[cfg(test)] mod tests { use super::*; use crate::quantity::{Information, Time}; #[test] fn large_changes_include_factors() { let reference = Stats { count: 1, mean: 100.0, stddev: None, median: 100.0, min: 100.0, max: 100.0, }; for (mean, metric, expected) in [ (130.0, Metric::TimeWallClock, "+30.0% (1.3x slower)"), (70.0, Metric::TimeCpu, "-30.0% (1.4x faster)"), (10.0, Metric::Instructions, "-90.0% (10.0x less)"), (1000.0, Metric::MemoryPeakResident, "+900.0% (10.0x more)"), (129.99, Metric::TimeWallClock, "+30.0%"), (70.01, Metric::Instructions, "-30.0%"), (100.0, Metric::Instructions, "0.0%"), (0.0, Metric::Instructions, "-100.0% (∞x less)"), ] { let stats = Stats { mean, ..reference }; let (percentage, factor) = expected .split_once(" (") .map_or((expected, String::new()), |(p, f)| (p, format!("({f}"))); assert_eq!(stats.format_change_from(&reference), percentage); assert_eq!(stats.format_factor_from(&reference, metric), factor); } let zero = Stats { mean: 0.0, ..reference }; assert_eq!(reference.format_change_from(&zero), "N/A"); } #[test] fn cpu_time_statistics_use_per_run_totals() { let measurements = Measurements::new( [(1.0, 3.0), (3.0, 1.0)] .iter() .map(|&(user, system)| Measurement { time_cpu: Time::new::(user + system), time_user: Time::new::(user), time_system: Time::new::(system), ..Measurement::default() }) .collect(), ); let selection = MetricSelection::parse_list("time_cpu:ms").unwrap()[0]; let stats = selection.metric.summarize(&measurements).unwrap(); assert_eq!(selection.csv_unit().format(stats.mean), "4000.0 ms"); // User and system time vary, but their per-run total is constant. assert_eq!(stats.stddev, Some(0.0)); assert_eq!(stats.min, 4.0); assert_eq!(stats.max, 4.0); let json = serde_json::to_value(&measurements).unwrap(); for sample in json["measurements"].as_array().unwrap() { assert_eq!(sample["time_cpu"]["value"], 4.0); assert_eq!(sample["time_cpu"]["unit"], "second"); } } #[test] fn selection_units_and_validation() { let selections = MetricSelection::parse_list( "instructions:B,memory_peak_resident:B,time_wall_clock:ms,time_user:us", ) .unwrap(); assert_eq!(selections[0].csv_unit().scale, 1e9); assert_eq!(selections[1].csv_unit().scale, 1.0); assert_eq!(selections[2].csv_unit().format(0.125), "125.0 ms"); assert_eq!(selections[3].csv_unit().symbol, "µs"); let instructions = MetricSelection::parse_list("instructions").unwrap()[0]; assert_eq!(instructions.csv_unit().symbol, "count"); assert_eq!( instructions.display_unit(143.2e9).format(143.2e9), "143.2 B" ); for invalid in [ "", "unknown", "time_wall_clock,", "instructions,instructions:M", "instructions:MiB", "memory_peak_resident:ms", "time_wall_clock:", ] { assert!(MetricSelection::parse_list(invalid).is_err(), "{}", invalid); } } #[test] fn summaries_require_complete_samples() { let mut measurements = Measurements::new(vec![Measurement { memory_peak_resident: Some(Information::new::(10.0)), ..Measurement::default() }]); let single = Metric::MemoryPeakResident.summarize(&measurements).unwrap(); assert_eq!(single.stddev, None); measurements.push(Measurement::default()); assert!(Metric::MemoryPeakResident .summarize(&measurements) .is_none()); measurements.measurements[1].memory_peak_resident = Some(Information::new::(30.0)); let stats = Metric::MemoryPeakResident.summarize(&measurements).unwrap(); assert_eq!(stats.mean, 20.0); assert_eq!(stats.change_from(&single), Some(100.0)); assert_eq!(stats.format_change_from(&single), "+100.0%"); assert_eq!(single.format_change_from(&stats), "-50.0%"); assert_eq!( Stats { mean: 9.999, ..single } .format_change_from(&single), "0.0%" ); approx::assert_relative_eq!(stats.stddev.unwrap(), 200.0_f64.sqrt()); let zero = Metric::TimeWallClock.summarize(&measurements).unwrap(); assert_eq!(zero.change_from(&zero), None); assert_eq!(zero.format_change_from(&zero), "N/A"); } }