use std::cmp::Ordering; use super::benchmark_result::BenchmarkResult; use crate::{ options::SortOrder, quantity::{self, Ratio, Time, Zero}, }; #[derive(Debug)] pub struct BenchmarkResultWithRelativeSpeed<'a> { pub result: &'a BenchmarkResult, pub relative_speed: f64, pub relative_speed_stddev: Option, pub is_reference: bool, // Less means faster pub relative_ordering: Ordering, } impl BenchmarkResultWithRelativeSpeed<'_> { pub fn reference_label(&self) -> &'static str { if self.is_reference { " (reference)" } else { match self.relative_ordering { Ordering::Less => " (faster)", Ordering::Equal => " (same speed)", Ordering::Greater => " (slower)", } } } } pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering { l.mean().partial_cmp(&r.mean()).unwrap_or(Ordering::Equal) } pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult { results .iter() .min_by(|&l, &r| compare_mean_time(l, r)) .expect("at least one benchmark result") } fn compute_relative_speeds<'a>( results: &'a [BenchmarkResult], reference: &'a BenchmarkResult, sort_order: SortOrder, ) -> Vec> { let mut results: Vec<_> = results .iter() .map(|result| { // Separate benchmarks can have identical names and measurements. let is_reference = std::ptr::eq(result, reference); let relative_ordering = compare_mean_time(result, reference); if result.mean_wall_clock_time() == Time::zero() || reference.mean_wall_clock_time() == Time::zero() { return BenchmarkResultWithRelativeSpeed { result, relative_speed: if is_reference { 1.0 } else { f64::INFINITY }, relative_speed_stddev: None, is_reference, relative_ordering, }; } let ratio = match relative_ordering { Ordering::Less => reference.mean() / result.mean(), Ordering::Equal => 1.0, Ordering::Greater => result.mean() / reference.mean(), }; // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas // Covariance assumed to be 0, i.e. variables are assumed to be independent let ratio_stddev = match ( result.measurements.stddev(), reference.measurements.stddev(), ) { (Some(result_stddev), Some(reference_stddev)) => Some( ratio * ((result_stddev / result.mean_wall_clock_time()) .powi(uom::typenum::P2::new()) + (reference_stddev / reference.mean_wall_clock_time()) .powi(uom::typenum::P2::new())) .sqrt(), ), _ => None, }; BenchmarkResultWithRelativeSpeed { result, relative_speed: ratio.get::(), relative_speed_stddev: ratio_stddev.map(|r| r.get::()), is_reference, relative_ordering, } }) .collect(); match sort_order { SortOrder::Command => {} SortOrder::MeanTime => { results.sort_unstable_by(|r1, r2| compare_mean_time(r1.result, r2.result)); } } results } pub fn compute_with_check_from_reference<'a>( results: &'a [BenchmarkResult], reference: &'a BenchmarkResult, sort_order: SortOrder, ) -> Option>> { if fastest_of(results).mean() == 0.0 || reference.mean() == 0.0 { return None; } Some(compute_relative_speeds(results, reference, sort_order)) } pub fn compute_with_check( results: &[BenchmarkResult], sort_order: SortOrder, ) -> Option>> { let fastest = fastest_of(results); if fastest.mean() == 0.0 { return None; } Some(compute_relative_speeds(results, fastest, sort_order)) } /// Compute relative speeds against the given reference, or the fastest result. pub fn compute<'a>( results: &'a [BenchmarkResult], sort_order: SortOrder, reference: Option<&'a BenchmarkResult>, ) -> Vec> { let reference = reference.unwrap_or_else(|| fastest_of(results)); compute_relative_speeds(results, reference, sort_order) } #[cfg(test)] fn create_result(name: &str, mean: f64) -> BenchmarkResult { use std::collections::BTreeMap; use crate::benchmark::benchmark_result::BenchmarkRun; BenchmarkResult { command: name.into(), runs: vec![BenchmarkRun { wall_clock_time: mean, user_time: mean, system_time: 0., memory_usage_byte: 1024, exit_code: Some(0), }], parameters: BTreeMap::new(), } } #[test] fn test_compute_relative_speed() { use approx::assert_relative_eq; let results = vec![ create_result("cmd1", 3.0), create_result("cmd2", 2.0), create_result("cmd3", 5.0), ]; let annotated_results = compute_with_check(&results, SortOrder::Command).unwrap(); assert_relative_eq!(1.5, annotated_results[0].relative_speed); assert_relative_eq!(1.0, annotated_results[1].relative_speed); assert_relative_eq!(2.5, annotated_results[2].relative_speed); } #[test] fn test_compute_relative_speed_with_reference() { use approx::assert_relative_eq; let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)]; let reference = create_result("cmd2", 4.0); let annotated_results = compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap(); assert_relative_eq!(2.0, annotated_results[0].relative_speed); assert_relative_eq!(1.25, annotated_results[1].relative_speed); } #[test] fn test_compute_relative_speed_for_zero_times() { let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)]; let annotated_results = compute_with_check(&results, SortOrder::Command); assert!(annotated_results.is_none()); } #[test] fn reference_identity_distinguishes_equal_results() { let results = vec![create_result("same", 1.0), create_result("same", 1.0)]; let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command); assert!(entries[0].is_reference); assert!(!entries[1].is_reference); } #[test] fn reference_ratios_handle_zero_times() { let mut results = vec![create_result("reference", 2.0), create_result("zero", 0.0)]; for result in &mut results { let measurement = result.measurements.measurements[0]; result.measurements.measurements.push(measurement); } let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command); assert_eq!(entries[1].relative_speed, f64::INFINITY); let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command); assert_eq!(entries[0].relative_speed, f64::INFINITY); assert_eq!(entries[0].relative_speed_stddev, None); assert_eq!(entries[1].relative_speed, 1.0); }