hyperfine/src/benchmark/relative_speed.rs

use std::cmp::Ordering;

use super::benchmark_result::BenchmarkResult;
use crate::{
    options::SortOrder,
    quantity::{self, Ratio, Time, Zero},
};

#[derive(Debug)]
pub struct BenchmarkResultWithRelativeSpeed<'a> {
    pub result: &'a BenchmarkResult,
    pub relative_speed: f64,
    pub relative_speed_stddev: Option<f64>,
    pub is_reference: bool,
    // Less means faster
    pub relative_ordering: Ordering,
}

impl BenchmarkResultWithRelativeSpeed<'_> {
    pub fn reference_label(&self) -> &'static str {
        if self.is_reference {
            " (reference)"
        } else {
            match self.relative_ordering {
                Ordering::Less => " (faster)",
                Ordering::Equal => " (same speed)",
                Ordering::Greater => " (slower)",
            }
        }
    }
}

pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
    l.mean().partial_cmp(&r.mean()).unwrap_or(Ordering::Equal)
}

pub fn fastest_of(results: &[BenchmarkResult]) -> &BenchmarkResult {
    results
        .iter()
        .min_by(|&l, &r| compare_mean_time(l, r))
        .expect("at least one benchmark result")
}

fn compute_relative_speeds<'a>(
    results: &'a [BenchmarkResult],
    reference: &'a BenchmarkResult,
    sort_order: SortOrder,
) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
    let mut results: Vec<_> = results
        .iter()
        .map(|result| {
            // Separate benchmarks can have identical names and measurements.
            let is_reference = std::ptr::eq(result, reference);
            let relative_ordering = compare_mean_time(result, reference);

            if result.mean_wall_clock_time() == Time::zero()
                || reference.mean_wall_clock_time() == Time::zero()
            {
                return BenchmarkResultWithRelativeSpeed {
                    result,
                    relative_speed: if is_reference { 1.0 } else { f64::INFINITY },
                    relative_speed_stddev: None,
                    is_reference,
                    relative_ordering,
                };
            }

            let ratio = match relative_ordering {
                Ordering::Less => reference.mean() / result.mean(),
                Ordering::Equal => 1.0,
                Ordering::Greater => result.mean() / reference.mean(),
            };

            // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
            // Covariance assumed to be 0, i.e. variables are assumed to be independent
            let ratio_stddev = match (
                result.measurements.stddev(),
                reference.measurements.stddev(),
            ) {
                (Some(result_stddev), Some(reference_stddev)) => Some(
                    ratio
                        * ((result_stddev / result.mean_wall_clock_time())
                            .powi(uom::typenum::P2::new())
                            + (reference_stddev / reference.mean_wall_clock_time())
                                .powi(uom::typenum::P2::new()))
                        .sqrt(),
                ),
                _ => None,
            };

            BenchmarkResultWithRelativeSpeed {
                result,
                relative_speed: ratio.get::<quantity::ratio>(),
                relative_speed_stddev: ratio_stddev.map(|r| r.get::<quantity::ratio>()),
                is_reference,
                relative_ordering,
            }
        })
        .collect();

    match sort_order {
        SortOrder::Command => {}
        SortOrder::MeanTime => {
            results.sort_unstable_by(|r1, r2| compare_mean_time(r1.result, r2.result));
        }
    }

    results
}

pub fn compute_with_check_from_reference<'a>(
    results: &'a [BenchmarkResult],
    reference: &'a BenchmarkResult,
    sort_order: SortOrder,
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
    if fastest_of(results).mean() == 0.0 || reference.mean() == 0.0 {
        return None;
    }

    Some(compute_relative_speeds(results, reference, sort_order))
}

pub fn compute_with_check(
    results: &[BenchmarkResult],
    sort_order: SortOrder,
) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'_>>> {
    let fastest = fastest_of(results);

    if fastest.mean() == 0.0 {
        return None;
    }

    Some(compute_relative_speeds(results, fastest, sort_order))
}

/// Compute relative speeds against the given reference, or the fastest result.
pub fn compute<'a>(
    results: &'a [BenchmarkResult],
    sort_order: SortOrder,
    reference: Option<&'a BenchmarkResult>,
) -> Vec<BenchmarkResultWithRelativeSpeed<'a>> {
    let reference = reference.unwrap_or_else(|| fastest_of(results));
    compute_relative_speeds(results, reference, sort_order)
}

#[cfg(test)]
fn create_result(name: &str, mean: f64) -> BenchmarkResult {
    use std::collections::BTreeMap;

    use crate::benchmark::benchmark_result::BenchmarkRun;

    BenchmarkResult {
        command: name.into(),
        runs: vec![BenchmarkRun {
            wall_clock_time: mean,
            user_time: mean,
            system_time: 0.,
            memory_usage_byte: 1024,
            exit_code: Some(0),
        }],
        parameters: BTreeMap::new(),
    }
}

#[test]
fn test_compute_relative_speed() {
    use approx::assert_relative_eq;

    let results = vec![
        create_result("cmd1", 3.0),
        create_result("cmd2", 2.0),
        create_result("cmd3", 5.0),
    ];

    let annotated_results = compute_with_check(&results, SortOrder::Command).unwrap();

    assert_relative_eq!(1.5, annotated_results[0].relative_speed);
    assert_relative_eq!(1.0, annotated_results[1].relative_speed);
    assert_relative_eq!(2.5, annotated_results[2].relative_speed);
}

#[test]
fn test_compute_relative_speed_with_reference() {
    use approx::assert_relative_eq;

    let results = vec![create_result("cmd2", 2.0), create_result("cmd3", 5.0)];
    let reference = create_result("cmd2", 4.0);

    let annotated_results =
        compute_with_check_from_reference(&results, &reference, SortOrder::Command).unwrap();

    assert_relative_eq!(2.0, annotated_results[0].relative_speed);
    assert_relative_eq!(1.25, annotated_results[1].relative_speed);
}

#[test]
fn test_compute_relative_speed_for_zero_times() {
    let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)];

    let annotated_results = compute_with_check(&results, SortOrder::Command);

    assert!(annotated_results.is_none());
}

#[test]
fn reference_identity_distinguishes_equal_results() {
    let results = vec![create_result("same", 1.0), create_result("same", 1.0)];
    let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command);

    assert!(entries[0].is_reference);
    assert!(!entries[1].is_reference);
}

#[test]
fn reference_ratios_handle_zero_times() {
    let mut results = vec![create_result("reference", 2.0), create_result("zero", 0.0)];
    for result in &mut results {
        let measurement = result.measurements.measurements[0];
        result.measurements.measurements.push(measurement);
    }
    let entries = compute_relative_speeds(&results, &results[0], SortOrder::Command);
    assert_eq!(entries[1].relative_speed, f64::INFINITY);

    let entries = compute_relative_speeds(&results, &results[1], SortOrder::Command);
    assert_eq!(entries[0].relative_speed, f64::INFINITY);
    assert_eq!(entries[0].relative_speed_stddev, None);
    assert_eq!(entries[1].relative_speed, 1.0);
}