mirror of
https://github.com/sharkdp/hyperfine.git
synced 2026-11-02 17:11:28 +00:00
Support matrices of independent parameter lists
This patch permits the `-L`/`--parameter-list` argument to appear multiple times. Each additional parameter increases the dimensionality of the parameter matrix. One benchmark will be run for each combination of parameters (i.e., the benchmark space is the Cartesian product of the parameter spaces). For now, `--parameter-list` and `--parameter-scan` are still mutually exclusive. If desired, a follow-up change could similarly permit multiple occurrences of `--parameter-scan` and also permit use of both listed and scanned parameters together. (After all, `--parameter-scan` can be thought of as syntactic sugar for a parameter list.) This implementation is a little profligate with memory usage for `BenchmarkResult`s: each result contains a separate copy of all parameter names (as `String`s). This could be done away with by threading lifetimes through the `BenchmarkResult`s, or by ref-counting the names. Because `BenchmarkResult`s now have maps with arbitrary key sets, we can no longer use `serde` to serialize them to CSV. But a CSV writer is easy to write by hand, so we just do that. Fixes #253. Test Plan: Some unit tests included. As an integration test, try: ``` cargo run --release -- --runs 10 \ -L foo foo1,foo2 -L bar bar9,bar8 \ 'echo {foo} {bar}' \ 'printf "%s\n" {foo} {bar}' \ --export-csv /tmp/out ``` with all the export formats. wchargin-branch: param-list-matrix wchargin-source: 11051ab222c9de0fbc6ac6a080fbca48e05996c2
This commit is contained in:
1 parent
5a80a23bdc
commit
f6464a2943
9 files changed
+286
-78
No files matched your search
+55
-22
@@ -1,6 +1,7 @@
|
||||
use rust_decimal::Decimal;
|
||||
/// This module contains common internal types.
|
||||
use serde::*;
|
||||
use std::collections::BTreeMap;
|
||||
use std::fmt;
|
||||
|
||||
use crate::hyperfine::units::{Second, Unit};
|
||||
@@ -11,7 +12,7 @@ pub const DEFAULT_SHELL: &str = "sh";
|
||||
#[cfg(windows)]
|
||||
pub const DEFAULT_SHELL: &str = "cmd.exe";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Copy, PartialEq)]
|
||||
#[derive(Debug, Clone, Serialize, Copy, PartialEq, Eq)]
|
||||
#[serde(untagged)]
|
||||
pub enum NumericType {
|
||||
Int(i32),
|
||||
@@ -39,7 +40,7 @@ impl Into<NumericType> for Decimal {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ParameterValue {
|
||||
Text(String),
|
||||
Numeric(NumericType),
|
||||
@@ -55,49 +56,79 @@ impl<'a> ToString for ParameterValue {
|
||||
}
|
||||
|
||||
/// A command that should be benchmarked.
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Command<'a> {
|
||||
/// The command that should be executed (without parameter substitution)
|
||||
expression: &'a str,
|
||||
|
||||
/// A possible parameter value.
|
||||
parameter: Option<(&'a str, ParameterValue)>,
|
||||
/// Zero or more parameter values.
|
||||
parameters: Vec<(&'a str, ParameterValue)>,
|
||||
}
|
||||
|
||||
impl<'a> Command<'a> {
|
||||
pub fn new(expression: &'a str) -> Command<'a> {
|
||||
Command {
|
||||
expression,
|
||||
parameter: None,
|
||||
parameters: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_parametrized(
|
||||
expression: &'a str,
|
||||
parameter: &'a str,
|
||||
value: ParameterValue,
|
||||
parameters: Vec<(&'a str, ParameterValue)>,
|
||||
) -> Command<'a> {
|
||||
Command {
|
||||
expression,
|
||||
parameter: Some((parameter, value)),
|
||||
parameters,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_shell_command(&self) -> String {
|
||||
match &self.parameter {
|
||||
Some((param_name, param_value)) => self.expression.replace(
|
||||
&format!("{{{param_name}}}", param_name = param_name),
|
||||
¶m_value.to_string(),
|
||||
),
|
||||
None => self.expression.into(),
|
||||
let mut result = String::new();
|
||||
let mut replacements = BTreeMap::<String, String>::new();
|
||||
for (param_name, param_value) in &self.parameters {
|
||||
replacements.insert(
|
||||
format!("{{{param_name}}}", param_name = param_name),
|
||||
param_value.to_string(),
|
||||
);
|
||||
}
|
||||
let mut remaining = self.expression;
|
||||
// Manually replace consecutive occurrences to avoid double-replacing: e.g.,
|
||||
//
|
||||
// hyperfine -L foo 'a,{bar}' -L bar 'baz,quux' 'echo {foo} {bar}'
|
||||
//
|
||||
// should not ever run 'echo baz baz'. See `test_get_shell_command_nonoverlapping`.
|
||||
'outer: while let Some(head) = remaining.chars().next() {
|
||||
for (k, v) in &replacements {
|
||||
if remaining.starts_with(k.as_str()) {
|
||||
result.push_str(&v);
|
||||
remaining = &remaining[k.len()..];
|
||||
continue 'outer;
|
||||
}
|
||||
}
|
||||
result.push(head);
|
||||
remaining = &remaining[head.len_utf8()..];
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub fn get_parameter(&self) -> &Option<(&'a str, ParameterValue)> {
|
||||
&self.parameter
|
||||
pub fn get_parameters(&self) -> &Vec<(&'a str, ParameterValue)> {
|
||||
&self.parameters
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_shell_command_nonoverlapping() {
|
||||
let cmd = Command::new_parametrized(
|
||||
"echo {foo} {bar}",
|
||||
vec![
|
||||
("foo", ParameterValue::Text("{bar} baz".into())),
|
||||
("bar", ParameterValue::Text("quux".into())),
|
||||
],
|
||||
);
|
||||
assert_eq!(cmd.get_shell_command(), "echo {bar} baz quux");
|
||||
}
|
||||
|
||||
impl<'a> fmt::Display for Command<'a> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.get_shell_command())
|
||||
@@ -199,6 +230,8 @@ impl Default for HyperfineOptions {
|
||||
}
|
||||
|
||||
/// Set of values that will be exported.
|
||||
// NOTE: `serde` is used for JSON serialization, but not for CSV serialization due to the
|
||||
// `parameters` map. Update `src/hyperfine/export/csv.rs` with new fields, as appropriate.
|
||||
#[derive(Debug, Default, Clone, Serialize, PartialEq)]
|
||||
pub struct BenchmarkResult {
|
||||
/// The command that was run
|
||||
@@ -229,9 +262,9 @@ pub struct BenchmarkResult {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub times: Option<Vec<Second>>,
|
||||
|
||||
/// Any parameter used
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub parameter: Option<String>,
|
||||
/// Any parameter values used
|
||||
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub parameters: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl BenchmarkResult {
|
||||
@@ -246,7 +279,7 @@ impl BenchmarkResult {
|
||||
min: Second,
|
||||
max: Second,
|
||||
times: Vec<Second>,
|
||||
parameter: Option<String>,
|
||||
parameters: BTreeMap<String, String>,
|
||||
) -> Self {
|
||||
BenchmarkResult {
|
||||
command,
|
||||
@@ -258,7 +291,7 @@ impl BenchmarkResult {
|
||||
min,
|
||||
max,
|
||||
times: Some(times),
|
||||
parameter,
|
||||
parameters,
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user