Support matrices of independent parameter lists

This patch permits the `-L`/`--parameter-list` argument to appear
multiple times. Each additional parameter increases the dimensionality
of the parameter matrix. One benchmark will be run for each combination
of parameters (i.e., the benchmark space is the Cartesian product of the
parameter spaces).

For now, `--parameter-list` and `--parameter-scan` are still mutually
exclusive. If desired, a follow-up change could similarly permit
multiple occurrences of `--parameter-scan` and also permit use of both
listed and scanned parameters together. (After all, `--parameter-scan`
can be thought of as syntactic sugar for a parameter list.)

This implementation is a little profligate with memory usage for
`BenchmarkResult`s: each result contains a separate copy of all
parameter names (as `String`s). This could be done away with by
threading lifetimes through the `BenchmarkResult`s, or by ref-counting
the names.

Because `BenchmarkResult`s now have maps with arbitrary key sets, we can
no longer use `serde` to serialize them to CSV. But a CSV writer is easy
to write by hand, so we just do that.

Fixes #253.

Test Plan:
Some unit tests included. As an integration test, try:

```
cargo run --release -- --runs 10 \
    -L foo foo1,foo2 -L bar bar9,bar8 \
    'echo {foo} {bar}' \
    'printf "%s\n" {foo} {bar}' \
    --export-csv /tmp/out
```

with all the export formats.

wchargin-branch: param-list-matrix
wchargin-source: 11051ab222c9de0fbc6ac6a080fbca48e05996c2
This commit is contained in:
William Chargin authored and David Peter committed 2020-10-13 14:00:46 +02:00
1 parent 5a80a23bdc
commit f6464a2943
9 files changed
+286 -78

No files matched your search

+100 -13
View File
@@ -1,4 +1,5 @@
use std::cmp;
use std::collections::BTreeMap;
use std::env;
use std::io;
@@ -207,23 +208,60 @@ fn build_commands<'a>(matches: &'a ArgMatches<'_>) -> Vec<Command<'a>> {
Ok(commands) => commands,
Err(e) => error(&e.to_string()),
}
} else if let Some(mut args) = matches.values_of("parameter-list") {
let param_name = args.next().unwrap();
let param_list_str = args.next().unwrap();
let param_list = tokenize(param_list_str);
} else if let Some(args) = matches.values_of("parameter-list") {
let args: Vec<_> = args.collect();
let param_names_and_values: Vec<(&str, Vec<String>)> = args
.chunks_exact(2)
.map(|pair| {
let name = pair[0];
let list_str = pair[1];
(name, tokenize(list_str))
})
.collect();
{
let dupes = find_dupes(param_names_and_values.iter().map(|(name, _)| *name));
if !dupes.is_empty() {
error(&format!("duplicate parameter names: {}", &dupes.join(", ")))
}
}
let command_list = command_strings.collect::<Vec<&str>>();
let mut commands = Vec::with_capacity(param_list.len() * command_list.len());
let dimensions: Vec<usize> = std::iter::once(command_list.len())
.chain(
param_names_and_values
.iter()
.map(|(_, values)| values.len()),
)
.collect();
let param_space_size = dimensions.iter().product();
if param_space_size == 0 {
return Vec::new();
}
for value in param_list {
for cmd in &command_list {
commands.push(Command::new_parametrized(
cmd,
param_name,
ParameterValue::Text(value.clone()),
));
let mut commands = Vec::with_capacity(param_space_size);
let mut index = vec![0usize; dimensions.len()];
'outer: loop {
let (command_index, params_indices) = index.split_first().unwrap();
let parameters = param_names_and_values
.iter()
.zip(params_indices)
.map(|((name, values), i)| (*name, ParameterValue::Text(values[*i].clone())))
.collect();
commands.push(Command::new_parametrized(
command_list[*command_index],
parameters,
));
// Increment index, exiting loop on overflow.
for (i, n) in index.iter_mut().zip(dimensions.iter()) {
*i += 1;
if *i < *n {
continue 'outer;
} else {
*i = 0;
}
}
break 'outer;
}
commands
@@ -231,3 +269,52 @@ fn build_commands<'a>(matches: &'a ArgMatches<'_>) -> Vec<Command<'a>> {
command_strings.map(Command::new).collect()
}
}
/// Finds all the strings that appear multiple times in the input iterator, returning them in
/// sorted order. If no string appears more than once, the result is an empty vector.
fn find_dupes<'a, I: IntoIterator<Item = &'a str>>(i: I) -> Vec<&'a str> {
let mut counts = BTreeMap::<&'a str, usize>::new();
for s in i {
*counts.entry(s).or_default() += 1;
}
counts
.into_iter()
.filter_map(|(k, n)| if n > 1 { Some(k) } else { None })
.collect()
}
#[test]
fn test_build_commands_cross_product() {
let matches = get_arg_matches(vec![
"hyperfine",
"-L",
"foo",
"a,b",
"-L",
"bar",
"z,y",
"echo {foo} {bar}",
"printf '%s\n' {foo} {bar}",
]);
let result = build_commands(&matches);
// Iteration order: command list first, then parameters in listed order (here, "foo" before
// "bar", which is distinct from their sorted order), with parameter values in listed order.
let pv = |s: &str| ParameterValue::Text(s.to_string());
let cmd = |cmd: usize, foo: &str, bar: &str| {
let expression = ["echo {foo} {bar}", "printf '%s\n' {foo} {bar}"][cmd];
let params = vec![("foo", pv(foo)), ("bar", pv(bar))];
Command::new_parametrized(expression, params)
};
let expected = vec![
cmd(0, "a", "z"),
cmd(1, "a", "z"),
cmd(0, "b", "z"),
cmd(1, "b", "z"),
cmd(0, "a", "y"),
cmd(1, "a", "y"),
cmd(0, "b", "y"),
cmd(1, "b", "y"),
];
assert_eq!(result, expected);
}