From 3fcc6a187bc8da7622ae11670512f76c1e253c98 Mon Sep 17 00:00:00 2001 From: Jordi Chauzi Date: Sun, 28 Feb 2021 10:22:21 +0100 Subject: [PATCH] Export exit codes to JSON output --- CHANGELOG.md | 1 + src/hyperfine/benchmark.rs | 36 +++++++++++++++++++++++++++----- src/hyperfine/export/asciidoc.rs | 4 ++++ src/hyperfine/export/csv.rs | 4 +++- src/hyperfine/export/markdown.rs | 8 +++++++ src/hyperfine/internal.rs | 1 + src/hyperfine/types.rs | 5 +++++ 7 files changed, 53 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0599f1..0d8354a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # unreleased - Add command exit code to output if it fails, see #342 (@KaindlJulian) +- Export command exit code to JSON output, see #371 (@JordiChauzi) ## Features diff --git a/src/hyperfine/benchmark.rs b/src/hyperfine/benchmark.rs index a066366..9eb8a49 100644 --- a/src/hyperfine/benchmark.rs +++ b/src/hyperfine/benchmark.rs @@ -1,6 +1,6 @@ use std::cmp; use std::io; -use std::process::Stdio; +use std::process::{ExitStatus, Stdio}; use colored::*; use statistical::{mean, median, standard_deviation}; @@ -46,7 +46,7 @@ pub fn time_shell_command( show_output: bool, failure_action: CmdFailureAction, shell_spawning_time: Option, -) -> io::Result<(TimingResult, bool)> { +) -> io::Result<(TimingResult, ExitStatus)> { let (stdout, stderr) = if show_output { (Stdio::inherit(), Stdio::inherit()) } else { @@ -88,7 +88,7 @@ pub fn time_shell_command( time_user, time_system, }, - result.status.success(), + result.status, )) } @@ -201,6 +201,26 @@ fn run_cleanup_command( run_intermediate_command(shell, command, show_output, error_output) } +#[cfg(unix)] +fn extract_exit_code(status: ExitStatus) -> i32 { + use std::os::unix::process::ExitStatusExt; + + /* From the ExitStatus::code documentation: + "On Unix, this will return None if the process was terminated by a signal." + In that case, ExitStatusExt::signal should never return None. + */ + status.code().unwrap_or_else(|| status.signal().unwrap()) +} + +#[cfg(not(unix))] +fn extract_exit_code(status: ExitStatus) -> i32 { + /* From the ExitStatus::code documentation: + "On Unix, this will return None if the process was terminated by a signal." + On the other configurations, ExitStatus::code should never return None. + */ + status.code().unwrap() +} + /// Run the benchmark for a single shell command pub fn run_benchmark( num: usize, @@ -228,6 +248,7 @@ pub fn run_benchmark( let mut times_real: Vec = vec![]; let mut times_user: Vec = vec![]; let mut times_system: Vec = vec![]; + let mut exit_codes: Vec = vec![]; let mut all_succeeded = true; // Run init command @@ -280,13 +301,14 @@ pub fn run_benchmark( let prepare_res = run_preparation_command(&options.shell, &prepare_cmd, options.show_output)?; // Initial timing run - let (res, success) = time_shell_command( + let (res, status) = time_shell_command( &options.shell, cmd, options.show_output, options.failure_action, Some(shell_spawning_time), )?; + let success = status.success(); // Determine number of benchmark runs let runs_in_min_time = (options.min_time_sec @@ -310,6 +332,7 @@ pub fn run_benchmark( times_real.push(res.time_real); times_user.push(res.time_user); times_system.push(res.time_system); + exit_codes.push(extract_exit_code(status)); all_succeeded = all_succeeded && success; @@ -328,17 +351,19 @@ pub fn run_benchmark( progress_bar.as_ref().map(|bar| bar.set_message(msg.to_owned())); - let (res, success) = time_shell_command( + let (res, status) = time_shell_command( &options.shell, cmd, options.show_output, options.failure_action, Some(shell_spawning_time), )?; + let success = status.success(); times_real.push(res.time_real); times_user.push(res.time_user); times_system.push(res.time_system); + exit_codes.push(extract_exit_code(status)); all_succeeded = all_succeeded && success; @@ -438,6 +463,7 @@ pub fn run_benchmark( t_min, t_max, times_real, + exit_codes, cmd.get_parameters() .iter() .map(|(name, value)| ((*name).to_string(), value.to_string())) diff --git a/src/hyperfine/export/asciidoc.rs b/src/hyperfine/export/asciidoc.rs index cae6075..0a7bd06 100644 --- a/src/hyperfine/export/asciidoc.rs +++ b/src/hyperfine/export/asciidoc.rs @@ -99,6 +99,7 @@ fn test_asciidoc_table_row() { 0.10745223440000001, 0.10697327940000001, ], + vec![0, 0, 0], // exit codes BTreeMap::new(), // param ); @@ -151,6 +152,7 @@ fn test_asciidoc_table_row_command_escape() { 0.10745223440000001, 0.10697327940000001, ], + vec![0, 0, 0], // exit codes BTreeMap::new(), // param ); let exps = format!( @@ -185,6 +187,7 @@ fn test_asciidoc() { 5.0, 6.0, vec![7.0, 8.0, 9.0], + vec![0, 0, 0], { let mut params = BTreeMap::new(); params.insert("foo".into(), "1".into()); @@ -202,6 +205,7 @@ fn test_asciidoc() { 15.0, 16.0, vec![17.0, 18.0, 19.0], + vec![0, 0, 0], { let mut params = BTreeMap::new(); params.insert("foo".into(), "1".into()); diff --git a/src/hyperfine/export/csv.rs b/src/hyperfine/export/csv.rs index 28593a4..ef2c382 100644 --- a/src/hyperfine/export/csv.rs +++ b/src/hyperfine/export/csv.rs @@ -17,7 +17,7 @@ impl Exporter for CsvExporter { { let mut headers: Vec> = [ - // The list of times cannot be exported to the CSV file - omit it. + // The list of times and exit codes cannot be exported to the CSV file - omit them. "command", "mean", "stddev", "median", "user", "system", "min", "max", ] .iter() @@ -68,6 +68,7 @@ fn test_csv() { 5.0, 6.0, vec![7.0, 8.0, 9.0], + vec![0, 0, 0], { let mut params = BTreeMap::new(); params.insert("foo".into(), "one".into()); @@ -85,6 +86,7 @@ fn test_csv() { 15.0, 16.5, vec![17.0, 18.0, 19.0], + vec![0, 0, 0], { let mut params = BTreeMap::new(); params.insert("foo".into(), "one".into()); diff --git a/src/hyperfine/export/markdown.rs b/src/hyperfine/export/markdown.rs index 40b3a6a..a21a815 100644 --- a/src/hyperfine/export/markdown.rs +++ b/src/hyperfine/export/markdown.rs @@ -102,6 +102,7 @@ fn test_markdown_format_ms() { 0.1023, // min 0.1080, // max vec![0.1, 0.1, 0.1], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); @@ -115,6 +116,7 @@ fn test_markdown_format_ms() { 2.0020, // min 2.0080, // max vec![2.0, 2.0, 2.0], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); @@ -150,6 +152,7 @@ fn test_markdown_format_s() { 2.0020, // min 2.0080, // max vec![2.0, 2.0, 2.0], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); @@ -163,6 +166,7 @@ fn test_markdown_format_s() { 0.1023, // min 0.1080, // max vec![0.1, 0.1, 0.1], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); @@ -197,6 +201,7 @@ fn test_markdown_format_time_unit_s() { 0.1023, // min 0.1080, // max vec![0.1, 0.1, 0.1], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); @@ -210,6 +215,7 @@ fn test_markdown_format_time_unit_s() { 2.0020, // min 2.0080, // max vec![2.0, 2.0, 2.0], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); @@ -250,6 +256,7 @@ fn test_markdown_format_time_unit_ms() { 2.0020, // min 2.0080, // max vec![2.0, 2.0, 2.0], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); @@ -263,6 +270,7 @@ fn test_markdown_format_time_unit_ms() { 0.1023, // min 0.1080, // max vec![0.1, 0.1, 0.1], // times + vec![0, 0, 0], // exit codes BTreeMap::new(), // parameter )); diff --git a/src/hyperfine/internal.rs b/src/hyperfine/internal.rs index 5f2b7bf..5e1aaf0 100644 --- a/src/hyperfine/internal.rs +++ b/src/hyperfine/internal.rs @@ -157,6 +157,7 @@ fn create_result(name: &str, mean: Scalar) -> BenchmarkResult { min: mean, max: mean, times: None, + exit_codes: Vec::new(), parameters: BTreeMap::new(), } } diff --git a/src/hyperfine/types.rs b/src/hyperfine/types.rs index 98e5ea0..dd792e3 100644 --- a/src/hyperfine/types.rs +++ b/src/hyperfine/types.rs @@ -268,6 +268,9 @@ pub struct BenchmarkResult { #[serde(skip_serializing_if = "Option::is_none")] pub times: Option>, + /// All run exit codes + pub exit_codes: Vec, + /// Any parameter values used #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub parameters: BTreeMap, @@ -285,6 +288,7 @@ impl BenchmarkResult { min: Second, max: Second, times: Vec, + exit_codes: Vec, parameters: BTreeMap, ) -> Self { BenchmarkResult { @@ -297,6 +301,7 @@ impl BenchmarkResult { min, max, times: Some(times), + exit_codes, parameters, } }