Move tokenize to new file

src/benchmark.rs

@@ -9,9 +9,9 @@ use crate::benchmark_result::BenchmarkResult;
 use crate::command::Command;
 use crate::format::{format_duration, format_duration_unit};
 use crate::internal::{max, min, MIN_EXECUTION_TIME};
-use crate::progress_bar::get_progress_bar;
 use crate::options::{CmdFailureAction, HyperfineOptions, OutputStyleOption};
 use crate::outlier_detection::{modified_zscores, OUTLIER_THRESHOLD};
+use crate::progress_bar::get_progress_bar;
 use crate::shell::execute_and_time;
 use crate::timer::wallclocktimer::WallClockTimer;
 use crate::timer::{TimerStart, TimerStop};

src/internal.rs

@@ -164,64 +164,3 @@ fn test_compute_relative_speed_for_zero_times() {
 
     assert!(annotated_results.is_none());
 }
-
-pub fn tokenize(values: &str) -> Vec<String> {
-    let mut tokens = vec![];
-    let mut buf = String::new();
-
-    let mut iter = values.chars();
-    while let Some(c) = iter.next() {
-        match c {
-            '\\' => match iter.next() {
-                Some(c2 @ ',') | Some(c2 @ '\\') => {
-                    buf.push(c2);
-                }
-                Some(c2) => {
-                    buf.push('\\');
-                    buf.push(c2);
-                }
-                None => buf.push('\\'),
-            },
-            ',' => {
-                tokens.push(buf);
-                buf = String::new();
-            }
-            _ => {
-                buf.push(c);
-            }
-        };
-    }
-
-    tokens.push(buf);
-
-    tokens
-}
-
-#[test]
-fn test_tokenize_single_value() {
-    assert_eq!(tokenize(r""), vec![""]);
-    assert_eq!(tokenize(r"foo"), vec!["foo"]);
-    assert_eq!(tokenize(r" "), vec![" "]);
-    assert_eq!(tokenize(r"hello\, world!"), vec!["hello, world!"]);
-    assert_eq!(tokenize(r"\,"), vec![","]);
-    assert_eq!(tokenize(r"\,\,\,"), vec![",,,"]);
-    assert_eq!(tokenize(r"\n"), vec![r"\n"]);
-    assert_eq!(tokenize(r"\\"), vec![r"\"]);
-    assert_eq!(tokenize(r"\\\,"), vec![r"\,"]);
-}
-
-#[test]
-fn test_tokenize_multiple_values() {
-    assert_eq!(tokenize(r"foo,bar,baz"), vec!["foo", "bar", "baz"]);
-    assert_eq!(tokenize(r"hello world,foo"), vec!["hello world", "foo"]);
-
-    assert_eq!(tokenize(r"hello\,world!,baz"), vec!["hello,world!", "baz"]);
-}
-
-#[test]
-fn test_tokenize_empty_values() {
-    assert_eq!(tokenize(r"foo,,bar"), vec!["foo", "", "bar"]);
-    assert_eq!(tokenize(r",bar"), vec!["", "bar"]);
-    assert_eq!(tokenize(r"bar,"), vec!["bar", ""]);
-    assert_eq!(tokenize(r",,"), vec!["", "", ""]);
-}

src/main.rs

@@ -21,6 +21,7 @@ pub mod parameter_range;
 pub mod progress_bar;
 pub mod shell;
 pub mod timer;
+pub mod tokenize;
 pub mod types;
 pub mod units;
 pub mod warnings;
@@ -30,9 +31,10 @@ use benchmark::{mean_shell_spawning_time, run_benchmark};
 use command::Command;
 use error::OptionsError;
 use export::{ExportManager, ExportType};
-use internal::{tokenize, write_benchmark_comparison};
+use internal::write_benchmark_comparison;
 use options::{CmdFailureAction, HyperfineOptions, OutputStyleOption};
 use parameter_range::get_parameterized_commands;
+use tokenize::tokenize;
 use types::ParameterValue;
 use units::Unit;
 

src/tokenize.rs

@@ -0,0 +1,60 @@
+pub fn tokenize(values: &str) -> Vec<String> {
+    let mut tokens = vec![];
+    let mut buf = String::new();
+
+    let mut iter = values.chars();
+    while let Some(c) = iter.next() {
+        match c {
+            '\\' => match iter.next() {
+                Some(c2 @ ',') | Some(c2 @ '\\') => {
+                    buf.push(c2);
+                }
+                Some(c2) => {
+                    buf.push('\\');
+                    buf.push(c2);
+                }
+                None => buf.push('\\'),
+            },
+            ',' => {
+                tokens.push(buf);
+                buf = String::new();
+            }
+            _ => {
+                buf.push(c);
+            }
+        };
+    }
+
+    tokens.push(buf);
+
+    tokens
+}
+
+#[test]
+fn test_tokenize_single_value() {
+    assert_eq!(tokenize(r""), vec![""]);
+    assert_eq!(tokenize(r"foo"), vec!["foo"]);
+    assert_eq!(tokenize(r" "), vec![" "]);
+    assert_eq!(tokenize(r"hello\, world!"), vec!["hello, world!"]);
+    assert_eq!(tokenize(r"\,"), vec![","]);
+    assert_eq!(tokenize(r"\,\,\,"), vec![",,,"]);
+    assert_eq!(tokenize(r"\n"), vec![r"\n"]);
+    assert_eq!(tokenize(r"\\"), vec![r"\"]);
+    assert_eq!(tokenize(r"\\\,"), vec![r"\,"]);
+}
+
+#[test]
+fn test_tokenize_multiple_values() {
+    assert_eq!(tokenize(r"foo,bar,baz"), vec!["foo", "bar", "baz"]);
+    assert_eq!(tokenize(r"hello world,foo"), vec!["hello world", "foo"]);
+
+    assert_eq!(tokenize(r"hello\,world!,baz"), vec!["hello,world!", "baz"]);
+}
+
+#[test]
+fn test_tokenize_empty_values() {
+    assert_eq!(tokenize(r"foo,,bar"), vec!["foo", "", "bar"]);
+    assert_eq!(tokenize(r",bar"), vec!["", "bar"]);
+    assert_eq!(tokenize(r"bar,"), vec!["bar", ""]);
+    assert_eq!(tokenize(r",,"), vec!["", "", ""]);
+}