Move tokenize to new file

src/main.rs

@@ -21,6 +21,7 @@ pub mod parameter_range;
 pub mod progress_bar;
 pub mod shell;
 pub mod timer;
+pub mod tokenize;
 pub mod types;
 pub mod units;
 pub mod warnings;
@@ -30,9 +31,10 @@ use benchmark::{mean_shell_spawning_time, run_benchmark};
 use command::Command;
 use error::OptionsError;
 use export::{ExportManager, ExportType};
-use internal::{tokenize, write_benchmark_comparison};
+use internal::write_benchmark_comparison;
 use options::{CmdFailureAction, HyperfineOptions, OutputStyleOption};
 use parameter_range::get_parameterized_commands;
+use tokenize::tokenize;
 use types::ParameterValue;
 use units::Unit;