diff --git a/Cargo.lock b/Cargo.lock index 02c106e..7864aba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -99,15 +99,27 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.5.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "bitflags" -version = "2.13.2" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] [[package]] name = "borsh" @@ -214,7 +226,7 @@ dependencies = [ name = "clap_lex" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "colorchoice" @@ -328,6 +340,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "getrandom" version = "0.2.17" @@ -387,12 +405,11 @@ dependencies = [ "nix", "once_cell", "predicates", - "rand 0.10.3", + "rand", "rust_decimal", "serde", "serde_json", "shell-words", - "statistical", "tempfile", "thiserror", "uom", @@ -505,73 +522,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" -[[package]] -name = "num" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" -dependencies = [ - "autocfg 1.4.0", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-complex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" -dependencies = [ - "autocfg 1.4.0", - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg 1.4.0", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" -dependencies = [ - "autocfg 1.4.0", - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -685,34 +635,13 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.8" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.5", -] - -[[package]] -name = "rand" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" -dependencies = [ - "chacha20", - "getrandom 0.4.3", - "rand_core 0.10.1", + "rand_chacha", + "rand_core", ] [[package]] @@ -722,17 +651,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", + "rand_core", ] [[package]] @@ -744,21 +663,6 @@ dependencies = [ "getrandom 0.2.17", ] -[[package]] -name = "rand_core" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] - -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - [[package]] name = "regex" version = "1.13.1" @@ -798,8 +702,8 @@ dependencies = [ "borsh", "bytes", "num-traits", - "rand 0.8.8", - "rand 0.9.5", + "rand", + "rkyv", "serde", "serde_json", "wasm-bindgen", @@ -885,15 +789,6 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" -[[package]] -name = "statistical" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" -dependencies = [ - "vte", -] - [[package]] name = "strsim" version = "0.11.1" @@ -1066,19 +961,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasip2" -version = "1.0.4+wasi-0.2.12" +name = "windows-sys" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.129" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ "cfg-if", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index 9154fc5..d779d03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ windows_process_extensions_main_thread_handle = [] colored = "3.1" # Newer versions regress ETA estimates: https://github.com/console-rs/indicatif/issues/580 indicatif = "=0.17.4" -csv = "1.4" +csv = "1.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" rust_decimal = "1.43" diff --git a/src/quantity/statistics.rs b/src/quantity/statistics.rs index 3245e23..311eca5 100644 --- a/src/quantity/statistics.rs +++ b/src/quantity/statistics.rs @@ -33,14 +33,6 @@ impl UnsafeRawValue for Information { } } -/// A max function that assumes no NaNs and at least one element -pub fn max(values: impl IntoIterator) -> Q { - values - .into_iter() - .max_by(|a, b| a.partial_cmp(b).expect("No NaN values")) - .expect("'max' requires at least one element") -} - /// A min function that assumes no NaNs and at least one element pub fn min(values: impl IntoIterator) -> Q { values @@ -49,6 +41,14 @@ pub fn min(values: impl IntoIterator) -> Q { .expect("'min' requires at least one element") } +/// A max function that assumes no NaNs and at least one element +pub fn max(values: impl IntoIterator) -> Q { + values + .into_iter() + .max_by(|a, b| a.partial_cmp(b).expect("No NaN values")) + .expect("'max' requires at least one element") +} + /// A mean function that assumes at least one element pub fn mean(values: impl IntoIterator) -> Q where @@ -85,71 +85,140 @@ where } } +fn standard_deviation_f64(values: &[f64]) -> f64 { + let mean_value = mean(values.iter().copied()); + + let mut squared_deviations = 0.; + let mut count = 0; + for value in values { + let deviation = value - mean_value; + squared_deviations += deviation * deviation; + count += 1; + } + + (1. / ((count - 1) as f64) * squared_deviations).sqrt() +} + pub fn standard_deviation(values: &[Q]) -> Q { let values: Vec<_> = values.iter().map(|q| q.unsafe_raw_value()).collect(); - let result_value = { - let mean_value = statistical::mean(&values); - statistical::standard_deviation(&values, Some(mean_value)) - }; - Q::unsafe_from_raw_value(result_value) + let result = standard_deviation_f64(&values); + Q::unsafe_from_raw_value(result) +} + +/// Compute modifized Z-scores for a given sample. A (unmodified) Z-score is defined by +/// `(x_i - x_mean)/x_stddev` whereas the modified Z-score is defined by `(x_i - x_median)/MAD` +/// where MAD is the median absolute deviation. +/// +/// References: +/// - +pub fn modified_zscores_f64(xs: &[f64]) -> Vec { + assert!(!xs.is_empty()); + + // Compute sample median: + let x_median = median(xs.iter().copied()); + + // Compute the absolute deviations from the median: + let deviations: Vec = xs.iter().map(|x| (x - x_median).abs()).collect(); + + // Compute median absolute deviation: + let mad = median(deviations.iter().copied()); + + // Handle MAD == 0 case + let mad = if mad > 0.0 { mad } else { f64::EPSILON }; + + // Compute modified Z-scores (x_i - x_median) / MAD + xs.iter().map(|&x| (x - x_median) / mad).collect() } pub fn modified_zscores(values: &[Q]) -> Vec { let values: Vec<_> = values.iter().map(|q| q.unsafe_raw_value()).collect(); - crate::outlier_detection::modified_zscores(&values) + modified_zscores_f64(&values) } -#[test] -fn test_max() { - assert_eq!(1.0, max([1.0])); - assert_eq!(-1.0, max([-1.0])); - assert_eq!(-1.0, max([-2.0, -1.0])); - assert_eq!(1.0, max([-1.0, 1.0])); - assert_eq!(1.0, max([-1.0, 1.0, 0.0])); -} - -#[test] -fn test_mean() { - use crate::quantity::{FormatQuantity, TimeUnit}; - use uom::si::time::millisecond; - - let values = vec![ - Time::new::(123.4), - Time::new::(234.5), - ]; - let result = mean(values.into_iter()); - assert_eq!(result.format(TimeUnit::MilliSecond), "178.9 ms"); -} - -#[test] -fn statistics() { - use crate::quantity::{FormatQuantity, InformationUnit, TimeUnit}; - - let values = vec![ - Time::new::(1.0), - Time::new::(2.0), - Time::new::(3.0), - ]; - - assert_eq!( - mean(values.iter().copied()).format(TimeUnit::Second), - "2.000 s" - ); - assert_eq!( - median(values.iter().copied()).format(TimeUnit::Second), - "2.000 s" - ); - assert_eq!(min(&values).format(TimeUnit::Second), "1.000 s"); - assert_eq!(max(&values).format(TimeUnit::Second), "3.000 s"); - assert_eq!( - standard_deviation(&values).format(TimeUnit::Second), - "1.000 s" - ); - - let values = vec![ - Information::new::(1.0), - Information::new::(2.0), - Information::new::(3.0), - ]; - mean(values.iter().copied()).format(InformationUnit::Byte); +#[cfg(test)] +mod tests { + use super::*; + use approx::assert_relative_eq; + use uom::si::information::kibibyte; + use uom::si::time::{microsecond, millisecond}; + + #[test] + fn test_min() { + assert_eq!(1.0, min([1.0])); + assert_eq!(-1.0, min([-1.0])); + assert_eq!(-2.0, min([-2.0, -1.0])); + assert_eq!(-1.0, min([-1.0, 1.0])); + assert_eq!(-1.0, min([1.0, -1.0, 0.0])); + + let values = vec![ + Information::new::(1.0), + Information::new::(2.0), + Information::new::(3.0), + ]; + assert_eq!(min(&values).get::(), 2.0); + } + + #[test] + fn test_max() { + assert_eq!(1.0, max([1.0])); + assert_eq!(-1.0, max([-1.0])); + assert_eq!(-1.0, max([-2.0, -1.0])); + assert_eq!(1.0, max([-1.0, 1.0])); + assert_eq!(1.0, max([-1.0, 1.0, 0.0])); + + let values = vec![ + Information::new::(1.0), + Information::new::(2.0), + Information::new::(3.0), + ]; + assert_eq!(max(&values).get::(), 2.0); + } + + #[test] + fn test_mean() { + assert_eq!(1.0, mean([1.0])); + assert_relative_eq!(2.0, mean([1.0, 3.0])); + + let values = [ + Time::new::(100.0), + Time::new::(200.0), + Time::new::(600_000.0), + ]; + let result = mean(values); + assert_relative_eq!(result.get::(), 300.0); + } + + #[test] + fn test_median() { + assert_eq!(1.0, median([1.0])); + assert_relative_eq!(2.0, median([1.0, 3.0])); + + let values = [ + Time::new::(100.0), + Time::new::(200.0), + Time::new::(600_000.0), + ]; + let result = median(values); + assert_relative_eq!(result.get::(), 200.0); + + let values = [ + Time::new::(100.0), + Time::new::(200.0), + Time::new::(300_000.0), + Time::new::(600_000.0), + ]; + let result = median(values); + assert_relative_eq!(result.get::(), 250.0); + } + + #[test] + fn test_standard_deviation() { + let values = [ + Time::new::(100.0), + Time::new::(200.0), + Time::new::(300_000.0), + ]; + let result = standard_deviation(&values); + assert_relative_eq!(result.get::(), 100.0); + } }