From 8bc384e963591c60cbef02f8d593415420f732aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20B=C3=BCrger?= Date: Wed, 3 Jul 2024 08:00:08 +0200 Subject: [PATCH] add crate feature "nightly" to enable features in rust nightly - enables code using cow_is_borrowed - enables code using new_uninit - use Vec::chunk_by (stable since rust 1.70) - use divan for benchmarks - add function to create empty vec --- Cargo.lock | 80 +++++++++++++++++++++++ Cargo.toml | 37 +++++++++++ benches/concurrency.rs | 48 +++++++------- benches/dataset.rs | 14 +++-- benches/index.rs | 14 +++-- benches/large.rs | 54 ++++++++-------- benches/native.rs | 50 ++++++++------- benches/norkyst.rs | 20 +++--- benches/read.rs | 50 ++++++++------- benches/serialize.rs | 126 +++++++++++++++++++------------------ benches/stream.rs | 38 +++++------ src/filters/shuffle.rs | 38 +++++------ src/idx/chunk.rs | 10 +-- src/idx/dataset/dataset.rs | 8 +-- src/idx/dataset/mod.rs | 20 +++--- src/idx/serde.rs | 1 + src/lib.rs | 11 +--- src/reader/dataset.rs | 32 +++++++--- src/reader/direct.rs | 2 +- tests/read_dims.rs | 3 - tests/read_norkyst.rs | 2 - tests/read_strings.rs | 3 - tests/read_svim.rs | 3 - 23 files changed, 398 insertions(+), 266 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c9045d..05d3053 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "anstyle" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" + [[package]] name = "anyhow" version = "1.0.86" @@ -155,6 +161,32 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clap" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" +dependencies = [ + "anstyle", + "clap_lex", + "terminal_size", +] + +[[package]] +name = "clap_lex" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" + [[package]] name = "cmake" version = "0.1.50" @@ -164,6 +196,12 @@ dependencies = [ "cc", ] +[[package]] +name = "condtype" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" + [[package]] name = "core-foundation" version = "0.9.4" @@ -229,6 +267,31 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "divan" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d567df2c9c2870a43f3f2bd65aaeb18dbce1c18f217c3e564b4fbaeb3ee56c" +dependencies = [ + "cfg-if", + "clap", + "condtype", + "divan-macros", + "libc", + "regex-lite", +] + +[[package]] +name = "divan-macros" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27540baf49be0d484d8f0130d7d8da3011c32a44d4fc873368154f1510e574a2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.68", +] + [[package]] name = "either" version = "1.13.0" @@ -548,6 +611,7 @@ dependencies = [ "byte-slice-cast", "byteorder", "bytes", + "divan", "flexbuffers", "futures", "futures-core", @@ -1347,6 +1411,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.8.4" @@ -1637,6 +1707,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "terminal_size" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" +dependencies = [ + "rustix", + "windows-sys 0.48.0", +] + [[package]] name = "tinyvec" version = "1.6.1" diff --git a/Cargo.toml b/Cargo.toml index defdae8..76f0dab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,7 @@ features = ["sync", "macros", "rt-multi-thread"] version = "1" [dev-dependencies] +divan = "0.1.14" rand = "0.8" sled = "0.34.6" reqwest = { version = "0.11", features = [ "blocking" ] } @@ -71,4 +72,40 @@ fast-index = [] python = ["pyo3", "numpy"] extension-module = ["python", "pyo3/extension-module"] netcdf = [ "dep:netcdf" ] +unstable = [] +[[bench]] +name = "concurrency" +harness = false + +[[bench]] +name = "dataset" +harness = false + +[[bench]] +name = "index" +harness = false + +[[bench]] +name = "large" +harness = false + +[[bench]] +name = "native" +harness = false + +[[bench]] +name = "norkyst" +harness = false + +[[bench]] +name = "read" +harness = false + +[[bench]] +name = "serialize" +harness = false + +[[bench]] +name = "stream" +harness = false diff --git a/benches/concurrency.rs b/benches/concurrency.rs index 6890c1e..2019550 100644 --- a/benches/concurrency.rs +++ b/benches/concurrency.rs @@ -1,7 +1,5 @@ -#![feature(test)] -extern crate test; use std::sync::Arc; -use test::Bencher; +use divan::Bencher; use hidefix::prelude::*; @@ -30,56 +28,56 @@ mod shuffled_compressed { use super::*; #[ignore] - #[bench] - fn cache_sequential(b: &mut Bencher) { + #[divan::bench] + fn cache_sequential(b: Bencher) { let i = Index::index("tests/data/dmrpp/chunked_shufzip_twoD.h5").unwrap(); let mut r = i.reader("d_4_shufzip_chunks").unwrap(); - b.iter(|| { + b.bench_local(|| { for _ in 0..ITERATIONS { for _ in 0..REPETITIONS { - test::black_box(&r.values::(..).unwrap()); + divan::black_box(&r.values::(..).unwrap()); } } }) } #[ignore] - #[bench] - fn direct_sequential_parallel(b: &mut Bencher) { + #[divan::bench] + fn direct_sequential_parallel(b: Bencher) { let i = Index::index("tests/data/dmrpp/chunked_shufzip_twoD.h5").unwrap(); let ds = i.dataset("d_4_shufzip_chunks").unwrap(); let r = ds .as_par_reader(&"tests/data/dmrpp/chunked_shufzip_twoD.h5") .unwrap(); - b.iter(|| { + b.bench_local(|| { for _ in 0..ITERATIONS { for _ in 0..REPETITIONS { - test::black_box(&r.values_par::(..).unwrap()); + divan::black_box(&r.values_par::(..).unwrap()); } } }) } #[ignore] - #[bench] - fn native_sequential(b: &mut Bencher) { + #[divan::bench] + fn native_sequential(b: Bencher) { let h = hdf5::File::open("tests/data/dmrpp/chunked_shufzip_twoD.h5").unwrap(); let d = h.dataset("d_4_shufzip_chunks").unwrap(); - b.iter(|| { + b.bench_local(|| { for _ in 0..ITERATIONS { for _ in 0..REPETITIONS { - test::black_box(&d.read_raw::().unwrap()); + divan::black_box(&d.read_raw::().unwrap()); } } }) } #[ignore] - #[bench] - fn cache_concurrent_reads(b: &mut Bencher) { + #[divan::bench] + fn cache_concurrent_reads(b: Bencher) { let i = Arc::new(Index::index("tests/data/dmrpp/chunked_shufzip_twoD.h5").unwrap()); let pool = rayon::ThreadPoolBuilder::new() @@ -87,7 +85,7 @@ mod shuffled_compressed { .build() .unwrap(); - b.iter(move || { + b.bench_local(move || { let i = Arc::clone(&i); pool.scope(move |s| { for _ in 0..ITERATIONS { @@ -96,7 +94,7 @@ mod shuffled_compressed { s.spawn(move |_| { let mut r = i.reader("d_4_shufzip_chunks").unwrap(); for _ in 0..REPETITIONS { - test::black_box(&r.values::(..).unwrap()); + divan::black_box(&r.values::(..).unwrap()); } }); } @@ -105,8 +103,8 @@ mod shuffled_compressed { } #[ignore] - #[bench] - fn native_concurrent_reads(b: &mut Bencher) { + #[divan::bench] + fn native_concurrent_reads(b: Bencher) { let h = hdf5::File::open("tests/data/dmrpp/chunked_shufzip_twoD.h5").unwrap(); let d = Arc::new(h.dataset("d_4_shufzip_chunks").unwrap()); @@ -115,7 +113,7 @@ mod shuffled_compressed { .build() .unwrap(); - b.iter(move || { + b.bench_local(move || { let d = Arc::clone(&d); pool.scope(move |s| { for _ in 0..ITERATIONS { @@ -123,7 +121,7 @@ mod shuffled_compressed { s.spawn(move |_| { for _ in 0..REPETITIONS { - test::black_box(&d.read_raw::().unwrap()); + divan::black_box(&d.read_raw::().unwrap()); } }); } @@ -131,3 +129,7 @@ mod shuffled_compressed { }) } } + +fn main() { + divan::main(); +} diff --git a/benches/dataset.rs b/benches/dataset.rs index faed004..f11fd59 100644 --- a/benches/dataset.rs +++ b/benches/dataset.rs @@ -1,17 +1,19 @@ -#![feature(test)] -extern crate test; -use test::Bencher; +use divan::Bencher; use hidefix::idx::DatasetD; use hidefix::idx::Index; -#[bench] -fn slicer(b: &mut Bencher) { +#[divan::bench] +fn slicer(b: Bencher) { let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); let d = i.dataset("SST").unwrap(); if let DatasetD::D3(d) = d { - b.iter(|| d.chunk_slices(..).for_each(drop)) + b.bench_local(|| d.chunk_slices(..).for_each(drop)) } else { panic!() } } + +fn main() { + divan::main(); +} diff --git a/benches/index.rs b/benches/index.rs index 0b55717..1dda64f 100644 --- a/benches/index.rs +++ b/benches/index.rs @@ -1,10 +1,12 @@ -#![feature(test)] -extern crate test; -use test::Bencher; +use divan::Bencher; use hidefix::idx::Index; -#[bench] -fn chunked_1d(b: &mut Bencher) { - b.iter(|| Index::index("tests/data/dmrpp/chunked_oneD.h5").unwrap()) +#[divan::bench] +fn chunked_1d(b: Bencher) { + b.bench_local(|| Index::index("tests/data/dmrpp/chunked_oneD.h5").unwrap()) +} + +fn main() { + divan::main(); } diff --git a/benches/large.rs b/benches/large.rs index f117307..9d2cc68 100644 --- a/benches/large.rs +++ b/benches/large.rs @@ -1,6 +1,4 @@ -#![feature(test)] -extern crate test; -use test::Bencher; +use divan::Bencher; use std::path::PathBuf; use std::sync::Mutex; @@ -41,8 +39,8 @@ fn get_file() -> PathBuf { } #[ignore] -#[bench] -fn idx_small_slice(b: &mut Bencher) { +#[divan::bench] +fn idx_small_slice(b: Bencher) { let p = get_file(); let i = Index::index(&p).unwrap(); let mut r = i.reader(VAR).unwrap(); @@ -62,18 +60,18 @@ fn idx_small_slice(b: &mut Bencher) { r.values::((&[0, 0, 0, 0], &[2, 2, 1, 5])).unwrap() ); - b.iter(|| test::black_box(r.values::((&[0, 0, 0, 0], &[2, 2, 1, 5])).unwrap())); + b.bench_local(|| divan::black_box(r.values::((&[0, 0, 0, 0], &[2, 2, 1, 5])).unwrap())); } #[ignore] -#[bench] -fn native_small_slice(b: &mut Bencher) { +#[divan::bench] +fn native_small_slice(b: Bencher) { let p = get_file(); let h = hdf5::File::open(p).unwrap(); let d = h.dataset(VAR).unwrap(); - b.iter(|| { - test::black_box( + b.bench_local(|| { + divan::black_box( d.read_slice::(s![0..2, 0..2, 0..1, 0..5]) .unwrap(), ) @@ -81,8 +79,8 @@ fn native_small_slice(b: &mut Bencher) { } #[ignore] -#[bench] -fn idx_med_slice(b: &mut Bencher) { +#[divan::bench] +fn idx_med_slice(b: Bencher) { let p = get_file(); let i = Index::index(&p).unwrap(); let mut r = i.reader(VAR).unwrap(); @@ -103,8 +101,8 @@ fn idx_med_slice(b: &mut Bencher) { .unwrap() ); - b.iter(|| { - test::black_box( + b.bench_local(|| { + divan::black_box( r.values::((&[0, 0, 0, 0], &[10, 10, 1, 2602])) .unwrap(), ) @@ -112,14 +110,14 @@ fn idx_med_slice(b: &mut Bencher) { } #[ignore] -#[bench] -fn native_med_slice(b: &mut Bencher) { +#[divan::bench] +fn native_med_slice(b: Bencher) { let p = get_file(); let h = hdf5::File::open(p).unwrap(); let d = h.dataset(VAR).unwrap(); - b.iter(|| { - test::black_box( + b.bench_local(|| { + divan::black_box( d.read_slice::(s![0..10, 0..10, 0..1, 0..2602]) .unwrap(), ) @@ -127,8 +125,8 @@ fn native_med_slice(b: &mut Bencher) { } #[ignore] -#[bench] -fn idx_big_slice(b: &mut Bencher) { +#[divan::bench] +fn idx_big_slice(b: Bencher) { let p = get_file(); let i = Index::index(&p).unwrap(); let mut r = i.reader(VAR).unwrap(); @@ -149,8 +147,8 @@ fn idx_big_slice(b: &mut Bencher) { .unwrap() ); - b.iter(|| { - test::black_box( + b.bench_local(|| { + divan::black_box( r.values::((&[0, 0, 0, 0], &[24, 16, 1, 2602])) .unwrap(), ) @@ -158,16 +156,20 @@ fn idx_big_slice(b: &mut Bencher) { } #[ignore] -#[bench] -fn native_big_slice(b: &mut Bencher) { +#[divan::bench] +fn native_big_slice(b: Bencher) { let p = get_file(); let h = hdf5::File::open(p).unwrap(); let d = h.dataset(VAR).unwrap(); - b.iter(|| { - test::black_box( + b.bench_local(|| { + divan::black_box( d.read_slice::(s![0..24, 0..16, 0..1, 0..2602]) .unwrap(), ) }) } + +fn main() { + divan::main(); +} diff --git a/benches/native.rs b/benches/native.rs index 96da0cb..343fcad 100644 --- a/benches/native.rs +++ b/benches/native.rs @@ -1,59 +1,61 @@ -#![feature(test)] -extern crate test; -use test::Bencher; +use divan::Bencher; -#[bench] -fn read_2d_shuffled(b: &mut Bencher) { +#[divan::bench] +fn read_2d_shuffled(b: Bencher) { let h = hdf5::File::open("tests/data/dmrpp/chunked_shuffled_twoD.h5").unwrap(); let d = h.dataset("d_4_shuffled_chunks").unwrap(); - b.iter(|| d.read_raw::().unwrap()) + b.bench_local(|| d.read_raw::().unwrap()) } -#[bench] -fn read_2d_compressed(b: &mut Bencher) { +#[divan::bench] +fn read_2d_compressed(b: Bencher) { let h = hdf5::File::open("tests/data/dmrpp/chunked_gzipped_twoD.h5").unwrap(); let d = h.dataset("d_4_gzipped_chunks").unwrap(); - b.iter(|| d.read_raw::().unwrap()) + b.bench_local(|| d.read_raw::().unwrap()) } -#[bench] -fn read_2d_shuffled_compressed(b: &mut Bencher) { +#[divan::bench] +fn read_2d_shuffled_compressed(b: Bencher) { let h = hdf5::File::open("tests/data/dmrpp/chunked_shufzip_twoD.h5").unwrap(); let d = h.dataset("d_4_shufzip_chunks").unwrap(); - b.iter(|| d.read_raw::().unwrap()) + b.bench_local(|| d.read_raw::().unwrap()) } -#[bench] -fn read_2d_chunked(b: &mut Bencher) { +#[divan::bench] +fn read_2d_chunked(b: Bencher) { let h = hdf5::File::open("tests/data/dmrpp/chunked_oneD.h5").unwrap(); let d = h.dataset("d_4_chunks").unwrap(); - b.iter(|| d.read_raw::().unwrap()) + b.bench_local(|| d.read_raw::().unwrap()) } -#[bench] -fn read_t_float32(b: &mut Bencher) { +#[divan::bench] +fn read_t_float32(b: Bencher) { let h = hdf5::File::open("tests/data/dmrpp/t_float.h5").unwrap(); let d = h.dataset("d32_1").unwrap(); - b.iter(|| d.read_raw::().unwrap()) + b.bench_local(|| d.read_raw::().unwrap()) } -#[bench] -fn read_chunked_1d(b: &mut Bencher) { +#[divan::bench] +fn read_chunked_1d(b: Bencher) { let h = hdf5::File::open("tests/data/dmrpp/chunked_oneD.h5").unwrap(); let d = h.dataset("d_4_chunks").unwrap(); - b.iter(|| d.read_raw::().unwrap()) + b.bench_local(|| d.read_raw::().unwrap()) } -#[bench] -fn coads_values(b: &mut Bencher) { +#[divan::bench] +fn coads_values(b: Bencher) { let h = hdf5::File::open("tests/data/coads_climatology.nc4").unwrap(); let d = h.dataset("SST").unwrap(); - b.iter(|| d.read_raw::().unwrap()) + b.bench_local(|| d.read_raw::().unwrap()) +} + +fn main() { + divan::main(); } diff --git a/benches/norkyst.rs b/benches/norkyst.rs index 8a1cfa7..c544ebe 100644 --- a/benches/norkyst.rs +++ b/benches/norkyst.rs @@ -1,6 +1,4 @@ -#![feature(test)] -extern crate test; -use test::Bencher; +use divan::Bencher; use std::path::PathBuf; use std::sync::Mutex; @@ -37,21 +35,25 @@ fn get_file() -> PathBuf { } #[ignore] -#[bench] -fn idx_big_slice(b: &mut Bencher) { +#[divan::bench] +fn idx_big_slice(b: Bencher) { let p = get_file(); let i = Index::index(&p).unwrap(); let mut u = i.reader("u_eastward").unwrap(); - b.iter(|| test::black_box(u.values::(..).unwrap())); + b.bench_local(|| divan::black_box(u.values::(..).unwrap())); } #[ignore] -#[bench] -fn native_big_slice(b: &mut Bencher) { +#[divan::bench] +fn native_big_slice(b: Bencher) { let p = get_file(); let h = hdf5::File::open(p).unwrap(); let d = h.dataset("u_eastward").unwrap(); - b.iter(|| test::black_box(d.read_raw::().unwrap())) + b.bench_local(|| divan::black_box(d.read_raw::().unwrap())) +} + +fn main() { + divan::main(); } diff --git a/benches/read.rs b/benches/read.rs index 70d713a..dd0bed9 100644 --- a/benches/read.rs +++ b/benches/read.rs @@ -1,58 +1,56 @@ -#![feature(test)] -extern crate test; use hidefix::prelude::*; -use test::Bencher; +use divan::Bencher; -#[bench] -fn read_2d_chunked(b: &mut Bencher) { +#[divan::bench] +fn read_2d_chunked(b: Bencher) { let i = Index::index("tests/data/dmrpp/chunked_oneD.h5").unwrap(); let mut r = i.reader("d_4_chunks").unwrap(); - b.iter(|| r.values::(..).unwrap()) + b.bench_local(|| r.values::(..).unwrap()) } -#[bench] -fn read_2d_shuffled(b: &mut Bencher) { +#[divan::bench] +fn read_2d_shuffled(b: Bencher) { let i = Index::index("tests/data/dmrpp/chunked_shuffled_twoD.h5").unwrap(); let mut r = i.reader("d_4_shuffled_chunks").unwrap(); - b.iter(|| r.values::(..).unwrap()) + b.bench_local(|| r.values::(..).unwrap()) } -#[bench] -fn read_2d_compressed(b: &mut Bencher) { +#[divan::bench] +fn read_2d_compressed(b: Bencher) { let i = Index::index("tests/data/dmrpp/chunked_gzipped_twoD.h5").unwrap(); let mut r = i.reader("d_4_gzipped_chunks").unwrap(); - b.iter(|| r.values::(..).unwrap()) + b.bench_local(|| r.values::(..).unwrap()) } -#[bench] -fn read_2d_shuffled_compressed(b: &mut Bencher) { +#[divan::bench] +fn read_2d_shuffled_compressed(b: Bencher) { let i = Index::index("tests/data/dmrpp/chunked_shufzip_twoD.h5").unwrap(); let mut r = i.reader("d_4_shufzip_chunks").unwrap(); - b.iter(|| r.values::(..).unwrap()) + b.bench_local(|| r.values::(..).unwrap()) } -#[bench] -fn read_t_float32(b: &mut Bencher) { +#[divan::bench] +fn read_t_float32(b: Bencher) { let i = Index::index("tests/data/dmrpp/t_float.h5").unwrap(); let mut r = i.reader("d32_1").unwrap(); - b.iter(|| r.values::(..).unwrap()) + b.bench_local(|| r.values::(..).unwrap()) } -#[bench] -fn read_chunked_1d(b: &mut Bencher) { +#[divan::bench] +fn read_chunked_1d(b: Bencher) { let i = Index::index("tests/data/dmrpp/chunked_oneD.h5").unwrap(); let mut r = i.reader("d_4_chunks").unwrap(); - b.iter(|| r.values::(..).unwrap()) + b.bench_local(|| r.values::(..).unwrap()) } -#[bench] -fn coads(b: &mut Bencher) { +#[divan::bench] +fn coads(b: Bencher) { let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); let mut r = i.reader("SST").unwrap(); @@ -66,5 +64,9 @@ fn coads(b: &mut Bencher) { ); } - b.iter(|| r.values::(..).unwrap()) + b.bench_local(|| r.values::(..).unwrap()) +} + +fn main() { + divan::main(); } diff --git a/benches/serialize.rs b/benches/serialize.rs index a08dec1..a61e812 100644 --- a/benches/serialize.rs +++ b/benches/serialize.rs @@ -1,6 +1,4 @@ -#![feature(test)] -extern crate test; -use test::Bencher; +use divan::Bencher; use hidefix::idx::Index; @@ -10,67 +8,67 @@ const FILE: Option<&'static str> = option_env!("HIDEFIX_LARGE_FILE"); mod serde_bincode { use super::*; - #[bench] - fn serialize_coads(b: &mut Bencher) { + #[divan::bench] + fn serialize_coads(b: Bencher) { let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); - b.iter(|| bincode::serialize(&i).unwrap()) + b.bench_local(|| bincode::serialize(&i).unwrap()) } - #[bench] - fn deserialize_coads(b: &mut Bencher) { + #[divan::bench] + fn deserialize_coads(b: Bencher) { let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); let bb = bincode::serialize(&i).unwrap(); - b.iter(|| bincode::deserialize::(&bb).unwrap()) + b.bench_local(|| bincode::deserialize::(&bb).unwrap()) } - #[bench] - fn serialize_coads_file(b: &mut Bencher) { + #[divan::bench] + fn serialize_coads_file(b: Bencher) { let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); - b.iter(|| { + b.bench_local(|| { let f = std::fs::File::create("/tmp/coads.idx.bc").unwrap(); let w = std::io::BufWriter::new(f); bincode::serialize_into(w, &i).unwrap() }) } - #[bench] - fn deserialize_coads_file(b: &mut Bencher) { + #[divan::bench] + fn deserialize_coads_file(b: Bencher) { let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); let f = std::fs::File::create("/tmp/coads.idx.bc").unwrap(); bincode::serialize_into(f, &i).unwrap(); - b.iter(|| { + b.bench_local(|| { let b = std::fs::read("/tmp/coads.idx.bc").unwrap(); bincode::deserialize::(&b).unwrap(); }) } #[ignore] - #[bench] - fn serialize_large_bincode(b: &mut Bencher) { + #[divan::bench] + fn serialize_large_bincode(b: Bencher) { let i = Index::index(FILE.unwrap()).unwrap(); - b.iter(|| bincode::serialize(&i).unwrap()) + b.bench_local(|| bincode::serialize(&i).unwrap()) } #[ignore] - #[bench] - fn deserialize_large_bincode(b: &mut Bencher) { + #[divan::bench] + fn deserialize_large_bincode(b: Bencher) { let i = Index::index(FILE.unwrap()).unwrap(); let bb = bincode::serialize(&i).unwrap(); - b.iter(|| bincode::deserialize::(&bb).unwrap()) + b.bench_local(|| bincode::deserialize::(&bb).unwrap()) } #[ignore] - #[bench] - fn serialize_large_bincode_file(b: &mut Bencher) { + #[divan::bench] + fn serialize_large_bincode_file(b: Bencher) { let i = Index::index(FILE.unwrap()).unwrap(); - b.iter(|| { + b.bench_local(|| { let f = std::fs::File::create("/tmp/large.idx.bc").unwrap(); let w = std::io::BufWriter::new(f); bincode::serialize_into(w, &i).unwrap() @@ -78,21 +76,21 @@ mod serde_bincode { } #[ignore] - #[bench] - fn deserialize_large_bincode_file(b: &mut Bencher) { + #[divan::bench] + fn deserialize_large_bincode_file(b: Bencher) { let i = Index::index(FILE.unwrap()).unwrap(); let f = std::fs::File::create("/tmp/large.idx.bc").unwrap(); bincode::serialize_into(f, &i).unwrap(); - b.iter(|| { + b.bench_local(|| { let b = std::fs::read("/tmp/large.idx.bc").unwrap(); bincode::deserialize::(&b).unwrap(); }) } #[ignore] - #[bench] - fn deserialize_large_bincode_db_sled(b: &mut Bencher) { + #[divan::bench] + fn deserialize_large_bincode_db_sled(b: Bencher) { let i = Index::index(FILE.unwrap()).unwrap(); let bts = bincode::serialize(&i).unwrap(); @@ -105,15 +103,15 @@ mod serde_bincode { db.insert("large", bts).unwrap(); - b.iter(|| { + b.bench_local(|| { let bts = db.get("large").unwrap().unwrap(); - test::black_box(bincode::deserialize::(&bts).unwrap()); + divan::black_box(bincode::deserialize::(&bts).unwrap()); }) } #[ignore] - #[bench] - fn deserialize_large_bincode_db_sled_only_read(b: &mut Bencher) { + #[divan::bench] + fn deserialize_large_bincode_db_sled_only_read(b: Bencher) { let i = Index::index(FILE.unwrap()).unwrap(); let bts = bincode::serialize(&i).unwrap(); @@ -126,20 +124,20 @@ mod serde_bincode { db.insert("large", bts).unwrap(); - b.iter(|| { - test::black_box(db.get("large").unwrap().unwrap()); + b.bench_local(|| { + divan::black_box(db.get("large").unwrap().unwrap()); }) } #[ignore] - #[bench] - fn deserialize_large_file_only_read(b: &mut Bencher) { + #[divan::bench] + fn deserialize_large_file_only_read(b: Bencher) { let i = Index::index(FILE.unwrap()).unwrap(); let f = std::fs::File::create("/tmp/large.idx.bc").unwrap(); bincode::serialize_into(f, &i).unwrap(); - b.iter(|| { - test::black_box(std::fs::read("/tmp/large.idx.bc").unwrap()); + b.bench_local(|| { + divan::black_box(std::fs::read("/tmp/large.idx.bc").unwrap()); }) } } @@ -149,87 +147,91 @@ mod serde_flexbuffers { use flexbuffers::FlexbufferSerializer as ser; use serde::ser::Serialize; - #[bench] - fn serialize_coads(b: &mut Bencher) { + #[divan::bench] + fn serialize_coads(b: Bencher) { let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); - b.iter(|| { + b.bench_local(|| { let mut s = ser::new(); i.serialize(&mut s).unwrap(); }) } - #[bench] - fn deserialize_coads(b: &mut Bencher) { + #[divan::bench] + fn deserialize_coads(b: Bencher) { let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); let mut s = ser::new(); i.serialize(&mut s).unwrap(); - b.iter(|| { + b.bench_local(|| { flexbuffers::from_slice::(s.view()).unwrap(); }) } - #[bench] - fn serialize_coads_file(b: &mut Bencher) { + #[divan::bench] + fn serialize_coads_file(b: Bencher) { let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); - b.iter(|| { + b.bench_local(|| { let mut s = ser::new(); i.serialize(&mut s).unwrap(); std::fs::write("/tmp/coads.idx.fx", s.view()).unwrap(); }) } - #[bench] - fn deserialize_coads_file(b: &mut Bencher) { + #[divan::bench] + fn deserialize_coads_file(b: Bencher) { let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); let mut s = ser::new(); i.serialize(&mut s).unwrap(); std::fs::write("/tmp/coads.idx.fx", s.view()).unwrap(); - b.iter(|| { + b.bench_local(|| { let b = std::fs::read("/tmp/coads.idx.fx").unwrap(); flexbuffers::from_slice::(&b).unwrap(); }) } - #[bench] - fn deserialize_coads_file_only_read(b: &mut Bencher) { + #[divan::bench] + fn deserialize_coads_file_only_read(b: Bencher) { let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); let mut s = ser::new(); i.serialize(&mut s).unwrap(); std::fs::write("/tmp/coads.idx.fx", s.view()).unwrap(); - b.iter(|| { - test::black_box(std::fs::read("/tmp/coads.idx.fx").unwrap()); + b.bench_local(|| { + divan::black_box(std::fs::read("/tmp/coads.idx.fx").unwrap()); }) } #[ignore] - #[bench] - fn deserialize_large_file(b: &mut Bencher) { + #[divan::bench] + fn deserialize_large_file(b: Bencher) { let i = Index::index(FILE.unwrap()).unwrap(); let mut s = ser::new(); i.serialize(&mut s).unwrap(); std::fs::write("/tmp/large.idx.fx", s.view()).unwrap(); - b.iter(|| { + b.bench_local(|| { let b = std::fs::read("/tmp/large.idx.fx").unwrap(); flexbuffers::from_slice::(&b).unwrap(); }) } #[ignore] - #[bench] - fn deserialize_large_file_only_read(b: &mut Bencher) { + #[divan::bench] + fn deserialize_large_file_only_read(b: Bencher) { let i = Index::index(FILE.unwrap()).unwrap(); let mut s = ser::new(); i.serialize(&mut s).unwrap(); std::fs::write("/tmp/large.idx.fx", s.view()).unwrap(); - b.iter(|| { - test::black_box(std::fs::read("/tmp/large.idx.fx").unwrap()); + b.bench_local(|| { + divan::black_box(std::fs::read("/tmp/large.idx.fx").unwrap()); }) } } + +fn main() { + divan::main(); +} diff --git a/benches/stream.rs b/benches/stream.rs index d64734e..d4659b5 100644 --- a/benches/stream.rs +++ b/benches/stream.rs @@ -1,9 +1,7 @@ -#![feature(test)] -extern crate test; use futures::executor::block_on_stream; use futures::{pin_mut, Stream, StreamExt}; use hidefix::prelude::*; -use test::Bencher; +use divan::Bencher; fn consume_stream(rt: &mut tokio::runtime::Runtime, s: S) { rt.block_on(async move { @@ -12,32 +10,32 @@ fn consume_stream(rt: &mut tokio::runtime::Runtime, s: S) { }); } -#[bench] -fn chunked_1d_values(b: &mut Bencher) { +#[divan::bench] +fn chunked_1d_values(b: Bencher) { let mut rt = tokio::runtime::Runtime::new().unwrap(); let i = Index::index("tests/data/dmrpp/chunked_oneD.h5").unwrap(); let r = i.streamer("d_4_chunks").unwrap(); - b.iter(|| { + b.bench_local(|| { let v = r.stream_values::(..); consume_stream(&mut rt, v); }) } -#[bench] -fn gzip_shuffle_2d_bytes(b: &mut Bencher) { +#[divan::bench] +fn gzip_shuffle_2d_bytes(b: Bencher) { let mut rt = tokio::runtime::Runtime::new().unwrap(); let i = Index::index("tests/data/gzip_shuffle_2d.h5").unwrap(); let r = i.streamer("data").unwrap(); - b.iter(|| { + b.bench_local(|| { let v = r.stream(&Extents::All); consume_stream(&mut rt, v); }) } -#[bench] -fn coads_values(b: &mut Bencher) { +#[divan::bench] +fn coads_values(b: Bencher) { let mut rt = tokio::runtime::Runtime::new().unwrap(); let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); let r = i.streamer("SST").unwrap(); @@ -51,26 +49,26 @@ fn coads_values(b: &mut Bencher) { assert_eq!(d.read_raw::().unwrap(), vs); } - b.iter(|| { + b.bench_local(|| { let v = r.stream_values::(..); consume_stream(&mut rt, v); }) } -#[bench] -fn coads_bytes(b: &mut Bencher) { +#[divan::bench] +fn coads_bytes(b: Bencher) { let mut rt = tokio::runtime::Runtime::new().unwrap(); let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); let r = i.streamer("SST").unwrap(); - b.iter(|| { + b.bench_local(|| { let v = r.stream(&Extents::All); consume_stream(&mut rt, v); }) } -#[bench] -fn coads_async_read(b: &mut Bencher) { +#[divan::bench] +fn coads_async_read(b: Bencher) { use futures::executor::block_on; use futures::io::AsyncReadExt; use futures::stream::TryStreamExt; @@ -78,7 +76,7 @@ fn coads_async_read(b: &mut Bencher) { let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); let r = i.streamer("SST").unwrap(); - b.iter(|| { + b.bench_local(|| { block_on(async { let v = r .stream(&Extents::All) @@ -89,3 +87,7 @@ fn coads_async_read(b: &mut Bencher) { }) }) } + +fn main() { + divan::main(); +} diff --git a/src/filters/shuffle.rs b/src/filters/shuffle.rs index 1329e64..88298d8 100644 --- a/src/filters/shuffle.rs +++ b/src/filters/shuffle.rs @@ -130,7 +130,7 @@ where mod tests { use super::*; use byteorder::{BigEndian, ByteOrder}; - use test::Bencher; + use divan::Bencher; #[test] fn shuffle_hdf5_example() { @@ -158,30 +158,30 @@ mod tests { ); } - #[bench] - fn shuffle_4kb(b: &mut Bencher) { + #[divan::bench] + fn shuffle_4kb(b: Bencher) { use rand::distributions::Standard; use rand::{thread_rng, Rng}; let v: Vec = thread_rng().sample_iter(Standard).take(1024).collect(); let mut d = vec![0_u8; 1024 * 4]; - b.iter(|| shuffle(&v, &mut d)) + b.bench_local(|| shuffle(&v, &mut d)) } - #[bench] - fn unshuffle_4kb(b: &mut Bencher) { + #[divan::bench] + fn unshuffle_4kb(b: Bencher) { use rand::distributions::Standard; use rand::{thread_rng, Rng}; let v: Vec = thread_rng().sample_iter(Standard).take(1024).collect(); let mut d = vec![0_u8; 1024 * 4]; - b.iter(|| shuffle(&v, &mut d)) + b.bench_local(|| shuffle(&v, &mut d)) } - #[bench] - fn shuffle_4mb(b: &mut Bencher) { + #[divan::bench] + fn shuffle_4mb(b: Bencher) { use rand::distributions::Standard; use rand::{thread_rng, Rng}; @@ -191,11 +191,11 @@ mod tests { .collect(); let mut d = vec![0_i32; 1024 * 1024]; - b.iter(|| unshuffle(&v, &mut d)) + b.bench_local(|| unshuffle(&v, &mut d)) } - #[bench] - fn unshuffle_4mb(b: &mut Bencher) { + #[divan::bench] + fn unshuffle_4mb(b: Bencher) { use rand::distributions::Standard; use rand::{thread_rng, Rng}; @@ -205,21 +205,21 @@ mod tests { .collect(); let mut d = vec![0_i32; 1024 * 1024]; - b.iter(|| unshuffle(&v, &mut d)) + b.bench_local(|| unshuffle(&v, &mut d)) } - #[bench] - fn unshuffle_structured_4kb(b: &mut Bencher) { + #[divan::bench] + fn unshuffle_structured_4kb(b: Bencher) { use rand::distributions::Standard; use rand::{thread_rng, Rng}; let v: Vec = thread_rng().sample_iter(Standard).take(4 * 1024).collect(); - b.iter(|| unshuffle_sized(&v, 4)) + b.bench_local(|| unshuffle_sized(&v, 4)) } - #[bench] - fn unshuffle_structured_4mb(b: &mut Bencher) { + #[divan::bench] + fn unshuffle_structured_4mb(b: Bencher) { use rand::distributions::Standard; use rand::{thread_rng, Rng}; @@ -228,7 +228,7 @@ mod tests { .take(4 * 1024 * 1024) .collect(); - b.iter(|| unshuffle_sized(&v, 4)) + b.bench_local(|| unshuffle_sized(&v, 4)) } #[test] diff --git a/src/idx/chunk.rs b/src/idx/chunk.rs index 31f7933..b7fd60b 100644 --- a/src/idx/chunk.rs +++ b/src/idx/chunk.rs @@ -228,7 +228,7 @@ mod tests { mod serde { use super::*; - use test::Bencher; + use divan::Bencher; #[test] fn as_u64s() { @@ -405,8 +405,8 @@ mod tests { assert_eq!(dcs, cs); } - #[bench] - fn slice_from_u64s_10k_3d(b: &mut Bencher) { + #[divan::bench] + fn slice_from_u64s_10k_3d(b: Bencher) { let chunks: Vec> = (0..10000) .map(|i| Chunk::new(i * 10, 300, [i * 10, i * 100, i * 10000])) .collect(); @@ -423,8 +423,8 @@ mod tests { assert_eq!(dechunks.len(), chunks.len()); assert_eq!(dechunks, chunks.as_slice()); - b.iter(|| { - test::black_box(Chunk::<3>::slice_from_u64s(slice)); + b.bench_local(|| { + divan::black_box(Chunk::<3>::slice_from_u64s(slice)); }); } } diff --git a/src/idx/dataset/dataset.rs b/src/idx/dataset/dataset.rs index 15944a5..d45b8cd 100644 --- a/src/idx/dataset/dataset.rs +++ b/src/idx/dataset/dataset.rs @@ -671,15 +671,15 @@ impl<'a, const D: usize> Iterator for ChunkSlicer<'a, D> { mod tests { use super::super::tests::test_dataset; use super::*; - use test::Bencher; + use divan::Bencher; - #[bench] - fn chunk_start(b: &mut Bencher) { + #[divan::bench] + fn chunk_start(b: Bencher) { let dim_sz = [10, 1]; let coords = [20, 10]; let ch_offset = [ULE::new(20), ULE::new(10)]; - b.iter(|| test::black_box(ChunkSlicer::chunk_start(&coords, &ch_offset, &dim_sz))) + b.bench_local(|| divan::black_box(ChunkSlicer::chunk_start(&coords, &ch_offset, &dim_sz))) } #[test] diff --git a/src/idx/dataset/mod.rs b/src/idx/dataset/mod.rs index 21500a8..60228bf 100644 --- a/src/idx/dataset/mod.rs +++ b/src/idx/dataset/mod.rs @@ -14,7 +14,7 @@ mod tests { use crate::filters::byteorder::Order as ByteOrder; use itertools::izip; use serde::{Deserialize, Serialize}; - use test::Bencher; + use divan::Bencher; pub(crate) fn test_dataset() -> Dataset<'static, 2> { Dataset::new( @@ -83,22 +83,22 @@ mod tests { assert_eq!(d.chunk_slices((&[0, 0], &[1, 0])).next(), None); } - #[bench] - fn chunk_slices_range(b: &mut Bencher) { + #[divan::bench] + fn chunk_slices_range(b: Bencher) { let d = test_dataset(); - b.iter(|| d.chunk_slices(..).for_each(drop)); + b.bench_local(|| d.chunk_slices(..).for_each(drop)); } - #[bench] - fn make_chunk_slices_iterator(b: &mut Bencher) { + #[divan::bench] + fn make_chunk_slices_iterator(b: Bencher) { let d = test_dataset(); - b.iter(|| test::black_box(d.chunk_slices(..))) + b.bench_local(|| divan::black_box(d.chunk_slices(..))) } - #[bench] - fn chunk_at_coord(b: &mut Bencher) { + #[divan::bench] + fn chunk_at_coord(b: Bencher) { let d = test_dataset(); println!("chunks: {:#?}", d.chunks); @@ -127,7 +127,7 @@ mod tests { [ULE::new(10), ULE::new(0)] ); - b.iter(|| test::black_box(d.chunk_at_coord(&[15, 1]))) + b.bench_local(|| divan::black_box(d.chunk_at_coord(&[15, 1]))) } #[test] diff --git a/src/idx/serde.rs b/src/idx/serde.rs index 305df25..17004a7 100644 --- a/src/idx/serde.rs +++ b/src/idx/serde.rs @@ -92,6 +92,7 @@ pub mod chunks_u64s { let chunks: &'a [Chunk] = Chunk::::slice_from_u64s(slice); let chunks = Cow::<'a, [Chunk]>::from(chunks); + #[cfg(feature="unstable")] debug_assert!(chunks.is_borrowed()); Ok(chunks) diff --git a/src/lib.rs b/src/lib.rs index 89ae784..3838eab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -85,16 +85,7 @@ //! hdf5-src = { git = "https://github.com/magnusuMET/hdf5-rust", branch = "hidefix_jul_2023" } //! ``` -#![allow(incomplete_features)] -#![recursion_limit = "1024"] -#![feature(test)] -#![feature(cow_is_borrowed, array_methods)] -#![feature(assert_matches)] -#![feature(slice_group_by)] -#![feature(mutex_unlock)] -#![feature(new_uninit)] - -extern crate test; +#![cfg_attr(feature = "unstable", feature(new_uninit, cow_is_borrowed))] pub mod extent; pub mod filters; diff --git a/src/reader/dataset.rs b/src/reader/dataset.rs index 095ddc7..3ab44d2 100644 --- a/src/reader/dataset.rs +++ b/src/reader/dataset.rs @@ -21,6 +21,26 @@ pub trait Reader { fn shape(&self) -> &[u64]; } +#[cfg(feature = "unstable")] +fn empty_vec(vsz: usize) -> Vec +where + T: Default, +{ + let values = Box::<[T]>::new_uninit_slice(vsz); + let values = unsafe { values.assume_init() }; + values.into_vec() +} + +#[cfg(not(feature = "unstable"))] +fn empty_vec(vsz: usize) -> Vec +where + T: Default, +{ + let mut values = Vec::with_capacity(vsz); + values.resize_with(vsz, T::default); + values +} + pub trait ReaderExt: Reader { /// Reads values into desitination slice. Returns values read. fn values_to(&mut self, extents: E, dst: &mut [T]) -> Result @@ -40,7 +60,7 @@ pub trait ReaderExt: Reader { /// Reads slice of dataset into `Vec`. fn values(&mut self, extents: E) -> Result, anyhow::Error> where - T: ToMutByteSlice, + T: ToMutByteSlice + Default, [T]: ToNative, E: TryInto, E::Error: Into, @@ -59,9 +79,7 @@ pub trait ReaderExt: Reader { ensure!((dsz * vsz) % std::mem::align_of::() == 0, "alignment of datatype ({}) not a multiple of datatype size and length {}*{}={}, alignment may not match and result in unsoundness", std::mem::align_of::(), dsz, vsz, vsz * dsz); - let values = Box::<[T]>::new_uninit_slice(vsz); - let values = unsafe { values.assume_init() }; - let mut values = values.into_vec(); + let mut values = empty_vec(vsz); self.values_to(extents, values.as_mut_slice())?; // XXX: take maybeuninit Ok(values) @@ -93,7 +111,7 @@ pub trait ParReaderExt: Reader + ParReader { /// Reads slice of dataset into `Vec`. fn values_par(&self, extents: E) -> Result, anyhow::Error> where - T: ToMutByteSlice, + T: ToMutByteSlice + Default, [T]: ToNative, E: TryInto, E::Error: Into, @@ -112,9 +130,7 @@ pub trait ParReaderExt: Reader + ParReader { ensure!((dsz * vsz) % std::mem::align_of::() == 0, "alignment of datatype ({}) not a multiple of datatype size and length {}*{}={}, alignment may not match and result in unsoundness", std::mem::align_of::(), dsz, vsz, vsz * dsz); - let values = Box::<[T]>::new_uninit_slice(vsz); - let values = unsafe { values.assume_init() }; - let mut values = values.into_vec(); + let mut values = empty_vec(vsz); self.values_to_par(extents, values.as_mut_slice())?; Ok(values) diff --git a/src/reader/direct.rs b/src/reader/direct.rs index cd2c76a..547d146 100644 --- a/src/reader/direct.rs +++ b/src/reader/direct.rs @@ -46,7 +46,7 @@ impl<'a, const D: usize> ParReader for Direct<'a, D> { ); let groups = self.ds.group_chunk_slices(extents); - let groups = groups.group_by(|a, b| a.0.addr == b.0.addr); + let groups = groups.chunk_by(|a, b| a.0.addr == b.0.addr); let groups = groups.collect::>(); groups.par_iter().try_for_each_init( diff --git a/tests/read_dims.rs b/tests/read_dims.rs index 92fa5a6..4150cab 100644 --- a/tests/read_dims.rs +++ b/tests/read_dims.rs @@ -1,6 +1,3 @@ -#![feature(test)] -extern crate test; - use hidefix::prelude::*; use ndarray::s; diff --git a/tests/read_norkyst.rs b/tests/read_norkyst.rs index 1eee4f9..0d05dd0 100644 --- a/tests/read_norkyst.rs +++ b/tests/read_norkyst.rs @@ -1,6 +1,4 @@ -#![feature(test)] #![allow(non_snake_case)] -extern crate test; use hidefix::idx::{Dataset, DatasetD}; use hidefix::prelude::*; diff --git a/tests/read_strings.rs b/tests/read_strings.rs index a082871..2411234 100644 --- a/tests/read_strings.rs +++ b/tests/read_strings.rs @@ -1,6 +1,3 @@ -#![feature(test)] -extern crate test; - use hidefix::prelude::*; #[test] diff --git a/tests/read_svim.rs b/tests/read_svim.rs index 9ce7a20..fee9f66 100644 --- a/tests/read_svim.rs +++ b/tests/read_svim.rs @@ -1,6 +1,3 @@ -#![feature(test)] -extern crate test; - use futures::executor::block_on_stream; use futures::pin_mut; use hidefix::prelude::*;