diff --git a/Cargo.toml b/Cargo.toml index a972f3f..c0cb685 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ num-bigint = "0.4" num-complex = { version = "0.4", optional = true } arrayvec = { version = "0.7.2", optional = true } half = { version = "2.1.0", optional = true } - +bytemuck = {version = "1.15.0", optional = true} [dependencies.npyz-derive] path = "derive" @@ -70,6 +70,7 @@ arrayvec = ["dep:arrayvec"] complex = ["dep:num-complex"] half = ["dep:half"] npz = ["dep:zip"] +bytemuck= ["dep:bytemuck","half/bytemuck"] [[bench]] name = "bench" diff --git a/src/read.rs b/src/read.rs index 9fe5698..7c3845a 100644 --- a/src/read.rs +++ b/src/read.rs @@ -4,6 +4,7 @@ use std::io; use crate::header::{Value, DType, read_header, convert_value_to_shape}; use crate::serialize::{Deserialize, TypeRead, DTypeError}; + /// Object for reading an `npy` file. /// /// This type represents a partially read `npy` file, where the header has been parsed @@ -276,8 +277,12 @@ impl NpyFile { /// /// This is a convenience wrapper around [`Self::data`] and [`Iterator::collect`]. pub fn into_vec(self) -> io::Result> { - match self.data() { - Ok(r) => r.collect(), + match self.data::() { + Ok(r) => { + let n = r.header.n_records as usize; + let reader: T::TypeReader = r.type_reader; + return reader.read_many(r.reader_and_current_index.0,n); + }, Err(e) => Err(invalid_data(e)), } } @@ -410,6 +415,7 @@ impl NpyReader where R: io::Seek { } } + #[allow(deprecated)] impl<'a, T: Deserialize> NpyData<'a, T> { /// Deserialize a NPY file represented as bytes diff --git a/src/serialize/mod.rs b/src/serialize/mod.rs index 1eb3490..3d389f8 100644 --- a/src/serialize/mod.rs +++ b/src/serialize/mod.rs @@ -16,9 +16,11 @@ mod traits; pub use slice::*; mod slice; +#[allow(unused_imports)] pub use primitive::*; mod primitive; +#[allow(unused_imports)] pub use array_member::*; mod array_member; diff --git a/src/serialize/primitive.rs b/src/serialize/primitive.rs index 09e26db..8706ac7 100644 --- a/src/serialize/primitive.rs +++ b/src/serialize/primitive.rs @@ -18,6 +18,14 @@ pub trait PrimitiveReadWrite: Sized { #[doc(hidden)] fn primitive_read_one(reader: R, swap_bytes: bool) -> io::Result; #[doc(hidden)] + fn primitive_read_many(mut reader: R, swap_bytes: bool,n:usize) -> io::Result> { + let mut vec = Vec::with_capacity(n); + for _ in 0..n { + vec.push(Self::primitive_read_one(&mut reader, swap_bytes)?); + } + Ok(vec) + } + #[doc(hidden)] fn primitive_write_one(&self, writer: W, swap_bytes: bool) -> io::Result<()>; } @@ -38,6 +46,25 @@ macro_rules! derive_int_primitive_read_write { } } + #[cfg(feature = "bytemuck")] + #[inline] + fn primitive_read_many(mut reader: R, swap_bytes: bool,n:usize) -> io::Result> { + if !swap_bytes{ + use std::mem::size_of; + + let mut buf:Vec = vec![0u8; size_of::<$int>()*n]; + reader.read_exact(&mut buf)?; + + Ok(bytemuck::cast_slice(&buf).to_vec()) + }else{ + let mut vec = Vec::with_capacity(n); + for _ in 0..n { + vec.push(Self::primitive_read_one(&mut reader, swap_bytes)?); + } + Ok(vec) + } + } + #[inline] fn primitive_write_one(&self, mut writer: W, swap_bytes: bool) -> io::Result<()> { let swapped = match swap_bytes { @@ -62,6 +89,25 @@ macro_rules! derive_float_primitive_read_write { Ok(<$float>::from_bits(bits)) } + #[cfg(feature = "bytemuck")] + #[inline] + fn primitive_read_many(mut reader: R, swap_bytes: bool,n:usize) -> io::Result> { + if !swap_bytes{ + use std::mem::size_of; + + let mut buf:Vec = vec![0u8; size_of::<$int>()*n]; + reader.read_exact(&mut buf)?; + Ok(bytemuck::cast_slice(&buf).to_vec()) + }else{ + let mut vec = Vec::with_capacity(n); + for _ in 0..n { + vec.push(Self::primitive_read_one(&mut reader, swap_bytes)?); + } + Ok(vec) + } + } + + #[inline] fn primitive_write_one(&self, writer: W, swap_bytes: bool) -> io::Result<()> { self.to_bits().primitive_write_one(writer, swap_bytes) @@ -131,6 +177,11 @@ impl TypeRead for PrimitiveReader { fn read_one(&self, reader: R) -> io::Result { T::primitive_read_one(reader, self.swap_bytes) } + + fn read_many(&self, mut bytes: R,n:usize) -> io::Result> { + T::primitive_read_many(&mut bytes, self.swap_bytes,n) + } + } impl TypeWrite for PrimitiveWriter { diff --git a/src/serialize/traits.rs b/src/serialize/traits.rs index 04ea1e9..c2ef032 100644 --- a/src/serialize/traits.rs +++ b/src/serialize/traits.rs @@ -2,7 +2,7 @@ use std::io; use std::fmt; use crate::header::DType; -use crate::type_str::{TypeStr}; +use crate::type_str::TypeStr; #[allow(unused)] // used by docstrings use crate::type_matchup_docs; @@ -111,7 +111,17 @@ pub trait TypeRead { /// The function. fn read_one(&self, bytes: R) -> io::Result - where Self: Sized; + where Self: Sized; + + /// read n values from the reader + fn read_many(&self, mut bytes: R,n:usize) -> io::Result> + where Self: Sized{ + let mut vec = Vec::with_capacity(n); + for _ in 0..n { + vec.push(self.read_one(&mut bytes)?); + } + Ok(vec) + } } /// Like some sort of `for Fn(W, &T) -> io::Result<()>`. @@ -141,6 +151,14 @@ pub trait TypeWrite { pub trait TypeReadDyn: TypeRead { #[doc(hidden)] fn read_one_dyn(&self, writer: &mut dyn io::Read) -> io::Result; + #[doc(hidden)] + fn read_many_dyn(&self, mut reader: &mut dyn io::Read,n:usize) -> io::Result>{ + let mut data = Vec::with_capacity(n); + for _ in 0..n { + data.push(self.read_one_dyn(&mut reader)?); + } + Ok(data) + } } impl TypeReadDyn for T { @@ -148,6 +166,10 @@ impl TypeReadDyn for T { fn read_one_dyn(&self, reader: &mut dyn io::Read) -> io::Result { self.read_one(reader) } + + fn read_many_dyn(&self, reader: &mut dyn io::Read,n:usize) -> io::Result> { + self.read_many(reader, n) + } } impl TypeRead for Box> {