Fast reading for primitive types

Cargo.toml

@@ -30,7 +30,7 @@ num-bigint = "0.4"
 num-complex = { version = "0.4", optional = true }
 arrayvec = { version = "0.7.2", optional = true }
 half = { version = "2.1.0", optional = true }
-
+bytemuck = {version = "1.15.0", optional = true}
 
 [dependencies.npyz-derive]
 path = "derive"
@@ -70,6 +70,7 @@ arrayvec = ["dep:arrayvec"]
 complex = ["dep:num-complex"]
 half = ["dep:half"]
 npz = ["dep:zip"]
+bytemuck= ["dep:bytemuck","half/bytemuck"]
 
 [[bench]]
 name = "bench"

src/read.rs

@@ -4,6 +4,7 @@ use std::io;
 use crate::header::{Value, DType, read_header, convert_value_to_shape};
 use crate::serialize::{Deserialize, TypeRead, DTypeError};
 
+
 /// Object for reading an `npy` file.
 ///
 /// This type represents a partially read `npy` file, where the header has been parsed
@@ -276,8 +277,12 @@ impl<R: io::Read> NpyFile<R> {
     ///
     /// This is a convenience wrapper around [`Self::data`] and [`Iterator::collect`].
     pub fn into_vec<T: Deserialize>(self) -> io::Result<Vec<T>> {
-        match self.data() {
-            Ok(r) => r.collect(),
+        match self.data::<T>() {
+            Ok(r) => {
+                let n = r.header.n_records as usize;
+                let reader: T::TypeReader = r.type_reader;
+                return reader.read_many(r.reader_and_current_index.0,n);
+            },
             Err(e) => Err(invalid_data(e)),
         }
     }
@@ -410,6 +415,7 @@ impl<R: io::Read, T: Deserialize> NpyReader<T, R> where R: io::Seek {
     }
 }
 
+
 #[allow(deprecated)]
 impl<'a, T: Deserialize> NpyData<'a, T> {
     /// Deserialize a NPY file represented as bytes

src/serialize/mod.rs

@@ -16,9 +16,11 @@ mod traits;
 pub use slice::*;
 mod slice;
 
+#[allow(unused_imports)]
 pub use primitive::*;
 mod primitive;
 
+#[allow(unused_imports)]
 pub use array_member::*;
 mod array_member;
 

src/serialize/primitive.rs

@@ -18,6 +18,14 @@ pub trait PrimitiveReadWrite: Sized {
     #[doc(hidden)]
     fn primitive_read_one<R: io::Read>(reader: R, swap_bytes: bool) -> io::Result<Self>;
     #[doc(hidden)]
+    fn primitive_read_many<R:io::Read>(mut reader: R, swap_bytes: bool,n:usize) -> io::Result<Vec<Self>> {
+        let mut vec = Vec::with_capacity(n);
+        for _ in 0..n {
+            vec.push(Self::primitive_read_one(&mut reader, swap_bytes)?);
+        }
+        Ok(vec)
+    }
+    #[doc(hidden)]
     fn primitive_write_one<W: io::Write>(&self, writer: W, swap_bytes: bool) -> io::Result<()>;
 }
 
@@ -38,6 +46,25 @@ macro_rules! derive_int_primitive_read_write {
                 }
             }
 
+            #[cfg(feature = "bytemuck")]
+            #[inline]
+            fn primitive_read_many<R:io::Read>(mut reader: R, swap_bytes: bool,n:usize) -> io::Result<Vec<$int>> {
+                if !swap_bytes{
+                    use std::mem::size_of;
+
+                    let mut buf:Vec<u8> = vec![0u8; size_of::<$int>()*n];
+                    reader.read_exact(&mut buf)?;
+
+                   Ok(bytemuck::cast_slice(&buf).to_vec())
+                }else{
+                    let mut vec = Vec::with_capacity(n);
+                    for _ in 0..n {
+                        vec.push(Self::primitive_read_one(&mut reader, swap_bytes)?);
+                    }
+                    Ok(vec)
+                }
+            }
+
             #[inline]
             fn primitive_write_one<W: io::Write>(&self, mut writer: W, swap_bytes: bool) -> io::Result<()> {
                 let swapped = match swap_bytes {
@@ -62,6 +89,25 @@ macro_rules! derive_float_primitive_read_write {
                 Ok(<$float>::from_bits(bits))
             }
 
+            #[cfg(feature = "bytemuck")]
+            #[inline]
+            fn primitive_read_many<R:io::Read>(mut reader: R, swap_bytes: bool,n:usize) -> io::Result<Vec<$float>> {
+                if !swap_bytes{
+                    use std::mem::size_of;
+
+                    let mut buf:Vec<u8> = vec![0u8; size_of::<$int>()*n];
+                    reader.read_exact(&mut buf)?;
+                    Ok(bytemuck::cast_slice(&buf).to_vec())
+                }else{
+                    let mut vec = Vec::with_capacity(n);
+                    for _ in 0..n {
+                        vec.push(Self::primitive_read_one(&mut reader, swap_bytes)?);
+                    }
+                    Ok(vec)
+                }
+            }
+
+
             #[inline]
             fn primitive_write_one<W: io::Write>(&self, writer: W, swap_bytes: bool) -> io::Result<()> {
                 self.to_bits().primitive_write_one(writer, swap_bytes)
@@ -131,6 +177,11 @@ impl<T: PrimitiveReadWrite> TypeRead for PrimitiveReader<T> {
     fn read_one<R: io::Read>(&self, reader: R) -> io::Result<Self::Value> {
         T::primitive_read_one(reader, self.swap_bytes)
     }
+
+    fn read_many<R: io::Read>(&self, mut bytes: R,n:usize) -> io::Result<Vec<Self::Value>> {
+        T::primitive_read_many(&mut bytes, self.swap_bytes,n)
+    }
+
 }
 
 impl<T: PrimitiveReadWrite> TypeWrite for PrimitiveWriter<T> {

src/serialize/traits.rs

@@ -2,7 +2,7 @@ use std::io;
 use std::fmt;
 
 use crate::header::DType;
-use crate::type_str::{TypeStr};
+use crate::type_str::TypeStr;
 
 #[allow(unused)] // used by docstrings
 use crate::type_matchup_docs;
@@ -111,7 +111,17 @@ pub trait TypeRead {
 
     /// The function.
     fn read_one<R: io::Read>(&self, bytes: R) -> io::Result<Self::Value>
-        where Self: Sized;
+    where Self: Sized;
+
+    /// read n values from the reader
+    fn read_many<R: io::Read>(&self, mut bytes: R,n:usize) -> io::Result<Vec<Self::Value>>
+        where Self: Sized{
+        let mut vec = Vec::with_capacity(n);
+        for _ in 0..n {
+            vec.push(self.read_one(&mut bytes)?);
+        }
+        Ok(vec)
+    }
 }
 
 /// Like some sort of `for<W: io::Write> Fn(W, &T) -> io::Result<()>`.
@@ -141,13 +151,25 @@ pub trait TypeWrite {
 pub trait TypeReadDyn: TypeRead {
     #[doc(hidden)]
     fn read_one_dyn(&self, writer: &mut dyn io::Read) -> io::Result<Self::Value>;
+    #[doc(hidden)]
+    fn read_many_dyn(&self, mut reader: &mut dyn io::Read,n:usize) -> io::Result<Vec<Self::Value>>{
+        let mut data = Vec::with_capacity(n);
+        for _ in 0..n {
+            data.push(self.read_one_dyn(&mut reader)?);
+        }
+        Ok(data)
+    }
 }
 
 impl<T: TypeRead> TypeReadDyn for T {
     #[inline(always)]
     fn read_one_dyn(&self, reader: &mut dyn io::Read) -> io::Result<Self::Value> {
         self.read_one(reader)
     }
+
+    fn read_many_dyn(&self,  reader: &mut dyn io::Read,n:usize) -> io::Result<Vec<Self::Value>> {
+        self.read_many(reader, n)
+    }
 }
 
 impl<T> TypeRead for Box<dyn TypeReadDyn<Value=T>> {