Shallow copy of npy_array_t (#31)

* add interface to build npy_array_list objects from C

* rename to deepcopy

* deepcopy and shallow copy

* fix readme

* add copying example to workflow

* left a note in copying.c for having the same data as copying_read.c

---------

Co-authored-by: alcubierre-drive <>

Author: alcubierre-drive

.github/workflows/make_and_exmples.yml

@@ -33,7 +33,9 @@ jobs:
          ./how_to_save_npz
          ./how_to_load my_4_by_3_array_shortcut.npy
          ./how_to_load_npz iarray_and_darray.npz
-    - name: Test install 
+         ./copying
+         ./copying_read
+    - name: Test install
       run: |
          sudo make install
          unset LD_LIBRARY_PATH

README.md

@@ -119,12 +119,13 @@ And the linked list structure for `.npz` files:
     } npy_array_list_t;
 
 ## API
-The API is really simple. There is only ~~eleven~~twelve public functions:
+The API is really simple. There is only 13 public functions:
 
     /* These are the four functions for loading and saving .npy files */
     npy_array_t*      npy_array_load        ( const char *filename);
     npy_array_t*      npy_array_mmap        ( const char *filename);
-    npy_array_t*      npy_array_alloc       ( const npy_array_t *m );
+    npy_array_t*      npy_array_deepcopy    ( const npy_array_t *m );
+    npy_array_t*      npy_array_copy        ( const npy_array_t *m );
     void              npy_array_dump        ( const npy_array_t *m );
     void              npy_array_save        ( const char *filename, const npy_array_t *m );
     void              npy_array_free        ( npy_array_t *m );
@@ -167,13 +168,22 @@ compile this with:
 You can the run example_list with a filename (_NumPy_ compressed) as argument.
 
     #include "npy_array_list.h"
+
     int main(int argc, char *argv[])
     {
         if( argc != 2 ) return -1;
+
         double data[] = {0,1,2,3,4,5};
+
         npy_array_list_t* list = NULL;
-        list = npy_array_list_append( list, NPY_ARRAY_ALLOCATOR(data, SHAPE(3,2), NPY_DTYPE_FLOAT64), "matrix" );
-        list = npy_array_list_append( list, NPY_ARRAY_ALLOCATOR(data, SHAPE(2,1,2), NPY_DTYPE_FLOAT64), "tensor" );
+
+        // the first npy_array_t* holds a reference to the data array
+        list = npy_array_list_append( list,
+            NPY_ARRAY_BUILDER_COPY(data, SHAPE(3,2), NPY_DTYPE_FLOAT64), "matrix" );
+        // the second npy_array_t* holds a copy of the data array (hence DEEPCOPY)
+        list = npy_array_list_append( list,
+            NPY_ARRAY_BUILDER_DEEPCOPY(data, SHAPE(2,1,2), NPY_DTYPE_FLOAT64), "tensor" );
+
         npy_array_list_save_compressed( argv[1], list, ZIP_CM_DEFAULT, 0 );
         npy_array_list_free( list );
     }

example/Makefile

@@ -1,9 +1,9 @@
-CC = gcc
-CFLAGS = -std=c99 -Wall -Wextra -O3 -I.. 
-LDLIBS = -L.. -lnpy_array
+CC ?= gcc
+CFLAGS += -std=c99 -Wall -Wextra -O3 -I..
+LDLIBS += -L.. -lnpy_array
 
 SOURCES = $(wildcard *.c)
-PROGRAMS = $(patsubst %.c,%,$(SOURCES)) 
+PROGRAMS = $(patsubst %.c,%,$(SOURCES))
 
 all: $(PROGRAMS)
 

example/copying.c

@@ -0,0 +1,25 @@
+#include "npy_array_list.h"
+
+// the first block must be the same as in copying_read.c
+int main(int argc, char *argv[])
+{
+    const char* fname = (argc != 2) ? "copy.npz" : argv[1];
+
+    const double data[] = {0,1,2, 3,4,5};
+    const int32_t idata[] = {0,1, 2,3,
+                             4,5, 6,7,
+                             8,9, 10,11};
+    const char names[][8] = { "double", "int" };
+
+    npy_array_list_t* list = NULL;
+
+    // the first npy_array_t* holds a reference to the data array
+    list = npy_array_list_append( list,
+        NPY_ARRAY_BUILDER_COPY(data, SHAPE(2,3), NPY_DTYPE_FLOAT64), names[0] );
+    // the second npy_array_t* holds a copy of the data array (hence DEEPCOPY)
+    list = npy_array_list_append( list,
+        NPY_ARRAY_BUILDER_DEEPCOPY(idata, SHAPE(3,2,2), NPY_DTYPE_INT32), names[1] );
+
+    npy_array_list_save_compressed( fname, list, ZIP_CM_DEFAULT, 0 );
+    npy_array_list_free( list );
+}

example/copying_read.c

@@ -0,0 +1,58 @@
+#include "npy_array_list.h"
+
+#include <string.h>
+#define MIN(x,y) ((x)<(y)?(x):(y))
+
+// the first block must be the same as in copying.c
+int main(int argc, char *argv[])
+{
+    const char* fname = (argc != 2) ? "copy.npz" : argv[1];
+
+    const double data[] = {0,1,2, 3,4,5};
+    const int32_t idata[] = {0,1, 2,3,
+                             4,5, 6,7,
+                             8,9, 10,11};
+    const char names[][8] = { "double", "int" };
+
+    // we load the file and check whether it contains the same data
+    npy_array_list_t *list = npy_array_list_load( fname );
+
+    int errors = 0;
+    int visited_data = 0,
+        visited_idata = 0;
+    npy_array_list_t *elem = list;
+    while (elem != NULL) {
+        npy_array_t* ary = elem->array;
+        size_t ary_sz = npy_array_calculate_datasize( ary );
+
+        char* cmp = NULL;
+        size_t cmp_sz = 0;
+
+        if (!strcmp( elem->filename, names[0] )) {
+            cmp = (char*)data;
+            cmp_sz = sizeof(data);
+            visited_data++;
+        } else if (!strcmp( elem->filename, names[1] )) {
+            cmp = (char*)idata;
+            cmp_sz = sizeof(idata);
+            visited_idata++;
+        }
+
+        errors += (cmp_sz != ary_sz);
+        if (cmp != NULL) {
+            errors += (memcmp( ary->data, cmp, MIN(cmp_sz,ary_sz) ) != 0);
+        } else {
+            errors++;
+        }
+
+        elem = elem->next;
+    }
+
+    errors += (visited_data != 1);
+    errors += (visited_idata != 1);
+
+    npy_array_list_free( list );
+
+    // we return the number of errors/inconsistencies
+    return errors;
+}

npy_array.c

@@ -257,7 +257,7 @@ npy_array_t * _read_matrix( void *fp, reader_func read_func )
         return m;
     }
 
-    m->data = malloc( n_elements * m->elem_size );
+    m->memory = m->data = malloc( n_elements * m->elem_size );
     if ( !m->data ){
         fprintf(stderr, "Cannot allocate memory for matrix data.\n");
         free( m );
@@ -392,8 +392,9 @@ void npy_array_free( npy_array_t *m )
 
         size_t len = NPY_ARRAY_PREHEADER_LENGTH + header_length + npy_array_calculate_datasize(m);
         munmap( m->map_addr, len );
-    } else
-        free( m->data );
+    } else if ( m->memory ) {
+        free( m->memory );
+    }
 
     free( m );
 }
@@ -409,7 +410,7 @@ npy_array_t* npy_array_deepcopy( const npy_array_t* m ) {
     memcpy( ary->shape, m->shape, sizeof(ary->shape) );
     ary->typechar = m->typechar;
     ary->elem_size = m->elem_size;
-    ary->data = malloc( npy_array_calculate_datasize(ary) );
+    ary->memory = ary->data = malloc( npy_array_calculate_datasize(ary) );
     if (!ary->data) {
         fprintf(stderr, "Cannot allocate memory!\n");
         free(ary);
@@ -418,3 +419,17 @@ npy_array_t* npy_array_deepcopy( const npy_array_t* m ) {
     memcpy( ary->data, m->data, npy_array_calculate_datasize(ary) );
     return ary;
 }
+
+npy_array_t* npy_array_copy( const npy_array_t* m ) {
+    npy_array_t* ary = calloc( 1, sizeof(*ary) );
+    if (!ary) {
+        fprintf(stderr, "Cannot allocate data structure!\n");
+        return NULL;
+    }
+    ary->ndim = MIN( m->ndim, NPY_ARRAY_MAX_DIMENSIONS );
+    memcpy( ary->shape, m->shape, sizeof(ary->shape) );
+    ary->typechar = m->typechar;
+    ary->elem_size = m->elem_size;
+    ary->data = m->data;
+    return ary;
+}

npy_array.h

@@ -53,14 +53,16 @@ typedef struct _npy_array_t {
     char              typechar;
     size_t            elem_size;
     bool              fortran_order;
-    /* Consider map_addr as a private member. Do modify this pointer! Used for unmap() */
+    /* Consider map_addr and memory as a private members. Do not modify these pointers! */
     void             *map_addr;      /* pointer to the map if array is mmap()'ed -- else NULL */
+    char             *memory;        /* pointer to memory if array owns it -- else NULL */
 } npy_array_t;
 
 
 npy_array_t*      npy_array_load       ( const char *filename );
 npy_array_t*      npy_array_mmap       ( const char *filename );
 npy_array_t*      npy_array_deepcopy   ( const npy_array_t *m );
+npy_array_t*      npy_array_copy       ( const npy_array_t *m );
 void              npy_array_dump       ( const npy_array_t *m );
 void              npy_array_save       ( const char *filename, const npy_array_t *m );
 void              npy_array_free       ( npy_array_t *m );
@@ -87,7 +89,9 @@ npy_array_t *     _read_matrix( void *fp, reader_func read_func );
 #define SHAPE(...) .shape = {__VA_ARGS__}, .ndim = _NARG(__VA_ARGS__)
 #define NPY_ARRAY_BUILDER(_data,_shape,...) \
     &(npy_array_t){ .data=(char*)_data, _shape, __VA_ARGS__ } 
-#define NPY_ARRAY_ALLOCATOR(...) \
+#define NPY_ARRAY_BUILDER_COPY(...) \
+    npy_array_copy(NPY_ARRAY_BUILDER(__VA_ARGS__))
+#define NPY_ARRAY_BUILDER_DEEPCOPY(...) \
     npy_array_deepcopy(NPY_ARRAY_BUILDER(__VA_ARGS__))
 
 #define NPY_DTYPE_FLOAT16    .typechar='f', .elem_size=2