object array handling

[espenhgn]

Seems like datasets from object arrays leaves files written to file that cannot be overwritten using another datatype:

import exdir
import numpy as np
import os
fname = 'test.exdir'
os.system('rm -f {}'.format(fname))
data = np.array(['ole', 'dole', 'doffen'])
f = exdir.File(fname, 'w', allow_remove=True)
f.require_dataset('bros', data=data) # all swell
try:
    f.require_dataset('nephews', data=data.astype('object')) # not so swell
except ValueError:
    pass # Ignore that it failed, try storing as str array instead:
f.require_dataset('nephews', data=data.astype(str))
f.close()

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-12-e5b64964c3dc> in <module>()
     11 except ValueError:
     12     pass # Ignore that it failed, try storing as str instead:
---> 13 f.require_dataset('nephews', data=data.astype(str))
     14 f.close()

~/COBRA/expipe-dev/exdir/exdir/core/group.py in require_dataset(self, name, shape, dtype, exact, data, fillvalue)
    144             )
    145 
--> 146         current_object = self[name]
    147 
    148         if not isinstance(current_object, ds.Dataset):

~/COBRA/expipe-dev/exdir/exdir/core/exdir_file.py in __getitem__(self, name)
     96         if len(path.parts) < 1:
     97             return self
---> 98         return super().__getitem__(path)
     99 
    100     def __contains__(self, name):

~/COBRA/expipe-dev/exdir/exdir/core/group.py in __getitem__(self, name)
    223                 object_name=name,
    224                 io_mode=self.io_mode,
--> 225                 validate_name=self.validate_name
    226             )
    227         elif meta_data[exob.EXDIR_METANAME][exob.TYPE_METANAME] == exob.GROUP_TYPENAME:

~/COBRA/expipe-dev/exdir/exdir/core/dataset.py in __init__(self, root_directory, parent_path, object_name, io_mode, validate_name)
     72 
     73         self.data_filename = str(_dataset_filename(self.directory))
---> 74         self._reload()
     75 
     76     def __getitem__(self, args):

~/COBRA/expipe-dev/exdir/exdir/core/dataset.py in _reload(self)
     98 
     99     def _reload(self):
--> 100         self._data = np.load(self.data_filename, mmap_mode=self._mmap_mode)
    101 
    102     def _reset(self, value):

~/anaconda3/envs/phy/lib/python3.5/site-packages/numpy/lib/npyio.py in load(file, mmap_mode, allow_pickle, fix_imports, encoding)
    414             # .npy file
    415             if mmap_mode:
--> 416                 return format.open_memmap(file, mode=mmap_mode)
    417             else:
    418                 return format.read_array(fid, allow_pickle=allow_pickle,

~/anaconda3/envs/phy/lib/python3.5/site-packages/numpy/lib/format.py in open_memmap(filename, mode, dtype, shape, fortran_order, version)
    774             if dtype.hasobject:
    775                 msg = "Array can't be memory-mapped: Python objects in dtype."
--> 776                 raise ValueError(msg)
    777             offset = fp.tell()
    778         finally:

ValueError: Array can't be memory-mapped: Python objects in dtype.

[espenhgn]

Btw., this was occurring on https://github.com/espenhgn/expipe-dev/tree/kilosort, but I haven't touched the exdir subrepo.

[dragly]

This should not be allowed. We'll track this issue in #47