Implement soft link and external link

exdir/__init__.py

@@ -1,7 +1,10 @@
 from . import core
 from . import plugin_interface
 from . import plugins
-from .core import File, validation, Attribute, Dataset, Group, Raw, Object
+from .core import (
+    File, validation, Attribute, Dataset, Group, Raw, Object, SoftLink,
+    ExternalLink
+)
 
 # TODO remove versioneer
 from ._version import get_versions

exdir/core/__init__.py

@@ -11,3 +11,4 @@
 from .dataset import Dataset
 from .group import Group
 from .raw import Raw
+from .links import SoftLink, ExternalLink

exdir/core/constants.py

@@ -2,6 +2,15 @@
 EXDIR_METANAME = "exdir"
 TYPE_METANAME = "type"
 VERSION_METANAME = "version"
+LINK_METANAME = "link"
+TARGET_METANAME = "target"
+
+#links
+LINK_TYPENAME = "link"
+LINK_TARGETNAME = "target"
+LINK_EXTERNALNAME = "external"
+LINK_SOFTNAME = "soft"
+LINK_FILENAME = "file"
 
 # filenames
 META_FILENAME = "exdir.yaml"

exdir/core/exdir_file.py

@@ -71,7 +71,7 @@ class File(Group):
     def __init__(self, directory, mode=None, allow_remove=False,
                  name_validation=None, plugins=None):
         self._open_datasets = weakref.WeakValueDictionary({})
-        directory = pathlib.Path(directory) #.resolve()
+        directory = pathlib.Path(directory).absolute() #.resolve()
         if directory.suffix != ".exdir":
             directory = directory.with_suffix(directory.suffix + ".exdir")
         self.user_mode = mode = mode or 'a'
@@ -220,6 +220,12 @@ def __getitem__(self, name):
             return self
         return super(File, self).__getitem__(path)
 
+    def __setitem__(self, name, value):
+        path = utils.path.remove_root(name)
+        if len(path.parts) < 1:
+            return self
+        return super(File, self).__setitem__(path, value)
+
     def __contains__(self, name):
         path = utils.path.remove_root(name)
         return super(File, self).__contains__(path)

exdir/core/exdir_object.py

@@ -112,7 +112,8 @@ def is_nonraw_object_directory(directory):
             return False
         if TYPE_METANAME not in meta_data[EXDIR_METANAME]:
             return False
-        valid_types = [DATASET_TYPENAME, FILE_TYPENAME, GROUP_TYPENAME]
+        valid_types = [
+            DATASET_TYPENAME, FILE_TYPENAME, GROUP_TYPENAME, LINK_METANAME]
         if meta_data[EXDIR_METANAME][TYPE_METANAME] not in valid_types:
             return False
     return True

exdir/core/group.py

@@ -19,12 +19,15 @@
     import collections as abc
 
 from .exdir_object import Object
+from .links import Link, SoftLink, ExternalLink
 from .mode import assert_file_open, OpenMode, assert_file_writable
 from . import exdir_object as exob
+from . import exdir_file as exfile
 from . import dataset as ds
 from . import raw
 from .. import utils
 
+
 def _data_to_shape_and_dtype(data, shape, dtype):
     if data is not None:
         if shape is None:
@@ -36,6 +39,7 @@ def _data_to_shape_and_dtype(data, shape, dtype):
         dtype = np.float32
     return shape, dtype
 
+
 def _assert_data_shape_dtype_match(data, shape, dtype):
     if data is not None:
         if shape is not None and np.product(shape) != np.product(data.shape):
@@ -53,6 +57,7 @@ def _assert_data_shape_dtype_match(data, shape, dtype):
             )
         return
 
+
 class Group(Object):
     """
     Container of other groups and datasets.
@@ -403,6 +408,8 @@ def __getitem__(self, name):
             return self._dataset(name)
         elif meta_data[exob.EXDIR_METANAME][exob.TYPE_METANAME] == exob.GROUP_TYPENAME:
             return self._group(name)
+        elif meta_data[exob.EXDIR_METANAME][exob.TYPE_METANAME] == exob.LINK_TYPENAME:
+            return self._link(name)
         else:
             error_string = (
                 "Object {name} has data type {type}.\n"
@@ -413,6 +420,25 @@ def __getitem__(self, name):
             )
             raise NotImplementedError(error_string)
 
+    def _link(self, name, get_link=False):
+        link_meta = self._group(name).meta[exob.EXDIR_METANAME][exob.LINK_METANAME]
+        print(link_meta)
+        if link_meta[exob.TYPE_METANAME] == exob.LINK_SOFTNAME:
+            if get_link:
+                result = SoftLink(link_meta[exob.LINK_TARGETNAME])
+            else:
+                result = self[link_meta[exob.LINK_TARGETNAME]]
+        elif link_meta[exob.TYPE_METANAME] == exob.LINK_EXTERNALNAME:
+            if get_link:
+                result = ExternalLink(
+                    link_meta[exob.LINK_FILENAME],
+                    link_meta[exob.LINK_TARGETNAME])
+            else:
+                external_file = exfile.File(
+                    link_meta[exob.LINK_FILENAME], 'r')
+                result = external_file[link_meta[exob.LINK_TARGETNAME]]
+        return result
+
     def _dataset(self, name):
         return ds.Dataset(
             root_directory=self.root_directory,
@@ -439,6 +465,13 @@ def __setitem__(self, name, value):
             self[path.parent][path.name] = value
             return
 
+        if isinstance(value, Link):
+            link_group = self.create_group(name)
+            # if value.path not in self.file:
+            #     return # TODO works when merging with lepmik/close
+            link_group.meta[exob.EXDIR_METANAME].update(value._link)
+            return
+
         if name not in self:
             self.create_dataset(name, data=value)
             return
@@ -509,20 +542,22 @@ def __len__(self):
         assert_file_open(self.file)
         return len([a for a in self])
 
-    def get(self, key):
+    def get(self, name, get_link=False):
         """
         Get an object in the group.
         Parameters
         ----------
-        key : str
-            The key of the desired object
+        name : str
+            The name of the desired object
         Returns
         -------
         Value or None if object does not exist.
         """
         assert_file_open(self.file)
-        if key in self:
-            return self[key]
+        if name in self:
+            if get_link:
+                return self._link(name, get_link)
+            return self[name]
         else:
             return None
 

exdir/core/links.py

@@ -0,0 +1,69 @@
+try:
+    import pathlib
+except ImportError as e:
+    try:
+        import pathlib2 as pathlib
+    except ImportError:
+        raise e
+from . import exdir_file
+from .exdir_object import Object, is_nonraw_object_directory
+from .constants import *
+
+
+class Link(Object):
+    """
+    Super class for link objects
+    """
+    def __init__(self, path):
+        self.path = path
+
+    @property
+    def _link(self):
+        return {TYPE_METANAME: LINK_TYPENAME}
+
+    def __eq__(self, other):
+        return self._link.get(LINK_METANAME) == other._link.get(LINK_METANAME)
+
+
+class SoftLink(Link):
+    def __init__(self, path):
+        super(SoftLink, self).__init__(
+            path=path
+        )
+
+    @property
+    def _link(self):
+        result = {
+            TYPE_METANAME: LINK_TYPENAME,
+            LINK_METANAME: {
+                TYPE_METANAME: LINK_SOFTNAME,
+                LINK_TARGETNAME: self.path
+             }
+        }
+        return result
+
+    def __repr__(self):
+        return "Exdir SoftLink '{}' at {}".format(self.path, id(self))
+
+
+class ExternalLink(Link):
+    def __init__(self, filename, path):
+        super(ExternalLink, self).__init__(
+            path=path
+        )
+        self.filename = filename
+
+    @property
+    def _link(self):
+        result = {
+            TYPE_METANAME: LINK_TYPENAME,
+            LINK_METANAME: {
+                TYPE_METANAME: LINK_EXTERNALNAME,
+                LINK_TARGETNAME: self.path,
+                LINK_FILENAME: str(self.filename)
+             }
+        }
+        return result
+
+    def __repr__(self):
+        return "Exdir SoftLink '{}' at {}".format(self.path, id(self))