diff --git a/exdir/__init__.py b/exdir/__init__.py index 2092ccb..16e7125 100644 --- a/exdir/__init__.py +++ b/exdir/__init__.py @@ -1,7 +1,10 @@ from . import core from . import plugin_interface from . import plugins -from .core import File, validation, Attribute, Dataset, Group, Raw, Object +from .core import ( + File, validation, Attribute, Dataset, Group, Raw, Object, SoftLink, + ExternalLink +) # TODO remove versioneer from ._version import get_versions diff --git a/exdir/core/__init__.py b/exdir/core/__init__.py index 474ff48..11caae6 100644 --- a/exdir/core/__init__.py +++ b/exdir/core/__init__.py @@ -11,3 +11,4 @@ from .dataset import Dataset from .group import Group from .raw import Raw +from .links import SoftLink, ExternalLink diff --git a/exdir/core/constants.py b/exdir/core/constants.py index eb833aa..8edda4f 100644 --- a/exdir/core/constants.py +++ b/exdir/core/constants.py @@ -2,6 +2,15 @@ EXDIR_METANAME = "exdir" TYPE_METANAME = "type" VERSION_METANAME = "version" +LINK_METANAME = "link" +TARGET_METANAME = "target" + +#links +LINK_TYPENAME = "link" +LINK_TARGETNAME = "target" +LINK_EXTERNALNAME = "external" +LINK_SOFTNAME = "soft" +LINK_FILENAME = "file" # filenames META_FILENAME = "exdir.yaml" diff --git a/exdir/core/exdir_file.py b/exdir/core/exdir_file.py index 82a07d9..097e29b 100644 --- a/exdir/core/exdir_file.py +++ b/exdir/core/exdir_file.py @@ -71,7 +71,7 @@ class File(Group): def __init__(self, directory, mode=None, allow_remove=False, name_validation=None, plugins=None): self._open_datasets = weakref.WeakValueDictionary({}) - directory = pathlib.Path(directory) #.resolve() + directory = pathlib.Path(directory).absolute() #.resolve() if directory.suffix != ".exdir": directory = directory.with_suffix(directory.suffix + ".exdir") self.user_mode = mode = mode or 'a' @@ -220,6 +220,12 @@ def __getitem__(self, name): return self return super(File, self).__getitem__(path) + def __setitem__(self, name, value): + path = utils.path.remove_root(name) + if len(path.parts) < 1: + return self + return super(File, self).__setitem__(path, value) + def __contains__(self, name): path = utils.path.remove_root(name) return super(File, self).__contains__(path) diff --git a/exdir/core/exdir_object.py b/exdir/core/exdir_object.py index a4b4bcf..e6994a0 100644 --- a/exdir/core/exdir_object.py +++ b/exdir/core/exdir_object.py @@ -112,7 +112,8 @@ def is_nonraw_object_directory(directory): return False if TYPE_METANAME not in meta_data[EXDIR_METANAME]: return False - valid_types = [DATASET_TYPENAME, FILE_TYPENAME, GROUP_TYPENAME] + valid_types = [ + DATASET_TYPENAME, FILE_TYPENAME, GROUP_TYPENAME, LINK_METANAME] if meta_data[EXDIR_METANAME][TYPE_METANAME] not in valid_types: return False return True diff --git a/exdir/core/group.py b/exdir/core/group.py index 53be5e6..106895f 100644 --- a/exdir/core/group.py +++ b/exdir/core/group.py @@ -19,12 +19,15 @@ import collections as abc from .exdir_object import Object +from .links import Link, SoftLink, ExternalLink from .mode import assert_file_open, OpenMode, assert_file_writable from . import exdir_object as exob +from . import exdir_file as exfile from . import dataset as ds from . import raw from .. import utils + def _data_to_shape_and_dtype(data, shape, dtype): if data is not None: if shape is None: @@ -36,6 +39,7 @@ def _data_to_shape_and_dtype(data, shape, dtype): dtype = np.float32 return shape, dtype + def _assert_data_shape_dtype_match(data, shape, dtype): if data is not None: if shape is not None and np.product(shape) != np.product(data.shape): @@ -53,6 +57,7 @@ def _assert_data_shape_dtype_match(data, shape, dtype): ) return + class Group(Object): """ Container of other groups and datasets. @@ -403,6 +408,8 @@ def __getitem__(self, name): return self._dataset(name) elif meta_data[exob.EXDIR_METANAME][exob.TYPE_METANAME] == exob.GROUP_TYPENAME: return self._group(name) + elif meta_data[exob.EXDIR_METANAME][exob.TYPE_METANAME] == exob.LINK_TYPENAME: + return self._link(name) else: error_string = ( "Object {name} has data type {type}.\n" @@ -413,6 +420,25 @@ def __getitem__(self, name): ) raise NotImplementedError(error_string) + def _link(self, name, get_link=False): + link_meta = self._group(name).meta[exob.EXDIR_METANAME][exob.LINK_METANAME] + print(link_meta) + if link_meta[exob.TYPE_METANAME] == exob.LINK_SOFTNAME: + if get_link: + result = SoftLink(link_meta[exob.LINK_TARGETNAME]) + else: + result = self[link_meta[exob.LINK_TARGETNAME]] + elif link_meta[exob.TYPE_METANAME] == exob.LINK_EXTERNALNAME: + if get_link: + result = ExternalLink( + link_meta[exob.LINK_FILENAME], + link_meta[exob.LINK_TARGETNAME]) + else: + external_file = exfile.File( + link_meta[exob.LINK_FILENAME], 'r') + result = external_file[link_meta[exob.LINK_TARGETNAME]] + return result + def _dataset(self, name): return ds.Dataset( root_directory=self.root_directory, @@ -439,6 +465,13 @@ def __setitem__(self, name, value): self[path.parent][path.name] = value return + if isinstance(value, Link): + link_group = self.create_group(name) + # if value.path not in self.file: + # return # TODO works when merging with lepmik/close + link_group.meta[exob.EXDIR_METANAME].update(value._link) + return + if name not in self: self.create_dataset(name, data=value) return @@ -509,20 +542,22 @@ def __len__(self): assert_file_open(self.file) return len([a for a in self]) - def get(self, key): + def get(self, name, get_link=False): """ Get an object in the group. Parameters ---------- - key : str - The key of the desired object + name : str + The name of the desired object Returns ------- Value or None if object does not exist. """ assert_file_open(self.file) - if key in self: - return self[key] + if name in self: + if get_link: + return self._link(name, get_link) + return self[name] else: return None diff --git a/exdir/core/links.py b/exdir/core/links.py new file mode 100644 index 0000000..dbe0312 --- /dev/null +++ b/exdir/core/links.py @@ -0,0 +1,69 @@ +try: + import pathlib +except ImportError as e: + try: + import pathlib2 as pathlib + except ImportError: + raise e +from . import exdir_file +from .exdir_object import Object, is_nonraw_object_directory +from .constants import * + + +class Link(Object): + """ + Super class for link objects + """ + def __init__(self, path): + self.path = path + + @property + def _link(self): + return {TYPE_METANAME: LINK_TYPENAME} + + def __eq__(self, other): + return self._link.get(LINK_METANAME) == other._link.get(LINK_METANAME) + + +class SoftLink(Link): + def __init__(self, path): + super(SoftLink, self).__init__( + path=path + ) + + @property + def _link(self): + result = { + TYPE_METANAME: LINK_TYPENAME, + LINK_METANAME: { + TYPE_METANAME: LINK_SOFTNAME, + LINK_TARGETNAME: self.path + } + } + return result + + def __repr__(self): + return "Exdir SoftLink '{}' at {}".format(self.path, id(self)) + + +class ExternalLink(Link): + def __init__(self, filename, path): + super(ExternalLink, self).__init__( + path=path + ) + self.filename = filename + + @property + def _link(self): + result = { + TYPE_METANAME: LINK_TYPENAME, + LINK_METANAME: { + TYPE_METANAME: LINK_EXTERNALNAME, + LINK_TARGETNAME: self.path, + LINK_FILENAME: str(self.filename) + } + } + return result + + def __repr__(self): + return "Exdir SoftLink '{}' at {}".format(self.path, id(self)) diff --git a/tests/test_links.py b/tests/test_links.py new file mode 100644 index 0000000..c0b9386 --- /dev/null +++ b/tests/test_links.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- + +# This file is part of Exdir, the Experimental Directory Structure. +# +# Copyright 2019 Mikkel Lepperød +# +# License: MIT, see "LICENSE" file for the full license terms. +# +# This file contains code from h5py, a Python interface to the HDF5 library, +# licensed under a standard 3-clause BSD license +# with copyright Andrew Collette and contributors. +# See http://www.h5py.org and the "3rdparty/h5py-LICENSE" file for details. + +from exdir import SoftLink, ExternalLink, File +import pytest +import numpy as np +try: + import ruamel_yaml as yaml +except ImportError: + import ruamel.yaml as yaml + + +def test_soft_links(setup_teardown_file): + """ Broken softlinks are contained, but their members are not """ + f = setup_teardown_file[3] + f.create_group('mongoose') + f.create_group('grp') + f['/grp/soft'] = SoftLink('/mongoose') + assert '/grp/soft' in f + assert '/grp/soft/something' not in f + + +def test_external_links(setup_teardown_file): + """ Broken softlinks are contained, but their members are not """ + f = setup_teardown_file[3] + g = File(setup_teardown_file[0] / 'mongoose.exdir', 'w') + g.create_group('mongoose') + f.create_group('grp') + f['/grp/external'] = ExternalLink('mongoose.exdir', '/mongoose') + assert '/grp/external' in f + assert '/grp/external/something' not in f + + +def test_get_link(setup_teardown_file): + """ Get link values """ + f = setup_teardown_file[3] + g = File(setup_teardown_file[0] / 'somewhere.exdir') + f.create_group('mongoose') + g.create_group('mongoose') + sl = SoftLink('/mongoose') + el = ExternalLink('somewhere.exdir', 'mongoose') + + f['soft'] = sl + f['external'] = el + + out_sl = f.get('soft', get_link=True) + out_el = f.get('external', get_link=True) + + assert isinstance(out_sl, SoftLink) + assert out_sl == sl + assert isinstance(out_el, ExternalLink) + assert out_el == el + + +# Feature: Create and manage soft links with the high-level interface +def test_soft_path(setup_teardown_file): + """ SoftLink directory attribute """ + sl = SoftLink('/foo') + assert sl.path == '/foo' + + +def test_soft_repr(setup_teardown_file): + """ SoftLink path repr """ + sl = SoftLink('/foo') + assert isinstance(repr(sl), str) + + +def test_linked_group_equal(setup_teardown_file): + """ Create new soft link by assignment """ + f = setup_teardown_file[3] + g = f.create_group('new') + sl = SoftLink('/new') + f['alias'] = sl + g2 = f['alias'] + assert g == g2 + + +def test_exc(setup_teardown_file): + """ Opening dangling soft link results in KeyError """ + f = setup_teardown_file[3] + f['alias'] = SoftLink('new') + with pytest.raises(KeyError): + f['alias'] + + +# Feature: Create and manage external links +def test_external_path(setup_teardown_file): + """ External link paths attributes """ + external_path = setup_teardown_file[0] / 'foo.exdir' + g = File(external_path, 'w') + egrp = g.create_group('foo') + el = ExternalLink(external_path, '/foo') + assert el.filename == external_path + assert el.path == '/foo' + + +def test_external_repr(setup_teardown_file): + """ External link repr """ + external_path = setup_teardown_file[0] / 'foo.exdir' + g = File(external_path, 'w') + el = ExternalLink(external_path, '/foo') + assert isinstance(repr(el), str) + + +def test_create(setup_teardown_file): + """ Creating external links """ + external_path = setup_teardown_file[0] / 'foo.exdir' + f = setup_teardown_file[3] + g = File(external_path, 'w') + egrp = g.require_group('external') + f['ext'] = ExternalLink(external_path, '/external') + grp = f['ext'] + ef = grp.file + assert ef != f + assert grp.name == '/external' + + +def test_broken_external_link(setup_teardown_file): + """ KeyError raised when attempting to open broken link """ + external_path = setup_teardown_file[0] / 'foo.exdir' + f = setup_teardown_file[3] + g = File(external_path, 'w') + f['ext'] = ExternalLink(external_path, '/missing') + with pytest.raises(KeyError): + f['ext'] + + +def test_exc_missingfile(setup_teardown_file): + """ KeyError raised when attempting to open missing file """ + f = setup_teardown_file[3] + f['ext'] = ExternalLink('mongoose.exdir','/foo') + with pytest.raises(RuntimeError): + f['ext'] + + +def test_close_file(setup_teardown_file): + """ Files opened by accessing external links can be closed + """ + external_path = setup_teardown_file[0] / 'foo.exdir' + f = setup_teardown_file[3] + g = File(external_path, 'w') + f['ext'] = ExternalLink(external_path, '/') + grp = f['ext'] + f2 = grp.file + f2.close() + assert not f2 + +# TODO uncomment if we start accepting unicode names +# def test_unicode_encode(setup_teardown_file): +# """ +# Check that external links encode unicode filenames properly +# """ +# external_path = setup_teardown_file[0] / u"α.exdir" +# with File(external_path, "w") as ext_file: +# ext_file.create_group('external') +# f['ext'] = ExternalLink(external_path, '/external') +# +# +# def test_unicode_decode(setup_teardown_file): +# """ +# Check that external links decode unicode filenames properly +# """ +# external_path = setup_teardown_file[0] / u"α.exdir" +# with File(external_path, "w") as ext_file: +# ext_file.create_group('external') +# ext_file["external"].attrs["ext_attr"] = "test" +# f['ext'] = ExternalLink(external_path, '/external') +# assert f["ext"].attrs["ext_attr"] == "test" +# +# +# def test_unicode_exdir_path(setup_teardown_file): +# """ +# Check that external links handle unicode exdir paths properly +# """ +# external_path = setup_teardown_file[0] / u"external.exdir" +# with File(external_path, "w") as ext_file: +# ext_file.create_group(u'α') +# ext_file[u"α"].attrs["ext_attr"] = "test" +# f['ext'] = ExternalLink(external_path, u'/α') +# assertEqual(f["ext"].attrs["ext_attr"], "test")