diff --git a/pyilastik/ilastik_storage_version_01.py b/pyilastik/ilastik_storage_version_01.py index 675a5db..58e7bcf 100644 --- a/pyilastik/ilastik_storage_version_01.py +++ b/pyilastik/ilastik_storage_version_01.py @@ -69,6 +69,27 @@ def _get_slices_for(p_pos, q_pos, p_shape, q_shape): return [slice(start, stop) for start, stop in q_slice] +def normalize_dim_order(dim_order, data=None, reverse=False): + ''' + transpose tile data to dimension order zyxc or yxc + ''' + n_dims = len(dim_order) + assert n_dims in [3, 4] + + ref_order = 'zyxc' + if n_dims == 3: + ref_order = 'yxc' + + mapping = tuple([dim_order.find(k) for k in ref_order]) + if reverse: + mapping = tuple([ref_order.find(k) for k in dim_order]) + if data is None: + return mapping + + assert n_dims == len(data.shape) + return np.transpose(data, mapping) + + class IlastikStorageVersion01(object): def __init__(self, h5_handle, image_path=None, prediction=False, @@ -86,9 +107,8 @@ def __init__(self, h5_handle, image_path=None, prediction=False, assert version == '0.2' def ilastik_version(self): - - version_str = self.f.get('ilastikVersion')[()].decode() - return int(version_str.replace('.','')[:3]) + version_str = self.f.get('ilastikVersion')[()].decode() + return int(version_str.replace('.', '')[:3]) def __iter__(self): ''' @@ -153,12 +173,16 @@ def __getitem__(self, i): prediction = None # TODO # 1st get the (approximate) labeled image size - shape = self.shape_of_labelmatrix(i) + shape = self.shape_of_original_labelmatrix(i) n_dims = len(shape) tile_slice = np.array([[0, s] for s in shape]) + labels = self.tile_inner(i, tile_slice) - labels = self.tile(i, tile_slice) + if len(labels) > 0: + labels = normalize_dim_order( + self.original_dimension_order(), + data=labels) msg = 'dimensions of labelmatrix should be 4 (zyxc) or 3 (yxc)' assert n_dims in [3, 4], msg @@ -167,16 +191,12 @@ def __getitem__(self, i): # add z dimension if missing labels = np.expand_dims(labels, axis=0) - version = self.ilastik_version() if self.skip_image: - if version >= 133: - # if version>=1.3.3 - labels = np.transpose(labels, (0, 2, 3, 1)) return original_path, (None, labels, prediction) msg = ('ilastik versions > 1.3.2 are not supported. ' 'Set skip_image=True or use older ilastik version.') - assert version < 133, msg + assert self.ilastik_version() < 133, msg fname = utils.basename(path) if self.image_path is not None: @@ -202,7 +222,6 @@ def __getitem__(self, i): labels = np.pad(labels, padding, mode='constant', constant_values=0) - return original_path, (img, labels, prediction) def n_dims(self, item_index): @@ -219,7 +238,51 @@ def n_dims(self, item_index): else: return 0 + def original_dimension_order(self): + ''' + Dimension orders of label matrices depend on dimensionality + of the pixel dataset (zstack vs 2d, monochannel vs multichannel) and + the ilastik version. Dimension order handling was changed in + ilastik version 1.3.3 (both staorage version 01). + + ''' + s = self.shape_of_original_labelmatrix(0) + + assert len(s) in [3, 4] + + if len(s) == 4: + + assert s[3] == 1 or s[1] == 1 + + if self.ilastik_version() < 133: + order = 'zyxc' + else: + if s[1] == 1: + order = 'zcyx' + elif s[3] == 1: + order = 'zyxc' + if len(s) == 3: + if self.ilastik_version() < 133: + + assert s[2] == 1 + order = 'yxc' + else: + assert s[0] == 1 or s[2] == 1 + if s[0] == 1: + order = 'cyx' + else: + order = 'yxc' + + return order + def shape_of_labelmatrix(self, item_index): + original_shape = self.shape_of_original_labelmatrix(item_index) + dim_mapping = list(normalize_dim_order( + self.original_dimension_order())) + + return original_shape[dim_mapping] + + def shape_of_original_labelmatrix(self, item_index): ''' Label matrix shape is retrieved from label data. @@ -295,6 +358,18 @@ def load_block_data(self, item_index, block_index): return block[()] def tile(self, item_index, tile_slice): + + ordering = list(normalize_dim_order(self.original_dimension_order(), + reverse=True)) + slices_corr = tile_slice[ordering] + + t = self.tile_inner(item_index, slices_corr) + t_corr = normalize_dim_order(self.original_dimension_order(), + reverse=False, + data=t) + return t_corr + + def tile_inner(self, item_index, tile_slice): ''' Order is (Z, Y, X, C) or (Y, X, C) where C size of C dimension is always 1 (only one label channel implemented, i.e. @@ -320,7 +395,6 @@ def tile(self, item_index, tile_slice): p_slice = _get_slices_for(pos_q, pos_p, shape_q, shape_p) labels_q[q_slice] = labels_p[p_slice] - return labels_q @lru_cache(maxsize=None) diff --git a/pyilastik/tests/data/dimensionstest/x15_y10_z1_c1_classes2_ilastik1.3.3.ilp b/pyilastik/tests/data/dimensionstest/x15_y10_z1_c1_classes2_ilastik1.3.3.ilp new file mode 100644 index 0000000..c3a1d3c Binary files /dev/null and b/pyilastik/tests/data/dimensionstest/x15_y10_z1_c1_classes2_ilastik1.3.3.ilp differ diff --git a/pyilastik/tests/data/dimensionstest/x15_y10_z1_c2_classes2_ilastik1.3.3.ilp b/pyilastik/tests/data/dimensionstest/x15_y10_z1_c2_classes2_ilastik1.3.3.ilp new file mode 100644 index 0000000..c446196 Binary files /dev/null and b/pyilastik/tests/data/dimensionstest/x15_y10_z1_c2_classes2_ilastik1.3.3.ilp differ diff --git a/pyilastik/tests/data/dimensionstest/x15_y10_z2_c1_classes2_ilastik1.3.3.ilp b/pyilastik/tests/data/dimensionstest/x15_y10_z2_c1_classes2_ilastik1.3.3.ilp new file mode 100644 index 0000000..857ed17 Binary files /dev/null and b/pyilastik/tests/data/dimensionstest/x15_y10_z2_c1_classes2_ilastik1.3.3.ilp differ diff --git a/pyilastik/tests/data/dimensionstest/x15_y10_z2_c4_classes2_ilastik1.3.3.ilp b/pyilastik/tests/data/dimensionstest/x15_y10_z2_c4_classes2_ilastik1.3.3.ilp new file mode 100644 index 0000000..96233ae Binary files /dev/null and b/pyilastik/tests/data/dimensionstest/x15_y10_z2_c4_classes2_ilastik1.3.3.ilp differ diff --git a/pyilastik/tests/data/dimensionstest/x502_y251_z5_c1_classes2_ilastik1.3.3.ilp b/pyilastik/tests/data/dimensionstest/x502_y251_z5_c1_classes2_ilastik1.3.3.ilp new file mode 100644 index 0000000..f64f9c2 Binary files /dev/null and b/pyilastik/tests/data/dimensionstest/x502_y251_z5_c1_classes2_ilastik1.3.3.ilp differ diff --git a/pyilastik/tests/test_label_import.py b/pyilastik/tests/test_label_import.py index b6d37cb..2a69ef7 100644 --- a/pyilastik/tests/test_label_import.py +++ b/pyilastik/tests/test_label_import.py @@ -73,29 +73,16 @@ def test_multi_channel_multi_z(self): self.assertEqual(labels.shape, (2, 8, 10, 1)) assert_array_equal(labels[:, :, :, 0], val_multi_z) - # print(np.transpose(labels, (3, 0, 2, 1)).astype(int)) - # assert False - - def test_caching_of_block_slices(self): p = os.path.join(path, 'dimensionstest/x15_y10_z2_c4_classes2.ilp') ilp = pyilastik.read_project(p, skip_image=True) - s = np.array([[0, 2], [0, 10], [0, 15], [0, 1]]) - bs_before = ilp._get_block_slices(0).copy() - tile = ilp.tile(0, s) bs_after = ilp._get_block_slices(0) print(bs_before) assert_array_equal(bs_before, bs_after) - - - - - - def test_multi_channel_single_z(self): p = os.path.join(path, 'dimensionstest/x15_y10_z1_c2_classes2.ilp') @@ -162,7 +149,6 @@ def test_get_block_slices(self): localpath = os.path.join(path, 'purkinjetest') p = os.path.join(localpath, 'ilastik-1.2.2post1mac.ilp') - #p = os.path.join(path, 'dimensionstest/x15_y10_z2_rgb_classes2.ilp') ilp = pyilastik.read_project(p, skip_image=True) b = ilp._get_block_slices(0) @@ -177,7 +163,6 @@ def test_get_block_slices(self): def test_blocks_in_tile(self): - localpath = os.path.join(path, 'purkinjetest') p = os.path.join(localpath, 'ilastik-1.2.2post1mac.ilp') @@ -192,7 +177,6 @@ def test_blocks_in_tile(self): self.assertFalse(b[1]) self.assertEqual(len(b), 2) - p = os.path.join(path, 'dimensionstest/x502_y251_z5_c1_classes2.ilp') ilp = pyilastik.read_project(p, skip_image=True) @@ -203,21 +187,18 @@ def test_blocks_in_tile(self): assert_array_equal(ilp._blocks_in_tile(0, tile_slices), np.array([True, False, False])) - - def test_label_order_is_consistent_between_ilastik_versions(self): - item_index = 1 p12 = os.path.join(path, 'multiim/ilastik-multiim-1.2.ilp') p133 = os.path.join(path, 'multiim/ilastik-multiim-1.3.3.ilp') p132 = os.path.join(path, 'multiim/ilastik-multiim-1.3.2.ilp') - fname = 'pixels_ilastik-multiim-1.2/34width_28height_2slices_2channels.tif' + fname = \ + 'pixels_ilastik-multiim-1.2/34width_28height_2slices_2channels.tif' ilp12 = pyilastik.read_project(p12, skip_image=True) (_, (_, labels_12, _)) = ilp12[fname] - ilp13 = pyilastik.read_project(p133, skip_image=True) (_, (_, labels_133, _)) = ilp13[fname] @@ -227,6 +208,221 @@ def test_label_order_is_consistent_between_ilastik_versions(self): assert_array_equal(labels_12, labels_133) assert_array_equal(labels_12, labels_132) + def test_tile_c1_z1(self): + + tile_slice = np.array([[0, 8], + [0, 10], + [0, 1]]) + + p12flat = os.path.join( + path, 'dimensionstest/x15_y10_z1_c1_classes2.ilp') + ilp12flat = pyilastik.read_project(p12flat, skip_image=True) + (_, (_, labels_12_flat, _)) = ilp12flat['x15_y10_z1_c1.tif'] + t12flat = ilp12flat.tile(0, tile_slice) + print('labelmat shape 12flat z1 c1') + print(ilp12flat.shape_of_labelmatrix(0)) + print(labels_12_flat.shape) + ilp12flat.original_dimension_order() + + p133flat = os.path.join( + path, 'dimensionstest/x15_y10_z1_c1_classes2_ilastik1.3.3.ilp') + ilp133flat = pyilastik.read_project(p133flat, skip_image=True) + print(ilp133flat.image_path_list()) + (_, (_, labels_133_flat, _)) = ilp133flat['x15_y10_z1_c1.tif'] + t133flat = ilp133flat.tile(0, tile_slice) + print('labelmat shape 133flat z1 c1') + print(ilp133flat.shape_of_labelmatrix(0)) + print(labels_133_flat.shape) + ilp133flat.original_dimension_order() + + assert_array_equal(t12flat, t133flat) + assert_array_equal(labels_12_flat, labels_133_flat) + + def test_tile_c2_z1(self): + + tile_slice = np.array([[0, 8], + [0, 10], + [0, 1]]) + + p12flat = os.path.join( + path, 'dimensionstest/x15_y10_z1_c2_classes2.ilp') + ilp12flat = pyilastik.read_project(p12flat, skip_image=True) + t12flat = ilp12flat.tile(0, tile_slice) + ilp12flat.original_dimension_order() + + p133flat = os.path.join( + path, 'dimensionstest/x15_y10_z1_c2_classes2_ilastik1.3.3.ilp') + ilp133flat = pyilastik.read_project(p133flat, skip_image=True) + t133flat = ilp133flat.tile(0, tile_slice) + + assert_array_equal(t12flat, t133flat) + + def test_tile_c1_z2(self): + + tile_slice = np.array([[0, 2], + [0, 8], + [0, 10], + [0, 1]]) + + p12flat = os.path.join( + path, 'dimensionstest/x15_y10_z2_c1_classes2.ilp') + ilp12flat = pyilastik.read_project(p12flat, skip_image=True) + t12flat = ilp12flat.tile(0, tile_slice) + + p133flat = os.path.join( + path, 'dimensionstest/x15_y10_z2_c1_classes2_ilastik1.3.3.ilp') + ilp133flat = pyilastik.read_project(p133flat, skip_image=True) + t133flat = ilp133flat.tile(0, tile_slice) + + assert_array_equal(t12flat, t133flat) + + def test_tile_c4_z2(self): + + tile_slice = np.array([[0, 2], + [0, 8], + [0, 10], + [0, 1]]) + + p = os.path.join( + path, 'dimensionstest/x15_y10_z2_c4_classes2.ilp') + ilp12 = pyilastik.read_project(p, skip_image=True) + t12 = ilp12.tile(0, tile_slice) + + p = os.path.join( + path, 'dimensionstest/x15_y10_z2_c4_classes2_ilastik1.3.3.ilp') + ilp133 = pyilastik.read_project(p, skip_image=True) + t133 = ilp133.tile(0, tile_slice) + + assert_array_equal(t12, t133) + + def test_get_labels(self): + + p = os.path.join( + path, 'dimensionstest/x15_y10_z2_c4_classes2.ilp') + ilp12 = pyilastik.read_project(p, skip_image=True) + (_, (_, labels_12, _)) = ilp12[0] + p = os.path.join( + path, 'dimensionstest/x15_y10_z2_c4_classes2_ilastik1.3.3.ilp') + ilp133 = pyilastik.read_project(p, skip_image=True) + (_, (_, labels_133, _)) = ilp133[0] + print(labels_12.shape) + print(labels_133.shape) + assert_array_equal(labels_133, labels_12) + + p = os.path.join( + path, 'dimensionstest/x15_y10_z1_c1_classes2.ilp') + ilp12 = pyilastik.read_project(p, skip_image=True) + (_, (_, labels_12, _)) = ilp12[0] + p = os.path.join( + path, 'dimensionstest/x15_y10_z1_c1_classes2_ilastik1.3.3.ilp') + ilp133 = pyilastik.read_project(p, skip_image=True) + (_, (_, labels_133, _)) = ilp133[0] + assert_array_equal(labels_133, labels_12) + + p = os.path.join(path, 'dimensionstest/x15_y10_z1_c2_classes2.ilp') + ilp12 = pyilastik.read_project(p, skip_image=True) + (_, (_, labels_12, _)) = ilp12[0] + p = os.path.join( + path, 'dimensionstest/x15_y10_z1_c2_classes2_ilastik1.3.3.ilp') + ilp133 = pyilastik.read_project(p, skip_image=True) + (_, (_, labels_133, _)) = ilp133[0] + assert_array_equal(labels_133, labels_12) + + p = os.path.join(path, 'dimensionstest/x15_y10_z2_c1_classes2.ilp') + ilp12 = pyilastik.read_project(p, skip_image=True) + (_, (_, labels_12, _)) = ilp12[0] + p = os.path.join( + path, 'dimensionstest/x15_y10_z2_c1_classes2_ilastik1.3.3.ilp') + ilp133 = pyilastik.read_project(p, skip_image=True) + (_, (_, labels_133, _)) = ilp133[0] + assert_array_equal(labels_133, labels_12) + + p = os.path.join(path, 'dimensionstest/x502_y251_z5_c1_classes2.ilp') + ilp12 = pyilastik.read_project(p, skip_image=True) + (_, (_, labels_12, _)) = ilp12[0] + p = os.path.join( + path, 'dimensionstest/x502_y251_z5_c1_classes2_ilastik1.3.3.ilp') + ilp133 = pyilastik.read_project(p, skip_image=True) + (_, (_, labels_133, _)) = ilp133[0] + assert_array_equal(labels_133, labels_12) + + def test_original_dimension_order(self): + + p = os.path.join( + path, 'dimensionstest/x15_y10_z2_c4_classes2.ilp') + ilp = pyilastik.read_project(p, skip_image=True) + assert ilp.original_dimension_order() == 'zyxc' + + p = os.path.join( + path, 'dimensionstest/x15_y10_z2_c4_classes2_ilastik1.3.3.ilp') + ilp = pyilastik.read_project(p, skip_image=True) + assert ilp.original_dimension_order() == 'zcyx' + + p = os.path.join( + path, 'dimensionstest/x15_y10_z1_c1_classes2.ilp') + ilp = pyilastik.read_project(p, skip_image=True) + assert ilp.original_dimension_order() == 'yxc' + + p = os.path.join( + path, 'dimensionstest/x15_y10_z1_c1_classes2_ilastik1.3.3.ilp') + ilp = pyilastik.read_project(p, skip_image=True) + assert ilp.original_dimension_order() == 'yxc' + + p = os.path.join( + path, 'dimensionstest/x15_y10_z1_c2_classes2.ilp') + ilp = pyilastik.read_project(p, skip_image=True) + print('z1 c2 1.2.2') + assert ilp.original_dimension_order() == 'yxc' + + p = os.path.join( + path, 'dimensionstest/x15_y10_z1_c2_classes2_ilastik1.3.3.ilp') + ilp = pyilastik.read_project(p, skip_image=True) + print('z1 c2 1.3.3') + assert ilp.original_dimension_order() == 'cyx' + + p = os.path.join( + path, 'dimensionstest/x15_y10_z2_c1_classes2.ilp') + ilp = pyilastik.read_project(p, skip_image=True) + assert ilp.original_dimension_order() == 'zyxc' + + p = os.path.join( + path, 'dimensionstest/x15_y10_z2_c1_classes2_ilastik1.3.3.ilp') + ilp = pyilastik.read_project(p, skip_image=True) + assert ilp.original_dimension_order() == 'zyxc' + + p = os.path.join( + path, 'dimensionstest/x502_y251_z5_c1_classes2.ilp') + ilp = pyilastik.read_project(p, skip_image=True) + assert ilp.original_dimension_order() == 'zyxc' + + p = os.path.join( + path, 'dimensionstest/x502_y251_z5_c1_classes2_ilastik1.3.3.ilp') + ilp = pyilastik.read_project(p, skip_image=True) + assert ilp.original_dimension_order() == 'zyxc' + + def test_normalize_dim_order(self): + + dim_order = 'cyx' + res = ils.normalize_dim_order(dim_order) + assert res == (1, 2, 0) + + data = np.zeros((1, 5, 4)) + res = ils.normalize_dim_order(dim_order, data=data) + assert res.shape == (5, 4, 1) + + dim_order = 'zcyx' + res = ils.normalize_dim_order(dim_order) + assert res == (0, 2, 3, 1) + + data = np.zeros((3, 1, 5, 4)) + res = ils.normalize_dim_order(dim_order, data=data) + assert res.shape == (3, 5, 4, 1) + + dim_order = 'cyx' + data = np.zeros((1, 8, 10)) + res = ils.normalize_dim_order(dim_order, data=data) + assert res.shape == (8, 10, 1) + def test_tile(self): p = os.path.join(path, 'dimensionstest/x502_y251_z5_c1_classes2.ilp') @@ -286,8 +482,65 @@ def test_tile(self): assert_array_equal(t[0, :, :, 0], val_3) + def test_tile_ilastik133(self): + + p = os.path.join( + path, 'dimensionstest/x502_y251_z5_c1_classes2_ilastik1.3.3.ilp') + ilp = pyilastik.read_project(p, skip_image=True) + + tile_slice = np.array([[0, 1], + [0, 10], + [0, 7], + [0, 1]]) + + t = ilp.tile(0, tile_slice) + + val_1 = np.array([[0., 0., 0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 0., 1., 1., 1., 0., 0.], + [0., 0., 1., 1., 1., 0., 0.], + [0., 0., 1., 1., 1., 0., 0.], + [0., 0., 1., 1., 1., 0., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0., 0., 0.]]) + + assert_array_equal(t[0, :, :, 0], val_1) + + tile_slice = np.array([[0, 1], + [92, 101], + [157, 164], + [0, 1]]) + + t = ilp.tile(0, tile_slice) + + val_2 = np.array([[0., 0., 0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0., 0., 0.], + [0., 0., 1., 1., 1., 1., 0.], + [0., 0., 1., 1., 1., 1., 0.], + [0., 0., 2., 2., 0., 0., 0.], + [0., 0., 2., 2., 0., 0., 0.], + [0., 0., 2., 2., 0., 0., 0.], + [0., 0., 2., 2., 0., 0., 0.], + [0., 0., 0., 0., 0., 0., 0.]]) + + assert_array_equal(t[0, :, :, 0], val_2) + + tile_slice = np.array([[0, 1], + [50, 55], + [50, 55], + [0, 1]]) + + t = ilp.tile(0, tile_slice) + val_3 = np.array([[0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0.]]) + assert_array_equal(t[0, :, :, 0], val_3) def test_tile_for_selected_blocks(self): @@ -326,11 +579,6 @@ def test_tile_for_selected_blocks(self): assert_array_equal(tile_1, val_1) assert_array_equal(tile_2, val_2) - - - - - def test_is_overlap(self): tpl = np.array([[2, 13], @@ -355,7 +603,6 @@ def test_is_overlap(self): b9 = np.array([[0, 3], [4, 7]]) - self.assertTrue(ils.is_overlap(tpl, b1)) self.assertTrue(ils.is_overlap(tpl, b2)) self.assertFalse(ils.is_overlap(tpl, b3)) @@ -371,8 +618,6 @@ def test_is_overlap(self): b1 = np.array([[2, 3], [4, 10]]) self.assertTrue(ils.is_overlap(tpl, b1)) - # assert False - def test_tile_loc_from_slices(self): diff --git a/setup.py b/setup.py index 697ee18..4b9236a 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ def readme(): setup(name='pyilastik', - version='0.0.6', + version='0.0.7', description='Read ilastik labels in python', author='Manuel Schoelling', author_email='manuel.schoelling@dzne.de',