From 4e2b97908271c01afc260d7e519466a6eea676c0 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 15 Feb 2024 10:49:52 +0100 Subject: [PATCH 1/6] put find_indexes outside of decorator --- polytope/datacube/backends/fdb.py | 2 +- polytope/datacube/backends/xarray.py | 29 +++++++++++-------- polytope/datacube/datacube_axis.py | 27 ++++++++++++++++- .../datacube_cyclic/datacube_cyclic.py | 3 ++ .../datacube_mappers/datacube_mappers.py | 7 +++++ .../datacube_mappers/mapper_axis_decorator.py | 2 +- .../datacube_merger/datacube_merger.py | 4 +++ .../datacube_merger/merger_axis_decorator.py | 2 +- .../datacube_reverse/datacube_reverse.py | 7 +++++ .../reverse_axis_decorator.py | 2 +- .../datacube_transformations.py | 8 +++++ .../datacube_type_change.py | 4 +++ .../type_change_axis_decorator.py | 2 +- 13 files changed, 81 insertions(+), 18 deletions(-) diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index eda6f24e9..aeb00a885 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -244,7 +244,7 @@ def sort_fdb_request_ranges(self, range_lengths, current_start_idx, lat_length): return (original_indices, sorted_request_ranges) def datacube_natural_indexes(self, axis, subarray): - indexes = subarray[axis.name] + indexes = subarray.get(axis.name, None) return indexes def select(self, path, unmapped_path): diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index ab73fe603..83655ee50 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -67,17 +67,7 @@ def get(self, requests: IndexTree): path.update(unmapped_path) unmapped_path = {} - for key in path.keys(): - if key not in self.dataarray.dims: - path.pop(key) - if key not in self.dataarray.coords.dtypes: - unmapped_path.update({key: path[key]}) - path.pop(key) - for key in self.dataarray.coords.dtypes: - key_dtype = self.dataarray.coords.dtypes[key] - if key_dtype.type is np.str_ and key in path.keys(): - unmapped_path.update({key: path[key]}) - path.pop(key) + self.refit_path(path, unmapped_path, path) subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmapped_path) @@ -97,8 +87,23 @@ def datacube_natural_indexes(self, axis, subarray): indexes = subarray[axis.name].values return indexes + def refit_path(self, path_copy, unmapped_path, path): + for key in path.keys(): + if key not in self.dataarray.dims: + path_copy.pop(key) + if key not in self.dataarray.coords.dtypes: + unmapped_path.update({key: path[key]}) + path_copy.pop(key) + for key in self.dataarray.coords.dtypes: + key_dtype = self.dataarray.coords.dtypes[key] + if key_dtype.type is np.str_ and key in path.keys(): + unmapped_path.update({key: path[key]}) + path_copy.pop(key, None) + def select(self, path, unmapped_path): - subarray = self.dataarray.sel(path, method="nearest") + path_copy = deepcopy(path) + self.refit_path(path_copy, unmapped_path, path) + subarray = self.dataarray.sel(path_copy, method="nearest") subarray = subarray.sel(unmapped_path) return subarray diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index ebe4a7ef1..ca325bfd7 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -12,6 +12,12 @@ from .transformations.datacube_reverse.reverse_axis_decorator import reverse from .transformations.datacube_type_change.type_change_axis_decorator import type_change +from .transformations.datacube_cyclic.datacube_cyclic import DatacubeAxisCyclic +from .transformations.datacube_mappers.datacube_mappers import DatacubeMapper +from .transformations.datacube_merger.datacube_merger import DatacubeAxisMerger +from .transformations.datacube_reverse.datacube_reverse import DatacubeAxisReverse +from .transformations.datacube_type_change.datacube_type_change import DatacubeAxisTypeChange + class DatacubeAxis(ABC): is_cyclic = False @@ -20,6 +26,13 @@ class DatacubeAxis(ABC): reorder = False type_change = False + def order_tranformations(self): + self.transformations = sorted(self.transformations, key=lambda x: transformations_order[type(x)]) + + def give_transformations_parents(self): + for i, transform in enumerate(self.transformations[1:]): + transform.parent = self.transformations[i-1] + def update_axis(self): if self.is_cyclic: self = cyclic(self) @@ -52,7 +65,7 @@ def remap(self, range: List) -> Any: def unmap_to_datacube(self, path, unmapped_path): return (path, unmapped_path) - def find_indexes(self, path, datacube): + def find_standard_indexes(self, path, datacube): unmapped_path = {} path_copy = deepcopy(path) for key in path_copy: @@ -61,6 +74,14 @@ def find_indexes(self, path, datacube): subarray = datacube.select(path, unmapped_path) return datacube.datacube_natural_indexes(self, subarray) + def find_indexes(self, path, datacube): + indexes = self.find_standard_indexes(path, datacube) + # transform = self.transformations[-1] + # TODO + for transformation in self.transformations[::-1]: + indexes = transformation.find_modified_indexes(indexes, path, datacube, self) + return indexes + def offset(self, value): return 0 @@ -122,6 +143,10 @@ def check_axis_type(name, values): raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") +transformations_order = [DatacubeAxisMerger, DatacubeAxisReverse, DatacubeAxisCyclic, DatacubeMapper, DatacubeAxisTypeChange] +transformations_order = {key: i for i, key in enumerate(transformations_order)} + + @reverse @cyclic @mapper diff --git a/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py index 86113aa2d..a0518f040 100644 --- a/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py @@ -23,3 +23,6 @@ def blocked_axes(self): def unwanted_axes(self): return [] + + def find_modified_indexes(self, indexes, path, datacube, axis): + return indexes diff --git a/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py index a2034bfba..4d08fab85 100644 --- a/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py @@ -75,6 +75,13 @@ def unmap_first_val_to_start_line_idx(self, first_val): def unmap(self, first_val, second_val): return self._final_transformation.unmap(first_val, second_val) + def find_modified_indexes(self, indexes, path, datacube, axis): + if axis.name == self._mapped_axes()[0]: + return self.first_axis_vals() + if axis.name == self._mapped_axes()[1]: + first_val = path[self._mapped_axes()[0]] + return self.second_axis_vals(first_val) + _type_to_datacube_mapper_lookup = { "octahedral": "OctahedralGridMapper", diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py index aa4910c99..e5ed23dc9 100644 --- a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py @@ -33,7 +33,7 @@ def unmap_path_key(key_value_path, leaf_path, unwanted_path): key_value_path[transform.old_axis] = unmapped_idx return (key_value_path, leaf_path, unwanted_path) - cls.find_indexes = find_indexes + # cls.find_indexes = find_indexes cls.unmap_path_key = unmap_path_key return cls diff --git a/polytope/datacube/transformations/datacube_merger/datacube_merger.py b/polytope/datacube/transformations/datacube_merger/datacube_merger.py index 310036f04..cddeef31b 100644 --- a/polytope/datacube/transformations/datacube_merger/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger/datacube_merger.py @@ -71,3 +71,7 @@ def unmerge(self, merged_val): def change_val_type(self, axis_name, values): new_values = pd.to_datetime(values) return new_values + + def find_modified_indexes(self, indexes, path, datacube, axis): + if axis.name == self._first_axis: + return self.merged_values(datacube) diff --git a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py index 907e58ab7..36d15eec5 100644 --- a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py @@ -26,7 +26,7 @@ def unmap_path_key(key_value_path, leaf_path, unwanted_path): new_key_value_path[transform._second_axis] = second_val return (new_key_value_path, leaf_path, unwanted_path) - cls.find_indexes = find_indexes + # cls.find_indexes = find_indexes cls.unmap_path_key = unmap_path_key return cls diff --git a/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py index 6e60de872..200922902 100644 --- a/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py @@ -20,3 +20,10 @@ def blocked_axes(self): def unwanted_axes(self): return [] + + def find_modified_indexes(self, indexes, path, datacube, axis): + if axis.name in datacube.complete_axes: + ordered_indices = indexes.sort_values() + else: + ordered_indices = indexes + return ordered_indices diff --git a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py index 44ec0e0ca..6a18150f1 100644 --- a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py @@ -55,7 +55,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): return indexes_between_ranges # TODO: maybe don't need find_indexes? - cls.find_indexes = find_indexes + # cls.find_indexes = find_indexes cls.find_indices_between = find_indices_between return cls diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 2077f3466..3cca1f814 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -4,6 +4,10 @@ class DatacubeAxisTransformation(ABC): + + def __init__(self): + self.parent = None + @staticmethod def create_transform(name, transformation_type_key, transformation_options): transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] @@ -43,6 +47,10 @@ def transformation_axes_final(self): def change_val_type(self, axis_name, values): pass + @abstractmethod + def find_modified_indexes(self, indexes, path, datacube, axis): + pass + _type_to_datacube_transformation_lookup = { "mapper": "DatacubeMapper", diff --git a/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py index 8ba2ef0f7..eff363101 100644 --- a/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py @@ -37,6 +37,10 @@ def blocked_axes(self): def unwanted_axes(self): return [] + def find_modified_indexes(self, indexes, path, datacube, axis): + if axis.name == self.name: + return self.change_val_type(axis.name, indexes) + class TypeChangeStrToInt(DatacubeAxisTypeChange): def __init__(self, axis_name, new_type): diff --git a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py index 77b3de830..a9f490c31 100644 --- a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py @@ -25,7 +25,7 @@ def unmap_path_key(key_value_path, leaf_path, unwanted_path): key_value_path[cls.name] = unchanged_val return (key_value_path, leaf_path, unwanted_path) - cls.find_indexes = find_indexes + # cls.find_indexes = find_indexes cls.unmap_path_key = unmap_path_key return cls From c8a03c94726705e4e1c59fb713b2a8a29b8f8cd5 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 15 Feb 2024 13:44:30 +0100 Subject: [PATCH 2/6] remove all functions from the decorators and put them into the transformation --- polytope/datacube/datacube_axis.py | 53 ++++---- .../datacube_cyclic/cyclic_axis_decorator.py | 126 ------------------ .../datacube_cyclic/datacube_cyclic.py | 119 +++++++++++++++++ .../datacube_mappers/datacube_mappers.py | 28 ++++ .../datacube_mappers/mapper_axis_decorator.py | 36 ----- .../datacube_merger/datacube_merger.py | 24 ++++ .../datacube_merger/merger_axis_decorator.py | 29 ---- .../datacube_reverse/datacube_reverse.py | 52 ++++++++ .../reverse_axis_decorator.py | 58 -------- .../datacube_transformations.py | 24 ++++ .../datacube_type_change.py | 22 +++ .../type_change_axis_decorator.py | 28 ---- tests/test_axis_mappers.py | 50 ++++--- 13 files changed, 330 insertions(+), 319 deletions(-) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index ca325bfd7..be7b879ff 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -7,16 +7,17 @@ import pandas as pd from .transformations.datacube_cyclic.cyclic_axis_decorator import cyclic -from .transformations.datacube_mappers.mapper_axis_decorator import mapper -from .transformations.datacube_merger.merger_axis_decorator import merge -from .transformations.datacube_reverse.reverse_axis_decorator import reverse -from .transformations.datacube_type_change.type_change_axis_decorator import type_change - from .transformations.datacube_cyclic.datacube_cyclic import DatacubeAxisCyclic from .transformations.datacube_mappers.datacube_mappers import DatacubeMapper +from .transformations.datacube_mappers.mapper_axis_decorator import mapper from .transformations.datacube_merger.datacube_merger import DatacubeAxisMerger +from .transformations.datacube_merger.merger_axis_decorator import merge from .transformations.datacube_reverse.datacube_reverse import DatacubeAxisReverse -from .transformations.datacube_type_change.datacube_type_change import DatacubeAxisTypeChange +from .transformations.datacube_reverse.reverse_axis_decorator import reverse +from .transformations.datacube_type_change.datacube_type_change import ( + DatacubeAxisTypeChange, +) +from .transformations.datacube_type_change.type_change_axis_decorator import type_change class DatacubeAxis(ABC): @@ -57,10 +58,16 @@ def serialize(self, value: Any) -> Any: pass def to_intervals(self, range): - return [range] + intervals = [range] + for transformation in self.transformations[::-1]: + intervals = transformation.to_intervals(range, intervals, self) + return intervals def remap(self, range: List) -> Any: - return [range] + ranges = [range] + for transformation in self.transformations[::-1]: + ranges = transformation.remap(range, ranges, self) + return ranges def unmap_to_datacube(self, path, unmapped_path): return (path, unmapped_path) @@ -76,22 +83,27 @@ def find_standard_indexes(self, path, datacube): def find_indexes(self, path, datacube): indexes = self.find_standard_indexes(path, datacube) - # transform = self.transformations[-1] - # TODO for transformation in self.transformations[::-1]: indexes = transformation.find_modified_indexes(indexes, path, datacube, self) return indexes def offset(self, value): - return 0 + offset = 0 + for transformation in self.transformations[::-1]: + offset = transformation.offset(value, self, offset) + return offset def unmap_path_key(self, key_value_path, leaf_path, unwanted_path): + for transformation in self.transformations[::-1]: + (key_value_path, leaf_path, unwanted_path) = transformation.unmap_path_key(key_value_path, leaf_path, unwanted_path, self) return (key_value_path, leaf_path, unwanted_path) def _remap_val_to_axis_range(self, value): + for transformation in self.transformations[::-1]: + value = transformation._remap_val_to_axis_range(value, self) return value - def find_indices_between(self, index_ranges, low, up, datacube, method=None): + def find_standard_indices_between(self, index_ranges, low, up, datacube, method=None): indexes_between_ranges = [] for indexes in index_ranges: if self.name in datacube.complete_axes and self.name not in datacube.transformed_axes: @@ -125,6 +137,12 @@ def find_indices_between(self, index_ranges, low, up, datacube, method=None): indexes_between_ranges.append(indexes_between) return indexes_between_ranges + def find_indices_between(self, indexes_ranges, low, up, datacube, method=None): + indexes_between_ranges = self.find_standard_indices_between(indexes_ranges, low, up, datacube, method) + for transformation in self.transformations[::-1]: + indexes_between_ranges = transformation.find_indices_between(indexes_ranges, low, up, datacube, method, indexes_between_ranges, self) + return indexes_between_ranges + @staticmethod def create_standard(name, values, datacube): values = np.array(values) @@ -147,10 +165,6 @@ def check_axis_type(name, values): transformations_order = {key: i for i, key in enumerate(transformations_order)} -@reverse -@cyclic -@mapper -@type_change class IntDatacubeAxis(DatacubeAxis): def __init__(self): self.name = None @@ -172,10 +186,6 @@ def serialize(self, value): return value -@reverse -@cyclic -@mapper -@type_change class FloatDatacubeAxis(DatacubeAxis): def __init__(self): self.name = None @@ -197,7 +207,6 @@ def serialize(self, value): return value -@merge class PandasTimestampDatacubeAxis(DatacubeAxis): def __init__(self): self.name = None @@ -227,7 +236,6 @@ def offset(self, value): return None -@merge class PandasTimedeltaDatacubeAxis(DatacubeAxis): def __init__(self): self.name = None @@ -257,7 +265,6 @@ def offset(self, value): return None -@type_change class UnsliceableDatacubeAxis(DatacubeAxis): def __init__(self): self.name = None diff --git a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py index e0fdb533f..3cda42d21 100644 --- a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py @@ -1,129 +1,3 @@ -import math -from copy import deepcopy -from typing import List - -from .datacube_cyclic import DatacubeAxisCyclic - - def cyclic(cls): - if cls.is_cyclic: - - def update_range(): - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisCyclic): - transformation = transform - cls.range = transformation.range - - def to_intervals(range): - update_range() - if range[0] == -math.inf: - range[0] = cls.range[0] - if range[1] == math.inf: - range[1] = cls.range[1] - axis_lower = cls.range[0] - axis_upper = cls.range[1] - axis_range = axis_upper - axis_lower - lower = range[0] - upper = range[1] - intervals = [] - if lower < axis_upper: - # In this case, we want to go from lower to the first remapped cyclic axis upper - # or the asked upper range value. - # For example, if we have cyclic range [0,360] and we want to break [-270,180] into intervals, - # we first want to obtain [-270, 0] as the first range, where 0 is the remapped cyclic axis upper - # but if we wanted to break [-270, -180] into intervals, we would want to get [-270,-180], - # where -180 is the asked upper range value. - loops = int((axis_upper - lower) / axis_range) - remapped_up = axis_upper - (loops) * axis_range - new_upper = min(upper, remapped_up) - else: - # In this case, since lower >= axis_upper, we need to either go to the asked upper range - # or we need to go to the first remapped cyclic axis upper which is higher than lower - new_upper = min(axis_upper + axis_range, upper) - while new_upper < lower: - new_upper = min(new_upper + axis_range, upper) - intervals.append([lower, new_upper]) - # Now that we have established what the first interval should be, we should just jump from cyclic range - # to cyclic range until we hit the asked upper range value. - new_up = deepcopy(new_upper) - while new_up < upper: - new_upper = new_up - new_up = min(upper, new_upper + axis_range) - intervals.append([new_upper, new_up]) - # Once we have added all the in-between ranges, we need to add the last interval - intervals.append([new_up, upper]) - return intervals - - def _remap_range_to_axis_range(range): - update_range() - axis_lower = cls.range[0] - axis_upper = cls.range[1] - axis_range = axis_upper - axis_lower - lower = range[0] - upper = range[1] - if lower < axis_lower: - # In this case we need to calculate the number of loops between the axis lower - # and the lower to recenter the lower - loops = int((axis_lower - lower - cls.tol) / axis_range) - return_lower = lower + (loops + 1) * axis_range - return_upper = upper + (loops + 1) * axis_range - elif lower >= axis_upper: - # In this case we need to calculate the number of loops between the axis upper - # and the lower to recenter the lower - loops = int((lower - axis_upper) / axis_range) - return_lower = lower - (loops + 1) * axis_range - return_upper = upper - (loops + 1) * axis_range - else: - # In this case, the lower value is already in the right range - return_lower = lower - return_upper = upper - return [return_lower, return_upper] - - def _remap_val_to_axis_range(value): - return_range = _remap_range_to_axis_range([value, value]) - return return_range[0] - - def remap(range: List): - update_range() - if cls.range[0] - cls.tol <= range[0] <= cls.range[1] + cls.tol: - if cls.range[0] - cls.tol <= range[1] <= cls.range[1] + cls.tol: - # If we are already in the cyclic range, return it - return [range] - elif abs(range[0] - range[1]) <= 2 * cls.tol: - # If we have a range that is just one point, then it should still be counted - # and so we should take a small interval around it to find values inbetween - range = [ - _remap_val_to_axis_range(range[0]) - cls.tol, - _remap_val_to_axis_range(range[0]) + cls.tol, - ] - return [range] - range_intervals = cls.to_intervals(range) - ranges = [] - for interval in range_intervals: - if abs(interval[0] - interval[1]) > 0: - # If the interval is not just a single point, we remap it to the axis range - range = _remap_range_to_axis_range([interval[0], interval[1]]) - up = range[1] - low = range[0] - if up < low: - # Make sure we remap in the right order - ranges.append([up - cls.tol, low + cls.tol]) - else: - ranges.append([low - cls.tol, up + cls.tol]) - return ranges - - def offset(range): - # We first unpad the range by the axis tolerance to make sure that - # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. - # Also, it's safer that we find the offset of a value inside the range instead of on the border - unpadded_range = [range[0] + 1.5 * cls.tol, range[1] - 1.5 * cls.tol] - cyclic_range = _remap_range_to_axis_range(unpadded_range) - offset = unpadded_range[0] - cyclic_range[0] - return offset - - cls.to_intervals = to_intervals - cls.remap = remap - cls.offset = offset - cls._remap_val_to_axis_range = _remap_val_to_axis_range return cls diff --git a/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py index a0518f040..2a6636498 100644 --- a/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py @@ -1,3 +1,6 @@ +import math +from copy import deepcopy + from ..datacube_transformations import DatacubeAxisTransformation @@ -26,3 +29,119 @@ def unwanted_axes(self): def find_modified_indexes(self, indexes, path, datacube, axis): return indexes + + def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): + return (key_value_path, leaf_path, unwanted_path) + + def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis): + return indexes_between_ranges + + def update_range(self, axis): + axis.range = self.range + + def _remap_range_to_axis_range(self, range, axis): + self.update_range(axis) + axis_lower = axis.range[0] + axis_upper = axis.range[1] + axis_range = axis_upper - axis_lower + lower = range[0] + upper = range[1] + if lower < axis_lower: + # In this case we need to calculate the number of loops between the axis lower + # and the lower to recenter the lower + loops = int((axis_lower - lower - axis.tol) / axis_range) + return_lower = lower + (loops + 1) * axis_range + return_upper = upper + (loops + 1) * axis_range + elif lower >= axis_upper: + # In this case we need to calculate the number of loops between the axis upper + # and the lower to recenter the lower + loops = int((lower - axis_upper) / axis_range) + return_lower = lower - (loops + 1) * axis_range + return_upper = upper - (loops + 1) * axis_range + else: + # In this case, the lower value is already in the right range + return_lower = lower + return_upper = upper + return [return_lower, return_upper] + + def _remap_val_to_axis_range(self, value, axis): + value = self._remap_range_to_axis_range([value, value], axis) + return value[0] + + def offset(self, range, axis, offset): + # We first unpad the range by the axis tolerance to make sure that + # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. + # Also, it's safer that we find the offset of a value inside the range instead of on the border + unpadded_range = [range[0] + 1.5 * axis.tol, range[1] - 1.5 * axis.tol] + cyclic_range = self._remap_range_to_axis_range(unpadded_range, axis) + offset = unpadded_range[0] - cyclic_range[0] + return offset + + def remap(self, range, ranges, axis): + self.update_range(axis) + if axis.range[0] - axis.tol <= range[0] <= axis.range[1] + axis.tol: + if axis.range[0] - axis.tol <= range[1] <= axis.range[1] + axis.tol: + # If we are already in the cyclic range, return it + return [range] + elif abs(range[0] - range[1]) <= 2 * axis.tol: + # If we have a range that is just one point, then it should still be counted + # and so we should take a small interval around it to find values inbetween + range = [ + self._remap_val_to_axis_range(range[0], axis) - axis.tol, + self._remap_val_to_axis_range(range[0], axis) + axis.tol, + ] + return [range] + range_intervals = self.to_intervals(range, [[]], axis) + ranges = [] + for interval in range_intervals: + if abs(interval[0] - interval[1]) > 0: + # If the interval is not just a single point, we remap it to the axis range + range = self._remap_range_to_axis_range([interval[0], interval[1]], axis) + up = range[1] + low = range[0] + if up < low: + # Make sure we remap in the right order + ranges.append([up - axis.tol, low + axis.tol]) + else: + ranges.append([low - axis.tol, up + axis.tol]) + return ranges + + def to_intervals(self, range, intervals, axis): + self.update_range(axis) + if range[0] == -math.inf: + range[0] = axis.range[0] + if range[1] == math.inf: + range[1] = axis.range[1] + axis_lower = axis.range[0] + axis_upper = axis.range[1] + axis_range = axis_upper - axis_lower + lower = range[0] + upper = range[1] + intervals = [] + if lower < axis_upper: + # In this case, we want to go from lower to the first remapped cyclic axis upper + # or the asked upper range value. + # For example, if we have cyclic range [0,360] and we want to break [-270,180] into intervals, + # we first want to obtain [-270, 0] as the first range, where 0 is the remapped cyclic axis upper + # but if we wanted to break [-270, -180] into intervals, we would want to get [-270,-180], + # where -180 is the asked upper range value. + loops = int((axis_upper - lower) / axis_range) + remapped_up = axis_upper - (loops) * axis_range + new_upper = min(upper, remapped_up) + else: + # In this case, since lower >= axis_upper, we need to either go to the asked upper range + # or we need to go to the first remapped cyclic axis upper which is higher than lower + new_upper = min(axis_upper + axis_range, upper) + while new_upper < lower: + new_upper = min(new_upper + axis_range, upper) + intervals.append([lower, new_upper]) + # Now that we have established what the first interval should be, we should just jump from cyclic range + # to cyclic range until we hit the asked upper range value. + new_up = deepcopy(new_upper) + while new_up < upper: + new_upper = new_up + new_up = min(upper, new_upper + axis_range) + intervals.append([new_upper, new_up]) + # Once we have added all the in-between ranges, we need to add the last interval + intervals.append([new_up, upper]) + return intervals diff --git a/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py index 4d08fab85..475bdd403 100644 --- a/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py @@ -82,6 +82,34 @@ def find_modified_indexes(self, indexes, path, datacube, axis): first_val = path[self._mapped_axes()[0]] return self.second_axis_vals(first_val) + def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): + value = key_value_path[axis.name] + if axis.name == self._mapped_axes()[0]: + unwanted_val = key_value_path[self._mapped_axes()[0]] + unwanted_path[axis.name] = unwanted_val + if axis.name == self._mapped_axes()[1]: + first_val = unwanted_path[self._mapped_axes()[0]] + unmapped_idx = self.unmap(first_val, value) + leaf_path.pop(self._mapped_axes()[0], None) + key_value_path.pop(axis.name) + key_value_path[self.old_axis] = unmapped_idx + return (key_value_path, leaf_path, unwanted_path) + + def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis): + return indexes_between_ranges + + def _remap_val_to_axis_range(self, value, axis): + return value + + def offset(self, range, axis, offset): + return offset + + def remap(self, range, ranges, axis): + return ranges + + def to_intervals(self, range, intervals, axis): + return intervals + _type_to_datacube_mapper_lookup = { "octahedral": "OctahedralGridMapper", diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py index e5ed23dc9..ba0ca5e56 100644 --- a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py @@ -1,39 +1,3 @@ -from .datacube_mappers import DatacubeMapper - - def mapper(cls): - if cls.has_mapper: - - def find_indexes(path, datacube): - # first, find the relevant transformation object that is a mapping in the cls.transformation dico - for transform in cls.transformations: - if isinstance(transform, DatacubeMapper): - transformation = transform - if cls.name == transformation._mapped_axes()[0]: - return transformation.first_axis_vals() - if cls.name == transformation._mapped_axes()[1]: - first_val = path[transformation._mapped_axes()[0]] - return transformation.second_axis_vals(first_val) - - old_unmap_path_key = cls.unmap_path_key - - def unmap_path_key(key_value_path, leaf_path, unwanted_path): - key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeMapper): - if cls.name == transform._mapped_axes()[0]: - unwanted_val = key_value_path[transform._mapped_axes()[0]] - unwanted_path[cls.name] = unwanted_val - if cls.name == transform._mapped_axes()[1]: - first_val = unwanted_path[transform._mapped_axes()[0]] - unmapped_idx = transform.unmap(first_val, value) - leaf_path.pop(transform._mapped_axes()[0], None) - key_value_path.pop(cls.name) - key_value_path[transform.old_axis] = unmapped_idx - return (key_value_path, leaf_path, unwanted_path) - - # cls.find_indexes = find_indexes - cls.unmap_path_key = unmap_path_key return cls diff --git a/polytope/datacube/transformations/datacube_merger/datacube_merger.py b/polytope/datacube/transformations/datacube_merger/datacube_merger.py index cddeef31b..a7f20aec7 100644 --- a/polytope/datacube/transformations/datacube_merger/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger/datacube_merger.py @@ -75,3 +75,27 @@ def change_val_type(self, axis_name, values): def find_modified_indexes(self, indexes, path, datacube, axis): if axis.name == self._first_axis: return self.merged_values(datacube) + + def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): + new_key_value_path = {} + value = key_value_path[axis.name] + if axis.name == self._first_axis: + (first_val, second_val) = self.unmerge(value) + new_key_value_path[self._first_axis] = first_val + new_key_value_path[self._second_axis] = second_val + return (new_key_value_path, leaf_path, unwanted_path) + + def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis): + return indexes_between_ranges + + def _remap_val_to_axis_range(self, value, axis): + return value + + def offset(self, range, axis, offset): + return offset + + def remap(self, range, ranges, axis): + return ranges + + def to_intervals(self, range, intervals, axis): + return intervals diff --git a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py index 36d15eec5..f0bcd1c26 100644 --- a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py @@ -1,32 +1,3 @@ -from .datacube_merger import DatacubeAxisMerger - - def merge(cls): - if cls.has_merger: - - def find_indexes(path, datacube): - # first, find the relevant transformation object that is a mapping in the cls.transformation dico - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - transformation = transform - if cls.name == transformation._first_axis: - return transformation.merged_values(datacube) - - old_unmap_path_key = cls.unmap_path_key - - def unmap_path_key(key_value_path, leaf_path, unwanted_path): - key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) - new_key_value_path = {} - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - if cls.name == transform._first_axis: - (first_val, second_val) = transform.unmerge(value) - new_key_value_path[transform._first_axis] = first_val - new_key_value_path[transform._second_axis] = second_val - return (new_key_value_path, leaf_path, unwanted_path) - - # cls.find_indexes = find_indexes - cls.unmap_path_key = unmap_path_key return cls diff --git a/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py index 200922902..40cf416b6 100644 --- a/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py @@ -1,3 +1,4 @@ +from ....utility.list_tools import bisect_left_cmp, bisect_right_cmp from ..datacube_transformations import DatacubeAxisTransformation @@ -27,3 +28,54 @@ def find_modified_indexes(self, indexes, path, datacube, axis): else: ordered_indices = indexes return ordered_indices + + def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): + return (key_value_path, leaf_path, unwanted_path) + + def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis): + indexes_between_ranges = [] + if axis.name == self.name: + for indexes in indexes_ranges: + if axis.name in datacube.complete_axes: + # Find the range of indexes between lower and upper + # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html + # Assumes the indexes are already sorted (could sort to be sure) and monotonically + # increasing + if method == "surrounding" or method == "nearest": + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.append(indexes_between) + else: + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.append(indexes_between) + else: + if method == "surrounding" or method == "nearest": + end_idx = bisect_left_cmp(indexes, low, cmp=lambda x, y: x > y) + 1 + start_idx = bisect_right_cmp(indexes, up, cmp=lambda x, y: x > y) + start = max(start_idx - 1, 0) + end = min(end_idx + 1, len(indexes)) + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + else: + end_idx = bisect_left_cmp(indexes, low, cmp=lambda x, y: x > y) + 1 + start_idx = bisect_right_cmp(indexes, up, cmp=lambda x, y: x > y) + indexes_between = indexes[start_idx:end_idx] + indexes_between_ranges.append(indexes_between) + return indexes_between_ranges + + def _remap_val_to_axis_range(self, value, axis): + return value + + def offset(self, range, axis, offset): + return offset + + def remap(self, range, ranges, axis): + return ranges + + def to_intervals(self, range, intervals, axis): + return intervals diff --git a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py index 6a18150f1..c8cbeaeda 100644 --- a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py @@ -1,61 +1,3 @@ -from ....utility.list_tools import bisect_left_cmp, bisect_right_cmp -from .datacube_reverse import DatacubeAxisReverse - - def reverse(cls): - if cls.reorder: - old_find_indexes = cls.find_indexes - - def find_indexes(path, datacube): - # first, find the relevant transformation object that is a mapping in the cls.transformation dico - unordered_indices = old_find_indexes(path, datacube) - if cls.name in datacube.complete_axes: - ordered_indices = unordered_indices.sort_values() - else: - ordered_indices = unordered_indices - return ordered_indices - - def find_indices_between(index_ranges, low, up, datacube, method=None): - indexes_between_ranges = [] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisReverse): - transformation = transform - if cls.name == transformation.name: - for indexes in index_ranges: - if cls.name in datacube.complete_axes: - # Find the range of indexes between lower and upper - # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html - # Assumes the indexes are already sorted (could sort to be sure) and monotonically - # increasing - if method == "surrounding" or method == "nearest": - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end].to_list() - indexes_between_ranges.append(indexes_between) - else: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - indexes_between = indexes[start:end].to_list() - indexes_between_ranges.append(indexes_between) - else: - if method == "surrounding" or method == "nearest": - end_idx = bisect_left_cmp(indexes, low, cmp=lambda x, y: x > y) + 1 - start_idx = bisect_right_cmp(indexes, up, cmp=lambda x, y: x > y) - start = max(start_idx - 1, 0) - end = min(end_idx + 1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - else: - end_idx = bisect_left_cmp(indexes, low, cmp=lambda x, y: x > y) + 1 - start_idx = bisect_right_cmp(indexes, up, cmp=lambda x, y: x > y) - indexes_between = indexes[start_idx:end_idx] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - # TODO: maybe don't need find_indexes? - # cls.find_indexes = find_indexes - cls.find_indices_between = find_indices_between return cls diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 3cca1f814..c3316b313 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -51,6 +51,30 @@ def change_val_type(self, axis_name, values): def find_modified_indexes(self, indexes, path, datacube, axis): pass + @abstractmethod + def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): + pass + + @abstractmethod + def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis): + pass + + @abstractmethod + def _remap_val_to_axis_range(self, value, axis): + pass + + @abstractmethod + def offset(self, range, axis, offset): + pass + + @abstractmethod + def remap(self, range, ranges, axis): + pass + + @abstractmethod + def to_intervals(self, range, intervals, axis): + pass + _type_to_datacube_transformation_lookup = { "mapper": "DatacubeMapper", diff --git a/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py index eff363101..f0ad4ffd8 100644 --- a/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py @@ -41,6 +41,28 @@ def find_modified_indexes(self, indexes, path, datacube, axis): if axis.name == self.name: return self.change_val_type(axis.name, indexes) + def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): + value = key_value_path[axis.name] + if axis.name == self.name: + unchanged_val = self.make_str(value) + key_value_path[axis.name] = unchanged_val + return (key_value_path, leaf_path, unwanted_path) + + def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis): + return indexes_between_ranges + + def _remap_val_to_axis_range(self, value, axis): + return value + + def offset(self, range, axis, offset): + return offset + + def remap(self, range, ranges, axis): + return ranges + + def to_intervals(self, range, intervals, axis): + return intervals + class TypeChangeStrToInt(DatacubeAxisTypeChange): def __init__(self, axis_name, new_type): diff --git a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py index a9f490c31..d75268ad0 100644 --- a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py @@ -1,31 +1,3 @@ -from .datacube_type_change import DatacubeAxisTypeChange - - def type_change(cls): - if cls.type_change: - old_find_indexes = cls.find_indexes - - def find_indexes(path, datacube): - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - transformation = transform - if cls.name == transformation.name: - original_vals = old_find_indexes(path, datacube) - return transformation.change_val_type(cls.name, original_vals) - - old_unmap_path_key = cls.unmap_path_key - - def unmap_path_key(key_value_path, leaf_path, unwanted_path): - key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - if cls.name == transform.name: - unchanged_val = transform.make_str(value) - key_value_path[cls.name] = unchanged_val - return (key_value_path, leaf_path, unwanted_path) - - # cls.find_indexes = find_indexes - cls.unmap_path_key = unmap_path_key return cls diff --git a/tests/test_axis_mappers.py b/tests/test_axis_mappers.py index 991dbb1f0..41c1a4c1a 100644 --- a/tests/test_axis_mappers.py +++ b/tests/test_axis_mappers.py @@ -1,6 +1,7 @@ import pandas as pd from polytope.datacube.datacube_axis import ( + DatacubeAxisCyclic, FloatDatacubeAxis, IntDatacubeAxis, PandasTimedeltaDatacubeAxis, @@ -36,25 +37,36 @@ def test_float_axis_cyclic(self): assert axis.to_float(2) == 2.0 assert axis.from_float(2) == 2.0 assert axis.serialize(2.0) == 2.0 + transformation = DatacubeAxisCyclic("", {}) # Test the to_intervals function - axis.range = [1, 3] - assert axis.to_intervals([4, 7]) == [[4, 5], [5, 7], [7, 7]] + transformation.range = [1, 3] + assert transformation.to_intervals([4, 7], [[]], axis) == [[4, 5], [5, 7], [7, 7]] # Test the cyclic_remap function - axis.range = [0, 1] - assert axis.remap([0, 2]) == [[-1e-12, 1.000000000001], [-1e-12, 1.000000000001]] - axis.range = [1, 2] - assert axis.remap([1, 3]) == [[0.999999999999, 2.000000000001], [0.999999999999, 2.000000000001]] - axis.range = [1, 3] - assert axis.remap([1, 4]) == [[0.999999999999, 3.000000000001], [0.999999999999, 2.000000000001]] - axis.range = [2, 4] - assert axis.remap([0, 5]) == [ + + transformation.range = [0, 1] + assert transformation.remap([0, 2], [[]], axis) == [[-1e-12, 1.000000000001], [-1e-12, 1.000000000001]] + + transformation.range = [1, 2] + assert transformation.remap([1, 3], [[]], axis) == [ + [0.999999999999, 2.000000000001], + [0.999999999999, 2.000000000001], + ] + + transformation.range = [1, 3] + assert transformation.remap([1, 4], [[]], axis) == [ + [0.999999999999, 3.000000000001], + [0.999999999999, 2.000000000001], + ] + + transformation.range = [2, 4] + assert transformation.remap([0, 5], [[]], axis) == [ [1.999999999999, 4.000000000001], [1.999999999999, 4.000000000001], [1.999999999999, 3.000000000001], ] - axis.range = [2.3, 4.6] - remapped = axis.remap([0.3, 5.7]) + transformation.range = [2.3, 4.6] + remapped = transformation.remap([0.3, 5.7], [[]], axis) assert remapped == [ [2.5999999999989996, 4.600000000001], [2.2999999999989997, 4.600000000001], @@ -62,22 +74,22 @@ def test_float_axis_cyclic(self): ] # Test the to_cyclic_value function - axis.range = [1, 3] - remapped = axis.remap([0, 7]) + transformation.range = [1, 3] + remapped = transformation.remap([0, 7], [[]], axis) assert remapped == [ [1.999999999999, 3.000000000001], [0.999999999999, 3.000000000001], [0.999999999999, 3.000000000001], [0.999999999999, 3.000000000001], ] - remapped = axis.remap([-1, 2]) + remapped = transformation.remap([-1, 2], [[]], axis) assert remapped == [[0.999999999999, 3.000000000001], [0.999999999999, 2.000000000001]] # Test the cyclic_offset function - assert axis.offset([3.05, 3.1]) == 2 - assert axis.offset([1.05, 1.1]) == 0 - assert axis.offset([-5.0, -4.95]) == -6 - assert axis.offset([5.05, 5.1]) == 4 + assert transformation.offset([3.05, 3.1], axis, 0) == 2 + assert transformation.offset([1.05, 1.1], axis, 0) == 0 + assert transformation.offset([-5.0, -4.95], axis, 0) == -6 + assert transformation.offset([5.05, 5.1], axis, 0) == 4 def test_timedelta_axis(self): axis = PandasTimedeltaDatacubeAxis() From 0d8eaee223ad370f7e2dcbbb53fea1a09205f813 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 15 Feb 2024 13:49:00 +0100 Subject: [PATCH 3/6] remove decorators completely when we create the axes/transformations --- polytope/datacube/backends/datacube.py | 5 +--- polytope/datacube/datacube_axis.py | 28 +++++++++---------- .../datacube_cyclic/cyclic_axis_decorator.py | 1 - .../datacube_mappers/mapper_axis_decorator.py | 1 - .../datacube_merger/merger_axis_decorator.py | 1 - .../reverse_axis_decorator.py | 1 - .../datacube_transformations.py | 1 - .../type_change_axis_decorator.py | 1 - tests/test_axis_mappers.py | 1 - 9 files changed, 15 insertions(+), 25 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 2fb585452..509895c48 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -1,4 +1,3 @@ -import importlib import logging import math from abc import ABC, abstractmethod @@ -53,9 +52,7 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt # factory inside datacube_transformations to set the has_transform, is_cyclic etc axis properties # add the specific transformation handled here to the relevant axes # Modify the axis to update with the tag - decorator_module = importlib.import_module("polytope.datacube.datacube_axis") - decorator = getattr(decorator_module, transformation_type_key) - decorator(self._axes[axis_name]) + if transformation not in self._axes[axis_name].transformations: # Avoids duplicates being stored self._axes[axis_name].transformations.append(transformation) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index be7b879ff..36209f21b 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -6,18 +6,13 @@ import numpy as np import pandas as pd -from .transformations.datacube_cyclic.cyclic_axis_decorator import cyclic from .transformations.datacube_cyclic.datacube_cyclic import DatacubeAxisCyclic from .transformations.datacube_mappers.datacube_mappers import DatacubeMapper -from .transformations.datacube_mappers.mapper_axis_decorator import mapper from .transformations.datacube_merger.datacube_merger import DatacubeAxisMerger -from .transformations.datacube_merger.merger_axis_decorator import merge from .transformations.datacube_reverse.datacube_reverse import DatacubeAxisReverse -from .transformations.datacube_reverse.reverse_axis_decorator import reverse from .transformations.datacube_type_change.datacube_type_change import ( DatacubeAxisTypeChange, ) -from .transformations.datacube_type_change.type_change_axis_decorator import type_change class DatacubeAxis(ABC): @@ -32,12 +27,7 @@ def order_tranformations(self): def give_transformations_parents(self): for i, transform in enumerate(self.transformations[1:]): - transform.parent = self.transformations[i-1] - - def update_axis(self): - if self.is_cyclic: - self = cyclic(self) - return self + transform.parent = self.transformations[i - 1] # Convert from user-provided value to CONTINUOUS type (e.g. float, pd.timestamp) @abstractmethod @@ -95,7 +85,9 @@ def offset(self, value): def unmap_path_key(self, key_value_path, leaf_path, unwanted_path): for transformation in self.transformations[::-1]: - (key_value_path, leaf_path, unwanted_path) = transformation.unmap_path_key(key_value_path, leaf_path, unwanted_path, self) + (key_value_path, leaf_path, unwanted_path) = transformation.unmap_path_key( + key_value_path, leaf_path, unwanted_path, self + ) return (key_value_path, leaf_path, unwanted_path) def _remap_val_to_axis_range(self, value): @@ -140,7 +132,9 @@ def find_standard_indices_between(self, index_ranges, low, up, datacube, method= def find_indices_between(self, indexes_ranges, low, up, datacube, method=None): indexes_between_ranges = self.find_standard_indices_between(indexes_ranges, low, up, datacube, method) for transformation in self.transformations[::-1]: - indexes_between_ranges = transformation.find_indices_between(indexes_ranges, low, up, datacube, method, indexes_between_ranges, self) + indexes_between_ranges = transformation.find_indices_between( + indexes_ranges, low, up, datacube, method, indexes_between_ranges, self + ) return indexes_between_ranges @staticmethod @@ -161,7 +155,13 @@ def check_axis_type(name, values): raise ValueError(f"Could not create a mapper for index type {values.dtype.type} for axis {name}") -transformations_order = [DatacubeAxisMerger, DatacubeAxisReverse, DatacubeAxisCyclic, DatacubeMapper, DatacubeAxisTypeChange] +transformations_order = [ + DatacubeAxisMerger, + DatacubeAxisReverse, + DatacubeAxisCyclic, + DatacubeMapper, + DatacubeAxisTypeChange, +] transformations_order = {key: i for i, key in enumerate(transformations_order)} diff --git a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py index 3cda42d21..8ec80a923 100644 --- a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py @@ -1,3 +1,2 @@ def cyclic(cls): - return cls diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py index ba0ca5e56..d4c024109 100644 --- a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py @@ -1,3 +1,2 @@ def mapper(cls): - return cls diff --git a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py index f0bcd1c26..bdd33aae6 100644 --- a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py @@ -1,3 +1,2 @@ def merge(cls): - return cls diff --git a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py index c8cbeaeda..ef59a3ac5 100644 --- a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py @@ -1,3 +1,2 @@ def reverse(cls): - return cls diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index c3316b313..bfe4a6440 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -4,7 +4,6 @@ class DatacubeAxisTransformation(ABC): - def __init__(self): self.parent = None diff --git a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py index d75268ad0..e4aad07e5 100644 --- a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py @@ -1,3 +1,2 @@ def type_change(cls): - return cls diff --git a/tests/test_axis_mappers.py b/tests/test_axis_mappers.py index 41c1a4c1a..6196251e8 100644 --- a/tests/test_axis_mappers.py +++ b/tests/test_axis_mappers.py @@ -32,7 +32,6 @@ def test_float_axis(self): def test_float_axis_cyclic(self): axis = FloatDatacubeAxis() axis.is_cyclic = True - axis = axis.update_axis() assert axis.parse(2) == 2.0 assert axis.to_float(2) == 2.0 assert axis.from_float(2) == 2.0 From 82cf208421c899cf8f57f2f4f0615cae1aa222e3 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 15 Feb 2024 13:54:14 +0100 Subject: [PATCH 4/6] completely remove decorator files and null transformation --- .../datacube_cyclic/cyclic_axis_decorator.py | 2 -- .../datacube_mappers/mapper_axis_decorator.py | 2 -- .../datacube_merger/merger_axis_decorator.py | 2 -- .../datacube_null_transformation.py | 22 ------------------- .../null_axis_decorator.py | 22 ------------------- .../reverse_axis_decorator.py | 2 -- .../datacube_transformations.py | 3 --- .../type_change_axis_decorator.py | 2 -- 8 files changed, 57 deletions(-) delete mode 100644 polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py delete mode 100644 polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py delete mode 100644 polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py delete mode 100644 polytope/datacube/transformations/datacube_null_transformation/datacube_null_transformation.py delete mode 100644 polytope/datacube/transformations/datacube_null_transformation/null_axis_decorator.py delete mode 100644 polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py delete mode 100644 polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py diff --git a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py deleted file mode 100644 index 8ec80a923..000000000 --- a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py +++ /dev/null @@ -1,2 +0,0 @@ -def cyclic(cls): - return cls diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py deleted file mode 100644 index d4c024109..000000000 --- a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py +++ /dev/null @@ -1,2 +0,0 @@ -def mapper(cls): - return cls diff --git a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py deleted file mode 100644 index bdd33aae6..000000000 --- a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py +++ /dev/null @@ -1,2 +0,0 @@ -def merge(cls): - return cls diff --git a/polytope/datacube/transformations/datacube_null_transformation/datacube_null_transformation.py b/polytope/datacube/transformations/datacube_null_transformation/datacube_null_transformation.py deleted file mode 100644 index 43dccbbed..000000000 --- a/polytope/datacube/transformations/datacube_null_transformation/datacube_null_transformation.py +++ /dev/null @@ -1,22 +0,0 @@ -from ..datacube_transformations import DatacubeAxisTransformation - - -class DatacubeNullTransformation(DatacubeAxisTransformation): - def __init__(self, name, mapper_options): - self.name = name - self.transformation_options = mapper_options - - def generate_final_transformation(self): - return self - - def transformation_axes_final(self): - return [self.name] - - def change_val_type(self, axis_name, values): - return values - - def blocked_axes(self): - return [] - - def unwanted_axes(self): - return [] diff --git a/polytope/datacube/transformations/datacube_null_transformation/null_axis_decorator.py b/polytope/datacube/transformations/datacube_null_transformation/null_axis_decorator.py deleted file mode 100644 index 10e2644df..000000000 --- a/polytope/datacube/transformations/datacube_null_transformation/null_axis_decorator.py +++ /dev/null @@ -1,22 +0,0 @@ -def null(cls): - if cls.type_change: - old_find_indexes = cls.find_indexes - - def find_indexes(path, datacube): - return old_find_indexes(path, datacube) - - def find_indices_between(index_ranges, low, up, datacube, method=None): - indexes_between_ranges = [] - for indexes in index_ranges: - indexes_between = [i for i in indexes if low <= i <= up] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - def remap(range): - return [range] - - cls.remap = remap - cls.find_indexes = find_indexes - cls.find_indices_between = find_indices_between - - return cls diff --git a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py deleted file mode 100644 index ef59a3ac5..000000000 --- a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py +++ /dev/null @@ -1,2 +0,0 @@ -def reverse(cls): - return cls diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index bfe4a6440..c329c5fe5 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -81,7 +81,6 @@ def to_intervals(self, range, intervals, axis): "merge": "DatacubeAxisMerger", "reverse": "DatacubeAxisReverse", "type_change": "DatacubeAxisTypeChange", - "null": "DatacubeNullTransformation", } _type_to_transformation_file_lookup = { @@ -90,7 +89,6 @@ def to_intervals(self, range, intervals, axis): "merge": "merger", "reverse": "reverse", "type_change": "type_change", - "null": "null_transformation", } has_transform = { @@ -99,5 +97,4 @@ def to_intervals(self, range, intervals, axis): "merge": "has_merger", "reverse": "reorder", "type_change": "type_change", - "null": "null", } diff --git a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py deleted file mode 100644 index e4aad07e5..000000000 --- a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py +++ /dev/null @@ -1,2 +0,0 @@ -def type_change(cls): - return cls From 79d818de29c6cf86037e3aab44d1af4744cb2594 Mon Sep 17 00:00:00 2001 From: mathleur Date: Fri, 16 Feb 2024 09:44:36 +0100 Subject: [PATCH 5/6] put base functions in the transformations in the generic transformation class --- .../datacube_cyclic/datacube_cyclic.py | 9 -------- .../datacube_mappers/datacube_mappers.py | 15 ------------- .../datacube_merger/datacube_merger.py | 15 ------------- .../datacube_reverse/datacube_reverse.py | 15 ------------- .../datacube_transformations.py | 21 +++++++------------ .../datacube_type_change.py | 15 ------------- 6 files changed, 7 insertions(+), 83 deletions(-) diff --git a/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py b/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py index 2a6636498..0dad0ca29 100644 --- a/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py +++ b/polytope/datacube/transformations/datacube_cyclic/datacube_cyclic.py @@ -27,15 +27,6 @@ def blocked_axes(self): def unwanted_axes(self): return [] - def find_modified_indexes(self, indexes, path, datacube, axis): - return indexes - - def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): - return (key_value_path, leaf_path, unwanted_path) - - def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis): - return indexes_between_ranges - def update_range(self, axis): axis.range = self.range diff --git a/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py index 475bdd403..6dad1d7ab 100644 --- a/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers/datacube_mappers.py @@ -95,21 +95,6 @@ def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): key_value_path[self.old_axis] = unmapped_idx return (key_value_path, leaf_path, unwanted_path) - def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis): - return indexes_between_ranges - - def _remap_val_to_axis_range(self, value, axis): - return value - - def offset(self, range, axis, offset): - return offset - - def remap(self, range, ranges, axis): - return ranges - - def to_intervals(self, range, intervals, axis): - return intervals - _type_to_datacube_mapper_lookup = { "octahedral": "OctahedralGridMapper", diff --git a/polytope/datacube/transformations/datacube_merger/datacube_merger.py b/polytope/datacube/transformations/datacube_merger/datacube_merger.py index a7f20aec7..91d680197 100644 --- a/polytope/datacube/transformations/datacube_merger/datacube_merger.py +++ b/polytope/datacube/transformations/datacube_merger/datacube_merger.py @@ -84,18 +84,3 @@ def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): new_key_value_path[self._first_axis] = first_val new_key_value_path[self._second_axis] = second_val return (new_key_value_path, leaf_path, unwanted_path) - - def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis): - return indexes_between_ranges - - def _remap_val_to_axis_range(self, value, axis): - return value - - def offset(self, range, axis, offset): - return offset - - def remap(self, range, ranges, axis): - return ranges - - def to_intervals(self, range, intervals, axis): - return intervals diff --git a/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py b/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py index 40cf416b6..ec2e84da3 100644 --- a/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py +++ b/polytope/datacube/transformations/datacube_reverse/datacube_reverse.py @@ -29,9 +29,6 @@ def find_modified_indexes(self, indexes, path, datacube, axis): ordered_indices = indexes return ordered_indices - def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): - return (key_value_path, leaf_path, unwanted_path) - def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis): indexes_between_ranges = [] if axis.name == self.name: @@ -67,15 +64,3 @@ def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexe indexes_between = indexes[start_idx:end_idx] indexes_between_ranges.append(indexes_between) return indexes_between_ranges - - def _remap_val_to_axis_range(self, value, axis): - return value - - def offset(self, range, axis, offset): - return offset - - def remap(self, range, ranges, axis): - return ranges - - def to_intervals(self, range, intervals, axis): - return intervals diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index c329c5fe5..dabf4ff9d 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -46,33 +46,26 @@ def transformation_axes_final(self): def change_val_type(self, axis_name, values): pass - @abstractmethod def find_modified_indexes(self, indexes, path, datacube, axis): - pass + return indexes - @abstractmethod def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): - pass + return (key_value_path, leaf_path, unwanted_path) - @abstractmethod def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis): - pass + return indexes_between_ranges - @abstractmethod def _remap_val_to_axis_range(self, value, axis): - pass + return value - @abstractmethod def offset(self, range, axis, offset): - pass + return offset - @abstractmethod def remap(self, range, ranges, axis): - pass + return ranges - @abstractmethod def to_intervals(self, range, intervals, axis): - pass + return intervals _type_to_datacube_transformation_lookup = { diff --git a/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py b/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py index f0ad4ffd8..137ed8a40 100644 --- a/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py +++ b/polytope/datacube/transformations/datacube_type_change/datacube_type_change.py @@ -48,21 +48,6 @@ def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): key_value_path[axis.name] = unchanged_val return (key_value_path, leaf_path, unwanted_path) - def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis): - return indexes_between_ranges - - def _remap_val_to_axis_range(self, value, axis): - return value - - def offset(self, range, axis, offset): - return offset - - def remap(self, range, ranges, axis): - return ranges - - def to_intervals(self, range, intervals, axis): - return intervals - class TypeChangeStrToInt(DatacubeAxisTypeChange): def __init__(self, axis_name, new_type): From f8247a12da697d9c3c768f9235f0b39102ff9c41 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 4 Mar 2024 09:46:31 +0100 Subject: [PATCH 6/6] merge develop --- polytope/datacube/transformations/datacube_cyclic/__init__.py | 1 - polytope/datacube/transformations/datacube_mappers/__init__.py | 1 - polytope/datacube/transformations/datacube_merger/__init__.py | 1 - .../transformations/datacube_null_transformation/__init__.py | 2 -- polytope/datacube/transformations/datacube_reverse/__init__.py | 1 - .../datacube/transformations/datacube_type_change/__init__.py | 1 - 6 files changed, 7 deletions(-) delete mode 100644 polytope/datacube/transformations/datacube_null_transformation/__init__.py diff --git a/polytope/datacube/transformations/datacube_cyclic/__init__.py b/polytope/datacube/transformations/datacube_cyclic/__init__.py index adfaf9d8f..4d172088d 100644 --- a/polytope/datacube/transformations/datacube_cyclic/__init__.py +++ b/polytope/datacube/transformations/datacube_cyclic/__init__.py @@ -1,2 +1 @@ -from .cyclic_axis_decorator import * from .datacube_cyclic import * diff --git a/polytope/datacube/transformations/datacube_mappers/__init__.py b/polytope/datacube/transformations/datacube_mappers/__init__.py index ad69c9c5a..3cc477326 100644 --- a/polytope/datacube/transformations/datacube_mappers/__init__.py +++ b/polytope/datacube/transformations/datacube_mappers/__init__.py @@ -1,2 +1 @@ from .datacube_mappers import * -from .mapper_axis_decorator import * diff --git a/polytope/datacube/transformations/datacube_merger/__init__.py b/polytope/datacube/transformations/datacube_merger/__init__.py index 71a59acb1..085c4dfe1 100644 --- a/polytope/datacube/transformations/datacube_merger/__init__.py +++ b/polytope/datacube/transformations/datacube_merger/__init__.py @@ -1,2 +1 @@ from .datacube_merger import * -from .merger_axis_decorator import * diff --git a/polytope/datacube/transformations/datacube_null_transformation/__init__.py b/polytope/datacube/transformations/datacube_null_transformation/__init__.py deleted file mode 100644 index 13395c4ad..000000000 --- a/polytope/datacube/transformations/datacube_null_transformation/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .datacube_null_transformation import * -from .null_axis_decorator import * diff --git a/polytope/datacube/transformations/datacube_reverse/__init__.py b/polytope/datacube/transformations/datacube_reverse/__init__.py index 73cb9d862..913c2c9bb 100644 --- a/polytope/datacube/transformations/datacube_reverse/__init__.py +++ b/polytope/datacube/transformations/datacube_reverse/__init__.py @@ -1,2 +1 @@ from .datacube_reverse import * -from .reverse_axis_decorator import * diff --git a/polytope/datacube/transformations/datacube_type_change/__init__.py b/polytope/datacube/transformations/datacube_type_change/__init__.py index 89dab3aef..a209dcfb6 100644 --- a/polytope/datacube/transformations/datacube_type_change/__init__.py +++ b/polytope/datacube/transformations/datacube_type_change/__init__.py @@ -1,2 +1 @@ from .datacube_type_change import * -from .type_change_axis_decorator import *