From 543c3930298d5321477dfe6d78df6103add066af Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 16 Jan 2025 11:43:45 +0100 Subject: [PATCH 1/2] fix duplicate merging --- polytope_feature/datacube/backends/datacube.py | 7 ++----- .../transformations/datacube_cyclic/datacube_cyclic.py | 2 +- .../datacube_mappers/datacube_mappers.py | 2 +- .../transformations/datacube_merger/datacube_merger.py | 8 +++++--- .../datacube_reverse/datacube_reverse.py | 2 +- .../transformations/datacube_transformations.py | 10 +++++----- .../datacube_type_change/datacube_type_change.py | 2 +- 7 files changed, 16 insertions(+), 17 deletions(-) diff --git a/polytope_feature/datacube/backends/datacube.py b/polytope_feature/datacube/backends/datacube.py index 97a8c7c77..6dcd52f98 100644 --- a/polytope_feature/datacube/backends/datacube.py +++ b/polytope_feature/datacube/backends/datacube.py @@ -48,11 +48,8 @@ def validate(self, axes): def _create_axes(self, name, values, transformation_type_key, transformation_options): # first check what the final axes are for this axis name given transformations transformation_options = transformation_type_key - final_axis_names = DatacubeAxisTransformation.get_final_axes( - name, transformation_type_key.name, transformation_options - ) - transformation = DatacubeAxisTransformation.create_transform( - name, transformation_type_key.name, transformation_options + (final_axis_names, transformation) = DatacubeAxisTransformation.get_final_axes( + name, transformation_type_key.name, transformation_options, self ) # do not compress merged axes diff --git a/polytope_feature/datacube/transformations/datacube_cyclic/datacube_cyclic.py b/polytope_feature/datacube/transformations/datacube_cyclic/datacube_cyclic.py index 3373dd082..90613b359 100644 --- a/polytope_feature/datacube/transformations/datacube_cyclic/datacube_cyclic.py +++ b/polytope_feature/datacube/transformations/datacube_cyclic/datacube_cyclic.py @@ -8,7 +8,7 @@ class DatacubeAxisCyclic(DatacubeAxisTransformation): # The transformation here will be to point the old axes to the new cyclic axes - def __init__(self, name, cyclic_options): + def __init__(self, name, cyclic_options, datacube=None): self.name = name self.transformation_options = cyclic_options self.range = cyclic_options.range diff --git a/polytope_feature/datacube/transformations/datacube_mappers/datacube_mappers.py b/polytope_feature/datacube/transformations/datacube_mappers/datacube_mappers.py index e28566e69..583da76a0 100644 --- a/polytope_feature/datacube/transformations/datacube_mappers/datacube_mappers.py +++ b/polytope_feature/datacube/transformations/datacube_mappers/datacube_mappers.py @@ -7,7 +7,7 @@ class DatacubeMapper(DatacubeAxisTransformation): # Needs to implements DatacubeAxisTransformation methods - def __init__(self, name, mapper_options): + def __init__(self, name, mapper_options, datacube=None): self.transformation_options = mapper_options self.grid_type = mapper_options.type self.grid_resolution = mapper_options.resolution diff --git a/polytope_feature/datacube/transformations/datacube_merger/datacube_merger.py b/polytope_feature/datacube/transformations/datacube_merger/datacube_merger.py index 5b0516638..5a86724f4 100644 --- a/polytope_feature/datacube/transformations/datacube_merger/datacube_merger.py +++ b/polytope_feature/datacube/transformations/datacube_merger/datacube_merger.py @@ -7,12 +7,13 @@ class DatacubeAxisMerger(DatacubeAxisTransformation): - def __init__(self, name, merge_options): + def __init__(self, name, merge_options, datacube=None): self.transformation_options = merge_options self.name = name self._first_axis = name self._second_axis = merge_options.other_axis self._linkers = merge_options.linkers + self._merged_values = self.merged_values(datacube) def blocked_axes(self): return [self._second_axis] @@ -38,6 +39,7 @@ def merged_values(self, datacube): val_to_add = val_to_add.astype("datetime64[s]") merged_values.append(val_to_add) merged_values = np.array(merged_values) + merged_values.sort() logging.info( f"Merged values {first_ax_vals} on axis {self.name} and \ values {second_ax_vals} on axis {second_ax_name} to values {merged_values}" @@ -59,7 +61,7 @@ def unmerge(self, merged_val): first_val = merged_val[:first_idx] first_linker_size = len(self._linkers[0]) second_linked_size = len(self._linkers[1]) - second_val = merged_val[first_idx + first_linker_size : -second_linked_size] + second_val = merged_val[first_idx + first_linker_size: -second_linked_size] # TODO: maybe replacing like this is too specific to time/dates? first_val = str(first_val).replace("-", "") @@ -78,7 +80,7 @@ def change_val_type(self, axis_name, values): def find_modified_indexes(self, indexes, path, datacube, axis): if axis.name == self._first_axis: - return self.merged_values(datacube) + return self._merged_values def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): new_key_value_path = {} diff --git a/polytope_feature/datacube/transformations/datacube_reverse/datacube_reverse.py b/polytope_feature/datacube/transformations/datacube_reverse/datacube_reverse.py index eed563c9e..baa38009d 100644 --- a/polytope_feature/datacube/transformations/datacube_reverse/datacube_reverse.py +++ b/polytope_feature/datacube/transformations/datacube_reverse/datacube_reverse.py @@ -3,7 +3,7 @@ class DatacubeAxisReverse(DatacubeAxisTransformation): - def __init__(self, name, mapper_options): + def __init__(self, name, mapper_options, datacube=None): self.name = name self.transformation_options = mapper_options diff --git a/polytope_feature/datacube/transformations/datacube_transformations.py b/polytope_feature/datacube/transformations/datacube_transformations.py index f174bf1aa..58467b81d 100644 --- a/polytope_feature/datacube/transformations/datacube_transformations.py +++ b/polytope_feature/datacube/transformations/datacube_transformations.py @@ -8,25 +8,25 @@ def __init__(self): self.parent = None @staticmethod - def create_transform(name, transformation_type_key, transformation_options): + def create_transform(name, transformation_type_key, transformation_options, datacube): transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key] file_name = ".datacube_" + transformation_file_name module = import_module("polytope_feature.datacube.transformations" + file_name + file_name) constructor = getattr(module, transformation_type) transformation_type_option = transformation_options - new_transformation = deepcopy(constructor(name, transformation_type_option)) + new_transformation = deepcopy(constructor(name, transformation_type_option, datacube)) new_transformation.name = name return new_transformation @staticmethod - def get_final_axes(name, transformation_type_key, transformation_options): + def get_final_axes(name, transformation_type_key, transformation_options, datacube): new_transformation = DatacubeAxisTransformation.create_transform( - name, transformation_type_key, transformation_options + name, transformation_type_key, transformation_options, datacube ) transformation_axis_names = new_transformation.transformation_axes_final() - return transformation_axis_names + return (transformation_axis_names, new_transformation) def name(self): pass diff --git a/polytope_feature/datacube/transformations/datacube_type_change/datacube_type_change.py b/polytope_feature/datacube/transformations/datacube_type_change/datacube_type_change.py index e9fa287db..5e395c95f 100644 --- a/polytope_feature/datacube/transformations/datacube_type_change/datacube_type_change.py +++ b/polytope_feature/datacube/transformations/datacube_type_change/datacube_type_change.py @@ -7,7 +7,7 @@ class DatacubeAxisTypeChange(DatacubeAxisTransformation): # The transformation here will be to point the old axes to the new cyclic axes - def __init__(self, name, type_options): + def __init__(self, name, type_options, datacube=None): self.name = name self.transformation_options = type_options self.new_type = type_options.type From 68732e3a401a7bb21234216fbc76b2653c95264f Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 16 Jan 2025 12:00:50 +0100 Subject: [PATCH 2/2] black --- .../datacube_mappers/mapper_types/healpix_nested.py | 8 +++++--- .../transformations/datacube_merger/datacube_merger.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py index dca87a7a3..334805d42 100644 --- a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py +++ b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py @@ -222,6 +222,8 @@ def int_sqrt(self, i): # md5 grid hash in form {resolution : hash} -_md5_hash = {1024: "cbda19e48d4d7e5e22641154878b9b22", - 512: "9533855ee8e38314e19aaa0434c310da", - 128: "f3dfeb7a5bbbdd13a20d10fdb3797c71"} +_md5_hash = { + 1024: "cbda19e48d4d7e5e22641154878b9b22", + 512: "9533855ee8e38314e19aaa0434c310da", + 128: "f3dfeb7a5bbbdd13a20d10fdb3797c71", +} diff --git a/polytope_feature/datacube/transformations/datacube_merger/datacube_merger.py b/polytope_feature/datacube/transformations/datacube_merger/datacube_merger.py index 5a86724f4..3d373ee5c 100644 --- a/polytope_feature/datacube/transformations/datacube_merger/datacube_merger.py +++ b/polytope_feature/datacube/transformations/datacube_merger/datacube_merger.py @@ -61,7 +61,7 @@ def unmerge(self, merged_val): first_val = merged_val[:first_idx] first_linker_size = len(self._linkers[0]) second_linked_size = len(self._linkers[1]) - second_val = merged_val[first_idx + first_linker_size: -second_linked_size] + second_val = merged_val[first_idx + first_linker_size : -second_linked_size] # TODO: maybe replacing like this is too specific to time/dates? first_val = str(first_val).replace("-", "")