diff --git a/.gitignore b/.gitignore index 7fcdd7723..4634ed76f 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ new_updated_numpy_venv newest-polytope-venv serializedTree new_polytope_venv +*.json \ No newline at end of file diff --git a/polytope_feature/datacube/backends/datacube.py b/polytope_feature/datacube/backends/datacube.py index 97a8c7c77..6dcd52f98 100644 --- a/polytope_feature/datacube/backends/datacube.py +++ b/polytope_feature/datacube/backends/datacube.py @@ -48,11 +48,8 @@ def validate(self, axes): def _create_axes(self, name, values, transformation_type_key, transformation_options): # first check what the final axes are for this axis name given transformations transformation_options = transformation_type_key - final_axis_names = DatacubeAxisTransformation.get_final_axes( - name, transformation_type_key.name, transformation_options - ) - transformation = DatacubeAxisTransformation.create_transform( - name, transformation_type_key.name, transformation_options + (final_axis_names, transformation) = DatacubeAxisTransformation.get_final_axes( + name, transformation_type_key.name, transformation_options, self ) # do not compress merged axes diff --git a/polytope_feature/datacube/transformations/datacube_cyclic/datacube_cyclic.py b/polytope_feature/datacube/transformations/datacube_cyclic/datacube_cyclic.py index 3373dd082..90613b359 100644 --- a/polytope_feature/datacube/transformations/datacube_cyclic/datacube_cyclic.py +++ b/polytope_feature/datacube/transformations/datacube_cyclic/datacube_cyclic.py @@ -8,7 +8,7 @@ class DatacubeAxisCyclic(DatacubeAxisTransformation): # The transformation here will be to point the old axes to the new cyclic axes - def __init__(self, name, cyclic_options): + def __init__(self, name, cyclic_options, datacube=None): self.name = name self.transformation_options = cyclic_options self.range = cyclic_options.range diff --git a/polytope_feature/datacube/transformations/datacube_mappers/datacube_mappers.py b/polytope_feature/datacube/transformations/datacube_mappers/datacube_mappers.py index e28566e69..583da76a0 100644 --- a/polytope_feature/datacube/transformations/datacube_mappers/datacube_mappers.py +++ b/polytope_feature/datacube/transformations/datacube_mappers/datacube_mappers.py @@ -7,7 +7,7 @@ class DatacubeMapper(DatacubeAxisTransformation): # Needs to implements DatacubeAxisTransformation methods - def __init__(self, name, mapper_options): + def __init__(self, name, mapper_options, datacube=None): self.transformation_options = mapper_options self.grid_type = mapper_options.type self.grid_resolution = mapper_options.resolution diff --git a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py index c71f59f5b..334805d42 100644 --- a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py +++ b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py @@ -222,4 +222,8 @@ def int_sqrt(self, i): # md5 grid hash in form {resolution : hash} -_md5_hash = {} +_md5_hash = { + 1024: "cbda19e48d4d7e5e22641154878b9b22", + 512: "9533855ee8e38314e19aaa0434c310da", + 128: "f3dfeb7a5bbbdd13a20d10fdb3797c71", +} diff --git a/polytope_feature/datacube/transformations/datacube_merger/datacube_merger.py b/polytope_feature/datacube/transformations/datacube_merger/datacube_merger.py index 5b0516638..aa53e111d 100644 --- a/polytope_feature/datacube/transformations/datacube_merger/datacube_merger.py +++ b/polytope_feature/datacube/transformations/datacube_merger/datacube_merger.py @@ -7,12 +7,13 @@ class DatacubeAxisMerger(DatacubeAxisTransformation): - def __init__(self, name, merge_options): + def __init__(self, name, merge_options, datacube=None): self.transformation_options = merge_options self.name = name self._first_axis = name self._second_axis = merge_options.other_axis self._linkers = merge_options.linkers + self._merged_values = self.merged_values(datacube) def blocked_axes(self): return [self._second_axis] @@ -24,20 +25,17 @@ def _mapped_axes(self): return self._first_axis def merged_values(self, datacube): - first_ax_vals = datacube.ax_vals(self.name) + first_ax_vals = np.array(datacube.ax_vals(self.name)) second_ax_name = self._second_axis - second_ax_vals = datacube.ax_vals(second_ax_name) + second_ax_vals = np.array(datacube.ax_vals(second_ax_name)) linkers = self._linkers - merged_values = [] - for i in range(len(first_ax_vals)): - first_val = first_ax_vals[i] - for j in range(len(second_ax_vals)): - second_val = second_ax_vals[j] - val_to_add = pd.to_datetime("".join([first_val, linkers[0], second_val, linkers[1]])) - val_to_add = val_to_add.to_numpy() - val_to_add = val_to_add.astype("datetime64[s]") - merged_values.append(val_to_add) + first_grid, second_grid = np.meshgrid(first_ax_vals, second_ax_vals, indexing="ij") + combined_strings = np.char.add( + np.char.add(first_grid.ravel(), linkers[0]), np.char.add(second_grid.ravel(), linkers[1]) + ) + merged_values = pd.to_datetime(combined_strings).to_numpy().astype("datetime64[s]") merged_values = np.array(merged_values) + merged_values.sort() logging.info( f"Merged values {first_ax_vals} on axis {self.name} and \ values {second_ax_vals} on axis {second_ax_name} to values {merged_values}" @@ -78,7 +76,7 @@ def change_val_type(self, axis_name, values): def find_modified_indexes(self, indexes, path, datacube, axis): if axis.name == self._first_axis: - return self.merged_values(datacube) + return self._merged_values def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): new_key_value_path = {} diff --git a/polytope_feature/datacube/transformations/datacube_reverse/datacube_reverse.py b/polytope_feature/datacube/transformations/datacube_reverse/datacube_reverse.py index eed563c9e..baa38009d 100644 --- a/polytope_feature/datacube/transformations/datacube_reverse/datacube_reverse.py +++ b/polytope_feature/datacube/transformations/datacube_reverse/datacube_reverse.py @@ -3,7 +3,7 @@ class DatacubeAxisReverse(DatacubeAxisTransformation): - def __init__(self, name, mapper_options): + def __init__(self, name, mapper_options, datacube=None): self.name = name self.transformation_options = mapper_options diff --git a/polytope_feature/datacube/transformations/datacube_transformations.py b/polytope_feature/datacube/transformations/datacube_transformations.py index f174bf1aa..58467b81d 100644 --- a/polytope_feature/datacube/transformations/datacube_transformations.py +++ b/polytope_feature/datacube/transformations/datacube_transformations.py @@ -8,25 +8,25 @@ def __init__(self): self.parent = None @staticmethod - def create_transform(name, transformation_type_key, transformation_options): + def create_transform(name, transformation_type_key, transformation_options, datacube): transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key] transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key] file_name = ".datacube_" + transformation_file_name module = import_module("polytope_feature.datacube.transformations" + file_name + file_name) constructor = getattr(module, transformation_type) transformation_type_option = transformation_options - new_transformation = deepcopy(constructor(name, transformation_type_option)) + new_transformation = deepcopy(constructor(name, transformation_type_option, datacube)) new_transformation.name = name return new_transformation @staticmethod - def get_final_axes(name, transformation_type_key, transformation_options): + def get_final_axes(name, transformation_type_key, transformation_options, datacube): new_transformation = DatacubeAxisTransformation.create_transform( - name, transformation_type_key, transformation_options + name, transformation_type_key, transformation_options, datacube ) transformation_axis_names = new_transformation.transformation_axes_final() - return transformation_axis_names + return (transformation_axis_names, new_transformation) def name(self): pass diff --git a/polytope_feature/datacube/transformations/datacube_type_change/datacube_type_change.py b/polytope_feature/datacube/transformations/datacube_type_change/datacube_type_change.py index e9fa287db..5e395c95f 100644 --- a/polytope_feature/datacube/transformations/datacube_type_change/datacube_type_change.py +++ b/polytope_feature/datacube/transformations/datacube_type_change/datacube_type_change.py @@ -7,7 +7,7 @@ class DatacubeAxisTypeChange(DatacubeAxisTransformation): # The transformation here will be to point the old axes to the new cyclic axes - def __init__(self, name, type_options): + def __init__(self, name, type_options, datacube=None): self.name = name self.transformation_options = type_options self.new_type = type_options.type diff --git a/polytope_feature/version.py b/polytope_feature/version.py index c916e6809..edb5a8e7b 100644 --- a/polytope_feature/version.py +++ b/polytope_feature/version.py @@ -1 +1 @@ -__version__ = "1.0.21" +__version__ = "1.0.22"