From a0360baa761725ae2795aab5dcf3f8e1a746c404 Mon Sep 17 00:00:00 2001 From: Mathilde Leuridan Date: Wed, 15 Nov 2023 14:01:37 +0100 Subject: [PATCH 01/26] small improvements --- performance/fdb_performance_3D.py | 1 + polytope/datacube/datacube_axis.py | 181 +++++++----------- polytope/datacube/index_tree.py | 11 +- .../transformations/datacube_mappers.py | 5 +- .../datacube_transformations.py | 3 - polytope/utility/geometry.py | 3 +- 6 files changed, 72 insertions(+), 132 deletions(-) diff --git a/performance/fdb_performance_3D.py b/performance/fdb_performance_3D.py index 2cfec94cf..c8723c179 100644 --- a/performance/fdb_performance_3D.py +++ b/performance/fdb_performance_3D.py @@ -20,6 +20,7 @@ def setup_method(self, method): "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, "levelist": {"transformation": {"type_change": "int"}}, + "longitude": {"transformation": {"cyclic": [0, 360]}} } self.config = {"class": "od", "expver": "0001", "levtype": "sfc"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 77ad107e0..b249a1b14 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -196,12 +196,11 @@ def find_indexes(path, datacube): # first, find the relevant transformation object that is a mapping in the cls.transformation dico for transform in cls.transformations: if isinstance(transform, DatacubeMapper): - transformation = transform - if cls.name == transformation._mapped_axes()[0]: - return transformation.first_axis_vals() - if cls.name == transformation._mapped_axes()[1]: - first_val = path[transformation._mapped_axes()[0]] - return transformation.second_axis_vals(first_val) + if cls.name == transform._mapped_axes()[0]: + return transform._first_axis_vals + if cls.name == transform._mapped_axes()[1]: + first_val = path[transform._mapped_axes()[0]] + return transform.second_axis_vals(first_val) old_unmap_to_datacube = cls.unmap_to_datacube @@ -255,8 +254,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between_ranges = [] for transform in cls.transformations: if isinstance(transform, DatacubeMapper): - transformation = transform - if cls.name in transformation._mapped_axes(): + if cls.name in transform._mapped_axes(): for idxs in index_ranges: if method == "surrounding": start = idxs.index(low) @@ -296,9 +294,8 @@ def find_indexes(path, datacube): # first, find the relevant transformation object that is a mapping in the cls.transformation dico for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): - transformation = transform - if cls.name == transformation._first_axis: - return transformation.merged_values(datacube) + if cls.name == transform._first_axis: + return transform.merged_values(datacube) old_n_unmap_path_key = cls.n_unmap_path_key @@ -320,13 +317,12 @@ def unmap_to_datacube(path, unmapped_path): (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): - transformation = transform - if cls.name == transformation._first_axis: + if cls.name == transform._first_axis: old_val = path.get(cls.name, None) - (first_val, second_val) = transformation.unmerge(old_val) + (first_val, second_val) = transform.unmerge(old_val) path.pop(cls.name, None) - path[transformation._first_axis] = first_val - path[transformation._second_axis] = second_val + path[transform._first_axis] = first_val + path[transform._second_axis] = second_val return (path, unmapped_path) def find_indices_between(index_ranges, low, up, datacube, method=None): @@ -334,8 +330,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between_ranges = [] for transform in cls.transformations: if isinstance(transform, DatacubeAxisMerger): - transformation = transform - if cls.name in transformation._mapped_axes(): + if cls.name in transform._mapped_axes(): for indexes in index_ranges: if method == "surrounding": start = indexes.index(low) @@ -351,10 +346,6 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between_ranges.append(indexes_between) return indexes_between_ranges - def remap(range): - return [range] - - cls.remap = remap cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between @@ -383,45 +374,39 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between_ranges = [] for transform in cls.transformations: if isinstance(transform, DatacubeAxisReverse): - transformation = transform - if cls.name == transformation.name: - for indexes in index_ranges: - if cls.name in datacube.complete_axes: - # Find the range of indexes between lower and upper - # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html - # Assumes the indexes are already sorted (could sort to be sure) and monotonically - # increasing - if method == "surrounding": - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end].to_list() - indexes_between_ranges.append(indexes_between) - else: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - indexes_between = indexes[start:end].to_list() - indexes_between_ranges.append(indexes_between) + for indexes in index_ranges: + if cls.name in datacube.complete_axes: + # Find the range of indexes between lower and upper + # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html + # Assumes the indexes are already sorted (could sort to be sure) and monotonically + # increasing + if method == "surrounding": + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.append(indexes_between) else: - if method == "surrounding": - start = indexes.index(low) - end = indexes.index(up) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - else: - lower_idx = bisect.bisect_left(indexes, low) - upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx:upper_idx] - indexes_between_ranges.append(indexes_between) + start = indexes.searchsorted(low, "left") + end = indexes.searchsorted(up, "right") + indexes_between = indexes[start:end].to_list() + indexes_between_ranges.append(indexes_between) + else: + if method == "surrounding": + start = indexes.index(low) + end = indexes.index(up) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + else: + lower_idx = bisect.bisect_left(indexes, low) + upper_idx = bisect.bisect_right(indexes, up) + indexes_between = indexes[lower_idx:upper_idx] + indexes_between_ranges.append(indexes_between) return indexes_between_ranges - def remap(range): - return [range] - - cls.remap = remap cls.find_indexes = find_indexes cls.find_indices_between = find_indices_between @@ -437,10 +422,8 @@ def type_change(cls): def find_indexes(path, datacube): for transform in cls.transformations: if isinstance(transform, DatacubeAxisTypeChange): - transformation = transform - if cls.name == transformation.name: - original_vals = old_find_indexes(path, datacube) - return transformation.change_val_type(cls.name, original_vals) + original_vals = old_find_indexes(path, datacube) + return transform.change_val_type(cls.name, original_vals) old_n_unmap_path_key = cls.n_unmap_path_key @@ -449,21 +432,18 @@ def n_unmap_path_key(key_value_path, leaf_path, unwanted_path): value = key_value_path[cls.name] for transform in cls.transformations: if isinstance(transform, DatacubeAxisTypeChange): - if cls.name == transform.name: - unchanged_val = transform.make_str(value) - key_value_path[cls.name] = unchanged_val + unchanged_val = transform.make_str(value) + key_value_path[cls.name] = unchanged_val return (key_value_path, leaf_path, unwanted_path) def unmap_to_datacube(path, unmapped_path): for transform in cls.transformations: if isinstance(transform, DatacubeAxisTypeChange): - transformation = transform - if cls.name == transformation.name: - changed_val = path.get(cls.name, None) - unchanged_val = transformation.make_str(changed_val) - if cls.name in path: - path.pop(cls.name, None) - unmapped_path[cls.name] = unchanged_val + changed_val = path.get(cls.name, None) + unchanged_val = transform.make_str(changed_val) + if cls.name in path: + path.pop(cls.name, None) + unmapped_path[cls.name] = unchanged_val return (path, unmapped_path) def find_indices_between(index_ranges, low, up, datacube, method=None): @@ -471,27 +451,21 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between_ranges = [] for transform in cls.transformations: if isinstance(transform, DatacubeAxisTypeChange): - transformation = transform - if cls.name == transformation.name: - for indexes in index_ranges: - if method == "surrounding": - start = indexes.index(low) - end = indexes.index(up) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - else: - lower_idx = bisect.bisect_left(indexes, low) - upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx:upper_idx] - indexes_between_ranges.append(indexes_between) + for indexes in index_ranges: + if method == "surrounding": + start = indexes.index(low) + end = indexes.index(up) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + else: + lower_idx = bisect.bisect_left(indexes, low) + upper_idx = bisect.bisect_right(indexes, up) + indexes_between = indexes[lower_idx:upper_idx] + indexes_between_ranges.append(indexes_between) return indexes_between_ranges - def remap(range): - return [range] - - cls.remap = remap cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between @@ -500,30 +474,6 @@ def remap(range): return cls -def null(cls): - if cls.type_change: - old_find_indexes = cls.find_indexes - - def find_indexes(path, datacube): - return old_find_indexes(path, datacube) - - def find_indices_between(index_ranges, low, up, datacube, method=None): - indexes_between_ranges = [] - for indexes in index_ranges: - indexes_between = [i for i in indexes if low <= i <= up] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - - def remap(range): - return [range] - - cls.remap = remap - cls.find_indexes = find_indexes - cls.find_indices_between = find_indices_between - - return cls - - class DatacubeAxis(ABC): is_cyclic = False has_mapper = False @@ -638,6 +588,7 @@ def __init__(self): self.name = None self.tol = 1e-12 self.range = None + # TODO: Maybe here, store transformations as a dico instead self.transformations = [] self.type = 0 diff --git a/polytope/datacube/index_tree.py b/polytope/datacube/index_tree.py index 9054cd0c5..ffd813046 100644 --- a/polytope/datacube/index_tree.py +++ b/polytope/datacube/index_tree.py @@ -45,12 +45,6 @@ def leaves_with_ancestors(self): self._collect_leaf_nodes(leaves) return leaves - def _collect_leaf_nodes_old(self, leaves): - if len(self.children) == 0: - leaves.append(self) - for n in self.children: - n._collect_leaf_nodes(leaves) - def _collect_leaf_nodes(self, leaves): # NOTE: leaves_and_ancestors is going to be a list of tuples, where first entry is leaf and second entry is a # list of its ancestors @@ -84,13 +78,10 @@ def __eq__(self, other): else: if other.value == self.value: return True - if other.value - 2 * other.axis.tol <= self.value <= other.value + 2 * other.axis.tol: - return True - elif self.value - 2 * self.axis.tol <= other.value <= self.value + 2 * self.axis.tol: + if abs(other.value - self.value) <= 2 * other.axis.tol: return True else: return False - # return (self.axis.name, self.value) == (other.axis.name, other.value) def __lt__(self, other): return (self.axis.name, self.value) < (other.axis.name, other.value) diff --git a/polytope/datacube/transformations/datacube_mappers.py b/polytope/datacube/transformations/datacube_mappers.py index 34af16bdc..44b705cfa 100644 --- a/polytope/datacube/transformations/datacube_mappers.py +++ b/polytope/datacube/transformations/datacube_mappers.py @@ -19,6 +19,7 @@ def __init__(self, name, mapper_options): self._final_transformation = self.generate_final_transformation() self._final_mapped_axes = self._final_transformation._mapped_axes self._axis_reversed = self._final_transformation._axis_reversed + self._first_axis_vals = self._final_transformation._first_axis_vals def generate_final_transformation(self): map_type = _type_to_datacube_mapper_lookup[self.grid_type] @@ -83,6 +84,7 @@ def __init__(self, base_axis, mapped_axes, resolution): self._resolution = resolution self.deg_increment = 90 / self._resolution self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} + self._first_axis_vals = self.first_axis_vals() def first_axis_vals(self): first_ax_vals = [-90 + i * self.deg_increment for i in range(2 * self._resolution)] @@ -134,6 +136,7 @@ def __init__(self, base_axis, mapped_axes, resolution): self._base_axis = base_axis self._resolution = resolution self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} + self._first_axis_vals = self.first_axis_vals() def first_axis_vals(self): rad2deg = 180 / math.pi @@ -319,8 +322,6 @@ def gauss_first_guess(self): def get_precomputed_values_N1280(self): lats = [0] * 2560 - # lats = SortedList() - # lats = {} lats[0] = 89.946187715665616 lats[1] = 89.876478353332288 lats[2] = 89.806357319542244 diff --git a/polytope/datacube/transformations/datacube_transformations.py b/polytope/datacube/transformations/datacube_transformations.py index 900ad16b6..f4ce357a9 100644 --- a/polytope/datacube/transformations/datacube_transformations.py +++ b/polytope/datacube/transformations/datacube_transformations.py @@ -50,7 +50,6 @@ def change_val_type(self, axis_name, values): "merge": "DatacubeAxisMerger", "reverse": "DatacubeAxisReverse", "type_change": "DatacubeAxisTypeChange", - "null": "DatacubeNullTransformation", } _type_to_transformation_file_lookup = { @@ -59,7 +58,6 @@ def change_val_type(self, axis_name, values): "merge": "merger", "reverse": "reverse", "type_change": "type_change", - "null": "null_transformation", } has_transform = { @@ -68,5 +66,4 @@ def change_val_type(self, axis_name, values): "merge": "has_merger", "reverse": "reorder", "type_change": "type_change", - "null": "null", } diff --git a/polytope/utility/geometry.py b/polytope/utility/geometry.py index bbbb75152..f1dc42eb6 100644 --- a/polytope/utility/geometry.py +++ b/polytope/utility/geometry.py @@ -1,4 +1,3 @@ def lerp(a, b, value): - direction = [a - b for a, b in zip(a, b)] - intersect = [b + value * d for b, d in zip(b, direction)] + intersect = [b + (a-b) * value for a, b in zip(a, b)] return intersect From 9e45b9e74c1eea889bb4e31cbc39a628994a9876 Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 7 Feb 2024 12:50:32 +0000 Subject: [PATCH 02/26] add meteosuisse local grid test --- tests/test_local_swiss_grid.py | 89 ++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tests/test_local_swiss_grid.py diff --git a/tests/test_local_swiss_grid.py b/tests/test_local_swiss_grid.py new file mode 100644 index 000000000..69d1984d3 --- /dev/null +++ b/tests/test_local_swiss_grid.py @@ -0,0 +1,89 @@ +# import geopandas as gpd +# import matplotlib.pyplot as plt +import pandas as pd +import pytest +from eccodes import codes_grib_find_nearest, codes_grib_new_from_file + +from polytope.engine.hullslicer import HullSlicer +from polytope.polytope import Polytope, Request +from polytope.shapes import Box, Select + + +class TestSlicingFDBDatacube: + def setup_method(self, method): + from polytope.datacube.backends.fdb import FDBDatacube + + # Create a dataarray with 3 labelled axes using different index types + self.options = { + "values": { + "mapper": { + "type": "local_regular", + "resolution": [193, 417], + "axes": ["latitude", "longitude"], + "local": [45.485, 48.1, 5.28985, 10.9087], + } + }, + "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, + "step": {"type_change": "int"}, + "number": {"type_change": "int"}, + "levelist": {"type_change": "int"}, + } + + self.config = {"param": "3008"} + self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) + self.slicer = HullSlicer() + self.API = Polytope(datacube=self.fdbdatacube, engine=self.slicer, axis_options=self.options) + + # Testing different shapes + @pytest.mark.fdb + @pytest.mark.skip("Non-accessible data") + def test_fdb_datacube(self): + request = Request( + Select("step", [0]), + Select("levtype", ["unknown"]), + Select("date", [pd.Timestamp("20211102T120000")]), + Select("param", ["3008"]), + Select("levelist", [1]), + Box(["latitude", "longitude"], [47.38, 7], [47.5, 7.14]), + ) + result = self.API.retrieve(request) + # result.pprint_2() + assert len(result.leaves) == 99 + + lats = [] + lons = [] + eccodes_lats = [] + eccodes_lons = [] + tol = 1e-4 + f = open("./tests/data/hhl_geo.grib", "rb") + messages = [] + message = codes_grib_new_from_file(f) + messages.append(message) + + leaves = result.leaves + for i in range(len(leaves)): + cubepath = leaves[i].flatten() + lat = cubepath["latitude"] + lon = cubepath["longitude"] + del cubepath + lats.append(lat) + lons.append(lon) + nearest_points = codes_grib_find_nearest(message, lat, lon)[0] + eccodes_lat = nearest_points.lat + eccodes_lon = nearest_points.lon + eccodes_lats.append(eccodes_lat) + eccodes_lons.append(eccodes_lon) + assert eccodes_lat - tol <= lat + assert lat <= eccodes_lat + tol + assert eccodes_lon - tol <= lon + assert lon <= eccodes_lon + tol + f.close() + + # worldmap = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) + # fig, ax = plt.subplots(figsize=(12, 6)) + # worldmap.plot(color="darkgrey", ax=ax) + + # plt.scatter(lons, lats, s=18, c="red", cmap="YlOrRd") + # plt.scatter(eccodes_lons, eccodes_lats, s=6, c="green") + # plt.colorbar(label="Temperature") + # plt.show() From 0009eb1749d852f065fe8f2aacfb16bdc8f41a2a Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 8 Feb 2024 14:42:02 +0000 Subject: [PATCH 03/26] update requirements --- .gitattributes | 4 ---- .gitconfig | 2 -- docs/requirements.txt | 2 +- examples/requirements_examples.txt | 2 +- requirements.txt | 2 +- requirements_example.txt | 1 - 6 files changed, 3 insertions(+), 10 deletions(-) delete mode 100644 .gitattributes delete mode 100644 .gitconfig delete mode 100644 requirements_example.txt diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index edc195ff1..000000000 --- a/.gitattributes +++ /dev/null @@ -1,4 +0,0 @@ -examples/data/*.grib filter=lfs diff=lfs merge=lfs -text -examples/data/*.jpg filter=lfs diff=lfs merge=lfs -text -examples/data/*.shp filter=lfs diff=lfs merge=lfs -text -tests/data/*.grib filter=lfs diff=lfs merge=lfs -text diff --git a/.gitconfig b/.gitconfig deleted file mode 100644 index e2bbefe3b..000000000 --- a/.gitconfig +++ /dev/null @@ -1,2 +0,0 @@ -[lfs] - fetchexclude = * \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index 03024ec5d..091f576d3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,3 @@ -jinja2<3.1.0 +jinja2>=3.1.3 Markdown<3.2 mkdocs>=1.0 \ No newline at end of file diff --git a/examples/requirements_examples.txt b/examples/requirements_examples.txt index 288d25581..8c1f9a3cc 100644 --- a/examples/requirements_examples.txt +++ b/examples/requirements_examples.txt @@ -3,7 +3,7 @@ matplotlib==3.6.2 matplotlib-inline==0.1.6 -Pillow==9.3.0 +Pillow>=10.2.0 Shapely==1.8.5.post1 shp==1.0.2 Fiona==1.8.22 diff --git a/requirements.txt b/requirements.txt index 50594c4fc..dade23e27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ numpy==1.23.5 pandas==1.5.2 pypi==2.1 requests==2.28.1 -scipy==1.9.3 +scipy==1.11.4 sortedcontainers==2.4.0 tripy==1.0.0 typing==3.7.4.3 diff --git a/requirements_example.txt b/requirements_example.txt deleted file mode 100644 index 19717dacf..000000000 --- a/requirements_example.txt +++ /dev/null @@ -1 +0,0 @@ -cfgrib==0.9.10.3 \ No newline at end of file From 9810972e3bd4f979417ea989681829ac1835115e Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 8 Feb 2024 15:02:13 +0000 Subject: [PATCH 04/26] black --- tests/test_point_nearest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py index 3bc5f93ec..c5d634c40 100644 --- a/tests/test_point_nearest.py +++ b/tests/test_point_nearest.py @@ -110,7 +110,7 @@ def test_fdb_datacube_true_point_5(self): Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["fc"]), - Point(["latitude", "longitude"], [[0.035149384216, 360-0.01]], method="nearest"), + Point(["latitude", "longitude"], [[0.035149384216, 360 - 0.01]], method="nearest"), ) result = self.API.retrieve(request) result.pprint() From af28bcb29baeedc9080ca5e4878d1bd8ef8adb8d Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 12 Feb 2024 11:42:18 +0100 Subject: [PATCH 05/26] remove unnecessary transformation function --- polytope/datacube/backends/datacube.py | 4 ++++ polytope/datacube/backends/fdb.py | 1 + polytope/datacube/backends/xarray.py | 1 + .../datacube_cyclic/cyclic_axis_decorator.py | 6 ------ .../datacube_type_change/type_change_axis_decorator.py | 1 - 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index f79be284d..da63e2e0d 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -16,6 +16,10 @@ class Datacube(ABC): + + # def __init__(self): + # self.is_xarray = False + @abstractmethod def get(self, requests: IndexTree) -> Any: """Return data given a set of request trees""" diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 2cd6da8d6..351808874 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -29,6 +29,7 @@ def __init__(self, config=None, axis_options=None, datacube_options=None): self.nearest_search = {} self.coupled_axes = [] self.axis_with_identical_structure_after = datacube_options.get("identical structure after") + # self.is_xarray = False partial_request = config # Find values in the level 3 FDB datacube diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 5412ca4f1..33032c95e 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -24,6 +24,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options=None, datacube_options= self.nearest_search = None self.coupled_axes = [] self.axis_with_identical_structure_after = datacube_options.get("identical structure after") + # self.is_xarray = True for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: diff --git a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py index 972d6d1aa..5278b3c26 100644 --- a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py @@ -113,11 +113,6 @@ def remap(range: List): ranges.append([low - cls.tol, up + cls.tol]) return ranges - old_find_indexes = cls.find_indexes - - def find_indexes(path, datacube): - return old_find_indexes(path, datacube) - old_unmap_path_key = cls.unmap_path_key def unmap_path_key(key_value_path, leaf_path, unwanted_path): @@ -180,7 +175,6 @@ def offset(range): cls.to_intervals = to_intervals cls.remap = remap cls.offset = offset - cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between cls.unmap_path_key = unmap_path_key diff --git a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py index 0e3669825..e55117542 100644 --- a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py @@ -40,7 +40,6 @@ def unmap_to_datacube(path, unmapped_path): return (path, unmapped_path) def find_indices_between(index_ranges, low, up, datacube, method=None): - # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: if isinstance(transform, DatacubeAxisTypeChange): From 53f4795a2d9f1ba8a7c036d05bc10f5c8a454484 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 12 Feb 2024 15:25:56 +0100 Subject: [PATCH 06/26] add TODO --- .../transformations/datacube_mappers/mapper_axis_decorator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py index 994321860..b8d3aa2d9 100644 --- a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py @@ -66,7 +66,7 @@ def unmap_path_key(key_value_path, leaf_path, unwanted_path): return (key_value_path, leaf_path, unwanted_path) def find_indices_between(index_ranges, low, up, datacube, method=None): - # TODO: add method for snappping + # TODO: untangle the reverse transformation from here... indexes_between_ranges = [] for transform in cls.transformations: if isinstance(transform, DatacubeMapper): From 06d95a092ee6b0f918fb02590266da3b607507b7 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 12 Feb 2024 16:07:46 +0100 Subject: [PATCH 07/26] simplify transformations --- polytope/datacube/backends/datacube.py | 1 + polytope/datacube/backends/fdb.py | 1 + polytope/datacube/backends/xarray.py | 1 + polytope/datacube/datacube_axis.py | 6 ++-- .../datacube_merger/merger_axis_decorator.py | 30 ++++++++---------- .../type_change_axis_decorator.py | 31 +++++++++---------- 6 files changed, 34 insertions(+), 36 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index da63e2e0d..685bbd17f 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -67,6 +67,7 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt def _add_all_transformation_axes(self, options, name, values): for transformation_type_key in options.keys(): + self.transformed_axes.append(name) self._create_axes(name, values, transformation_type_key, options) def _check_and_add_axes(self, options, name, values): diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 351808874..4721c7a52 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -30,6 +30,7 @@ def __init__(self, config=None, axis_options=None, datacube_options=None): self.coupled_axes = [] self.axis_with_identical_structure_after = datacube_options.get("identical structure after") # self.is_xarray = False + self.transformed_axes = [] partial_request = config # Find values in the level 3 FDB datacube diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 33032c95e..54f963c63 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -24,6 +24,7 @@ def __init__(self, dataarray: xr.DataArray, axis_options=None, datacube_options= self.nearest_search = None self.coupled_axes = [] self.axis_with_identical_structure_after = datacube_options.get("identical structure after") + self.transformed_axes = [] # self.is_xarray = True for name, values in dataarray.coords.variables.items(): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 5bed5dfe8..86afa71f0 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -1,3 +1,4 @@ +import bisect from abc import ABC, abstractmethod from copy import deepcopy from typing import Any, List @@ -70,7 +71,6 @@ def _remap_val_to_axis_range(self, value): return value def find_indices_between(self, index_ranges, low, up, datacube, method=None): - # TODO: add method for snappping indexes_between_ranges = [] for indexes in index_ranges: if self.name in datacube.complete_axes: @@ -98,7 +98,9 @@ def find_indices_between(self, index_ranges, low, up, datacube, method=None): indexes_between = indexes[start:end] indexes_between_ranges.append(indexes_between) else: - indexes_between = [i for i in indexes if low <= i <= up] + lower_idx = bisect.bisect_left(indexes, low) + upper_idx = bisect.bisect_right(indexes, up) + indexes_between = indexes[lower_idx:upper_idx] indexes_between_ranges.append(indexes_between) return indexes_between_ranges diff --git a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py index 16d816391..5958f8aca 100644 --- a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py @@ -46,23 +46,19 @@ def unmap_to_datacube(path, unmapped_path): def find_indices_between(index_ranges, low, up, datacube, method=None): # TODO: add method for snappping indexes_between_ranges = [] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - transformation = transform - if cls.name in transformation._mapped_axes(): - for indexes in index_ranges: - if method == "surrounding" or method == "nearest": - start = indexes.index(low) - end = indexes.index(up) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - else: - lower_idx = bisect.bisect_left(indexes, low) - upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx:upper_idx] - indexes_between_ranges.append(indexes_between) + for indexes in index_ranges: + if method == "surrounding" or method == "nearest": + start = indexes.index(low) + end = indexes.index(up) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + else: + lower_idx = bisect.bisect_left(indexes, low) + upper_idx = bisect.bisect_right(indexes, up) + indexes_between = indexes[lower_idx:upper_idx] + indexes_between_ranges.append(indexes_between) return indexes_between_ranges def remap(range): diff --git a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py index e55117542..a5a473241 100644 --- a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py @@ -40,24 +40,21 @@ def unmap_to_datacube(path, unmapped_path): return (path, unmapped_path) def find_indices_between(index_ranges, low, up, datacube, method=None): + # TODO: make all the find_indices_between functions the same indexes_between_ranges = [] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - transformation = transform - if cls.name == transformation.name: - for indexes in index_ranges: - if method == "surrounding" or method == "nearest": - start = indexes.index(low) - end = indexes.index(up) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - else: - lower_idx = bisect.bisect_left(indexes, low) - upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx:upper_idx] - indexes_between_ranges.append(indexes_between) + for indexes in index_ranges: + if method == "surrounding" or method == "nearest": + start = indexes.index(low) + end = indexes.index(up) + start = max(start - 1, 0) + end = min(end + 1, len(indexes)) + indexes_between = indexes[start:end] + indexes_between_ranges.append(indexes_between) + else: + lower_idx = bisect.bisect_left(indexes, low) + upper_idx = bisect.bisect_right(indexes, up) + indexes_between = indexes[lower_idx:upper_idx] + indexes_between_ranges.append(indexes_between) return indexes_between_ranges def remap(range): From 86bf99efe5f16a35434a131a68fc7223601a8828 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 12 Feb 2024 16:24:13 +0100 Subject: [PATCH 08/26] remove unnecessary find_indexes_between function in merge and type change transformations --- polytope/datacube/backends/datacube.py | 3 ++- polytope/datacube/datacube_axis.py | 2 +- .../datacube_merger/merger_axis_decorator.py | 21 ------------------- .../type_change_axis_decorator.py | 21 ------------------- 4 files changed, 3 insertions(+), 44 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 685bbd17f..e7ed76e9e 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -67,7 +67,8 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt def _add_all_transformation_axes(self, options, name, values): for transformation_type_key in options.keys(): - self.transformed_axes.append(name) + if transformation_type_key != "cyclic": + self.transformed_axes.append(name) self._create_axes(name, values, transformation_type_key, options) def _check_and_add_axes(self, options, name, values): diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 86afa71f0..bab7e6adb 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -73,7 +73,7 @@ def _remap_val_to_axis_range(self, value): def find_indices_between(self, index_ranges, low, up, datacube, method=None): indexes_between_ranges = [] for indexes in index_ranges: - if self.name in datacube.complete_axes: + if self.name in datacube.complete_axes and self.name not in datacube.transformed_axes: # Find the range of indexes between lower and upper # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing diff --git a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py index 5958f8aca..ed09bef2c 100644 --- a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py @@ -1,5 +1,3 @@ -import bisect - from .datacube_merger import DatacubeAxisMerger @@ -43,31 +41,12 @@ def unmap_to_datacube(path, unmapped_path): path[transformation._second_axis] = second_val return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method=None): - # TODO: add method for snappping - indexes_between_ranges = [] - for indexes in index_ranges: - if method == "surrounding" or method == "nearest": - start = indexes.index(low) - end = indexes.index(up) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - else: - lower_idx = bisect.bisect_left(indexes, low) - upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx:upper_idx] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - def remap(range): return [range] cls.remap = remap cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube - cls.find_indices_between = find_indices_between cls.unmap_path_key = unmap_path_key return cls diff --git a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py index a5a473241..1e1c21f25 100644 --- a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py @@ -1,5 +1,3 @@ -import bisect - from .datacube_type_change import DatacubeAxisTypeChange @@ -39,31 +37,12 @@ def unmap_to_datacube(path, unmapped_path): unmapped_path[cls.name] = unchanged_val return (path, unmapped_path) - def find_indices_between(index_ranges, low, up, datacube, method=None): - # TODO: make all the find_indices_between functions the same - indexes_between_ranges = [] - for indexes in index_ranges: - if method == "surrounding" or method == "nearest": - start = indexes.index(low) - end = indexes.index(up) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - else: - lower_idx = bisect.bisect_left(indexes, low) - upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx:upper_idx] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - def remap(range): return [range] cls.remap = remap cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube - cls.find_indices_between = find_indices_between cls.unmap_path_key = unmap_path_key return cls From a16124fbea7588bc228bec8e60693d5a505cdc6b Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 12 Feb 2024 16:40:33 +0100 Subject: [PATCH 09/26] remove unnecessary cyclic transformation decorator function --- .../datacube_cyclic/cyclic_axis_decorator.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py index 5278b3c26..0d3e42e01 100644 --- a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py @@ -125,12 +125,6 @@ def unmap_path_key(key_value_path, leaf_path, unwanted_path): key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) return (key_value_path, leaf_path, unwanted_path) - old_unmap_to_datacube = cls.unmap_to_datacube - - def unmap_to_datacube(path, unmapped_path): - (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) - return (path, unmapped_path) - old_find_indices_between = cls.find_indices_between def find_indices_between(index_ranges, low, up, datacube, method=None): @@ -175,7 +169,6 @@ def offset(range): cls.to_intervals = to_intervals cls.remap = remap cls.offset = offset - cls.unmap_to_datacube = unmap_to_datacube cls.find_indices_between = find_indices_between cls.unmap_path_key = unmap_path_key cls._remap_val_to_axis_range = _remap_val_to_axis_range From 0eba71c6152c9b987c8710280cfaf7d4498cd38c Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 12 Feb 2024 16:43:14 +0100 Subject: [PATCH 10/26] remove cyclic find_indices_between function --- .../datacube_cyclic/cyclic_axis_decorator.py | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py index 0d3e42e01..04cf4cd6c 100644 --- a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py @@ -1,4 +1,3 @@ -import bisect import math from copy import deepcopy from typing import List @@ -125,38 +124,6 @@ def unmap_path_key(key_value_path, leaf_path, unwanted_path): key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) return (key_value_path, leaf_path, unwanted_path) - old_find_indices_between = cls.find_indices_between - - def find_indices_between(index_ranges, low, up, datacube, method=None): - update_range() - indexes_between_ranges = [] - - if method != "surrounding" or method != "nearest": - return old_find_indices_between(index_ranges, low, up, datacube, method) - else: - for indexes in index_ranges: - if cls.name in datacube.complete_axes: - start = indexes.searchsorted(low, "left") - end = indexes.searchsorted(up, "right") - else: - start = bisect.bisect_left(indexes, low) - end = bisect.bisect_right(indexes, up) - - if start - 1 < 0: - index_val_found = indexes[-1:][0] - indexes_between_ranges.append([index_val_found]) - if end + 1 > len(indexes): - index_val_found = indexes[:2][0] - indexes_between_ranges.append([index_val_found]) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) - if cls.name in datacube.complete_axes: - indexes_between = indexes[start:end].to_list() - else: - indexes_between = indexes[start:end] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - def offset(range): # We first unpad the range by the axis tolerance to make sure that # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. @@ -169,7 +136,6 @@ def offset(range): cls.to_intervals = to_intervals cls.remap = remap cls.offset = offset - cls.find_indices_between = find_indices_between cls.unmap_path_key = unmap_path_key cls._remap_val_to_axis_range = _remap_val_to_axis_range From 2633ed203cdf716ce8dbc7af69ce5f6897722b14 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 12 Feb 2024 16:46:22 +0100 Subject: [PATCH 11/26] remove remap_path_key function for cyclic decorator --- .../datacube_cyclic/cyclic_axis_decorator.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py index 04cf4cd6c..e0fdb533f 100644 --- a/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_cyclic/cyclic_axis_decorator.py @@ -112,18 +112,6 @@ def remap(range: List): ranges.append([low - cls.tol, up + cls.tol]) return ranges - old_unmap_path_key = cls.unmap_path_key - - def unmap_path_key(key_value_path, leaf_path, unwanted_path): - value = key_value_path[cls.name] - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisCyclic): - if cls.name == transform.name: - new_val = _remap_val_to_axis_range(value) - key_value_path[cls.name] = new_val - key_value_path, leaf_path, unwanted_path = old_unmap_path_key(key_value_path, leaf_path, unwanted_path) - return (key_value_path, leaf_path, unwanted_path) - def offset(range): # We first unpad the range by the axis tolerance to make sure that # we find the wanted range of the cyclic axis since we padded by the axis tolerance before. @@ -136,7 +124,6 @@ def offset(range): cls.to_intervals = to_intervals cls.remap = remap cls.offset = offset - cls.unmap_path_key = unmap_path_key cls._remap_val_to_axis_range = _remap_val_to_axis_range return cls From c37e7a764d2c83bdc83daccfb8aa8b73205c565b Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 13 Feb 2024 13:59:11 +0100 Subject: [PATCH 12/26] separate mappers from reverse transformation and make better reverse transformation --- polytope/datacube/backends/datacube.py | 1 - polytope/datacube/backends/fdb.py | 1 + polytope/datacube/datacube_axis.py | 6 ++- .../datacube_mappers/mapper_axis_decorator.py | 39 ------------------- .../reverse_axis_decorator.py | 21 +++++----- tests/test_cyclic_axis_over_negative_vals.py | 2 - tests/test_cyclic_axis_slicer_not_0.py | 2 - tests/test_cyclic_axis_slicing.py | 2 - tests/test_cyclic_nearest.py | 1 + tests/test_cyclic_simple.py | 2 - tests/test_cyclic_snapping.py | 2 - tests/test_datacube_axes_init.py | 3 +- tests/test_ecmwf_oper_data_fdb.py | 1 + tests/test_fdb_datacube.py | 1 + tests/test_float_type.py | 2 - tests/test_healpix_mapper.py | 2 - tests/test_local_grid_cyclic.py | 1 + tests/test_local_regular_grid.py | 1 + tests/test_merge_cyclic_octahedral.py | 2 - tests/test_merge_octahedral_one_axis.py | 3 +- tests/test_merge_transformation.py | 2 - tests/test_multiple_param_fdb.py | 1 + tests/test_octahedral_grid.py | 5 +-- tests/test_point_nearest.py | 4 +- tests/test_point_shape.py | 2 - tests/test_regular_grid.py | 1 + tests/test_reverse_transformation.py | 2 - tests/test_shapes.py | 1 + tests/test_slice_date_range_fdb.py | 1 + tests/test_slice_date_range_fdb_v2.py | 1 + tests/test_slicer_era5.py | 2 - tests/test_slicing_unsliceable_axis.py | 2 - tests/test_slicing_xarray_4D.py | 2 - tests/test_snapping.py | 2 - tests/test_snapping_real_data.py | 2 - tests/test_type_change_transformation.py | 2 - 36 files changed, 31 insertions(+), 96 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index e7ed76e9e..ef3519b4b 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -16,7 +16,6 @@ class Datacube(ABC): - # def __init__(self): # self.is_xarray = False diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 4721c7a52..8fa3009e7 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -31,6 +31,7 @@ def __init__(self, config=None, axis_options=None, datacube_options=None): self.axis_with_identical_structure_after = datacube_options.get("identical structure after") # self.is_xarray = False self.transformed_axes = [] + self.dataarray = [] partial_request = config # Find values in the level 3 FDB datacube diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index bab7e6adb..2879dfb41 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -91,8 +91,10 @@ def find_indices_between(self, index_ranges, low, up, datacube, method=None): indexes_between_ranges.append(indexes_between) else: if method == "surrounding" or method == "nearest": - start = indexes.index(low) - end = indexes.index(up) + # start = indexes.index(low) + # end = indexes.index(up) + start = bisect.bisect_left(indexes, low) + end = bisect.bisect_right(indexes, up) start = max(start - 1, 0) end = min(end + 1, len(indexes)) indexes_between = indexes[start:end] diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py index b8d3aa2d9..c2d0703b9 100644 --- a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py @@ -1,6 +1,3 @@ -import bisect - -from ....utility.list_tools import bisect_left_cmp, bisect_right_cmp from .datacube_mappers import DatacubeMapper @@ -65,44 +62,8 @@ def unmap_path_key(key_value_path, leaf_path, unwanted_path): key_value_path[transform.old_axis] = unmapped_idx return (key_value_path, leaf_path, unwanted_path) - def find_indices_between(index_ranges, low, up, datacube, method=None): - # TODO: untangle the reverse transformation from here... - indexes_between_ranges = [] - for transform in cls.transformations: - if isinstance(transform, DatacubeMapper): - transformation = transform - if cls.name in transformation._mapped_axes(): - for idxs in index_ranges: - if method == "surrounding" or method == "nearest": - axis_reversed = transform._axis_reversed[cls.name] - if not axis_reversed: - start = bisect.bisect_left(idxs, low) - end = bisect.bisect_right(idxs, up) - else: - # TODO: do the custom bisect - end = bisect_left_cmp(idxs, low, cmp=lambda x, y: x > y) + 1 - start = bisect_right_cmp(idxs, up, cmp=lambda x, y: x > y) - start = max(start - 1, 0) - end = min(end + 1, len(idxs)) - indexes_between = idxs[start:end] - indexes_between_ranges.append(indexes_between) - else: - axis_reversed = transform._axis_reversed[cls.name] - if not axis_reversed: - lower_idx = bisect.bisect_left(idxs, low) - upper_idx = bisect.bisect_right(idxs, up) - indexes_between = idxs[lower_idx:upper_idx] - else: - # TODO: do the custom bisect - end_idx = bisect_left_cmp(idxs, low, cmp=lambda x, y: x > y) + 1 - start_idx = bisect_right_cmp(idxs, up, cmp=lambda x, y: x > y) - indexes_between = idxs[start_idx:end_idx] - indexes_between_ranges.append(indexes_between) - return indexes_between_ranges - cls.find_indexes = find_indexes cls.unmap_to_datacube = unmap_to_datacube - cls.find_indices_between = find_indices_between cls.unmap_path_key = unmap_path_key return cls diff --git a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py index 18bc8bd63..b2554e523 100644 --- a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py @@ -1,15 +1,14 @@ -import bisect - +from ....utility.list_tools import bisect_left_cmp, bisect_right_cmp from .datacube_reverse import DatacubeAxisReverse def reverse(cls): if cls.reorder: + old_find_indexes = cls.find_indexes def find_indexes(path, datacube): # first, find the relevant transformation object that is a mapping in the cls.transformation dico - subarray = datacube.dataarray.sel(path, method="nearest") - unordered_indices = datacube.datacube_natural_indexes(cls, subarray) + unordered_indices = old_find_indexes(path, datacube) if cls.name in datacube.complete_axes: ordered_indices = unordered_indices.sort_values() else: @@ -43,16 +42,16 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between_ranges.append(indexes_between) else: if method == "surrounding" or method == "nearest": - start = indexes.index(low) - end = indexes.index(up) - start = max(start - 1, 0) - end = min(end + 1, len(indexes)) + end_idx = bisect_left_cmp(indexes, low, cmp=lambda x, y: x > y) + 1 + start_idx = bisect_right_cmp(indexes, up, cmp=lambda x, y: x > y) + start = max(start_idx - 1, 0) + end = min(end_idx + 1, len(indexes)) indexes_between = indexes[start:end] indexes_between_ranges.append(indexes_between) else: - lower_idx = bisect.bisect_left(indexes, low) - upper_idx = bisect.bisect_right(indexes, up) - indexes_between = indexes[lower_idx:upper_idx] + end_idx = bisect_left_cmp(indexes, low, cmp=lambda x, y: x > y) + 1 + start_idx = bisect_right_cmp(indexes, up, cmp=lambda x, y: x > y) + indexes_between = indexes[start_idx:end_idx] indexes_between_ranges.append(indexes_between) return indexes_between_ranges diff --git a/tests/test_cyclic_axis_over_negative_vals.py b/tests/test_cyclic_axis_over_negative_vals.py index 52cd41c21..c11bcb098 100644 --- a/tests/test_cyclic_axis_over_negative_vals.py +++ b/tests/test_cyclic_axis_over_negative_vals.py @@ -2,7 +2,6 @@ import pandas as pd import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -25,7 +24,6 @@ def setup_method(self, method): "long": {"cyclic": [-1.1, -0.1]}, "level": {"cyclic": [1, 129]}, } - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_cyclic_axis_slicer_not_0.py b/tests/test_cyclic_axis_slicer_not_0.py index 8266c0a09..526473beb 100644 --- a/tests/test_cyclic_axis_slicer_not_0.py +++ b/tests/test_cyclic_axis_slicer_not_0.py @@ -2,7 +2,6 @@ import pandas as pd import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -25,7 +24,6 @@ def setup_method(self, method): "long": {"cyclic": [-1.1, -0.1]}, "level": {"cyclic": [1, 129]}, } - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_cyclic_axis_slicing.py b/tests/test_cyclic_axis_slicing.py index c0ac914f5..5a49be0b1 100644 --- a/tests/test_cyclic_axis_slicing.py +++ b/tests/test_cyclic_axis_slicing.py @@ -2,7 +2,6 @@ import pandas as pd import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -25,7 +24,6 @@ def setup_method(self, method): "long": {"cyclic": [0, 1.0]}, "level": {"cyclic": [1, 129]}, } - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_cyclic_nearest.py b/tests/test_cyclic_nearest.py index 9367ebfea..2dcf6aabd 100644 --- a/tests/test_cyclic_nearest.py +++ b/tests/test_cyclic_nearest.py @@ -23,6 +23,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, "longitude": {"cyclic": [0, 360]}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper", "type": "fc"} self.datacube_options = {"identical structure after": "number"} diff --git a/tests/test_cyclic_simple.py b/tests/test_cyclic_simple.py index f3bd1e28b..f900cac1b 100644 --- a/tests/test_cyclic_simple.py +++ b/tests/test_cyclic_simple.py @@ -2,7 +2,6 @@ import pandas as pd import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -22,7 +21,6 @@ def setup_method(self, method): }, ) options = {"long": {"cyclic": [0, 1.0]}, "level": {"cyclic": [1, 129]}} - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_cyclic_snapping.py b/tests/test_cyclic_snapping.py index adb3a7959..fa10fbbd3 100644 --- a/tests/test_cyclic_snapping.py +++ b/tests/test_cyclic_snapping.py @@ -1,6 +1,5 @@ import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Select @@ -17,7 +16,6 @@ def setup_method(self, method): }, ) options = {"long": {"cyclic": [0, 1.0]}} - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index f3739e5bf..d2ec327de 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -2,7 +2,6 @@ from earthkit import data from helper_functions import download_test_data -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.datacube.datacube_axis import FloatDatacubeAxis from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request @@ -17,9 +16,9 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/foo.grib") latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) latlon_array = latlon_array.t2m - self.xarraydatacube = XArrayDatacube(latlon_array) self.options = { "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "latitude": {"reverse": {True}}, } self.slicer = HullSlicer() self.API = Polytope(datacube=latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 692924f15..f52d41acf 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -15,6 +15,7 @@ def setup_method(self, method): "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "type": "fc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index bf4452a35..093750926 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -19,6 +19,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, "number": {"type_change": "int"}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_float_type.py b/tests/test_float_type.py index 7fb6f94ab..6593234b3 100644 --- a/tests/test_float_type.py +++ b/tests/test_float_type.py @@ -2,7 +2,6 @@ import pytest import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Select, Span @@ -20,7 +19,6 @@ def setup_method(self, method): "alt": np.arange(0.0, 20.0, 0.1), }, ) - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index ea6c4e6f9..8b6121da7 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -2,7 +2,6 @@ from earthkit import data from helper_functions import download_test_data, find_nearest_latlon -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -15,7 +14,6 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/healpix.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(time=0).isel(isobaricInhPa=0).z - self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { "values": {"mapper": {"type": "healpix", "resolution": 32, "axes": ["latitude", "longitude"]}}, "longitude": {"cyclic": [0, 360]}, diff --git a/tests/test_local_grid_cyclic.py b/tests/test_local_grid_cyclic.py index 025986321..da8b71f25 100644 --- a/tests/test_local_grid_cyclic.py +++ b/tests/test_local_grid_cyclic.py @@ -24,6 +24,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, "longitude": {"cyclic": [-180, 180]}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_local_regular_grid.py b/tests/test_local_regular_grid.py index dfac410e5..62be20962 100644 --- a/tests/test_local_regular_grid.py +++ b/tests/test_local_regular_grid.py @@ -23,6 +23,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, "number": {"type_change": "int"}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_merge_cyclic_octahedral.py b/tests/test_merge_cyclic_octahedral.py index 2714c36c6..e6cde2689 100644 --- a/tests/test_merge_cyclic_octahedral.py +++ b/tests/test_merge_cyclic_octahedral.py @@ -2,7 +2,6 @@ import pytest import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select, Span @@ -27,7 +26,6 @@ def setup_method(self, method): "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, "step": {"cyclic": [0, 2]}, } - self.xarraydatacube = XArrayDatacube(self.array) self.slicer = HullSlicer() self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index 7c3764904..82d95fb22 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -2,7 +2,6 @@ from earthkit import data from helper_functions import download_test_data -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -16,10 +15,10 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/foo.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) self.latlon_array = self.latlon_array.t2m - self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, "longitude": {"cyclic": [0, 360.0]}, + "latitude": {"reverse": {True}}, } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_merge_transformation.py b/tests/test_merge_transformation.py index 28f075d56..71d6f3129 100644 --- a/tests/test_merge_transformation.py +++ b/tests/test_merge_transformation.py @@ -2,7 +2,6 @@ import pandas as pd import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Select @@ -20,7 +19,6 @@ def setup_method(self, method): }, ) self.options = {"date": {"merge": {"with": "time", "linkers": ["T", "00"]}}} - self.xarraydatacube = XArrayDatacube(self.array) self.slicer = HullSlicer() self.API = Polytope(datacube=self.array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_multiple_param_fdb.py b/tests/test_multiple_param_fdb.py index 04b8e7127..dd1df749e 100644 --- a/tests/test_multiple_param_fdb.py +++ b/tests/test_multiple_param_fdb.py @@ -15,6 +15,7 @@ def setup_method(self, method): "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper", "type": "fc"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index adc251684..e041b0e53 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -2,7 +2,6 @@ from earthkit import data from helper_functions import download_test_data, find_nearest_latlon -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -16,9 +15,9 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/foo.grib") self.latlon_array = ds.to_xarray().isel(step=0).isel(number=0).isel(surface=0).isel(time=0) self.latlon_array = self.latlon_array.t2m - self.xarraydatacube = XArrayDatacube(self.latlon_array) self.options = { - "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}} + "values": {"mapper": {"type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"]}}, + "latitude": {"reverse": {True}}, } self.slicer = HullSlicer() self.API = Polytope(datacube=self.latlon_array, engine=self.slicer, axis_options=self.options) diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py index fa53b9809..6c5bc90d4 100644 --- a/tests/test_point_nearest.py +++ b/tests/test_point_nearest.py @@ -17,6 +17,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, "longitude": {"cyclic": [0, 360]}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) @@ -39,7 +40,6 @@ def test_fdb_datacube(self): Point(["latitude", "longitude"], [[0.16, 0.176]], method="nearest"), ) result = self.API.retrieve(request) - # result.pprint() assert len(result.leaves) == 1 @pytest.mark.fdb @@ -113,7 +113,7 @@ def test_fdb_datacube_true_point_4(self): Point(["latitude", "longitude"], [[0.035149384216, 359.97]], method="nearest"), ) result = self.API.retrieve(request) - result.pprint() + # result.pprint_2() assert len(result.leaves) == 1 assert result.leaves[0].value == 359.929906542056 assert result.leaves[0].axis.name == "longitude" diff --git a/tests/test_point_shape.py b/tests/test_point_shape.py index aa1caf79d..95ce48f99 100644 --- a/tests/test_point_shape.py +++ b/tests/test_point_shape.py @@ -2,7 +2,6 @@ import pandas as pd import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Point, Select @@ -20,7 +19,6 @@ def setup_method(self, method): "level": range(1, 130), }, ) - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index b955ee827..a28f005a8 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -22,6 +22,7 @@ def setup_method(self, method): "step": {"type_change": "int"}, "number": {"type_change": "int"}, "longitude": {"cyclic": [0, 360]}, + "latitude": {"reverse": {True}}, } self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "step": "0"} self.datacube_options = {"identical structure after": "number"} diff --git a/tests/test_reverse_transformation.py b/tests/test_reverse_transformation.py index e501dfdfc..9e4d6f8b5 100644 --- a/tests/test_reverse_transformation.py +++ b/tests/test_reverse_transformation.py @@ -1,7 +1,6 @@ import numpy as np import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Select @@ -18,7 +17,6 @@ def setup_method(self, method): }, ) options = {"lat": {"reverse": {True}}} - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_shapes.py b/tests/test_shapes.py index 092154bce..b0daa8b10 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -47,6 +47,7 @@ def test_all_mapper_cyclic(self): "step": {"type_change": "int"}, "number": {"type_change": "int"}, "longitude": {"cyclic": [0, 360]}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "step": "11"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index 319bc2c24..fae61798e 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -16,6 +16,7 @@ def setup_method(self, method): "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, "number": {"type_change": "int"}, + "latitude": {"reverse": {True}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_slice_date_range_fdb_v2.py b/tests/test_slice_date_range_fdb_v2.py index 63ce5e678..7fe005450 100644 --- a/tests/test_slice_date_range_fdb_v2.py +++ b/tests/test_slice_date_range_fdb_v2.py @@ -15,6 +15,7 @@ def setup_method(self, method): "values": {"mapper": {"type": "regular", "resolution": 30, "axes": ["latitude", "longitude"]}}, "date": {"merge": {"with": "time", "linkers": ["T", "00"]}}, "step": {"type_change": "int"}, + "latitude": {"reverse": {True}}, } self.config = {"class": "ea", "expver": "0001", "levtype": "pl", "stream": "enda"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 15d355cb1..f9651edc4 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -3,7 +3,6 @@ from earthkit import data from helper_functions import download_test_data -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -16,7 +15,6 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() options = {"lat": {"reverse": {True}}} self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) diff --git a/tests/test_slicing_unsliceable_axis.py b/tests/test_slicing_unsliceable_axis.py index 9d02abf76..a5fa1fcca 100644 --- a/tests/test_slicing_unsliceable_axis.py +++ b/tests/test_slicing_unsliceable_axis.py @@ -3,7 +3,6 @@ import pytest import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -18,7 +17,6 @@ def setup_method(self, method): dims=("date", "variable", "level"), coords={"date": pd.date_range("2000-01-01", "2000-01-03", 3), "variable": ["a"], "level": range(1, 130)}, ) - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) diff --git a/tests/test_slicing_xarray_4D.py b/tests/test_slicing_xarray_4D.py index 301bd874c..a19c260d4 100644 --- a/tests/test_slicing_xarray_4D.py +++ b/tests/test_slicing_xarray_4D.py @@ -3,7 +3,6 @@ import pytest import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.datacube.index_tree import IndexTree from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request @@ -34,7 +33,6 @@ def setup_method(self, method): "lat": np.around(np.arange(0.0, 10.0, 0.1), 15), }, ) - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) diff --git a/tests/test_snapping.py b/tests/test_snapping.py index 83d472203..41492f06f 100644 --- a/tests/test_snapping.py +++ b/tests/test_snapping.py @@ -1,7 +1,6 @@ import numpy as np import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Select @@ -18,7 +17,6 @@ def setup_method(self, method): "step": [1, 3, 5], }, ) - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer) diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index acad24fbc..1f113dfca 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -6,7 +6,6 @@ from earthkit import data from helper_functions import download_test_data -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Box, Select @@ -19,7 +18,6 @@ def setup_method(self, method): ds = data.from_source("file", "./tests/data/era5-levels-members.grib") array = ds.to_xarray().isel(step=0).t - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() options = { "latitude": {"reverse": {True}}, diff --git a/tests/test_type_change_transformation.py b/tests/test_type_change_transformation.py index d88a03aca..5291e4180 100644 --- a/tests/test_type_change_transformation.py +++ b/tests/test_type_change_transformation.py @@ -1,7 +1,6 @@ import numpy as np import xarray as xr -from polytope.datacube.backends.xarray import XArrayDatacube from polytope.engine.hullslicer import HullSlicer from polytope.polytope import Polytope, Request from polytope.shapes import Select @@ -19,7 +18,6 @@ def setup_method(self, method): ) self.array = array options = {"step": {"type_change": "int"}} - self.xarraydatacube = XArrayDatacube(array) self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, axis_options=options) From b439d662e7293d7b2b5d2de56b046dfe25047c90 Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 13 Feb 2024 15:12:50 +0100 Subject: [PATCH 13/26] black --- performance/fdb_performance_3D.py | 2 +- polytope/utility/geometry.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/performance/fdb_performance_3D.py b/performance/fdb_performance_3D.py index 3af0a2b03..88fb85c92 100644 --- a/performance/fdb_performance_3D.py +++ b/performance/fdb_performance_3D.py @@ -20,7 +20,7 @@ def setup_method(self, method): "date": {"transformation": {"merge": {"with": "time", "linkers": [" ", "00"]}}}, "step": {"transformation": {"type_change": "int"}}, "levelist": {"transformation": {"type_change": "int"}}, - "longitude": {"transformation": {"cyclic": [0, 360]}} + "longitude": {"transformation": {"cyclic": [0, 360]}}, } self.config = {"class": "od", "expver": "0001", "levtype": "sfc"} self.fdbdatacube = FDBDatacube(self.config, axis_options=self.options) diff --git a/polytope/utility/geometry.py b/polytope/utility/geometry.py index b4b7852fc..2906fed19 100644 --- a/polytope/utility/geometry.py +++ b/polytope/utility/geometry.py @@ -2,7 +2,7 @@ def lerp(a, b, value): - intersect = [b + (a-b) * value for a, b in zip(a, b)] + intersect = [b + (a - b) * value for a, b in zip(a, b)] return intersect From 76b8b3432c9a8133aef0b67ad2a313b5df3a8546 Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 14 Feb 2024 11:15:35 +0100 Subject: [PATCH 14/26] simplify transformations --- polytope/datacube/backends/datacube.py | 2 ++ polytope/datacube/backends/xarray.py | 25 ++++++++++++++-- polytope/datacube/datacube_axis.py | 2 -- .../datacube_mappers/mapper_axis_decorator.py | 30 ------------------- .../datacube_merger/merger_axis_decorator.py | 20 ------------- .../reverse_axis_decorator.py | 6 +--- .../type_change_axis_decorator.py | 17 ----------- tests/test_datacube_axes_init.py | 8 ++--- 8 files changed, 29 insertions(+), 81 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index ef3519b4b..fc1ce4b16 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -101,6 +101,7 @@ def get_indices(self, path: DatacubePath, axis, lower, upper, method=None): """ path = self.fit_path(path) indexes = axis.find_indexes(path, self) + # TODO: this could also be handled by axis/transformations? search_ranges = axis.remap([lower, upper]) original_search_ranges = axis.to_intervals([lower, upper]) # Find the offsets for each interval in the requested range, which we will need later @@ -120,6 +121,7 @@ def get_indices(self, path: DatacubePath, axis, lower, upper, method=None): def _look_up_datacube(self, search_ranges, search_ranges_offset, indexes, axis, method): idx_between = [] + # TODO: maybe this can all go inside find_indices_between for the different cyclic and other transformations for i in range(len(search_ranges)): r = search_ranges[i] offset = search_ranges_offset[i] diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 54f963c63..16f68f008 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -1,5 +1,6 @@ from copy import deepcopy +import numpy as np import xarray as xr from .datacube import Datacube, IndexTree @@ -60,9 +61,27 @@ def get(self, requests: IndexTree): path_copy = deepcopy(path) for key in path_copy: axis = self._axes[key] - (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path) - # TODO: here do nearest point search - path = self.fit_path(path) + key_value_path = {key: path_copy[key]} + # (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path) + (key_value_path, path, unmapped_path) = axis.unmap_path_key( + key_value_path, path, unmapped_path + ) + path.update(key_value_path) + path.update(unmapped_path) + + unmapped_path = {} + for key in path.keys(): + if key not in self.dataarray.dims: + path.pop(key) + if key not in self.dataarray.coords.dtypes: + unmapped_path.update({key: path[key]}) + path.pop(key) + for key in self.dataarray.coords.dtypes: + key_dtype = self.dataarray.coords.dtypes[key] + if key_dtype.type is np.str_ and key in path.keys(): + unmapped_path.update({key: path[key]}) + path.pop(key) + subxarray = self.dataarray.sel(path, method="nearest") subxarray = subxarray.sel(unmapped_path) value = subxarray.item() diff --git a/polytope/datacube/datacube_axis.py b/polytope/datacube/datacube_axis.py index 2879dfb41..ebe4a7ef1 100644 --- a/polytope/datacube/datacube_axis.py +++ b/polytope/datacube/datacube_axis.py @@ -91,8 +91,6 @@ def find_indices_between(self, index_ranges, low, up, datacube, method=None): indexes_between_ranges.append(indexes_between) else: if method == "surrounding" or method == "nearest": - # start = indexes.index(low) - # end = indexes.index(up) start = bisect.bisect_left(indexes, low) end = bisect.bisect_right(indexes, up) start = max(start - 1, 0) diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py index c2d0703b9..aa4910c99 100644 --- a/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_mappers/mapper_axis_decorator.py @@ -15,35 +15,6 @@ def find_indexes(path, datacube): first_val = path[transformation._mapped_axes()[0]] return transformation.second_axis_vals(first_val) - old_unmap_to_datacube = cls.unmap_to_datacube - - def unmap_to_datacube(path, unmapped_path): - (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) - for transform in cls.transformations: - if isinstance(transform, DatacubeMapper): - if cls.name == transform._mapped_axes()[0]: - # if we are on the first axis, then need to add the first val to unmapped_path - first_val = path.get(cls.name, None) - path.pop(cls.name, None) - if cls.name not in unmapped_path: - # if for some reason, the unmapped_path already has the first axis val, then don't update - unmapped_path[cls.name] = first_val - if cls.name == transform._mapped_axes()[1]: - # if we are on the second axis, then the val of the first axis is stored - # inside unmapped_path so can get it from there - second_val = path.get(cls.name, None) - path.pop(cls.name, None) - first_val = unmapped_path.get(transform._mapped_axes()[0], None) - unmapped_path.pop(transform._mapped_axes()[0], None) - # if the first_val was not in the unmapped_path, then it's still in path - if first_val is None: - first_val = path.get(transform._mapped_axes()[0], None) - path.pop(transform._mapped_axes()[0], None) - if first_val is not None and second_val is not None: - unmapped_idx = transform.unmap(first_val, second_val) - unmapped_path[transform.old_axis] = unmapped_idx - return (path, unmapped_path) - old_unmap_path_key = cls.unmap_path_key def unmap_path_key(key_value_path, leaf_path, unwanted_path): @@ -63,7 +34,6 @@ def unmap_path_key(key_value_path, leaf_path, unwanted_path): return (key_value_path, leaf_path, unwanted_path) cls.find_indexes = find_indexes - cls.unmap_to_datacube = unmap_to_datacube cls.unmap_path_key = unmap_path_key return cls diff --git a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py index ed09bef2c..907e58ab7 100644 --- a/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_merger/merger_axis_decorator.py @@ -26,27 +26,7 @@ def unmap_path_key(key_value_path, leaf_path, unwanted_path): new_key_value_path[transform._second_axis] = second_val return (new_key_value_path, leaf_path, unwanted_path) - old_unmap_to_datacube = cls.unmap_to_datacube - - def unmap_to_datacube(path, unmapped_path): - (path, unmapped_path) = old_unmap_to_datacube(path, unmapped_path) - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisMerger): - transformation = transform - if cls.name == transformation._first_axis: - old_val = path.get(cls.name, None) - (first_val, second_val) = transformation.unmerge(old_val) - path.pop(cls.name, None) - path[transformation._first_axis] = first_val - path[transformation._second_axis] = second_val - return (path, unmapped_path) - - def remap(range): - return [range] - - cls.remap = remap cls.find_indexes = find_indexes - cls.unmap_to_datacube = unmap_to_datacube cls.unmap_path_key = unmap_path_key return cls diff --git a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py index b2554e523..44ec0e0ca 100644 --- a/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_reverse/reverse_axis_decorator.py @@ -16,7 +16,6 @@ def find_indexes(path, datacube): return ordered_indices def find_indices_between(index_ranges, low, up, datacube, method=None): - # TODO: add method for snappping indexes_between_ranges = [] for transform in cls.transformations: if isinstance(transform, DatacubeAxisReverse): @@ -55,10 +54,7 @@ def find_indices_between(index_ranges, low, up, datacube, method=None): indexes_between_ranges.append(indexes_between) return indexes_between_ranges - def remap(range): - return [range] - - cls.remap = remap + # TODO: maybe don't need find_indexes? cls.find_indexes = find_indexes cls.find_indices_between = find_indices_between diff --git a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py index 1e1c21f25..77b3de830 100644 --- a/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py +++ b/polytope/datacube/transformations/datacube_type_change/type_change_axis_decorator.py @@ -25,24 +25,7 @@ def unmap_path_key(key_value_path, leaf_path, unwanted_path): key_value_path[cls.name] = unchanged_val return (key_value_path, leaf_path, unwanted_path) - def unmap_to_datacube(path, unmapped_path): - for transform in cls.transformations: - if isinstance(transform, DatacubeAxisTypeChange): - transformation = transform - if cls.name == transformation.name: - changed_val = path.get(cls.name, None) - unchanged_val = transformation.make_str(changed_val) - if cls.name in path: - path.pop(cls.name, None) - unmapped_path[cls.name] = unchanged_val - return (path, unmapped_path) - - def remap(range): - return [range] - - cls.remap = remap cls.find_indexes = find_indexes - cls.unmap_to_datacube = unmap_to_datacube cls.unmap_path_key = unmap_path_key return cls diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index d2ec327de..311fb5a0b 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -68,13 +68,13 @@ def test_created_axes(self): assert len(self.datacube._axes["longitude"].find_indexes({"latitude": 89.94618771566562}, self.datacube)) == 20 lon_ax = self.datacube._axes["longitude"] lat_ax = self.datacube._axes["latitude"] - (path, unmapped_path) = lat_ax.unmap_to_datacube({"latitude": 89.94618771566562}, {}) + (path_key, path, unmapped_path) = lat_ax.unmap_path_key({"latitude": 89.94618771566562}, {}, {}) assert path == {} assert unmapped_path == {"latitude": 89.94618771566562} - assert unmapped_path == {"latitude": 89.94618771566562} - (path, unmapped_path) = lon_ax.unmap_to_datacube({"longitude": 0.0}, {"latitude": 89.94618771566562}) + (path_key, path, unmapped_path) = lon_ax.unmap_path_key({"longitude": 0.0}, {}, {"latitude": 89.94618771566562}) assert path == {} - assert unmapped_path == {"values": 0} + assert unmapped_path == {'latitude': 89.94618771566562} + assert path_key == {"values": 0} assert lat_ax.find_indices_between([[89.94618771566562, 89.87647835333229]], 89.87, 90, self.datacube, 0) == [ [89.94618771566562, 89.87647835333229] ] From cfd098392c60632ebb002274a9dcb9666093bace Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 14 Feb 2024 11:18:28 +0100 Subject: [PATCH 15/26] clean up --- polytope/datacube/backends/datacube.py | 3 --- polytope/datacube/backends/fdb.py | 1 - polytope/datacube/backends/xarray.py | 5 +---- tests/test_datacube_axes_init.py | 2 +- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index fc1ce4b16..3d29a64fd 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -16,9 +16,6 @@ class Datacube(ABC): - # def __init__(self): - # self.is_xarray = False - @abstractmethod def get(self, requests: IndexTree) -> Any: """Return data given a set of request trees""" diff --git a/polytope/datacube/backends/fdb.py b/polytope/datacube/backends/fdb.py index 8fa3009e7..eda6f24e9 100644 --- a/polytope/datacube/backends/fdb.py +++ b/polytope/datacube/backends/fdb.py @@ -29,7 +29,6 @@ def __init__(self, config=None, axis_options=None, datacube_options=None): self.nearest_search = {} self.coupled_axes = [] self.axis_with_identical_structure_after = datacube_options.get("identical structure after") - # self.is_xarray = False self.transformed_axes = [] self.dataarray = [] diff --git a/polytope/datacube/backends/xarray.py b/polytope/datacube/backends/xarray.py index 16f68f008..ab73fe603 100644 --- a/polytope/datacube/backends/xarray.py +++ b/polytope/datacube/backends/xarray.py @@ -26,7 +26,6 @@ def __init__(self, dataarray: xr.DataArray, axis_options=None, datacube_options= self.coupled_axes = [] self.axis_with_identical_structure_after = datacube_options.get("identical structure after") self.transformed_axes = [] - # self.is_xarray = True for name, values in dataarray.coords.variables.items(): if name in dataarray.dims: @@ -63,9 +62,7 @@ def get(self, requests: IndexTree): axis = self._axes[key] key_value_path = {key: path_copy[key]} # (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path) - (key_value_path, path, unmapped_path) = axis.unmap_path_key( - key_value_path, path, unmapped_path - ) + (key_value_path, path, unmapped_path) = axis.unmap_path_key(key_value_path, path, unmapped_path) path.update(key_value_path) path.update(unmapped_path) diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index 311fb5a0b..5612ac305 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -73,7 +73,7 @@ def test_created_axes(self): assert unmapped_path == {"latitude": 89.94618771566562} (path_key, path, unmapped_path) = lon_ax.unmap_path_key({"longitude": 0.0}, {}, {"latitude": 89.94618771566562}) assert path == {} - assert unmapped_path == {'latitude': 89.94618771566562} + assert unmapped_path == {"latitude": 89.94618771566562} assert path_key == {"values": 0} assert lat_ax.find_indices_between([[89.94618771566562, 89.87647835333229]], 89.87, 90, self.datacube, 0) == [ [89.94618771566562, 89.87647835333229] From c78c3d5854bda2ff6fbd3476c40b57715fef250c Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 14 Feb 2024 16:49:31 +0100 Subject: [PATCH 16/26] clean up --- polytope/datacube/backends/datacube.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/polytope/datacube/backends/datacube.py b/polytope/datacube/backends/datacube.py index 3d29a64fd..2fb585452 100644 --- a/polytope/datacube/backends/datacube.py +++ b/polytope/datacube/backends/datacube.py @@ -46,9 +46,7 @@ def _create_axes(self, name, values, transformation_type_key, transformation_opt # first need to change the values so that we have right type values = transformation.change_val_type(axis_name, values) - if self._axes is None: - DatacubeAxis.create_standard(axis_name, values, self) - elif axis_name not in self._axes.keys(): + if self._axes is None or axis_name not in self._axes.keys(): DatacubeAxis.create_standard(axis_name, values, self) # add transformation tag to axis, as well as transformation options for later setattr(self._axes[axis_name], has_transform[transformation_type_key], True) # where has_transform is a @@ -72,9 +70,7 @@ def _check_and_add_axes(self, options, name, values): self._add_all_transformation_axes(options, name, values) else: if name not in self.blocked_axes: - if self._axes is None: - DatacubeAxis.create_standard(name, values, self) - elif name not in self._axes.keys(): + if self._axes is None or name not in self._axes.keys(): DatacubeAxis.create_standard(name, values, self) def has_index(self, path: DatacubePath, axis, index): From 811629252c5115ec530acd61b08c425c07097c70 Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 28 Feb 2024 15:08:13 +0100 Subject: [PATCH 17/26] update readme --- readme.md | 28 +++++++++++++++------------- requirements_example.txt | 0 2 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 requirements_example.txt diff --git a/readme.md b/readme.md index 48dba68f6..d66ec8507 100644 --- a/readme.md +++ b/readme.md @@ -22,21 +22,23 @@ Documentation

-Polytope is a library for extracting complex data from datacubes. It provides an API for non-orthogonal access to data, where the stencil used to extract data from the datacube can be any arbitrary n-dimensional polygon (called a *polytope*). This can be used to efficiently extract complex features from a datacube, such as polygon regions or spatio-temporal paths. +Polytope is a library for extracting complex data from datacubes. It provides an API for non-orthogonal access to data, where the stencil used to extract data from the datacube can be any arbitrary *n*-dimensional polygon (called a *polytope*). This can be used to efficiently extract complex features from a datacube, such as polygon regions or spatio-temporal paths. -Polytope is designed to extend different datacube backends. -* Xarray dataarrays -* FDB object stores (coming soon) +Polytope is designed to extend different datacube backends: +* XArray dataarrays +* FDB object stores (through the GribJump software) -Polytope supports datacubes which have branching, non-uniform indexing, and even cyclic axes. If the datacube backend supports byte-addressability and efficient random access (either in-memory or direct from storage), *polytope* can be used to dramatically decrease overall I/O load. +Polytope supports datacubes which have branching, non-uniform indexing, and even cyclic axes. If the datacube backend supports byte-addressability and efficient random access (either in-memory or direct from storage), **polytope** can be used to dramatically decrease overall I/O load. +> [!WARNING] +> This project is BETA and will be experimental for the forseable future. Interfaces and functionality are likely to change, and the project itself may be scrapped. DO NOT use this software in any project/software that is operational. -| :warning: This project is BETA and will be experimental for the foreseeable future. Interfaces and functionality are likely to change. DO NOT use this software in any project/software that is operational. | -|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| + ## Concept -Polytope is designed to enable extraction of arbitrary extraction of data from a datacube. Instead of the typical range-based bounding-box approach, Polytope can extract any shape of data from a datacube using a "polytope" (n-dimensional polygon) stencil. +Polytope is designed to enable extraction of arbitrary extraction of data from a datacube. Instead of the typical range-based bounding-box approach, Polytope can extract any shape of data from a datacube using a "polytope" (*n*-dimensional polygon) stencil.

@@ -93,11 +95,11 @@ Here is a step-by-step example of how to use this software. We then construct the Polytope object, passing in some additional metadata describing properties of the longitude axis. ```Python - options = {"longitude": {"Cyclic": [0, 360.0]}} + options = {"longitude": {"cyclic": [0, 360.0]}} from polytope.polytope import Polytope - p = Polytope(datacube=array, options=options) + p = Polytope(datacube=array, axis_options=options) ``` 2. Next, we create a request shape to extract from the datacube. @@ -146,18 +148,18 @@ TODO: populate requirements.txt --> ## Testing -#### Git Large File Storage + #### Additional Dependencies The Polytope tests and examples require additional Python packages compared to the main Polytope algorithm. -The additional dependencies are provided in the requirements_test.txt and requirements_examples.txt files, which can respectively be found in the examples and tests folders. +The additional dependencies are provided in the requirements_test.txt and requirements_examples.txt files, which can respectively be found in the tests and examples folders. Moreover, Polytope's tests and examples also require the installation of eccodes and GDAL. It is possible to install both of these dependencies using either a package manager or manually. diff --git a/requirements_example.txt b/requirements_example.txt new file mode 100644 index 000000000..e69de29bb From 0100a7710178d2905ecf960d1ab78671fc4d1a42 Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 28 Feb 2024 15:12:11 +0100 Subject: [PATCH 18/26] small fix --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index d66ec8507..ff357c990 100644 --- a/readme.md +++ b/readme.md @@ -28,7 +28,7 @@ Polytope is designed to extend different datacube backends: * XArray dataarrays * FDB object stores (through the GribJump software) -Polytope supports datacubes which have branching, non-uniform indexing, and even cyclic axes. If the datacube backend supports byte-addressability and efficient random access (either in-memory or direct from storage), **polytope** can be used to dramatically decrease overall I/O load. +Polytope supports datacubes which have branching, non-uniform indexing, and even cyclic axes. If the datacube backend supports byte-addressability and efficient random access (either in-memory or direct from storage), **Polytope** can be used to dramatically decrease overall I/O load. > [!WARNING] > This project is BETA and will be experimental for the forseable future. Interfaces and functionality are likely to change, and the project itself may be scrapped. DO NOT use this software in any project/software that is operational. From 85437c3c60798e715b7cefcbc1c7108141aaedfa Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 28 Feb 2024 15:13:08 +0100 Subject: [PATCH 19/26] remove unused comments --- readme.md | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/readme.md b/readme.md index ff357c990..2f3ec4390 100644 --- a/readme.md +++ b/readme.md @@ -33,39 +33,30 @@ Polytope supports datacubes which have branching, non-uniform indexing, and even > [!WARNING] > This project is BETA and will be experimental for the forseable future. Interfaces and functionality are likely to change, and the project itself may be scrapped. DO NOT use this software in any project/software that is operational. - - ## Concept Polytope is designed to enable extraction of arbitrary extraction of data from a datacube. Instead of the typical range-based bounding-box approach, Polytope can extract any shape of data from a datacube using a "polytope" (*n*-dimensional polygon) stencil. -

Polytope Concept

- The Polytope algorithm can for example be used to extract: - 2D cut-outs, such as country cut-outs, from a datacube -

Greece cut-out

- - timeseries from a datacube

Timeseries

- - more complicated spatio-temporal paths, such as flight paths, from a datacube

Flight path

- - and many more high-dimensional shapes in arbitrary dimensions... @@ -141,21 +132,8 @@ Here is a step-by-step example of how to use this software. ↳longitude=1.0 ``` - - ## Testing - - #### Additional Dependencies The Polytope tests and examples require additional Python packages compared to the main Polytope algorithm. From df734b379c9e926fbaba68ef38cf3461b3c5b96c Mon Sep 17 00:00:00 2001 From: mathleur Date: Fri, 1 Mar 2024 14:12:17 +0100 Subject: [PATCH 20/26] remove numpy version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a6c7f3b39..b2bd6f0fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy==1.23.5 +numpy pandas==1.5.2 scipy==1.9.3 sortedcontainers==2.4.0 From 23ce604fec2be3e6a7b45ea3ed966617d7348138 Mon Sep 17 00:00:00 2001 From: mathleur Date: Fri, 1 Mar 2024 15:10:19 +0100 Subject: [PATCH 21/26] add __init__.py to subfolders --- .../datacube/transformations/datacube_cyclic/__init__.py | 2 ++ .../datacube/transformations/datacube_mappers/__init__.py | 2 ++ .../datacube_mappers/mapper_types/__init__.py | 5 +++++ .../datacube/transformations/datacube_merger/__init__.py | 2 ++ .../transformations/datacube_null_transformation/__init__.py | 2 ++ .../datacube/transformations/datacube_reverse/__init__.py | 2 ++ .../transformations/datacube_type_change/__init__.py | 2 ++ 7 files changed, 17 insertions(+) create mode 100644 polytope/datacube/transformations/datacube_cyclic/__init__.py create mode 100644 polytope/datacube/transformations/datacube_mappers/__init__.py create mode 100644 polytope/datacube/transformations/datacube_mappers/mapper_types/__init__.py create mode 100644 polytope/datacube/transformations/datacube_merger/__init__.py create mode 100644 polytope/datacube/transformations/datacube_null_transformation/__init__.py create mode 100644 polytope/datacube/transformations/datacube_reverse/__init__.py create mode 100644 polytope/datacube/transformations/datacube_type_change/__init__.py diff --git a/polytope/datacube/transformations/datacube_cyclic/__init__.py b/polytope/datacube/transformations/datacube_cyclic/__init__.py new file mode 100644 index 000000000..adfaf9d8f --- /dev/null +++ b/polytope/datacube/transformations/datacube_cyclic/__init__.py @@ -0,0 +1,2 @@ +from .cyclic_axis_decorator import * +from .datacube_cyclic import * diff --git a/polytope/datacube/transformations/datacube_mappers/__init__.py b/polytope/datacube/transformations/datacube_mappers/__init__.py new file mode 100644 index 000000000..ad69c9c5a --- /dev/null +++ b/polytope/datacube/transformations/datacube_mappers/__init__.py @@ -0,0 +1,2 @@ +from .datacube_mappers import * +from .mapper_axis_decorator import * diff --git a/polytope/datacube/transformations/datacube_mappers/mapper_types/__init__.py b/polytope/datacube/transformations/datacube_mappers/mapper_types/__init__.py new file mode 100644 index 000000000..ba9a7b339 --- /dev/null +++ b/polytope/datacube/transformations/datacube_mappers/mapper_types/__init__.py @@ -0,0 +1,5 @@ +from .healpix import * +from .local_regular import * +from .octahedral import * +from .reduced_ll import * +from .regular import * diff --git a/polytope/datacube/transformations/datacube_merger/__init__.py b/polytope/datacube/transformations/datacube_merger/__init__.py new file mode 100644 index 000000000..71a59acb1 --- /dev/null +++ b/polytope/datacube/transformations/datacube_merger/__init__.py @@ -0,0 +1,2 @@ +from .datacube_merger import * +from .merger_axis_decorator import * diff --git a/polytope/datacube/transformations/datacube_null_transformation/__init__.py b/polytope/datacube/transformations/datacube_null_transformation/__init__.py new file mode 100644 index 000000000..13395c4ad --- /dev/null +++ b/polytope/datacube/transformations/datacube_null_transformation/__init__.py @@ -0,0 +1,2 @@ +from .datacube_null_transformation import * +from .null_axis_decorator import * diff --git a/polytope/datacube/transformations/datacube_reverse/__init__.py b/polytope/datacube/transformations/datacube_reverse/__init__.py new file mode 100644 index 000000000..73cb9d862 --- /dev/null +++ b/polytope/datacube/transformations/datacube_reverse/__init__.py @@ -0,0 +1,2 @@ +from .datacube_reverse import * +from .reverse_axis_decorator import * diff --git a/polytope/datacube/transformations/datacube_type_change/__init__.py b/polytope/datacube/transformations/datacube_type_change/__init__.py new file mode 100644 index 000000000..89dab3aef --- /dev/null +++ b/polytope/datacube/transformations/datacube_type_change/__init__.py @@ -0,0 +1,2 @@ +from .datacube_type_change import * +from .type_change_axis_decorator import * From bd0eb3d6e578a1e7a090210b85500adaa99ca474 Mon Sep 17 00:00:00 2001 From: mathleur Date: Fri, 1 Mar 2024 14:15:33 +0100 Subject: [PATCH 22/26] remove requirement versions --- requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index b2bd6f0fe..afbacd4ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ numpy -pandas==1.5.2 -scipy==1.9.3 -sortedcontainers==2.4.0 -tripy==1.0.0 -xarray==2022.12.0 +pandas +scipy +sortedcontainers +tripy +xarray From b117f83d90ff5328d6393b3bdeb97bf1b43911a6 Mon Sep 17 00:00:00 2001 From: mathleur Date: Fri, 1 Mar 2024 14:21:34 +0100 Subject: [PATCH 23/26] remove test requirement versions --- tests/requirements_test.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt index e09e60e35..0df9637b0 100644 --- a/tests/requirements_test.txt +++ b/tests/requirements_test.txt @@ -1,8 +1,8 @@ -r ../requirements.txt -pytest==7.2.0 -cffi==1.15.1 -eccodes==1.5.2 -h5netcdf==1.1.0 -h5py==3.8.0 -earthkit==0.0.1 -earthkit-data==0.1.3 \ No newline at end of file +pytest +cffi +eccodes +h5netcdf +h5py +earthkit +earthkit-data \ No newline at end of file From 2b8dc06b9a9a183137d93dc1d39fbf52b0c09b60 Mon Sep 17 00:00:00 2001 From: mathleur Date: Fri, 1 Mar 2024 14:23:33 +0100 Subject: [PATCH 24/26] remove example requirement versions --- examples/requirements_examples.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/requirements_examples.txt b/examples/requirements_examples.txt index cb2087e6e..5c46d4d64 100644 --- a/examples/requirements_examples.txt +++ b/examples/requirements_examples.txt @@ -1,13 +1,13 @@ -r ../requirements.txt -r ../tests/requirements_test.txt -matplotlib==3.6.2 -matplotlib-inline==0.1.6 -Pillow==9.3.0 -Shapely==1.8.5.post1 -shp==1.0.2 -Fiona==1.8.22 -geopandas==0.12.2 -plotly==5.11.0 -pyshp==2.3.1 -cfgrib==0.9.10.3 \ No newline at end of file +matplotlib +matplotlib-inline +Pillow +Shapely +shp +Fiona +geopandas +plotly +pyshp +cfgrib \ No newline at end of file From a0c56db2d3c27fc12d43f0ff1a7aadcf0813c995 Mon Sep 17 00:00:00 2001 From: James Hawkes Date: Fri, 1 Mar 2024 15:17:55 +0000 Subject: [PATCH 25/26] Version 1.0.3 --- polytope/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope/version.py b/polytope/version.py index 7863915fa..976498ab9 100644 --- a/polytope/version.py +++ b/polytope/version.py @@ -1 +1 @@ -__version__ = "1.0.2" +__version__ = "1.0.3" From 0294f09b34332641b4e8a2953b024cf79a0e8609 Mon Sep 17 00:00:00 2001 From: mathleur Date: Fri, 16 Feb 2024 09:51:37 +0100 Subject: [PATCH 26/26] fix pypi step in CI --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 56b8e7ee0..fb1be91ea 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -155,8 +155,8 @@ jobs: pip install setuptools wheel twine - name: Build and publish env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + TWINE_USERNAME: "__token__" + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: | python setup.py sdist twine upload dist/* \ No newline at end of file