From 1625b61004da3a55f33dd5c3138a355bb64f3730 Mon Sep 17 00:00:00 2001 From: Darius Couchard Date: Wed, 12 Jun 2024 15:50:27 +0200 Subject: [PATCH 1/5] Added METEO data, dependant on GFMAP --- src/worldcereal/openeo/preprocessing.py | 36 ++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/worldcereal/openeo/preprocessing.py b/src/worldcereal/openeo/preprocessing.py index b5ca0987..65bebbe3 100644 --- a/src/worldcereal/openeo/preprocessing.py +++ b/src/worldcereal/openeo/preprocessing.py @@ -11,9 +11,10 @@ TemporalContext, ) from openeo_gfmap.fetching.generic import build_generic_extractor +from openeo_gfmap.fetching.meteo import build_meteo_extractor from openeo_gfmap.fetching.s1 import build_sentinel1_grd_extractor from openeo_gfmap.fetching.s2 import build_sentinel2_l2a_extractor -from openeo_gfmap.preprocessing.compositing import mean_compositing, median_compositing +from openeo_gfmap.preprocessing.compositing import mean_compositing, median_compositing, sum_compositing from openeo_gfmap.preprocessing.sar import compress_backscatter_uint16 from openeo_gfmap.utils.catalogue import UncoveredS1Exception, select_S1_orbitstate @@ -229,11 +230,10 @@ def raw_datacube_METEO( temporal_extent: TemporalContext, fetch_type: FetchType, ) -> DataCube: - extractor = build_generic_extractor( + extractor = build_meteo_extractor( backend_context=backend_context, bands=["AGERA5-TMEAN", "AGERA5-PRECIP"], - fetch_type=fetch_type, - collection_name="AGERA5", + fetch_type=fetch_type ) return extractor.get_cube(connection, spatial_extent, temporal_extent) @@ -302,24 +302,24 @@ def worldcereal_preprocessed_inputs_gfmap( dem_data = dem_data.linear_scale_range(0, 65534, 0, 65534) - # meteo_data = raw_datacube_METEO( - # connection=connection, - # backend_context=backend_context, - # spatial_extent=spatial_extent, - # temporal_extent=temporal_extent, - # fetch_type=FetchType.TILE, - # ) + meteo_data = raw_datacube_METEO( + connection=connection, + backend_context=backend_context, + spatial_extent=spatial_extent, + temporal_extent=temporal_extent, + fetch_type=FetchType.TILE, + ) - # # Perform compositing differently depending on the bands - # mean_temperature = meteo_data.band("AGERA5-TMEAN") - # mean_temperature = mean_compositing(mean_temperature, period="month") + # Perform compositing differently depending on the bands + mean_temperature = meteo_data.band("AGERA5-TMEAN") + mean_temperature = mean_compositing(mean_temperature, period="month") - # total_precipitation = meteo_data.band("AGERA5-PRECIP") - # total_precipitation = sum_compositing(total_precipitation, period="month") + total_precipitation = meteo_data.band("AGERA5-PRECIP") + total_precipitation = sum_compositing(total_precipitation, period="month") data = s2_data.merge_cubes(s1_data) data = data.merge_cubes(dem_data) - # data = data.merge_cubes(mean_temperature) - # data = data.merge_cubes(total_precipitation) + data = data.merge_cubes(mean_temperature) + data = data.merge_cubes(total_precipitation) return data From f407e47ee9594faf7c6d948925a0b341540fc04b Mon Sep 17 00:00:00 2001 From: Darius Couchard Date: Tue, 18 Jun 2024 10:44:14 +0200 Subject: [PATCH 2/5] Added meteo data from preprocessed catalogue in inference --- src/worldcereal/openeo/feature_extractor.py | 4 +- src/worldcereal/openeo/preprocessing.py | 48 +++++++++++++++------ 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/src/worldcereal/openeo/feature_extractor.py b/src/worldcereal/openeo/feature_extractor.py index 19037055..1e884f26 100644 --- a/src/worldcereal/openeo/feature_extractor.py +++ b/src/worldcereal/openeo/feature_extractor.py @@ -103,7 +103,9 @@ def execute(self, inarr: xr.DataArray) -> xr.DataArray: ) self.logger.info("Extracting presto features") - features = get_presto_features(inarr, presto_model_url, self.epsg) + features = get_presto_features( + inarr, presto_model_url, self.epsg, batch_size=8192 + ) return features def _execute(self, cube: XarrayDataCube, parameters: dict) -> XarrayDataCube: diff --git a/src/worldcereal/openeo/preprocessing.py b/src/worldcereal/openeo/preprocessing.py index 65bebbe3..101a7832 100644 --- a/src/worldcereal/openeo/preprocessing.py +++ b/src/worldcereal/openeo/preprocessing.py @@ -10,6 +10,7 @@ SpatialContext, TemporalContext, ) +from openeo_gfmap.fetching.meteo import build_meteo_extractor from openeo_gfmap.fetching.generic import build_generic_extractor from openeo_gfmap.fetching.meteo import build_meteo_extractor from openeo_gfmap.fetching.s1 import build_sentinel1_grd_extractor @@ -238,6 +239,39 @@ def raw_datacube_METEO( return extractor.get_cube(connection, spatial_extent, temporal_extent) +def precomposited_datacube_METEO( + connection: Connection, + spatial_extent: SpatialContext, + temporal_extent: TemporalContext, +) -> DataCube: + """Extract the precipitation and temperature AGERA5 data from a + pre-composited and pre-processed collection. The data is stored in the + CloudFerro S3 stoage, allowing faster access and processing from the CDSE + backend. + + Limitations: + - Only monthly composited data is available. + - Only two bands are available: precipitation-flux and temperature-mean. + - This function do not support fetching points or polygons, but only + tiles. + """ + temporal_extent = [temporal_extent.start_date, temporal_extent.end_date] + spatial_extent = dict(spatial_extent) + + # Monthly composited METEO data + cube = connection.load_stac( + "https://s3.waw3-1.cloudferro.com/swift/v1/agera/stac/collection.json", + spatial_extent=spatial_extent, + temporal_extent=temporal_extent, + bands=["precipitation-flux", "temperature-mean"], + ) + cube = cube.rename_labels(dimension="bands", target=[ + "AGERA5-PRECIP", "AGERA5-TMEAN" + ]) + + return cube + + def worldcereal_preprocessed_inputs_gfmap( connection: Connection, backend_context: BackendContext, @@ -302,24 +336,14 @@ def worldcereal_preprocessed_inputs_gfmap( dem_data = dem_data.linear_scale_range(0, 65534, 0, 65534) - meteo_data = raw_datacube_METEO( + meteo_data = precomposited_datacube_METEO( connection=connection, - backend_context=backend_context, spatial_extent=spatial_extent, temporal_extent=temporal_extent, - fetch_type=FetchType.TILE, ) - # Perform compositing differently depending on the bands - mean_temperature = meteo_data.band("AGERA5-TMEAN") - mean_temperature = mean_compositing(mean_temperature, period="month") - - total_precipitation = meteo_data.band("AGERA5-PRECIP") - total_precipitation = sum_compositing(total_precipitation, period="month") - data = s2_data.merge_cubes(s1_data) data = data.merge_cubes(dem_data) - data = data.merge_cubes(mean_temperature) - data = data.merge_cubes(total_precipitation) + data = data.merge_cubes(meteo_data) return data From 2ae43b3cfa23b4f2ddbf3752f73ab66b07d12d9b Mon Sep 17 00:00:00 2001 From: Darius Couchard Date: Tue, 18 Jun 2024 10:47:31 +0200 Subject: [PATCH 3/5] Fix feature extractor --- src/worldcereal/openeo/feature_extractor.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/worldcereal/openeo/feature_extractor.py b/src/worldcereal/openeo/feature_extractor.py index b39ff8a0..40dde71a 100644 --- a/src/worldcereal/openeo/feature_extractor.py +++ b/src/worldcereal/openeo/feature_extractor.py @@ -104,11 +104,7 @@ def execute(self, inarr: xr.DataArray) -> xr.DataArray: self.logger.info("Extracting presto features") features = get_presto_features( -<<<<<<< HEAD - inarr, presto_model_url, self.epsg, batch_size=8192 -======= inarr, presto_model_url, self.epsg, batch_size=4096 ->>>>>>> main ) return features From 06c4349b162c1724f529d1930c3df7d080aa7cbe Mon Sep 17 00:00:00 2001 From: Darius Couchard Date: Tue, 18 Jun 2024 11:06:11 +0200 Subject: [PATCH 4/5] Fixed linting --- src/worldcereal/openeo/preprocessing.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/worldcereal/openeo/preprocessing.py b/src/worldcereal/openeo/preprocessing.py index 101a7832..f838d93e 100644 --- a/src/worldcereal/openeo/preprocessing.py +++ b/src/worldcereal/openeo/preprocessing.py @@ -10,12 +10,14 @@ SpatialContext, TemporalContext, ) -from openeo_gfmap.fetching.meteo import build_meteo_extractor from openeo_gfmap.fetching.generic import build_generic_extractor from openeo_gfmap.fetching.meteo import build_meteo_extractor from openeo_gfmap.fetching.s1 import build_sentinel1_grd_extractor from openeo_gfmap.fetching.s2 import build_sentinel2_l2a_extractor -from openeo_gfmap.preprocessing.compositing import mean_compositing, median_compositing, sum_compositing +from openeo_gfmap.preprocessing.compositing import ( + mean_compositing, + median_compositing, +) from openeo_gfmap.preprocessing.sar import compress_backscatter_uint16 from openeo_gfmap.utils.catalogue import UncoveredS1Exception, select_S1_orbitstate From a98a8403deef307532f44ef7f7d29843d2160b65 Mon Sep 17 00:00:00 2001 From: Darius Couchard Date: Tue, 18 Jun 2024 11:10:16 +0200 Subject: [PATCH 5/5] Fix linting and this time you will pass --- src/worldcereal/openeo/preprocessing.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/worldcereal/openeo/preprocessing.py b/src/worldcereal/openeo/preprocessing.py index f838d93e..a9025383 100644 --- a/src/worldcereal/openeo/preprocessing.py +++ b/src/worldcereal/openeo/preprocessing.py @@ -14,10 +14,7 @@ from openeo_gfmap.fetching.meteo import build_meteo_extractor from openeo_gfmap.fetching.s1 import build_sentinel1_grd_extractor from openeo_gfmap.fetching.s2 import build_sentinel2_l2a_extractor -from openeo_gfmap.preprocessing.compositing import ( - mean_compositing, - median_compositing, -) +from openeo_gfmap.preprocessing.compositing import mean_compositing, median_compositing from openeo_gfmap.preprocessing.sar import compress_backscatter_uint16 from openeo_gfmap.utils.catalogue import UncoveredS1Exception, select_S1_orbitstate @@ -236,7 +233,7 @@ def raw_datacube_METEO( extractor = build_meteo_extractor( backend_context=backend_context, bands=["AGERA5-TMEAN", "AGERA5-PRECIP"], - fetch_type=fetch_type + fetch_type=fetch_type, ) return extractor.get_cube(connection, spatial_extent, temporal_extent) @@ -267,9 +264,9 @@ def precomposited_datacube_METEO( temporal_extent=temporal_extent, bands=["precipitation-flux", "temperature-mean"], ) - cube = cube.rename_labels(dimension="bands", target=[ - "AGERA5-PRECIP", "AGERA5-TMEAN" - ]) + cube = cube.rename_labels( + dimension="bands", target=["AGERA5-PRECIP", "AGERA5-TMEAN"] + ) return cube