Merge pull request #17 from GEOSYS/dev

v0.0.1-rc4
earthdaily · Oct 19, 2023 · 8fce3df · 8fce3df
2 parents 0cba3d3 + 91734c7
commit 8fce3df
Show file tree

Hide file tree

Showing 7 changed files with 56 additions and 118 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,11 +4,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.0.1-rc4] 
+
+### Changed
+
+- When asking `ag_cloud_mask`, keep only sensor items that have `eda:ag_cloud_mask_available=True`.
+
 ## [0.0.1-rc3] 2023-10-18
 
 ### Added
 
-- `earthdaily.earthdatastore.cube_utils.zonal_stats_numpy` to compute local statistics using numpy functions.
+- `earthdaily.earthdatastore.cube_utils.zonal_stats_numpy` to compute local statistics using numpy functions. Best when high number of geometries.
 
 ### Changed
 
@@ -17,7 +23,7 @@ really faster faster. Previous behavior is available by selecting `method="stand
 
 ### Fixed
 
-- Fix when number of ag_cloud_mask is lower than number of sensor items
+- Fix when number of `ag_cloud_mask` is lower than number of sensor items
 
 ## [0.0.1-rc2] 2023-10-12
 

diff --git a/earthdaily/__init__.py b/earthdaily/__init__.py
@@ -1,3 +1,3 @@
 from earthdaily import earthdatastore
 
-__version__ = "0.0.1-rc3"
+__version__ = "0.0.1-rc4"
diff --git a/earthdaily/earthdatastore/__init__.py b/earthdaily/earthdatastore/__init__.py
@@ -32,7 +32,9 @@ def post_query_items(items, query):
                                 items_.append(item)
                                 item_already_checked = True
                     else:
-                        operation = operator.__dict__[v_op](item.properties[k], v_val)
+                        operation = operator.__dict__[v_op](
+                            item.properties[k], v_val
+                        )
                     if operation:
                         items_.append(item)
                         item_already_checked = True
@@ -81,7 +83,9 @@ def enhance_assets(
                 if use_http_url:
                     href = item.assets[asset].to_dict().get("href", {})
                     if href:
-                        items[idx].assets[asset].href = _cloud_path_to_http(href)
+                        items[idx].assets[asset].href = _cloud_path_to_http(
+                            href
+                        )
                 if add_default_scale_factor:
                     scale_factor_collection = (
                         _scales_collections.scale_factor_collections.get(
@@ -103,15 +107,15 @@ def enhance_assets(
                                 .extra_fields["raster:bands"][0]
                                 .get("scale")
                             ):
-                                items[idx].assets[asset].extra_fields["raster:bands"][
-                                    0
-                                ]["scale"] = scales_collection["scale"]
-                                items[idx].assets[asset].extra_fields["raster:bands"][
-                                    0
-                                ]["offset"] = scales_collection["offset"]
-                                items[idx].assets[asset].extra_fields["raster:bands"][
-                                    0
-                                ]["nodata"] = scales_collection["nodata"]
+                                items[idx].assets[asset].extra_fields[
+                                    "raster:bands"
+                                ][0]["scale"] = scales_collection["scale"]
+                                items[idx].assets[asset].extra_fields[
+                                    "raster:bands"
+                                ][0]["offset"] = scales_collection["offset"]
+                                items[idx].assets[asset].extra_fields[
+                                    "raster:bands"
+                                ][0]["nodata"] = scales_collection["nodata"]
 
     return items
 
@@ -126,7 +130,9 @@ def _get_client(config=None):
     auth_url = config("EDS_AUTH_URL")
     secret = config("EDS_SECRET")
     client_id = config("EDS_CLIENT_ID")
-    eds_url = config("EDS_API_URL", "https://api.eds.earthdaily.com/archive/v1/stac/v1")
+    eds_url = config(
+        "EDS_API_URL", "https://api.eds.earthdaily.com/archive/v1/stac/v1"
+    )
     if auth_url is None or secret is None or client_id is None:
         raise AttributeError(
             "You need to have env : EDS_AUTH_URL, EDS_SECRET and EDS_CLIENT_ID"
@@ -166,7 +172,10 @@ def __first_item(self):
 
     @property
     def item_properties(self):
-        return {k: self.item.properties[k] for k in sorted(self.item.properties.keys())}
+        return {
+            k: self.item.properties[k]
+            for k in sorted(self.item.properties.keys())
+        }
 
     def assets(self, asset_name=None):
         if asset_name:
@@ -340,9 +349,9 @@ def explore(self, collection: str = None):
         """
         if collection:
             if collection not in self._staccollectionexplorer.keys():
-                self._staccollectionexplorer[collection] = StacCollectionExplorer(
-                    self.client, collection
-                )
+                self._staccollectionexplorer[
+                    collection
+                ] = StacCollectionExplorer(self.client, collection)
             return self._staccollectionexplorer.get(collection)
         return sorted(c.id for c in self.client.get_all_collections())
 
@@ -368,11 +377,19 @@ def datacube(
                 )
             if isinstance(collections, list):
                 if len(collections) > 1:
-                    raise ValueError("Mask_with only manage one collection at a time.")
+                    raise ValueError(
+                        "Mask_with only manage one collection at a time."
+                    )
                 collection = collections[0]
             else:
                 collection = collections
 
+        if mask_with == "ag_cloud_mask":
+            # to get only items that have a ag_cloud_mask
+            query = search_kwargs.get("query", {})
+            query.update({"eda:ag_cloud_mask_available": {"eq": True}})
+            search_kwargs["query"] = query
+
         items = self.search(
             collections=collections,
             bbox=bbox,
@@ -388,7 +405,9 @@ def datacube(
         )
         if mask_with:
             if mask_with == "native":
-                mask_with = mask._native_mask_def_mapping.get(collection, None)
+                mask_with = mask._native_mask_def_mapping.get(
+                    collection, None
+                )
                 if mask_with is None:
                     raise ValueError(
                         f"Sorry, there's no native mask available for {collection}. Only these collections have native cloudmask : {list(mask._native_mask_mapping.keys())}."
@@ -406,6 +425,7 @@ def datacube(
                 )
                 xr_datacube["time"] = xr_datacube.time.astype("M8[s]")
                 acm_datacube["time"] = acm_datacube.time.astype("M8[s]")
+
                 mask_kwargs.update(acm_datacube=acm_datacube)
             else:
                 mask_assets = mask._native_mask_asset_mapping[collections]
@@ -576,7 +596,9 @@ def ag_cloud_mask_from_items(items):
             for item in items:
                 if not item.properties.get("eda:ag_cloud_mask_available"):
                     continue
-                collection = item.properties["eda:ag_cloud_mask_collection_id"]
+                collection = item.properties[
+                    "eda:ag_cloud_mask_collection_id"
+                ]
                 if products.get(collection, None) is None:
                     products[collection] = []
                 products[collection].append(

diff --git a/earthdaily/earthdatastore/mask/__init__.py b/earthdaily/earthdatastore/mask/__init__.py
@@ -55,7 +55,9 @@ def ag_cloud_mask(
         )  # rm nano second
         self._obj["time"] = self._obj.time.dt.round("s")  # rm nano second
         #
-        dc = self._obj.where(acm_datacube["agriculture-cloud-mask"] == 1)
+        self._obj = self._obj.where(
+            acm_datacube["agriculture-cloud-mask"] == 1
+        )
         if add_mask_var:
             self._obj["agriculture-cloud-mask"] = acm_datacube[
                 "agriculture-cloud-mask"

diff --git a/examples/first_steps_create_datacube.py b/examples/first_steps_create_datacube.py
@@ -56,7 +56,9 @@
 ###########################################################
 # Request items
 
-items = eds.search("sentinel-2-l2a", intersects=geometry, datetime=["2022-07"])
+items = eds.search(
+    "sentinel-2-l2a", intersects=geometry, datetime=["2022-07"]
+)
 
 ###########################################################
 # Creata datacube (independent from being log into earthdatastore)

diff --git a/examples/mask_scl_stats.py b/examples/mask_scl_stats.py
diff --git a/examples/s2_datacube_SCL_masked_county_steel.py b/examples/s2_datacube_SCL_masked_county_steel.py