From 71b3cc5afd7690a417420e1b048b99ced220c663 Mon Sep 17 00:00:00 2001 From: Antsalacia Date: Thu, 28 Nov 2024 15:40:45 +0100 Subject: [PATCH 01/13] adding compression option --- earthspy/earthspy.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/earthspy/earthspy.py b/earthspy/earthspy.py index de6581d..66fcfe6 100644 --- a/earthspy/earthspy.py +++ b/earthspy/earthspy.py @@ -85,6 +85,7 @@ def set_query_parameters( download_mode: str = "SM", remove_splitboxes: bool = True, verbose: bool = True, + compression: str = None ) -> None: """Define a set of parameters used for the API request. @@ -171,6 +172,10 @@ def set_query_parameters( # set and correct resolution self.set_correct_resolution() + + # set compress mode + + self.compress_mode = compression # set post-processing attributes self.get_evaluation_script(evaluation_script) @@ -188,6 +193,14 @@ def set_query_parameters( return None + def get_raster_compression(self) -> None: + + try: + assert self.compress_mode in ['DEFLATE','LZW','PACKBITS','JPEG', + 'WEBP','LZMA','ZSTD'] + except: + print("Not a valid compression keyword") + def get_data_collection(self) -> shb.DataCollection: """Get Sentinel Hub DataCollection object from data collection name. @@ -1059,6 +1072,7 @@ def merge_rasters(self) -> None: "height": mosaic.shape[1], "width": mosaic.shape[2], "transform": output_transform, + "compress": self.compress_mode } ) From c7dbb65952c52238d9ef6c6a97bb8ef191f98503 Mon Sep 17 00:00:00 2001 From: Antsalacia Date: Fri, 29 Nov 2024 11:33:35 +0100 Subject: [PATCH 02/13] Adding modification and verification in method --- earthspy/earthspy.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/earthspy/earthspy.py b/earthspy/earthspy.py index 66fcfe6..b242513 100644 --- a/earthspy/earthspy.py +++ b/earthspy/earthspy.py @@ -174,8 +174,7 @@ def set_query_parameters( self.set_correct_resolution() # set compress mode - - self.compress_mode = compression + self.get_raster_compression() # set post-processing attributes self.get_evaluation_script(evaluation_script) @@ -193,13 +192,18 @@ def set_query_parameters( return None - def get_raster_compression(self) -> None: + def get_raster_compression(self) -> str: + """Verify valid keyword for raster compression - try: - assert self.compress_mode in ['DEFLATE','LZW','PACKBITS','JPEG', - 'WEBP','LZMA','ZSTD'] - except: - print("Not a valid compression keyword") + :return: Compression mode + """ + + if compress_mode in ['DEFLATE','LZW','PACKBITS','JPEG', 'WEBP','LZMA','ZSTD']: + self.compress_mode = compress_mode + else: + raise KeyError("Compression mode not found") + + return self.compress_mode def get_data_collection(self) -> shb.DataCollection: """Get Sentinel Hub DataCollection object from data collection name. From b1e9d192bcfe38af71caa7fdd42f18f24c33e246 Mon Sep 17 00:00:00 2001 From: Antsalacia Date: Fri, 29 Nov 2024 12:09:44 +0100 Subject: [PATCH 03/13] add test_geojson method to test the geojson file --- tests/test_earthspy.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_earthspy.py b/tests/test_earthspy.py index ffe06c7..2808bf4 100644 --- a/tests/test_earthspy.py +++ b/tests/test_earthspy.py @@ -12,7 +12,7 @@ import pandas as pd import requests import sentinelhub as shb - +import json class TestEarthspy: # create local variables from environment secrets for convenience @@ -330,6 +330,20 @@ def test_sentinelhub_request(self) -> None: # # check that a Sentinel Hub request was created assert isinstance(sr2, shb.SentinelHubRequest) + def test_geojson(self) -> None: + """Test geojson files""" + + with open("earthspy/data/ilulissat.geojson", "r+") as files_geo: + data = json.load(files_geo) + array_coord = np.array(data["features"][0]["geometry"] + ['coordinates'][0]) + + assert data["features"][0]["geometry"]['type'] == "Polygon" + assert array_coord.shape == (5, 2) + for n in array_coord: + for i in n: + assert (-90 <= i <= 90) + def test_rename_output_files(self) -> None: """Test output renaming""" From 0167d58d840a2110930515863bb0acef70536f86 Mon Sep 17 00:00:00 2001 From: Antsalacia Date: Fri, 29 Nov 2024 16:39:22 +0100 Subject: [PATCH 04/13] restore earthspy file --- earthspy/earthspy.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/earthspy/earthspy.py b/earthspy/earthspy.py index b242513..de6581d 100644 --- a/earthspy/earthspy.py +++ b/earthspy/earthspy.py @@ -85,7 +85,6 @@ def set_query_parameters( download_mode: str = "SM", remove_splitboxes: bool = True, verbose: bool = True, - compression: str = None ) -> None: """Define a set of parameters used for the API request. @@ -172,9 +171,6 @@ def set_query_parameters( # set and correct resolution self.set_correct_resolution() - - # set compress mode - self.get_raster_compression() # set post-processing attributes self.get_evaluation_script(evaluation_script) @@ -192,19 +188,6 @@ def set_query_parameters( return None - def get_raster_compression(self) -> str: - """Verify valid keyword for raster compression - - :return: Compression mode - """ - - if compress_mode in ['DEFLATE','LZW','PACKBITS','JPEG', 'WEBP','LZMA','ZSTD']: - self.compress_mode = compress_mode - else: - raise KeyError("Compression mode not found") - - return self.compress_mode - def get_data_collection(self) -> shb.DataCollection: """Get Sentinel Hub DataCollection object from data collection name. @@ -1076,7 +1059,6 @@ def merge_rasters(self) -> None: "height": mosaic.shape[1], "width": mosaic.shape[2], "transform": output_transform, - "compress": self.compress_mode } ) From 300cfdf08e722883245faab556edf9dd599334e3 Mon Sep 17 00:00:00 2001 From: Antsalacia Date: Fri, 29 Nov 2024 17:34:21 +0100 Subject: [PATCH 05/13] Add loop over file and test --- tests/test_earthspy.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_earthspy.py b/tests/test_earthspy.py index 2808bf4..651457b 100644 --- a/tests/test_earthspy.py +++ b/tests/test_earthspy.py @@ -333,16 +333,16 @@ def test_sentinelhub_request(self) -> None: def test_geojson(self) -> None: """Test geojson files""" - with open("earthspy/data/ilulissat.geojson", "r+") as files_geo: - data = json.load(files_geo) - array_coord = np.array(data["features"][0]["geometry"] - ['coordinates'][0]) - - assert data["features"][0]["geometry"]['type'] == "Polygon" - assert array_coord.shape == (5, 2) - for n in array_coord: - for i in n: - assert (-90 <= i <= 90) + for file in os.listdir("earthspy/data"): + with open(f"earthspy/data/{file}", "r+") as files_geo: + data = json.load(files_geo) + array_coord = np.array(data["features"][0]["geometry"] + ['coordinates'][0]) + + assert data["features"][0]["geometry"]['type'] == "Polygon" + assert array_coord.shape == (5, 2) + assert ((array_coord >= -90) & (array_coord <= 90)).all() + assert array_coord[0, 0] == array_coord[-1, 0] def test_rename_output_files(self) -> None: """Test output renaming""" From d6d91f2fab6bf51dfb909f309fa0e7ef90294cee Mon Sep 17 00:00:00 2001 From: Antsalacia Date: Mon, 2 Dec 2024 10:31:26 +0100 Subject: [PATCH 06/13] Adding comments and use of glob --- tests/test_earthspy.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_earthspy.py b/tests/test_earthspy.py index 651457b..e24e132 100644 --- a/tests/test_earthspy.py +++ b/tests/test_earthspy.py @@ -13,6 +13,7 @@ import requests import sentinelhub as shb import json +import glob class TestEarthspy: # create local variables from environment secrets for convenience @@ -333,15 +334,20 @@ def test_sentinelhub_request(self) -> None: def test_geojson(self) -> None: """Test geojson files""" - for file in os.listdir("earthspy/data"): - with open(f"earthspy/data/{file}", "r+") as files_geo: + folder_path = glob.glob("earthspy/data/*", recursive=True) + for file in folder_path: + with open(file, "r+") as files_geo: data = json.load(files_geo) array_coord = np.array(data["features"][0]["geometry"] ['coordinates'][0]) + # check if the figure is a Polygon assert data["features"][0]["geometry"]['type'] == "Polygon" + # check if the coordinates are in the right format assert array_coord.shape == (5, 2) + # check if the coordinates are between -90 and 90 assert ((array_coord >= -90) & (array_coord <= 90)).all() + # check if the first and the last coordinates are the same assert array_coord[0, 0] == array_coord[-1, 0] def test_rename_output_files(self) -> None: From 017e5aab70bc6ff0da0375a7194a91edd199dd8d Mon Sep 17 00:00:00 2001 From: Antsalacia Date: Tue, 3 Dec 2024 10:46:46 +0100 Subject: [PATCH 07/13] add pre-commit file and last change in test_earthspy --- .pre-commit-config.yaml | 112 ++++++++++++++++++++++++++++++++++++++++ tests/test_earthspy.py | 23 +++++---- 2 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..210299f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,112 @@ +# based on Glaciohack pre-commit config +ci: + autofix_prs: false + autoupdate_schedule: quarterly +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + exclude: \.txt$ + - id: trailing-whitespace # Remove trailing + # whitespaces + - id: check-merge-conflict + # Fix common spelling mistakes + - repo: https://github.com/codespell-project/codespell + rev: v2.3.0 + hooks: + - id: codespell + args: [ + '--ignore-words-list', 'alos,inout,vor', + '--ignore-regex', '\bnin\b', + '--' + ] + types_or: [python, rst, markdown] + files: ^(earthspy|docs|tests)/ + + # Replace relative imports + - repo: https://github.com/MarcoGorelli/absolufy-imports + rev: v0.3.1 + hooks: + - id: absolufy-imports + + # Format the code aggressively using black + - repo: https://github.com/psf/black + rev: 24.10.0 + hooks: + - id: black + args: [--line-length=88] + + # Lint the code using flake8 + - repo: https://github.com/pycqa/flake8 + rev: 7.1.1 + hooks: + - id: flake8 + args: [ + '--max-line-length', '122', + '--extend-ignore', 'E203,B028', # flake8 + # disagrees + # with + # black, so + # this + # should be + # ignored. + '--' + ] + additional_dependencies: + - flake8-comprehensions + - flake8-bugbear + files: ^(earthspy|tests) + + # Sort imports using isort + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + args: [ "--profile", "black" ] + + # Automatically upgrade syntax to a minimum version + - repo: https://github.com/asottile/pyupgrade + rev: v3.19.0 + hooks: + - id: pyupgrade + args: [--py37-plus] + + # Various formattings + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + # Detect common mistake of using single backticks when + # writing rst + - id: rst-backticks + # Detect mistake of rst directive not ending with + # double colon or space before the double colon + - id: rst-directive-colons + types: [text] + types_or: [python, rst] + # Detect mistake of inline code touching normal text + # in rst + - id: rst-inline-touching-normal + types: [text] + types_or: [python, rst] + # Eval should never be used (can do arbitrary code + # execution) + - id: python-no-eval + # Enforce the use of type annotations instead of + # docstring type comments + - id: python-use-type-annotations + + + - repo: local + hooks: + # Generate pip's dev-requirements.txt from conda's + # dev-environment.yml to run snyk (snyk doesn't currently + # support conda) + - id: pip-to-conda + name: Generate pip dependency from conda + language: python + entry: .github/scripts/generate_pip_deps_from_conda.py + files: ^(dev-environment.yml|requirements-dev.txt)$ + pass_filenames: false + additional_dependencies: [tomli, pyyaml] \ No newline at end of file diff --git a/tests/test_earthspy.py b/tests/test_earthspy.py index e24e132..10773e0 100644 --- a/tests/test_earthspy.py +++ b/tests/test_earthspy.py @@ -1,19 +1,21 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ @author: Adrien Wehrlé, EO-IO, University of Zurich, Switzerland """ -import earthspy.earthspy as es -import numpy as np +import glob +import json import os + +import numpy as np import pandas as pd import requests import sentinelhub as shb -import json -import glob + +import earthspy.earthspy as es + class TestEarthspy: # create local variables from environment secrets for convenience @@ -338,16 +340,17 @@ def test_geojson(self) -> None: for file in folder_path: with open(file, "r+") as files_geo: data = json.load(files_geo) - array_coord = np.array(data["features"][0]["geometry"] - ['coordinates'][0]) - + array_coord = np.array( + data["features"][0]["geometry"]["coordinates"][0] + ) + # check if the figure is a Polygon - assert data["features"][0]["geometry"]['type'] == "Polygon" + assert data["features"][0]["geometry"]["type"] == "Polygon" # check if the coordinates are in the right format assert array_coord.shape == (5, 2) # check if the coordinates are between -90 and 90 assert ((array_coord >= -90) & (array_coord <= 90)).all() - # check if the first and the last coordinates are the same + # check if the first and the last coordinates are the same assert array_coord[0, 0] == array_coord[-1, 0] def test_rename_output_files(self) -> None: From 0163f80953446b79016925654eb8ab397dc203ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Wehrl=C3=A9?= Date: Tue, 3 Dec 2024 11:29:18 +0100 Subject: [PATCH 08/13] Modify comments and doc of `test_geojson` --- tests/test_earthspy.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/test_earthspy.py b/tests/test_earthspy.py index ff43205..c117939 100644 --- a/tests/test_earthspy.py +++ b/tests/test_earthspy.py @@ -335,24 +335,32 @@ def test_sentinelhub_request(self) -> None: assert isinstance(sr2, shb.SentinelHubRequest) def test_geojson(self) -> None: - """Test geojson files""" - - folder_path = glob.glob("earthspy/data/*", recursive=True) - for file in folder_path: - with open(file, "r+") as files_geo: - data = json.load(files_geo) - array_coord = np.array( + """Test fields of GEOJSON files""" + + # list all geojson files available in earthspy + geojson_files = glob.glob("earthspy/data/*") + + for file in geojson_files: + + with open(file, "r+") as f: + data = json.load(f) + + # extract coordinates of geometry nodes + geometry_coordinates = np.array( data["features"][0]["geometry"]["coordinates"][0] ) - # check if the figure is a Polygon + # check if feature is a polygon assert data["features"][0]["geometry"]["type"] == "Polygon" - # check if the coordinates are in the right format - assert array_coord.shape == (5, 2) - # check if the coordinates are between -90 and 90 - assert ((array_coord >= -90) & (array_coord <= 90)).all() - # check if the first and the last coordinates are the same - assert array_coord[0, 0] == array_coord[-1, 0] + + # check if coordinates match a 4-point square + assert geometry_coordinates.shape == (5, 2) + + # check if the coordinates are valid longitude/latitude coordinates + assert ((geometry_coordinates >= -90) & (geometry_coordinates <= 90)).all() + + # check if first coordinate is repeated on the last element + assert geometry_coordinates[0, 0] == geometry_coordinates[-1, 0] def test_rename_output_files(self) -> None: """Test output renaming""" From de8a3581979034511ec1b85ac2d830bbe26d8ef8 Mon Sep 17 00:00:00 2001 From: AdrienWehrle Date: Tue, 3 Dec 2024 14:05:24 +0100 Subject: [PATCH 09/13] reran precommit --- tests/test_earthspy.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_earthspy.py b/tests/test_earthspy.py index c117939..48adc81 100644 --- a/tests/test_earthspy.py +++ b/tests/test_earthspy.py @@ -339,9 +339,9 @@ def test_geojson(self) -> None: # list all geojson files available in earthspy geojson_files = glob.glob("earthspy/data/*") - + for file in geojson_files: - + with open(file, "r+") as f: data = json.load(f) @@ -352,13 +352,15 @@ def test_geojson(self) -> None: # check if feature is a polygon assert data["features"][0]["geometry"]["type"] == "Polygon" - + # check if coordinates match a 4-point square assert geometry_coordinates.shape == (5, 2) - + # check if the coordinates are valid longitude/latitude coordinates - assert ((geometry_coordinates >= -90) & (geometry_coordinates <= 90)).all() - + assert ( + (geometry_coordinates >= -90) & (geometry_coordinates <= 90) + ).all() + # check if first coordinate is repeated on the last element assert geometry_coordinates[0, 0] == geometry_coordinates[-1, 0] From e75a08f2ea20796b6775f48707be779ddb11aa8e Mon Sep 17 00:00:00 2001 From: AdrienWehrle Date: Tue, 3 Dec 2024 14:08:30 +0100 Subject: [PATCH 10/13] bring tests out of file context --- tests/test_earthspy.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tests/test_earthspy.py b/tests/test_earthspy.py index 48adc81..94fab87 100644 --- a/tests/test_earthspy.py +++ b/tests/test_earthspy.py @@ -345,24 +345,22 @@ def test_geojson(self) -> None: with open(file, "r+") as f: data = json.load(f) - # extract coordinates of geometry nodes - geometry_coordinates = np.array( - data["features"][0]["geometry"]["coordinates"][0] - ) + # extract coordinates of geometry nodes + geometry_coordinates = np.array( + data["features"][0]["geometry"]["coordinates"][0] + ) - # check if feature is a polygon - assert data["features"][0]["geometry"]["type"] == "Polygon" + # check if feature is a polygon + assert data["features"][0]["geometry"]["type"] == "Polygon" - # check if coordinates match a 4-point square - assert geometry_coordinates.shape == (5, 2) + # check if coordinates match a 4-point square + assert geometry_coordinates.shape == (5, 2) - # check if the coordinates are valid longitude/latitude coordinates - assert ( - (geometry_coordinates >= -90) & (geometry_coordinates <= 90) - ).all() + # check if the coordinates are valid longitude/latitude coordinates + assert ((geometry_coordinates >= -90) & (geometry_coordinates <= 90)).all() - # check if first coordinate is repeated on the last element - assert geometry_coordinates[0, 0] == geometry_coordinates[-1, 0] + # check if first coordinate is repeated on the last element + assert geometry_coordinates[0, 0] == geometry_coordinates[-1, 0] def test_rename_output_files(self) -> None: """Test output renaming""" From 81a990c96d8b457bd564103bd25d3366a435968e Mon Sep 17 00:00:00 2001 From: AdrienWehrle Date: Tue, 3 Dec 2024 14:24:39 +0100 Subject: [PATCH 11/13] check that geojson files are found --- tests/test_earthspy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_earthspy.py b/tests/test_earthspy.py index 94fab87..a382943 100644 --- a/tests/test_earthspy.py +++ b/tests/test_earthspy.py @@ -338,7 +338,10 @@ def test_geojson(self) -> None: """Test fields of GEOJSON files""" # list all geojson files available in earthspy - geojson_files = glob.glob("earthspy/data/*") + geojson_files = glob.glob("./earthspy/data/*") + + # check that files were found + assert len(geojson_files) > 0 for file in geojson_files: From c6be13c3d0a872baabda0d03fa9115dc6640950c Mon Sep 17 00:00:00 2001 From: AdrienWehrle Date: Tue, 3 Dec 2024 14:29:21 +0100 Subject: [PATCH 12/13] change tests on --- .github/workflows/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1ae9a63..81b7c71 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,9 @@ name: CI -on: [push] +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] jobs: miniconda: From 9fbafd4ec6b15e1f717616f09d3e50fa73d75da4 Mon Sep 17 00:00:00 2001 From: AdrienWehrle Date: Wed, 4 Dec 2024 22:19:08 +0100 Subject: [PATCH 13/13] rename test method --- tests/test_earthspy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_earthspy.py b/tests/test_earthspy.py index a382943..28a3ced 100644 --- a/tests/test_earthspy.py +++ b/tests/test_earthspy.py @@ -334,7 +334,7 @@ def test_sentinelhub_request(self) -> None: # # check that a Sentinel Hub request was created assert isinstance(sr2, shb.SentinelHubRequest) - def test_geojson(self) -> None: + def test_geojson_files(self) -> None: """Test fields of GEOJSON files""" # list all geojson files available in earthspy