From 2f49382bc0f01cea89e57b1856b47a785f4255b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Wehrl=C3=A9?= Date: Sat, 23 Nov 2024 22:43:28 +0100 Subject: [PATCH 01/13] Update workflow actions (#102) * update workflow actions * fix codecov token * fix codecov token * fix repository not found with slug * use codecov action v4 * use codecov action v4 * update snyk action * update snyk action * fix snyk action * setup snyk action for python * setup snyk action for python * setup snyk action for python * setup snyk action for python * setup snyk action for python --- .github/workflows/main.yml | 38 +++++++++++++++++++------------------- environment.yml | 5 +---- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9b67109..5f36cdb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,13 +9,13 @@ jobs: matrix: os: ["ubuntu-latest"] steps: - - uses: actions/checkout@v2 - - uses: snyk/actions/setup@master - - uses: conda-incubator/setup-miniconda@v2 + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 with: activate-environment: test environment-file: environment.yml - python-version: 3.8 + python-version: 3.9 + use-mamba: true auto-activate-base: true - shell: bash -l {0} @@ -29,7 +29,7 @@ jobs: pip install -e . - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: # Full git history is needed to get a proper list of changed files within `super-linter` fetch-depth: 0 @@ -48,26 +48,26 @@ jobs: run: | pip install pytest pip install pytest-cov - pytest --cov=./ --cov-report=xml + pytest --cov --junitxml=junit.xml -o junit_family=legacy env: SH_CLIENT_ID: ${{ secrets.SH_CLIENT_ID }} SH_CLIENT_SECRET: ${{ secrets.SH_CLIENT_SECRET }} - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + - name: Upload test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 with: - directory: ./coverage/reports/ - fail_ci_if_error: true - files: ./coverage.xml - flags: unittests - name: codecov-umbrella - path_to_write_report: ./coverage/codecov_report.txt - verbose: true + token: ${{ secrets.CODECOV_TOKEN }} - - name: Run Snyk to check for vulnerabilities + - name: Install dev requirements + shell: bash -l {0} run: | pip install -r dev-requirements.txt - snyk test --file=dev-requirements.txt --package-manager=pip --severity-threshold=high + + - name: Run Snyk to check for vulnerabilities + uses: snyk/actions/python@master + continue-on-error: true env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + args: --file=dev-requirements.txt --package-manager=pip --severity-threshold=high --skip-unresolved=true diff --git a/environment.yml b/environment.yml index fce44eb..65dd7cd 100644 --- a/environment.yml +++ b/environment.yml @@ -1,11 +1,8 @@ name: earthspy channels: - - conda-forge/label/cf202003 - - https://conda.software.inl.gov/public - - defaults - conda-forge dependencies: - - python=3 + - python>=3.9 - numpy - objectpath - pandas From 4dd794893cf75001c8a4f9d222beb58031096f54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Wehrl=C3=A9?= Date: Sun, 24 Nov 2024 20:43:16 +0100 Subject: [PATCH 02/13] expand testing on python 3.10-3.12 * add macos and python versions * remove python 3.9 * keep only ubuntu-latest for now --- .github/workflows/main.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5f36cdb..0cbb670 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,15 +6,16 @@ jobs: name: Setup and test runs-on: ${{ matrix.os }} strategy: - matrix: - os: ["ubuntu-latest"] + matrix: + os: ["ubuntu-latest"] + python-version: ["3.10", "3.11", "3.12"] + steps: - uses: actions/checkout@v4 - uses: conda-incubator/setup-miniconda@v3 with: activate-environment: test environment-file: environment.yml - python-version: 3.9 use-mamba: true auto-activate-base: true From f44ef666eec535903039c53984b7d09f5dc9202b Mon Sep 17 00:00:00 2001 From: Antsalacia Date: Tue, 26 Nov 2024 15:04:27 +0100 Subject: [PATCH 03/13] Add metadata to tiff files (#103) * add metadata to tiff files * add the **kwargs in variable --- earthspy/earthspy.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/earthspy/earthspy.py b/earthspy/earthspy.py index de6581d..0296aba 100644 --- a/earthspy/earthspy.py +++ b/earthspy/earthspy.py @@ -228,8 +228,13 @@ def get_available_data(self) -> list: # and update service_url if dowload failed try: # store metadata of available scenes - self.metadata = [list(iterator) for iterator in search_iterator] - + self.metadata = {} + for iterator in search_iterator: + iterator_list = list(iterator) + if len(iterator_list) > 0: + date = iterator_list[0]["properties"]["datetime"].split("T")[0] + self.metadata[date] = iterator_list + except shb.exceptions.DownloadFailedException: # set specific base URL of deployment self.catalog_config.sh_base_url = shb.DataCollection[ @@ -250,8 +255,13 @@ def get_available_data(self) -> list: ] # store metadata of available scenes - self.metadata = [list(iterator) for iterator in search_iterator] - + self.metadata = {} + for iterator in search_iterator: + iterator_list = list(iterator) + if len(iterator_list) > 0: + date = iterator_list[0]["properties"]["datetime"].split("T")[0] + self.metadata[date] = iterator_list + # create date +-1 hour around acquisition time time_difference = timedelta(hours=1) @@ -1061,11 +1071,11 @@ def merge_rasters(self) -> None: "transform": output_transform, } ) - + id_dict = {k:self.metadata[date][0][k] for k in ["id"]} # write mosaic with rasterio.open(date_output_filename, "w", **output_meta) as dst: dst.write(mosaic) - + dst.update_tags(**id_dict) # save file name of merged raster self.output_filenames_renamed.append(date_output_filename) From f2bfed8434f9cd44823b69cfe06dbbd265e1a895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Wehrl=C3=A9?= Date: Tue, 26 Nov 2024 16:28:09 +0100 Subject: [PATCH 04/13] remove WIP badge (#105) --- README.org | 1 - 1 file changed, 1 deletion(-) diff --git a/README.org b/README.org index c187917..06d0cd7 100644 --- a/README.org +++ b/README.org @@ -1,4 +1,3 @@ -[[https://www.repostatus.org/badges/latest/wip.svg][https://www.repostatus.org/badges/latest/wip.svg]] [[https://www.gnu.org/licenses/gpl-3.0][https://img.shields.io/badge/License-GPLv3-blue.svg]] [[https://github.com/AdrienWehrle/earthspy/actions][file:https://github.com/AdrienWehrle/earthspy/workflows/CI/badge.svg]] [[https://github.com/AdrienWehrle/earthspy/actions/workflows/codeql.yml][https://github.com/AdrienWehrle/earthspy/actions/workflows/codeql.yml/badge.svg]] From 5ff5a9a370d3e6b85cf5751b2bd02cc52d07b2f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Wehrl=C3=A9?= Date: Sun, 1 Dec 2024 12:12:01 +0100 Subject: [PATCH 05/13] Enable pre-commit and add action (#108) * enable pre-commit * add pre-commit action * fix test * fix path * remove line length in flake8 * set flake8 line length high enough * set flake8 line length high enough --- .github/workflows/main.yml | 16 +-- .github/workflows/pre-commit.yml | 15 +++ .gitignore | 2 +- .pre-commit-config.yaml | 98 +++++++++++++++++++ README.org | 16 ++- docs/_build/html/_static/alabaster.css | 2 +- docs/_build/html/_static/basic.css | 2 +- .../html/_static/documentation_options.js | 2 +- docs/_build/html/_static/language_data.js | 2 - docs/_build/html/_static/pygments.css | 2 +- docs/_build/html/_static/underscore.js | 2 +- docs/_build/html/genindex.html | 26 ++--- docs/_build/html/index.html | 24 ++--- docs/_build/html/search.html | 48 ++++----- docs/_build/html/searchindex.js | 2 +- docs/api.rst | 2 +- docs/generated/earthspy.rst | 19 ---- docs/index.rst | 2 +- earthspy/__init__.py | 1 - earthspy/earthspy.py | 63 ++++++------ earthspy/operational/earthspy_NRT.py | 1 - setup.py | 1 + tests/__init__.py | 1 - tests/test_earthspy.py | 20 ++-- 24 files changed, 234 insertions(+), 135 deletions(-) create mode 100644 .github/workflows/pre-commit.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0cbb670..72237e3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,7 +9,7 @@ jobs: matrix: os: ["ubuntu-latest"] python-version: ["3.10", "3.11", "3.12"] - + steps: - uses: actions/checkout@v4 - uses: conda-incubator/setup-miniconda@v3 @@ -18,23 +18,23 @@ jobs: environment-file: environment.yml use-mamba: true auto-activate-base: true - + - shell: bash -l {0} run: | conda info conda list - + - name: Install earthspy shell: bash -l {0} run: | pip install -e . - + - name: Checkout code uses: actions/checkout@v4 with: # Full git history is needed to get a proper list of changed files within `super-linter` - fetch-depth: 0 - + fetch-depth: 0 + - name: Lint code base uses: github/super-linter/slim@v4.9.0 env: @@ -64,11 +64,11 @@ jobs: shell: bash -l {0} run: | pip install -r dev-requirements.txt - + - name: Run Snyk to check for vulnerabilities uses: snyk/actions/python@master continue-on-error: true env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} with: args: --file=dev-requirements.txt --package-manager=pip --severity-threshold=high --skip-unresolved=true diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..25ac772 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,15 @@ +name: Linting and formatting (pre-commit) + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - uses: pre-commit/action@v3.0.1 diff --git a/.gitignore b/.gitignore index f294af6..cef40bf 100644 --- a/.gitignore +++ b/.gitignore @@ -142,4 +142,4 @@ auth.txt *.gcno *.gcda *.gcov -test_calculator \ No newline at end of file +test_calculator diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..aa9d759 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,98 @@ +# based on Glaciohack pre-commit config +ci: + autofix_prs: false + autoupdate_schedule: quarterly +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + exclude: \.txt$ + - id: trailing-whitespace # Remove trailing + # whitespaces + - id: check-merge-conflict + # Fix common spelling mistakes + - repo: https://github.com/codespell-project/codespell + rev: v2.3.0 + hooks: + - id: codespell + args: [ + '--ignore-words-list', 'alos,inout,vor', + '--ignore-regex', '\bnin\b', + '--' + ] + types_or: [python, rst, markdown] + files: ^(earthspy|docs|tests)/ + + # Replace relative imports + - repo: https://github.com/MarcoGorelli/absolufy-imports + rev: v0.3.1 + hooks: + - id: absolufy-imports + + # Format the code aggressively using black + - repo: https://github.com/psf/black + rev: 24.10.0 + hooks: + - id: black + args: [--line-length=88] + + # Lint the code using flake8 + - repo: https://github.com/pycqa/flake8 + rev: 7.1.1 + hooks: + - id: flake8 + args: [ + '--max-line-length', '122', + '--extend-ignore', 'E203,B028', # flake8 + # disagrees + # with + # black, so + # this + # should be + # ignored. + '--' + ] + additional_dependencies: + - flake8-comprehensions + - flake8-bugbear + files: ^(earthspy|tests) + + # Sort imports using isort + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + args: [ "--profile", "black" ] + + # Automatically upgrade syntax to a minimum version + - repo: https://github.com/asottile/pyupgrade + rev: v3.19.0 + hooks: + - id: pyupgrade + args: [--py37-plus] + + # Various formattings + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + # Detect common mistake of using single backticks when + # writing rst + - id: rst-backticks + # Detect mistake of rst directive not ending with + # double colon or space before the double colon + - id: rst-directive-colons + types: [text] + types_or: [python, rst] + # Detect mistake of inline code touching normal text + # in rst + - id: rst-inline-touching-normal + types: [text] + types_or: [python, rst] + # Eval should never be used (can do arbitrary code + # execution) + - id: python-no-eval + # Enforce the use of type annotations instead of + # docstring type comments + - id: python-use-type-annotations diff --git a/README.org b/README.org index 06d0cd7..32a9000 100644 --- a/README.org +++ b/README.org @@ -8,13 +8,13 @@ =earthspy= is a wrapper around methods for the download of satellite data offered in the [[https://github.com/sentinel-hub/sentinelhub-py][sentinelhub Python package]]. This tool makes the monitoring and study of any place on Earth simple, ready to use and easily deployable for operational purposes and automated Near-Real Time (NRT) applications. -Some useful capabilities: +Some useful capabilities: - Data download in multiprocessing - - Data download at optimized resolutions with the Direct (D) download mode + - Data download at optimized resolutions with the Direct (D) download mode - Data download at native resolutions with the Split and Merge (SM) downlodad mode - Data storage with efficient structure and file naming -As =earthspy= is built on top of the [[https://www.sentinel-hub.com/][Sentinel Hub services]], it includes e.g. the data pre-processing through [[https://docs.sentinel-hub.com/api/latest/evalscript/][custom scripts]] allowing the user to process and download only the products needed (such as high-level indices) therefore optimizing download time and local storage. +As =earthspy= is built on top of the [[https://www.sentinel-hub.com/][Sentinel Hub services]], it includes e.g. the data pre-processing through [[https://docs.sentinel-hub.com/api/latest/evalscript/][custom scripts]] allowing the user to process and download only the products needed (such as high-level indices) therefore optimizing download time and local storage. * Table of Contents :toc_2:noexport: - [[#earthspy-%EF%B8%8F-earth_africa-earth_americas-earth_asia][earthspy]] @@ -66,7 +66,12 @@ job = es.EarthSpy("/path/to/auth.txt") # as simple as it gets job.set_query_parameters( - bounding_box=[-51.13, 69.204, -51.06, 69.225], # format from doc: [min_x, min_y, max_x, max_y] + bounding_box=[ + -51.13, + 69.204, + -51.06, + 69.225, + ], # format from doc: [min_x, min_y, max_x, max_y] time_interval=["2019-08-03", "2019-08-10"], evaluation_script="https://custom-scripts.sentinel-hub.com/custom-scripts/sentinel-2/true_color/script.js", data_collection="SENTINEL2_L2A", @@ -74,6 +79,7 @@ job.set_query_parameters( # and off it goes! job.send_sentinelhub_requests() + #+end_src Homemade custom evalscripts can also be passed without effort to e.g. compute high-level indices (NDVI, NDSI...). @@ -141,7 +147,7 @@ job.send_sentinelhub_requests() #+end_src -* Operational Near Real-Time (NRT) deployment +* Operational Near Real-Time (NRT) deployment =earthspy= can be easily deployed for NRT monitoring. The setup is as simple as wrapping the query parameters in a short python script such as [[https://github.com/AdrienWehrle/earthspy/blob/main/earthspy/operational/earthspy_NRT.py][earthspy_NRT.py]] and including it in a cron job. See an example below where Sentinel-2 images of Ilulissat, Greenland acquired over the past three days are downloaded everyday at noon. #+BEGIN_SRC bash :results verbatim diff --git a/docs/_build/html/_static/alabaster.css b/docs/_build/html/_static/alabaster.css index 0eddaeb..969ce31 100644 --- a/docs/_build/html/_static/alabaster.css +++ b/docs/_build/html/_static/alabaster.css @@ -698,4 +698,4 @@ nav#breadcrumbs li+li:before { div.related { display: none; } -} \ No newline at end of file +} diff --git a/docs/_build/html/_static/basic.css b/docs/_build/html/_static/basic.css index bf18350..1c769a5 100644 --- a/docs/_build/html/_static/basic.css +++ b/docs/_build/html/_static/basic.css @@ -903,4 +903,4 @@ div.math:hover a.headerlink { #top-link { display: none; } -} \ No newline at end of file +} diff --git a/docs/_build/html/_static/documentation_options.js b/docs/_build/html/_static/documentation_options.js index f1df40e..bef71d2 100644 --- a/docs/_build/html/_static/documentation_options.js +++ b/docs/_build/html/_static/documentation_options.js @@ -9,4 +9,4 @@ var DOCUMENTATION_OPTIONS = { HAS_SOURCE: true, SOURCELINK_SUFFIX: '.txt', NAVIGATION_WITH_KEYS: false -}; \ No newline at end of file +}; diff --git a/docs/_build/html/_static/language_data.js b/docs/_build/html/_static/language_data.js index ebe2f03..7dbd658 100644 --- a/docs/_build/html/_static/language_data.js +++ b/docs/_build/html/_static/language_data.js @@ -293,5 +293,3 @@ function splitQuery(query) { } return result; } - - diff --git a/docs/_build/html/_static/pygments.css b/docs/_build/html/_static/pygments.css index 87f8bd1..b35b845 100644 --- a/docs/_build/html/_static/pygments.css +++ b/docs/_build/html/_static/pygments.css @@ -79,4 +79,4 @@ span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: .highlight .vg { color: #000000 } /* Name.Variable.Global */ .highlight .vi { color: #000000 } /* Name.Variable.Instance */ .highlight .vm { color: #000000 } /* Name.Variable.Magic */ -.highlight .il { color: #990000 } /* Literal.Number.Integer.Long */ \ No newline at end of file +.highlight .il { color: #990000 } /* Literal.Number.Integer.Long */ diff --git a/docs/_build/html/_static/underscore.js b/docs/_build/html/_static/underscore.js index cf177d4..542fe3c 100644 --- a/docs/_build/html/_static/underscore.js +++ b/docs/_build/html/_static/underscore.js @@ -3,4 +3,4 @@ // https://underscorejs.org // (c) 2009-2021 Jeremy Ashkenas, Julian Gonggrijp, and DocumentCloud and Investigative Reporters & Editors // Underscore may be freely distributed under the MIT license. -var n="1.13.1",r="object"==typeof self&&self.self===self&&self||"object"==typeof global&&global.global===global&&global||Function("return this")()||{},t=Array.prototype,e=Object.prototype,u="undefined"!=typeof Symbol?Symbol.prototype:null,o=t.push,i=t.slice,a=e.toString,f=e.hasOwnProperty,c="undefined"!=typeof ArrayBuffer,l="undefined"!=typeof DataView,s=Array.isArray,p=Object.keys,v=Object.create,h=c&&ArrayBuffer.isView,y=isNaN,d=isFinite,g=!{toString:null}.propertyIsEnumerable("toString"),b=["valueOf","isPrototypeOf","toString","propertyIsEnumerable","hasOwnProperty","toLocaleString"],m=Math.pow(2,53)-1;function j(n,r){return r=null==r?n.length-1:+r,function(){for(var t=Math.max(arguments.length-r,0),e=Array(t),u=0;u=0&&t<=m}}function J(n){return function(r){return null==r?void 0:r[n]}}var G=J("byteLength"),H=K(G),Q=/\[object ((I|Ui)nt(8|16|32)|Float(32|64)|Uint8Clamped|Big(I|Ui)nt64)Array\]/;var X=c?function(n){return h?h(n)&&!q(n):H(n)&&Q.test(a.call(n))}:C(!1),Y=J("length");function Z(n,r){r=function(n){for(var r={},t=n.length,e=0;e":">",'"':""","'":"'","`":"`"},Cn=Ln($n),Kn=Ln(_n($n)),Jn=tn.templateSettings={evaluate:/<%([\s\S]+?)%>/g,interpolate:/<%=([\s\S]+?)%>/g,escape:/<%-([\s\S]+?)%>/g},Gn=/(.)^/,Hn={"'":"'","\\":"\\","\r":"r","\n":"n","\u2028":"u2028","\u2029":"u2029"},Qn=/\\|'|\r|\n|\u2028|\u2029/g;function Xn(n){return"\\"+Hn[n]}var Yn=/^\s*(\w|\$)+\s*$/;var Zn=0;function nr(n,r,t,e,u){if(!(e instanceof r))return n.apply(t,u);var o=Mn(n.prototype),i=n.apply(o,u);return _(i)?i:o}var rr=j((function(n,r){var t=rr.placeholder,e=function(){for(var u=0,o=r.length,i=Array(o),a=0;a1)ur(a,r-1,t,e),u=e.length;else for(var f=0,c=a.length;f0&&(t=r.apply(this,arguments)),n<=1&&(r=null),t}}var lr=rr(cr,2);function sr(n,r,t){r=qn(r,t);for(var e,u=nn(n),o=0,i=u.length;o0?0:u-1;o>=0&&o0?a=o>=0?o:Math.max(o+f,a):f=o>=0?Math.min(o+1,f):o+f+1;else if(t&&o&&f)return e[o=t(e,u)]===u?o:-1;if(u!=u)return(o=r(i.call(e,a,f),$))>=0?o+a:-1;for(o=n>0?a:f-1;o>=0&&o0?0:i-1;for(u||(e=r[o?o[a]:a],a+=n);a>=0&&a=3;return r(n,Fn(t,u,4),e,o)}}var Ar=wr(1),xr=wr(-1);function Sr(n,r,t){var e=[];return r=qn(r,t),jr(n,(function(n,t,u){r(n,t,u)&&e.push(n)})),e}function Or(n,r,t){r=qn(r,t);for(var e=!er(n)&&nn(n),u=(e||n).length,o=0;o=0}var Br=j((function(n,r,t){var e,u;return D(r)?u=r:(r=Nn(r),e=r.slice(0,-1),r=r[r.length-1]),_r(n,(function(n){var o=u;if(!o){if(e&&e.length&&(n=In(n,e)),null==n)return;o=n[r]}return null==o?o:o.apply(n,t)}))}));function Nr(n,r){return _r(n,Rn(r))}function Ir(n,r,t){var e,u,o=-1/0,i=-1/0;if(null==r||"number"==typeof r&&"object"!=typeof n[0]&&null!=n)for(var a=0,f=(n=er(n)?n:jn(n)).length;ao&&(o=e);else r=qn(r,t),jr(n,(function(n,t,e){((u=r(n,t,e))>i||u===-1/0&&o===-1/0)&&(o=n,i=u)}));return o}function Tr(n,r,t){if(null==r||t)return er(n)||(n=jn(n)),n[Wn(n.length-1)];var e=er(n)?En(n):jn(n),u=Y(e);r=Math.max(Math.min(r,u),0);for(var o=u-1,i=0;i1&&(e=Fn(e,r[1])),r=an(n)):(e=qr,r=ur(r,!1,!1),n=Object(n));for(var u=0,o=r.length;u1&&(t=r[1])):(r=_r(ur(r,!1,!1),String),e=function(n,t){return!Er(r,t)}),Ur(n,e,t)}));function zr(n,r,t){return i.call(n,0,Math.max(0,n.length-(null==r||t?1:r)))}function Lr(n,r,t){return null==n||n.length<1?null==r||t?void 0:[]:null==r||t?n[0]:zr(n,n.length-r)}function $r(n,r,t){return i.call(n,null==r||t?1:r)}var Cr=j((function(n,r){return r=ur(r,!0,!0),Sr(n,(function(n){return!Er(r,n)}))})),Kr=j((function(n,r){return Cr(n,r)}));function Jr(n,r,t,e){A(r)||(e=t,t=r,r=!1),null!=t&&(t=qn(t,e));for(var u=[],o=[],i=0,a=Y(n);ir?(e&&(clearTimeout(e),e=null),a=c,i=n.apply(u,o),e||(u=o=null)):e||!1===t.trailing||(e=setTimeout(f,l)),i};return c.cancel=function(){clearTimeout(e),a=0,e=u=o=null},c},debounce:function(n,r,t){var e,u,o,i,a,f=function(){var c=zn()-u;r>c?e=setTimeout(f,r-c):(e=null,t||(i=n.apply(a,o)),e||(o=a=null))},c=j((function(c){return a=this,o=c,u=zn(),e||(e=setTimeout(f,r),t&&(i=n.apply(a,o))),i}));return c.cancel=function(){clearTimeout(e),e=o=a=null},c},wrap:function(n,r){return rr(r,n)},negate:fr,compose:function(){var n=arguments,r=n.length-1;return function(){for(var t=r,e=n[r].apply(this,arguments);t--;)e=n[t].call(this,e);return e}},after:function(n,r){return function(){if(--n<1)return r.apply(this,arguments)}},before:cr,once:lr,findKey:sr,findIndex:vr,findLastIndex:hr,sortedIndex:yr,indexOf:gr,lastIndexOf:br,find:mr,detect:mr,findWhere:function(n,r){return mr(n,Dn(r))},each:jr,forEach:jr,map:_r,collect:_r,reduce:Ar,foldl:Ar,inject:Ar,reduceRight:xr,foldr:xr,filter:Sr,select:Sr,reject:function(n,r,t){return Sr(n,fr(qn(r)),t)},every:Or,all:Or,some:Mr,any:Mr,contains:Er,includes:Er,include:Er,invoke:Br,pluck:Nr,where:function(n,r){return Sr(n,Dn(r))},max:Ir,min:function(n,r,t){var e,u,o=1/0,i=1/0;if(null==r||"number"==typeof r&&"object"!=typeof n[0]&&null!=n)for(var a=0,f=(n=er(n)?n:jn(n)).length;ae||void 0===t)return 1;if(t=0&&t<=m}}function J(n){return function(r){return null==r?void 0:r[n]}}var G=J("byteLength"),H=K(G),Q=/\[object ((I|Ui)nt(8|16|32)|Float(32|64)|Uint8Clamped|Big(I|Ui)nt64)Array\]/;var X=c?function(n){return h?h(n)&&!q(n):H(n)&&Q.test(a.call(n))}:C(!1),Y=J("length");function Z(n,r){r=function(n){for(var r={},t=n.length,e=0;e":">",'"':""","'":"'","`":"`"},Cn=Ln($n),Kn=Ln(_n($n)),Jn=tn.templateSettings={evaluate:/<%([\s\S]+?)%>/g,interpolate:/<%=([\s\S]+?)%>/g,escape:/<%-([\s\S]+?)%>/g},Gn=/(.)^/,Hn={"'":"'","\\":"\\","\r":"r","\n":"n","\u2028":"u2028","\u2029":"u2029"},Qn=/\\|'|\r|\n|\u2028|\u2029/g;function Xn(n){return"\\"+Hn[n]}var Yn=/^\s*(\w|\$)+\s*$/;var Zn=0;function nr(n,r,t,e,u){if(!(e instanceof r))return n.apply(t,u);var o=Mn(n.prototype),i=n.apply(o,u);return _(i)?i:o}var rr=j((function(n,r){var t=rr.placeholder,e=function(){for(var u=0,o=r.length,i=Array(o),a=0;a1)ur(a,r-1,t,e),u=e.length;else for(var f=0,c=a.length;f0&&(t=r.apply(this,arguments)),n<=1&&(r=null),t}}var lr=rr(cr,2);function sr(n,r,t){r=qn(r,t);for(var e,u=nn(n),o=0,i=u.length;o0?0:u-1;o>=0&&o0?a=o>=0?o:Math.max(o+f,a):f=o>=0?Math.min(o+1,f):o+f+1;else if(t&&o&&f)return e[o=t(e,u)]===u?o:-1;if(u!=u)return(o=r(i.call(e,a,f),$))>=0?o+a:-1;for(o=n>0?a:f-1;o>=0&&o0?0:i-1;for(u||(e=r[o?o[a]:a],a+=n);a>=0&&a=3;return r(n,Fn(t,u,4),e,o)}}var Ar=wr(1),xr=wr(-1);function Sr(n,r,t){var e=[];return r=qn(r,t),jr(n,(function(n,t,u){r(n,t,u)&&e.push(n)})),e}function Or(n,r,t){r=qn(r,t);for(var e=!er(n)&&nn(n),u=(e||n).length,o=0;o=0}var Br=j((function(n,r,t){var e,u;return D(r)?u=r:(r=Nn(r),e=r.slice(0,-1),r=r[r.length-1]),_r(n,(function(n){var o=u;if(!o){if(e&&e.length&&(n=In(n,e)),null==n)return;o=n[r]}return null==o?o:o.apply(n,t)}))}));function Nr(n,r){return _r(n,Rn(r))}function Ir(n,r,t){var e,u,o=-1/0,i=-1/0;if(null==r||"number"==typeof r&&"object"!=typeof n[0]&&null!=n)for(var a=0,f=(n=er(n)?n:jn(n)).length;ao&&(o=e);else r=qn(r,t),jr(n,(function(n,t,e){((u=r(n,t,e))>i||u===-1/0&&o===-1/0)&&(o=n,i=u)}));return o}function Tr(n,r,t){if(null==r||t)return er(n)||(n=jn(n)),n[Wn(n.length-1)];var e=er(n)?En(n):jn(n),u=Y(e);r=Math.max(Math.min(r,u),0);for(var o=u-1,i=0;i1&&(e=Fn(e,r[1])),r=an(n)):(e=qr,r=ur(r,!1,!1),n=Object(n));for(var u=0,o=r.length;u1&&(t=r[1])):(r=_r(ur(r,!1,!1),String),e=function(n,t){return!Er(r,t)}),Ur(n,e,t)}));function zr(n,r,t){return i.call(n,0,Math.max(0,n.length-(null==r||t?1:r)))}function Lr(n,r,t){return null==n||n.length<1?null==r||t?void 0:[]:null==r||t?n[0]:zr(n,n.length-r)}function $r(n,r,t){return i.call(n,null==r||t?1:r)}var Cr=j((function(n,r){return r=ur(r,!0,!0),Sr(n,(function(n){return!Er(r,n)}))})),Kr=j((function(n,r){return Cr(n,r)}));function Jr(n,r,t,e){A(r)||(e=t,t=r,r=!1),null!=t&&(t=qn(t,e));for(var u=[],o=[],i=0,a=Y(n);ir?(e&&(clearTimeout(e),e=null),a=c,i=n.apply(u,o),e||(u=o=null)):e||!1===t.trailing||(e=setTimeout(f,l)),i};return c.cancel=function(){clearTimeout(e),a=0,e=u=o=null},c},debounce:function(n,r,t){var e,u,o,i,a,f=function(){var c=zn()-u;r>c?e=setTimeout(f,r-c):(e=null,t||(i=n.apply(a,o)),e||(o=a=null))},c=j((function(c){return a=this,o=c,u=zn(),e||(e=setTimeout(f,r),t&&(i=n.apply(a,o))),i}));return c.cancel=function(){clearTimeout(e),e=o=a=null},c},wrap:function(n,r){return rr(r,n)},negate:fr,compose:function(){var n=arguments,r=n.length-1;return function(){for(var t=r,e=n[r].apply(this,arguments);t--;)e=n[t].call(this,e);return e}},after:function(n,r){return function(){if(--n<1)return r.apply(this,arguments)}},before:cr,once:lr,findKey:sr,findIndex:vr,findLastIndex:hr,sortedIndex:yr,indexOf:gr,lastIndexOf:br,find:mr,detect:mr,findWhere:function(n,r){return mr(n,Dn(r))},each:jr,forEach:jr,map:_r,collect:_r,reduce:Ar,foldl:Ar,inject:Ar,reduceRight:xr,foldr:xr,filter:Sr,select:Sr,reject:function(n,r,t){return Sr(n,fr(qn(r)),t)},every:Or,all:Or,some:Mr,any:Mr,contains:Er,includes:Er,include:Er,invoke:Br,pluck:Nr,where:function(n,r){return Sr(n,Dn(r))},max:Ir,min:function(n,r,t){var e,u,o=1/0,i=1/0;if(null==r||"number"==typeof r&&"object"!=typeof n[0]&&null!=n)for(var a=0,f=(n=er(n)?n:jn(n)).length;ae||void 0===t)return 1;if(t - + - - + + - +
- +
- +

Index

E | M - +

E

@@ -67,7 +67,7 @@

M

- + - - + + - \ No newline at end of file + diff --git a/docs/_build/html/index.html b/docs/_build/html/index.html index 1bef618..eba98a3 100644 --- a/docs/_build/html/index.html +++ b/docs/_build/html/index.html @@ -16,22 +16,22 @@ - + - - + + - +
- +
- +

Welcome to earthspy’s documentation!

@@ -52,7 +52,7 @@

Indices and tables @@ -104,18 +104,18 @@

Quick search

- - + + - \ No newline at end of file + diff --git a/docs/_build/html/search.html b/docs/_build/html/search.html index 6ccb10a..a5d9bab 100644 --- a/docs/_build/html/search.html +++ b/docs/_build/html/search.html @@ -8,7 +8,7 @@ Search — earthspy v0.1.0 documentation - + @@ -18,26 +18,26 @@ - - + + - - + + - +
- +
- +

Search

- + - - + +

Searching for multiple words only shows matches that contain all words.

- - + +
- - - + + +
- +
- +
- +
- - + + - \ No newline at end of file + diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index 7263ea1..4b1bedd 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({docnames:["api","generated/earthspy","index"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,sphinx:56},filenames:["api.rst","generated/earthspy.rst","index.rst"],objects:{"":[[1,0,0,"-","earthspy"]]},objnames:{"0":["py","module","Python module"]},objtypes:{"0":"py:module"},terms:{api:2,index:2,modul:2,page:2,search:2},titles:["API","earthspy","Welcome to earthspy\u2019s documentation!"],titleterms:{api:0,content:2,document:2,earthspi:[1,2],indic:2,s:2,tabl:2,welcom:2}}) \ No newline at end of file +Search.setIndex({docnames:["api","generated/earthspy","index"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,sphinx:56},filenames:["api.rst","generated/earthspy.rst","index.rst"],objects:{"":[[1,0,0,"-","earthspy"]]},objnames:{"0":["py","module","Python module"]},objtypes:{"0":"py:module"},terms:{api:2,index:2,modul:2,page:2,search:2},titles:["API","earthspy","Welcome to earthspy\u2019s documentation!"],titleterms:{api:0,content:2,document:2,earthspi:[1,2],indic:2,s:2,tabl:2,welcom:2}}) diff --git a/docs/api.rst b/docs/api.rst index ced4568..fa15c80 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -9,7 +9,7 @@ API :recursive: earthspy - + .. automodule earthspy diff --git a/docs/generated/earthspy.rst b/docs/generated/earthspy.rst index 9e1db12..efebd77 100644 --- a/docs/generated/earthspy.rst +++ b/docs/generated/earthspy.rst @@ -2,22 +2,3 @@ ======== .. automodule:: earthspy - - - - - - - - - - - - - - - - - - - diff --git a/docs/index.rst b/docs/index.rst index 4fbde63..8176788 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ .. earthspy documentation master file, created by sphinx-quickstart on Mon May 2 20:38:22 2022. You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. + contain the root ``toctree`` directive. Welcome to earthspy's documentation! ==================================== diff --git a/earthspy/__init__.py b/earthspy/__init__.py index 8b13789..e69de29 100644 --- a/earthspy/__init__.py +++ b/earthspy/__init__.py @@ -1 +0,0 @@ - diff --git a/earthspy/earthspy.py b/earthspy/earthspy.py index 0296aba..b0d9a22 100644 --- a/earthspy/earthspy.py +++ b/earthspy/earthspy.py @@ -1,45 +1,46 @@ -# -*- coding: utf-8 -*- """ @author: Adrien Wehrlé, EO-IO, University of Zurich, Switzerland """ -from collections import Counter -from datetime import datetime, timedelta import glob import json +import os +import shutil +import tarfile +import time +from collections import Counter +from datetime import datetime, timedelta from multiprocessing import cpu_count +from pathlib import Path +from typing import Tuple, Union + import numpy as np import objectpath -import os import pandas as pd -from pathlib import Path import rasterio -from rasterio.merge import merge import requests import sentinelhub as shb -import shutil -import tarfile -import time -from typing import Union, Tuple import validators +from rasterio.merge import merge class EarthSpy: - """Monitor and study any place on Earth and in Near Real-Time (NRT) using the - SentinelHub services. + """Monitor and study any place on Earth and in Near Real-Time + (NRT) using the SentinelHub services. + """ def __init__(self, CLIENT_credentials_file: str) -> None: """ :param CLIENT_credentials_file: full path to file containing credentials - with User's OAuth client ID (1st row) secrect (2nd row). + with User's OAuth client ID (1st row) secret (2nd row). :type CLIENT_credentials_file: str """ # read credentials stored in text file - with open(CLIENT_credentials_file, "r") as file: + with open(CLIENT_credentials_file) as file: credentials = file.read().splitlines() # extract credentials from lines @@ -49,8 +50,6 @@ def __init__(self, CLIENT_credentials_file: str) -> None: # setup connection self.configure_connection() - return None - def configure_connection(self) -> shb.SHConfig: """Build a shb configuration class for the connection to Sentinel Hub services. @@ -225,7 +224,7 @@ def get_available_data(self) -> list: ] # some data sets require a difference service_url, test search_iterator - # and update service_url if dowload failed + # and update service_url if download failed try: # store metadata of available scenes self.metadata = {} @@ -233,8 +232,8 @@ def get_available_data(self) -> list: iterator_list = list(iterator) if len(iterator_list) > 0: date = iterator_list[0]["properties"]["datetime"].split("T")[0] - self.metadata[date] = iterator_list - + self.metadata[date] = iterator_list + except shb.exceptions.DownloadFailedException: # set specific base URL of deployment self.catalog_config.sh_base_url = shb.DataCollection[ @@ -260,8 +259,8 @@ def get_available_data(self) -> list: iterator_list = list(iterator) if len(iterator_list) > 0: date = iterator_list[0]["properties"]["datetime"].split("T")[0] - self.metadata[date] = iterator_list - + self.metadata[date] = iterator_list + # create date +-1 hour around acquisition time time_difference = timedelta(hours=1) @@ -320,7 +319,7 @@ def get_raw_data_collection_resolution(self) -> int: return self.raw_data_collection_resolution def set_number_of_cores(self, nb_cores) -> int: - """Set number of cores if not specificed by user. + """Set number of cores if not specified by user. :return: Number of cores to use in multithreading. :rtype: int @@ -353,7 +352,8 @@ def get_date_range( :rtype: pd.core.indexes.datetimes.DatetimeIndex """ - # if an integer, create a datetimeIndex with the number of days from present date + # if an integer, create a datetimeIndex with the number of days + # from present date if isinstance(time_interval, int): # keep time_interval positive if time_interval < 0: @@ -403,12 +403,12 @@ def get_bounding_box(self, bounding_box: Union[list, str]) -> shb.geometry.BBox: :rtype: sentinelhub.geometry.BBox """ - # if a list, set Sentinel Hub BBox wit bounding_box + # if a list, set Sentinel Hub BBox with bounding_box if isinstance(bounding_box, list): # create Sentinel Hub BBox self.bounding_box = shb.BBox(bbox=bounding_box, crs=shb.CRS.WGS84) - # cant guess name, so set to None + # can't guess name, so set to None self.bounding_box_name = None # if a string, extract bounding box from corresponding GEOJSON file @@ -472,7 +472,7 @@ def get_store_folder(self, store_folder: Union[str, None]) -> str: if self.algorithm: store_folder += f"{os.sep}{self.algorithm}" - # create subfolder if doesnt exist + # create subfolder if doesn't exist if not os.path.exists(store_folder): os.makedirs(store_folder) @@ -482,7 +482,7 @@ def get_store_folder(self, store_folder: Union[str, None]) -> str: return self.store_folder def convert_bounding_box_coordinates(self) -> Tuple[shb.geometry.BBox, list]: - """Convert bounding boxe coordinates to a Geodetic Parameter Dataset (EPSG) in + """Convert bounding box coordinates to a Geodetic Parameter Dataset (EPSG) in meter unit, default to EPSG:3413 (NSIDC Sea Ice Polar Stereographic North). @@ -568,7 +568,7 @@ def set_correct_resolution(self) -> int: elif not self.resolution and self.download_mode == "SM": self.resolution = self.raw_data_collection_resolution - # resolution cant be higher than max resolution in D download mode + # resolution can't be higher than max resolution in D download mode if self.download_mode == "D" and self.resolution < max_resolution: self.resolution = max_resolution if self.verbose: @@ -618,7 +618,8 @@ def get_optimal_box_split(self) -> Tuple[int, int]: boxes_pixels_x = (dx / trial_split_boxes) / self.resolution boxes_pixels_y = (dy / trial_split_boxes) / self.resolution - # get minimum number of split boxes needed to stay below Sentinel Hub max dimensions + # get minimum number of split boxes needed to stay below Sentinel Hub + # max dimensions min_nb_boxes_x = int(trial_split_boxes[np.where(boxes_pixels_x <= 2500)[0][0]]) min_nb_boxes_y = int(trial_split_boxes[np.where(boxes_pixels_y <= 2500)[0][0]]) @@ -681,7 +682,7 @@ def set_split_boxes_ids(self) -> dict: """ # store split boxes ids in dict - self.split_boxes_ids = {i: sb for i, sb in enumerate(self.split_boxes)} + self.split_boxes_ids = dict(zip(range(len(self.split_boxes)), self.split_boxes)) return self.split_boxes_ids @@ -1071,7 +1072,7 @@ def merge_rasters(self) -> None: "transform": output_transform, } ) - id_dict = {k:self.metadata[date][0][k] for k in ["id"]} + id_dict = {k: self.metadata[date][0][k] for k in ["id"]} # write mosaic with rasterio.open(date_output_filename, "w", **output_meta) as dst: dst.write(mosaic) diff --git a/earthspy/operational/earthspy_NRT.py b/earthspy/operational/earthspy_NRT.py index d24644e..ab859df 100644 --- a/earthspy/operational/earthspy_NRT.py +++ b/earthspy/operational/earthspy_NRT.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ @author: Adrien Wehrlé, EO-IO, University of Zurich, Switzerland diff --git a/setup.py b/setup.py index 8bf1ba9..6068493 100644 --- a/setup.py +++ b/setup.py @@ -1,2 +1,3 @@ from setuptools import setup + setup() diff --git a/tests/__init__.py b/tests/__init__.py index 8b13789..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +0,0 @@ - diff --git a/tests/test_earthspy.py b/tests/test_earthspy.py index ffe06c7..f0d3da6 100644 --- a/tests/test_earthspy.py +++ b/tests/test_earthspy.py @@ -1,18 +1,19 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ @author: Adrien Wehrlé, EO-IO, University of Zurich, Switzerland """ -import earthspy.earthspy as es -import numpy as np import os + +import numpy as np import pandas as pd import requests import sentinelhub as shb +import earthspy.earthspy as es + class TestEarthspy: # create local variables from environment secrets for convenience @@ -36,7 +37,8 @@ class TestEarthspy: // Set gain for visualisation let gain = 2.5; // Return RGB - return [sample.B04 * gain, sample.B03 * gain, sample.B02 * gain, sample.dataMask]; + return [sample.B04 * gain, sample.B03 * gain, sample.B02 * gain, + sample.dataMask]; } """ @@ -100,7 +102,7 @@ def test_init(self) -> None: assert self.t1.config.sh_client_secret == os.environ["SH_CLIENT_SECRET"] def test_set_query_parameters(self) -> None: - """Test direct attribute assignement.""" + """Test direct attribute assignment.""" # check if attributes were set accordingly assert self.t1.download_mode is not None @@ -249,9 +251,9 @@ def test_set_correct_resolution(self) -> None: r2 = self.t3.set_correct_resolution() # check that query resolution was set correctly - assert r1 == 10 + assert r2 == 11 # check that download mode was set correctly - assert isinstance(self.t1.download_mode, str) + assert isinstance(self.t3.download_mode, str) def test_list_requests(self) -> None: """Test request listing""" @@ -299,9 +301,9 @@ def test_set_split_boxes_ids(self) -> None: """Test split box ID generation""" sbi1 = self.t1.set_split_boxes_ids() - # check that split box ids were saved in dictionnary + # check that split box ids were saved in dictionary assert isinstance(sbi1, dict) - # check that dictionnary has the right shape + # check that dictionary has the right shape assert len(sbi1) == 4 def test_get_evaluation_script(self) -> None: From 68437b46abc1f01e3189c66d6ac46a45d2744546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Wehrl=C3=A9?= Date: Sun, 1 Dec 2024 14:40:48 +0100 Subject: [PATCH 06/13] Add CONTRIBUTING (#109) * add isort badge * bix isort badge * bix isort badge * bix isort badge * refix badge * add hyperlink to badge * add hyperlink to badge * add hyperlink to badge * skeleton of CONTRIBUTING * fix emojis * fix precommit * add precommit badge * fix workflow * add pre-commit hook to convert yml * fix install of dev requirements * fix contributing headers --- .../scripts/generate_pip_deps_from_conda.py | 152 ++++++++++++++++++ .github/workflows/main.yml | 4 +- .pre-commit-config.yaml | 14 ++ CONTRIBUTING.org | 131 +++++++++++++++ README.org | 3 + dev-environment.yml | 22 +++ dev-requirements.txt => requirements-dev.txt | 10 +- 7 files changed, 331 insertions(+), 5 deletions(-) create mode 100755 .github/scripts/generate_pip_deps_from_conda.py create mode 100644 CONTRIBUTING.org create mode 100644 dev-environment.yml rename dev-requirements.txt => requirements-dev.txt (50%) diff --git a/.github/scripts/generate_pip_deps_from_conda.py b/.github/scripts/generate_pip_deps_from_conda.py new file mode 100755 index 0000000..ac74bd5 --- /dev/null +++ b/.github/scripts/generate_pip_deps_from_conda.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +""" +(Copied from pandas: +https://github.com/pandas-dev/pandas/blob/main/scripts/generate_pip_deps_from_conda.py) +Convert the conda environment.yml to the pip requirements-dev.txt, or +check that they have the same packages (for the CI) + +Usage: + + Generate `requirements-dev.txt` + $ python scripts/generate_pip_deps_from_conda.py + + Compare and fail (exit status != 0) if `requirements-dev.txt` has not been + generated with this script: + $ python scripts/generate_pip_deps_from_conda.py --compare + +""" +import argparse +import pathlib +import re +import sys + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib +import yaml + +EXCLUDE = {"python"} +REMAP_VERSION = {"tzdata": "2022.1"} +RENAME = {} + + +def conda_package_to_pip(package: str): + """ + Convert a conda package to its pip equivalent. + + In most cases they are the same, those are the exceptions: + - Packages that should be excluded (in `EXCLUDE`) + - Packages that should be renamed (in `RENAME`) + - A package requiring a specific version, in conda is defined with a single + equal (e.g. ``pandas=1.0``) and in pip with two (e.g. ``pandas==1.0``) + """ + package = re.sub("(?<=[^<>])=", "==", package).strip() + print(package) + + for compare in ("<=", ">=", "=="): + if compare in package: + pkg, version = package.split(compare) + if pkg in EXCLUDE: + return + if pkg in REMAP_VERSION: + return "".join((pkg, compare, REMAP_VERSION[pkg])) + if pkg in RENAME: + return "".join((RENAME[pkg], compare, version)) + + if package in EXCLUDE: + return + + if package in RENAME: + return RENAME[package] + + return package + + +def generate_pip_from_conda( + conda_path: pathlib.Path, pip_path: pathlib.Path, compare: bool = False +) -> bool: + """ + Generate the pip dependencies file from the conda file, or compare that + they are synchronized (``compare=True``). + + Parameters + ---------- + conda_path : pathlib.Path + Path to the conda file with dependencies (e.g. `environment.yml`). + pip_path : pathlib.Path + Path to the pip file with dependencies (e.g. `requirements-dev.txt`). + compare : bool, default False + Whether to generate the pip file (``False``) or to compare if the + pip file has been generated with this script and the last version + of the conda file (``True``). + + Returns + ------- + bool + True if the comparison fails, False otherwise + """ + with conda_path.open() as file: + deps = yaml.safe_load(file)["dependencies"] + + pip_deps = [] + for dep in deps: + if isinstance(dep, str): + conda_dep = conda_package_to_pip(dep) + if conda_dep: + pip_deps.append(conda_dep) + elif isinstance(dep, dict) and len(dep) == 1 and "pip" in dep: + pip_deps.extend(dep["pip"]) + else: + raise ValueError(f"Unexpected dependency {dep}") + + header = ( + f"# This file is auto-generated from {conda_path.name}, do not modify.\n" + "# See that file for comments about the need/usage of each dependency.\n\n" + ) + pip_content = header + "\n".join(pip_deps) + "\n" + + # Add setuptools to requirements-dev.txt + + # with open(pathlib.Path(conda_path.parent, "pyproject.toml"), "rb") as fd: + # meta = tomllib.load(fd) + # for requirement in meta["build-system"]["requires"]: + # if "setuptools" in requirement: + # pip_content += requirement + # pip_content += "\n" + + if compare: + with pip_path.open() as file: + return pip_content != file.read() + + with pip_path.open("w") as file: + file.write(pip_content) + return False + + +if __name__ == "__main__": + argparser = argparse.ArgumentParser( + description="convert (or compare) conda file to pip" + ) + argparser.add_argument( + "--compare", + action="store_true", + help="compare whether the two files are equivalent", + ) + args = argparser.parse_args() + + conda_fname = "dev-environment.yml" + pip_fname = "requirements-dev.txt" + repo_path = pathlib.Path(__file__).parent.parent.parent.absolute() + res = generate_pip_from_conda( + pathlib.Path(repo_path, conda_fname), + pathlib.Path(repo_path, pip_fname), + compare=args.compare, + ) + if res: + msg = ( + f"`{pip_fname}` has to be generated with `{__file__}` after " + f"`{conda_fname}` is modified.\n" + ) + sys.stderr.write(msg) + sys.exit(res) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 72237e3..729f4bf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -63,7 +63,7 @@ jobs: - name: Install dev requirements shell: bash -l {0} run: | - pip install -r dev-requirements.txt + pip install -r requirements-dev.txt - name: Run Snyk to check for vulnerabilities uses: snyk/actions/python@master @@ -71,4 +71,4 @@ jobs: env: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} with: - args: --file=dev-requirements.txt --package-manager=pip --severity-threshold=high --skip-unresolved=true + args: --file=requirements-dev.txt --package-manager=pip --severity-threshold=high --skip-unresolved=true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa9d759..4848f16 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -96,3 +96,17 @@ repos: # Enforce the use of type annotations instead of # docstring type comments - id: python-use-type-annotations + + + - repo: local + hooks: + # Generate pip's dev-requirements.txt from conda's + # dev-environment.yml to run snyk (snyk doesn't currently + # support conda) + - id: pip-to-conda + name: Generate pip dependency from conda + language: python + entry: .github/scripts/generate_pip_deps_from_conda.py + files: ^(dev-environment.yml|requirements-dev.txt)$ + pass_filenames: false + additional_dependencies: [tomli, pyyaml] diff --git a/CONTRIBUTING.org b/CONTRIBUTING.org new file mode 100644 index 0000000..d2cde9f --- /dev/null +++ b/CONTRIBUTING.org @@ -0,0 +1,131 @@ +* Introduction + +First off, thank you for considering contributing to =earthspy=! 😊 + +Following the guidelines presented here helps to communicate that you +respect the efforts the developers managing and developing this open +source project have put to foster community development. In return, +they should reciprocate that respect in addressing your issue, +assessing changes, and helping you finalize your pull requests. 🌻 + +Any kind of contribution is more than welcome. Improving +documentation, bug triaging, implementing new features or writing +tutorials are all examples of helpful contributions. 🚀 + +* Making a contribution + +** Overview + + 1. Fork =AdrienWehrle/earthspy= and clone your fork repository locally. + 2. Set up the development environment (section below). + 3. Create a branch for the new feature or bug fix. + 4. Make your changes, and add or modify related tests in =tests/=. + 5. Commit, making sure to run pre-commit separately if not installed as git hook. + 6. Push to your fork. + 7. Open a pull request from GitHub to discuss and eventually merge. + + +** Step by step + +*** Setup + +Clone the git repository and create a [[https://docs.conda.io/projects/conda/en/latest/index.html][conda]] or [[https://mamba.readthedocs.io/en/latest/index.html][mamba]] environment. If +you use mamba (mamba is great), just replace =conda= for =mamba= +below. Steps below are similar to those presented in the [[https://github.com/AdrienWehrle/earthspy/tree/main][README]] except +=dev-environment.yml= is used here instead of =environment.yml= as it +contains necessary dependencies for continuous integration and +continuous development. + +#+begin_src shell :results verbatim + +# clone repository +git clone git@github.com:AdrienWehrle/earthspy.git + +# move into earthspy directory +cd earthspy + +# create conda environment +conda env create -f dev-environment.yml + +# activate conda environment +conda activate earthspy-dev + +# install earthspy +pip install -e . + +#+end_src + +*** Tests + +If your PR targets the implementation of a new feature or the +improvement of existing ones, please add at least one test per feature +(in the associated =tests/test_earthspy.py= file) and include them in +your PR, using =pytests= (see existing tests for examples). + +To run the entire test suite, run pytest in the current directory: + +#+begin_src shell :results verbatim +pytest +#+end_src + + +*** Formatting and linting + +Before you make your commit, please install =pre-commit= (see +[[https://pre-commit.com/][pre-commit documentation]]), which will use =.pre-commit-config.yaml= to +verify spelling errors, import sorting, type checking, formatting and +linting. + +You can then run =pre-commit= manually: + +#+begin_src shell :results verbatim +pre-commit run --all-files +#+end_src + +In your PR, make sure to note if all =pre-commit= checks passed or +not. Other developers will gladly help you getting rid of remaining +red signs in your terminal if you feel stuck! 🌿 + +Optionally, =pre-commit= can also be installed as a git hook to +automatically ensure checks have to pass before committing. + +Once all =pre-commit= checks have passed (or you need help), +please commit your changes with short and clear messages. + + +* Reporting a bug + +** Security disclosures + +If you find a security vulnerability, please do not open an +issue. Email adrien.wehrle@hotmail.fr instead. + +In order to determine whether you are dealing with a security issue, +ask yourself these two questions: + +- Can I access something that's not mine, or something I shouldn't + have access to? +- Can I disable something for other people? + +** How to file a bug report + +When filing an issue at +[[earthspy/issues][https://github.com/AdrienWehrle/earthspy/issues]], make sure to answer +these five questions: + +- Have you installed =earthspy= using =environment.yml=? If not, what + version of Python and dependencies are you using? +- What operating system and processor architecture are you using? +- What did you do? +- What did you expect to see? +- What did you see instead? + + +* Rights + +The license (see =LICENSE=) applies to all contributions. + +* Credits + +Part of this document is based on [[https://github.com/nayafia][nayafia]]'s [[https://github.com/nayafia/contributing-template/blob/master/CONTRIBUTING-template.md][CONTRIBUTING]] template and +[[https://github.com/GlacioHack/geoutils/blob/main/CONTRIBUTING.md][geoutils CONTRIBUTING]]. diff --git a/README.org b/README.org index 32a9000..9fff53a 100644 --- a/README.org +++ b/README.org @@ -1,8 +1,11 @@ [[https://www.gnu.org/licenses/gpl-3.0][https://img.shields.io/badge/License-GPLv3-blue.svg]] [[https://github.com/AdrienWehrle/earthspy/actions][file:https://github.com/AdrienWehrle/earthspy/workflows/CI/badge.svg]] [[https://github.com/AdrienWehrle/earthspy/actions/workflows/codeql.yml][https://github.com/AdrienWehrle/earthspy/actions/workflows/codeql.yml/badge.svg]] +[[https://pre-commit.com/][https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit.svg]] [[https://codecov.io/gh/AdrienWehrle/earthspy][https://codecov.io/gh/AdrienWehrle/earthspy/branch/main/graph/badge.svg]] [[https://github.com/psf/black][https://img.shields.io/badge/code%20style-black-000000.svg]] +[[https://pycqa.github.io/isort/][https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336.svg]] + * earthspy 🛰️ :earth_africa: :earth_americas: :earth_asia: :moon: diff --git a/dev-environment.yml b/dev-environment.yml new file mode 100644 index 0000000..52e488e --- /dev/null +++ b/dev-environment.yml @@ -0,0 +1,22 @@ +name: earthspy-dev +channels: + - conda-forge +dependencies: + - python>=3.9 + - numpy + - objectpath + - pandas + - rasterio + - validators + - sentinelhub >= 3.4 + - pip + - flake8 + - pytest + - pytest-lazy-fixture + - sphinx + - sphinx_rtd_theme + - pylint + - sphinxcontrib_programoutput + - sphinx_autodoc_typehints + - pre-commit + - sphinx-gallery diff --git a/dev-requirements.txt b/requirements-dev.txt similarity index 50% rename from dev-requirements.txt rename to requirements-dev.txt index 8d3ac43..23edc01 100644 --- a/dev-requirements.txt +++ b/requirements-dev.txt @@ -1,8 +1,13 @@ +# This file is auto-generated from dev-environment.yml, do not modify. +# See that file for comments about the need/usage of each dependency. + numpy +objectpath pandas -validators rasterio -sentinelhub +validators +sentinelhub >= 3.4 +pip flake8 pytest pytest-lazy-fixture @@ -13,4 +18,3 @@ sphinxcontrib_programoutput sphinx_autodoc_typehints pre-commit sphinx-gallery - From 9e3e1887cdc8bb8a6cc382481f5786e0ce85927f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Wehrl=C3=A9?= Date: Sun, 1 Dec 2024 14:47:46 +0100 Subject: [PATCH 07/13] Add dependabot (#110) --- .github/dependabot.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100755 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100755 index 0000000..ac27a84 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" From 649bfef69c02d71a3df0cbca215bab51e3263c9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Wehrl=C3=A9?= Date: Sun, 1 Dec 2024 15:56:21 +0100 Subject: [PATCH 08/13] Fix doc requirements (#114) * fix docs * fix dependencies in dev-environment.yml * fix environment * fix environment * switch back to pip for doc * switch back to pip for doc * fix requirements --- .readthedocs.yml | 12 +++++++++--- dev-environment.yml | 6 ++++-- environment.yml | 1 + requirements-dev.txt | 4 ++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index f0b29af..fb19340 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,17 +5,23 @@ # Required version: 2 +# Set the OS, Python version and other tools you might need + +build: + os: ubuntu-22.04 + tools: + python: "3.12" + # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/conf.py fail_on_warning: false # Optionally build your docs in additional formats such as PDF and ePub -formats: [] +formats: [pdf] python: - version: 3.7 install: - - requirements: dev-requirements.txt + - requirements: requirements-dev.txt - method: pip path: . diff --git a/dev-environment.yml b/dev-environment.yml index 52e488e..18c217f 100644 --- a/dev-environment.yml +++ b/dev-environment.yml @@ -1,6 +1,8 @@ name: earthspy-dev channels: + - defaults - conda-forge + - bioconda dependencies: - python>=3.9 - numpy @@ -16,7 +18,7 @@ dependencies: - sphinx - sphinx_rtd_theme - pylint - - sphinxcontrib_programoutput - - sphinx_autodoc_typehints + - sphinxcontrib-programoutput + - sphinx-autodoc-typehints - pre-commit - sphinx-gallery diff --git a/environment.yml b/environment.yml index 65dd7cd..56402e9 100644 --- a/environment.yml +++ b/environment.yml @@ -1,5 +1,6 @@ name: earthspy channels: + - defaults - conda-forge dependencies: - python>=3.9 diff --git a/requirements-dev.txt b/requirements-dev.txt index 23edc01..2d62fed 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -14,7 +14,7 @@ pytest-lazy-fixture sphinx sphinx_rtd_theme pylint -sphinxcontrib_programoutput -sphinx_autodoc_typehints +sphinxcontrib-programoutput +sphinx-autodoc-typehints pre-commit sphinx-gallery From 13f786c39d1c43706f64f16f9ef1a1d22c840447 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Dec 2024 16:14:56 +0100 Subject: [PATCH 09/13] Bump github/super-linter from 4.9.0 to 5.0.0 (#113) Bumps [github/super-linter](https://github.com/github/super-linter) from 4.9.0 to 5.0.0. - [Release notes](https://github.com/github/super-linter/releases) - [Changelog](https://github.com/github/super-linter/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/super-linter/compare/v4.9.0...v5.0.0) --- updated-dependencies: - dependency-name: github/super-linter dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 729f4bf..bf53d1b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,7 +36,7 @@ jobs: fetch-depth: 0 - name: Lint code base - uses: github/super-linter/slim@v4.9.0 + uses: github/super-linter/slim@v5.0.0 env: VALIDATE_ALL_CODEBASE: true VALIDATE_JSON: true From c6adb35721e3df683d30f438b2add8eefde96f43 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Dec 2024 16:15:10 +0100 Subject: [PATCH 10/13] Bump actions/checkout from 3 to 4 (#112) Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 2da8421..32fbd0f 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Initialize CodeQL uses: github/codeql-action/init@v2 From e6b9a14763b71880a00879c301f617109b0e09d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Dec 2024 16:15:23 +0100 Subject: [PATCH 11/13] Bump github/codeql-action from 2 to 3 (#111) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 32fbd0f..483a9d9 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -27,15 +27,15 @@ jobs: uses: actions/checkout@v4 - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} queries: +security-and-quality - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: category: "/language:${{ matrix.language }}" From d01f7d5038b576ded107e5e3ff7b05ff9f67628d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Wehrl=C3=A9?= Date: Sun, 1 Dec 2024 23:26:17 +0100 Subject: [PATCH 12/13] Translate README and CONTRIBUTING to markdown (#116) * converting README org to md * fixing missing translation * fixing badges * fixing badges * fix table of content * add contributing * translate CONTRIBUTING * fix table of contents --- CONTRIBUTING.md | 140 +++++++++++++++++++++++++++++++++++++++ CONTRIBUTING.org | 131 ------------------------------------- README.md | 167 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 307 insertions(+), 131 deletions(-) create mode 100644 CONTRIBUTING.md delete mode 100644 CONTRIBUTING.org create mode 100644 README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..21e22a4 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,140 @@ +# Introduction + +First off, thank you for considering contributing to `earthspy`! 😊 + +Following the guidelines presented here helps to communicate that you +respect the efforts the developers managing and developing this open +source project have put to foster community development. In return, +they should reciprocate that respect in addressing your issue, +assessing changes, and helping you finalize your pull requests. 🌻 + +Any kind of contribution is more than welcome. Improving +documentation, bug triaging, implementing new features or writing +tutorials are all examples of helpful contributions. 🚀 + +# Making a contribution + +## Overview + + 1. Fork `AdrienWehrle/earthspy` and clone your fork repository locally. + 2. Set up the development environment (section below). + 3. Create a branch for the new feature or bug fix. + 4. Make your changes, and add or modify related tests in + [`tests/`](https://github.com/AdrienWehrle/earthspy/tree/main/tests). + 5. Commit, making sure to run pre-commit separately if not installed as git hook. + 6. Push to your fork. + 7. Open a pull request from GitHub to discuss and eventually merge. + + +## Step by step + +### Setup + +Clone the git repository and create a +[conda](https://docs.conda.io/projects/conda/en/latest/index.html) or +[mamba](https://mamba.readthedocs.io/en/latest/index.html) +environment. If you use mamba (mamba is great), just replace `conda` +for `mamba` below. Steps below are similar to those presented in the +[README](https://github.com/AdrienWehrle/earthspy/tree/main) except +[`dev-environment.yml`](https://github.com/AdrienWehrle/earthspy/blob/main/dev-environment.yml) +is used here instead of +[`environment.yml`](https://github.com/AdrienWehrle/earthspy/blob/main/dev-environment.yml) +as it contains necessary dependencies for continuous integration and +continuous development. + +```bash + +# clone repository +git clone git@github.com:AdrienWehrle/earthspy.git + +# move into earthspy directory +cd earthspy + +# create conda environment +conda env create -f dev-environment.yml + +# activate conda environment +conda activate earthspy-dev + +# install earthspy +pip install -e . +``` + +### Tests + +If your PR targets the implementation of a new feature or the +improvement of existing ones, please add at least one test per feature +(in the associated +[`tests/test_earthspy.py`](https://github.com/AdrienWehrle/earthspy/blob/main/LICENSE) +file) and include them in your PR, using `pytest` (see existing tests +for examples). + +To run the entire test suite, run pytest in the current directory: + +```bash +pytest +``` + + +### Formatting and linting + +Before you make your commit, please install `pre-commit` (see +[pre-commit documentation](https://pre-commit.com/)), which will use +`.pre-commit-config.yaml` to verify spelling errors, import sorting, +type checking, formatting and linting. + +You can then run `pre-commit` manually: + +```bash +pre-commit run --all-files +``` + +In your PR, make sure to note if all `pre-commit` checks passed or +not. Other developers will gladly help you getting rid of remaining +red signs in your terminal if you feel stuck! 🌿 + +Optionally, `pre-commit` can also be installed as a git hook to +automatically ensure checks have to pass before committing. + +Once all `pre-commit` checks have passed (or you need help), +please commit your changes with short and clear messages. + + +# Reporting a bug + +## Security disclosures + +If you find a security vulnerability, please do not open an +issue. Email adrien.wehrle@hotmail.fr instead. + +In order to determine whether you are dealing with a security issue, +ask yourself these two questions: + +- Can I access something that's not mine, or something I shouldn't + have access to? +- Can I disable something for other people? + +## How to file a bug report + +When filing an issue at +[earthspy/issues](https://github.com/AdrienWehrle/earthspy/issues), +make sure to answer these five questions: + +- Have you installed `earthspy` using + [`environment.yml`](https://github.com/AdrienWehrle/earthspy/blob/main/LICENSE)? + If not, what version of Python and dependencies are you using? +- What operating system and processor architecture are you using? +- What did you do? +- What did you expect to see? +- What did you see instead? + + +# Rights + +The license (see +[`LICENSE`](https://github.com/AdrienWehrle/earthspy/blob/main/LICENSE)) +applies to all contributions. + +# Credits + +Part of this document is based on [nayafia](https://github.com/nayafia)'s [CONTRIBUTING](https://github.com/nayafia) template and [geoutils CONTRIBUTING](https://github.com/GlacioHack/geoutils/blob/main/CONTRIBUTING.md). diff --git a/CONTRIBUTING.org b/CONTRIBUTING.org deleted file mode 100644 index d2cde9f..0000000 --- a/CONTRIBUTING.org +++ /dev/null @@ -1,131 +0,0 @@ -* Introduction - -First off, thank you for considering contributing to =earthspy=! 😊 - -Following the guidelines presented here helps to communicate that you -respect the efforts the developers managing and developing this open -source project have put to foster community development. In return, -they should reciprocate that respect in addressing your issue, -assessing changes, and helping you finalize your pull requests. 🌻 - -Any kind of contribution is more than welcome. Improving -documentation, bug triaging, implementing new features or writing -tutorials are all examples of helpful contributions. 🚀 - -* Making a contribution - -** Overview - - 1. Fork =AdrienWehrle/earthspy= and clone your fork repository locally. - 2. Set up the development environment (section below). - 3. Create a branch for the new feature or bug fix. - 4. Make your changes, and add or modify related tests in =tests/=. - 5. Commit, making sure to run pre-commit separately if not installed as git hook. - 6. Push to your fork. - 7. Open a pull request from GitHub to discuss and eventually merge. - - -** Step by step - -*** Setup - -Clone the git repository and create a [[https://docs.conda.io/projects/conda/en/latest/index.html][conda]] or [[https://mamba.readthedocs.io/en/latest/index.html][mamba]] environment. If -you use mamba (mamba is great), just replace =conda= for =mamba= -below. Steps below are similar to those presented in the [[https://github.com/AdrienWehrle/earthspy/tree/main][README]] except -=dev-environment.yml= is used here instead of =environment.yml= as it -contains necessary dependencies for continuous integration and -continuous development. - -#+begin_src shell :results verbatim - -# clone repository -git clone git@github.com:AdrienWehrle/earthspy.git - -# move into earthspy directory -cd earthspy - -# create conda environment -conda env create -f dev-environment.yml - -# activate conda environment -conda activate earthspy-dev - -# install earthspy -pip install -e . - -#+end_src - -*** Tests - -If your PR targets the implementation of a new feature or the -improvement of existing ones, please add at least one test per feature -(in the associated =tests/test_earthspy.py= file) and include them in -your PR, using =pytests= (see existing tests for examples). - -To run the entire test suite, run pytest in the current directory: - -#+begin_src shell :results verbatim -pytest -#+end_src - - -*** Formatting and linting - -Before you make your commit, please install =pre-commit= (see -[[https://pre-commit.com/][pre-commit documentation]]), which will use =.pre-commit-config.yaml= to -verify spelling errors, import sorting, type checking, formatting and -linting. - -You can then run =pre-commit= manually: - -#+begin_src shell :results verbatim -pre-commit run --all-files -#+end_src - -In your PR, make sure to note if all =pre-commit= checks passed or -not. Other developers will gladly help you getting rid of remaining -red signs in your terminal if you feel stuck! 🌿 - -Optionally, =pre-commit= can also be installed as a git hook to -automatically ensure checks have to pass before committing. - -Once all =pre-commit= checks have passed (or you need help), -please commit your changes with short and clear messages. - - -* Reporting a bug - -** Security disclosures - -If you find a security vulnerability, please do not open an -issue. Email adrien.wehrle@hotmail.fr instead. - -In order to determine whether you are dealing with a security issue, -ask yourself these two questions: - -- Can I access something that's not mine, or something I shouldn't - have access to? -- Can I disable something for other people? - -** How to file a bug report - -When filing an issue at -[[earthspy/issues][https://github.com/AdrienWehrle/earthspy/issues]], make sure to answer -these five questions: - -- Have you installed =earthspy= using =environment.yml=? If not, what - version of Python and dependencies are you using? -- What operating system and processor architecture are you using? -- What did you do? -- What did you expect to see? -- What did you see instead? - - -* Rights - -The license (see =LICENSE=) applies to all contributions. - -* Credits - -Part of this document is based on [[https://github.com/nayafia][nayafia]]'s [[https://github.com/nayafia/contributing-template/blob/master/CONTRIBUTING-template.md][CONTRIBUTING]] template and -[[https://github.com/GlacioHack/geoutils/blob/main/CONTRIBUTING.md][geoutils CONTRIBUTING]]. diff --git a/README.md b/README.md new file mode 100644 index 0000000..6b3f482 --- /dev/null +++ b/README.md @@ -0,0 +1,167 @@ +[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) +[![Continuous integration](https://github.com/AdrienWehrle/earthspy/workflows/CI/badge.svg)](https://github.com/AdrienWehrle/earthspy/actions) +[![CodeQL](https://github.com/AdrienWehrle/earthspy/actions/workflows/codeql.yml/badge.svg)](https://github.com/AdrienWehrle/earthspy/actions/workflows/codeql.yml) +[![readthedocs](https://readthedocs.org/projects/earthspy/badge/?version=latest)]() +[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://pre-commit.com/) +[![codecov](https://codecov.io/gh/AdrienWehrle/earthspy/branch/main/graph/badge.svg)](https://codecov.io/gh/AdrienWehrle/earthspy) +[![black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) + + +# earthspy 🛰️ :earth_africa: :earth_americas: :earth_asia: + +`earthspy` is a wrapper around methods for the download of satellite data offered in the [sentinelhub-py Python package](https://github.com/sentinel-hub/sentinelhub-py). This tool makes the monitoring and study of any place on Earth simple, ready to use and easily deployable for operational purposes and automated Near-Real Time (NRT) applications. + +Some useful capabilities: + - Data download in multiprocessing + - Data download at optimized resolutions with the Direct (D) download mode + - Data download at native resolutions with the Split and Merge (SM) downlodad mode + - Data storage with efficient structure and file naming + +As `earthspy` is built on top of the [Sentinel Hub services](https://www.sentinel-hub.com/), it includes e.g. the data pre-processing through [custom scripts](https://docs.sentinel-hub.com/api/latest/evalscript/) allowing the user to process and download only the products needed (such as high-level indices) therefore optimizing download time and local storage. + +# Table of Contents +- [earthspy](#earthspy-%EF%B8%8F-earth_africa-earth_americas-earth_asia) +- [Installation](#installation) +- [Usage](#usage) +- [Operational Near Real-Time (NRT) deployment](#operational-near-real-time-nrt-deployment) +- [Documentation](#documentation) +- [Contributing](#contributing) + +# Installation + +It is recommended to install `earthspy` via [Github](https://github.com/), with [conda](https://docs.conda.io/en/latest/) and [pip](https://pip.pypa.io/en/stable/): + +```bash + +# clone repository +git clone git@github.com:AdrienWehrle/earthspy.git + +# move into earthspy directory +cd earthspy + +# create conda environment +conda env create -f environment.yml + +# activate conda environment +conda activate earthspy + +# install earthspy +pip install -e . +``` + +- Using `pip` together with `conda` is usually a bad idea, but here `conda` installs all the dependencies and `pip` only sets up the associated paths, that's all! :+1: +- Installation can be sped up using the fast cross-platform package manager [mamba](https://mamba.readthedocs.io/en/latest/) (reimplementation of the conda package manager in C++), simply use `mamba` instead of `conda` in the instructions above! + + +# Usage +At present `earthspy` can be run within a couple of lines of Python code that execute three main tasks: +- set up a Sentinel Hub connection (for a given Sentinel Hub account) +- set query parameters including Sentinel Hub API variables and `earthspy` additional ones (mainly for download efficiency) +- send request + +Below is presented a simple application of `earthspy` for the download of Sentinel-2 data download around Ilulissat, Greenland for a few days in August 2019 using a True Color custom script available on Sentinel Hub's [custom script online repository](https://custom-scripts.sentinel-hub.com). All other available data collections can be found [here](https://sentinelhub-py.readthedocs.io/en/latest/examples/data_collections.html). + +```python +import earthspy.earthspy as es + +# auth.txt should contain username and password (first and second row) +job = es.EarthSpy("/path/to/auth.txt") + +# as simple as it gets +job.set_query_parameters( + bounding_box=[ + -51.13, + 69.204, + -51.06, + 69.225, + ], # format from doc: [min_x, min_y, max_x, max_y] + time_interval=["2019-08-03", "2019-08-10"], + evaluation_script="https://custom-scripts.sentinel-hub.com/custom-scripts/sentinel-2/true_color/script.js", + data_collection="SENTINEL2_L2A", +) + +# and off it goes! +job.send_sentinelhub_requests() +``` + +Homemade custom evalscripts can also be passed without effort to e.g. compute high-level indices (NDVI, NDSI...). +Below is presented an example with the default evaluation script used above (to keep it short): + +```python +import earthspy.earthspy as es + +# Sentinel-2 default True Color script +example_evalscript = """ + //VERSION=3 + function setup(){ + return{ + input: ["B02", "B03", "B04", "dataMask"], + output: {bands: 4} + } + } + + function evaluatePixel(sample){ + // Set gain for visualisation + let gain = 2.5; + // Return RGB + return [sample.B04 * gain, sample.B03 * gain, sample.B02 * gain, sample.dataMask]; + } + + """ + +# auth.txt should contain username and password (first and second row) +job = es.EarthSpy("/path/to/auth.txt") + +# pass string to evaluation_script +job.set_query_parameters( + bounding_box=[-51.13, 69.204, -51.06, 69.225], + time_interval=["2019-08-03", "2019-08-10"], + evaluation_script=example_evalscript, + data_collection="SENTINEL2_L2A", +) + +# and off it goes! +job.send_sentinelhub_requests() +``` + +[GEOJSON](https://geojson.org/) files containing a polygon corresponding to a given region of interest +and its associated name can also be created at [geojson.io](https://geojson.io/#map=2/20.0/0.0) and stored in [`./data`](https://github.com/AdrienWehrle/earthspy/tree/main/data). +In this way, the name of the region can be directly passed to the `bounding_box` +query parameter. See below for a simple example with the [ilulissat.geojson](https://github.com/AdrienWehrle/earthspy/tree/main/data/ilulissat.geojson) example file. + +```python +import earthspy.earthspy as es + +# auth.txt should contain username and password (first and second row) +job = es.EarthSpy("/path/to/auth.txt") + +# as simple as it gets +job.set_query_parameters( + bounding_box="Ilulissat", + time_interval=["2019-08-03", "2019-08-10"], + evaluation_script="https://custom-scripts.sentinel-hub.com/custom-scripts/sentinel-2/true_color/script.js", + data_collection="SENTINEL2_L2A", +) + +# and off it goes! +job.send_sentinelhub_requests() +``` + + +# Operational Near Real-Time (NRT) deployment + +`earthspy` can be easily deployed for NRT monitoring. The setup is as simple as wrapping the query parameters in a short python script such as [earthspy_NRT.py](https://github.com/AdrienWehrle/earthspy/blob/main/earthspy/operational/earthspy_NRT.py) and including it in a cron job. See an example below where Sentinel-2 images of Ilulissat, Greenland acquired over the past three days are downloaded everyday at noon. +```bash +# m h dom mon dow command +00 12 * * * /bin/bash -c "/path/to/earthspy_NRT.py" > /path/to/log/log_earthspy_NRT.txt +``` + +# Documentation + +The documentation of `earthspy` is hosted on [readthedocs](https://earthspy.readthedocs.io/en/latest/). + +# Contributing + +Contributions to `earthspy` are more than welcome! Guidelines are +presented in [CONTRIBUTING.md](https://github.com/AdrienWehrle/earthspy/blob/main/CONTRIBUTING.md). From 07967b4f6d8b9b5c2666686560f5f9f48b6d43de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 20:47:56 +0100 Subject: [PATCH 13/13] Bump github/super-linter from 5.0.0 to 7 (#117) Bumps [github/super-linter](https://github.com/github/super-linter) from 5.0.0 to 7. - [Release notes](https://github.com/github/super-linter/releases) - [Changelog](https://github.com/github/super-linter/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/super-linter/compare/v5.0.0...v7) --- updated-dependencies: - dependency-name: github/super-linter dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bf53d1b..1ae9a63 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,7 +36,7 @@ jobs: fetch-depth: 0 - name: Lint code base - uses: github/super-linter/slim@v5.0.0 + uses: github/super-linter/slim@v7 env: VALIDATE_ALL_CODEBASE: true VALIDATE_JSON: true