diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100755 index 0000000..ac27a84 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" diff --git a/.github/scripts/generate_pip_deps_from_conda.py b/.github/scripts/generate_pip_deps_from_conda.py new file mode 100755 index 0000000..ac74bd5 --- /dev/null +++ b/.github/scripts/generate_pip_deps_from_conda.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +""" +(Copied from pandas: +https://github.com/pandas-dev/pandas/blob/main/scripts/generate_pip_deps_from_conda.py) +Convert the conda environment.yml to the pip requirements-dev.txt, or +check that they have the same packages (for the CI) + +Usage: + + Generate `requirements-dev.txt` + $ python scripts/generate_pip_deps_from_conda.py + + Compare and fail (exit status != 0) if `requirements-dev.txt` has not been + generated with this script: + $ python scripts/generate_pip_deps_from_conda.py --compare + +""" +import argparse +import pathlib +import re +import sys + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib +import yaml + +EXCLUDE = {"python"} +REMAP_VERSION = {"tzdata": "2022.1"} +RENAME = {} + + +def conda_package_to_pip(package: str): + """ + Convert a conda package to its pip equivalent. + + In most cases they are the same, those are the exceptions: + - Packages that should be excluded (in `EXCLUDE`) + - Packages that should be renamed (in `RENAME`) + - A package requiring a specific version, in conda is defined with a single + equal (e.g. ``pandas=1.0``) and in pip with two (e.g. ``pandas==1.0``) + """ + package = re.sub("(?<=[^<>])=", "==", package).strip() + print(package) + + for compare in ("<=", ">=", "=="): + if compare in package: + pkg, version = package.split(compare) + if pkg in EXCLUDE: + return + if pkg in REMAP_VERSION: + return "".join((pkg, compare, REMAP_VERSION[pkg])) + if pkg in RENAME: + return "".join((RENAME[pkg], compare, version)) + + if package in EXCLUDE: + return + + if package in RENAME: + return RENAME[package] + + return package + + +def generate_pip_from_conda( + conda_path: pathlib.Path, pip_path: pathlib.Path, compare: bool = False +) -> bool: + """ + Generate the pip dependencies file from the conda file, or compare that + they are synchronized (``compare=True``). + + Parameters + ---------- + conda_path : pathlib.Path + Path to the conda file with dependencies (e.g. `environment.yml`). + pip_path : pathlib.Path + Path to the pip file with dependencies (e.g. `requirements-dev.txt`). + compare : bool, default False + Whether to generate the pip file (``False``) or to compare if the + pip file has been generated with this script and the last version + of the conda file (``True``). + + Returns + ------- + bool + True if the comparison fails, False otherwise + """ + with conda_path.open() as file: + deps = yaml.safe_load(file)["dependencies"] + + pip_deps = [] + for dep in deps: + if isinstance(dep, str): + conda_dep = conda_package_to_pip(dep) + if conda_dep: + pip_deps.append(conda_dep) + elif isinstance(dep, dict) and len(dep) == 1 and "pip" in dep: + pip_deps.extend(dep["pip"]) + else: + raise ValueError(f"Unexpected dependency {dep}") + + header = ( + f"# This file is auto-generated from {conda_path.name}, do not modify.\n" + "# See that file for comments about the need/usage of each dependency.\n\n" + ) + pip_content = header + "\n".join(pip_deps) + "\n" + + # Add setuptools to requirements-dev.txt + + # with open(pathlib.Path(conda_path.parent, "pyproject.toml"), "rb") as fd: + # meta = tomllib.load(fd) + # for requirement in meta["build-system"]["requires"]: + # if "setuptools" in requirement: + # pip_content += requirement + # pip_content += "\n" + + if compare: + with pip_path.open() as file: + return pip_content != file.read() + + with pip_path.open("w") as file: + file.write(pip_content) + return False + + +if __name__ == "__main__": + argparser = argparse.ArgumentParser( + description="convert (or compare) conda file to pip" + ) + argparser.add_argument( + "--compare", + action="store_true", + help="compare whether the two files are equivalent", + ) + args = argparser.parse_args() + + conda_fname = "dev-environment.yml" + pip_fname = "requirements-dev.txt" + repo_path = pathlib.Path(__file__).parent.parent.parent.absolute() + res = generate_pip_from_conda( + pathlib.Path(repo_path, conda_fname), + pathlib.Path(repo_path, pip_fname), + compare=args.compare, + ) + if res: + msg = ( + f"`{pip_fname}` has to be generated with `{__file__}` after " + f"`{conda_fname}` is modified.\n" + ) + sys.stderr.write(msg) + sys.exit(res) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 2da8421..483a9d9 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -24,18 +24,18 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} queries: +security-and-quality - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9b67109..1ae9a63 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,36 +6,37 @@ jobs: name: Setup and test runs-on: ${{ matrix.os }} strategy: - matrix: - os: ["ubuntu-latest"] + matrix: + os: ["ubuntu-latest"] + python-version: ["3.10", "3.11", "3.12"] + steps: - - uses: actions/checkout@v2 - - uses: snyk/actions/setup@master - - uses: conda-incubator/setup-miniconda@v2 + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 with: activate-environment: test environment-file: environment.yml - python-version: 3.8 + use-mamba: true auto-activate-base: true - + - shell: bash -l {0} run: | conda info conda list - + - name: Install earthspy shell: bash -l {0} run: | pip install -e . - + - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: # Full git history is needed to get a proper list of changed files within `super-linter` - fetch-depth: 0 - + fetch-depth: 0 + - name: Lint code base - uses: github/super-linter/slim@v4.9.0 + uses: github/super-linter/slim@v7 env: VALIDATE_ALL_CODEBASE: true VALIDATE_JSON: true @@ -48,26 +49,26 @@ jobs: run: | pip install pytest pip install pytest-cov - pytest --cov=./ --cov-report=xml + pytest --cov --junitxml=junit.xml -o junit_family=legacy env: SH_CLIENT_ID: ${{ secrets.SH_CLIENT_ID }} SH_CLIENT_SECRET: ${{ secrets.SH_CLIENT_SECRET }} - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + - name: Upload test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 with: - directory: ./coverage/reports/ - fail_ci_if_error: true - files: ./coverage.xml - flags: unittests - name: codecov-umbrella - path_to_write_report: ./coverage/codecov_report.txt - verbose: true + token: ${{ secrets.CODECOV_TOKEN }} - - name: Run Snyk to check for vulnerabilities + - name: Install dev requirements + shell: bash -l {0} run: | - pip install -r dev-requirements.txt - snyk test --file=dev-requirements.txt --package-manager=pip --severity-threshold=high + pip install -r requirements-dev.txt + + - name: Run Snyk to check for vulnerabilities + uses: snyk/actions/python@master + continue-on-error: true env: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - + with: + args: --file=requirements-dev.txt --package-manager=pip --severity-threshold=high --skip-unresolved=true diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..25ac772 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,15 @@ +name: Linting and formatting (pre-commit) + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - uses: pre-commit/action@v3.0.1 diff --git a/.gitignore b/.gitignore index f294af6..cef40bf 100644 --- a/.gitignore +++ b/.gitignore @@ -142,4 +142,4 @@ auth.txt *.gcno *.gcda *.gcov -test_calculator \ No newline at end of file +test_calculator diff --git a/.readthedocs.yml b/.readthedocs.yml index f0b29af..fb19340 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,17 +5,23 @@ # Required version: 2 +# Set the OS, Python version and other tools you might need + +build: + os: ubuntu-22.04 + tools: + python: "3.12" + # Build documentation in the docs/ directory with Sphinx sphinx: configuration: docs/conf.py fail_on_warning: false # Optionally build your docs in additional formats such as PDF and ePub -formats: [] +formats: [pdf] python: - version: 3.7 install: - - requirements: dev-requirements.txt + - requirements: requirements-dev.txt - method: pip path: . diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..21e22a4 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,140 @@ +# Introduction + +First off, thank you for considering contributing to `earthspy`! 😊 + +Following the guidelines presented here helps to communicate that you +respect the efforts the developers managing and developing this open +source project have put to foster community development. In return, +they should reciprocate that respect in addressing your issue, +assessing changes, and helping you finalize your pull requests. 🌻 + +Any kind of contribution is more than welcome. Improving +documentation, bug triaging, implementing new features or writing +tutorials are all examples of helpful contributions. 🚀 + +# Making a contribution + +## Overview + + 1. Fork `AdrienWehrle/earthspy` and clone your fork repository locally. + 2. Set up the development environment (section below). + 3. Create a branch for the new feature or bug fix. + 4. Make your changes, and add or modify related tests in + [`tests/`](https://github.com/AdrienWehrle/earthspy/tree/main/tests). + 5. Commit, making sure to run pre-commit separately if not installed as git hook. + 6. Push to your fork. + 7. Open a pull request from GitHub to discuss and eventually merge. + + +## Step by step + +### Setup + +Clone the git repository and create a +[conda](https://docs.conda.io/projects/conda/en/latest/index.html) or +[mamba](https://mamba.readthedocs.io/en/latest/index.html) +environment. If you use mamba (mamba is great), just replace `conda` +for `mamba` below. Steps below are similar to those presented in the +[README](https://github.com/AdrienWehrle/earthspy/tree/main) except +[`dev-environment.yml`](https://github.com/AdrienWehrle/earthspy/blob/main/dev-environment.yml) +is used here instead of +[`environment.yml`](https://github.com/AdrienWehrle/earthspy/blob/main/dev-environment.yml) +as it contains necessary dependencies for continuous integration and +continuous development. + +```bash + +# clone repository +git clone git@github.com:AdrienWehrle/earthspy.git + +# move into earthspy directory +cd earthspy + +# create conda environment +conda env create -f dev-environment.yml + +# activate conda environment +conda activate earthspy-dev + +# install earthspy +pip install -e . +``` + +### Tests + +If your PR targets the implementation of a new feature or the +improvement of existing ones, please add at least one test per feature +(in the associated +[`tests/test_earthspy.py`](https://github.com/AdrienWehrle/earthspy/blob/main/LICENSE) +file) and include them in your PR, using `pytest` (see existing tests +for examples). + +To run the entire test suite, run pytest in the current directory: + +```bash +pytest +``` + + +### Formatting and linting + +Before you make your commit, please install `pre-commit` (see +[pre-commit documentation](https://pre-commit.com/)), which will use +`.pre-commit-config.yaml` to verify spelling errors, import sorting, +type checking, formatting and linting. + +You can then run `pre-commit` manually: + +```bash +pre-commit run --all-files +``` + +In your PR, make sure to note if all `pre-commit` checks passed or +not. Other developers will gladly help you getting rid of remaining +red signs in your terminal if you feel stuck! 🌿 + +Optionally, `pre-commit` can also be installed as a git hook to +automatically ensure checks have to pass before committing. + +Once all `pre-commit` checks have passed (or you need help), +please commit your changes with short and clear messages. + + +# Reporting a bug + +## Security disclosures + +If you find a security vulnerability, please do not open an +issue. Email adrien.wehrle@hotmail.fr instead. + +In order to determine whether you are dealing with a security issue, +ask yourself these two questions: + +- Can I access something that's not mine, or something I shouldn't + have access to? +- Can I disable something for other people? + +## How to file a bug report + +When filing an issue at +[earthspy/issues](https://github.com/AdrienWehrle/earthspy/issues), +make sure to answer these five questions: + +- Have you installed `earthspy` using + [`environment.yml`](https://github.com/AdrienWehrle/earthspy/blob/main/LICENSE)? + If not, what version of Python and dependencies are you using? +- What operating system and processor architecture are you using? +- What did you do? +- What did you expect to see? +- What did you see instead? + + +# Rights + +The license (see +[`LICENSE`](https://github.com/AdrienWehrle/earthspy/blob/main/LICENSE)) +applies to all contributions. + +# Credits + +Part of this document is based on [nayafia](https://github.com/nayafia)'s [CONTRIBUTING](https://github.com/nayafia) template and [geoutils CONTRIBUTING](https://github.com/GlacioHack/geoutils/blob/main/CONTRIBUTING.md). diff --git a/README.md b/README.md new file mode 100644 index 0000000..6b3f482 --- /dev/null +++ b/README.md @@ -0,0 +1,167 @@ +[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) +[![Continuous integration](https://github.com/AdrienWehrle/earthspy/workflows/CI/badge.svg)](https://github.com/AdrienWehrle/earthspy/actions) +[![CodeQL](https://github.com/AdrienWehrle/earthspy/actions/workflows/codeql.yml/badge.svg)](https://github.com/AdrienWehrle/earthspy/actions/workflows/codeql.yml) +[![readthedocs](https://readthedocs.org/projects/earthspy/badge/?version=latest)]() +[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://pre-commit.com/) +[![codecov](https://codecov.io/gh/AdrienWehrle/earthspy/branch/main/graph/badge.svg)](https://codecov.io/gh/AdrienWehrle/earthspy) +[![black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) + + +# earthspy 🛰️ :earth_africa: :earth_americas: :earth_asia: + +`earthspy` is a wrapper around methods for the download of satellite data offered in the [sentinelhub-py Python package](https://github.com/sentinel-hub/sentinelhub-py). This tool makes the monitoring and study of any place on Earth simple, ready to use and easily deployable for operational purposes and automated Near-Real Time (NRT) applications. + +Some useful capabilities: + - Data download in multiprocessing + - Data download at optimized resolutions with the Direct (D) download mode + - Data download at native resolutions with the Split and Merge (SM) downlodad mode + - Data storage with efficient structure and file naming + +As `earthspy` is built on top of the [Sentinel Hub services](https://www.sentinel-hub.com/), it includes e.g. the data pre-processing through [custom scripts](https://docs.sentinel-hub.com/api/latest/evalscript/) allowing the user to process and download only the products needed (such as high-level indices) therefore optimizing download time and local storage. + +# Table of Contents +- [earthspy](#earthspy-%EF%B8%8F-earth_africa-earth_americas-earth_asia) +- [Installation](#installation) +- [Usage](#usage) +- [Operational Near Real-Time (NRT) deployment](#operational-near-real-time-nrt-deployment) +- [Documentation](#documentation) +- [Contributing](#contributing) + +# Installation + +It is recommended to install `earthspy` via [Github](https://github.com/), with [conda](https://docs.conda.io/en/latest/) and [pip](https://pip.pypa.io/en/stable/): + +```bash + +# clone repository +git clone git@github.com:AdrienWehrle/earthspy.git + +# move into earthspy directory +cd earthspy + +# create conda environment +conda env create -f environment.yml + +# activate conda environment +conda activate earthspy + +# install earthspy +pip install -e . +``` + +- Using `pip` together with `conda` is usually a bad idea, but here `conda` installs all the dependencies and `pip` only sets up the associated paths, that's all! :+1: +- Installation can be sped up using the fast cross-platform package manager [mamba](https://mamba.readthedocs.io/en/latest/) (reimplementation of the conda package manager in C++), simply use `mamba` instead of `conda` in the instructions above! + + +# Usage +At present `earthspy` can be run within a couple of lines of Python code that execute three main tasks: +- set up a Sentinel Hub connection (for a given Sentinel Hub account) +- set query parameters including Sentinel Hub API variables and `earthspy` additional ones (mainly for download efficiency) +- send request + +Below is presented a simple application of `earthspy` for the download of Sentinel-2 data download around Ilulissat, Greenland for a few days in August 2019 using a True Color custom script available on Sentinel Hub's [custom script online repository](https://custom-scripts.sentinel-hub.com). All other available data collections can be found [here](https://sentinelhub-py.readthedocs.io/en/latest/examples/data_collections.html). + +```python +import earthspy.earthspy as es + +# auth.txt should contain username and password (first and second row) +job = es.EarthSpy("/path/to/auth.txt") + +# as simple as it gets +job.set_query_parameters( + bounding_box=[ + -51.13, + 69.204, + -51.06, + 69.225, + ], # format from doc: [min_x, min_y, max_x, max_y] + time_interval=["2019-08-03", "2019-08-10"], + evaluation_script="https://custom-scripts.sentinel-hub.com/custom-scripts/sentinel-2/true_color/script.js", + data_collection="SENTINEL2_L2A", +) + +# and off it goes! +job.send_sentinelhub_requests() +``` + +Homemade custom evalscripts can also be passed without effort to e.g. compute high-level indices (NDVI, NDSI...). +Below is presented an example with the default evaluation script used above (to keep it short): + +```python +import earthspy.earthspy as es + +# Sentinel-2 default True Color script +example_evalscript = """ + //VERSION=3 + function setup(){ + return{ + input: ["B02", "B03", "B04", "dataMask"], + output: {bands: 4} + } + } + + function evaluatePixel(sample){ + // Set gain for visualisation + let gain = 2.5; + // Return RGB + return [sample.B04 * gain, sample.B03 * gain, sample.B02 * gain, sample.dataMask]; + } + + """ + +# auth.txt should contain username and password (first and second row) +job = es.EarthSpy("/path/to/auth.txt") + +# pass string to evaluation_script +job.set_query_parameters( + bounding_box=[-51.13, 69.204, -51.06, 69.225], + time_interval=["2019-08-03", "2019-08-10"], + evaluation_script=example_evalscript, + data_collection="SENTINEL2_L2A", +) + +# and off it goes! +job.send_sentinelhub_requests() +``` + +[GEOJSON](https://geojson.org/) files containing a polygon corresponding to a given region of interest +and its associated name can also be created at [geojson.io](https://geojson.io/#map=2/20.0/0.0) and stored in [`./data`](https://github.com/AdrienWehrle/earthspy/tree/main/data). +In this way, the name of the region can be directly passed to the `bounding_box` +query parameter. See below for a simple example with the [ilulissat.geojson](https://github.com/AdrienWehrle/earthspy/tree/main/data/ilulissat.geojson) example file. + +```python +import earthspy.earthspy as es + +# auth.txt should contain username and password (first and second row) +job = es.EarthSpy("/path/to/auth.txt") + +# as simple as it gets +job.set_query_parameters( + bounding_box="Ilulissat", + time_interval=["2019-08-03", "2019-08-10"], + evaluation_script="https://custom-scripts.sentinel-hub.com/custom-scripts/sentinel-2/true_color/script.js", + data_collection="SENTINEL2_L2A", +) + +# and off it goes! +job.send_sentinelhub_requests() +``` + + +# Operational Near Real-Time (NRT) deployment + +`earthspy` can be easily deployed for NRT monitoring. The setup is as simple as wrapping the query parameters in a short python script such as [earthspy_NRT.py](https://github.com/AdrienWehrle/earthspy/blob/main/earthspy/operational/earthspy_NRT.py) and including it in a cron job. See an example below where Sentinel-2 images of Ilulissat, Greenland acquired over the past three days are downloaded everyday at noon. +```bash +# m h dom mon dow command +00 12 * * * /bin/bash -c "/path/to/earthspy_NRT.py" > /path/to/log/log_earthspy_NRT.txt +``` + +# Documentation + +The documentation of `earthspy` is hosted on [readthedocs](https://earthspy.readthedocs.io/en/latest/). + +# Contributing + +Contributions to `earthspy` are more than welcome! Guidelines are +presented in [CONTRIBUTING.md](https://github.com/AdrienWehrle/earthspy/blob/main/CONTRIBUTING.md). diff --git a/README.org b/README.org index c187917..9fff53a 100644 --- a/README.org +++ b/README.org @@ -1,21 +1,23 @@ -[[https://www.repostatus.org/badges/latest/wip.svg][https://www.repostatus.org/badges/latest/wip.svg]] [[https://www.gnu.org/licenses/gpl-3.0][https://img.shields.io/badge/License-GPLv3-blue.svg]] [[https://github.com/AdrienWehrle/earthspy/actions][file:https://github.com/AdrienWehrle/earthspy/workflows/CI/badge.svg]] [[https://github.com/AdrienWehrle/earthspy/actions/workflows/codeql.yml][https://github.com/AdrienWehrle/earthspy/actions/workflows/codeql.yml/badge.svg]] +[[https://pre-commit.com/][https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit.svg]] [[https://codecov.io/gh/AdrienWehrle/earthspy][https://codecov.io/gh/AdrienWehrle/earthspy/branch/main/graph/badge.svg]] [[https://github.com/psf/black][https://img.shields.io/badge/code%20style-black-000000.svg]] +[[https://pycqa.github.io/isort/][https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336.svg]] + * earthspy 🛰️ :earth_africa: :earth_americas: :earth_asia: :moon: =earthspy= is a wrapper around methods for the download of satellite data offered in the [[https://github.com/sentinel-hub/sentinelhub-py][sentinelhub Python package]]. This tool makes the monitoring and study of any place on Earth simple, ready to use and easily deployable for operational purposes and automated Near-Real Time (NRT) applications. -Some useful capabilities: +Some useful capabilities: - Data download in multiprocessing - - Data download at optimized resolutions with the Direct (D) download mode + - Data download at optimized resolutions with the Direct (D) download mode - Data download at native resolutions with the Split and Merge (SM) downlodad mode - Data storage with efficient structure and file naming -As =earthspy= is built on top of the [[https://www.sentinel-hub.com/][Sentinel Hub services]], it includes e.g. the data pre-processing through [[https://docs.sentinel-hub.com/api/latest/evalscript/][custom scripts]] allowing the user to process and download only the products needed (such as high-level indices) therefore optimizing download time and local storage. +As =earthspy= is built on top of the [[https://www.sentinel-hub.com/][Sentinel Hub services]], it includes e.g. the data pre-processing through [[https://docs.sentinel-hub.com/api/latest/evalscript/][custom scripts]] allowing the user to process and download only the products needed (such as high-level indices) therefore optimizing download time and local storage. * Table of Contents :toc_2:noexport: - [[#earthspy-%EF%B8%8F-earth_africa-earth_americas-earth_asia][earthspy]] @@ -67,7 +69,12 @@ job = es.EarthSpy("/path/to/auth.txt") # as simple as it gets job.set_query_parameters( - bounding_box=[-51.13, 69.204, -51.06, 69.225], # format from doc: [min_x, min_y, max_x, max_y] + bounding_box=[ + -51.13, + 69.204, + -51.06, + 69.225, + ], # format from doc: [min_x, min_y, max_x, max_y] time_interval=["2019-08-03", "2019-08-10"], evaluation_script="https://custom-scripts.sentinel-hub.com/custom-scripts/sentinel-2/true_color/script.js", data_collection="SENTINEL2_L2A", @@ -75,6 +82,7 @@ job.set_query_parameters( # and off it goes! job.send_sentinelhub_requests() + #+end_src Homemade custom evalscripts can also be passed without effort to e.g. compute high-level indices (NDVI, NDSI...). @@ -142,7 +150,7 @@ job.send_sentinelhub_requests() #+end_src -* Operational Near Real-Time (NRT) deployment +* Operational Near Real-Time (NRT) deployment =earthspy= can be easily deployed for NRT monitoring. The setup is as simple as wrapping the query parameters in a short python script such as [[https://github.com/AdrienWehrle/earthspy/blob/main/earthspy/operational/earthspy_NRT.py][earthspy_NRT.py]] and including it in a cron job. See an example below where Sentinel-2 images of Ilulissat, Greenland acquired over the past three days are downloaded everyday at noon. #+BEGIN_SRC bash :results verbatim diff --git a/dev-environment.yml b/dev-environment.yml new file mode 100644 index 0000000..18c217f --- /dev/null +++ b/dev-environment.yml @@ -0,0 +1,24 @@ +name: earthspy-dev +channels: + - defaults + - conda-forge + - bioconda +dependencies: + - python>=3.9 + - numpy + - objectpath + - pandas + - rasterio + - validators + - sentinelhub >= 3.4 + - pip + - flake8 + - pytest + - pytest-lazy-fixture + - sphinx + - sphinx_rtd_theme + - pylint + - sphinxcontrib-programoutput + - sphinx-autodoc-typehints + - pre-commit + - sphinx-gallery diff --git a/dev-requirements.txt b/dev-requirements.txt deleted file mode 100644 index 8d3ac43..0000000 --- a/dev-requirements.txt +++ /dev/null @@ -1,16 +0,0 @@ -numpy -pandas -validators -rasterio -sentinelhub -flake8 -pytest -pytest-lazy-fixture -sphinx -sphinx_rtd_theme -pylint -sphinxcontrib_programoutput -sphinx_autodoc_typehints -pre-commit -sphinx-gallery - diff --git a/docs/_build/html/_static/alabaster.css b/docs/_build/html/_static/alabaster.css index 0eddaeb..969ce31 100644 --- a/docs/_build/html/_static/alabaster.css +++ b/docs/_build/html/_static/alabaster.css @@ -698,4 +698,4 @@ nav#breadcrumbs li+li:before { div.related { display: none; } -} \ No newline at end of file +} diff --git a/docs/_build/html/_static/basic.css b/docs/_build/html/_static/basic.css index bf18350..1c769a5 100644 --- a/docs/_build/html/_static/basic.css +++ b/docs/_build/html/_static/basic.css @@ -903,4 +903,4 @@ div.math:hover a.headerlink { #top-link { display: none; } -} \ No newline at end of file +} diff --git a/docs/_build/html/_static/documentation_options.js b/docs/_build/html/_static/documentation_options.js index f1df40e..bef71d2 100644 --- a/docs/_build/html/_static/documentation_options.js +++ b/docs/_build/html/_static/documentation_options.js @@ -9,4 +9,4 @@ var DOCUMENTATION_OPTIONS = { HAS_SOURCE: true, SOURCELINK_SUFFIX: '.txt', NAVIGATION_WITH_KEYS: false -}; \ No newline at end of file +}; diff --git a/docs/_build/html/_static/language_data.js b/docs/_build/html/_static/language_data.js index ebe2f03..7dbd658 100644 --- a/docs/_build/html/_static/language_data.js +++ b/docs/_build/html/_static/language_data.js @@ -293,5 +293,3 @@ function splitQuery(query) { } return result; } - - diff --git a/docs/_build/html/_static/pygments.css b/docs/_build/html/_static/pygments.css index 87f8bd1..b35b845 100644 --- a/docs/_build/html/_static/pygments.css +++ b/docs/_build/html/_static/pygments.css @@ -79,4 +79,4 @@ span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: .highlight .vg { color: #000000 } /* Name.Variable.Global */ .highlight .vi { color: #000000 } /* Name.Variable.Instance */ .highlight .vm { color: #000000 } /* Name.Variable.Magic */ -.highlight .il { color: #990000 } /* Literal.Number.Integer.Long */ \ No newline at end of file +.highlight .il { color: #990000 } /* Literal.Number.Integer.Long */ diff --git a/docs/_build/html/_static/underscore.js b/docs/_build/html/_static/underscore.js index cf177d4..542fe3c 100644 --- a/docs/_build/html/_static/underscore.js +++ b/docs/_build/html/_static/underscore.js @@ -3,4 +3,4 @@ // https://underscorejs.org // (c) 2009-2021 Jeremy Ashkenas, Julian Gonggrijp, and DocumentCloud and Investigative Reporters & Editors // Underscore may be freely distributed under the MIT license. -var n="1.13.1",r="object"==typeof self&&self.self===self&&self||"object"==typeof global&&global.global===global&&global||Function("return this")()||{},t=Array.prototype,e=Object.prototype,u="undefined"!=typeof Symbol?Symbol.prototype:null,o=t.push,i=t.slice,a=e.toString,f=e.hasOwnProperty,c="undefined"!=typeof ArrayBuffer,l="undefined"!=typeof DataView,s=Array.isArray,p=Object.keys,v=Object.create,h=c&&ArrayBuffer.isView,y=isNaN,d=isFinite,g=!{toString:null}.propertyIsEnumerable("toString"),b=["valueOf","isPrototypeOf","toString","propertyIsEnumerable","hasOwnProperty","toLocaleString"],m=Math.pow(2,53)-1;function j(n,r){return r=null==r?n.length-1:+r,function(){for(var t=Math.max(arguments.length-r,0),e=Array(t),u=0;u=0&&t<=m}}function J(n){return function(r){return null==r?void 0:r[n]}}var G=J("byteLength"),H=K(G),Q=/\[object ((I|Ui)nt(8|16|32)|Float(32|64)|Uint8Clamped|Big(I|Ui)nt64)Array\]/;var X=c?function(n){return h?h(n)&&!q(n):H(n)&&Q.test(a.call(n))}:C(!1),Y=J("length");function Z(n,r){r=function(n){for(var r={},t=n.length,e=0;e":">",'"':""","'":"'","`":"`"},Cn=Ln($n),Kn=Ln(_n($n)),Jn=tn.templateSettings={evaluate:/<%([\s\S]+?)%>/g,interpolate:/<%=([\s\S]+?)%>/g,escape:/<%-([\s\S]+?)%>/g},Gn=/(.)^/,Hn={"'":"'","\\":"\\","\r":"r","\n":"n","\u2028":"u2028","\u2029":"u2029"},Qn=/\\|'|\r|\n|\u2028|\u2029/g;function Xn(n){return"\\"+Hn[n]}var Yn=/^\s*(\w|\$)+\s*$/;var Zn=0;function nr(n,r,t,e,u){if(!(e instanceof r))return n.apply(t,u);var o=Mn(n.prototype),i=n.apply(o,u);return _(i)?i:o}var rr=j((function(n,r){var t=rr.placeholder,e=function(){for(var u=0,o=r.length,i=Array(o),a=0;a1)ur(a,r-1,t,e),u=e.length;else for(var f=0,c=a.length;f0&&(t=r.apply(this,arguments)),n<=1&&(r=null),t}}var lr=rr(cr,2);function sr(n,r,t){r=qn(r,t);for(var e,u=nn(n),o=0,i=u.length;o0?0:u-1;o>=0&&o0?a=o>=0?o:Math.max(o+f,a):f=o>=0?Math.min(o+1,f):o+f+1;else if(t&&o&&f)return e[o=t(e,u)]===u?o:-1;if(u!=u)return(o=r(i.call(e,a,f),$))>=0?o+a:-1;for(o=n>0?a:f-1;o>=0&&o0?0:i-1;for(u||(e=r[o?o[a]:a],a+=n);a>=0&&a=3;return r(n,Fn(t,u,4),e,o)}}var Ar=wr(1),xr=wr(-1);function Sr(n,r,t){var e=[];return r=qn(r,t),jr(n,(function(n,t,u){r(n,t,u)&&e.push(n)})),e}function Or(n,r,t){r=qn(r,t);for(var e=!er(n)&&nn(n),u=(e||n).length,o=0;o=0}var Br=j((function(n,r,t){var e,u;return D(r)?u=r:(r=Nn(r),e=r.slice(0,-1),r=r[r.length-1]),_r(n,(function(n){var o=u;if(!o){if(e&&e.length&&(n=In(n,e)),null==n)return;o=n[r]}return null==o?o:o.apply(n,t)}))}));function Nr(n,r){return _r(n,Rn(r))}function Ir(n,r,t){var e,u,o=-1/0,i=-1/0;if(null==r||"number"==typeof r&&"object"!=typeof n[0]&&null!=n)for(var a=0,f=(n=er(n)?n:jn(n)).length;ao&&(o=e);else r=qn(r,t),jr(n,(function(n,t,e){((u=r(n,t,e))>i||u===-1/0&&o===-1/0)&&(o=n,i=u)}));return o}function Tr(n,r,t){if(null==r||t)return er(n)||(n=jn(n)),n[Wn(n.length-1)];var e=er(n)?En(n):jn(n),u=Y(e);r=Math.max(Math.min(r,u),0);for(var o=u-1,i=0;i1&&(e=Fn(e,r[1])),r=an(n)):(e=qr,r=ur(r,!1,!1),n=Object(n));for(var u=0,o=r.length;u1&&(t=r[1])):(r=_r(ur(r,!1,!1),String),e=function(n,t){return!Er(r,t)}),Ur(n,e,t)}));function zr(n,r,t){return i.call(n,0,Math.max(0,n.length-(null==r||t?1:r)))}function Lr(n,r,t){return null==n||n.length<1?null==r||t?void 0:[]:null==r||t?n[0]:zr(n,n.length-r)}function $r(n,r,t){return i.call(n,null==r||t?1:r)}var Cr=j((function(n,r){return r=ur(r,!0,!0),Sr(n,(function(n){return!Er(r,n)}))})),Kr=j((function(n,r){return Cr(n,r)}));function Jr(n,r,t,e){A(r)||(e=t,t=r,r=!1),null!=t&&(t=qn(t,e));for(var u=[],o=[],i=0,a=Y(n);ir?(e&&(clearTimeout(e),e=null),a=c,i=n.apply(u,o),e||(u=o=null)):e||!1===t.trailing||(e=setTimeout(f,l)),i};return c.cancel=function(){clearTimeout(e),a=0,e=u=o=null},c},debounce:function(n,r,t){var e,u,o,i,a,f=function(){var c=zn()-u;r>c?e=setTimeout(f,r-c):(e=null,t||(i=n.apply(a,o)),e||(o=a=null))},c=j((function(c){return a=this,o=c,u=zn(),e||(e=setTimeout(f,r),t&&(i=n.apply(a,o))),i}));return c.cancel=function(){clearTimeout(e),e=o=a=null},c},wrap:function(n,r){return rr(r,n)},negate:fr,compose:function(){var n=arguments,r=n.length-1;return function(){for(var t=r,e=n[r].apply(this,arguments);t--;)e=n[t].call(this,e);return e}},after:function(n,r){return function(){if(--n<1)return r.apply(this,arguments)}},before:cr,once:lr,findKey:sr,findIndex:vr,findLastIndex:hr,sortedIndex:yr,indexOf:gr,lastIndexOf:br,find:mr,detect:mr,findWhere:function(n,r){return mr(n,Dn(r))},each:jr,forEach:jr,map:_r,collect:_r,reduce:Ar,foldl:Ar,inject:Ar,reduceRight:xr,foldr:xr,filter:Sr,select:Sr,reject:function(n,r,t){return Sr(n,fr(qn(r)),t)},every:Or,all:Or,some:Mr,any:Mr,contains:Er,includes:Er,include:Er,invoke:Br,pluck:Nr,where:function(n,r){return Sr(n,Dn(r))},max:Ir,min:function(n,r,t){var e,u,o=1/0,i=1/0;if(null==r||"number"==typeof r&&"object"!=typeof n[0]&&null!=n)for(var a=0,f=(n=er(n)?n:jn(n)).length;ae||void 0===t)return 1;if(t=0&&t<=m}}function J(n){return function(r){return null==r?void 0:r[n]}}var G=J("byteLength"),H=K(G),Q=/\[object ((I|Ui)nt(8|16|32)|Float(32|64)|Uint8Clamped|Big(I|Ui)nt64)Array\]/;var X=c?function(n){return h?h(n)&&!q(n):H(n)&&Q.test(a.call(n))}:C(!1),Y=J("length");function Z(n,r){r=function(n){for(var r={},t=n.length,e=0;e":">",'"':""","'":"'","`":"`"},Cn=Ln($n),Kn=Ln(_n($n)),Jn=tn.templateSettings={evaluate:/<%([\s\S]+?)%>/g,interpolate:/<%=([\s\S]+?)%>/g,escape:/<%-([\s\S]+?)%>/g},Gn=/(.)^/,Hn={"'":"'","\\":"\\","\r":"r","\n":"n","\u2028":"u2028","\u2029":"u2029"},Qn=/\\|'|\r|\n|\u2028|\u2029/g;function Xn(n){return"\\"+Hn[n]}var Yn=/^\s*(\w|\$)+\s*$/;var Zn=0;function nr(n,r,t,e,u){if(!(e instanceof r))return n.apply(t,u);var o=Mn(n.prototype),i=n.apply(o,u);return _(i)?i:o}var rr=j((function(n,r){var t=rr.placeholder,e=function(){for(var u=0,o=r.length,i=Array(o),a=0;a1)ur(a,r-1,t,e),u=e.length;else for(var f=0,c=a.length;f0&&(t=r.apply(this,arguments)),n<=1&&(r=null),t}}var lr=rr(cr,2);function sr(n,r,t){r=qn(r,t);for(var e,u=nn(n),o=0,i=u.length;o0?0:u-1;o>=0&&o0?a=o>=0?o:Math.max(o+f,a):f=o>=0?Math.min(o+1,f):o+f+1;else if(t&&o&&f)return e[o=t(e,u)]===u?o:-1;if(u!=u)return(o=r(i.call(e,a,f),$))>=0?o+a:-1;for(o=n>0?a:f-1;o>=0&&o0?0:i-1;for(u||(e=r[o?o[a]:a],a+=n);a>=0&&a=3;return r(n,Fn(t,u,4),e,o)}}var Ar=wr(1),xr=wr(-1);function Sr(n,r,t){var e=[];return r=qn(r,t),jr(n,(function(n,t,u){r(n,t,u)&&e.push(n)})),e}function Or(n,r,t){r=qn(r,t);for(var e=!er(n)&&nn(n),u=(e||n).length,o=0;o=0}var Br=j((function(n,r,t){var e,u;return D(r)?u=r:(r=Nn(r),e=r.slice(0,-1),r=r[r.length-1]),_r(n,(function(n){var o=u;if(!o){if(e&&e.length&&(n=In(n,e)),null==n)return;o=n[r]}return null==o?o:o.apply(n,t)}))}));function Nr(n,r){return _r(n,Rn(r))}function Ir(n,r,t){var e,u,o=-1/0,i=-1/0;if(null==r||"number"==typeof r&&"object"!=typeof n[0]&&null!=n)for(var a=0,f=(n=er(n)?n:jn(n)).length;ao&&(o=e);else r=qn(r,t),jr(n,(function(n,t,e){((u=r(n,t,e))>i||u===-1/0&&o===-1/0)&&(o=n,i=u)}));return o}function Tr(n,r,t){if(null==r||t)return er(n)||(n=jn(n)),n[Wn(n.length-1)];var e=er(n)?En(n):jn(n),u=Y(e);r=Math.max(Math.min(r,u),0);for(var o=u-1,i=0;i1&&(e=Fn(e,r[1])),r=an(n)):(e=qr,r=ur(r,!1,!1),n=Object(n));for(var u=0,o=r.length;u1&&(t=r[1])):(r=_r(ur(r,!1,!1),String),e=function(n,t){return!Er(r,t)}),Ur(n,e,t)}));function zr(n,r,t){return i.call(n,0,Math.max(0,n.length-(null==r||t?1:r)))}function Lr(n,r,t){return null==n||n.length<1?null==r||t?void 0:[]:null==r||t?n[0]:zr(n,n.length-r)}function $r(n,r,t){return i.call(n,null==r||t?1:r)}var Cr=j((function(n,r){return r=ur(r,!0,!0),Sr(n,(function(n){return!Er(r,n)}))})),Kr=j((function(n,r){return Cr(n,r)}));function Jr(n,r,t,e){A(r)||(e=t,t=r,r=!1),null!=t&&(t=qn(t,e));for(var u=[],o=[],i=0,a=Y(n);ir?(e&&(clearTimeout(e),e=null),a=c,i=n.apply(u,o),e||(u=o=null)):e||!1===t.trailing||(e=setTimeout(f,l)),i};return c.cancel=function(){clearTimeout(e),a=0,e=u=o=null},c},debounce:function(n,r,t){var e,u,o,i,a,f=function(){var c=zn()-u;r>c?e=setTimeout(f,r-c):(e=null,t||(i=n.apply(a,o)),e||(o=a=null))},c=j((function(c){return a=this,o=c,u=zn(),e||(e=setTimeout(f,r),t&&(i=n.apply(a,o))),i}));return c.cancel=function(){clearTimeout(e),e=o=a=null},c},wrap:function(n,r){return rr(r,n)},negate:fr,compose:function(){var n=arguments,r=n.length-1;return function(){for(var t=r,e=n[r].apply(this,arguments);t--;)e=n[t].call(this,e);return e}},after:function(n,r){return function(){if(--n<1)return r.apply(this,arguments)}},before:cr,once:lr,findKey:sr,findIndex:vr,findLastIndex:hr,sortedIndex:yr,indexOf:gr,lastIndexOf:br,find:mr,detect:mr,findWhere:function(n,r){return mr(n,Dn(r))},each:jr,forEach:jr,map:_r,collect:_r,reduce:Ar,foldl:Ar,inject:Ar,reduceRight:xr,foldr:xr,filter:Sr,select:Sr,reject:function(n,r,t){return Sr(n,fr(qn(r)),t)},every:Or,all:Or,some:Mr,any:Mr,contains:Er,includes:Er,include:Er,invoke:Br,pluck:Nr,where:function(n,r){return Sr(n,Dn(r))},max:Ir,min:function(n,r,t){var e,u,o=1/0,i=1/0;if(null==r||"number"==typeof r&&"object"!=typeof n[0]&&null!=n)for(var a=0,f=(n=er(n)?n:jn(n)).length;ae||void 0===t)return 1;if(t - + - - + + - +
- +
- +

Index

E | M - +

E

@@ -67,7 +67,7 @@

M

- +
©2022, Adrien Wehrlé. - + | Powered by Sphinx 4.4.0 & Alabaster 0.7.12 - +
- - + + - \ No newline at end of file + diff --git a/docs/_build/html/index.html b/docs/_build/html/index.html index 1bef618..eba98a3 100644 --- a/docs/_build/html/index.html +++ b/docs/_build/html/index.html @@ -16,22 +16,22 @@ - + - - + + - +
- +
- +

Welcome to earthspy’s documentation!

@@ -52,7 +52,7 @@

Indices and tables @@ -104,18 +104,18 @@

Quick search

©2022, Adrien Wehrlé. - + | Powered by Sphinx 4.4.0 & Alabaster 0.7.12 - + | Page source
- - + + - \ No newline at end of file + diff --git a/docs/_build/html/search.html b/docs/_build/html/search.html index 6ccb10a..a5d9bab 100644 --- a/docs/_build/html/search.html +++ b/docs/_build/html/search.html @@ -8,7 +8,7 @@ Search — earthspy v0.1.0 documentation - + @@ -18,26 +18,26 @@ - - + + - - + + - +
- +
- +

Search

- + - - + +

Searching for multiple words only shows matches that contain all words.

- - + +
- - - + + +
- +
- +
- +
©2022, Adrien Wehrlé. - + | Powered by Sphinx 4.4.0 & Alabaster 0.7.12 - +
- - + + - \ No newline at end of file + diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index 7263ea1..4b1bedd 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({docnames:["api","generated/earthspy","index"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,sphinx:56},filenames:["api.rst","generated/earthspy.rst","index.rst"],objects:{"":[[1,0,0,"-","earthspy"]]},objnames:{"0":["py","module","Python module"]},objtypes:{"0":"py:module"},terms:{api:2,index:2,modul:2,page:2,search:2},titles:["API","earthspy","Welcome to earthspy\u2019s documentation!"],titleterms:{api:0,content:2,document:2,earthspi:[1,2],indic:2,s:2,tabl:2,welcom:2}}) \ No newline at end of file +Search.setIndex({docnames:["api","generated/earthspy","index"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,sphinx:56},filenames:["api.rst","generated/earthspy.rst","index.rst"],objects:{"":[[1,0,0,"-","earthspy"]]},objnames:{"0":["py","module","Python module"]},objtypes:{"0":"py:module"},terms:{api:2,index:2,modul:2,page:2,search:2},titles:["API","earthspy","Welcome to earthspy\u2019s documentation!"],titleterms:{api:0,content:2,document:2,earthspi:[1,2],indic:2,s:2,tabl:2,welcom:2}}) diff --git a/docs/api.rst b/docs/api.rst index ced4568..fa15c80 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -9,7 +9,7 @@ API :recursive: earthspy - + .. automodule earthspy diff --git a/docs/generated/earthspy.rst b/docs/generated/earthspy.rst index 9e1db12..efebd77 100644 --- a/docs/generated/earthspy.rst +++ b/docs/generated/earthspy.rst @@ -2,22 +2,3 @@ ======== .. automodule:: earthspy - - - - - - - - - - - - - - - - - - - diff --git a/docs/index.rst b/docs/index.rst index 4fbde63..8176788 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ .. earthspy documentation master file, created by sphinx-quickstart on Mon May 2 20:38:22 2022. You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. + contain the root ``toctree`` directive. Welcome to earthspy's documentation! ==================================== diff --git a/earthspy/__init__.py b/earthspy/__init__.py index 8b13789..e69de29 100644 --- a/earthspy/__init__.py +++ b/earthspy/__init__.py @@ -1 +0,0 @@ - diff --git a/earthspy/earthspy.py b/earthspy/earthspy.py index e1e1b5b..8da0556 100644 --- a/earthspy/earthspy.py +++ b/earthspy/earthspy.py @@ -27,8 +27,9 @@ class EarthSpy: - """Monitor and study any place on Earth and in Near Real-Time (NRT) using the - SentinelHub services. + """Monitor and study any place on Earth and in Near Real-Time + (NRT) using the SentinelHub services. + """ def __init__(self, CLIENT_credentials_file: str) -> None: @@ -49,8 +50,6 @@ def __init__(self, CLIENT_credentials_file: str) -> None: # setup connection self.configure_connection() - return None - def configure_connection(self) -> shb.SHConfig: """Build a shb configuration class for the connection to Sentinel Hub services. @@ -262,7 +261,12 @@ def get_available_data(self) -> list: # and update service_url if download failed try: # store metadata of available scenes - self.metadata = [list(iterator) for iterator in search_iterator] + self.metadata = {} + for iterator in search_iterator: + iterator_list = list(iterator) + if len(iterator_list) > 0: + date = iterator_list[0]["properties"]["datetime"].split("T")[0] + self.metadata[date] = iterator_list except shb.exceptions.DownloadFailedException: # set specific base URL of deployment @@ -284,7 +288,12 @@ def get_available_data(self) -> list: ] # store metadata of available scenes - self.metadata = [list(iterator) for iterator in search_iterator] + self.metadata = {} + for iterator in search_iterator: + iterator_list = list(iterator) + if len(iterator_list) > 0: + date = iterator_list[0]["properties"]["datetime"].split("T")[0] + self.metadata[date] = iterator_list # create date +-1 hour around acquisition time time_difference = timedelta(hours=1) @@ -344,7 +353,7 @@ def get_raw_data_collection_resolution(self) -> int: return self.raw_data_collection_resolution def set_number_of_cores(self, nb_cores) -> int: - """Set number of cores if not specificed by user. + """Set number of cores if not specified by user. :return: Number of cores to use in multithreading. :rtype: int @@ -377,7 +386,8 @@ def get_date_range( :rtype: pd.core.indexes.datetimes.DatetimeIndex """ - # if an integer, create a datetimeIndex with the number of days from present date + # if an integer, create a datetimeIndex with the number of days + # from present date if isinstance(time_interval, int): # keep time_interval positive if time_interval < 0: @@ -506,7 +516,7 @@ def get_store_folder(self, store_folder: Union[str, None]) -> str: return self.store_folder def convert_bounding_box_coordinates(self) -> Tuple[shb.geometry.BBox, list]: - """Convert bounding boxes coordinates to a Geodetic Parameter Dataset (EPSG) in + """Convert bounding box coordinates to a Geodetic Parameter Dataset (EPSG) in meter unit, default to EPSG:3413 (NSIDC Sea Ice Polar Stereographic North). @@ -642,7 +652,8 @@ def get_optimal_box_split(self) -> Tuple[int, int]: boxes_pixels_x = (dx / trial_split_boxes) / self.resolution boxes_pixels_y = (dy / trial_split_boxes) / self.resolution - # get minimum number of split boxes needed to stay below Sentinel Hub max dimensions + # get minimum number of split boxes needed to stay below Sentinel Hub + # max dimensions min_nb_boxes_x = int(trial_split_boxes[np.where(boxes_pixels_x <= 2500)[0][0]]) min_nb_boxes_y = int(trial_split_boxes[np.where(boxes_pixels_y <= 2500)[0][0]]) @@ -705,7 +716,7 @@ def set_split_boxes_ids(self) -> dict: """ # store split boxes ids in dict - self.split_boxes_ids = {i: sb for i, sb in enumerate(self.split_boxes)} + self.split_boxes_ids = dict(zip(range(len(self.split_boxes)), self.split_boxes)) return self.split_boxes_ids @@ -1096,17 +1107,22 @@ def merge_rasters(self) -> None: } ) + # update dictionary if compression set if self.raster_compression is not None: output_meta.update( { "compress":self.raster_compression } ) - + + # extract scene id + id_dict = {k: self.metadata[date][0][k] for k in ["id"]} + # write mosaic with rasterio.open(date_output_filename, "w", **output_meta) as dst: dst.write(mosaic) - + dst.update_tags(**id_dict) + # save file name of merged raster self.output_filenames_renamed.append(date_output_filename) diff --git a/earthspy/operational/earthspy_NRT.py b/earthspy/operational/earthspy_NRT.py index d24644e..ab859df 100644 --- a/earthspy/operational/earthspy_NRT.py +++ b/earthspy/operational/earthspy_NRT.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ @author: Adrien Wehrlé, EO-IO, University of Zurich, Switzerland diff --git a/environment.yml b/environment.yml index fce44eb..56402e9 100644 --- a/environment.yml +++ b/environment.yml @@ -1,11 +1,9 @@ name: earthspy channels: - - conda-forge/label/cf202003 - - https://conda.software.inl.gov/public - defaults - conda-forge dependencies: - - python=3 + - python>=3.9 - numpy - objectpath - pandas diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..2d62fed --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,20 @@ +# This file is auto-generated from dev-environment.yml, do not modify. +# See that file for comments about the need/usage of each dependency. + +numpy +objectpath +pandas +rasterio +validators +sentinelhub >= 3.4 +pip +flake8 +pytest +pytest-lazy-fixture +sphinx +sphinx_rtd_theme +pylint +sphinxcontrib-programoutput +sphinx-autodoc-typehints +pre-commit +sphinx-gallery diff --git a/setup.py b/setup.py index 8bf1ba9..6068493 100644 --- a/setup.py +++ b/setup.py @@ -1,2 +1,3 @@ from setuptools import setup + setup() diff --git a/tests/__init__.py b/tests/__init__.py index 8b13789..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +0,0 @@ - diff --git a/tests/test_earthspy.py b/tests/test_earthspy.py index ffe06c7..f0d3da6 100644 --- a/tests/test_earthspy.py +++ b/tests/test_earthspy.py @@ -1,18 +1,19 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- """ @author: Adrien Wehrlé, EO-IO, University of Zurich, Switzerland """ -import earthspy.earthspy as es -import numpy as np import os + +import numpy as np import pandas as pd import requests import sentinelhub as shb +import earthspy.earthspy as es + class TestEarthspy: # create local variables from environment secrets for convenience @@ -36,7 +37,8 @@ class TestEarthspy: // Set gain for visualisation let gain = 2.5; // Return RGB - return [sample.B04 * gain, sample.B03 * gain, sample.B02 * gain, sample.dataMask]; + return [sample.B04 * gain, sample.B03 * gain, sample.B02 * gain, + sample.dataMask]; } """ @@ -100,7 +102,7 @@ def test_init(self) -> None: assert self.t1.config.sh_client_secret == os.environ["SH_CLIENT_SECRET"] def test_set_query_parameters(self) -> None: - """Test direct attribute assignement.""" + """Test direct attribute assignment.""" # check if attributes were set accordingly assert self.t1.download_mode is not None @@ -249,9 +251,9 @@ def test_set_correct_resolution(self) -> None: r2 = self.t3.set_correct_resolution() # check that query resolution was set correctly - assert r1 == 10 + assert r2 == 11 # check that download mode was set correctly - assert isinstance(self.t1.download_mode, str) + assert isinstance(self.t3.download_mode, str) def test_list_requests(self) -> None: """Test request listing""" @@ -299,9 +301,9 @@ def test_set_split_boxes_ids(self) -> None: """Test split box ID generation""" sbi1 = self.t1.set_split_boxes_ids() - # check that split box ids were saved in dictionnary + # check that split box ids were saved in dictionary assert isinstance(sbi1, dict) - # check that dictionnary has the right shape + # check that dictionary has the right shape assert len(sbi1) == 4 def test_get_evaluation_script(self) -> None: