diff --git a/.gitignore b/.gitignore index 2be5118..8e68576 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,24 @@ *.tif +other/out/* +other/datasets/* +other/old/* +src/examples/getHagenBrae.py +src/examples/analyse.py +src/examples/getHagenBrae_dask.py +src/examples/getMargin.py +src/examples/getMarginGIMP.py +src/examples/getTimeSeries.py +src/examples/margin.py +src/examples/process_old.py +src/examples/sar_water.py +*.docx +src/griml/examples/analyse.py +src/griml/examples/getHagenBrae.py +src/griml/examples/getHagenBrae_dask.py +src/griml/examples/getMargin.py +src/griml/examples/getMarginGIMP.py +src/griml/examples/getTimeSeries.py +src/griml/examples/margin.py +src/griml/examples/process_old.py +src/griml/examples/sar_water.py diff --git a/README.md b/README.md index 59506be..83fdc5d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # Investigating Greenland's ice marginal lakes under a changing climate (GrIML) +[![PyPI version](https://badge.fury.io/py/griml.svg)](https://badge.fury.io/py/griml) +[![Documentation Status](https://readthedocs.org/projects/griml/badge/?version=latest)](https://griml.readthedocs.io/en/latest/?badge=latest) + A repository for all project-related materials, funded under the ESA Living Planet Fellowship. **Project aim:** To examine ice marginal lake changes across Greenland using a multi-method and multi-sensor remote sensing approach, refined with in situ validation. diff --git a/dist/griml-0.0.1-py3-none-any.whl b/dist/griml-0.0.1-py3-none-any.whl new file mode 100644 index 0000000..b01b9ba Binary files /dev/null and b/dist/griml-0.0.1-py3-none-any.whl differ diff --git a/dist/griml-0.0.1.tar.gz b/dist/griml-0.0.1.tar.gz new file mode 100644 index 0000000..31bbdf9 Binary files /dev/null and b/dist/griml-0.0.1.tar.gz differ diff --git a/docs/requirements.txt b/docs/requirements.txt index 4c5e642..c82983c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,6 @@ ee geopandas +griml numpy pandas scipy diff --git a/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_filtered.dbf b/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_filtered.dbf index 2926172..77cef9a 100644 Binary files a/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_filtered.dbf and b/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_filtered.dbf differ diff --git a/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_filtered.shp b/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_filtered.shp index 30ca781..e4ab2d0 100644 Binary files a/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_filtered.shp and b/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_filtered.shp differ diff --git a/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_filtered.shx b/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_filtered.shx index a599f10..526a4b6 100644 Binary files a/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_filtered.shx and b/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_filtered.shx differ diff --git a/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.dbf b/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.dbf index 25a0427..7ade172 100644 Binary files a/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.dbf and b/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.dbf differ diff --git a/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.prj b/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.prj index 212c2e0..2e49174 100644 --- a/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.prj +++ b/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.prj @@ -1 +1 @@ -PROJCS["WGS_1984_UTM_Zone_24N",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",-39.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]] \ No newline at end of file +PROJCS["WGS_1984_NSIDC_Sea_Ice_Polar_Stereographic_North",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Stereographic_North_Pole"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",-45.0],PARAMETER["Standard_Parallel_1",70.0],UNIT["Meter",1.0]] \ No newline at end of file diff --git a/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.shp b/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.shp index 6198193..5848742 100644 Binary files a/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.shp and b/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.shp differ diff --git a/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.shx b/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.shx index 63e88f6..44cc651 100644 Binary files a/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.shx and b/other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.shx differ diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..ad51564 --- /dev/null +++ b/setup.py @@ -0,0 +1,32 @@ +import setuptools + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + +setuptools.setup( + name="griml", + version="0.0.1", + author="Penelope How", + author_email="pho@geus.dk", + description="A workflow for classifying lakes from satellite imagery and compiling lake inventories", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/PennyHow/GrIML", + project_urls={ + "Bug Tracker": "https://github.com/PennyHow/GrIML/issues", + }, + keywords="glaciology ice lake ESA", + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "Natural Language :: English", + "Topic :: Scientific/Engineering", + "Operating System :: OS Independent", + ], + package_dir={"": "src"}, + packages=setuptools.find_packages(where="src"), + python_requires=">=3.6", + install_requires=['earthengine-api', 'geopandas', 'numpy', 'pandas', 'scipy', 'Shapely'], +) diff --git a/src/README.md b/src/README.md index d6052c1..1b039ed 100644 --- a/src/README.md +++ b/src/README.md @@ -1,30 +1,45 @@ -# GrIML workflow +# The GrIML python package -This space is for storing and documenting GrIML's processing chain, primarily programmed in Python. +[![PyPI version](https://badge.fury.io/py/griml.svg)](https://badge.fury.io/py/griml) +[![Documentation Status](https://readthedocs.org/projects/griml/badge/?version=latest)](https://griml.readthedocs.io/en/latest/?badge=latest) -## Proposed workflow +A GrIML workflow for classifying water bodies from satellite imagery using a multi-sensor, multi-method approach. This workflow is part of the ESA GrIML project. -GrIML will build upon existing workflows from the ESA Glaciers CCI (Option 6, An Inventory of Ice-Marginal Lakes in Greenland), refined here to form a unified processing chain that will be shared openly on Github and pip. -The proposed GrIML workflow. +## Quickstart + +The GrIML package can be installed using pip: + +```python +pip install griml +``` +Or cloned from the Github repository: + +```python +git clone https://github.com/PennyHow/GrIML +``` -## Online processing -It is intended to perform the satellite data retrieval and basic binary classification of water bodies with cloud processing and/or in-memory processing. By doing so, the workflow will avoid the handling of heavy data downloads. +## Workflow + +GrIML builds on the existing workflows from the ESA Glaciers CCI (Option 6, An Inventory of Ice-Marginal Lakes in Greenland), refined here to form a unified processing chain that is shared openly on Github and pip. + +The proposed GrIML workflow. + -Online processing strategies will include: +## Cloud processing -+ Using cloud processing APIs such as the Google Earth Engine Python API -+ Utilising data retrieval from urls, using readily available functions like this +Primary processing is performed using the Google Earth Engine Python API, including satellite data retrieval and binary classification from multiple sensors. By doing so, the workflow avoids the handling of heavy data downloads and operations. -Subject to funding, add-on modules to the workflow will take advantage of the cloud processing capabilities provided by the SentinelHub APIs. SentinelHub is a cloud processing platform that can be used to retrieve and process data from many satellite products. +Subject to funding, it is intended to include add-on modules to the workflow, which take advantage of the cloud processing capabilities provided by the SentinelHub APIs. SentinelHub is a cloud processing platform that can be used to retrieve and process data from many satellite products. -By having the option to retrieve data from URL or SentinelHub, I envisage that the workflow can be used by all regardless of whether they have a paid license for SentinelHub or not. ## Offline processing Key Python packages that will be used in the offline components of the workflow: -+ [xarray](https://xarray.pydata.org/en/stable/) - for large data handling and parralel processing -+ [rasterio](https://rasterio.readthedocs.io/en/latest/) - for raster processing -+ [geopandas](https://geopandas.org/en/stable/) - for vector data handling ++ [geopandas](https://geopandas.org/en/stable/) - for vector dataset handling ++ ['numpy'](https://numpy.org/) - for numerical operations ++ ['pandas'](https://pandas.pydata.org/) - for dataframe handling ++ ['scipy'](https://docs.scipy.org/doc/scipy/index.html) - for matrix operations ++ ['shapely'](https://shapely.readthedocs.io/en/stable/manual.html) - for geometric operations diff --git a/src/griml.egg-info/PKG-INFO b/src/griml.egg-info/PKG-INFO new file mode 100644 index 0000000..f83ffdc --- /dev/null +++ b/src/griml.egg-info/PKG-INFO @@ -0,0 +1,76 @@ +Metadata-Version: 2.1 +Name: griml +Version: 0.0.1 +Summary: A workflow for classifying lakes from satellite imagery and compiling lake inventories +Home-page: https://github.com/PennyHow/GrIML +Author: Penelope How +Author-email: pho@geus.dk +License: UNKNOWN +Project-URL: Bug Tracker, https://github.com/PennyHow/GrIML/issues +Keywords: glaciology ice lake ESA +Platform: UNKNOWN +Classifier: Programming Language :: Python :: 3 +Classifier: License :: OSI Approved :: MIT License +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Science/Research +Classifier: Natural Language :: English +Classifier: Topic :: Scientific/Engineering +Classifier: Operating System :: OS Independent +Requires-Python: >=3.6 +Description-Content-Type: text/markdown +License-File: LICENSE + +# Investigating Greenland's ice marginal lakes under a changing climate (GrIML) + +[![Documentation Status](https://readthedocs.org/projects/griml/badge/?version=latest)](https://griml.readthedocs.io/en/latest/?badge=latest) + +A repository for all project-related materials, funded under the ESA Living Planet Fellowship. + +**Project aim:** To examine ice marginal lake changes across Greenland using a multi-method and multi-sensor remote sensing approach, refined with in situ validation. + +## Background + + + +Sea level is predicted to rise drastically by 2100, with significant contribution from the melting of the Greenland Ice Sheet (GrIS). In these predictions, melt runoff is assumed to contribute directly to sea level change, with little consideration for meltwater storage at the terrestrial margin of the ice sheet; such as ice marginal lakes. + +In 2017, 3347 ice marginal lakes were identified in Greenland along the ice margin (How et al., 2021, see map figure for all mapped lakes). Globally, these ice marginal lakes hold up to 0.43 mm of sea level equivalent, which could have a marked impact on future predictions (Shugar et al., 2021). Therefore, they need to be monitored to understand how changes in ice marginal lake water storage affect melt contribution, and how their dynamics evolve under a changing climate. + +**GrIML** proposes to examine ice marginal lake changes across Greenland using a multi-sensor and multi-method remote sensing approach to better address their influence on sea level contribution forecasting. + +1. Greenland-wide inventories of ice marginal lakes will be generated for selected years during the satellite era, building upon established classification methods in a unified cloud processing workflow + +2. Detailed time-series analysis will be conducted on chosen ice marginal lakes to assess changes in their flooding dynamics; focusing on lakes with societal and scientific importance + +3. The findings from this work will be validated against in situ observations - namely hydrological measurements and terrestrial time-lapse images - to evaluate whether the remote sensing workflow adequately captures ice marginal lake dynamics + + +## Methodology + +Ice marginal lakes will be detected using a remote sensing approach, based on offline workflows developed within the ESA Glaciers CCI (Option 6, An Inventory of Ice-Marginal Lakes in Greenland) (see workflow below). Lake extents were defined through a multi-sensor approach, using multi-spectral indices classification from Sentinel-2 optical imagery, backscatter classification from Sentinel-1 SAR (synthetic aperture radar) imagery, and sink detection from ArcticDEM digital elevation models (How et al., 2021). + +The proposed GrIML workflow. + +The intent in GrIML is to build upon this pre-existing workflow with new and innovative solutions to form a unified and automated processing chain, with the option to integrate it into a cloud processing platform for efficient big data analysis. + +These developments will alleviate the current challenges associated with data-heavy processing (i.e. multi-sensor integration and data retrieval), and ensure detection accuracy with a merged and collated set of established methodologies. These developments will include: + +1. Incorporation of the multi-spectral indices classification, backscatter classification, and sink detection methods into one unified processing chain + +2. Integration of image analysis from additional sensors into the processing chain, such as Landsat 4-8 + +3. Automisation of post-processing filtering to remove detached water bodies and misclassifications, using the already-existing 2017 inventory (How et al., 2021) as a training dataset to automatically match and retain existing lakes + +4. Adaptation of the workflow for easy transference between offline and cloud processing platforms + +## Links + +- ESA project outline and fellow information + +- Information about the ESA Living Planet Fellowship + +- GrIML project description + +- 2017 ice marginal lake inventory Scientific Reports paper and dataset + + diff --git a/src/griml.egg-info/SOURCES.txt b/src/griml.egg-info/SOURCES.txt new file mode 100644 index 0000000..b38c4c2 --- /dev/null +++ b/src/griml.egg-info/SOURCES.txt @@ -0,0 +1,15 @@ +LICENSE +README.md +setup.py +src/griml/__init__.py +src/griml/dem.py +src/griml/lake.py +src/griml/process.py +src/griml/retrieve.py +src/griml/sar.py +src/griml/vis.py +src/griml.egg-info/PKG-INFO +src/griml.egg-info/SOURCES.txt +src/griml.egg-info/dependency_links.txt +src/griml.egg-info/requires.txt +src/griml.egg-info/top_level.txt \ No newline at end of file diff --git a/src/griml.egg-info/dependency_links.txt b/src/griml.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/griml.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/src/griml.egg-info/requires.txt b/src/griml.egg-info/requires.txt new file mode 100644 index 0000000..fe00572 --- /dev/null +++ b/src/griml.egg-info/requires.txt @@ -0,0 +1,6 @@ +earthengine-api +geopandas +numpy +pandas +scipy +Shapely diff --git a/src/griml.egg-info/top_level.txt b/src/griml.egg-info/top_level.txt new file mode 100644 index 0000000..ba4ef6c --- /dev/null +++ b/src/griml.egg-info/top_level.txt @@ -0,0 +1 @@ +griml diff --git a/src/__init__.py b/src/griml/__init__.py similarity index 100% rename from src/__init__.py rename to src/griml/__init__.py diff --git a/src/dem.py b/src/griml/dem.py similarity index 100% rename from src/dem.py rename to src/griml/dem.py diff --git a/src/environment.yml b/src/griml/environment.yml similarity index 100% rename from src/environment.yml rename to src/griml/environment.yml diff --git a/src/examples/getInventory_funcs.py b/src/griml/examples/getInventory_funcs.py similarity index 100% rename from src/examples/getInventory_funcs.py rename to src/griml/examples/getInventory_funcs.py diff --git a/src/examples/getInventory_obj.py b/src/griml/examples/getInventory_obj.py similarity index 100% rename from src/examples/getInventory_obj.py rename to src/griml/examples/getInventory_obj.py diff --git a/src/lake.py b/src/griml/lake.py similarity index 100% rename from src/lake.py rename to src/griml/lake.py diff --git a/src/process.py b/src/griml/process.py similarity index 93% rename from src/process.py rename to src/griml/process.py index dbf040f..c6300d5 100644 --- a/src/process.py +++ b/src/griml/process.py @@ -5,14 +5,23 @@ """ import ee -from retrieve import getScenes, getScene, getInt, getSmooth, getMosaic, \ - getMean, maskImage, splitBBox, getFeatures, getFeaturesSplit -from sar import filterSARscenes, classifySARimage -from vis import filterS2scenes, maskS2clouds, filterLSscenes, maskL8clouds, \ - maskL7clouds, getNDWI, getMNDWI, getAWEISH, getAWEINSH, getBRIGHT, \ - getClassification -from dem import getElevation, getSinks +try: + from griml.retrieve import getScenes, getScene, getInt, getSmooth, getMosaic, \ + getMean, maskImage, splitBBox, getFeatures, getFeaturesSplit + from griml.sar import filterSARscenes, classifySARimage + from griml.vis import filterS2scenes, maskS2clouds, filterLSscenes, maskL8clouds, \ + maskL7clouds, getNDWI, getMNDWI, getAWEISH, getAWEINSH, getBRIGHT, \ + getClassification + from griml.dem import getElevation, getSinks +except: + from retrieve import getScenes, getScene, getInt, getSmooth, getMosaic, \ + getMean, maskImage, splitBBox, getFeatures, getFeaturesSplit + from sar import filterSARscenes, classifySARimage + from vis import filterS2scenes, maskS2clouds, filterLSscenes, maskL8clouds, \ + maskL7clouds, getNDWI, getMNDWI, getAWEISH, getAWEINSH, getBRIGHT, \ + getClassification + from dem import getElevation, getSinks class gee(object): '''Class object for rigid classification workflows using the diff --git a/src/retrieve.py b/src/griml/retrieve.py similarity index 100% rename from src/retrieve.py rename to src/griml/retrieve.py diff --git a/src/sar.py b/src/griml/sar.py similarity index 100% rename from src/sar.py rename to src/griml/sar.py diff --git a/src/sentinelhub_addon/detectIceMargin.py b/src/griml/sentinelhub_addon/detectIceMargin.py similarity index 100% rename from src/sentinelhub_addon/detectIceMargin.py rename to src/griml/sentinelhub_addon/detectIceMargin.py diff --git a/src/sentinelhub_addon/loadURL.py b/src/griml/sentinelhub_addon/loadURL.py similarity index 100% rename from src/sentinelhub_addon/loadURL.py rename to src/griml/sentinelhub_addon/loadURL.py diff --git a/src/sentinelhub_addon/retrieveSentinelHub.py b/src/griml/sentinelhub_addon/retrieveSentinelHub.py similarity index 100% rename from src/sentinelhub_addon/retrieveSentinelHub.py rename to src/griml/sentinelhub_addon/retrieveSentinelHub.py diff --git a/src/sentinelhub_addon/sentinelhub_test.py b/src/griml/sentinelhub_addon/sentinelhub_test.py similarity index 100% rename from src/sentinelhub_addon/sentinelhub_test.py rename to src/griml/sentinelhub_addon/sentinelhub_test.py diff --git a/src/vis.py b/src/griml/vis.py similarity index 99% rename from src/vis.py rename to src/griml/vis.py index 238409d..7622143 100644 --- a/src/vis.py +++ b/src/griml/vis.py @@ -300,6 +300,8 @@ def getClassification(ndwi, mndwi, aweish, aweinsh, bright): # ndwi_t, mndwi_t, aweish_t1, aweish_t2, aweinsh_t1, aweinsh_t2, bright_t): '''Generate classification from thresholded multi-spectral indices + Parameters + ---------- ndwi : ee.Image NDWI band mndwi : ee.Image @@ -310,6 +312,11 @@ def getClassification(ndwi, mndwi, aweish, aweinsh, bright): AWEInsh band bright : ee.Image BRIGHT band + + Returns + ------- + ee.Image + Binary classification image ''' classified = ee.Image().expression("(BRIGHT > 5000) ? 0" ": (NDWI > 0.3) ? 1 " @@ -639,4 +646,4 @@ def testLS(self): self.assertIsNotNone(scenes.getInfo()) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main()