Merge pull request #2 from PennyHow/dev

Readthedocs and pypi distribution merge

.gitignore

@@ -1,2 +1,24 @@
 
 *.tif
+other/out/*
+other/datasets/*
+other/old/*
+src/examples/getHagenBrae.py
+src/examples/analyse.py
+src/examples/getHagenBrae_dask.py
+src/examples/getMargin.py
+src/examples/getMarginGIMP.py
+src/examples/getTimeSeries.py
+src/examples/margin.py
+src/examples/process_old.py
+src/examples/sar_water.py
+*.docx
+src/griml/examples/analyse.py
+src/griml/examples/getHagenBrae.py
+src/griml/examples/getHagenBrae_dask.py
+src/griml/examples/getMargin.py
+src/griml/examples/getMarginGIMP.py
+src/griml/examples/getTimeSeries.py
+src/griml/examples/margin.py
+src/griml/examples/process_old.py
+src/griml/examples/sar_water.py

README.md

@@ -1,5 +1,8 @@
 # Investigating Greenland's ice marginal lakes under a changing climate (GrIML)
 
+[![PyPI version](https://badge.fury.io/py/griml.svg)](https://badge.fury.io/py/griml)
+[![Documentation Status](https://readthedocs.org/projects/griml/badge/?version=latest)](https://griml.readthedocs.io/en/latest/?badge=latest)
+
 A repository for all project-related materials, funded under the ESA Living Planet Fellowship.
 
 **Project aim:** To examine ice marginal lake changes across Greenland using a multi-method and multi-sensor remote sensing approach, refined with in situ validation.

docs/requirements.txt

@@ -1,5 +1,6 @@
 ee
 geopandas
+griml
 numpy
 pandas
 scipy

other/out/iiml_2017-07-01_2017-08-31_-50.94_67.98_unfiltered.prj

@@ -1 +1 @@
-PROJCS["WGS_1984_UTM_Zone_24N",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",-39.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]]
+PROJCS["WGS_1984_NSIDC_Sea_Ice_Polar_Stereographic_North",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Stereographic_North_Pole"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",-45.0],PARAMETER["Standard_Parallel_1",70.0],UNIT["Meter",1.0]]

setup.py

@@ -0,0 +1,32 @@
+import setuptools
+
+with open("README.md", "r", encoding="utf-8") as fh:
+    long_description = fh.read()
+
+setuptools.setup(
+    name="griml",
+    version="0.0.1",
+    author="Penelope How",
+    author_email="pho@geus.dk",
+    description="A workflow for classifying lakes from satellite imagery and compiling lake inventories",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/PennyHow/GrIML",
+    project_urls={
+        "Bug Tracker": "https://github.com/PennyHow/GrIML/issues",
+    },
+    keywords="glaciology ice lake ESA",
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Development Status :: 4 - Beta",
+        "Intended Audience :: Science/Research",
+        "Natural Language :: English",
+        "Topic :: Scientific/Engineering",
+        "Operating System :: OS Independent",
+    ],
+    package_dir={"": "src"},
+    packages=setuptools.find_packages(where="src"),
+    python_requires=">=3.6",
+    install_requires=['earthengine-api', 'geopandas', 'numpy', 'pandas', 'scipy', 'Shapely'],
+)

src/README.md

@@ -1,30 +1,45 @@
-# GrIML workflow
+# The GrIML python package
 
-This space is for storing and documenting GrIML's processing chain, primarily programmed in Python.
+[![PyPI version](https://badge.fury.io/py/griml.svg)](https://badge.fury.io/py/griml)
+[![Documentation Status](https://readthedocs.org/projects/griml/badge/?version=latest)](https://griml.readthedocs.io/en/latest/?badge=latest)
 
-## Proposed workflow
+A GrIML workflow for classifying water bodies from satellite imagery using a multi-sensor, multi-method approach. This workflow is part of the <a href="https://eo4society.esa.int/projects/griml/">ESA GrIML project</a>.
 
-GrIML will build upon existing workflows from the <a href="https://catalogue.ceda.ac.uk/uuid/7ea7540135f441369716ef867d217519">ESA Glaciers CCI</a> (Option 6, An Inventory of Ice-Marginal Lakes in Greenland), refined here to form a unified processing chain that will be shared openly on Github and pip.
 
-<img src="https://github.com/PennyHow/pennyhow.github.io/blob/master/assets/images/griml_workflow.png?raw=true" alt="The proposed GrIML workflow." width="1500" align="aligncenter">
+## Quickstart
+
+The GrIML package can be installed using pip: 
+
+```python
+pip install griml
+```
+Or cloned from the Github repository: 
+
+```python
+git clone https://github.com/PennyHow/GrIML
+```
 
-## Online processing
 
-It is intended to perform the satellite data retrieval and basic binary classification of water bodies with cloud processing and/or in-memory processing. By doing so, the workflow will avoid the handling of heavy data downloads. 
+## Workflow
+
+GrIML builds on the existing workflows from the <a href="https://catalogue.ceda.ac.uk/uuid/7ea7540135f441369716ef867d217519">ESA Glaciers CCI</a> (Option 6, An Inventory of Ice-Marginal Lakes in Greenland), refined here to form a unified processing chain that is shared openly on Github and pip.
+
+<img src="https://github.com/PennyHow/pennyhow.github.io/blob/master/assets/images/griml_workflow.png?raw=true" alt="The proposed GrIML workflow." width="1500" align="aligncenter">
+
 
-Online processing strategies will include:
+## Cloud processing
 
-+ Using cloud processing APIs such as the <a href="https://developers.google.com/earth-engine/guides/python_install">Google Earth Engine Python API</a>
-+ Utilising data retrieval from urls, using readily available functions like <a href="https://rasterio.readthedocs.io/en/latest/api/rasterio.html?highlight=URL#rasterio.open">this</a>
+Primary processing is performed using the <a href="https://developers.google.com/earth-engine/guides/python_install">Google Earth Engine Python API</a>, including satellite data retrieval and binary classification from multiple sensors. By doing so, the workflow avoids the handling of heavy data downloads and operations. 
 
-Subject to funding, add-on modules to the workflow will take advantage of the cloud processing capabilities provided by the <a href="https://sentinelhub-py.readthedocs.io/en/latest/">SentinelHub APIs</a>. SentinelHub is a cloud processing platform that can be used to retrieve and process data from many satellite products.
+Subject to funding, it is intended to include add-on modules to the workflow, which take advantage of the cloud processing capabilities provided by the <a href="https://sentinelhub-py.readthedocs.io/en/latest/">SentinelHub APIs</a>. SentinelHub is a cloud processing platform that can be used to retrieve and process data from many satellite products.
 
-By having the option to retrieve data from URL or SentinelHub, I envisage that the workflow can be used by all regardless of whether they have a paid license for SentinelHub or not.
 
 ## Offline processing
 
 Key Python packages that will be used in the offline components of the workflow:
 
-+ [xarray](https://xarray.pydata.org/en/stable/) - for large data handling and parralel processing
-+ [rasterio](https://rasterio.readthedocs.io/en/latest/) - for raster processing
-+ [geopandas](https://geopandas.org/en/stable/) - for vector data handling
++ [geopandas](https://geopandas.org/en/stable/) - for vector dataset handling
++ ['numpy'](https://numpy.org/) - for numerical operations
++ ['pandas'](https://pandas.pydata.org/) - for dataframe handling
++ ['scipy'](https://docs.scipy.org/doc/scipy/index.html) - for matrix operations
++ ['shapely'](https://shapely.readthedocs.io/en/stable/manual.html) - for geometric operations

src/griml.egg-info/PKG-INFO

@@ -0,0 +1,76 @@
+Metadata-Version: 2.1
+Name: griml
+Version: 0.0.1
+Summary: A workflow for classifying lakes from satellite imagery and compiling lake inventories
+Home-page: https://github.com/PennyHow/GrIML
+Author: Penelope How
+Author-email: pho@geus.dk
+License: UNKNOWN
+Project-URL: Bug Tracker, https://github.com/PennyHow/GrIML/issues
+Keywords: glaciology ice lake ESA
+Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Natural Language :: English
+Classifier: Topic :: Scientific/Engineering
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# Investigating Greenland's ice marginal lakes under a changing climate (GrIML)
+
+[![Documentation Status](https://readthedocs.org/projects/griml/badge/?version=latest)](https://griml.readthedocs.io/en/latest/?badge=latest)
+
+A repository for all project-related materials, funded under the ESA Living Planet Fellowship.
+
+**Project aim:** To examine ice marginal lake changes across Greenland using a multi-method and multi-sensor remote sensing approach, refined with in situ validation.
+
+## Background
+
+<img src="https://media.springernature.com/full/springer-static/image/art%3A10.1038%2Fs41598-021-83509-1/MediaObjects/41598_2021_83509_Fig1_HTML.png?raw=true" align="right" width="400">
+
+Sea level is predicted to rise drastically by 2100, with significant contribution from the melting of the Greenland Ice Sheet (GrIS). In these predictions, melt runoff is assumed to contribute directly to sea level change, with little consideration for meltwater storage at the terrestrial margin of the ice sheet; such as ice marginal lakes. 
+
+In 2017, 3347 ice marginal lakes were identified in Greenland along the ice margin (<a href="https://www.nature.com/articles/s41598-021-83509-1">How et al., 2021</a>, see map figure for all mapped lakes). Globally, these ice marginal lakes hold up to 0.43 mm of sea level equivalent, which could have a marked impact on future predictions (<a href="https://www.nature.com/articles/s41558-020-0855-4">Shugar et al., 2021</a>). Therefore, they need to be monitored to understand how changes in ice marginal lake water storage affect melt contribution, and how their dynamics evolve under a changing climate.
+
+**GrIML** proposes to examine ice marginal lake changes across Greenland using a multi-sensor and multi-method remote sensing approach to better address their influence on sea level contribution forecasting.
+
+1. Greenland-wide inventories of ice marginal lakes will be generated for selected years during the satellite era, building upon established classification methods in a unified cloud processing workflow
+
+2. Detailed time-series analysis will be conducted on chosen ice marginal lakes to assess changes in their flooding dynamics; focusing on lakes with societal and scientific importance
+
+3. The findings from this work will be validated against in situ observations - namely hydrological measurements and terrestrial time-lapse images - to evaluate whether the remote sensing workflow adequately captures ice marginal lake dynamics
+
+
+## Methodology
+
+Ice marginal lakes will be detected using a remote sensing approach, based on offline workflows developed within the <a href="https://catalogue.ceda.ac.uk/uuid/7ea7540135f441369716ef867d217519">ESA Glaciers CCI</a> (Option 6, An Inventory of Ice-Marginal Lakes in Greenland) (see workflow below). Lake extents were defined through a multi-sensor approach, using multi-spectral indices classification from Sentinel-2 optical imagery, backscatter classification from Sentinel-1 SAR (synthetic aperture radar) imagery, and sink detection from ArcticDEM digital elevation models (<a href="https://www.nature.com/articles/s41598-021-83509-1">How et al., 2021</a>). 
+
+<img src="https://github.com/PennyHow/pennyhow.github.io/blob/master/assets/images/griml_workflow.png?raw=true" alt="The proposed GrIML workflow." width="1500" align="aligncenter" />
+
+The intent in GrIML is to build upon this pre-existing workflow with new and innovative solutions to form a unified and automated processing chain, with the option to integrate it into a cloud processing platform for efficient big data analysis. 
+
+These developments will alleviate the current challenges associated with data-heavy processing (i.e. multi-sensor integration and data retrieval), and ensure detection accuracy with a merged and collated set of established methodologies. These developments will include:
+
+1. Incorporation of the multi-spectral indices classification, backscatter classification, and sink detection methods into one unified processing chain
+
+2. Integration of image analysis from additional sensors into the processing chain, such as Landsat 4-8
+
+3. Automisation of post-processing filtering to remove detached water bodies and misclassifications, using the already-existing 2017 inventory (How et al., 2021) as a training dataset to automatically match and retain existing lakes 
+
+4. Adaptation of the workflow for easy transference between offline and cloud processing platforms
+
+## Links
+
+- ESA <a href="https://eo4society.esa.int/projects/griml/">project outline</a> and <a href="https://eo4society.esa.int/lpf/penelope-how/">fellow information</a>
+
+- Information about the <a href="https://eo4society.esa.int/communities/scientists/living-planet-fellowship/">ESA Living Planet Fellowship</a>
+
+- <a href="https://pennyhow.github.io/blog/investigating-griml/">GrIML project description</a>
+
+- 2017 ice marginal lake inventory <a href="https://www.nature.com/articles/s41598-021-83509-1">Scientific Reports paper</a> and <a href="https://catalogue.ceda.ac.uk/uuid/7ea7540135f441369716ef867d217519">dataset</a>
+
+

src/griml.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+LICENSE
+README.md
+setup.py
+src/griml/__init__.py
+src/griml/dem.py
+src/griml/lake.py
+src/griml/process.py
+src/griml/retrieve.py
+src/griml/sar.py
+src/griml/vis.py
+src/griml.egg-info/PKG-INFO
+src/griml.egg-info/SOURCES.txt
+src/griml.egg-info/dependency_links.txt
+src/griml.egg-info/requires.txt
+src/griml.egg-info/top_level.txt

src/griml.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

src/griml.egg-info/requires.txt

@@ -0,0 +1,6 @@
+earthengine-api
+geopandas
+numpy
+pandas
+scipy
+Shapely

src/griml.egg-info/top_level.txt

@@ -0,0 +1 @@
+griml

src/process.py → src/griml/process.py

@@ -5,14 +5,23 @@
 """
 
 import ee
-from retrieve import getScenes, getScene, getInt, getSmooth, getMosaic, \
-    getMean, maskImage, splitBBox, getFeatures, getFeaturesSplit
-from sar import filterSARscenes, classifySARimage
-from vis import filterS2scenes, maskS2clouds,  filterLSscenes, maskL8clouds, \
-    maskL7clouds, getNDWI, getMNDWI, getAWEISH, getAWEINSH, getBRIGHT, \
-    getClassification
-from dem import getElevation, getSinks 
 
+try:
+    from griml.retrieve import getScenes, getScene, getInt, getSmooth, getMosaic, \
+        getMean, maskImage, splitBBox, getFeatures, getFeaturesSplit
+    from griml.sar import filterSARscenes, classifySARimage
+    from griml.vis import filterS2scenes, maskS2clouds,  filterLSscenes, maskL8clouds, \
+        maskL7clouds, getNDWI, getMNDWI, getAWEISH, getAWEINSH, getBRIGHT, \
+        getClassification
+    from griml.dem import getElevation, getSinks 
+except:
+    from retrieve import getScenes, getScene, getInt, getSmooth, getMosaic, \
+        getMean, maskImage, splitBBox, getFeatures, getFeaturesSplit
+    from sar import filterSARscenes, classifySARimage
+    from vis import filterS2scenes, maskS2clouds,  filterLSscenes, maskL8clouds, \
+        maskL7clouds, getNDWI, getMNDWI, getAWEISH, getAWEINSH, getBRIGHT, \
+        getClassification
+    from dem import getElevation, getSinks 
 
 class gee(object):
     '''Class object for rigid classification workflows using the 

src/vis.py → src/griml/vis.py

@@ -300,6 +300,8 @@ def getClassification(ndwi, mndwi, aweish, aweinsh, bright):
                       # ndwi_t, mndwi_t, aweish_t1, aweish_t2, aweinsh_t1, aweinsh_t2, bright_t):  
     '''Generate classification from thresholded multi-spectral indices
     
+    Parameters
+    ----------
     ndwi : ee.Image
         NDWI band
     mndwi : ee.Image
@@ -310,6 +312,11 @@ def getClassification(ndwi, mndwi, aweish, aweinsh, bright):
         AWEInsh band
     bright : ee.Image
         BRIGHT band
+        
+    Returns
+    -------
+    ee.Image
+    	Binary classification image
     '''
     classified =  ee.Image().expression("(BRIGHT > 5000) ? 0"
                                         ": (NDWI > 0.3) ? 1 "
@@ -639,4 +646,4 @@ def testLS(self):
         self.assertIsNotNone(scenes.getInfo())                 
 
 if __name__ == "__main__": 
-    unittest.main() 
+    unittest.main()