Merge pull request #6 from mapbox/kwarg-be-gone

Kwarg be gone

Author: dnomadb

README.md

@@ -27,7 +27,7 @@ pip install -e .
 with riomucho.RioMucho([{inputs}], {output}, {run function},
     windows={windows},
     global_args={global arguments}, 
-    kwargs={kwargs to write}) as rios:
+    options={options to write}) as rios:
 
     rios.run({processes})
 ```
@@ -80,9 +80,9 @@ global_args = {
 }
 ```
 
-#### `kwargs={keyword args}`
+#### `options={keyword args}`
 
-The kwargs to pass to the output. `[Default = srcs[0].kwargs`
+The options to pass to the writing output. `[Default = srcs[0].meta`
 
 ## Example
 
@@ -100,9 +100,9 @@ def basic_run(data, window, ij, g_args):
 with rasterio.open('/tmp/test_1.tif') as src:
     ## grabbing the windows as an example. Default behavior is identical.
     windows = [[window, ij] for ij, window in src.block_windows()]
-    kwargs = src.meta
+    options = src.meta
     # since we are only writing to 2 bands
-    kwargs.update(count=2)
+    options.update(count=2)
 
 global_args = {
     'divide': 2
@@ -114,7 +114,7 @@ processes = 4
 with riomucho.RioMucho(['input1.tif','input2.tif'], 'output.tif', basic_run,
     windows=windows,
     global_args=global_args, 
-    kwargs=kwargs) as rm:
+    options=options) as rm:
 
     rm.run(processes)
 

README.rst

@@ -5,68 +5,170 @@ Parallel processing wrapper for rasterio
 
 |Build Status|
 
+Install
+-------
+
+From pypi:
+
+``pip install rio-mucho --pre``
+
+From github (usually for a branch / dev):
+
+``pip install pip install git+ssh://git@github.com/mapbox/rio-mucho.git@<branch>``
+
+Development:
+
+::
+
+    git clone git@github.com:mapbox/rio-mucho.git
+    cd rio-mucho
+    pip install -e .
+
 Usage
 -----
 
-1. Define a function to be applied to each window chunk. This should
-   have input arguments of:
+.. code:: python
+
+    with riomucho.RioMucho([{inputs}], {output}, {run function},
+        windows={windows},
+        global_args={global arguments}, 
+        meta={meta to write}) as rios:
+
+        rios.run({processes})
+
+Arguments
+~~~~~~~~~
+
+``inputs``
+^^^^^^^^^^
+
+An list of file paths to open and read.
+
+``output``
+^^^^^^^^^^
+
+What file to write to.
+
+``run_function``
+^^^^^^^^^^^^^^^^
+
+A function to be applied to each window chunk. This should have input
+arguments of:
+
+1. A data input, which can be one of:
+
+-  A list of numpy arrays of shape (x,y,z), one for each file as
+   specified in input file list ``mode="simple_read" [default]``
+-  A numpy array of shape ({*n* input files x *n* band count}, {window
+   rows}, {window cols}) ``mode=array_read"``
+-  A list of open sources for reading ``mode="manual_read"``
 
--  A list of numpy arrays (one for each file as specified in input file
-   list) of shape ``({bands}, {window rows}, {window cols})``
--  A ``rasterio`` window tuple
--  A ``rasterio`` window index (``ij``)
--  A global arguments object that you can use to pass in global
+2. A ``rasterio`` window tuple
+3. A ``rasterio`` window index (``ij``)
+4. A global arguments object that you can use to pass in global
    arguments
 
+This should return:
+
+1. An output array of ({count}, {window rows}, {window cols}) shape, and
+   of the correct data type for writing
+
 .. code:: python
 
-    def basic_run(data, window, ij, g_args):
-        return data[0]
+    def basic_run({data}, {window}, {ij}, {global args}):
+        ## do something
+        return {out}
+
+Keyword arguments
+~~~~~~~~~~~~~~~~~
+
+``windows={windows}``
+^^^^^^^^^^^^^^^^^^^^^
+
+A list of ``rasterio`` (window, ij) tuples to operate on.
+``[Default = src[0].block_windows()]``
 
-2. Alternatively, for more flexibility, you can use a "manual read"
-   where you read each raster in this function. This is useful if you
-   want to read / write different window sizes (eg for pansharpening, or
-   buffered window reading). Here, instead of a list of arrays, the
-   function is passed an array of rasters open for reading.
+``global_args={global arguments}``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Since this is working in parallel, any other objects / values that you
+want to be accessible in the ``run_function``. ``[Default = {}]``
 
 .. code:: python
 
-    def basic_run(open_files, window, ij, g_args):
-        return numpy.array([f.read(window=window)[0] for f in open_files]) / g_args['divide']
+    global_args = {
+        'divide_value': 2
+    }
 
-For both of these, an array of identical shape to the destination window
-should be returned.
+``meta={keyword args}``
+^^^^^^^^^^^^^^^^^^^^^^^
 
-3. To run, make some windows, get or make some keyword args for writing,
-   and pass these and the above function into ``riomucho``: \`\`\`python
-   import riomucho, rasterio, numpy
+The meta to pass to the output. ``[Default = srcs[0].meta``
 
-get windows from an input
-=========================
+Example
+-------
 
-with rasterio.open('/tmp/test\_1.tif') as src: windows = [[window, ij]
-for ij, window in src.block\_windows()] kwargs = src.meta # since we are
-only writing to 2 bands kwargs.update(count=2)
+.. code:: python
 
-global\_args = { 'divide': 2 }
+    import riomucho, rasterio, numpy
 
-processes = 4
+    def basic_run(data, window, ij, g_args):
+        ## do something
+        out = np.array(
+            [d[0] /= global_args['divide'] for d in data]
+            )
+        return out
 
-run it
-======
+    # get windows from an input
+    with rasterio.open('/tmp/test_1.tif') as src:
+        ## grabbing the windows as an example. Default behavior is identical.
+        windows = [[window, ij] for ij, window in src.block_windows()]
+        meta = src.meta
+        # since we are only writing to 2 bands
+        meta.update(count=2)
 
-with riomucho.RioMucho(['input1.tif','input2, input2.tif'],
-'output.tif', basic\_run, windows=windows, global\_args=global\_args,
-kwargs=kwargs) as rm:
+    global_args = {
+        'divide': 2
+    }
 
-::
+    processes = 4
+
+    # run it
+    with riomucho.RioMucho(['input1.tif','input2.tif'], 'output.tif', basic_run,
+        windows=windows,
+        global_args=global_args, 
+        meta=meta) as rm:
 
-    rm.run(processes)
+        rm.run(processes)
+
+Utility functions
+-----------------
+
+\`riomucho.utils.array\_stack([array, array, array,...])
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Given a list of ({depth}, {rows}, {cols}) numpy arrays, stack into a
+single (l{list length \* each image depth}, {rows}, {cols}) array. This
+is useful for handling variation between ``rgb`` inputs of a single
+file, or separate files for each.
+
+One RGB file
+^^^^^^^^^^^^
+
+.. code:: python
+
+    files = ['rgb.tif']
+    open_files = [rasterio.open(f) for f in files]
+    rgb = `riomucho.utils.array_stack([src.read() for src in open_files])
+
+Separate RGB files
+^^^^^^^^^^^^^^^^^^
+
+.. code:: python
 
-\`\`\` - If no windows are specified, rio-mucho uses the block windows
-of the first input raster - If no kwargs are specified, rio-mucho uses
-the kwargs of the first input dataset to write to output - If no global
-args are specified, an empty object is passed.
+    files = ['r.tif', 'g.tif', 'b.tif']
+    open_files = [rasterio.open(f) for f in files]
+    rgb = `riomucho.utils.array_stack([src.read() for src in open_files])
 
 .. |Build Status| image:: https://travis-ci.org/mapbox/rio-mucho.svg?branch=master
    :target: https://travis-ci.org/mapbox/rio-mucho

riomucho/__init__.py

@@ -4,6 +4,7 @@
 import numpy as np
 import click
 import riomucho.scripts.riomucho_utils as utils
+import traceback
 
 work_func = None
 global_args = None
@@ -23,19 +24,30 @@ def main_worker(inpaths, g_work_func, g_args):
         return
 
 def manualRead(args):
-    window, ij = args
-    return work_func(srcs, window, ij, global_args), window
+    try:
+        window, ij = args
+        return work_func(srcs, window, ij, global_args), window
+    except Exception as e:
+        traceback.print_exc()
+        raise e
 
 def arrayRead(args):
     window, ij = args
-    return work_func(utils.array_stack(
-        [src.read(window=window) for src in srcs]),
-        window, ij, global_args), window
+    try:
+        return work_func(utils.array_stack(
+            [src.read(window=window) for src in srcs]),
+            window, ij, global_args), window
+    except Exception as e:
+        traceback.print_exc()
+        raise e
 
 def simpleRead(args):
     window, ij = args
-    return work_func([src.read(window=window) for src in srcs], window, ij, global_args), window
-
+    try:
+        return work_func([src.read(window=window) for src in srcs], window, ij, global_args), window
+    except Exception as e:
+        traceback.print_exc()
+        raise e
 
 class RioMucho:
     def __init__(self, inpaths, outpath, run_function, **kwargs):
@@ -45,16 +57,17 @@ def __init__(self, inpaths, outpath, run_function, **kwargs):
         else:
             self.windows = kwargs['windows']
 
-        if not 'kwargs' in kwargs:
-            self.kwargs = utils.getKwargs(inpaths[0])
+        if not 'options' in kwargs:
+            self.options = utils.getOptions(inpaths[0])
         else:
-            self.kwargs = kwargs['kwargs']
+            self.options = kwargs['options']
 
         if not 'global_args' in kwargs:
             self.global_args = {}
         else:
             self.global_args = kwargs['global_args']
 
+
         if not 'mode' in kwargs or kwargs['mode'] == 'simple_read':
             self.mode = 'simple_read'
         elif kwargs['mode'] == 'array_read':
@@ -74,10 +87,10 @@ def __exit__(self, ext_t, ext_v, trace):
             click.echo("in __exit__")
 
     def run(self, processes=4):
-        pool = Pool(processes, main_worker, (self.inpaths, self.run_function, self.global_args))
+        self.pool = Pool(processes, main_worker, (self.inpaths, self.run_function, self.global_args))
 
         ##shh
-        self.kwargs['transform'] = self.kwargs['affine']
+        self.options['transform'] = self.options['affine']
 
         if self.mode == 'manual_read':
             reader_worker = manualRead
@@ -87,13 +100,12 @@ def run(self, processes=4):
             reader_worker = simpleRead
 
         ## Open an output file, work through the function in parallel, and write out the data
-        with rio.open(self.outpath, 'w', **self.kwargs) as dst:   
-            for data, window in pool.imap_unordered(reader_worker, self.windows):
+        with rio.open(self.outpath, 'w', **self.options) as dst:   
+            for data, window in self.pool.imap_unordered(reader_worker, self.windows):
                 dst.write(data, window=window)
 
-        pool.close()
-        pool.join()
-
+        self.pool.close()
+        self.pool.join()
         return
 
 

riomucho/scripts/riomucho_utils.py

@@ -3,7 +3,7 @@
 import click
 
 
-def getKwargs(input):
+def getOptions(input):
     with rio.open(input) as src:
         return src.meta
 

setup.cfg

@@ -1,2 +1 @@
-[egg_info]
-tag_build = dev
+[egg_info]

setup.py

@@ -8,7 +8,7 @@
 
 
 setup(name='rio-mucho',
-      version='0.0.2',
+      version='0.1.1',
       description=u"Windowed multiprocessing wrapper for rasterio",
       long_description=long_description,
       classifiers=[],