diff --git a/.travis.yml b/.travis.yml index c73dec6..bff0eb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,35 +1,54 @@ language: python python: - - "2.7" - - "3.4" - - "3.5" - - "3.6" + - 2.7 + - 3.5 + - 3.6 + +env: + - NCO_VERSION=4.5.5 + - NCO_VERSION=4.6.1 + - NCO_VERSION=latest before_install: - - sudo apt-get update - - sudo apt-get install libnetcdf-dev nco - # We do this conditionally because it saves us some downloading if the - # version is the same. - - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then - wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; - else - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - fi + # Install miniconda + # ----------------- + - export CONDA_BASE=http://repo.continuum.io/miniconda/Miniconda + - wget ${CONDA_BASE}3-latest-Linux-x86_64.sh -O miniconda.sh; - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - - hash -r + + # Create the basic testing environment + # ------------------------------------ - conda config --set always_yes yes --set changeps1 no - - conda update -q conda - # Useful for debugging any issues with conda - - conda info -a + - conda config --set show_channel_urls True + - conda config --add create_default_packages pip + - conda update --quiet conda + + # Add 3rd party channels + # ------------------------------------ + - conda config --add channels conda-forge - # Make Conda test environment - - conda env create --file ci/requirements-$TRAVIS_PYTHON_VERSION.yml + # Create our environment + # ------------------------------------ + - conda env create --name test_env --quiet --file ci/requirements-$TRAVIS_PYTHON_VERSION.yml - source activate test_env + - if [ "${NCO_VERSION}" == "latest" ]; + then conda install nco; + else conda install nco==${NCO_VERSION}; + fi + + # Debugging information + # ------------------------------------ + - conda info -a + - conda list --export + - which ncks ncdump + + # Install pynco + # ------------------------------------ - python setup.py install # command to run tests script: - python -c "from nco import Nco" - - py.test + - py.test -rxs -s -v diff --git a/README.md b/README.md index 3309da1..819ee74 100644 --- a/README.md +++ b/README.md @@ -1,192 +1,262 @@ -pynco -============ +# pynco Python bindings for [NCO](http://nco.sourceforge.net/). A fork from Ralf Mueller's [cdo-bindings](https://github.com/Try2Code/cdo-bindings). ## `pynco` - Use Python to access the power of [NCO](http://nco.sourceforge.net/) -This package contains the module python `nco`, which implements a python style access to -the [NetCDF Operators (NCO)](http://nco.sourceforge.net/). NCO is a command line tool for processing -netCDF data. Its main focus is climate data, but it can by used for other -purposes too. +This package contains the module python `nco`, which implements a python style access to the [NetCDF Operators (NCO)](http://nco.sourceforge.net/). NCO is a command line tool for processing netCDF data. Its main focus is climate data, but it can by used for other purposes too. + ## Installation -### Python Installation: +### Conda Installation (recommended) + +This will install all required and optional dependencies and is the quickest and easiest way to a working `pynco` installation. + +```bash +conda install -c conda-forge pynco +``` + +### Pypi Installation + +Please see [the requirements](#requirements) before installing - python setup.py install +```bash +pip install nco +``` -### Pypi Installation: +### Python Installation - pip install nco +Please see [the requirements](#requirements) before installing -### Conda Installation: +```bash +python setup.py install +``` - conda install -c conda-forge pynco ### Requirements -- ***Platform***: Unix or Mac OS (Windows has not bee tested) -- [NetCDF Operators (NCO)](http://nco.sourceforge.net/) - Version 4.2 or later -- Python 2.7, 3.4, or later +**Mandatory** + +- ***Platform***: Unix or Mac OS (Windows has not bee tested) +- [NetCDF Operators (NCO)](http://nco.sourceforge.net/) - Version 4.2 or later. We don't test against every NCO version. +- Python 2.7, 3.4, 3.5, or 3.6 + +**Recommended** + +These will allow `pynco` operations to return `numpy` arrays + +- [scipy](http://docs.scipy.org/doc/scipy/reference/generated/scipy.io.netcdf.netcdf_file.html) +- [netCDF-4](https://code.google.com/p/netcdf4-python/) +- [numpy](http://www.numpy.org/) -**Recommended dependencies for returning `numpy` arrays from `nco` operations** -- [scipy](http://docs.scipy.org/doc/scipy/reference/generated/scipy.io.netcdf.netcdf_file.html) -- [netCDF-4](https://code.google.com/p/netcdf4-python/) -- [numpy](http://www.numpy.org/) ## Usage #### Importing the Nco class - `from nco import Nco` +```python +from nco import Nco +``` ### Run operators For python an instance has to be created first - from nco import Nco - nco = Nco() +```python +from nco import Nco +nco = Nco() +``` -Now any NCO command (i.e. ncks, ncra, ...) can be called as a method of `nco`. +Now any NCO command (i.e. `ncks`, `ncra`, ...) can be called as a method of `nco`. -* Required argument - - input - Input netcdf file name, str +* Required arguments + - `input` - Input netcdf file name, str * Optional arguments - - `output` - String or list of strings representing input netCDF filenames. If not provided and operator returns a file (not an array or stdout text), the method will return a temporary file. - - `debug` - bool or int, if <0 or True, debug statements will be turned on for NCO and NCOpy (default=False) - - `returnCdf` - return a netCDF file handle, bool (default=False) - - `returnArray` - return a numpy array of variable name, str (default='') - - `returnMaArray` - return a numpy masked array of variable name, str (default='') - - `options` - a string of NCO input options, for example options='-7 -L 1' (default='') - - `Atted` - a wrapper object to be used for ncatted. Atted objects can be included in the options list - - `Limit` - a wrapper object for the hyperslab ( -d ) command line option - - `Rename` - a wrapper object for the -d, -a, -v, -g command line options in ncrename - - `**kwargs` - any kwarg will be passed as a key, value pair to the nco command `--{key}={value}`. This allows the user to pass any number of long name commands list in the nco help pages. + + - `output` - `str` or `list` of strings representing input netCDF filenames. If not provided and operator returns a file (not an array or stdout text), the method will return a temporary file. + - `debug` - `bool` or `int`, if less than 0 or True, debug statements will be turned on for NCO and NCOpy (default: `False`) + - `returnCdf` - `bool`, return a netCDF file handle (default: `False`) + - `returnArray` - `str`. return a numpy array of variable name (default: `''`) + - `returnMaArray` - `str`. return a numpy masked array of variable name (default: `''`) + - `options` - `list`, NCO input options, for example `options=['-7', '-L 1']` (default: `[]`) + - `Atted` - a wrapper object to be used for ncatted. Atted objects can be included in the options list + - `Limit` - a wrapper object for the hyperslab (`-d`) command line option + - `Rename` - a wrapper object for the `-d`, `-a`, `-v`, and `-g` command line options in `ncrename` + - `**kwargs` - any kwarg will be passed as a key, value pair to the nco command `--{key}={value}`. This allows the user to pass any number of long name commands list in the nco help pages. + ### Examples -* File information: +* File information: - ncdump_string = nco.ncdump(input=ifile) + ```python + ncdump_string = nco.ncdump(input=ifile) + ``` -* Operators with user defined regular output files: +* Operators with user defined regular output files: - nco.ncra(input=ifile, output=ofile) + ```python + nco.ncra(input=ifile, output=ofile) + ``` -* Use temporary output files: +* Use temporary output files: - temp_ofile = nco.ncrcat(input=ifile) + ```python + temp_ofile = nco.ncrcat(input=ifile) + ``` -* Set global NCO options: +* Set global NCO options: - nco.ncks(input=ifile, output=ofile, options="--netcdf4") + ```python + nco.ncks(input=ifile, output=ofile, options="--netcdf4") + ``` -* Return multi-dimension arrays: +* Return multi-dimension arrays: - temperatures = nco.ncra(input=ifile, returnArray=True).variables['T'][:] - temperatures = nco.ncra(input=ifile, returnCdf=True).variables['T'][:] - temperatures = nco.ncra(input=ifile, returnArray='T') + ```python + temperatures = nco.ncra(input=ifile, returnArray=True).variables['T'][:] + temperatures = nco.ncra(input=ifile, returnCdf=True).variables['T'][:] + temperatures = nco.ncra(input=ifile, returnArray='T') + ``` * Wrapper Objects - The Atted opject is a convienent wrapper object to the `-a` command-line switch in ncatted. - The Limit object is a wrapper to the `-d` command-line switch. - The Rename is a wrapper for the `-a, -v, -d , -g ` switches in ncrename. + The Atted opject is a convienent wrapper object to the `-a` command-line switch in ncatted. + The Limit object is a wrapper to the `-d` command-line switch. + The Rename is a wrapper for the `-a, -v, -d , -g ` switches in ncrename. + + e.g the following are equivalent: - e.g the following are equivalent: - ``` - ncatted -a _FillValue,three_dmn,o,d,-9.91e+33 in.nc - nco.ncatted(input="in.nc",options=[c.atted("overwrite","_FillValue","three_dmn",-9.91e+33,'d')]) - ``` - see more examples below + ``` + ncatted -a _FillValue,three_dmn,o,d,-9.91e+33 in.nc + nco.ncatted(input="in.nc",options=[c.atted("overwrite","_FillValue","three_dmn",-9.91e+33,'d')]) + ``` ## Tempfile helpers `pynco` includes a simple tempfile wrapper, which makes life easier. In the -absence of a specified output file, `Nco()` will create a temporary file to allow the results of the task to be returned to the user. For example: +absence of a specified output file, `pynco` will create a temporary file to allow the results of the task to be returned to the user. For example: + +```python +temperatures = nco.ncra(input=ifile, returnArray='T') +``` - temperatures = nco.ncra(input=ifile, returnArray='T') is equivalent to: - temperatures = nco.ncra(input=ifile, output=tempfile.mktemp(), returnArray='T') +``` +temperatures = nco.ncra(input=ifile, output=tempfile.mktemp(), returnArray='T') +``` -* Atted wrapper +## Atted wrapper - It is sometimes more tidy to define the atted objects in a separate list then add that list the options in the nco call +It is sometimes more tidy to define the atted objects in a separate list then add that list the options in the nco call - ``` - opt=[ - c.Atted("o", "units", "temperature", "Kelvin"), - c.Atted("c", "min", "temperature", 0.16,'d' ), - c.Atted("m", "max", "temperature", 283.01,'float64'), - c.Atted("c", "bnds","time",[0.5,1.5],'f') - ] - nco.ncatted(input="in.nc",options=opt) - ``` +```python +opt = [ + c.Atted("o", "units", "temperature", "Kelvin"), + c.Atted("c", "min", "temperature", 0.16, 'd'), + c.Atted("m", "max", "temperature", 283.01, 'float64'), + c.Atted("c", "bnds", "time", [0.5, 1.5], 'f') +] +nco.ncatted(input="in.nc", options=opt) +``` - You can also use keyword arguments in the call so the above options become +You can also use keyword arguments in the call so the above options become - ``` - opt=[c.Atted(mode="o", attName="units", varName="temperature", Value="Kelvin",sType="c"), - c.Atted(mode="create", attName="min", varName="temperature", Value=0.16,sType='d' ), - c.Atted(mode="modify", attName="max", varName="temperature", Value=283.01,sType='float64'), - c.Atted(mode="create", attName="bnds", varName="time", Value=[0.5,1.5],sType='float32') - ] - ``` +```python +opt = [ + c.Atted(mode="o", attName="units", varName="temperature", Value="Kelvin", sType="c"), + c.Atted(mode="create", attName="min", varName="temperature", Value=0.16, sType='d' ), + c.Atted(mode="modify", attName="max", varName="temperature", Value=283.01, sType='float64'), + c.Atted(mode="create", attName="bnds", varName="time", Value=[0.5, 1.5], sType='float32') +] +nco.ncatted(input="in.nc", options=opt) +``` - Value can be a single value or a list ( or any python iterable type or a numpy array). +#### `Value` - If sType is NOT included then the type is inferred from the first value in the list
- if sType is included then any values in the list are NOT of sType are converted to sType

+Can be a single value or a list (or any python iterable type or a numpy array). - For sType you can use the following: - f, d, l/i, s, b, ub, us, u, ll, ull - Or their numpy equivalents - float32, float64, int32, int16, byte, ubyte, uint16, uint32, int64, uint64 +* If sType **is not** included then the type is inferred from the first value in the list +* If sType **is** included then any values in the list are **not** of sType are converted to the sType - For a netCDF3 character string use "c"or "char"
- For netCDF4 string(s) use "sng" or "string"
+#### `sType` - For mode you can use the single character abbreviations as per ncatted or the following words:
- (a)ppend, (c)reate, (d)elete, (m)odify, (n)append, (o)verwrite
+You can use the following: `f, d, l/i, s, b, ub, us, u, ll, ull` +Or their numpy equivalents: `float32, float64, int32, int16, byte, ubyte, uint16, uint32, int64, uint64` -* Limit and LimitSingle wrapper +For a netCDF3 character string use `c` or `char` +For netCDF4 string(s) use `sng` or `string` - the following are equivalent - ``` - ncks -d time,0,8,2 -d time,10 -d lat,-20.0,20.0 -d lon,50.0,350.0 -d lev,,,4 - and - opt=[ - c.Limit("time",0,8,2), - c.LimitSingle("time",10), - c.Limit("lat",-20.0,20.0), - c.Limit(dmn_name="lon",srt=50.0,end=350.0), - c.Limit(dmn_name="lev",srd=4) - ] +#### `mode` + +For mode you can use the single character abbreviations as per `ncatted`: `a, c, d, m, n, o` or the following words: `a)ppend, create, delete, modify, overwrite` + + +## Limit and LimitSingle wrapper + +The following are equivalent: + +**nco** + +```bash +ncks -d time,0,8,2 -d time,10 -d lat,-20.0,20.0 -d lon,50.0,350.0 -d lev,,,4 +``` + +**pynco** + +```python +opt = [ + c.Limit("time", 0, 8, 2), + c.LimitSingle("time", 10), + c.Limit("lat", -20.0, 20.0), + c.Limit(dmn_name="lon", srt=50.0, end=350.0), + c.Limit(dmn_name="lev", srd=4) +] + +nco.ncks(input="in.nc", output="out.nc", options=opt) +``` + +## Rename wrapper + +The following are equivalent: + +**`nco`** + +```bash +ncrename -v p,pressure -v t,temperature in.nc +``` + +**`pynco`** - nco.ncks(input="in.nc", output="out.nc", options=opt) - ``` +```python +rDict = { + 'p': 'pressure', + 't': 'temperature' +} +nco.ncrename(input="in.nc", options=[ c.Rename("variable", rDict) ]) +``` -* Rename wrapper +Also equivalent: - the following are equivalent: - ``` - ncrename -v p,pressure -v t,temperature in.nc +**`nco`** - rDict={'p':'pressure', 't':'temperature'} - nco.ncrename(input="in.nc", options=[ c.Rename("variable",rDict)]) - ``` +```bash +ncrename -d lon,longitude -d lat,latitude -v lon,longitude -v lat,latitude in.nc +``` - rename coordinate variables (dim & var) - ``` - ncrename -d lon,longitude -d lat,latitude -v lon,longitude -v lat,latitude in.nc +**`pynco`** - rDict={'lon':'longitude', 'lat':'latitude'} - nco.ncrename(input="in.nc", options=[c.Rename("d",rDict), c.Rename("v",rDict)]) - ``` +```python +rDict = { + 'lon': 'longitude', + 'lat': 'latitude' +} +nco.ncrename(input="in.nc", options=[ c.Rename("d", rDict), c.Rename("v", rDict) ]) +``` ## Support, Issues, Bugs, ... @@ -199,7 +269,6 @@ For usage questions, please use [Stack Overflow](https://stackoverflow.com/quest `pynco` makes use of the GPLv2 License, see LICENSE.txt. ---- ## Other stuff diff --git a/ci/requirements-2.7.yml b/ci/requirements-2.7.yml index c173559..c1bb473 100644 --- a/ci/requirements-2.7.yml +++ b/ci/requirements-2.7.yml @@ -1,9 +1,8 @@ -name: test_env channels: - conda-forge dependencies: - python=2.7 - - dateutil + - python-dateutil - h5py - netCDF4 - numpy diff --git a/ci/requirements-3.4.yml b/ci/requirements-3.4.yml index e1f4828..6fd71ff 100644 --- a/ci/requirements-3.4.yml +++ b/ci/requirements-3.4.yml @@ -1,9 +1,8 @@ -name: test_env channels: - conda-forge dependencies: - python=3.4 - - dateutil + - python-dateutil - h5py - netCDF4 - numpy diff --git a/ci/requirements-3.5.yml b/ci/requirements-3.5.yml index e96e5af..184febe 100644 --- a/ci/requirements-3.5.yml +++ b/ci/requirements-3.5.yml @@ -1,9 +1,8 @@ -name: test_env channels: - conda-forge dependencies: - python=3.5 - - dateutil + - python-dateutil - h5py - netCDF4 - numpy diff --git a/ci/requirements-3.6.yml b/ci/requirements-3.6.yml index 879e472..a93960b 100644 --- a/ci/requirements-3.6.yml +++ b/ci/requirements-3.6.yml @@ -1,9 +1,8 @@ -name: test_env channels: - conda-forge dependencies: - python=3.6 - - dateutil + - python-dateutil - h5py - netCDF4 - numpy diff --git a/docs/index.md b/docs/index.md index 0b8a0ed..0ca1ce1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,111 +1,263 @@ -pynco -============ -Use Python to access the power of [NCO](http://nco.sourceforge.net/) +# pynco Python bindings for [NCO](http://nco.sourceforge.net/). A fork from Ralf Mueller's [cdo-bindings](https://github.com/Try2Code/cdo-bindings). -This package contains the module python `nco`, which implements a python style access to -the [NetCDF Operators (NCO)](http://nco.sourceforge.net/). NCO is a command line tool for processing -netCDF data. Its main focus is climate data, but it can by used for other -purposes too. +## `pynco` - Use Python to access the power of [NCO](http://nco.sourceforge.net/) + +This package contains the module python `nco`, which implements a python style access to the [NetCDF Operators (NCO)](http://nco.sourceforge.net/). NCO is a command line tool for processing netCDF data. Its main focus is climate data, but it can by used for other purposes too. + ## Installation -### Python Installation: +### Conda Installation (recommended) - python setup.py install +This will install all required and optional dependencies and is the quickest and easiest way to a working `pynco` installation. -### PyPI Installation: +```bash +conda install -c conda-forge pynco +``` - pip install nco +### Pypi Installation - ### Conda Installation: +Please see [the requirements](#requirements) before installing + +```bash +pip install nco +``` + +### Python Installation + +Please see [the requirements](#requirements) before installing + +```bash +python setup.py install +``` - conda install -c conda-forge pynco ### Requirements -- ***Platform***: Unix or Mac OS (Windows has not bee tested) -- [NetCDF Operators (NCO)](http://nco.sourceforge.net/) - Version 4.2 or later -- Python 2.7, 3.4, or later +**Mandatory** + +- ***Platform***: Unix or Mac OS (Windows has not bee tested) +- [NetCDF Operators (NCO)](http://nco.sourceforge.net/) - Version 4.2 or later. We don't test against every NCO version. +- Python 2.7, 3.4, 3.5, or 3.6 + +**Recommended** + +These will allow `pynco` operations to return `numpy` arrays + +- [scipy](http://docs.scipy.org/doc/scipy/reference/generated/scipy.io.netcdf.netcdf_file.html) +- [netCDF-4](https://code.google.com/p/netcdf4-python/) +- [numpy](http://www.numpy.org/) -**Recommended dependencies for returning `numpy` arrays from `nco` operations** -- [scipy](http://docs.scipy.org/doc/scipy/reference/generated/scipy.io.netcdf.netcdf_file.html) -- [netCDF-4](https://code.google.com/p/netcdf4-python/) -- [numpy](http://www.numpy.org/) ## Usage #### Importing the Nco class - `from nco import Nco` +```python +from nco import Nco +``` ### Run operators For python an instance has to be created first -```Python +```python from nco import Nco nco = Nco() ``` Now any NCO command (i.e. `ncks`, `ncra`, ...) can be called as a method of `nco`. -* Required argument - - input - Input netCDF file name, str +* Required arguments + - `input` - Input netcdf file name, str * Optional arguments - - `output` - String or list of strings representing input netCDF filenames. If not provided and operator returns a file (not an array or stdout text), the method will return a temporary file. - - `debug` - bool or int, if <0 or True, debug statements will be turned on for NCO and NCOpy (default=False) - - `returnCdf` - return a netCDF file handle, bool (default=False) - - `returnArray` - return a numpy array of variable name, str (default='') - - `returnMaArray` - return a numpy masked array of variable name, str (default='') - - `options` - a string of NCO input options, for example options='-7 -L 1' (default='') - - `**kwargs` - any kwarg will be passed as a key, value pair to the nco command `--{key}={value}`. This allows the user to pass any number of long name commands list in the nco help pages. + + - `output` - `str` or `list` of strings representing input netCDF filenames. If not provided and operator returns a file (not an array or stdout text), the method will return a temporary file. + - `debug` - `bool` or `int`, if less than 0 or True, debug statements will be turned on for NCO and NCOpy (default: `False`) + - `returnCdf` - `bool`, return a netCDF file handle (default: `False`) + - `returnArray` - `str`. return a numpy array of variable name (default: `''`) + - `returnMaArray` - `str`. return a numpy masked array of variable name (default: `''`) + - `options` - `list`, NCO input options, for example `options=['-7', '-L 1']` (default: `[]`) + - `Atted` - a wrapper object to be used for ncatted. Atted objects can be included in the options list + - `Limit` - a wrapper object for the hyperslab (`-d`) command line option + - `Rename` - a wrapper object for the `-d`, `-a`, `-v`, and `-g` command line options in `ncrename` + - `**kwargs` - any kwarg will be passed as a key, value pair to the nco command `--{key}={value}`. This allows the user to pass any number of long name commands list in the nco help pages. + ### Examples -* Operators with user defined regular output files: +* File information: -```Python -nco.ncra(input=ifile, output=ofile) -``` + ```python + ncdump_string = nco.ncdump(input=ifile) + ``` -* Use temporary output files: +* Operators with user defined regular output files: -```Python -temp_ofile = nco.ncrcat(input=ifile) -``` + ```python + nco.ncra(input=ifile, output=ofile) + ``` -* Set global NCO options: +* Use temporary output files: -```Python -nco.ncks(input=ifile, output=ofile, options="--netcdf4") -``` + ```python + temp_ofile = nco.ncrcat(input=ifile) + ``` -* Return multi-dimension arrays: +* Set global NCO options: -```Python -temperatures = nco.ncra(input=ifile, returnArray=True).variables['T'][:] -temperatures = nco.ncra(input=ifile, returnCdf=True).variables['T'][:] -temperatures = nco.ncra(input=ifile, returnArray='T') -``` + ```python + nco.ncks(input=ifile, output=ofile, options="--netcdf4") + ``` + +* Return multi-dimension arrays: + + ```python + temperatures = nco.ncra(input=ifile, returnArray=True).variables['T'][:] + temperatures = nco.ncra(input=ifile, returnCdf=True).variables['T'][:] + temperatures = nco.ncra(input=ifile, returnArray='T') + ``` + +* Wrapper Objects + + The Atted opject is a convienent wrapper object to the `-a` command-line switch in ncatted. + The Limit object is a wrapper to the `-d` command-line switch. + The Rename is a wrapper for the `-a, -v, -d , -g ` switches in ncrename. + + e.g the following are equivalent: + + ``` + ncatted -a _FillValue,three_dmn,o,d,-9.91e+33 in.nc + nco.ncatted(input="in.nc",options=[c.atted("overwrite","_FillValue","three_dmn",-9.91e+33,'d')]) + ``` ## Tempfile helpers `pynco` includes a simple tempfile wrapper, which makes life easier. In the -absence of a specified output file, `Nco()` will create a temporary file to allow the results of the task to be returned to the user. For example: +absence of a specified output file, `pynco` will create a temporary file to allow the results of the task to be returned to the user. For example: -```Python +```python temperatures = nco.ncra(input=ifile, returnArray='T') ``` is equivalent to: -```Python +``` temperatures = nco.ncra(input=ifile, output=tempfile.mktemp(), returnArray='T') ``` + +## Atted wrapper + +It is sometimes more tidy to define the atted objects in a separate list then add that list the options in the nco call + +```python +opt = [ + c.Atted("o", "units", "temperature", "Kelvin"), + c.Atted("c", "min", "temperature", 0.16, 'd'), + c.Atted("m", "max", "temperature", 283.01, 'float64'), + c.Atted("c", "bnds", "time", [0.5, 1.5], 'f') +] +nco.ncatted(input="in.nc", options=opt) +``` + +You can also use keyword arguments in the call so the above options become + +```python +opt = [ + c.Atted(mode="o", attName="units", varName="temperature", Value="Kelvin", sType="c"), + c.Atted(mode="create", attName="min", varName="temperature", Value=0.16, sType='d' ), + c.Atted(mode="modify", attName="max", varName="temperature", Value=283.01, sType='float64'), + c.Atted(mode="create", attName="bnds", varName="time", Value=[0.5, 1.5], sType='float32') +] +nco.ncatted(input="in.nc", options=opt) +``` + +#### `Value` + +Can be a single value or a list (or any python iterable type or a numpy array). + +* If sType **is not** included then the type is inferred from the first value in the list +* If sType **is** included then any values in the list are **not** of sType are converted to the sType + +#### `sType` + +You can use the following: `f, d, l/i, s, b, ub, us, u, ll, ull` +Or their numpy equivalents: `float32, float64, int32, int16, byte, ubyte, uint16, uint32, int64, uint64` + +For a netCDF3 character string use `c` or `char` +For netCDF4 string(s) use `sng` or `string` + +#### `mode` + +For mode you can use the single character abbreviations as per `ncatted`: `a, c, d, m, n, o` or the following words: `a)ppend, create, delete, modify, overwrite` + + +## Limit and LimitSingle wrapper + +The following are equivalent: + +**nco** + +```bash +ncks -d time,0,8,2 -d time,10 -d lat,-20.0,20.0 -d lon,50.0,350.0 -d lev,,,4 +``` + +**pynco** + +```python +opt = [ + c.Limit("time", 0, 8, 2), + c.LimitSingle("time", 10), + c.Limit("lat", -20.0, 20.0), + c.Limit(dmn_name="lon", srt=50.0, end=350.0), + c.Limit(dmn_name="lev", srd=4) +] + +nco.ncks(input="in.nc", output="out.nc", options=opt) +``` + +## Rename wrapper + +The following are equivalent: + +**`nco`** + +```bash +ncrename -v p,pressure -v t,temperature in.nc +``` + +**`pynco`** + +```python +rDict = { + 'p': 'pressure', + 't': 'temperature' +} +nco.ncrename(input="in.nc", options=[ c.Rename("variable", rDict) ]) +``` + +Also equivalent: + +**`nco`** + +```bash +ncrename -d lon,longitude -d lat,latitude -v lon,longitude -v lat,latitude in.nc +``` + +**`pynco`** + +```python +rDict = { + 'lon': 'longitude', + 'lat': 'latitude' +} +nco.ncrename(input="in.nc", options=[ c.Rename("d", rDict), c.Rename("v", rDict) ]) +``` + ## Support, Issues, Bugs, ... Please use the github page to report issues/bugs/features: https://github.com/nco/pynco. @@ -116,7 +268,6 @@ For usage questions, please use [Stack Overflow](https://stackoverflow.com/quest `pynco` makes use of the GPLv2 License, see LICENSE.txt. ---- ## Other stuff diff --git a/nco/nco.py b/nco/nco.py index a708e42..89f1b18 100644 --- a/nco/nco.py +++ b/nco/nco.py @@ -360,6 +360,8 @@ def version(self): stdout=subprocess.PIPE) ret = proc.communicate() ncra_help = ret[1] + if isinstance(ncra_help, bytes): + ncra_help = ncra_help.decode('utf-8') match = re.search('NCO netCDF Operators version (\d.*) ', ncra_help) # some versions write version information in quotation marks diff --git a/tests/conftest.py b/tests/conftest.py index 8d50394..a2c67df 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,6 @@ import pytest import tempfile import numpy as np -# import h5py import netCDF4 import datetime from dateutil import relativedelta @@ -37,7 +36,7 @@ def random_field(): @pytest.fixture(scope="module") def mask4tests(): - return np.random.random_integers(0, 1, (1, 5, 5)) + return np.random.randint(1, size=(1, 5, 5)) @pytest.fixture(scope="module") @@ -47,15 +46,19 @@ def random_masked_field(mask4tests): return field -# @pytest.fixture(scope="module") -# def hdf_file(random_field, tempsrcdir): -# filename = os.path.join(tempsrcdir, 'testhdf.hdf5') -# f = h5py.File(filename, 'w') -# shape = random_field.shape -# dset = f.create_dataset("random_field", shape, dtype='f') -# dset[:, :] = random_field -# f.close() -# return filename +@pytest.fixture(scope="module") +def hdf_file(random_field, tempsrcdir): + try: + import h5py + filename = os.path.join(tempsrcdir, 'testhdf.hdf5') + f = h5py.File(filename, 'w') + shape = random_field.shape + dset = f.create_dataset("random_field", shape, dtype='f') + dset[:, :] = random_field + f.close() + return filename + except (ImportError, AttributeError): + return None @pytest.fixture(scope="module") @@ -154,12 +157,12 @@ def testfiles8589(random_field, tempsrcdir): shape = random_field.shape f.createDimension('dim0', shape[0]) f.createDimension('dim1', shape[1]) - time = f.createDimension('time', 1) + f.createDimension('time') var = f.createVariable('random', 'f8', ('time', 'dim0', 'dim1',)) time = f.createVariable('time', 'f8', ('time')) time.units = stdtimeunits time.calendar = noleapcalendar - var[:, :, :] = random_field + var[0, :, :] = random_field time[:] = netCDF4.date2num(date, stdtimeunits, calendar=noleapcalendar) f.close() diff --git a/tests/test_nco.py b/tests/test_nco.py index bb88b0a..63a243a 100644 --- a/tests/test_nco.py +++ b/tests/test_nco.py @@ -86,10 +86,10 @@ def test_use_list_inputs(foo_nc, bar_nc): def test_use_list_options(foo_nc): nco = Nco(debug=True) options = [] - options.extend(['-a', 'units,time,o,c,days since 1999-01-01']) + options.extend(['-a', 'units,time,o,c,"days since 1999-01-01"']) options.extend(['-a', 'long_name,time,o,c,time']) options.extend(['-a', 'calendar,time,o,c,noleap']) - nco.ncrcat(input=foo_nc, output='out.nc', options=options) + nco.ncatted(input=foo_nc, output='out.nc', options=options) @pytest.mark.usefixtures("foo_nc", "bar_nc") diff --git a/tests/test_nco_examples.py b/tests/test_nco_examples.py index 508c8d7..51b6bd9 100644 --- a/tests/test_nco_examples.py +++ b/tests/test_nco_examples.py @@ -34,6 +34,8 @@ def test_ncks_hdf2nc(hdf_file): ncks fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.4.0+, netCDF4.3.1+) ncks --hdf4 fl.hdf fl.nc # Convert HDF4->netCDF4 (NCO 4.3.7-4.3.9) """ + if hdf_file is None: + pytest.skip('Skipped because h5py is not installed') nco = Nco(debug=True) nco.ncks(input=hdf_file, output='foo.nc') nco.ncks(input=hdf_file, output='foo.nc', hdf4=True) @@ -51,11 +53,13 @@ def test_ncks_hdf2nc3(hdf_file): ncks --hdf4 -3 fl.hdf fl.nc # HDF4->netCDF3 (netCDF 4.3.0-) ncks --hdf4 -7 fl.hdf fl.nc # HDF4->netCDF4 classic (netCDF 4.3.0-) """ + if hdf_file is None: + pytest.skip('Skipped because h5py is not installed') nco = Nco(debug=True) - nco.ncks(input=hdf_file, output='foo.nc', options='-3') - nco.ncks(input=hdf_file, output='foo.nc', options='-7 -L 1') - nco.ncks(input=hdf_file, output='foo.nc', options='-3', hdf4=True) - nco.ncks(input=hdf_file, output='foo.nc', options='-7', hdf4=True) + nco.ncks(input=hdf_file, output='foo.nc', options=['-3']) + nco.ncks(input=hdf_file, output='foo.nc', options=['-7 -L 1']) + nco.ncks(input=hdf_file, output='foo.nc', options=['-3'], hdf4=True) + nco.ncks(input=hdf_file, output='foo.nc', options=['-7'], hdf4=True) def test_temp_output_files(foo_nc): @@ -116,9 +120,9 @@ def test_command_line_options(foo_nc): ncks --dbg_lvl 3 in.nc # Long option, alternate form """ nco = Nco(debug=True) - nco.ncks(input=foo_nc, options='-D 3') - nco.ncks(input=foo_nc, options='--dbg_lvl=3') - nco.ncks(input=foo_nc, options='--dbg_lvl 3') + nco.ncks(input=foo_nc, options=['-D 3']) + nco.ncks(input=foo_nc, options=['--dbg_lvl=3']) + nco.ncks(input=foo_nc, options=['--dbg_lvl 3']) nco.ncks(input=foo_nc, dbg_lvl=3) @@ -131,13 +135,16 @@ def test_specifying_input_files(testfiles8589): ncra -p input-path 85.nc 86.nc 87.nc 88.nc 89.nc 8589.nc ncra -n 5,2,1 85.nc 8589.nc """ - inputs = ['85.nc', '86.nc', '87.nc', '88.nc', '89.nc'] nco = Nco(debug=True) - nco.ncra(input=inputs, ouptut='8589.nc') - nco.ncra(input='8[56789].nc', ouptut='8589.nc') - srcdir = os.path.split(inputs[0]) - nco.ncra(input=inputs, ouptut='8589.nc', path=srcdir) - nco.ncra(input=inputs, ouptut='8589.nc', nintap='5,2,1') + nco.ncra(input=testfiles8589, output='8589.nc') + nco.ncra(input=testfiles8589, output='8589.nc', nintap='5,2,1') + + srcdir = os.path.dirname(testfiles8589[0]) + basenames = [ os.path.basename(x) for x in testfiles8589 ] + nco.ncra(input=basenames, output='8589.nc', path=srcdir) + + regpath = os.path.join(srcdir, '8[56789].nc') + nco.ncra(input=regpath, output='8589.nc') def test_determining_file_format(foo3c, foo364, foo4c, hdf_file): @@ -156,12 +163,14 @@ def test_determining_file_format(foo3c, foo364, foo4c, hdf_file): nco.ncks(input=foo3c, options=['-M']) nco.ncks(input=foo364, options=['-M']) nco.ncks(input=foo4c, options=['-M']) - assert os.path.isfile(hdf_file) - nco.ncks(input=hdf_file, options=['-D 2 -M']) - # nco.ncks(input='http://thredds-test.ucar.edu/thredds/dodsC/testdods/in.nc', - # options='-D 2 -M') # does not work from command line either nco.ncks(input=foo4c, options=['-D 2 -M']) + if hdf_file is not None: + assert os.path.isfile(hdf_file) + nco.ncks(input=hdf_file, options=['-D 2 -M']) + # nco.ncks(input='http://thredds-test.ucar.edu/thredds/dodsC/testdods/in.nc', + # options=['-D 2', '-M']) # does not work from command line either + def test_file_conversion(foo3c, foo4c): """ @@ -185,14 +194,14 @@ def test_file_conversion(foo3c, foo4c): nco.ncks(input=foo4c, output='foo_364.nc', fl_fmt='64bit') nco.ncks(input=foo3c, output='foo_4c.nc', fl_fmt='netcdf4_classic') nco.ncks(input=foo3c, output='foo_4.nc', fl_fmt='netcdf4') - nco.ncks(input=foo4c, output='foo_3c.nc', options='-3') - nco.ncks(input=foo4c, output='foo_3c.nc', options='--3') - nco.ncks(input=foo3c, output='foo_364c.nc', options='-6') - nco.ncks(input=foo3c, output='foo_364c.nc', options='--64') - nco.ncks(input=foo3c, output='foo_4.nc', options='-4') - nco.ncks(input=foo3c, output='foo_4.nc', options='--4') - nco.ncks(input=foo3c, output='foo_4c.nc', options='-7') - nco.ncks(input=foo3c, output='foo_4c.nc', options='--7') + nco.ncks(input=foo4c, output='foo_3c.nc', options=['-3']) + nco.ncks(input=foo4c, output='foo_3c.nc', options=['--3']) + nco.ncks(input=foo3c, output='foo_364c.nc', options=['-6']) + nco.ncks(input=foo3c, output='foo_364c.nc', options=['--64bit_offset']) + nco.ncks(input=foo3c, output='foo_4.nc', options=['-4']) + nco.ncks(input=foo3c, output='foo_4.nc', options=['--4']) + nco.ncks(input=foo3c, output='foo_4c.nc', options=['-7']) + nco.ncks(input=foo3c, output='foo_4c.nc', options=['--7']) def test_hyperslabs(testfileglobal):