From 4d2ef4b8971da4010a4484eb055723a228845a12 Mon Sep 17 00:00:00 2001 From: Chris Barnard Date: Wed, 18 Jan 2023 14:26:19 +0000 Subject: [PATCH 1/9] Add SEEMHEWS parameters --- configuration/global/parameters.json | 48 +++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/configuration/global/parameters.json b/configuration/global/parameters.json index 222da16..3fc2c55 100644 --- a/configuration/global/parameters.json +++ b/configuration/global/parameters.json @@ -142,6 +142,17 @@ "@id": "tomm", "@unit": "mm" } + }, + "pev": { + "@description": "Potential Evaporation", + "@shortName": "pev", + "@unit": "m", + "Conversion": { + "@cutOffNegative": true, + "@function": "x=-x*1000", + "@id": "tomm", + "@unit": "mm" + } }, "fis": { "@description": "Geopotential cosmo", @@ -260,5 +271,40 @@ "@description": "Convective inhibition", "@shortName": "cin", "@unit": "J kg**-1" + }, + "swvl1": { + "@description": "Soil Moisture", + "@shortName": "swvl1", + "@unit": "percentage" + }, + "sd": { + "@description": "soil water equivalent", + "@shortName": "scov", + "@unit": "m", + "Conversion":{ + "@function":"x=1000*x", + "@id":"tomm", + "@unit":"mm" + } + }, + "sf": { + "@description": "snowmelt", + "@shortName": "smelt", + "@unit": "m", + "Conversion":{ + "@function":"x=1000*x", + "@id":"tomm", + "@unit":"mm" + } + }, + "smlt": { + "@description": "snowmelt", + "@shortName": "smelt", + "@unit": "m", + "Conversion":{ + "@function":"x=1000*x", + "@id":"tomm", + "@unit":"mm" + } } -} \ No newline at end of file +} From ff9941021db87a0cf6472d932acf7ce26407e646 Mon Sep 17 00:00:00 2001 From: Chris Barnard Date: Wed, 18 Jan 2023 14:27:00 +0000 Subject: [PATCH 2/9] Add SEEMHWS intertables and geopotentials --- configuration/global/geopotentials.json | 6 +- configuration/global/intertables.json | 88 +++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/configuration/global/geopotentials.json b/configuration/global/geopotentials.json index 6929e48..6bcc0c0 100644 --- a/configuration/global/geopotentials.json +++ b/configuration/global/geopotentials.json @@ -44,5 +44,9 @@ "-40$80$M$214$41826$reduced_gg":"O320_p80_m40_p20_p80.grb", "-40$80$M$214$58717$reduced_gg":"efhs_58717.grb", "-40$80$M$106$14996$reduced_gg":"efhs_14996.grb", - "0$359.723$M$640$421120$reduced_gg":"O320_421120.grb" + "0$359.723$M$640$421120$reduced_gg":"O320_421120.grb", + "-25$15$1001$601$601601$rotated_ll": "cosmo_z_seemhews.grib", + "7$49$1556$1066$1644290$regular_ll": "nmmb_z_seemhews.grib", + "9.4$44.98$1187$1049$1244967$regular_ll": "alsmws_verif_geop_seemhews.grb2", + "9.4$45$1425$961$1369425$regular_ll": "icon_gp_seemhews.grib" } diff --git a/configuration/global/intertables.json b/configuration/global/intertables.json index 4a369ba..10c6e35 100644 --- a/configuration/global/intertables.json +++ b/configuration/global/intertables.json @@ -1046,6 +1046,28 @@ 3600 ] }, + "I-15.75_16.125_511_415_212065_rotated_ll_2614000_4078000_1000_-1000_32.03_59.80_2614000_4078000_1000_-1000_-10.48_36.57scipy_nearest":{ + "filename": "cos_212065_to_2878x3479_nearest.npy", + "method": "nearest", + "source_shape": [ + 212065 + ], + "target_shape": [ + 2878, + 3479 + ] + }, + "I-15.75_16.125_511_415_212065_rotated_ll_-1700000_1360000_1000_-1000_0.00_60.22_-1700000_1360000_1000_-1000_-10.32_37.93scipy_nearest":{ + "filename":"cos_212065_to_2878x3479_nearest.npy", + "method":"nearest", + "source_shape":[ + 212065 + ], + "target_shape":[ + 2980, + 3570 + ] + }, "I-15.75_16.125_511_415_212065_rotated_ll_2568000_4163000_1000_-1000_31.63_60.56_2568000_4163000_1000_-1000_-19.97_40.95scipy_nearest":{ "filename":"cos_212065_to_2981x3571_nearest.npy", "method":"nearest", @@ -1133,5 +1155,71 @@ 1500, 3600 ] + }, + "I-25_15_1001_601_601601_rotated_ll_4790000_2550000_5000_-5000_43.34_45.87_4790000_2550000_5000_-5000_15.83_19.90scipy_nearest": { + "filename": "tbl_csm_3111_scipy_nearest.npy", + "method": "nearest", + "source_shape": [ + 601601 + ], + "target_shape": [ + 51, + 61 + ] + }, + "I0_359.929_M_2560_6599680_reduced_gg_4790000_2550000_5000_-5000_43.34_45.87_4790000_2550000_5000_-5000_15.83_19.90scipy_nearest": { + "filename": "tbl_1_2021061700_3111_scipy_nearest.npy", + "method": "nearest", + "source_shape": [ + 6599680 + ], + "target_shape": [ + 51, + 61 + ] + }, + "I7_49_1556_1066_1644290_regular_ll_4790000_2550000_5000_-5000_43.34_45.87_4790000_2550000_5000_-5000_15.83_19.90scipy_nearest": { + "filename": "tbl_nmmb_3111_scipy_nearest.npy", + "method": "nearest", + "source_shape": [ + 1658696 + ], + "target_shape": [ + 51, + 61 + ] + }, + "I7_49_1556_1066_1658696_regular_ll_4790000_2550000_5000_-5000_43.34_45.87_4790000_2550000_5000_-5000_15.83_19.90scipy_nearest": { + "filename": "tbl_1_nmmb_3111_scipy_nearest.npy", + "method": "nearest", + "source_shape": [ + 1658696 + ], + "target_shape": [ + 51, + 61 + ] + }, + "I9.4_44.98_1187_1049_1244967_regular_ll_4790000_2550000_5000_-5000_43.34_45.87_4790000_2550000_5000_-5000_15.83_19.90scipy_nearest": { + "filename": "tbl_2_2021061700_3111_scipy_nearest.npy", + "method": "nearest", + "source_shape": [ + 1245163 + ], + "target_shape": [ + 51, + 61 + ] + }, + "I9.4_45_1425_961_1369425_regular_ll_4790000_2550000_5000_-5000_43.34_45.87_4790000_2550000_5000_-5000_15.83_19.90scipy_nearest": { + "filename": "tbl_3_2021061700_3111_scipy_nearest.npy", + "method": "nearest", + "source_shape": [ + 1369425 + ], + "target_shape": [ + 51, + 61 + ] } } From 657fbf2b6afc3381d19c28584f57f020312438d0 Mon Sep 17 00:00:00 2001 From: Chris Barnard Date: Wed, 18 Jan 2023 14:27:22 +0000 Subject: [PATCH 3/9] Update ecmwf module install script --- ecmwf_install_as_module | 60 ++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/ecmwf_install_as_module b/ecmwf_install_as_module index 60b1674..f5d7b63 100755 --- a/ecmwf_install_as_module +++ b/ecmwf_install_as_module @@ -1,54 +1,70 @@ #!/bin/bash -set -ex +module unload python +module unload python3 +module load python3 + +set -eux + +mod_version=${1:-3.0.0.post1} +if [[ $# -gt 0 ]]; then shift; fi +hosts=${@:-deploy@aa-login deploy@ab-login deploy@ac-login deploy@ad-login} -mod_version=${1:-2.1.1} assets_version=$mod_version +echo "module version to be installed: $mod_version" +echo "on hosts: $hosts" + # Where is pyg2p source code src_dir=$PWD src_host=$HOST +src_host=hpc-login # where to install it + dest_dir=/usr/local/apps/pyg2p/$mod_version +eccodes=eccodes/2.24.0 +for dest_host in $hosts; do -#for host in localhost lxc ecgb lxop cca ccb; do -for dest_host in lxc lxop cca ccb; do case $dest_host in - lxc* ) static_data_root=/gpfs/lxc/efas/emos/data/pyg2p/$assets_version ;; - lxop* ) static_data_root=/gpfs/lxop/efas/emos/data/pyg2p/$assets_version ;; - cca* ) static_data_root=/sc1/tcwork/emos/emos_data/efas/assets/pyg2p/$assets_version ;; - ccb* ) static_data_root=/sc2/tcwork/emos/emos_data/efas/assets/pyg2p/$assets_version ;; - * ) "unexpected target host $dest_host"; false ;; + + deploy@a?-login ) dest_dir=/usr/local/apps/pyg2p/$mod_version + static_data_root=/usr/local/apps/pyg2p/data + eccodes=ecmwf-toolbox/2022.08.0.0 ;; + * ) echo "unexpected target host $dest_host"; false ;; esac echo installing pyg2p/$mod_version in $dest_host .... - if ssh $dest_host [[ -d $dest_dir ]]; then + if ssh -x $dest_host [[ -d $dest_dir ]]; then case $mod_version in - dev | test ) echo "reinstalling pyg2p/$mod_version" ;; + *dev* | *test | 3.0.0-01 | 3.0.1 | 3.0.0.post1 ) echo "reinstalling pyg2p/$mod_version" ;; * ) echo "module pyg2p/$mod_version is already installed on $dest_host. skipping."; continue ;; esac fi - ssh $dest_host bash -l << END - module unload python - module load python/2.7.15-01 - umask 022 + ssh -x $dest_host bash -l << END set -eux - echo \$TMPDIR - cd \$TMPDIR - rsync -avz --exclude='.git/' $src_host:$src_dir . + module unload python3 python gdal grib_api eccodes || : + module load $eccodes + module load gdal/3.2.1 + module load python3/3.8.8-01 + module load gcc || : # on HPC ;; + umask 022 + mkdir -p \$SCRATCH/tmp_pyg2p + cd \$SCRATCH/tmp_pyg2p/ + rsync -avz --exclude='.git/' -e ssh $src_host:$src_dir . cd pyg2p echo '{ "geopotentials": "$static_data_root/geopotentials", "intertables": "$static_data_root/intertables" } ' > configuration/global/global_conf.json - export HTTPS_PROXY=http://proxy.ecmwf.int:3333 - export PYTHONPATH=/usr/local/apps/pyg2p/$mod_version/lib/python2.7/site-packages - python setup.py clean --all + python_version=\$(python3 -c "import sys; print(f'python{sys.version_info.major}.{sys.version_info.minor}')") + export PYTHONPATH=$dest_dir/lib/\$python_version/site-packages + python3 setup.py clean --all mkdir -p \$PYTHONPATH - python setup.py install --prefix=/usr/local/apps/pyg2p/$mod_version + PYTHONPATH=\$ECCODES_DIR/lib/\$python_version/site-packages/:\$PYTHONPATH + python3 setup.py install --prefix=$dest_dir END echo done done From 5b6286ace229cc378313192c82103fea84b72b07 Mon Sep 17 00:00:00 2001 From: Chris Barnard Date: Fri, 20 Jan 2023 16:37:51 +0000 Subject: [PATCH 4/9] Add new glofas bilinear intertables --- configuration/global/intertables.json | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/configuration/global/intertables.json b/configuration/global/intertables.json index 10c6e35..6e624fb 100644 --- a/configuration/global/intertables.json +++ b/configuration/global/intertables.json @@ -1221,5 +1221,38 @@ 51, 61 ] + }, + "I0_359.722_M_640_421120_reduced_gg_-179_-89_0_0_-89.97_89.97_-179_-89_0_0_-179.97_179.97scipy_bilinear": { + "filename": "tbl_fcroo6400_25920000_scipy_bilinear.npy.gz", + "method": "bilinear", + "source_shape": [ + 421120 + ], + "target_shape": [ + 3600, + 7200 + ] + }, + "I0_359.723_M_640_421120_reduced_gg_-179_-89_0_0_-89.97_89.97_-179_-89_0_0_-179.97_179.97scipy_bilinear": { + "filename": "tbl_1_fcinsto640_25920000_scipy_bilinear.npy.gz", + "method": "bilinear", + "source_shape": [ + 421120 + ], + "target_shape": [ + 3600, + 7200 + ] + }, + "I0_359.86_M_1280_1661440_reduced_gg_-179_-89_0_0_-89.97_89.97_-179_-89_0_0_-179.97_179.97scipy_bilinear": { + "filename": "tbl_fcacco6400_25920000_scipy_bilinear.npy.gz", + "method": "bilinear", + "source_shape": [ + 1661440 + ], + "target_shape": [ + 3600, + 7200 + ] } } From 9d6f7fe540782eeddb672e86fd52cb44f0e0e6f8 Mon Sep 17 00:00:00 2001 From: Chris Barnard Date: Fri, 20 Jan 2023 16:43:10 +0000 Subject: [PATCH 5/9] Add ssrd to parameters.json --- configuration/global/parameters.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configuration/global/parameters.json b/configuration/global/parameters.json index 3fc2c55..19fcd06 100644 --- a/configuration/global/parameters.json +++ b/configuration/global/parameters.json @@ -205,6 +205,11 @@ "@shortName": "ssr", "@unit": "W s/m^2" }, + "ssrd": { + "@description": "Surface solar radiation downwards", + "@shortName": "ssrd", + "@unit": "W s/m^2" + }, "ssro": { "@description": "Sub-Surface Runoff", "@shortName": "ssro", From 1f50097120b715bdfa0bba08834a649ae06d5eb5 Mon Sep 17 00:00:00 2001 From: Chris Barnard Date: Fri, 27 Jan 2023 16:41:48 +0000 Subject: [PATCH 6/9] Add intertables without antarctica and ERA5 intertables --- configuration/global/intertables.json | 44 +++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/configuration/global/intertables.json b/configuration/global/intertables.json index 6e624fb..e164b50 100644 --- a/configuration/global/intertables.json +++ b/configuration/global/intertables.json @@ -1244,6 +1244,17 @@ 7200 ] }, + "I0_359.86_M_1280_1661440_reduced_gg_-179_-59_0_0_-59.97_89.97_-179_-59_0_0_-179.97_179.97scipy_bilinear": { + "filename": "tbl_fcroo6400_21600000_scipy_bilinear.npy.gz", + "method": "bilinear", + "source_shape": [ + 1661440 + ], + "target_shape": [ + 3000, + 7200 + ] + }, "I0_359.86_M_1280_1661440_reduced_gg_-179_-89_0_0_-89.97_89.97_-179_-89_0_0_-179.97_179.97scipy_bilinear": { "filename": "tbl_fcacco6400_25920000_scipy_bilinear.npy.gz", "method": "bilinear", @@ -1254,5 +1265,38 @@ 3600, 7200 ] + }, + "I0_359.722_M_640_421120_reduced_gg_-179_-59_0_0_-59.97_89.97_-179_-59_0_0_-179.97_179.97scipy_bilinear": { + "filename": "tbl_1_fcroo6400_21600000_scipy_bilinear.npy.gz", + "method": "bilinear", + "source_shape": [ + 421120 + ], + "target_shape": [ + 3000, + 7200 + ] + }, + "I0_359.723_M_640_421120_reduced_gg_-179_-59_0_0_-59.97_89.97_-179_-59_0_0_-179.97_179.97scipy_bilinear": { + "filename": "tbl_1_fcinsto640_21600000_scipy_bilinear.npy.gz", + "method": "bilinear", + "source_shape": [ + 421120 + ], + "target_shape": [ + 3000, + 7200 + ] + }, + "I0_359.718_M_640_542080_reduced_gg_-179_-59_0_0_-59.97_89.97_-179_-59_0_0_-179.97_179.97scipy_bilinear": { + "filename": "tbl_1_det0to1_21600000_scipy_bilinear.npy.gz", + "method": "bilinear", + "source_shape": [ + 542080 + ], + "target_shape": [ + 3000, + 7200 + ] } } From 6975a7eb206305703a6623986682b1f424320725 Mon Sep 17 00:00:00 2001 From: Carlo Russo Date: Fri, 12 May 2023 14:46:00 +0200 Subject: [PATCH 7/9] Small fix to PCRaster writer for gribfile missing values --- src/pyg2p/main/writers/pcr.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pyg2p/main/writers/pcr.py b/src/pyg2p/main/writers/pcr.py index a9ebf96..ea55173 100644 --- a/src/pyg2p/main/writers/pcr.py +++ b/src/pyg2p/main/writers/pcr.py @@ -1,5 +1,6 @@ from osgeo import gdal import numpy.ma as ma +import numpy as np from pyg2p.main.writers import Writer @@ -29,6 +30,7 @@ def __init__(self, *args): self._mask = ma.getmask(rs) def write(self, output_map_name, values): + values[np.isnan(values)]=self.mv drv = gdal.GetDriverByName(self.FORMAT) masked_values = self._mask_values(values) n = ma.count_masked(masked_values) From 8a3def5a1a1675e3ce6ca96dd77828069467892f Mon Sep 17 00:00:00 2001 From: Carlo Russo Date: Wed, 17 May 2023 16:38:21 +0200 Subject: [PATCH 8/9] small fix in setup tool --- .gitignore | 1 + setup.py | 32 +++++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 83cfa7d..5f0a0e3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .idea/ +.vscode/ *.py[c|o] *~*.* /build diff --git a/setup.py b/setup.py index 5f595b4..ba44bbf 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,6 @@ current_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(os.path.join(current_dir, './src/')) -import pyg2p.util.files as fm - version_file = os.path.join(current_dir, 'src/pyg2p/VERSION') with open(version_file, 'r') as f: @@ -82,17 +80,41 @@ def run(self): sys.exit() +def delete_files_from_dir(dir_path, prefix_=''): + # Gather directory contents + if is_dir(dir_path): + contents = [os.path.join(dir_path, i) for i in os.listdir(dir_path)] + # Iterate and remove each item in the appropriate manner + [os.unlink(i) for i in contents if i.startswith(prefix_)] + +def create_dir(pathname, recreate=False): + if not os.path.exists(pathname): + os.makedirs(pathname) + elif recreate: + delete_files_from_dir(pathname) + os.rmdir(pathname) + os.makedirs(pathname) + +def is_dir(pathname): + return os.path.isdir(pathname) and pathname not in ('.', '..', './', '../') + +def exists(pathname, is_folder=False): + return os.path.exists(pathname) and (os.path.isdir(pathname) if is_folder else os.path.isfile(pathname)) + +def filename(pathname): + return os.path.basename(pathname) + def setup_data_files(setup_args_): user_conf_dir = f'{os.path.expanduser("~")}/.pyg2p/' - fm.create_dir(user_conf_dir) + create_dir(user_conf_dir) list_files = {t: [os.path.join(t, f) for f in os.listdir(t) if f.endswith('.json')] for t in ('./templates', './configuration', './configuration/global')} for_user_to_copy = [f for f in list_files['./configuration'] if - not fm.exists(os.path.join(user_conf_dir, fm.filename(f)))] + not exists(os.path.join(user_conf_dir, filename(f)))] templates_to_copy = [f for f in list_files['./templates'] if - not fm.exists(os.path.join(user_conf_dir, 'templates_samples', fm.filename(f)))] + not exists(os.path.join(user_conf_dir, 'templates_samples', filename(f)))] data_files = [('pyg2p/configuration/', list_files['./configuration/global'])] if for_user_to_copy: From 5b8eb16979940c247c3304f6df6bfaaf6e0eacf3 Mon Sep 17 00:00:00 2001 From: Carlo Russo Date: Fri, 19 May 2023 11:23:39 +0200 Subject: [PATCH 9/9] Updated version number and Changelog --- CHANGE_LOG.rst | 20 +++++++++++++++++++- src/pyg2p/VERSION | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGE_LOG.rst b/CHANGE_LOG.rst index 0ea7e83..d9d1e6b 100644 --- a/CHANGE_LOG.rst +++ b/CHANGE_LOG.rst @@ -1,12 +1,30 @@ CHANGE LOG ========== +v 3.2.4 +----- +* Added values in parameters, intertables and geopotentials config files +* Fix to PCRaster writer for gribfile missing values +* Fix in setup tool + +v 3.2.3 +----- +* Fixed scale/offset use in netCDF writing function +* Using netCDF default_fillvals as missing values +* Added choice of output stepUnit value. +* Updated documentation +* Optimized scipi invdist interpolation + +v 3.2.2 +----- +* New interpolation methods: bilinear interpolation on irregular grids, Delaunay triangulation, bilinear interpolation based on Delaunay triangulation (best performances on irregular and projected grids) +* Additional average computation method on Aggregator +* NetCDF lat/lon map reading. Scale factor, offset, variable type options in NetCDF output map writing v 3.1 ----- * **74** API is much more flexible. Check documentation on how to use pyg2p programmatically. * **73** Intertables are saved as .gz now, to save disk space. It's possible to compress all existing intertables and use them without changes in configuration. If pyg2p doesn't find the intertable, it will try to read it with suffix .gz. - v 3.0 ----- * **72** Pruned old test suite and test runner script. Using pytest with oracle data from latest version (v 2.1). diff --git a/src/pyg2p/VERSION b/src/pyg2p/VERSION index 06eda28..9b7a431 100644 --- a/src/pyg2p/VERSION +++ b/src/pyg2p/VERSION @@ -1 +1 @@ -3.2.3 \ No newline at end of file +3.2.4 \ No newline at end of file