Merge pull request #1 from Tristanovsk/v1.3

V1.3

Makefile

@@ -1,4 +1,4 @@
-export F2PY=f2py3.8
+export F2PY=f2py
 
 DIRS = grs/fortran/grs grs/fortran/grs_a grs/landsat_angles/OLI grs/landsat_angles/TM
 

README.md

@@ -107,13 +107,35 @@ python setup.py install --user
 
 If the installation is successful, you should have:
 ```
+
 $ grs
 Usage:
   grs <input_file> [--sensor <sensor>] [-o <ofile>] [--odir <odir>] [--shape <shp>] [--wkt <wktfile>]   [--longlat <longmax,longmin,latmax,latmin> ]    [--altitude=alt] [--dem] [--aerosol=DB] [--aeronet=<afile>]    [--aot550=aot] [--angstrom=ang] [--output param]   [--resolution=res] [--levname <lev>] [--no_clobber] [--memory_safe] [--unzip]
   grs -h | --help
   grs -v | --version
 ```
 
+### On the PBS cluster : installing from sources with conda on the cluster CNES
+
+Create the conda environment using the definition file available in the conda folder :
+```
+conda env create -f conda/grs_conda_3.6.yml -p /work/scratch/$user/grs_py3.6
+````
+The option -p set the directory where the conda environment will be installed
+
+To install the package grs in conda :
+
+```
+source conda/conda_grs.sh -ci
+```
+
+
+To launch GRS on a pbs node :
+
+```
+qsub launch_grs_exemple.pbs
+```
+
 ## Running the tests
 From terminal:
 ```

amalthee_launcher.pbs

@@ -0,0 +1,27 @@
+# launch with qsub (PBS job)
+#!/bin/bash
+
+#PBS -V
+#PBS -N amalth
+#PBS -l select=1:ncpus=2:mpiprocs=2:mem=2gb
+#PBS -l walltime=24:00:00
+
+module load conda
+
+# set file path with list of images to process
+# format/header of the file in csv:
+# process (yes if 1),Site Name,start_date,end_date,satellite,tile,Latitude,Longitude,resolution (m)
+list_file=list_grs_cnes_obs2mod.csv
+#list_file=list_grs_cnes_guimaraes.csv
+#list_file=list_grs_cnes_seine.csv
+#list_file=list_grs_gernez_juillet_2021.csv
+
+GRSHOME=/home/eh/harmelt/scratch/dev/grs2
+list_filepath=$GRSHOME/exe/$list_file
+
+# ------------------------
+# first section: load L1C and L2A images with Amalthee
+# ------------------------
+
+conda activate /softs/rh7/conda-envs/amalthee_prod_client
+/softs/rh7/conda-envs/amalthee_prod_client/bin/python $GRSHOME/exe/call_amalthee.py $list_filepath

conda/conda_grs.sh

@@ -0,0 +1,117 @@
+#!/bin/sh
+
+# Après installation de snap, faire un lien symbolique de esasnappy vers snappy (qui se trouve dans le .snap du home
+
+#desactivate conda s'il est activé
+conda deactivate 2>/dev/null
+
+#On récupère la connexion au proxy
+[ "$_" = "$0" ] && echo "Ce script doit être sourcé!" && exit 127
+
+read -s -p "Please enter your proxy password ? " _passwd || exit 127
+echo
+
+_user=${_user:-$USER}
+
+if [ -z ${_passwd} ]
+then
+	echo "No password"
+else
+
+	export http_proxy="http://${_user}:${_passwd}@proxy-surf.loc.cnes.fr:8050"
+	export https_proxy="http://${_user}:${_passwd}@proxy-surf.loc.cnes.fr:8050"
+
+#echo 'http_proxy variable has to be set : '$http_proxy
+
+	export ftp_proxy="${http_proxy}"
+	export no_proxy=cnes.fr,sis.cnes.fr,gitlab.cnes.fr
+
+	unset _passwd
+	unset _user
+fi
+
+#load avail module for conda, snap and jdk
+module load snap/8.0 conda jdk/1.8.0_112
+
+#otb-depends/7.0-python3.6.5
+
+#switch to gcc/4.8.5
+#module unload gcc
+
+#set the variable environment for je jpy and snap use
+export JDK_HOME=$JDKHOME
+export JAVAHOME=$JAVA_HOME
+export SNAP_HOME=$SNAPHOME
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:.
+
+
+#activate conda environement python 3.6
+conda activate /work/scratch/$USER/grs_py3.6
+echo $USER
+
+#configuration for snap
+# go to the $SNAPHOME environment and configure snappy
+pythonpath=$(which python)
+echo $pythonpath
+
+pippath=$(which pip)
+echo $pippath
+
+#Install snappy
+#$SNAPHOME/bin/snappy-conf $pythonpath
+
+if [ ! -d $CONDA_PREFIX/lib/python3.6/site-packages/esasnappy ]
+then
+	ln -s /work/ALT/swot/aval/OBS2CO/snap/snappy $CONDA_PREFIX/lib/python3.6/site-packages/esasnappy
+fi
+
+
+echo $'snap.versionCheck.interval=NEVER\nsnap.jai.tileCacheSize=1024' > $HOME/.snap/etc/snap.properties
+
+#Install the jpy package used by snappy
+pip install /work/ALT/swot/aval/OBS2CO/snap/snappy/lib/jpy-0.9.0-cp36-cp36m-linux_x86_64.whl nco
+
+#Modifier la config jpy pour java
+cat $CONDA_PREFIX/lib/python3.6/site-packages/jpyconfig.py | grep '#java_home'
+
+if [ $? == 1 ]
+then
+	sed -i 's/java_home/#java_home/g' $CONDA_PREFIX/lib/python3.6/site-packages/jpyconfig.py
+	sed -i 's/jvm_dll/#jvm_dll/g' $CONDA_PREFIX/lib/python3.6/site-packages/jpyconfig.py
+fi
+
+#if it is not already installed
+#conda install GDAL
+#compilation
+
+cd /work/ALT/swot/aval/OBS2CO/git/grs2
+
+#catch egm96 data
+mkdir -p $HOME/.snap/auxdata/dem/egm96
+cp /work/ALT/swot/aval/OBS2CO/git/grsdata/dem/ww15mgh_b.zip $HOME/.snap/auxdata/dem/egm96/ww15mgh_b.zip
+
+
+if [ $# != 0 ]
+then
+	if [ $1 = "-c" ]
+	then
+        	echo "Compilation"
+        	make
+	else 
+		if [ $1 = "-i" ]
+		then
+        		echo "install"
+        		python setup.py install
+		else
+			if [ $1 = "-ci" ]
+			then
+				echo 'compilation and installation'
+				make
+				python setup.py install
+			fi
+		fi
+	fi
+fi
+
+
+echo "Tester GRS sur une tuile sentinel 2 : grs /work/ALT/swot/aval/OBS2CO/git/grsdata/INPUT_DATA/S2B_MSIL1C_20180927T103019_N0206_R108_T31TGK_20180927T143835.SAFE --shape test/data/shape/SPO04.shp --odir test/results/ --aerosol cams_forecast --dem --resolution 20 \nPreciser le répertoire de sortie dans --odir"

conda/grs_conda_3.6.yml

@@ -0,0 +1,82 @@
+name: grs_py3.6
+channels:
+  - anaconda
+  - conda-forge
+  - orfeotoolbox
+  - defaults
+dependencies:
+  - ca-certificates=2020.10.14=0
+  - certifi=2020.6.20=py36_0
+  - openssl=1.1.1h=h7b6447c_0
+  - pip=20.2.4=py36_0
+  - _libgcc_mutex=0.1=main
+  - blas=1.0=mkl
+  - bzip2=1.0.8=h7b6447c_0
+  - cairo=1.14.12=h8948797_3
+  - cfitsio=3.470=hb7c8383_2
+  - curl=7.67.0=hbc83047_0
+  - expat=2.2.10=he6710b0_2
+  - fontconfig=2.13.0=h9420a91_0
+  - freetype=2.10.4=h5ab3b9f_0
+  - freexl=1.0.5=h14c3975_0
+  - gdal=3.0.2=py36hbb2a789_0
+  - geos=3.8.0=he6710b0_0
+  - geotiff=1.5.1=h21e8280_1
+  - giflib=5.1.4=h14c3975_1
+  - glib=2.63.1=h5a9c865_0
+  - hdf4=4.2.13=h3ca952b_2
+  - hdf5=1.10.4=hb1b8bf9_0
+  - icu=58.2=he6710b0_3
+  - intel-openmp=2020.2=254
+  - jpeg=9b=h024ee3a_2
+  - json-c=0.13.1=h1bed415_0
+  - kealib=1.4.7=hd0c454d_6
+  - krb5=1.16.4=h173b8e3_0
+  - ld_impl_linux-64=2.33.1=h53a641e_7
+  - libboost=1.67.0=h46d08c1_4
+  - libcurl=7.67.0=h20c2e04_0
+  - libdap4=3.19.1=h6ec2957_0
+  - libedit=3.1.20191231=h14c3975_1
+  - libffi=3.2.1=hf484d3e_1007
+  - libgcc-ng=9.1.0=hdf63c60_0
+  - libgdal=3.0.2=h27ab9cc_0
+  - libgfortran-ng=7.3.0=hdf63c60_0
+  - libkml=1.3.0=h590aaf7_4
+  - libnetcdf=4.6.1=h11d0813_2
+  - libpng=1.6.37=hbc83047_0
+  - libpq=11.2=h20c2e04_0
+  - libspatialite=4.3.0a=h793db0d_0
+  - libssh2=1.9.0=h1ba5d50_1
+  - libstdcxx-ng=9.1.0=hdf63c60_0
+  - libtiff=4.1.0=h2733197_0
+  - libuuid=1.0.3=h1bed415_2
+  - libxcb=1.14=h7b6447c_0
+  - libxml2=2.9.10=hb55368b_3
+  - lz4-c=1.8.1.2=h14c3975_0
+  - mkl=2020.2=256
+  - mkl-service=2.3.0=py36he904b0f_0
+  - mkl_fft=1.2.0=py36h23d657b_0
+  - mkl_random=1.1.1=py36h0573a6f_0
+  - ncurses=6.2=he6710b0_1
+  - numpy=1.19.1=py36hbc911f0_0
+  - numpy-base=1.19.1=py36hfa32c7d_0
+  - openjpeg=2.3.0=h05c96fa_1
+  - pcre=8.44=he6710b0_0
+  - pixman=0.40.0=h7b6447c_0
+  - poppler=0.65.0=h581218d_1
+  - poppler-data=0.4.9=0
+  - postgresql=11.2=h20c2e04_0
+  - proj=6.2.1=haa6030c_0
+  - python=3.6.10=h191fe78_1
+  - readline=7.0=h7b6447c_5
+  - setuptools=50.3.0=py36hb0f4dca_1
+  - six=1.15.0=py_0
+  - sqlite=3.33.0=h62c20be_0
+  - tbb=2018.0.5=h6bb024c_0
+  - tiledb=1.6.3=h1fb8f14_0
+  - tk=8.6.10=hbc83047_0
+  - wheel=0.35.1=py_0
+  - xerces-c=3.2.3=h780794e_0
+  - xz=5.2.5=h7b6447c_0
+  - zlib=1.2.11=h7b6447c_3
+  - zstd=1.3.7=h0b5b093_0

ecmwf/download_cams_all.py

@@ -0,0 +1,103 @@
+import os, sys
+
+import argparse
+from datetime import date
+import calendar
+from ecmwfapi import ECMWFDataServer
+
+"""
+This function download the cams data used in grs 
+
+The main program takes one argument : the mode cams_reanalysis or cams_forecast needed
+
+"""
+
+
+def main(dic):
+    data_type = dic['mode']
+    month = dic['month']
+    year_start = int(dic['year_start'])
+    year_end = int(dic['year_end'])
+    area = "90/-180/-90/180"
+
+    if month != 'all':
+        today = date.today()
+        d1 = today.strftime("%Y%m%d")
+
+    server = ECMWFDataServer()
+
+    # dataset will be download by default from 2017 to 2020
+    step = '0'
+    if data_type == 'cams_forecast':
+        class_ = 'mc'
+        dataset = 'cams_nrealtime'
+        time = '00:00:00'
+        step = '0/6/12/18'
+        type = 'fc'
+
+    # data will be download by default from 2000 to 2017
+    elif data_type == 'cams_reanalysis':
+        class_ = 'mc'
+        dataset = 'cams_reanalysis'
+        # date = '20150301/TO/20170101'
+        time = '00:00:00/06:00:00/12:00:00/18:00:00'
+        type = 'an'
+
+    elif data_type == 'interim':
+        class_ = 'ei'
+        dataset = 'interim'
+        type = 'an'
+    else:
+        print('Error: not appropriate dataset for ecmwf/cams download')
+        sys.exit()
+
+    # specify the period to catch data
+    # try:
+    for year in range(year_start, year_end + 1):
+        odir = "/datalake/watcal/ECMWF/CAMS/" + str(year) + "/"
+        if not os.path.exists(odir):
+            os.makedirs(odir)
+        for month in range(1, 13):
+            target = odir + str(year) + "-" + str(month).zfill(
+                2) + "_month_" + data_type + ".nc"
+
+            if os.path.exists(target):
+                continue
+
+            numberOfDays = calendar.monthrange(year, month)[1]
+            date = str(year) + str(month).zfill(2) + "01/TO/" + str(year) + str(month).zfill(2) + str(numberOfDays)
+            print(date)
+            server.retrieve({
+                'class': class_,
+                'dataset': dataset,
+                'date': date,
+                'grid': "0.125/0.125",
+                'levtype': 'sfc',
+                'param': "125.210/137.128/151.128/165.128/166.128/167.128/206.128/207.210/213.210/214.210/215.210/216.210",
+                'stream': 'oper',
+                'step': step,
+                'time': time,
+                'type': type,
+                'format': 'netcdf',
+                'area': "90/-180/-90/180",
+                'target': target
+            })
+    return
+    # except:
+    #    print('Error: not appropriate cams settings for download. Refers to ecmwf.')
+    #    sys.exit()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Download Cams datasets.')
+    parser.add_argument('mode',
+                        help='the cams_forecast or cams_reanalysis mode')
+    parser.add_argument('month',
+                        help='the month from which data will be downloaded. Set all for downloading data from 2000 to today. Set a date in %Y%m%d format if you want to download data from this date until now.')
+    parser.add_argument('year_start',
+                        help='the start year of period to be downloaded in case of month = all ')
+    parser.add_argument('year_end',
+                        help='the end year of period to be downloaded in case of month = all ')
+    args = parser.parse_args()
+
+    main(vars(args))