Merge pull request #84 from mvdebolskiy/fix/download-data

Fix download test data script.

Author: alejandrobodas

.gitignore

@@ -0,0 +1,10 @@
+**.o
+**.a
+**.mod
+driver/run/cosp2_test
+driver/data/inputs/UKMO/*_global.nc
+driver/data/outputs/UKMO/*kgo.v*.nc
+driver/data/outputs/UKMO/*.nc
+!driver/data/outputs/cosp2_output_um.gfortran.kgo.nc
+driver/data/outputs/UKMO/*md5
+driver/data/outputs/UKMO/*.out

README.md

@@ -50,12 +50,12 @@ The offline drivers read sample snapshots from the Met Office Unified Model, use
 
       `./cosp2_test cosp2_input_nl.um_global.txt`
 
-Currently, there are 2 input namelists: `cosp2_input_nl.txt` uses a small test input file (`cosp_input.nc`), stored in the github repository; `cosp2_input_nl.um_global.txt` uses a global field (`cosp_input.um_global.nc`). The global input file and its associated known good output (`cosp2_output.um_global.gfortran.kgo.nc`) are stored externally in Google Drive, and they can be retrieved by running `download_test_data.sh` from within the `driver/` directory.
+Currently, there are 2 input namelists: `cosp2_input_nl.txt` uses a small test input file (`cosp_input.nc`), stored in the github repository; `cosp2_input_nl.um_global.txt` uses a global field (`cosp_input.um_global.nc`). The global input file and its associated known good output (`cosp2_output.um_global.gfortran.kgo.vXXX.nc`) are stored externally in Google Drive, and they can be retrieved by running `download_test_data.sh` from within the `driver/` directory.`vXXX` represents the version of the known good output files, f.e. `v002`. You can set an environmet variable (f.e. if you are in bash shell: `export KGO_VERSION=v002`)
 
 1. Regression testing (comparing to reference data)
    1. Reference data or known good output (KGO) for a small test case is provided with COSP2. The data can be found at `driver/data/outputs/UKMO/`.
-   1. To compare driver output to the KGO, in `driver/`, invoke Python 3 script `compare_to_kgo.py`. This script requires the following Python modules: numpy, netCDF4, argparse, and sys. The following example shows how to call this script from the command line:
+   1. To compare driver output to the KGO, in `driver/`, invoke Python 3 script `compare_to_kgo.py`. This script requires the following Python modules: numpy, netCDF4, argparse, and sys. The following example shows how to call this script from the command line. Replace `vXXX` with the latest version you have acquired through `./download_test_data.sh`:
 
-      `python compare_to_kgo.py data/outputs/UKMO/cosp2_output_um.gfortran.kgo.nc data/outputs/UKMO/cosp2_output_um.nc --atol=1.0e-20 --rtol=0.0005`
+      `python compare_to_kgo.py data/outputs/UKMO/cosp2_output.um_global.gfortran.kgo.vXXX.nc data/outputs/UKMO/cosp2_output.um_global.nc --atol=1.0e-20 --rtol=0.0005`
 
    The script accepts thresholds for absolute and relative tolerances, named `atol` and `rtol` respectively. By default the script will report all differences, i.e. `--atol=0.0 --rtol=0.0`. The example above allows for a relative tolerance of 0.05\% in the subset of absolute differences larger or equal than 1.0e-20. Previous tests indicate that these thresholds are appropriate when gfortran is used to build the driver program. If a different compiler is available, it is encouraged to build the driver using that compiler to increase the robustness of the tests. For non-gfortran compilers, the differences may be larger. This is not necessarily an issue, but further investigations may be required.

driver/download_test_data.sh

@@ -1,25 +1,40 @@
-KGO_VERSION=v001
+#/bin/bash
+
+if [[ -n ${KGO_VERSION} ]] ; then
+    echo "KGO_VERSION=$KGO_VERSION"
+else
+    export KGO_VERSION=v003
+    echo "set KGO_VERSION=$KGO_VERSION"
+fi
+
+
+kgo_links=( "https://docs.google.com/uc?export=download&id=1oQBJGFg0F8k-LhRGsCYn-qWzmMMVfQ6K" \
+            "https://docs.google.com/uc?export=download&id=1b7qwJWqDzoZGcIP0qyUprTV_LErCGpT6" \
+            "https://docs.google.com/uc?export=download&id=1NvTo3bYaGpz-FUpZ4jta_kRzA04xTCsn" )
+
+out_type=( "cosp2_output_um.gfortran.kgo" \
+           "cosp2_output.um_global.gfortran.kgo" \
+           "cosp2_output.um_global_model_levels.gfortran.kgo" )
+
 # Input: global UM
 DRIVER_DIR=${PWD}
 GDFILE='https://docs.google.com/uc?export=download&id=17eK4_DVEvFOE9Uf6siXJDpWZJKT1aqkU'
 OUTPATH=data/inputs/UKMO/cosp_input.um_global.nc.gz
 wget --no-check-certificate $GDFILE -O $OUTPATH
-gunzip ${OUTPATH}
+gunzip -f ${OUTPATH}
 cd data/inputs/UKMO
 md5sum -c cosp_input.um_global.nc.md5
-# KGO: global UM
-cd ${DRIVER_DIR}
-GDFILE='https://docs.google.com/uc?export=download&id=1uQBPUEXlniQWEp2nU3iC6d8CO3GM9cvP'
-OUTPATH=data/outputs/UKMO/cosp2_output.um_global.gfortran.kgo.$KGO_VERSION.nc.gz
-wget --no-check-certificate $GDFILE -O $OUTPATH
-gunzip ${OUTPATH}
-cd data/outputs/UKMO
-md5sum -c cosp2_output.um_global.gfortran.kgo.$KGO_VERSION.nc.md5
-# KGO:  UM
-cd ${DRIVER_DIR}
-GDFILE='https://docs.google.com/uc?export=download&id=1gSEdJJpqhfElsFNcTF_r4A_0vIMEWGla'
-OUTPATH=data/outputs/UKMO/cosp2_output_um.gfortran.kgo.$KGO_VERSION.nc.gz
-wget --no-check-certificate $GDFILE -O $OUTPATH
-gunzip ${OUTPATH}
-cd data/outputs/UKMO
-md5sum -c cosp2_output_um.gfortran.kgo.$KGO_VERSION.nc.md5
+
+if [[ ${KGO_VERSION} == "v002" ]] ; then
+  for i in ${!kgo_links[@]}; do
+    cd ${DRIVER_DIR}
+    GDFILE=${kgo_links[$i]}
+    OUTPATH=data/outputs/UKMO/${out_type[$i]}.$KGO_VERSION.nc.gz
+    wget --no-check-certificate $GDFILE -O $OUTPATH
+    gunzip -f ${OUTPATH}
+    cd data/outputs/UKMO
+    md5sum -c ${out_type[$i]}.$KGO_VERSION.nc.md5
+  done
+else
+  echo "wrong KGO_VERSION supplied, try 'export KGO_VERSION=v002'"
+fi