Accumulated changes including reorganization to main (#235)

Code runs in single precision with differences up to 0.13 W/m2 in example problems. A new class represents MERRA aerosol optical properties. The directory layout is reorganized (front-ends, kernels) and data moved to a separate repo. Citation information added. See PR #235 for all changes. Closes #39 #128 #156 #160 #193 #203 #214
earth-system-radiation · Nov 27, 2023 · 3ac0636 · 3ac0636
1 parent 74a0e09
commit 3ac0636
Show file tree

Hide file tree

Showing 82 changed files with 3,498 additions and 1,540 deletions.
diff --git a/.github/workflows/containerized-ci.yml b/.github/workflows/containerized-ci.yml
@@ -10,63 +10,70 @@ on:
 jobs:
   Containerized-CI:
     runs-on: ubuntu-22.04
+    continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
         fortran-compiler: [ifort, ifx, nvfortran]
-        rte-kernels: [default, openacc]
+        rte-kernels: [default, accel]
+        fpmodel: [DP, SP]
         include:
+        # The tests are not experimental by default:
+        - experimental: false
+        # Set flags for Intel Fortran Compiler Classic
         - fortran-compiler: ifort
-          fcflags: "-m64 -g  -traceback -heap-arrays -assume realloc_lhs -extend-source 132 -check bounds,uninit,pointers,stack -stand f08"
-          image: "earthsystemradiation/rte-rrtmgp-ci:ifort"
+          fcflags: -m64 -g -traceback -heap-arrays -assume realloc_lhs -extend-source 132 -check bounds,uninit,pointers,stack -stand f08
+        # Set flags for Intel Fortran Compiler
         - fortran-compiler: ifx
-          fcflags: "-m64 -g  -traceback -heap-arrays -assume realloc_lhs -extend-source 132 -check bounds,uninit,pointers,stack -stand f08"
-          image: "earthsystemradiation/rte-rrtmgp-ci:ifort"
+          rte-kernels: default
+          fcflags: -debug -traceback -O0 -heap-arrays -assume realloc_lhs -extend-source 132 -stand f08
+        - fortran-compiler: ifx
+          rte-kernels: accel
+          fcflags: -debug -traceback -O0 -heap-arrays -assume realloc_lhs -extend-source 132 -stand f08 -fiopenmp -fopenmp-targets=spir64
+          experimental: true
+        # Set flags for NVIDIA Fortran compiler
+        - fortran-compiler: nvfortran
+          rte-kernels: default
+          fcflags: -Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk
+        - fortran-compiler: nvfortran
+          rte-kernels: accel
+          fcflags: -Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk -acc
+        # Set container images
+        - fortran-compiler: ifort
+          image: ghcr.io/earth-system-radiation/rte-rrtmgp-ci:ifort
+        - fortran-compiler: ifx
+          image: ghcr.io/earth-system-radiation/rte-rrtmgp-ci:ifort
         - fortran-compiler: nvfortran
-          fcflags: "-Mallocatable=03 -Mstandard -Mbounds -Mchkptr -Kieee -Mchkstk"
-          image: "earthsystemradiation/rte-rrtmgp-ci:nvfortran"
+          image: ghcr.io/earth-system-radiation/rte-rrtmgp-ci:nvfortran
     container:
       image: ${{ matrix.image }}
     env:
       # Core variables:
       FC: ${{ matrix.fortran-compiler }}
-      FCFLAGS: ${{ matrix.fcflags }}
+      FCFLAGS: ${{ matrix.fcflags }} -DRTE_USE_${{ matrix.fpmodel}}
       # Make variables:
-      NCHOME: /dummy
       NFHOME: /opt/netcdf-fortran
       RRTMGP_ROOT: ${{ github.workspace }}
+      RRTMGP_DATA: ${{ github.workspace }}/rrtmgp-data
       RTE_KERNELS: ${{ matrix.rte-kernels }}
       RUN_CMD:
-      # Auxiliary variables:
-      RFMIP_CACHEDIR: /home/runner/rfmip-files
+      # https://github.com/earth-system-radiation/rte-rrtmgp/issues/194
+      OMP_TARGET_OFFLOAD: DISABLED
+      FAILURE_THRESHOLD: 7.e-4
+
     steps:
     #
     # Checks-out repository under $GITHUB_WORKSPACE
     #
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     #
-    # Cache RFMIP files
+    # Check out data
     #
-    - name: Cache RFMIP files
-      uses: actions/cache@v3
+    - name: Check out data
+      uses: actions/checkout@v4
       with:
-        path: ${{ env.RFMIP_CACHEDIR }}
-        key: rfmip-files
-    #
-    # Stage RFMIP files
-    #
-    - name: Stage RFMIP files
-      run: |
-        if test ! -d "${RFMIP_CACHEDIR}"; then
-          mkdir -p "${RFMIP_CACHEDIR}" && cd "${RFMIP_CACHEDIR}"
-          python "${GITHUB_WORKSPACE}/examples/rfmip-clear-sky/stage_files.py"
-        fi
-        for file in "${RFMIP_CACHEDIR}"/*; do
-          if test ! -d "${file}"; then
-            echo "copying '${file}'..."
-            cp "${file}" "${GITHUB_WORKSPACE}/examples/rfmip-clear-sky/"
-          fi
-        done
+        repository: earth-system-radiation/rrtmgp-data
+        path: rrtmgp-data
     #
     # Build libraries, examples and tests
     #
@@ -81,6 +88,12 @@ jobs:
     - name: Run examples and tests
       run: make tests
     #
+    # Relax failure thresholds for single precision
+    #
+    - name: Relax failure threshold for single precision
+      if: matrix.fpmodel == 'SP'
+      run: echo "FAILURE_THRESHOLD=3.5e-1" >> $GITHUB_ENV
+    #
     # Compare the results
     #
     - name: Compare the results
@@ -89,14 +102,14 @@ jobs:
     # Generate validation plots
     #
     - name: Generate validation plots
-      if: matrix.fortran-compiler == 'ifort' && matrix.rte-kernels == 'default'
+      if: matrix.fortran-compiler == 'ifort' && matrix.rte-kernels == 'default' && matrix.fpmodel == 'DP'
       working-directory: tests
       run: python validation-plots.py
     #
     # Upload validation plots
     #
     - name: Upload validation plots
-      if: matrix.fortran-compiler == 'ifort' && matrix.rte-kernels == 'default'
+      if: matrix.fortran-compiler == 'ifort' && matrix.rte-kernels == 'default' && matrix.fpmodel == 'DP'
       uses: actions/upload-artifact@v3
       with:
         name: valdiation-plot

diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml
@@ -19,24 +19,37 @@ jobs:
       fail-fast: false
       matrix:
         fortran-compiler: [gfortran-10, gfortran-11, gfortran-12]
-        rte-kernels: [default, openacc]
+        fpmodel: [DP, SP]
     env:
       # Core variables:
       FC: ${{ matrix.fortran-compiler }}
-      FCFLAGS: "-ffree-line-length-none -m64 -std=f2008 -march=native -fbounds-check -finit-real=nan -g -DRTE_USE_CBOOL"
+      FCFLAGS: "-ffree-line-length-none -m64 -std=f2008 -march=native -fbounds-check -fmodule-private -fimplicit-none -finit-real=nan -g -DRTE_USE_CBOOL -DRTE_USE_${{ matrix.fpmodel }}"
       # Make variables:
-      NCHOME: /dummy
       NFHOME: /usr
       RRTMGP_ROOT: ${{ github.workspace }}
-      RTE_KERNELS: ${{ matrix.rte-kernels }}
+      RRTMGP_DATA: ${{ github.workspace }}/rrtmgp-data
       RUN_CMD:
-      # Auxiliary variables:
-      RFMIP_CACHEDIR: /home/runner/rfmip-files
+      FAILURE_THRESHOLD: 7.e-4
     steps:
     #
-    # Checks-out repository under $GITHUB_WORKSPACE
+    # Relax failure thresholds for single precision
     #
-    - uses: actions/checkout@v3
+    - name: Relax failure threshold for single precision
+      if: matrix.fpmodel == 'SP'
+      run: echo "FAILURE_THRESHOLD=3.5e-1" >> $GITHUB_ENV
+    #
+    # Check out repository under $GITHUB_WORKSPACE
+    #
+    - name: Check out code
+      uses: actions/checkout@v4
+    #
+    # Check out data
+    #
+    - name: Check out data
+      uses: actions/checkout@v4
+      with:
+        repository: earth-system-radiation/rrtmgp-data
+        path: rrtmgp-data
     #
     # Synchronize the package index
     #
@@ -69,29 +82,6 @@ jobs:
         # Use the cache properly:
         use-only-tar-bz2: true
     #
-    # Cache RFMIP files
-    #
-    - name: Cache RFMIP files
-      uses: actions/cache@v3
-      with:
-        path: ${{ env.RFMIP_CACHEDIR }}
-        key: rfmip-files
-    #
-    # Stage RFMIP files
-    #
-    - name: Stage RFMIP files
-      run: |
-        if test ! -d "${RFMIP_CACHEDIR}"; then
-          mkdir -p "${RFMIP_CACHEDIR}" && cd "${RFMIP_CACHEDIR}"
-          python "${GITHUB_WORKSPACE}/examples/rfmip-clear-sky/stage_files.py"
-        fi
-        for file in "${RFMIP_CACHEDIR}"/*; do
-          if test ! -d "${file}"; then
-            echo "copying '${file}'..."
-            cp "${file}" "${GITHUB_WORKSPACE}/examples/rfmip-clear-sky/"
-          fi
-        done
-    #
     # Build libraries, examples and tests
     #
     - name: Build libraries, examples and tests

diff --git a/.github/workflows/doc-deployment.yml b/.github/workflows/doc-deployment.yml
@@ -12,7 +12,7 @@ jobs:
     #
     # Checks-out repository under $GITHUB_WORKSPACE
     #
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     #
     # Synchronize the package index
     #
@@ -70,7 +70,7 @@ jobs:
     # Deploy documentation
     #
     - name: Deploy API Documentation
-      uses: JamesIves/github-pages-deploy-action@v4.4.1
+      uses: JamesIves/github-pages-deploy-action@v4.4.3
       if: ${{ github.event_name == 'push'  &&  github.ref == 'refs/heads/main' }}
       with:
         branch: gh-pages

diff --git a/.github/workflows/self-hosted-ci.yml b/.github/workflows/self-hosted-ci.yml
@@ -13,46 +13,57 @@ defaults:
 
 jobs:
   CI:
-    runs-on: daint
+    runs-on:
+      labels: cscs-ci
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
+        config-name: [nvidia-gpu-openacc, cce-cpu-icon-production, cce-gpu-openmp]
+        fpmodel: [DP, SP]
         include:
-        - config-name: nvidia-gpu-acc
-          rte-kernels: openacc
+        # The tests are not experimental by default:
+        - experimental: false
+        - config-name: nvidia-gpu-openacc
+          rte-kernels: accel
           compiler-modules: "PrgEnv-nvidia nvidia craype-accel-nvidia60 cdt-cuda/21.09 !cray-libsci_acc"
           # Generic accelerator flag
           fcflags: "-O3 -acc -Mallocatable=03 -gopt"
-          experimental: false
         - config-name: cce-cpu-icon-production
           rte-kernels: default
           compiler-modules: "PrgEnv-cray"
           # Production flags for Icon model
           fcflags: "-hadd_paren -r am -Ktrap=divz,ovf,inv -hflex_mp=intolerant -hfp1 -hnoacc -O1,cache0"
-          experimental: false
         - config-name: cce-gpu-openmp
-          rte-kernels: openacc
-          compiler-modules: "PrgEnv-cray craype-accel-nvidia60 cdt-cuda/22.05"
+          rte-kernels: accel
+          compiler-modules: "PrgEnv-cray craype-accel-nvidia60 cdt-cuda/22.05 cudatoolkit/11.2.0_3.39-2.1__gf93aa1c"
           # OpenMP flags from Nichols Romero (Argonne)
           fcflags: "-hnoacc -homp -O0"
           experimental: true
     env:
       # Core variables:
       FC: ftn
-      FCFLAGS: ${{ matrix.fcflags }}
+      FCFLAGS: ${{ matrix.fcflags }} -DRTE_USE_${{ matrix.fpmodel}}
       # Make variables:
-      NCHOME: /dummy
-      NFHOME: /dummy
       RRTMGP_ROOT: ${{ github.workspace }}
+      RRTMGP_DATA: ${{ github.workspace }}/rrtmgp-data
       RTE_KERNELS: ${{ matrix.rte-kernels }}
       RUN_CMD: "srun -C gpu -A d56 -p cscsci -t 15:00"
+      FAILURE_THRESHOLD: 7.e-4
     steps:
     #
     # Checks-out repository under $GITHUB_WORKSPACE
     #
     - uses: actions/checkout@v3
     #
+    # Check out data
+    #
+    - name: Check out data
+      uses: actions/checkout@v3
+      with:
+        repository: earth-system-radiation/rrtmgp-data
+        path: rrtmgp-data
+    #
     # Finalize build environment
     #
     - name: Finalize build environment
@@ -68,12 +79,18 @@ jobs:
         #     python3 -m venv /scratch/snx3000/rpincus/rte-rrtmgp-python
         #     /scratch/snx3000/rpincus/rte-rrtmgp-python/bin/pip3 install --upgrade pip
         #     /scratch/snx3000/rpincus/rte-rrtmgp-python/bin/pip3 install dask[array] netCDF4 numpy xarray
-        echo "PATH=\"/scratch/snx3000/rpincus/rte-rrtmgp-python/bin:\${PATH}\"" >> "${BASH_ENV}"
+        echo 'PATH="/scratch/snx3000/rpincus/rte-rrtmgp-python/bin:${PATH}"' >> "${BASH_ENV}"
         # Make bash run the above on startup:
         echo "BASH_ENV=${BASH_ENV}" >> "${GITHUB_ENV}"
         # Compiler needs more temporary space than normally available:
         tmpdir='${{ github.workspace }}/tmp'
         mkdir "${tmpdir}" && echo "TMPDIR=${tmpdir}" >> "${GITHUB_ENV}"
+        # We use the "non-default products" for the tests
+        # (see https://support.hpe.com/hpesc/public/docDisplay?docId=a00113984en_us&page=Modify_Linking_Behavior_to_Use_Non-default_Libraries.html):
+        echo 'LD_LIBRARY_PATH="${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH}"' >> "${BASH_ENV}"
+        # SLURM jobs, user home directories and HDF5 file locking are
+        # incompatible on Daint:
+        echo 'HDF5_USE_FILE_LOCKING=FALSE' >> "${GITHUB_ENV}"
     #
     # Build libraries, examples and tests
     #
@@ -88,6 +105,12 @@ jobs:
     - name: Run examples and tests
       run: make tests
     #
+    # Relax failure thresholds for single precision
+    #
+    - name: Relax failure threshold for single precision
+      if: matrix.fpmodel == 'SP'
+      run: echo "FAILURE_THRESHOLD=3.5e-1" >> $GITHUB_ENV
+    #
     # Compare the results
     #
     - name: Compare the results

diff --git a/CITATION.cff b/CITATION.cff
@@ -0,0 +1,66 @@
+cff-version: 1.2.0
+title: "RTE+RRTMGP"
+authors:
+  - family-names: Pincus
+    given-names: Robert
+    orcid: "https://orcid.org/0000-0002-0016-3470"
+  - family-names: Iacono
+    given-names: Michael J. 
+    orcid: "https://orcid.org/0000-0002-9578-0649"
+  - family-names: Alexeev
+    given-names: Dmitry 
+    orcid: "https://orcid.org/0000-0002-6425-2181"
+  - family-names: Adamidis
+    given-names: Panos
+  - family-names: Hillman
+    given-names: Benjamin R.
+    orcid: "https://orcid.org/0000-0002-9264-9872"
+  - family-names: Norman
+    given-names: Matthew
+    orcid: "https://orcid.org/0000-0003-4764-3348"
+  - family-names: Pfister
+    given-names: Erik 
+    orcid: "http://orcid.org/0009-0002-2688-138X"
+  - family-names: Polonsky
+    given-names: Igor N.
+  - family-names: Romero
+    given-names: Nicols A. 
+  - family-names: Wehe
+    given-names: Andre
+type: software
+repository-code: "https://github.com/earth-system-radiaton/rte-rrtmgp"
+license: BSD-3-Clause
+date-released: "2023-05-30"
+version: 1.7
+
+abstract: "RTE+RRTMGP is a set of codes for computing radiative fluxes in planetary atmospheres. 
+  RRTMGP uses a k-distribution to provide an optical description (absorption and possibly Rayleigh optical depth) of the gaseous atmosphere, along with the relevant source functions, on a pre-determined spectral grid given temperatures, pressures, and gas concentration. RTE computes fluxes given spectrally-resolved optical descriptions and source functions. The fluxes are normally summarized or reduced via a user extensible class."
+identifiers:
+  - description: "All versions"
+    type: doi
+    value: 10.5281/zenodo.3403172
+keywords:
+  - "RTE+RRTMGP"
+  - "atmospheric radiation"
+  - "radiative transfer"
+  - "climate model"
+  - research
+message: "If you use this software, please cite it using these metadata."
+
+references:
+  - authors:
+    - family-names: Pincus
+      given-names: Robert
+    - family-names: Mlawer
+      given-names: Eli J.
+    - family-names: Delamere
+      given-names: Jennifer S.
+    doi: 10.1029/2019MS001621
+    journal: "Journal of Advances in Modeling Earth Systems"
+    scope: "This paper describes the design, concepts, and version 1.0 of the software"
+    title: "Balancing Accuracy, Efficiency, and Flexibility in Radiation Calculations for Dynamical Models"
+    type: article
+    volume: 11
+    number: 10
+    pages: 3074-3089
+    year: 2019