Add some testing of YAML formatting checks and start adding marine YAMLs

.github/run_jcb_basic_testing.yaml

@@ -0,0 +1,61 @@
+name: Run JCB basic testing with jcb-gdas changes
+
+on:
+  push:
+    branches:
+      - develop
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    env:
+      JCB_REPO: https://github.com/NOAA-EMC/jcb.git
+      JCB_DEVELOP_BRANCH: develop
+
+    steps:
+    - name: Checkout submodule repository
+      uses: actions/checkout@v3
+      with:
+        submodules: true
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'  # specify your python version
+
+    - name: Determine branch to use
+      id: determine-branch
+      run: |
+        # Determine if the branch exists in the jcb repo
+        BRANCH_NAME=${{ github.ref_name }}
+        git ls-remote --exit-code --heads $JCB_REPO $BRANCH_NAME
+        if [ $? -eq 0 ]; then
+          echo "Branch $BRANCH_NAME exists in jcb repo."
+          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
+        else
+          echo "Branch $BRANCH_NAME does not exist in jcb repo. Using develop branch."
+          echo "BRANCH_NAME=$JCB_DEVELOP_BRANCH" >> $GITHUB_ENV
+        fi
+
+    - name: Clone jcb repository
+      run: |
+        git clone --branch ${{ env.BRANCH_NAME }} $JCB_REPO jcb_repo
+        cd jcb_repo/src/jcb/configuration/apps/gdas
+        git checkout ${{ env.BRANCH_NAME }}
+        cd -
+
+    - name: Install dependencies
+      run: |
+        cd jcb_repo
+        pip install .[testing]
+
+    - name: Run unit tests
+      run: |
+        cd $GITHUB_WORKSPACE
+        pytest -v

model/atmosphere/background.yaml.j2

@@ -1,4 +1,4 @@
-datapath: {{ background_path}}
+datapath: {{ atm_background_path}}
 filetype: fms restart
 datetime: "{{ atm_background_time_iso }}"
 filename is datetime templated: true

model/atmosphere/background_ensemble.yaml.j2

@@ -6,13 +6,13 @@ members from template:
     state variables: [ua,va,t,DZ,delp,ps,sphum,ice_wat,liq_wat,o3mr,phis,
                      slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph,
                      u_srf,v_srf,f10m]
-    datapath: {{ background_ensemble_path}}
+    datapath: {{ atm_background_ensemble_path}}
     filename is datetime templated: true
     filename_core: "%yyyy%mm%dd.%hh%MM%ss.fv_core.res.nc"
     filename_trcr: "%yyyy%mm%dd.%hh%MM%ss.fv_tracer.res.nc"
     filename_sfcd: "%yyyy%mm%dd.%hh%MM%ss.sfc_data.nc"
     filename_sfcw: "%yyyy%mm%dd.%hh%MM%ss.fv_srf_wnd.res.nc"
     filename_cplr: "%yyyy%mm%dd.%hh%MM%ss.coupler.res"
-  pattern: "{{ ensemble_pattern | default("%mem%", true) }}"
-  nmembers: {{ number_ensemble_members }}
+  pattern: "{{ atm_ensemble_pattern | default("%mem%", true) }}"
+  nmembers: {{ atm_number_ensemble_members }}
   zero padding: 3

model/atmosphere/background_error_hybrid_bump_bump.yaml.j2

@@ -98,15 +98,15 @@ components:
         datetime: "{{ atm_background_time_iso }}"
         filetype: fms restart
         state variables: *3dvars
-        datapath: {{ background_ensemble_path}}
+        datapath: {{ atm_background_ensemble_path}}
         filename is datetime templated: true
         filename_core: "%yyyy%mm%dd.%hh%MM%ss.fv_core.res.nc"
         filename_trcr: "%yyyy%mm%dd.%hh%MM%ss.fv_tracer.res.nc"
         filename_sfcd: "%yyyy%mm%dd.%hh%MM%ss.sfc_data.nc"
         filename_sfcw: "%yyyy%mm%dd.%hh%MM%ss.fv_srf_wnd.res.nc"
         filename_cplr: "%yyyy%mm%dd.%hh%MM%ss.coupler.res"
-      pattern: "{{ ensemble_pattern | default("%mem%", true) }}"
-      nmembers: {{number_ensemble_members}}
+      pattern: "{{ atm_ensemble_pattern | default("%mem%", true) }}"
+      nmembers: {{atm_number_ensemble_members}}
       zero padding: 3
 #    localization:
 #      localization method: SABER

model/atmosphere/background_error_hybrid_gsibec_bump.yaml.j2

@@ -34,15 +34,15 @@ components:
         datetime: "{{ atm_background_time_iso }}"
         filetype: fms restart
         state variables: *3dvars_anal
-        datapath: {{ background_ensemble_path}}
+        datapath: {{ atm_background_ensemble_path}}
         filename is datetime templated: true
         filename_core: "%yyyy%mm%dd.%hh%MM%ss.fv_core.res.nc"
         filename_trcr: "%yyyy%mm%dd.%hh%MM%ss.fv_tracer.res.nc"
         filename_sfcd: "%yyyy%mm%dd.%hh%MM%ss.sfc_data.nc"
         filename_sfcw: "%yyyy%mm%dd.%hh%MM%ss.fv_srf_wnd.res.nc"
         filename_cplr: "%yyyy%mm%dd.%hh%MM%ss.coupler.res"
-      pattern: "{{ ensemble_pattern | default("%mem%", true) }}"
-      nmembers: {{number_ensemble_members}}
+      pattern: "{{ atm_ensemble_pattern | default("%mem%", true) }}"
+      nmembers: {{atm_number_ensemble_members}}
       zero padding: 3
     localization:
       localization method: SABER

model/atmosphere/background_error_static_bump.yaml.j2

@@ -8,7 +8,7 @@ saber blocks:
   active variables: &active_vars [psi,chi,t,ps,rh,liq_wat,o3mr]
   bump:
     io:
-      data directory: &staticb_dir {{bump_data_directory}}
+      data directory: &staticb_dir {{atm_bump_data_directory}}
     drivers:
       multivariate strategy: specific_univariate
       read local nicas: true

model/atmosphere/final_increment_gaussian.yaml.j2

@@ -2,7 +2,7 @@ output:
   state component:
     filetype: auxgrid
     gridtype: gaussian
-    filename: {{final_increment_prefix}}
+    filename: {{atm_final_increment_prefix}}
 geometry:
   fms initialization:
     namelist filename: "{{fv3jedi_files_path}}/fmsmpp.nml"

model/atmosphere/model_pseudo.yaml.j2

@@ -1,13 +1,13 @@
 name: PSEUDO
-datapath: {{ background_path}}
+datapath: {{ atm_background_path}}
 filetype: fms restart
 filename is datetime templated: true
 filename_core: '%yyyy%mm%dd.%hh%MM%ss.fv_core.res.nc'
 filename_trcr: '%yyyy%mm%dd.%hh%MM%ss.fv_tracer.res.nc'
 filename_sfcd: '%yyyy%mm%dd.%hh%MM%ss.sfc_data.nc'
 filename_sfcw: '%yyyy%mm%dd.%hh%MM%ss.fv_srf_wnd.res.nc'
 filename_cplr: '%yyyy%mm%dd.%hh%MM%ss.coupler.res'
-tstep: {{forecast_timestep}}
+tstep: {{atm_forecast_timestep}}
 model variables: [ua,va,t,delp,sphum,ice_wat,liq_wat,o3mr,phis,
                   slmsk,sheleg,tsea,vtype,stype,vfrac,stc,smc,snwdph,
                   u_srf,v_srf,f10m]

model/atmosphere/output_ensemble_increments_gaussian.yaml.j2

@@ -1,3 +1,3 @@
 filetype: auxgrid
 gridtype: gaussian
-filename: {{ensemble_increment_prefix}}
+filename: {{atm_ensemble_increment_prefix}}

model/atmosphere/posterior_output_gaussian.yaml.j2

@@ -1,3 +1,3 @@
 filetype: auxgrid
 gridtype: gaussian
-filename: {{posterior_output_gaussian}}
+filename: {{atm_posterior_output_gaussian}}

model/marine/3dfgat_outer_loop_1.yaml.j2

@@ -0,0 +1,7 @@
+- ninner: {{ninner_1}}
+  gradient norm reduction: {{grad_red_1}}
+  geometry:
+    mom6_input_nml: mom_input.nml
+    fields metadata: {{soca_files_path}}/fields_metadata.yaml
+  diagnostics:
+    departures: ombg

model/marine/background.yaml.j2

@@ -0,0 +1,6 @@
+read_from_file: 1
+basename: {{ marine_background_path}}
+ocn_filename: MOM.res.nc
+ice_filename: cice.res.nc
+date: "{{ marine_background_time }}"
+state variables: {{&soca_model_vars [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh, hocn, mld, layer_depth]}}

model/marine/background_error_hybrid_diffusion_diffusion.yaml.j2

@@ -0,0 +1,92 @@
+covariance model: hybrid
+components:
+- covariance:
+    covariance model: SABER
+    saber central block:
+      saber block name: EXPLICIT_DIFFUSION
+      active variables: [tocn, socn, ssh, cicen]
+      geometry:
+        mom6_input_nml: mom_input.nml
+        fields metadata: ./fields_metadata.yaml
+      group mapping:
+      - name: ocean
+        variables:
+        - tocn
+        - socn
+        - ssh
+      - name: ice
+        variables:
+        - cicen
+      read:
+        groups:
+        - name: ocean
+          horizontal:
+            filename: hz_ocean.nc
+          vertical:
+            filename: vt_ocean.nc
+        - name: ice
+          horizontal:
+            filename: hz_ice.nc
+
+    saber outer blocks:
+    - saber block name: StdDev
+      read:
+        model file:
+          date: '{{window_middle}}'
+          basename: ./
+          ocn_filename: 'ocn.bkgerr_stddev.incr.{{window_middle}}.nc'
+          ice_filename: 'ice.bkgerr_stddev.incr.{{window_middle}}.nc'
+          read_from_file: 3
+
+    linear variable change:
+      input variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh]
+      output variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh]
+      linear variable changes:
+      - linear variable change name: BalanceSOCA
+
+  weight:
+    value: 1.00
+
+- covariance:
+    covariance model: ensemble
+    members from template:
+      template:
+        read_from_file: 1
+        date: '{{window_middle}}'
+        basename: ./static_ens/
+        ocn_filename: 'ocn.pert.steric.%mem%.nc'
+        ice_filename: 'ice.%mem%.nc'
+        state variables: [tocn, socn, ssh, uocn, vocn, cicen, hicen, hsnon]
+      pattern: '%mem%'
+      nmembers: {{marine_number_ensemble_members}}
+    localization:
+      localization method: SABER
+      saber central block:
+        saber block name: EXPLICIT_DIFFUSION
+        active variables: [tocn, socn, ssh]
+        geometry:
+          mom6_input_nml: mom_input.nml
+          fields metadata: ./fields_metadata.yaml
+        group mapping:
+        - name: ocean
+          variables: [tocn, socn, ssh, uocn, vocn]
+        - name: ice
+          variables: [cicen, hicen, hsnon]
+        read:
+          groups:
+          - name: ocean
+            multivariate strategy: duplicated
+            horizontal:
+              filename: hz_ocean.nc
+            vertical:
+              strategy: duplicated
+          - name: ice
+            horizontal:
+              filename: hz_ice.nc
+
+  weight:
+    read_from_file: 3
+    basename: ./
+    ocn_filename: 'ocn.ens_weights.incr.{{window_middle}}.nc'
+    ice_filename: 'ice.ens_weights.incr.{{window_middle}}.nc'
+    date: '{{window_middle}}'

model/marine/background_error_static_diffusion.yaml.j2

@@ -0,0 +1,42 @@
+covariance model: SABER
+saber central block:
+  saber block name: EXPLICIT_DIFFUSION
+  active variables: [tocn, socn, ssh, cicen]
+  geometry:
+    mom6_input_nml: mom_input.nml
+    fields metadata: ./fields_metadata.yaml
+  group mapping:
+  - name: ocean
+    variables:
+    - tocn
+    - socn
+    - ssh
+  - name: ice
+    variables:
+    - cicen
+  read:
+    groups:
+    - name: ocean
+      horizontal:
+        filename: hz_ocean.nc
+      vertical:
+        filename: vt_ocean.nc
+    - name: ice
+      horizontal:
+        filename: hz_ice.nc
+
+saber outer blocks:
+- saber block name: StdDev
+  read:
+    model file:
+      date: '{{marine_stddev_time}}'
+      basename: ./
+      ocn_filename: 'ocn.bkgerr_stddev.incr.{{marine_stddev_time}}.nc'
+      ice_filename: 'ice.bkgerr_stddev.incr.{{marine_stddev_time}}.nc'
+      read_from_file: 3
+
+linear variable change:
+  input variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh]
+  output variables: [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh]
+  linear variable changes:
+  - linear variable change name: BalanceSOCA

model/marine/final_increment.yaml.j2

@@ -0,0 +1,9 @@
+output:
+  state component:
+    datadir: {{final_increment_dir}}
+    date: '{{window_begin}}'
+    exp: {{final_increment_exp}}
+    type: incr
+geometry:
+  mom6_input_nml: mom_input.nml
+  fields metadata: {{soca_files_path}}/fields_metadata.yaml

model/marine/geometry_background.yaml.j2

@@ -0,0 +1,2 @@
+mom6_input_nml: mom_input.nml
+fields metadata: {{soca_files_path}}/fields_metadata.yaml

model/marine/model_pseudo.yaml.j2

@@ -0,0 +1,10 @@
+name: PseudoModel
+tstep: {{marine_forecast_timestep}}
+states:
+{% for pseudo_model_state in pseudo_model_states %}
+- date: '{{ pseudo_model_state.date }}'
+  basename: '{{ pseudo_model_state.basename }}'
+  ocn_filename: '{{ pseudo_model_state.ocn_filename }}'
+  ice_filename: '{{ pseudo_model_state.ice_filename }}'
+  read_from_file: {{ pseudo_model_state.read_from_file }}
+{% endfor %}

model/marine/output.yaml.j2

@@ -0,0 +1,4 @@
+datadir: {{output_dir}}
+exp: {{output_exp}}
+type: an
+frequency: {{output_freq}}