From 5998ec66a55b239d18799b7e2f94fac77e7ab221 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Thu, 22 Oct 2020 15:01:06 -0600 Subject: [PATCH 001/142] Sync jcsda and jcsda-internal. (#941) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update CI (#5) * update travis-ci token * update codebuild and clone.sh * Feature/new ewok (#3) * changes to configs for new ewok * with extra files Co-authored-by: Claude Gibert Co-authored-by: Yannick Trémolet <30638944+ytremolet@users.noreply.github.com> Co-authored-by: Maryam Abdi Co-authored-by: Claude Gibert Co-authored-by: Claude Gibert --- .travis.yml | 9 ++++----- CI/buildspec_clang.yml | 3 +-- CI/buildspec_gnu.yml | 6 +++--- CI/buildspec_intel.yml | 6 +++--- CI/clone.sh | 4 ++-- ewok/forecast.yaml | 2 +- l95/defaults/an.yaml | 2 +- l95/defaults/anout.yaml | 3 ++- l95/defaults/bg.yaml | 2 +- l95/defaults/fc.yaml | 1 + l95/defaults/fcout.yaml | 2 +- l95/defaults/model.py | 13 +++++++------ l95/defaults/ob.yaml | 1 + l95/defaults/obs.yaml | 7 ++++--- qg/configs/an.yaml | 2 +- qg/configs/anout.yaml | 2 +- qg/configs/bg.yaml | 2 +- qg/configs/fc.yaml | 2 ++ qg/configs/fcout.yaml | 2 +- qg/configs/ob.yaml | 1 + qg/configs/stream.yaml | 9 +++++---- qg/configs/wind.yaml | 9 +++++---- qg/configs/wspeed.yaml | 7 ++++--- 23 files changed, 53 insertions(+), 44 deletions(-) create mode 100644 l95/defaults/fc.yaml create mode 100644 l95/defaults/ob.yaml create mode 100644 qg/configs/fc.yaml create mode 100644 qg/configs/ob.yaml diff --git a/.travis.yml b/.travis.yml index a31a8b7b7..72dce28f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,10 +3,9 @@ #====================================================================== env: global: - - secure: XyvtxajZUw3ObIpcTecY3qEPDNNhIkbVlVY5QBgymkZqJZGwXBYHagHMywB9lVC1ACasogzarnx3Yxf3G7aSoSHGRGEo9bsgNPB3fm66h/VrjYLYhT1XWgAJbuywVRwEvqicHF5FKPouAafVyMKYcFka/IUm+eJRGqmC9Uap0yR9AXjQT93UcoJseGn/quNS5qrTjkIV543fKvZ2Pr2AlanDepyQKzqaM0KN4GG0hFxl/E9E9PU+eUN/N7nt3/QgtFVgZpeYx+P3zZsav0ByjMEmAP8dzpPge44Qz6TiEomMF52D5ry1ok6V/iPOgfW0jPscppGbzj58ee3YBlY+GQp+LXe8RNradkCJIbSVrigNv3vv1Xv4mpYi7uu2JROx+9VX179H80Ca/y1C9mnPvRWSmFg2gUpYRuYeHb+krg/iRKOvh0N9T38j/P+HA3YQyD0J+xMSImMvi5XOuvhi3B27PLnGMptbxE1gz62ccgEKmdjiDL4BDYzeOqew1nmBXnqAgVniUGR+Yux2+tralYpawO0u1iYJyrTdkxoDeR2SAzDocJm6/ri5V1fP7cEPc4eSHxwFrqNITBeyh7vZHU+ZArQrC5J40WJh1uo23TdLiA8sH1XGreSkUudmPxnt2rZ8ATAbuS85lWDQEegaQH+DYd+F3/akEJxLuH8MQYM= - - secure: lqVsqiUeblhERNDdgJU7dU5EojE31qsj2u5MGlpjghPGVJqhJ5ZyMTJ7zHsCAS8sldzrkaUGH+2tvTTa+LocEqBgJuxRBuucJA48VyoWYmbi0baqWxHoxWsW42KvVWxOCT3TfpfanDsVWI7ShuOlES3SxJaZdjD7CeYIilh3ks61YuxxPI6KZi8w6+phI+miQ3ZUHJMHGdaw3J2Bg3TXlaXJjRDHoweXEe4TnOfU10eokkNmvPj6EL1l50ilBSe+ZicAt4ZACkrWmkjivg9A2cP8gfJPxTp4wKKXOelusTVu9IueAg36fxHhNpQQsGND60rq5xYfcCEc60RxA8PJ8K/wvkrRLeEuEoQNNrwi0FgOb5fhWdsmGG9Bxqlgb3B69rDItI/RqiiKpFxdCnB2wCaD/Jqbw8JwXjIgkf4pJ2RFWo7MM7qj854XOYOV97S2Fc0C9yz3SOHW9gGu0rcGoT2sJivzWUTaKMkNcECJetMugCJwh4Puy6kj9vdh2RjPBkJtQXXYuR+BG2K5WUiggG7oI5hnNq2lxakbqxHJtolZuzuaAjRQhvTgX+7muUii9SNSo/Cjj0B3XPgmWp4ieqAg9OgeauKxc+BuqfNJk8R3LtlbXonAtZLjFSgY9pwNNwEKdF3KT36f2lQO3sumCFjk9OtcH7GU5Y6BGjZPnNA= - - secure: Oax5AhsSipTMVFsBKOHbNM4jjBsFMJXAr7lfdUwfnpHuh85jfS5gNcoJALtRFJeWXAEFgntBAUHLToMxb+9hIa3B+dDYfN2eUGxu+VSS0sYJWr8e4KzJf15jRaDzSh+BmocsCzBYgPA1qHcuIHRfD9/MOa1AdaJF8mpejTrrukGmhJ24xXkUsJChqoX/VeDCRcYoS9LG0AbQZkdjtRCXt+jkCHTcw8Jdjhf1qAMeUvZABohg/Mw1Psn60B1bfab7vjwaSZWng2C2Ji2NriPYr8oe3rt1Uwa3q/qZeVTo7wlhP6FZUzgnXfoCWqfx7c+Z32vxtl7+vml6xKDUZiEwI8tS/gx9drsMxAOzb+g3hVyu55uld3OlhQN97htYQszcTyJ5NzGsD4ym0jM1IKEEXTad9tcxbINQ2+rO79EGsNLflj37f7t4EQ+wnLC6xu3gm16OsW+EMxVMLTZHp5VQWEDFqM0LMGlg8D8V75gORhXPNlbAEIAm4U+4vu/8Wjb7pKicbNZV45HJT3Cn6Ri5JU0BzpTP/GlSLbwOXcl5ggmcjpgtdDOhSZ8BHDs4bJMhNWrvnJLLzh9V+kH4EfHI0UxIcR+vW5gt04IM78qmPkeAhjLl/Yh7a2bRCEU3QCzqssLiTg7izBxN6r+0t14tEAbpzeNFsIiVc7ewO6DpWRY= - + - secure: ztTCO/DzSfshSaPMJ6BzySkaD2TSeAITKUFaMYbJ7fUWn9bYSDGzxXI90jpusAn7r2Z6exQC0kGfgIWwqVnAhTed9kBk+vyeskB6lsGdpJ1EGjtxhPKiAITIMuwEhmfaqBmuC20f8UkClpxEPmYEYxBR0wamAIQ4FAe/1QGdYZ+uEbNYKbjcyPCZNe/A/xeOYIjo88urY3jCNfFH1DShgEXV+GhbQl/nNSgSGp4n2u8ay5IVr7dzh/NG51y77D43p8e0Rqd255uDt7hro+6O7XakDLjABmkO2fTs6+osmRqHOOSbv3fLz7O1dcdHVDFPI7U7u6GZX6dCwuvRAWi/l62MqJ//+tXpLr81AyH75eoMHovbMdbkkbftrcCqXTIMLoqt3hZ0XQtiYv0+aOKb/67l09UNz1szSImhxiFxNB7SfK6vVmJ0vzqB835NCIbxpW0iJuVRxBNIssd9oU+9wvaBCpGUxJBL/anoxEGhFt4olmewVZzCbiwCuFlp8cZAFwUVNXnGah87QZW76KD+qaDHWaWYp6KzdBmLeWFu5hBOBIGVSEZyxeAgMUyOi8fYfsw2q/dGLPryXfLALEui1uN4XOwAuOGnmqecqXOpaFeRIhJbc03JizxdISteRQRdYXshYgeM7/9VPXyYhzg2Z8sijvbY7MBZwTtEPipXkNM= + - secure: a7T55NgRK0dmjPb+5vddqpXuAwJ9l/VNCbFlisVG7uYrmfeIybNwE2rTxU4wTy3GX+mjB8VjsxtjPk5jutA16Xb/8tg3EzLn6VHpA2uHVlzap0PAaPLnvpTKvRtYa0J/AJr4GPLV69Dnl022h8LdBNVXqIVoYZPEv2R27BvVZAN2TFucCSXKZXYUW/fpMZdbe++45V9UDGTBEoq+csyesM6ent2fp4dSmuTxXVzOW4VEeBD3MxKwYaEz7Ay4nDvi1EqLtXb8sQIPC+PCRrQobLXTML7aeDiwHDnYK5m74rc0BHaUz6syJTh4eBL6mjvB4d3bcX+Y5tLLkV6fMKr44L382bK/zayxOpfAKpyxaWQwnomkw7RMisLaDNCACxl/7yQkRnj5tZO2HbXuMZH3MO7WWeZaLtV28bj4ZbWAVLWRqgGb1paA0jKqEMtylzfekRSFXg995qvTJCnzkfO9lkG/agXp/ljbYqc8Ts54EqvjdlR6PvUkWJZCAOcXiUw1rNKq9k4czNsyzpM0xwzguNs9HYThKIDwxBDg/8vsISO4Gt8JYnVOk+JMXxqtyFmnzdu7E3vB02QSZsdNvipRrPwZ+rcM6qiwfMFejt5eayG2ow5rYU0BPN0BLWYv8b0kpZP2e6o9+8qSEmxQ7tm7R7RDOQEv4RviKD4F/GoJmtE= + - secure: S6E4UYFyTAQP19KEYT9r08O+yPJIquX5kITqKsj04N7q6smuEoKv6lletWpqhMcZtWdfsV49LvmU5XgGLj3jNTiU0GsBJYl7H0cF3E4/3QQgK7Qm1gbxmd50odQXloi1UXaMVprgixD/TFroPhyE2IsaV/TxuBdJIL732wRt+I2qzhrYbbK4DqFPjKqxXHQgwLVc9ZbmaZLHSf/WQ4IaYLtT7d6ChvZvxkbLlf3/V2OpKo85VwPk1oLdVCL9GbZSKqMFPpliOg2PoZCaAz9To5dAcbWoKEDKOVaf0DwUVgbjshZWeXvbVYRqxp5NeS4xmL2+7FdgfzAcfhWLMfZrYAN+0SmTjS+6F3+WSVF90bkPzORJTyga28LtN5zV8H6Dc4YkYGlotjTIdgH5a/p5DooQ0K87fF7JItypt7TuaVHxM7oy5vTCy0xghJD5lJ8JTy7NfCmGncK+Dh3GdhWCDVSCRspL62dqu8UX7Ekz0+x0HHkAL0ZhswzCTiEZoQFXIhAsJw8BA+YRAmHBGG4fpEqJlq1EjDm9Ca9AHBRvf1f4npM12znBhhjF/e7JSVOHgbDP9YrvZWLRC8YWuFcOaRZU3JtrG1bbCDtZXJ4UK4gQv7Ru5rbtVk/q6d56qaMvC//6RFYxSYxpA+WD8Crcb/OsK3eKHrOzgsbtm1PYCRM= branches: only: - develop @@ -31,7 +30,7 @@ before_install: - cd CI - REPO_SOURCE_DIR=${TRAVIS_BUILD_DIR}/jcsda/src_repo - - git clone https://${GH_TOKEN}@github.com/jcsda/jedi-build-package.git ${REPO_SOURCE_DIR}/jedi_build_package + - git clone https://${GH_TOKEN}@github.com/jcsda-internal/jedi-build-package.git ${REPO_SOURCE_DIR}/jedi_build_package - docker pull jcsda/docker-clang-mpich-dev - docker images diff --git a/CI/buildspec_clang.yml b/CI/buildspec_clang.yml index ae2361359..c11629b0d 100644 --- a/CI/buildspec_clang.yml +++ b/CI/buildspec_clang.yml @@ -40,7 +40,7 @@ phases: - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > artifacts/commit_sha.txt # get jedi-build-package - - git clone https://$GIT_USER:$GIT_PASS@github.com/jcsda/jedi-build-package + - git clone https://$GIT_USER:$GIT_PASS@github.com/jcsda-internal/jedi-build-package - cd jedi-build-package - pip install --user -e . @@ -51,5 +51,4 @@ phases: artifacts: files: - 'artifacts/*' -# - '**/*' name: artifact-oops-clang diff --git a/CI/buildspec_gnu.yml b/CI/buildspec_gnu.yml index 6c0e9fafc..fd674f757 100644 --- a/CI/buildspec_gnu.yml +++ b/CI/buildspec_gnu.yml @@ -71,11 +71,11 @@ phases: fi # oops - - ./clone.sh $GIT_USER $GIT_PASS oops $CODEBUILD_GIT_BRANCH oops /jcsda/oops-bundle develop + - ./clone.sh $GIT_USER $GIT_PASS jcsda/oops $CODEBUILD_GIT_BRANCH oops /jcsda/oops-bundle develop # fckit - - ./clone.sh $GIT_USER $GIT_PASS fckit $CODEBUILD_GIT_BRANCH_FORK fckit /jcsda/oops-bundle release-stable + - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/fckit $CODEBUILD_GIT_BRANCH_FORK fckit /jcsda/oops-bundle release-stable # atlas - - ./clone.sh $GIT_USER $GIT_PASS atlas $CODEBUILD_GIT_BRANCH_FORK atlas /jcsda/oops-bundle release-stable + - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/atlas $CODEBUILD_GIT_BRANCH_FORK atlas /jcsda/oops-bundle release-stable - cd /jcsda/oops-bundle - ls diff --git a/CI/buildspec_intel.yml b/CI/buildspec_intel.yml index 5ece9806c..193b6d50b 100644 --- a/CI/buildspec_intel.yml +++ b/CI/buildspec_intel.yml @@ -86,11 +86,11 @@ phases: fi # oops - - ./clone.sh $GIT_USER $GIT_PASS oops $CODEBUILD_GIT_BRANCH oops /jcsda/oops-bundle develop + - ./clone.sh $GIT_USER $GIT_PASS jcsda/oops $CODEBUILD_GIT_BRANCH oops /jcsda/oops-bundle develop # fckit - - ./clone.sh $GIT_USER $GIT_PASS fckit $CODEBUILD_GIT_BRANCH_FORK fckit /jcsda/oops-bundle release-stable + - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/fckit $CODEBUILD_GIT_BRANCH_FORK fckit /jcsda/oops-bundle release-stable # atlas - - ./clone.sh $GIT_USER $GIT_PASS atlas $CODEBUILD_GIT_BRANCH_FORK atlas /jcsda/oops-bundle release-stable + - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/atlas $CODEBUILD_GIT_BRANCH_FORK atlas /jcsda/oops-bundle release-stable - cd /jcsda/oops-bundle - ls diff --git a/CI/clone.sh b/CI/clone.sh index 970b03012..c724fb368 100755 --- a/CI/clone.sh +++ b/CI/clone.sh @@ -15,7 +15,7 @@ echo "========================================================================== echo "Clone " $repo_name echo "===============================================================================" -git ls-remote --heads --exit-code https://$git_user:$git_token@github.com/jcsda/$repo_name $branch_name +git ls-remote --heads --exit-code https://$git_user:$git_token@github.com/$repo_name $branch_name exit_code=$? if test "${exit_code}" == "0"; then @@ -27,6 +27,6 @@ else echo "clone " ${branch_name_clone} fi -git clone -b $branch_name_clone https://$git_user:$git_token@github.com/jcsda/$repo_name $save_dir/$save_name +git clone -b $branch_name_clone https://$git_user:$git_token@github.com/$repo_name $save_dir/$save_name diff --git a/ewok/forecast.yaml b/ewok/forecast.yaml index 87dd12db3..b0246d3b3 100644 --- a/ewok/forecast.yaml +++ b/ewok/forecast.yaml @@ -1,6 +1,6 @@ forecast length: $(forecast_length) initial condition: - $(INITIAL_CONDITION) + $(AN_TEMPLATE) geometry: $(GEOMETRY) model: diff --git a/l95/defaults/an.yaml b/l95/defaults/an.yaml index f104cf20e..bba1af04d 100644 --- a/l95/defaults/an.yaml +++ b/l95/defaults/an.yaml @@ -1,2 +1,2 @@ date: '{{current_cycle}}' -filename: '$(expdir)/{{current_cycle}}/$(experiment).an.{{current_cycle}}' +filename: '$(experiment_dir)/{{current_cycle}}/$(experiment).an.{{current_cycle}}' diff --git a/l95/defaults/anout.yaml b/l95/defaults/anout.yaml index a54d0934b..3008726e5 100644 --- a/l95/defaults/anout.yaml +++ b/l95/defaults/anout.yaml @@ -1,4 +1,5 @@ -datadir: '$(expdir)/{{current_cycle}}' +datadir: '$(experiment_dir)/{{current_cycle}}' +filename: '$(experiment_dir)/{{current_cycle}}/$(experiment).an.{{current_cycle}}' exp: $(experiment) ##first: '{% $(window_length) - $(window_offset) %}' first: PT3H diff --git a/l95/defaults/bg.yaml b/l95/defaults/bg.yaml index 7413014a3..c2eaf19bd 100644 --- a/l95/defaults/bg.yaml +++ b/l95/defaults/bg.yaml @@ -1,2 +1,2 @@ date: '{{window_begin}}' -filename: '$(expdir)/{{current_cycle}}/$(experiment).bg.{{current_cycle}}.l95' +filename: '$(experiment_dir)/{{current_cycle}}/$(experiment).bg.{{current_cycle}}.l95' diff --git a/l95/defaults/fc.yaml b/l95/defaults/fc.yaml new file mode 100644 index 000000000..84db6a09e --- /dev/null +++ b/l95/defaults/fc.yaml @@ -0,0 +1 @@ +filename: $(experiment_dir)/{{current_cycle}}/$(experiment).fc.{{current_cycle}}.${step} diff --git a/l95/defaults/fcout.yaml b/l95/defaults/fcout.yaml index cb23292dc..31394cd02 100644 --- a/l95/defaults/fcout.yaml +++ b/l95/defaults/fcout.yaml @@ -1,4 +1,4 @@ -datadir: '$(expdir)/{{current_cycle}}' +datadir: $(experiment_dir)/{{current_cycle}} date: '{{current_cycle}}' exp: $(experiment) frequency: PT3H diff --git a/l95/defaults/model.py b/l95/defaults/model.py index 48a43c879..313fc8572 100644 --- a/l95/defaults/model.py +++ b/l95/defaults/model.py @@ -1,5 +1,5 @@ import os -import ewok +from solo.date import JediDate, DateIncrement __all__ = ["fc_file", "obs_file", "r2d2_obsfile", "r2d2_anfile"] @@ -7,7 +7,8 @@ def fc_file(fcout, step): fc = {} fc['date'] = fcout['date'] - keys = [fcout['exp'], fcout['type'], fcout['date'], ewok.jediformat(step)] + step = DateIncrement(duration=step) + keys = [fcout['exp'], fcout['type'], fcout['date'], str(step)] fname = '.'.join(keys) fc['filename'] = os.path.join(fcout['datadir'], fname) return fc @@ -19,15 +20,15 @@ def obs_file(conf): def r2d2_obsfile(conf, date): - sdate = ewok.jediformat(date) - r2d2keys = ['l95', conf['source'], sdate, 'obt'] + sdate = JediDate(date) + r2d2keys = ['l95', conf['source'], str(sdate), 'obt'] r2d2file = '.'.join(r2d2keys) return r2d2file def r2d2_anfile(conf, date): - sdate = ewok.jediformat(date) - r2d2keys = [conf['exp'], conf['type'], sdate, 'l95'] + sdate = JediDate(date) + r2d2keys = [conf['exp'], conf['type'], str(sdate), 'l95'] r2d2file = '.'.join(r2d2keys) return r2d2file diff --git a/l95/defaults/ob.yaml b/l95/defaults/ob.yaml new file mode 100644 index 000000000..79ff5c62b --- /dev/null +++ b/l95/defaults/ob.yaml @@ -0,0 +1 @@ +obs_file: obs space.obsdatain diff --git a/l95/defaults/obs.yaml b/l95/defaults/obs.yaml index 25bad5027..38010f1f2 100644 --- a/l95/defaults/obs.yaml +++ b/l95/defaults/obs.yaml @@ -1,8 +1,9 @@ -obs operator: {} +obs operator: + obs type: Obs obs space: source: truth filetype: obt - obsdatain: '$(expdir)/{{current_cycle}}/input.obs.{{current_cycle}}.obt' - obsdataout: '$(expdir)/{{current_cycle}}/$(experiment).obs.{{current_cycle}}.obt' + obsdatain: '$(experiment_dir)/{{current_cycle}}/input.obs.{{current_cycle}}.obt' + obsdataout: '$(experiment_dir)/{{current_cycle}}/$(experiment).obs.{{current_cycle}}.obt' obs error: covariance model: diagonal diff --git a/qg/configs/an.yaml b/qg/configs/an.yaml index e47862b03..a9904e7dd 100644 --- a/qg/configs/an.yaml +++ b/qg/configs/an.yaml @@ -1,2 +1,2 @@ date: '{{current_cycle}}' -filename: '$(expdir)/{{current_cycle}}/$(experiment).an.{{current_cycle}}.nc' +filename: '$(experiment_dir)/{{current_cycle}}/$(experiment).an.{{current_cycle}}.nc' diff --git a/qg/configs/anout.yaml b/qg/configs/anout.yaml index 7bc409ebd..035f69d4c 100644 --- a/qg/configs/anout.yaml +++ b/qg/configs/anout.yaml @@ -1,4 +1,4 @@ -datadir: $(expdir)/{{current_cycle}} +datadir: $(experiment_dir)/{{current_cycle}} exp: $(experiment) #first: '{% $(window_length) - $(window_offset) %}' first: PT3H diff --git a/qg/configs/bg.yaml b/qg/configs/bg.yaml index 676050161..f659e39d1 100644 --- a/qg/configs/bg.yaml +++ b/qg/configs/bg.yaml @@ -1,2 +1,2 @@ date: '{{window_begin}}' -filename: '$(expdir)/{{current_cycle}}/$(experiment).bg.{{current_cycle}}.nc' +filename: '$(experiment_dir)/{{current_cycle}}/$(experiment).bg.{{current_cycle}}.nc' diff --git a/qg/configs/fc.yaml b/qg/configs/fc.yaml new file mode 100644 index 000000000..d81245250 --- /dev/null +++ b/qg/configs/fc.yaml @@ -0,0 +1,2 @@ +date: '{{current_cycle}}' +filename: $(experiment_dir)/{{current_cycle}}/$(experiment).fc.{{current_cycle}}.${step}.nc diff --git a/qg/configs/fcout.yaml b/qg/configs/fcout.yaml index cb23292dc..c95330ead 100644 --- a/qg/configs/fcout.yaml +++ b/qg/configs/fcout.yaml @@ -1,4 +1,4 @@ -datadir: '$(expdir)/{{current_cycle}}' +datadir: '$(experiment_dir)/{{current_cycle}}' date: '{{current_cycle}}' exp: $(experiment) frequency: PT3H diff --git a/qg/configs/ob.yaml b/qg/configs/ob.yaml new file mode 100644 index 000000000..911c776bd --- /dev/null +++ b/qg/configs/ob.yaml @@ -0,0 +1 @@ +obs_file: obs space.obsdatain.obsfile diff --git a/qg/configs/stream.yaml b/qg/configs/stream.yaml index c07653b4c..17bd18234 100644 --- a/qg/configs/stream.yaml +++ b/qg/configs/stream.yaml @@ -2,10 +2,11 @@ obs operator: obs type: Stream obs space: source: truth - obsdatain: - obsfile: $(expdir)/{{current_cycle}}/obs.stream.{{current_cycle}}.nc - obsdataout: - obsfile: $(expdir)/{{current_cycle}}/obs.stream.$(experiment).{{current_cycle}}.nc + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/obs.stream.{{current_cycle}}.nc + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/obs.stream.$(experiment).{{current_cycle}}.nc obs type: Stream obs error: covariance model: diagonal +filename: obs space.obsdatain.obsfile diff --git a/qg/configs/wind.yaml b/qg/configs/wind.yaml index 44fe6430a..bbd214085 100644 --- a/qg/configs/wind.yaml +++ b/qg/configs/wind.yaml @@ -2,10 +2,11 @@ obs operator: obs type: Wind obs space: source: truth - obsdatain: - obsfile: $(expdir)/{{current_cycle}}/obs.wind.{{current_cycle}}.nc - obsdataout: - obsfile: $(expdir)/{{current_cycle}}/obs.wind.$(experiment).{{current_cycle}}.nc + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/obs.wind.{{current_cycle}}.nc + obsdataout: + obsfile: $(experiment_dir)/{{current_cycle}}/obs.wind.$(experiment).{{current_cycle}}.nc obs type: Wind obs error: covariance model: diagonal +filename: obs space.obsdatain.obsfile diff --git a/qg/configs/wspeed.yaml b/qg/configs/wspeed.yaml index 7c5348ac3..182fa1a9a 100644 --- a/qg/configs/wspeed.yaml +++ b/qg/configs/wspeed.yaml @@ -2,10 +2,11 @@ obs operator: obs type: WSpeed obs space: source: truth - obsdatain: - obsfile: $(expdir)/{{current_cycle}}/obs.wspeed.{{current_cycle}}.nc + obsdatain: + obsfile: $(experiment_dir)/{{current_cycle}}/obs.wspeed.{{current_cycle}}.nc obsdataout: - obsfile: $(expdir)/{{current_cycle}}/obs.wspeed.$(experiment).{{current_cycle}}.nc + obsfile: $(experiment_dir)/{{current_cycle}}/obs.wspeed.$(experiment).{{current_cycle}}.nc obs type: WSpeed obs error: covariance model: diagonal +filename: obs space.obsdatain.obsfile From ef2818e9a9c29a5e44e38de02a78022e6141c955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Fri, 23 Oct 2020 10:34:21 -0600 Subject: [PATCH 002/142] Removed deprecated ATLASIFIED flag (#2) (#942) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove ATLASIFIED flag and unstructured grid, add empty ATLAS interfaces in L95 * Remove unstructured grid reintroduced in previous merge with develop Co-authored-by: Yannick Trémolet <30638944+ytremolet@users.noreply.github.com> Co-authored-by: Benjamin Menetrier <30638301+benjaminmenetrier@users.noreply.github.com> --- l95/src/lorenz95/CMakeLists.txt | 1 - l95/src/lorenz95/FieldL95.cc | 13 - l95/src/lorenz95/FieldL95.h | 9 - l95/src/lorenz95/IncrementL95.cc | 17 +- l95/src/lorenz95/IncrementL95.h | 11 +- qg/model/CMakeLists.txt | 2 - qg/model/IncrementQG.h | 10 - src/CMakeLists.txt | 5 - src/oops/assimilation/Increment4D.h | 37 +- src/oops/generic/UnstructuredGrid.cc | 113 ------ src/oops/generic/UnstructuredGrid.h | 71 ---- src/oops/generic/unstructured_grid_f.h | 34 -- .../generic/unstructured_grid_interface.F90 | 252 ------------- src/oops/generic/unstructured_grid_mod.F90 | 330 ------------------ src/oops/interface/Geometry.h | 2 - src/oops/interface/Increment.h | 31 -- 16 files changed, 16 insertions(+), 922 deletions(-) delete mode 100644 src/oops/generic/UnstructuredGrid.cc delete mode 100644 src/oops/generic/UnstructuredGrid.h delete mode 100644 src/oops/generic/unstructured_grid_f.h delete mode 100644 src/oops/generic/unstructured_grid_interface.F90 delete mode 100644 src/oops/generic/unstructured_grid_mod.F90 diff --git a/l95/src/lorenz95/CMakeLists.txt b/l95/src/lorenz95/CMakeLists.txt index f22e7b774..8290f2431 100644 --- a/l95/src/lorenz95/CMakeLists.txt +++ b/l95/src/lorenz95/CMakeLists.txt @@ -74,4 +74,3 @@ ecbuild_add_library( TARGET lorenz95 #TODO: Reenable cxx_std_14 when supported by intel jedi-stack modules (jedi-stack #125) target_compile_features( lorenz95 PUBLIC cxx_std_11 ) target_include_directories( lorenz95 PUBLIC $ ) -target_compile_definitions( lorenz95 PUBLIC ATLASIFIED=0 ) diff --git a/l95/src/lorenz95/FieldL95.cc b/l95/src/lorenz95/FieldL95.cc index c6be63794..1276b6fe7 100644 --- a/l95/src/lorenz95/FieldL95.cc +++ b/l95/src/lorenz95/FieldL95.cc @@ -21,7 +21,6 @@ #include "lorenz95/GomL95.h" #include "lorenz95/LocsL95.h" #include "lorenz95/Resolution.h" -#include "oops/generic/UnstructuredGrid.h" #include "oops/util/abor1_cpp.h" #include "oops/util/Logger.h" #include "oops/util/Random.h" @@ -150,18 +149,6 @@ void FieldL95::random() { for (int jj = 0; jj < resol_; ++jj) x_[jj] = xx[jj]; } // ----------------------------------------------------------------------------- -void FieldL95::ug_coord(oops::UnstructuredGrid & ug) const { - ABORT("FieldL95 unstructured grid setup not implemented."); -} -// ----------------------------------------------------------------------------- -void FieldL95::field_to_ug(oops::UnstructuredGrid & ug, const int & its) const { - ABORT("FieldL95 conversion to unstructured grid not implemented."); -} -// ----------------------------------------------------------------------------- -void FieldL95::field_from_ug(const oops::UnstructuredGrid & ug, const int & its) { - ABORT("FieldL95 conversion from unstructured grid not implemented."); -} -// ----------------------------------------------------------------------------- void FieldL95::read(std::ifstream & fin) { fin.precision(std::numeric_limits::digits10); for (int jj = 0; jj < resol_; ++jj) fin >> x_[jj]; diff --git a/l95/src/lorenz95/FieldL95.h b/l95/src/lorenz95/FieldL95.h index 061221a8d..f03fb53d6 100644 --- a/l95/src/lorenz95/FieldL95.h +++ b/l95/src/lorenz95/FieldL95.h @@ -23,10 +23,6 @@ namespace eckit { class Configuration; } -namespace oops { - class UnstructuredGrid; -} - namespace lorenz95 { class LocsL95; class GomL95; @@ -72,11 +68,6 @@ class FieldL95 : public util::Printable, std::vector & asVector() {return x_;} const std::vector & asVector() const {return x_;} -/// Unstructured grid - void ug_coord(oops::UnstructuredGrid &) const; - void field_to_ug(oops::UnstructuredGrid &, const int &) const; - void field_from_ug(const oops::UnstructuredGrid &, const int &); - /// Serialize and deserialize size_t serialSize() const override; void serialize(std::vector &) const override; diff --git a/l95/src/lorenz95/IncrementL95.cc b/l95/src/lorenz95/IncrementL95.cc index 59e94ce28..bea4c7313 100644 --- a/l95/src/lorenz95/IncrementL95.cc +++ b/l95/src/lorenz95/IncrementL95.cc @@ -14,9 +14,10 @@ #include #include +#include "atlas/field.h" + #include "eckit/exception/Exceptions.h" -#include "oops/generic/UnstructuredGrid.h" #include "oops/util/abor1_cpp.h" #include "oops/util/DateTime.h" #include "oops/util/dot_product.h" @@ -211,18 +212,18 @@ void IncrementL95::setLocal(const oops::LocalIncrement & gp, const Iterator & i) fld_[i.index()] = vals[0]; } // ----------------------------------------------------------------------------- -/// Convert to/from unstructured grid +/// Convert to/from ATLAS fieldset // ----------------------------------------------------------------------------- -void IncrementL95::ug_coord(oops::UnstructuredGrid & ug) const { - fld_.ug_coord(ug); +void IncrementL95::setAtlas(atlas::FieldSet *) const { + ABORT("FieldL95 setAtlas not implemented"); } // ----------------------------------------------------------------------------- -void IncrementL95::field_to_ug(oops::UnstructuredGrid & ug, const int & its) const { - fld_.field_to_ug(ug, its); +void IncrementL95::toAtlas(atlas::FieldSet *) const { + ABORT("FieldL95 toAtlas not implemented"); } // ----------------------------------------------------------------------------- -void IncrementL95::field_from_ug(const oops::UnstructuredGrid & ug, const int & its) { - fld_.field_from_ug(ug, its); +void IncrementL95::fromAtlas(atlas::FieldSet *) { + ABORT("FieldL95 fromAtlas not implemented"); } // ----------------------------------------------------------------------------- /// Serialize - deserialize diff --git a/l95/src/lorenz95/IncrementL95.h b/l95/src/lorenz95/IncrementL95.h index 08837dd7a..2c6cb4f51 100644 --- a/l95/src/lorenz95/IncrementL95.h +++ b/l95/src/lorenz95/IncrementL95.h @@ -17,6 +17,8 @@ #include #include +#include "atlas/field.h" + #include "lorenz95/FieldL95.h" #include "lorenz95/Iterator.h" #include "lorenz95/Resolution.h" @@ -35,7 +37,6 @@ namespace eckit { namespace oops { class LocalIncrement; - class UnstructuredGrid; class Variables; } @@ -81,10 +82,10 @@ class IncrementL95 : public util::Printable, void schur_product_with(const IncrementL95 &); void random(); -/// Unstructured grid - void ug_coord(oops::UnstructuredGrid &) const; - void field_to_ug(oops::UnstructuredGrid &, const int &) const; - void field_from_ug(const oops::UnstructuredGrid &, const int &); +/// ATLAS + void setAtlas(atlas::FieldSet *) const; + void toAtlas(atlas::FieldSet *) const; + void fromAtlas(atlas::FieldSet *); // Utilities void read(const eckit::Configuration &); diff --git a/qg/model/CMakeLists.txt b/qg/model/CMakeLists.txt index 2701969d9..61f198e52 100644 --- a/qg/model/CMakeLists.txt +++ b/qg/model/CMakeLists.txt @@ -126,8 +126,6 @@ ecbuild_add_library( TARGET qg INSTALL_HEADERS LISTED LINKER_LANGUAGE CXX ) -target_compile_definitions(qg PUBLIC ATLASIFIED=1) - #Configure include directory layout for build-tree to match install-tree set(QG_BUILD_DIR_INCLUDE_PATH ${CMAKE_BINARY_DIR}/${PROJECT_NAME}/include) add_custom_target(qg_headers ALL COMMAND ${CMAKE_COMMAND} -E make_directory "${QG_BUILD_DIR_INCLUDE_PATH}/oops" diff --git a/qg/model/IncrementQG.h b/qg/model/IncrementQG.h index dc0b5a665..d7f625550 100644 --- a/qg/model/IncrementQG.h +++ b/qg/model/IncrementQG.h @@ -23,9 +23,6 @@ #include "oops/base/GeneralizedDepartures.h" #include "oops/base/LocalIncrement.h" -#if !ATLASIFIED -#include "oops/generic/UnstructuredGrid.h" -#endif #include "oops/util/DateTime.h" #include "oops/util/dot_product.h" #include "oops/util/Duration.h" @@ -105,13 +102,6 @@ class IncrementQG : public oops::GeneralizedDepartures, void toAtlas(atlas::FieldSet *) const; void fromAtlas(atlas::FieldSet *); -#if !ATLASIFIED -/// Unstructured grid (doing nothing, just to check compilation if ATLASIFIED=0) - void ug_coord(oops::UnstructuredGrid &) const {} - void field_to_ug(oops::UnstructuredGrid &) const {} - void field_from_ug(const oops::UnstructuredGrid &) {} -#endif - /// Access to fields FieldsQG & fields() {return *fields_;} const FieldsQG & fields() const {return *fields_;} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 455d00e39..8e7107eef 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -175,12 +175,7 @@ oops/generic/LinearModelId.h oops/generic/LocalObsErrorDiag.h oops/generic/ObsErrorDiag.h oops/generic/PseudoModel.h -oops/generic/unstructured_grid_f.h -oops/generic/unstructured_grid_interface.F90 -oops/generic/unstructured_grid_mod.F90 oops/generic/unstructured_interpolation_mod.F90 -oops/generic/UnstructuredGrid.cc -oops/generic/UnstructuredGrid.h oops/generic/VerticalLocEV.h oops/interface/AnalyticInit.h diff --git a/src/oops/assimilation/Increment4D.h b/src/oops/assimilation/Increment4D.h index e544051c2..9da43016a 100644 --- a/src/oops/assimilation/Increment4D.h +++ b/src/oops/assimilation/Increment4D.h @@ -19,7 +19,7 @@ #include -#include "atlas/field/FieldSet.h" +#include "atlas/field.h" #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" @@ -27,9 +27,6 @@ #include "oops/assimilation/CostJbState.h" #include "oops/assimilation/State4D.h" #include "oops/base/Variables.h" -#if !ATLASIFIED -#include "oops/generic/UnstructuredGrid.h" -#endif #include "oops/interface/Geometry.h" #include "oops/interface/Increment.h" #include "oops/util/DateTime.h" @@ -91,17 +88,10 @@ template class Increment4D : public util::Printable, /// Get geometry Geometry_ geometry() const {return this->get(first_).geometry();} -#if ATLASIFIED /// ATLAS FieldSet void setAtlas(atlas::FieldSet *) const; void toAtlas(atlas::FieldSet *) const; void fromAtlas(atlas::FieldSet *); -#else -/// Unstructured grid - void ug_coord(UnstructuredGrid &) const; - void field_to_ug(UnstructuredGrid &) const; - void field_from_ug(const UnstructuredGrid &); -#endif /// Get model space control variable Increment_ & operator[](const int ii) {return this->get(ii);} @@ -314,8 +304,6 @@ void Increment4D::write(const eckit::Configuration & config) const { } } // ----------------------------------------------------------------------------- -#if ATLASIFIED -// ----------------------------------------------------------------------------- template void Increment4D::setAtlas(atlas::FieldSet * afieldset) const { for (icst_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { @@ -337,29 +325,6 @@ void Increment4D::fromAtlas(atlas::FieldSet * afieldset) { } } // ----------------------------------------------------------------------------- -#else -// ----------------------------------------------------------------------------- -template -void Increment4D::ug_coord(UnstructuredGrid & ug) const { - incr4d_.begin()->second->increment().ug_coord(ug); -} -// ----------------------------------------------------------------------------- -template -void Increment4D::field_to_ug(UnstructuredGrid & ug) const { - for (icst_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - jsub->second->increment().field_to_ug(ug, jsub->first); - } -} -// ----------------------------------------------------------------------------- -template -void Increment4D::field_from_ug(const UnstructuredGrid & ug) { - for (iter_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - jsub->second->increment().field_from_ug(ug, jsub->first); - } -} -// ----------------------------------------------------------------------------- -#endif -// ----------------------------------------------------------------------------- template void Increment4D::print(std::ostream & outs) const { for (icst_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { diff --git a/src/oops/generic/UnstructuredGrid.cc b/src/oops/generic/UnstructuredGrid.cc deleted file mode 100644 index 5e6d0140e..000000000 --- a/src/oops/generic/UnstructuredGrid.cc +++ /dev/null @@ -1,113 +0,0 @@ -/* - * (C) Copyright 2017 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include -#include -#include - -#include "atlas/field.h" -#include "atlas/functionspace.h" -#include "atlas/grid.h" -#include "atlas/mesh.h" -#include "atlas/meshgenerator.h" -#include "atlas/util/Config.h" - -#include "eckit/config/LocalConfiguration.h" - -#include "oops/base/Variables.h" -#include "oops/generic/unstructured_grid_f.h" -#include "oops/generic/UnstructuredGrid.h" -#include "oops/util/DateTime.h" - -namespace oops { - -// ----------------------------------------------------------------------------- -UnstructuredGrid::UnstructuredGrid(const int & colocated, const int & nts) : - keyUGrid_(0), atlasFunctionSpace_(), atlasFieldSet_() { - create_ug_f90(keyUGrid_, colocated, nts); -} -// ----------------------------------------------------------------------------- -UnstructuredGrid::UnstructuredGrid(UnstructuredGrid & other) : - keyUGrid_(other.keyUGrid_), atlasFunctionSpace_(std::move(other.atlasFunctionSpace_)), - atlasFieldSet_(std::move(other.atlasFieldSet_)) { - other.keyUGrid_ = 0; -} -// ----------------------------------------------------------------------------- -UnstructuredGrid::~UnstructuredGrid() { - delete_ug_f90(keyUGrid_); -} -// ----------------------------------------------------------------------------- -void UnstructuredGrid::zero() { -} -// ----------------------------------------------------------------------------- -void UnstructuredGrid::random() { -} -// ----------------------------------------------------------------------------- -void UnstructuredGrid::defineGeometry() { - // Set ATLAS lon/lat field - atlasFieldSet_.reset(new atlas::FieldSet()); - ug_set_atlas_lonlat_f90(keyUGrid_, atlasFieldSet_->get()); - atlas::Field atlasField = atlasFieldSet_->field("lonlat"); - - // Create ATLAS function space - atlasFunctionSpace_.reset(new atlas::functionspace::PointCloud(atlasField)); - - // Set ATLAS function space pointer in Fortran - ug_set_atlas_functionspace_pointer_f90(keyUGrid_, atlasFunctionSpace_->get()); - - // Fill ATLAS fieldset - atlasFieldSet_.reset(new atlas::FieldSet()); - ug_fill_atlas_fieldset_f90(keyUGrid_, atlasFieldSet_->get()); - - // Set ATLAS fieldset in Fortran - ug_set_atlas_fieldset_pointer_f90(keyUGrid_, atlasFieldSet_->get()); -} -// ----------------------------------------------------------------------------- -void UnstructuredGrid::defineGrids(std::vector & grids) const { - int ngrid; - ug_get_ngrid_f90(keyUGrid_, ngrid); - for (int jgrid = 0; jgrid < ngrid; ++jgrid) { - int nl, nv, nts; - ug_get_dims_f90(keyUGrid_, jgrid, nl, nv, nts); - std::vector variables; - for (int iv = 0; iv < nv; ++iv) { - std::ostringstream ss; - ss << std::setw(2) << std::setfill('0') << iv+1; - variables.push_back("var_" + ss.str()); - } - eckit::LocalConfiguration grid; - grid.set("grid_index", jgrid); - grid.set("nl", nl); - grid.set("variables", variables); - grid.set("lev2d", "first"); - grids.push_back(grid); - } -} -// ----------------------------------------------------------------------------- -void UnstructuredGrid::setAtlas(atlas::FieldSet * atlasFieldSet) const { - ug_set_atlas_f90(keyUGrid_, atlasFieldSet->get()); -} -// ----------------------------------------------------------------------------- -void UnstructuredGrid::toAtlas(atlas::FieldSet * atlasFieldSet) const { - ug_to_atlas_f90(keyUGrid_, atlasFieldSet->get()); -} -// ----------------------------------------------------------------------------- -void UnstructuredGrid::fromAtlas(atlas::FieldSet * atlasFieldSet) { - ug_from_atlas_f90(keyUGrid_, atlasFieldSet->get()); -} -// ----------------------------------------------------------------------------- -double UnstructuredGrid::dot_product_with(const UnstructuredGrid & other) const { - double zz = 0.0; - return zz; -} -// ----------------------------------------------------------------------------- -void UnstructuredGrid::print(std::ostream & os) const { - os << " UnstructuredGrid: print not implemented yet."; -} -// ----------------------------------------------------------------------------- - -} // namespace oops diff --git a/src/oops/generic/UnstructuredGrid.h b/src/oops/generic/UnstructuredGrid.h deleted file mode 100644 index 28e82d4f3..000000000 --- a/src/oops/generic/UnstructuredGrid.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * (C) Copyright 2017 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef OOPS_GENERIC_UNSTRUCTUREDGRID_H_ -#define OOPS_GENERIC_UNSTRUCTUREDGRID_H_ - -#include -#include -#include - -#include - -#include "atlas/field.h" -#include "atlas/functionspace.h" - -#include "eckit/config/LocalConfiguration.h" - -#include "oops/base/Variables.h" -#include "oops/util/DateTime.h" -#include "oops/util/ObjectCounter.h" -#include "oops/util/Printable.h" - -namespace oops { - -// ----------------------------------------------------------------------------- - -class UnstructuredGrid : public util::Printable, - private boost::noncopyable, - private util::ObjectCounter { - public: - static const std::string classname() {return "oops::UnstructuredGrid";} - - explicit UnstructuredGrid(const int &, const int &); - explicit UnstructuredGrid(UnstructuredGrid &); - ~UnstructuredGrid(); - -// Will be useful for tests - void zero(); - void random(); - double dot_product_with(const UnstructuredGrid &) const; - -// Will be useful for tests - int & toFortran() {return keyUGrid_;} - const int & toFortran() const {return keyUGrid_;} - -/// ATLAS-related methods - void defineGeometry(); - void defineGrids(std::vector &) const; - atlas::FunctionSpace * atlasFunctionSpace() const {return atlasFunctionSpace_.get();} - atlas::FieldSet * atlasFieldSet() const {return atlasFieldSet_.get();} - void setAtlas(atlas::FieldSet *) const; - void toAtlas(atlas::FieldSet *) const; - void fromAtlas(atlas::FieldSet *); - - private: - void print(std::ostream &) const; - - int keyUGrid_; - std::unique_ptr atlasFunctionSpace_; - std::unique_ptr atlasFieldSet_; -}; - -// ----------------------------------------------------------------------------- - -} // namespace oops - -#endif // OOPS_GENERIC_UNSTRUCTUREDGRID_H_ diff --git a/src/oops/generic/unstructured_grid_f.h b/src/oops/generic/unstructured_grid_f.h deleted file mode 100644 index 46a57c0d2..000000000 --- a/src/oops/generic/unstructured_grid_f.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * (C) Copyright 2017 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef OOPS_GENERIC_UNSTRUCTURED_GRID_F_H_ -#define OOPS_GENERIC_UNSTRUCTURED_GRID_F_H_ - -#include "atlas/field.h" -#include "atlas/functionspace.h" - -#include "oops/util/DateTime.h" - -namespace oops { -extern "C" { - void create_ug_f90(int &, const int &, const int &); - void delete_ug_f90(int &); - void ug_get_ngrid_f90(const int &, int &); - void ug_get_dims_f90(const int &, const int &, int &, int &, int &); - void ug_set_atlas_lonlat_f90(const int &, atlas::field::FieldSetImpl *); - void ug_set_atlas_functionspace_pointer_f90(const int &, - atlas::functionspace::FunctionSpaceImpl *); - void ug_fill_atlas_fieldset_f90(const int &, atlas::field::FieldSetImpl *); - void ug_set_atlas_fieldset_pointer_f90(const int &, - atlas::field::FieldSetImpl *); - void ug_set_atlas_f90(const int &, atlas::field::FieldSetImpl *); - void ug_to_atlas_f90(const int &, atlas::field::FieldSetImpl *); - void ug_from_atlas_f90(const int &, atlas::field::FieldSetImpl *); -} -} // namespace oops - -#endif // OOPS_GENERIC_UNSTRUCTURED_GRID_F_H_ diff --git a/src/oops/generic/unstructured_grid_interface.F90 b/src/oops/generic/unstructured_grid_interface.F90 deleted file mode 100644 index f9c598bbb..000000000 --- a/src/oops/generic/unstructured_grid_interface.F90 +++ /dev/null @@ -1,252 +0,0 @@ -! (C) Copyright 2017 UCAR -! -! This software is licensed under the terms of the Apache Licence Version 2.0 -! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -module unstructured_grid_interface - -use atlas_module, only: atlas_fieldset, atlas_functionspace_pointcloud -use iso_c_binding -use unstructured_grid_mod - -implicit none - -private -!------------------------------------------------------------------------------- -contains -!------------------------------------------------------------------------------- -!> Create unstructured grid -subroutine create_ug_c(key,colocated,nts) bind(c,name='create_ug_f90') - -implicit none - -! Passed variables -integer(c_int),intent(inout) :: key !< Unstructured grid -integer(c_int),intent(in) :: colocated !< Colocated flag -integer(c_int),intent(in) :: nts !< Number of timeslots - -! Local variables -type(unstructured_grid),pointer :: self - -! Interface -call unstructured_grid_registry%init() -call unstructured_grid_registry%add(key) -call unstructured_grid_registry%get(key,self) - -! Call Fortran -call create_ug(self,colocated,nts) - -end subroutine create_ug_c -! ------------------------------------------------------------------------------ -!> Delete unstructured grid -subroutine delete_ug_c(key) bind(c,name='delete_ug_f90') - -implicit none - -! Passed variables -integer(c_int),intent(inout) :: key !< Unstructured grid - -! Local variables -type(unstructured_grid),pointer :: self - -! Interface -call unstructured_grid_registry%get(key,self) - -! Call Fortran -call delete_ug(self) - -! Clear interface -call unstructured_grid_registry%remove(key) - -end subroutine delete_ug_c -! ------------------------------------------------------------------------------ -!> Get number of grids -subroutine ug_get_ngrid_c(key,ngrid) bind(c,name='ug_get_ngrid_f90') - -implicit none - -! Passed variables -integer(c_int),intent(in) :: key !< Unstructured grid -integer(c_int),intent(out) :: ngrid !< Number of grids - -! Local variables -type(unstructured_grid),pointer :: self - -! Interface -call unstructured_grid_registry%get(key,self) - -! Call Fortran -ngrid = self%ngrid - -end subroutine ug_get_ngrid_c -! ------------------------------------------------------------------------------ -!> Get dimensions for a given grid -subroutine ug_get_dims_c(key,c_igrid,nl,nv,nts) bind(c,name='ug_get_dims_f90') - -implicit none - -! Passed variables -integer(c_int),intent(in) :: key !< Unstructured grid -integer(c_int),intent(in) :: c_igrid !< Grid index -integer(c_int),intent(out) :: nl !< Number of levels -integer(c_int),intent(out) :: nv !< Number of variables -integer(c_int),intent(out) :: nts !< Number of timeslots - -! Local variables -type(unstructured_grid),pointer :: self -integer :: igrid - -! Interface -call unstructured_grid_registry%get(key,self) -igrid = c_igrid+1 - -! Call Fortran -nl = self%grid(igrid)%nl0 -nv = self%grid(igrid)%nv -nts = 1 - -end subroutine ug_get_dims_c -! ------------------------------------------------------------------------------ -!> Set ATLAS grid lon/lat in fieldset -subroutine ug_set_atlas_lonlat_c(key,c_afieldset) bind(c,name='ug_set_atlas_lonlat_f90') - -implicit none - -! Passed variables -integer(c_int),intent(in) :: key !< Unstructured grid -type(c_ptr),intent(in),value :: c_afieldset !< ATLAS fieldset pointer - -! Local variables -type(unstructured_grid),pointer :: self -type(atlas_fieldset) :: afieldset - -! Interface -call unstructured_grid_registry%get(key,self) -afieldset = atlas_fieldset(c_afieldset) - -! Call Fortran -call ug_set_atlas_lonlat(self,afieldset) - -end subroutine ug_set_atlas_lonlat_c -! ------------------------------------------------------------------------------ -!> Set ATLAS function space pointer -subroutine ug_set_atlas_functionspace_pointer_c(key,c_afunctionspace) bind(c,name='ug_set_atlas_functionspace_pointer_f90') - -! Passed variables -integer(c_int),intent(in) :: key !< Unstructured grid -type(c_ptr),intent(in),value :: c_afunctionspace !< ATLAS function space pointer - -! Local variables -type(unstructured_grid),pointer :: self - -! Interface -call unstructured_grid_registry%get(key,self) -self%afunctionspace = atlas_functionspace_pointcloud(c_afunctionspace) - -end subroutine ug_set_atlas_functionspace_pointer_c -! ------------------------------------------------------------------------------ -!> Fill ATLAS fieldset from unstructured grid -subroutine ug_fill_atlas_fieldset_c(key,c_afieldset) bind(c,name='ug_fill_atlas_fieldset_f90') - -implicit none - -! Passed variables -integer(c_int),intent(in) :: key !< Unstructured grid -type(c_ptr),intent(in),value :: c_afieldset !< ATLAS fieldset pointer - -! Local variables -type(unstructured_grid),pointer :: self -type(atlas_fieldset) :: afieldset - -! Interface -call unstructured_grid_registry%get(key,self) -afieldset = atlas_fieldset(c_afieldset) - -! Call Fortran -call ug_fill_atlas_fieldset(self,afieldset) - -end subroutine ug_fill_atlas_fieldset_c -! ------------------------------------------------------------------------------ -!> Set ATLAS fieldset pointer -subroutine ug_set_atlas_fieldset_pointer_c(key,c_afieldset) bind(c,name='ug_set_atlas_fieldset_pointer_f90') - -! Passed variables -integer(c_int),intent(in) :: key !< Unstructured grid -type(c_ptr),intent(in),value :: c_afieldset !< ATLAS function space pointer - -! Local variables -type(unstructured_grid),pointer :: self - -! Interface -call unstructured_grid_registry%get(key,self) -self%afieldset = atlas_fieldset(c_afieldset) - -end subroutine ug_set_atlas_fieldset_pointer_c -! ------------------------------------------------------------------------------ -!> Set ATLAS fieldset -subroutine ug_set_atlas_c(key,c_afieldset) bind(c,name='ug_set_atlas_f90') - -implicit none - -! Passed variables -integer(c_int),intent(in) :: key !< Unstructured grid -type(c_ptr),intent(in),value :: c_afieldset !< ATLAS fieldset pointer - -! Local variables -type(unstructured_grid),pointer :: self -type(atlas_fieldset) :: afieldset - -! Interface -call unstructured_grid_registry%get(key,self) -afieldset = atlas_fieldset(c_afieldset) - -! Call Fortran -call ug_set_atlas(self,afieldset) - -end subroutine ug_set_atlas_c -! ------------------------------------------------------------------------------ -!> Unstructured grid to ATLAS fieldset -subroutine ug_to_atlas_c(key,c_afieldset) bind(c,name='ug_to_atlas_f90') - -implicit none - -! Passed variables -integer(c_int),intent(in) :: key !< Unstructured grid -type(c_ptr),intent(in),value :: c_afieldset !< ATLAS fieldset pointer - -! Local variables -type(unstructured_grid),pointer :: self -type(atlas_fieldset) :: afieldset - -! Interface -call unstructured_grid_registry%get(key,self) -afieldset = atlas_fieldset(c_afieldset) - -! Call Fortran -call ug_to_atlas(self,afieldset) - -end subroutine ug_to_atlas_c -! ------------------------------------------------------------------------------ -!> Unstructured grid from ATLAS fieldset -subroutine ug_from_atlas_c(key,c_afieldset) bind(c,name='ug_from_atlas_f90') - -implicit none - -! Passed variables -integer(c_int),intent(in) :: key !< Unstructured grid -type(c_ptr),intent(in),value :: c_afieldset !< ATLAS fieldset pointer - -! Local variables -type(unstructured_grid),pointer :: self -type(atlas_fieldset) :: afieldset - -! Interface -call unstructured_grid_registry%get(key,self) -afieldset = atlas_fieldset(c_afieldset) - -! Call Fortran -call ug_from_atlas(self,afieldset) - -end subroutine ug_from_atlas_c -! ------------------------------------------------------------------------------ -end module unstructured_grid_interface diff --git a/src/oops/generic/unstructured_grid_mod.F90 b/src/oops/generic/unstructured_grid_mod.F90 deleted file mode 100644 index 33039d08a..000000000 --- a/src/oops/generic/unstructured_grid_mod.F90 +++ /dev/null @@ -1,330 +0,0 @@ -! (C) Copyright 2017 UCAR -! -! This software is licensed under the terms of the Apache Licence Version 2.0 -! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -module unstructured_grid_mod - -use atlas_module, only: atlas_field, atlas_fieldset, atlas_integer, atlas_real, atlas_functionspace_pointcloud -use datetime_mod -use iso_c_binding -use kinds -use oops_variables_mod - -implicit none - -private -public :: unstructured_grid -public :: unstructured_grid_registry -public :: create_ug,delete_ug,ug_set_atlas_lonlat,ug_fill_atlas_fieldset,ug_set_atlas,ug_to_atlas,ug_from_atlas -public :: allocate_unstructured_grid_coord,allocate_unstructured_grid_field -! ------------------------------------------------------------------------------ -type grid_type - integer :: igrid !> Index of the grid - integer :: nmga !> Number of gridpoints (on a given MPI task) - integer :: nl0 !> Number of levels - integer :: nv !> Number of variables - integer :: nts !> Number of timeslots - real(kind=kind_real),allocatable :: lon(:) !> Longitude (in degrees: -180 to 180) - real(kind=kind_real),allocatable :: lat(:) !> Latitude (in degrees: -90 to 90) - real(kind=kind_real),allocatable :: area(:) !> Area (in m^2) - real(kind=kind_real),allocatable :: vunit(:,:) !> Vertical unit - logical,allocatable :: lmask(:,:) !> Mask - real(kind=kind_real),allocatable :: fld(:,:,:,:) !> Data -end type grid_type - -type unstructured_grid - integer :: colocated !> Colocation flag - integer :: nts !> Number of timeslots - integer :: ngrid !> Number of different grids - type(grid_type),allocatable :: grid(:) !> Grid instance - type(atlas_functionspace_pointcloud) :: afunctionspace !< ATLAS function space - type(atlas_fieldset) :: afieldset !< ATLAS fieldset -end type unstructured_grid - -#define LISTED_TYPE unstructured_grid - -!> Linked list interface - defines registry_t type -#include "oops/util/linkedList_i.f" - -!> Global registry -type(registry_t) :: unstructured_grid_registry -!------------------------------------------------------------------------------- -contains -!------------------------------------------------------------------------------- -!> Linked list implementation -#include "oops/util/linkedList_c.f" -!------------------------------------------------------------------------------- -!> Create unstructured grid -subroutine create_ug(self,colocated,nts) - -implicit none - -! Passed variables -type(unstructured_grid),intent(inout) :: self !< Unstructured grid -integer,intent(in) :: colocated !< Colocated flag -integer,intent(in) :: nts !< Number of timeslots - -! Set colocated flag -self%colocated = colocated - -! Set number of timeslots -if (nts>1) call abor1_ftn('create_ug: number of timeslots should be one now') -self%nts = 1 - -end subroutine create_ug -!------------------------------------------------------------------------------- -!> Delete unstructured grid -subroutine delete_ug(self) - -implicit none - -! Passed variables -type(unstructured_grid),intent(inout) :: self !< Unstructured grid - -! Local variables -integer :: igrid - -! Release memory -if (allocated(self%grid)) then - do igrid=1,self%ngrid - if (allocated(self%grid(igrid)%lon)) deallocate(self%grid(igrid)%lon) - if (allocated(self%grid(igrid)%lat)) deallocate(self%grid(igrid)%lat) - if (allocated(self%grid(igrid)%area)) deallocate(self%grid(igrid)%area) - if (allocated(self%grid(igrid)%vunit)) deallocate(self%grid(igrid)%vunit) - if (allocated(self%grid(igrid)%lmask)) deallocate(self%grid(igrid)%lmask) - if (allocated(self%grid(igrid)%fld)) deallocate(self%grid(igrid)%fld) - enddo - deallocate(self%grid) -endif -call self%afunctionspace%final() -call self%afieldset%final() - -end subroutine delete_ug -! ------------------------------------------------------------------------------ -!> Set ATLAS grid lon/lat in fieldset -subroutine ug_set_atlas_lonlat(self,afieldset) - -! Passed variables -type(unstructured_grid),intent(inout) :: self !< Unstructured grid -type(atlas_fieldset),intent(inout) :: afieldset !< ATLAS fieldset - -! Local variables -integer :: imga -real(kind_real),parameter :: pi = 4.d0*atan(1.0) -real(kind_real),parameter :: rad2deg = 180.0/pi -real(kind_real),pointer :: real_ptr(:,:) -type(atlas_field) :: afield - -! Create lon/lat field -afield = atlas_field(name="lonlat",kind=atlas_real(kind_real),shape=(/2,self%grid(1)%nmga/)) -call afield%data(real_ptr) -do imga=1,self%grid(1)%nmga - real_ptr(1,imga) = self%grid(1)%lon(imga) - real_ptr(2,imga) = self%grid(1)%lat(imga) -end do -call afieldset%add(afield) - -end subroutine ug_set_atlas_lonlat -! ------------------------------------------------------------------------------ -!> Fill ATLAS fieldset from unstructured grid -subroutine ug_fill_atlas_fieldset(self,afieldset) - -! Passed variables -type(unstructured_grid),intent(inout) :: self !< Unstructured grid -type(atlas_fieldset),intent(inout) :: afieldset !< ATLAS fieldset - -! Local variables -integer :: imga,il0 -integer,pointer :: int_ptr_2(:,:) -real(kind_real),pointer :: real_ptr_1(:),real_ptr_2(:,:) -type(atlas_field) :: afield - -! Add area -afield = self%afunctionspace%create_field(name='area',kind=atlas_real(kind_real),levels=0) -call afield%data(real_ptr_1) -real_ptr_1(1:self%grid(1)%nmga) = self%grid(1)%area -call afieldset%add(afield) -call afield%final() - -! Add vertical unit -afield = self%afunctionspace%create_field(name='vunit',kind=atlas_real(kind_real),levels=self%grid(1)%nl0) -call afield%data(real_ptr_2) -real_ptr_2(1:self%grid(1)%nl0,1:self%grid(1)%nmga) = transpose(self%grid(1)%vunit) -call afieldset%add(afield) -call afield%final() - -! Add geometry mask -afield = self%afunctionspace%create_field(name='gmask',kind=atlas_integer(kind_int),levels=self%grid(1)%nl0) -call afield%data(int_ptr_2) -do il0=1,self%grid(1)%nl0 - do imga=1,self%grid(1)%nmga - if (self%grid(1)%lmask(imga,il0)) then - int_ptr_2(il0,imga) = 1 - else - int_ptr_2(il0,imga) = 0 - endif - enddo -enddo -call afieldset%add(afield) -call afield%final() - -end subroutine ug_fill_atlas_fieldset -! ------------------------------------------------------------------------------ -!> Set ATLAS fieldset -subroutine ug_set_atlas(self,afieldset) - -implicit none - -! Passed variables -type(unstructured_grid),intent(in) :: self !< Unstructured grid -type(atlas_fieldset),intent(inout) :: afieldset !< ATLAS fieldset - -! Local variables -integer :: iv,igrid -character(len=1024) :: fieldname -type(atlas_field) :: afield - -! Copy fields -do igrid=1,self%ngrid - do iv=1,self%grid(igrid)%nv - ! Get or create field - write(fieldname,'(a,i2.2)') 'var_',iv - if (afieldset%has_field(trim(fieldname))) then - ! Get field - afield = afieldset%field(trim(fieldname)) - else - ! Create field - afield = self%afunctionspace%create_field(name=trim(fieldname),kind=atlas_real(kind_real),levels=self%grid(igrid)%nl0) - - ! Add field - call afieldset%add(afield) - endif - - ! Release pointer - call afield%final() - enddo -enddo - -end subroutine ug_set_atlas -! ------------------------------------------------------------------------------ -!> Unstructured grid to ATLAS fieldset -subroutine ug_to_atlas(self,afieldset) - -implicit none - -! Passed variables -type(unstructured_grid),intent(in) :: self !< Unstructured grid -type(atlas_fieldset),intent(inout) :: afieldset !< ATLAS fieldset - -! Local variables -integer :: iv,igrid -real(kind_real),pointer :: real_ptr_2(:,:) -character(len=1024) :: fieldname -type(atlas_field) :: afield - -! Copy fields -do igrid=1,self%ngrid - do iv=1,self%grid(igrid)%nv - ! Get or create field - write(fieldname,'(a,i2.2)') 'var_',iv - if (afieldset%has_field(trim(fieldname))) then - ! Get field - afield = afieldset%field(trim(fieldname)) - else - ! Create field - afield = self%afunctionspace%create_field(name=trim(fieldname),kind=atlas_real(kind_real),levels=self%grid(igrid)%nl0) - - ! Add field - call afieldset%add(afield) - endif - - ! Copy data - call afield%data(real_ptr_2) - real_ptr_2(1:self%grid(igrid)%nl0,1:self%grid(igrid)%nmga) = transpose(self%grid(igrid)%fld(:,:,iv,1)) - - ! Release pointer - call afield%final() - enddo -enddo - -end subroutine ug_to_atlas -! ------------------------------------------------------------------------------ -!> Convert ATLAS fieldset to unstructured grid -subroutine ug_from_atlas(self,afieldset) - -implicit none - -! Passed variables -type(unstructured_grid),intent(inout) :: self !< Unstructured grid -type(atlas_fieldset),intent(inout) :: afieldset !< ATLAS fieldset - -! Local variables -integer :: iv,igrid -real(kind_real),pointer :: real_ptr_2(:,:) -character(len=1024) :: fieldname -type(atlas_field) :: afield - -! Allocation -call allocate_unstructured_grid_field(self) - -! Copy fields -do igrid=1,self%ngrid - do iv=1,self%grid(igrid)%nv - ! Get field - write(fieldname,'(a,i2.2)') 'var_',iv - afield = afieldset%field(trim(fieldname)) - - ! Copy data - call afield%data(real_ptr_2) - self%grid(igrid)%fld(:,:,iv,1) = transpose(real_ptr_2(1:self%grid(igrid)%nl0,1:self%grid(igrid)%nmga)) - - ! Release pointer - call afield%final() - enddo -enddo - -end subroutine ug_from_atlas -! ------------------------------------------------------------------------------ -!> Allocate unstructured grid coordinates -subroutine allocate_unstructured_grid_coord(self) - -implicit none - -! Passed variables -type(unstructured_grid),intent(inout) :: self !< Unstructured grid - -! Local variables -integer :: igrid - -! Allocation -do igrid=1,self%ngrid - if (.not.allocated(self%grid(igrid)%lon)) allocate(self%grid(igrid)%lon(self%grid(igrid)%nmga)) - if (.not.allocated(self%grid(igrid)%lat)) allocate(self%grid(igrid)%lat(self%grid(igrid)%nmga)) - if (.not.allocated(self%grid(igrid)%area)) allocate(self%grid(igrid)%area(self%grid(igrid)%nmga)) - if (.not.allocated(self%grid(igrid)%vunit)) allocate(self%grid(igrid)%vunit(self%grid(igrid)%nmga,self%grid(igrid)%nl0)) - if (.not.allocated(self%grid(igrid)%lmask)) allocate(self%grid(igrid)%lmask(self%grid(igrid)%nmga,self%grid(igrid)%nl0)) -enddo - -end subroutine allocate_unstructured_grid_coord -! ------------------------------------------------------------------------------ -!> Allocate unstructured grid fields -subroutine allocate_unstructured_grid_field(self) - -implicit none - -! Passed variables -type(unstructured_grid),intent(inout) :: self !< Unstructured grid - -! Local variables -integer :: igrid - -! Allocation -do igrid=1,self%ngrid - if (.not.allocated(self%grid(igrid)%fld)) allocate(self%grid(igrid)%fld(self%grid(igrid)%nmga,self%grid(igrid)%nl0,& - & self%grid(igrid)%nv,1)) -enddo - -end subroutine allocate_unstructured_grid_field -!------------------------------------------------------------------------------- -end module unstructured_grid_mod diff --git a/src/oops/interface/Geometry.h b/src/oops/interface/Geometry.h index e52f0090e..605c68621 100644 --- a/src/oops/interface/Geometry.h +++ b/src/oops/interface/Geometry.h @@ -85,10 +85,8 @@ class Geometry : public util::Printable, /// Accessor to the geometry communicator const eckit::mpi::Comm & getComm() const {return geom_->getComm();} -#if ATLASIFIED atlas::FunctionSpace * atlasFunctionSpace() const {return geom_->atlasFunctionSpace();} atlas::FieldSet * atlasFieldSet() const {return geom_->atlasFieldSet();} -#endif protected: std::shared_ptr geom_; /// pointer to the Geometry implementation diff --git a/src/oops/interface/Increment.h b/src/oops/interface/Increment.h index f46a37f72..4deeb10bd 100644 --- a/src/oops/interface/Increment.h +++ b/src/oops/interface/Increment.h @@ -24,9 +24,6 @@ #include "oops/interface/Geometry.h" #include "oops/interface/GeometryIterator.h" #include "oops/interface/State.h" -#if !ATLASIFIED -#include "oops/generic/UnstructuredGrid.h" -#endif #include "oops/mpi/mpi.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" @@ -411,13 +408,7 @@ template void Increment::setAtlas(atlas::FieldSet * atlasFieldSet) const { Log::trace() << "Increment::setAtlas starting" << std::endl; util::Timer timer(classname(), "setAtlas"); -#if ATLASIFIED increment_->setAtlas(atlasFieldSet); -#else - oops::UnstructuredGrid ug(1, 1); - increment_->ug_coord(ug); - ug.setAtlas(atlasFieldSet); -#endif Log::trace() << "Increment::setAtlas done" << std::endl; } @@ -427,14 +418,7 @@ template void Increment::toAtlas(atlas::FieldSet * atlasFieldSet) const { Log::trace() << "Increment::toAtlas starting" << std::endl; util::Timer timer(classname(), "toAtlas"); -#if ATLASIFIED increment_->toAtlas(atlasFieldSet); -#else - oops::UnstructuredGrid ug(1, 1); - increment_->ug_coord(ug); - increment_->field_to_ug(ug); - ug.toAtlas(atlasFieldSet); -#endif Log::trace() << "Increment::toAtlas done" << std::endl; } @@ -444,15 +428,7 @@ template void Increment::fromAtlas(atlas::FieldSet * atlasFieldSet) { Log::trace() << "Increment::fromAtlas starting" << std::endl; util::Timer timer(classname(), "fromAtlas"); -#if ATLASIFIED increment_->fromAtlas(atlasFieldSet); -#else - oops::UnstructuredGrid ug(1, 1); - increment_->ug_coord(ug); - increment_->field_to_ug(ug); - ug.fromAtlas(atlasFieldSet); - increment_->field_from_ug(ug); -#endif Log::trace() << "Increment::fromAtlas done" << std::endl; } @@ -461,14 +437,7 @@ void Increment::fromAtlas(atlas::FieldSet * atlasFieldSet) { template void Increment::toAtlas() { Log::trace() << "Increment::toAtlas starting" << std::endl; -#if ATLASIFIED increment_->toAtlas(&atlasFieldSet_); -#else - oops::UnstructuredGrid ug(1, 1); - increment_->ug_coord(ug); - increment_->field_to_ug(ug); - ug.toAtlas(&atlasFieldSet_); -#endif increment_.reset(); Log::trace() << "Increment::toAtlas done" << std::endl; } From 2b0576866440f26ec7b4db4f5bfb9d838a4822b8 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Mon, 2 Nov 2020 15:23:22 -0700 Subject: [PATCH 003/142] remove unused functions and methods (#946) --- src/oops/base/LocalIncrement.cc | 16 ---------------- src/oops/base/LocalIncrement.h | 3 --- src/oops/util/IntSetParser.cc | 12 ------------ 3 files changed, 31 deletions(-) diff --git a/src/oops/base/LocalIncrement.cc b/src/oops/base/LocalIncrement.cc index e5b75d434..2c9e49961 100644 --- a/src/oops/base/LocalIncrement.cc +++ b/src/oops/base/LocalIncrement.cc @@ -11,22 +11,6 @@ namespace oops { - LocalIncrement & LocalIncrement::operator +=(const LocalIncrement & rhs) { - ASSERT(vars_ == rhs.vars_ && varlens_ == rhs.varlens_); - ASSERT(vals_.size() == rhs.vals_.size()); - for (unsigned i=0; i < vals_.size(); ++i) { - vals_[i] += rhs.vals_[i]; - } - return *this; - } - - LocalIncrement & LocalIncrement::operator *=(const double & zz) { - for (unsigned i=0; i < vals_.size(); ++i) { - vals_[i] *= zz; - } - return *this; - } - LocalIncrement & LocalIncrement::operator *=(const std::vector & rhs) { ASSERT(vals_.size() == rhs.size()); for (unsigned i=0; i < vals_.size(); ++i) { diff --git a/src/oops/base/LocalIncrement.h b/src/oops/base/LocalIncrement.h index b870e4e82..e29165ed0 100644 --- a/src/oops/base/LocalIncrement.h +++ b/src/oops/base/LocalIncrement.h @@ -28,11 +28,8 @@ class LocalIncrement: public util::Printable { void setVals(std::vector &); /// Linear algebra operators - LocalIncrement & operator+=(const LocalIncrement &); - LocalIncrement & operator*=(const double &); LocalIncrement & operator*=(const std::vector &); - private: void print(std::ostream & os) const { os << "LocalIncrement, size: " << vals_.size() << ", first element: " diff --git a/src/oops/util/IntSetParser.cc b/src/oops/util/IntSetParser.cc index 4a18f8c48..f6440122e 100644 --- a/src/oops/util/IntSetParser.cc +++ b/src/oops/util/IntSetParser.cc @@ -88,17 +88,5 @@ std::set parseIntSet(const std::string & str) { return channels; } -// ----------------------------------------------------------------------------- - -void splitVarGroup(const std::string & vargrp, std::string & var, std::string & grp) { - const size_t at = vargrp.find("@"); - var = vargrp.substr(0, at); - if (at != std::string::npos) { - grp = vargrp.substr(at + 1, std::string::npos); - const size_t no_at = grp.find("@"); - ASSERT(no_at == std::string::npos); - } -} - // ----------------------------------------------------------------------------- } // namespace oops From be9e7a1bbe0b707c52352ad58893447a309dfa47 Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Tue, 3 Nov 2020 16:09:08 -0700 Subject: [PATCH 004/142] update clone.sh (#953) --- CI/clone.sh | 61 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/CI/clone.sh b/CI/clone.sh index 3386a44ea..171c3b9fc 100755 --- a/CI/clone.sh +++ b/CI/clone.sh @@ -5,28 +5,61 @@ git_user=$1 git_token=$2 -repo_name=$3 +org_repo_name=$3 branch_name=$4 save_name=$5 save_dir=$6 branch_name_default=$7 -echo "===============================================================================" -echo "Clone " $repo_name -echo "===============================================================================" +repo_name="$(cut -d'/' -f2 <<<$org_repo_name)" +org_name="$(cut -d'/' -f1 <<<$org_repo_name)" -git ls-remote --heads --exit-code https://$git_user:$git_token@github.com/$repo_name $branch_name -exit_code=$? -if test "${exit_code}" == "0"; then - branch_name_clone=${branch_name} - echo ${branch_name} " branch found" -else +if [ "${branch_name}" = "develop" ] || [ "${branch_name}" = "master" ]; then + + echo "merging into develop or master" branch_name_clone=${branch_name_default} - echo ${branch_name} " branch does not exist" - echo "clone " ${branch_name_clone} -fi + org_repo_name_clone=${org_name} + echo "===============================================================================" + echo " Merge into develop or master" + echo " Clone " $org_repo_name_clone "/" $repo_name + echo "===============================================================================" + git clone -b $branch_name_clone https://$git_user:$git_token@github.com/$org_repo_name_clone/$repo_name $save_dir/$save_name + +else + + # check jcsda-internal for branch + git ls-remote --heads --exit-code https://$git_user:$git_token@github.com/jcsda-internal/$repo_name $branch_name + exit_code_internal=$? -git clone --depth 1 -b $branch_name_clone https://$git_user:$git_token@github.com/$repo_name $save_dir/$save_name + git ls-remote --heads --exit-code https://$git_user:$git_token@github.com/jcsda/$repo_name $branch_name + exit_code=$? + + # if branch exists in both jcsda-internal and jcsda it will clone jcsda-internal + # it searches in jcsda only if org_name is jcsda + + if test "${exit_code_internal}" == "0"; then + echo ${branch_name} " branch found in jcsda-internal" + branch_name_clone=${branch_name} + org_repo_name_clone="jcsda-internal" + + # search in jcsda only if cloning from jcsda + elif [ "${exit_code}" == "0" ] && [ "${org_name}" == "jcsda" ]; then + echo ${branch_name} " branch found in jcsda" + branch_name_clone=${branch_name} + org_repo_name_clone="jcsda" + + else + echo ${branch_name} " branch does not exist in jcsda-internal or jcsda" + echo "clone " ${branch_name_clone} + branch_name_clone=${branch_name_default} + org_repo_name_clone=${org_name} + fi + echo "===============================================================================" + echo "Clone " $org_repo_name_clone "/" $repo_name " branch " $branch_name_clone + echo "===============================================================================" + git clone --depth 1 -b $branch_name_clone https://$git_user:$git_token@github.com/$org_repo_name_clone/$repo_name $save_dir/$save_name + +fi From 8cbc88d3429920845df0f3a6b14ec0d6739c55f9 Mon Sep 17 00:00:00 2001 From: Benjamin Menetrier <30638301+benjaminmenetrier@users.noreply.github.com> Date: Wed, 4 Nov 2020 17:03:24 +0100 Subject: [PATCH 005/142] Remove GNU 9.3.0 compiler warnings (#957) * Remove GNU 9.3.0 compiler warnings * Change conversion from size_t to int64_t --- src/oops/assimilation/CostFct4DEnsVar.h | 2 +- src/oops/assimilation/CostFctWeak.h | 2 +- src/oops/assimilation/CostJbJq.h | 2 +- src/oops/assimilation/JqTermTLAD.h | 8 ++++---- src/oops/mpi/mpi.cc | 2 +- src/test/mpi/mpi.h | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/oops/assimilation/CostFct4DEnsVar.h b/src/oops/assimilation/CostFct4DEnsVar.h index c83befb1e..1e7312060 100644 --- a/src/oops/assimilation/CostFct4DEnsVar.h +++ b/src/oops/assimilation/CostFct4DEnsVar.h @@ -108,7 +108,7 @@ CostFct4DEnsVar::CostFct4DEnsVar(const eckit::Configuration & config subWinLength_ = util::Duration(config.getString("subwindow")); nsubwin_ = windowLength.toSeconds() / subWinLength_.toSeconds() + 1; // Not like WC - ASSERT(windowLength.toSeconds() == subWinLength_.toSeconds() * (nsubwin_ - 1)); + ASSERT(windowLength.toSeconds() == subWinLength_.toSeconds() * (int64_t)(nsubwin_ - 1)); size_t ntasks = comm.size(); ASSERT(ntasks % nsubwin_ == 0); diff --git a/src/oops/assimilation/CostFctWeak.h b/src/oops/assimilation/CostFctWeak.h index f88207d74..14b32dae7 100644 --- a/src/oops/assimilation/CostFctWeak.h +++ b/src/oops/assimilation/CostFctWeak.h @@ -116,7 +116,7 @@ CostFctWeak::CostFctWeak(const eckit::Configuration & config, subWinLength_ = util::Duration(config.getString("subwindow")); nsubwin_ = windowLength.toSeconds() / subWinLength_.toSeconds(); - ASSERT(windowLength.toSeconds() == subWinLength_.toSeconds()*nsubwin_); + ASSERT(windowLength.toSeconds() == subWinLength_.toSeconds()*(int64_t)nsubwin_); size_t ntasks = comm.size(); ASSERT(ntasks % nsubwin_ == 0); diff --git a/src/oops/assimilation/CostJbJq.h b/src/oops/assimilation/CostJbJq.h index 80ca2dc7e..a66a54107 100644 --- a/src/oops/assimilation/CostJbJq.h +++ b/src/oops/assimilation/CostJbJq.h @@ -127,7 +127,7 @@ void CostJbJq::computeIncrement(const State_ & xb, const State_ & fg, con Increment_ & dx) const { Log::trace() << "CostJbJq::computeIncrement start" << std::endl; static int tag = 13579; - int mytime = commTime_.rank(); + size_t mytime = commTime_.rank(); State_ mxim1(fg); // Send values of M(x_i) at end of my subwindow to next subwindow diff --git a/src/oops/assimilation/JqTermTLAD.h b/src/oops/assimilation/JqTermTLAD.h index 37f7f7851..3a7a1455b 100644 --- a/src/oops/assimilation/JqTermTLAD.h +++ b/src/oops/assimilation/JqTermTLAD.h @@ -130,7 +130,7 @@ State & JqTermTLAD::getMxi() const { template void JqTermTLAD::doFinalizeTL(const Increment_ & dx) { Log::trace() << "JqTermTLAD::doFinalizeTL start" << std::endl; - int mytime = commTime_.rank(); + size_t mytime = commTime_.rank(); if (mytime + 1 < commTime_.size()) oops::mpi::send(commTime_, dx, mytime+1, 2468); Log::trace() << "JqTermTLAD::doFinalizeTL done" << std::endl; } @@ -141,7 +141,7 @@ template void JqTermTLAD::computeModelErrorTL(Increment_ & dx) { Log::trace() << "JqTermTLAD::computeModelErrorTL start" << std::endl; // Compute x_i - M(x_{i-1}) - int mytime = commTime_.rank(); + size_t mytime = commTime_.rank(); if (mytime > 0) { Increment_ mxim1(dx, false); oops::mpi::receive(commTime_, mxim1, mytime-1, 2468); @@ -156,7 +156,7 @@ void JqTermTLAD::computeModelErrorTL(Increment_ & dx) { template void JqTermTLAD::setupAD(const Increment_ & dx) { Log::trace() << "JqTermTLAD::setupAD start" << std::endl; - int mytime = commTime_.rank(); + size_t mytime = commTime_.rank(); if (mytime > 0) oops::mpi::send(commTime_, dx, mytime-1, 8642); Log::trace() << "JqTermTLAD::setupAD done" << std::endl; } @@ -167,7 +167,7 @@ template void JqTermTLAD::doFirstAD(Increment_ & dx, const util::DateTime &, const util::Duration &) { Log::trace() << "JqTermTLAD::doFirstAD start" << std::endl; - int mytime = commTime_.rank(); + size_t mytime = commTime_.rank(); if (mytime + 1 < commTime_.size()) { Increment_ xip1(dx, false); oops::mpi::receive(commTime_, xip1, mytime+1, 8642); diff --git a/src/oops/mpi/mpi.cc b/src/oops/mpi/mpi.cc index c4fcbf5e3..7eed3dc99 100644 --- a/src/oops/mpi/mpi.cc +++ b/src/oops/mpi/mpi.cc @@ -33,7 +33,7 @@ const eckit::mpi::Comm & myself() { void gather(const eckit::mpi::Comm & comm, const std::vector & send, std::vector & recv, const size_t root) { - int ntasks = comm.size(); + size_t ntasks = comm.size(); if (ntasks > 1) { int mysize = send.size(); std::vector sizes(ntasks); diff --git a/src/test/mpi/mpi.h b/src/test/mpi/mpi.h index baaf9630a..135c936d1 100644 --- a/src/test/mpi/mpi.h +++ b/src/test/mpi/mpi.h @@ -115,13 +115,13 @@ CASE("mpi/mpi/gatherSerializable") { std::vector globalValues(numGlobalValues); util::DateTime zeroDate("0001-01-01T00:00:00Z"); - for (int ii = 0; ii < numGlobalValues; ++ii) { + for (size_t ii = 0; ii < numGlobalValues; ++ii) { globalValues[ii] = zeroDate; } std::vector zeroValues = globalValues; - int root_gather = conf.getInt("root for gathering", 0); + size_t root_gather = conf.getInt("root for gathering", 0); oops::mpi::gather(comm, localValues, globalValues, root_gather); if (rank == root_gather) { @@ -148,7 +148,7 @@ CASE("mpi/mpi/gatherDouble") { std::vector globalDouble(numGlobalDouble, 0.0); std::vector zerosDouble = globalDouble; - int root_gather = conf.getInt("root for gathering", 0); + size_t root_gather = conf.getInt("root for gathering", 0); oops::mpi::gather(comm, localDouble, globalDouble, root_gather); From d66605e5433decc7b4c777dd84cd12a797fe2c56 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 4 Nov 2020 14:49:03 -0700 Subject: [PATCH 006/142] change ObsOperator::locations() signature (#947) * simplify L95 geovals and locs * simplify qg locs & geovals; rollback accidental l95 change * change ObsOp::locations signature Co-authored-by: Anna Shlyaeva --- l95/src/lorenz95/ObsTableView.cc | 35 +++-------- l95/src/lorenz95/ObsTableView.h | 3 +- l95/src/lorenz95/ObservationL95.cc | 6 +- l95/src/lorenz95/ObservationL95.h | 6 +- qg/model/ObsOpBaseQG.h | 4 +- qg/model/ObsOperatorQG.cc | 6 +- qg/model/ObsOperatorQG.h | 6 +- qg/model/ObsSpaceQG.cc | 8 +-- qg/model/ObsSpaceQG.h | 5 +- qg/model/ObsStreamQG.cc | 5 +- qg/model/ObsStreamQG.h | 3 +- qg/model/ObsWSpeedQG.cc | 5 +- qg/model/ObsWSpeedQG.h | 3 +- qg/model/ObsWindQG.cc | 5 +- qg/model/ObsWindQG.h | 3 +- qg/model/QgFortran.h | 1 - qg/model/qg_obsdb_interface.F90 | 9 +-- qg/model/qg_obsdb_mod.F90 | 80 ++------------------------ src/oops/base/Observer.h | 12 ++-- src/oops/base/ObserverTLAD.h | 13 +++-- src/oops/interface/ObsOperator.h | 8 +-- src/test/interface/LinearObsOperator.h | 5 +- src/test/interface/ObsOperator.h | 5 +- 23 files changed, 59 insertions(+), 177 deletions(-) diff --git a/l95/src/lorenz95/ObsTableView.cc b/l95/src/lorenz95/ObsTableView.cc index ef16583f1..e0c4b38d7 100644 --- a/l95/src/lorenz95/ObsTableView.cc +++ b/l95/src/lorenz95/ObsTableView.cc @@ -39,7 +39,7 @@ ObsTableView::ObsTableView(const ObsTableView & obstable, const eckit::Configuration & conf) : obstable_(obstable.obstable_), localobs_(), obsdist_() { - std::vector locations = obstable.locations(); + std::vector locations = obstable.obstable_->locations(); const double dist = conf.getDouble("lengthscale"); for (unsigned int jj = 0; jj < obstable.nobs(); ++jj) { double curdist = std::abs(center[0] - locations[jj]); @@ -162,18 +162,6 @@ unsigned int ObsTableView::nobs() const { // ----------------------------------------------------------------------------- -std::vector ObsTableView::locations() const { - std::vector full = obstable_->locations(); - std::vector local(nobs()); - for (unsigned int i = 0; i < nobs(); i++) { - local[i] = full[localobs_[i]]; - } - oops::Log::trace() << "ObsTableView::locations done" << std::endl; - return local; -} - -// ----------------------------------------------------------------------------- - void ObsTableView::generateDistribution(const eckit::Configuration & conf) { obstable_->generateDistribution(conf); int nobs = obstable_->nobs(); @@ -184,24 +172,17 @@ void ObsTableView::generateDistribution(const eckit::Configuration & conf) { // ----------------------------------------------------------------------------- -std::unique_ptr ObsTableView::locations(const util::DateTime & t1, - const util::DateTime & t2) const { +std::unique_ptr ObsTableView::locations() const { // get times and locations from the obsspace std::vector all_times = obstable_->times(); std::vector all_locs = obstable_->locations(); - // find local times that are within t1 and t2 - std::vector mask; - for (unsigned int i = 0; i < nobs(); i++) { - if (all_times[localobs_[i]] > t1 && all_times[localobs_[i]] <= t2) - mask.push_back(i); - } // set up locations - const unsigned int nobs_t = mask.size(); - std::vector locs(nobs_t); - std::vector times(nobs_t); - for (unsigned int i = 0; i < nobs_t; i++) { - locs[i] = all_locs[localobs_[mask[i]]]; - times[i] = all_times[localobs_[mask[i]]]; + const unsigned int nobs = localobs_.size(); + std::vector locs(nobs); + std::vector times(nobs); + for (unsigned int i = 0; i < nobs; i++) { + locs[i] = all_locs[localobs_[i]]; + times[i] = all_times[localobs_[i]]; } oops::Log::trace() << "ObsTableView::locations done" << std::endl; return std::unique_ptr(new LocsL95(locs, times)); diff --git a/l95/src/lorenz95/ObsTableView.h b/l95/src/lorenz95/ObsTableView.h index 64a622e5d..ec67221eb 100644 --- a/l95/src/lorenz95/ObsTableView.h +++ b/l95/src/lorenz95/ObsTableView.h @@ -55,9 +55,8 @@ class ObsTableView : public util::Printable, void random(std::vector &) const; unsigned int nobs() const; - std::vector locations() const; void generateDistribution(const eckit::Configuration &); - std::unique_ptr locations(const util::DateTime & t1, const util::DateTime & t2) const; + std::unique_ptr locations() const; void printJo(const ObsVec1D &, const ObsVec1D &); size_t index(const size_t ii) const {return localobs_[ii];} diff --git a/l95/src/lorenz95/ObservationL95.cc b/l95/src/lorenz95/ObservationL95.cc index 7c989a10d..230eeb82c 100644 --- a/l95/src/lorenz95/ObservationL95.cc +++ b/l95/src/lorenz95/ObservationL95.cc @@ -19,7 +19,6 @@ #include "lorenz95/ObsDiags1D.h" #include "lorenz95/ObsVec1D.h" #include "oops/base/Variables.h" -#include "oops/util/DateTime.h" #include "oops/util/Logger.h" // ----------------------------------------------------------------------------- @@ -45,9 +44,8 @@ void ObservationL95::simulateObs(const GomL95 & gom, ObsVec1D & ovec, // ----------------------------------------------------------------------------- -std::unique_ptr ObservationL95::locations(const util::DateTime & t1, - const util::DateTime & t2) const { - return obsdb_.locations(t1, t2); +std::unique_ptr ObservationL95::locations() const { + return obsdb_.locations(); } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObservationL95.h b/l95/src/lorenz95/ObservationL95.h index 301e56bf0..45148bfcf 100644 --- a/l95/src/lorenz95/ObservationL95.h +++ b/l95/src/lorenz95/ObservationL95.h @@ -29,10 +29,6 @@ namespace eckit { class Configuration; } -namespace util { - class DateTime; -} - namespace lorenz95 { class GomL95; class LocsL95; @@ -61,7 +57,7 @@ class ObservationL95 : public util::Printable, // Other const oops::Variables & requiredVars() const {return inputs_;} - std::unique_ptr locations(const util::DateTime &, const util::DateTime &) const; + std::unique_ptr locations() const; const ObsTableView & table() const {return obsdb_;} diff --git a/qg/model/ObsOpBaseQG.h b/qg/model/ObsOpBaseQG.h index d64855c29..62c2fab34 100644 --- a/qg/model/ObsOpBaseQG.h +++ b/qg/model/ObsOpBaseQG.h @@ -18,7 +18,6 @@ #include "oops/base/Variables.h" #include "oops/util/abor1_cpp.h" -#include "oops/util/DateTime.h" #include "oops/util/Printable.h" #include "oops/qg/ObsSpaceQG.h" @@ -42,8 +41,7 @@ class ObsOpBaseQG : public util::Printable, /// Other virtual const oops::Variables & requiredVars() const = 0; // Required from Model - virtual std::unique_ptr locations(const util::DateTime &, - const util::DateTime &) const = 0; + virtual std::unique_ptr locations() const = 0; private: virtual void print(std::ostream &) const = 0; diff --git a/qg/model/ObsOperatorQG.cc b/qg/model/ObsOperatorQG.cc index 333ef8df3..3d14a46ee 100644 --- a/qg/model/ObsOperatorQG.cc +++ b/qg/model/ObsOperatorQG.cc @@ -19,7 +19,6 @@ #include "model/ObsSpaceQG.h" #include "model/ObsVecQG.h" #include "oops/base/Variables.h" -#include "oops/util/DateTime.h" namespace qg { @@ -48,9 +47,8 @@ const oops::Variables & ObsOperatorQG::requiredVars() const { // ----------------------------------------------------------------------------- -std::unique_ptr ObsOperatorQG::locations(const util::DateTime & t1, - const util::DateTime & t2) const { - return oper_->locations(t1, t2); +std::unique_ptr ObsOperatorQG::locations() const { + return oper_->locations(); } // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsOperatorQG.h b/qg/model/ObsOperatorQG.h index b7f8f8503..d03cb0275 100644 --- a/qg/model/ObsOperatorQG.h +++ b/qg/model/ObsOperatorQG.h @@ -24,10 +24,6 @@ namespace eckit { class Configuration; } -namespace util { - class DateTime; -} - namespace qg { class GomQG; class LocationsQG; @@ -50,7 +46,7 @@ class ObsOperatorQG : public util::Printable, /// Other const oops::Variables & requiredVars() const; // Required input requiredVars from Model - std::unique_ptr locations(const util::DateTime &, const util::DateTime &) const; + std::unique_ptr locations() const; private: void print(std::ostream &) const; diff --git a/qg/model/ObsSpaceQG.cc b/qg/model/ObsSpaceQG.cc index d425f0990..b9a1bbae3 100644 --- a/qg/model/ObsSpaceQG.cc +++ b/qg/model/ObsSpaceQG.cc @@ -123,7 +123,7 @@ ObsSpaceQG::ObsSpaceQG(const ObsSpaceQG & obsdb, const double dist = conf.getDouble("lengthscale"); // get locations of all obs - std::unique_ptr locs = locations(winbgn_, winend_); + std::unique_ptr locs = locations(); atlas::Field field_lonlat = locs->lonlat(); auto lonlat = make_view(field_lonlat); @@ -179,12 +179,10 @@ bool ObsSpaceQG::has(const std::string & col) const { // ----------------------------------------------------------------------------- -std::unique_ptr ObsSpaceQG::locations(const util::DateTime & t1, - const util::DateTime & t2) const { +std::unique_ptr ObsSpaceQG::locations() const { atlas::FieldSet fields; std::vector times; - qg_obsdb_locations_f90(key_, obsname_.size(), obsname_.c_str(), t1, t2, - fields.get(), times); + qg_obsdb_locations_f90(key_, obsname_.size(), obsname_.c_str(), fields.get(), times); return std::unique_ptr(new LocationsQG(fields, std::move(times))); } diff --git a/qg/model/ObsSpaceQG.h b/qg/model/ObsSpaceQG.h index 79ee3796e..d0efb42d2 100644 --- a/qg/model/ObsSpaceQG.h +++ b/qg/model/ObsSpaceQG.h @@ -59,9 +59,8 @@ class ObsSpaceQG : public oops::ObsSpaceBase { /// check if variable is in ObsSpace bool has(const std::string & col) const; - /// create locations between times (\p t1, \p t2] - std::unique_ptr locations(const util::DateTime & t1, - const util::DateTime & t2) const; + /// create locations for the whole time window + std::unique_ptr locations() const; void printJo(const ObsVecQG &, const ObsVecQG &) const; diff --git a/qg/model/ObsStreamQG.cc b/qg/model/ObsStreamQG.cc index 7704f65b1..545bdd4e0 100644 --- a/qg/model/ObsStreamQG.cc +++ b/qg/model/ObsStreamQG.cc @@ -41,9 +41,8 @@ void ObsStreamQG::simulateObs(const GomQG & gom, ObsVecQG & ovec, // ----------------------------------------------------------------------------- -std::unique_ptr ObsStreamQG::locations(const util::DateTime & t1, - const util::DateTime & t2) const { - return obsdb_.locations(t1, t2); +std::unique_ptr ObsStreamQG::locations() const { + return obsdb_.locations(); } // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsStreamQG.h b/qg/model/ObsStreamQG.h index 4ae32707a..f34ef6fc8 100644 --- a/qg/model/ObsStreamQG.h +++ b/qg/model/ObsStreamQG.h @@ -47,8 +47,7 @@ class ObsStreamQG : public ObsOpBaseQG, // Other const oops::Variables & requiredVars() const override {return varin_;} - std::unique_ptr locations(const util::DateTime &, - const util::DateTime &) const override; + std::unique_ptr locations() const override; private: void print(std::ostream &) const override; diff --git a/qg/model/ObsWSpeedQG.cc b/qg/model/ObsWSpeedQG.cc index ed8d8cb30..c73ad1938 100644 --- a/qg/model/ObsWSpeedQG.cc +++ b/qg/model/ObsWSpeedQG.cc @@ -42,9 +42,8 @@ void ObsWSpeedQG::simulateObs(const GomQG & gom, ObsVecQG & ovec, // ----------------------------------------------------------------------------- -std::unique_ptr ObsWSpeedQG::locations(const util::DateTime & t1, - const util::DateTime & t2) const { - return obsdb_.locations(t1, t2); +std::unique_ptr ObsWSpeedQG::locations() const { + return obsdb_.locations(); } // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsWSpeedQG.h b/qg/model/ObsWSpeedQG.h index 90589c4a9..96400519e 100644 --- a/qg/model/ObsWSpeedQG.h +++ b/qg/model/ObsWSpeedQG.h @@ -48,8 +48,7 @@ class ObsWSpeedQG : public ObsOpBaseQG, // Other const oops::Variables & requiredVars() const override {return varin_;} - std::unique_ptr locations(const util::DateTime &, - const util::DateTime &) const override; + std::unique_ptr locations() const override; private: void print(std::ostream &) const override; diff --git a/qg/model/ObsWindQG.cc b/qg/model/ObsWindQG.cc index ff2ad3c07..830e1b906 100644 --- a/qg/model/ObsWindQG.cc +++ b/qg/model/ObsWindQG.cc @@ -42,9 +42,8 @@ void ObsWindQG::simulateObs(const GomQG & gom, ObsVecQG & ovec, // ----------------------------------------------------------------------------- -std::unique_ptr ObsWindQG::locations(const util::DateTime & t1, - const util::DateTime & t2) const { - return obsdb_.locations(t1, t2); +std::unique_ptr ObsWindQG::locations() const { + return obsdb_.locations(); } // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsWindQG.h b/qg/model/ObsWindQG.h index bf7fafe57..19e6f7bff 100644 --- a/qg/model/ObsWindQG.h +++ b/qg/model/ObsWindQG.h @@ -48,8 +48,7 @@ class ObsWindQG : public ObsOpBaseQG, // Other const oops::Variables & requiredVars() const override {return varin_;} - std::unique_ptr locations(const util::DateTime &, - const util::DateTime &) const override; + std::unique_ptr locations() const override; private: void print(std::ostream &) const override; diff --git a/qg/model/QgFortran.h b/qg/model/QgFortran.h index 62258486f..4dcd5daee 100644 --- a/qg/model/QgFortran.h +++ b/qg/model/QgFortran.h @@ -211,7 +211,6 @@ extern "C" { void qg_obsdb_has_f90(const F90odb &, const int &, const char *, const int &, const char *, int &); void qg_obsdb_locations_f90(const F90odb &, const int &, const char *, - const util::DateTime &, const util::DateTime &, atlas::field::FieldSetImpl *, std::vector &); void qg_obsdb_generate_f90(const F90odb &, const int &, const char *, const eckit::Configuration &, const util::DateTime &, diff --git a/qg/model/qg_obsdb_interface.F90 b/qg/model/qg_obsdb_interface.F90 index 6a0908c5a..5afec8f69 100644 --- a/qg/model/qg_obsdb_interface.F90 +++ b/qg/model/qg_obsdb_interface.F90 @@ -198,7 +198,7 @@ subroutine qg_obsdb_has_c(c_key_self,lgrp,c_grp,lcol,c_col,c_has) bind(c,name='q end subroutine qg_obsdb_has_c ! ------------------------------------------------------------------------------ !> Get locations from observation data -subroutine qg_obsdb_locations_c(c_key_self,lgrp,c_grp,c_t1,c_t2,c_fields,c_times) bind(c,name='qg_obsdb_locations_f90') +subroutine qg_obsdb_locations_c(c_key_self,lgrp,c_grp,c_fields,c_times) bind(c,name='qg_obsdb_locations_f90') implicit none @@ -206,26 +206,21 @@ subroutine qg_obsdb_locations_c(c_key_self,lgrp,c_grp,c_t1,c_t2,c_fields,c_times integer(c_int),intent(in) :: c_key_self !< Observation data integer(c_int),intent(in) :: lgrp !< Group size character(kind=c_char,len=1),intent(in) :: c_grp(lgrp+1) !< Group name -type(c_ptr),value,intent(in) :: c_t1 !< Time 1 -type(c_ptr),value,intent(in) :: c_t2 !< Time 2 type(c_ptr), intent(in), value :: c_fields !< Locations fieldset type(c_ptr), intent(in), value :: c_times !< times ! Local variables type(qg_obsdb),pointer :: self character(len=lgrp) :: grp -type(datetime) :: t1,t2 type(atlas_fieldset) :: fields ! Interface call qg_obsdb_registry%get(c_key_self,self) call c_f_string(c_grp,grp) -call c_f_datetime(c_t1,t1) -call c_f_datetime(c_t2,t2) fields = atlas_fieldset(c_fields) ! Call Fortran -call qg_obsdb_locations(self,grp,t1,t2,fields,c_times) +call qg_obsdb_locations(self,grp,fields,c_times) call fields%final() diff --git a/qg/model/qg_obsdb_mod.F90 b/qg/model/qg_obsdb_mod.F90 index 01a0688f0..9804a1f73 100644 --- a/qg/model/qg_obsdb_mod.F90 +++ b/qg/model/qg_obsdb_mod.F90 @@ -304,35 +304,28 @@ subroutine qg_obsdb_has(self,grp,col,has) end subroutine qg_obsdb_has ! ------------------------------------------------------------------------------ !> Get locations from observation data -subroutine qg_obsdb_locations(self,grp,t1,t2,fields,c_times) +subroutine qg_obsdb_locations(self,grp,fields,c_times) implicit none ! Passed variables type(qg_obsdb),intent(in) :: self !< Observation data character(len=*),intent(in) :: grp !< Group -type(datetime),intent(in) :: t1 !< Time 1 -type(datetime),intent(in) :: t2 !< Time 2 type(atlas_fieldset), intent(inout) :: fields !< Locations FieldSet type(c_ptr), intent(in), value :: c_times !< pointer to times array in C++ ! Local variables integer :: nlocs, jo -integer,allocatable :: indx(:) character(len=8),parameter :: col = 'Location' type(group_data),pointer :: jgrp type(column_data),pointer :: jcol type(atlas_field) :: field_z, field_lonlat real(kind_real), pointer :: z(:), lonlat(:,:) -! Count observations -call qg_obsdb_count_time(self,grp,t1,t2,nlocs) -allocate(indx(nlocs)) -call qg_obsdb_count_indx(self,grp,t1,t2,indx) - ! Find observation group call qg_obsdb_find_group(self,grp,jgrp) if (.not.associated(jgrp)) call abor1_ftn('qg_obsdb_locations: obs group not found') +nlocs = jgrp%nobs ! Find observation column call qg_obsdb_find_column(jgrp,col,jcol) @@ -348,10 +341,10 @@ subroutine qg_obsdb_locations(self,grp,t1,t2,fields,c_times) ! Copy coordinates do jo = 1, nlocs - lonlat(1,jo) = jcol%values(1,indx(jo)) - lonlat(2,jo) = jcol%values(2,indx(jo)) - z(jo) = jcol%values(3,indx(jo)) - call f_c_push_to_datetime_vector(c_times, jgrp%times(indx(jo))) + lonlat(1,jo) = jcol%values(1,jo) + lonlat(2,jo) = jcol%values(2,jo) + z(jo) = jcol%values(3,jo) + call f_c_push_to_datetime_vector(c_times, jgrp%times(jo)) enddo call fields%add(field_lonlat) @@ -361,8 +354,6 @@ subroutine qg_obsdb_locations(self,grp,t1,t2,fields,c_times) call field_lonlat%final() call field_z%final() -deallocate(indx) - end subroutine qg_obsdb_locations ! ------------------------------------------------------------------------------ !> Generate observation data @@ -820,63 +811,4 @@ subroutine qg_obsdb_create(self,grp,times,locs) end subroutine qg_obsdb_create ! ------------------------------------------------------------------------------ -!> Get observation data size between times -subroutine qg_obsdb_count_time(self,grp,t1,t2,kobs) - -implicit none - -! Passed variables -type(qg_obsdb),intent(in) :: self !< Observation data -character(len=*),intent(in) :: grp !< Group -type(datetime),intent(in) :: t1 !< Time 1 -type(datetime),intent(in) :: t2 !< Time 2 -integer,intent(inout) :: kobs !< Number of observations - -! Local variables -type(group_data),pointer :: jgrp -integer :: jobs - -! Find observation group -call qg_obsdb_find_group(self,grp,jgrp) -if (.not.associated(jgrp)) call abor1_ftn('qg_obsdb_count_time: obs group not found') - -! Time selection -kobs = 0 -do jobs=1,jgrp%nobs - if ((t1 Get observation data index -subroutine qg_obsdb_count_indx(self,grp,t1,t2,indx) - -implicit none - -! Passed variables -type(qg_obsdb),intent(in) :: self !< Observation data -character(len=*),intent(in) :: grp !< Group -type(datetime),intent(in) :: t1 !< Time 1 -type(datetime),intent(in) :: t2 !< Time 2 -integer,intent(inout) :: indx(:) !< Observation index - -! Local variables -type(group_data),pointer :: jgrp -integer :: jobs,io - -! Find observation group -call qg_obsdb_find_group(self,grp,jgrp) -if (.not.associated(jgrp)) call abor1_ftn('qg_obsdb_count_indx: obs group not found') - -! Time selection -io = 0 -do jobs=1,jgrp%nobs - if ((t1 class Observer : public util::Printable { typedef GeoVaLs GeoVaLs_; + typedef Locations Locations_; typedef ObsDiagnostics ObsDiags_; typedef ObsSpace ObsSpace_; typedef GetValues GetValues_; @@ -71,6 +73,7 @@ class Observer : public util::Printable { ObsFilters_ filters_; Variables geovars_; // Variables needed from model (through geovals) + Locations_ locs_; std::unique_ptr getvals_; std::shared_ptr gvals_; }; @@ -82,7 +85,8 @@ Observer::Observer(const eckit::Configuration & conf, const ObsSpace const ObsAuxCtrl_ & ybias, ObsVector_ & yobs, ObsDataPtr_ qcflags, ObsDataPtr_ obserr) : hop_(obsdb, eckit::LocalConfiguration(conf, "obs operator")), - obsdb_(obsdb), yobs_(yobs), ybias_(ybias), filters_(obsdb, conf, qcflags, obserr) + obsdb_(obsdb), yobs_(yobs), ybias_(ybias), filters_(obsdb, conf, qcflags, obserr), + locs_(hop_.locations()) { Log::trace() << "Observer::Observer starting" << std::endl; geovars_ += hop_.requiredVars(); @@ -108,8 +112,8 @@ void Observer::doInitialize(const State_ & xx, const util::DateTime & end) { Log::trace() << "Observer::doInitialize start" << std::endl; filters_.preProcess(); - getvals_.reset(new GetValues_(xx.geometry(), hop_.locations(begin, end))); - gvals_.reset(new GeoVaLs_(hop_.locations(begin, end), geovars_)); + getvals_.reset(new GetValues_(xx.geometry(), locs_)); + gvals_.reset(new GeoVaLs_(locs_, geovars_)); Log::trace() << "Observer::doInitialize done" << std::endl; } @@ -134,7 +138,7 @@ void Observer::doFinalize() { oops::Variables vars; vars += filters_.requiredHdiagnostics(); vars += ybias_.requiredHdiagnostics(); - ObsDiags_ ydiags(obsdb_, hop_.locations(obsdb_.windowStart(), obsdb_.windowEnd()), vars); + ObsDiags_ ydiags(obsdb_, locs_, vars); hop_.simulateObs(*gvals_, yobs_, ybias_, ydiags); filters_.postFilter(yobs_, ydiags); Log::trace() << "Observer::doFinalize done" << std::endl; diff --git a/src/oops/base/ObserverTLAD.h b/src/oops/base/ObserverTLAD.h index c970c57c7..f0796d1da 100644 --- a/src/oops/base/ObserverTLAD.h +++ b/src/oops/base/ObserverTLAD.h @@ -19,6 +19,7 @@ #include "oops/interface/Increment.h" #include "oops/interface/LinearGetValues.h" #include "oops/interface/LinearObsOperator.h" +#include "oops/interface/Locations.h" #include "oops/interface/ObsAuxControl.h" #include "oops/interface/ObsAuxIncrement.h" #include "oops/interface/ObsDiagnostics.h" @@ -39,6 +40,7 @@ class ObserverTLAD { typedef Increment Increment_; typedef LinearGetValues LinearGetValues_; typedef LinearObsOperator LinearObsOperator_; + typedef Locations Locations_; typedef ObsAuxControl ObsAuxCtrl_; typedef ObsAuxIncrement ObsAuxIncr_; typedef ObsDiagnostics ObsDiags_; @@ -73,6 +75,7 @@ class ObserverTLAD { const ObsAuxCtrl_ & ybias_; Variables geovars_; + Locations_ locs_; std::unique_ptr lingetvals_; std::shared_ptr gvals_; @@ -85,7 +88,7 @@ ObserverTLAD::ObserverTLAD(const eckit::Configuration & config, const ObsAuxCtrl_ & ybias) : obsdb_(obsdb), hop_(obsdb, eckit::LocalConfiguration(config, "obs operator")), hoptlad_(obsdb, eckit::LocalConfiguration(config, "linear obs operator")), - ybias_(ybias), geovars_(), lingetvals_(), gvals_() + ybias_(ybias), geovars_(), locs_(hop_.locations()), lingetvals_(), gvals_() { geovars_ += hop_.requiredVars(); geovars_ += ybias_.requiredVars(); @@ -97,8 +100,8 @@ void ObserverTLAD::doInitializeTraj(const State_ & xx, const util::DateTime & winbgn, const util::DateTime & winend) { Log::trace() << "ObserverTLAD::doInitializeTraj start" << std::endl; - lingetvals_.reset(new LinearGetValues_(xx.geometry(), hop_.locations(winbgn, winend))); - gvals_.reset(new GeoVaLs_(hop_.locations(winbgn, winend), geovars_)); + lingetvals_.reset(new LinearGetValues_(xx.geometry(), locs_)); + gvals_.reset(new GeoVaLs_(locs_, geovars_)); Log::trace() << "ObserverTLAD::doInitializeTraj done" << std::endl; } // ----------------------------------------------------------------------------- @@ -124,7 +127,7 @@ void ObserverTLAD::doInitializeTL(const Increment_ & dx, const util::DateTime & winbgn, const util::DateTime & winend) { Log::trace() << "ObserverTLAD::doInitializeTL start" << std::endl; - gvals_.reset(new GeoVaLs_(hop_.locations(winbgn, winend), hoptlad_.requiredVars())); + gvals_.reset(new GeoVaLs_(locs_, hoptlad_.requiredVars())); Log::trace() << "ObserverTLAD::doInitializeTL done" << std::endl; } // ----------------------------------------------------------------------------- @@ -152,7 +155,7 @@ void ObserverTLAD::doFirstAD(Increment_ & dx, const ObsVector_ & yde const util::DateTime & winbgn, const util::DateTime & winend) { Log::trace() << "ObserverTLAD::doFirstAD start" << std::endl; - gvals_.reset(new GeoVaLs_(hop_.locations(winbgn, winend), hoptlad_.requiredVars())); + gvals_.reset(new GeoVaLs_(locs_, hoptlad_.requiredVars())); hoptlad_.simulateObsAD(*gvals_, ydepad, ybiasad); Log::trace() << "ObserverTLAD::doFirstAD done" << std::endl; } diff --git a/src/oops/interface/ObsOperator.h b/src/oops/interface/ObsOperator.h index 2c2197348..86611aa95 100644 --- a/src/oops/interface/ObsOperator.h +++ b/src/oops/interface/ObsOperator.h @@ -22,7 +22,6 @@ #include "oops/interface/ObsDiagnostics.h" #include "oops/interface/ObsSpace.h" #include "oops/interface/ObsVector.h" -#include "oops/util/DateTime.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" @@ -58,7 +57,7 @@ class ObsOperator : public util::Printable, /// Other const Variables & requiredVars() const; // Required input variables from Model - Locations_ locations(const util::DateTime &, const util::DateTime &) const; + Locations_ locations() const; private: void print(std::ostream &) const; @@ -109,11 +108,10 @@ const Variables & ObsOperator::requiredVars() const { // ----------------------------------------------------------------------------- template -Locations ObsOperator::locations(const util::DateTime & t1, - const util::DateTime & t2) const { +Locations ObsOperator::locations() const { Log::trace() << "ObsOperator::locations starting" << std::endl; util::Timer timer(classname(), "locations"); - return Locations_(oper_->locations(t1, t2)); + return Locations_(oper_->locations()); } // ----------------------------------------------------------------------------- diff --git a/src/test/interface/LinearObsOperator.h b/src/test/interface/LinearObsOperator.h index 515d00d82..bb871c92f 100644 --- a/src/test/interface/LinearObsOperator.h +++ b/src/test/interface/LinearObsOperator.h @@ -269,10 +269,7 @@ template void testTangentLinear() { // create obsdatavector to hold diags oops::Variables diagvars; diagvars += ybias0.requiredHdiagnostics(); - ObsDiags_ ydiag(Test_::obspace()[jj], - hop.locations(Test_::obspace()[jj].windowStart(), - Test_::obspace()[jj].windowEnd()), - diagvars); + ObsDiags_ ydiag(Test_::obspace()[jj], hop.locations(), diagvars); // y1 = hop(x0, ybias0) hop.simulateObs(x0, y1, ybias0, ydiag); diff --git a/src/test/interface/ObsOperator.h b/src/test/interface/ObsOperator.h index de68e53d8..ac3615ab8 100644 --- a/src/test/interface/ObsOperator.h +++ b/src/test/interface/ObsOperator.h @@ -84,10 +84,7 @@ template void testSimulateObs() { // create diagnostics to hold HofX diags oops::Variables diagvars; diagvars += ybias.requiredHdiagnostics(); - ObsDiags_ diags(Test_::obspace()[jj], - hop.locations(Test_::obspace()[jj].windowStart(), - Test_::obspace()[jj].windowEnd()), - diagvars); + ObsDiags_ diags(Test_::obspace()[jj], hop.locations(), diagvars); // call H(x), save result in the output file as @hofx hop.simulateObs(gval, hofx, ybias, diags); From 5a1c5cbdb05a85bda6c842dbca48eb5dc0de7234 Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Thu, 5 Nov 2020 08:25:08 -0700 Subject: [PATCH 007/142] switch to jcsda-internal (#958) * switch to jcsda-internal * update webhook_branchname * bugfix * bugfix --- CI/CMakeLists.txt | 6 ++-- CI/buildspec_gnu.yml | 2 +- CI/buildspec_intel.yml | 2 +- CI/update_webhook_branchname.py | 52 ++++++++++++++++----------------- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/CI/CMakeLists.txt b/CI/CMakeLists.txt index 80f20190d..63e5e718d 100644 --- a/CI/CMakeLists.txt +++ b/CI/CMakeLists.txt @@ -25,8 +25,8 @@ ecbuild_add_option( FEATURE OMP DEFAULT ON DESCRIPTION "Use OpenMP" ) -ecbuild_bundle( PROJECT fckit GIT "https://github.com/JCSDA/fckit.git" ) -ecbuild_bundle( PROJECT atlas GIT "https://github.com/JCSDA/atlas.git" ) -ecbuild_bundle( PROJECT oops GIT "https://github.com/JCSDA/oops.git" ) +ecbuild_bundle( PROJECT fckit GIT "https://github.com/JCSDA-internal/fckit.git" ) +ecbuild_bundle( PROJECT atlas GIT "https://github.com/JCSDA-internal/atlas.git" ) +ecbuild_bundle( PROJECT oops GIT "https://github.com/JCSDA-internal/oops.git" ) ecbuild_bundle_finalize() diff --git a/CI/buildspec_gnu.yml b/CI/buildspec_gnu.yml index 27397dd78..1a6ced445 100644 --- a/CI/buildspec_gnu.yml +++ b/CI/buildspec_gnu.yml @@ -76,7 +76,7 @@ phases: fi # oops - - ./clone.sh $GIT_USER $GIT_PASS jcsda/oops $CODEBUILD_GIT_BRANCH oops /jcsda/oops-bundle develop + - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/oops $CODEBUILD_GIT_BRANCH oops /jcsda/oops-bundle develop # fckit - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/fckit $CODEBUILD_GIT_BRANCH_FORK fckit /jcsda/oops-bundle release-stable # atlas diff --git a/CI/buildspec_intel.yml b/CI/buildspec_intel.yml index 606a86d28..7e09a3101 100644 --- a/CI/buildspec_intel.yml +++ b/CI/buildspec_intel.yml @@ -92,7 +92,7 @@ phases: fi # oops - - ./clone.sh $GIT_USER $GIT_PASS jcsda/oops $CODEBUILD_GIT_BRANCH oops /jcsda/oops-bundle develop + - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/oops $CODEBUILD_GIT_BRANCH oops /jcsda/oops-bundle develop # fckit - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/fckit $CODEBUILD_GIT_BRANCH_FORK fckit /jcsda/oops-bundle release-stable # atlas diff --git a/CI/update_webhook_branchname.py b/CI/update_webhook_branchname.py index f79e7a2e2..201c84201 100644 --- a/CI/update_webhook_branchname.py +++ b/CI/update_webhook_branchname.py @@ -7,9 +7,8 @@ """ Codebuild Runs this script to check for branches with -same names accross downstream repos. Then creates a +same names accross dependent repos. Then creates a webhook on PR page to notify users if such a branch exists. - Call as: update_webhook_branchname.py branchName commitId """ @@ -24,30 +23,31 @@ # check other repos -repoList = ["ufo", "saber", "ioda", "soca", "fv3-jedi", "mpas-jedi", "shallow-water"] +repoList = ["atlas", "saber", "ioda", "ufo", "soca", "fv3-jedi", "mpas-jedi"] token = os.getenv('GIT_PASS', '...') g = Github(token) -owner = "JCSDA" - -oopsRepo = g.get_repo("jcsda/oops") - -for repoName in repoList: - branchExists = True - repo = g.get_repo(f"{owner}/{repoName}") - branchList = list(repo.get_branches()) - try: - repo.get_branch(branchName) - except Exception: - pass - branchExists = False - - if (branchExists): - print(branchName + ' exists in '+repoName) - stageDescription= branchName+" exists in "+ repoName - commitStatus = oopsRepo.get_commit(sha=commitId).create_status( - state="pending", - description=stageDescription, - context="Branch Check-"+repoName) - else: - print(branchName + ' does not exist in '+repoName) +ownerList = ["JCSDA", "jcsda-internal"] + +oopsRepo = g.get_repo("jcsda-internal/oops") + +for owner in ownerList: + for repoName in repoList: + branchExists = True + repo = g.get_repo(f"{owner}/{repoName}") + branchList = list(repo.get_branches()) + try: + repo.get_branch(branchName) + except Exception: + pass + branchExists = False + + if (branchExists): + print(branchName + ' exists in '+owner+'/'+repoName) + stageDescription= branchName+" exists in "+owner+'/'+ repoName + commitStatus = oopsRepo.get_commit(sha=commitId).create_status( + state="pending", + description=stageDescription, + context="Branch Check-"+owner+'/'+repoName) + else: + print(branchName + ' does not exist in '+owner+'/'+repoName) From dca01da87fdb7bc9ccc2d9fe04c6d3d31658447f Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Thu, 5 Nov 2020 09:37:14 -0700 Subject: [PATCH 008/142] Pass "observations" config section to obs-related classes (#951) * pass observations config section where appropriate * pass iteration (variational config) explicitly to ObsFilters * pass iteration index as int to ObsFilters --- l95/test/testinput/eda.3dfgat.1.yaml | 2 +- l95/test/testinput/eda.3dfgat.2.yaml | 2 +- l95/test/testinput/eda.3dfgat.3.yaml | 2 +- l95/test/testinput/eda.3dfgat.4.yaml | 2 +- l95/test/testinput/eda.3dvar.1.yaml | 2 +- l95/test/testinput/eda.3dvar.2.yaml | 2 +- l95/test/testinput/eda.3dvar.3.yaml | 2 +- l95/test/testinput/eda.3dvar.4.yaml | 2 +- l95/test/testinput/eda.4dvar.3.yaml | 2 +- qg/test/testinput/eda_3dfgat_1.yaml | 4 +- qg/test/testinput/eda_3dfgat_2.yaml | 4 +- qg/test/testinput/eda_3dfgat_3.yaml | 4 +- qg/test/testinput/eda_3dfgat_4.yaml | 4 +- qg/test/testinput/eda_3dvar_2.yaml | 4 +- qg/test/testinput/eda_4dvar_1.yaml | 4 +- qg/test/testinput/eda_4dvar_2.yaml | 4 +- qg/test/testinput/eda_4dvar_3.yaml | 4 +- qg/test/testinput/eda_4dvar_4.yaml | 4 +- src/oops/assimilation/CalcHofX.h | 2 +- src/oops/assimilation/ControlVariable.h | 4 +- src/oops/assimilation/CostFunction.h | 6 +- src/oops/assimilation/CostJbTotal.h | 2 +- src/oops/assimilation/CostJo.h | 5 +- src/oops/assimilation/LocalEnsembleSolver.h | 2 +- src/oops/base/ObsAuxControls.h | 3 +- src/oops/base/ObsAuxCovariances.h | 5 +- src/oops/base/ObsAuxIncrements.h | 3 +- src/oops/base/ObsErrors.h | 5 +- src/oops/base/ObsFilters.h | 51 ++++++--------- src/oops/base/ObsSpaces.h | 8 +-- src/oops/base/Observer.h | 9 +-- src/oops/base/Observers.h | 12 ++-- src/oops/base/ObserversTLAD.h | 3 +- src/oops/runs/HofX.h | 5 +- src/oops/runs/HofXNoModel.h | 3 +- src/oops/runs/LocalEnsembleDA.h | 3 +- src/test/interface/GeoVaLs.h | 48 +++----------- src/test/interface/LinearObsOperator.h | 72 ++++++++++----------- src/test/interface/ObsAuxControl.h | 8 +-- src/test/interface/ObsAuxCovariance.h | 4 +- src/test/interface/ObsAuxIncrement.h | 51 +++++++-------- src/test/interface/ObsOperator.h | 29 ++++----- src/test/interface/ObsTestsFixture.h | 19 ++++-- 43 files changed, 185 insertions(+), 231 deletions(-) diff --git a/l95/test/testinput/eda.3dfgat.1.yaml b/l95/test/testinput/eda.3dfgat.1.yaml index 06d26d65e..3ed0a63ea 100644 --- a/l95/test/testinput/eda.3dfgat.1.yaml +++ b/l95/test/testinput/eda.3dfgat.1.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 1 cost type: 4D-Var window begin: 2010-01-01T21:00:00Z window length: PT6H @@ -25,6 +24,7 @@ cost function: obs space: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obsdataout: Data/mem001.eda.3dfgat.2010-01-02T00:00:00Z.obt + obs perturbations seed: 1 obs operator: {} variational: minimizer: diff --git a/l95/test/testinput/eda.3dfgat.2.yaml b/l95/test/testinput/eda.3dfgat.2.yaml index b0d2329b0..cb80db463 100644 --- a/l95/test/testinput/eda.3dfgat.2.yaml +++ b/l95/test/testinput/eda.3dfgat.2.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 2 cost type: 4D-Var window begin: 2010-01-01T21:00:00Z window length: PT6H @@ -25,6 +24,7 @@ cost function: obs space: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obsdataout: Data/mem002.eda.3dfgat.2010-01-02T00:00:00Z.obt + obs perturbations seed: 2 obs operator: {} variational: minimizer: diff --git a/l95/test/testinput/eda.3dfgat.3.yaml b/l95/test/testinput/eda.3dfgat.3.yaml index b43997590..d8a3db094 100644 --- a/l95/test/testinput/eda.3dfgat.3.yaml +++ b/l95/test/testinput/eda.3dfgat.3.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 3 cost type: 4D-Var window begin: 2010-01-01T21:00:00Z window length: PT6H @@ -25,6 +24,7 @@ cost function: obs space: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obsdataout: Data/mem003.eda.3dfgat.2010-01-02T00:00:00Z.obt + obs perturbations seed: 3 obs operator: {} variational: minimizer: diff --git a/l95/test/testinput/eda.3dfgat.4.yaml b/l95/test/testinput/eda.3dfgat.4.yaml index bc38d6b16..1262bbe8d 100644 --- a/l95/test/testinput/eda.3dfgat.4.yaml +++ b/l95/test/testinput/eda.3dfgat.4.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 4 cost type: 4D-Var window begin: 2010-01-01T21:00:00Z window length: PT6H @@ -25,6 +24,7 @@ cost function: obs space: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obsdataout: Data/mem004.eda.3dfgat.2010-01-02T00:00:00Z.obt + obs perturbations seed: 4 obs operator: {} variational: minimizer: diff --git a/l95/test/testinput/eda.3dvar.1.yaml b/l95/test/testinput/eda.3dvar.1.yaml index f6d25a811..d7354b7a7 100644 --- a/l95/test/testinput/eda.3dvar.1.yaml +++ b/l95/test/testinput/eda.3dvar.1.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 1 cost type: 3D-Var window begin: 2010-01-01T21:00:00Z window length: PT6H @@ -21,6 +20,7 @@ cost function: obs space: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obsdataout: Data/mem001.eda.3dvar.2010-01-02T00:00:00Z.obt + obs perturbations seed: 1 obs operator: {} variational: minimizer: diff --git a/l95/test/testinput/eda.3dvar.2.yaml b/l95/test/testinput/eda.3dvar.2.yaml index bc6d08c4a..bfa6ad04b 100644 --- a/l95/test/testinput/eda.3dvar.2.yaml +++ b/l95/test/testinput/eda.3dvar.2.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 2 cost type: 3D-Var window begin: 2010-01-01T21:00:00Z window length: PT6H @@ -21,6 +20,7 @@ cost function: obs space: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obsdataout: Data/mem002.eda.3dvar.2010-01-02T00:00:00Z.obt + obs perturbations seed: 2 obs operator: {} variational: minimizer: diff --git a/l95/test/testinput/eda.3dvar.3.yaml b/l95/test/testinput/eda.3dvar.3.yaml index 1a5a99a70..5e1281a1d 100644 --- a/l95/test/testinput/eda.3dvar.3.yaml +++ b/l95/test/testinput/eda.3dvar.3.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 3 cost type: 3D-Var window begin: 2010-01-01T21:00:00Z window length: PT6H @@ -21,6 +20,7 @@ cost function: obs space: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obsdataout: Data/mem003.eda.3dvar.2010-01-02T00:00:00Z.obt + obs perturbations seed: 3 obs operator: {} variational: minimizer: diff --git a/l95/test/testinput/eda.3dvar.4.yaml b/l95/test/testinput/eda.3dvar.4.yaml index 5176b6ece..2a207be1a 100644 --- a/l95/test/testinput/eda.3dvar.4.yaml +++ b/l95/test/testinput/eda.3dvar.4.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 4 cost type: 3D-Var window begin: 2010-01-01T21:00:00Z window length: PT6H @@ -21,6 +20,7 @@ cost function: obs space: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obsdataout: Data/mem004.eda.3dvar.2010-01-02T00:00:00Z.obt + obs perturbations seed: 4 obs operator: {} variational: minimizer: diff --git a/l95/test/testinput/eda.4dvar.3.yaml b/l95/test/testinput/eda.4dvar.3.yaml index 6b050ce66..28c7f7149 100644 --- a/l95/test/testinput/eda.4dvar.3.yaml +++ b/l95/test/testinput/eda.4dvar.3.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 3 cost type: 4D-Var window begin: 2010-01-01T03:00:00Z window length: P1D @@ -23,6 +22,7 @@ cost function: obs space: obsdatain: Data/l95.truth4d.2010-01-02T00:00:00Z.obt obsdataout: Data/mem003.eda.4dvar.2010-01-02T00:00:00Z.obt + obs perturbations seed: 3 obs error: covariance model: diagonal random amplitude: 0.2 diff --git a/qg/test/testinput/eda_3dfgat_1.yaml b/qg/test/testinput/eda_3dfgat_1.yaml index c9ea4b65e..b44b9a425 100644 --- a/qg/test/testinput/eda_3dfgat_1.yaml +++ b/qg/test/testinput/eda_3dfgat_1.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 1 cost type: 4D-Var window begin: 2010-01-01T09:00:00Z window length: PT6H @@ -32,6 +31,7 @@ cost function: obsdataout: obsfile: Data/mem001.eda_3dfgat.obs3d.nc obs type: Stream + obs perturbations seed: 1 - obs error: covariance model: diagonal random amplitude: 0.5 @@ -43,6 +43,7 @@ cost function: obsdataout: obsfile: Data/mem001.eda_3dfgat.obs3d.nc obs type: Wind + obs perturbations seed: 1 - obs error: covariance model: diagonal random amplitude: 0.2 @@ -54,6 +55,7 @@ cost function: obsdataout: obsfile: Data/mem001.eda_3dfgat.obs3d.nc obs type: WSpeed + obs perturbations seed: 1 variational: minimizer: algorithm: DRIPCG diff --git a/qg/test/testinput/eda_3dfgat_2.yaml b/qg/test/testinput/eda_3dfgat_2.yaml index 3e3fbb200..e06aca56a 100644 --- a/qg/test/testinput/eda_3dfgat_2.yaml +++ b/qg/test/testinput/eda_3dfgat_2.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 2 cost type: 4D-Var window begin: 2010-01-01T09:00:00Z window length: PT6H @@ -32,6 +31,7 @@ cost function: obsdataout: obsfile: Data/mem002.eda_3dfgat.obs3d.nc obs type: Stream + obs perturbations seed: 2 - obs error: covariance model: diagonal random amplitude: 0.5 @@ -43,6 +43,7 @@ cost function: obsdataout: obsfile: Data/mem002.eda_3dfgat.obs3d.nc obs type: Wind + obs perturbations seed: 2 - obs error: covariance model: diagonal random amplitude: 0.2 @@ -54,6 +55,7 @@ cost function: obsdataout: obsfile: Data/mem002.eda_3dfgat.obs3d.nc obs type: WSpeed + obs perturbations seed: 2 variational: minimizer: algorithm: DRIPCG diff --git a/qg/test/testinput/eda_3dfgat_3.yaml b/qg/test/testinput/eda_3dfgat_3.yaml index 52d05522f..7529ddb6f 100644 --- a/qg/test/testinput/eda_3dfgat_3.yaml +++ b/qg/test/testinput/eda_3dfgat_3.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 3 cost type: 4D-Var window begin: 2010-01-01T09:00:00Z window length: PT6H @@ -32,6 +31,7 @@ cost function: obsdataout: obsfile: Data/mem003.eda_3dfgat.obs3d.nc obs type: Stream + obs perturbations seed: 3 - obs error: covariance model: diagonal random amplitude: 0.5 @@ -43,6 +43,7 @@ cost function: obsdataout: obsfile: Data/mem003.eda_3dfgat.obs3d.nc obs type: Wind + obs perturbations seed: 3 - obs error: covariance model: diagonal random amplitude: 0.2 @@ -54,6 +55,7 @@ cost function: obsdataout: obsfile: Data/mem003.eda_3dfgat.obs3d.nc obs type: WSpeed + obs perturbations seed: 3 variational: minimizer: algorithm: DRIPCG diff --git a/qg/test/testinput/eda_3dfgat_4.yaml b/qg/test/testinput/eda_3dfgat_4.yaml index d94c029bd..ec66dfbb4 100644 --- a/qg/test/testinput/eda_3dfgat_4.yaml +++ b/qg/test/testinput/eda_3dfgat_4.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 4 cost type: 4D-Var window begin: 2010-01-01T09:00:00Z window length: PT6H @@ -32,6 +31,7 @@ cost function: obsdataout: obsfile: Data/mem004.eda_3dfgat.obs3d.nc obs type: Stream + obs perturbations seed: 4 - obs error: covariance model: diagonal random amplitude: 0.5 @@ -43,6 +43,7 @@ cost function: obsdataout: obsfile: Data/mem004.eda_3dfgat.obs3d.nc obs type: Wind + obs perturbations seed: 4 - obs error: covariance model: diagonal random amplitude: 0.2 @@ -54,6 +55,7 @@ cost function: obsdataout: obsfile: Data/mem004.eda_3dfgat.obs3d.nc obs type: WSpeed + obs perturbations seed: 4 variational: minimizer: algorithm: DRIPCG diff --git a/qg/test/testinput/eda_3dvar_2.yaml b/qg/test/testinput/eda_3dvar_2.yaml index 5afdaed8f..182c60278 100644 --- a/qg/test/testinput/eda_3dvar_2.yaml +++ b/qg/test/testinput/eda_3dvar_2.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 2 cost type: 3D-Var window begin: 2010-01-01T09:00:00Z window length: PT6H @@ -29,6 +28,7 @@ cost function: obsdataout: obsfile: Data/mem002.eda_3dvar.obs3d.nc obs type: Stream + obs perturbations seed: 2 - obs error: covariance model: diagonal random amplitude: 0.5 @@ -40,6 +40,7 @@ cost function: obsdataout: obsfile: Data/mem002.eda_3dvar.obs3d.nc obs type: Wind + obs perturbations seed: 2 - obs error: covariance model: diagonal random amplitude: 0.2 @@ -51,6 +52,7 @@ cost function: obsdataout: obsfile: Data/mem002.eda_3dvar.obs3d.nc obs type: WSpeed + obs perturbations seed: 2 variational: minimizer: algorithm: DRIPCG diff --git a/qg/test/testinput/eda_4dvar_1.yaml b/qg/test/testinput/eda_4dvar_1.yaml index cedc7cc03..ca51bb939 100644 --- a/qg/test/testinput/eda_4dvar_1.yaml +++ b/qg/test/testinput/eda_4dvar_1.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 1 cost type: 4D-Var window begin: 2010-01-01T00:00:00Z window length: P1D @@ -29,6 +28,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/mem001.eda_4dvar.obs4d_24h.nc + obs perturbations seed: 1 obs error: covariance model: diagonal random amplitude: 0.4 @@ -40,6 +40,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/mem001.eda_4dvar.obs4d_24h.nc + obs perturbations seed: 1 obs error: covariance model: diagonal random amplitude: 0.5 @@ -51,6 +52,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/mem001.eda_4dvar.obs4d_24h.nc + obs perturbations seed: 1 obs error: covariance model: diagonal random amplitude: 0.2 diff --git a/qg/test/testinput/eda_4dvar_2.yaml b/qg/test/testinput/eda_4dvar_2.yaml index c520227cc..5f8c7d341 100644 --- a/qg/test/testinput/eda_4dvar_2.yaml +++ b/qg/test/testinput/eda_4dvar_2.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 2 cost type: 4D-Var window begin: 2010-01-01T00:00:00Z window length: P1D @@ -29,6 +28,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/mem002.eda_4dvar.obs4d_24h.nc + obs perturbations seed: 2 obs error: covariance model: diagonal random amplitude: 0.4 @@ -40,6 +40,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/mem002.eda_4dvar.obs4d_24h.nc + obs perturbations seed: 2 obs error: covariance model: diagonal random amplitude: 0.5 @@ -51,6 +52,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/mem002.eda_4dvar.obs4d_24h.nc + obs perturbations seed: 2 obs error: covariance model: diagonal random amplitude: 0.2 diff --git a/qg/test/testinput/eda_4dvar_3.yaml b/qg/test/testinput/eda_4dvar_3.yaml index decedda01..937a9abc5 100644 --- a/qg/test/testinput/eda_4dvar_3.yaml +++ b/qg/test/testinput/eda_4dvar_3.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 3 cost type: 4D-Var window begin: 2010-01-01T00:00:00Z window length: P1D @@ -29,6 +28,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/mem003.eda_4dvar.obs4d_24h.nc + obs perturbations seed: 3 obs error: covariance model: diagonal random amplitude: 0.4 @@ -40,6 +40,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/mem003.eda_4dvar.obs4d_24h.nc + obs perturbations seed: 3 obs error: covariance model: diagonal random amplitude: 0.5 @@ -51,6 +52,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/mem003.eda_4dvar.obs4d_24h.nc + obs perturbations seed: 3 obs error: covariance model: diagonal random amplitude: 0.2 diff --git a/qg/test/testinput/eda_4dvar_4.yaml b/qg/test/testinput/eda_4dvar_4.yaml index 4d9850f39..60cfcec00 100644 --- a/qg/test/testinput/eda_4dvar_4.yaml +++ b/qg/test/testinput/eda_4dvar_4.yaml @@ -1,5 +1,4 @@ cost function: - obs perturbations seed: 4 cost type: 4D-Var window begin: 2010-01-01T00:00:00Z window length: P1D @@ -29,6 +28,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/mem004.eda_4dvar.obs4d_24h.nc + obs perturbations seed: 4 obs error: covariance model: diagonal random amplitude: 0.4 @@ -40,6 +40,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/mem004.eda_4dvar.obs4d_24h.nc + obs perturbations seed: 4 obs error: covariance model: diagonal random amplitude: 0.5 @@ -51,6 +52,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/mem004.eda_4dvar.obs4d_24h.nc + obs perturbations seed: 4 obs error: covariance model: diagonal random amplitude: 0.2 diff --git a/src/oops/assimilation/CalcHofX.h b/src/oops/assimilation/CalcHofX.h index 086950a0d..c5287657a 100644 --- a/src/oops/assimilation/CalcHofX.h +++ b/src/oops/assimilation/CalcHofX.h @@ -86,7 +86,7 @@ class CalcHofX { template CalcHofX::CalcHofX(const ObsSpaces_ & obspaces, const Geometry_ & geometry, const eckit::Configuration & config) : - obsconf_(config), obspaces_(obspaces), ybias_(obspaces_, obsconf_), + obsconf_(config, "observations"), obspaces_(obspaces), ybias_(obspaces_, obsconf_), geometry_(geometry), moderr_(geometry_, config.getSubConfiguration("model aux control")), winbgn_(config.getString("window begin")), winlen_(config.getString("window length")) {} diff --git a/src/oops/assimilation/ControlVariable.h b/src/oops/assimilation/ControlVariable.h index b991b9b11..bc48eb4f7 100644 --- a/src/oops/assimilation/ControlVariable.h +++ b/src/oops/assimilation/ControlVariable.h @@ -93,7 +93,7 @@ ControlVariable::ControlVariable(const eckit::Configuration & conf, const Geometry_ & resol, const ObsSpaces_ & odb) : state_(resol, eckit::LocalConfiguration(conf, "background")), modbias_(resol, conf.getSubConfiguration("model aux control")), - obsbias_(odb, conf) + obsbias_(odb, conf.getSubConfiguration("observations")) { Log::trace() << "ControlVariable contructed" << std::endl; } @@ -105,7 +105,7 @@ ControlVariable::ControlVariable(const eckit::Configuration & conf, const State_ & statein, const ObsSpaces_ & odb) : state_(statein), modbias_(statein.geometry(), conf.getSubConfiguration("model aux control")), - obsbias_(odb, conf) + obsbias_(odb, conf.getSubConfiguration("observations")) { Log::trace() << "ControlVariable contructed" << std::endl; } diff --git a/src/oops/assimilation/CostFunction.h b/src/oops/assimilation/CostFunction.h index dfa97681f..5f5cc17e3 100644 --- a/src/oops/assimilation/CostFunction.h +++ b/src/oops/assimilation/CostFunction.h @@ -196,7 +196,8 @@ void CostFunction::setupTerms(const eckit::Configuration & config) { Log::trace() << "CostFunction::setupTerms start" << std::endl; // Jo - CostJo * jo = this->newJo(config); + eckit::LocalConfiguration obsconf(config, "observations"); + CostJo * jo = this->newJo(obsconf); jterms_.push_back(jo); Log::trace() << "CostFunction::setupTerms Jo added" << std::endl; @@ -226,7 +227,8 @@ void CostFunction::setupTerms(const eckit::Configuration & config, Log::trace() << "CostFunction::setupTerms start" << std::endl; // Jo - CostJo * jo = this->newJo(config); + eckit::LocalConfiguration obsconf(config, "observations"); + CostJo * jo = this->newJo(obsconf); jterms_.push_back(jo); Log::trace() << "CostFunction::setupTerms Jo added" << std::endl; diff --git a/src/oops/assimilation/CostJbTotal.h b/src/oops/assimilation/CostJbTotal.h index b31b99386..f26b2f347 100644 --- a/src/oops/assimilation/CostJbTotal.h +++ b/src/oops/assimilation/CostJbTotal.h @@ -120,7 +120,7 @@ CostJbTotal::CostJbTotal(const CtrlVar_ & xb, JbState_ * jb, const Geometry_ & resol, const ObsSpaces_ & odb) : xb_(xb), jb_(jb), jbModBias_(conf.getSubConfiguration("model aux error"), resol), - jbObsBias_(odb, conf), dxFG_(), resol_(), + jbObsBias_(odb, conf.getSubConfiguration("observations")), dxFG_(), resol_(), windowBegin_(conf.getString("window begin")), windowEnd_(windowBegin_ + util::Duration(conf.getString("window length"))) { diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index 96bfdb7c3..62049d220 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -173,9 +173,8 @@ CostJo::initialize(const CtrlVar_ & xx, const eckit::Configuration & Log::trace() << "CostJo::initialize start" << std::endl; currentConf_.reset(new eckit::LocalConfiguration(conf)); - const int iterout = currentConf_->getInt("iteration"); - obsconf_.set("iteration", iterout); - pobs_.reset(new Observers_(obsconf_, obspace_, xx.obsVar(), qc_)); + pobs_.reset(new Observers_(obsconf_, obspace_, xx.obsVar(), qc_, + currentConf_->getInt("iteration"))); Log::trace() << "CostJo::initialize done" << std::endl; return pobs_; } diff --git a/src/oops/assimilation/LocalEnsembleSolver.h b/src/oops/assimilation/LocalEnsembleSolver.h index e0d8070ea..9fc111dec 100644 --- a/src/oops/assimilation/LocalEnsembleSolver.h +++ b/src/oops/assimilation/LocalEnsembleSolver.h @@ -78,7 +78,7 @@ template LocalEnsembleSolver::LocalEnsembleSolver(ObsSpaces_ & obspaces, const Geometry_ & geometry, const eckit::Configuration & config, size_t nens) - : obsconf_(config), obspaces_(obspaces), + : obsconf_(config, "observations"), obspaces_(obspaces), hofx_(obspaces, geometry, config), omb_(obspaces_), Yb_(obspaces_, nens) { diff --git a/src/oops/base/ObsAuxControls.h b/src/oops/base/ObsAuxControls.h index 09a01cbe6..400563bb2 100644 --- a/src/oops/base/ObsAuxControls.h +++ b/src/oops/base/ObsAuxControls.h @@ -57,8 +57,7 @@ template ObsAuxControls::ObsAuxControls(const ObsSpaces_ & odb, const eckit::Configuration & conf) : auxs_(0) { - std::vector obsconf; - conf.get("observations", obsconf); + std::vector obsconf = conf.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsconf.size(); ++jobs) { auxs_.push_back( std::unique_ptr(new ObsAuxControl_(odb[jobs], obsconf[jobs]))); diff --git a/src/oops/base/ObsAuxCovariances.h b/src/oops/base/ObsAuxCovariances.h index 1ea00a10b..80171adc4 100644 --- a/src/oops/base/ObsAuxCovariances.h +++ b/src/oops/base/ObsAuxCovariances.h @@ -61,12 +61,11 @@ class ObsAuxCovariances : public util::Printable, template ObsAuxCovariances::ObsAuxCovariances(const ObsSpaces_ & odb, - const eckit::Configuration & conf) + const eckit::Configuration & conf) : cov_(0), odb_(odb), conf_(conf) { Log::trace() << "ObsAuxCovariances::ObsAuxCovariances starting" << std::endl; - std::vector obsconf; - conf.get("observations", obsconf); + std::vector obsconf = conf.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsconf.size(); ++jobs) { cov_.push_back( std::unique_ptr(new ObsAuxCovariance_(odb[jobs], obsconf[jobs]))); diff --git a/src/oops/base/ObsAuxIncrements.h b/src/oops/base/ObsAuxIncrements.h index 4778a55b8..3d3a7006d 100644 --- a/src/oops/base/ObsAuxIncrements.h +++ b/src/oops/base/ObsAuxIncrements.h @@ -89,8 +89,7 @@ template ObsAuxIncrements::ObsAuxIncrements(const ObsSpaces_ & odb, const eckit::Configuration & conf) : auxs_(0) { - std::vector obsconf; - conf.get("observations", obsconf); + std::vector obsconf = conf.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsconf.size(); ++jobs) { auxs_.push_back( std::unique_ptr(new ObsAuxIncrement_(odb[jobs], obsconf[jobs]))); diff --git a/src/oops/base/ObsErrors.h b/src/oops/base/ObsErrors.h index d3da217ee..8489bf208 100644 --- a/src/oops/base/ObsErrors.h +++ b/src/oops/base/ObsErrors.h @@ -66,9 +66,8 @@ class ObsErrors : public util::Printable, template ObsErrors::ObsErrors(const eckit::Configuration & config, - const ObsSpaces_ & os) : err_() { - std::vector obsconf; - config.get("observations", obsconf); + const ObsSpaces_ & os) : err_() { + std::vector obsconf = config.getSubConfigurations(); for (size_t jj = 0; jj < os.size(); ++jj) { eckit::LocalConfiguration conf(obsconf[jj], "obs error"); err_.emplace_back(ObsErrorFactory::create(conf, os[jj])); diff --git a/src/oops/base/ObsFilters.h b/src/oops/base/ObsFilters.h index 856085a06..3e33e75a3 100644 --- a/src/oops/base/ObsFilters.h +++ b/src/oops/base/ObsFilters.h @@ -36,18 +36,19 @@ class ObsFilters : public util::Printable, private boost::noncopyable { typedef GeoVaLs GeoVaLs_; typedef ObsDiagnostics ObsDiags_; - typedef ObsFilterBase ObsFilterBase_; typedef ObsSpace ObsSpace_; typedef ObsVector ObsVector_; typedef std::shared_ptr > ObsFilterPtr_; template using ObsDataPtr_ = std::shared_ptr >; public: + /// Initialize all filters for \p obspace, from configuration, using + /// \p qcflags and \p obserr (observation error variances) + /// \p iteration argument indicates outer loop iteration in the variational + /// assimilation ObsFilters(const ObsSpace_ &, const eckit::Configuration &, - ObsDataPtr_ qcflags = ObsDataPtr_(), - ObsDataPtr_ obserr = ObsDataPtr_()); - ObsFilters(); - ~ObsFilters(); + ObsDataPtr_ qcflags, ObsDataPtr_ obserr, + const int iteration = 0); void preProcess() const; void priorFilter(const GeoVaLs_ &) const; @@ -57,7 +58,7 @@ class ObsFilters : public util::Printable, Variables requiredHdiagnostics() const {return diagvars_;} private: - void print(std::ostream &) const; + void print(std::ostream &) const override; std::vector filters_; Variables geovars_; @@ -68,7 +69,8 @@ class ObsFilters : public util::Printable, template ObsFilters::ObsFilters(const ObsSpace_ & os, const eckit::Configuration & conf, - ObsDataPtr_ qcflags, ObsDataPtr_ obserr) + ObsDataPtr_ qcflags, ObsDataPtr_ obserr, + const int iteration) : filters_(), geovars_(), diagvars_() { Log::trace() << "ObsFilters::ObsFilters starting " << conf << std::endl; @@ -85,13 +87,12 @@ ObsFilters::ObsFilters(const ObsSpace_ & os, const eckit::Configuration & c // Create the filters, only at 0-th iteration, or at iterations specified in "apply at iterations" for (std::size_t jj = 0; jj < confs.size(); ++jj) { - // Only create filters for the 0-th iteration - const int iter = conf.getInt("iteration"); - bool apply = (iter == 0); + // Only create filters for the 0-th iteration by default + bool apply = (iteration == 0); // If "apply at iterations" is set, check if this is the right iteration if (confs[jj].has("apply at iterations")) { std::set iters = parseIntSet(confs[jj].getString("apply at iterations")); - apply = contains(iters, iter); + apply = contains(iters, iteration); } if (apply) { ObsFilterPtr_ tmp(FilterFactory::create(os, confs[jj], qcflags, obserr)); @@ -106,22 +107,10 @@ ObsFilters::ObsFilters(const ObsSpace_ & os, const eckit::Configuration & c // ----------------------------------------------------------------------------- -template -ObsFilters::ObsFilters() : filters_(), geovars_(), diagvars_() {} - -// ----------------------------------------------------------------------------- - -template -ObsFilters::~ObsFilters() { - Log::trace() << "ObsFilters::~ObsFilters destructed" << std::endl; -} - -// ----------------------------------------------------------------------------- - template void ObsFilters::preProcess() const { - for (std::size_t jj = 0; jj < filters_.size(); ++jj) { - filters_.at(jj)->preProcess(); + for (const auto & filter : filters_) { + filter->preProcess(); } } @@ -129,8 +118,8 @@ void ObsFilters::preProcess() const { template void ObsFilters::priorFilter(const GeoVaLs_ & gv) const { - for (std::size_t jj = 0; jj < filters_.size(); ++jj) { - filters_.at(jj)->priorFilter(gv); + for (const auto & filter : filters_) { + filter->priorFilter(gv); } } @@ -138,8 +127,8 @@ void ObsFilters::priorFilter(const GeoVaLs_ & gv) const { template void ObsFilters::postFilter(const ObsVector_ & hofx, const ObsDiags_ & diags) const { - for (std::size_t jj = 0; jj < filters_.size(); ++jj) { - filters_.at(jj)->postFilter(hofx, diags); + for (const auto & filter : filters_) { + filter->postFilter(hofx, diags); } } @@ -148,8 +137,8 @@ void ObsFilters::postFilter(const ObsVector_ & hofx, const ObsDiags_ & diag template void ObsFilters::print(std::ostream & os) const { os << "ObsFilters: " << filters_.size() << " elements:" << std::endl; - for (std::size_t jj = 0; jj < filters_.size(); ++jj) { - os << *filters_.at(jj) << std::endl; + for (const auto & filter : filters_) { + os << *filter << std::endl; } } diff --git a/src/oops/base/ObsSpaces.h b/src/oops/base/ObsSpaces.h index a1dc09707..c9d58170d 100644 --- a/src/oops/base/ObsSpaces.h +++ b/src/oops/base/ObsSpaces.h @@ -81,12 +81,9 @@ ObsSpaces::ObsSpaces(const eckit::Configuration & conf, const eckit::mpi::C const eckit::mpi::Comm & time) : spaces_(0), wbgn_(bgn), wend_(end) { - const int seed_member = conf.getInt("obs perturbations seed", 0); - std::vector typeconfs; - conf.get("observations", typeconfs); + std::vector typeconfs = conf.getSubConfigurations(); for (std::size_t jj = 0; jj < typeconfs.size(); ++jj) { eckit::LocalConfiguration obsconf(typeconfs[jj], "obs space"); - obsconf.set("obs perturbations seed", seed_member); Log::debug() << "ObsSpaces::ObsSpaces : conf " << obsconf << std::endl; std::shared_ptr tmp(new ObsSpace_(obsconf, comm, bgn, end, time)); spaces_.push_back(tmp); @@ -101,8 +98,7 @@ ObsSpaces::ObsSpaces(const ObsSpaces & obss, const eckit::geometry::Po const eckit::Configuration & conf) : spaces_(0), wbgn_(obss.wbgn_), wend_(obss.wend_) { - std::vector typeconfs; - conf.get("observations", typeconfs); + std::vector typeconfs = conf.getSubConfigurations(); for (std::size_t jj = 0; jj < obss.size(); ++jj) { eckit::LocalConfiguration locconf(typeconfs[jj], "obs error.localization"); std::shared_ptr tmp(new ObsSpace_(obss[jj], center, locconf)); diff --git a/src/oops/base/Observer.h b/src/oops/base/Observer.h index 94fa1fe44..641eb45f8 100644 --- a/src/oops/base/Observer.h +++ b/src/oops/base/Observer.h @@ -52,8 +52,8 @@ class Observer : public util::Printable { public: Observer(const eckit::Configuration &, const ObsSpace_ &, const ObsAuxCtrl_ &, - ObsVector_ &, ObsDataPtr_ qcflags = ObsDataPtr_(), - ObsDataPtr_ obserr = ObsDataPtr_()); + ObsVector_ &, ObsDataPtr_ qcflags, ObsDataPtr_ obserr, + const int iteration = 0); ~Observer(); void doInitialize(const State_ &, const util::DateTime &, const util::DateTime &); @@ -83,9 +83,10 @@ class Observer : public util::Printable { template Observer::Observer(const eckit::Configuration & conf, const ObsSpace_ & obsdb, const ObsAuxCtrl_ & ybias, ObsVector_ & yobs, - ObsDataPtr_ qcflags, ObsDataPtr_ obserr) + ObsDataPtr_ qcflags, ObsDataPtr_ obserr, + const int iteration) : hop_(obsdb, eckit::LocalConfiguration(conf, "obs operator")), - obsdb_(obsdb), yobs_(yobs), ybias_(ybias), filters_(obsdb, conf, qcflags, obserr), + obsdb_(obsdb), yobs_(yobs), ybias_(ybias), filters_(obsdb, conf, qcflags, obserr, iteration), locs_(hop_.locations()) { Log::trace() << "Observer::Observer starting" << std::endl; diff --git a/src/oops/base/Observers.h b/src/oops/base/Observers.h index ca6d4ed22..bea22f903 100644 --- a/src/oops/base/Observers.h +++ b/src/oops/base/Observers.h @@ -45,7 +45,7 @@ class Observers : public PostBase> { public: Observers(const eckit::Configuration &, const ObsSpaces_ & obsdb, const ObsAuxCtrls_ &, - QCData_ &); + QCData_ &, const int iteration = 0); ~Observers() {} const Observations_ & hofx() {return yobs_;} @@ -71,22 +71,18 @@ class Observers : public PostBase> { template Observers::Observers(const eckit::Configuration & conf, const ObsSpaces_ & obsdb, - const ObsAuxCtrls_ & ybias, QCData_ & qc) + const ObsAuxCtrls_ & ybias, QCData_ & qc, const int iteration) : PostBase(), obspace_(obsdb), yobs_(obsdb), winbgn_(obsdb.windowStart()), winend_(obsdb.windowEnd()), hslot_(0), observers_(0) { Log::trace() << "Observers::Observers starting" << std::endl; - - const int iterout = conf.getInt("iteration", 0); - std::vector typeconf; - conf.get("observations", typeconf); + std::vector typeconf = conf.getSubConfigurations(); ASSERT(obsdb.size() == typeconf.size()); observers_.reserve(obsdb.size()); for (size_t jj = 0; jj < obsdb.size(); ++jj) { - typeconf[jj].set("iteration", iterout); observers_.emplace_back(new Observer_(typeconf[jj], obsdb[jj], - ybias[jj], yobs_[jj], qc.qcFlags(jj), qc.obsErrors(jj))); + ybias[jj], yobs_[jj], qc.qcFlags(jj), qc.obsErrors(jj), iteration)); } Log::trace() << "Observers::Observers done" << std::endl; } diff --git a/src/oops/base/ObserversTLAD.h b/src/oops/base/ObserversTLAD.h index 422871b9b..0015c5be1 100644 --- a/src/oops/base/ObserversTLAD.h +++ b/src/oops/base/ObserversTLAD.h @@ -98,8 +98,7 @@ ObserversTLAD::ObserversTLAD(const eckit::Configuration & config, hslot_(0), hslottraj_(0) { // setup observers - std::vector typeconf; - config.get("observations", typeconf); + std::vector typeconf = config.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsdb.size(); ++jobs) { // Set LinearObsOperator section to ObsOperator section if not available if (!typeconf[jobs].has("linear obs operator")) { diff --git a/src/oops/runs/HofX.h b/src/oops/runs/HofX.h index 01c894f80..abe97e64f 100644 --- a/src/oops/runs/HofX.h +++ b/src/oops/runs/HofX.h @@ -92,7 +92,8 @@ template class HofX : public Application { post.enrollProcessor(new StateInfo("fc", prtConf)); // Setup observations - ObsSpaces_ obspace(fullConfig, this->getComm(), winbgn, winend); + const eckit::LocalConfiguration obsConfig(fullConfig, "observations"); + ObsSpaces_ obspace(obsConfig, this->getComm(), winbgn, winend); // Setup and run observer CalcHofX hofx(obspace, geometry, fullConfig); @@ -107,7 +108,7 @@ template class HofX : public Application { // as ObsValue if "hofx group name" == ObsValue. bool obspert = fullConfig.getBool("obs perturbations", false); if (obspert) { - ObsErrors_ matR(fullConfig, obspace); + ObsErrors_ matR(obsConfig, obspace); yobs.perturb(matR); Log::test() << "Perturbed H(x): " << std::endl << yobs << "End Perturbed H(x)" << std::endl; } diff --git a/src/oops/runs/HofXNoModel.h b/src/oops/runs/HofXNoModel.h index c2b837ecd..a8779a385 100644 --- a/src/oops/runs/HofXNoModel.h +++ b/src/oops/runs/HofXNoModel.h @@ -57,7 +57,8 @@ template class HofXNoModel : public Application { Log::test() << "Initial state: " << xx[0] << std::endl; // Setup observations - ObsSpaces_ obspace(fullConfig, this->getComm(), winbgn, winend); + const eckit::LocalConfiguration obsConfig(fullConfig, "observations"); + ObsSpaces_ obspace(obsConfig, this->getComm(), winbgn, winend); // Setup and run observer CalcHofX hofx(obspace, geometry, fullConfig); diff --git a/src/oops/runs/LocalEnsembleDA.h b/src/oops/runs/LocalEnsembleDA.h index 5bfe6df96..c4ca3529e 100644 --- a/src/oops/runs/LocalEnsembleDA.h +++ b/src/oops/runs/LocalEnsembleDA.h @@ -73,7 +73,8 @@ template class LocalEnsembleDA : public Applicati const Geometry_ geometry(geometryConfig, this->getComm()); // Setup observations - ObsSpaces_ obsdb(fullConfig, this->getComm(), winbgn, winend); + const eckit::LocalConfiguration obsConfig(fullConfig, "observations"); + ObsSpaces_ obsdb(obsConfig, this->getComm(), winbgn, winend); Observations_ yobs(obsdb, "ObsValue"); // Get background configurations diff --git a/src/test/interface/GeoVaLs.h b/src/test/interface/GeoVaLs.h index 6006d75b0..256ce53e0 100644 --- a/src/test/interface/GeoVaLs.h +++ b/src/test/interface/GeoVaLs.h @@ -26,49 +26,19 @@ #include "oops/runs/Test.h" #include "oops/util/dot_product.h" #include "oops/util/Logger.h" +#include "test/interface/ObsTestsFixture.h" #include "test/TestEnvironment.h" namespace test { // ----------------------------------------------------------------------------- -template -class GeoVaLsFixture : private boost::noncopyable { - typedef oops::ObsSpaces ObsSpaces_; - - public: - static ObsSpaces_ & obspace() {return *getInstance().ospaces_;} - static eckit::LocalConfiguration conf(const size_t ii) {return getInstance().confs_[ii];} - static void reset() {getInstance().ospaces_.reset();} - - private: - static GeoVaLsFixture& getInstance() { - static GeoVaLsFixture theGeoVaLsFixture; - return theGeoVaLsFixture; - } - - GeoVaLsFixture(): ospaces_() { - const util::DateTime tbgn(TestEnvironment::config().getString("window begin")); - const util::DateTime tend(TestEnvironment::config().getString("window end")); - - ospaces_.reset(new ObsSpaces_(TestEnvironment::config(), oops::mpi::world(), tbgn, tend)); - TestEnvironment::config().get("observations", confs_); - } - - ~GeoVaLsFixture() {} - - std::unique_ptr ospaces_; - std::vector confs_; -}; - -// ----------------------------------------------------------------------------- - template void testConstructor() { - typedef GeoVaLsFixture Test_; + typedef ObsTestsFixture Test_; typedef oops::GeoVaLs GeoVaLs_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - eckit::LocalConfiguration gconf(Test_::conf(jj), "geovals"); + eckit::LocalConfiguration gconf(Test_::config(jj), "geovals"); oops::Variables geovars(gconf, "state variables"); std::unique_ptr ov(new GeoVaLs_(gconf, Test_::obspace()[jj], geovars)); EXPECT(ov.get()); @@ -81,13 +51,13 @@ template void testConstructor() { // ----------------------------------------------------------------------------- template void testUtils() { - typedef GeoVaLsFixture Test_; + typedef ObsTestsFixture Test_; typedef oops::GeoVaLs GeoVaLs_; const double tol = 1e-6; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - eckit::LocalConfiguration gconf(Test_::conf(jj), "geovals"); + eckit::LocalConfiguration gconf(Test_::config(jj), "geovals"); oops::Variables geovars(gconf, "state variables"); GeoVaLs_ gval(gconf, Test_::obspace()[jj], geovars); @@ -158,12 +128,12 @@ template void testUtils() { // ----------------------------------------------------------------------------- template void testRead() { - typedef GeoVaLsFixture Test_; - typedef oops::GeoVaLs GeoVaLs_; + typedef ObsTestsFixture Test_; + typedef oops::GeoVaLs GeoVaLs_; const double tol = 1.0e-9; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - eckit::LocalConfiguration gconf(Test_::conf(jj), "geovals"); + eckit::LocalConfiguration gconf(Test_::config(jj), "geovals"); oops::Variables geovars(gconf, "state variables"); GeoVaLs_ gval(gconf, Test_::obspace()[jj], geovars); @@ -181,7 +151,7 @@ template void testRead() { template class GeoVaLs : public oops::Test { - typedef GeoVaLsFixture Test_; + typedef ObsTestsFixture Test_; public: GeoVaLs() {} virtual ~GeoVaLs() {} diff --git a/src/test/interface/LinearObsOperator.h b/src/test/interface/LinearObsOperator.h index bb871c92f..b94441151 100644 --- a/src/test/interface/LinearObsOperator.h +++ b/src/test/interface/LinearObsOperator.h @@ -37,14 +37,12 @@ template void testConstructor() { typedef ObsTestsFixture Test_; typedef oops::LinearObsOperator LinearObsOperator_; - std::vector conf; - TestEnvironment::config().get("observations", conf); - for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + const eckit::LocalConfiguration & conf = Test_::config(jj); // Use ObsOperator section of yaml unless LinearObsOperator is specified std::string confname = "obs operator"; - if (conf[jj].has("linear obs operator")) confname = "linear obs operator"; - eckit::LocalConfiguration linobsopconf(conf[jj], confname); + if (conf.has("linear obs operator")) confname = "linear obs operator"; + eckit::LocalConfiguration linobsopconf(conf, confname); std::unique_ptr ov( new LinearObsOperator_(Test_::obspace()[jj], linobsopconf)); EXPECT(ov.get()); @@ -69,34 +67,33 @@ template void testLinearity() { const double zero = 0.0; const double coef = 3.14; const double tol = 1.0e-11; - std::vector conf; - TestEnvironment::config().get("observations", conf); for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + const eckit::LocalConfiguration & conf = Test_::config(jj); // initialize observation operator (set variables requested from the model, // variables simulated by the observation operator, other init) - eckit::LocalConfiguration obsopconf(conf[jj], "obs operator"); + eckit::LocalConfiguration obsopconf(conf, "obs operator"); ObsOperator_ hop(Test_::obspace()[jj], obsopconf); // initialize TL/AD observation operator (set model variables for Jacobian), // other init) // Use ObsOperator section of yaml unless LinearObsOperator is specified std::string confname = "obs operator"; - if (conf[jj].has("linear obs operator")) confname = "linear obs operator"; - eckit::LocalConfiguration linobsopconf(conf[jj], confname); + if (conf.has("linear obs operator")) confname = "linear obs operator"; + eckit::LocalConfiguration linobsopconf(conf, confname); LinearObsOperator_ hoptl(Test_::obspace()[jj], linobsopconf); // initialize obs bias - const ObsAuxCtrl_ ybias(Test_::obspace()[jj], conf[jj]); - ObsAuxIncr_ ybinc(Test_::obspace()[jj], conf[jj]); + const ObsAuxCtrl_ ybias(Test_::obspace()[jj], conf); + ObsAuxIncr_ ybinc(Test_::obspace()[jj], conf); // read geovals from the file - const eckit::LocalConfiguration gconf(conf[jj], "geovals"); + const eckit::LocalConfiguration gconf(conf, "geovals"); oops::Variables hopvars = hop.requiredVars(); hopvars += ybias.requiredVars(); const GeoVaLs_ gval(gconf, Test_::obspace()[jj], hopvars); // initialize Obs. Bias Covariance - const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], conf[jj]); + const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], conf); // set trajectory for TL/AD to be the geovals from the file hoptl.setTrajectory(gval, ybias); @@ -144,33 +141,32 @@ template void testAdjoint() { typedef oops::ObsVector ObsVector_; const double zero = 0.0; - std::vector conf; - TestEnvironment::config().get("observations", conf); for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + const eckit::LocalConfiguration & conf = Test_::config(jj); // initialize observation operator (set variables requested from the model, // variables simulated by the observation operator, other init) - eckit::LocalConfiguration obsopconf(conf[jj], "obs operator"); + eckit::LocalConfiguration obsopconf(conf, "obs operator"); ObsOperator_ hop(Test_::obspace()[jj], obsopconf); // initialize TL/AD observation operator (set model variables for Jacobian), // other init) // Use ObsOperator section of yaml unless LinearObsOperator is specified std::string confname = "obs operator"; - if (conf[jj].has("linear obs operator")) confname = "linear obs operator"; - eckit::LocalConfiguration linobsopconf(conf[jj], confname); + if (conf.has("linear obs operator")) confname = "linear obs operator"; + eckit::LocalConfiguration linobsopconf(conf, confname); LinearObsOperator_ hoptl(Test_::obspace()[jj], linobsopconf); - const double tol = conf[jj].getDouble("linear obs operator test.tolerance AD"); + const double tol = conf.getDouble("linear obs operator test.tolerance AD"); // initialize bias correction - const ObsAuxCtrl_ ybias(Test_::obspace()[jj], conf[jj]); - ObsAuxIncr_ ybinc1(Test_::obspace()[jj], conf[jj]); // TL - ObsAuxIncr_ ybinc2(Test_::obspace()[jj], conf[jj]); // AD + const ObsAuxCtrl_ ybias(Test_::obspace()[jj], conf); + ObsAuxIncr_ ybinc1(Test_::obspace()[jj], conf); // TL + ObsAuxIncr_ ybinc2(Test_::obspace()[jj], conf); // AD // initialize Obs. Bias Covariance - const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], conf[jj]); + const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], conf); // read geovals from the file - eckit::LocalConfiguration gconf(conf[jj], "geovals"); + eckit::LocalConfiguration gconf(conf, "geovals"); oops::Variables hopvars = hop.requiredVars(); hopvars += ybias.requiredVars(); const GeoVaLs_ gval(gconf, Test_::obspace()[jj], hopvars); @@ -224,35 +220,33 @@ template void testTangentLinear() { typedef oops::ObsOperator ObsOperator_; typedef oops::ObsVector ObsVector_; - std::vector conf; - TestEnvironment::config().get("observations", conf); - for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + const eckit::LocalConfiguration & conf = Test_::config(jj); // initialize observation operator (set variables requested from the model, // variables simulated by the observation operator, other init) - eckit::LocalConfiguration obsopconf(conf[jj], "obs operator"); + eckit::LocalConfiguration obsopconf(conf, "obs operator"); ObsOperator_ hop(Test_::obspace()[jj], obsopconf); // initialize TL/AD observation operator (set model variables for Jacobian), // other init) // Use ObsOperator section of yaml unless LinearObsOperator is specified std::string confname = "obs operator"; - if (conf[jj].has("linear obs operator")) confname = "linear obs operator"; - eckit::LocalConfiguration linobsopconf(conf[jj], confname); + if (conf.has("linear obs operator")) confname = "linear obs operator"; + eckit::LocalConfiguration linobsopconf(conf, confname); LinearObsOperator_ hoptl(Test_::obspace()[jj], linobsopconf); - const double tol = conf[jj].getDouble("linear obs operator test.tolerance TL"); - const double alpha = conf[jj].getDouble("linear obs operator test.coef TL", 0.1); - const int iter = conf[jj].getInt("linear obs operator test.iterations TL", 1); + const double tol = conf.getDouble("linear obs operator test.tolerance TL"); + const double alpha = conf.getDouble("linear obs operator test.coef TL", 0.1); + const int iter = conf.getInt("linear obs operator test.iterations TL", 1); // initialize obs bias from file - const ObsAuxCtrl_ ybias0(Test_::obspace()[jj], conf[jj]); - ObsAuxCtrl_ ybias(Test_::obspace()[jj], conf[jj]); + const ObsAuxCtrl_ ybias0(Test_::obspace()[jj], conf); + ObsAuxCtrl_ ybias(Test_::obspace()[jj], conf); // initialize Obs. Bias Covariance - const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], conf[jj]); + const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], conf); // read geovals from the file - const eckit::LocalConfiguration gconf(conf[jj], "geovals"); + const eckit::LocalConfiguration gconf(conf, "geovals"); oops::Variables hopvars = hop.requiredVars(); hopvars += ybias0.requiredVars(); const GeoVaLs_ x0(gconf, Test_::obspace()[jj], hopvars); @@ -277,7 +271,7 @@ template void testTangentLinear() { // randomize dx and ybinc GeoVaLs_ dx(gconf, Test_::obspace()[jj], hoptl.requiredVars()); dx.random(); - ObsAuxIncr_ ybinc(Test_::obspace()[jj], conf[jj]); + ObsAuxIncr_ ybinc(Test_::obspace()[jj], conf); Bobsbias.randomize(ybinc); // scale dx by x0 diff --git a/src/test/interface/ObsAuxControl.h b/src/test/interface/ObsAuxControl.h index b09371de9..af13a1bca 100644 --- a/src/test/interface/ObsAuxControl.h +++ b/src/test/interface/ObsAuxControl.h @@ -34,10 +34,8 @@ template void testConstructor() { typedef ObsTestsFixture Test_; typedef oops::ObsAuxControl ObsAux_; - std::vector oconf; - TestEnvironment::config().get("observations", oconf); for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - std::unique_ptr bias(new ObsAux_(Test_::obspace()[jj], oconf[jj])); + std::unique_ptr bias(new ObsAux_(Test_::obspace()[jj], Test_::config(jj))); EXPECT(bias.get()); bias.reset(); @@ -51,10 +49,8 @@ template void testCopyConstructor() { typedef ObsTestsFixture Test_; typedef oops::ObsAuxControl ObsAux_; - std::vector oconf; - TestEnvironment::config().get("observations", oconf); for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - std::unique_ptr bias(new ObsAux_(Test_::obspace()[jj], oconf[jj])); + std::unique_ptr bias(new ObsAux_(Test_::obspace()[jj], Test_::config(jj))); std::unique_ptr other(new ObsAux_(*bias)); EXPECT(other.get()); diff --git a/src/test/interface/ObsAuxCovariance.h b/src/test/interface/ObsAuxCovariance.h index 6ff18ff43..d3fdd85f5 100644 --- a/src/test/interface/ObsAuxCovariance.h +++ b/src/test/interface/ObsAuxCovariance.h @@ -33,10 +33,8 @@ template void testConstructor() { typedef ObsTestsFixture Test_; typedef oops::ObsAuxCovariance Covariance_; - std::vector oconf; - TestEnvironment::config().get("observations", oconf); for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - std::unique_ptr bias(new Covariance_(Test_::obspace()[jj], oconf[jj])); + std::unique_ptr bias(new Covariance_(Test_::obspace()[jj], Test_::config(jj))); EXPECT(bias.get()); bias.reset(); diff --git a/src/test/interface/ObsAuxIncrement.h b/src/test/interface/ObsAuxIncrement.h index b1a436333..ef1bf12f9 100644 --- a/src/test/interface/ObsAuxIncrement.h +++ b/src/test/interface/ObsAuxIncrement.h @@ -39,11 +39,8 @@ namespace test { template class ObsAuxIncrementFixture : private boost::noncopyable { typedef ObsTestsFixture Test_; typedef oops::ObsAuxCovariance Covariance_; - typedef oops::ObsAuxControl ObsAux_; - typedef oops::ObsAuxIncrement AuxIncr_; public: - static const eckit::Configuration & config(const size_t ii) {return getInstance().conf_.at(ii);} static const Covariance_ & covariance(const size_t ii) {return *getInstance().covar_.at(ii);} private: @@ -53,16 +50,14 @@ template class ObsAuxIncrementFixture : private boost::noncopyabl } ObsAuxIncrementFixture() { - TestEnvironment::config().get("observations", conf_); for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - std::shared_ptr tmp(new Covariance_(Test_::obspace()[jj], conf_[jj])); + std::shared_ptr tmp(new Covariance_(Test_::obspace()[jj], Test_::config(jj))); covar_.push_back(tmp); } } ~ObsAuxIncrementFixture() {} - std::vector conf_; std::vector > covar_; }; @@ -70,11 +65,10 @@ template class ObsAuxIncrementFixture : private boost::noncopyabl template void testObsAuxIncrementConstructor() { typedef ObsTestsFixture Test_; - typedef ObsAuxIncrementFixture AuxTest_; typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - AuxIncr_ dx(Test_::obspace()[jj], AuxTest_::config(jj)); + AuxIncr_ dx(Test_::obspace()[jj], Test_::config(jj)); EXPECT(dx.norm() == 0.0); } } @@ -83,12 +77,11 @@ template void testObsAuxIncrementConstructor() { template void testObsAuxIncrementCopyConstructor() { typedef ObsTestsFixture Test_; - typedef ObsAuxIncrementFixture AuxTest_; typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - if (AuxTest_::config(jj).has("obs bias")) { - AuxIncr_ dx1(Test_::obspace()[jj], AuxTest_::config(jj)); + if (Test_::config(jj).has("obs bias")) { + AuxIncr_ dx1(Test_::obspace()[jj], Test_::config(jj)); ObsAuxIncrementFixture::covariance(jj).randomize(dx1); AuxIncr_ dx2(dx1); @@ -110,11 +103,11 @@ template void testObsAuxIncrementChangeRes() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - if (AuxTest_::config(jj).has("obs bias")) { - AuxIncr_ dx1(Test_::obspace()[jj], AuxTest_::config(jj)); - ObsAuxIncrementFixture::covariance(jj).randomize(dx1); + if (Test_::config(jj).has("obs bias")) { + AuxIncr_ dx1(Test_::obspace()[jj], Test_::config(jj)); + AuxTest_::covariance(jj).randomize(dx1); - AuxIncr_ dx2(dx1, AuxTest_::config(jj)); + AuxIncr_ dx2(dx1, Test_::config(jj)); EXPECT(dx2.norm() > 0.0); EXPECT(dx2.norm() == dx1.norm()); @@ -133,10 +126,10 @@ template void testObsAuxIncrementTriangle() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - if (AuxTest_::config(jj).has("obs bias")) { - AuxIncr_ dx1(Test_::obspace()[jj], AuxTest_::config(jj)); - ObsAuxIncrementFixture::covariance(jj).randomize(dx1); - AuxIncr_ dx2(Test_::obspace()[jj], AuxTest_::config(jj)); + if (Test_::config(jj).has("obs bias")) { + AuxIncr_ dx1(Test_::obspace()[jj], Test_::config(jj)); + AuxTest_::covariance(jj).randomize(dx1); + AuxIncr_ dx2(Test_::obspace()[jj], Test_::config(jj)); ObsAuxIncrementFixture::covariance(jj).randomize(dx2); // test triangle inequality @@ -163,8 +156,8 @@ template void testObsAuxIncrementOpPlusEq() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - AuxIncr_ dx1(Test_::obspace()[jj], AuxTest_::config(jj)); - ObsAuxIncrementFixture::covariance(jj).randomize(dx1); + AuxIncr_ dx1(Test_::obspace()[jj], Test_::config(jj)); + AuxTest_::covariance(jj).randomize(dx1); AuxIncr_ dx2(dx1); // test *= and += @@ -184,10 +177,10 @@ template void testObsAuxIncrementDotProduct() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - AuxIncr_ dx1(Test_::obspace()[jj], AuxTest_::config(jj)); - ObsAuxIncrementFixture::covariance(jj).randomize(dx1); - AuxIncr_ dx2(Test_::obspace()[jj], AuxTest_::config(jj)); - ObsAuxIncrementFixture::covariance(jj).randomize(dx2); + AuxIncr_ dx1(Test_::obspace()[jj], Test_::config(jj)); + AuxTest_::covariance(jj).randomize(dx1); + AuxIncr_ dx2(Test_::obspace()[jj], Test_::config(jj)); + AuxTest_::covariance(jj).randomize(dx2); // test symmetry of dot product double zz1 = dot_product(dx1, dx2); @@ -205,8 +198,8 @@ template void testObsAuxIncrementZero() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - AuxIncr_ dx(Test_::obspace()[jj], AuxTest_::config(jj)); - ObsAuxIncrementFixture::covariance(jj).randomize(dx); + AuxIncr_ dx(Test_::obspace()[jj], Test_::config(jj)); + AuxTest_::covariance(jj).randomize(dx); EXPECT(dx.norm() > 0.0); // test zero @@ -223,8 +216,8 @@ template void testObsAuxIncrementAxpy() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - AuxIncr_ dx1(Test_::obspace()[jj], AuxTest_::config(jj)); - ObsAuxIncrementFixture::covariance(jj).randomize(dx1); + AuxIncr_ dx1(Test_::obspace()[jj], Test_::config(jj)); + AuxTest_::covariance(jj).randomize(dx1); // test axpy AuxIncr_ dx2(dx1); diff --git a/src/test/interface/ObsOperator.h b/src/test/interface/ObsOperator.h index ac3615ab8..f963bdeff 100644 --- a/src/test/interface/ObsOperator.h +++ b/src/test/interface/ObsOperator.h @@ -37,11 +37,8 @@ template void testConstructor() { typedef ObsTestsFixture Test_; typedef oops::ObsOperator ObsOperator_; - std::vector conf; - TestEnvironment::config().get("observations", conf); - for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - eckit::LocalConfiguration obsopconf(conf[jj], "obs operator"); + eckit::LocalConfiguration obsopconf(Test_::config(jj), "obs operator"); std::unique_ptr hop(new ObsOperator_(Test_::obspace()[jj], obsopconf)); EXPECT(hop.get()); @@ -60,20 +57,18 @@ template void testSimulateObs() { typedef oops::ObsOperator ObsOperator_; typedef oops::ObsVector ObsVector_; - std::vector conf; - TestEnvironment::config().get("observations", conf); - for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + const eckit::LocalConfiguration & conf = Test_::config(jj); // initialize observation operator (set variables requested from the model, // variables simulated by the observation operator, other init) - eckit::LocalConfiguration obsopconf(conf[jj], "obs operator"); + eckit::LocalConfiguration obsopconf(conf, "obs operator"); ObsOperator_ hop(Test_::obspace()[jj], obsopconf); // initialize bias correction - const ObsAuxCtrl_ ybias(Test_::obspace()[jj], conf[jj]); + const ObsAuxCtrl_ ybias(Test_::obspace()[jj], conf); // read geovals from the file - eckit::LocalConfiguration gconf(conf[jj], "geovals"); + eckit::LocalConfiguration gconf(conf, "geovals"); oops::Variables hopvars = hop.requiredVars(); hopvars += ybias.requiredVars(); const GeoVaLs_ gval(gconf, Test_::obspace()[jj], hopvars); @@ -91,25 +86,25 @@ template void testSimulateObs() { hofx.save("hofx"); // apply bias correction if it is required - if (conf[jj].has("obs bias")) { + if (conf.has("obs bias")) { const ObsVector_ bias(Test_::obspace()[jj], "ObsBias", false); hofx += bias; } - const double tol = conf[jj].getDouble("tolerance"); - if (conf[jj].has("vector ref")) { + const double tol = conf.getDouble("tolerance"); + if (conf.has("vector ref")) { // if reference h(x) is saved in file as a vector, read from file // and compare the norm of difference to zero - ObsVector_ obsref(Test_::obspace()[jj], conf[jj].getString("vector ref")); + ObsVector_ obsref(Test_::obspace()[jj], conf.getString("vector ref")); obsref -= hofx; const double zz = obsref.rms(); oops::Log::info() << "Vector difference between reference and computed: " << obsref; EXPECT(zz < 100*tol); // change tol from percent to actual value. // tol used in is_close is relative - } else if (conf[jj].has("norm ref")) { + } else if (conf.has("norm ref")) { // if reference h(x) is saved in file as a vector, read from file // and compare the difference, normalised by the reference values to zero - ObsVector_ obsref(Test_::obspace()[jj], conf[jj].getString("norm ref")); + ObsVector_ obsref(Test_::obspace()[jj], conf.getString("norm ref")); obsref -= hofx; obsref /= hofx; const double zz = obsref.rms(); @@ -120,7 +115,7 @@ template void testSimulateObs() { } else { // else compare h(x) norm to the norm from the config const double zz = hofx.rms(); - const double xx = conf[jj].getDouble("rms ref"); + const double xx = conf.getDouble("rms ref"); oops::Log::debug() << "zz: " << std::fixed << std::setprecision(8) << zz << std::endl; oops::Log::debug() << "xx: " << std::fixed << std::setprecision(8) << xx << std::endl; diff --git a/src/test/interface/ObsTestsFixture.h b/src/test/interface/ObsTestsFixture.h index d671d4e94..a00bfe570 100644 --- a/src/test/interface/ObsTestsFixture.h +++ b/src/test/interface/ObsTestsFixture.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2017-2018 UCAR + * (C) Copyright 2017-2020 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -9,7 +9,7 @@ #define TEST_INTERFACE_OBSTESTSFIXTURE_H_ #include -#include +#include #include @@ -18,20 +18,25 @@ #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" #include "oops/util/DateTime.h" -#include "oops/util/Logger.h" #include "test/TestEnvironment.h" namespace test { // ----------------------------------------------------------------------------- - +/// Fixture for observations-related tests +/// Gets created only once per test runs, ObsSpaces, configuration under `observations` +/// and window characteristics get saved template class ObsTestsFixture : private boost::noncopyable { typedef oops::ObsSpaces ObsSpaces_; public: + /// accessors to observation window static const util::DateTime & tbgn() {return *getInstance().tbgn_;} static const util::DateTime & tend() {return *getInstance().tend_;} + /// accessor to a jj-th obs type config + static eckit::LocalConfiguration & config(size_t jj) {return getInstance().configs_.at(jj);} + /// accessor to a all obs spaces static ObsSpaces_ & obspace() {return *getInstance().ospaces_;} static void reset() { @@ -44,7 +49,10 @@ class ObsTestsFixture : private boost::noncopyable { ObsTestsFixture(): tbgn_(), tend_(), ospaces_() { tbgn_.reset(new util::DateTime(TestEnvironment::config().getString("window begin"))); tend_.reset(new util::DateTime(TestEnvironment::config().getString("window end"))); - ospaces_.reset(new ObsSpaces_(TestEnvironment::config(), oops::mpi::world(), *tbgn_, *tend_)); + configs_ = TestEnvironment::config().getSubConfigurations("observations"); + eckit::LocalConfiguration obsconfig = + TestEnvironment::config().getSubConfiguration("observations"); + ospaces_.reset(new ObsSpaces_(obsconfig, oops::mpi::world(), *tbgn_, *tend_)); } ~ObsTestsFixture() {} @@ -56,6 +64,7 @@ class ObsTestsFixture : private boost::noncopyable { std::unique_ptr tbgn_; std::unique_ptr tend_; + std::vector configs_; std::unique_ptr ospaces_; }; From 041635167f97d1c04079a6d5b5955ce9846a5d95 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Thu, 5 Nov 2020 11:18:15 -0700 Subject: [PATCH 009/142] add zero-obs test to L95 (commented for now) (#956) * add zero-obs test to L95; try to fix DRIPCGMinimizer * remove changes in minimizer; comment the test out for now --- l95/test/CMakeLists.txt | 9 ++++++ l95/test/testdata/noobs.obt | 6 ++++ l95/test/testinput/3dvar_noobs.yaml | 45 ++++++++++++++++++++++++++++ l95/test/testoutput/3dvar_noobs.test | 17 +++++++++++ 4 files changed, 77 insertions(+) create mode 100644 l95/test/testdata/noobs.obt create mode 100644 l95/test/testinput/3dvar_noobs.yaml create mode 100644 l95/test/testoutput/3dvar_noobs.test diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index e0a7dc2af..84141293b 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -2,6 +2,7 @@ include ( oops_add_test ) list( APPEND l95_test_input testinput/3dvar.yaml + testinput/3dvar_noobs.yaml testinput/3dvar_qc.yaml testinput/3dvar_qc_obserr.yaml testinput/3dvar_qc_iterations.yaml @@ -95,10 +96,12 @@ list( APPEND l95_test_data truth.2010-01-02T00:00:00Z.obt simplifiedl95.fc simplifiedl95.truth.obt + noobs.obt ) list( APPEND l95_testoutput testoutput/3dvar.test + testoutput/3dvar_noobs.test testoutput/3dvar_qc.test testoutput/3dvar_qc_obserr.test testoutput/3dvar_qc_iterations.test @@ -469,6 +472,12 @@ oops_add_test( TESTNAME 3dvar EXENAME l95_4dvar.x TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) +#oops_add_test( TESTNAME 3dvar_noobs +# MODELNAME l95 +# YAMLNAME testinput/3dvar_noobs.yaml +# EXENAME l95_4dvar.x +# TEST_DEPENDS test_l95_forecast ) + oops_add_test( TESTNAME 3dvar_qc MODELNAME l95 YAMLNAME testinput/3dvar_qc.yaml diff --git a/l95/test/testdata/noobs.obt b/l95/test/testdata/noobs.obt new file mode 100644 index 000000000..205809877 --- /dev/null +++ b/l95/test/testdata/noobs.obt @@ -0,0 +1,6 @@ +4 +EffectiveError +EffectiveQC +ObsError +ObsValue +0 diff --git a/l95/test/testinput/3dvar_noobs.yaml b/l95/test/testinput/3dvar_noobs.yaml new file mode 100644 index 000000000..288f0ce07 --- /dev/null +++ b/l95/test/testinput/3dvar_noobs.yaml @@ -0,0 +1,45 @@ +cost function: + cost type: 3D-Var + window begin: 2010-01-01T21:00:00Z + window length: PT6H + geometry: + resol: 40 + analysis variables: [x] + background: + date: 2010-01-02T00:00:00Z + filename: Data/test.fc.2010-01-01T00:00:00Z.P1D + background error: + covariance model: L95Error + date: 2010-01-02T00:00:00Z + length_scale: 1.0 + standard_deviation: 0.6 + observations: + - obs operator: {} + obs space: + obsdatain: Data/noobs.obt + obs error: + covariance model: diagonal +variational: + minimizer: + algorithm: DRIPCG + iterations: + - ninner: 10 + gradient norm reduction: 1e-10 + geometry: + resol: 40 + diagnostics: + departures: ombg + test: on + - ninner: 10 + gradient norm reduction: 1e-10 + geometry: + resol: 40 + test: on +final: + diagnostics: + departures: oman +output: + datadir: Data + exp: test + frequency: PT6H + type: an diff --git a/l95/test/testoutput/3dvar_noobs.test b/l95/test/testoutput/3dvar_noobs.test new file mode 100644 index 000000000..f5053155d --- /dev/null +++ b/l95/test/testoutput/3dvar_noobs.test @@ -0,0 +1,17 @@ +Test : CostJb : Nonlinear Jb = 0 +Test : CostJo : Nonlinear Jo(Lorenz 95) = 0 --- No Observations +Test : CostFunction: Nonlinear J = 0 +Test : DRIPCGMinimizer: reduction in residual norm = 1 +Test : CostFunction::addIncrement: Analysis: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=6.65953, Max=9.39191, Average=7.97085 +Test : CostJb : Nonlinear Jb = 0 +Test : CostJo : Nonlinear Jo(Lorenz 95) = 0 --- No Observations +Test : CostFunction: Nonlinear J = 0 +Test : DRIPCGMinimizer: reduction in residual norm = 1 +Test : CostFunction::addIncrement: Analysis: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=6.65953, Max=9.39191, Average=7.97085 +Test : CostJb : Nonlinear Jb = 0 +Test : CostJo : Nonlinear Jo(Lorenz 95) = 0 --- No Observations +Test : CostFunction: Nonlinear J = 0 From 0a4f641314378bef6b57f53b14ee6241e91f368b Mon Sep 17 00:00:00 2001 From: Steven Vahl <42982505+svahl991@users.noreply.github.com> Date: Tue, 10 Nov 2020 09:14:22 -0700 Subject: [PATCH 010/142] Split lines on equal symbol during compare (#959) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Split lines on equal symbol during compare * trigger pipeline Co-authored-by: Yannick Trémolet <30638944+ytremolet@users.noreply.github.com> --- tools/compare.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/compare.py b/tools/compare.py index 4e4401b08..7864824b1 100755 --- a/tools/compare.py +++ b/tools/compare.py @@ -30,9 +30,9 @@ # and compares them one by one def line_diff(line1,line2,lnum,ftol,idif): - #Split line by whitespace - sline1 = re.split('\s+', line1) - sline2 = re.split('\s+', line2) + #Split line by whitespace or '=' + sline1 = re.split('\s+|=', line1) + sline2 = re.split('\s+|=', line2) lineerror = 0 From 349e1ac8b29477f7cb6ed8037de57cf018dce7c0 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Tue, 10 Nov 2020 15:04:11 -0700 Subject: [PATCH 011/142] add l95 letkf with qc test (#963) * add l95 letkf with qc test (+ bugfix) * rollback masking H(x) with QC flags --- l95/src/lorenz95/ObsLocGC99.cc | 13 ++++----- l95/src/lorenz95/ObsLocGC99.h | 3 +- l95/test/CMakeLists.txt | 8 ++++++ l95/test/testinput/letkf_qc.yaml | 47 ++++++++++++++++++++++++++++++ l95/test/testoutput/letkf_qc.test | 48 +++++++++++++++++++++++++++++++ src/oops/base/Observations.h | 12 ++++++++ 6 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 l95/test/testinput/letkf_qc.yaml create mode 100644 l95/test/testoutput/letkf_qc.test diff --git a/l95/src/lorenz95/ObsLocGC99.cc b/l95/src/lorenz95/ObsLocGC99.cc index 1d5072ab8..d6f73c4b0 100644 --- a/l95/src/lorenz95/ObsLocGC99.cc +++ b/l95/src/lorenz95/ObsLocGC99.cc @@ -15,6 +15,7 @@ #include "oops/generic/gc99.h" #include "oops/interface/ObsLocalization.h" +#include "oops/util/missingValues.h" // ----------------------------------------------------------------------------- namespace lorenz95 { @@ -31,16 +32,14 @@ ObsLocGC99::ObsLocGC99(const eckit::Configuration & config, const ObsTableView & // ----------------------------------------------------------------------------- -ObsLocGC99::~ObsLocGC99() {} - -// ----------------------------------------------------------------------------- - void ObsLocGC99::multiply(ObsVec1D & dy) const { const std::vector & obsdist = obsdb_.obsdist(); - double gc; + double missing = util::missingValue(missing); for (unsigned int ii=0; ii < dy.nobs(); ++ii) { - gc = oops::gc99(obsdist[ii]/rscale_); - dy[ii] = dy[ii]*gc; + if (dy[ii] != missing) { + double gc = oops::gc99(obsdist[ii]/rscale_); + dy[ii] = dy[ii]*gc; + } } } diff --git a/l95/src/lorenz95/ObsLocGC99.h b/l95/src/lorenz95/ObsLocGC99.h index bee4d944b..3b3cb7777 100644 --- a/l95/src/lorenz95/ObsLocGC99.h +++ b/l95/src/lorenz95/ObsLocGC99.h @@ -31,11 +31,10 @@ class ObsLocGC99: public util::Printable { static const std::string classname() {return "lorenz95::ObsLocGC99";} ObsLocGC99(const eckit::Configuration &, const ObsTableView &); - ~ObsLocGC99(); void multiply(ObsVec1D &) const; private: - void print(std::ostream &) const; + void print(std::ostream &) const override; const ObsTableView & obsdb_; const double rscale_; }; diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index 84141293b..f74f28089 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -64,6 +64,7 @@ list( APPEND l95_test_input testinput/identitymodel.yaml testinput/letkfGSI.yaml testinput/letkf.yaml + testinput/letkf_qc.yaml testinput/interfaces.yaml testinput/linearmodelfactory.yaml testinput/makeobs3d.yaml @@ -145,6 +146,7 @@ list( APPEND l95_testoutput testoutput/hofx.nomodel.test testoutput/hofx_for_getkf_nomodel.test testoutput/letkf.test + testoutput/letkf_qc.test testoutput/letkf_gsi.test testoutput/localhofx.test testoutput/makeobs3d.test @@ -724,6 +726,12 @@ oops_add_test( TESTNAME letkf EXENAME l95_letkf.x TEST_DEPENDS test_l95_makeobs3d test_l95_genenspert ) +oops_add_test( TESTNAME letkf_qc + MODELNAME l95 + YAMLNAME testinput/letkf_qc.yaml + EXENAME l95_letkf.x + TEST_DEPENDS test_l95_makeobs3d test_l95_genenspert ) + oops_add_test( TESTNAME letkf_gsi MODELNAME l95 YAMLNAME testinput/letkfGSI.yaml diff --git a/l95/test/testinput/letkf_qc.yaml b/l95/test/testinput/letkf_qc.yaml new file mode 100644 index 000000000..93675cfce --- /dev/null +++ b/l95/test/testinput/letkf_qc.yaml @@ -0,0 +1,47 @@ +window begin: 2010-01-01T21:00:00Z +window length: PT6H + +geometry: + resol: 40 + +# use 3D for middle of the window +background: + members: + - date: &date 2010-01-02T00:00:00Z + filename: Data/test.ens.1.2010-01-01T00:00:00Z.P1D + - date: *date + filename: Data/test.ens.2.2010-01-01T00:00:00Z.P1D + - date: *date + filename: Data/test.ens.3.2010-01-01T00:00:00Z.P1D + - date: *date + filename: Data/test.ens.4.2010-01-01T00:00:00Z.P1D + - date: *date + filename: Data/test.ens.5.2010-01-01T00:00:00Z.P1D + +observations: +- obs error: + covariance model: localized diagonal + localization: + localization method: Gaspari-Cohn + lengthscale: .1 + obs space: + obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt + obs operator: {} + obs filters: + - filter: Background Check + threshold: 3.0 + +driver: + +local ensemble DA: + solver: LETKF + inflation: + rtps: 0.5 + rtpp: 0.5 + mult: 1.1 + +output: + datadir: Data + date: *date + exp: letkf.%{member}% + type: an diff --git a/l95/test/testoutput/letkf_qc.test b/l95/test/testoutput/letkf_qc.test new file mode 100644 index 000000000..3b9b72a5a --- /dev/null +++ b/l95/test/testoutput/letkf_qc.test @@ -0,0 +1,48 @@ +Test : Initial state for member 1: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=3.53609, Max=11.1974, Average=7.90978 +Test : Initial state for member 2: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=3.77507, Max=11.1844, Average=7.75047 +Test : Initial state for member 3: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=5.36882, Max=10.844, Average=7.80576 +Test : Initial state for member 4: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=4.14639, Max=11.6734, Average=7.76001 +Test : Initial state for member 5: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=3.17563, Max=10.8183, Average=7.60243 +Test : H(x) for member 1: +Test : Lorenz 95 nobs= 120 Min=3.53609, Max=11.1974, Average=7.90978 +Test : H(x) for member 2: +Test : Lorenz 95 nobs= 120 Min=3.77507, Max=11.1844, Average=7.75047 +Test : H(x) for member 3: +Test : Lorenz 95 nobs= 120 Min=5.36882, Max=10.844, Average=7.80576 +Test : H(x) for member 4: +Test : Lorenz 95 nobs= 120 Min=4.14639, Max=11.6734, Average=7.76001 +Test : H(x) for member 5: +Test : Lorenz 95 nobs= 120 Min=3.17563, Max=10.8183, Average=7.60243 +Test : H(x) ensemble background mean: +Test : Lorenz 95 nobs= 120 Min=5.25199, Max=9.50208, Average=7.76569 +Test : background y - H(x): +Test : Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 +Test : Background mean : +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=5.25199, Max=9.50208, Average=7.76569 +Test : H(x) for member 1: +Test : Lorenz 95 nobs= 120 Min=6.31712, Max=10.1291, Average=8.09238 +Test : H(x) for member 2: +Test : Lorenz 95 nobs= 120 Min=5.62763, Max=9.94295, Average=7.96969 +Test : H(x) for member 3: +Test : Lorenz 95 nobs= 120 Min=5.98634, Max=10.5924, Average=8.01273 +Test : H(x) for member 4: +Test : Lorenz 95 nobs= 120 Min=4.57542, Max=11.5295, Average=7.98536 +Test : H(x) for member 5: +Test : Lorenz 95 nobs= 120 Min=4.96477, Max=9.69106, Average=7.857 +Test : H(x) ensemble analysis mean: +Test : Lorenz 95 nobs= 120 Min=6.73044, Max=9.19002, Average=7.98343 +Test : analysis y - H(x): +Test : Lorenz 95 nobs= 120 Min=-0.3032, Max=0.441464, Average=0.0279604 +Test : ombg RMS: 0.864096 +Test : oman RMS: 0.149894 diff --git a/src/oops/base/Observations.h b/src/oops/base/Observations.h index abc57310d..78dd40722 100644 --- a/src/oops/base/Observations.h +++ b/src/oops/base/Observations.h @@ -20,6 +20,7 @@ #include "oops/base/Departures.h" #include "oops/base/ObsErrors.h" #include "oops/base/ObsSpaces.h" +#include "oops/base/QCData.h" #include "oops/interface/ObsVector.h" #include "oops/util/Logger.h" #include "oops/util/Printable.h" @@ -37,6 +38,7 @@ template class Observations : public util::Printable { typedef ObsErrors ObsErrors_; typedef ObsSpaces ObsSpaces_; typedef ObsVector ObsVector_; + typedef QCData QCData_; public: /// \brief create Observations for all obs (read from ObsSpace if name is specified) @@ -72,6 +74,9 @@ template class Observations : public util::Printable { /// Perturbations void perturb(const ObsErrors_ &); +/// Mask out departures where the passed in qc flags are > 0 + void mask(const QCData_ &); + private: void print(std::ostream &) const; size_t nobs() const; @@ -194,6 +199,13 @@ size_t Observations::nobs() const { return nobs; } // ----------------------------------------------------------------------------- +template +void Observations::mask(const QCData_ & qc) { + for (size_t ii = 0; ii < obs_.size(); ++ii) { + obs_[ii].mask(*qc.qcFlags(ii)); + } +} +// ----------------------------------------------------------------------------- template void Observations::perturb(const ObsErrors_ & Rmat) { Departures_ ypert(obsdb_); From f02918c28ee39c91366b961cef558df8cbc62ed2 Mon Sep 17 00:00:00 2001 From: Fabio L R Diniz <45880035+fabiolrdiniz@users.noreply.github.com> Date: Tue, 10 Nov 2020 19:37:59 -0300 Subject: [PATCH 012/142] Feature/save increment (#955) * adding to DRMinimizer.h * moving write increment from DRMinimizer to Minimizer * adding scaling factor back * removing scaling factor * adding a test for l95 * adding a test for qg * removing additional tests for save increment Co-authored-by: Anna Shlyaeva --- l95/test/testinput/3dvar.yaml | 14 ++++++++++++++ qg/test/testinput/3dvar.yaml | 15 +++++++++++++++ src/oops/assimilation/Minimizer.h | 31 +++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/l95/test/testinput/3dvar.yaml b/l95/test/testinput/3dvar.yaml index 8cf5df123..63a382223 100644 --- a/l95/test/testinput/3dvar.yaml +++ b/l95/test/testinput/3dvar.yaml @@ -31,11 +31,25 @@ variational: diagnostics: departures: ombg test: on + online diagnostics: + write increment: true + increment: + datadir: Data + date: 2010-01-02T00:00:00Z + exp: test.iter1 + type: in - ninner: 10 gradient norm reduction: 1e-10 geometry: resol: 40 test: on + online diagnostics: + write increment: true + increment: + datadir: Data + date: 2010-01-02T00:00:00Z + exp: test.iter2 + type: in final: diagnostics: departures: oman diff --git a/qg/test/testinput/3dvar.yaml b/qg/test/testinput/3dvar.yaml index dbd22ea23..a89267046 100644 --- a/qg/test/testinput/3dvar.yaml +++ b/qg/test/testinput/3dvar.yaml @@ -60,6 +60,13 @@ variational: ny: 10 depths: [4500.0, 5500.0] test: on + online diagnostics: + write increment: true + increment: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: 3dvar.iter1 + type: in - diagnostics: departures: ombg gradient norm reduction: 1.0e-10 @@ -69,6 +76,14 @@ variational: ny: 20 depths: [4500.0, 5500.0] test: on + online diagnostics: + write increment: true + increment: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: 3dvar.iter2 + type: in + analysis variables: [x] final: diagnostics: departures: oman diff --git a/src/oops/assimilation/Minimizer.h b/src/oops/assimilation/Minimizer.h index bab8d0169..38c6f5115 100644 --- a/src/oops/assimilation/Minimizer.h +++ b/src/oops/assimilation/Minimizer.h @@ -59,6 +59,7 @@ template class Minimizer : private boost::noncopya void tlmApproxTest(const H_ &); void tlmTaylorTest(const H_ &); + void writeIncrement(const eckit::Configuration & config, const CtrlInc_ &); void tlmPropagTest(const eckit::Configuration & config, const CtrlInc_ &); const CostFct_ & J_; @@ -79,6 +80,9 @@ Minimizer::minimize(const eckit::Configuration & config) { // Minimize ControlIncrement * dx = this->doMinimize(config); + // Write increment + this->writeIncrement(config, *dx); + // TLM propagation test this->tlmPropagTest(config, *dx); @@ -90,6 +94,33 @@ Minimizer::minimize(const eckit::Configuration & config) { // ----------------------------------------------------------------------------- +template +void Minimizer::writeIncrement(const eckit::Configuration & config, + const CtrlInc_ & dx) { +// Write out the increment + + if (config.has("online diagnostics")) { + const eckit::LocalConfiguration onlineDiag(config, "online diagnostics"); + bool writeinc = onlineDiag.getBool("write increment", false); + + if (writeinc) { + // print log + Log::info() << "Write Increment - starting: " << outerIteration_ + << std::endl << std::endl; + + const eckit::LocalConfiguration incConf(config, "increment"); + + // write increment + dx.write(incConf); + + // print log + Log::info() << std::endl << "Write Increment: done." << std::endl; + } + } +} + +// ----------------------------------------------------------------------------- + template void Minimizer::tlmTests(const eckit::Configuration & config) { // Tangent Linear tests From 0ff544d0c3d5b31a4eb2ec54348211707be854b9 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Fri, 13 Nov 2020 16:27:19 -0700 Subject: [PATCH 013/142] let's see OOPS_TRACE again! (#968) --- src/oops/base/ModelBase.h | 2 -- src/oops/runs/Run.cc | 1 - 2 files changed, 3 deletions(-) diff --git a/src/oops/base/ModelBase.h b/src/oops/base/ModelBase.h index 7ff89ef57..6df48e688 100644 --- a/src/oops/base/ModelBase.h +++ b/src/oops/base/ModelBase.h @@ -196,12 +196,10 @@ class ModelMaker : public ModelFactory { template ModelFactory::ModelFactory(const std::string & name) { - Log::trace() << "ModelFactory::ModelFactory starting" << std::endl; if (getMakers().find(name) != getMakers().end()) { throw std::runtime_error(name + " already registered in the model factory."); } getMakers()[name] = this; - Log::trace() << "ModelFactory::ModelFactory done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/runs/Run.cc b/src/oops/runs/Run.cc index da4229c36..5a04d837d 100644 --- a/src/oops/runs/Run.cc +++ b/src/oops/runs/Run.cc @@ -104,7 +104,6 @@ Run::Run(int argc, char** argv) : eckit::Main(argc, argv, "OOPS_HOME"), config_( Run::~Run() { LibOOPS::instance().finalise(); // Finalize MPI and logs - Log::trace() << "Oops::Run destructed" << std::endl; } // ----------------------------------------------------------------------------- From 924019e1d90d9178fd15657b0b7e143f47a53d7e Mon Sep 17 00:00:00 2001 From: Claude Gibert Date: Tue, 24 Nov 2020 19:08:32 +0100 Subject: [PATCH 014/142] changes of metadata for ewok. (#975) --- l95/defaults/an.yaml | 2 -- l95/defaults/anout.yaml | 7 ---- l95/defaults/bg.yaml | 2 -- l95/defaults/model.py | 34 ------------------- l95/defaults/obs.yaml | 9 ----- l95/{defaults/README => ewok/README.md} | 0 l95/ewok/defaults/an.yaml | 2 ++ l95/ewok/defaults/an_output.yaml | 7 ++++ l95/ewok/defaults/bg.yaml | 2 ++ l95/{ => ewok}/defaults/bstatic.yaml | 0 l95/{ => ewok}/defaults/drplanclmp.yaml | 0 l95/{ => ewok}/defaults/drplanczos.yaml | 0 l95/ewok/defaults/fc.yaml | 2 ++ .../defaults/fc_output.yaml} | 2 +- l95/{ => ewok}/defaults/jc.yaml | 0 l95/{ => ewok}/defaults/model.yaml | 0 l95/{ => ewok}/defaults/ob.yaml | 0 l95/ewok/defaults/obs.yaml | 9 +++++ l95/{ => ewok}/defaults/resol.yaml | 0 l95/{ => ewok}/defaults/tlm.yaml | 0 qg/configs/fc.yaml | 2 -- qg/configs/model.py | 33 ------------------ qg/{configs/README => ewok/README.md} | 0 qg/{configs => ewok/defaults}/an.yaml | 1 - .../defaults/an_output.yaml} | 0 qg/{configs => ewok/defaults}/bg.yaml | 0 qg/{configs => ewok/defaults}/bstatic.yaml | 0 qg/{configs => ewok/defaults}/drplanclmp.yaml | 0 qg/{configs => ewok/defaults}/drplanczos.yaml | 0 {l95 => qg/ewok}/defaults/fc.yaml | 2 +- .../defaults/fc_output.yaml} | 0 qg/{configs => ewok/defaults}/geom.40x20.yaml | 0 qg/{configs => ewok/defaults}/jc.yaml | 0 qg/{configs => ewok/defaults}/model.yaml | 0 qg/{configs => ewok/defaults}/ob.yaml | 0 qg/{configs => ewok/defaults}/stream.yaml | 0 qg/{configs => ewok/defaults}/tlm.yaml | 0 qg/{configs => ewok/defaults}/wind.yaml | 0 qg/{configs => ewok/defaults}/wspeed.yaml | 0 39 files changed, 24 insertions(+), 92 deletions(-) delete mode 100644 l95/defaults/an.yaml delete mode 100644 l95/defaults/anout.yaml delete mode 100644 l95/defaults/bg.yaml delete mode 100644 l95/defaults/model.py delete mode 100644 l95/defaults/obs.yaml rename l95/{defaults/README => ewok/README.md} (100%) create mode 100644 l95/ewok/defaults/an.yaml create mode 100644 l95/ewok/defaults/an_output.yaml create mode 100644 l95/ewok/defaults/bg.yaml rename l95/{ => ewok}/defaults/bstatic.yaml (100%) rename l95/{ => ewok}/defaults/drplanclmp.yaml (100%) rename l95/{ => ewok}/defaults/drplanczos.yaml (100%) create mode 100644 l95/ewok/defaults/fc.yaml rename l95/{defaults/fcout.yaml => ewok/defaults/fc_output.yaml} (60%) rename l95/{ => ewok}/defaults/jc.yaml (100%) rename l95/{ => ewok}/defaults/model.yaml (100%) rename l95/{ => ewok}/defaults/ob.yaml (100%) create mode 100644 l95/ewok/defaults/obs.yaml rename l95/{ => ewok}/defaults/resol.yaml (100%) rename l95/{ => ewok}/defaults/tlm.yaml (100%) delete mode 100644 qg/configs/fc.yaml delete mode 100644 qg/configs/model.py rename qg/{configs/README => ewok/README.md} (100%) rename qg/{configs => ewok/defaults}/an.yaml (76%) rename qg/{configs/anout.yaml => ewok/defaults/an_output.yaml} (100%) rename qg/{configs => ewok/defaults}/bg.yaml (100%) rename qg/{configs => ewok/defaults}/bstatic.yaml (100%) rename qg/{configs => ewok/defaults}/drplanclmp.yaml (100%) rename qg/{configs => ewok/defaults}/drplanczos.yaml (100%) rename {l95 => qg/ewok}/defaults/fc.yaml (69%) rename qg/{configs/fcout.yaml => ewok/defaults/fc_output.yaml} (100%) rename qg/{configs => ewok/defaults}/geom.40x20.yaml (100%) rename qg/{configs => ewok/defaults}/jc.yaml (100%) rename qg/{configs => ewok/defaults}/model.yaml (100%) rename qg/{configs => ewok/defaults}/ob.yaml (100%) rename qg/{configs => ewok/defaults}/stream.yaml (100%) rename qg/{configs => ewok/defaults}/tlm.yaml (100%) rename qg/{configs => ewok/defaults}/wind.yaml (100%) rename qg/{configs => ewok/defaults}/wspeed.yaml (100%) diff --git a/l95/defaults/an.yaml b/l95/defaults/an.yaml deleted file mode 100644 index bba1af04d..000000000 --- a/l95/defaults/an.yaml +++ /dev/null @@ -1,2 +0,0 @@ -date: '{{current_cycle}}' -filename: '$(experiment_dir)/{{current_cycle}}/$(experiment).an.{{current_cycle}}' diff --git a/l95/defaults/anout.yaml b/l95/defaults/anout.yaml deleted file mode 100644 index 3008726e5..000000000 --- a/l95/defaults/anout.yaml +++ /dev/null @@ -1,7 +0,0 @@ -datadir: '$(experiment_dir)/{{current_cycle}}' -filename: '$(experiment_dir)/{{current_cycle}}/$(experiment).an.{{current_cycle}}' -exp: $(experiment) -##first: '{% $(window_length) - $(window_offset) %}' -first: PT3H -frequency: PT06H -type: an diff --git a/l95/defaults/bg.yaml b/l95/defaults/bg.yaml deleted file mode 100644 index c2eaf19bd..000000000 --- a/l95/defaults/bg.yaml +++ /dev/null @@ -1,2 +0,0 @@ -date: '{{window_begin}}' -filename: '$(experiment_dir)/{{current_cycle}}/$(experiment).bg.{{current_cycle}}.l95' diff --git a/l95/defaults/model.py b/l95/defaults/model.py deleted file mode 100644 index 313fc8572..000000000 --- a/l95/defaults/model.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -from solo.date import JediDate, DateIncrement - -__all__ = ["fc_file", "obs_file", "r2d2_obsfile", "r2d2_anfile"] - - -def fc_file(fcout, step): - fc = {} - fc['date'] = fcout['date'] - step = DateIncrement(duration=step) - keys = [fcout['exp'], fcout['type'], fcout['date'], str(step)] - fname = '.'.join(keys) - fc['filename'] = os.path.join(fcout['datadir'], fname) - return fc - - -def obs_file(conf): - obsfile = conf['obsdatain'] - return obsfile - - -def r2d2_obsfile(conf, date): - sdate = JediDate(date) - r2d2keys = ['l95', conf['source'], str(sdate), 'obt'] - r2d2file = '.'.join(r2d2keys) - return r2d2file - - -def r2d2_anfile(conf, date): - sdate = JediDate(date) - r2d2keys = [conf['exp'], conf['type'], str(sdate), 'l95'] - r2d2file = '.'.join(r2d2keys) - return r2d2file - diff --git a/l95/defaults/obs.yaml b/l95/defaults/obs.yaml deleted file mode 100644 index 38010f1f2..000000000 --- a/l95/defaults/obs.yaml +++ /dev/null @@ -1,9 +0,0 @@ -obs operator: - obs type: Obs -obs space: - source: truth - filetype: obt - obsdatain: '$(experiment_dir)/{{current_cycle}}/input.obs.{{current_cycle}}.obt' - obsdataout: '$(experiment_dir)/{{current_cycle}}/$(experiment).obs.{{current_cycle}}.obt' -obs error: - covariance model: diagonal diff --git a/l95/defaults/README b/l95/ewok/README.md similarity index 100% rename from l95/defaults/README rename to l95/ewok/README.md diff --git a/l95/ewok/defaults/an.yaml b/l95/ewok/defaults/an.yaml new file mode 100644 index 000000000..34895a157 --- /dev/null +++ b/l95/ewok/defaults/an.yaml @@ -0,0 +1,2 @@ +date: '{{current_cycle}}' +filename: '$(current_dir)/$(experiment).an.{{current_cycle}}' diff --git a/l95/ewok/defaults/an_output.yaml b/l95/ewok/defaults/an_output.yaml new file mode 100644 index 000000000..022f9c00d --- /dev/null +++ b/l95/ewok/defaults/an_output.yaml @@ -0,0 +1,7 @@ +datadir: '$(current_dir)' +filename: '$(current_dir)/$(experiment).an.{{current_cycle}}' +exp: $(experiment) +##first: '{% $(window_length) - $(window_offset) %}' +first: PT3H +frequency: PT06H +type: an diff --git a/l95/ewok/defaults/bg.yaml b/l95/ewok/defaults/bg.yaml new file mode 100644 index 000000000..021233df2 --- /dev/null +++ b/l95/ewok/defaults/bg.yaml @@ -0,0 +1,2 @@ +date: '{{window_begin}}' +filename: '$(current_dir)/$(experiment).bg.{{current_cycle}}.l95' diff --git a/l95/defaults/bstatic.yaml b/l95/ewok/defaults/bstatic.yaml similarity index 100% rename from l95/defaults/bstatic.yaml rename to l95/ewok/defaults/bstatic.yaml diff --git a/l95/defaults/drplanclmp.yaml b/l95/ewok/defaults/drplanclmp.yaml similarity index 100% rename from l95/defaults/drplanclmp.yaml rename to l95/ewok/defaults/drplanclmp.yaml diff --git a/l95/defaults/drplanczos.yaml b/l95/ewok/defaults/drplanczos.yaml similarity index 100% rename from l95/defaults/drplanczos.yaml rename to l95/ewok/defaults/drplanczos.yaml diff --git a/l95/ewok/defaults/fc.yaml b/l95/ewok/defaults/fc.yaml new file mode 100644 index 000000000..c29dd966e --- /dev/null +++ b/l95/ewok/defaults/fc.yaml @@ -0,0 +1,2 @@ +date: '{{current_cycle}}' +filename: $(current_dir)/$(experiment).fc.{{current_cycle}}.$(step) diff --git a/l95/defaults/fcout.yaml b/l95/ewok/defaults/fc_output.yaml similarity index 60% rename from l95/defaults/fcout.yaml rename to l95/ewok/defaults/fc_output.yaml index 31394cd02..b958e5abe 100644 --- a/l95/defaults/fcout.yaml +++ b/l95/ewok/defaults/fc_output.yaml @@ -1,4 +1,4 @@ -datadir: $(experiment_dir)/{{current_cycle}} +datadir: $(current_dir) date: '{{current_cycle}}' exp: $(experiment) frequency: PT3H diff --git a/l95/defaults/jc.yaml b/l95/ewok/defaults/jc.yaml similarity index 100% rename from l95/defaults/jc.yaml rename to l95/ewok/defaults/jc.yaml diff --git a/l95/defaults/model.yaml b/l95/ewok/defaults/model.yaml similarity index 100% rename from l95/defaults/model.yaml rename to l95/ewok/defaults/model.yaml diff --git a/l95/defaults/ob.yaml b/l95/ewok/defaults/ob.yaml similarity index 100% rename from l95/defaults/ob.yaml rename to l95/ewok/defaults/ob.yaml diff --git a/l95/ewok/defaults/obs.yaml b/l95/ewok/defaults/obs.yaml new file mode 100644 index 000000000..f3f332e8c --- /dev/null +++ b/l95/ewok/defaults/obs.yaml @@ -0,0 +1,9 @@ +obs operator: + obs type: Obs +obs space: + source: truth + filetype: obt + obsdatain: '$(current_dir)/input.obs.{{current_cycle}}.obt' + obsdataout: '$(current_dir)/$(experiment).obs.{{current_cycle}}.obt' +obs error: + covariance model: diagonal diff --git a/l95/defaults/resol.yaml b/l95/ewok/defaults/resol.yaml similarity index 100% rename from l95/defaults/resol.yaml rename to l95/ewok/defaults/resol.yaml diff --git a/l95/defaults/tlm.yaml b/l95/ewok/defaults/tlm.yaml similarity index 100% rename from l95/defaults/tlm.yaml rename to l95/ewok/defaults/tlm.yaml diff --git a/qg/configs/fc.yaml b/qg/configs/fc.yaml deleted file mode 100644 index d81245250..000000000 --- a/qg/configs/fc.yaml +++ /dev/null @@ -1,2 +0,0 @@ -date: '{{current_cycle}}' -filename: $(experiment_dir)/{{current_cycle}}/$(experiment).fc.{{current_cycle}}.${step}.nc diff --git a/qg/configs/model.py b/qg/configs/model.py deleted file mode 100644 index 6d5282a3e..000000000 --- a/qg/configs/model.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -import ewok - -__all__ = ["fc_file", "obs_file", "r2d2_obsfile", "r2d2_anfile"] - - -def fc_file(fcout, step): - fc = {} - fc['date'] = fcout['date'] - keys = [fcout['exp'], fcout['type'], fcout['date'], ewok.jediformat(step), 'nc'] - fname = '.'.join(keys) - fc['filename'] = os.path.join(fcout['datadir'], fname) - return fc - - -def obs_file(conf): - obsfile = conf['obsdatain']['obsfile'] - return obsfile - - -def r2d2_obsfile(conf, date): - sdate = ewok.jediformat(date) - r2d2keys = [conf['source'], 'qg', sdate, conf['obs type'], 'obs', 'nc'] - r2d2file = '.'.join(r2d2keys) - return r2d2file - - -def r2d2_anfile(conf, date): - sdate = ewok.jediformat(date) - r2d2keys = [conf['exp'], 'qg', conf['type'], sdate, 'nc'] - r2d2file = '.'.join(r2d2keys) - return r2d2file - diff --git a/qg/configs/README b/qg/ewok/README.md similarity index 100% rename from qg/configs/README rename to qg/ewok/README.md diff --git a/qg/configs/an.yaml b/qg/ewok/defaults/an.yaml similarity index 76% rename from qg/configs/an.yaml rename to qg/ewok/defaults/an.yaml index a9904e7dd..91e469433 100644 --- a/qg/configs/an.yaml +++ b/qg/ewok/defaults/an.yaml @@ -1,2 +1 @@ -date: '{{current_cycle}}' filename: '$(experiment_dir)/{{current_cycle}}/$(experiment).an.{{current_cycle}}.nc' diff --git a/qg/configs/anout.yaml b/qg/ewok/defaults/an_output.yaml similarity index 100% rename from qg/configs/anout.yaml rename to qg/ewok/defaults/an_output.yaml diff --git a/qg/configs/bg.yaml b/qg/ewok/defaults/bg.yaml similarity index 100% rename from qg/configs/bg.yaml rename to qg/ewok/defaults/bg.yaml diff --git a/qg/configs/bstatic.yaml b/qg/ewok/defaults/bstatic.yaml similarity index 100% rename from qg/configs/bstatic.yaml rename to qg/ewok/defaults/bstatic.yaml diff --git a/qg/configs/drplanclmp.yaml b/qg/ewok/defaults/drplanclmp.yaml similarity index 100% rename from qg/configs/drplanclmp.yaml rename to qg/ewok/defaults/drplanclmp.yaml diff --git a/qg/configs/drplanczos.yaml b/qg/ewok/defaults/drplanczos.yaml similarity index 100% rename from qg/configs/drplanczos.yaml rename to qg/ewok/defaults/drplanczos.yaml diff --git a/l95/defaults/fc.yaml b/qg/ewok/defaults/fc.yaml similarity index 69% rename from l95/defaults/fc.yaml rename to qg/ewok/defaults/fc.yaml index 84db6a09e..db0154035 100644 --- a/l95/defaults/fc.yaml +++ b/qg/ewok/defaults/fc.yaml @@ -1 +1 @@ -filename: $(experiment_dir)/{{current_cycle}}/$(experiment).fc.{{current_cycle}}.${step} +filename: $(experiment_dir)/{{current_cycle}}/$(experiment).fc.{{current_cycle}}.$(step).nc diff --git a/qg/configs/fcout.yaml b/qg/ewok/defaults/fc_output.yaml similarity index 100% rename from qg/configs/fcout.yaml rename to qg/ewok/defaults/fc_output.yaml diff --git a/qg/configs/geom.40x20.yaml b/qg/ewok/defaults/geom.40x20.yaml similarity index 100% rename from qg/configs/geom.40x20.yaml rename to qg/ewok/defaults/geom.40x20.yaml diff --git a/qg/configs/jc.yaml b/qg/ewok/defaults/jc.yaml similarity index 100% rename from qg/configs/jc.yaml rename to qg/ewok/defaults/jc.yaml diff --git a/qg/configs/model.yaml b/qg/ewok/defaults/model.yaml similarity index 100% rename from qg/configs/model.yaml rename to qg/ewok/defaults/model.yaml diff --git a/qg/configs/ob.yaml b/qg/ewok/defaults/ob.yaml similarity index 100% rename from qg/configs/ob.yaml rename to qg/ewok/defaults/ob.yaml diff --git a/qg/configs/stream.yaml b/qg/ewok/defaults/stream.yaml similarity index 100% rename from qg/configs/stream.yaml rename to qg/ewok/defaults/stream.yaml diff --git a/qg/configs/tlm.yaml b/qg/ewok/defaults/tlm.yaml similarity index 100% rename from qg/configs/tlm.yaml rename to qg/ewok/defaults/tlm.yaml diff --git a/qg/configs/wind.yaml b/qg/ewok/defaults/wind.yaml similarity index 100% rename from qg/configs/wind.yaml rename to qg/ewok/defaults/wind.yaml diff --git a/qg/configs/wspeed.yaml b/qg/ewok/defaults/wspeed.yaml similarity index 100% rename from qg/configs/wspeed.yaml rename to qg/ewok/defaults/wspeed.yaml From 5598b57fdbd81ee99025b11e5d4aa1b1fc7434b3 Mon Sep 17 00:00:00 2001 From: Fabio L R Diniz <45880035+fabiolrdiniz@users.noreply.github.com> Date: Tue, 1 Dec 2020 14:01:18 -0300 Subject: [PATCH 015/142] Feature/read fcst sensitivity (#969) * adding to DRMinimizer.h * moving inputFcstSens to be nested inside the fsoi section * moving inputFcstSens to be nested inside the fsoi section * adding modification for PrimalMinimizer.h --- src/oops/assimilation/DRMinimizer.h | 10 ++++++++-- src/oops/assimilation/PrimalMinimizer.h | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/oops/assimilation/DRMinimizer.h b/src/oops/assimilation/DRMinimizer.h index a4bdac561..f42a6b4c9 100644 --- a/src/oops/assimilation/DRMinimizer.h +++ b/src/oops/assimilation/DRMinimizer.h @@ -93,8 +93,14 @@ DRMinimizer::doMinimize(const eckit::Configuration & config) { // Compute RHS (sum B^{-1} dx_{i}) + H^T R^{-1} d // dx_i = x_i - x_{i-1}; dx_1 = x_1 - x_b CtrlInc_ rhs(J_.jb()); - J_.computeGradientFG(rhs); - J_.jb().addGradientFG(rhs, *gradJb_); + if (config.has("fsoi")) { + const eckit::LocalConfiguration FcSensitivityConfig(config, "fsoi.input forecast sensitivity"); + rhs.read(FcSensitivityConfig); + Log::info() << classname() << " rhs has forecast sensitivity" << std::endl; + } else { + J_.computeGradientFG(rhs); + J_.jb().addGradientFG(rhs, *gradJb_); + } rhs *= -1.0; Log::info() << classname() << " rhs" << rhs << std::endl; diff --git a/src/oops/assimilation/PrimalMinimizer.h b/src/oops/assimilation/PrimalMinimizer.h index 847eb651a..fc3167d26 100644 --- a/src/oops/assimilation/PrimalMinimizer.h +++ b/src/oops/assimilation/PrimalMinimizer.h @@ -70,8 +70,14 @@ PrimalMinimizer::doMinimize(const eckit::Configuration & config) { // Compute RHS CtrlInc_ rhs(J_.jb()); - J_.computeGradientFG(rhs); - J_.jb().addGradientFG(rhs); + if (config.has("fsoi")) { + const eckit::LocalConfiguration FcSensitivityConfig(config, "fsoi.input forecast sensitivity"); + rhs.read(FcSensitivityConfig); + Log::info() << classname() << " rhs has forecast sensitivity" << std::endl; + } else { + J_.computeGradientFG(rhs); + J_.jb().addGradientFG(rhs); + } rhs *= -1.0; Log::info() << classname() << " rhs" << rhs << std::endl; From 6bd7f9f2ad782be9572b8ef02b1371a41db07b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Tue, 1 Dec 2020 19:44:42 +0000 Subject: [PATCH 016/142] Parameters subclass for ObsFilter (#977) * Allow passing Parameters rather than Configuration objects to constructors of implementations of ObsFilter. * Made lorenz95::BackgroundCheck's constructor take a Parameters object rather than a Configuration object. * Construct ObsFilters and Observer by passing Parameters rather than Configuration objects. Observers * Corrected copy-and-paste errors. Co-authored-by: Anna Shlyaeva --- l95/src/lorenz95/BackgroundCheck.cc | 5 +- l95/src/lorenz95/BackgroundCheck.h | 14 +-- src/CMakeLists.txt | 1 + src/oops/base/ObsFilterBase.h | 145 ++++++++++++++++++++++-- src/oops/base/ObsFilterParametersBase.h | 34 ++++++ src/oops/base/ObsFilters.h | 26 ++--- src/oops/base/Observer.h | 26 ++++- src/oops/base/Observers.h | 11 +- src/oops/interface/ObsFilter.h | 46 +++++++- 9 files changed, 261 insertions(+), 47 deletions(-) create mode 100644 src/oops/base/ObsFilterParametersBase.h diff --git a/l95/src/lorenz95/BackgroundCheck.cc b/l95/src/lorenz95/BackgroundCheck.cc index 77cd5b944..131d8404e 100644 --- a/l95/src/lorenz95/BackgroundCheck.cc +++ b/l95/src/lorenz95/BackgroundCheck.cc @@ -19,11 +19,10 @@ static oops::FilterMaker > makerBackgroundCheck_("Background Check"); // ----------------------------------------------------------------------------- -BackgroundCheck::BackgroundCheck(const ObsTableView & obsdb, const eckit::Configuration & conf, +BackgroundCheck::BackgroundCheck(const ObsTableView & obsdb, const Parameters_ & parameters, std::shared_ptr > qcflags, std::shared_ptr > obserr) - : obsdb_(obsdb), qcflags_(qcflags), obserr_(obserr), novars_() + : obsdb_(obsdb), options_(parameters), qcflags_(qcflags), obserr_(obserr), novars_() { - options_.deserialize(conf); } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/BackgroundCheck.h b/l95/src/lorenz95/BackgroundCheck.h index 9f57a6a62..44fce1e68 100644 --- a/l95/src/lorenz95/BackgroundCheck.h +++ b/l95/src/lorenz95/BackgroundCheck.h @@ -11,11 +11,9 @@ #include #include -#include "eckit/config/LocalConfiguration.h" - +#include "oops/base/ObsFilterParametersBase.h" #include "oops/base/Variables.h" #include "oops/util/parameters/OptionalParameter.h" -#include "oops/util/parameters/Parameters.h" #include "oops/util/parameters/RequiredParameter.h" #include "oops/util/Printable.h" @@ -28,8 +26,8 @@ namespace lorenz95 { /// Parameters for L95 BackgroundCheck /// background check: all obs for which {|y-H(x)| < threshold} pass QC -class BackgroundCheckParameters : public oops::Parameters { - OOPS_CONCRETE_PARAMETERS(BackgroundCheckParameters, Parameters) +class BackgroundCheckParameters : public oops::ObsFilterParametersBase { + OOPS_CONCRETE_PARAMETERS(BackgroundCheckParameters, ObsFilterParametersBase) public: /// threshold for background check @@ -44,7 +42,9 @@ class BackgroundCheckParameters : public oops::Parameters { /// Simple background check: all obs for which {|y-H(x)| < threshold} pass QC class BackgroundCheck : public util::Printable { public: - BackgroundCheck(const ObsTableView &, const eckit::Configuration &, + typedef BackgroundCheckParameters Parameters_; + + BackgroundCheck(const ObsTableView &, const Parameters_ &, std::shared_ptr >, std::shared_ptr >); void preProcess() const {} @@ -58,7 +58,7 @@ class BackgroundCheck : public util::Printable { void print(std::ostream & os) const; const ObsTableView & obsdb_; - BackgroundCheckParameters options_; + Parameters_ options_; std::shared_ptr > qcflags_; // QC flags std::shared_ptr > obserr_; // obs error stddev const oops::Variables novars_; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8e7107eef..a934838a0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -121,6 +121,7 @@ oops/base/Observers.h oops/base/ObserversTLAD.h oops/base/ObserverTLAD.h oops/base/ObsFilterBase.h +oops/base/ObsFilterParametersBase.h oops/base/ObsFilters.h oops/base/ObsLocalizationBase.h oops/base/ObsSpaceBase.cc diff --git a/src/oops/base/ObsFilterBase.h b/src/oops/base/ObsFilterBase.h index 5831749b2..bcc9fa580 100644 --- a/src/oops/base/ObsFilterBase.h +++ b/src/oops/base/ObsFilterBase.h @@ -11,21 +11,30 @@ #include #include #include +#include +#include #include +#include "oops/base/ObsFilterParametersBase.h" #include "oops/base/Variables.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/ObsDataVector.h" #include "oops/interface/ObsDiagnostics.h" #include "oops/interface/ObsSpace.h" #include "oops/interface/ObsVector.h" +#include "oops/util/AssociativeContainers.h" +#include "oops/util/parameters/ConfigurationParameter.h" +#include "oops/util/parameters/HasParameters_.h" +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/ParametersOrConfiguration.h" +#include "oops/util/parameters/RequiredPolymorphicParameter.h" #include "oops/util/Printable.h" namespace oops { /// Base class for QC filters applied to observations - // ----------------------------------------------------------------------------- template @@ -52,22 +61,95 @@ class ObsFilterBase : public util::Printable, // ============================================================================= +template +class FilterFactory; + +// ----------------------------------------------------------------------------- + +/// \brief A subclass of ObsFilterParametersBase storing the values of all options in a +/// single Configuration object. +/// +/// This object can be accessed by calling the value() method of the \p config member variable. +/// +/// The ConfigurationParameter class does not perform any parameter validation; filters using +/// GenericFilterParameters should therefore ideally be refactored, replacing this +/// class with a dedicated subclass of ObsFilterParametersBase storing each parameter in +/// a separate (Optional/Required)Parameter object. +class GenericObsFilterParameters : public ObsFilterParametersBase { + OOPS_CONCRETE_PARAMETERS(GenericObsFilterParameters, ObsFilterParametersBase) + public: + ConfigurationParameter config{this}; +}; + +// ----------------------------------------------------------------------------- + +/// \brief Contains a polymorphic parameter holding an instance of a subclass of +/// ObsFilterParametersBase. +template +class ObsFilterParametersWrapper : public Parameters { + OOPS_CONCRETE_PARAMETERS(ObsFilterParametersWrapper, Parameters) + public: + /// After deserialization, holds an instance of a subclass of ObsFilterParametersBase + /// controlling the behavior of an observation filter. The type of the subclass is determined + /// by the value of the "filter" key in the Configuration object from which this object + /// is deserialized. + RequiredPolymorphicParameter> filterParameters{ + "filter", this}; + + /// Indices of iterations at which this filter should be applied. + OptionalParameter applyAtIterations{"apply at iterations", this}; +}; + +// ============================================================================= + /// ObsFilter Factory template class FilterFactory { typedef ObsSpace ObsSpace_; template using ObsDataPtr_ = std::shared_ptr >; + public: + /// \brief Create and return a new observation filter. + /// + /// The type of the filter is determined by the `Filter` attribute of \p parameters. \p params + /// must be an instance of the subclass of ObsFilterParametersBase associated with that filter, + /// otherwise an exception will be thrown. + static std::shared_ptr> create(const ObsSpace_ &, + const ObsFilterParametersBase & params, + ObsDataPtr_ flags = ObsDataPtr_(), + ObsDataPtr_ obserr = ObsDataPtr_()); + + /// \brief Create and return a new observation filter. + /// + /// Deprecated overload taking a Configuration instead of an ObsFilterParametersBase. static std::shared_ptr> create(const ObsSpace_ &, const eckit::Configuration &, ObsDataPtr_ flags = ObsDataPtr_(), ObsDataPtr_ obserr = ObsDataPtr_()); + + /// \brief Create and return an instance of the subclass of ObsFilterParametersBase + /// storing parameters of observation filters of the specified type. + static std::unique_ptr createParameters( + const std::string &name); + + /// \brief Return the names of all filters that can be created by one of the + /// registered makers. + static std::vector getMakerNames() { + return keys(getMakers()); + } + virtual ~FilterFactory() = default; + protected: - explicit FilterFactory(const std::string &); + /// \brief Register a maker able to create observation filters of type \p name. + explicit FilterFactory(const std::string &name); + private: - virtual ObsFilterBase * make(const ObsSpace_ &, const eckit::Configuration &, + virtual ObsFilterBase * make(const ObsSpace_ &, const ObsFilterParametersBase &, ObsDataPtr_ &, ObsDataPtr_ &) = 0; + + virtual std::unique_ptr makeParameters() const = 0; + static std::map < std::string, FilterFactory * > & getMakers() { static std::map < std::string, FilterFactory * > makers_; return makers_; @@ -78,11 +160,26 @@ class FilterFactory { template class FilterMaker : public FilterFactory { + /// Defined as T::Parameters_ if T defines a Parameters_ type; otherwise as + /// GenericObsFilterParameters. + typedef TParameters_IfAvailableElseFallbackType_t Parameters_; + typedef ObsSpace ObsSpace_; template using ObsDataPtr_ = std::shared_ptr >; - virtual ObsFilterBase * make(const ObsSpace_ & os, const eckit::Configuration & conf, - ObsDataPtr_ & flags, ObsDataPtr_ & obserr) - { return new T(os, conf, flags, obserr); } + + ObsFilterBase * make(const ObsSpace_ & os, const ObsFilterParametersBase & params, + ObsDataPtr_ & flags, ObsDataPtr_ & obserr) override { + const auto &stronglyTypedParams = dynamic_cast(params); + return new T(os, + parametersOrConfiguration::value>(stronglyTypedParams), + flags, + obserr); + } + + std::unique_ptr makeParameters() const override { + return boost::make_unique(); + } + public: explicit FilterMaker(const std::string & name) : FilterFactory(name) {} }; @@ -101,10 +198,10 @@ FilterFactory::FilterFactory(const std::string & name) { template std::shared_ptr> -FilterFactory::create(const ObsSpace_ & os, const eckit::Configuration & conf, - ObsDataPtr_ flags, ObsDataPtr_ obserr) { - Log::trace() << "ObsFilterBase::create starting" << std::endl; - const std::string id = conf.getString("filter"); +FilterFactory::create(const ObsSpace_ & os, const ObsFilterParametersBase & params, + ObsDataPtr_ flags, ObsDataPtr_ obserr) { + Log::trace() << "FilterFactory::create starting" << std::endl; + const std::string &id = params.filter.value().value(); typename std::map*>::iterator jloc = getMakers().find(id); if (jloc == getMakers().end()) { @@ -116,13 +213,37 @@ FilterFactory::create(const ObsSpace_ & os, const eckit::Configuration & co } throw std::runtime_error(id + " does not exist in obs filter factory."); } - std::shared_ptr> ptr(jloc->second->make(os, conf, flags, obserr)); - Log::trace() << "ObsFilterBase::create done" << std::endl; + std::shared_ptr> ptr(jloc->second->make(os, params, flags, obserr)); + Log::trace() << "FilterFactory::create done" << std::endl; return ptr; } // ----------------------------------------------------------------------------- +template +std::shared_ptr> +FilterFactory::create(const ObsSpace_ & os, const eckit::Configuration & conf, + ObsDataPtr_ flags, ObsDataPtr_ obserr) { + ObsFilterParametersWrapper parameters; + parameters.validateAndDeserialize(conf); + return create(os, parameters.filterParameters, flags, obserr); +} + +// ----------------------------------------------------------------------------- + +template +std::unique_ptr +FilterFactory::createParameters(const std::string &name) { + typename std::map*>::iterator it = + getMakers().find(name); + if (it == getMakers().end()) { + throw std::runtime_error(name + " does not exist in ObsFilterFactory"); + } + return it->second->makeParameters(); +} + +// ----------------------------------------------------------------------------- + } // namespace oops #endif // OOPS_BASE_OBSFILTERBASE_H_ diff --git a/src/oops/base/ObsFilterParametersBase.h b/src/oops/base/ObsFilterParametersBase.h new file mode 100644 index 000000000..801f8d0ae --- /dev/null +++ b/src/oops/base/ObsFilterParametersBase.h @@ -0,0 +1,34 @@ +/* + * (C) Copyright 2020 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_BASE_OBSFILTERPARAMETERSBASE_H_ +#define OOPS_BASE_OBSFILTERPARAMETERSBASE_H_ + +#include + +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameters.h" + +namespace oops { + +/// \brief Base class of classes storing parameters controlling specific observation filters. +class ObsFilterParametersBase : public Parameters { + OOPS_ABSTRACT_PARAMETERS(ObsFilterParametersBase, Parameters) + public: + /// \brief Observation filter type. + /// + /// \note This parameter is marked as optional because it is only required in certain + /// circumstances (e.g. when observation filter parameters are deserialized into an + /// ObsFilterParametersWrapper and used by FilterFactory to instantiate a filter whose type is + /// determined at runtime), but not others (e.g. in tests written with a particular filter in + /// mind). ObsFilterParametersWrapper will throw an exception if this parameter is not provided. + OptionalParameter filter{"filter", this}; +}; + +} // namespace oops + +#endif // OOPS_BASE_OBSFILTERPARAMETERSBASE_H_ diff --git a/src/oops/base/ObsFilters.h b/src/oops/base/ObsFilters.h index 3e33e75a3..9d048d868 100644 --- a/src/oops/base/ObsFilters.h +++ b/src/oops/base/ObsFilters.h @@ -42,11 +42,11 @@ class ObsFilters : public util::Printable, template using ObsDataPtr_ = std::shared_ptr >; public: - /// Initialize all filters for \p obspace, from configuration, using + /// Initialize all filters for \p obspace, from parameters, using /// \p qcflags and \p obserr (observation error variances) /// \p iteration argument indicates outer loop iteration in the variational /// assimilation - ObsFilters(const ObsSpace_ &, const eckit::Configuration &, + ObsFilters(const ObsSpace_ &, const std::vector> &, ObsDataPtr_ qcflags, ObsDataPtr_ obserr, const int iteration = 0); @@ -68,34 +68,34 @@ class ObsFilters : public util::Printable, // ----------------------------------------------------------------------------- template -ObsFilters::ObsFilters(const ObsSpace_ & os, const eckit::Configuration & conf, +ObsFilters::ObsFilters(const ObsSpace_ & os, + const std::vector> & filtersParams, ObsDataPtr_ qcflags, ObsDataPtr_ obserr, const int iteration) : filters_(), geovars_(), diagvars_() { - Log::trace() << "ObsFilters::ObsFilters starting " << conf << std::endl; - -// Get filters configuration - std::vector confs; - conf.get("obs filters", confs); + Log::trace() << "ObsFilters::ObsFilters starting:\n"; + for (const ObsFilterParametersWrapper &filterParams : filtersParams) + Log::trace() << " " << filterParams << std::endl; // Prepare QC handling and statistics if any filters are present - if (confs.size() > 0) { + if (filtersParams.size() > 0) { eckit::LocalConfiguration preconf; preconf.set("filter", "QCmanager"); filters_.push_back(FilterFactory::create(os, preconf, qcflags, obserr)); } // Create the filters, only at 0-th iteration, or at iterations specified in "apply at iterations" - for (std::size_t jj = 0; jj < confs.size(); ++jj) { + for (const ObsFilterParametersWrapper &filterParams : filtersParams) { // Only create filters for the 0-th iteration by default bool apply = (iteration == 0); // If "apply at iterations" is set, check if this is the right iteration - if (confs[jj].has("apply at iterations")) { - std::set iters = parseIntSet(confs[jj].getString("apply at iterations")); + if (filterParams.applyAtIterations.value() != boost::none) { + std::set iters = parseIntSet(*filterParams.applyAtIterations.value()); apply = contains(iters, iteration); } if (apply) { - ObsFilterPtr_ tmp(FilterFactory::create(os, confs[jj], qcflags, obserr)); + ObsFilterPtr_ tmp(FilterFactory::create(os, filterParams.filterParameters, + qcflags, obserr)); geovars_ += tmp->requiredVars(); diagvars_ += tmp->requiredHdiagnostics(); filters_.push_back(tmp); diff --git a/src/oops/base/Observer.h b/src/oops/base/Observer.h index 641eb45f8..6c05102ac 100644 --- a/src/oops/base/Observer.h +++ b/src/oops/base/Observer.h @@ -27,15 +27,30 @@ #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" #include "oops/util/Printable.h" namespace oops { -/// Computes observation equivalent for a single ObsType +// ----------------------------------------------------------------------------- + +/// \brief Parameters controlling an Observer. + +template +class ObserverParameters : public Parameters { + OOPS_CONCRETE_PARAMETERS(ObserverParameters, Parameters) + public: + oops::RequiredParameter obsOperator{"obs operator", this}; + oops::Parameter>> obsFilters{"obs filters", {}, this}; +}; // ----------------------------------------------------------------------------- +/// Computes observation equivalent for a single ObsType + template class Observer : public util::Printable { typedef GeoVaLs GeoVaLs_; @@ -51,7 +66,7 @@ class Observer : public util::Printable { template using ObsDataPtr_ = std::shared_ptr >; public: - Observer(const eckit::Configuration &, const ObsSpace_ &, const ObsAuxCtrl_ &, + Observer(const ObserverParameters &, const ObsSpace_ &, const ObsAuxCtrl_ &, ObsVector_ &, ObsDataPtr_ qcflags, ObsDataPtr_ obserr, const int iteration = 0); ~Observer(); @@ -81,12 +96,13 @@ class Observer : public util::Printable { // ----------------------------------------------------------------------------- template -Observer::Observer(const eckit::Configuration & conf, const ObsSpace_ & obsdb, +Observer::Observer(const ObserverParameters & params, const ObsSpace_ & obsdb, const ObsAuxCtrl_ & ybias, ObsVector_ & yobs, ObsDataPtr_ qcflags, ObsDataPtr_ obserr, const int iteration) - : hop_(obsdb, eckit::LocalConfiguration(conf, "obs operator")), - obsdb_(obsdb), yobs_(yobs), ybias_(ybias), filters_(obsdb, conf, qcflags, obserr, iteration), + : hop_(obsdb, params.obsOperator), + obsdb_(obsdb), yobs_(yobs), ybias_(ybias), + filters_(obsdb, params.obsFilters, qcflags, obserr, iteration), locs_(hop_.locations()) { Log::trace() << "Observer::Observer starting" << std::endl; diff --git a/src/oops/base/Observers.h b/src/oops/base/Observers.h index bea22f903..213767f42 100644 --- a/src/oops/base/Observers.h +++ b/src/oops/base/Observers.h @@ -81,7 +81,16 @@ Observers::Observers(const eckit::Configuration & conf, const ObsSpa ASSERT(obsdb.size() == typeconf.size()); observers_.reserve(obsdb.size()); for (size_t jj = 0; jj < obsdb.size(); ++jj) { - observers_.emplace_back(new Observer_(typeconf[jj], obsdb[jj], + // typeconf[jj] contains not only options controlling the obs operator and filters (known to + // Observer) but also those controlling the obs space (unknown to it). So we can't call + // validateAndDeserialize() here, since "obs space" would be treated as an unrecognized + // keyword. In the long term the code constructing the Observers will probably need to split + // the contents of the "observations" vector into two vectors, one containing the "obs space" + // sections and the other the "obs operator" and "obs filters" sections, and pass the former to + // the constructor of ObsSpaces and the latter to the constructor of Observers. + ObserverParameters observerParams; + observerParams.deserialize(typeconf[jj]); + observers_.emplace_back(new Observer_(observerParams, obsdb[jj], ybias[jj], yobs_[jj], qc.qcFlags(jj), qc.obsErrors(jj), iteration)); } Log::trace() << "Observers::Observers done" << std::endl; diff --git a/src/oops/interface/ObsFilter.h b/src/oops/interface/ObsFilter.h index ac33f9907..4c7f7e023 100644 --- a/src/oops/interface/ObsFilter.h +++ b/src/oops/interface/ObsFilter.h @@ -21,11 +21,28 @@ #include "oops/interface/ObsVector.h" #include "oops/util/dot_product.h" #include "oops/util/Logger.h" +#include "oops/util/parameters/HasParameters_.h" +#include "oops/util/parameters/ParametersOrConfiguration.h" namespace oops { // ----------------------------------------------------------------------------- +/// Note: implementations of this interface can opt to extract their settings either from +/// a Configuration object or from a subclass of ObsFilterParametersBase. +/// +/// In the former case, they should provide a constructor with the following signature: +/// +/// ObsFilter(const ObsSpace_ &, const eckit::Configuration &, +/// ObsDataPtr_, ObsDataPtr_); +/// +/// In the latter case, the implementer should first define a subclass of ObsFilterParametersBase +/// holding the settings of the filter in question. The implementation of the ObsFilter interface +/// should then typedef `Parameters_` to the name of that subclass and provide a constructor with +/// the following signature: +/// +/// ObsFilter(const ObsSpace_ &, const Parameters_ &, +/// ObsDataPtr_, ObsDataPtr_); template class ObsFilter : public ObsFilterBase { typedef GeoVaLs GeoVaLs_; @@ -36,8 +53,14 @@ class ObsFilter : public ObsFilterBase { template using ObsDataVec_ = typename OBS::template ObsDataVector; public: + /// Defined as FILTER::Parameters_ if FILTER defines a Parameters_ type; otherwise as + /// GenericObsFilterParameters + typedef TParameters_IfAvailableElseFallbackType_t Parameters_; + static const std::string classname() {return "oops::ObsFilter";} + ObsFilter(const ObsSpace_ &, const Parameters_ &, + ObsDataPtr_, ObsDataPtr_); ObsFilter(const ObsSpace_ &, const eckit::Configuration &, ObsDataPtr_, ObsDataPtr_); ~ObsFilter(); @@ -53,7 +76,7 @@ class ObsFilter : public ObsFilterBase { void print(std::ostream &) const override; ObsSpace_ obsdb_; - const eckit::LocalConfiguration conf_; + const std::unique_ptr parameters_; std::unique_ptr ofilt_; }; @@ -61,9 +84,9 @@ class ObsFilter : public ObsFilterBase { template ObsFilter::ObsFilter(const ObsSpace_ & os, - const eckit::Configuration & conf, - ObsDataPtr_ flags, ObsDataPtr_ obserr) - : obsdb_(os), conf_(conf), ofilt_() + const Parameters_ & parameters, + ObsDataPtr_ flags, ObsDataPtr_ obserr) + : obsdb_(os), parameters_(parameters.clone()), ofilt_() { Log::trace() << "ObsFilter::ObsFilter Configuration starting" << std::endl; util::Timer timer(classname(), "ObsFilter"); @@ -73,12 +96,23 @@ ObsFilter::ObsFilter(const ObsSpace_ & os, if (flags) qc = flags->obsdatavectorptr(); if (obserr) oberr = obserr->obsdatavectorptr(); - ofilt_.reset(new FILTER(obsdb_.obsspace(), conf, qc, oberr)); + ofilt_.reset(new FILTER(obsdb_.obsspace(), + parametersOrConfiguration::value>(parameters), + qc, oberr)); Log::trace() << "ObsFilter::ObsFilter Configuration done" << std::endl; } // ----------------------------------------------------------------------------- +template +ObsFilter::ObsFilter(const ObsSpace_ & os, + const eckit::Configuration & conf, + ObsDataPtr_ flags, ObsDataPtr_ obserr) + : ObsFilter(os, validateAndDeserialize(conf), flags, obserr) +{} + +// ----------------------------------------------------------------------------- + template ObsFilter::~ObsFilter() { Log::trace() << "ObsFilter::~ObsFilter starting" << std::endl; @@ -137,7 +171,7 @@ Variables ObsFilter::requiredHdiagnostics() const { template void ObsFilter::print(std::ostream & os) const { - os << "ObsFilter " << conf_; + os << "ObsFilter " << *parameters_; } // ----------------------------------------------------------------------------- From 86c450a26bf3b3ccebaa5f555ffe41aee384a7fc Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Thu, 3 Dec 2020 13:26:45 -0700 Subject: [PATCH 017/142] update codecov token (#985) * update codecov token * make them executable * debug * change file name * caps for jcsda * clean up --- CI/buildspec_gnu.yml | 4 +++- CI/codecov_script_JCSDA-internal.sh | 2 ++ CI/{codecov_script.sh => codecov_script_JCSDA.sh} | 0 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100755 CI/codecov_script_JCSDA-internal.sh rename CI/{codecov_script.sh => codecov_script_JCSDA.sh} (100%) mode change 100644 => 100755 diff --git a/CI/buildspec_gnu.yml b/CI/buildspec_gnu.yml index 1a6ced445..c8bd831d3 100644 --- a/CI/buildspec_gnu.yml +++ b/CI/buildspec_gnu.yml @@ -12,6 +12,8 @@ phases: - echo $CODEBUILD_RESOLVED_SOURCE_VERSION - echo $CODEBUILD_SOURCE_REPO_URL + - org_name=$(echo $CODEBUILD_SOURCE_REPO_URL | awk '{split($0,org,"/"); print org[4]}') + - echo $org_name - echo $CODEBUILD_SOURCE_VERSION - echo $CODEBUILD_WEBHOOK_MERGE_COMMIT @@ -129,4 +131,4 @@ phases: - cd /build_container/oops - pwd - ls - - bash /jcsda/oops-bundle/oops/CI/codecov_script.sh + - bash /jcsda/oops-bundle/oops/CI/codecov_script_$org_name.sh diff --git a/CI/codecov_script_JCSDA-internal.sh b/CI/codecov_script_JCSDA-internal.sh new file mode 100755 index 000000000..b04eb7e49 --- /dev/null +++ b/CI/codecov_script_JCSDA-internal.sh @@ -0,0 +1,2 @@ +#/bin/bash +bash <(curl -s https://codecov.io/bash) -t 53f87271-b490-453c-b891-afd39cb658af -R /jcsda/oops-bundle/oops diff --git a/CI/codecov_script.sh b/CI/codecov_script_JCSDA.sh old mode 100644 new mode 100755 similarity index 100% rename from CI/codecov_script.sh rename to CI/codecov_script_JCSDA.sh From e25c5197a5b1885b62ebe2175a5d372c61200480 Mon Sep 17 00:00:00 2001 From: JJ Guerrette Date: Thu, 3 Dec 2020 14:24:59 -0700 Subject: [PATCH 018/142] Rename test cases (#986) Co-authored-by: Anna Shlyaeva --- src/test/interface/GetValues.h | 6 +++--- src/test/interface/LinearGetValues.h | 10 +++++----- src/test/interface/LinearModel.h | 12 ++++++------ src/test/interface/LinearObsOperator.h | 8 ++++---- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/test/interface/GetValues.h b/src/test/interface/GetValues.h index 03663405b..918fe3245 100644 --- a/src/test/interface/GetValues.h +++ b/src/test/interface/GetValues.h @@ -237,11 +237,11 @@ class GetValues : public oops::Test { void register_tests() const override { std::vector& ts = eckit::testing::specification(); - ts.emplace_back(CASE("interface/GeometryIterator/testGetValuesConstructor") + ts.emplace_back(CASE("interface/GetValues/testGetValuesConstructor") { testGetValuesConstructor(); }); - ts.emplace_back(CASE("interface/GeometryIterator/testGetValuesMultiWindow") + ts.emplace_back(CASE("interface/GetValues/testGetValuesMultiWindow") { testGetValuesMultiWindow(); }); - ts.emplace_back(CASE("interface/GeometryIterator/testGetValuesInterpolation") + ts.emplace_back(CASE("interface/GetValues/testGetValuesInterpolation") { testGetValuesInterpolation(); }); } diff --git a/src/test/interface/LinearGetValues.h b/src/test/interface/LinearGetValues.h index f15bd98c7..2509eda0b 100644 --- a/src/test/interface/LinearGetValues.h +++ b/src/test/interface/LinearGetValues.h @@ -315,15 +315,15 @@ class LinearGetValues : public oops::Test { void register_tests() const override { std::vector& ts = eckit::testing::specification(); - ts.emplace_back(CASE("interface/GeometryIterator/testLinearGetValuesConstructor") + ts.emplace_back(CASE("interface/LinearGetValues/testLinearGetValuesConstructor") { testLinearGetValuesConstructor(); }); - ts.emplace_back(CASE("interface/GeometryIterator/testLinearGetValuesZeroPert") + ts.emplace_back(CASE("interface/LinearGetValues/testLinearGetValuesZeroPert") { testLinearGetValuesZeroPert(); }); - ts.emplace_back(CASE("interface/GeometryIterator/testLinearGetValuesLinearity") + ts.emplace_back(CASE("interface/LinearGetValues/testLinearGetValuesLinearity") { testLinearGetValuesLinearity(); }); - ts.emplace_back(CASE("interface/GeometryIterator/testLinearGetValuesLinearApproximation") + ts.emplace_back(CASE("interface/LinearGetValues/testLinearGetValuesLinearApproximation") { testLinearGetValuesLinearApproximation(); }); - ts.emplace_back(CASE("interface/GeometryIterator/testLinearGetValuesAdjoint") + ts.emplace_back(CASE("interface/LinearGetValues/testLinearGetValuesAdjoint") { testLinearGetValuesAdjoint(); }); } diff --git a/src/test/interface/LinearModel.h b/src/test/interface/LinearModel.h index 6d27affaf..55d443877 100644 --- a/src/test/interface/LinearModel.h +++ b/src/test/interface/LinearModel.h @@ -365,17 +365,17 @@ class LinearModel : public oops::Test { void register_tests() const override { std::vector& ts = eckit::testing::specification(); - ts.emplace_back(CASE("interface/GeometryIterator/testLinearModelConstructor") + ts.emplace_back(CASE("interface/LinearModel/testLinearModelConstructor") { testLinearModelConstructor(); }); - ts.emplace_back(CASE("interface/GeometryIterator/testLinearModelZeroLength") + ts.emplace_back(CASE("interface/LinearModel/testLinearModelZeroLength") { testLinearModelZeroLength(); }); - ts.emplace_back(CASE("interface/GeometryIterator/testLinearModelZeroPert") + ts.emplace_back(CASE("interface/LinearModel/testLinearModelZeroPert") { testLinearModelZeroPert(); }); - ts.emplace_back(CASE("interface/GeometryIterator/testLinearModelLinearity") + ts.emplace_back(CASE("interface/LinearModel/testLinearModelLinearity") { testLinearModelLinearity(); }); - ts.emplace_back(CASE("interface/GeometryIterator/testLinearApproximation") + ts.emplace_back(CASE("interface/LinearModel/testLinearApproximation") { testLinearApproximation(); }); - ts.emplace_back(CASE("interface/GeometryIterator/testLinearModelAdjoint") + ts.emplace_back(CASE("interface/LinearModel/testLinearModelAdjoint") { testLinearModelAdjoint(); }); } diff --git a/src/test/interface/LinearObsOperator.h b/src/test/interface/LinearObsOperator.h index b94441151..57b1b1b10 100644 --- a/src/test/interface/LinearObsOperator.h +++ b/src/test/interface/LinearObsOperator.h @@ -316,13 +316,13 @@ class LinearObsOperator : public oops::Test { void register_tests() const override { std::vector& ts = eckit::testing::specification(); - ts.emplace_back(CASE("interface/GeometryIterator/testConstructor") + ts.emplace_back(CASE("interface/LinearObsOperator/testConstructor") { testConstructor(); }); - ts.emplace_back(CASE("interface/GeometryIterator/testLinearity") + ts.emplace_back(CASE("interface/LinearObsOperator/testLinearity") { testLinearity(); }); - ts.emplace_back(CASE("interface/GeometryIterator/testTangentLinear") + ts.emplace_back(CASE("interface/LinearObsOperator/testTangentLinear") { testTangentLinear(); }); - ts.emplace_back(CASE("interface/GeometryIterator/testAdjoint") + ts.emplace_back(CASE("interface/LinearObsOperator/testAdjoint") { testAdjoint(); }); } From 9578f8c9ac336db635d509186da17b561fd5ac69 Mon Sep 17 00:00:00 2001 From: Claude Gibert Date: Tue, 8 Dec 2020 17:16:21 +0000 Subject: [PATCH 019/142] syncing (#987) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Yannick Trémolet <30638944+ytremolet@users.noreply.github.com> --- ewok/hofx.yaml | 12 ++++++++++++ l95/ewok/defaults/ob.yaml | 1 - l95/ewok/defaults/obs.yaml | 1 + qg/ewok/defaults/ob.yaml | 1 - qg/ewok/defaults/stream.yaml | 1 - qg/ewok/defaults/wind.yaml | 1 - qg/ewok/defaults/wspeed.yaml | 1 - 7 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 ewok/hofx.yaml diff --git a/ewok/hofx.yaml b/ewok/hofx.yaml new file mode 100644 index 000000000..bc128b8cd --- /dev/null +++ b/ewok/hofx.yaml @@ -0,0 +1,12 @@ +forecast length: + $(forecast_length) +geometry: + $(GEOMETRY) +initial condition: + $(AN_TEMPLATE) +model: + $(MODEL) +observations: + $(OBSERVATIONS) +window begin: $(window begin) +window length: $(window length) diff --git a/l95/ewok/defaults/ob.yaml b/l95/ewok/defaults/ob.yaml index 79ff5c62b..e69de29bb 100644 --- a/l95/ewok/defaults/ob.yaml +++ b/l95/ewok/defaults/ob.yaml @@ -1 +0,0 @@ -obs_file: obs space.obsdatain diff --git a/l95/ewok/defaults/obs.yaml b/l95/ewok/defaults/obs.yaml index f3f332e8c..57a3e4401 100644 --- a/l95/ewok/defaults/obs.yaml +++ b/l95/ewok/defaults/obs.yaml @@ -7,3 +7,4 @@ obs space: obsdataout: '$(current_dir)/$(experiment).obs.{{current_cycle}}.obt' obs error: covariance model: diagonal + diff --git a/qg/ewok/defaults/ob.yaml b/qg/ewok/defaults/ob.yaml index 911c776bd..e69de29bb 100644 --- a/qg/ewok/defaults/ob.yaml +++ b/qg/ewok/defaults/ob.yaml @@ -1 +0,0 @@ -obs_file: obs space.obsdatain.obsfile diff --git a/qg/ewok/defaults/stream.yaml b/qg/ewok/defaults/stream.yaml index 17bd18234..84182d4a9 100644 --- a/qg/ewok/defaults/stream.yaml +++ b/qg/ewok/defaults/stream.yaml @@ -9,4 +9,3 @@ obs space: obs type: Stream obs error: covariance model: diagonal -filename: obs space.obsdatain.obsfile diff --git a/qg/ewok/defaults/wind.yaml b/qg/ewok/defaults/wind.yaml index bbd214085..525878d56 100644 --- a/qg/ewok/defaults/wind.yaml +++ b/qg/ewok/defaults/wind.yaml @@ -9,4 +9,3 @@ obs space: obs type: Wind obs error: covariance model: diagonal -filename: obs space.obsdatain.obsfile diff --git a/qg/ewok/defaults/wspeed.yaml b/qg/ewok/defaults/wspeed.yaml index 182fa1a9a..b67034cbd 100644 --- a/qg/ewok/defaults/wspeed.yaml +++ b/qg/ewok/defaults/wspeed.yaml @@ -9,4 +9,3 @@ obs space: obs type: WSpeed obs error: covariance model: diagonal -filename: obs space.obsdatain.obsfile From a78444e446689ffbcd42896575a04fa791b922a9 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 9 Dec 2020 08:35:21 -0700 Subject: [PATCH 020/142] add GetValues postprocessor, refactor H(x) computations (#965) * create GetValues postprocessor, move H(x) to CalcHofX * change constructors (problems with filters and eda still) * figure out how to fix ObsFilters + iterations; coding norms * fix QC * remove unused parameters in CostJo * refactor PostTimer * change ObsOp::locations() to not take times * have CalcHofX interface Locations, not ObsOperators * simplify L95 geovals and locs * simplify qg locs & geovals; rollback accidental l95 change * change ObsOp::locations signature * bring back LETKF developments that this branch was behind on * add l95 letkf with qc test (+ bugfix) * resurrect letkf/getkf * change test results (no filters after letkf by default) * read QC flags from file when LETKF is reading H(x) data from file * fix obsbias issue + better comments * Update src/oops/assimilation/CalcHofX.h Co-authored-by: Clementine Gas <43183478+cmgas@users.noreply.github.com> * remove unused conf parameter & unused mutable * fix comments and remove GetValuesPost from HofXNoModel * bugfix: add missing includes; trigger pipeline Co-authored-by: Anna Shlyaeva Co-authored-by: Clementine Gas <43183478+cmgas@users.noreply.github.com> --- l95/src/lorenz95/ObsData1D.h | 6 + qg/model/ObsDataQG.h | 6 + src/CMakeLists.txt | 4 +- src/oops/assimilation/CalcHofX.h | 274 ++++++++++++-------- src/oops/assimilation/CostJo.h | 49 ++-- src/oops/assimilation/GETKFSolver.h | 10 +- src/oops/assimilation/LocalEnsembleSolver.h | 41 +-- src/oops/base/Departures.h | 12 +- src/oops/base/GetValuesPost.h | 143 ++++++++++ src/oops/base/Observations.h | 12 - src/oops/base/Observer.h | 173 ------------ src/oops/base/ObserverTLAD.h | 1 - src/oops/base/Observers.h | 144 ---------- src/oops/base/QCData.h | 64 ----- src/oops/interface/ObsDataVector.h | 9 + src/oops/runs/HofX.h | 25 +- src/oops/runs/HofXNoModel.h | 65 ++++- 17 files changed, 472 insertions(+), 566 deletions(-) create mode 100644 src/oops/base/GetValuesPost.h delete mode 100644 src/oops/base/Observer.h delete mode 100644 src/oops/base/Observers.h delete mode 100644 src/oops/base/QCData.h diff --git a/l95/src/lorenz95/ObsData1D.h b/l95/src/lorenz95/ObsData1D.h index 2111916c3..99a6112df 100644 --- a/l95/src/lorenz95/ObsData1D.h +++ b/l95/src/lorenz95/ObsData1D.h @@ -48,6 +48,7 @@ class ObsData1D : public util::Printable, const DATATYPE & operator[] (const size_t ii) const {return data_.at(ii);} // I/O + void read(const std::string &); void save(const std::string &) const; private: @@ -96,6 +97,11 @@ void ObsData1D::mask(const ObsData1D & mask) { } // ----------------------------------------------------------------------------- template +void ObsData1D::read(const std::string & name) { + obsdb_.getdb(name, data_); +} +// ----------------------------------------------------------------------------- +template void ObsData1D::save(const std::string & name) const { obsdb_.putdb(name, data_); } diff --git a/qg/model/ObsDataQG.h b/qg/model/ObsDataQG.h index 89e373e56..57013cfdd 100644 --- a/qg/model/ObsDataQG.h +++ b/qg/model/ObsDataQG.h @@ -43,6 +43,7 @@ class ObsDataQG : public util::Printable, void mask(const ObsDataQG); // I/O + void read(const std::string &); void save(const std::string &) const; private: @@ -77,6 +78,11 @@ void ObsDataQG::mask(const ObsDataQG) { } // ----------------------------------------------------------------------------- template +void ObsDataQG::read(const std::string & name) { + data_.read(name); +} +// ----------------------------------------------------------------------------- +template void ObsDataQG::save(const std::string & name) const { data_.save(name); } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a934838a0..564f0f669 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -92,6 +92,7 @@ oops/base/DolphChebyshev.h oops/base/EnsembleCovariance.h oops/base/GeneralizedDepartures.h oops/base/GeoVaLsWriter.h +oops/base/GetValuesPost.h oops/base/LocalIncrement.cc oops/base/LocalIncrement.h oops/base/HybridCovariance.h @@ -116,8 +117,6 @@ oops/base/ObsEnsemble.h oops/base/ObsErrorBase.h oops/base/ObsErrors.h oops/base/Observations.h -oops/base/Observer.h -oops/base/Observers.h oops/base/ObserversTLAD.h oops/base/ObserverTLAD.h oops/base/ObsFilterBase.h @@ -136,7 +135,6 @@ oops/base/PostProcessorTLAD.h oops/base/PostTimer.cc oops/base/PostTimer.h oops/base/PostTimerParameters.h -oops/base/QCData.h oops/base/StateEnsemble.h oops/base/StateEnsemble4D.h oops/base/StateInfo.h diff --git a/src/oops/assimilation/CalcHofX.h b/src/oops/assimilation/CalcHofX.h index c5287657a..ff65926bf 100644 --- a/src/oops/assimilation/CalcHofX.h +++ b/src/oops/assimilation/CalcHofX.h @@ -13,167 +13,227 @@ #include "eckit/config/LocalConfiguration.h" -#include "oops/assimilation/State4D.h" #include "oops/base/ObsAuxControls.h" #include "oops/base/Observations.h" -#include "oops/base/Observers.h" +#include "oops/base/ObsFilters.h" #include "oops/base/ObsSpaces.h" -#include "oops/base/PostProcessor.h" -#include "oops/base/QCData.h" -#include "oops/base/StateInfo.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/Model.h" -#include "oops/interface/ModelAuxControl.h" -#include "oops/interface/State.h" -#include "oops/util/Duration.h" +#include "oops/base/Variables.h" +#include "oops/interface/GeoVaLs.h" +#include "oops/interface/Locations.h" +#include "oops/interface/ObsDataVector.h" +#include "oops/interface/ObsDiagnostics.h" +#include "oops/interface/ObsOperator.h" #include "oops/util/Logger.h" +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" namespace oops { -// ----------------------------------------------------------------------------- +template +class CalcHofXParameters : public Parameters { + OOPS_CONCRETE_PARAMETERS(CalcHofXParameters, Parameters) -/// \brief Computes observation operator (while running model, or with State4D) + public: + oops::RequiredParameter obsOperator{"obs operator", this}; + oops::Parameter>> obsFilters{"obs filters", {}, this}; +}; -template +// ----------------------------------------------------------------------------- + +/// \brief Computes observation operator (from GeoVaLs), applies bias correction +/// and runs QC filters +template class CalcHofX { - typedef Geometry Geometry_; - typedef Model Model_; - typedef ModelAuxControl ModelAux_; + typedef GeoVaLs GeoVaLs_; + typedef Locations Locations_; typedef ObsAuxControls ObsAuxCtrls_; + typedef ObsDiagnostics ObsDiags_; typedef Observations Observations_; + typedef ObsFilters ObsFilters_; + typedef ObsOperator ObsOperator_; typedef ObsSpaces ObsSpaces_; - typedef State State_; - typedef State4D State4D_; - typedef PostProcessor PostProcessor_; - typedef QCData QCData_; template using ObsData_ = ObsDataVector; + template using ObsDataVec_ = std::vector>>; - public: -/// \brief Initializes Observers - CalcHofX(const ObsSpaces_ &, const Geometry_ &, const eckit::Configuration &); - -/// \brief Computes 4D H(x) (running the model) - const Observations_ & compute(const Model_ &, State_ &, PostProcessor_ &, const util::Duration &); -/// \brief Computes 4D H(x) (using State4D) - const Observations_ & compute(const State4D_ &); + typedef std::vector> GeoVaLsVec_; + typedef std::vector> LocationsVec_; + typedef std::vector> ObsFiltersVec_; + typedef std::vector> ObsOperatorVec_; + typedef std::vector VariablesVec_; -/// \brief saves QC flags to ObsSpaces + public: +/// \brief Initializes ObsOperators, Locations, and QC data + CalcHofX(const ObsSpaces_ &, const eckit::Configuration &); + +/// \brief Initializes variables, obs bias, obs filters (could be different for +/// different iterations + void initialize(const ObsAuxCtrls_ &, const int iteration = 0); + +/// \brief Computes H(x) from the filled in GeoVaLs + Observations_ compute(const GeoVaLsVec_ &); + +/// \brief accessor to the locations + const LocationsVec_ & locations() const { return locations_; } +/// \brief accessor to variables required from the model + const VariablesVec_ & requiredVars() const { return geovars_; } +/// \brief accessor to QC flags + const ObsDataVec_ & qcflags() const { return qcflags_; } + +/// \brief read QC flags from \p qcname variable from file + void readQcFlags(const std::string & qcname); +/// \brief reset QC flags and ObsErrors + void resetQc(); +/// \brief save QC flags to ObsSpaces void saveQcFlags(const std::string &) const; -/// \brief saves obs error variances (modified in QC) to ObsSpaces +/// \brief save obs error variances (modified in QC) to ObsSpaces void saveObsErrors(const std::string &) const; -/// Mask out the obs errors where the passed in qc flags are > 0 - void maskObsErrors(const QCData_ &); - - std::shared_ptr qc() const {return qc_;} +/// \brief mask obs errors with QC flags + void maskObsErrors(); private: -/// \brief helper method to initialize qc flags and observer - void initObserver(); - - const eckit::LocalConfiguration obsconf_; // configuration for observer - const ObsSpaces_ & obspaces_; // ObsSpaces used in H(x) - ObsAuxCtrls_ ybias_; // obs bias - const Geometry_ & geometry_; // Model Geometry - ModelAux_ moderr_; // model bias - const util::DateTime winbgn_; // window for assimilation - const util::Duration winlen_; - std::shared_ptr qc_; // QC-related (flags and obserrors) - std::shared_ptr > pobs_; // Observer + eckit::LocalConfiguration obsconfig_; + const ObsSpaces_ & obspaces_; // ObsSpaces used in H(x) + ObsOperatorVec_ obsops_; // Obs operators + LocationsVec_ locations_; // locations + const ObsAuxCtrls_ * ybias_; // Obs bias + ObsDataVec_ qcflags_; // QC flags + ObsDataVec_ obserrs_; // Obs error variances (used in QC filters) + ObsFiltersVec_ filters_; // QC filters + VariablesVec_ geovars_; // variables required from the model }; // ----------------------------------------------------------------------------- -template -CalcHofX::CalcHofX(const ObsSpaces_ & obspaces, const Geometry_ & geometry, - const eckit::Configuration & config) : - obsconf_(config, "observations"), obspaces_(obspaces), ybias_(obspaces_, obsconf_), - geometry_(geometry), moderr_(geometry_, config.getSubConfiguration("model aux control")), - winbgn_(config.getString("window begin")), - winlen_(config.getString("window length")) {} +template +CalcHofX::CalcHofX(const ObsSpaces_ & obspaces, + const eckit::Configuration & config) : + obsconfig_(config), obspaces_(obspaces), obsops_(), locations_(), + ybias_(nullptr), qcflags_(), obserrs_(), filters_(), + geovars_(obspaces_.size()) +{ + std::vector obsconfs = config.getSubConfigurations(); + for (size_t jj = 0; jj < obspaces_.size(); ++jj) { + // obsconfs[jj] contains not only options controlling the obs operator and filters (known to + // CalcHofX) but also those controlling the obs space (unknown to it). So we can't call + // validateAndDeserialize() here, since "obs space" would be treated as an unrecognized + // keyword. In the long term the code constructing the CalcHofX will probably need to split + // the contents of the "observations" vector into two vectors, one containing the "obs space" + // sections and the other the "obs operator" and "obs filters" sections, and pass the former to + // the constructor of ObsSpaces and the latter to the constructor of CalcHofX. + CalcHofXParameters observerParams; + observerParams.deserialize(obsconfs[jj]); + /// Set up observation operators + obsops_.emplace_back(new ObsOperator_(obspaces_[jj], observerParams.obsOperator)); + locations_.emplace_back(new Locations_(obsops_[jj]->locations())); + + /// Allocate QC flags + qcflags_.emplace_back(std::make_shared>(obspaces[jj], + obspaces[jj].obsvariables())); + /// Allocate and read initial obs error + obserrs_.emplace_back(std::make_shared>(obspaces[jj], + obspaces[jj].obsvariables(), "ObsError")); + } + Log::trace() << "CalcHofX constructed" << std::endl; +} // ----------------------------------------------------------------------------- -template -void CalcHofX::initObserver() { - qc_.reset(new QCData_(obspaces_)); -// Setup Observers - pobs_.reset(new Observers(obsconf_, obspaces_, ybias_, *qc_)); + +template +void CalcHofX::initialize(const ObsAuxCtrls_ & obsaux, const int iteration) { + std::vector obsconfs = obsconfig_.getSubConfigurations(); + ybias_ = &obsaux; + filters_.clear(); + geovars_.clear(); + for (size_t jj = 0; jj < obspaces_.size(); ++jj) { + CalcHofXParameters observerParams; + observerParams.deserialize(obsconfs[jj]); + /// Set up QC filters and run preprocess + filters_.emplace_back(new ObsFilters_(obspaces_[jj], observerParams.obsFilters, + qcflags_[jj], obserrs_[jj], iteration)); + filters_[jj]->preProcess(); + + /// Set up variables requested from the model + geovars_[jj] += obsops_[jj]->requiredVars(); + geovars_[jj] += (*ybias_)[jj].requiredVars(); + geovars_[jj] += filters_[jj]->requiredVars(); + } + Log::trace() << "CalcHofX::initialize done" << std::endl; } // ----------------------------------------------------------------------------- -template -const Observations & CalcHofX::compute(const Model_ & model, State_ & xx, - PostProcessor_ & post, const util::Duration & length) { - oops::Log::trace() << "CalcHofX::compute (model) start" << std::endl; +template +Observations CalcHofX::compute(const GeoVaLsVec_ & geovals) { + oops::Log::trace() << "CalcHofX::compute start" << std::endl; - this->initObserver(); -// run the model and compute H(x) - post.enrollProcessor(pobs_); - model.forecast(xx, moderr_, length, post); + Observations yobs(obspaces_); + for (size_t jj = 0; jj < obspaces_.size(); ++jj) { + /// call prior filters + filters_[jj]->priorFilter(*geovals[jj]); + /// compute H(x) + oops::Variables vars; + vars += filters_[jj]->requiredHdiagnostics(); + vars += (*ybias_)[jj].requiredHdiagnostics(); + ObsDiags_ ydiags(obspaces_[jj], *locations_[jj], vars); + obsops_[jj]->simulateObs(*geovals[jj], yobs[jj], (*ybias_)[jj], ydiags); + /// call posterior filters + filters_[jj]->postFilter(yobs[jj], ydiags); + } + oops::Log::trace() << "CalcHofX::compute done" << std::endl; + return yobs; +} - oops::Log::trace() << "CalcHofX::compute (model) done" << std::endl; - return pobs_->hofx(); +// ----------------------------------------------------------------------------- + +template +void CalcHofX::readQcFlags(const std::string & qcname) { + for (const auto & qcflag : qcflags_) { + qcflag->read(qcname); + } } // ----------------------------------------------------------------------------- -template -const Observations & CalcHofX::compute(const State4D_ & xx) { - oops::Log::trace() << "CalcHofX::compute (state4D) start" << std::endl; - - this->initObserver(); - size_t nstates = xx.size(); - util::DateTime winend = winbgn_ + winlen_; - util::Duration tstep = winlen_; // for a single state - // if using several states, compute the timestep and check that it's the same - // for all states - if (nstates > 1) { - tstep = xx[1].validTime() - xx[0].validTime(); - for (size_t ii = 1; ii < xx.size(); ++ii) { - ASSERT(tstep == (xx[ii].validTime() - xx[ii-1].validTime())); - } +template +void CalcHofX::resetQc() { + for (const auto & qcflag : qcflags_) { + qcflag->zero(); } - // check that initial and last states have valid times - ASSERT(xx[0].validTime() <= (winbgn_ + tstep/2)); - ASSERT(xx[nstates-1].validTime() >= (winend - tstep/2)); - - // run Observer looping through all the states - pobs_->initialize(xx[0], winend, tstep); - for (size_t ii = 0; ii < xx.size(); ++ii) { - pobs_->process(xx[ii]); + for (const auto & obserr : obserrs_) { + obserr->read("ObsError"); } - pobs_->finalize(xx[nstates-1]); - - oops::Log::trace() << "CalcHofX::compute (state4D) done" << std::endl; - return pobs_->hofx(); } // ----------------------------------------------------------------------------- -template -void CalcHofX::saveQcFlags(const std::string & name) const { - for (size_t jj = 0; jj < obspaces_.size(); ++jj) { - qc_->qcFlags(jj)->save(name); +template +void CalcHofX::saveQcFlags(const std::string & name) const { + for (const auto & qcflag : qcflags_) { + qcflag->save(name); } } // ----------------------------------------------------------------------------- -template -void CalcHofX::saveObsErrors(const std::string & name) const { - for (size_t jj = 0; jj < obspaces_.size(); ++jj) { - qc_->obsErrors(jj)->save(name); +template +void CalcHofX::saveObsErrors(const std::string & name) const { + for (const auto & obserr : obserrs_) { + obserr->save(name); } } // ----------------------------------------------------------------------------- -template -void CalcHofX::maskObsErrors(const QCData_ & qcMask) { - for (size_t jj = 0; jj < obspaces_.size(); ++jj) { - qc_->obsErrors(jj)->mask(*qcMask.qcFlags(jj)); +template +void CalcHofX::maskObsErrors() { + for (size_t jj = 0; jj < obserrs_.size(); ++jj) { + obserrs_[jj]->mask(*qcflags_[jj]); } } + +// ----------------------------------------------------------------------------- + } // namespace oops #endif // OOPS_ASSIMILATION_CALCHOFX_H_ diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index 62049d220..f73e24822 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -19,18 +19,19 @@ #include #include "eckit/config/LocalConfiguration.h" +#include "oops/assimilation/CalcHofX.h" #include "oops/assimilation/ControlIncrement.h" #include "oops/assimilation/ControlVariable.h" #include "oops/assimilation/CostTermBase.h" #include "oops/base/Departures.h" +#include "oops/base/GetValuesPost.h" #include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" -#include "oops/base/Observers.h" #include "oops/base/ObserversTLAD.h" +#include "oops/base/ObsFilters.h" #include "oops/base/ObsSpaces.h" #include "oops/base/PostBase.h" #include "oops/base/PostBaseTLAD.h" -#include "oops/base/QCData.h" #include "oops/interface/Geometry.h" #include "oops/interface/Increment.h" #include "oops/interface/State.h" @@ -56,15 +57,16 @@ template class CostJo : public CostTermBase CtrlInc_; typedef Departures Departures_; typedef Observations Observations_; + typedef ObsFilters ObsFilters_; typedef Geometry Geometry_; typedef State State_; typedef Increment Increment_; typedef ObsErrors ObsErrors_; typedef ObsSpaces ObsSpaces_; - typedef Observers Observers_; + typedef GetValuesPost GetValuesPost_; + typedef CalcHofX CalcHofX_; typedef ObserversTLAD ObserversTLAD_; typedef PostBaseTLAD PostBaseTLAD_; - typedef QCData QCData_; public: /// Construct \f$ J_o\f$ from \f$ R\f$ and \f$ y_{obs}\f$. @@ -127,14 +129,14 @@ template class CostJo : public CostTermBase gradFG_; - /// Observers passed by \f$ J_o\f$ to the model during integration. - mutable std::shared_ptr pobs_; + /// Postprocessor passed by \f$ J_o\f$ to the model during integration. + std::shared_ptr getvals_; + + /// Used for computing H(x) and running QC filters + CalcHofX_ calchofx_; /// Linearized observation operators. std::shared_ptr pobstlad_; - - /// Storage for QC flags and obs error - QCData_ qc_; }; // ============================================================================= @@ -145,8 +147,8 @@ CostJo::CostJo(const eckit::Configuration & joConf, const eckit::mpi const eckit::mpi::Comm & ctime) : obsconf_(joConf), obspace_(obsconf_, comm, winbgn, winend, ctime), yobs_(obspace_, "ObsValue"), - Rmat_(), currentConf_(), gradFG_(), pobs_(), - pobstlad_(), qc_(obspace_) + Rmat_(), currentConf_(), gradFG_(), getvals_(), calchofx_(obspace_, obsconf_), + pobstlad_() { Log::trace() << "CostJo::CostJo done" << std::endl; } @@ -159,8 +161,8 @@ CostJo::CostJo(const eckit::Configuration & joConf, const ObsSpaces_ & localobs) : obsconf_(joConf), obspace_(localobs), yobs_(obspace_, "ObsValue"), - Rmat_(), currentConf_(), gradFG_(), pobs_(), - pobstlad_(), qc_(obspaces) + Rmat_(), currentConf_(), gradFG_(), getvals_(), calchofx_(obspace_, obsconf_), + pobstlad_() { Log::trace() << "CostJo::CostJo using local obs spaces done" << std::endl; } @@ -173,10 +175,10 @@ CostJo::initialize(const CtrlVar_ & xx, const eckit::Configuration & Log::trace() << "CostJo::initialize start" << std::endl; currentConf_.reset(new eckit::LocalConfiguration(conf)); - pobs_.reset(new Observers_(obsconf_, obspace_, xx.obsVar(), qc_, - currentConf_->getInt("iteration"))); + calchofx_.initialize(xx.obsVar(), currentConf_->getInt("iteration")); + getvals_.reset(new GetValuesPost_(obspace_, calchofx_.locations(), calchofx_.requiredVars())); Log::trace() << "CostJo::initialize done" << std::endl; - return pobs_; + return getvals_; } // ----------------------------------------------------------------------------- @@ -184,7 +186,7 @@ CostJo::initialize(const CtrlVar_ & xx, const eckit::Configuration & template double CostJo::finalize() { Log::trace() << "CostJo::finalize start" << std::endl; - const Observations_ & yeqv = pobs_->hofx(); + Observations_ yeqv = calchofx_.compute(getvals_->geovals()); Log::info() << "Jo Observation Equivalent:" << std::endl << yeqv << "End Jo Observation Equivalent" << std::endl; const int iterout = currentConf_->getInt("iteration"); @@ -194,12 +196,11 @@ double CostJo::finalize() { const std::string qcname = "EffectiveQC" + std::to_string(iterout); const std::string errname = "EffectiveError" + std::to_string(iterout); yeqv.save(obsname); - for (size_t jj = 0; jj < obspace_.size(); ++jj) { - qc_.obsErrors(jj)->mask(*qc_.qcFlags(jj)); - qc_.qcFlags(jj)->save(qcname); - qc_.obsErrors(jj)->save(errname); - qc_.obsErrors(jj)->save("EffectiveError"); // Obs error covariance is looking for that for now - } + + calchofx_.maskObsErrors(); + calchofx_.saveQcFlags(qcname); + calchofx_.saveObsErrors(errname); + calchofx_.saveObsErrors("EffectiveError"); // Obs error covariance is looking for that for now // Set observation error covariance Rmat_.reset(new ObsErrors_(obsconf_, obspace_)); @@ -236,7 +237,7 @@ double CostJo::finalize() { ydep.save(depname); } - pobs_.reset(); + getvals_.reset(); currentConf_.reset(); Log::trace() << "CostJo::finalize done" << std::endl; return zjo; diff --git a/src/oops/assimilation/GETKFSolver.h b/src/oops/assimilation/GETKFSolver.h index f5ea75481..63bedf798 100644 --- a/src/oops/assimilation/GETKFSolver.h +++ b/src/oops/assimilation/GETKFSolver.h @@ -20,6 +20,7 @@ #include "oops/assimilation/State4D.h" #include "oops/base/Departures.h" #include "oops/base/DeparturesEnsemble.h" +#include "oops/base/GetValuesPost.h" #include "oops/base/IncrementEnsemble4D.h" #include "oops/base/LocalIncrement.h" #include "oops/base/ObsEnsemble.h" @@ -49,6 +50,7 @@ class GETKFSolver : public LocalEnsembleSolver { typedef DeparturesEnsemble DeparturesEnsemble_; typedef Geometry Geometry_; typedef GeometryIterator GeometryIterator_; + typedef GetValuesPost GetValuesPost_; typedef IncrementEnsemble4D IncrementEnsemble4D_; typedef ObsEnsemble ObsEnsemble_; typedef ObsErrors ObsErrors_; @@ -137,7 +139,7 @@ GETKFSolver::GETKFSolver(ObsSpaces_ & obspaces, const Geometry_ & ge // ----------------------------------------------------------------------------- template Observations GETKFSolver::computeHofX(const StateEnsemble4D_ & ens_xx, - size_t iteration, bool readFromFile) { + size_t iteration, bool readFromFile) { util::Timer timer(classname(), "computeHofX"); // compute/read H(x) for the original ensemble members @@ -171,7 +173,11 @@ Observations GETKFSolver::computeHofX(const StateEnsemble4D_ & for (size_t ieig = 0; ieig < neig_; ++ieig) { State4D_ tmpState = xx_mean; tmpState += Ztmp[ieig]; - Observations_ tmpObs = this->hofx_.compute(tmpState); + GetValuesPost_ getvals(this->obspaces_, this->hofx_.locations(), + this->hofx_.requiredVars()); + getvals.fill(tmpState); + // compute H(x) on filled in geovals and run the filters + Observations_ tmpObs = this->hofx_.compute(getvals.geovals()); HZb_[ii] = tmpObs - yb_mean; tmpObs.save("hofxm"+std::to_string(iteration)+"_"+std::to_string(ieig+1)+ "_"+std::to_string(iens+1)); diff --git a/src/oops/assimilation/LocalEnsembleSolver.h b/src/oops/assimilation/LocalEnsembleSolver.h index 9fc111dec..e520cdc0b 100644 --- a/src/oops/assimilation/LocalEnsembleSolver.h +++ b/src/oops/assimilation/LocalEnsembleSolver.h @@ -16,11 +16,12 @@ #include "oops/assimilation/CalcHofX.h" #include "oops/base/Departures.h" #include "oops/base/DeparturesEnsemble.h" +#include "oops/base/GetValuesPost.h" #include "oops/base/IncrementEnsemble4D.h" +#include "oops/base/ObsAuxControls.h" #include "oops/base/ObsEnsemble.h" #include "oops/base/Observations.h" #include "oops/base/ObsSpaces.h" -#include "oops/base/QCData.h" #include "oops/base/StateEnsemble4D.h" #include "oops/interface/Geometry.h" #include "oops/interface/GeometryIterator.h" @@ -32,16 +33,17 @@ namespace oops { /// \brief Base class for LETKF-type solvers template class LocalEnsembleSolver { - typedef CalcHofX CalcHofX_; + typedef CalcHofX CalcHofX_; typedef Departures Departures_; typedef DeparturesEnsemble DeparturesEnsemble_; typedef Geometry Geometry_; typedef GeometryIterator GeometryIterator_; + typedef GetValuesPost GetValuesPost_; typedef IncrementEnsemble4D IncrementEnsemble4D_; + typedef ObsAuxControls ObsAuxControls_; typedef ObsEnsemble ObsEnsemble_; typedef Observations Observations_; typedef ObsSpaces ObsSpaces_; - typedef QCData QCData_; typedef StateEnsemble4D StateEnsemble4D_; public: @@ -66,10 +68,11 @@ class LocalEnsembleSolver { protected: const eckit::LocalConfiguration obsconf_; // configuration for observations - const ObsSpaces_ & obspaces_; // ObsSpaces - CalcHofX_ hofx_; // observer - Departures_ omb_; // obs - mean(H(x)) - DeparturesEnsemble_ Yb_; // ensemble perturbations in the observation space + const ObsSpaces_ & obspaces_; // ObsSpaces + const ObsAuxControls_ obsaux_; // Obs bias + CalcHofX_ hofx_; // observer + Departures_ omb_; // obs - mean(H(x)) + DeparturesEnsemble_ Yb_; // ensemble perturbations in the observation space }; // ----------------------------------------------------------------------------- @@ -78,9 +81,8 @@ template LocalEnsembleSolver::LocalEnsembleSolver(ObsSpaces_ & obspaces, const Geometry_ & geometry, const eckit::Configuration & config, size_t nens) - : obsconf_(config, "observations"), obspaces_(obspaces), - hofx_(obspaces, geometry, config), - omb_(obspaces_), Yb_(obspaces_, nens) + : obsconf_(config, "observations"), obspaces_(obspaces), obsaux_(obspaces_, obsconf_), + hofx_(obspaces, obsconf_), omb_(obspaces_), Yb_(obspaces_, nens) { } @@ -95,7 +97,6 @@ Observations LocalEnsembleSolver::computeHofX(const StateEnsemb const size_t nens = ens_xx.size(); ObsEnsemble_ obsens(obspaces_, nens); - std::shared_ptr qc; if (readFromDisk) { // read hofx from disk @@ -104,21 +105,25 @@ Observations LocalEnsembleSolver::computeHofX(const StateEnsemb obsens[jj].read("hofx"+std::to_string(iteration)+"_"+std::to_string(jj+1)); Log::test() << "H(x) for member " << jj+1 << ":" << std::endl << obsens[jj] << std::endl; } - qc.reset(new QCData_(obspaces_, "EffectiveQC", "EffectiveError")); - + hofx_.readQcFlags("EffectiveQC"); } else { // compute and save H(x) Log::debug() << "Computing H(X) online" << std::endl; + hofx_.initialize(obsaux_, iteration); for (size_t jj = 0; jj < nens; ++jj) { - obsens[jj] = hofx_.compute(ens_xx[jj]); + hofx_.resetQc(); + // fill in geovals + GetValuesPost_ getvals(obspaces_, hofx_.locations(), hofx_.requiredVars()); + getvals.fill(ens_xx[jj]); + // compute H(x) on filled in geovals and run the filters + obsens[jj] = hofx_.compute(getvals.geovals()); Log::test() << "H(x) for member " << jj+1 << ":" << std::endl << obsens[jj] << std::endl; obsens[jj].save("hofx"+std::to_string(iteration)+"_"+std::to_string(jj+1)); } // QC flags and Obs errors are set to that of the last ensemble member // TODO(someone) combine qc flags from all ensemble members - qc = hofx_.qc(); hofx_.saveQcFlags("EffectiveQC"); - hofx_.maskObsErrors(*qc); + hofx_.maskObsErrors(); hofx_.saveObsErrors("EffectiveError"); } @@ -128,13 +133,13 @@ Observations LocalEnsembleSolver::computeHofX(const StateEnsemb // calculate H(x) ensemble perturbations for (size_t iens = 0; iens < nens; ++iens) { Yb_[iens] = obsens[iens] - yb_mean; - Yb_[iens].mask(*qc); + Yb_[iens].mask(hofx_.qcflags()); } // calculate obs departures and mask with qc flag Observations_ yobs(obspaces_, "ObsValue"); omb_ = yobs - yb_mean; - omb_.mask(*qc); + omb_.mask(hofx_.qcflags()); // return mean H(x) return yb_mean; diff --git a/src/oops/base/Departures.h b/src/oops/base/Departures.h index e21261bf4..ed6f62240 100644 --- a/src/oops/base/Departures.h +++ b/src/oops/base/Departures.h @@ -14,12 +14,13 @@ #include #include #include +#include #include #include #include "oops/base/GeneralizedDepartures.h" #include "oops/base/ObsSpaces.h" -#include "oops/base/QCData.h" +#include "oops/interface/ObsDataVector.h" #include "oops/interface/ObsVector.h" #include "oops/util/dot_product.h" #include "oops/util/Logger.h" @@ -44,7 +45,8 @@ class Departures : public util::Printable, public GeneralizedDepartures { typedef ObsSpaces ObsSpaces_; typedef ObsVector ObsVector_; - typedef QCData QCData_; + template using ObsData_ = ObsDataVector; + template using ObsDataVec_ = std::vector>>; public: /// \brief create Departures for all obs (read from ObsSpace if name is specified) @@ -74,7 +76,7 @@ class Departures : public util::Printable, size_t nobs() const; /// Mask out departures where the passed in qc flags are > 0 - void mask(const QCData_ &); + void mask(ObsDataVec_); /// Pack departures in an Eigen vector (excluding departures that are masked out) Eigen::VectorXd packEigen() const; @@ -204,9 +206,9 @@ size_t Departures::nobs() const { } // ----------------------------------------------------------------------------- template -void Departures::mask(const QCData_ & qc) { +void Departures::mask(ObsDataVec_ qcflags) { for (size_t ii = 0; ii < dep_.size(); ++ii) { - dep_[ii].mask(*qc.qcFlags(ii)); + dep_[ii].mask(*qcflags[ii]); } } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/GetValuesPost.h b/src/oops/base/GetValuesPost.h new file mode 100644 index 000000000..a7e4e10c4 --- /dev/null +++ b/src/oops/base/GetValuesPost.h @@ -0,0 +1,143 @@ +/* + * (C) Copyright 2009-2016 ECMWF. + * (C) Copyright 2020-2020 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#ifndef OOPS_BASE_GETVALUESPOST_H_ +#define OOPS_BASE_GETVALUESPOST_H_ + +#include +#include +#include +#include + +#include "oops/assimilation/State4D.h" +#include "oops/base/ObsSpaces.h" +#include "oops/base/PostBase.h" +#include "oops/base/Variables.h" +#include "oops/interface/GeoVaLs.h" +#include "oops/interface/GetValues.h" +#include "oops/interface/Locations.h" +#include "oops/interface/State.h" +#include "oops/util/DateTime.h" +#include "oops/util/Duration.h" +#include "oops/util/Logger.h" + +namespace oops { + +/// \brief Fills GeoVaLs with requested variables at requested locations: +/// - as a postprocessor during model run. +/// - as a method fill() on State4D +template +class GetValuesPost : public PostBase> { + typedef GeoVaLs GeoVaLs_; + typedef Locations Locations_; + typedef ObsSpaces ObsSpaces_; + typedef State State_; + typedef State4D State4D_; + typedef GetValues GetValues_; + + typedef std::vector> GetValuesVec_; + typedef std::vector> GeoVaLsVec_; + typedef std::vector> LocationsVec_; + typedef std::vector VariablesVec_; + + public: +/// \brief Saves Locations and Variables to be processed + GetValuesPost(const ObsSpaces_ &, const LocationsVec_ &, const VariablesVec_ &); + +/// \brief Returns geovals filled in during the model run + const GeoVaLsVec_ & geovals() const {return geovals_;} + +/// \brief fills in GeoVaLs looping through State4D + void fill(const State4D_ &); + + private: +/// \brief initialization before model run: sets up GetValues and allocate GeoVaLs + void doInitialize(const State_ &, const util::DateTime &, const util::Duration &) override; +/// \brief called at each model step: fill in GeoVaLs for the current time slot + void doProcessing(const State_ &) override; + +// Data + util::DateTime winbgn_; /// Begining of assimilation window + util::DateTime winend_; /// End of assimilation window + util::Duration hslot_; /// Half time slot + + const LocationsVec_ & locations_; /// locations of observations + const VariablesVec_ & geovars_; /// Variables needed from model + GetValuesVec_ getvals_; /// GetValues used to fill in GeoVaLs + GeoVaLsVec_ geovals_; /// GeoVaLs that are filled in +}; + +// ----------------------------------------------------------------------------- + +template +GetValuesPost::GetValuesPost(const ObsSpaces_ & obsdb, const LocationsVec_ & locations, + const VariablesVec_ & vars) + : PostBase(), + winbgn_(obsdb.windowStart()), winend_(obsdb.windowEnd()), hslot_(), + locations_(locations), geovars_(vars) +{ + Log::trace() << "GetValuesPost::GetValuesPost done" << std::endl; +} + +// ----------------------------------------------------------------------------- +template +void GetValuesPost::fill(const State4D_ & xx) { + const size_t nstates = xx.size(); + util::Duration tstep = winend_ - winbgn_; // for a single state + // if using several states, compute the timestep and check that it's the same + // for all states + if (nstates > 1) { + tstep = xx[1].validTime() - xx[0].validTime(); + for (size_t ii = 1; ii < nstates; ++ii) { + ASSERT(tstep == (xx[ii].validTime() - xx[ii-1].validTime())); + } + } + // run GetValues postprocessor looping through all the states + doInitialize(xx[0], xx[nstates-1].validTime(), tstep); + for (size_t ii = 0; ii < nstates; ++ii) { + doProcessing(xx[ii]); + } +} + +// ----------------------------------------------------------------------------- + +template +void GetValuesPost::doInitialize(const State_ & xx, const util::DateTime & end, + const util::Duration & tstep) { + Log::trace() << "GetValuesPost::doInitialize start" << std::endl; + hslot_ = tstep/2; + for (size_t jj = 0; jj < locations_.size(); ++jj) { + getvals_.emplace_back(new GetValues_(xx.geometry(), *locations_[jj])); + geovals_.emplace_back(new GeoVaLs_(*locations_[jj], geovars_[jj])); + } + Log::trace() << "GetValuesPost::doInitialize done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void GetValuesPost::doProcessing(const State_ & xx) { + Log::trace() << "GetValuesPost::doProcessing start" << std::endl; + util::DateTime t1 = std::max(xx.validTime()-hslot_, winbgn_); + util::DateTime t2 = std::min(xx.validTime()+hslot_, winend_); + +// Get state variables at obs locations + for (size_t jj = 0; jj < getvals_.size(); ++jj) { + getvals_[jj]->fillGeoVaLs(xx, t1, t2, *geovals_[jj]); + } + Log::trace() << "GetValuesPost::doProcessing done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_BASE_GETVALUESPOST_H_ diff --git a/src/oops/base/Observations.h b/src/oops/base/Observations.h index 78dd40722..abc57310d 100644 --- a/src/oops/base/Observations.h +++ b/src/oops/base/Observations.h @@ -20,7 +20,6 @@ #include "oops/base/Departures.h" #include "oops/base/ObsErrors.h" #include "oops/base/ObsSpaces.h" -#include "oops/base/QCData.h" #include "oops/interface/ObsVector.h" #include "oops/util/Logger.h" #include "oops/util/Printable.h" @@ -38,7 +37,6 @@ template class Observations : public util::Printable { typedef ObsErrors ObsErrors_; typedef ObsSpaces ObsSpaces_; typedef ObsVector ObsVector_; - typedef QCData QCData_; public: /// \brief create Observations for all obs (read from ObsSpace if name is specified) @@ -74,9 +72,6 @@ template class Observations : public util::Printable { /// Perturbations void perturb(const ObsErrors_ &); -/// Mask out departures where the passed in qc flags are > 0 - void mask(const QCData_ &); - private: void print(std::ostream &) const; size_t nobs() const; @@ -199,13 +194,6 @@ size_t Observations::nobs() const { return nobs; } // ----------------------------------------------------------------------------- -template -void Observations::mask(const QCData_ & qc) { - for (size_t ii = 0; ii < obs_.size(); ++ii) { - obs_[ii].mask(*qc.qcFlags(ii)); - } -} -// ----------------------------------------------------------------------------- template void Observations::perturb(const ObsErrors_ & Rmat) { Departures_ ypert(obsdb_); diff --git a/src/oops/base/Observer.h b/src/oops/base/Observer.h deleted file mode 100644 index 6c05102ac..000000000 --- a/src/oops/base/Observer.h +++ /dev/null @@ -1,173 +0,0 @@ -/* - * (C) Copyright 2019 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef OOPS_BASE_OBSERVER_H_ -#define OOPS_BASE_OBSERVER_H_ - -#include -#include -#include - -#include "oops/base/ObsFilters.h" -#include "oops/base/Variables.h" -#include "oops/interface/GeoVaLs.h" -#include "oops/interface/GetValues.h" -#include "oops/interface/Locations.h" -#include "oops/interface/ObsAuxControl.h" -#include "oops/interface/ObsDataVector.h" -#include "oops/interface/ObsDiagnostics.h" -#include "oops/interface/ObsOperator.h" -#include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" -#include "oops/interface/State.h" -#include "oops/util/DateTime.h" -#include "oops/util/Duration.h" -#include "oops/util/Logger.h" -#include "oops/util/parameters/Parameter.h" -#include "oops/util/parameters/Parameters.h" -#include "oops/util/parameters/RequiredParameter.h" -#include "oops/util/Printable.h" - -namespace oops { - -// ----------------------------------------------------------------------------- - -/// \brief Parameters controlling an Observer. - -template -class ObserverParameters : public Parameters { - OOPS_CONCRETE_PARAMETERS(ObserverParameters, Parameters) - - public: - oops::RequiredParameter obsOperator{"obs operator", this}; - oops::Parameter>> obsFilters{"obs filters", {}, this}; -}; - -// ----------------------------------------------------------------------------- - -/// Computes observation equivalent for a single ObsType - -template -class Observer : public util::Printable { - typedef GeoVaLs GeoVaLs_; - typedef Locations Locations_; - typedef ObsDiagnostics ObsDiags_; - typedef ObsSpace ObsSpace_; - typedef GetValues GetValues_; - typedef ObsAuxControl ObsAuxCtrl_; - typedef ObsFilters ObsFilters_; - typedef ObsOperator ObsOperator_; - typedef ObsVector ObsVector_; - typedef State State_; - template using ObsDataPtr_ = std::shared_ptr >; - - public: - Observer(const ObserverParameters &, const ObsSpace_ &, const ObsAuxCtrl_ &, - ObsVector_ &, ObsDataPtr_ qcflags, ObsDataPtr_ obserr, - const int iteration = 0); - ~Observer(); - - void doInitialize(const State_ &, const util::DateTime &, const util::DateTime &); - void doProcessing(const State_ &, const util::DateTime &, const util::DateTime &); - void doFinalize(); - - private: - void print(std::ostream &) const override; - -// Obs operator - ObsOperator_ hop_; - -// Data - const ObsSpace_ & obsdb_; - ObsVector_ & yobs_; - const ObsAuxCtrl_ & ybias_; - - ObsFilters_ filters_; - Variables geovars_; // Variables needed from model (through geovals) - Locations_ locs_; - std::unique_ptr getvals_; - std::shared_ptr gvals_; -}; - -// ----------------------------------------------------------------------------- - -template -Observer::Observer(const ObserverParameters & params, const ObsSpace_ & obsdb, - const ObsAuxCtrl_ & ybias, ObsVector_ & yobs, - ObsDataPtr_ qcflags, ObsDataPtr_ obserr, - const int iteration) - : hop_(obsdb, params.obsOperator), - obsdb_(obsdb), yobs_(yobs), ybias_(ybias), - filters_(obsdb, params.obsFilters, qcflags, obserr, iteration), - locs_(hop_.locations()) -{ - Log::trace() << "Observer::Observer starting" << std::endl; - geovars_ += hop_.requiredVars(); - geovars_ += ybias_.requiredVars(); - geovars_ += filters_.requiredVars(); - Log::trace() << "Observer::Observer done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -Observer::~Observer() { - Log::trace() << "Observer::~Observer starting" << std::endl; - gvals_.reset(); - Log::trace() << "Observer::~Observer done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void Observer::doInitialize(const State_ & xx, - const util::DateTime & begin, - const util::DateTime & end) { - Log::trace() << "Observer::doInitialize start" << std::endl; - filters_.preProcess(); - getvals_.reset(new GetValues_(xx.geometry(), locs_)); - gvals_.reset(new GeoVaLs_(locs_, geovars_)); - Log::trace() << "Observer::doInitialize done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void Observer::doProcessing(const State_ & xx, - const util::DateTime & t1, - const util::DateTime & t2) { - Log::trace() << "Observer::doProcessing start" << std::endl; -// Get state variables at obs locations - getvals_->fillGeoVaLs(xx, t1, t2, *gvals_); - Log::trace() << "Observer::doProcessing done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void Observer::doFinalize() { - Log::trace() << "Observer::doFinalize start" << std::endl; - filters_.priorFilter(*gvals_); - oops::Variables vars; - vars += filters_.requiredHdiagnostics(); - vars += ybias_.requiredHdiagnostics(); - ObsDiags_ ydiags(obsdb_, locs_, vars); - hop_.simulateObs(*gvals_, yobs_, ybias_, ydiags); - filters_.postFilter(yobs_, ydiags); - Log::trace() << "Observer::doFinalize done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void Observer::print(std::ostream &) const {} - -// ----------------------------------------------------------------------------- - -} // namespace oops - -#endif // OOPS_BASE_OBSERVER_H_ diff --git a/src/oops/base/ObserverTLAD.h b/src/oops/base/ObserverTLAD.h index f0796d1da..16b431178 100644 --- a/src/oops/base/ObserverTLAD.h +++ b/src/oops/base/ObserverTLAD.h @@ -28,7 +28,6 @@ #include "oops/interface/ObsVector.h" #include "oops/interface/State.h" #include "oops/util/DateTime.h" -#include "oops/util/Duration.h" namespace oops { diff --git a/src/oops/base/Observers.h b/src/oops/base/Observers.h deleted file mode 100644 index 213767f42..000000000 --- a/src/oops/base/Observers.h +++ /dev/null @@ -1,144 +0,0 @@ -/* - * (C) Copyright 2009-2016 ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -#ifndef OOPS_BASE_OBSERVERS_H_ -#define OOPS_BASE_OBSERVERS_H_ - -#include -#include -#include -#include - -#include "oops/base/ObsAuxControls.h" -#include "oops/base/Observations.h" -#include "oops/base/Observer.h" -#include "oops/base/ObsSpaces.h" -#include "oops/base/PostBase.h" -#include "oops/base/QCData.h" -#include "oops/interface/State.h" -#include "oops/util/DateTime.h" -#include "oops/util/Duration.h" -#include "oops/util/Logger.h" - -namespace oops { - -/// Computes observation equivalent during model run. - -// ----------------------------------------------------------------------------- - -template -class Observers : public PostBase> { - typedef GeoVaLs GeoVaLs_; - typedef ObsAuxControls ObsAuxCtrls_; - typedef Observations Observations_; - typedef Observer Observer_; - typedef ObsSpaces ObsSpaces_; - typedef QCData QCData_; - typedef State State_; - - public: - Observers(const eckit::Configuration &, const ObsSpaces_ & obsdb, const ObsAuxCtrls_ &, - QCData_ &, const int iteration = 0); - ~Observers() {} - - const Observations_ & hofx() {return yobs_;} - - private: -// Methods - void doInitialize(const State_ &, const util::DateTime &, const util::Duration &) override; - void doProcessing(const State_ &) override; - void doFinalize(const State_ &) override; - -// Data - ObsSpaces_ obspace_; - Observations_ yobs_; - - util::DateTime winbgn_; //!< Begining of assimilation window - util::DateTime winend_; //!< End of assimilation window - util::Duration hslot_; //!< Half time slot - - std::vector> observers_; -}; - -// ----------------------------------------------------------------------------- - -template -Observers::Observers(const eckit::Configuration & conf, const ObsSpaces_ & obsdb, - const ObsAuxCtrls_ & ybias, QCData_ & qc, const int iteration) - : PostBase(), - obspace_(obsdb), yobs_(obsdb), - winbgn_(obsdb.windowStart()), winend_(obsdb.windowEnd()), hslot_(0), observers_(0) -{ - Log::trace() << "Observers::Observers starting" << std::endl; - std::vector typeconf = conf.getSubConfigurations(); - ASSERT(obsdb.size() == typeconf.size()); - observers_.reserve(obsdb.size()); - for (size_t jj = 0; jj < obsdb.size(); ++jj) { - // typeconf[jj] contains not only options controlling the obs operator and filters (known to - // Observer) but also those controlling the obs space (unknown to it). So we can't call - // validateAndDeserialize() here, since "obs space" would be treated as an unrecognized - // keyword. In the long term the code constructing the Observers will probably need to split - // the contents of the "observations" vector into two vectors, one containing the "obs space" - // sections and the other the "obs operator" and "obs filters" sections, and pass the former to - // the constructor of ObsSpaces and the latter to the constructor of Observers. - ObserverParameters observerParams; - observerParams.deserialize(typeconf[jj]); - observers_.emplace_back(new Observer_(observerParams, obsdb[jj], - ybias[jj], yobs_[jj], qc.qcFlags(jj), qc.obsErrors(jj), iteration)); - } - Log::trace() << "Observers::Observers done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void Observers::doInitialize(const State_ & xx, const util::DateTime & end, - const util::Duration & tstep) { - Log::trace() << "Observers::doInitialize start" << std::endl; - const util::DateTime bgn(xx.validTime()); - hslot_ = tstep/2; - - for (size_t jj = 0; jj < observers_.size(); ++jj) { - observers_[jj]->doInitialize(xx, winbgn_, winend_); - } - Log::trace() << "Observers::doInitialize done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void Observers::doProcessing(const State_ & xx) { - Log::trace() << "Observers::doProcessing start" << std::endl; - util::DateTime t1 = std::max(xx.validTime()-hslot_, winbgn_); - util::DateTime t2 = std::min(xx.validTime()+hslot_, winend_); - -// Get state variables at obs locations - for (size_t jj = 0; jj < observers_.size(); ++jj) { - observers_[jj]->doProcessing(xx, t1, t2); - } - Log::trace() << "Observers::doProcessing done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void Observers::doFinalize(const State_ &) { - Log::trace() << "Observers::doFinalize start" << std::endl; - for (size_t jj = 0; jj < observers_.size(); ++jj) { - observers_[jj]->doFinalize(); - } - Log::trace() << "Observers::doFinalize done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -} // namespace oops - -#endif // OOPS_BASE_OBSERVERS_H_ diff --git a/src/oops/base/QCData.h b/src/oops/base/QCData.h deleted file mode 100644 index d136a5cb0..000000000 --- a/src/oops/base/QCData.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * (C) Copyright 2020 UCAR. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ -#ifndef OOPS_BASE_QCDATA_H_ -#define OOPS_BASE_QCDATA_H_ - -#include -#include -#include - -#include "oops/base/ObsSpaces.h" -#include "oops/interface/ObsDataVector.h" - -namespace oops { - -// ----------------------------------------------------------------------------- - -/// \brief container for QC-related things (flags & obserrors) - -template -class QCData { - typedef ObsSpaces ObsSpaces_; - template using ObsData_ = ObsDataVector; - template using ObsDataPtr_ = std::shared_ptr >; - - public: -/// \brief Initializes QC data, optionally setting the variable names to read. - explicit QCData(const ObsSpaces_ &, const std::string qcName = "", - const std::string errName = "ObsError"); - -/// \brief accessor to QC flag - const ObsDataPtr_ qcFlags(const size_t ii) const {return qcflags_[ii];} -/// \brief accessor to Obs errors - const ObsDataPtr_ obsErrors(const size_t ii) const {return obserr_[ii];} - - private: - std::vector > qcflags_; // QC flags - std::vector > obserr_; // Obs Errors -}; - - -// ----------------------------------------------------------------------------- - -template -QCData::QCData(const ObsSpaces_ & obspaces, const std::string qcName, - const std::string errName) { - qcflags_.reserve(obspaces.size()); - obserr_.reserve(obspaces.size()); - for (size_t jj = 0; jj < obspaces.size(); ++jj) { -// Allocate QC flags - qcflags_.emplace_back(std::shared_ptr>(new ObsData_(obspaces[jj], - obspaces[jj].obsvariables(), qcName))); -// Allocate and read initial obs error - obserr_.emplace_back(std::shared_ptr>(new ObsData_(obspaces[jj], - obspaces[jj].obsvariables(), errName))); - } -} - -} // namespace oops - -#endif // OOPS_BASE_QCDATA_H_ diff --git a/src/oops/interface/ObsDataVector.h b/src/oops/interface/ObsDataVector.h index 15731313a..0754546e6 100644 --- a/src/oops/interface/ObsDataVector.h +++ b/src/oops/interface/ObsDataVector.h @@ -49,6 +49,7 @@ class ObsDataVector : public util::Printable, unsigned int nobs() const {return data_->nobs();} // I/O + void read(const std::string &); void save(const std::string &) const; private: @@ -118,6 +119,14 @@ void ObsDataVector::print(std::ostream & os) const { } // ----------------------------------------------------------------------------- template +void ObsDataVector::read(const std::string & name) { + Log::trace() << "ObsDataVector::read starting " << name << std::endl; + util::Timer timer(classname(), "read"); + data_->read(name); + Log::trace() << "ObsDataVector::read done" << std::endl; +} +// ----------------------------------------------------------------------------- +template void ObsDataVector::save(const std::string & name) const { Log::trace() << "ObsDataVector::save starting " << name << std::endl; util::Timer timer(classname(), "save"); diff --git a/src/oops/runs/HofX.h b/src/oops/runs/HofX.h index abe97e64f..036cbecf9 100644 --- a/src/oops/runs/HofX.h +++ b/src/oops/runs/HofX.h @@ -12,12 +12,15 @@ #ifndef OOPS_RUNS_HOFX_H_ #define OOPS_RUNS_HOFX_H_ +#include #include #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" #include "oops/assimilation/CalcHofX.h" +#include "oops/base/GetValuesPost.h" #include "oops/base/instantiateObsFilterFactory.h" +#include "oops/base/ObsAuxControls.h" #include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" #include "oops/base/ObsSpaces.h" @@ -26,6 +29,7 @@ #include "oops/generic/instantiateObsErrorFactory.h" #include "oops/interface/Geometry.h" #include "oops/interface/Model.h" +#include "oops/interface/ModelAuxControl.h" #include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" @@ -40,7 +44,10 @@ namespace oops { /// H(x) is perturbed. It is saved as "hofx" by default, or as specified "hofx group name" template class HofX : public Application { typedef Geometry Geometry_; + typedef GetValuesPost GetValuesPost_; typedef Model Model_; + typedef ModelAuxControl ModelAux_; + typedef ObsAuxControls ObsAux_; typedef Observations Observations_; typedef ObsErrors ObsErrors_; typedef ObsSpaces ObsSpaces_; @@ -73,6 +80,7 @@ template class HofX : public Application { // Setup initial state const eckit::LocalConfiguration initialConfig(fullConfig, "initial condition"); State_ xx(geometry, initialConfig); + ModelAux_ moderr(geometry, initialConfig); const util::Duration flength(fullConfig.getString("forecast length")); Log::test() << "Initial state: " << xx << std::endl; @@ -93,11 +101,20 @@ template class HofX : public Application { // Setup observations const eckit::LocalConfiguration obsConfig(fullConfig, "observations"); - ObsSpaces_ obspace(obsConfig, this->getComm(), winbgn, winend); + ObsSpaces_ obspaces(obsConfig, this->getComm(), winbgn, winend); + ObsAux_ obsaux(obspaces, obsConfig); // Setup and run observer - CalcHofX hofx(obspace, geometry, fullConfig); - Observations_ yobs = hofx.compute(model, xx, post, flength); + CalcHofX hofx(obspaces, obsConfig); + hofx.initialize(obsaux); + +// run the model and compute H(x) + std::shared_ptr + getvals(new GetValuesPost_(obspaces, hofx.locations(), hofx.requiredVars())); + post.enrollProcessor(getvals); + model.forecast(xx, moderr, flength, post); + + Observations_ yobs = hofx.compute(getvals->geovals()); hofx.saveQcFlags("EffectiveQC"); hofx.saveObsErrors("EffectiveError"); @@ -108,7 +125,7 @@ template class HofX : public Application { // as ObsValue if "hofx group name" == ObsValue. bool obspert = fullConfig.getBool("obs perturbations", false); if (obspert) { - ObsErrors_ matR(obsConfig, obspace); + ObsErrors_ matR(obsConfig, obspaces); yobs.perturb(matR); Log::test() << "Perturbed H(x): " << std::endl << yobs << "End Perturbed H(x)" << std::endl; } diff --git a/src/oops/runs/HofXNoModel.h b/src/oops/runs/HofXNoModel.h index a8779a385..0ef814d17 100644 --- a/src/oops/runs/HofXNoModel.h +++ b/src/oops/runs/HofXNoModel.h @@ -8,7 +8,10 @@ #ifndef OOPS_RUNS_HOFXNOMODEL_H_ #define OOPS_RUNS_HOFXNOMODEL_H_ +#include +#include #include +#include #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/CalcHofX.h" @@ -16,7 +19,11 @@ #include "oops/base/instantiateObsFilterFactory.h" #include "oops/base/Observations.h" #include "oops/base/ObsSpaces.h" +#include "oops/base/Variables.h" #include "oops/interface/Geometry.h" +#include "oops/interface/GeoVaLs.h" +#include "oops/interface/GetValues.h" +#include "oops/interface/Locations.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" @@ -26,11 +33,21 @@ namespace oops { template class HofXNoModel : public Application { + typedef CalcHofX CalcHofX_; typedef Geometry Geometry_; + typedef GeoVaLs GeoVaLs_; + typedef GetValues GetValues_; + typedef Locations Locations_; + typedef ObsAuxControls ObsAux_; typedef Observations Observations_; typedef ObsSpaces ObsSpaces_; typedef State4D State4D_; + typedef std::vector> GeoVaLsVec_; + typedef std::vector> LocationsVec_; + typedef std::vector VariablesVec_; + + public: // ----------------------------------------------------------------------------- explicit HofXNoModel(const eckit::mpi::Comm & comm = oops::mpi::world()) : Application(comm) { @@ -40,34 +57,64 @@ template class HofXNoModel : public Application { virtual ~HofXNoModel() {} // ----------------------------------------------------------------------------- int execute(const eckit::Configuration & fullConfig) const { -// Setup observation window + // Setup observation window const util::Duration winlen(fullConfig.getString("window length")); const util::DateTime winbgn(fullConfig.getString("window begin")); const util::DateTime winend(winbgn + winlen); Log::info() << "Observation window from " << winbgn << " to " << winend << std::endl; -// Setup geometry + // Setup geometry const eckit::LocalConfiguration geometryConfig(fullConfig, "geometry"); const Geometry_ geometry(geometryConfig, this->getComm()); -// Setup states for H(x) + // Setup states for H(x) const eckit::LocalConfiguration stateConfig(fullConfig, "forecasts"); Log::info() << "States configuration is:" << stateConfig << std::endl; State4D_ xx(geometry, stateConfig); Log::test() << "Initial state: " << xx[0] << std::endl; -// Setup observations + // Setup observations const eckit::LocalConfiguration obsConfig(fullConfig, "observations"); - ObsSpaces_ obspace(obsConfig, this->getComm(), winbgn, winend); + ObsSpaces_ obspaces(obsConfig, this->getComm(), winbgn, winend); + ObsAux_ obsaux(obspaces, obsConfig); + CalcHofX_ hofx(obspaces, obsConfig); + hofx.initialize(obsaux); + + // Setup and check time steps between the states in 4D state + const size_t nstates = xx.size(); + util::Duration tstep = winlen; // for a single state + // if using several states, compute the timestep and check that it's the same for all states + if (nstates > 1) { + tstep = xx[1].validTime() - xx[0].validTime(); + for (size_t ii = 1; ii < nstates; ++ii) { + ASSERT(tstep == (xx[ii].validTime() - xx[ii-1].validTime())); + } + } + + // fill in GeoVaLs + GeoVaLsVec_ geovals; + const LocationsVec_ & locations = hofx.locations(); + const VariablesVec_ & vars = hofx.requiredVars(); + // loop over all observation types + for (size_t jj = 0; jj < obspaces.size(); ++jj) { + GetValues_ getvals(geometry, *locations[jj]); + // add GeoVaLs for this obs type + geovals.emplace_back(new GeoVaLs_(*locations[jj], vars[jj])); + // fill in by looping through 4D state + for (size_t ii = 0; ii < nstates; ++ii) { + util::DateTime t1 = std::max(xx[ii].validTime()-tstep/2, winbgn); + util::DateTime t2 = std::min(xx[ii].validTime()+tstep/2, winend); + getvals.fillGeoVaLs(xx[ii], t1, t2, *geovals[jj]); + } + } -// Setup and run observer - CalcHofX hofx(obspace, geometry, fullConfig); - const Observations_ & yobs = hofx.compute(xx); + // Compute H(x) on filled in geovals and run the filters + Observations_ yobs = hofx.compute(geovals); hofx.saveQcFlags("EffectiveQC"); hofx.saveObsErrors("EffectiveError"); Log::test() << "Final state: " << xx[xx.size()-1] << std::endl; -// Save H(x) + // Save H(x) Log::test() << "H(x): " << std::endl << yobs << "End H(x)" << std::endl; yobs.save("hofx"); From b9047a36338800c233e8494bfd8b26a75c478715 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 9 Dec 2020 09:37:46 -0700 Subject: [PATCH 021/142] make minimizer logs consistent (#988) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Yannick Trémolet <30638944+ytremolet@users.noreply.github.com> --- src/CMakeLists.txt | 2 ++ src/oops/assimilation/DRGMRESRMinimizer.h | 10 +++--- src/oops/assimilation/DRIPCGMinimizer.h | 19 +++++------- src/oops/assimilation/DRPCGMinimizer.h | 13 +++----- src/oops/assimilation/DRPFOMMinimizer.h | 13 +++----- src/oops/assimilation/DRPLanczosMinimizer.h | 13 +++----- src/oops/assimilation/FGMRES.h | 5 +-- src/oops/assimilation/FullGMRES.h | 5 +-- src/oops/assimilation/GMRESR.h | 5 +-- src/oops/assimilation/IPCG.h | 10 +++--- src/oops/assimilation/MINRES.h | 5 +-- src/oops/assimilation/MinimizerUtils.cc | 34 +++++++++++++++++++++ src/oops/assimilation/MinimizerUtils.h | 24 +++++++++++++++ src/oops/assimilation/PCG.h | 5 +-- src/oops/assimilation/PLanczos.h | 5 +-- src/oops/assimilation/RPCGMinimizer.h | 5 +-- src/oops/assimilation/RPLanczosMinimizer.h | 5 +-- 17 files changed, 116 insertions(+), 62 deletions(-) create mode 100644 src/oops/assimilation/MinimizerUtils.cc create mode 100644 src/oops/assimilation/MinimizerUtils.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 564f0f669..0684d0d86 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -59,6 +59,8 @@ oops/assimilation/LETKFSolverParameters.h oops/assimilation/linsysteigen.h oops/assimilation/LocalEnsembleSolver.h oops/assimilation/Minimizer.h +oops/assimilation/MinimizerUtils.cc +oops/assimilation/MinimizerUtils.h oops/assimilation/MINRES.h oops/assimilation/MINRESMinimizer.h oops/assimilation/PCG.h diff --git a/src/oops/assimilation/DRGMRESRMinimizer.h b/src/oops/assimilation/DRGMRESRMinimizer.h index dda92fd3a..d8715e827 100644 --- a/src/oops/assimilation/DRGMRESRMinimizer.h +++ b/src/oops/assimilation/DRGMRESRMinimizer.h @@ -20,6 +20,7 @@ #include "oops/assimilation/CostFunction.h" #include "oops/assimilation/DRMinimizer.h" #include "oops/assimilation/HtRinvHMatrix.h" +#include "oops/assimilation/MinimizerUtils.h" #include "oops/base/IdentityMatrix.h" #include "oops/util/dot_product.h" #include "oops/util/formats.h" @@ -104,7 +105,7 @@ double DRGMRESRMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, CtrlIn CtrlInc_ zz(xh); CtrlInc_ zh(xh); - double dotRr0 = dot_product(rr, rr); + double rrnorm0 = sqrt(dot_product(rr, rr)); double normReduction = 1.0; Log::info() << std::endl; @@ -136,9 +137,10 @@ double DRGMRESRMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, CtrlIn xh.axpy(cdotr, uh[jiter]); rr.axpy(-cdotr, c[jiter]); - normReduction = sqrt(dot_product(rr, rr)/dotRr0); - Log::info() << "DRGMRESR end of iteration " << jiter+1 << ". Norm reduction= " - << util::full_precision(normReduction) << std::endl << std::endl; + double rrnorm = sqrt(dot_product(rr, rr)); + normReduction = rrnorm/rrnorm0; + Log::info() << "DRGMRESR end of iteration " << jiter+1 << std::endl; + printNormReduction(jiter+1, rrnorm, normReduction); if (normReduction < tolerance) { Log::info() << "DRGMRESR: Achieved required reduction in residual norm." << std::endl; diff --git a/src/oops/assimilation/DRIPCGMinimizer.h b/src/oops/assimilation/DRIPCGMinimizer.h index 5cf4e8161..c495be918 100644 --- a/src/oops/assimilation/DRIPCGMinimizer.h +++ b/src/oops/assimilation/DRIPCGMinimizer.h @@ -20,6 +20,7 @@ #include "oops/assimilation/CostFunction.h" #include "oops/assimilation/DRMinimizer.h" #include "oops/assimilation/HtRinvHMatrix.h" +#include "oops/assimilation/MinimizerUtils.h" #include "oops/assimilation/QNewtonLMP.h" #include "oops/util/dot_product.h" #include "oops/util/formats.h" @@ -125,7 +126,7 @@ double DRIPCGMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, CtrlInc_ lmp_.multiply(rr, sh); B.multiply(sh, ss); - double dotRr0 = dot_product(rr, rr); + double rrnorm0 = sqrt(dot_product(rr, rr)); double dotSr0 = dot_product(rr, ss); double normReduction = 1.0; double rdots = dotSr0; @@ -188,16 +189,12 @@ double DRIPCGMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, CtrlInc_ // double rdotr = dot_product(rr, rr); // Log::info() << "DRIPCGMinimizer rdots = " << rdots // << ", sdots = " << sdots << ", rdotr = " << rdotr << std::endl; - normReduction = sqrt(dot_product(rr, rr)/dotRr0); - - Log::info() << "DRIPCG end of iteration " << jiter+1 << ". Norm reduction= " - << util::full_precision(normReduction) << std::endl - << " Quadratic cost function: J (" << jiter+1 << ") = " - << util::full_precision(costJ) << std::endl - << " Quadratic cost function: Jb (" << jiter+1 << ") = " - << util::full_precision(costJb) << std::endl - << " Quadratic cost function: JoJc(" << jiter+1 << ") = " - << util::full_precision(costJoJc) << std::endl << std::endl; + double rrnorm = sqrt(dot_product(rr, rr)); + normReduction = rrnorm/rrnorm0; + + Log::info() << "DRIPCG end of iteration " << jiter+1 << std::endl; + printNormReduction(jiter+1, rrnorm, normReduction); + printQuadraticCostFunction(jiter+1, costJ, costJb, costJoJc); // Save the pairs for preconditioning lmp_.push(pp, ph, ap, rho); diff --git a/src/oops/assimilation/DRPCGMinimizer.h b/src/oops/assimilation/DRPCGMinimizer.h index c618b9daa..90d3ce3f8 100644 --- a/src/oops/assimilation/DRPCGMinimizer.h +++ b/src/oops/assimilation/DRPCGMinimizer.h @@ -21,6 +21,7 @@ #include "oops/assimilation/CostFunction.h" #include "oops/assimilation/DRMinimizer.h" #include "oops/assimilation/HtRinvHMatrix.h" +#include "oops/assimilation/MinimizerUtils.h" #include "oops/assimilation/QNewtonLMP.h" #include "oops/util/dot_product.h" #include "oops/util/formats.h" @@ -215,15 +216,9 @@ double DRPCGMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, CtrlInc_ // r_{i+1}^T z_{i+1} / r_{0}^T z_{0} normReduction = sqrt(rdots/dotRr0); - Log::info() << "DRPCG end of iteration " << jiter+1 << std::endl - << " Norm reduction (" << std::setw(2) << jiter+1 << ") = " - << util::full_precision(normReduction) << std::endl - << " Quadratic cost function: J (" << std::setw(2) << jiter+1 << ") = " - << util::full_precision(costJ) << std::endl - << " Quadratic cost function: Jb (" << std::setw(2) << jiter+1 << ") = " - << util::full_precision(costJb) << std::endl - << " Quadratic cost function: JoJc(" << std::setw(2) << jiter+1 << ") = " - << util::full_precision(costJoJc) << std::endl << std::endl; + Log::info() << "DRPCG end of iteration " << jiter+1 << std::endl; + printNormReduction(jiter+1, sqrt(rdots), normReduction); + printQuadraticCostFunction(jiter+1, costJ, costJb, costJoJc); // Save the pairs for preconditioning lmp_.push(pp, hh, qq, rho); diff --git a/src/oops/assimilation/DRPFOMMinimizer.h b/src/oops/assimilation/DRPFOMMinimizer.h index a2306df18..832769072 100644 --- a/src/oops/assimilation/DRPFOMMinimizer.h +++ b/src/oops/assimilation/DRPFOMMinimizer.h @@ -22,6 +22,7 @@ #include "oops/assimilation/CostFunction.h" #include "oops/assimilation/DRMinimizer.h" #include "oops/assimilation/HtRinvHMatrix.h" +#include "oops/assimilation/MinimizerUtils.h" #include "oops/assimilation/SpectralLMP.h" #include "oops/assimilation/UpHessSolve.h" #include "oops/util/dot_product.h" @@ -231,15 +232,9 @@ double DRPFOMMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, CtrlInc double rznorm = beta*std::abs(ss[jiter]); normReduction = rznorm/beta0; - Log::info() << "DRPFOM end of iteration " << jiter+1 << std::endl - << " Norm reduction (" << std::setw(2) << jiter+1 << ") = " - << util::full_precision(normReduction) << std::endl - << " Quadratic cost function: J (" << std::setw(2) << jiter+1 << ") = " - << util::full_precision(costJ) << std::endl - << " Quadratic cost function: Jb (" << std::setw(2) << jiter+1 << ") = " - << util::full_precision(costJb) << std::endl - << " Quadratic cost function: JoJc(" << std::setw(2) << jiter+1 << ") = " - << util::full_precision(costJoJc) << std::endl << std::endl; + Log::info() << "DRPFOM end of iteration " << jiter+1 << std::endl; + printNormReduction(jiter+1, rznorm, normReduction); + printQuadraticCostFunction(jiter+1, costJ, costJb, costJoJc); if (normReduction < tolerance) { Log::info() << "DRPFOM: Achieved required reduction in residual norm." << std::endl; diff --git a/src/oops/assimilation/DRPLanczosMinimizer.h b/src/oops/assimilation/DRPLanczosMinimizer.h index f75235e1c..68cc682ac 100644 --- a/src/oops/assimilation/DRPLanczosMinimizer.h +++ b/src/oops/assimilation/DRPLanczosMinimizer.h @@ -22,6 +22,7 @@ #include "oops/assimilation/CostFunction.h" #include "oops/assimilation/DRMinimizer.h" #include "oops/assimilation/HtRinvHMatrix.h" +#include "oops/assimilation/MinimizerUtils.h" #include "oops/assimilation/SpectralLMP.h" #include "oops/assimilation/TriDiagSolve.h" #include "oops/util/dot_product.h" @@ -223,15 +224,9 @@ double DRPLanczosMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, Ctr double rznorm = beta*std::abs(ss[jiter]); normReduction = rznorm/beta0; - Log::info() << "DRPLanczos end of iteration " << jiter+1 << std::endl - << " Norm reduction (" << std::setw(2) << jiter+1 << ") = " - << util::full_precision(normReduction) << std::endl - << " Quadratic cost function: J (" << std::setw(2) << jiter+1 << ") = " - << util::full_precision(costJ) << std::endl - << " Quadratic cost function: Jb (" << std::setw(2) << jiter+1 << ") = " - << util::full_precision(costJb) << std::endl - << " Quadratic cost function: JoJc(" << std::setw(2) << jiter+1 << ") = " - << util::full_precision(costJoJc) << std::endl << std::endl; + Log::info() << "DRPLanczos end of iteration " << jiter+1 << std::endl; + printNormReduction(jiter+1, rznorm, normReduction); + printQuadraticCostFunction(jiter+1, costJ, costJb, costJoJc); if (normReduction < tolerance) { Log::info() << "DRPLanczos: Achieved required reduction in residual norm." << std::endl; diff --git a/src/oops/assimilation/FGMRES.h b/src/oops/assimilation/FGMRES.h index e8e369d67..50680d3f6 100644 --- a/src/oops/assimilation/FGMRES.h +++ b/src/oops/assimilation/FGMRES.h @@ -15,6 +15,7 @@ #include #include +#include "oops/assimilation/MinimizerUtils.h" #include "oops/assimilation/rotmat.h" #include "oops/assimilation/UpTriSolve.h" #include "oops/util/dot_product.h" @@ -170,8 +171,8 @@ double FGMRES(VECTOR & x, const VECTOR & b, } normReduction = std::abs(s[jiter+1])/bnrm2; - Log::info() << "FGMRES end of iteration " << jiter+1 << ". PNorm reduction= " - << util::full_precision(normReduction) << std::endl << std::endl; + Log::info() << "FGMRES end of iteration " << jiter+1 << std::endl; + printNormReduction(jiter+1, std::abs(s[jiter+1]), normReduction); if (normReduction <= tolerance) { Log::info() << "FGMRES: Achieved required reduction in residual norm." << std::endl; diff --git a/src/oops/assimilation/FullGMRES.h b/src/oops/assimilation/FullGMRES.h index 9ddde19a5..b8f7fcf35 100644 --- a/src/oops/assimilation/FullGMRES.h +++ b/src/oops/assimilation/FullGMRES.h @@ -15,6 +15,7 @@ #include #include +#include "oops/assimilation/MinimizerUtils.h" #include "oops/assimilation/rotmat.h" #include "oops/assimilation/UpTriSolve.h" #include "oops/util/dot_product.h" @@ -171,8 +172,8 @@ double FullGMRES(VECTOR & xx, const VECTOR & bb, const AMATRIX & A, ss[jiter] = temp; normReduction = std::abs(ss[jiter+1])/znrm2; - Log::info() << "FullGMRES end of iteration " << jiter+1 << ". PNorm reduction= " - << util::full_precision(normReduction) << std::endl << std::endl; + Log::info() << "FullGMRES end of iteration " << jiter+1 << std::endl; + printNormReduction(jiter+1, abs(ss[jiter+1]), normReduction); if (normReduction <= tolerance) { Log::info() << "FullGMRES: Achieved required reduction in presidual norm." << std::endl; diff --git a/src/oops/assimilation/GMRESR.h b/src/oops/assimilation/GMRESR.h index 5b7053040..44625f810 100644 --- a/src/oops/assimilation/GMRESR.h +++ b/src/oops/assimilation/GMRESR.h @@ -14,6 +14,7 @@ #include #include +#include "oops/assimilation/MinimizerUtils.h" #include "oops/util/dot_product.h" #include "oops/util/formats.h" #include "oops/util/Logger.h" @@ -113,8 +114,8 @@ double GMRESR(VECTOR & xx, const VECTOR & bb, rrnorm = sqrt(dot_product(rr, rr)); normReduction = rrnorm/rrnorm0; - Log::info() << "GMRESR end of iteration " << jiter+1 << ". Norm reduction= " - << util::full_precision(normReduction) << std::endl << std::endl; + Log::info() << "GMRESR end of iteration " << jiter+1 << std::endl; + printNormReduction(jiter+1, rrnorm, normReduction); if (normReduction < tolerance) { Log::info() << "GMRESR: Achieved required reduction in residual norm." << std::endl; diff --git a/src/oops/assimilation/IPCG.h b/src/oops/assimilation/IPCG.h index 514328bfe..7f4dd772c 100644 --- a/src/oops/assimilation/IPCG.h +++ b/src/oops/assimilation/IPCG.h @@ -14,6 +14,7 @@ #include #include +#include "oops/assimilation/MinimizerUtils.h" #include "oops/util/dot_product.h" #include "oops/util/formats.h" #include "oops/util/Logger.h" @@ -88,7 +89,7 @@ double IPCG(VECTOR & x, const VECTOR & b, // s = precond r precond.multiply(r, s); - double dotRr0 = dot_product(r, r); + double rnorm0 = sqrt(dot_product(r, r)); double dotSr0 = dot_product(r, s); double normReduction = 1.0; double rdots_old = dotSr0; @@ -143,9 +144,10 @@ double IPCG(VECTOR & x, const VECTOR & b, vVEC.push_back(v); zVEC.push_back(z); - normReduction = sqrt(dot_product(r, r)/dotRr0); - Log::info() << "IPCG end of iteration " << jiter+1 << ". Norm reduction= " - << util::full_precision(normReduction) << std::endl << std::endl; + double rnorm = sqrt(dot_product(r, r)); + normReduction = rnorm/rnorm0; + Log::info() << "IPCG end of iteration " << jiter+1 << std::endl; + printNormReduction(jiter+1, rnorm, normReduction); if (normReduction < tolerance) { Log::info() << "IPCG: Achieved required reduction in residual norm." << std::endl; diff --git a/src/oops/assimilation/MINRES.h b/src/oops/assimilation/MINRES.h index 49ca535d9..cdca5d0ac 100644 --- a/src/oops/assimilation/MINRES.h +++ b/src/oops/assimilation/MINRES.h @@ -15,6 +15,7 @@ #include #include +#include "oops/assimilation/MinimizerUtils.h" #include "oops/util/dot_product.h" #include "oops/util/formats.h" #include "oops/util/Logger.h" @@ -146,8 +147,8 @@ double MINRES(VECTOR & x, const VECTOR & b, normReduction = phibar/ynrm2; - Log::info() << "MINRES end of iteration " << jiter+1 << ". PNorm reduction= " - << util::full_precision(normReduction) << std::endl << std::endl; + Log::info() << "MINRES end of iteration " << jiter+1 << std::endl; + printNormReduction(jiter+1, phibar, normReduction); if (normReduction <= tolerance) { Log::info() << "MINRES: Achieved required reduction in residual norm." << std::endl; diff --git a/src/oops/assimilation/MinimizerUtils.cc b/src/oops/assimilation/MinimizerUtils.cc new file mode 100644 index 000000000..a1d22c929 --- /dev/null +++ b/src/oops/assimilation/MinimizerUtils.cc @@ -0,0 +1,34 @@ +/* + * (C) Copyright 2020 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/assimilation/MinimizerUtils.h" + +#include "oops/util/formats.h" +#include "oops/util/Logger.h" + +namespace oops { + +void printNormReduction(int iteration, const double & grad, const double & norm) { + Log::info() << " Gradient reduction (" << std::setw(2) << iteration << ") = " + << util::full_precision(grad) << std::endl + << " Norm reduction (" << std::setw(2) << iteration << ") = " + << util::full_precision(norm) << std::endl << std::endl; +} + +void printQuadraticCostFunction(int iteration, const double & costJ, + const double & costJb, const double & costJoJc) { + Log::info() << " Quadratic cost function: J (" << std::setw(2) << iteration << ") = " + << util::full_precision(costJ) << std::endl + << " Quadratic cost function: Jb (" << std::setw(2) << iteration << ") = " + << util::full_precision(costJb) << std::endl + << " Quadratic cost function: JoJc(" << std::setw(2) << iteration << ") = " + << util::full_precision(costJoJc) << std::endl << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace oops diff --git a/src/oops/assimilation/MinimizerUtils.h b/src/oops/assimilation/MinimizerUtils.h new file mode 100644 index 000000000..70e8a8070 --- /dev/null +++ b/src/oops/assimilation/MinimizerUtils.h @@ -0,0 +1,24 @@ +/* + * (C) Copyright 2020 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_ASSIMILATION_MINIMIZERUTILS_H_ +#define OOPS_ASSIMILATION_MINIMIZERUTILS_H_ + +namespace oops { + +/// Prints to Log::info gradient reduction \p grad and normalized gradient reduction \p norm +/// for iteration \p iteration +void printNormReduction(int iteration, const double & grad, const double & norm); +/// Prints to Log::info cost function values for \p costJ, \p costJb, \p costJoJc for +/// iteration \p iteration +void printQuadraticCostFunction(int iteration, const double & costJ, const double & costJb, + const double & costJoJc); +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_ASSIMILATION_MINIMIZERUTILS_H_ diff --git a/src/oops/assimilation/PCG.h b/src/oops/assimilation/PCG.h index 6e99b9b6e..84109778c 100644 --- a/src/oops/assimilation/PCG.h +++ b/src/oops/assimilation/PCG.h @@ -14,6 +14,7 @@ #include #include +#include "oops/assimilation/MinimizerUtils.h" #include "oops/util/dot_product.h" #include "oops/util/formats.h" #include "oops/util/Logger.h" @@ -141,8 +142,8 @@ double PCG(VECTOR & x, const VECTOR & b, normReduction = sqrt(rdots/dotRr0); - Log::info() << "PCG end of iteration " << jiter+1 << ". Norm reduction= " - << util::full_precision(normReduction) << std::endl << std::endl; + Log::info() << "PCG end of iteration " << jiter+1 << std::endl; + printNormReduction(jiter+1, sqrt(rdots), normReduction); if (normReduction < tolerance) { Log::info() << "PCG: Achieved required reduction in residual norm." << std::endl; diff --git a/src/oops/assimilation/PLanczos.h b/src/oops/assimilation/PLanczos.h index 9e74f41c0..cc04bad8a 100644 --- a/src/oops/assimilation/PLanczos.h +++ b/src/oops/assimilation/PLanczos.h @@ -14,6 +14,7 @@ #include #include +#include "oops/assimilation/MinimizerUtils.h" #include "oops/assimilation/TriDiagSolve.h" #include "oops/util/dot_product.h" #include "oops/util/formats.h" @@ -164,8 +165,8 @@ double PLanczos(VECTOR & xx, const VECTOR & bb, betas.push_back(beta); - Log::info() << "PLanczos end of iteration " << jiter+1 << ". Norm reduction= " - << util::full_precision(normReduction) << std::endl << std::endl; + Log::info() << "PLanczos end of iteration " << jiter+1 << std::endl; + printNormReduction(jiter+1, rznorm, normReduction); ++jiter; diff --git a/src/oops/assimilation/RPCGMinimizer.h b/src/oops/assimilation/RPCGMinimizer.h index 03c22e2b0..18a3ea40c 100644 --- a/src/oops/assimilation/RPCGMinimizer.h +++ b/src/oops/assimilation/RPCGMinimizer.h @@ -18,6 +18,7 @@ #include "oops/assimilation/DualMinimizer.h" #include "oops/assimilation/DualVector.h" #include "oops/assimilation/HBHtMatrix.h" +#include "oops/assimilation/MinimizerUtils.h" #include "oops/assimilation/RinvMatrix.h" #include "oops/base/IdentityMatrix.h" #include "oops/util/dot_product.h" @@ -230,8 +231,8 @@ double RPCGMinimizer::solve(Dual_ & vv, double & vvp, Dual_ & rr, normReduction = sqrt(dotwr/dotw0r0); - Log::info() << "RPCG end of iteration " << jiter+1 << ". Norm reduction= " - << util::full_precision(normReduction) << std::endl << std::endl; + Log::info() << "RPCG end of iteration " << jiter+1 << std::endl; + printNormReduction(jiter+1, sqrt(dotwr), normReduction); if (normReduction < tolerance) { Log::info() << "RPCG: Achieved required reduction in residual norm." << std::endl; diff --git a/src/oops/assimilation/RPLanczosMinimizer.h b/src/oops/assimilation/RPLanczosMinimizer.h index 7ed214aec..6afda13e5 100644 --- a/src/oops/assimilation/RPLanczosMinimizer.h +++ b/src/oops/assimilation/RPLanczosMinimizer.h @@ -19,6 +19,7 @@ #include "oops/assimilation/DualMinimizer.h" #include "oops/assimilation/DualVector.h" #include "oops/assimilation/HBHtMatrix.h" +#include "oops/assimilation/MinimizerUtils.h" #include "oops/assimilation/RinvMatrix.h" #include "oops/base/IdentityMatrix.h" #include "oops/util/dot_product.h" @@ -235,8 +236,8 @@ double RPLanczosMinimizer::solve(Dual_ & vv, double & vvp, Dual_ & r betas.push_back(beta); - Log::info() << "RPLanczos end of iteration " << jiter+1 << ". Norm reduction= " - << util::full_precision(normReduction) << std::endl << std::endl; + Log::info() << "RPLanczos end of iteration " << jiter+1 << std::endl; + printNormReduction(jiter+1, rznorm, normReduction); ++jiter; From 3224e84068d9d529f7e28ef50522cbc9295eade3 Mon Sep 17 00:00:00 2001 From: Clementine Gas <43183478+cmgas@users.noreply.github.com> Date: Mon, 14 Dec 2020 14:29:55 -0700 Subject: [PATCH 022/142] Remove boost::ptr_vector from minimizers and lmp (#993) * Remove boost::ptr_vector from minimizers and lmp * use emplace_back(std::unique_ptr(new CtrlInc_(..))) * Change ww declaration and use std::move * add #include * Changes for tests (emplace_back of std::unique_ptr) --- src/oops/assimilation/DRPFOMMinimizer.h | 41 +++++++------- src/oops/assimilation/DRPLanczosMinimizer.h | 39 +++++++------ src/oops/assimilation/SpectralLMP.h | 62 ++++++++++----------- src/test/assimilation/SpectralLMP.h | 45 ++++++++------- 4 files changed, 92 insertions(+), 95 deletions(-) diff --git a/src/oops/assimilation/DRPFOMMinimizer.h b/src/oops/assimilation/DRPFOMMinimizer.h index 832769072..9ee21044a 100644 --- a/src/oops/assimilation/DRPFOMMinimizer.h +++ b/src/oops/assimilation/DRPFOMMinimizer.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -12,11 +12,10 @@ #define OOPS_ASSIMILATION_DRPFOMMINIMIZER_H_ #include +#include #include #include -#include - #include "oops/assimilation/BMatrix.h" #include "oops/assimilation/ControlIncrement.h" #include "oops/assimilation/CostFunction.h" @@ -83,9 +82,9 @@ template class DRPFOMMinimizer : public DRMinimize SpectralLMP lmp_; // !!!!! Needs to be generalized for Hessenberg Matrix. - boost::ptr_vector hvecs_; - boost::ptr_vector vvecs_; - boost::ptr_vector zvecs_; + std::vector> hvecs_; + std::vector> vvecs_; + std::vector> zvecs_; std::vector alphas_; std::vector betas_; }; @@ -144,11 +143,11 @@ double DRPFOMMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, CtrlInc zz *= 1/beta; // hvecs[0] = pr_{1} --> required for solution - hvecs_.push_back(new CtrlInc_(pr)); + hvecs_.emplace_back(std::unique_ptr(new CtrlInc_(pr))); // zvecs[0] = z_{1} ---> for re-orthogonalization - zvecs_.push_back(new CtrlInc_(zz)); + zvecs_.emplace_back(std::unique_ptr(new CtrlInc_(zz))); // vvecs[0] = v_{1} ---> for re-orthogonalization - vvecs_.push_back(new CtrlInc_(vv)); + vvecs_.emplace_back(std::unique_ptr(new CtrlInc_(vv))); // Initialiaze (maxiter + 1) by maxiter matrix H Hess.resize(maxiter); @@ -170,8 +169,8 @@ double DRPFOMMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, CtrlInc vv += pr; // Arnoldi Process for (int jj = 0; jj <= jiter; ++jj) { - Hess[jiter][jj] = dot_product(zvecs_[jj], vv); - vv.axpy(-Hess[jiter][jj], vvecs_[jj]); + Hess[jiter][jj] = dot_product(*zvecs_[jj], vv); + vv.axpy(-Hess[jiter][jj], *vvecs_[jj]); } // z_{i+1} = B LMP v_{i+1} @@ -192,18 +191,18 @@ double DRPFOMMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, CtrlInc zz *= 1/beta; // hvecs[i+1] =pr_{i+1} - hvecs_.push_back(new CtrlInc_(pr)); + hvecs_.emplace_back(std::unique_ptr(new CtrlInc_(pr))); // zvecs[i+1] = z_{i+1} - zvecs_.push_back(new CtrlInc_(zz)); + zvecs_.emplace_back(std::unique_ptr(new CtrlInc_(zz))); // vvecs[i+1] = v_{i+1} - vvecs_.push_back(new CtrlInc_(vv)); + vvecs_.emplace_back(std::unique_ptr(new CtrlInc_(vv))); if (jiter == 0) { ss.push_back(beta0/Hess[0][0]); dd.push_back(beta0); } else { // Solve the upper Hessenberg system H_{i} s_{i} = beta0 * e_1 - dd.push_back(beta0*dot_product(zvecs_[0], vv)); + dd.push_back(beta0*dot_product(*zvecs_[0], vv)); UpHess = Hess; UpHess.resize(jiter+1); for (int ii = 0; ii <= jiter; ii++) { @@ -223,8 +222,8 @@ double DRPFOMMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, CtrlInc double costJ = costJ0; double costJb = costJ0Jb; for (int jj = 0; jj < jiter+1; ++jj) { - costJ -= 0.5 * ss[jj] * dot_product(zvecs_[jj], rr); - costJb += 0.5 * ss[jj] * dot_product(vvecs_[jj], zvecs_[jj]) * ss[jj]; + costJ -= 0.5 * ss[jj] * dot_product(*zvecs_[jj], rr); + costJb += 0.5 * ss[jj] * dot_product(*vvecs_[jj], *zvecs_[jj]) * ss[jj]; } double costJoJc = costJ - costJb; @@ -244,8 +243,8 @@ double DRPFOMMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, CtrlInc // Calculate the solution (dxh = Binv dx) for (unsigned int jj = 0; jj < ss.size(); ++jj) { - dx.axpy(ss[jj], zvecs_[jj]); - dxh.axpy(ss[jj], hvecs_[jj]); + dx.axpy(ss[jj], *zvecs_[jj]); + dxh.axpy(ss[jj], *hvecs_[jj]); } return normReduction; diff --git a/src/oops/assimilation/DRPLanczosMinimizer.h b/src/oops/assimilation/DRPLanczosMinimizer.h index 68cc682ac..f2c103810 100644 --- a/src/oops/assimilation/DRPLanczosMinimizer.h +++ b/src/oops/assimilation/DRPLanczosMinimizer.h @@ -12,11 +12,10 @@ #define OOPS_ASSIMILATION_DRPLANCZOSMINIMIZER_H_ #include +#include #include #include -#include - #include "oops/assimilation/BMatrix.h" #include "oops/assimilation/ControlIncrement.h" #include "oops/assimilation/CostFunction.h" @@ -88,9 +87,9 @@ template class DRPLanczosMinimizer : public DRMini SpectralLMP lmp_; - boost::ptr_vector hvecs_; - boost::ptr_vector vvecs_; - boost::ptr_vector zvecs_; + std::vector> hvecs_; + std::vector> vvecs_; + std::vector> zvecs_; std::vector alphas_; std::vector betas_; }; @@ -142,11 +141,11 @@ double DRPLanczosMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, Ctr zz *= 1/beta; // hvecs[0] = pr_{1} --> required for solution - hvecs_.push_back(new CtrlInc_(pr)); + hvecs_.emplace_back(std::unique_ptr(new CtrlInc_(pr))); // zvecs[0] = z_{1} ---> for re-orthogonalization - zvecs_.push_back(new CtrlInc_(zz)); + zvecs_.emplace_back(std::unique_ptr(new CtrlInc_(zz))); // vvecs[0] = v_{1} ---> for re-orthogonalization - vvecs_.push_back(new CtrlInc_(vv)); + vvecs_.emplace_back(std::unique_ptr(new CtrlInc_(vv))); double normReduction = 1.0; @@ -159,19 +158,19 @@ double DRPLanczosMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, Ctr vv += pr; if (jiter > 0) { - vv.axpy(-beta, vvecs_[jiter-1]); + vv.axpy(-beta, *vvecs_[jiter-1]); } // alpha_{i} = v_{i+1}^T z_{i} double alpha = dot_product(zz, vv); // v_{i+1} = v_{i+1} - alpha_{i} v_{i} - vv.axpy(-alpha, vvecs_[jiter]); // vv = vv - alpha * v_j + vv.axpy(-alpha, *vvecs_[jiter]); // vv = vv - alpha * v_j // Re-orthogonalization for (int jj = 0; jj < jiter; ++jj) { - double proj = dot_product(vv, zvecs_[jj]); - vv.axpy(-proj, vvecs_[jj]); + double proj = dot_product(vv, *zvecs_[jj]); + vv.axpy(-proj, *vvecs_[jj]); } // z_{i+1} = B LMP v_{i+1} @@ -189,11 +188,11 @@ double DRPLanczosMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, Ctr zz *= 1/beta; // hvecs[i+1] =pr_{i+1} - hvecs_.push_back(new CtrlInc_(pr)); + hvecs_.emplace_back(std::unique_ptr(new CtrlInc_(pr))); // zvecs[i+1] = z_{i+1} - zvecs_.push_back(new CtrlInc_(zz)); + zvecs_.emplace_back(std::unique_ptr(new CtrlInc_(zz))); // vvecs[i+1] = v_{i+1} - vvecs_.push_back(new CtrlInc_(vv)); + vvecs_.emplace_back(std::unique_ptr(new CtrlInc_(vv))); alphas_.push_back(alpha); @@ -202,7 +201,7 @@ double DRPLanczosMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, Ctr dd.push_back(beta0); } else { // Solve the tridiagonal system T_{i} s_{i} = beta0 * e_1 - dd.push_back(beta0*dot_product(zvecs_[0], vv)); + dd.push_back(beta0*dot_product(*zvecs_[0], vv)); TriDiagSolve(alphas_, betas_, dd, ss); } @@ -215,8 +214,8 @@ double DRPLanczosMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, Ctr double costJb = costJ0Jb; for (int jj = 0; jj < jiter+1; ++jj) { - costJ -= 0.5 * ss[jj] * dot_product(zvecs_[jj], rr); - costJb += 0.5 * ss[jj] * dot_product(vvecs_[jj], zvecs_[jj]) * ss[jj]; + costJ -= 0.5 * ss[jj] * dot_product(*zvecs_[jj], rr); + costJb += 0.5 * ss[jj] * dot_product(*vvecs_[jj], *zvecs_[jj]) * ss[jj]; } double costJoJc = costJ - costJb; @@ -239,8 +238,8 @@ double DRPLanczosMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, Ctr // Calculate the solution (dxh = Binv dx) for (unsigned int jj = 0; jj < ss.size(); ++jj) { - dx.axpy(ss[jj], zvecs_[jj]); - dxh.axpy(ss[jj], hvecs_[jj]); + dx.axpy(ss[jj], *zvecs_[jj]); + dxh.axpy(ss[jj], *hvecs_[jj]); } return normReduction; diff --git a/src/oops/assimilation/SpectralLMP.h b/src/oops/assimilation/SpectralLMP.h index 4d9b0635d..490725aa5 100644 --- a/src/oops/assimilation/SpectralLMP.h +++ b/src/oops/assimilation/SpectralLMP.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -13,11 +13,11 @@ #include #include +#include #include +#include #include -#include - #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/TriDiagSpectrum.h" #include "oops/util/dot_product.h" @@ -52,8 +52,8 @@ template class SpectralLMP { explicit SpectralLMP(const eckit::Configuration &); ~SpectralLMP() {} - void update(boost::ptr_vector &, boost::ptr_vector &, - boost::ptr_vector &, std::vector &, std::vector &); + void update(std::vector> &, std::vector> &, + std::vector> &, std::vector &, std::vector &); void multiply(const VECTOR &, VECTOR &) const; @@ -64,14 +64,14 @@ template class SpectralLMP { int maxouter_; int update_; - boost::ptr_vector X_; - boost::ptr_vector U_; + std::vector> X_; + std::vector> U_; std::vector eigvals_; std::vector omega_; // For RitzPrecond - boost::ptr_vector Y_; - boost::ptr_vector S_; + std::vector> Y_; + std::vector> S_; std::vector Zlast_; std::vector Zhlast_; std::vector usedpairIndx_; @@ -110,9 +110,9 @@ SpectralLMP::SpectralLMP(const eckit::Configuration & conf) // ----------------------------------------------------------------------------- template -void SpectralLMP::update(boost::ptr_vector & Zv, - boost::ptr_vector & Zhl, - boost::ptr_vector & Zl, +void SpectralLMP::update(std::vector> & Zv, + std::vector> & Zhl, + std::vector> & Zl, std::vector & alphas, std::vector & betas) { // If useoldpairs = false, use only current information @@ -205,25 +205,25 @@ void SpectralLMP::update(boost::ptr_vector & Zv, U_.erase(U_.begin(), U_.begin() + std::min(xsize, minsize)); } for (unsigned jiter = 0; jiter < convIndx.size(); ++jiter) { - VECTOR * ww = new VECTOR(Zl[0], false); + std::unique_ptr ww(new VECTOR(*Zl[0], false)); for (unsigned iiter = 0; iiter < nvec-1; ++iiter) { - ww->axpy(evecs[convIndx[jiter]][iiter], Zl[iiter]); + ww->axpy(evecs[convIndx[jiter]][iiter], *Zl[iiter]); } // Add new information - X_.push_back(ww); + X_.emplace_back(std::move(ww)); } for (unsigned jiter = 0; jiter < convIndx.size(); ++jiter) { - VECTOR * ww = new VECTOR(Zl[0], false); + std::unique_ptr ww(new VECTOR(*Zl[0], false)); for (unsigned iiter = 0; iiter < nvec-1; ++iiter) { - ww->axpy(evecs[convIndx[jiter]][iiter], Zhl[iiter]); + ww->axpy(evecs[convIndx[jiter]][iiter], *Zhl[iiter]); } // Add new information - U_.push_back(ww); + U_.emplace_back(std::move(ww)); } if (RitzPrecond_) { - Zlast_.push_back(Zl[nvec-1]); - Zhlast_.push_back(Zhl[nvec-1]); + Zlast_.push_back(*Zl[nvec-1]); + Zhlast_.push_back(*Zhl[nvec-1]); // Calculate the matrix Y = [U1*omega1, ..., Uk*omegak] Y_.clear(); @@ -232,12 +232,12 @@ void SpectralLMP::update(boost::ptr_vector & Zv, for (unsigned kiter = 0; kiter < usedpairIndx_.size(); ++kiter) { if (usedpairIndx_[kiter] != 0) { zcount.push_back(kiter); - VECTOR * ww = new VECTOR(Zl[0], false); + std::unique_ptr ww(new VECTOR(*Zl[0], false)); for (unsigned jiter = 0; jiter < usedpairIndx_[kiter]; ++jiter) { - ww->axpy(omega_[jiter + kk], U_[jiter + kk]); + ww->axpy(omega_[jiter + kk], *U_[jiter + kk]); } kk += usedpairIndx_[kiter]; - Y_.push_back(ww); + Y_.emplace_back(std::move(ww)); } } @@ -246,12 +246,12 @@ void SpectralLMP::update(boost::ptr_vector & Zv, kk = 0; for (unsigned kiter = 0; kiter < usedpairIndx_.size(); ++kiter) { if (usedpairIndx_[kiter] != 0) { - VECTOR * ww = new VECTOR(Zl[0], false); + std::unique_ptr ww(new VECTOR(*Zl[0], false)); for (unsigned jiter = 0; jiter < usedpairIndx_[kiter]; ++jiter) { - ww->axpy(omega_[jiter + kk], X_[jiter + kk]); + ww->axpy(omega_[jiter + kk], *X_[jiter + kk]); } kk += usedpairIndx_[kiter]; - S_.push_back(ww); + S_.emplace_back(std::move(ww)); } } } @@ -275,7 +275,7 @@ void SpectralLMP::multiply(const VECTOR & a, VECTOR & b) const { double zeval = std::min(10.0, eigvals_[iiter]); // double zeval = eigvals_[iiter]; double zz = 1.0/zeval - 1.0; - b.axpy(zz*dot_product(a, X_[iiter]), U_[iiter]); + b.axpy(zz*dot_product(a, *X_[iiter]), *U_[iiter]); } if (RitzPrecond_ && eigvals_.size() != 0) { @@ -283,14 +283,14 @@ void SpectralLMP::multiply(const VECTOR & a, VECTOR & b) const { std::vector sta; sta.clear(); for (unsigned iiter = 0; iiter < S_.size(); ++iiter) { - sta.push_back(dot_product(S_[iiter], a)); + sta.push_back(dot_product(*S_[iiter], a)); } // Yk (sta(k) - Zlast' a) for (unsigned kiter = 0; kiter < S_.size(); ++kiter) { for (unsigned iiter = 0; iiter < eigvals_.size(); ++iiter) { double wxap = sta[kiter] - dot_product(a, Zlast_[zcount[kiter]]); - b.axpy(wxap, Y_[kiter]); + b.axpy(wxap, *Y_[kiter]); } } diff --git a/src/test/assimilation/SpectralLMP.h b/src/test/assimilation/SpectralLMP.h index e0d215a3a..d8133d8ed 100644 --- a/src/test/assimilation/SpectralLMP.h +++ b/src/test/assimilation/SpectralLMP.h @@ -8,11 +8,10 @@ #ifndef TEST_ASSIMILATION_SPECTRALLMP_H_ #define TEST_ASSIMILATION_SPECTRALLMP_H_ +#include #include #include -#include - #define ECKIT_TESTING_SELF_REGISTER_CASES 0 #include "eckit/config/LocalConfiguration.h" @@ -33,22 +32,22 @@ namespace test { oops::SpectralLMP spectralLMP(conf); // assign vectors following DRPLanczosMinimizer.h - boost::ptr_vector hvecs; - boost::ptr_vector vvecs; - boost::ptr_vector zvecs; + std::vector> hvecs; + std::vector> vvecs; + std::vector> zvecs; std::vector alphas; std::vector betas; // Simple case - hvecs.push_back(new Vector3D(1, 1, 1)); - hvecs.push_back(new Vector3D(1, 1, 1)); - hvecs.push_back(new Vector3D(1, 1, 1)); - vvecs.push_back(new Vector3D(1, 1, 1)); - vvecs.push_back(new Vector3D(1, 1, 1)); - vvecs.push_back(new Vector3D(1, 1, 1)); - zvecs.push_back(new Vector3D(1, 1, 1)); - zvecs.push_back(new Vector3D(1, 1, 1)); - zvecs.push_back(new Vector3D(1, 1, 1)); + hvecs.emplace_back(std::unique_ptr(new Vector3D(1, 1, 1))); + hvecs.emplace_back(std::unique_ptr(new Vector3D(1, 1, 1))); + hvecs.emplace_back(std::unique_ptr(new Vector3D(1, 1, 1))); + vvecs.emplace_back(std::unique_ptr(new Vector3D(1, 1, 1))); + vvecs.emplace_back(std::unique_ptr(new Vector3D(1, 1, 1))); + vvecs.emplace_back(std::unique_ptr(new Vector3D(1, 1, 1))); + zvecs.emplace_back(std::unique_ptr(new Vector3D(1, 1, 1))); + zvecs.emplace_back(std::unique_ptr(new Vector3D(1, 1, 1))); + zvecs.emplace_back(std::unique_ptr(new Vector3D(1, 1, 1))); alphas.push_back(1.0); alphas.push_back(1.0); @@ -68,15 +67,15 @@ namespace test { zvecs.clear(); alphas.clear(); betas.clear(); - hvecs.push_back(new Vector3D(10, 1, -10)); - hvecs.push_back(new Vector3D(1, 10, 100)); - hvecs.push_back(new Vector3D(-5, 5, 20)); - vvecs.push_back(new Vector3D(1, 3, 11)); - vvecs.push_back(new Vector3D(-1, 2, 100)); - vvecs.push_back(new Vector3D(6, 77, 7)); - zvecs.push_back(new Vector3D(1, 10, 1)); - zvecs.push_back(new Vector3D(1, 21, 21)); - zvecs.push_back(new Vector3D(100, 3, 70)); + hvecs.emplace_back(std::unique_ptr(new Vector3D(10, 1, -10))); + hvecs.emplace_back(std::unique_ptr(new Vector3D(1, 10, 100))); + hvecs.emplace_back(std::unique_ptr(new Vector3D(-5, 5, 20))); + vvecs.emplace_back(std::unique_ptr(new Vector3D(1, 3, 11))); + vvecs.emplace_back(std::unique_ptr(new Vector3D(-1, 2, 100))); + vvecs.emplace_back(std::unique_ptr(new Vector3D(6, 77, 7))); + zvecs.emplace_back(std::unique_ptr(new Vector3D(1, 10, 1))); + zvecs.emplace_back(std::unique_ptr(new Vector3D(1, 21, 21))); + zvecs.emplace_back(std::unique_ptr(new Vector3D(100, 3, 70))); alphas.push_back(1000.0); alphas.push_back(33.0); From 47ae0ecff9055f69a518ba884a53c70fc083ba1d Mon Sep 17 00:00:00 2001 From: Benjamin Menetrier <30638301+benjaminmenetrier@users.noreply.github.com> Date: Wed, 16 Dec 2020 17:05:51 +0100 Subject: [PATCH 023/142] Correct hybrid weights (#995) * Correct hybrid weights * Correct hybrid weights --- qg/test/testinput/3dvar_hybrid.yaml | 4 +- qg/test/testinput/4densvar_hybrid.yaml | 4 +- .../testinput/4dvar_drplanczos_hybrid.yaml | 4 +- qg/test/testinput/dirac_hyb.yaml | 4 +- qg/test/testoutput/3dvar_hybrid.test | 28 +++++----- qg/test/testoutput/4densvar_hybrid.test | 52 +++++++++---------- .../testoutput/4dvar_drplanczos_hybrid.test | 32 ++++++------ qg/test/testoutput/dirac_hyb.test | 2 +- 8 files changed, 65 insertions(+), 65 deletions(-) diff --git a/qg/test/testinput/3dvar_hybrid.yaml b/qg/test/testinput/3dvar_hybrid.yaml index 94e0a3f87..8f7731720 100644 --- a/qg/test/testinput/3dvar_hybrid.yaml +++ b/qg/test/testinput/3dvar_hybrid.yaml @@ -14,7 +14,7 @@ cost function: maximum_condition_number: 1.0e6 standard_deviation: 1.8e7 vertical_length_scale: 15000.0 - static weight: 0.707 + static weight: 0.5 ensemble: date: 2010-01-01T12:00:00Z localization: @@ -34,7 +34,7 @@ cost function: filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT12H.nc - date: 2010-01-01T12:00:00Z filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT12H.nc - ensemble weight: 0.707 + ensemble weight: 0.5 observations: - obs operator: obs type: Stream diff --git a/qg/test/testinput/4densvar_hybrid.yaml b/qg/test/testinput/4densvar_hybrid.yaml index f625489d6..1b97fbfcf 100644 --- a/qg/test/testinput/4densvar_hybrid.yaml +++ b/qg/test/testinput/4densvar_hybrid.yaml @@ -180,14 +180,14 @@ cost function: filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT5H.nc - date: 2010-01-01T06:00:00Z filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT6H.nc - ensemble weight: 0.707 + ensemble weight: 0.5 static: covariance model: QgError horizontal_length_scale: 2.2e6 maximum_condition_number: 1.0e6 standard_deviation: 1.8e7 vertical_length_scale: 15000.0 - static weight: 0.707 + static weight: 0.5 observations: - obs error: covariance model: diagonal diff --git a/qg/test/testinput/4dvar_drplanczos_hybrid.yaml b/qg/test/testinput/4dvar_drplanczos_hybrid.yaml index 9ce66336a..6eb5cd81e 100644 --- a/qg/test/testinput/4dvar_drplanczos_hybrid.yaml +++ b/qg/test/testinput/4dvar_drplanczos_hybrid.yaml @@ -43,14 +43,14 @@ cost function: filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1D.nc - date: 2010-01-01T00:00:00Z filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1D.nc - ensemble weight: 0.707 + ensemble weight: 0.5 static: covariance model: QgError horizontal_length_scale: 2.2e6 maximum_condition_number: 1.0e6 standard_deviation: 1.8e7 vertical_length_scale: 15000.0 - static weight: 0.707 + static weight: 0.5 observations: - obs error: covariance model: diagonal diff --git a/qg/test/testinput/dirac_hyb.yaml b/qg/test/testinput/dirac_hyb.yaml index a9f603026..f8cd33e1d 100644 --- a/qg/test/testinput/dirac_hyb.yaml +++ b/qg/test/testinput/dirac_hyb.yaml @@ -28,14 +28,14 @@ background error: filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT12H.nc - date: 2010-01-01T12:00:00Z filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT12H.nc - ensemble weight: 0.707 + ensemble weight: 0.5 static: covariance model: QgError horizontal_length_scale: 2.2e6 maximum_condition_number: 1.0e6 standard_deviation: 1.8e7 vertical_length_scale: 15000.0 - static weight: 0.707 + static weight: 0.5 dirac: date: 2010-01-01T12:00:00Z ixdir: [20] diff --git a/qg/test/testoutput/3dvar_hybrid.test b/qg/test/testoutput/3dvar_hybrid.test index 5f92ed0dd..2d5b5b201 100644 --- a/qg/test/testoutput/3dvar_hybrid.test +++ b/qg/test/testoutput/3dvar_hybrid.test @@ -3,31 +3,31 @@ Test : CostJo : Nonlinear Jo(Stream) = 7820.89, nobs = 600, Jo/n = 13.0348 Test : CostJo : Nonlinear Jo(Wind) = 1065.61, nobs = 600, Jo/n = 1.77601, err = 6 Test : CostJo : Nonlinear Jo(WSpeed) = 144.328, nobs = 300, Jo/n = 0.481094, err = 12 Test : CostFunction: Nonlinear J = 9030.82 -Test : DRIPCGMinimizer: reduction in residual norm = 0.123574 +Test : DRIPCGMinimizer: reduction in residual norm = 0.121211 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.4449, Max= 1.0463, RMS= 1.7728 +Test : Scaling= 1e+08, Min= -4.4444, Max= 1.0456, RMS= 1.7727 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 -Test : CostJb : Nonlinear Jb = 24.08 -Test : CostJo : Nonlinear Jo(Stream) = 480.3, nobs = 600, Jo/n = 0.8005, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 201.9, nobs = 600, Jo/n = 0.3366, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 30.37, nobs = 300, Jo/n = 0.1012, err = 12 -Test : CostFunction: Nonlinear J = 736.7 -Test : DRIPCGMinimizer: reduction in residual norm = 0.5646 +Test : CostJb : Nonlinear Jb = 33.64 +Test : CostJo : Nonlinear Jo(Stream) = 480.1, nobs = 600, Jo/n = 0.8002, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 202.7, nobs = 600, Jo/n = 0.3379, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 30.36, nobs = 300, Jo/n = 0.1012, err = 12 +Test : CostFunction: Nonlinear J = 746.8 +Test : DRIPCGMinimizer: reduction in residual norm = 0.5399 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.4290, Max= 1.0241, RMS= 1.7719 +Test : Scaling= 1e+08, Min= -4.4287, Max= 1.0243, RMS= 1.7719 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 -Test : CostJb : Nonlinear Jb = 30.5 -Test : CostJo : Nonlinear Jo(Stream) = 425.3, nobs = 600, Jo/n = 0.7089, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 139, nobs = 600, Jo/n = 0.2317, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 24.36, nobs = 300, Jo/n = 0.08119, err = 12 -Test : CostFunction: Nonlinear J = 619.2 +Test : CostJb : Nonlinear Jb = 41.85 +Test : CostJo : Nonlinear Jo(Stream) = 425.7, nobs = 600, Jo/n = 0.7095, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 141.4, nobs = 600, Jo/n = 0.2356, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 24.37, nobs = 300, Jo/n = 0.08125, err = 12 +Test : CostFunction: Nonlinear J = 633.3 diff --git a/qg/test/testoutput/4densvar_hybrid.test b/qg/test/testoutput/4densvar_hybrid.test index 99e27cfc8..dc61bf759 100644 --- a/qg/test/testoutput/4densvar_hybrid.test +++ b/qg/test/testoutput/4densvar_hybrid.test @@ -3,115 +3,115 @@ Test : CostJo : Nonlinear Jo(Stream) = 1613.47, nobs = 150, Jo/n = 10.7565 Test : CostJo : Nonlinear Jo(Wind) = 76.701, nobs = 80, Jo/n = 0.958762, err = 6 Test : CostJo : Nonlinear Jo(WSpeed) = 44.0485, nobs = 150, Jo/n = 0.293656, err = 12 Test : CostFunction: Nonlinear J = 1734.22 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.0195677 +Test : DRPLanczosMinimizer: reduction in residual norm = 0.0171331 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.8023, Max= 0.9705, RMS= 1.8301 +Test : Scaling= 1e+08, Min= -4.8004, Max= 0.9701, RMS= 1.8293 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 Test : Valid time: 2010-01-01T01:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.9215, Max= 1.0688, RMS= 1.8663 +Test : Scaling= 1e+08, Min= -4.9089, Max= 1.0599, RMS= 1.8624 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 Test : Valid time: 2010-01-01T02:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.9421, Max= 1.0020, RMS= 1.8736 +Test : Scaling= 1e+08, Min= -4.9400, Max= 0.9989, RMS= 1.8726 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 Test : Valid time: 2010-01-01T03:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.7254, Max= 0.9767, RMS= 1.8112 +Test : Scaling= 1e+08, Min= -4.7220, Max= 0.9734, RMS= 1.8096 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 Test : Valid time: 2010-01-01T04:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.8576, Max= 1.0105, RMS= 1.8505 +Test : Scaling= 1e+08, Min= -4.8569, Max= 1.0071, RMS= 1.8495 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 Test : Valid time: 2010-01-01T05:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.6831, Max= 0.9791, RMS= 1.7915 +Test : Scaling= 1e+08, Min= -4.6799, Max= 0.9759, RMS= 1.7900 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.7784, Max= 1.0172, RMS= 1.8293 +Test : Scaling= 1e+08, Min= -4.7783, Max= 1.0136, RMS= 1.8282 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 -Test : CostJb : Nonlinear Jb = 34.5224 -Test : CostJo : Nonlinear Jo(Stream) = 2.62629, nobs = 150, Jo/n = 0.0175086, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 5.3584, nobs = 80, Jo/n = 0.0669801, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 7.25016, nobs = 150, Jo/n = 0.0483344, err = 12 -Test : CostFunction: Nonlinear J = 49.7573 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.174323 +Test : CostJb : Nonlinear Jb = 46.203 +Test : CostJo : Nonlinear Jo(Stream) = 3.32558, nobs = 150, Jo/n = 0.0221705, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 6.39078, nobs = 80, Jo/n = 0.0798848, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 7.99854, nobs = 150, Jo/n = 0.0533236, err = 12 +Test : CostFunction: Nonlinear J = 63.9179 +Test : DRPLanczosMinimizer: reduction in residual norm = 0.139746 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.8192, Max= 0.9702, RMS= 1.8305 +Test : Scaling= 1e+08, Min= -4.8138, Max= 0.9698, RMS= 1.8296 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 Test : Valid time: 2010-01-01T01:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.9567, Max= 1.0554, RMS= 1.8685 +Test : Scaling= 1e+08, Min= -4.9362, Max= 1.0496, RMS= 1.8642 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 Test : Valid time: 2010-01-01T02:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.9463, Max= 1.0025, RMS= 1.8721 +Test : Scaling= 1e+08, Min= -4.9436, Max= 0.9989, RMS= 1.8715 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 Test : Valid time: 2010-01-01T03:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.7456, Max= 0.9903, RMS= 1.8150 +Test : Scaling= 1e+08, Min= -4.7373, Max= 0.9837, RMS= 1.8125 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 Test : Valid time: 2010-01-01T04:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.8519, Max= 1.0114, RMS= 1.8492 +Test : Scaling= 1e+08, Min= -4.8526, Max= 1.0074, RMS= 1.8485 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 Test : Valid time: 2010-01-01T05:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.7035, Max= 0.9932, RMS= 1.7954 +Test : Scaling= 1e+08, Min= -4.6955, Max= 0.9866, RMS= 1.7930 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -4.7729, Max= 1.0183, RMS= 1.8282 +Test : Scaling= 1e+08, Min= -4.7744, Max= 1.0139, RMS= 1.8274 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 -Test : CostJb : Nonlinear Jb = 36.789 -Test : CostJo : Nonlinear Jo(Stream) = 1.34736, nobs = 150, Jo/n = 0.0089824, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 3.10299, nobs = 80, Jo/n = 0.0387873, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 4.52308, nobs = 150, Jo/n = 0.0301539, err = 12 -Test : CostFunction: Nonlinear J = 45.7624 +Test : CostJb : Nonlinear Jb = 48.2511 +Test : CostJo : Nonlinear Jo(Stream) = 2.37944, nobs = 150, Jo/n = 0.0158629, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 4.56144, nobs = 80, Jo/n = 0.0570181, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 5.81766, nobs = 150, Jo/n = 0.0387844, err = 12 +Test : CostFunction: Nonlinear J = 61.0096 diff --git a/qg/test/testoutput/4dvar_drplanczos_hybrid.test b/qg/test/testoutput/4dvar_drplanczos_hybrid.test index 8d643ce02..18d90ce84 100644 --- a/qg/test/testoutput/4dvar_drplanczos_hybrid.test +++ b/qg/test/testoutput/4dvar_drplanczos_hybrid.test @@ -4,33 +4,33 @@ Test : CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, Test : CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 Test : CostJcDFI: Nonlinear Jc = 57.9613 Test : CostFunction: Nonlinear J = 12656.8 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.0147078 +Test : DRPLanczosMinimizer: reduction in residual norm = 0.0142818 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -5.1729, Max= 1.0492, RMS= 1.8898 +Test : Scaling= 1e+08, Min= -5.1710, Max= 1.0488, RMS= 1.8894 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 -Test : CostJb : Nonlinear Jb = 30.17 -Test : CostJo : Nonlinear Jo(Stream) = 557.1, nobs = 800, Jo/n = 0.6964, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 146, nobs = 800, Jo/n = 0.1825, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 28.32, nobs = 400, Jo/n = 0.0708, err = 12 -Test : CostJcDFI: Nonlinear Jc = 64.67 -Test : CostFunction: Nonlinear J = 826.2 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.1135 +Test : CostJb : Nonlinear Jb = 41.74 +Test : CostJo : Nonlinear Jo(Stream) = 542.1, nobs = 800, Jo/n = 0.6776, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 144.3, nobs = 800, Jo/n = 0.1804, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 27.98, nobs = 400, Jo/n = 0.06996, err = 12 +Test : CostJcDFI: Nonlinear Jc = 64.52 +Test : CostFunction: Nonlinear J = 820.7 +Test : DRPLanczosMinimizer: reduction in residual norm = 0.1142 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -5.2206, Max= 1.0535, RMS= 1.9038 +Test : Scaling= 1e+08, Min= -5.2175, Max= 1.0535, RMS= 1.9034 Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 -Test : CostJb : Nonlinear Jb = 28.23 -Test : CostJo : Nonlinear Jo(Stream) = 87.26, nobs = 800, Jo/n = 0.1091, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 62.57, nobs = 800, Jo/n = 0.07821, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 10.29, nobs = 400, Jo/n = 0.02572, err = 12 -Test : CostJcDFI: Nonlinear Jc = 67.15 -Test : CostFunction: Nonlinear J = 255.5 +Test : CostJb : Nonlinear Jb = 39.05 +Test : CostJo : Nonlinear Jo(Stream) = 86.94, nobs = 800, Jo/n = 0.1087, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 62.92, nobs = 800, Jo/n = 0.07865, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 10.27, nobs = 400, Jo/n = 0.02567, err = 12 +Test : CostJcDFI: Nonlinear Jc = 67.21 +Test : CostFunction: Nonlinear J = 266.4 diff --git a/qg/test/testoutput/dirac_hyb.test b/qg/test/testoutput/dirac_hyb.test index d26fc1750..f8d746c5c 100644 --- a/qg/test/testoutput/dirac_hyb.test +++ b/qg/test/testoutput/dirac_hyb.test @@ -9,7 +9,7 @@ Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are not activated -Test : Scaling= 1e+13, Min= -1.8875, Max= 44.1265, RMS= 8.3084 +Test : Scaling= 1e+13, Min= -1.3348, Max= 31.2069, RMS= 5.8758 Test : Localized Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 From 7d5346dd0fdf5c05bea70954afd7ad6f0965a28d Mon Sep 17 00:00:00 2001 From: Claude Gibert Date: Tue, 22 Dec 2020 22:42:00 +0000 Subject: [PATCH 024/142] changes for observations for qg and l95 + date management (#998) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Yannick Trémolet <30638944+ytremolet@users.noreply.github.com> --- l95/ewok/defaults/obs.yaml | 5 +++-- qg/ewok/defaults/fc.yaml | 2 +- qg/ewok/defaults/fc_output.yaml | 2 +- qg/ewok/defaults/stream.yaml | 5 +++-- qg/ewok/defaults/wind.yaml | 5 +++-- qg/ewok/defaults/wspeed.yaml | 5 +++-- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/l95/ewok/defaults/obs.yaml b/l95/ewok/defaults/obs.yaml index 57a3e4401..5df78463e 100644 --- a/l95/ewok/defaults/obs.yaml +++ b/l95/ewok/defaults/obs.yaml @@ -3,8 +3,9 @@ obs operator: obs space: source: truth filetype: obt - obsdatain: '$(current_dir)/input.obs.{{current_cycle}}.obt' - obsdataout: '$(current_dir)/$(experiment).obs.{{current_cycle}}.obt' + obsdatain: '$(current_dir)/input.obs.{{window_begin}}.obt' + obsdataout: '$(current_dir)/$(experiment).obs.{{window_begin}}.obt' obs error: covariance model: diagonal +r2d2_type: l95_obs diff --git a/qg/ewok/defaults/fc.yaml b/qg/ewok/defaults/fc.yaml index db0154035..937896083 100644 --- a/qg/ewok/defaults/fc.yaml +++ b/qg/ewok/defaults/fc.yaml @@ -1 +1 @@ -filename: $(experiment_dir)/{{current_cycle}}/$(experiment).fc.{{current_cycle}}.$(step).nc +filename: $(current_dir)/$(experiment).fc.{{current_cycle}}.$(step).nc diff --git a/qg/ewok/defaults/fc_output.yaml b/qg/ewok/defaults/fc_output.yaml index c95330ead..2b808d5a9 100644 --- a/qg/ewok/defaults/fc_output.yaml +++ b/qg/ewok/defaults/fc_output.yaml @@ -1,4 +1,4 @@ -datadir: '$(experiment_dir)/{{current_cycle}}' +datadir: '$(current_dir)' date: '{{current_cycle}}' exp: $(experiment) frequency: PT3H diff --git a/qg/ewok/defaults/stream.yaml b/qg/ewok/defaults/stream.yaml index 84182d4a9..03f411474 100644 --- a/qg/ewok/defaults/stream.yaml +++ b/qg/ewok/defaults/stream.yaml @@ -3,9 +3,10 @@ obs operator: obs space: source: truth obsdatain: - obsfile: $(experiment_dir)/{{current_cycle}}/obs.stream.{{current_cycle}}.nc + obsfile: $(current_dir)/obs.stream.{{window_begin}}.nc obsdataout: - obsfile: $(experiment_dir)/{{current_cycle}}/obs.stream.$(experiment).{{current_cycle}}.nc + obsfile: $(current_dir)/obs.stream.$(experiment).{{window_begin}}.nc obs type: Stream obs error: covariance model: diagonal +r2d2_type: qg_stream diff --git a/qg/ewok/defaults/wind.yaml b/qg/ewok/defaults/wind.yaml index 525878d56..59a703053 100644 --- a/qg/ewok/defaults/wind.yaml +++ b/qg/ewok/defaults/wind.yaml @@ -3,9 +3,10 @@ obs operator: obs space: source: truth obsdatain: - obsfile: $(experiment_dir)/{{current_cycle}}/obs.wind.{{current_cycle}}.nc + obsfile: $(current_dir)/obs.wind.{{window_begin}}.nc obsdataout: - obsfile: $(experiment_dir)/{{current_cycle}}/obs.wind.$(experiment).{{current_cycle}}.nc + obsfile: $(current_dir)/obs.wind.$(experiment).{{window_begin}}.nc obs type: Wind obs error: covariance model: diagonal +r2d2_type: qg_wind diff --git a/qg/ewok/defaults/wspeed.yaml b/qg/ewok/defaults/wspeed.yaml index b67034cbd..e6cb622dc 100644 --- a/qg/ewok/defaults/wspeed.yaml +++ b/qg/ewok/defaults/wspeed.yaml @@ -3,9 +3,10 @@ obs operator: obs space: source: truth obsdatain: - obsfile: $(experiment_dir)/{{current_cycle}}/obs.wspeed.{{current_cycle}}.nc + obsfile: $(current_dir)/obs.wspeed.{{window_begin}}.nc obsdataout: - obsfile: $(experiment_dir)/{{current_cycle}}/obs.wspeed.$(experiment).{{current_cycle}}.nc + obsfile: $(current_dir)/obs.wspeed.$(experiment).{{window_begin}}.nc obs type: WSpeed obs error: covariance model: diagonal +r2d2_type: qg_wspeed From 174da1b7aa9ce008e8b3e9c520a27c004557665e Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Mon, 28 Dec 2020 13:10:54 -0700 Subject: [PATCH 025/142] call print in interface tests (#999) * call print in interface tests * Bugfix for the geometry pointer in qg_fields_create_from_other * remove extra blank lines * output to test log * remove some newlines, and update some prints in l95/qg * more test output to log:test (instead of info) * a couple of forgotten prints (thank you, codecov) Co-authored-by: Benjamin Menetrier --- l95/src/lorenz95/BackgroundCheck.cc | 2 +- l95/src/lorenz95/ErrorCovarianceL95.cc | 2 +- l95/src/lorenz95/GetValuesL95.cc | 2 +- l95/src/lorenz95/GetValuesTLAD.cc | 2 +- l95/src/lorenz95/LocalizationMatrixL95.cc | 2 +- l95/src/lorenz95/ModelBiasCovariance.cc | 2 +- l95/src/lorenz95/ObsLocGC99.cc | 2 +- l95/src/lorenz95/ObsTable.cc | 2 +- l95/src/lorenz95/ObsTableView.cc | 3 ++- l95/src/lorenz95/ObservationL95.cc | 2 +- l95/src/lorenz95/ObservationTLAD.cc | 2 +- l95/src/lorenz95/TLML95.cc | 4 ++-- qg/model/FieldsQG.cc | 4 ++-- qg/model/GeometryQG.cc | 2 +- qg/model/GeometryQGIterator.cc | 2 +- qg/model/GetValuesQG.cc | 2 +- qg/model/GetValuesTLAD.cc | 2 +- qg/model/ModelQG.cc | 2 +- qg/model/ObsSpaceQG.cc | 2 +- qg/model/ObsStreamQG.cc | 2 +- qg/model/ObsStreamTLAD.cc | 2 +- qg/model/ObsWSpeedQG.cc | 2 +- qg/model/ObsWSpeedTLAD.cc | 2 +- qg/model/ObsWindQG.cc | 2 +- qg/model/ObsWindTLAD.cc | 2 +- qg/model/QgFortran.h | 2 +- qg/model/TlmIdQG.cc | 2 +- qg/model/qg_error_covariance_mod.F90 | 8 +++---- qg/model/qg_fields_interface.F90 | 7 ++++-- qg/model/qg_fields_mod.F90 | 11 +++++----- src/test/base/ObsErrorCovariance.h | 2 +- src/test/interface/ErrorCovariance.h | 10 ++++----- src/test/interface/GeoVaLs.h | 12 +++++------ src/test/interface/Geometry.h | 3 ++- src/test/interface/GeometryIterator.h | 8 +++---- src/test/interface/GetValues.h | 4 ++-- src/test/interface/Increment.h | 5 ++++- src/test/interface/LinearGetValues.h | 4 ++-- src/test/interface/LinearModel.h | 2 ++ src/test/interface/LinearObsOperator.h | 16 +++++++------- src/test/interface/LinearVariableChange.h | 26 +++++++++++------------ src/test/interface/Localization.h | 2 ++ src/test/interface/Locations.h | 4 ++-- src/test/interface/Model.h | 2 ++ src/test/interface/ModelAuxControl.h | 4 ++-- src/test/interface/ModelAuxCovariance.h | 12 +++++------ src/test/interface/ModelAuxIncrement.h | 4 ++-- src/test/interface/ObsAuxControl.h | 4 ++-- src/test/interface/ObsAuxCovariance.h | 12 +++++------ src/test/interface/ObsAuxIncrement.h | 5 +++-- src/test/interface/ObsOperator.h | 8 +++---- src/test/interface/ObsSpace.h | 3 ++- src/test/interface/ObsVector.h | 5 +++-- src/test/interface/State.h | 4 +++- src/test/interface/VariableChange.h | 3 ++- 55 files changed, 135 insertions(+), 114 deletions(-) diff --git a/l95/src/lorenz95/BackgroundCheck.cc b/l95/src/lorenz95/BackgroundCheck.cc index 131d8404e..888a6855b 100644 --- a/l95/src/lorenz95/BackgroundCheck.cc +++ b/l95/src/lorenz95/BackgroundCheck.cc @@ -44,7 +44,7 @@ void BackgroundCheck::postFilter(const ObsVec1D & hofx, const ObsDiags1D &) cons // ----------------------------------------------------------------------------- void BackgroundCheck::print(std::ostream & os) const { - os << "L95 Background check with absolute threshold " << options_.threshold << std::endl; + os << "L95 Background check with absolute threshold " << options_.threshold; } } // namespace lorenz95 diff --git a/l95/src/lorenz95/ErrorCovarianceL95.cc b/l95/src/lorenz95/ErrorCovarianceL95.cc index eba10c2ea..0c41c3520 100644 --- a/l95/src/lorenz95/ErrorCovarianceL95.cc +++ b/l95/src/lorenz95/ErrorCovarianceL95.cc @@ -90,7 +90,7 @@ void ErrorCovarianceL95::randomize(IncrementL95 & dx) const { // ----------------------------------------------------------------------------- void ErrorCovarianceL95::print(std::ostream & os) const { os << "ErrorCovarianceL95: time = " << time_ << ", std dev = " << sigmab_ - << ", length scale = " << 1.0/rscale_ << std::endl; + << ", length scale = " << 1.0/rscale_; } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/GetValuesL95.cc b/l95/src/lorenz95/GetValuesL95.cc index c336c2444..1cc9f7724 100644 --- a/l95/src/lorenz95/GetValuesL95.cc +++ b/l95/src/lorenz95/GetValuesL95.cc @@ -49,7 +49,7 @@ void GetValuesL95::fillGeoVaLs(const StateL95 & state, const util::DateTime & t1 } // ----------------------------------------------------------------------------- void GetValuesL95::print(std::ostream & os) const { - os << " GetValues for L95" << std::endl; + os << "Nearest neighbor interpolation GetValues"; } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/GetValuesTLAD.cc b/l95/src/lorenz95/GetValuesTLAD.cc index 5ba1f98e3..df866d373 100644 --- a/l95/src/lorenz95/GetValuesTLAD.cc +++ b/l95/src/lorenz95/GetValuesTLAD.cc @@ -75,7 +75,7 @@ void GetValuesTLAD::fillGeoVaLsAD(IncrementL95 & inc, const util::DateTime & t1, } // ----------------------------------------------------------------------------- void GetValuesTLAD::print(std::ostream & os) const { - os << " GetValuesTLAD for L95 " << std::endl; + os << "Nearest neighbor interpolation GetValues TL/AD "; } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/LocalizationMatrixL95.cc b/l95/src/lorenz95/LocalizationMatrixL95.cc index c8a943c66..24a002d80 100644 --- a/l95/src/lorenz95/LocalizationMatrixL95.cc +++ b/l95/src/lorenz95/LocalizationMatrixL95.cc @@ -64,7 +64,7 @@ void LocalizationMatrixL95::multiply(IncrementL95 & dx) const { // ----------------------------------------------------------------------------- void LocalizationMatrixL95::print(std::ostream & os) const { - os << "LocalizationMatrixL95::print not implemented"; + os << "Localization with Gaussian, lengthscale = " << 1.0/rscale_; } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ModelBiasCovariance.cc b/l95/src/lorenz95/ModelBiasCovariance.cc index 5f80b2496..382bf548a 100644 --- a/l95/src/lorenz95/ModelBiasCovariance.cc +++ b/l95/src/lorenz95/ModelBiasCovariance.cc @@ -64,7 +64,7 @@ void ModelBiasCovariance::randomize(ModelBiasCorrection & dx) const { // ----------------------------------------------------------------------------- void ModelBiasCovariance::print(std::ostream & os) const { if (active_) { - os << "ModelBiasCovariance: variance = " << variance_ << std::endl; + os << "ModelBiasCovariance: variance = " << variance_; } else { os << "ModelBiasCovariance not active" << std::endl; } diff --git a/l95/src/lorenz95/ObsLocGC99.cc b/l95/src/lorenz95/ObsLocGC99.cc index d6f73c4b0..d2d18b0ac 100644 --- a/l95/src/lorenz95/ObsLocGC99.cc +++ b/l95/src/lorenz95/ObsLocGC99.cc @@ -46,7 +46,7 @@ void ObsLocGC99::multiply(ObsVec1D & dy) const { // ----------------------------------------------------------------------------- void ObsLocGC99::print(std::ostream & os) const { - os << "ObsLocGC99::print not implemented"; + os << "Gaspari-Cohn localization with lengthscale=" << rscale_; } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObsTable.cc b/l95/src/lorenz95/ObsTable.cc index d32da14b8..56780f7d6 100644 --- a/l95/src/lorenz95/ObsTable.cc +++ b/l95/src/lorenz95/ObsTable.cc @@ -351,7 +351,7 @@ void ObsTable::otWrite(const std::string & filename) const { void ObsTable::print(std::ostream & os) const { os << "ObsTable: assimilation window = " << winbgn_ << " to " << winend_ << std::endl; - os << "ObsTable: file in = " << nameIn_ << ", file out = " << nameOut_ << std::endl; + os << "ObsTable: file in = " << nameIn_ << ", file out = " << nameOut_; } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObsTableView.cc b/l95/src/lorenz95/ObsTableView.cc index e0c4b38d7..3e1171aa0 100644 --- a/l95/src/lorenz95/ObsTableView.cc +++ b/l95/src/lorenz95/ObsTableView.cc @@ -197,7 +197,8 @@ void ObsTableView::printJo(const ObsVec1D & x1, const ObsVec1D & x2) { // ----------------------------------------------------------------------------- void ObsTableView::print(std::ostream & os) const { - os << "Local observation indices: " << localobs_ << std::endl; + os << *obstable_ << std::endl; + os << "Local observation indices: " << localobs_; } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObservationL95.cc b/l95/src/lorenz95/ObservationL95.cc index 230eeb82c..5811bbd9e 100644 --- a/l95/src/lorenz95/ObservationL95.cc +++ b/l95/src/lorenz95/ObservationL95.cc @@ -51,7 +51,7 @@ std::unique_ptr ObservationL95::locations() const { // ----------------------------------------------------------------------------- void ObservationL95::print(std::ostream & os) const { - os << "ObservationL95: Lorenz 95 Obs Operator"; + os << "Lorenz 95: Identity obs operator"; } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObservationTLAD.cc b/l95/src/lorenz95/ObservationTLAD.cc index ab76ce973..7d6e7fe04 100644 --- a/l95/src/lorenz95/ObservationTLAD.cc +++ b/l95/src/lorenz95/ObservationTLAD.cc @@ -56,7 +56,7 @@ void ObservationTLAD::simulateObsAD(GomL95 & gom, const ObsVec1D & ovec, // ----------------------------------------------------------------------------- void ObservationTLAD::print(std::ostream & os) const { - os << "ObservationTLAD: Lorenz 95 Linear Obs Operator"; + os << "Lorenz 95: Identity linear obs operator"; } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/TLML95.cc b/l95/src/lorenz95/TLML95.cc index a443ff090..5b66fbe23 100644 --- a/l95/src/lorenz95/TLML95.cc +++ b/l95/src/lorenz95/TLML95.cc @@ -186,8 +186,8 @@ void TLML95::tendenciesAD(FieldL95 & xx, double & bias, #endif // ----------------------------------------------------------------------------- void TLML95::print(std::ostream & os) const { - os << "TLML95: resol = " << resol_ << ", tstep = " << tstep_; - os << "L95 Model Trajectory, nstep=" << traj_.size() << "\n"; + os << "TLML95: resol = " << resol_ << ", tstep = " << tstep_ << std::endl; + os << "L95 Model Trajectory, nstep=" << traj_.size() << std::endl; typedef std::map< util::DateTime, ModelTrajectory * >::const_iterator trajICst; if (traj_.size() > 0) { os << "L95 Model Trajectory: times are:"; diff --git a/qg/model/FieldsQG.cc b/qg/model/FieldsQG.cc index 198f96ac5..747aba06e 100644 --- a/qg/model/FieldsQG.cc +++ b/qg/model/FieldsQG.cc @@ -43,7 +43,7 @@ FieldsQG::FieldsQG(const GeometryQG & geom, const oops::Variables & vars, FieldsQG::FieldsQG(const FieldsQG & other, const bool copy) : geom_(other.geom_), vars_(other.vars_), lbc_(other.lbc_), time_(other.time_) { - qg_fields_create_from_other_f90(keyFlds_, other.keyFlds_); + qg_fields_create_from_other_f90(keyFlds_, other.keyFlds_, geom_->toFortran()); if (copy) { qg_fields_copy_f90(keyFlds_, other.keyFlds_); } @@ -52,7 +52,7 @@ FieldsQG::FieldsQG(const FieldsQG & other, const bool copy) FieldsQG::FieldsQG(const FieldsQG & other) : geom_(other.geom_), vars_(other.vars_), lbc_(other.lbc_), time_(other.time_) { - qg_fields_create_from_other_f90(keyFlds_, other.keyFlds_); + qg_fields_create_from_other_f90(keyFlds_, other.keyFlds_, geom_->toFortran()); qg_fields_copy_f90(keyFlds_, other.keyFlds_); } // ----------------------------------------------------------------------------- diff --git a/qg/model/GeometryQG.cc b/qg/model/GeometryQG.cc index 2fe3ec27c..3ca705afd 100644 --- a/qg/model/GeometryQG.cc +++ b/qg/model/GeometryQG.cc @@ -110,7 +110,7 @@ void GeometryQG::print(std::ostream & os) const { qg_geom_info_f90(keyGeom_, nx, ny, nz, deltax, deltay); os << "Geometry:" << std::endl; os << "nx = " << nx << ", ny = " << ny << ", nz = " << nz << std::endl; - os << "deltax = " << deltax << ", deltay = " << deltay << std::endl; + os << "deltax = " << deltax << ", deltay = " << deltay; } // ----------------------------------------------------------------------------- } // namespace qg diff --git a/qg/model/GeometryQGIterator.cc b/qg/model/GeometryQGIterator.cc index 9c85d4439..1b7ac1723 100644 --- a/qg/model/GeometryQGIterator.cc +++ b/qg/model/GeometryQGIterator.cc @@ -70,7 +70,7 @@ GeometryQGIterator& GeometryQGIterator::operator++() { // ----------------------------------------------------------------------------- void GeometryQGIterator::print(std::ostream & os) const { - os << "GeometryQGIterator, key: " << keyIter_ << std::endl; + os << "GeometryQGIterator, key: " << keyIter_; } // ----------------------------------------------------------------------------- diff --git a/qg/model/GetValuesQG.cc b/qg/model/GetValuesQG.cc index 44f81493e..19343a722 100644 --- a/qg/model/GetValuesQG.cc +++ b/qg/model/GetValuesQG.cc @@ -34,7 +34,7 @@ void GetValuesQG::fillGeoVaLs(const StateQG & state, const util::DateTime & t1, } // ----------------------------------------------------------------------------- void GetValuesQG::print(std::ostream & os) const { - os << "GetValues" << std::endl; + os << "QG GetValues"; } // ----------------------------------------------------------------------------- diff --git a/qg/model/GetValuesTLAD.cc b/qg/model/GetValuesTLAD.cc index 7593a196d..f81fd0aa9 100644 --- a/qg/model/GetValuesTLAD.cc +++ b/qg/model/GetValuesTLAD.cc @@ -47,7 +47,7 @@ void GetValuesTLAD::fillGeoVaLsAD(IncrementQG & inc, const util::DateTime & t1, } // ----------------------------------------------------------------------------- void GetValuesTLAD::print(std::ostream & os) const { - os << "GetValuesTLAD " << std::endl; + os << "QG GetValues TL/AD"; } // ----------------------------------------------------------------------------- diff --git a/qg/model/ModelQG.cc b/qg/model/ModelQG.cc index e3a1b8a4e..8eb3caecc 100644 --- a/qg/model/ModelQG.cc +++ b/qg/model/ModelQG.cc @@ -62,7 +62,7 @@ int ModelQG::saveTrajectory(StateQG & xx, const ModelBias &) const { ASSERT(xx.fields().isForModel(true)); int ftraj = 0; oops::Log::debug() << "ModelQG::saveTrajectory fields in" << xx.fields() << std::endl; - qg_fields_create_from_other_f90(ftraj, xx.fields().toFortran()); + qg_fields_create_from_other_f90(ftraj, xx.fields().toFortran(), geom_.toFortran()); qg_fields_copy_f90(ftraj, xx.fields().toFortran()); ASSERT(ftraj != 0); oops::Log::debug() << "ModelQG::saveTrajectory fields out" << xx.fields() << std::endl; diff --git a/qg/model/ObsSpaceQG.cc b/qg/model/ObsSpaceQG.cc index b9a1bbae3..acfeea566 100644 --- a/qg/model/ObsSpaceQG.cc +++ b/qg/model/ObsSpaceQG.cc @@ -206,7 +206,7 @@ int ObsSpaceQG::nobs() const { // ----------------------------------------------------------------------------- void ObsSpaceQG::print(std::ostream & os) const { - os << "ObsSpace for " << obsname_ << ", " << this->nobs() << " obs" << std::endl; + os << "ObsSpace for " << obsname_ << ", " << this->nobs() << " obs"; } // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsStreamQG.cc b/qg/model/ObsStreamQG.cc index 545bdd4e0..f004bdea7 100644 --- a/qg/model/ObsStreamQG.cc +++ b/qg/model/ObsStreamQG.cc @@ -48,7 +48,7 @@ std::unique_ptr ObsStreamQG::locations() const { // ----------------------------------------------------------------------------- void ObsStreamQG::print(std::ostream & os) const { - os << "ObsStreamQG::print not implemented"; + os << "QG Stream observation operator TL/AD"; } // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsStreamTLAD.cc b/qg/model/ObsStreamTLAD.cc index bbb5e846a..1ba3761fe 100644 --- a/qg/model/ObsStreamTLAD.cc +++ b/qg/model/ObsStreamTLAD.cc @@ -55,7 +55,7 @@ void ObsStreamTLAD::simulateObsAD(GomQG & gom, const ObsVecQG & ovec, // ----------------------------------------------------------------------------- void ObsStreamTLAD::print(std::ostream & os) const { - os << "ObsStreamTLAD::print not implemented" << std::endl; + os << "QG Stream observation operator TL/AD"; } // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsWSpeedQG.cc b/qg/model/ObsWSpeedQG.cc index c73ad1938..5457314ae 100644 --- a/qg/model/ObsWSpeedQG.cc +++ b/qg/model/ObsWSpeedQG.cc @@ -49,7 +49,7 @@ std::unique_ptr ObsWSpeedQG::locations() const { // ----------------------------------------------------------------------------- void ObsWSpeedQG::print(std::ostream & os) const { - os << "ObsWSpeedQG::print not implemented"; + os << "QG wind speed observation operator TL/AD"; } // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsWSpeedTLAD.cc b/qg/model/ObsWSpeedTLAD.cc index 5c81bee06..f70c89deb 100644 --- a/qg/model/ObsWSpeedTLAD.cc +++ b/qg/model/ObsWSpeedTLAD.cc @@ -61,7 +61,7 @@ void ObsWSpeedTLAD::simulateObsAD(GomQG & gom, const ObsVecQG & ovec, // ----------------------------------------------------------------------------- void ObsWSpeedTLAD::print(std::ostream & os) const { - os << "ObsStreamTLAD::print not implemented" << std::endl; + os << "QG wind speed observation operator TL/AD"; } // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsWindQG.cc b/qg/model/ObsWindQG.cc index 830e1b906..41ea16957 100644 --- a/qg/model/ObsWindQG.cc +++ b/qg/model/ObsWindQG.cc @@ -49,7 +49,7 @@ std::unique_ptr ObsWindQG::locations() const { // ----------------------------------------------------------------------------- void ObsWindQG::print(std::ostream & os) const { - os << "ObsWindQG::print not implemented"; + os << "QG wind components (u and v) observation operator"; } // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsWindTLAD.cc b/qg/model/ObsWindTLAD.cc index ea31dd8dc..4ae764d0f 100644 --- a/qg/model/ObsWindTLAD.cc +++ b/qg/model/ObsWindTLAD.cc @@ -55,7 +55,7 @@ void ObsWindTLAD::simulateObsAD(GomQG & gom, const ObsVecQG & ovec, // ----------------------------------------------------------------------------- void ObsWindTLAD::print(std::ostream & os) const { - os << "ObsStreamTLAD::print not implemented" << std::endl; + os << "QG wind components (u and v) observation operator TL/AD"; } // ----------------------------------------------------------------------------- diff --git a/qg/model/QgFortran.h b/qg/model/QgFortran.h index 4dcd5daee..ac3908cd7 100644 --- a/qg/model/QgFortran.h +++ b/qg/model/QgFortran.h @@ -89,7 +89,7 @@ extern "C" { // Fields // ----------------------------------------------------------------------------- void qg_fields_create_f90(F90flds &, const F90geom &, const oops::Variables &, const bool &); - void qg_fields_create_from_other_f90(F90flds &, const F90flds &); + void qg_fields_create_from_other_f90(F90flds &, const F90flds &, const F90geom &); void qg_fields_delete_f90(F90flds &); void qg_fields_zero_f90(const F90flds &); void qg_fields_ones_f90(const F90flds &); diff --git a/qg/model/TlmIdQG.cc b/qg/model/TlmIdQG.cc index 06c829d0a..60a0e41eb 100644 --- a/qg/model/TlmIdQG.cc +++ b/qg/model/TlmIdQG.cc @@ -73,7 +73,7 @@ void TlmIdQG::finalizeAD(IncrementQG & dx) const { } // ----------------------------------------------------------------------------- void TlmIdQG::print(std::ostream & os) const { - os << "QG IdTLM" << std::endl; + os << "QG Identity TLM"; } // ----------------------------------------------------------------------------- } // namespace qg diff --git a/qg/model/qg_error_covariance_mod.F90 b/qg/model/qg_error_covariance_mod.F90 index 39d4b4530..dcc04ffaa 100644 --- a/qg/model/qg_error_covariance_mod.F90 +++ b/qg/model/qg_error_covariance_mod.F90 @@ -248,7 +248,7 @@ subroutine qg_error_covariance_mult(conf,fld_in,fld_out) type(qg_fields) :: fld_tmp ! Initialization -call qg_fields_create_from_other(fld_tmp,fld_in) +call qg_fields_create_from_other(fld_tmp,fld_in,fld_in%geom) call qg_fields_copy(fld_tmp,fld_in) ! Apply covariance matrix @@ -271,7 +271,7 @@ subroutine qg_error_covariance_randomize(conf,fld_out) type(qg_fields) :: fld_tmp ! Initialize temporary field -call qg_fields_create_from_other(fld_tmp,fld_out) +call qg_fields_create_from_other(fld_tmp,fld_out,fld_out%geom) call qg_fields_random(fld_tmp) ! Apply square-root of the covariance matrix @@ -404,7 +404,7 @@ subroutine qg_error_covariance_sqrt_mult(conf,fld_in,fld_out) type(qg_fields) :: fld_tmp ! Initialization -call qg_fields_create_from_other(fld_tmp,fld_in) +call qg_fields_create_from_other(fld_tmp,fld_in,fld_in%geom) call qg_fields_copy(fld_tmp,fld_in) ! Multiply by symmetric square-root of vertical correlation matrix @@ -447,7 +447,7 @@ subroutine qg_error_covariance_sqrt_mult_ad(conf,fld_in,fld_out) type(qg_fields) :: fld_tmp ! Initialization -call qg_fields_create_from_other(fld_tmp,fld_in) +call qg_fields_create_from_other(fld_tmp,fld_in,fld_in%geom) call qg_fields_copy(fld_tmp,fld_in) ! Multiply by standard deviation diff --git a/qg/model/qg_fields_interface.F90 b/qg/model/qg_fields_interface.F90 index 5f071009d..e8e047bf6 100644 --- a/qg/model/qg_fields_interface.F90 +++ b/qg/model/qg_fields_interface.F90 @@ -57,26 +57,29 @@ subroutine qg_fields_create_c(c_key_self,c_key_geom,c_vars,c_lbc) bind(c,name='q end subroutine qg_fields_create_c ! ------------------------------------------------------------------------------ !> Create fields from another one -subroutine qg_fields_create_from_other_c(c_key_self,c_key_other) bind(c,name='qg_fields_create_from_other_f90') +subroutine qg_fields_create_from_other_c(c_key_self,c_key_other,c_key_geom) bind(c,name='qg_fields_create_from_other_f90') implicit none ! Passed variables integer(c_int),intent(inout) :: c_key_self !< Fields integer(c_int),intent(in) :: c_key_other !< Other fields +integer(c_int),intent(in) :: c_key_geom !< Geometry ! Local variables type(qg_fields),pointer :: self type(qg_fields),pointer :: other +type(qg_geom),pointer :: geom ! Interface call qg_fields_registry%get(c_key_other,other) call qg_fields_registry%init() call qg_fields_registry%add(c_key_self) call qg_fields_registry%get(c_key_self,self) +call qg_geom_registry%get(c_key_geom,geom) ! Call Fortran -call qg_fields_create_from_other(self,other) +call qg_fields_create_from_other(self,other,geom) end subroutine qg_fields_create_from_other_c ! ------------------------------------------------------------------------------ diff --git a/qg/model/qg_fields_mod.F90 b/qg/model/qg_fields_mod.F90 index 3870c5c43..8b3f380a8 100644 --- a/qg/model/qg_fields_mod.F90 +++ b/qg/model/qg_fields_mod.F90 @@ -123,7 +123,7 @@ subroutine qg_fields_create(self,geom,vars,lbc) call qg_fields_zero(self) end subroutine qg_fields_create - +! ------------------------------------------------------------------------------ !> Create fields from geometry (x) subroutine qg_fields_create_default(self,geom,lbc) @@ -164,16 +164,17 @@ subroutine qg_fields_create_default(self,geom,lbc) end subroutine qg_fields_create_default ! ------------------------------------------------------------------------------ !> Create fields from another one -subroutine qg_fields_create_from_other(self,other) +subroutine qg_fields_create_from_other(self,other,geom) implicit none ! Passed variables -type(qg_fields),intent(inout) :: self !< Fields -type(qg_fields),intent(in) :: other !< Other fields +type(qg_fields),intent(inout) :: self !< Fields +type(qg_fields),intent(in) :: other !< Other fields +type(qg_geom),target,intent(in) :: geom !< Geometry ! Associate geometry -self%geom => other%geom +self%geom => geom ! Copy attributes self%lq = other%lq diff --git a/src/test/base/ObsErrorCovariance.h b/src/test/base/ObsErrorCovariance.h index de6d88da6..ae4f68d47 100644 --- a/src/test/base/ObsErrorCovariance.h +++ b/src/test/base/ObsErrorCovariance.h @@ -47,7 +47,7 @@ template void testConstructor() { std::unique_ptr R( oops::ObsErrorFactory::create(rconf, Test_::obspace()[jj])); EXPECT(R.get()); - + oops::Log::test() << "Testing ObsError: " << *R << std::endl; R.reset(); EXPECT(!R.get()); } diff --git a/src/test/interface/ErrorCovariance.h b/src/test/interface/ErrorCovariance.h index 181e2fd34..ab7f87a2d 100644 --- a/src/test/interface/ErrorCovariance.h +++ b/src/test/interface/ErrorCovariance.h @@ -97,7 +97,7 @@ template void testErrorCovarianceZero() { Increment_ dx2(Test_::resol(), Test_::ctlvars(), Test_::time()); Test_::covariance().randomize(dx2); - oops::Log::info() << "dx2.norm()=" << dx2.norm() << std::endl; + oops::Log::test() << "dx2.norm()=" << dx2.norm() << std::endl; EXPECT(dx1.norm() == 0.0); EXPECT(dx2.norm() > 0.0); @@ -108,7 +108,7 @@ template void testErrorCovarianceZero() { const bool testinverse = Test_::test().getBool("testinverse", true); if (testinverse) { - oops::Log::info() << "Doing zero test for inverse" << std::endl; + oops::Log::test() << "Doing zero test for inverse" << std::endl; dx1.zero(); Test_::covariance().randomize(dx2); EXPECT(dx1.norm() == 0.0); @@ -116,7 +116,7 @@ template void testErrorCovarianceZero() { Test_::covariance().inverseMultiply(dx1, dx2); EXPECT(dx2.norm() == 0.0); } else { - oops::Log::info() << "Not doing zero test for inverse" << std::endl; + oops::Log::test() << "Not doing zero test for inverse" << std::endl; } } @@ -166,9 +166,9 @@ template void testErrorCovarianceSym() { Test_::covariance().multiply(dy, Bdy); const double zz1 = dot_product(dx, Bdy); const double zz2 = dot_product(Bdx, dy); - oops::Log::info() << "-/=" + oops::Log::test() << "-/=" << (zz1-zz2)/zz1 << std::endl; - oops::Log::info() << "-/=" + oops::Log::test() << "-/=" << (zz1-zz2)/zz2 << std::endl; const double tol = Test_::test().getDouble("tolerance"); EXPECT(oops::is_close(zz1, zz2, tol)); diff --git a/src/test/interface/GeoVaLs.h b/src/test/interface/GeoVaLs.h index 256ce53e0..7d96f42af 100644 --- a/src/test/interface/GeoVaLs.h +++ b/src/test/interface/GeoVaLs.h @@ -32,7 +32,7 @@ namespace test { // ----------------------------------------------------------------------------- - +/// \brief Tests test-constructor and print method template void testConstructor() { typedef ObsTestsFixture Test_; typedef oops::GeoVaLs GeoVaLs_; @@ -40,11 +40,11 @@ template void testConstructor() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { eckit::LocalConfiguration gconf(Test_::config(jj), "geovals"); oops::Variables geovars(gconf, "state variables"); - std::unique_ptr ov(new GeoVaLs_(gconf, Test_::obspace()[jj], geovars)); - EXPECT(ov.get()); - - ov.reset(); - EXPECT(!ov.get()); + std::unique_ptr geovals(new GeoVaLs_(gconf, Test_::obspace()[jj], geovars)); + EXPECT(geovals.get()); + oops::Log::test() << "Testing GeoVaLs: " << *geovals << std::endl; + geovals.reset(); + EXPECT(!geovals.get()); } } diff --git a/src/test/interface/Geometry.h b/src/test/interface/Geometry.h index 39a323063..dd52be533 100644 --- a/src/test/interface/Geometry.h +++ b/src/test/interface/Geometry.h @@ -31,6 +31,7 @@ namespace test { // ----------------------------------------------------------------------------- +/// \brief Tests constructor and print method template void testConstructor() { typedef oops::Geometry Geometry_; @@ -38,7 +39,7 @@ template void testConstructor() { oops::mpi::world(), oops::mpi::myself())); EXPECT(geom.get()); - + oops::Log::test() << "Testing geometry: " << *geom << std::endl; geom.reset(); EXPECT(!geom.get()); } diff --git a/src/test/interface/GeometryIterator.h b/src/test/interface/GeometryIterator.h index dc2d3ea3c..4ac4c48e0 100644 --- a/src/test/interface/GeometryIterator.h +++ b/src/test/interface/GeometryIterator.h @@ -59,7 +59,7 @@ template void testBasic() { EXPECT(iter1 != iter2); // At least test that nothing fails on print - oops::Log::info() << "Geometry::begin " << iter1 << std::endl; + oops::Log::test() << "Geometry::begin " << iter1 << std::endl; } @@ -85,12 +85,12 @@ template void testGetSetLocal() { Increment_ dx1(Test_::resol(), Test_::ctlvars(), Test_::time()); dx1.random(); EXPECT(dx1.norm() != 0.0); - oops::Log::info() << "Increment dx1 (random): " << dx1 << std::endl; + oops::Log::test() << "Increment dx1 (random): " << dx1 << std::endl; // zero out increment dx2 Increment_ dx2(Test_::resol(), Test_::ctlvars(), Test_::time()); dx2.zero(); EXPECT(dx2.norm() == 0.0); - oops::Log::info() << "Increment dx2 (zero): " << dx2 << std::endl; + oops::Log::test() << "Increment dx2 (zero): " << dx2 << std::endl; for (GeometryIterator_ i = geom.begin(); i != geom.end(); ++i) { // get value for i-th gridpoint from dx1 @@ -99,7 +99,7 @@ template void testGetSetLocal() { // set this value for i-th gridpoint in dx2 dx2.setLocal(gp, i); } - oops::Log::info() << "Increment dx2 after dx2=dx1 (at every point): " << dx2 << std::endl; + oops::Log::test() << "Increment dx2 after dx2=dx1 (at every point): " << dx2 << std::endl; EXPECT(dx2.norm() != 0.0); // compare two increments dx2 -= dx1; diff --git a/src/test/interface/GetValues.h b/src/test/interface/GetValues.h index 918fe3245..d6ddf489e 100644 --- a/src/test/interface/GetValues.h +++ b/src/test/interface/GetValues.h @@ -105,14 +105,14 @@ template class GetValuesFixture : private boost:: }; // ================================================================================================= - +/// \brief tests constructor and print method template void testGetValuesConstructor() { typedef GetValuesFixture Test_; typedef oops::GetValues GetValues_; std::unique_ptr GetValues(new GetValues_(Test_::resol(), Test_::locs())); EXPECT(GetValues.get()); - + oops::Log::test() << "Testing GetValues: " << *GetValues << std::endl; GetValues.reset(); EXPECT(!GetValues.get()); } diff --git a/src/test/interface/Increment.h b/src/test/interface/Increment.h index bdaff6222..149255c63 100644 --- a/src/test/interface/Increment.h +++ b/src/test/interface/Increment.h @@ -86,12 +86,13 @@ template class IncrementFixture : private boost::noncopyable { }; // ============================================================================= - +/// \brief tests Increment constructor and print method template void testIncrementConstructor() { typedef IncrementFixture Test_; typedef oops::Increment Increment_; Increment_ dx(Test_::resol(), Test_::ctlvars(), Test_::time()); + oops::Log::test() << "Printing zero increment: " << dx << std::endl; EXPECT(dx.norm() == 0.0); } @@ -104,6 +105,8 @@ template void testIncrementCopyConstructor() { Increment_ dx1(Test_::resol(), Test_::ctlvars(), Test_::time()); dx1.random(); + oops::Log::test() << "Printing random increment: " << dx1 << std::endl; + EXPECT(dx1.norm() > 0.0); Increment_ dx2(dx1); diff --git a/src/test/interface/LinearGetValues.h b/src/test/interface/LinearGetValues.h index 2509eda0b..1370e56f3 100644 --- a/src/test/interface/LinearGetValues.h +++ b/src/test/interface/LinearGetValues.h @@ -131,7 +131,7 @@ template class LinearGetValuesFixture : private b }; // ================================================================================================= - +/// \brief tests constructor and print method template void testLinearGetValuesConstructor() { typedef LinearGetValuesFixture Test_; typedef oops::LinearGetValues LinearGetValues_; @@ -139,7 +139,7 @@ template void testLinearGetValuesConstructor() { std::unique_ptr lineargetvalues(new LinearGetValues_(Test_::resol(), Test_::locs())); EXPECT(lineargetvalues.get()); - + oops::Log::test() << "Testing LinearGetValues: " << *lineargetvalues << std::endl; lineargetvalues.reset(); EXPECT(!lineargetvalues.get()); } diff --git a/src/test/interface/LinearModel.h b/src/test/interface/LinearModel.h index 55d443877..4c025bc41 100644 --- a/src/test/interface/LinearModel.h +++ b/src/test/interface/LinearModel.h @@ -149,11 +149,13 @@ template class LinearModelFixture : private boost::noncopyable // ============================================================================= +/// \brief tests constructor, timeResolution() method and print method template void testLinearModelConstructor() { typedef LinearModelFixture Test_; const util::Duration zero(0); EXPECT(Test_::tlm().timeResolution() > zero); + oops::Log::test() << "Testing LinearModel: " << Test_::tlm() << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/test/interface/LinearObsOperator.h b/src/test/interface/LinearObsOperator.h index 57b1b1b10..b5a5903bf 100644 --- a/src/test/interface/LinearObsOperator.h +++ b/src/test/interface/LinearObsOperator.h @@ -32,7 +32,7 @@ namespace test { // ----------------------------------------------------------------------------- - +/// \brief tests constructor and print method template void testConstructor() { typedef ObsTestsFixture Test_; typedef oops::LinearObsOperator LinearObsOperator_; @@ -43,12 +43,12 @@ template void testConstructor() { std::string confname = "obs operator"; if (conf.has("linear obs operator")) confname = "linear obs operator"; eckit::LocalConfiguration linobsopconf(conf, confname); - std::unique_ptr ov( + std::unique_ptr linobsop( new LinearObsOperator_(Test_::obspace()[jj], linobsopconf)); - EXPECT(ov.get()); - - ov.reset(); - EXPECT(!ov.get()); + EXPECT(linobsop.get()); + oops::Log::test() << "Testing LinearObsOperator: " << *linobsop << std::endl; + linobsop.reset(); + EXPECT(!linobsop.get()); } } @@ -197,7 +197,7 @@ template void testAdjoint() { const double zz1 = dot_product(dx1, dx2) + dot_product(ybinc1, ybinc2); const double zz2 = dot_product(dy1, dy2); - oops::Log::info() << "Adjoint test result: (-)/ = " + oops::Log::test() << "Adjoint test result: (-)/ = " << (zz1-zz2)/zz2 << std::endl; EXPECT(zz1 != zero); @@ -295,7 +295,7 @@ template void testTangentLinear() { y2 -= y3; double test_norm = y2.rms(); - oops::Log::info() << "Iter:" << jter << " ||(h(x+alpha*dx)-h(x)-h'*(alpha*dx))||=" + oops::Log::test() << "Iter:" << jter << " ||(h(x+alpha*dx)-h(x)-h'*(alpha*dx))||=" << test_norm << std::endl; } EXPECT(y2.rms() < tol); diff --git a/src/test/interface/LinearVariableChange.h b/src/test/interface/LinearVariableChange.h index aac02ddc0..1da4347bb 100644 --- a/src/test/interface/LinearVariableChange.h +++ b/src/test/interface/LinearVariableChange.h @@ -95,7 +95,7 @@ template void testLinearVariableChangeZero() { std::unique_ptr changevar(LinearVariableChangeFactory_::create( Test_::xx(), Test_::xx(), Test_::resol(), Test_::confs()[jj])); - + oops::Log::test() << "Testing linear variable change: " << *changevar << std::endl; Increment_ dxinTlIAd(Test_::resol(), varin, Test_::time()); Increment_ dxinAdInv(Test_::resol(), varout, Test_::time()); Increment_ dxoutTlIAd(Test_::resol(), varout, Test_::time()); @@ -114,7 +114,7 @@ template void testLinearVariableChangeZero() { const bool testinverse = Test_::confs()[jj].getBool("test inverse", true); if (testinverse) { - oops::Log::info() << "Doing zero test for inverse" << std::endl; + oops::Log::test() << "Doing zero test for inverse" << std::endl; dxinTlIAd.zero(); dxoutTlIAd = changevar->multiplyInverseAD(dxinTlIAd); EXPECT(dxoutTlIAd.norm() == 0.0); @@ -123,7 +123,7 @@ template void testLinearVariableChangeZero() { dxoutAdInv = changevar->multiplyInverse(dxinAdInv); EXPECT(dxoutAdInv.norm() == 0.0); } else { - oops::Log::info() << "Not doing zero test for inverse" << std::endl; + oops::Log::test() << "Not doing zero test for inverse" << std::endl; } } } @@ -162,16 +162,16 @@ template void testLinearVariableChangeAdjoint() { // zz2 = double zz2 = dot_product(dxinTlIAd0, dxoutAdInv); - oops::Log::info() << "-/=" + oops::Log::test() << "-/=" << (zz1-zz2)/zz1 << std::endl; - oops::Log::info() << "-/=" + oops::Log::test() << "-/=" << (zz1-zz2)/zz2 << std::endl; const double tol = 1e-10; EXPECT(oops::is_close(zz1, zz2, tol)); const bool testinverse = Test_::confs()[jj].getBool("test inverse", true); if (testinverse) { - oops::Log::info() << "Doing adjoint test for inverse" << std::endl; + oops::Log::test() << "Doing adjoint test for inverse" << std::endl; dxoutAdInv.zero(); dxoutTlIAd.zero(); dxinAdInv.random(); @@ -182,13 +182,13 @@ template void testLinearVariableChangeAdjoint() { dxoutAdInv = changevar->multiplyInverse(dxinAdInv); zz1 = dot_product(dxoutTlIAd, dxinAdInv0); zz2 = dot_product(dxinTlIAd0, dxoutAdInv); - oops::Log::info() << "-/=" + oops::Log::test() << "-/=" << (zz1-zz2)/zz1 << std::endl; - oops::Log::info() << "-/=" + oops::Log::test() << "-/=" << (zz1-zz2)/zz2 << std::endl; EXPECT(oops::is_close(zz1, zz2, tol)); } else { - oops::Log::info() << "Not doing adjoint test for inverse" << std::endl; + oops::Log::test() << "Not doing adjoint test for inverse" << std::endl; } } } @@ -210,7 +210,7 @@ template void testLinearVariableChangeInverse() { const bool testinverse = Test_::confs()[jj].getBool("test inverse", false); if (testinverse) { - oops::Log::info() << "Testing multiplyInverse" << std::endl; + oops::Log::test() << "Testing multiplyInverse" << std::endl; std::unique_ptr changevar(LinearVariableChangeFactory_::create( Test_::xx(), Test_::xx(), Test_::resol(), Test_::confs()[jj])); @@ -227,12 +227,12 @@ template void testLinearVariableChangeInverse() { const double zz1 = dxinInv.norm(); const double zz2 = dxout.norm(); - oops::Log::info() << ", =" << zz1 << " " << zz2 << std::endl; - oops::Log::info() << "-=" << zz1-zz2 << std::endl; + oops::Log::test() << ", =" << zz1 << " " << zz2 << std::endl; + oops::Log::test() << "-=" << zz1-zz2 << std::endl; EXPECT((zz1-zz2) < tol); } else { - oops::Log::info() << "multiplyInverse test not executed" << std::endl; + oops::Log::test() << "multiplyInverse test not executed" << std::endl; EXPECT(1.0 < 2.0); } } diff --git a/src/test/interface/Localization.h b/src/test/interface/Localization.h index 93230aa13..e513cfddb 100644 --- a/src/test/interface/Localization.h +++ b/src/test/interface/Localization.h @@ -65,6 +65,8 @@ template class LocalizationFixture : private boost::noncopyable // Setup the localization matrix const eckit::LocalConfiguration conf(TestEnvironment::config(), "localization"); local_ = oops::LocalizationFactory::create(*resol_, *time_, conf); + + oops::Log::test() << "Testing localization: " << *local_ << std::endl; } ~LocalizationFixture() {} diff --git a/src/test/interface/Locations.h b/src/test/interface/Locations.h index 86d457345..534ea9690 100644 --- a/src/test/interface/Locations.h +++ b/src/test/interface/Locations.h @@ -29,14 +29,14 @@ namespace test { // ----------------------------------------------------------------------------- - +/// \brief tests test constructor and print method template void testConstructor() { typedef oops::Locations Locations_; const eckit::LocalConfiguration conf(TestEnvironment::config(), "locations"); std::unique_ptr locs(new Locations_(conf, oops::mpi::world())); EXPECT(locs.get()); - + oops::Log::test() << "Testing locations: " << *locs << std::endl; locs.reset(); EXPECT(!locs.get()); } diff --git a/src/test/interface/Model.h b/src/test/interface/Model.h index de173d8df..73f59b99f 100644 --- a/src/test/interface/Model.h +++ b/src/test/interface/Model.h @@ -91,11 +91,13 @@ template class ModelFixture : private boost::noncopyable { // ================================================================================================= +/// \brief tests constructor, timeResolution() method and print method template void testModelConstructor() { typedef ModelFixture Test_; const util::Duration zero(0); EXPECT(Test_::model().timeResolution() > zero); + oops::Log::test() << "Testing Model: " << Test_::model() << std::endl; } // ------------------------------------------------------------------------------------------------- diff --git a/src/test/interface/ModelAuxControl.h b/src/test/interface/ModelAuxControl.h index e8f267168..4545d5d88 100644 --- a/src/test/interface/ModelAuxControl.h +++ b/src/test/interface/ModelAuxControl.h @@ -58,14 +58,14 @@ template class ModelAuxControlFixture : private boost::noncopya }; // ----------------------------------------------------------------------------- - +/// \brief testing constructor and print method template void testConstructor() { typedef ModelAuxControlFixture Test_; typedef oops::ModelAuxControl ModelAux_; std::unique_ptr bias(new ModelAux_(Test_::resol(), Test_::config())); EXPECT(bias.get()); - + oops::Log::test() << "Testing ModelAuxControl: " << *bias << std::endl; bias.reset(); EXPECT(!bias.get()); } diff --git a/src/test/interface/ModelAuxCovariance.h b/src/test/interface/ModelAuxCovariance.h index a5e0acd19..157b71909 100644 --- a/src/test/interface/ModelAuxCovariance.h +++ b/src/test/interface/ModelAuxCovariance.h @@ -58,16 +58,16 @@ template class ModelAuxCovarianceFixture : private boost::nonco }; // ----------------------------------------------------------------------------- - +/// \brief tests constructor and print method template void testConstructor() { typedef ModelAuxCovarianceFixture Test_; typedef oops::ModelAuxCovariance Covariance_; - std::unique_ptr bias(new Covariance_(Test_::config(), Test_::resol())); - EXPECT(bias.get()); - - bias.reset(); - EXPECT(!bias.get()); + std::unique_ptr cov(new Covariance_(Test_::config(), Test_::resol())); + EXPECT(cov.get()); + oops::Log::test() << "Testing ModelAuxCovariance: " << *cov << std::endl; + cov.reset(); + EXPECT(!cov.get()); } // ----------------------------------------------------------------------------- diff --git a/src/test/interface/ModelAuxIncrement.h b/src/test/interface/ModelAuxIncrement.h index 3e6648c0b..082459c95 100644 --- a/src/test/interface/ModelAuxIncrement.h +++ b/src/test/interface/ModelAuxIncrement.h @@ -72,13 +72,13 @@ template class ModelAuxIncrementFixture : private boost::noncop }; // ============================================================================= - +/// \brief tests constructor and print method template void testModelAuxIncrementConstructor() { typedef ModelAuxIncrementFixture Test_; typedef oops::ModelAuxIncrement AuxIncr_; AuxIncr_ dx(Test_::resol(), Test_::config()); - + oops::Log::test() << "Testing ModelAuxIncrement: " << dx << std::endl; EXPECT(dx.norm() == 0.0); } diff --git a/src/test/interface/ObsAuxControl.h b/src/test/interface/ObsAuxControl.h index af13a1bca..e7eff791f 100644 --- a/src/test/interface/ObsAuxControl.h +++ b/src/test/interface/ObsAuxControl.h @@ -29,7 +29,7 @@ namespace test { // ----------------------------------------------------------------------------- - +/// \brief test constructor and print method template void testConstructor() { typedef ObsTestsFixture Test_; typedef oops::ObsAuxControl ObsAux_; @@ -37,7 +37,7 @@ template void testConstructor() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { std::unique_ptr bias(new ObsAux_(Test_::obspace()[jj], Test_::config(jj))); EXPECT(bias.get()); - + oops::Log::test() << "Testing ObsAuxControl: " << *bias << std::endl; bias.reset(); EXPECT(!bias.get()); } diff --git a/src/test/interface/ObsAuxCovariance.h b/src/test/interface/ObsAuxCovariance.h index d3fdd85f5..9edd0b5ee 100644 --- a/src/test/interface/ObsAuxCovariance.h +++ b/src/test/interface/ObsAuxCovariance.h @@ -28,17 +28,17 @@ namespace test { // ----------------------------------------------------------------------------- - +/// \brief testing constructor and print method template void testConstructor() { typedef ObsTestsFixture Test_; typedef oops::ObsAuxCovariance Covariance_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - std::unique_ptr bias(new Covariance_(Test_::obspace()[jj], Test_::config(jj))); - EXPECT(bias.get()); - - bias.reset(); - EXPECT(!bias.get()); + std::unique_ptr cov(new Covariance_(Test_::obspace()[jj], Test_::config(jj))); + EXPECT(cov.get()); + oops::Log::test() << "Testing ObsAuxCovariance: " << *cov << std::endl; + cov.reset(); + EXPECT(!cov.get()); } } diff --git a/src/test/interface/ObsAuxIncrement.h b/src/test/interface/ObsAuxIncrement.h index ef1bf12f9..38396159f 100644 --- a/src/test/interface/ObsAuxIncrement.h +++ b/src/test/interface/ObsAuxIncrement.h @@ -62,13 +62,14 @@ template class ObsAuxIncrementFixture : private boost::noncopyabl }; // ============================================================================= - +/// \brief test constructor and print method template void testObsAuxIncrementConstructor() { typedef ObsTestsFixture Test_; typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { AuxIncr_ dx(Test_::obspace()[jj], Test_::config(jj)); + oops::Log::test() << "Printing zero ObsAuxIncrement: " << dx << std::endl; EXPECT(dx.norm() == 0.0); } } @@ -83,7 +84,7 @@ template void testObsAuxIncrementCopyConstructor() { if (Test_::config(jj).has("obs bias")) { AuxIncr_ dx1(Test_::obspace()[jj], Test_::config(jj)); ObsAuxIncrementFixture::covariance(jj).randomize(dx1); - + oops::Log::test() << "Printing random ObsAuxIncrement: " << dx1 << std::endl; AuxIncr_ dx2(dx1); EXPECT(dx2.norm() > 0.0); EXPECT(dx2.norm() == dx1.norm()); diff --git a/src/test/interface/ObsOperator.h b/src/test/interface/ObsOperator.h index f963bdeff..fccbcde87 100644 --- a/src/test/interface/ObsOperator.h +++ b/src/test/interface/ObsOperator.h @@ -32,7 +32,7 @@ namespace test { // ----------------------------------------------------------------------------- - +/// \brief tests constructor and print method template void testConstructor() { typedef ObsTestsFixture Test_; typedef oops::ObsOperator ObsOperator_; @@ -41,7 +41,7 @@ template void testConstructor() { eckit::LocalConfiguration obsopconf(Test_::config(jj), "obs operator"); std::unique_ptr hop(new ObsOperator_(Test_::obspace()[jj], obsopconf)); EXPECT(hop.get()); - + oops::Log::test() << "Testing ObsOperator: " << *hop << std::endl; hop.reset(); EXPECT(!hop.get()); } @@ -98,7 +98,7 @@ template void testSimulateObs() { ObsVector_ obsref(Test_::obspace()[jj], conf.getString("vector ref")); obsref -= hofx; const double zz = obsref.rms(); - oops::Log::info() << "Vector difference between reference and computed: " << obsref; + oops::Log::test() << "Vector difference between reference and computed: " << obsref; EXPECT(zz < 100*tol); // change tol from percent to actual value. // tol used in is_close is relative } else if (conf.has("norm ref")) { @@ -108,7 +108,7 @@ template void testSimulateObs() { obsref -= hofx; obsref /= hofx; const double zz = obsref.rms(); - oops::Log::info() << "Normalised vector difference between reference and computed: " + oops::Log::test() << "Normalised vector difference between reference and computed: " << obsref; EXPECT(zz < 100*tol); // change tol from percent to actual value. // tol used in is_close is relative diff --git a/src/test/interface/ObsSpace.h b/src/test/interface/ObsSpace.h index fd96d92ac..1823be9d5 100644 --- a/src/test/interface/ObsSpace.h +++ b/src/test/interface/ObsSpace.h @@ -29,11 +29,12 @@ namespace test { // ----------------------------------------------------------------------------- - +/// \brief tests constructor, window accessor methods and print template void testConstructor() { typedef ObsTestsFixture Test_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + oops::Log::test() << "Testing ObsSpace: " << Test_::obspace()[jj] << std::endl; EXPECT(Test_::obspace()[jj].windowStart() == Test_::tbgn()); EXPECT(Test_::obspace()[jj].windowEnd() == Test_::tend()); } diff --git a/src/test/interface/ObsVector.h b/src/test/interface/ObsVector.h index e0be4148b..ca2f250d4 100644 --- a/src/test/interface/ObsVector.h +++ b/src/test/interface/ObsVector.h @@ -28,7 +28,7 @@ namespace test { // ----------------------------------------------------------------------------- - +/// \brief tests constructor and print method template void testConstructor() { typedef ObsTestsFixture Test_; typedef oops::ObsVector ObsVector_; @@ -36,7 +36,7 @@ template void testConstructor() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { std::unique_ptr ov(new ObsVector_(Test_::obspace()[jj])); EXPECT(ov.get()); - + oops::Log::test() << "Printing zero ObsVector: " << *ov << std::endl; ov.reset(); EXPECT(!ov.get()); } @@ -52,6 +52,7 @@ template void testCopyConstructor() { std::unique_ptr ov(new ObsVector_(Test_::obspace()[jj])); ov->random(); + oops::Log::test() << "Printing random ObsVector: " << *ov << std::endl; std::unique_ptr other(new ObsVector_(*ov)); EXPECT(other.get()); diff --git a/src/test/interface/State.h b/src/test/interface/State.h index 45688dcff..66ad09166 100644 --- a/src/test/interface/State.h +++ b/src/test/interface/State.h @@ -65,7 +65,7 @@ template class StateFixture : private boost::noncopyable { }; // ----------------------------------------------------------------------------- - +/// \brief tests constructors and print method template void testStateConstructors() { typedef StateFixture Test_; typedef oops::State State_; @@ -79,6 +79,7 @@ template void testStateConstructors() { std::unique_ptr xx1(new State_(Test_::resol(), conf)); EXPECT(xx1.get()); + oops::Log::test() << "Printing State from yaml: " << *xx1 << std::endl; const double norm1 = xx1->norm(); EXPECT(oops::is_close(norm1, norm, tol)); EXPECT(xx1->validTime() == vt); @@ -100,6 +101,7 @@ template void testStateConstructors() { // Test State(const Geometry_ &, const Variables &, const util::DateTime &) constructor oops::Variables vars(xx1->variables()); State_ xx3(Test_::resol(), vars, vt); + oops::Log::test() << "Printing empty State: " << xx3 << std::endl; EXPECT(xx3.norm() == 0); EXPECT(xx3.validTime() == vt); EXPECT(xx3.variables() == vars); diff --git a/src/test/interface/VariableChange.h b/src/test/interface/VariableChange.h index cf0ee17ee..7a127fbdf 100644 --- a/src/test/interface/VariableChange.h +++ b/src/test/interface/VariableChange.h @@ -80,6 +80,7 @@ template void testVariableChangeInverse() { std::unique_ptr \ changevar(VariableChangeFactory_::create(Test_::confs()[jj], Test_::resol())); + oops::Log::test() << "Testing VariableChange: " << *changevar << std::endl; // User specified tolerance for pass/fail const double tol = Test_::confs()[jj].getDouble("tolerance inverse"); @@ -112,7 +113,7 @@ template void testVariableChangeInverse() { const double xxnorm_tst = xx.norm(); // Print the input and final state - oops::Log::info() << ", , (-)/>=" << xxnorm_ref << + oops::Log::test() << ", , (-)/>=" << xxnorm_ref << " " << xxnorm_tst << " " << (xxnorm_ref - xxnorm_tst)/xxnorm_ref < Date: Tue, 29 Dec 2020 14:26:10 -0300 Subject: [PATCH 026/142] fix for letkf & getkf test dependencies (#1005) Co-authored-by: Anna Shlyaeva --- l95/test/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index f74f28089..6130bd629 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -749,16 +749,16 @@ oops_add_test( TESTNAME getkf oops_add_test( TESTNAME hofx_for_getkf_nomodel MODELNAME l95 YAMLNAME testinput/hofx_for_getkf_nomodel.yaml - EXENAME l95_letkf.x + EXENAME l95_letkf.x CTOL 1e-3 - TEST_DEPENDS letkf ) + TEST_DEPENDS test_l95_letkf ) oops_add_test( TESTNAME getkf_offline_hofx MODELNAME l95 YAMLNAME testinput/getkf_offline_hofx.yaml EXENAME l95_letkf.x CTOL 1e-3 - TEST_DEPENDS hofx_for_getkf_nomodel ) + TEST_DEPENDS test_l95_hofx_for_getkf_nomodel ) ##################################################################### From 177c90be6a45a802d807744a3cfbdbc6c93fef6f Mon Sep 17 00:00:00 2001 From: Clementine Gas <43183478+cmgas@users.noreply.github.com> Date: Tue, 29 Dec 2020 15:11:11 -0600 Subject: [PATCH 027/142] Adding block Lanczos minimizer (#992) * Add yaml files for block minimizer * block is repaired * Add correct files qg block * Replace boost and remove extra copies * More efficient block (less copies) and works as expected * Finished clean up of Block but still has too much prints * Coding norms * Add cout for all members * More clean up and timing the mpi * Remove useless file * Correct CMakelist * Correct name... * Add better option for emplace_back * Change yaml for qg * Remove model section l95 block 3dvar * change qg background + correct typo * Remove useless include * Fix mpi test * Add compare for block test * Add files * Correct dependencies * Address some comments * Lorenz yaml for block without extra indent --- l95/test/CMakeLists.txt | 11 + l95/test/testinput/eda.3dvar.block.1.yaml | 51 +++ l95/test/testinput/eda.3dvar.block.2.yaml | 51 +++ l95/test/testinput/eda.3dvar.block.yaml | 3 + l95/test/testoutput/eda_3dvar_block.test | 17 + qg/test/CMakeLists.txt | 13 + qg/test/testinput/eda_3dvar_block.yaml | 5 + qg/test/testinput/eda_3dvar_block_1.yaml | 87 ++++ qg/test/testinput/eda_3dvar_block_2.yaml | 87 ++++ qg/test/testinput/eda_3dvar_block_3.yaml | 87 ++++ qg/test/testinput/eda_3dvar_block_4.yaml | 87 ++++ qg/test/testoutput/eda_3dvar_block.test | 33 ++ src/CMakeLists.txt | 2 +- .../assimilation/BlockBLanczosMinimizer.h | 384 ++++++++++++++++++ src/oops/assimilation/TriDiagSolve.h | 5 +- src/oops/assimilation/instantiateMinFactory.h | 3 + src/oops/assimilation/linsysteigen.h | 36 -- src/oops/mpi/mpi.cc | 4 +- src/oops/mpi/mpi.h | 6 +- src/test/mpi/mpi.h | 27 +- 20 files changed, 947 insertions(+), 52 deletions(-) create mode 100644 l95/test/testinput/eda.3dvar.block.1.yaml create mode 100644 l95/test/testinput/eda.3dvar.block.2.yaml create mode 100644 l95/test/testinput/eda.3dvar.block.yaml create mode 100644 l95/test/testoutput/eda_3dvar_block.test create mode 100644 qg/test/testinput/eda_3dvar_block.yaml create mode 100644 qg/test/testinput/eda_3dvar_block_1.yaml create mode 100644 qg/test/testinput/eda_3dvar_block_2.yaml create mode 100644 qg/test/testinput/eda_3dvar_block_3.yaml create mode 100644 qg/test/testinput/eda_3dvar_block_4.yaml create mode 100644 qg/test/testoutput/eda_3dvar_block.test create mode 100644 src/oops/assimilation/BlockBLanczosMinimizer.h delete mode 100644 src/oops/assimilation/linsysteigen.h diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index 6130bd629..f26229b04 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -45,6 +45,9 @@ list( APPEND l95_test_input testinput/eda.3dvar.3.yaml testinput/eda.3dvar.4.yaml testinput/eda.3dvar.yaml + testinput/eda.3dvar.block.1.yaml + testinput/eda.3dvar.block.2.yaml + testinput/eda.3dvar.block.yaml testinput/eda.4dvar.3.yaml testinput/eda.4dvar.yaml testinput/enshofx.1.yaml @@ -135,6 +138,7 @@ list( APPEND l95_testoutput testoutput/diffstates.test testoutput/addincrement.test testoutput/addincrement_scaled.test + testoutput/eda_3dvar_block.test testoutput/enshofx.test testoutput/forecast.test testoutput/forecast_pseudomodel.test @@ -692,6 +696,13 @@ ecbuild_add_test( TARGET test_l95_eda_4dvar DEPENDS l95_eda.x TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) +oops_add_test( TESTNAME eda_3dvar_block + MPI 4 + MODELNAME l95 + YAMLNAME testinput/eda.3dvar.block.yaml + EXENAME l95_eda.x + COMPARE + TEST_DEPENDS test_l95_genenspert test_l95_makeobs3d ) ##################################################################### # state-related tests diff --git a/l95/test/testinput/eda.3dvar.block.1.yaml b/l95/test/testinput/eda.3dvar.block.1.yaml new file mode 100644 index 000000000..db45c5618 --- /dev/null +++ b/l95/test/testinput/eda.3dvar.block.1.yaml @@ -0,0 +1,51 @@ +cost function: + cost type: 3D-Var + window begin: 2010-01-01T21:00:00Z + window length: PT6H + variable change: Identity + geometry: + resol: 40 + analysis variables: [x] + background: + date: 2010-01-02T00:00:00Z + filename: Data/test.ens.1.2010-01-01T00:00:00Z.P1D + background error: + covariance model: L95Error + date: 2010-01-02T00:00:00Z + length_scale: 1.0 + standard_deviation: 0.6 + observations: + - obs error: + covariance model: diagonal + random amplitude: 0.2 + obs space: + obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt + obsdataout: Data/mem001.eda.block.3dvar.2010-01-02T00:00:00Z.obt + obs perturbations seed: 1 + obs operator: {} +variational: + minimizer: + algorithm: BlockBLanczos + members: 2 + iterations: + - geometry: + resol: 40 + ninner: 10 + gradient norm reduction: 1e-10 + diagnostics: + departures: ombg + test: on + obs perturbations: false + - geometry: + resol: 40 + ninner: 10 + gradient norm reduction: 1e-10 + test: on +final: + diagnostics: + departures: oman +output: + datadir: Data + exp: mem001.eda.block.3dvar + frequency: PT6H + type: an diff --git a/l95/test/testinput/eda.3dvar.block.2.yaml b/l95/test/testinput/eda.3dvar.block.2.yaml new file mode 100644 index 000000000..1d2cf201c --- /dev/null +++ b/l95/test/testinput/eda.3dvar.block.2.yaml @@ -0,0 +1,51 @@ +cost function: + cost type: 3D-Var + window begin: 2010-01-01T21:00:00Z + window length: PT6H + variable change: Identity + geometry: + resol: 40 + analysis variables: [x] + background: + date: 2010-01-02T00:00:00Z + filename: Data/test.ens.2.2010-01-01T00:00:00Z.P1D + background error: + covariance model: L95Error + date: 2010-01-02T00:00:00Z + length_scale: 1.0 + standard_deviation: 0.6 + observations: + - obs error: + covariance model: diagonal + random amplitude: 0.2 + obs space: + obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt + obsdataout: Data/mem002.eda.block.3dvar.2010-01-02T00:00:00Z.obt + obs perturbations seed: 2 + obs operator: {} +variational: + minimizer: + algorithm: BlockBLanczos + members: 2 + iterations: + - geometry: + resol: 40 + ninner: 10 + gradient norm reduction: 1e-10 + diagnostics: + departures: ombg + test: on + obs perturbations: true + - geometry: + resol: 40 + ninner: 10 + gradient norm reduction: 1e-10 + test: on +final: + diagnostics: + departures: oman +output: + datadir: Data + exp: mem002.eda.block.3dvar + frequency: PT6H + type: an diff --git a/l95/test/testinput/eda.3dvar.block.yaml b/l95/test/testinput/eda.3dvar.block.yaml new file mode 100644 index 000000000..869049f7c --- /dev/null +++ b/l95/test/testinput/eda.3dvar.block.yaml @@ -0,0 +1,3 @@ +files: +- "testinput/eda.3dvar.block.1.yaml" +- "testinput/eda.3dvar.block.2.yaml" diff --git a/l95/test/testoutput/eda_3dvar_block.test b/l95/test/testoutput/eda_3dvar_block.test new file mode 100644 index 000000000..dc711fe33 --- /dev/null +++ b/l95/test/testoutput/eda_3dvar_block.test @@ -0,0 +1,17 @@ +Test : CostJb : Nonlinear Jb = 0 +Test : CostJo : Nonlinear Jo(Lorenz 95) = 1003.27, nobs = 120, Jo/n = 8.36061, err = 0.4 +Test : CostFunction: Nonlinear J = 1003.27 +Test : BlockBLanczosMinimizer: reduction in residual norm = 0.000587397 +Test : CostFunction::addIncrement: Analysis: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=6.77364, Max=9.00861, Average=8.0056 +Test : CostJb : Nonlinear Jb = 97.3394 +Test : CostJo : Nonlinear Jo(Lorenz 95) = 15.3786, nobs = 120, Jo/n = 0.128155, err = 0.4 +Test : CostFunction: Nonlinear J = 112.718 +Test : BlockBLanczosMinimizer: reduction in residual norm = 0.00319633 +Test : CostFunction::addIncrement: Analysis: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=6.77193, Max=9.00912, Average=8.00572 +Test : CostJb : Nonlinear Jb = 97.3419 +Test : CostJo : Nonlinear Jo(Lorenz 95) = 15.3755, nobs = 120, Jo/n = 0.128129, err = 0.4 +Test : CostFunction: Nonlinear J = 112.717 diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index 0a1a67fb1..24dca6a10 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -35,6 +35,11 @@ list( APPEND qg_testinput testinput/eda_3dfgat.yaml testinput/eda_3dvar_2.yaml testinput/eda_3dvar.yaml + testinput/eda_3dvar_block_1.yaml + testinput/eda_3dvar_block_2.yaml + testinput/eda_3dvar_block_3.yaml + testinput/eda_3dvar_block_4.yaml + testinput/eda_3dvar_block.yaml testinput/eda_4dvar_1.yaml testinput/eda_4dvar_2.yaml testinput/eda_4dvar_3.yaml @@ -103,6 +108,7 @@ list( APPEND qg_testoutput testoutput/dirac_loc_4d.test testoutput/dirac_no_loc.test # testoutput/eda_4dvar.test + testoutput/eda_3dvar_block.test testoutput/ens_forecast.test testoutput/ens_hofx.test testoutput/ens_recenter.test @@ -701,6 +707,13 @@ ecbuild_add_test( TARGET test_qg_eda_4dvar DEPENDS qg_eda.x TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) +oops_add_test( TESTNAME eda_3dvar_block + MODELNAME qg + MPI 4 + YAMLNAME testinput/eda_3dvar_block.yaml + EXENAME qg_eda.x + COMPARE + TEST_DEPENDS test_qg_make_obs_3d test_qg_gen_ens_pert_B ) ##################################################################### # state-related tests diff --git a/qg/test/testinput/eda_3dvar_block.yaml b/qg/test/testinput/eda_3dvar_block.yaml new file mode 100644 index 000000000..0dd142b19 --- /dev/null +++ b/qg/test/testinput/eda_3dvar_block.yaml @@ -0,0 +1,5 @@ +files: +- "testinput/eda_3dvar_block_1.yaml" +- "testinput/eda_3dvar_block_2.yaml" +- "testinput/eda_3dvar_block_3.yaml" +- "testinput/eda_3dvar_block_4.yaml" diff --git a/qg/test/testinput/eda_3dvar_block_1.yaml b/qg/test/testinput/eda_3dvar_block_1.yaml new file mode 100644 index 000000000..be50fac61 --- /dev/null +++ b/qg/test/testinput/eda_3dvar_block_1.yaml @@ -0,0 +1,87 @@ +cost function: + cost type: 3D-Var + window begin: 2010-01-01T09:00:00Z + window length: PT6H + analysis variables: [x] + geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + background: + date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT12H.nc + background error: + covariance model: QgError + horizontal_length_scale: 2.2e6 + maximum_condition_number: 1.0e6 + standard_deviation: 1.8e7 + vertical_length_scale: 15000.0 + observations: + - obs error: + covariance model: diagonal + random amplitude: 0.4 + obs operator: + obs type: Stream + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/mem001.block_eda_3dvar.obs3d.nc + obs type: Stream + obs perturbations seed: 1 + - obs error: + covariance model: diagonal + random amplitude: 0.5 + obs operator: + obs type: Wind + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/mem001.block_eda_3dvar.obs3d.nc + obs type: Wind + obs perturbations seed: 1 + - obs error: + covariance model: diagonal + random amplitude: 0.2 + obs operator: + obs type: WSpeed + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/mem001.block_eda_3dvar.obs3d.nc + obs type: WSpeed + obs perturbations seed: 1 +variational: + minimizer: + algorithm: BlockBLanczos + members: 4 + iterations: + - diagnostics: + departures: ombg + gradient norm reduction: 1.0e-10 + ninner: 10 + geometry: + nx: 20 + ny: 10 + depths: [4500.0, 5500.0] + test: on + obs perturbations: false + - diagnostics: + departures: ombg + gradient norm reduction: 1.0e-10 + ninner: 10 + geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + test: on +final: + diagnostics: + departures: oman +output: + datadir: Data + exp: mem001.block_eda_3dvar + frequency: PT6H + type: an diff --git a/qg/test/testinput/eda_3dvar_block_2.yaml b/qg/test/testinput/eda_3dvar_block_2.yaml new file mode 100644 index 000000000..9c084f5d2 --- /dev/null +++ b/qg/test/testinput/eda_3dvar_block_2.yaml @@ -0,0 +1,87 @@ +cost function: + cost type: 3D-Var + window begin: 2010-01-01T09:00:00Z + window length: PT6H + analysis variables: [x] + geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + background: + date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT12H.nc + background error: + covariance model: QgError + horizontal_length_scale: 2.2e6 + maximum_condition_number: 1.0e6 + standard_deviation: 1.8e7 + vertical_length_scale: 15000.0 + observations: + - obs error: + covariance model: diagonal + random amplitude: 0.4 + obs operator: + obs type: Stream + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/mem002.block_eda_3dvar.obs3d.nc + obs type: Stream + obs perturbations seed: 2 + - obs error: + covariance model: diagonal + random amplitude: 0.5 + obs operator: + obs type: Wind + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/mem002.block_eda_3dvar.obs3d.nc + obs type: Wind + obs perturbations seed: 2 + - obs error: + covariance model: diagonal + random amplitude: 0.2 + obs operator: + obs type: WSpeed + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/mem002.block_eda_3dvar.obs3d.nc + obs type: WSpeed + obs perturbations seed: 2 +variational: + minimizer: + algorithm: BlockBLanczos + members: 4 + iterations: + - diagnostics: + departures: ombg + gradient norm reduction: 1.0e-10 + ninner: 10 + geometry: + nx: 20 + ny: 10 + depths: [4500.0, 5500.0] + test: on + obs perturbations: true + - diagnostics: + departures: ombg + gradient norm reduction: 1.0e-10 + ninner: 10 + geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + test: on +final: + diagnostics: + departures: oman +output: + datadir: Data + exp: mem002.block_eda_3dvar + frequency: PT6H + type: an diff --git a/qg/test/testinput/eda_3dvar_block_3.yaml b/qg/test/testinput/eda_3dvar_block_3.yaml new file mode 100644 index 000000000..c0d9c8386 --- /dev/null +++ b/qg/test/testinput/eda_3dvar_block_3.yaml @@ -0,0 +1,87 @@ +cost function: + cost type: 3D-Var + window begin: 2010-01-01T09:00:00Z + window length: PT6H + analysis variables: [x] + geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + background: + date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT12H.nc + background error: + covariance model: QgError + horizontal_length_scale: 2.2e6 + maximum_condition_number: 1.0e6 + standard_deviation: 1.8e7 + vertical_length_scale: 15000.0 + observations: + - obs error: + covariance model: diagonal + random amplitude: 0.4 + obs operator: + obs type: Stream + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/mem003.block_eda_3dvar.obs3d.nc + obs type: Stream + obs perturbations seed: 3 + - obs error: + covariance model: diagonal + random amplitude: 0.5 + obs operator: + obs type: Wind + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/mem003.block_eda_3dvar.obs3d.nc + obs type: Wind + obs perturbations seed: 3 + - obs error: + covariance model: diagonal + random amplitude: 0.2 + obs operator: + obs type: WSpeed + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/mem003.block_eda_3dvar.obs3d.nc + obs type: WSpeed + obs perturbations seed: 3 +variational: + minimizer: + algorithm: BlockBLanczos + members: 4 + iterations: + - diagnostics: + departures: ombg + gradient norm reduction: 1.0e-10 + ninner: 10 + geometry: + nx: 20 + ny: 10 + depths: [4500.0, 5500.0] + test: on + obs perturbations: true + - diagnostics: + departures: ombg + gradient norm reduction: 1.0e-10 + ninner: 10 + geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + test: on +final: + diagnostics: + departures: oman +output: + datadir: Data + exp: mem003.block_eda_3dvar + frequency: PT6H + type: an diff --git a/qg/test/testinput/eda_3dvar_block_4.yaml b/qg/test/testinput/eda_3dvar_block_4.yaml new file mode 100644 index 000000000..a82ed446c --- /dev/null +++ b/qg/test/testinput/eda_3dvar_block_4.yaml @@ -0,0 +1,87 @@ +cost function: + cost type: 3D-Var + window begin: 2010-01-01T09:00:00Z + window length: PT6H + analysis variables: [x] + geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + background: + date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT12H.nc + background error: + covariance model: QgError + horizontal_length_scale: 2.2e6 + maximum_condition_number: 1.0e6 + standard_deviation: 1.8e7 + vertical_length_scale: 15000.0 + observations: + - obs error: + covariance model: diagonal + random amplitude: 0.4 + obs operator: + obs type: Stream + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/mem004.block_eda_3dvar.obs3d.nc + obs type: Stream + obs perturbations seed: 4 + - obs error: + covariance model: diagonal + random amplitude: 0.5 + obs operator: + obs type: Wind + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/mem004.block_eda_3dvar.obs3d.nc + obs type: Wind + obs perturbations seed: 4 + - obs error: + covariance model: diagonal + random amplitude: 0.2 + obs operator: + obs type: WSpeed + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/mem004.block_eda_3dvar.obs3d.nc + obs type: WSpeed + obs perturbations seed: 4 +variational: + minimizer: + algorithm: BlockBLanczos + members: 4 + iterations: + - diagnostics: + departures: ombg + gradient norm reduction: 1.0e-10 + ninner: 10 + geometry: + nx: 20 + ny: 10 + depths: [4500.0, 5500.0] + test: on + obs perturbations: true + - diagnostics: + departures: ombg + gradient norm reduction: 1.0e-10 + ninner: 10 + geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + test: on +final: + diagnostics: + departures: oman +output: + datadir: Data + exp: mem004.block_eda_3dvar + frequency: PT6H + type: an diff --git a/qg/test/testoutput/eda_3dvar_block.test b/qg/test/testoutput/eda_3dvar_block.test new file mode 100644 index 000000000..53e394fb1 --- /dev/null +++ b/qg/test/testoutput/eda_3dvar_block.test @@ -0,0 +1,33 @@ +Test : CostJb : Nonlinear Jb = 0 +Test : CostJo : Nonlinear Jo(Stream) = 10879.7, nobs = 600, Jo/n = 18.1329, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 1663.07, nobs = 600, Jo/n = 2.77178, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 163.034, nobs = 300, Jo/n = 0.543446, err = 12 +Test : CostFunction: Nonlinear J = 12705.8 +Test : BlockBLanczosMinimizer: reduction in residual norm = 0.0230185 +Test : CostFunction::addIncrement: Analysis: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are activated +Test : Scaling= 1e+08, Min= -4.4863, Max= 1.0335, RMS= 1.7694 +Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 +Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 +Test : CostJb : Nonlinear Jb = 151.4 +Test : CostJo : Nonlinear Jo(Stream) = 688.6, nobs = 600, Jo/n = 1.148, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 596.2, nobs = 600, Jo/n = 0.9936, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 91.81, nobs = 300, Jo/n = 0.306, err = 12 +Test : CostFunction: Nonlinear J = 1528 +Test : BlockBLanczosMinimizer: reduction in residual norm = 0.2453 +Test : CostFunction::addIncrement: Analysis: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are activated +Test : Scaling= 1e+08, Min= -4.5389, Max= 1.0534, RMS= 1.7720 +Test : Scaling= 1e+08, Min= -4.0032, Max= -0.0000, RMS= 2.0632 +Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 +Test : CostJb : Nonlinear Jb = 162.8 +Test : CostJo : Nonlinear Jo(Stream) = 620, nobs = 600, Jo/n = 1.033, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 435.2, nobs = 600, Jo/n = 0.7254, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 56.05, nobs = 300, Jo/n = 0.1868, err = 12 +Test : CostFunction: Nonlinear J = 1274 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0684d0d86..48757a18f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,6 +2,7 @@ include_directories( ${CMAKE_CURRENT_SOURCE_DIR} ) list( APPEND oops_src_files +oops/assimilation/BlockBLanczosMinimizer.h oops/assimilation/BMatrix.h oops/assimilation/CalcHofX.h oops/assimilation/ControlIncrement.h @@ -56,7 +57,6 @@ oops/assimilation/LBMinimizer.h oops/assimilation/LETKFSolver.h oops/assimilation/LETKFSolverGSI.h oops/assimilation/LETKFSolverParameters.h -oops/assimilation/linsysteigen.h oops/assimilation/LocalEnsembleSolver.h oops/assimilation/Minimizer.h oops/assimilation/MinimizerUtils.cc diff --git a/src/oops/assimilation/BlockBLanczosMinimizer.h b/src/oops/assimilation/BlockBLanczosMinimizer.h new file mode 100644 index 000000000..9e2921d24 --- /dev/null +++ b/src/oops/assimilation/BlockBLanczosMinimizer.h @@ -0,0 +1,384 @@ +/* + * (C) Copyright 2018 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_ASSIMILATION_BLOCKBLANCZOSMINIMIZER_H_ +#define OOPS_ASSIMILATION_BLOCKBLANCZOSMINIMIZER_H_ + +#include + +#include +#include +#include +#include +#include + +#include "eckit/config/Configuration.h" +#include "eckit/mpi/Comm.h" + +#include "oops/assimilation/BMatrix.h" +#include "oops/assimilation/ControlIncrement.h" +#include "oops/assimilation/CostFunction.h" +#include "oops/assimilation/DRMinimizer.h" +#include "oops/assimilation/HtRinvHMatrix.h" +#include "oops/assimilation/MinimizerUtils.h" +#include "oops/assimilation/TriDiagSolve.h" +#include "oops/mpi/mpi.h" +#include "oops/util/dot_product.h" +#include "oops/util/formats.h" +#include "oops/util/Logger.h" + +namespace oops { + +template class BlockBLanczosMinimizer : + public DRMinimizer { + typedef BMatrix Bmat_; + typedef CostFunction CostFct_; + typedef ControlIncrement CtrlInc_; + typedef HtRinvHMatrix HtRinvH_; + typedef Eigen::VectorXd eigenvec_; + typedef Eigen::MatrixXd eigenmat_; + + public: + const std::string classname() const override {return "BlockBLanczosMinimizer";} + BlockBLanczosMinimizer(const eckit::Configuration &, const CostFct_ &); + ~BlockBLanczosMinimizer() {} + + private: + double solve(CtrlInc_ &, CtrlInc_ &, CtrlInc_ &, const Bmat_ &, const HtRinvH_ &, const double, + const double, const int, const double) override; + + // other functions: + void get_proj(const CtrlInc_ &, const CtrlInc_ &, eigenmat_ &, int &, + const eckit::mpi::Comm &, CtrlInc_ &); + void apply_proj(CtrlInc_ &, const CtrlInc_ &, const eigenmat_ &, int &, + const eckit::mpi::Comm &, CtrlInc_ &); + void mqrgs(CtrlInc_ &, CtrlInc_ &, eigenmat_ &, const CtrlInc_ &, int &, + const eckit::mpi::Comm &, CtrlInc_ &, CtrlInc_ &); + void HtRinvH0(const CtrlInc_ &, CtrlInc_ &, const HtRinvH_ &, int &, + const eckit::mpi::Comm &, CtrlInc_ &); + + const int members_; + const int ntasks_; + const int tasks_per_member_; + const int global_task_; + const int mymember_; + const int local_task_; +}; + +// =============================================================================================== +// Block-B-Lanczos algorithm +// Primal space +// Lanczos algorithm with complete reorthogonalization. +// MPI version (storage of partial Krylov base on each processor) + +template +BlockBLanczosMinimizer::BlockBLanczosMinimizer(const eckit::Configuration & conf, + const CostFct_ & J) + : DRMinimizer(J), members_(conf.getInt("members")), + ntasks_(oops::mpi::world().size()), + tasks_per_member_(ntasks_/members_), global_task_(oops::mpi::world().rank()), + mymember_(global_task_ / tasks_per_member_), local_task_(global_task_%tasks_per_member_) {} + +// ----------------------------------------------------------------------------------------------- + +template +double BlockBLanczosMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, CtrlInc_ & rr, + const Bmat_ & B, const HtRinvH_ & HtRinvH, + const double costJ0Jb, const double costJ0JoJc, + const int maxiter, const double tolerance) { + eigenvec_ zerov = Eigen::VectorXd::Zero(members_); + eigenmat_ zeromm = Eigen::MatrixXd::Zero(members_, members_); + + CtrlInc_ ww(xh); // Current w_i + CtrlInc_ vv(rr); // Current v_i + CtrlInc_ zz(xh); // Current z_i + + CtrlInc_ temp1(xh); // Current w_i + CtrlInc_ temp2(xh); // Current w_i + + int gestag = 0; + + std::vector> Zbase; // store the zz during iterative process + std::vector> Vbase; // store the vv during iterative process + + eigenmat_ projsol(members_, members_); // projectors (w,z) for residuals calculation. + + eigenmat_ alpha(members_, members_); + eigenmat_ beta = zeromm; + eigenmat_ beta0 = zeromm; + + std::vector ALPHAS; // store the diagonal blocks of the Arnoldi matrix + std::vector BETAS; // store the underdiagonal blocks of the Arnoldi matrix + + eigenmat_ ss; // contains the solution + eigenvec_ ss_loc = zerov; + + double norm_iiter = 0; + double norm0 = 0; + double normReduction = 1; + bool complexValues = false; + + eigenvec_ norm_red_loc(maxiter); + eigenmat_ norm_red_all(maxiter, members_); + + eigenvec_ costj_loc(maxiter); + eigenmat_ costj_all(maxiter, members_); + +// ----------------------------------------------------------------------------------------------- +// Creating the proper communicator (geographic area) for send and receive + std::string CommGeoStr = "comm_geo_" + std::to_string(local_task_); + char const *CommGeoName = CommGeoStr.c_str(); + const eckit::mpi::Comm & CommGeo = oops::mpi::world().split(local_task_, CommGeoName); + Log::info() << "Geo communicators created, for task 0 of member 0: " + << CommGeo.name() << " of size " << CommGeo.size() << std::endl; + +// ----------------------------------------------------------------------------------------------- + B.multiply(rr, zz); // z_0 = B * r_0 + norm0 = sqrt(dot_product(zz, rr)); + + // QR decomposition + mqrgs(zz, vv, beta0, rr, gestag, CommGeo, temp1, temp2); // [z_1, v_1, b0] = qr[r_0, v_0] + + Vbase.emplace_back(std::unique_ptr(new CtrlInc_(vv))); + Zbase.emplace_back(std::unique_ptr(new CtrlInc_(zz))); + + for (int iiter = 0; iiter < maxiter; ++iiter) { + Log::info() << "BlockBLanczos starting iteration " << iiter << " for rank: " << mymember_ + << std::endl; + + // Hessian application: w_i = v_i + HtRinvH * B*v_i = v_i + HtRinvH * z_i + // --> new search directions + // HtRinvH.multiply(zz, ww); + HtRinvH0(zz, ww, HtRinvH, gestag, CommGeo, temp1); + + ww += vv; + + // Orthogonalize ww against previous base vectors + for (int jiter = 0; jiter < iiter + 1; ++jiter) { + get_proj(ww, *Zbase[jiter], alpha, gestag, CommGeo, temp1); + apply_proj(ww, *Vbase[jiter], alpha, gestag, CommGeo, temp1); + } + + B.multiply(ww, zz); + get_proj(ww, zz, projsol, gestag, CommGeo, temp1); // projectors for residuals calculation + + vv = ww; + mqrgs(zz, vv, beta, ww, gestag, CommGeo, temp1, temp2); + + Zbase.emplace_back(std::unique_ptr(new CtrlInc_(zz))); + Vbase.emplace_back(std::unique_ptr(new CtrlInc_(vv))); + ALPHAS.push_back(alpha); + BETAS.push_back(beta); + + // solve T ss = beta0 * e1 + blockTriDiagSolve(ALPHAS, BETAS, beta0, ss, complexValues, members_); + + eigenvec_ ss_loc = (ss.block(iiter*members_, 0, members_, members_)).col(mymember_); + + eigenvec_ temp = projsol * ss_loc; + norm_iiter = sqrt(temp.dot(ss_loc)); + normReduction = norm_iiter / norm0; + + xh.zero(); + xx.zero(); + + const double costJ0 = costJ0Jb + costJ0JoJc; + double costJ = costJ0; + double costJb = 0; + double costJoJc = 0; + + eigenmat_ SSLK; + for (int ll = 0; ll < iiter+1; ++ll) { + SSLK = - (ss.block(ll*members_, 0, members_, members_)); + apply_proj(xh, *Vbase[ll], SSLK, gestag, CommGeo, temp1); + apply_proj(xx, *Zbase[ll], SSLK, gestag, CommGeo, temp1); + // Compute the quadratic cost function + // J[du_{i}] = J[0] - 0.5 s_{i}^T Z_{i}^T r_{0} + // Jb[du_{i}] = 0.5 s_{i}^T V_{i}^T Z_{i} s_{i} + temp2.zero(); + apply_proj(temp2, *Zbase[ll], SSLK, gestag, CommGeo, temp1); + costJ -= 0.5 * dot_product(temp2, rr); + } // this loop can be moved outside the main loop when we don't need the diagnostics anymore + + Log::info() << "BlockBLanczos end of iteration " << iiter+1 << std::endl; + printNormReduction(iiter+1, norm_iiter, normReduction); + printQuadraticCostFunction(iiter+1, costJ, costJb, costJoJc); + + norm_red_loc[iiter] = normReduction; + costj_loc[iiter] = costJ; + } // main loop iiter + + oops::mpi::allGather(CommGeo, norm_red_loc, norm_red_all); + oops::mpi::allGather(CommGeo, costj_loc, costj_all); + + Log::info() << "Norm reduction for all members:\n" << norm_red_all << std::endl; + Log::info() << "Quadratic costJ for all members:\n" << costj_all << std::endl; + + eckit::mpi::deleteComm(CommGeoName); + return normReduction; +} + +// ----------------------------------------------------------------------------------------------- +// OTHER FUNCTIONS +// ----------------------------------------------------------------------------------------------- + +template +void BlockBLanczosMinimizer::get_proj(const CtrlInc_ & www, + const CtrlInc_ & incr_tosend, + eigenmat_ & alpha_mat, int & tag, + const eckit::mpi::Comm & comm, + CtrlInc_ & incr_rcv) { + // Computes mat_proj = Zt.W + // with incr_tosend(member_i) the columns of matrix Z + // with www(member_i) the columns of matrix W + eigenvec_ alpha_loc = Eigen::VectorXd::Zero(members_); + int tag_rcv; + int tag_send; + + for (int p = 0; p < mymember_; p++) { + tag_rcv = p * members_ + mymember_ + tag * members_ * members_; + tag_send = mymember_ * members_ + p + tag * members_ * members_; + oops::mpi::send(comm, incr_tosend, p, tag_send); + oops::mpi::receive(comm, incr_rcv, p, tag_rcv); + alpha_loc(p) = dot_product(incr_rcv, www); + } + + alpha_loc(mymember_) = dot_product(incr_tosend, www); + + for (int p = mymember_ + 1; p < members_; p++) { + tag_rcv = p * members_ + mymember_ + tag * members_ * members_; + tag_send = mymember_ * members_ + p + tag * members_ * members_; + oops::mpi::receive(comm, incr_rcv, p, tag_rcv); + oops::mpi::send(comm, incr_tosend, p, tag_send); + alpha_loc(p) = dot_product(incr_rcv, www); + } + oops::mpi::allGather(comm, alpha_loc, alpha_mat); + + tag += 2; +} + +// ----------------------------------------------------------------------------------------------- + +template +void BlockBLanczosMinimizer::apply_proj(CtrlInc_ & incr_tochange, + const CtrlInc_ & incr_tosend, + const eigenmat_ & alpha_mat, int & tag, + const eckit::mpi::Comm & comm, + CtrlInc_ & incr_rcv) { + // Computes W = W - V.alpha + // with incr_tochange(member_i) the columns of matrix W + // with incr_tosend(member_i) the columns of matrix V + eigenvec_ alpha_loc = alpha_mat.col(mymember_); + int tag_rcv; + int tag_send; + + for (int p = 0; p < mymember_; p++) { + tag_rcv = p * members_ + mymember_ + tag * members_ * members_; + tag_send = mymember_ * members_ + p + tag * members_ * members_; + oops::mpi::send(comm, incr_tosend, p, tag_send); + oops::mpi::receive(comm, incr_rcv, p, tag_rcv); + incr_tochange.axpy(-alpha_loc(p), incr_rcv); + } + + incr_tochange.axpy(-alpha_loc(mymember_), incr_tosend); + + for (int p = mymember_ + 1; p < members_; p++) { + tag_rcv = p * members_ + mymember_ + tag * members_ * members_; + tag_send = mymember_ * members_ + p + tag * members_ * members_; + oops::mpi::receive(comm, incr_rcv, p, tag_rcv); + oops::mpi::send(comm, incr_tosend, p, tag_send); + incr_tochange.axpy(-alpha_loc(p), incr_rcv); + } + tag += 2; +} + +// ----------------------------------------------------------------------------------------------- + +template +void BlockBLanczosMinimizer::mqrgs(CtrlInc_ & zzz, CtrlInc_ & vvv, + eigenmat_ & beta_mat, const CtrlInc_ & www, + int & tag, + const eckit::mpi::Comm & comm, + CtrlInc_ & v_other, CtrlInc_ & z_other) { + // QR decomposition: [zzz, vvv, beta_mat] = qr[www, vvv] using Gram-Schmidt + eigenvec_ beta_loc = Eigen::VectorXd::Zero(members_); + int tag_rcv_v; + int tag_rcv_z; + int tag_send_v; + int tag_send_z; + + // Orthogonalization + for (int p = 0; p < mymember_; p++) { + tag_rcv_v = p * members_ + mymember_ + tag * members_ * members_; + tag_rcv_z = p * members_ + mymember_ + (tag + 1) * members_ * members_; + oops::mpi::receive(comm, v_other, p, tag_rcv_v); + oops::mpi::receive(comm, z_other, p, tag_rcv_z); + beta_loc(p) = dot_product(www, z_other) / dot_product(v_other, z_other); + vvv.axpy(-beta_loc(p), v_other); + zzz.axpy(-beta_loc(p), z_other); + } + + for (int p = mymember_ + 1; p < members_; p++) { + tag_send_v = mymember_ * members_ + p + tag * members_ * members_; + tag_send_z = mymember_ * members_ + p + (tag + 1) * members_ * members_; + oops::mpi::send(comm, vvv, p, tag_send_v); + oops::mpi::send(comm, zzz, p, tag_send_z); + } + + // Normalization (TODO: needs better writing) + beta_loc(mymember_) = sqrt(std::max(dot_product(vvv, zzz), 1e-15)); + + vvv *= (1 / beta_loc(mymember_)); + zzz *= (1 / beta_loc(mymember_)); + + oops::mpi::allGather(comm, beta_loc, beta_mat); + + for (int ii = 0; ii < members_; ++ii) { + for (int jj = ii + 1; jj < members_; ++jj) { + beta_mat(ii, jj) = beta_mat(ii, jj) * beta_mat(ii, ii); + } + } + + tag += 2; +} + +// ----------------------------------------------------------------------------------------------- + +template +void BlockBLanczosMinimizer::HtRinvH0(const CtrlInc_ & z_loc, CtrlInc_ & w_out, + const HtRinvH_ & HtRinvH, int & tag, + const eckit::mpi::Comm & comm, CtrlInc_ & z_other) { +// send z_loc to task 0, process HtRinvH_0 * z_loc and send the result back to original task + int tag_send_w; + int tag_rcv_w; + int tag_send_z; + int tag_rcv_z; + int dest = 0; + + if (mymember_ == 0) { + for (int ii = 1; ii < members_; ++ii) { + tag_rcv_z = members_ * ii + tag * members_ * members_; + oops::mpi::receive(comm, z_other, ii, tag_rcv_z); + HtRinvH.multiply(z_other, w_out); + tag_send_w = members_ * ii + (tag + 1) * members_ * members_; + oops::mpi::send(comm, w_out, ii, tag_send_w); // send to task ii + } + HtRinvH.multiply(z_loc, w_out); + } else { + tag_send_z = mymember_ * members_ + tag * members_ * members_; + oops::mpi::send(comm, z_loc, dest, tag_send_z); // send to task 0 + tag_rcv_w = mymember_ * members_ + (tag + 1) * members_ * members_; + oops::mpi::receive(comm, w_out, dest, tag_rcv_w); + } + tag += 2; +} + +// ----------------------------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_ASSIMILATION_BLOCKBLANCZOSMINIMIZER_H_ diff --git a/src/oops/assimilation/TriDiagSolve.h b/src/oops/assimilation/TriDiagSolve.h index 65255e81e..90db63176 100644 --- a/src/oops/assimilation/TriDiagSolve.h +++ b/src/oops/assimilation/TriDiagSolve.h @@ -16,9 +16,9 @@ #include -#include "oops/assimilation/linsysteigen.h" // for solving small full linear systems. #include "oops/mpi/mpi.h" #include "oops/util/dot_product.h" +#include "oops/util/Logger.h" namespace oops { @@ -74,6 +74,9 @@ void blockTriDiagSolve(const std::vector & alphas, Eigen::VectorXcd eivals = TT.eigenvalues(); Eigen::MatrixXd eivalsImag = eivals.imag(); complexValues = (eivalsImag.array() != 0.0).any(); + if (complexValues) { + throw eckit::BadValue("T matrix has complex values."); + } } diff --git a/src/oops/assimilation/instantiateMinFactory.h b/src/oops/assimilation/instantiateMinFactory.h index 18a5882a0..a6050267b 100644 --- a/src/oops/assimilation/instantiateMinFactory.h +++ b/src/oops/assimilation/instantiateMinFactory.h @@ -11,6 +11,7 @@ #ifndef OOPS_ASSIMILATION_INSTANTIATEMINFACTORY_H_ #define OOPS_ASSIMILATION_INSTANTIATEMINFACTORY_H_ +#include "oops/assimilation/BlockBLanczosMinimizer.h" #include "oops/assimilation/DRGMRESRMinimizer.h" #include "oops/assimilation/DRIPCGMinimizer.h" #include "oops/assimilation/DRPCGMinimizer.h" @@ -47,6 +48,8 @@ template void instantiateMinFactory() { static MinMaker > makerRPLanczos_("RPLanczos"); static MinMaker > makerMINRES_("MINRES"); static MinMaker > makerFGMRES_("FGMRES"); + static MinMaker > + makerBlockBLanczos_("BlockBLanczos"); } } // namespace oops diff --git a/src/oops/assimilation/linsysteigen.h b/src/oops/assimilation/linsysteigen.h deleted file mode 100644 index dc3e70ab6..000000000 --- a/src/oops/assimilation/linsysteigen.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * (C) Copyright 2018 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef OOPS_ASSIMILATION_LINSYSTEIGEN_H_ -#define OOPS_ASSIMILATION_LINSYSTEIGEN_H_ - -#include - -namespace oops { - -typedef Eigen::MatrixXd eigenmat_; -typedef Eigen::VectorXd eigenvec_; - -eigenmat_ linsysteigen(const eigenmat_ & AA, const eigenmat_ & YY) { - // Solve AX=Y (unknown X), A full square matrix, Y matrix - return AA.fullPivLu().solve(YY); -} - -eigenvec_ linsysteigen(const eigenmat_ & AA, const eigenvec_ & yy) { - // Solve Ax=y (unknown x), A full square matrix, y vector - return AA.fullPivLu().solve(yy); -} - -eigenmat_ linsysteigentrans(const eigenmat_ & UU, const eigenmat_ & II) { - // Solve LU=I (unknown L), U & I full square matrixes - eigenmat_ AA = UU.transpose(); - eigenmat_ BB = II.transpose(); - return linsysteigen(AA, BB).transpose(); -} - -} // namespace oops -#endif // OOPS_ASSIMILATION_LINSYSTEIGEN_H_ diff --git a/src/oops/mpi/mpi.cc b/src/oops/mpi/mpi.cc index 7eed3dc99..d5c46a4bd 100644 --- a/src/oops/mpi/mpi.cc +++ b/src/oops/mpi/mpi.cc @@ -56,7 +56,7 @@ void gather(const eckit::mpi::Comm & comm, const std::vector & send, // ------------------------------------------------------------------------------------------------ void allGather(const eckit::mpi::Comm & comm, - const Eigen::VectorXd & sendbuf, std::vector & recvbuf) { + const Eigen::VectorXd & sendbuf, Eigen::MatrixXd & recvbuf) { const int ntasks = comm.size(); int buf_size = sendbuf.size(); @@ -77,7 +77,7 @@ void allGather(const eckit::mpi::Comm & comm, vbuf_total.begin() + (ii + 1) * buf_size); Eigen::VectorXd my_vect = Eigen::Map(vloc.data(), vloc.size()); - recvbuf[ii] = my_vect; + recvbuf.col(ii) = my_vect; } } diff --git a/src/oops/mpi/mpi.h b/src/oops/mpi/mpi.h index 4e8762b7d..0650a3e4e 100644 --- a/src/oops/mpi/mpi.h +++ b/src/oops/mpi/mpi.h @@ -17,6 +17,8 @@ #include "eckit/exception/Exceptions.h" #include "eckit/mpi/Comm.h" +#include "oops/util/Timer.h" + namespace oops { namespace mpi { @@ -35,6 +37,7 @@ const eckit::mpi::Comm & myself(); template void send(const eckit::mpi::Comm & comm, const SERIALIZABLE & sendobj, const int dest, const int tag) { + util::Timer timer("oops::mpi", "send"); std::vector sendbuf; sendobj.serialize(sendbuf); comm.send(sendbuf.data(), sendbuf.size(), dest, tag); @@ -45,6 +48,7 @@ void send(const eckit::mpi::Comm & comm, const SERIALIZABLE & sendobj, template void receive(const eckit::mpi::Comm & comm, SERIALIZABLE & recvobj, const int source, const int tag) { + util::Timer timer("oops::mpi", "receive"); size_t sz = recvobj.serialSize(); std::vector recvbuf(sz); eckit::mpi::Status status = comm.receive(recvbuf.data(), sz, source, tag); @@ -84,7 +88,7 @@ void gather(const eckit::mpi::Comm & comm, const std::vector & sen // allGather for eigen vectors // ------------------------------------------------------------------------------------------------ void allGather(const eckit::mpi::Comm & comm, - const Eigen::VectorXd &, std::vector &); + const Eigen::VectorXd &, Eigen::MatrixXd &); /// \brief A wrapper around the MPI *all gather* operation for serializable types. /// diff --git a/src/test/mpi/mpi.h b/src/test/mpi/mpi.h index 135c936d1..ed9e73c8d 100644 --- a/src/test/mpi/mpi.h +++ b/src/test/mpi/mpi.h @@ -162,18 +162,23 @@ CASE("mpi/mpi/gatherDouble") { CASE("mpi/mpi/allGatherEigen") { const eckit::mpi::Comm &comm = oops::mpi::world(); const size_t rank = comm.rank(); - Eigen::VectorXd localEigen = rank * Eigen::VectorXd::Ones(5); - std::vector globalEigen = {Eigen::VectorXd::Zero(5), Eigen::VectorXd::Zero(5), - Eigen::VectorXd::Zero(5), Eigen::VectorXd::Zero(5)}; - std::vector expectedEigen = {0*Eigen::VectorXd::Ones(5), - 1*Eigen::VectorXd::Ones(5), - 2*Eigen::VectorXd::Ones(5), - 3*Eigen::VectorXd::Ones(5)}; + + Eigen::VectorXd localEigen = rank * Eigen::VectorXd::Ones(4); + + Eigen::MatrixXd globalEigen(4, 4); + globalEigen << Eigen::VectorXd::Zero(4), + Eigen::VectorXd::Zero(4), + Eigen::VectorXd::Zero(4), + Eigen::VectorXd::Zero(4); + + Eigen::MatrixXd expectedEigen(4, 4); + expectedEigen << 0*Eigen::VectorXd::Ones(4), + 1*Eigen::VectorXd::Ones(4), + 2*Eigen::VectorXd::Ones(4), + 3*Eigen::VectorXd::Ones(4); + oops::mpi::allGather(comm, localEigen, globalEigen); - EXPECT_EQUAL(expectedEigen[0], globalEigen[0]); - EXPECT_EQUAL(expectedEigen[1], globalEigen[1]); - EXPECT_EQUAL(expectedEigen[2], globalEigen[2]); - EXPECT_EQUAL(expectedEigen[3], globalEigen[3]); + EXPECT_EQUAL(expectedEigen, globalEigen); } // ----------------------------------------------------------------------------------------------- From a5055fff2332e86c2e9990c6f4b706caf9aaddec Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 30 Dec 2020 12:57:14 -0700 Subject: [PATCH 028/142] Simplification of State4D/Increment4D related classes (#1000) * remove unused State/Inc4D code * simplify Increment4D implementation --- src/oops/assimilation/Increment4D.h | 324 +++------------------------- src/oops/assimilation/State4D.h | 64 +----- src/oops/base/IncrementEnsemble4D.h | 137 ++---------- 3 files changed, 48 insertions(+), 477 deletions(-) diff --git a/src/oops/assimilation/Increment4D.h b/src/oops/assimilation/Increment4D.h index 9da43016a..61e9ce21f 100644 --- a/src/oops/assimilation/Increment4D.h +++ b/src/oops/assimilation/Increment4D.h @@ -11,117 +11,59 @@ #ifndef OOPS_ASSIMILATION_INCREMENT4D_H_ #define OOPS_ASSIMILATION_INCREMENT4D_H_ -#include #include -#include #include #include -#include - -#include "atlas/field.h" - #include "eckit/config/LocalConfiguration.h" -#include "eckit/exception/Exceptions.h" -#include "oops/assimilation/CostJbState.h" #include "oops/assimilation/State4D.h" #include "oops/base/Variables.h" #include "oops/interface/Geometry.h" #include "oops/interface/Increment.h" #include "oops/util/DateTime.h" #include "oops/util/dot_product.h" -#include "oops/util/Duration.h" #include "oops/util/Logger.h" #include "oops/util/Printable.h" -#include "oops/util/Serializable.h" namespace oops { -/// State increment -/*! - * The state increment contains the increment to the 3D or 4D state part of - * the VDA control variable. - */ - -// ----------------------------------------------------------------------------- -template class Increment4D : public util::Printable, - public util::Serializable { +/// 4D model state Increment (vector of 3D Increments) +template class Increment4D : public util::Printable { typedef Geometry Geometry_; typedef Increment Increment_; - typedef CostJbState JbState_; typedef State4D State4D_; public: static const std::string classname() {return "Increment4D";} -/// Constructor, destructor - explicit Increment4D(const JbState_ &); - explicit Increment4D(const Increment_ &); + /// Constructor for specified times Increment4D(const Geometry_ &, const Variables &, const std::vector &); - Increment4D(const Increment4D &, const bool copy = true); - Increment4D(const Geometry_ &, const Increment4D &); - ~Increment4D(); - -/// Interfacing - Increment_ & incr4d() {return *incr4d_;} - const Increment_ & incr4d() const {return *incr4d_;} -/// Linear algebra operators + /// Linear algebra operators void diff(const State4D_ &, const State4D_ &); void zero(); void random(); void ones(); - void dirac(std::vector); - Increment4D & operator=(const Increment4D &); - Increment4D & operator+=(const Increment4D &); - Increment4D & operator-=(const Increment4D &); - Increment4D & operator*=(const double); - void axpy(const double, const Increment4D &, const bool check = true); double dot_product_with(const Increment4D &) const; void schur_product_with(const Increment4D &); -/// I/O and diagnostics - void read(const eckit::Configuration &); - void write(const eckit::Configuration &) const; - -/// Get geometry - Geometry_ geometry() const {return this->get(first_).geometry();} + /// Get geometry + Geometry_ geometry() const {return incr4d_[0].geometry();} -/// ATLAS FieldSet - void setAtlas(atlas::FieldSet *) const; - void toAtlas(atlas::FieldSet *) const; - void fromAtlas(atlas::FieldSet *); - -/// Get model space control variable - Increment_ & operator[](const int ii) {return this->get(ii);} - const Increment_ & operator[](const int ii) const {return this->get(ii);} - int first() const {return first_;} - int last() const {return last_;} - size_t size() const {return last_-first_+1;} - -/// To be removed - void shift_forward(); - void shift_backward(); - -/// Serialize and deserialize - size_t serialSize() const override; - void serialize(std::vector &) const override; - void deserialize(const std::vector &, size_t &) override; + /// Get 3D increments + Increment_ & operator[](const int ii) {return incr4d_[ii];} + const Increment_ & operator[](const int ii) const {return incr4d_[ii];} + size_t size() const {return incr4d_.size();} private: - Increment_ & get(const int); - const Increment_ & get(const int) const; void print(std::ostream &) const override; - typedef typename boost::ptr_map::iterator iter_; - typedef typename boost::ptr_map::const_iterator icst_; - boost::ptr_map incr4d_; - int first_; - int last_; + std::vector incr4d_; }; // ============================================================================= +/// "Increment" 4D State \p xx with 4D Increment \p dx template State4D & operator+=(State4D & xx, const Increment4D & dx) { Log::trace() << "operator+=(State4D, Increment4D) starting" << std::endl; @@ -131,276 +73,70 @@ State4D & operator+=(State4D & xx, const Increment4D & dx) Log::trace() << "operator+=(State4D, Increment4D) done" << std::endl; return xx; } -// ---------------------------------------------------------------------------- -template -Increment4D::Increment4D(const JbState_ & jb) - : incr4d_(), first_(0), last_(jb.nstates() - 1) -{ - for (int jsub = 0; jsub <= last_; ++jsub) { - Increment_ * incr = jb.newStateIncrement(jsub); - incr4d_.insert(jsub, incr); - } - Log::trace() << "Increment4D:Increment4D created." << std::endl; -} -// ----------------------------------------------------------------------------- -template -Increment4D::Increment4D(const Increment_ & dx) - : incr4d_(), first_(0), last_(0) -{ - Increment_ * incr = new Increment_(dx); - incr4d_.insert(0, incr); - Log::trace() << "Increment4D:Increment4D created." << std::endl; -} + // ----------------------------------------------------------------------------- template Increment4D::Increment4D(const Geometry_ & resol, const Variables & vars, const std::vector & timeslots) - : incr4d_(), first_(0), last_(timeslots.size() - 1) + : incr4d_() { - for (int jsub = 0; jsub <= last_; ++jsub) { - Increment_ * incr = new Increment_(resol, vars, timeslots[jsub]); - incr4d_.insert(jsub, incr); + for (int jtime = 0; jtime < timeslots.size(); ++jtime) { + incr4d_.emplace_back(resol, vars, timeslots[jtime]); } Log::trace() << "Increment4D:Increment4D created." << std::endl; } // ----------------------------------------------------------------------------- template -Increment4D::Increment4D(const Increment4D & other, const bool copy) - : incr4d_(), first_(other.first_), last_(other.last_) -{ - for (icst_ jsub = other.incr4d_.begin(); jsub != other.incr4d_.end(); ++jsub) { - int isub = jsub->first; - Increment_ * tmp = new Increment_(*jsub->second, copy); - incr4d_.insert(isub, tmp); - } - Log::trace() << "Increment4D:Increment4D copied." << std::endl; -} -// ----------------------------------------------------------------------------- -// ----------------------------------------------------------------------------- -template -Increment4D::Increment4D(const Geometry_ & geom, const Increment4D & other) - : incr4d_(), first_(other.first_), last_(other.last_) -{ - for (icst_ jsub = other.incr4d_.begin(); jsub != other.incr4d_.end(); ++jsub) { - int isub = jsub->first; - Increment_ * tmp = new Increment_(geom, *jsub->second); - incr4d_.insert(isub, tmp); - } - Log::trace() << "Increment4D:Increment4D copied." << std::endl; -} -// ----------------------------------------------------------------------------- -template -Increment & Increment4D::get(const int ii) { - iter_ it = incr4d_.find(ii); - ASSERT(it != incr4d_.end()); - return *it->second; -} -// ----------------------------------------------------------------------------- -template -const Increment & Increment4D::get(const int ii) const { - icst_ it = incr4d_.find(ii); - ASSERT(it != incr4d_.end()); - return *it->second; -} -// ----------------------------------------------------------------------------- -template -Increment4D::~Increment4D() {} -// ----------------------------------------------------------------------------- -template void Increment4D::zero() { - for (iter_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - jsub->second->zero(); + for (auto & incr : incr4d_) { + incr.zero(); } } // ----------------------------------------------------------------------------- template void Increment4D::random() { - for (iter_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - jsub->second->random(); + for (auto & incr : incr4d_) { + incr.random(); } } // ----------------------------------------------------------------------------- template void Increment4D::ones() { - for (iter_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - jsub->second->ones(); - } -} -// ----------------------------------------------------------------------------- -template -void Increment4D::dirac(std::vector confs) { - this->zero(); - for (const auto & conf : confs) { - const util::DateTime date(conf.getString("date")); - for (iter_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - if (date == jsub->second->validTime()) { - jsub->second->dirac(conf); - } - } + for (auto & incr : incr4d_) { + incr.ones(); } } // ----------------------------------------------------------------------------- template void Increment4D::diff(const State4D_ & cv1, const State4D_ & cv2) { - for (iter_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - jsub->second->diff(cv1[jsub->first], cv2[jsub->first]); - } -} -// ----------------------------------------------------------------------------- -template -Increment4D & Increment4D::operator=(const Increment4D & rhs) { - incr4d_ = rhs.incr4d_; - return *this; -} -// ----------------------------------------------------------------------------- -template -Increment4D & Increment4D::operator+=(const Increment4D & rhs) { - for (int jsub = rhs.first(); jsub <= rhs.last(); ++jsub) { - this->get(jsub) += rhs[jsub]; - } - return *this; -} -// ----------------------------------------------------------------------------- -template -Increment4D & Increment4D::operator-=(const Increment4D & rhs) { - for (int jsub = rhs.first(); jsub <= rhs.last(); ++jsub) { - this->get(jsub) -= rhs[jsub]; - } - return *this; -} -// ----------------------------------------------------------------------------- -template -Increment4D & Increment4D::operator*=(const double zz) { - for (iter_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - *jsub->second *= zz; - } - return *this; -} -// ----------------------------------------------------------------------------- -template -void Increment4D::axpy(const double zz, const Increment4D & rhs, const bool check) { - for (iter_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - jsub->second->axpy(zz, rhs[jsub->first], check); - } -} -// ----------------------------------------------------------------------------- -template -void Increment4D::read(const eckit::Configuration & config) { - for (iter_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - std::stringstream ss; - ss << jsub->first+1; - std::string query = "increment[@indx='" + ss.str() + "']"; - eckit::LocalConfiguration fileConfig(config, query); - jsub->second->read(fileConfig); - Log::info() << "Increment4D:read increment" << *jsub->second << std::endl; - } -} -// ----------------------------------------------------------------------------- -template -void Increment4D::write(const eckit::Configuration & config) const { - for (icst_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - jsub->second->write(config); - } -} -// ----------------------------------------------------------------------------- -template -void Increment4D::setAtlas(atlas::FieldSet * afieldset) const { - for (icst_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - jsub->second->setAtlas(afieldset); - } -} -// ----------------------------------------------------------------------------- -template -void Increment4D::toAtlas(atlas::FieldSet * afieldset) const { - for (icst_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - jsub->second->toAtlas(afieldset); - } -} -// ----------------------------------------------------------------------------- -template -void Increment4D::fromAtlas(atlas::FieldSet * afieldset) { - for (iter_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - jsub->second->fromAtlas(afieldset); + for (size_t jtime = 0; jtime < incr4d_.size(); ++jtime) { + incr4d_[jtime].diff(cv1[jtime], cv2[jtime]); } } // ----------------------------------------------------------------------------- template void Increment4D::print(std::ostream & outs) const { - for (icst_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - outs << *jsub->second << std::endl; + for (const auto & incr : incr4d_) { + outs << incr << std::endl; } } // ----------------------------------------------------------------------------- template double Increment4D::dot_product_with(const Increment4D & x2) const { double zz = 0.0; - for (icst_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - zz += dot_product(*jsub->second, x2[jsub->first]); + for (size_t jtime = 0; jtime < incr4d_.size(); ++jtime) { + zz += dot_product(incr4d_[jtime], x2[jtime]); } return zz; } // ----------------------------------------------------------------------------- template void Increment4D::schur_product_with(const Increment4D & x2) { - for (iter_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - jsub->second->schur_product_with(x2[jsub->first]); - } -} -// ----------------------------------------------------------------------------- -template -void Increment4D::shift_forward() { - typedef typename boost::ptr_map::reverse_iterator rit; - for (rit jsub = incr4d_.rbegin(); jsub != incr4d_.rend(); ++jsub) { - const int isub = jsub->first; - if (isub > first_) this->get(isub) = this->get(isub-1); - } - incr4d_.erase(first_); - Log::info() << "Increment4D::shift_forward erased " << first_ << std::endl; - first_ += 1; -} -// ----------------------------------------------------------------------------- -template -void Increment4D::shift_backward() { - for (iter_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - const int isub = jsub->first; - if (isub < last_) this->get(isub) = this->get(isub+1); + for (size_t jtime = 0; jtime < incr4d_.size(); ++jtime) { + incr4d_[jtime].schur_product_with(x2[jtime]); } - incr4d_.erase(last_); - Log::info() << "Increment4D::shift_backward erased " << last_ << std::endl; - last_ -= 1; } -// ----------------------------------------------------------------------------- -template -size_t Increment4D::serialSize() const { - size_t ss = 1; - for (icst_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - ss += jsub->second->serialSize(); - ++ss; - } - return ss; -} -// ----------------------------------------------------------------------------- -template -void Increment4D::serialize(std::vector & vect) const { - vect.push_back(-98765.4321); - for (icst_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - jsub->second->serialize(vect); - vect.push_back(-98765.4321); - } -} -// ----------------------------------------------------------------------------- -template -void Increment4D::deserialize(const std::vector & vect, size_t & current) { - ASSERT(vect.at(current) == -98765.4321); - ++current; - for (iter_ jsub = incr4d_.begin(); jsub != incr4d_.end(); ++jsub) { - jsub->second->deserialize(vect, current); - ASSERT(vect.at(current) == -98765.4321); - ++current; - } -} - // ----------------------------------------------------------------------------- } // namespace oops diff --git a/src/oops/assimilation/State4D.h b/src/oops/assimilation/State4D.h index daa8208cd..9c0035fa6 100644 --- a/src/oops/assimilation/State4D.h +++ b/src/oops/assimilation/State4D.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2009-2016 ECMWF. + * (C) Copyright 2020 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -11,13 +12,11 @@ #ifndef OOPS_ASSIMILATION_STATE4D_H_ #define OOPS_ASSIMILATION_STATE4D_H_ -#include #include #include #include #include "eckit/config/LocalConfiguration.h" -#include "eckit/exception/Exceptions.h" #include "oops/interface/Geometry.h" #include "oops/interface/State.h" @@ -26,12 +25,7 @@ namespace oops { -/// Four dimensional state -/*! - * The 4D state is mostly used as part of the VDA control variable. - */ - -// ----------------------------------------------------------------------------- +/// Four dimensional state (vector of 3D States) template class State4D : public util::Printable { typedef Geometry Geometry_; typedef State State_; @@ -39,17 +33,13 @@ template class State4D : public util::Printable { public: static const std::string classname() {return "State4D";} -/// The arguments define the number of sub-windows and the resolution + /// The arguments define all states in 4D and their resolution State4D(const Geometry_ &, const eckit::Configuration &); - explicit State4D(const State_ &); -/// I/O and diagnostics - void read(const eckit::Configuration &); + /// I/O void write(const eckit::Configuration &) const; - double norm() const; -/// Get model space control variable - bool checkStatesNumber(const unsigned int nn) const {return state4d_.size() == nn;} + /// Get 3D model state size_t size() const {return state4d_.size();} State_ & operator[](const int ii) {return state4d_[ii];} const State_ & operator[](const int ii) const {return state4d_[ii];} @@ -58,7 +48,7 @@ template class State4D : public util::Printable { const Variables & variables() const {return state4d_[0].variables();} const std::vector validTimes() const; -/// Accumulator + /// Accumulator void zero(); void accumul(const double &, const State4D &); @@ -93,36 +83,6 @@ State4D::State4D(const Geometry_ & resol, const eckit::Configuration & co // ----------------------------------------------------------------------------- -template -State4D::State4D(const State_ & state3d) { - state4d_.emplace_back(state3d); - Log::trace() << "State4D constructed." << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void State4D::read(const eckit::Configuration & config) { - // 4D state - if (config.has("states")) { - std::vector confs; - config.get("states", confs); - ASSERT(state4d_.size() == confs.size()); - for (size_t jj = 0; jj < state4d_.size(); ++jj) { - state4d_[jj].read(confs[jj]);; - } - } else { - // 3D state - ASSERT(state4d_.size() == 1); - state4d_[0].read(config); - } - for (size_t jj = 1; jj < state4d_.size(); ++jj) { - ASSERT(state4d_[jj].variables() == state4d_[0].variables()); - } -} - -// ----------------------------------------------------------------------------- - template void State4D::write(const eckit::Configuration & config) const { // 4D state @@ -187,18 +147,6 @@ void State4D::print(std::ostream & outs) const { // ----------------------------------------------------------------------------- -template -double State4D::norm() const { - double zn = 0.0; - for (const State_ & state : state4d_) { - double zz = state.norm(); - zn += zz * zz; - } - return sqrt(zn); -} - -// ----------------------------------------------------------------------------- - } // namespace oops #endif // OOPS_ASSIMILATION_STATE4D_H_ diff --git a/src/oops/base/IncrementEnsemble4D.h b/src/oops/base/IncrementEnsemble4D.h index 9e128a2a3..ebd8f2b20 100644 --- a/src/oops/base/IncrementEnsemble4D.h +++ b/src/oops/base/IncrementEnsemble4D.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2009-2016 ECMWF. + * (C) Copyright 2020 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -11,18 +12,12 @@ #ifndef OOPS_BASE_INCREMENTENSEMBLE4D_H_ #define OOPS_BASE_INCREMENTENSEMBLE4D_H_ -#include #include -#include #include -#include - #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/Increment4D.h" #include "oops/assimilation/State4D.h" -#include "oops/base/Accumulator.h" -#include "oops/base/LinearVariableChangeBase.h" #include "oops/base/StateEnsemble4D.h" #include "oops/base/Variables.h" #include "oops/interface/Geometry.h" @@ -33,50 +28,30 @@ namespace oops { // ----------------------------------------------------------------------------- -/// \brief Ensemble of 4D inrements +/// \brief Ensemble of 4D increments template class IncrementEnsemble4D { - typedef LinearVariableChangeBase LinearVariableChangeBase_; typedef Geometry Geometry_; typedef State4D State4D_; typedef StateEnsemble4D StateEnsemble4D_; typedef Increment4D Increment4D_; - typedef typename boost::ptr_vector ChvarVec_; - typedef typename ChvarVec_::const_reverse_iterator ircst_; - public: /// Constructor IncrementEnsemble4D(const Geometry_ & resol, - const Variables & vars, - const std::vector &, - const int rank); + const Variables & vars, + const std::vector &, + const int rank); /// \brief construct ensemble of perturbations as \p ens - \p mean; holding // \p vars variables IncrementEnsemble4D(const StateEnsemble4D_ & ens, const State4D_ & mean, - const Variables & vars); - IncrementEnsemble4D(const eckit::Configuration &, - const State4D_ &, const State4D_ &, const Geometry_ &, const Variables &); + const Variables & vars); /// Accessors - unsigned int size() const { - return ensemblePerturbs_.size(); - } - Increment4D_ & operator[](const int ii) { - return ensemblePerturbs_[ii]; - } - const Increment4D_ & operator[](const int ii) const { - return ensemblePerturbs_[ii]; - } - - /// Control variables - const Variables & controlVariables() const {return vars_;} - - /// Release / reset - void releaseMember(); - void resetMember(const Increment4D_ &); + size_t size() const {return ensemblePerturbs_.size();} + Increment4D_ & operator[](const size_t ii) {return ensemblePerturbs_[ii];} + const Increment4D_ & operator[](const size_t ii) const {return ensemblePerturbs_[ii];} private: - const Variables vars_; std::vector ensemblePerturbs_; }; @@ -86,11 +61,11 @@ template IncrementEnsemble4D::IncrementEnsemble4D(const Geometry_ & resol, const Variables & vars, const std::vector & timeslots, const int rank) - : vars_(vars), ensemblePerturbs_() + : ensemblePerturbs_() { ensemblePerturbs_.reserve(rank); for (int m = 0; m < rank; ++m) { - ensemblePerturbs_.emplace_back(resol, vars_, timeslots); + ensemblePerturbs_.emplace_back(resol, vars, timeslots); } Log::trace() << "IncrementEnsemble4D:contructor done" << std::endl; } @@ -100,7 +75,7 @@ IncrementEnsemble4D::IncrementEnsemble4D(const Geometry_ & resol, const V template IncrementEnsemble4D::IncrementEnsemble4D(const StateEnsemble4D_ & ensemble, const State4D_ & mean, const Variables & vars) - : vars_(vars), ensemblePerturbs_() + : ensemblePerturbs_() { ensemblePerturbs_.reserve(ensemble.size()); for (size_t ii = 0; ii < ensemble.size(); ++ii) { @@ -113,94 +88,6 @@ IncrementEnsemble4D::IncrementEnsemble4D(const StateEnsemble4D_ & ensembl // ----------------------------------------------------------------------------- -template -IncrementEnsemble4D::IncrementEnsemble4D(const eckit::Configuration & conf, - const State4D_ & xb, const State4D_ & fg, - const Geometry_ & resol, const Variables & vars) - : vars_(vars), ensemblePerturbs_() -{ - // Get rank from config - std::vector memberConfig; - conf.get("members", memberConfig); - - // Check sizes and fill in timeslots - ASSERT(xb.size() == fg.size()); - std::vector timeslots(xb.size()); - for (unsigned jsub = 0; jsub < xb.size(); ++jsub) { - ASSERT(xb[jsub].validTime() == fg[jsub].validTime()); - timeslots[jsub] = xb[jsub].validTime(); - } - - // Read inflation field - std::unique_ptr inflationField; - if (conf.has("inflation field")) { - const eckit::LocalConfiguration inflationConfig(conf, "inflation field"); - inflationField.reset(new Increment4D_(resol, vars, timeslots)); - inflationField->read(inflationConfig); - } - - // Get inflation value - double inflationValue = 1; - if (conf.has("inflation value")) { - conf.get("inflation value", inflationValue); - } - - // Setup change of variable - ChvarVec_ chvars; - if (conf.has("variable changes")) { - std::vector chvarconfs; - conf.get("variable changes", chvarconfs); - for (const auto & conf : chvarconfs) { - chvars.push_back(LinearVariableChangeFactory::create(xb[0], fg[0], resol, conf)); - } - } - // TODO(Benjamin): one change of variable for each timeslot - - // Read ensemble - StateEnsemble4D_ ensemble(resol, conf); - State4D_ bgmean = ensemble.mean(); - - ensemblePerturbs_.reserve(ensemble.size()); - for (unsigned int ie = 0; ie < ensemble.size(); ++ie) { - // Ensemble will be centered around ensemble mean - Increment4D_ dx(resol, vars_, timeslots); - dx.diff(ensemble[ie], bgmean); - - // Apply inflation - if (conf.has("inflation field")) { - dx.schur_product_with(*inflationField); - } - dx *= inflationValue; - - // Apply inverse of the linear balance operator - for (unsigned jsub = 0; jsub < timeslots.size(); ++jsub) { - // K_1^{-1} K_2^{-1} .. K_N^{-1} - for (ircst_ it = chvars.rbegin(); it != chvars.rend(); ++it) { - dx[jsub] = it->multiplyInverse(dx[jsub]); - } - } - - ensemblePerturbs_.emplace_back(std::move(dx)); - } - Log::trace() << "IncrementEnsemble4D:contructor done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -void IncrementEnsemble4D::releaseMember() { - ensemblePerturbs_.erase(ensemblePerturbs_.begin()); -} - -// ----------------------------------------------------------------------------- - -template -void IncrementEnsemble4D::resetMember(const Increment4D_ & dx) { - ensemblePerturbs_.emplace_back(dx); -} - -// ----------------------------------------------------------------------------- - } // namespace oops #endif // OOPS_BASE_INCREMENTENSEMBLE4D_H_ From 4ad4e4d58629724a577434e69a5b608a064ce34c Mon Sep 17 00:00:00 2001 From: Ryan Honeyager Date: Wed, 30 Dec 2020 17:01:28 -0500 Subject: [PATCH 029/142] Update ParameterTraitsVariables.cc (#1011) Co-authored-by: Anna Shlyaeva --- src/oops/base/ParameterTraitsVariables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/oops/base/ParameterTraitsVariables.cc b/src/oops/base/ParameterTraitsVariables.cc index 98f13a5df..5bb517d8d 100644 --- a/src/oops/base/ParameterTraitsVariables.cc +++ b/src/oops/base/ParameterTraitsVariables.cc @@ -39,7 +39,7 @@ std::vector getVariableNamesWithoutChannelSuffix(const Variables &v const char channelSeparator = '_'; std::vector uniqueBaseNames; std::map channelsPerVariable; - for (const std::string name : variables.variables()) { + for (const std::string& name : variables.variables()) { const std::string::size_type separatorPosition = name.find_last_of(channelSeparator); if (separatorPosition == std::string::npos) throwException(); // no channel suffix From 373da3ec36d31c64e948ee6a16c0f63dad8d2a89 Mon Sep 17 00:00:00 2001 From: Claude Gibert Date: Mon, 4 Jan 2021 15:44:27 +0000 Subject: [PATCH 030/142] small changes to make all ewok experiments work. (#1015) * small changes to make all ewok experiement work. * removed model from hovx3d.yaml --- ewok/forecast.yaml | 2 +- ewok/hofx.yaml | 12 ------------ ewok/hofx3d.yaml | 9 +++++---- 3 files changed, 6 insertions(+), 17 deletions(-) delete mode 100644 ewok/hofx.yaml diff --git a/ewok/forecast.yaml b/ewok/forecast.yaml index b0246d3b3..ca7450e7b 100644 --- a/ewok/forecast.yaml +++ b/ewok/forecast.yaml @@ -1,6 +1,6 @@ forecast length: $(forecast_length) initial condition: - $(AN_TEMPLATE) + $(FC_INPUT) geometry: $(GEOMETRY) model: diff --git a/ewok/hofx.yaml b/ewok/hofx.yaml deleted file mode 100644 index bc128b8cd..000000000 --- a/ewok/hofx.yaml +++ /dev/null @@ -1,12 +0,0 @@ -forecast length: - $(forecast_length) -geometry: - $(GEOMETRY) -initial condition: - $(AN_TEMPLATE) -model: - $(MODEL) -observations: - $(OBSERVATIONS) -window begin: $(window begin) -window length: $(window length) diff --git a/ewok/hofx3d.yaml b/ewok/hofx3d.yaml index 411436437..3f85a2b2f 100644 --- a/ewok/hofx3d.yaml +++ b/ewok/hofx3d.yaml @@ -1,9 +1,10 @@ -window begin: '{% current_cycle + $(window_offset) - $(window_length) %}' -window length: $(window_length) +forecast length: + $(forecast_length) geometry: $(GEOMETRY) forecasts: - states: - - $(BACKGROUND) + $(BACKGROUND) observations: $(OBSERVATIONS) +window begin: '{{window_begin}}' +window length: $(window length) From 149dc03862035f4df33c370fc2c7be34909774c7 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Mon, 4 Jan 2021 09:14:35 -0700 Subject: [PATCH 031/142] remove useless Geometry copyctor test; add State::geometry test (#982) * remove useless Geometry copyctor test; add State::geometry test * bugfix for Geometry print test --- src/test/interface/Geometry.h | 23 +---------------------- src/test/interface/State.h | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/test/interface/Geometry.h b/src/test/interface/Geometry.h index dd52be533..9f1af202a 100644 --- a/src/test/interface/Geometry.h +++ b/src/test/interface/Geometry.h @@ -17,14 +17,12 @@ #define ECKIT_TESTING_SELF_REGISTER_CASES 0 -#include - - #include "eckit/config/Configuration.h" #include "eckit/testing/Test.h" #include "oops/interface/Geometry.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" +#include "oops/util/Logger.h" #include "test/interface/GeometryFixture.h" #include "test/TestEnvironment.h" @@ -37,29 +35,12 @@ template void testConstructor() { std::unique_ptr geom(new Geometry_(GeometryFixture::getParameters(), oops::mpi::world(), oops::mpi::myself())); - EXPECT(geom.get()); oops::Log::test() << "Testing geometry: " << *geom << std::endl; geom.reset(); EXPECT(!geom.get()); } -// ----------------------------------------------------------------------------- -template void testCopyConstructor() { - typedef oops::Geometry Geometry_; - std::unique_ptr geom(new Geometry_(GeometryFixture::getParameters(), - oops::mpi::world(), oops::mpi::myself())); - - - std::unique_ptr other(new Geometry_(*geom)); - EXPECT(other.get()); - - other.reset(); - EXPECT(!other.get()); - - EXPECT(geom.get()); -} - // ----------------------------------------------------------------------------- template class Geometry : public oops::Test { public: @@ -73,8 +54,6 @@ template class Geometry : public oops::Test { ts.emplace_back(CASE("interface/Geometry/testConstructor") { testConstructor(); }); - ts.emplace_back(CASE("interface/Geometry/testCopyConstructor") - { testCopyConstructor(); }); } void clear() const override {} diff --git a/src/test/interface/State.h b/src/test/interface/State.h index 66ad09166..7a637c3e5 100644 --- a/src/test/interface/State.h +++ b/src/test/interface/State.h @@ -118,6 +118,30 @@ template void testStateConstructors() { EXPECT(xx5.variables() == xx1->variables()); } +// ----------------------------------------------------------------------------- +/*! \brief Tests State::geometry() and Geometry copy constructors + */ + +template void testStateGeometry() { + typedef StateFixture Test_; + typedef oops::Geometry Geometry_; + typedef oops::State State_; + + const double norm = Test_::test().getDouble("norm file"); + const double tol = Test_::test().getDouble("tolerance"); + const util::DateTime vt(Test_::test().getString("date")); + + const eckit::LocalConfiguration conf(Test_::test(), "statefile"); + State_ xx1(Test_::resol(), conf); + + // get geometry from xx1 and initialize xx2 (xx2 & xx1 should be the same) + const Geometry_ & geometry = xx1.geometry(); + State_ xx2(geometry, conf); + + const double norm2 = xx2.norm(); + EXPECT(oops::is_close(norm2, norm, tol)); +} + // ----------------------------------------------------------------------------- /*! \brief Interpolation test @@ -306,6 +330,7 @@ class State : public oops::Test { public: State() {} virtual ~State() {} + private: std::string testid() const override {return "test::State<" + MODEL::name() + ">";} @@ -314,6 +339,8 @@ class State : public oops::Test { ts.emplace_back(CASE("interface/State/testStateConstructors") { testStateConstructors(); }); + ts.emplace_back(CASE("interface/State/testStateGeometry") + { testStateGeometry(); }); ts.emplace_back(CASE("interface/State/testStateAnalyticInitialCondition") { testStateAnalyticInitialCondition(); }); ts.emplace_back(CASE("interface/State/testStateZeroAndAccumul") From c4b4cccb35fef66254cb20e0691f799ceee49730 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Mon, 4 Jan 2021 10:02:34 -0700 Subject: [PATCH 032/142] Add zero-obs LETKF test (#1012) * add zero-obs LETKF test * output analysis mean to Log::test for comparison --- l95/test/CMakeLists.txt | 8 ++++ l95/test/testinput/letkf_noobs.yaml | 44 ++++++++++++++++++ l95/test/testoutput/getkf.test | 3 ++ l95/test/testoutput/getkf_offline_hofx.test | 3 ++ l95/test/testoutput/letkf.test | 3 ++ l95/test/testoutput/letkf_gsi.test | 3 ++ l95/test/testoutput/letkf_noobs.test | 51 +++++++++++++++++++++ l95/test/testoutput/letkf_qc.test | 3 ++ qg/test/testoutput/letkf.test | 22 +++++++++ src/oops/base/Departures.h | 4 +- src/oops/runs/LocalEnsembleDA.h | 2 +- 11 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 l95/test/testinput/letkf_noobs.yaml create mode 100644 l95/test/testoutput/letkf_noobs.test diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index f26229b04..1d26c6409 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -67,6 +67,7 @@ list( APPEND l95_test_input testinput/identitymodel.yaml testinput/letkfGSI.yaml testinput/letkf.yaml + testinput/letkf_noobs.yaml testinput/letkf_qc.yaml testinput/interfaces.yaml testinput/linearmodelfactory.yaml @@ -150,6 +151,7 @@ list( APPEND l95_testoutput testoutput/hofx.nomodel.test testoutput/hofx_for_getkf_nomodel.test testoutput/letkf.test + testoutput/letkf_noobs.test testoutput/letkf_qc.test testoutput/letkf_gsi.test testoutput/localhofx.test @@ -737,6 +739,12 @@ oops_add_test( TESTNAME letkf EXENAME l95_letkf.x TEST_DEPENDS test_l95_makeobs3d test_l95_genenspert ) +oops_add_test( TESTNAME letkf_noobs + MODELNAME l95 + YAMLNAME testinput/letkf_noobs.yaml + EXENAME l95_letkf.x + TEST_DEPENDS test_l95_forecast test_l95_genenspert ) + oops_add_test( TESTNAME letkf_qc MODELNAME l95 YAMLNAME testinput/letkf_qc.yaml diff --git a/l95/test/testinput/letkf_noobs.yaml b/l95/test/testinput/letkf_noobs.yaml new file mode 100644 index 000000000..fb9646a67 --- /dev/null +++ b/l95/test/testinput/letkf_noobs.yaml @@ -0,0 +1,44 @@ +window begin: 2010-01-01T21:00:00Z +window length: PT6H + +geometry: + resol: 40 + +# use 3D for middle of the window +background: + members: + - date: &date 2010-01-02T00:00:00Z + filename: Data/test.ens.1.2010-01-01T00:00:00Z.P1D + - date: *date + filename: Data/test.ens.2.2010-01-01T00:00:00Z.P1D + - date: *date + filename: Data/test.ens.3.2010-01-01T00:00:00Z.P1D + - date: *date + filename: Data/test.ens.4.2010-01-01T00:00:00Z.P1D + - date: *date + filename: Data/test.ens.5.2010-01-01T00:00:00Z.P1D + +observations: +- obs error: + covariance model: localized diagonal + localization: + localization method: Gaspari-Cohn + lengthscale: .1 + obs space: + obsdatain: Data/noobs.obt + obs operator: {} + +driver: + +local ensemble DA: + solver: LETKF + inflation: + rtps: 0.5 + rtpp: 0.5 + mult: 1.1 + +output: + datadir: Data + date: *date + exp: letkf.%{member}% + type: an diff --git a/l95/test/testoutput/getkf.test b/l95/test/testoutput/getkf.test index 1fcf61a01..7c4f417d3 100644 --- a/l95/test/testoutput/getkf.test +++ b/l95/test/testoutput/getkf.test @@ -40,6 +40,9 @@ Test : Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 Test : Background mean : Test : Valid time: 2010-01-02T00:00:00Z Test : Min=5.25199, Max=9.50208, Average=7.76569 +Test : Analysis mean : +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=6.73405, Max=9.18815, Average=7.98704 Test : H(x) for member 1: Test : Lorenz 95 nobs= 120 Min=6.77434, Max=9.33536, Average=8.03454 Test : H(x) for member 2: diff --git a/l95/test/testoutput/getkf_offline_hofx.test b/l95/test/testoutput/getkf_offline_hofx.test index 4ad12045e..82f81b96c 100644 --- a/l95/test/testoutput/getkf_offline_hofx.test +++ b/l95/test/testoutput/getkf_offline_hofx.test @@ -40,3 +40,6 @@ Test : Lorenz 95 nobs= 120 Min=-1.48378, Max=2.74405, Average=0.2457 Test : Background mean : Test : Valid time: 2010-01-02T00:00:00Z Test : Min=5.25199, Max=9.50208, Average=7.76569 +Test : Analysis mean : +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=6.73405, Max=9.18814, Average=7.98704 diff --git a/l95/test/testoutput/letkf.test b/l95/test/testoutput/letkf.test index d515b3f39..ecf5c9560 100644 --- a/l95/test/testoutput/letkf.test +++ b/l95/test/testoutput/letkf.test @@ -30,6 +30,9 @@ Test : Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 Test : Background mean : Test : Valid time: 2010-01-02T00:00:00Z Test : Min=5.25199, Max=9.50208, Average=7.76569 +Test : Analysis mean : +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=6.73044, Max=9.19002, Average=7.98832 Test : H(x) for member 1: Test : Lorenz 95 nobs= 120 Min=6.31712, Max=10.1291, Average=8.09746 Test : H(x) for member 2: diff --git a/l95/test/testoutput/letkf_gsi.test b/l95/test/testoutput/letkf_gsi.test index 28dc3c267..f8ef4eaa2 100644 --- a/l95/test/testoutput/letkf_gsi.test +++ b/l95/test/testoutput/letkf_gsi.test @@ -30,6 +30,9 @@ Test : Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 Test : Background mean : Test : Valid time: 2010-01-02T00:00:00Z Test : Min=5.25199, Max=9.50208, Average=7.76569 +Test : Analysis mean : +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=6.73405, Max=9.18815, Average=7.98704 Test : H(x) for member 1: Test : Lorenz 95 nobs= 120 Min=6.30807, Max=10.1297, Average=8.0962 Test : H(x) for member 2: diff --git a/l95/test/testoutput/letkf_noobs.test b/l95/test/testoutput/letkf_noobs.test new file mode 100644 index 000000000..a578482e7 --- /dev/null +++ b/l95/test/testoutput/letkf_noobs.test @@ -0,0 +1,51 @@ +Test : Initial state for member 1: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=3.53609, Max=11.1974, Average=7.90978 +Test : Initial state for member 2: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=3.77507, Max=11.1844, Average=7.75047 +Test : Initial state for member 3: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=5.36882, Max=10.844, Average=7.80576 +Test : Initial state for member 4: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=4.14639, Max=11.6734, Average=7.76001 +Test : Initial state for member 5: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=3.17563, Max=10.8183, Average=7.60243 +Test : H(x) for member 1: +Test : Lorenz 95 : No observations +Test : H(x) for member 2: +Test : Lorenz 95 : No observations +Test : H(x) for member 3: +Test : Lorenz 95 : No observations +Test : H(x) for member 4: +Test : Lorenz 95 : No observations +Test : H(x) for member 5: +Test : Lorenz 95 : No observations +Test : H(x) ensemble background mean: +Test : Lorenz 95 : No observations +Test : background y - H(x): +Test : Lorenz 95 : No observations +Test : Background mean : +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=5.25199, Max=9.50208, Average=7.76569 +Test : Analysis mean : +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=5.25199, Max=9.50208, Average=7.76569 +Test : H(x) for member 1: +Test : Lorenz 95 : No observations +Test : H(x) for member 2: +Test : Lorenz 95 : No observations +Test : H(x) for member 3: +Test : Lorenz 95 : No observations +Test : H(x) for member 4: +Test : Lorenz 95 : No observations +Test : H(x) for member 5: +Test : Lorenz 95 : No observations +Test : H(x) ensemble analysis mean: +Test : Lorenz 95 : No observations +Test : analysis y - H(x): +Test : Lorenz 95 : No observations +Test : ombg RMS: 0 +Test : oman RMS: 0 diff --git a/l95/test/testoutput/letkf_qc.test b/l95/test/testoutput/letkf_qc.test index 3b9b72a5a..53c68532e 100644 --- a/l95/test/testoutput/letkf_qc.test +++ b/l95/test/testoutput/letkf_qc.test @@ -30,6 +30,9 @@ Test : Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 Test : Background mean : Test : Valid time: 2010-01-02T00:00:00Z Test : Min=5.25199, Max=9.50208, Average=7.76569 +Test : Analysis mean : +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=6.73044, Max=9.19002, Average=7.98343 Test : H(x) for member 1: Test : Lorenz 95 nobs= 120 Min=6.31712, Max=10.1291, Average=8.09238 Test : H(x) for member 2: diff --git a/qg/test/testoutput/letkf.test b/qg/test/testoutput/letkf.test index bfc03147e..ff1dc2551 100644 --- a/qg/test/testoutput/letkf.test +++ b/qg/test/testoutput/letkf.test @@ -158,6 +158,28 @@ Test : Boundary conditions are activated Test : Scaling= 1e+08, Min= -4.3566, Max= 0.9538, RMS= 1.7157 Test : Scaling= 1e+08, Min= -4.0032, Max= 0.0000, RMS= 2.0632 Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 +Test : Analysis mean : +Test : Valid time: 2010-01-01T00:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are activated +Test : Scaling= 1e+08, Min= -4.9026, Max= 0.9586, RMS= 1.8792 +Test : Scaling= 1e+08, Min= -4.0032, Max= 0.0000, RMS= 2.0632 +Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 +Test : Valid time: 2010-01-01T06:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are activated +Test : Scaling= 1e+08, Min= -4.6092, Max= 0.9136, RMS= 1.8251 +Test : Scaling= 1e+08, Min= -4.0032, Max= 0.0000, RMS= 2.0632 +Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are activated +Test : Scaling= 1e+08, Min= -4.4655, Max= 0.9637, RMS= 1.7841 +Test : Scaling= 1e+08, Min= -4.0032, Max= 0.0000, RMS= 2.0632 +Test : Scaling= 0.001, Min= -0.6729, Max= 0.5790, RMS= 0.4464 Test : H(x) for member 1: Test : Stream nobs= 300 Scaling= 1e+08, Min= -5.4486, Max= 0.9183, Average= -1.2800 Test : Wind nobs= 160 Scaling= 10, Min= -9.6559, Max= 10.8809, Average= 1.1563 diff --git a/src/oops/base/Departures.h b/src/oops/base/Departures.h index ed6f62240..8396a3bf6 100644 --- a/src/oops/base/Departures.h +++ b/src/oops/base/Departures.h @@ -193,7 +193,9 @@ double Departures::dot_product_with(const Departures & other) const { // ----------------------------------------------------------------------------- template double Departures::rms() const { - return sqrt(dot_product_with(*this) / this->nobs()); + double zz = 0.0; + if (nobs() > 0) zz = sqrt(dot_product_with(*this) / this->nobs()); + return zz; } // ----------------------------------------------------------------------------- template diff --git a/src/oops/runs/LocalEnsembleDA.h b/src/oops/runs/LocalEnsembleDA.h index c4ca3529e..44c7c3633 100644 --- a/src/oops/runs/LocalEnsembleDA.h +++ b/src/oops/runs/LocalEnsembleDA.h @@ -138,7 +138,7 @@ template class LocalEnsembleDA : public Applicati // save the analysis mean State4D_ ana_mean = ens_xx.mean(); // calculate analysis mean - Log::info() << "Analysis mean :" << ana_mean << std::endl; + Log::test() << "Analysis mean :" << ana_mean << std::endl; eckit::LocalConfiguration outConfig(fullConfig, "output"); outConfig.set("member", 0); ana_mean.write(outConfig); From b9130de1b39ce710834c74c48ed7b6185dc68199 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Tue, 5 Jan 2021 10:52:11 -0700 Subject: [PATCH 033/142] add a test for ObsSpace on subwindows (#1010) --- qg/model/ObsSpaceQG.cc | 1 + src/test/interface/ObsSpace.h | 36 +++++++++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/qg/model/ObsSpaceQG.cc b/qg/model/ObsSpaceQG.cc index acfeea566..de3d1904e 100644 --- a/qg/model/ObsSpaceQG.cc +++ b/qg/model/ObsSpaceQG.cc @@ -69,6 +69,7 @@ ObsSpaceQG::ObsSpaceQG(const eckit::Configuration & config, const eckit::mpi::Co ABORT("Underspecified observation files."); } + ref = ref + bgn.toString() + end.toString(); otiter it = theObsFileRegister_.find(ref); if ( it == theObsFileRegister_.end() ) { // Open new file diff --git a/src/test/interface/ObsSpace.h b/src/test/interface/ObsSpace.h index 1823be9d5..28cd2cf08 100644 --- a/src/test/interface/ObsSpace.h +++ b/src/test/interface/ObsSpace.h @@ -1,6 +1,6 @@ /* * (C) Copyright 2009-2016 ECMWF. - * (C) Copyright 2017-2019 UCAR. + * (C) Copyright 2017-2020 UCAR. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -22,14 +22,17 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/testing/Test.h" #include "oops/interface/ObsSpace.h" +#include "oops/interface/ObsVector.h" #include "oops/runs/Test.h" +#include "oops/util/DateTime.h" +#include "oops/util/Duration.h" #include "test/interface/ObsTestsFixture.h" #include "test/TestEnvironment.h" namespace test { // ----------------------------------------------------------------------------- -/// \brief tests constructor, window accessor methods and print +/// \brief tests constructor, window accessor methods and prints template void testConstructor() { typedef ObsTestsFixture Test_; @@ -40,6 +43,33 @@ template void testConstructor() { } } +// ----------------------------------------------------------------------------- +/// \brief tests that ObsSpaces created on subwindows have the same number obs as +/// ObsSpaces created on the whole window +template void testSubwindows() { + typedef ObsTestsFixture Test_; + typedef oops::ObsSpace ObsSpace_; + typedef oops::ObsVector ObsVector_; + + util::DateTime tmid = Test_::tbgn() + (Test_::tend()-Test_::tbgn())/2; + oops::Log::test() << "Testing subwindows: " << Test_::tbgn() << " to " << tmid << " and " + << tmid << " to " << Test_::tend() << std::endl; + for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + eckit::LocalConfiguration obsconfig(Test_::config(jj), "obs space"); + ObsSpace_ obspace1(obsconfig, oops::mpi::world(), Test_::tbgn(), tmid); + ObsSpace_ obspace2(obsconfig, oops::mpi::world(), tmid, Test_::tend()); + + /// Create ObsVectors for each of the ObsSpaces, to compare nobs + ObsVector_ ovec(Test_::obspace()[jj]); + ObsVector_ ovec1(obspace1); + ObsVector_ ovec2(obspace2); + oops::Log::test() << Test_::obspace()[jj].obsname() << " nobs(all): " << ovec.nobs() + << " nobs(1st subwindow): " << ovec1.nobs() + << " nobs(2nd subwindow): " << ovec2.nobs() << std::endl; + EXPECT_EQUAL(ovec1.nobs() + ovec2.nobs(), ovec.nobs()); + } +} + // ----------------------------------------------------------------------------- template class ObsSpace : public oops::Test { @@ -55,6 +85,8 @@ template class ObsSpace : public oops::Test { ts.emplace_back(CASE("interface/ObsSpace/testConstructor") { testConstructor(); }); + ts.emplace_back(CASE("interface/ObsSpace/testSubwindows") + { testSubwindows(); }); } void clear() const override { From 20c0aa2369c8998b24ed1bec2da88a2405d1e293 Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Fri, 8 Jan 2021 20:57:26 +0000 Subject: [PATCH 034/142] size_t -> int (#1019) --- src/oops/assimilation/Increment4D.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/oops/assimilation/Increment4D.h b/src/oops/assimilation/Increment4D.h index 61e9ce21f..00c5b686e 100644 --- a/src/oops/assimilation/Increment4D.h +++ b/src/oops/assimilation/Increment4D.h @@ -81,7 +81,7 @@ Increment4D::Increment4D(const Geometry_ & resol, const std::vector & timeslots) : incr4d_() { - for (int jtime = 0; jtime < timeslots.size(); ++jtime) { + for (size_t jtime = 0; jtime < timeslots.size(); ++jtime) { incr4d_.emplace_back(resol, vars, timeslots[jtime]); } Log::trace() << "Increment4D:Increment4D created." << std::endl; From c08f6d8da04f53a5a536c45d7d9602775f6b8e7c Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Sat, 9 Jan 2021 09:26:42 -0700 Subject: [PATCH 035/142] make codecov a little more chill (thank you Travis for the idea) (#1021) --- .codecov.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.codecov.yml b/.codecov.yml index 33d203f67..7e69da931 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -6,6 +6,13 @@ coverage: precision: 2 round: down range: "70...100" + status: + project: + default: + threshold: 0.05% + patch: + default: + threshold: 2% ignore: - "src/test" From a47cc20415e3ec06693446f49ac1f913ee6763b4 Mon Sep 17 00:00:00 2001 From: KrissyRayD242 <59601469+KrissyRayD242@users.noreply.github.com> Date: Tue, 12 Jan 2021 16:06:47 +0000 Subject: [PATCH 036/142] add missing header (#1025) --- src/oops/util/CompareNVectors.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/oops/util/CompareNVectors.h b/src/oops/util/CompareNVectors.h index 753c457d6..860788da2 100644 --- a/src/oops/util/CompareNVectors.h +++ b/src/oops/util/CompareNVectors.h @@ -8,6 +8,7 @@ #ifndef OOPS_UTIL_COMPARENVECTORS_H_ #define OOPS_UTIL_COMPARENVECTORS_H_ +#include #include namespace oops { From 1cdee0b995bf2d405b8bd90998db0fd6f76e0494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Fri, 15 Jan 2021 00:11:54 +0000 Subject: [PATCH 037/142] Facilitate local (scoped) changes to environment variables (#1028) * Added a LocalEnvironment class to make it possible to temporarily modify env variables and restore them to original values afterwards. * Added a test verifying env variables are restored even if an exception occurs. --- src/CMakeLists.txt | 8 +++ src/oops/util/LocalEnvironment.cc | 37 ++++++++++++++ src/oops/util/LocalEnvironment.h | 43 ++++++++++++++++ src/test/util/LocalEnvironment.cc | 15 ++++++ src/test/util/LocalEnvironment.h | 84 +++++++++++++++++++++++++++++++ 5 files changed, 187 insertions(+) create mode 100644 src/oops/util/LocalEnvironment.cc create mode 100644 src/oops/util/LocalEnvironment.h create mode 100644 src/test/util/LocalEnvironment.cc create mode 100644 src/test/util/LocalEnvironment.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 48757a18f..590a16cd2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -272,6 +272,8 @@ oops/util/liboops_f.h oops/util/liboops_mod.F90 oops/util/LibOOPS.cc oops/util/LibOOPS.h +oops/util/LocalEnvironment.cc +oops/util/LocalEnvironment.h oops/util/Logger.h oops/util/missing_values_f.cc oops/util/missing_values_f.h @@ -399,6 +401,7 @@ test/util/CompositePath.h test/util/MissingValues.h test/util/PropertiesOfNVectors.h test/util/stringFunctions.h +test/util/LocalEnvironment.h ) list (APPEND oops_fheader_files @@ -630,6 +633,11 @@ ecbuild_add_test( TARGET test_util_propertiesofnvectors ARGS "test/testinput/empty.yaml" LIBS oops ) +ecbuild_add_test( TARGET test_util_localenvironment + SOURCES test/util/LocalEnvironment.cc + ARGS "test/testinput/empty.yaml" + LIBS oops ) + ecbuild_add_test( TARGET test_assimilation_fullgmres SOURCES test/assimilation/FullGMRES.cc ARGS "test/testinput/empty.yaml" diff --git a/src/oops/util/LocalEnvironment.cc b/src/oops/util/LocalEnvironment.cc new file mode 100644 index 000000000..75a04d2fd --- /dev/null +++ b/src/oops/util/LocalEnvironment.cc @@ -0,0 +1,37 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/util/LocalEnvironment.h" + +#include + +namespace util { + +LocalEnvironment::~LocalEnvironment() { + for (const auto &nameAndValue : variableNamesAndOriginalValues_) + setenv(nameAndValue.first.c_str(), nameAndValue.second.c_str(), 1 /*overwrite*/); + for (const auto &name : originallyUnsetVariables_) + unsetenv(name.c_str()); +} + +void LocalEnvironment::set(const std::string &variableName, const std::string &value) { + const bool originalValueAlreadyRecorded = + variableNamesAndOriginalValues_.find(variableName) != variableNamesAndOriginalValues_.end() || + originallyUnsetVariables_.find(variableName) != originallyUnsetVariables_.end(); + + if (!originalValueAlreadyRecorded) { + char* originalValue = getenv(variableName.c_str()); + if (originalValue != nullptr) + variableNamesAndOriginalValues_[variableName] = originalValue; + else + originallyUnsetVariables_.insert(variableName); + } + + setenv(variableName.c_str(), value.c_str(), 1 /*overwrite*/); +} + +} // namespace util diff --git a/src/oops/util/LocalEnvironment.h b/src/oops/util/LocalEnvironment.h new file mode 100644 index 000000000..dcfb2bd49 --- /dev/null +++ b/src/oops/util/LocalEnvironment.h @@ -0,0 +1,43 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_UTIL_LOCALENVIRONMENT_H_ +#define OOPS_UTIL_LOCALENVIRONMENT_H_ + +#include +#include +#include +#include + +namespace util { + +/// \brief Change environment variables, restoring their original values on destruction. +class LocalEnvironment { + public: + /// \brief Constructor. + LocalEnvironment() = default; + + LocalEnvironment(const LocalEnvironment &) = delete; + LocalEnvironment(LocalEnvironment &&) = delete; + LocalEnvironment& operator=(const LocalEnvironment &) = delete; + LocalEnvironment& operator=(LocalEnvironment &&) = delete; + + /// \brief Restore all environment variables changed by the set() function + /// to their original values. + ~LocalEnvironment(); + + /// Set the environment variable \p variableName to \p value. + void set(const std::string &variableName, const std::string &value); + + private: + std::map variableNamesAndOriginalValues_; + std::set originallyUnsetVariables_; +}; + +} // namespace util + +#endif // OOPS_UTIL_LOCALENVIRONMENT_H_ diff --git a/src/test/util/LocalEnvironment.cc b/src/test/util/LocalEnvironment.cc new file mode 100644 index 000000000..50f0cb2ed --- /dev/null +++ b/src/test/util/LocalEnvironment.cc @@ -0,0 +1,15 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/runs/Run.h" +#include "test/util/LocalEnvironment.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + test::LocalEnvironment tests; + return run.execute(tests); +} diff --git a/src/test/util/LocalEnvironment.h b/src/test/util/LocalEnvironment.h new file mode 100644 index 000000000..d78b41e36 --- /dev/null +++ b/src/test/util/LocalEnvironment.h @@ -0,0 +1,84 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef TEST_UTIL_LOCALENVIRONMENT_H_ +#define TEST_UTIL_LOCALENVIRONMENT_H_ + +#include + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/testing/Test.h" +#include "oops/../test/TestEnvironment.h" +#include "oops/mpi/mpi.h" +#include "oops/runs/Test.h" +#include "oops/util/LocalEnvironment.h" + +namespace test { + +CASE("util/LocalEnvironment/set") { + setenv("SOME_VARIABLE", "abcdef", 1 /*replace?*/); + setenv("ANOTHER_VARIABLE", "XYZ", 1 /*replace?*/); + + { + util::LocalEnvironment localEnv; + + localEnv.set("SOME_VARIABLE", "1234"); + EXPECT_EQUAL(std::string(::getenv("SOME_VARIABLE")), "1234"); + localEnv.set("NEW_VARIABLE", "PQR"); + EXPECT_EQUAL(std::string(::getenv("NEW_VARIABLE")), "PQR"); + + localEnv.set("SOME_VARIABLE", "5678"); + EXPECT_EQUAL(std::string(::getenv("SOME_VARIABLE")), "5678"); + localEnv.set("NEW_VARIABLE", "ijk"); + EXPECT_EQUAL(std::string(::getenv("NEW_VARIABLE")), "ijk"); + + // Sanity check: ANOTHER_VARIABLE should still be set to its original value + EXPECT_EQUAL(std::string(::getenv("ANOTHER_VARIABLE")), "XYZ"); + } + + // SOME_VARIABLE should now be restored to its original value and NEW_VARIABLE unset + EXPECT_EQUAL(std::string(::getenv("SOME_VARIABLE")), "abcdef"); + EXPECT(::getenv("NEW_VARIABLE") == nullptr); + + // Sanity check: ANOTHER_VARIABLE should still be set to its original value + EXPECT_EQUAL(std::string(::getenv("ANOTHER_VARIABLE")), "XYZ"); +} + +CASE("util/LocalEnvironment/exceptionSafety") { + setenv("SOME_VARIABLE", "abcdef", 1 /*replace?*/); + setenv("ANOTHER_VARIABLE", "XYZ", 1 /*replace?*/); + + try { + util::LocalEnvironment localEnv; + + localEnv.set("SOME_VARIABLE", "1234"); + EXPECT_EQUAL(std::string(::getenv("SOME_VARIABLE")), "1234"); + localEnv.set("NEW_VARIABLE", "PQR"); + EXPECT_EQUAL(std::string(::getenv("NEW_VARIABLE")), "PQR"); + + throw std::runtime_error("test"); + } catch (std::runtime_error &) { + // SOME_VARIABLE should now be restored to its original value and NEW_VARIABLE unset + EXPECT_EQUAL(std::string(::getenv("SOME_VARIABLE")), "abcdef"); + EXPECT(::getenv("NEW_VARIABLE") == nullptr); + + // Sanity check: ANOTHER_VARIABLE should still be set to its original value + EXPECT_EQUAL(std::string(::getenv("ANOTHER_VARIABLE")), "XYZ"); + } +} + +class LocalEnvironment : public oops::Test { + private: + std::string testid() const override {return "test::LocalEnvironment";} + + void register_tests() const override {} + void clear() const override {} +}; + +} // namespace test + +#endif // TEST_UTIL_LOCALENVIRONMENT_H_ From cf0b6092be034ffacb622688b298c84198ab7424 Mon Sep 17 00:00:00 2001 From: Mark Miesch Date: Fri, 15 Jan 2021 08:49:39 -0700 Subject: [PATCH 038/142] uninitialized variable (#1032) * uninitialized variable * clang container is update Co-authored-by: Maryam Abdi-Oskouei --- src/oops/assimilation/DualMinimizer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/oops/assimilation/DualMinimizer.h b/src/oops/assimilation/DualMinimizer.h index 0ebaa2441..9ce0d3e8c 100644 --- a/src/oops/assimilation/DualMinimizer.h +++ b/src/oops/assimilation/DualMinimizer.h @@ -92,7 +92,7 @@ DualMinimizer::doMinimize(const eckit::Configuration & config) { for (unsigned jj = 0; jj < J_.nterms(); ++jj) { vv.append(J_.jterm(jj).newDualVector()); } - double vvp; + double vvp = 0.0; // Get R^{-1} d Dual_ rr; From 1d86b758da92e239b75b34f294286c4942847c8f Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Fri, 15 Jan 2021 09:56:47 -0700 Subject: [PATCH 039/142] bugfix abs->std::abs + forgotten override (#1034) Co-authored-by: Maryam Abdi --- src/oops/assimilation/FullGMRES.h | 2 +- src/test/util/Random.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/oops/assimilation/FullGMRES.h b/src/oops/assimilation/FullGMRES.h index b8f7fcf35..781fbee98 100644 --- a/src/oops/assimilation/FullGMRES.h +++ b/src/oops/assimilation/FullGMRES.h @@ -173,7 +173,7 @@ double FullGMRES(VECTOR & xx, const VECTOR & bb, const AMATRIX & A, normReduction = std::abs(ss[jiter+1])/znrm2; Log::info() << "FullGMRES end of iteration " << jiter+1 << std::endl; - printNormReduction(jiter+1, abs(ss[jiter+1]), normReduction); + printNormReduction(jiter+1, std::abs(ss[jiter+1]), normReduction); if (normReduction <= tolerance) { Log::info() << "FullGMRES: Achieved required reduction in presidual norm." << std::endl; diff --git a/src/test/util/Random.h b/src/test/util/Random.h index e15373e06..38d3092da 100644 --- a/src/test/util/Random.h +++ b/src/test/util/Random.h @@ -145,9 +145,9 @@ class Random : public oops::Test { virtual ~Random() {} private: - std::string testid() const {return "test::Random";} + std::string testid() const override {return "test::Random";} - void register_tests() const { + void register_tests() const override { std::vector& ts = eckit::testing::specification(); ts.emplace_back(CASE("util/Random/testCppRandom") From bfe6e78e353156ac0808459eacd08b8fd8ed0ea2 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Tue, 19 Jan 2021 14:51:11 -0700 Subject: [PATCH 040/142] add subclass + comments to Increment class (#1006) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add subclass + comments to Increment class * add dots in doxygen comments for line breaks * return resetting the increment and move increment() to interface class Co-authored-by: Yannick Trémolet <30638944+ytremolet@users.noreply.github.com> --- src/oops/interface/Increment.h | 301 ++++++++++++++++++++++----------- 1 file changed, 206 insertions(+), 95 deletions(-) diff --git a/src/oops/interface/Increment.h b/src/oops/interface/Increment.h index 4deeb10bd..bf945d728 100644 --- a/src/oops/interface/Increment.h +++ b/src/oops/interface/Increment.h @@ -1,6 +1,6 @@ /* * (C) Copyright 2009-2016 ECMWF. - * (C) Copyright 2017-2019 UCAR. + * (C) Copyright 2017-2020 UCAR. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -35,120 +35,112 @@ namespace oops { -/// Increment Class: Difference between two states -/*! - * Some fields that are present in a State may not be present in an Increment. - */ - -// ----------------------------------------------------------------------------- +namespace interface { +/// Increment: Difference between two model states. +/// Some fields that are present in a State may not be present in an Increment. template class Increment : public oops::GeneralizedDepartures, public util::Printable, public util::Serializable, private util::ObjectCounter > { typedef typename MODEL::Increment Increment_; - typedef Geometry Geometry_; + typedef oops::Geometry Geometry_; typedef GeometryIterator GeometryIterator_; typedef State State_; public: static const std::string classname() {return "oops::Increment";} -/// Constructor, destructor - Increment(const Geometry_ &, const Variables &, const util::DateTime &); - Increment(const Geometry_ &, const Increment &); + /// Constructor for specified \p geometry, with \p variables, valid on \p date + Increment(const Geometry_ & geometry, const Variables & variables, const util::DateTime & date); + /// Copies \p other increment, changing its resolution to \p geometry + Increment(const Geometry_ & geometry, const Increment & other); + /// Creates Increment with the same geometry and variables as \p other. + /// Copies \p other if \p copy is true, otherwise creates zero increment Increment(const Increment &, const bool copy = true); - virtual ~Increment(); -/// Interfacing - Increment_ & increment() {return *increment_;} - const Increment_ & increment() const {return *increment_;} + /// Destructor (defined explicitly for timing and tracing) + virtual ~Increment(); -/// Interactions with State - void diff(const State_ &, const State_ &); + /// Set this Increment to be difference between \p state1 and \p state2 + void diff(const State_ & state1, const State_ & state2); -/// Time + /// Accessor to the time of this Increment const util::DateTime validTime() const {return increment_->validTime();} + /// Updates this Increment's valid time by \p dt (used in PseudoModel) void updateTime(const util::Duration & dt) {increment_->updateTime(dt);} -/// Linear algebra operators + /// Zero out this Increment void zero(); - void zero(const util::DateTime &); + /// Zero out this Increment and set its date to \p date + void zero(const util::DateTime & date); + /// Set this Increment to ones (used in tests) void ones(); + /// Set Increment according to the configuration (used in Dirac application) void dirac(const eckit::Configuration &); + + /// Assignment operator Increment & operator =(const Increment &); + /// Linear algebra operators Increment & operator+=(const Increment &); Increment & operator-=(const Increment &); Increment & operator*=(const double &); - void axpy(const double &, const Increment &, const bool check = true); - double dot_product_with(const Increment &) const; - void schur_product_with(const Increment &); + /// Add \p w * \p dx to the Increment. If \p check is set, check whether this and \p dx's + /// dates are the same + void axpy(const double & w, const Increment & dx, const bool check = true); + /// Compute dot product of this Increment with \p other + double dot_product_with(const Increment & other) const; + /// Compute Schur product of this Increment with \p other, assign to this Increment + void schur_product_with(const Increment & other); + + /// Randomize the Increment (used in tests) void random(); - void accumul(const double &, const State_ &); + /// Accumulate (add \p w * \p x to the increment), used in WeightedDiff with Accumulator + void accumul(const double & w, const State_ & x); -/// I/O and diagnostics + /// Read this Increment from file void read(const eckit::Configuration &); + /// Write this Increment out to file void write(const eckit::Configuration &) const; + /// Norm (used in tests) double norm() const; + /// Get local (at \p iter local volume) increment (used in LocalEnsembleSolver) LocalIncrement getLocal(const GeometryIterator_ & iter) const; + /// Set local (at \p iter local volume) increment to be \p gp (used in LocalEnsembleSolver) void setLocal(const LocalIncrement & gp, const GeometryIterator_ & iter); -/// Get geometry + /// Accessor to geometry associated with this Increment Geometry_ geometry() const; - const Variables & variables() const {return variables_;} -/// ATLAS FieldSet + /// ATLAS FieldSet (used in SABER) void setAtlas(atlas::FieldSet *) const; void toAtlas(atlas::FieldSet *) const; void fromAtlas(atlas::FieldSet *); -/// ATLAS fieldset - void toAtlas(); - atlas::FieldSet & atlas() { - return atlasFieldSet_; - } - const atlas::FieldSet & atlas() const { - return atlasFieldSet_; - } - -/// Serialize and deserialize + /// Serialize and deserialize (used in 4DEnVar, weak-constraint 4DVar and Block-Lanczos minimizer) size_t serialSize() const override; void serialize(std::vector &) const override; void deserialize(const std::vector &, size_t &) override; - void shift_forward(const util::DateTime &); - void shift_backward(const util::DateTime &); - const eckit::mpi::Comm & timeComm() const {return commTime_;} + /// Interfacing with other oops classes (returns reference to the implementation object) + const Increment_ & increment() const {return *this->increment_;} + Increment_ & increment() {return *this->increment_;} + + protected: + std::unique_ptr increment_; /// pointer to the Increment implementation private: void print(std::ostream &) const override; - std::unique_ptr increment_; - const Variables variables_; - const eckit::mpi::Comm & commTime_; - atlas::FieldSet atlasFieldSet_; }; // ----------------------------------------------------------------------------- -template -State & operator+=(State & xx, const Increment & dx) { - Log::trace() << "operator+=(State, Increment) starting" << std::endl; - util::Timer timer("oops::Increment", "operator+=(State, Increment)"); - xx.state() += dx.increment(); - Log::trace() << "operator+=(State, Increment) done" << std::endl; - return xx; -} - -// ============================================================================= -/// Constructor, destructor -// ----------------------------------------------------------------------------- - template Increment::Increment(const Geometry_ & resol, const Variables & vars, const util::DateTime & time) - : increment_(), variables_(vars), commTime_(resol.timeComm()) + : increment_() { Log::trace() << "Increment::Increment starting" << std::endl; util::Timer timer(classname(), "Increment"); @@ -160,7 +152,7 @@ Increment::Increment(const Geometry_ & resol, const Variables & vars, template Increment::Increment(const Geometry_ & resol, const Increment & other) - : increment_(), variables_(other.variables_), commTime_(other.commTime_) + : increment_() { Log::trace() << "Increment::Increment starting" << std::endl; util::Timer timer(classname(), "Increment"); @@ -172,7 +164,7 @@ Increment::Increment(const Geometry_ & resol, const Increment & other) template Increment::Increment(const Increment & other, const bool copy) - : increment_(), variables_(other.variables_), commTime_(other.commTime_) + : increment_() { Log::trace() << "Increment::Increment copy starting" << std::endl; util::Timer timer(classname(), "Increment"); @@ -301,7 +293,6 @@ double Increment::dot_product_with(const Increment & dx) const { Log::trace() << "Increment::dot_product_with starting" << std::endl; util::Timer timer(classname(), "dot_product_with"); double zz = increment_->dot_product_with(*dx.increment_); - commTime_.allReduceInPlace(zz, eckit::mpi::Operation::SUM); Log::trace() << "Increment::dot_product_with done" << std::endl; return zz; } @@ -384,9 +375,6 @@ double Increment::norm() const { Log::trace() << "Increment::norm starting" << std::endl; util::Timer timer(classname(), "norm"); double zz = increment_->norm(); - zz *= zz; - commTime_.allReduceInPlace(zz, eckit::mpi::Operation::SUM); - zz = sqrt(zz); Log::trace() << "Increment::norm done" << std::endl; return zz; } @@ -394,10 +382,10 @@ double Increment::norm() const { // ----------------------------------------------------------------------------- template -Geometry Increment::geometry() const { +oops::Geometry Increment::geometry() const { Log::trace() << "Increment::geometry starting" << std::endl; util::Timer timer(classname(), "geometry"); - Geometry geom(increment_->geometry()); + oops::Geometry geom(increment_->geometry()); Log::trace() << "Increment::geometry done" << std::endl; return geom; } @@ -434,16 +422,6 @@ void Increment::fromAtlas(atlas::FieldSet * atlasFieldSet) { // ----------------------------------------------------------------------------- -template -void Increment::toAtlas() { - Log::trace() << "Increment::toAtlas starting" << std::endl; - increment_->toAtlas(&atlasFieldSet_); - increment_.reset(); - Log::trace() << "Increment::toAtlas done" << std::endl; -} - -// ----------------------------------------------------------------------------- - template size_t Increment::serialSize() const { Log::trace() << "Increment::serialSize" << std::endl; @@ -473,23 +451,142 @@ void Increment::deserialize(const std::vector & vect, size_t & cu // ----------------------------------------------------------------------------- + +template +void Increment::print(std::ostream & os) const { + Log::trace() << "Increment::print starting" << std::endl; + util::Timer timer(classname(), "print"); + os << *increment_; + Log::trace() << "Increment::print done" << std::endl; +} + +} // namespace interface + +// ----------------------------------------------------------------------------- +/// \brief Increment class used in oops +/// +/// \details +/// Adds extra methods that do not need to be implemented in the implementations: +/// - timeComm() (accessor to the MPI communicator in time - collection of processes +/// holding the data needed to represent the state in a particular region +/// of space X_i and throughout the whole time interval for which DA is done) +/// - variables() (accessor to variables in this Increment) +/// - shift_forward +/// - shift_backward +/// - toAtlas, atlas +/// +/// Adds communication through time to the following Increment methods: +/// - dot_product_with +/// - norm +/// - print + +template +class Increment : public interface::Increment { + typedef Geometry Geometry_; + + public: + /// Constructor for specified \p geometry, with \p variables, valid on \p date + Increment(const Geometry_ & geometry, const Variables & variables, const util::DateTime & date); + /// Copies \p other increment, changing its resolution to \p geometry + Increment(const Geometry_ & geometry, const Increment & other); + /// Creates Increment with the same geometry and variables as \p other. + /// Copies \p other if \p copy is true, otherwise creates zero increment + Increment(const Increment & other, const bool copy = true); + + /// Accessor to the time communicator + const eckit::mpi::Comm & timeComm() const {return *timeComm_;} + /// Accessor to Variables stored in this increment + const Variables & variables() const {return variables_;} + + /// Shift forward in time by \p dt + void shift_forward(const util::DateTime & dt); + /// Shift backward in time by \p dt + void shift_backward(const util::DateTime & dt); + + /// Set ATLAS fieldset associated with this Increment internally + void toAtlas(); + /// Allow to access base class's method as well + using interface::Increment::toAtlas; + /// Accessors to the ATLAS fieldset + atlas::FieldSet & atlas() {return atlasFieldSet_;} + const atlas::FieldSet & atlas() const {return atlasFieldSet_;} + + /// dot product with the \p other increment + double dot_product_with(const Increment & other) const; + /// Norm for diagnostics + double norm() const; + + private: + void print(std::ostream &) const override; + + Variables variables_; /// Variables stored in this Increment + const eckit::mpi::Comm * timeComm_; /// pointer to the MPI communicator in time + atlas::FieldSet atlasFieldSet_; /// Atlas fields associated with this Increment +}; + +// ----------------------------------------------------------------------------- + +template +Increment::Increment(const Geometry_ & geometry, const Variables & variables, + const util::DateTime & date): + interface::Increment(geometry, variables, date), variables_(variables), + timeComm_(&geometry.timeComm()) +{} + +// ----------------------------------------------------------------------------- + +template +Increment::Increment(const Geometry_ & geometry, const Increment & other): + interface::Increment(geometry, other), variables_(other.variables_), + timeComm_(other.timeComm_) +{} + +// ----------------------------------------------------------------------------- + +template +Increment::Increment(const Increment & other, const bool copy): + interface::Increment(other, copy), variables_(other.variables_), + timeComm_(other.timeComm_) +{} + +// ----------------------------------------------------------------------------- + +template +double Increment::dot_product_with(const Increment & dx) const { + double zz = interface::Increment::dot_product_with(dx); + timeComm_->allReduceInPlace(zz, eckit::mpi::Operation::SUM); + return zz; +} + +// ----------------------------------------------------------------------------- + +template +double Increment::norm() const { + double zz = interface::Increment::norm(); + zz *= zz; + timeComm_->allReduceInPlace(zz, eckit::mpi::Operation::SUM); + zz = sqrt(zz); + return zz; +} + +// ----------------------------------------------------------------------------- + template void Increment::shift_forward(const util::DateTime & begin) { Log::trace() << "Increment::Increment shift_forward starting" << std::endl; - util::Timer timer(classname(), "shift_forward"); static int tag = 159357; - size_t mytime = commTime_.rank(); + size_t mytime = timeComm_->rank(); // Send values of M.dx_i at end of my subwindow to next subwindow - if (mytime + 1 < commTime_.size()) { - oops::mpi::send(commTime_, *this, mytime+1, tag); + if (mytime + 1 < timeComm_->size()) { + oops::mpi::send(*timeComm_, *this, mytime+1, tag); } // Receive values at beginning of my subwindow from previous subwindow if (mytime > 0) { - oops::mpi::receive(commTime_, *this, mytime-1, tag); + oops::mpi::receive(*timeComm_, *this, mytime-1, tag); } else { - increment_->zero(begin); + this->zero(begin); } ++tag; @@ -501,41 +598,55 @@ void Increment::shift_forward(const util::DateTime & begin) { template void Increment::shift_backward(const util::DateTime & end) { Log::trace() << "Increment::Increment shift_backward starting" << std::endl; - util::Timer timer(classname(), "shift_backward"); static int tag = 753951; - size_t mytime = commTime_.rank(); + size_t mytime = timeComm_->rank(); // Send values of dx_i at start of my subwindow to previous subwindow if (mytime > 0) { - oops::mpi::send(commTime_, *this, mytime-1, tag); + oops::mpi::send(*timeComm_, *this, mytime-1, tag); } // Receive values at end of my subwindow from next subwindow - if (mytime + 1 < commTime_.size()) { - oops::mpi::receive(commTime_, *this, mytime+1, tag); + if (mytime + 1 < timeComm_->size()) { + oops::mpi::receive(*timeComm_, *this, mytime+1, tag); } else { - increment_->zero(end); + this->zero(end); } ++tag; Log::trace() << "Increment::Increment shift_backward done" << std::endl; } + +// ----------------------------------------------------------------------------- +template +void Increment::toAtlas() { + interface::Increment::toAtlas(&atlasFieldSet_); + this->increment_.reset(); +} + // ----------------------------------------------------------------------------- template void Increment::print(std::ostream & os) const { - Log::trace() << "Increment::print starting" << std::endl; - util::Timer timer(classname(), "print"); - if (commTime_.size() > 1) { - gatherPrint(os, *increment_, commTime_); + if (timeComm_->size() > 1) { + gatherPrint(os, this->increment(), *timeComm_); } else { - os << *increment_; + os << this->increment(); } - Log::trace() << "Increment::print done" << std::endl; } // ----------------------------------------------------------------------------- +/// Add on \p dx incrment to model state \p xx +template +State & operator+=(State & xx, const Increment & dx) { + Log::trace() << "operator+=(State, Increment) starting" << std::endl; + util::Timer timer("oops::Increment", "operator+=(State, Increment)"); + xx.state() += dx.increment(); + Log::trace() << "operator+=(State, Increment) done" << std::endl; + return xx; +} + } // namespace oops From 8e1763ddf8ee5e71b8a0c68bbf4a4af75962fdf0 Mon Sep 17 00:00:00 2001 From: Marek Wlasak Date: Wed, 20 Jan 2021 14:42:14 +0000 Subject: [PATCH 041/142] Feature/getvalues config (#1002) * added T dx = TUTdx test; * allow all the tests. * making a separate sub test for tutdx=tdx in test inverse. * added log out in test * 100 character long format issue * end of day commit * disabling qg for now and going back on getvalues interface change * failing code with error message that does not make sense! * configuration now compiles - defaults to not including "getvalues" in yaml * removing additional code used fro umjedi test * simple removal of commented code. * change code to make configuration optional. There needs to be some more tidying up. * tidy up * multiple observations types now work * minor refactor * add whitespace * filtering at top level - removal of extra constructor * removal of l95 tests * remove redundant file * responding to Anna's comments * responding to Wojciech's comments * added error trap * using local config in GetValues test Co-authored-by: Anna Shlyaeva --- l95/src/lorenz95/GetValuesL95.cc | 6 ++-- l95/src/lorenz95/GetValuesL95.h | 6 +++- qg/model/GetValuesQG.cc | 34 ++++++++++++++++--- qg/model/GetValuesQG.h | 10 +++++- qg/test/testinput/3dvar.yaml | 6 ++++ qg/test/testinput/eda_3dfgat_1.yaml | 6 ++++ qg/test/testinput/hofx.yaml | 6 ++++ src/CMakeLists.txt | 2 ++ src/oops/assimilation/CostJo.h | 8 ++++- src/oops/assimilation/GETKFSolver.h | 8 +++-- src/oops/assimilation/LocalEnsembleSolver.h | 11 +++++-- src/oops/base/GetValuesPost.h | 28 ++++++++++------ src/oops/interface/GetValues.h | 10 ++++-- src/oops/runs/HofX.h | 8 ++++- src/oops/runs/HofXNoModel.h | 9 ++++-- src/oops/util/ConfigFunctions.cc | 33 +++++++++++++++++++ src/oops/util/ConfigFunctions.h | 36 +++++++++++++++++++++ src/test/interface/GetValues.h | 10 ++++-- src/test/interface/LinearGetValues.h | 5 ++- 19 files changed, 209 insertions(+), 33 deletions(-) create mode 100644 src/oops/util/ConfigFunctions.cc create mode 100644 src/oops/util/ConfigFunctions.h diff --git a/l95/src/lorenz95/GetValuesL95.cc b/l95/src/lorenz95/GetValuesL95.cc index 1cc9f7724..52c1883da 100644 --- a/l95/src/lorenz95/GetValuesL95.cc +++ b/l95/src/lorenz95/GetValuesL95.cc @@ -23,10 +23,12 @@ namespace lorenz95 { // ----------------------------------------------------------------------------- GetValuesL95::GetValuesL95(const Resolution & resol, - const LocsL95 & locs) - : resolidx_(locs.size()), times_(locs.times()) + const LocsL95 & locs, + const eckit::Configuration & conf) + : resolidx_(locs.size()), times_(locs.times()) { // find indices of gridpoints nearest to all observations (resolidx_) + const int npoints = resol.npoints(); const double dres = static_cast(npoints); for (size_t jobs = 0; jobs < locs.size(); ++jobs) { diff --git a/l95/src/lorenz95/GetValuesL95.h b/l95/src/lorenz95/GetValuesL95.h index 3a3a379ef..35f3f3587 100644 --- a/l95/src/lorenz95/GetValuesL95.h +++ b/l95/src/lorenz95/GetValuesL95.h @@ -16,6 +16,10 @@ #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" +namespace eckit { + class Configuration; +} + namespace lorenz95 { class GomL95; class LocsL95; @@ -30,7 +34,7 @@ class GetValuesL95 : public util::Printable, static const std::string classname() {return "lorenz95::GetValuesL95";} /// \brief computes indices resolidx_ of nearest gridpoints for all locations \p locs - GetValuesL95(const Resolution &, const LocsL95 & locs); + GetValuesL95(const Resolution &, const LocsL95 & locs, const eckit::Configuration &); /// \brief fills in \p geovals for all observations in the timeframe (\p t1, \p t2], /// \p geovals are equal to the value of \p state at the nearest gridpoint diff --git a/qg/model/GetValuesQG.cc b/qg/model/GetValuesQG.cc index 19343a722..03ac0f566 100644 --- a/qg/model/GetValuesQG.cc +++ b/qg/model/GetValuesQG.cc @@ -7,6 +7,9 @@ #include +#include "eckit/config/LocalConfiguration.h" +#include "eckit/exception/Exceptions.h" + #include "model/GetValuesQG.h" #include "oops/util/Logger.h" @@ -22,16 +25,37 @@ namespace qg { // ----------------------------------------------------------------------------- /// Constructor, destructor // ----------------------------------------------------------------------------- -GetValuesQG::GetValuesQG(const GeometryQG & geom, const LocationsQG & locs) - : locs_(locs) {} +GetValuesQG::GetValuesQG(const GeometryQG & geom, const LocationsQG & locs, + const eckit::Configuration & conf) + : locs_(locs), conf_(conf) +{ + oops::Log::trace() << "GetValuesQG constructor with config " + << conf_ << std::endl; +} + + // ----------------------------------------------------------------------------- /// Get state values at observation locations // ----------------------------------------------------------------------------- void GetValuesQG::fillGeoVaLs(const StateQG & state, const util::DateTime & t1, - const util::DateTime & t2, GomQG & gom) const { - qg_getvalues_interp_f90(locs_, state.fields().toFortran(), - t1, t2, gom.toFortran()); + const util::DateTime & t2, GomQG & gom) const +{ + // the below call is an example if one wanted a different interpolation type + const std::string interpType = conf_.getString("interpolation type", "default"); + + if (interpType == "default" || + (interpType.compare(0, 8, "default_") == 0)) { + oops::Log::trace() << "GetValuesQG config = " + << conf_ << std::endl; + qg_getvalues_interp_f90(locs_, state.fields().toFortran(), + t1, t2, gom.toFortran()); + } else { + std::string err_message("interpolation type option " + + interpType + " not supported"); + throw eckit::BadValue(err_message, Here()); + } } + // ----------------------------------------------------------------------------- void GetValuesQG::print(std::ostream & os) const { os << "QG GetValues"; diff --git a/qg/model/GetValuesQG.h b/qg/model/GetValuesQG.h index d96d16cc0..7da42ac23 100644 --- a/qg/model/GetValuesQG.h +++ b/qg/model/GetValuesQG.h @@ -12,6 +12,8 @@ #include #include +#include "eckit/config/LocalConfiguration.h" + #include "oops/util/DateTime.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" @@ -19,6 +21,10 @@ #include "oops/qg/LocationsQG.h" #include "oops/qg/QgFortran.h" +namespace eckit { + class Configuration; +} + namespace qg { class GomQG; class GeometryQG; @@ -32,7 +38,8 @@ class GetValuesQG : public util::Printable, static const std::string classname() {return "qg::GetValuesQG";} /// \brief saves all locations \p locs to use during filling GeoVaLs - GetValuesQG(const GeometryQG &, const LocationsQG & locs); + GetValuesQG(const GeometryQG &, const LocationsQG & locs, + const eckit::Configuration &); ~GetValuesQG() {} /// \brief fills in \p geovals for all observations in the timeframe (\p t1, \p t2], @@ -43,6 +50,7 @@ class GetValuesQG : public util::Printable, private: void print(std::ostream &) const; LocationsQG locs_; + eckit::LocalConfiguration conf_; }; // ----------------------------------------------------------------------------- diff --git a/qg/test/testinput/3dvar.yaml b/qg/test/testinput/3dvar.yaml index a89267046..d1b404553 100644 --- a/qg/test/testinput/3dvar.yaml +++ b/qg/test/testinput/3dvar.yaml @@ -27,6 +27,8 @@ cost function: obsdataout: obsfile: Data/3dvar.obs3d.nc obs type: Stream + get values: + interpolation type: default_1 - obs error: covariance model: diagonal obs operator: @@ -37,6 +39,8 @@ cost function: obsdataout: obsfile: Data/3dvar.obs3d.nc obs type: Wind + get values: + interpolation type: default_2 - obs error: covariance model: diagonal obs operator: @@ -47,6 +51,8 @@ cost function: obsdataout: obsfile: Data/3dvar.obs3d.nc obs type: WSpeed + get values: + interpolation type: default_3 variational: minimizer: algorithm: DRIPCG diff --git a/qg/test/testinput/eda_3dfgat_1.yaml b/qg/test/testinput/eda_3dfgat_1.yaml index b44b9a425..97e7bf548 100644 --- a/qg/test/testinput/eda_3dfgat_1.yaml +++ b/qg/test/testinput/eda_3dfgat_1.yaml @@ -32,6 +32,8 @@ cost function: obsfile: Data/mem001.eda_3dfgat.obs3d.nc obs type: Stream obs perturbations seed: 1 + get values: + interpolation type: default_1 - obs error: covariance model: diagonal random amplitude: 0.5 @@ -44,6 +46,8 @@ cost function: obsfile: Data/mem001.eda_3dfgat.obs3d.nc obs type: Wind obs perturbations seed: 1 + get values: + interpolation type: default_2 - obs error: covariance model: diagonal random amplitude: 0.2 @@ -56,6 +60,8 @@ cost function: obsfile: Data/mem001.eda_3dfgat.obs3d.nc obs type: WSpeed obs perturbations seed: 1 + get values: + interpolation type: default_3 variational: minimizer: algorithm: DRIPCG diff --git a/qg/test/testinput/hofx.yaml b/qg/test/testinput/hofx.yaml index 192b3f1d6..21cc88711 100644 --- a/qg/test/testinput/hofx.yaml +++ b/qg/test/testinput/hofx.yaml @@ -20,6 +20,8 @@ observations: obs type: Stream obs operator: obs type: Stream + get values: + interpolation type: default_1 - obs space: obsdatain: obsfile: Data/truth.obs4d_12h.nc @@ -28,6 +30,8 @@ observations: obs type: Wind obs operator: obs type: Wind + get values: + interpolation type: default_2 - obs space: obsdatain: obsfile: Data/truth.obs4d_12h.nc @@ -36,5 +40,7 @@ observations: obs type: WSpeed obs operator: obs type: WSpeed + get values: + interpolation type: default_3 prints: frequency: PT3H diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 590a16cd2..b671de69d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -241,6 +241,8 @@ oops/util/AssociativeContainers.h oops/util/CompareNVectors.h oops/util/CompositePath.cc oops/util/CompositePath.h +oops/util/ConfigFunctions.cc +oops/util/ConfigFunctions.h oops/util/dateFunctions.cc oops/util/dateFunctions.h oops/util/datetime_f.cc diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index f73e24822..c64fa938c 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -36,6 +36,7 @@ #include "oops/interface/Increment.h" #include "oops/interface/State.h" #include "oops/mpi/mpi.h" +#include "oops/util/ConfigFunctions.h" #include "oops/util/DateTime.h" #include "oops/util/Logger.h" #include "oops/util/missingValues.h" @@ -176,7 +177,12 @@ CostJo::initialize(const CtrlVar_ & xx, const eckit::Configuration & currentConf_.reset(new eckit::LocalConfiguration(conf)); calchofx_.initialize(xx.obsVar(), currentConf_->getInt("iteration")); - getvals_.reset(new GetValuesPost_(obspace_, calchofx_.locations(), calchofx_.requiredVars())); + + std::vector getValuesConfig = + util::oopsconfigfunctions::vectoriseAndFilter(obsconf_, "get values"); + + getvals_.reset(new GetValuesPost_(obspace_, calchofx_.locations(), + calchofx_.requiredVars(), getValuesConfig)); Log::trace() << "CostJo::initialize done" << std::endl; return getvals_; } diff --git a/src/oops/assimilation/GETKFSolver.h b/src/oops/assimilation/GETKFSolver.h index 63bedf798..9b2a2ec11 100644 --- a/src/oops/assimilation/GETKFSolver.h +++ b/src/oops/assimilation/GETKFSolver.h @@ -31,6 +31,7 @@ #include "oops/generic/VerticalLocEV.h" #include "oops/interface/Geometry.h" #include "oops/interface/GeometryIterator.h" +#include "oops/util/ConfigFunctions.h" #include "oops/util/Logger.h" #include "oops/util/Timer.h" @@ -84,7 +85,6 @@ class GETKFSolver : public LocalEnsembleSolver { private: LETKFSolverParameters options_; - // parameters size_t nens_; const Geometry_ & geometry_; @@ -173,8 +173,12 @@ Observations GETKFSolver::computeHofX(const StateEnsemble4D_ & for (size_t ieig = 0; ieig < neig_; ++ieig) { State4D_ tmpState = xx_mean; tmpState += Ztmp[ieig]; + + std::vector getValuesConfig = + util::oopsconfigfunctions::vectoriseAndFilter(this->obsconf_, "get values"); + GetValuesPost_ getvals(this->obspaces_, this->hofx_.locations(), - this->hofx_.requiredVars()); + this->hofx_.requiredVars(), getValuesConfig); getvals.fill(tmpState); // compute H(x) on filled in geovals and run the filters Observations_ tmpObs = this->hofx_.compute(getvals.geovals()); diff --git a/src/oops/assimilation/LocalEnsembleSolver.h b/src/oops/assimilation/LocalEnsembleSolver.h index e520cdc0b..b3175cd00 100644 --- a/src/oops/assimilation/LocalEnsembleSolver.h +++ b/src/oops/assimilation/LocalEnsembleSolver.h @@ -11,7 +11,9 @@ #include #include #include +#include +#include "eckit/config/Configuration.h" #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/CalcHofX.h" #include "oops/base/Departures.h" @@ -25,6 +27,7 @@ #include "oops/base/StateEnsemble4D.h" #include "oops/interface/Geometry.h" #include "oops/interface/GeometryIterator.h" +#include "oops/util/ConfigFunctions.h" #include "oops/util/Logger.h" #include "oops/util/Timer.h" @@ -81,7 +84,8 @@ template LocalEnsembleSolver::LocalEnsembleSolver(ObsSpaces_ & obspaces, const Geometry_ & geometry, const eckit::Configuration & config, size_t nens) - : obsconf_(config, "observations"), obspaces_(obspaces), obsaux_(obspaces_, obsconf_), + : obsconf_(config, "observations"), + obspaces_(obspaces), obsaux_(obspaces_, obsconf_), hofx_(obspaces, obsconf_), omb_(obspaces_), Yb_(obspaces_, nens) { } @@ -113,7 +117,10 @@ Observations LocalEnsembleSolver::computeHofX(const StateEnsemb for (size_t jj = 0; jj < nens; ++jj) { hofx_.resetQc(); // fill in geovals - GetValuesPost_ getvals(obspaces_, hofx_.locations(), hofx_.requiredVars()); + std::vector getValuesConfig = + util::oopsconfigfunctions::vectoriseAndFilter(obsconf_, "get values"); + + GetValuesPost_ getvals(obspaces_, hofx_.locations(), hofx_.requiredVars(), getValuesConfig); getvals.fill(ens_xx[jj]); // compute H(x) on filled in geovals and run the filters obsens[jj] = hofx_.compute(getvals.geovals()); diff --git a/src/oops/base/GetValuesPost.h b/src/oops/base/GetValuesPost.h index a7e4e10c4..34d4d17a6 100644 --- a/src/oops/base/GetValuesPost.h +++ b/src/oops/base/GetValuesPost.h @@ -29,6 +29,10 @@ #include "oops/util/Duration.h" #include "oops/util/Logger.h" +namespace eckit { + class Configuration; +} + namespace oops { /// \brief Fills GeoVaLs with requested variables at requested locations: @@ -50,7 +54,9 @@ class GetValuesPost : public PostBase> { public: /// \brief Saves Locations and Variables to be processed - GetValuesPost(const ObsSpaces_ &, const LocationsVec_ &, const VariablesVec_ &); + GetValuesPost(const ObsSpaces_ &, + const LocationsVec_ &, const VariablesVec_ &, + const std::vector &); /// \brief Returns geovals filled in during the model run const GeoVaLsVec_ & geovals() const {return geovals_;} @@ -69,23 +75,25 @@ class GetValuesPost : public PostBase> { util::DateTime winend_; /// End of assimilation window util::Duration hslot_; /// Half time slot + const LocationsVec_ & locations_; /// locations of observations const VariablesVec_ & geovars_; /// Variables needed from model GetValuesVec_ getvals_; /// GetValues used to fill in GeoVaLs GeoVaLsVec_ geovals_; /// GeoVaLs that are filled in + const std::vector getvalsconfs_; /// configuration object }; // ----------------------------------------------------------------------------- template -GetValuesPost::GetValuesPost(const ObsSpaces_ & obsdb, const LocationsVec_ & locations, - const VariablesVec_ & vars) +GetValuesPost::GetValuesPost(const ObsSpaces_ & obsdb, + const LocationsVec_ & locations, + const VariablesVec_ & vars, + const std::vector & confs) : PostBase(), winbgn_(obsdb.windowStart()), winend_(obsdb.windowEnd()), hslot_(), - locations_(locations), geovars_(vars) -{ - Log::trace() << "GetValuesPost::GetValuesPost done" << std::endl; -} + locations_(locations), geovars_(vars), + getvalsconfs_(confs) {} // ----------------------------------------------------------------------------- template @@ -111,13 +119,15 @@ void GetValuesPost::fill(const State4D_ & xx) { template void GetValuesPost::doInitialize(const State_ & xx, const util::DateTime & end, - const util::Duration & tstep) { + const util::Duration & tstep) { Log::trace() << "GetValuesPost::doInitialize start" << std::endl; hslot_ = tstep/2; + for (size_t jj = 0; jj < locations_.size(); ++jj) { - getvals_.emplace_back(new GetValues_(xx.geometry(), *locations_[jj])); + getvals_.emplace_back(new GetValues_(xx.geometry(), *locations_[jj], getvalsconfs_[jj])); geovals_.emplace_back(new GeoVaLs_(*locations_[jj], geovars_[jj])); } + Log::trace() << "GetValuesPost::doInitialize done" << std::endl; } diff --git a/src/oops/interface/GetValues.h b/src/oops/interface/GetValues.h index ab9a1af54..9b8ca2b41 100644 --- a/src/oops/interface/GetValues.h +++ b/src/oops/interface/GetValues.h @@ -13,6 +13,7 @@ #include "eckit/config/Configuration.h" +#include "eckit/config/LocalConfiguration.h" #include "oops/interface/Geometry.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/Locations.h" @@ -40,7 +41,7 @@ class GetValues : public util::Printable, static const std::string classname() {return "oops::GetValues";} /// Constructor, destructor - GetValues(const Geometry_ &, const Locations_ &); + GetValues(const Geometry_ &, const Locations_ &, const eckit::Configuration &); ~GetValues(); /// Interfacing @@ -59,14 +60,17 @@ class GetValues : public util::Printable, // ============================================================================= template -GetValues::GetValues(const Geometry_ & resol, const Locations_ & locs) : getvalues_() +GetValues::GetValues(const Geometry_ & resol, const Locations_ & locs, + const eckit::Configuration & conf) + :getvalues_() { Log::trace() << "GetValues::GetValues starting" << std::endl; util::Timer timer(classname(), "GetValues"); - getvalues_.reset(new GetValues_(resol.geometry(), locs.locations())); + getvalues_.reset(new GetValues_(resol.geometry(), locs.locations(), conf)); Log::trace() << "GetValues::GetValues done" << std::endl; } + // ----------------------------------------------------------------------------- template diff --git a/src/oops/runs/HofX.h b/src/oops/runs/HofX.h index 036cbecf9..156a35fee 100644 --- a/src/oops/runs/HofX.h +++ b/src/oops/runs/HofX.h @@ -14,6 +14,7 @@ #include #include +#include #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" @@ -33,6 +34,7 @@ #include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" +#include "oops/util/ConfigFunctions.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" @@ -109,8 +111,12 @@ template class HofX : public Application { hofx.initialize(obsaux); // run the model and compute H(x) + std::vector getValuesConfig = + util::oopsconfigfunctions::vectoriseAndFilter(obsConfig, "get values"); + std::shared_ptr - getvals(new GetValuesPost_(obspaces, hofx.locations(), hofx.requiredVars())); + getvals(new GetValuesPost_(obspaces, hofx.locations(), + hofx.requiredVars(), getValuesConfig)); post.enrollProcessor(getvals); model.forecast(xx, moderr, flength, post); diff --git a/src/oops/runs/HofXNoModel.h b/src/oops/runs/HofXNoModel.h index 0ef814d17..accb8fe87 100644 --- a/src/oops/runs/HofXNoModel.h +++ b/src/oops/runs/HofXNoModel.h @@ -26,6 +26,7 @@ #include "oops/interface/Locations.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" +#include "oops/util/ConfigFunctions.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" @@ -95,9 +96,13 @@ template class HofXNoModel : public Application { GeoVaLsVec_ geovals; const LocationsVec_ & locations = hofx.locations(); const VariablesVec_ & vars = hofx.requiredVars(); - // loop over all observation types + + std::vector getValuesConfig = + util::oopsconfigfunctions::vectoriseAndFilter(obsConfig, "get values"); + + // loop over all observation types for (size_t jj = 0; jj < obspaces.size(); ++jj) { - GetValues_ getvals(geometry, *locations[jj]); + GetValues_ getvals(geometry, *locations[jj], getValuesConfig[jj]); // add GeoVaLs for this obs type geovals.emplace_back(new GeoVaLs_(*locations[jj], vars[jj])); // fill in by looping through 4D state diff --git a/src/oops/util/ConfigFunctions.cc b/src/oops/util/ConfigFunctions.cc new file mode 100644 index 000000000..6d0d8155d --- /dev/null +++ b/src/oops/util/ConfigFunctions.cc @@ -0,0 +1,33 @@ + +/* + * (C) Copyright 2020 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/util/ConfigFunctions.h" + +namespace util { + + namespace oopsconfigfunctions { + + std::vector + vectoriseAndFilter(const eckit::Configuration & config, const std::string & tag) + { + const std::vector + ObsConfigVec(config.getSubConfigurations()); + std::vector filteredConfig; + for (auto conf : ObsConfigVec) { + eckit::LocalConfiguration tempConf; + if (conf.has(tag)) { + tempConf = eckit::LocalConfiguration(conf, tag); + } + filteredConfig.push_back(tempConf); + } + return filteredConfig; + } + + } // namespace oopsconfigfunctions + +} // namespace util diff --git a/src/oops/util/ConfigFunctions.h b/src/oops/util/ConfigFunctions.h new file mode 100644 index 000000000..5b9dc13c9 --- /dev/null +++ b/src/oops/util/ConfigFunctions.h @@ -0,0 +1,36 @@ +/* + * (C) Copyright 2020 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_UTIL_CONFIGFUNCTIONS_H_ +#define OOPS_UTIL_CONFIGFUNCTIONS_H_ + +#include +#include + +#include "eckit/config/Configuration.h" +#include "eckit/config/LocalConfiguration.h" + + +namespace util { + /// Namespace for oopsconfigfunctions + namespace oopsconfigfunctions { + + /// \brief vectoriseAndFilter takes a single configuration object that is + /// internally a list of subConfigurations and breaks them up + /// into a vector of subConfigurations. + /// Then it loops over all subConfigurations and searches + /// for sub(sub)Configurations that are identified by the string tag. + /// If they exist, they are copied into the correct index of a + /// vector of LocalConfigurations. + /// If they don't exist an empty configuration is copied in the vector element. + std::vector + vectoriseAndFilter(const eckit::Configuration &config, const std::string & tag); + + } // namespace oopsconfigfunctions +} // namespace util + +#endif // OOPS_UTIL_CONFIGFUNCTIONS_H_ diff --git a/src/test/interface/GetValues.h b/src/test/interface/GetValues.h index d6ddf489e..c856c0ee1 100644 --- a/src/test/interface/GetValues.h +++ b/src/test/interface/GetValues.h @@ -89,8 +89,10 @@ template class GetValuesFixture : private boost:: geovals_.reset(new GeoVaLs_(*locs_, *geovalvars_)); // GetValues - getvalues_.reset(new GetValues_(*resol_, *locs_)); - } + LocalConfig_ getvaluesConfig; + if (TestEnvironment::config().has("get values")) + getvaluesConfig = eckit::LocalConfiguration(TestEnvironment::config(), "get values"); + getvalues_.reset(new GetValues_( *resol_, *locs_, getvaluesConfig)); } ~GetValuesFixture() {} @@ -110,7 +112,9 @@ template void testGetValuesConstructor() { typedef GetValuesFixture Test_; typedef oops::GetValues GetValues_; - std::unique_ptr GetValues(new GetValues_(Test_::resol(), Test_::locs())); + std::unique_ptr + GetValues(new GetValues_(Test_::resol(), Test_::locs(), TestEnvironment::config())); + EXPECT(GetValues.get()); oops::Log::test() << "Testing GetValues: " << *GetValues << std::endl; GetValues.reset(); diff --git a/src/test/interface/LinearGetValues.h b/src/test/interface/LinearGetValues.h index 1370e56f3..5ab759535 100644 --- a/src/test/interface/LinearGetValues.h +++ b/src/test/interface/LinearGetValues.h @@ -96,7 +96,10 @@ template class LinearGetValuesFixture : private b geovals_.reset(new GeoVaLs_(*locs_, *geovalvars_)); // Nonlinear GetValues - getvalues_.reset(new GetValues_(*resol_, *locs_)); + LocalConfig_ getvaluesConfig; + if (TestEnvironment::config().has("get values")) + getvaluesConfig = eckit::LocalConfiguration(TestEnvironment::config(), "get values"); + getvalues_.reset(new GetValues_( *resol_, *locs_, getvaluesConfig)); // State const LocalConfig_ stateConfig(TestEnvironment::config(), "background"); From a5ea3877b1ec83568c78fcc71389c2e70963df12 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 20 Jan 2021 13:52:54 -0700 Subject: [PATCH 042/142] Add a base class for generic Model implemenations (#1038) * yet another attempt at refactoring model classes * rename ModelBase classes * a bit more comments * mark final methods final --- src/oops/base/ModelBase.h | 93 +++++++++++++++++++--- src/oops/generic/IdentityModel.h | 8 +- src/oops/generic/PseudoModel.h | 8 +- src/oops/generic/instantiateModelFactory.h | 4 +- src/oops/interface/Model.h | 8 +- 5 files changed, 94 insertions(+), 27 deletions(-) diff --git a/src/oops/base/ModelBase.h b/src/oops/base/ModelBase.h index 6df48e688..6ec517bae 100644 --- a/src/oops/base/ModelBase.h +++ b/src/oops/base/ModelBase.h @@ -38,19 +38,21 @@ namespace oops { // ----------------------------------------------------------------------------- -/// \brief Base class for the forecasting model +/// \brief Base class for the forecasting model. /// Defines the interfaces for a forecast model. +/// Use this class as a base class for generic implementations, +/// and ModelBase as a base calss for MODEL-specific implementations. template -class ModelBase : public util::Printable, - private boost::noncopyable { - typedef typename MODEL::ModelAuxControl ModelAux_; - typedef typename MODEL::State State_; +class GenericModelBase : public util::Printable, + private boost::noncopyable { + typedef ModelAuxControl ModelAux_; + typedef State State_; public: - static const std::string classname() {return "oops::ModelBase";} + static const std::string classname() {return "oops::GenericModelBase";} - ModelBase() = default; - virtual ~ModelBase() = default; + GenericModelBase() = default; + virtual ~GenericModelBase() = default; /// \brief Forecast initialization, called before every forecast run virtual void initialize(State_ &) const = 0; @@ -70,6 +72,40 @@ class ModelBase : public util::Printable, virtual void print(std::ostream &) const = 0; }; + +/// \brief Base class for MODEL-specific implementations of Model class +/// The complete interface that needs to be implemented is described in GenericModelBase. +/// ModelBase overrides GenericModelBase methods to pass MODEL-specific implementations +/// of State and ModelAuxControl to the MODEL-specific implementation of Model. +template +class ModelBase : public GenericModelBase { + typedef typename MODEL::ModelAuxControl ModelAux_; + typedef typename MODEL::State State_; + + public: + static const std::string classname() {return "oops::ModelBase";} + + ModelBase() = default; + virtual ~ModelBase() = default; + + /// Overrides for ModelBase classes, passing MODEL-specific classes to the + /// MODEL-specific implementations of Model + void initialize(State & xx) const final + { this->initialize(xx.state()); } + void step(State & xx, const ModelAuxControl & modelaux) const final + { this->step(xx.state(), modelaux.modelauxcontrol()); } + void finalize(State & xx) const final + { this->finalize(xx.state()); } + + /// \brief Forecast initialization, called before every forecast run + virtual void initialize(State_ &) const = 0; + /// \brief Forecast "step", called during forecast run; updates state to the next time + virtual void step(State_ &, const ModelAux_ &) const = 0; + /// \brief Forecast finalization; called after each forecast run + virtual void finalize(State_ &) const = 0; +}; + + // ============================================================================= template @@ -136,7 +172,8 @@ class ModelFactory { /// The model's type is determined by the \c name attribute of \p parameters. /// \p parameters must be an instance of the subclass of ModelParametersBase /// associated with that model type, otherwise an exception will be thrown. - static ModelBase * create(const Geometry_ &, const ModelParametersBase & parameters); + static GenericModelBase * create(const Geometry_ &, + const ModelParametersBase & parameters); /// \brief Create and return an instance of the subclass of ModelParametersBase /// storing parameters of models of the specified type. @@ -154,7 +191,7 @@ class ModelFactory { explicit ModelFactory(const std::string & name); private: - virtual ModelBase * make(const Geometry_ &, const ModelParametersBase &) = 0; + virtual GenericModelBase * make(const Geometry_ &, const ModelParametersBase &) = 0; virtual std::unique_ptr makeParameters() const = 0; @@ -167,7 +204,36 @@ class ModelFactory { // ----------------------------------------------------------------------------- /// \brief A subclass of ModelFactory able to create instances of T (a concrete subclass of -/// ModelBase). +/// GenericModelBase). Passes Geometry to the generic implementation of Model. +template +class GenericModelMaker : public ModelFactory { + private: + /// Defined as T::Parameters_ if T defines a Parameters_ type; otherwise as + /// GenericModelParameters. + typedef TParameters_IfAvailableElseFallbackType_t Parameters_; + + public: + typedef Geometry Geometry_; + + explicit GenericModelMaker(const std::string & name) : ModelFactory(name) {} + + GenericModelBase * make(const Geometry_ & geom, + const ModelParametersBase & parameters) override { + Log::trace() << "ModelBase::make starting" << std::endl; + const auto &stronglyTypedParameters = dynamic_cast(parameters); + return new T(geom, + parametersOrConfiguration::value>(stronglyTypedParameters)); + } + + std::unique_ptr makeParameters() const override { + return boost::make_unique(); + } +}; + +// ----------------------------------------------------------------------------- + +/// \brief A subclass of ModelFactory able to create instances of T (a concrete subclass of +/// ModelBase). Passes MODEL::Geometry to the MODEL-specific implementation of Model. template class ModelMaker : public ModelFactory { private: @@ -192,6 +258,7 @@ class ModelMaker : public ModelFactory { } }; + // ----------------------------------------------------------------------------- template @@ -205,7 +272,7 @@ ModelFactory::ModelFactory(const std::string & name) { // ----------------------------------------------------------------------------- template -ModelBase * ModelFactory::create(const Geometry_ & geom, +GenericModelBase * ModelFactory::create(const Geometry_ & geom, const ModelParametersBase & parameters) { Log::trace() << "ModelFactory::create starting" << std::endl; const std::string &id = parameters.name.value().value(); @@ -214,7 +281,7 @@ ModelBase * ModelFactory::create(const Geometry_ & geom, if (jerr == getMakers().end()) { throw std::runtime_error(id + " does not exist in the model factory"); } - ModelBase * ptr = jerr->second->make(geom, parameters); + GenericModelBase * ptr = jerr->second->make(geom, parameters); Log::trace() << "ModelFactory::create done" << std::endl; return ptr; } diff --git a/src/oops/generic/IdentityModel.h b/src/oops/generic/IdentityModel.h index cbfac40f7..8d3ccfe94 100644 --- a/src/oops/generic/IdentityModel.h +++ b/src/oops/generic/IdentityModel.h @@ -33,10 +33,10 @@ class IdentityModelParameters : public ModelParametersBase { /// Generic implementation of identity model template -class IdentityModel : public ModelBase { - typedef typename MODEL::Geometry Geometry_; - typedef typename MODEL::ModelAuxControl ModelAux_; - typedef typename MODEL::State State_; +class IdentityModel : public GenericModelBase { + typedef Geometry Geometry_; + typedef ModelAuxControl ModelAux_; + typedef State State_; public: typedef IdentityModelParameters Parameters_; diff --git a/src/oops/generic/PseudoModel.h b/src/oops/generic/PseudoModel.h index 45fc792ca..c9dcea409 100644 --- a/src/oops/generic/PseudoModel.h +++ b/src/oops/generic/PseudoModel.h @@ -26,10 +26,10 @@ namespace oops { /// Generic implementation of the pseudo model (steps through time by reading states) template -class PseudoModel : public ModelBase { - typedef typename MODEL::Geometry Geometry_; - typedef typename MODEL::ModelAuxControl ModelAux_; - typedef typename MODEL::State State_; +class PseudoModel : public GenericModelBase { + typedef Geometry Geometry_; + typedef ModelAuxControl ModelAux_; + typedef State State_; public: static const std::string classname() {return "oops::PseudoModel";} diff --git a/src/oops/generic/instantiateModelFactory.h b/src/oops/generic/instantiateModelFactory.h index 6e6f36def..b0211b03d 100644 --- a/src/oops/generic/instantiateModelFactory.h +++ b/src/oops/generic/instantiateModelFactory.h @@ -15,8 +15,8 @@ namespace oops { template void instantiateModelFactory() { - static ModelMaker > makerIdentityModel_("Identity"); - static ModelMaker > makerPseudoModel_("PseudoModel"); + static GenericModelMaker > makerIdentityModel_("Identity"); + static GenericModelMaker > makerPseudoModel_("PseudoModel"); } } // namespace oops diff --git a/src/oops/interface/Model.h b/src/oops/interface/Model.h index c5b438e8d..80c6b1b3e 100644 --- a/src/oops/interface/Model.h +++ b/src/oops/interface/Model.h @@ -52,7 +52,7 @@ template class Model : public util::Printable, private boost::noncopyable, private util::ObjectCounter > { - typedef ModelBase ModelBase_; + typedef GenericModelBase ModelBase_; typedef Geometry Geometry_; typedef ModelAuxControl ModelAux_; typedef State State_; @@ -150,7 +150,7 @@ template void Model::initialize(State_ & xx) const { Log::trace() << "Model::initialize starting" << std::endl; util::Timer timer(classname(), "initialize"); - model_->initialize(xx.state()); + model_->initialize(xx); Log::trace() << "Model::initialize done" << std::endl; } @@ -160,7 +160,7 @@ template void Model::step(State_ & xx, const ModelAux_ & maux) const { Log::trace() << "Model::step starting" << std::endl; util::Timer timer(classname(), "step"); - model_->step(xx.state(), maux.modelauxcontrol()); + model_->step(xx, maux); Log::trace() << "Model::step done" << std::endl; } @@ -170,7 +170,7 @@ template void Model::finalize(State_ & xx) const { Log::trace() << "Model::finalize starting" << std::endl; util::Timer timer(classname(), "finalize"); - model_->finalize(xx.state()); + model_->finalize(xx); Log::trace() << "Model::finalize done" << std::endl; } From 6973fbe7ba5bf017dedbbb2430fa9309e13c1187 Mon Sep 17 00:00:00 2001 From: Benjamin Menetrier <30638301+benjaminmenetrier@users.noreply.github.com> Date: Thu, 21 Jan 2021 15:04:09 +0100 Subject: [PATCH 043/142] Generic hybrid covariance (#996) * Generic hybrid using 3D weight * Add option to read hybrid weights from files * Update tests, add new ones * Change yaml structure to respect parameters checks * Save memory for scalar hybrid weights Co-authored-by: Anna Shlyaeva --- l95/test/testinput/4densvar.hybrid.yaml | 406 +++++++++--------- l95/test/testinput/4dvar.hybrid.yaml | 69 +-- qg/test/CMakeLists.txt | 35 +- qg/test/testinput/3dvar_hybrid.yaml | 56 +-- qg/test/testinput/4densvar_hybrid.yaml | 334 +++++++------- .../testinput/4dvar_drplanczos_hybrid.yaml | 74 ++-- qg/test/testinput/dirac_hyb.yaml | 62 --- qg/test/testinput/dirac_hyb_field.yaml | 68 +++ qg/test/testinput/dirac_hyb_value.yaml | 66 +++ .../ens_variance_inflation_field.yaml | 2 +- qg/test/testinput/uniform_field_hybrid.yaml | 21 + ...ield.yaml => uniform_field_inflation.yaml} | 2 +- .../{dirac_hyb.test => dirac_hyb_field.test} | 0 qg/test/testoutput/dirac_hyb_value.test | 18 + qg/test/testoutput/uniform_field_hybrid.test | 16 + ...ield.test => uniform_field_inflation.test} | 0 src/oops/base/HybridCovariance.h | 62 ++- src/oops/runs/Dirac.h | 25 +- 18 files changed, 753 insertions(+), 563 deletions(-) delete mode 100644 qg/test/testinput/dirac_hyb.yaml create mode 100644 qg/test/testinput/dirac_hyb_field.yaml create mode 100644 qg/test/testinput/dirac_hyb_value.yaml create mode 100644 qg/test/testinput/uniform_field_hybrid.yaml rename qg/test/testinput/{uniform_field.yaml => uniform_field_inflation.yaml} (91%) rename qg/test/testoutput/{dirac_hyb.test => dirac_hyb_field.test} (100%) create mode 100644 qg/test/testoutput/dirac_hyb_value.test create mode 100644 qg/test/testoutput/uniform_field_hybrid.test rename qg/test/testoutput/{uniform_field.test => uniform_field_inflation.test} (100%) diff --git a/l95/test/testinput/4densvar.hybrid.yaml b/l95/test/testinput/4densvar.hybrid.yaml index 08848f382..d311ef26d 100644 --- a/l95/test/testinput/4densvar.hybrid.yaml +++ b/l95/test/testinput/4densvar.hybrid.yaml @@ -35,208 +35,212 @@ cost function: filename: Data/test.fc.2010-01-01T00:00:00Z.PT15H background error: covariance model: hybrid - ensemble: - localization: + components: + - covariance: + covariance model: ensemble + localization: + length_scale: 1.0 + localization method: L95 + members: + - states: + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T04:30:00Z' + filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT4H30M + - date: '2010-01-01T06:00:00Z' + filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT6H + - date: '2010-01-01T07:30:00Z' + filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT7H30M + - date: '2010-01-01T09:00:00Z' + filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT9H + - date: '2010-01-01T10:30:00Z' + filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT10H30M + - date: '2010-01-01T12:00:00Z' + filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT12H + - date: '2010-01-01T13:30:00Z' + filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT13H30M + - date: '2010-01-01T15:00:00Z' + filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT15H + - states: + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T04:30:00Z' + filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT4H30M + - date: '2010-01-01T06:00:00Z' + filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT6H + - date: '2010-01-01T07:30:00Z' + filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT7H30M + - date: '2010-01-01T09:00:00Z' + filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT9H + - date: '2010-01-01T10:30:00Z' + filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT10H30M + - date: '2010-01-01T12:00:00Z' + filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT12H + - date: '2010-01-01T13:30:00Z' + filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT13H30M + - date: '2010-01-01T15:00:00Z' + filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT15H + - states: + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T04:30:00Z' + filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT4H30M + - date: '2010-01-01T06:00:00Z' + filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT6H + - date: '2010-01-01T07:30:00Z' + filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT7H30M + - date: '2010-01-01T09:00:00Z' + filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT9H + - date: '2010-01-01T10:30:00Z' + filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT10H30M + - date: '2010-01-01T12:00:00Z' + filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT12H + - date: '2010-01-01T13:30:00Z' + filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT13H30M + - date: '2010-01-01T15:00:00Z' + filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT15H + - states: + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T04:30:00Z' + filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT4H30M + - date: '2010-01-01T06:00:00Z' + filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT6H + - date: '2010-01-01T07:30:00Z' + filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT7H30M + - date: '2010-01-01T09:00:00Z' + filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT9H + - date: '2010-01-01T10:30:00Z' + filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT10H30M + - date: '2010-01-01T12:00:00Z' + filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT12H + - date: '2010-01-01T13:30:00Z' + filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT13H30M + - date: '2010-01-01T15:00:00Z' + filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT15H + - states: + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T04:30:00Z' + filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT4H30M + - date: '2010-01-01T06:00:00Z' + filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT6H + - date: '2010-01-01T07:30:00Z' + filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT7H30M + - date: '2010-01-01T09:00:00Z' + filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT9H + - date: '2010-01-01T10:30:00Z' + filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT10H30M + - date: '2010-01-01T12:00:00Z' + filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT12H + - date: '2010-01-01T13:30:00Z' + filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT13H30M + - date: '2010-01-01T15:00:00Z' + filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT15H + - states: + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T04:30:00Z' + filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT4H30M + - date: '2010-01-01T06:00:00Z' + filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT6H + - date: '2010-01-01T07:30:00Z' + filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT7H30M + - date: '2010-01-01T09:00:00Z' + filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT9H + - date: '2010-01-01T10:30:00Z' + filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT10H30M + - date: '2010-01-01T12:00:00Z' + filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT12H + - date: '2010-01-01T13:30:00Z' + filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT13H30M + - date: '2010-01-01T15:00:00Z' + filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT15H + - states: + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T04:30:00Z' + filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT4H30M + - date: '2010-01-01T06:00:00Z' + filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT6H + - date: '2010-01-01T07:30:00Z' + filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT7H30M + - date: '2010-01-01T09:00:00Z' + filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT9H + - date: '2010-01-01T10:30:00Z' + filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT10H30M + - date: '2010-01-01T12:00:00Z' + filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT12H + - date: '2010-01-01T13:30:00Z' + filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT13H30M + - date: '2010-01-01T15:00:00Z' + filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT15H + - states: + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T04:30:00Z' + filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT4H30M + - date: '2010-01-01T06:00:00Z' + filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT6H + - date: '2010-01-01T07:30:00Z' + filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT7H30M + - date: '2010-01-01T09:00:00Z' + filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT9H + - date: '2010-01-01T10:30:00Z' + filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT10H30M + - date: '2010-01-01T12:00:00Z' + filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT12H + - date: '2010-01-01T13:30:00Z' + filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT13H30M + - date: '2010-01-01T15:00:00Z' + filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT15H + - states: + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T04:30:00Z' + filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT4H30M + - date: '2010-01-01T06:00:00Z' + filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT6H + - date: '2010-01-01T07:30:00Z' + filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT7H30M + - date: '2010-01-01T09:00:00Z' + filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT9H + - date: '2010-01-01T10:30:00Z' + filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT10H30M + - date: '2010-01-01T12:00:00Z' + filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT12H + - date: '2010-01-01T13:30:00Z' + filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT13H30M + - date: '2010-01-01T15:00:00Z' + filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT15H + - states: + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T04:30:00Z' + filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT4H30M + - date: '2010-01-01T06:00:00Z' + filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT6H + - date: '2010-01-01T07:30:00Z' + filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT7H30M + - date: '2010-01-01T09:00:00Z' + filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT9H + - date: '2010-01-01T10:30:00Z' + filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT10H30M + - date: '2010-01-01T12:00:00Z' + filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT12H + - date: '2010-01-01T13:30:00Z' + filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT13H30M + - date: '2010-01-01T15:00:00Z' + filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT15H + weight: + value: 0.4 + - covariance: + covariance model: L95Error + date: '2010-01-01T03:00:00Z' length_scale: 1.0 - localization method: L95 - members: - - states: - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T04:30:00Z' - filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT4H30M - - date: '2010-01-01T06:00:00Z' - filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT6H - - date: '2010-01-01T07:30:00Z' - filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT7H30M - - date: '2010-01-01T09:00:00Z' - filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT9H - - date: '2010-01-01T10:30:00Z' - filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT10H30M - - date: '2010-01-01T12:00:00Z' - filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT12H - - date: '2010-01-01T13:30:00Z' - filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT13H30M - - date: '2010-01-01T15:00:00Z' - filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT15H - - states: - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T04:30:00Z' - filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT4H30M - - date: '2010-01-01T06:00:00Z' - filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT6H - - date: '2010-01-01T07:30:00Z' - filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT7H30M - - date: '2010-01-01T09:00:00Z' - filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT9H - - date: '2010-01-01T10:30:00Z' - filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT10H30M - - date: '2010-01-01T12:00:00Z' - filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT12H - - date: '2010-01-01T13:30:00Z' - filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT13H30M - - date: '2010-01-01T15:00:00Z' - filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT15H - - states: - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T04:30:00Z' - filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT4H30M - - date: '2010-01-01T06:00:00Z' - filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT6H - - date: '2010-01-01T07:30:00Z' - filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT7H30M - - date: '2010-01-01T09:00:00Z' - filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT9H - - date: '2010-01-01T10:30:00Z' - filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT10H30M - - date: '2010-01-01T12:00:00Z' - filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT12H - - date: '2010-01-01T13:30:00Z' - filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT13H30M - - date: '2010-01-01T15:00:00Z' - filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT15H - - states: - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T04:30:00Z' - filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT4H30M - - date: '2010-01-01T06:00:00Z' - filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT6H - - date: '2010-01-01T07:30:00Z' - filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT7H30M - - date: '2010-01-01T09:00:00Z' - filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT9H - - date: '2010-01-01T10:30:00Z' - filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT10H30M - - date: '2010-01-01T12:00:00Z' - filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT12H - - date: '2010-01-01T13:30:00Z' - filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT13H30M - - date: '2010-01-01T15:00:00Z' - filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT15H - - states: - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T04:30:00Z' - filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT4H30M - - date: '2010-01-01T06:00:00Z' - filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT6H - - date: '2010-01-01T07:30:00Z' - filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT7H30M - - date: '2010-01-01T09:00:00Z' - filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT9H - - date: '2010-01-01T10:30:00Z' - filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT10H30M - - date: '2010-01-01T12:00:00Z' - filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT12H - - date: '2010-01-01T13:30:00Z' - filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT13H30M - - date: '2010-01-01T15:00:00Z' - filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT15H - - states: - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T04:30:00Z' - filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT4H30M - - date: '2010-01-01T06:00:00Z' - filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT6H - - date: '2010-01-01T07:30:00Z' - filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT7H30M - - date: '2010-01-01T09:00:00Z' - filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT9H - - date: '2010-01-01T10:30:00Z' - filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT10H30M - - date: '2010-01-01T12:00:00Z' - filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT12H - - date: '2010-01-01T13:30:00Z' - filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT13H30M - - date: '2010-01-01T15:00:00Z' - filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT15H - - states: - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T04:30:00Z' - filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT4H30M - - date: '2010-01-01T06:00:00Z' - filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT6H - - date: '2010-01-01T07:30:00Z' - filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT7H30M - - date: '2010-01-01T09:00:00Z' - filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT9H - - date: '2010-01-01T10:30:00Z' - filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT10H30M - - date: '2010-01-01T12:00:00Z' - filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT12H - - date: '2010-01-01T13:30:00Z' - filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT13H30M - - date: '2010-01-01T15:00:00Z' - filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT15H - - states: - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T04:30:00Z' - filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT4H30M - - date: '2010-01-01T06:00:00Z' - filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT6H - - date: '2010-01-01T07:30:00Z' - filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT7H30M - - date: '2010-01-01T09:00:00Z' - filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT9H - - date: '2010-01-01T10:30:00Z' - filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT10H30M - - date: '2010-01-01T12:00:00Z' - filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT12H - - date: '2010-01-01T13:30:00Z' - filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT13H30M - - date: '2010-01-01T15:00:00Z' - filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT15H - - states: - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T04:30:00Z' - filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT4H30M - - date: '2010-01-01T06:00:00Z' - filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT6H - - date: '2010-01-01T07:30:00Z' - filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT7H30M - - date: '2010-01-01T09:00:00Z' - filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT9H - - date: '2010-01-01T10:30:00Z' - filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT10H30M - - date: '2010-01-01T12:00:00Z' - filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT12H - - date: '2010-01-01T13:30:00Z' - filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT13H30M - - date: '2010-01-01T15:00:00Z' - filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT15H - - states: - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T04:30:00Z' - filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT4H30M - - date: '2010-01-01T06:00:00Z' - filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT6H - - date: '2010-01-01T07:30:00Z' - filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT7H30M - - date: '2010-01-01T09:00:00Z' - filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT9H - - date: '2010-01-01T10:30:00Z' - filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT10H30M - - date: '2010-01-01T12:00:00Z' - filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT12H - - date: '2010-01-01T13:30:00Z' - filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT13H30M - - date: '2010-01-01T15:00:00Z' - filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT15H - ensemble weight: '0.4' - static: - covariance model: L95Error - date: '2010-01-01T03:00:00Z' - length_scale: 1.0 - standard_deviation: 0.6 - static weight: '0.6' + standard_deviation: 0.6 + weight: + value: 0.6 # constraints: # - jcdfi: # alpha: 1000.0 diff --git a/l95/test/testinput/4dvar.hybrid.yaml b/l95/test/testinput/4dvar.hybrid.yaml index daa4a7858..4eabc5c6f 100644 --- a/l95/test/testinput/4dvar.hybrid.yaml +++ b/l95/test/testinput/4dvar.hybrid.yaml @@ -14,40 +14,43 @@ cost function: filename: Data/test.fc.2010-01-01T00:00:00Z.PT3H background error: covariance model: hybrid - ensemble: - covariance model: ensemble - date: '2010-01-01T03:00:00Z' - localization: + components: + - covariance: + covariance model: ensemble + date: '2010-01-01T03:00:00Z' + localization: + length_scale: 1.0 + localization method: L95 + members: + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT3H + - date: '2010-01-01T03:00:00Z' + filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT3H + weight: + value: 0.4 + - covariance: + covariance model: L95Error + date: '2010-01-01T03:00:00Z' length_scale: 1.0 - localization method: L95 - members: - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.1.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.2.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.3.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.4.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.5.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.6.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.7.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.8.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.9.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T03:00:00Z' - filename: Data/test.ens.10.2010-01-01T00:00:00Z.PT3H - ensemble weight: '0.4' - static: - covariance model: L95Error - date: '2010-01-01T03:00:00Z' - length_scale: 1.0 - standard_deviation: 0.6 - static weight: '0.6' + standard_deviation: 0.6 + weight: + value: 0.6 observations: - obs error: covariance model: diagonal diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index 24dca6a10..e7e0722ae 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -24,7 +24,8 @@ list( APPEND qg_testinput testinput/dfi.yaml testinput/diffstates.yaml testinput/dirac_cov.yaml - testinput/dirac_hyb.yaml + testinput/dirac_hyb_field.yaml + testinput/dirac_hyb_value.yaml testinput/dirac_loc_3d.yaml testinput/dirac_loc_4d.yaml testinput/dirac_no_loc.yaml @@ -74,7 +75,8 @@ list( APPEND qg_testinput testinput/static_b_init.yaml testinput/truth.yaml testinput/verticallocev.yaml - testinput/uniform_field.yaml + testinput/uniform_field_hybrid.yaml + testinput/uniform_field_inflation.yaml ) list( APPEND qg_testoutput @@ -103,7 +105,8 @@ list( APPEND qg_testoutput testoutput/dfi.test testoutput/diffstates.test testoutput/dirac_cov.test - testoutput/dirac_hyb.test + testoutput/dirac_hyb_field.test + testoutput/dirac_hyb_value.test testoutput/dirac_loc_3d.test testoutput/dirac_loc_4d.test testoutput/dirac_no_loc.test @@ -127,7 +130,8 @@ list( APPEND qg_testoutput testoutput/rtpp.test testoutput/static_b_init.test testoutput/truth.test - testoutput/uniform_field.test + testoutput/uniform_field_hybrid.test + testoutput/uniform_field_inflation.test ) # Create data directory for test input and symlink all files @@ -450,10 +454,10 @@ oops_add_test( TESTNAME ens_recenter EXENAME qg_ens_recenter.x TEST_DEPENDS test_qg_gen_ens_pert_B ) -oops_add_test( TESTNAME uniform_field +oops_add_test( TESTNAME uniform_field_inflation MODELNAME qg OMP 2 - YAMLNAME testinput/uniform_field.yaml + YAMLNAME testinput/uniform_field_inflation.yaml EXENAME qg_forecast.x ) oops_add_test( TESTNAME ens_variance_inflation_field @@ -461,7 +465,7 @@ oops_add_test( TESTNAME ens_variance_inflation_field OMP 2 YAMLNAME testinput/ens_variance_inflation_field.yaml EXENAME qg_ens_variance.x - TEST_DEPENDS test_qg_gen_ens_pert_B test_qg_uniform_field ) + TEST_DEPENDS test_qg_gen_ens_pert_B test_qg_uniform_field_inflation ) oops_add_test( TESTNAME ens_variance_inflation_value MODELNAME qg @@ -513,13 +517,26 @@ oops_add_test( TESTNAME dirac_cov EXENAME qg_dirac.x TEST_DEPENDS test_qg_forecast ) -oops_add_test( TESTNAME dirac_hyb +oops_add_test( TESTNAME dirac_hyb_value MODELNAME qg OMP 2 - YAMLNAME testinput/dirac_hyb.yaml + YAMLNAME testinput/dirac_hyb_value.yaml EXENAME qg_dirac.x TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B ) +oops_add_test( TESTNAME uniform_field_hybrid + MODELNAME qg + OMP 2 + YAMLNAME testinput/uniform_field_hybrid.yaml + EXENAME qg_forecast.x ) + +oops_add_test( TESTNAME dirac_hyb_field + MODELNAME qg + OMP 2 + YAMLNAME testinput/dirac_hyb_field.yaml + EXENAME qg_dirac.x + TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_uniform_field_hybrid ) + oops_add_test( TESTNAME dirac_loc_3d MODELNAME qg OMP 2 diff --git a/qg/test/testinput/3dvar_hybrid.yaml b/qg/test/testinput/3dvar_hybrid.yaml index 8f7731720..51c6b2fb7 100644 --- a/qg/test/testinput/3dvar_hybrid.yaml +++ b/qg/test/testinput/3dvar_hybrid.yaml @@ -8,33 +8,37 @@ cost function: filename: Data/forecast.fc.2009-12-31T00:00:00Z.P1DT12H.nc background error: covariance model: hybrid - static: - covariance model: QgError - horizontal_length_scale: 2.2e6 - maximum_condition_number: 1.0e6 - standard_deviation: 1.8e7 - vertical_length_scale: 15000.0 - static weight: 0.5 - ensemble: - date: 2010-01-01T12:00:00Z - localization: - horizontal_length_scale: 4.0e6 - localization method: QG + components: + - covariance: + covariance model: QgError + horizontal_length_scale: 2.2e6 maximum_condition_number: 1.0e6 - standard_deviation: 1.0 - vertical_length_scale: 30000.0 - members: - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT12H.nc - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT12H.nc - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT12H.nc - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT12H.nc - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT12H.nc - ensemble weight: 0.5 + standard_deviation: 1.8e7 + vertical_length_scale: 15000.0 + weight: + value: 0.5 + - covariance: + covariance model: ensemble + date: 2010-01-01T12:00:00Z + localization: + horizontal_length_scale: 4.0e6 + localization method: QG + maximum_condition_number: 1.0e6 + standard_deviation: 1.0 + vertical_length_scale: 30000.0 + members: + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT12H.nc + weight: + value: 0.5 observations: - obs operator: obs type: Stream diff --git a/qg/test/testinput/4densvar_hybrid.yaml b/qg/test/testinput/4densvar_hybrid.yaml index 1b97fbfcf..1155abbb5 100644 --- a/qg/test/testinput/4densvar_hybrid.yaml +++ b/qg/test/testinput/4densvar_hybrid.yaml @@ -22,172 +22,176 @@ cost function: filename: Data/forecast.fc.2009-12-31T00:00:00Z.P1DT6H.nc background error: covariance model: hybrid - ensemble: - localization: - horizontal_length_scale: 2.0e6 - localization method: QG + components: + - covariance: + covariance model: ensemble + localization: + horizontal_length_scale: 2.0e6 + localization method: QG + maximum_condition_number: 1.0e6 + standard_deviation: 1.0 + vertical_length_scale: 3694.0 + members: + - states: + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T01:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT1H.nc + - date: 2010-01-01T02:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT2H.nc + - date: 2010-01-01T03:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT3H.nc + - date: 2010-01-01T04:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT4H.nc + - date: 2010-01-01T05:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT5H.nc + - date: 2010-01-01T06:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT6H.nc + - states: + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T01:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT1H.nc + - date: 2010-01-01T02:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT2H.nc + - date: 2010-01-01T03:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT3H.nc + - date: 2010-01-01T04:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT4H.nc + - date: 2010-01-01T05:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT5H.nc + - date: 2010-01-01T06:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT6H.nc + - states: + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T01:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT1H.nc + - date: 2010-01-01T02:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT2H.nc + - date: 2010-01-01T03:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT3H.nc + - date: 2010-01-01T04:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT4H.nc + - date: 2010-01-01T05:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT5H.nc + - date: 2010-01-01T06:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT6H.nc + - states: + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T01:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT1H.nc + - date: 2010-01-01T02:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT2H.nc + - date: 2010-01-01T03:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT3H.nc + - date: 2010-01-01T04:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT4H.nc + - date: 2010-01-01T05:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT5H.nc + - date: 2010-01-01T06:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT6H.nc + - states: + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T01:00:00Z + filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT1H.nc + - date: 2010-01-01T02:00:00Z + filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT2H.nc + - date: 2010-01-01T03:00:00Z + filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT3H.nc + - date: 2010-01-01T04:00:00Z + filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT4H.nc + - date: 2010-01-01T05:00:00Z + filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT5H.nc + - date: 2010-01-01T06:00:00Z + filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT6H.nc + - states: + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T01:00:00Z + filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT1H.nc + - date: 2010-01-01T02:00:00Z + filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT2H.nc + - date: 2010-01-01T03:00:00Z + filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT3H.nc + - date: 2010-01-01T04:00:00Z + filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT4H.nc + - date: 2010-01-01T05:00:00Z + filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT5H.nc + - date: 2010-01-01T06:00:00Z + filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT6H.nc + - states: + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T01:00:00Z + filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT1H.nc + - date: 2010-01-01T02:00:00Z + filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT2H.nc + - date: 2010-01-01T03:00:00Z + filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT3H.nc + - date: 2010-01-01T04:00:00Z + filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT4H.nc + - date: 2010-01-01T05:00:00Z + filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT5H.nc + - date: 2010-01-01T06:00:00Z + filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT6H.nc + - states: + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T01:00:00Z + filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT1H.nc + - date: 2010-01-01T02:00:00Z + filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT2H.nc + - date: 2010-01-01T03:00:00Z + filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT3H.nc + - date: 2010-01-01T04:00:00Z + filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT4H.nc + - date: 2010-01-01T05:00:00Z + filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT5H.nc + - date: 2010-01-01T06:00:00Z + filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT6H.nc + - states: + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T01:00:00Z + filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT1H.nc + - date: 2010-01-01T02:00:00Z + filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT2H.nc + - date: 2010-01-01T03:00:00Z + filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT3H.nc + - date: 2010-01-01T04:00:00Z + filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT4H.nc + - date: 2010-01-01T05:00:00Z + filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT5H.nc + - date: 2010-01-01T06:00:00Z + filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT6H.nc + - states: + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T01:00:00Z + filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT1H.nc + - date: 2010-01-01T02:00:00Z + filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT2H.nc + - date: 2010-01-01T03:00:00Z + filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT3H.nc + - date: 2010-01-01T04:00:00Z + filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT4H.nc + - date: 2010-01-01T05:00:00Z + filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT5H.nc + - date: 2010-01-01T06:00:00Z + filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT6H.nc + weight: + value: 0.5 + - covariance: + covariance model: QgError + horizontal_length_scale: 2.2e6 maximum_condition_number: 1.0e6 - standard_deviation: 1.0 - vertical_length_scale: 3694.0 - members: - - states: - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T01:00:00Z - filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT1H.nc - - date: 2010-01-01T02:00:00Z - filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT2H.nc - - date: 2010-01-01T03:00:00Z - filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT3H.nc - - date: 2010-01-01T04:00:00Z - filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT4H.nc - - date: 2010-01-01T05:00:00Z - filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT5H.nc - - date: 2010-01-01T06:00:00Z - filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT6H.nc - - states: - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T01:00:00Z - filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT1H.nc - - date: 2010-01-01T02:00:00Z - filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT2H.nc - - date: 2010-01-01T03:00:00Z - filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT3H.nc - - date: 2010-01-01T04:00:00Z - filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT4H.nc - - date: 2010-01-01T05:00:00Z - filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT5H.nc - - date: 2010-01-01T06:00:00Z - filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT6H.nc - - states: - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T01:00:00Z - filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT1H.nc - - date: 2010-01-01T02:00:00Z - filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT2H.nc - - date: 2010-01-01T03:00:00Z - filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT3H.nc - - date: 2010-01-01T04:00:00Z - filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT4H.nc - - date: 2010-01-01T05:00:00Z - filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT5H.nc - - date: 2010-01-01T06:00:00Z - filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT6H.nc - - states: - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T01:00:00Z - filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT1H.nc - - date: 2010-01-01T02:00:00Z - filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT2H.nc - - date: 2010-01-01T03:00:00Z - filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT3H.nc - - date: 2010-01-01T04:00:00Z - filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT4H.nc - - date: 2010-01-01T05:00:00Z - filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT5H.nc - - date: 2010-01-01T06:00:00Z - filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT6H.nc - - states: - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T01:00:00Z - filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT1H.nc - - date: 2010-01-01T02:00:00Z - filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT2H.nc - - date: 2010-01-01T03:00:00Z - filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT3H.nc - - date: 2010-01-01T04:00:00Z - filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT4H.nc - - date: 2010-01-01T05:00:00Z - filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT5H.nc - - date: 2010-01-01T06:00:00Z - filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT6H.nc - - states: - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T01:00:00Z - filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT1H.nc - - date: 2010-01-01T02:00:00Z - filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT2H.nc - - date: 2010-01-01T03:00:00Z - filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT3H.nc - - date: 2010-01-01T04:00:00Z - filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT4H.nc - - date: 2010-01-01T05:00:00Z - filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT5H.nc - - date: 2010-01-01T06:00:00Z - filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT6H.nc - - states: - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T01:00:00Z - filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT1H.nc - - date: 2010-01-01T02:00:00Z - filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT2H.nc - - date: 2010-01-01T03:00:00Z - filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT3H.nc - - date: 2010-01-01T04:00:00Z - filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT4H.nc - - date: 2010-01-01T05:00:00Z - filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT5H.nc - - date: 2010-01-01T06:00:00Z - filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT6H.nc - - states: - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T01:00:00Z - filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT1H.nc - - date: 2010-01-01T02:00:00Z - filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT2H.nc - - date: 2010-01-01T03:00:00Z - filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT3H.nc - - date: 2010-01-01T04:00:00Z - filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT4H.nc - - date: 2010-01-01T05:00:00Z - filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT5H.nc - - date: 2010-01-01T06:00:00Z - filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT6H.nc - - states: - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T01:00:00Z - filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT1H.nc - - date: 2010-01-01T02:00:00Z - filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT2H.nc - - date: 2010-01-01T03:00:00Z - filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT3H.nc - - date: 2010-01-01T04:00:00Z - filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT4H.nc - - date: 2010-01-01T05:00:00Z - filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT5H.nc - - date: 2010-01-01T06:00:00Z - filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT6H.nc - - states: - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T01:00:00Z - filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT1H.nc - - date: 2010-01-01T02:00:00Z - filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT2H.nc - - date: 2010-01-01T03:00:00Z - filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT3H.nc - - date: 2010-01-01T04:00:00Z - filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT4H.nc - - date: 2010-01-01T05:00:00Z - filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT5H.nc - - date: 2010-01-01T06:00:00Z - filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT6H.nc - ensemble weight: 0.5 - static: - covariance model: QgError - horizontal_length_scale: 2.2e6 - maximum_condition_number: 1.0e6 - standard_deviation: 1.8e7 - vertical_length_scale: 15000.0 - static weight: 0.5 + standard_deviation: 1.8e7 + vertical_length_scale: 15000.0 + weight: + value: 0.5 observations: - obs error: covariance model: diagonal diff --git a/qg/test/testinput/4dvar_drplanczos_hybrid.yaml b/qg/test/testinput/4dvar_drplanczos_hybrid.yaml index 6eb5cd81e..4bee9b610 100644 --- a/qg/test/testinput/4dvar_drplanczos_hybrid.yaml +++ b/qg/test/testinput/4dvar_drplanczos_hybrid.yaml @@ -15,42 +15,46 @@ cost function: filename: Data/forecast.fc.2009-12-31T00:00:00Z.P1D.nc background error: covariance model: hybrid - ensemble: - localization: - horizontal_length_scale: 4.0e6 - localization method: QG + components: + - covariance: + covariance model: ensemble + localization: + horizontal_length_scale: 4.0e6 + localization method: QG + maximum_condition_number: 1.0e6 + standard_deviation: 1.0 + vertical_length_scale: 30000.0 + members: + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1D.nc + - date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1D.nc + weight: + value: 0.5 + - covariance: + covariance model: QgError + horizontal_length_scale: 2.2e6 maximum_condition_number: 1.0e6 - standard_deviation: 1.0 - vertical_length_scale: 30000.0 - members: - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1D.nc - - date: 2010-01-01T00:00:00Z - filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1D.nc - ensemble weight: 0.5 - static: - covariance model: QgError - horizontal_length_scale: 2.2e6 - maximum_condition_number: 1.0e6 - standard_deviation: 1.8e7 - vertical_length_scale: 15000.0 - static weight: 0.5 + standard_deviation: 1.8e7 + vertical_length_scale: 15000.0 + weight: + value: 0.5 observations: - obs error: covariance model: diagonal diff --git a/qg/test/testinput/dirac_hyb.yaml b/qg/test/testinput/dirac_hyb.yaml deleted file mode 100644 index f8cd33e1d..000000000 --- a/qg/test/testinput/dirac_hyb.yaml +++ /dev/null @@ -1,62 +0,0 @@ -background error: - covariance model: hybrid - ensemble: - localization: - horizontal_length_scale: 4.0e6 - localization method: QG - maximum_condition_number: 1.0e6 - standard_deviation: 1.0 - vertical_length_scale: 30000.0 - members: - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT12H.nc - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT12H.nc - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT12H.nc - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT12H.nc - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT12H.nc - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT12H.nc - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT12H.nc - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT12H.nc - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT12H.nc - - date: 2010-01-01T12:00:00Z - filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT12H.nc - ensemble weight: 0.5 - static: - covariance model: QgError - horizontal_length_scale: 2.2e6 - maximum_condition_number: 1.0e6 - standard_deviation: 1.8e7 - vertical_length_scale: 15000.0 - static weight: 0.5 -dirac: - date: 2010-01-01T12:00:00Z - ixdir: [20] - iydir: [10] - izdir: [1] -geometry: - nx: 40 - ny: 20 - depths: [4500.0, 5500.0] -initial condition: - date: 2010-01-01T12:00:00Z - filename: Data/forecast.fc.2009-12-31T00:00:00Z.P1DT12H.nc -model: - tstep: PT1H -output B: - datadir: Data - date: 2010-01-01T12:00:00Z - exp: dirac_qg_hyb_B - type: an -output localization: - datadir: Data - date: 2010-01-01T12:00:00Z - exp: dirac_qg_hyb_localization - type: an diff --git a/qg/test/testinput/dirac_hyb_field.yaml b/qg/test/testinput/dirac_hyb_field.yaml new file mode 100644 index 000000000..87d86adc2 --- /dev/null +++ b/qg/test/testinput/dirac_hyb_field.yaml @@ -0,0 +1,68 @@ +background error: + covariance model: hybrid + components: + - covariance: + covariance model: ensemble + localization: + horizontal_length_scale: 4.0e6 + localization method: QG + maximum_condition_number: 1.0e6 + standard_deviation: 1.0 + vertical_length_scale: 30000.0 + members: + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT12H.nc + weight: + date: 2010-01-01T00:00:00Z + filename: Data/uniform_field_hybrid.fc.2010-01-01T00:00:00Z.PT0S.nc + - covariance: + covariance model: QgError + horizontal_length_scale: 2.2e6 + maximum_condition_number: 1.0e6 + standard_deviation: 1.8e7 + vertical_length_scale: 15000.0 + weight: + date: 2010-01-01T00:00:00Z + filename: Data/uniform_field_hybrid.fc.2010-01-01T00:00:00Z.PT0S.nc +dirac: + date: 2010-01-01T12:00:00Z + ixdir: [20] + iydir: [10] + izdir: [1] +geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] +initial condition: + date: 2010-01-01T12:00:00Z + filename: Data/forecast.fc.2009-12-31T00:00:00Z.P1DT12H.nc +model: + tstep: PT1H +output B: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: dirac_qg_hyb_field_B + type: an +output localization: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: dirac_qg_hyb_field_localization + type: an diff --git a/qg/test/testinput/dirac_hyb_value.yaml b/qg/test/testinput/dirac_hyb_value.yaml new file mode 100644 index 000000000..bfa053a8a --- /dev/null +++ b/qg/test/testinput/dirac_hyb_value.yaml @@ -0,0 +1,66 @@ +background error: + covariance model: hybrid + components: + - covariance: + covariance model: ensemble + localization: + horizontal_length_scale: 4.0e6 + localization method: QG + maximum_condition_number: 1.0e6 + standard_deviation: 1.0 + vertical_length_scale: 30000.0 + members: + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.6.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.7.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.8.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.9.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT12H.nc + weight: + value: 0.5 + - covariance: + covariance model: QgError + horizontal_length_scale: 2.2e6 + maximum_condition_number: 1.0e6 + standard_deviation: 1.8e7 + vertical_length_scale: 15000.0 + weight: + value: 0.5 +dirac: + date: 2010-01-01T12:00:00Z + ixdir: [20] + iydir: [10] + izdir: [1] +geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] +initial condition: + date: 2010-01-01T12:00:00Z + filename: Data/forecast.fc.2009-12-31T00:00:00Z.P1DT12H.nc +model: + tstep: PT1H +output B: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: dirac_qg_hyb_value_B + type: an +output localization: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: dirac_qg_hyb_value_localization + type: an diff --git a/qg/test/testinput/ens_variance_inflation_field.yaml b/qg/test/testinput/ens_variance_inflation_field.yaml index c9c2870c9..415e77d29 100644 --- a/qg/test/testinput/ens_variance_inflation_field.yaml +++ b/qg/test/testinput/ens_variance_inflation_field.yaml @@ -16,7 +16,7 @@ ensemble: filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1D.nc inflation field: date: 2010-01-01T00:00:00Z - filename: Data/uniform_field.fc.2010-01-01T00:00:00Z.PT0S.nc + filename: Data/uniform_field_inflation.fc.2010-01-01T00:00:00Z.PT0S.nc geometry: nx: 40 ny: 20 diff --git a/qg/test/testinput/uniform_field_hybrid.yaml b/qg/test/testinput/uniform_field_hybrid.yaml new file mode 100644 index 000000000..d60105ddf --- /dev/null +++ b/qg/test/testinput/uniform_field_hybrid.yaml @@ -0,0 +1,21 @@ +forecast length: PT0S +initial condition: + analytic_init: uniform_field + date: 2010-01-01T00:00:00Z + uval: 0.5 +model: + name: QG + tstep: PT10M +output: + datadir: Data + date: 2010-01-01T00:00:00Z + exp: uniform_field_hybrid + frequency: PT12H + type: fc +prints: + frequency: PT3H +geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + heating: false diff --git a/qg/test/testinput/uniform_field.yaml b/qg/test/testinput/uniform_field_inflation.yaml similarity index 91% rename from qg/test/testinput/uniform_field.yaml rename to qg/test/testinput/uniform_field_inflation.yaml index 096eb44df..23db796e7 100644 --- a/qg/test/testinput/uniform_field.yaml +++ b/qg/test/testinput/uniform_field_inflation.yaml @@ -9,7 +9,7 @@ model: output: datadir: Data date: 2010-01-01T00:00:00Z - exp: uniform_field + exp: uniform_field_inflation frequency: PT12H type: fc prints: diff --git a/qg/test/testoutput/dirac_hyb.test b/qg/test/testoutput/dirac_hyb_field.test similarity index 100% rename from qg/test/testoutput/dirac_hyb.test rename to qg/test/testoutput/dirac_hyb_field.test diff --git a/qg/test/testoutput/dirac_hyb_value.test b/qg/test/testoutput/dirac_hyb_value.test new file mode 100644 index 000000000..f8d746c5c --- /dev/null +++ b/qg/test/testoutput/dirac_hyb_value.test @@ -0,0 +1,18 @@ +Test : Input Dirac increment: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 0.1, Min= 0.0000, Max= 10.0000, RMS= 0.2500 +Test : B * Increment: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+13, Min= -1.3348, Max= 31.2069, RMS= 5.8758 +Test : Localized Increment: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1, Min= 0.0000, Max= 1.0000, RMS= 0.3433 diff --git a/qg/test/testoutput/uniform_field_hybrid.test b/qg/test/testoutput/uniform_field_hybrid.test new file mode 100644 index 000000000..50c4d4fa7 --- /dev/null +++ b/qg/test/testoutput/uniform_field_hybrid.test @@ -0,0 +1,16 @@ +Test : Initial state: +Test : Valid time: 2010-01-01T00:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are activated +Test : Scaling= 1, Min= 0.5000, Max= 0.5000, RMS= 0.5000 +Test : Scaling= 1, Min= 0.0000, Max= 0.0000, RMS= 0.0000 +Test : Scaling= 0.0001, Min= -0.7149, Max= 0.7863, RMS= 0.7514 +Test : Final state: +Test : Valid time: 2010-01-01T00:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are activated +Test : Scaling= 1, Min= 0.5000, Max= 0.5000, RMS= 0.5000 +Test : Scaling= 1, Min= 0.0000, Max= 0.0000, RMS= 0.0000 +Test : Scaling= 0.0001, Min= -0.7149, Max= 0.7863, RMS= 0.7514 diff --git a/qg/test/testoutput/uniform_field.test b/qg/test/testoutput/uniform_field_inflation.test similarity index 100% rename from qg/test/testoutput/uniform_field.test rename to qg/test/testoutput/uniform_field_inflation.test diff --git a/src/oops/base/HybridCovariance.h b/src/oops/base/HybridCovariance.h index f3d3de540..fb9db087a 100644 --- a/src/oops/base/HybridCovariance.h +++ b/src/oops/base/HybridCovariance.h @@ -12,6 +12,9 @@ #define OOPS_BASE_HYBRIDCOVARIANCE_H_ #include +#include +#include +#include #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" @@ -47,10 +50,10 @@ class HybridCovariance : public ModelSpaceCovarianceBase { void doMultiply(const Increment_ &, Increment_ &) const override; void doInverseMultiply(const Increment_ &, Increment_ &) const override; - std::unique_ptr< ModelSpaceCovarianceBase > static_; - std::unique_ptr< EnsembleCovariance > ensemble_; - double ensWeight_; - double staWeight_; + std::vector< std::unique_ptr< ModelSpaceCovarianceBase > > Bcomponents_; + std::vector< std::string > weightTypes_; + std::vector< double > valueWeights_; + std::vector< Increment_ > incrementWeights_; }; // ============================================================================= @@ -61,17 +64,31 @@ template HybridCovariance::HybridCovariance(const Geometry_ & resol, const Variables & vars, const eckit::Configuration & config, const State_ & xb, const State_ & fg) - : ModelSpaceCovarianceBase(xb, fg, resol, config), - static_(CovarianceFactory::create( - eckit::LocalConfiguration(config, "static"), resol, vars, xb, fg)) + : ModelSpaceCovarianceBase(xb, fg, resol, config) { - const eckit::LocalConfiguration ensConf(config, "ensemble"); - ensemble_.reset(new EnsembleCovariance(resol, vars, ensConf, xb, fg)); + std::vector confs; + config.get("components", confs); + for (const auto & conf : confs) { + // B component + const eckit::LocalConfiguration covarConf(conf, "covariance"); + std::unique_ptr< ModelSpaceCovarianceBase > B( + CovarianceFactory::create(covarConf, resol, vars, xb, fg)); + Bcomponents_.push_back(std::move(B)); - ensWeight_ = config.getDouble("ensemble weight"); - ASSERT(ensWeight_ > 0.0); - staWeight_ = config.getDouble("static weight"); - ASSERT(staWeight_ > 0.0); + // Weight + const eckit::LocalConfiguration weightConf(conf, "weight"); + if (weightConf.has("value")) { + // Scalar weight provided in the configuration + weightTypes_.push_back("value"); + valueWeights_.push_back(weightConf.getDouble("value")); + } else { + // 3D weight read from a file + weightTypes_.push_back("increment"); + Increment_ weight(resol, vars, xb.validTime()); + weight.read(weightConf); + incrementWeights_.push_back(weight); + } + } Log::trace() << "HybridCovariance created." << std::endl; } // ----------------------------------------------------------------------------- @@ -82,11 +99,22 @@ HybridCovariance::~HybridCovariance() { // ----------------------------------------------------------------------------- template void HybridCovariance::doMultiply(const Increment_ & dxi, Increment_ & dxo) const { - static_->multiply(dxi, dxo); - dxo *= staWeight_; + dxo.zero(); Increment_ tmp(dxo); - ensemble_->multiply(dxi, tmp); - dxo.axpy(ensWeight_, tmp); + int valueIndex = 0; + int incrementIndex = 0; + for (size_t jcomp = 0; jcomp < Bcomponents_.size(); ++jcomp) { + Bcomponents_[jcomp]->multiply(dxi, tmp); + if (weightTypes_[jcomp] == "value") { + tmp *= valueWeights_[valueIndex]; + valueIndex += 1; + } + if (weightTypes_[jcomp] == "increment") { + tmp.schur_product_with(incrementWeights_[incrementIndex]); + incrementIndex += 1; + } + dxo += tmp; + } } // ----------------------------------------------------------------------------- template diff --git a/src/oops/runs/Dirac.h b/src/oops/runs/Dirac.h index be93952ab..500f0935a 100644 --- a/src/oops/runs/Dirac.h +++ b/src/oops/runs/Dirac.h @@ -114,24 +114,23 @@ template class Dirac : public Application { Log::test() << "B * Increment: " << dxdirout << std::endl; // Setup localization and ensemble configurations - eckit::LocalConfiguration locConfig; - eckit::LocalConfiguration ensConfig; - bool hasLoc(false); + std::vector locConfigs; if (covarConfig.has("localization")) { - locConfig = eckit::LocalConfiguration(covarConfig, "localization"); - ensConfig = covarConfig; - hasLoc = true; + locConfigs.push_back(eckit::LocalConfiguration(covarConfig, "localization")); } else { - if (covarConfig.has("ensemble")) { - ensConfig = eckit::LocalConfiguration(covarConfig, "ensemble"); - if (ensConfig.has("localization")) { - locConfig = eckit::LocalConfiguration(ensConfig, "localization"); - hasLoc = true; + if (covarConfig.has("components")) { + std::vector confs; + covarConfig.get("components", confs); + for (const auto & conf : confs) { + const eckit::LocalConfiguration componentConf(conf, "covariance"); + if (componentConf.has("localization")) { + locConfigs.push_back(eckit::LocalConfiguration(componentConf, "localization")); + } } } } - if (hasLoc) { + for (size_t jcomp = 0; jcomp < locConfigs.size(); ++jcomp) { // Apply localization to Dirac // Setup Dirac @@ -141,7 +140,7 @@ template class Dirac : public Application { // Setup localization std::unique_ptr loc_ = - LocalizationFactory::create(resol, time, locConfig); + LocalizationFactory::create(resol, time, locConfigs[jcomp]); // Apply localization loc_->localize(dxdir); From e04b45d259be5c673c8a8030f386793dd719251e Mon Sep 17 00:00:00 2001 From: Claude Gibert Date: Thu, 21 Jan 2021 17:57:42 +0000 Subject: [PATCH 044/142] fixed infinite recursion. (#1041) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Yannick Trémolet <30638944+ytremolet@users.noreply.github.com> --- ewok/hofx3d.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ewok/hofx3d.yaml b/ewok/hofx3d.yaml index 3f85a2b2f..a3eca0826 100644 --- a/ewok/hofx3d.yaml +++ b/ewok/hofx3d.yaml @@ -7,4 +7,4 @@ forecasts: observations: $(OBSERVATIONS) window begin: '{{window_begin}}' -window length: $(window length) +window length: $(window_length) From 888b7d2d387d63033b263cfa546b4f899922c79c Mon Sep 17 00:00:00 2001 From: Benjamin Menetrier <30638301+benjaminmenetrier@users.noreply.github.com> Date: Fri, 22 Jan 2021 14:37:00 +0100 Subject: [PATCH 045/142] Get randomized variance (#1026) * Generic hybrid using 3D weight * Add option to read hybrid weights from files * Update tests, add new ones * Change yaml structure to respect parameters checks * Save memory for scalar hybrid weights * Initial commit * Empty interface for the L95 model * Implement randomization for L95 * Cleaning * Add randomization size in covariance parameters Co-authored-by: Anna Shlyaeva --- l95/src/lorenz95/LocalizationMatrixL95.cc | 18 +++-- l95/src/lorenz95/LocalizationMatrixL95.h | 1 + qg/model/LocalizationMatrixQG.cc | 4 ++ qg/model/LocalizationMatrixQG.h | 1 + qg/test/testinput/dirac_cov.yaml | 7 +- qg/test/testinput/dirac_hyb_value.yaml | 4 ++ qg/test/testinput/dirac_loc_3d.yaml | 12 +++- qg/test/testinput/dirac_loc_4d.yaml | 8 ++- qg/test/testinput/dirac_no_loc.yaml | 4 ++ qg/test/testoutput/dirac_cov.test | 10 ++- qg/test/testoutput/dirac_hyb_field.test | 6 +- qg/test/testoutput/dirac_hyb_value.test | 6 ++ qg/test/testoutput/dirac_loc_3d.test | 12 +++- qg/test/testoutput/dirac_loc_4d.test | 72 ++++++++++++++++++- qg/test/testoutput/dirac_no_loc.test | 10 ++- src/oops/base/EnsembleCovariance.h | 31 ++++++-- src/oops/base/HybridCovariance.h | 17 ++++- src/oops/base/LocalizationBase.h | 46 ++++++++++-- src/oops/base/ModelSpaceCovarianceBase.h | 27 +++++++ .../base/ModelSpaceCovarianceParametersBase.h | 1 + src/oops/interface/Localization.h | 21 ++++-- src/oops/runs/Dirac.h | 17 ++++- src/test/interface/Localization.h | 18 ++++- 23 files changed, 305 insertions(+), 48 deletions(-) diff --git a/l95/src/lorenz95/LocalizationMatrixL95.cc b/l95/src/lorenz95/LocalizationMatrixL95.cc index 24a002d80..66c31a4de 100644 --- a/l95/src/lorenz95/LocalizationMatrixL95.cc +++ b/l95/src/lorenz95/LocalizationMatrixL95.cc @@ -24,7 +24,6 @@ // ----------------------------------------------------------------------------- namespace lorenz95 { // ----------------------------------------------------------------------------- - LocalizationMatrixL95::LocalizationMatrixL95(const Resolution & resol, const eckit::Configuration & config) : resol_(resol.npoints()), @@ -47,9 +46,19 @@ LocalizationMatrixL95::LocalizationMatrixL95(const Resolution & resol, coefs_[jj] = std::real(four[jj]); } } - // ----------------------------------------------------------------------------- - +void LocalizationMatrixL95::randomize(IncrementL95 & dx) const { + dx.random(); + unsigned int size = resol_/2+1; + Eigen::FFT fft; + std::vector > four(size); + fft.fwd(four, dx.asVector()); + for (unsigned int jj = 0; jj < size; ++jj) { + four[jj] *= std::sqrt(coefs_[jj]); + } + fft.inv(dx.asVector(), four); +} +// ----------------------------------------------------------------------------- void LocalizationMatrixL95::multiply(IncrementL95 & dx) const { unsigned int size = resol_/2+1; Eigen::FFT fft; @@ -60,13 +69,10 @@ void LocalizationMatrixL95::multiply(IncrementL95 & dx) const { } fft.inv(dx.asVector(), four); } - // ----------------------------------------------------------------------------- - void LocalizationMatrixL95::print(std::ostream & os) const { os << "Localization with Gaussian, lengthscale = " << 1.0/rscale_; } - // ----------------------------------------------------------------------------- } // namespace lorenz95 diff --git a/l95/src/lorenz95/LocalizationMatrixL95.h b/l95/src/lorenz95/LocalizationMatrixL95.h index c8a5fbe18..0e29b43b4 100644 --- a/l95/src/lorenz95/LocalizationMatrixL95.h +++ b/l95/src/lorenz95/LocalizationMatrixL95.h @@ -37,6 +37,7 @@ class LocalizationMatrixL95: public util::Printable, static const std::string classname() {return "lorenz95::LocalizationMatrixL95";} LocalizationMatrixL95(const Resolution &, const eckit::Configuration &); + void randomize(IncrementL95 &) const; void multiply(IncrementL95 &) const; private: diff --git a/qg/model/LocalizationMatrixQG.cc b/qg/model/LocalizationMatrixQG.cc index c34e6a31d..5b36a9428 100644 --- a/qg/model/LocalizationMatrixQG.cc +++ b/qg/model/LocalizationMatrixQG.cc @@ -27,6 +27,10 @@ LocalizationMatrixQG::~LocalizationMatrixQG() { qg_error_covariance_delete_f90(keyLocal_); } // ----------------------------------------------------------------------------- +void LocalizationMatrixQG::randomize(IncrementQG & dx) const { + qg_error_covariance_randomize_f90(keyLocal_, dx.fields().toFortran()); +} +// ----------------------------------------------------------------------------- void LocalizationMatrixQG::multiply(IncrementQG & dx) const { IncrementQG dxtmp(dx); qg_error_covariance_mult_f90(keyLocal_, dxtmp.fields().toFortran(), dx.fields().toFortran()); diff --git a/qg/model/LocalizationMatrixQG.h b/qg/model/LocalizationMatrixQG.h index 1bec059d3..a6680f63a 100644 --- a/qg/model/LocalizationMatrixQG.h +++ b/qg/model/LocalizationMatrixQG.h @@ -41,6 +41,7 @@ class LocalizationMatrixQG: public util::Printable, LocalizationMatrixQG(const GeometryQG &, const eckit::Configuration &); ~LocalizationMatrixQG(); + void randomize(IncrementQG &) const; void multiply(IncrementQG &) const; private: diff --git a/qg/test/testinput/dirac_cov.yaml b/qg/test/testinput/dirac_cov.yaml index 33d89fe63..32be6d4a0 100644 --- a/qg/test/testinput/dirac_cov.yaml +++ b/qg/test/testinput/dirac_cov.yaml @@ -4,6 +4,7 @@ background error: maximum_condition_number: 1.0e6 standard_deviation: 1.8e7 vertical_length_scale: 15000.0 + randomization size: 1000 dirac: date: 2010-01-01T12:00:00Z ixdir: [20] @@ -20,5 +21,9 @@ model: tstep: PT1H output B: datadir: Data - exp: dirac_qg_cov + exp: dirac_cov_B + type: an +output variance: + datadir: Data + exp: dirac_cov_var type: an diff --git a/qg/test/testinput/dirac_hyb_value.yaml b/qg/test/testinput/dirac_hyb_value.yaml index bfa053a8a..fe7371b57 100644 --- a/qg/test/testinput/dirac_hyb_value.yaml +++ b/qg/test/testinput/dirac_hyb_value.yaml @@ -64,3 +64,7 @@ output localization: date: 2010-01-01T12:00:00Z exp: dirac_qg_hyb_value_localization type: an +output variance: + datadir: Data + exp: dirac_hyb_value_var + type: an diff --git a/qg/test/testinput/dirac_loc_3d.yaml b/qg/test/testinput/dirac_loc_3d.yaml index f89516b1c..5147512d4 100644 --- a/qg/test/testinput/dirac_loc_3d.yaml +++ b/qg/test/testinput/dirac_loc_3d.yaml @@ -44,10 +44,18 @@ model: output B: datadir: Data date: 2010-01-01T12:00:00Z - exp: dirac_qg_loc_3d_B + exp: dirac_loc_3d_B type: an output localization: datadir: Data date: 2010-01-01T00:00:00Z - exp: dirac_qg_loc_3d_localization + exp: dirac_loc_3d_localization + type: an +output variance: + datadir: Data + exp: dirac_loc_3d_var + type: an +output variance: + datadir: Data + exp: dirac_loc_3d_var type: an diff --git a/qg/test/testinput/dirac_loc_4d.yaml b/qg/test/testinput/dirac_loc_4d.yaml index cbc485da9..bda56e2e3 100644 --- a/qg/test/testinput/dirac_loc_4d.yaml +++ b/qg/test/testinput/dirac_loc_4d.yaml @@ -319,10 +319,14 @@ model: output B: datadir: Data date: 2010-01-01T00:00:00Z - exp: dirac_qg_loc_4d_B + exp: dirac_loc_4d_B type: an output localization: datadir: Data date: 2010-01-01T00:00:00Z - exp: dirac_qg_loc_4d_localization + exp: dirac_loc_4d_localization + type: an +output variance: + datadir: Data + exp: dirac_loc_4d_var type: an diff --git a/qg/test/testinput/dirac_no_loc.yaml b/qg/test/testinput/dirac_no_loc.yaml index 7f5dac238..bab5ad23e 100644 --- a/qg/test/testinput/dirac_no_loc.yaml +++ b/qg/test/testinput/dirac_no_loc.yaml @@ -40,3 +40,7 @@ output B: date: 2010-01-01T12:00:00Z exp: dirac_no_loc_B type: an +output variance: + datadir: Data + exp: dirac_no_loc_var + type: an diff --git a/qg/test/testoutput/dirac_cov.test b/qg/test/testoutput/dirac_cov.test index 7f96ac197..da2662bef 100644 --- a/qg/test/testoutput/dirac_cov.test +++ b/qg/test/testoutput/dirac_cov.test @@ -1,12 +1,18 @@ -Test : Input Dirac increment: +Test : Input Dirac increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are not activated Test : Scaling= 0.1, Min= 0.0000, Max= 10.0000, RMS= 0.2500 -Test : B * Increment: +Test : B * Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are not activated Test : Scaling= 1e+13, Min= -0.0000, Max= 32.4000, RMS= 6.2881 +Test : Randomized variance: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 2.8165, Max= 3.6315, RMS= 3.2388 diff --git a/qg/test/testoutput/dirac_hyb_field.test b/qg/test/testoutput/dirac_hyb_field.test index f8d746c5c..b18391e74 100644 --- a/qg/test/testoutput/dirac_hyb_field.test +++ b/qg/test/testoutput/dirac_hyb_field.test @@ -1,16 +1,16 @@ -Test : Input Dirac increment: +Test : Input Dirac increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are not activated Test : Scaling= 0.1, Min= 0.0000, Max= 10.0000, RMS= 0.2500 -Test : B * Increment: +Test : B * Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are not activated Test : Scaling= 1e+13, Min= -1.3348, Max= 31.2069, RMS= 5.8758 -Test : Localized Increment: +Test : Localized Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction diff --git a/qg/test/testoutput/dirac_hyb_value.test b/qg/test/testoutput/dirac_hyb_value.test index f8d746c5c..a9a75d89d 100644 --- a/qg/test/testoutput/dirac_hyb_value.test +++ b/qg/test/testoutput/dirac_hyb_value.test @@ -16,3 +16,9 @@ Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are not activated Test : Scaling= 1, Min= 0.0000, Max= 1.0000, RMS= 0.3433 +Test : Randomized variance: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 1.0521, Max= 9.2106, RMS= 3.1869 diff --git a/qg/test/testoutput/dirac_loc_3d.test b/qg/test/testoutput/dirac_loc_3d.test index 6d52b22a6..d87d47a55 100644 --- a/qg/test/testoutput/dirac_loc_3d.test +++ b/qg/test/testoutput/dirac_loc_3d.test @@ -1,18 +1,24 @@ -Test : Input Dirac increment: +Test : Input Dirac increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are not activated Test : Scaling= 0.1, Min= 0.0000, Max= 10.0000, RMS= 0.2500 -Test : B * Increment: +Test : B * Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are not activated Test : Scaling= 1e+13, Min= -6.5158, Max= 37.2551, RMS= 5.8877 -Test : Localized Increment: +Test : Localized Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are not activated Test : Scaling= 1, Min= 0.0000, Max= 1.0000, RMS= 0.3433 +Test : Randomized variance: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0133, Max= 14.2520, RMS= 3.6277 diff --git a/qg/test/testoutput/dirac_loc_4d.test b/qg/test/testoutput/dirac_loc_4d.test index 2203b586b..a9a8fd344 100644 --- a/qg/test/testoutput/dirac_loc_4d.test +++ b/qg/test/testoutput/dirac_loc_4d.test @@ -1,4 +1,4 @@ -Test : Input Dirac increment: +Test : Input Dirac increment: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction @@ -64,7 +64,7 @@ Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are not activated Test : Scaling= 1, Min= 0.0000, Max= 0.0000, RMS= 0.0000 -Test : B * Increment: +Test : B * Increment: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction @@ -130,7 +130,7 @@ Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are not activated Test : Scaling= 1e+13, Min= -2.2238, Max= 25.8411, RMS= 2.8326 -Test : Localized Increment: +Test : Localized Increment: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction @@ -196,3 +196,69 @@ Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are not activated Test : Scaling= 1, Min= -0.0000, Max= 1.0000, RMS= 0.1382 +Test : Randomized variance: +Test : Valid time: 2010-01-01T00:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0373, Max= 10.5384, RMS= 2.4663 +Test : Valid time: 2010-01-01T01:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0391, Max= 11.2991, RMS= 2.5694 +Test : Valid time: 2010-01-01T02:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0241, Max= 12.0195, RMS= 2.6754 +Test : Valid time: 2010-01-01T03:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0181, Max= 12.6787, RMS= 2.7848 +Test : Valid time: 2010-01-01T04:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0199, Max= 13.2664, RMS= 2.8969 +Test : Valid time: 2010-01-01T05:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0245, Max= 13.7668, RMS= 3.0102 +Test : Valid time: 2010-01-01T06:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0232, Max= 14.1630, RMS= 3.1227 +Test : Valid time: 2010-01-01T07:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0224, Max= 14.5678, RMS= 3.2326 +Test : Valid time: 2010-01-01T08:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0201, Max= 15.1222, RMS= 3.3397 +Test : Valid time: 2010-01-01T09:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0171, Max= 15.7176, RMS= 3.4435 +Test : Valid time: 2010-01-01T10:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0146, Max= 16.3334, RMS= 3.5419 +Test : Valid time: 2010-01-01T11:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0123, Max= 16.9441, RMS= 3.6352 +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0103, Max= 17.6977, RMS= 3.7255 diff --git a/qg/test/testoutput/dirac_no_loc.test b/qg/test/testoutput/dirac_no_loc.test index f48bb8cbd..2a3550ca9 100644 --- a/qg/test/testoutput/dirac_no_loc.test +++ b/qg/test/testoutput/dirac_no_loc.test @@ -1,12 +1,18 @@ -Test : Input Dirac increment: +Test : Input Dirac increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are not activated Test : Scaling= 0.1, Min= 0.0000, Max= 10.0000, RMS= 0.2500 -Test : B * Increment: +Test : B * Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are not activated Test : Scaling= 1e+14, Min= -3.0667, Max= 4.4252, RMS= 1.0739 +Test : Randomized variance: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Scaling= 1e+14, Min= 0.0119, Max= 17.1590, RMS= 3.9892 diff --git a/src/oops/base/EnsembleCovariance.h b/src/oops/base/EnsembleCovariance.h index 219d19ece..4f55a6de0 100644 --- a/src/oops/base/EnsembleCovariance.h +++ b/src/oops/base/EnsembleCovariance.h @@ -53,6 +53,7 @@ class EnsembleCovariance : public ModelSpaceCovarianceBase { EnsemblePtr_ ens_; std::unique_ptr loc_; + int seed_ = 7; // For reproducibility }; // ============================================================================= @@ -80,6 +81,27 @@ EnsembleCovariance::~EnsembleCovariance() { } // ----------------------------------------------------------------------------- template +void EnsembleCovariance::doRandomize(Increment_ & dx) const { + dx.zero(); + if (loc_) { + // Localized covariance matrix + for (unsigned int ie = 0; ie < ens_->size(); ++ie) { + Increment_ tmp(dx); + loc_->randomize(tmp); + tmp.schur_product_with((*ens_)[ie]); + dx.axpy(1.0, tmp, false); + } + } else { + // Raw covariance matrix + util::NormalDistribution normalDist(ens_->size(), 0.0, 1.0, seed_); + for (unsigned int ie = 0; ie < ens_->size(); ++ie) { + dx.axpy(normalDist[ie], (*ens_)[ie]); + } + } + dx *= 1.0/sqrt(static_cast(ens_->size()-1)); +} +// ----------------------------------------------------------------------------- +template void EnsembleCovariance::doMultiply(const Increment_ & dxi, Increment_ & dxo) const { dxo.zero(); for (unsigned int ie = 0; ie < ens_->size(); ++ie) { @@ -87,7 +109,7 @@ void EnsembleCovariance::doMultiply(const Increment_ & dxi, Increment_ & // Localized covariance matrix Increment_ dx(dxi); dx.schur_product_with((*ens_)[ie]); - loc_->localize(dx); + loc_->multiply(dx); dx.schur_product_with((*ens_)[ie]); dxo.axpy(1.0, dx, false); } else { @@ -96,7 +118,7 @@ void EnsembleCovariance::doMultiply(const Increment_ & dxi, Increment_ & dxo.axpy(wgt, (*ens_)[ie], false); } } - const double rk = 1.0/(static_cast(ens_->size()) - 1.0); + const double rk = 1.0/static_cast(ens_->size()-1); dxo *= rk; } // ----------------------------------------------------------------------------- @@ -107,11 +129,6 @@ void EnsembleCovariance::doInverseMultiply(const Increment_ & dxi, Increm GMRESR(dxo, dxi, *this, Id, 10, 1.0e-3); } // ----------------------------------------------------------------------------- -template -void EnsembleCovariance::doRandomize(Increment_ &) const { - throw eckit::NotImplemented("EnsembleCovariance::doRandomize: Would it make sense?", Here()); -} -// ----------------------------------------------------------------------------- } // namespace oops #endif // OOPS_BASE_ENSEMBLECOVARIANCE_H_ diff --git a/src/oops/base/HybridCovariance.h b/src/oops/base/HybridCovariance.h index fb9db087a..27460cd9d 100644 --- a/src/oops/base/HybridCovariance.h +++ b/src/oops/base/HybridCovariance.h @@ -125,8 +125,21 @@ void HybridCovariance::doInverseMultiply(const Increment_ & dxi, Incremen } // ----------------------------------------------------------------------------- template -void HybridCovariance::doRandomize(Increment_ &) const { - throw eckit::NotImplemented("HybridCovariance::doRandomize: Would it make sense?", Here()); +void HybridCovariance::doRandomize(Increment_ & dx) const { + dx.zero(); + Increment_ tmp(dx); + int valueIndex = 0; + for (size_t jcomp = 0; jcomp < Bcomponents_.size(); ++jcomp) { + Bcomponents_[jcomp]->randomize(tmp); + if (weightTypes_[jcomp] == "value") { + tmp *= std::sqrt(valueWeights_[valueIndex]); + valueIndex += 1; + } + if (weightTypes_[jcomp] == "increment") { + throw eckit::NotImplemented("HybridCovariance::doRandomize: no square-root", Here()); + } + dx += tmp; + } } // ----------------------------------------------------------------------------- } // namespace oops diff --git a/src/oops/base/LocalizationBase.h b/src/oops/base/LocalizationBase.h index 0f8776bd9..40b83d654 100644 --- a/src/oops/base/LocalizationBase.h +++ b/src/oops/base/LocalizationBase.h @@ -42,20 +42,51 @@ class LocalizationBase : public util::Printable, LocalizationBase() = default; virtual ~LocalizationBase() = default; - /// default implementation of 4D localization (used in model-specific implementations) - virtual void localize(Increment_ &) const; + /// 4D localization with the same localization for time blocks + virtual void randomize(Increment_ &) const; + virtual void multiply(Increment_ &) const; protected: - virtual void multiply(Increment_ &) const = 0; + virtual void doRandomize(Increment_ &) const = 0; + virtual void doMultiply(Increment_ &) const = 0; private: virtual void print(std::ostream &) const = 0; }; // ----------------------------------------------------------------------------- -/// Default implementation of 4D localization (used in model-specific implementations) +/// Randomize template -void LocalizationBase::localize(Increment_ & dx) const { +void LocalizationBase::randomize(Increment_ & dx) const { + Log::trace() << "LocalizationBase::randomize starting" << std::endl; + const eckit::mpi::Comm & comm = dx.timeComm(); + static int tag = 23456; + size_t nslots = comm.size(); + int mytime = comm.rank(); + + if (mytime > 0) { + util::DateTime dt = dx.validTime(); // Save original time value + dx.zero(); + oops::mpi::receive(comm, dx, 0, tag); + dx.updateTime(dt - dx.validTime()); // Set time back to original value + } else { + // Apply 3D localization + this->doRandomize(dx); + + // Copy result to all timeslots + for (size_t jj = 1; jj < nslots; ++jj) { + oops::mpi::send(comm, dx, jj, tag); + } + } + ++tag; + + Log::trace() << "LocalizationBase::randomize done" << std::endl; +} + +// ----------------------------------------------------------------------------- +/// Multiply +template +void LocalizationBase::multiply(Increment_ & dx) const { Log::trace() << "LocalizationBase::multiply starting" << std::endl; const eckit::mpi::Comm & comm = dx.timeComm(); static int tag = 23456; @@ -75,8 +106,10 @@ void LocalizationBase::localize(Increment_ & dx) const { oops::mpi::receive(comm, dxtmp, jj, tag); dx.axpy(1.0, dxtmp, false); } + // Apply 3D localization - this->multiply(dx); + this->doMultiply(dx); + // Copy result to all timeslots for (size_t jj = 1; jj < nslots; ++jj) { oops::mpi::send(comm, dx, jj, tag); @@ -87,7 +120,6 @@ void LocalizationBase::localize(Increment_ & dx) const { Log::trace() << "LocalizationBase::multiply done" << std::endl; } - // ============================================================================= /// Localization Factory diff --git a/src/oops/base/ModelSpaceCovarianceBase.h b/src/oops/base/ModelSpaceCovarianceBase.h index 5fb90fa9a..92d4946e2 100644 --- a/src/oops/base/ModelSpaceCovarianceBase.h +++ b/src/oops/base/ModelSpaceCovarianceBase.h @@ -36,6 +36,7 @@ #include "oops/util/parameters/OptionalParameter.h" #include "oops/util/parameters/Parameters.h" #include "oops/util/parameters/RequiredPolymorphicParameter.h" +#include "oops/util/Random.h" namespace util { class DateTime; @@ -78,6 +79,7 @@ class ModelSpaceCovarianceBase { void randomize(Increment_ &) const; void multiply(const Increment_ &, Increment_ &) const; void inverseMultiply(const Increment_ &, Increment_ &) const; + void getVariance(Increment_ &) const; private: virtual void doRandomize(Increment_ &) const = 0; @@ -85,6 +87,7 @@ class ModelSpaceCovarianceBase { virtual void doInverseMultiply(const Increment_ &, Increment_ &) const = 0; ChvarVec_ chvars_; + size_t randomizationSize_; }; // ============================================================================= @@ -302,6 +305,7 @@ ModelSpaceCovarianceBase::ModelSpaceCovarianceBase( chvars_.push_back(LinearVariableChangeFactory::create( bg, fg, resol, variableChange.variableChangeParameters)); } + randomizationSize_ = parameters.randomizationSize; } // ----------------------------------------------------------------------------- @@ -389,6 +393,29 @@ void ModelSpaceCovarianceBase::inverseMultiply(const Increment_ & dxi, // ----------------------------------------------------------------------------- +template +void ModelSpaceCovarianceBase::getVariance(Increment_ & variance) const { + Increment_ dx(variance); + Increment_ dxsq(variance); + Increment_ mean(variance); + mean.zero(); + variance.zero(); + for (int ie = 0; ie < randomizationSize_; ++ie) { + this->randomize(dx); + dx -= mean; + dxsq = dx; + dxsq.schur_product_with(dx); + double rk_var = static_cast(ie)/static_cast(ie+1); + double rk_mean = 1.0/static_cast(ie+1); + variance.axpy(rk_var, dxsq, false); + mean.axpy(rk_mean, dx, false); + } + double rk_norm = 1.0/static_cast(randomizationSize_-1); + variance *= rk_norm; +} + +// ----------------------------------------------------------------------------- + } // namespace oops #endif // OOPS_BASE_MODELSPACECOVARIANCEBASE_H_ diff --git a/src/oops/base/ModelSpaceCovarianceParametersBase.h b/src/oops/base/ModelSpaceCovarianceParametersBase.h index 084be5f99..cf481b542 100644 --- a/src/oops/base/ModelSpaceCovarianceParametersBase.h +++ b/src/oops/base/ModelSpaceCovarianceParametersBase.h @@ -37,6 +37,7 @@ class ModelSpaceCovarianceParametersBase : public Parameters { Parameter>> variableChanges{ "variable changes", {}, this}; + Parameter randomizationSize{"randomization size", 50, this}; }; } // namespace oops diff --git a/src/oops/interface/Localization.h b/src/oops/interface/Localization.h index 4f201b57f..41827bc50 100644 --- a/src/oops/interface/Localization.h +++ b/src/oops/interface/Localization.h @@ -36,7 +36,8 @@ class Localization : public LocalizationBase { Localization(const Geometry_ &, const util::DateTime &, const eckit::Configuration &); ~Localization(); - void multiply(Increment_ &) const override; + void doRandomize(Increment_ &) const override; + void doMultiply(Increment_ &) const override; private: void print(std::ostream &) const override; @@ -69,11 +70,21 @@ Localization::~Localization() { // ----------------------------------------------------------------------------- template -void Localization::multiply(Increment_ & dx) const { - Log::trace() << "Localization::multiply starting" << std::endl; - util::Timer timer(classname(), "multiply"); +void Localization::doRandomize(Increment_ & dx) const { + Log::trace() << "Localization::doRandomize starting" << std::endl; + util::Timer timer(classname(), "doRandomize"); + loc_->randomize(dx.increment()); + Log::trace() << "Localization::doRandomize done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void Localization::doMultiply(Increment_ & dx) const { + Log::trace() << "Localization::doMultiply starting" << std::endl; + util::Timer timer(classname(), "doMultiply"); loc_->multiply(dx.increment()); - Log::trace() << "Localization::multiply done" << std::endl; + Log::trace() << "Localization::doMultiply done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/runs/Dirac.h b/src/oops/runs/Dirac.h index 500f0935a..fa48ec28b 100644 --- a/src/oops/runs/Dirac.h +++ b/src/oops/runs/Dirac.h @@ -143,7 +143,7 @@ template class Dirac : public Application { LocalizationFactory::create(resol, time, locConfigs[jcomp]); // Apply localization - loc_->localize(dxdir); + loc_->multiply(dxdir); // Write increment const eckit::LocalConfiguration output_localization(fullConfig, "output localization"); @@ -151,6 +151,21 @@ template class Dirac : public Application { Log::test() << "Localized Increment: " << dxdir << std::endl; } + if (fullConfig.has("output variance")) { + // Variance configuration + const eckit::LocalConfiguration output_variance(fullConfig, "output variance"); + + // Setup variance + Increment_ variance(resol, vars, time); + + // Get variance + B->getVariance(variance); + + // Write increment + variance.write(output_variance); + Log::test() << "Randomized variance: " << variance << std::endl; + } + return 0; } // ----------------------------------------------------------------------------- diff --git a/src/test/interface/Localization.h b/src/test/interface/Localization.h index e513cfddb..9c1c82275 100644 --- a/src/test/interface/Localization.h +++ b/src/test/interface/Localization.h @@ -79,6 +79,18 @@ template class LocalizationFixture : private boost::noncopyable // ----------------------------------------------------------------------------- +template void testLocalizationRandomize() { + typedef LocalizationFixture Test_; + typedef oops::Increment Increment_; + + Increment_ dx(Test_::resol(), Test_::ctlvars(), Test_::time()); + + Test_::localization().randomize(dx); + EXPECT(dx.norm() > 0.0); +} + +// ----------------------------------------------------------------------------- + template void testLocalizationZero() { typedef LocalizationFixture Test_; typedef oops::Increment Increment_; @@ -86,7 +98,7 @@ template void testLocalizationZero() { Increment_ dx(Test_::resol(), Test_::ctlvars(), Test_::time()); EXPECT(dx.norm() == 0.0); - Test_::localization().localize(dx); + Test_::localization().multiply(dx); EXPECT(dx.norm() == 0.0); } @@ -100,7 +112,7 @@ template void testLocalizationMultiply() { dx.random(); EXPECT(dx.norm() > 0.0); - Test_::localization().localize(dx); + Test_::localization().multiply(dx); EXPECT(dx.norm() > 0.0); } @@ -116,6 +128,8 @@ template class Localization : public oops::Test { void register_tests() const override { std::vector& ts = eckit::testing::specification(); + ts.emplace_back(CASE("interface/Localization/testLocalizationRandomize") + { testLocalizationRandomize(); }); ts.emplace_back(CASE("interface/Localization/testLocalizationZero") { testLocalizationZero(); }); ts.emplace_back(CASE("interface/Localization/testLocalizationMultiply") From 335ed50c38f2c275581b992377c189ebc5b6a4ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Tue, 26 Jan 2021 16:05:28 +0000 Subject: [PATCH 046/142] Add a function testing if a string matches a wildcard pattern (#1045) * Added a function testing if a string matches a wildcard pattern (with * and ? characters interpreted in the usual way). * Corrected an URL in a comment. Co-authored-by: Anna Shlyaeva --- src/CMakeLists.txt | 7 +++++ src/oops/util/wildcard.cc | 58 +++++++++++++++++++++++++++++++++++++++ src/oops/util/wildcard.h | 23 ++++++++++++++++ src/test/util/wildcard.cc | 45 ++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+) create mode 100644 src/oops/util/wildcard.cc create mode 100644 src/oops/util/wildcard.h create mode 100644 src/test/util/wildcard.cc diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b671de69d..825e8273d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -310,6 +310,8 @@ oops/util/Timer.h oops/util/TimerHelper.cc oops/util/TimerHelper.h oops/util/utilNamespaceDoc.h +oops/util/wildcard.h +oops/util/wildcard.cc oops/util/parameters/ConfigurationParameter.cc oops/util/parameters/ConfigurationParameter.h @@ -640,6 +642,11 @@ ecbuild_add_test( TARGET test_util_localenvironment ARGS "test/testinput/empty.yaml" LIBS oops ) +ecbuild_add_test( TARGET test_util_wildcard + SOURCES test/util/wildcard.cc + ARGS "test/testinput/empty.yaml" + LIBS oops ) + ecbuild_add_test( TARGET test_assimilation_fullgmres SOURCES test/assimilation/FullGMRES.cc ARGS "test/testinput/empty.yaml" diff --git a/src/oops/util/wildcard.cc b/src/oops/util/wildcard.cc new file mode 100644 index 000000000..f84e84e83 --- /dev/null +++ b/src/oops/util/wildcard.cc @@ -0,0 +1,58 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/util/wildcard.h" + +namespace util { + +// An std::string-based implementation of the algorithm from Robert van Engelen, +// "Fast String Matching with Wildcards, Globs, and Gitignore-Style Globs - How Not to Blow it Up", +// https://www.codeproject.com/Articles/5163931/Fast-String-Matching-with-Wildcards-Globs-and-Giti +bool matchesWildcardPattern(const std::string &string, const std::string &pattern) { + const std::size_t stringSize = string.size(); + const std::size_t patternSize = pattern.size(); + + if (patternSize == 0) + return stringSize == 0; + + std::size_t stringPos = 0; + std::size_t patternPos = 0; + std::size_t stringBacktrackPos = std::string::npos; + std::size_t patternBacktrackPos = std::string::npos; + + while (stringPos != stringSize) { + if (patternPos != patternSize && pattern[patternPos] == '*') { + // Found a new * wildcard. Set new positions to use in case of backtracking + stringBacktrackPos = stringPos; + patternBacktrackPos = ++patternPos; + } else if (patternPos != patternSize && + (pattern[patternPos] == string[stringPos] || pattern[patternPos] == '?')) { + // Matched a single character + ++stringPos; + ++patternPos; + } else { + // Mismatch or end of pattern + if (patternBacktrackPos == std::string::npos) { + // There have been no * wildcards -- we can't backtrack. + // The string doesn't match the pattern. + return false; + } + // Backtrack: try to assign one more character to the most recent * wildcard. + stringPos = ++stringBacktrackPos; + patternPos = patternBacktrackPos; + } + } + + // Ignore any trailing * wildcards. + while (patternPos != patternSize && pattern[patternPos] == '*') + patternPos++; + + // Have we matched the complete pattern? + return patternPos == patternSize; +} + +} // namespace util diff --git a/src/oops/util/wildcard.h b/src/oops/util/wildcard.h new file mode 100644 index 000000000..0b3b970e5 --- /dev/null +++ b/src/oops/util/wildcard.h @@ -0,0 +1,23 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_UTIL_WILDCARD_H_ +#define OOPS_UTIL_WILDCARD_H_ + +#include + +namespace util { + +/// Returns true if \p string matches \p pattern, false otherwise. +/// +/// \p pattern may contain wildcards: ? stands for any character and * stands for any number of +/// characters. +bool matchesWildcardPattern(const std::string &string, const std::string &pattern); + +} // namespace util + +#endif // OOPS_UTIL_WILDCARD_H_ diff --git a/src/test/util/wildcard.cc b/src/test/util/wildcard.cc new file mode 100644 index 000000000..bd9991e4e --- /dev/null +++ b/src/test/util/wildcard.cc @@ -0,0 +1,45 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "eckit/testing/Test.h" +#include "oops/util/wildcard.h" + +namespace { + +CASE("util/wildcard") { + EXPECT(util::matchesWildcardPattern("", "")); + EXPECT(util::matchesWildcardPattern("", "*")); + EXPECT_NOT(util::matchesWildcardPattern("", "?")); + EXPECT(util::matchesWildcardPattern("1", "1")); + EXPECT(util::matchesWildcardPattern("12345", "12345")); + EXPECT(util::matchesWildcardPattern("12345", "1?3?5")); + EXPECT(util::matchesWildcardPattern("12345", "?2?4?")); + EXPECT_NOT(util::matchesWildcardPattern("12345", "?3?4?")); + EXPECT(util::matchesWildcardPattern("abcdefghijkl", "*cd*efg*k*")); + EXPECT(util::matchesWildcardPattern("abcdefghijkl", "ab*cde*kl")); + EXPECT_NOT(util::matchesWildcardPattern("abcdefghijkl", "*cd*defg*k*")); + EXPECT_NOT(util::matchesWildcardPattern("abcdefghijkl", "ab*cde*k")); + EXPECT_NOT(util::matchesWildcardPattern("abcdefghijkl", "b*cde*kl")); + EXPECT(util::matchesWildcardPattern("abcdefghijkl", "***cd***efg***k***")); + EXPECT(util::matchesWildcardPattern("abcdefghijkl", "ab***cde***kl")); + EXPECT_NOT(util::matchesWildcardPattern("abcdefghijkl", "***cd***defg***k***")); + EXPECT_NOT(util::matchesWildcardPattern("abcdefghijkl", "ab***cde***k")); + EXPECT_NOT(util::matchesWildcardPattern("abcdefghijkl", "b***cde***kl")); + EXPECT(util::matchesWildcardPattern("1212123", "*12123")); + EXPECT(util::matchesWildcardPattern("1212123", "***12123")); + EXPECT_NOT(util::matchesWildcardPattern("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "*a*a*a*b")); + EXPECT_NOT(util::matchesWildcardPattern("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", "*a*a*a*a")); + EXPECT(util::matchesWildcardPattern("xx", "*?")); + EXPECT(util::matchesWildcardPattern("xx", "?*")); +} + +} // anonymous namespace + +int main(int argc, char **argv) +{ + return eckit::testing::run_tests ( argc, argv ); +} From 4c14984892852bea21fb944907743f7f77ccd2de Mon Sep 17 00:00:00 2001 From: Mark Miesch Date: Tue, 26 Jan 2021 13:24:03 -0700 Subject: [PATCH 047/142] change mpi tag (#1052) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Yannick Trémolet <30638944+ytremolet@users.noreply.github.com> --- src/oops/interface/Increment.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/oops/interface/Increment.h b/src/oops/interface/Increment.h index bf945d728..886e2f537 100644 --- a/src/oops/interface/Increment.h +++ b/src/oops/interface/Increment.h @@ -598,7 +598,7 @@ void Increment::shift_forward(const util::DateTime & begin) { template void Increment::shift_backward(const util::DateTime & end) { Log::trace() << "Increment::Increment shift_backward starting" << std::endl; - static int tag = 753951; + static int tag = 30951; size_t mytime = timeComm_->rank(); // Send values of dx_i at start of my subwindow to previous subwindow From 8ad05d409a166a41fe14bd38b550441098f77e5a Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 27 Jan 2021 07:08:39 -0700 Subject: [PATCH 048/142] bugfix in VerticalLocEV: initialize covariance matrix with zeros. (#1042) * init cov matrix with zeros * normalize before doing eigen decomposition in geometry Co-authored-by: mmiesch --- qg/model/qg_geom_mod.F90 | 19 +++++++++++++------ src/oops/generic/VerticalLocEV.h | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/qg/model/qg_geom_mod.F90 b/qg/model/qg_geom_mod.F90 index 78e825195..e19e91c26 100644 --- a/qg/model/qg_geom_mod.F90 +++ b/qg/model/qg_geom_mod.F90 @@ -1,8 +1,8 @@ ! (C) Copyright 2009-2016 ECMWF. -! +! ! This software is licensed under the terms of the Apache Licence Version 2.0 -! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -! In applying this licence, ECMWF does not waive the privileges and immunities +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! In applying this licence, ECMWF does not waive the privileges and immunities ! granted to it by virtue of its status as an intergovernmental organisation nor ! does it submit to any jurisdiction. @@ -72,6 +72,7 @@ subroutine qg_geom_setup(self,f_conf) real(kind_real) :: mapfac,distx,disty,f real(kind_real),allocatable :: real_array(:),depths(:),wi(:),vl(:,:),work(:) real(kind_real),allocatable :: fsave(:,:),vrlu(:,:),vrlusave(:,:) +real(kind_real) :: norm character(len=1024) :: record logical :: htype character(len=:),allocatable :: str @@ -154,14 +155,15 @@ subroutine qg_geom_setup(self,f_conf) enddo ! Compute eigendecomposition of ff -fsave = self%f +norm=maxval(abs(self%f)) ! normalization for numerical stability +fsave = self%f / norm allocate(work(1)) call dgeev('V','V',self%nz,fsave,self%nz,self%f_d,wi,vl,self%nz,self%f_p,self%nz,work,-1,info) if (info/=0) call abor1_ftn('error in dgeev, first pass') lwork = int(work(1)) deallocate(work) allocate(work(lwork)) -fsave = self%f +fsave = self%f / norm call dgeev('V','V',self%nz,fsave,self%nz,self%f_d,wi,vl,self%nz,self%f_p,self%nz,work,lwork,info) if (info/=0) call abor1_ftn('error in dgeev, second pass') deallocate(work) @@ -184,12 +186,17 @@ subroutine qg_geom_setup(self,f_conf) if (info/=0) call abor1_ftn('error in dgetri, second pass') deallocate(work) +! re-apply normalization after eigendecomposition +self%f_d = self%f_d * norm +self%f_p = self%f_p * norm +self%f_pinv = self%f_pinv / norm + ! Beta coefficient do iy=1,self%ny self%bet(iy) = real(iy-(self%ny+1)/2,kind_real)*self%deltay*bet0 enddo -! Set heating term +! Set heating term call f_conf%get_or_die("heating",htype) if (.not. htype) then ! No heating term diff --git a/src/oops/generic/VerticalLocEV.h b/src/oops/generic/VerticalLocEV.h index ae1d8d58b..918064665 100644 --- a/src/oops/generic/VerticalLocEV.h +++ b/src/oops/generic/VerticalLocEV.h @@ -122,7 +122,7 @@ template size_t nlevs = vCoord.size(); // compute vertical correlations and eigen vectors - Eigen::MatrixXd cov(nlevs, nlevs); + Eigen::MatrixXd cov = Eigen::MatrixXd::Zero(nlevs, nlevs); for (size_t jj=0; jj < nlevs; ++jj) { for (size_t ii=jj; ii < nlevs; ++ii) { cov(ii, jj) = oops::gc99(std::abs(vCoord[jj]-vCoord[ii])/options_.VertLocDist); From 8f59f1e74fa025f57d73704671bf8464e765c532 Mon Sep 17 00:00:00 2001 From: Benjamin Menetrier <30638301+benjaminmenetrier@users.noreply.github.com> Date: Fri, 29 Jan 2021 17:08:42 +0100 Subject: [PATCH 049/142] Remove QG scaling and fix floating point exceptions (#1056) * Remove scaling in QG prints * Update reference files * Cleaning * Trim std::scientific and std::setprecision --- qg/model/FieldsQG.cc | 10 +- qg/model/GomQG.cc | 15 +- qg/model/ObsVecQG.cc | 12 +- qg/model/QgFortran.h | 4 +- qg/model/TlmQG.cc | 10 +- qg/model/qg_fields_interface.F90 | 2 +- qg/model/qg_fields_mod.F90 | 51 +---- qg/model/qg_gom_interface.F90 | 5 +- qg/model/qg_gom_mod.F90 | 31 +-- qg/model/qg_obsvec_interface.F90 | 5 +- qg/model/qg_obsvec_mod.F90 | 40 +--- qg/test/testoutput/3densvar.test | 12 +- qg/test/testoutput/3dfgat.test | 6 +- qg/test/testoutput/3dvar.test | 12 +- qg/test/testoutput/3dvar_change_var.test | 12 +- qg/test/testoutput/3dvar_hybrid.test | 12 +- qg/test/testoutput/4densvar.test | 84 +++---- qg/test/testoutput/4densvar_hybrid.test | 84 +++---- qg/test/testoutput/4dvar_change_var.test | 12 +- qg/test/testoutput/4dvar_dripcg.test | 12 +- qg/test/testoutput/4dvar_drpcg_lmp.test | 12 +- qg/test/testoutput/4dvar_drpfom.test | 12 +- qg/test/testoutput/4dvar_drplanczos.test | 12 +- .../testoutput/4dvar_drplanczos_hybrid.test | 12 +- qg/test/testoutput/4dvar_forcing.test | 50 ----- qg/test/testoutput/4dvar_ipcg.test | 12 +- qg/test/testoutput/4dvar_obs_biased.test | 12 +- qg/test/testoutput/4dvar_rpcg.test | 12 +- qg/test/testoutput/4dvar_saddlepoint.test | 24 +- qg/test/testoutput/addincrement.test | 14 +- qg/test/testoutput/addincrement_scaled.test | 16 +- qg/test/testoutput/analytic_forecast.test | 12 +- qg/test/testoutput/convertstate.test | 96 ++++---- qg/test/testoutput/dfi.test | 18 +- qg/test/testoutput/diffstates.test | 14 +- qg/test/testoutput/dirac_cov.test | 12 +- qg/test/testoutput/dirac_hyb_field.test | 12 +- qg/test/testoutput/dirac_hyb_value.test | 8 +- qg/test/testoutput/dirac_loc_3d.test | 16 +- qg/test/testoutput/dirac_loc_4d.test | 112 +++++----- qg/test/testoutput/dirac_no_loc.test | 12 +- qg/test/testoutput/eda_3dvar_block.test | 16 +- qg/test/testoutput/ens_forecast.test | 32 +-- qg/test/testoutput/ens_hofx.test | 110 ++------- qg/test/testoutput/ens_recenter.test | 66 +++--- qg/test/testoutput/ens_variance.test | 2 +- .../ens_variance_inflation_field.test | 2 +- .../ens_variance_inflation_value.test | 2 +- qg/test/testoutput/forecast.test | 12 +- qg/test/testoutput/gen_ens_pert_B.test | 66 +++--- qg/test/testoutput/hofx.test | 18 +- qg/test/testoutput/hofx3d.test | 18 +- qg/test/testoutput/letkf.test | 210 +++++++++--------- qg/test/testoutput/make_obs_3d.test | 18 +- qg/test/testoutput/make_obs_4d_12h.test | 18 +- qg/test/testoutput/make_obs_4d_24h.test | 18 +- qg/test/testoutput/make_obs_4d_biased.test | 18 +- qg/test/testoutput/rtpp.test | 24 +- qg/test/testoutput/static_b_init.test | 2 +- qg/test/testoutput/truth.test | 12 +- qg/test/testoutput/uniform_field_hybrid.test | 12 +- .../testoutput/uniform_field_inflation.test | 12 +- 62 files changed, 699 insertions(+), 920 deletions(-) diff --git a/qg/model/FieldsQG.cc b/qg/model/FieldsQG.cc index 747aba06e..74f033e1e 100644 --- a/qg/model/FieldsQG.cc +++ b/qg/model/FieldsQG.cc @@ -191,14 +191,14 @@ void FieldsQG::print(std::ostream & os) const { } else { os << std::endl << " Boundary conditions are not activated"; } - std::vector zstat(4*(1+nb)); + std::vector zstat(3*(1+nb)); qg_fields_gpnorm_f90(keyFlds_, nb, zstat[0]); for (int jj = 0; jj < 1+nb; ++jj) { std::ios_base::fmtflags f(os.flags()); - os << std::endl << " Scaling=" << std::setprecision(4) << std::setw(7) << zstat[4*jj] - << ", Min=" << std::fixed << std::setprecision(4) << std::setw(12) << zstat[4*jj+1] - << ", Max=" << std::fixed << std::setprecision(4) << std::setw(12) < zstat(4*(1+nb)); + std::vector zstat(3*(1+nb)); qg_fields_gpnorm_f90(jtra->second, nb, zstat[0]); for (int jj = 0; jj < 1+nb; ++jj) { std::ios_base::fmtflags f(os.flags()); - os << std::endl << " Scaling=" << std::setprecision(4) << std::setw(7) << zstat[4*jj] - << ", Min=" << std::fixed << std::setprecision(4) << std::setw(12) << zstat[4*jj+1] - << ", Max=" << std::fixed << std::setprecision(4) << std::setw(12) <has(col); -} - -// ----------------------------------------------------------------------------- - void ObsTableView::putdb(const std::string & col, const std::vector & vec) const { int missing; std::vector fullvec(obstable_->nobs(), util::missingValue(missing)); diff --git a/l95/src/lorenz95/ObsTableView.h b/l95/src/lorenz95/ObsTableView.h index ec67221eb..9a4e9683f 100644 --- a/l95/src/lorenz95/ObsTableView.h +++ b/l95/src/lorenz95/ObsTableView.h @@ -45,7 +45,6 @@ class ObsTableView : public util::Printable, const eckit::Configuration &); ~ObsTableView(); - bool has(const std::string &) const; void putdb(const std::string &, const std::vector &) const; void putdb(const std::string &, const std::vector &) const; void putdb(const std::string &, const std::vector &) const; diff --git a/l95/src/lorenz95/ObsVec1D.cc b/l95/src/lorenz95/ObsVec1D.cc index 167d38fb9..bfdb96bde 100644 --- a/l95/src/lorenz95/ObsVec1D.cc +++ b/l95/src/lorenz95/ObsVec1D.cc @@ -23,13 +23,11 @@ namespace lorenz95 { // ----------------------------------------------------------------------------- ObsVec1D::ObsVec1D(const ObsTableView & ot, - const std::string & name, const bool fail) + const std::string & name) : obsdb_(ot), data_(ot.nobs()), missing_(util::missingValue(missing_)) { for (double & val : data_) { val = 0.0; } - if (!name.empty()) { - if (fail || obsdb_.has(name)) obsdb_.getdb(name, data_); - } + if (!name.empty()) obsdb_.getdb(name, data_); } // ----------------------------------------------------------------------------- ObsVec1D::ObsVec1D(const ObsVec1D & other) diff --git a/l95/src/lorenz95/ObsVec1D.h b/l95/src/lorenz95/ObsVec1D.h index a838b2187..b7e7987df 100644 --- a/l95/src/lorenz95/ObsVec1D.h +++ b/l95/src/lorenz95/ObsVec1D.h @@ -35,7 +35,7 @@ class ObsVec1D : public util::Printable, public: static const std::string classname() {return "lorenz95::ObsVec1D";} - explicit ObsVec1D(const ObsTableView &, const std::string & name = "", const bool fail = true); + explicit ObsVec1D(const ObsTableView &, const std::string & name = ""); ObsVec1D(const ObsVec1D &); ObsVec1D(const ObsTableView &, const ObsVec1D &); ~ObsVec1D() = default; diff --git a/qg/model/ObsSpaceQG.cc b/qg/model/ObsSpaceQG.cc index de3d1904e..184dff46e 100644 --- a/qg/model/ObsSpaceQG.cc +++ b/qg/model/ObsSpaceQG.cc @@ -172,14 +172,6 @@ void ObsSpaceQG::putdb(const std::string & col, const int & keyData) const { // ----------------------------------------------------------------------------- -bool ObsSpaceQG::has(const std::string & col) const { - int ii; - qg_obsdb_has_f90(key_, obsname_.size(), obsname_.c_str(), col.size(), col.c_str(), ii); - return ii; -} - -// ----------------------------------------------------------------------------- - std::unique_ptr ObsSpaceQG::locations() const { atlas::FieldSet fields; std::vector times; diff --git a/qg/model/ObsSpaceQG.h b/qg/model/ObsSpaceQG.h index d0efb42d2..2a18bf415 100644 --- a/qg/model/ObsSpaceQG.h +++ b/qg/model/ObsSpaceQG.h @@ -56,9 +56,6 @@ class ObsSpaceQG : public oops::ObsSpaceBase { /// save data or metadata void putdb(const std::string &, const int &) const; - /// check if variable is in ObsSpace - bool has(const std::string & col) const; - /// create locations for the whole time window std::unique_ptr locations() const; diff --git a/qg/model/ObsVecQG.cc b/qg/model/ObsVecQG.cc index e7f37410f..db2ed261c 100644 --- a/qg/model/ObsVecQG.cc +++ b/qg/model/ObsVecQG.cc @@ -22,13 +22,11 @@ namespace qg { // ----------------------------------------------------------------------------- ObsVecQG::ObsVecQG(const ObsSpaceQG & obsdb, - const std::string & name, const bool fail) + const std::string & name) : obsdb_(obsdb), keyOvec_(0) { qg_obsvec_setup_f90(keyOvec_, obsdb.obsvariables().size(), obsdb.nobs()); - if (!name.empty()) { - if (fail || obsdb_.has(name)) obsdb_.getdb(name, keyOvec_); - } + if (!name.empty()) obsdb_.getdb(name, keyOvec_); } // ----------------------------------------------------------------------------- ObsVecQG::ObsVecQG(const ObsVecQG & other) diff --git a/qg/model/ObsVecQG.h b/qg/model/ObsVecQG.h index 4ee20717d..3a9393bd2 100644 --- a/qg/model/ObsVecQG.h +++ b/qg/model/ObsVecQG.h @@ -36,7 +36,7 @@ class ObsVecQG : public util::Printable, static const std::string classname() {return "qg::ObsVecQG";} ObsVecQG(const ObsSpaceQG &, - const std::string & name = "", const bool fail = true); + const std::string & name = ""); ObsVecQG(const ObsVecQG &); ObsVecQG(const ObsSpaceQG &, const ObsVecQG &); ~ObsVecQG(); diff --git a/qg/model/QgFortran.h b/qg/model/QgFortran.h index 669720782..1a471e233 100644 --- a/qg/model/QgFortran.h +++ b/qg/model/QgFortran.h @@ -208,8 +208,6 @@ extern "C" { const F90ovec &); void qg_obsdb_put_f90(const F90odb &, const int &, const char *, const int &, const char *, const F90ovec &); - void qg_obsdb_has_f90(const F90odb &, const int &, const char *, - const int &, const char *, int &); void qg_obsdb_locations_f90(const F90odb &, const int &, const char *, atlas::field::FieldSetImpl *, std::vector &); void qg_obsdb_generate_f90(const F90odb &, const int &, const char *, diff --git a/qg/model/qg_obsdb_interface.F90 b/qg/model/qg_obsdb_interface.F90 index 5afec8f69..29cbe5210 100644 --- a/qg/model/qg_obsdb_interface.F90 +++ b/qg/model/qg_obsdb_interface.F90 @@ -169,34 +169,6 @@ subroutine qg_obsdb_put_c(c_key_self,lgrp,c_grp,lcol,c_col,c_key_ovec) bind(c,na end subroutine qg_obsdb_put_c ! ------------------------------------------------------------------------------ -!> Test observation data existence -subroutine qg_obsdb_has_c(c_key_self,lgrp,c_grp,lcol,c_col,c_has) bind(c,name='qg_obsdb_has_f90') - -implicit none - -! Passed variables -integer(c_int),intent(in) :: c_key_self !< Observation data -integer(c_int),intent(in) :: lgrp !< Group size -character(kind=c_char,len=1),intent(in) :: c_grp(lgrp+1) !< Group name -integer(c_int),intent(in) :: lcol !< Column size -character(kind=c_char,len=1),intent(in) :: c_col(lcol+1) !< Column name -integer(c_int),intent(out) :: c_has !< Test flag - -! Local variables -type(qg_obsdb),pointer :: self -character(len=lgrp) :: grp -character(len=lcol) :: col - -! Interface -call qg_obsdb_registry%get(c_key_self,self) -call c_f_string(c_grp,grp) -call c_f_string(c_col,col) - -! Call Fortran -call qg_obsdb_has(self,trim(grp),trim(col),c_has) - -end subroutine qg_obsdb_has_c -! ------------------------------------------------------------------------------ !> Get locations from observation data subroutine qg_obsdb_locations_c(c_key_self,lgrp,c_grp,c_fields,c_times) bind(c,name='qg_obsdb_locations_f90') diff --git a/qg/model/qg_obsdb_mod.F90 b/qg/model/qg_obsdb_mod.F90 index 9804a1f73..0a4f58ab5 100644 --- a/qg/model/qg_obsdb_mod.F90 +++ b/qg/model/qg_obsdb_mod.F90 @@ -30,7 +30,7 @@ module qg_obsdb_mod private public :: qg_obsdb public :: qg_obsdb_registry -public :: qg_obsdb_setup,qg_obsdb_delete,qg_obsdb_get,qg_obsdb_put,qg_obsdb_has,qg_obsdb_locations,qg_obsdb_generate,qg_obsdb_nobs +public :: qg_obsdb_setup,qg_obsdb_delete,qg_obsdb_get,qg_obsdb_put,qg_obsdb_locations,qg_obsdb_generate,qg_obsdb_nobs ! ------------------------------------------------------------------------------ integer,parameter :: rseed = 1 !< Random seed (for reproducibility) @@ -275,34 +275,6 @@ subroutine qg_obsdb_put(self,grp,col,ovec) end subroutine qg_obsdb_put ! ------------------------------------------------------------------------------ -!> Test observation data existence -subroutine qg_obsdb_has(self,grp,col,has) - -implicit none - -! Passed variables -type(qg_obsdb),intent(in) :: self !< Observation data -character(len=*),intent(in) :: grp !< Group -character(len=*),intent(in) :: col !< Column -integer,intent(out) :: has !< Test flag - -! Passed variables -type(group_data),pointer :: jgrp -type(column_data),pointer :: jcol - -! Initialization -has = 0 - -! Find observation group -call qg_obsdb_find_group(self,grp,jgrp) -if (associated(jgrp)) then - ! Find observation column - call qg_obsdb_find_column(jgrp,col,jcol) - if (associated(jcol)) has = 1 -endif - -end subroutine qg_obsdb_has -! ------------------------------------------------------------------------------ !> Get locations from observation data subroutine qg_obsdb_locations(self,grp,fields,c_times) diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index c64fa938c..80efb96e7 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -220,11 +220,6 @@ double CostJo::finalize() { // Compute departures Departures_ ydep(yeqv - yobs_); - Log::info() << "Jo Departures:" << std::endl << ydep << "End Jo Departures" << std::endl; - -// Apply bias correction - Departures_ bias(obspace_, "ObsBias", false); - ydep += bias; Log::info() << "Jo Bias Corrected Departures:" << std::endl << ydep << "End Jo Bias Corrected Departures" << std::endl; diff --git a/src/oops/base/Departures.h b/src/oops/base/Departures.h index 8396a3bf6..820b50cd4 100644 --- a/src/oops/base/Departures.h +++ b/src/oops/base/Departures.h @@ -49,9 +49,8 @@ class Departures : public util::Printable, template using ObsDataVec_ = std::vector>>; public: -/// \brief create Departures for all obs (read from ObsSpace if name is specified) - Departures(const ObsSpaces_ &, - const std::string & name = "", const bool failIfNameNotFound = true); +/// \brief create Departures for all obs (read from ObsSpace if \p name is specified) + explicit Departures(const ObsSpaces_ &, const std::string & name = ""); /// \brief create local Departures Departures(const ObsSpaces_ &, const Departures &); @@ -95,11 +94,11 @@ class Departures : public util::Printable, template Departures::Departures(const ObsSpaces_ & obsdb, - const std::string & name, const bool fail): dep_() + const std::string & name): dep_() { dep_.reserve(obsdb.size()); for (size_t jj = 0; jj < obsdb.size(); ++jj) { - dep_.emplace_back(obsdb[jj], name, fail); + dep_.emplace_back(obsdb[jj], name); } Log::trace() << "Departures created" << std::endl; } diff --git a/src/oops/base/Observations.h b/src/oops/base/Observations.h index abc57310d..55a004fd3 100644 --- a/src/oops/base/Observations.h +++ b/src/oops/base/Observations.h @@ -89,7 +89,7 @@ Observations::Observations(const ObsSpaces_ & obsdb, { obs_.reserve(obsdb.size()); for (std::size_t jj = 0; jj < obsdb.size(); ++jj) { - obs_.emplace_back(obsdb[jj], name, true); + obs_.emplace_back(obsdb[jj], name); } Log::trace() << "Observations created" << std::endl; } diff --git a/src/oops/interface/ObsVector.h b/src/oops/interface/ObsVector.h index a4138f595..dabe8be94 100644 --- a/src/oops/interface/ObsVector.h +++ b/src/oops/interface/ObsVector.h @@ -45,7 +45,9 @@ class ObsVector : public util::Printable, public: static const std::string classname() {return "oops::ObsVector";} - explicit ObsVector(const ObsSpace &, const std::string name = "", const bool fail = true); + /// Creates vector from \p obsspace. If \p name is specified, reads the + /// specified \p name variable from \p obsspace. Otherwise, zero vector is created. + explicit ObsVector(const ObsSpace & obsspace, const std::string name = ""); explicit ObsVector(const ObsVector &); ObsVector(const ObsSpace &, const ObsVector &); ~ObsVector(); @@ -88,12 +90,12 @@ class ObsVector : public util::Printable, // ----------------------------------------------------------------------------- template -ObsVector::ObsVector(const ObsSpace & os, const std::string name, - const bool fail): data_(), commTime_(os.timeComm()) { +ObsVector::ObsVector(const ObsSpace & os, const std::string name) + : data_(), commTime_(os.timeComm()) { Log::trace() << "ObsVector::ObsVector starting " << name << std::endl; util::Timer timer(classname(), "ObsVector"); - data_.reset(new ObsVector_(os.obsspace(), name, fail)); + data_.reset(new ObsVector_(os.obsspace(), name)); Log::trace() << "ObsVector::ObsVector done" << std::endl; } diff --git a/src/test/interface/ObsOperator.h b/src/test/interface/ObsOperator.h index fccbcde87..fe863d6ba 100644 --- a/src/test/interface/ObsOperator.h +++ b/src/test/interface/ObsOperator.h @@ -85,12 +85,6 @@ template void testSimulateObs() { hop.simulateObs(gval, hofx, ybias, diags); hofx.save("hofx"); - // apply bias correction if it is required - if (conf.has("obs bias")) { - const ObsVector_ bias(Test_::obspace()[jj], "ObsBias", false); - hofx += bias; - } - const double tol = conf.getDouble("tolerance"); if (conf.has("vector ref")) { // if reference h(x) is saved in file as a vector, read from file From ce084a378cc2acf221fdd2507cec0c938b83c86e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Thu, 4 Feb 2021 23:50:17 +0000 Subject: [PATCH 052/142] Defined macros similar to OOPS_CONCRETE/ABSTRACT_PARAMETERS, but skipping the definitions of the non-copy and non-move constructors. (#1061) Co-authored-by: Anna Shlyaeva --- src/oops/util/parameters/Parameters.h | 52 +++++++++++++++++++-------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/src/oops/util/parameters/Parameters.h b/src/oops/util/parameters/Parameters.h index 5d16ffb90..fde3b2c12 100644 --- a/src/oops/util/parameters/Parameters.h +++ b/src/oops/util/parameters/Parameters.h @@ -24,6 +24,24 @@ namespace eckit { namespace oops { +/// \brief This macro may be invoked at the top of the declaration of an abstract subclass of +/// Parameters instead of OOPS_ABSTRACT_PARAMETERS() if the definition of the (non-copy and +/// non-move) constructors needs to be customized. See OOPS_ABSTRACT_PARAMETERS() for more +/// information. +#define OOPS_ABSTRACT_PARAMETERS_ENABLE_COPY_AND_MOVE(className, baseClassName) \ + protected: \ + className(const className &other) : className() { *this = other; } \ + className(className &&other) : className() { *this = std::move(other); } \ + className &operator=(const className &) = default; \ + className &operator=(className &&) = default; \ + private: \ + virtual className* cloneImpl() const = 0; \ + public: \ + std::unique_ptr clone() const { \ + return std::unique_ptr(cloneImpl()); \ + } \ + private: + /// \brief This macro needs to be invoked at the top of the declaration of each abstract subclass /// of Parameters, with \p className set to the name of the class being declared and \p /// baseClassName set to the name of the class from which it (directly) inherits. @@ -34,16 +52,30 @@ namespace oops { /// * the abstract subclass provides a clone() method; /// * the abstract subclass cannot be instantiated (the constructors are protected); /// * the assignment operators are protected to prevent slicing. +/// +/// If you need to customize the definition of the (non-copy and non-move) constructors, call +/// `OOPS_ABSTRACT_PARAMETERS_ENABLE_COPY_AND_MOVE(className, baseClassName)` instead of this macro +/// and define the constructors on your own. #define OOPS_ABSTRACT_PARAMETERS(className, baseClassName) \ protected: \ className() : baseClassName() {} \ explicit className(Parameters* parent) : baseClassName(parent) {} \ + OOPS_ABSTRACT_PARAMETERS_ENABLE_COPY_AND_MOVE(className, baseClassName) + +/// \brief This macro may be invoked at the top of the declaration of a concrete subclass of +/// Parameters instead of OOPS_CONCRETE_PARAMETERS() if the definition of the (non-copy and +/// non-move) constructors needs to be customized. See OOPS_CONCRETE_PARAMETERS() for more +/// information. +#define OOPS_CONCRETE_PARAMETERS_ENABLE_COPY_AND_MOVE(className, baseClassName) \ + public: \ className(const className &other) : className() { *this = other; } \ className(className &&other) : className() { *this = std::move(other); } \ className &operator=(const className &) = default; \ className &operator=(className &&) = default; \ private: \ - virtual className* cloneImpl() const = 0; \ + virtual className* cloneImpl() const { \ + return new className(*this); \ + } \ public: \ std::unique_ptr clone() const { \ return std::unique_ptr(cloneImpl()); \ @@ -62,23 +94,15 @@ namespace oops { /// * the children_ vector is set up correctly (for example, after copy construction it contains /// pointers to members the new Parameters instance rather than the instance that was copied); /// * the concrete subclass provides a clone() method. +/// +/// If you need to customize the definition of the (non-copy and non-move) constructors, call +/// `OOPS_CONCRETE_PARAMETERS_ENABLE_COPY_AND_MOVE(className, baseClassName)` instead of this macro +/// and define the constructors on your own. #define OOPS_CONCRETE_PARAMETERS(className, baseClassName) \ public: \ className() : baseClassName() {} \ explicit className(oops::Parameters* parent) : baseClassName(parent) {} \ - className(const className &other) : className() { *this = other; } \ - className(className &&other) : className() { *this = std::move(other); } \ - className &operator=(const className &) = default; \ - className &operator=(className &&) = default; \ - private: \ - virtual className* cloneImpl() const { \ - return new className(*this); \ - } \ - public: \ - std::unique_ptr clone() const { \ - return std::unique_ptr(cloneImpl()); \ - } \ - private: + OOPS_CONCRETE_PARAMETERS_ENABLE_COPY_AND_MOVE(className, baseClassName) /// \brief Abstract base class for collections of parameters located at the same level of the /// parameter hierarchy. From 7c00e2f0fc41fbd472f4f2aefbc547bbe42f4a84 Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Fri, 5 Feb 2021 15:02:38 -0700 Subject: [PATCH 053/142] use intel-oneapi for codebuild (#1057) * update codebuild intel; clean up other yamls * bugfix * bugfix * trigger tests * trigger tests * bugfix * qg_4dvar_saddlepoint test hangs; use 15G, 8CPUs for intel * exclude test_qg_4dvar_saddlepoint --- CI/buildspec_clang.yml | 6 ++- CI/buildspec_gnu.yml | 6 ++- CI/buildspec_intel.yml | 108 +++++++++++------------------------------ 3 files changed, 37 insertions(+), 83 deletions(-) diff --git a/CI/buildspec_clang.yml b/CI/buildspec_clang.yml index 0fb134e17..b43f2a7d7 100644 --- a/CI/buildspec_clang.yml +++ b/CI/buildspec_clang.yml @@ -21,7 +21,11 @@ phases: - echo $CODEBUILD_WEBHOOK_TRIGGER - echo $CODEBUILD_WEBHOOK_BASE_REF - - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PULL_REQUEST_MERGED" ]; + # Codebuild only runs on PUSH events if HEAD_REF + # is refs/heads/develop (merge to develop). In this + # case CODEBUILD_GIT_BRANCH="develop" + + - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PUSH" ]; then export CODEBUILD_GIT_BRANCH="develop"; echo "Merging to develop"; else export CODEBUILD_GIT_BRANCH=${CODEBUILD_WEBHOOK_HEAD_REF#refs/heads/}; diff --git a/CI/buildspec_gnu.yml b/CI/buildspec_gnu.yml index c8bd831d3..b49917cb6 100644 --- a/CI/buildspec_gnu.yml +++ b/CI/buildspec_gnu.yml @@ -24,7 +24,11 @@ phases: - echo $CODEBUILD_WEBHOOK_TRIGGER - echo $CODEBUILD_WEBHOOK_BASE_REF - - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PULL_REQUEST_MERGED" ]; + # Codebuild only runs on PUSH events if HEAD_REF + # is refs/heads/develop (merge to develop). In this + # case CODEBUILD_GIT_BRANCH="develop" + + - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PUSH" ]; then export CODEBUILD_GIT_BRANCH="develop"; echo "Merging to develop"; else export CODEBUILD_GIT_BRANCH=${CODEBUILD_WEBHOOK_HEAD_REF#refs/heads/}; diff --git a/CI/buildspec_intel.yml b/CI/buildspec_intel.yml index 7e09a3101..d4a6d1523 100644 --- a/CI/buildspec_intel.yml +++ b/CI/buildspec_intel.yml @@ -1,6 +1,7 @@ version: 0.2 env: + shell: bash parameter-store: GIT_USER: "/CodeBuild/Git_USER" GIT_PASS: "/CodeBuild/Git_PASS" @@ -22,59 +23,15 @@ phases: - echo $CODEBUILD_WEBHOOK_TRIGGER - echo $CODEBUILD_WEBHOOK_BASE_REF - - echo MPI setup for Docker - - mkdir -p /var/run/sshd - - ssh-keygen -A - - sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config - - sed -i 's/#RSAAuthentication yes/RSAAuthentication yes/g' /etc/ssh/sshd_config - - sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config - - - groupadd jcsda -g 9999 - - useradd jcsdauser - - mkdir -p /jcsda /build_container - - chown -R jcsdauser:jcsda /build_container /usr/local - - chmod 6755 /jcsda /build_container /usr/local - - - mkdir /jcsda/.ssh ; echo "StrictHostKeyChecking no" > /jcsda/.ssh/config - - mkdir -p /jcsda/.openmpi - - mkdir /jcsda/oops-bundle - - mkdir -p /home/jcsdauser/.openmpi - - - cp CI/default-mca-params.conf /home/jcsdauser/.openmpi/mca-params.conf - - cat /home/jcsdauser/.openmpi/mca-params.conf - - chown -R jcsdauser:jcsda /jcsda/ - - - su - jcsdauser -c "ssh-keygen -f /jcsda/.ssh/id_rsa -t rsa -N '' - && chmod 600 /jcsda/.ssh/config - && chmod 700 /jcsda/.ssh - && cp /jcsda/.ssh/id_rsa.pub /jcsda/.ssh/authorized_keys - && echo MPI setup for Docker done" - - su - jcsdauser -c "echo $CC - && echo $CXX - && echo $FC - && whereis mpicc" - - ## cannot source /etc/bash.bashrc so copy what's there for root - - sed '12s/INTEL_TARGET_ARCH=/INTEL_TARGET_ARCH=intel64/' /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh - - export COMPILERVARS_ARCHITECTURE=intel64 - - export COMPILERVARS_PLATFORM=linux - - . /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh - - export FC=mpiifort - - export CC=mpiicc - - export CXX=mpiicpc - - export PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - - export LD_LIBRARY_PATH=/usr/local/lib - - export LIBRARY_PATH=/usr/local/lib - - . /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh - pre_build: commands: - echo Executing pre_build phase - - git lfs install # creates .gitconfig - - cp ~/.gitconfig /home/jcsdauser/ - - cp CI/CMakeLists.txt /jcsda/oops-bundle - - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PULL_REQUEST_MERGED" ]; + # Codebuild only runs on PUSH events if HEAD_REF + # is refs/heads/develop (merge to develop). In this + # case CODEBUILD_GIT_BRANCH="develop" + # + - if [ "$CODEBUILD_WEBHOOK_EVENT" = "PUSH" ]; then export CODEBUILD_GIT_BRANCH="develop"; echo "Merging to develop"; else export CODEBUILD_GIT_BRANCH=${CODEBUILD_WEBHOOK_HEAD_REF#refs/heads/}; @@ -83,14 +40,15 @@ phases: - echo "CODEBUILD_GIT_BRANCH=${CODEBUILD_GIT_BRANCH}" - echo "CODEBUILD_SOURCE_VERSION=${CODEBUILD_SOURCE_VERSION}" - - cd CI - - if [ "$CODEBUILD_GIT_BRANCH" = "develop" ]; then export CODEBUILD_GIT_BRANCH_FORK="release-stable"; else export CODEBUILD_GIT_BRANCH_FORK=${CODEBUILD_GIT_BRANCH}; echo "CODEBUILD_GIT_BRANCH_FORK=${CODEBUILD_GIT_BRANCH_FORK}"; fi + - cd CI + - . /etc/profile.d/intel.sh + # oops - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/oops $CODEBUILD_GIT_BRANCH oops /jcsda/oops-bundle develop # fckit @@ -98,6 +56,10 @@ phases: # atlas - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/atlas $CODEBUILD_GIT_BRANCH_FORK atlas /jcsda/oops-bundle release-stable + # move CMakeLists.txt from oops/CI to bundle directory + - cp CMakeLists.txt /jcsda/oops-bundle + - chmod 777 -R /jcsda/oops-bundle + - cd /jcsda/oops-bundle - ls @@ -105,20 +67,14 @@ phases: commands: - echo Executing build phase ## cannot source /etc/bash.bashrc so copy what's there for jscdauser - - su - jcsdauser -c "export COMPILERVARS_ARCHITECTURE=intel64 - && export COMPILERVARS_PLATFORM=linux - && . /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh + - su - jedi -c "export CC=mpiicc && export FC=mpiifort - && export CC=mpiicc && export CXX=mpiicpc - && export PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - && export LD_LIBRARY_PATH=/usr/local/lib - && export LIBRARY_PATH=/usr/local/lib - && . /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh - && cd /build_container - && ls + && export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + && export PATH=/usr/local/lib:$PATH + && cd /home/jedi && ecbuild /jcsda/oops-bundle/ - && cd /build_container/oops + && cd /home/jedi/oops && make -j2" - export BUILD_STATUS="0" @@ -131,18 +87,13 @@ phases: fi - echo $BUILD_STATUS - - su - jcsdauser -c "export COMPILERVARS_ARCHITECTURE=intel64 - && export COMPILERVARS_PLATFORM=linux - && . /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh + - su - jedi -c "export CC=mpiicc && export FC=mpiifort - && export CC=mpiicc && export CXX=mpiicpc - && export PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - && export LD_LIBRARY_PATH=/usr/local/lib - && export LIBRARY_PATH=/usr/local/lib - && . /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh - && cd /build_container/oops - && ctest" + && export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + && export PATH=/usr/local/lib:$PATH + && cd /home/jedi/oops + && ctest -E test_qg_4dvar_saddlepoint" post_build: commands: @@ -151,17 +102,12 @@ phases: - if [ "$BUILD_STATUS" = "1" ] && [ "$CODEBUILD_BUILD_SUCCEEDING" = "0" ]; then echo "Build passed, rerun failed tests"; - su - jcsdauser -c "export COMPILERVARS_ARCHITECTURE=intel64 - && export COMPILERVARS_PLATFORM=linux - && . /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh + su - jedi -c "export CC=mpiicc && export FC=mpiifort - && export CC=mpiicc && export CXX=mpiicpc - && export PATH=/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - && export LD_LIBRARY_PATH=/usr/local/lib - && export LIBRARY_PATH=/usr/local/lib - && . /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh - && cd /build_container/oops/ + && export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + && export PATH=/usr/local/lib:$PATH + && cd /home/jedi/oops && ctest -VV --rerun-failed"; else echo "Build failed"; fi From 083a395fe91888dadc1663bf6b92bf1f3407ead9 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Mon, 8 Feb 2021 14:47:32 -0700 Subject: [PATCH 054/142] refactor VariableChange/VariableChangeBase (#1008) * refactor VariableChange/VariableChangeBase * bugfix for parameters; move comments; missed include * add VarChange base class (one for generic, one for MODEL-specific Varchanges) * add print for varin_,varout_ & remove obsolete comment --- qg/model/ChangeVarQG.h | 11 +- qg/model/QgTraits.h | 1 - qg/model/instantiateQgChangeVarFactory.h | 5 +- qg/test/testoutput/convertstate.test | 60 ++++-- src/oops/assimilation/CostFct4DVar.h | 15 +- src/oops/assimilation/CostFctWeak.h | 9 +- src/oops/base/ModelBase.h | 2 +- src/oops/base/VariableChangeBase.h | 190 +++++++++--------- src/oops/base/VariableChangeParametersBase.h | 10 +- src/oops/generic/IdVariableChange.h | 8 +- .../instantiateVariableChangeFactory.h | 3 +- src/oops/interface/VariableChange.h | 147 ++++++++++---- src/oops/runs/ConvertState.h | 18 +- src/test/interface/VariableChange.h | 22 +- 14 files changed, 290 insertions(+), 211 deletions(-) diff --git a/qg/model/ChangeVarQG.h b/qg/model/ChangeVarQG.h index ea94d8bf3..8a632c7ac 100644 --- a/qg/model/ChangeVarQG.h +++ b/qg/model/ChangeVarQG.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2017-2018 UCAR. + * (C) Copyright 2017-2021 UCAR. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -11,9 +11,10 @@ #include #include -#include "oops/util/Printable.h" +#include "oops/base/VariableChangeBase.h" #include "oops/qg/QgFortran.h" +#include "oops/qg/QgTraits.h" // Forward declarations namespace eckit { @@ -27,7 +28,7 @@ namespace qg { // ----------------------------------------------------------------------------- /// QG change of variable -class ChangeVarQG: public util::Printable { +class ChangeVarQG: public oops::VariableChangeBase { public: static const std::string classname() {return "qg::ChangeVarQG";} @@ -35,8 +36,8 @@ class ChangeVarQG: public util::Printable { ~ChangeVarQG(); /// Perform transforms - void changeVar(const StateQG &, StateQG &) const; - void changeVarInverse(const StateQG &, StateQG &) const; + void changeVar(const StateQG &, StateQG &) const override; + void changeVarInverse(const StateQG &, StateQG &) const override; private: void print(std::ostream &) const override; diff --git a/qg/model/QgTraits.h b/qg/model/QgTraits.h index d7b96bca4..3800dafc9 100644 --- a/qg/model/QgTraits.h +++ b/qg/model/QgTraits.h @@ -14,7 +14,6 @@ #include #include "oops/qg/AnalyticInit.h" -#include "oops/qg/ChangeVarQG.h" #include "oops/qg/ErrorCovarianceQG.h" #include "oops/qg/GeometryQG.h" #include "oops/qg/GeometryQGIterator.h" diff --git a/qg/model/instantiateQgChangeVarFactory.h b/qg/model/instantiateQgChangeVarFactory.h index 3c1572f63..05a3078e7 100644 --- a/qg/model/instantiateQgChangeVarFactory.h +++ b/qg/model/instantiateQgChangeVarFactory.h @@ -21,9 +21,8 @@ namespace qg { void instantiateQgChangeVarFactory() { - static oops::VariableChangeMaker > - makerChVarQG_("ChVarQG"); + static oops::VariableChangeMaker makerchangevar_("ChVarQG"); + static oops::LinearVariableChangeMaker > makerChLinVarQG_("ChVarQG"); diff --git a/qg/test/testoutput/convertstate.test b/qg/test/testoutput/convertstate.test index 42da00510..d75d8a02a 100644 --- a/qg/test/testoutput/convertstate.test +++ b/qg/test/testoutput/convertstate.test @@ -6,7 +6,10 @@ Test : Boundary conditions are activated Test : Min= -4.4304e+08, Max= 9.3153e+07, RMS= 1.7024e+08 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : State after QG change of variable transform: +Test : Variable transform: QG change of variable +Test : Variable change from: 1 variables: x +Test : Variable change to: 1 variables: q +Test : State after variable transform: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = potential vorticity @@ -14,7 +17,10 @@ Test : Boundary conditions are activated Test : Min= -5.8900e-04, Max= 9.6046e-04, RMS= 3.5956e-04 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : State after QG change of variable transform: +Test : Variable transform: QG change of variable +Test : Variable change from: 1 variables: q +Test : Variable change to: 1 variables: x +Test : State after variable transform: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = streamfunction @@ -22,7 +28,10 @@ Test : Boundary conditions are activated Test : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : State after QG change of variable transform: +Test : Variable transform: QG change of variable +Test : Variable change from: 1 variables: q +Test : Variable change to: 1 variables: x +Test : State after variable transform: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = potential vorticity @@ -30,7 +39,10 @@ Test : Boundary conditions are activated Test : Min= -5.8900e-04, Max= 9.6046e-04, RMS= 3.5956e-04 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : State after QG change of variable transform: +Test : Variable transform: QG change of variable +Test : Variable change from: 1 variables: x +Test : Variable change to: 1 variables: q +Test : State after variable transform: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = streamfunction @@ -38,7 +50,10 @@ Test : Boundary conditions are activated Test : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : State after IdVariableChange transform: +Test : Variable transform: IdVariableChange +Test : Variable change from: 1 variables: x +Test : Variable change to: 1 variables: x +Test : State after variable transform: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = streamfunction @@ -46,7 +61,10 @@ Test : Boundary conditions are activated Test : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : State after IdVariableChange transform: +Test : Variable transform: IdVariableChange +Test : Variable change from: 1 variables: x +Test : Variable change to: 1 variables: x +Test : State after variable transform: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = streamfunction @@ -70,7 +88,10 @@ Test : Boundary conditions are activated Test : Min= -4.3926e+08, Max= 9.3338e+07, RMS= 1.6950e+08 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : State after QG change of variable transform: +Test : Variable transform: QG change of variable +Test : Variable change from: 1 variables: x +Test : Variable change to: 1 variables: q +Test : State after variable transform: Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = potential vorticity @@ -78,7 +99,10 @@ Test : Boundary conditions are activated Test : Min= -5.9340e-04, Max= 9.5651e-04, RMS= 3.5978e-04 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : State after QG change of variable transform: +Test : Variable transform: QG change of variable +Test : Variable change from: 1 variables: q +Test : Variable change to: 1 variables: x +Test : State after variable transform: Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = streamfunction @@ -86,7 +110,10 @@ Test : Boundary conditions are activated Test : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : State after QG change of variable transform: +Test : Variable transform: QG change of variable +Test : Variable change from: 1 variables: q +Test : Variable change to: 1 variables: x +Test : State after variable transform: Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = potential vorticity @@ -94,7 +121,10 @@ Test : Boundary conditions are activated Test : Min= -5.9340e-04, Max= 9.5651e-04, RMS= 3.5978e-04 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : State after QG change of variable transform: +Test : Variable transform: QG change of variable +Test : Variable change from: 1 variables: x +Test : Variable change to: 1 variables: q +Test : State after variable transform: Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = streamfunction @@ -102,7 +132,10 @@ Test : Boundary conditions are activated Test : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : State after IdVariableChange transform: +Test : Variable transform: IdVariableChange +Test : Variable change from: 1 variables: x +Test : Variable change to: 1 variables: x +Test : State after variable transform: Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = streamfunction @@ -110,7 +143,10 @@ Test : Boundary conditions are activated Test : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : State after IdVariableChange transform: +Test : Variable transform: IdVariableChange +Test : Variable change from: 1 variables: x +Test : Variable change to: 1 variables: x +Test : State after variable transform: Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = streamfunction diff --git a/src/oops/assimilation/CostFct4DVar.h b/src/oops/assimilation/CostFct4DVar.h index 137b27491..91254f44d 100644 --- a/src/oops/assimilation/CostFct4DVar.h +++ b/src/oops/assimilation/CostFct4DVar.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2009-2016 ECMWF. + * (C) Copyright 2020-2021 UCAR. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -25,13 +26,13 @@ #include "oops/base/PostProcessorTLAD.h" #include "oops/base/StateInfo.h" #include "oops/base/TrajectorySaver.h" -#include "oops/base/VariableChangeBase.h" #include "oops/base/Variables.h" #include "oops/interface/Geometry.h" #include "oops/interface/Increment.h" #include "oops/interface/LinearModel.h" #include "oops/interface/Model.h" #include "oops/interface/State.h" +#include "oops/interface/VariableChange.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" @@ -56,7 +57,7 @@ template class CostFct4DVar : public CostFunction< typedef State State_; typedef Model Model_; typedef LinearModel LinearModel_; - typedef VariableChangeBase VarCha_; + typedef VariableChange VarCha_; typedef LinearVariableChangeBase LinVarCha_; public: @@ -93,7 +94,7 @@ template class CostFct4DVar : public CostFunction< Model_ model_; const Variables ctlvars_; std::shared_ptr tlm_; - std::unique_ptr an2model_; + VarCha_ an2model_; std::unique_ptr inc2model_; }; @@ -105,7 +106,8 @@ CostFct4DVar::CostFct4DVar(const eckit::Configuration & config, : CostFunction::CostFunction(config), comm_(comm), resol_(eckit::LocalConfiguration(config, "geometry"), comm), model_(resol_, eckit::LocalConfiguration(config, "model")), - ctlvars_(config, "analysis variables"), tlm_(), an2model_(), inc2model_() + ctlvars_(config, "analysis variables"), tlm_(), an2model_(resol_, config), + inc2model_() { Log::trace() << "CostFct4DVar:CostFct4DVar" << std::endl; windowLength_ = util::Duration(config.getString("window length")); @@ -113,7 +115,6 @@ CostFct4DVar::CostFct4DVar(const eckit::Configuration & config, windowEnd_ = windowBegin_ + windowLength_; this->setupTerms(config); // ASSERT(ctlvars_ <= this->background().state().variables()); - an2model_.reset(VariableChangeFactory::create(config, resol_)); Log::trace() << "CostFct4DVar constructed" << std::endl; } @@ -150,9 +151,9 @@ void CostFct4DVar::runNL(CtrlVar_ & xx, PostProcessor & post ASSERT(xx.state().validTime() == windowBegin_); State_ xm(xx.state().geometry(), model_.variables(), windowBegin_); - an2model_->changeVar(xx.state(), xm); + an2model_.changeVar(xx.state(), xm); model_.forecast(xm, xx.modVar(), windowLength_, post); - an2model_->changeVarInverse(xm, xx.state()); + an2model_.changeVarInverse(xm, xx.state()); ASSERT(xx.state().validTime() == windowEnd_); } diff --git a/src/oops/assimilation/CostFctWeak.h b/src/oops/assimilation/CostFctWeak.h index 14b32dae7..3dd523820 100644 --- a/src/oops/assimilation/CostFctWeak.h +++ b/src/oops/assimilation/CostFctWeak.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2009-2016 ECMWF. + * (C) Copyright 2020-2021 UCAR. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -15,6 +16,8 @@ #include #include +#include + #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/CostFunction.h" #include "oops/assimilation/CostJbJq.h" @@ -26,13 +29,13 @@ #include "oops/base/PostProcessorTLAD.h" #include "oops/base/StateInfo.h" #include "oops/base/TrajectorySaver.h" -#include "oops/base/VariableChangeBase.h" #include "oops/base/Variables.h" #include "oops/interface/Geometry.h" #include "oops/interface/Increment.h" #include "oops/interface/LinearModel.h" #include "oops/interface/Model.h" #include "oops/interface/State.h" +#include "oops/interface/VariableChange.h" #include "oops/mpi/mpi.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" @@ -56,7 +59,7 @@ template class CostFctWeak : public CostFunction State_; typedef Model Model_; typedef LinearModel LinearModel_; - typedef VariableChangeBase VarCha_; + typedef VariableChange VarCha_; typedef LinearVariableChangeBase LinVarCha_; public: @@ -146,7 +149,7 @@ CostFctWeak::CostFctWeak(const eckit::Configuration & config, model_.reset(new Model_(*resol_, eckit::LocalConfiguration(config, "model"))); this->setupTerms(config); - an2model_.reset(VariableChangeFactory::create(config, *resol_)); + an2model_ = boost::make_unique(*resol_, config); Log::trace() << "CostFctWeak constructed" << std::endl; } diff --git a/src/oops/base/ModelBase.h b/src/oops/base/ModelBase.h index 6ec517bae..539c0792b 100644 --- a/src/oops/base/ModelBase.h +++ b/src/oops/base/ModelBase.h @@ -73,7 +73,7 @@ class GenericModelBase : public util::Printable, }; -/// \brief Base class for MODEL-specific implementations of Model class +/// \brief Base class for MODEL-specific implementations of Model class. /// The complete interface that needs to be implemented is described in GenericModelBase. /// ModelBase overrides GenericModelBase methods to pass MODEL-specific implementations /// of State and ModelAuxControl to the MODEL-specific implementation of Model. diff --git a/src/oops/base/VariableChangeBase.h b/src/oops/base/VariableChangeBase.h index 2a11da8c8..b4487e495 100644 --- a/src/oops/base/VariableChangeBase.h +++ b/src/oops/base/VariableChangeBase.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2018 UCAR + * (C) Copyright 2018-2021 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -17,7 +17,7 @@ #include #include "oops/base/VariableChangeParametersBase.h" -#include "oops/base/Variables.h" +#include "oops/interface/Geometry.h" #include "oops/interface/State.h" #include "oops/util/AssociativeContainers.h" #include "oops/util/parameters/ConfigurationParameter.h" @@ -34,41 +34,69 @@ namespace oops { // ------------------------------------------------------------------------------------------------- -/// Base class for generic variable transform +/// Base class for a variable transforms, defining the interfaces. +/// Use this class as a base class for generic implementations, +/// and VariableChangeBase as a base class for MODEL-specific implementations. /// /// Note: subclasses can opt to extract their settings either from a Configuration object or from a /// subclass of Parameters. /// -/// In the former case, they should provide a constructor taking a const reference to an -/// eckit::Configuration object. In the latter case, the implementer should first define a subclass -/// of Parameters holding the settings of the variable change in question. The latter should -/// then typedef `Parameters_` to the name of that subclass and provide a constructor taking a -/// const reference to an instance of that subclass. +/// In the former case, they should provide a constructor with the following signature: +/// +/// VariableChange(const Geometry_ &, const eckit::Configuration &); +/// +/// In the latter case, the implementer should first define a subclass of +/// VariableChangeParametersBase holding the settings of the variable change in question. +/// The implementation of the VariableChange interface should then typedef `Parameters_` +/// to the name of that subclass and provide a constructor with the following signature: +/// +/// VariableChange(const Geometry_ &, const Parameters_ &); template -class VariableChangeBase : public util::Printable, - private boost::noncopyable { - typedef State State_; +class GenericVariableChangeBase : public util::Printable, + private boost::noncopyable { + typedef State State_; public: - explicit VariableChangeBase(const VariableChangeParametersBase &); - explicit VariableChangeBase(const eckit::Configuration &); - virtual ~VariableChangeBase() {} - - void setInputVariables(const Variables & vars) { varin_.reset(new Variables(vars)); } - void setOutputVariables(const Variables & vars) { varout_.reset(new Variables(vars)); } + GenericVariableChangeBase() = default; + virtual ~GenericVariableChangeBase() = default; - virtual void changeVar(const State_ &, State_ &) const = 0; - virtual void changeVarInverse(const State_ &, State_ &) const = 0; - - State_ changeVar(const State_ &) const; - State_ changeVarInverse(const State_ &) const; + /// change variables from state \p xin to \p xout + virtual void changeVar(const State_ & xin, State_ & xout) const = 0; + /// inverse of changeVar, change variables back from \p xout to \p xin + virtual void changeVarInverse(const State_ & xout, State_ & xin) const = 0; private: + /// Print, used for logging virtual void print(std::ostream &) const = 0; - std::unique_ptr varin_; - std::unique_ptr varout_; }; +/// \brief Base class for MODEL-specific implementations of VariableChange class. +/// The complete interface that needs to be implemented is described in +/// GenericVariableChangeBase. VariableChangeBase overrides GenericVariableChangeBase +/// methods to pass MODEL-specific implementations of State to the MODEL-specific +/// implementation of VariableChange. +template +class VariableChangeBase : public GenericVariableChangeBase { + typedef typename MODEL::State State_; + + public: + VariableChangeBase() = default; + virtual ~VariableChangeBase() = default; + + /// Overrides for VariableChangeBase classes, passing MODEL-specific classes to the + /// MODEL-specific implementations of VariableChange + void changeVar(const State & xin, State & xout) const final + { this->changeVar(xin.state(), xout.state()); } + void changeVarInverse(const State & xout, State & xin) const final + { this->changeVarInverse(xout.state(), xin.state()); } + + /// change variables from state \p xin to \p xout + virtual void changeVar(const State_ & xin, State_ & xout) const = 0; + /// inverse of changeVar, change variables back from \p xout to \p xin + virtual void changeVarInverse(const State_ & xout, State_ & xin) const = 0; +}; + + // ============================================================================= template @@ -122,14 +150,8 @@ class VariableChangeFactory { /// parameters. \p parameters must be an instance of the subclass of /// VariableChangeParametersBase associated with that variable change type, otherwise an /// exception will be thrown. - static VariableChangeBase * create(const VariableChangeParametersBase &, - const Geometry_ &); - - /// \brief Create and return a new variable change. - /// - /// Deprecated overload taking a Configuration instead of a VariableChangeParametersBase. - static VariableChangeBase * create(const eckit::Configuration &, const Geometry_ &); - + static GenericVariableChangeBase * create(const Geometry_ &, + const VariableChangeParametersBase &); /// \brief Create and return an instance of the subclass of VariableChangeParametersBase /// storing parameters of variable changes of the specified type. static std::unique_ptr createParameters(const std::string &name); @@ -147,8 +169,8 @@ class VariableChangeFactory { explicit VariableChangeFactory(const std::string &); private: - virtual VariableChangeBase * make(const VariableChangeParametersBase &, - const Geometry_ &) = 0; + virtual GenericVariableChangeBase * make(const Geometry_ &, + const VariableChangeParametersBase &) = 0; virtual std::unique_ptr makeParameters() const = 0; @@ -161,15 +183,15 @@ class VariableChangeFactory { // ------------------------------------------------------------------------------------------------- template -class VariableChangeMaker : public VariableChangeFactory { +class GenericVariableChangeMaker : public VariableChangeFactory { /// Defined as T::Parameters_ if T defines a Parameters_ type; otherwise as /// GenericVariableChangeParameters. typedef TParameters_IfAvailableElseFallbackType_t Parameters_; typedef Geometry Geometry_; - VariableChangeBase * make(const VariableChangeParametersBase & params, - const Geometry_ & resol) override { + GenericVariableChangeBase * make(const Geometry_ & resol, + const VariableChangeParametersBase & params) override { const auto &stronglyTypedParams = dynamic_cast(params); return new T(resol, parametersOrConfiguration::value>(stronglyTypedParams)); @@ -179,6 +201,33 @@ class VariableChangeMaker : public VariableChangeFactory { return boost::make_unique(); } + public: + explicit GenericVariableChangeMaker(const std::string & name) + : VariableChangeFactory(name) {} +}; + + +// ------------------------------------------------------------------------------------------------- + +template +class VariableChangeMaker : public VariableChangeFactory { + /// Defined as T::Parameters_ if T defines a Parameters_ type; otherwise as + /// GenericVariableChangeParameters. + typedef TParameters_IfAvailableElseFallbackType_t Parameters_; + + typedef Geometry Geometry_; + + VariableChangeBase * make(const Geometry_ & resol, + const VariableChangeParametersBase & params) override { + const auto &stronglyTypedParams = dynamic_cast(params); + return new T(resol.geometry(), + parametersOrConfiguration::value>(stronglyTypedParams)); + } + + std::unique_ptr makeParameters() const override { + return boost::make_unique(); + } + public: explicit VariableChangeMaker(const std::string & name) : VariableChangeFactory(name) {} @@ -197,32 +246,22 @@ VariableChangeFactory::VariableChangeFactory(const std::string & name) { // ------------------------------------------------------------------------------------------------- template -VariableChangeBase * VariableChangeFactory::create( - const VariableChangeParametersBase & params, const Geometry_ & resol) +GenericVariableChangeBase * VariableChangeFactory::create( + const Geometry_ & resol, const VariableChangeParametersBase & params) { Log::trace() << "VariableChangeBase::create starting" << std::endl; // Not good: should not create anything if no variable change required. YT - const std::string &id = params.variableChange.value().value(); + const std::string &id = params.variableChange.value(); typename std::map*>::iterator jerr = getMakers().find(id); if (jerr == getMakers().end()) { throw std::runtime_error(id + " does not exist in variable change factory."); } - VariableChangeBase * ptr = jerr->second->make(params, resol); + GenericVariableChangeBase * ptr = jerr->second->make(resol, params); Log::trace() << "VariableChangeBase::create done" << std::endl; return ptr; } -// ------------------------------------------------------------------------------------------------- - -template -VariableChangeBase * VariableChangeFactory::create(const eckit::Configuration & conf, - const Geometry_ & resol) { - VariableChangeParametersWrapper parameters; - parameters.validateAndDeserialize(conf); - return create(parameters.variableChangeParameters, resol); -} - // ----------------------------------------------------------------------------- template @@ -236,55 +275,6 @@ std::unique_ptr VariableChangeFactory::crea return it->second->makeParameters(); } -// ================================================================================================= - -template -VariableChangeBase::VariableChangeBase(const VariableChangeParametersBase & params) - : varin_(), varout_() -{ - if (params.inputVariables.value() != boost::none) { - varin_.reset(new Variables(*params.inputVariables.value())); - Log::trace() << "VariableChangeBase::VariableChangeBase input variables: " - << *varin_ << std::endl; - } - if (params.outputVariables.value() != boost::none) { - varout_.reset(new Variables(*params.outputVariables.value())); - Log::trace() << "VariableChangeBase::VariableChangeBase output variables: " - << *varout_ << std::endl; - } -} - -// ------------------------------------------------------------------------------------------------- - -template -VariableChangeBase::VariableChangeBase(const eckit::Configuration & conf) - : VariableChangeBase(validateAndDeserialize(conf)) -{} - -// ------------------------------------------------------------------------------------------------- - -template -State VariableChangeBase::changeVar(const State_ & xin) const { - Log::trace() << "VariableChangeBase::changeVar starting" << std::endl; - ASSERT(varout_); - State_ xout(xin.geometry(), *varout_, xin.validTime()); - this->changeVar(xin, xout); - Log::trace() << "VariableChangeBase::changeVar done" << std::endl; - return xout; -} - -// ------------------------------------------------------------------------------------------------- - -template -State VariableChangeBase::changeVarInverse(const State_ & xin) const { - Log::trace() << "VariableChangeBase::changeVarInverse starting" << std::endl; - ASSERT(varin_); - State_ xout(xin.geometry(), *varin_, xin.validTime()); - this->changeVarInverse(xin, xout); - Log::trace() << "VariableChangeBase::changeVarInverse done" << std::endl; - return xout; -} - // ------------------------------------------------------------------------------------------------- } // namespace oops diff --git a/src/oops/base/VariableChangeParametersBase.h b/src/oops/base/VariableChangeParametersBase.h index 4e53e32d7..66ad7baf5 100644 --- a/src/oops/base/VariableChangeParametersBase.h +++ b/src/oops/base/VariableChangeParametersBase.h @@ -13,6 +13,7 @@ #include "oops/base/ParameterTraitsVariables.h" #include "oops/base/Variables.h" #include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameter.h" #include "oops/util/parameters/Parameters.h" namespace oops { @@ -22,14 +23,7 @@ class VariableChangeParametersBase : public Parameters { OOPS_ABSTRACT_PARAMETERS(VariableChangeParametersBase, Parameters) public: /// \brief Variable change type. - /// - /// \note This parameter is marked as optional because it is only required in certain - /// circumstances (e.g. when variable change parameters are deserialized into a - /// VariableChangeParametersWrapper and used by VariableChangeFactory to instantiate a variable - /// change whose type is determined at runtime), but not others (e.g. in tests written with a - /// particular variable change in mind). VariableChangeParametersWrapper will throw an exception - /// if this parameter is not provided. - OptionalParameter variableChange{"variable change", this}; + Parameter variableChange{"variable change", "Identity", this}; OptionalParameter inputVariables{"input variables", this}; OptionalParameter outputVariables{"output variables", this}; diff --git a/src/oops/generic/IdVariableChange.h b/src/oops/generic/IdVariableChange.h index a5db2758c..8e51e720e 100644 --- a/src/oops/generic/IdVariableChange.h +++ b/src/oops/generic/IdVariableChange.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2017-2018 UCAR. + * (C) Copyright 2017-2021 UCAR. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -27,15 +27,13 @@ namespace oops { /// No change of variable template -class IdVariableChange : public VariableChangeBase { +class IdVariableChange : public GenericVariableChangeBase { typedef Geometry Geometry_; typedef State State_; public: static const std::string classname() {return "oops::IdVariableChange";} - IdVariableChange(const Geometry_ &, const eckit::Configuration & conf) - : VariableChangeBase(conf) {} - virtual ~IdVariableChange() {} + IdVariableChange(const Geometry_ &, const eckit::Configuration &) {} /// Perform identity change of variable void changeVar(const State_ & x1, State_ & x2) const override {x2 = x1;} diff --git a/src/oops/generic/instantiateVariableChangeFactory.h b/src/oops/generic/instantiateVariableChangeFactory.h index e2edd29d9..0ce80ae03 100644 --- a/src/oops/generic/instantiateVariableChangeFactory.h +++ b/src/oops/generic/instantiateVariableChangeFactory.h @@ -1,5 +1,6 @@ /* * (C) Copyright 2009-2016 ECMWF. + * (C) Copyright 2021 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -21,7 +22,7 @@ namespace oops { template void instantiateVariableChangeFactory() { // Nonlinear change of variables - static VariableChangeMaker > makerId_("Identity"); + static GenericVariableChangeMaker > makerId_("Identity"); // Linear change of variables static LinearVariableChangeMaker > makerIdLin_("Identity"); diff --git a/src/oops/interface/VariableChange.h b/src/oops/interface/VariableChange.h index 6a4dcdd14..ba92c1ff8 100644 --- a/src/oops/interface/VariableChange.h +++ b/src/oops/interface/VariableChange.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2018 UCAR + * (C) Copyright 2018-2021 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -11,9 +11,8 @@ #include #include -#include - #include "oops/base/VariableChangeBase.h" +#include "oops/base/VariableChangeParametersBase.h" #include "oops/base/Variables.h" #include "oops/interface/Geometry.h" #include "oops/interface/State.h" @@ -28,86 +27,146 @@ namespace eckit { namespace oops { -// ----------------------------------------------------------------------------- -/// Wrapper for change of variable - -template -class VariableChange : public oops::VariableChangeBase { +/// \brief Encapsulates the nonlinear variable change +/// Note: to see methods that need to be implemented in the implementation, +/// see VariableChangeBase class. +template +class VariableChange : public util::Printable, + private util::ObjectCounter > { typedef Geometry Geometry_; typedef State State_; + typedef GenericVariableChangeBase VariableChangeBase_; public: - /// Defined as CHVAR::Parameters_ if CHVAR defines a Parameters_ type; otherwise as - /// GenericVariableChangeParameters - typedef TParameters_IfAvailableElseFallbackType_t< - CHVAR, GenericVariableChangeParameters> Parameters_; - static const std::string classname() {return "oops::VariableChange";} - VariableChange(const Geometry_ &, const Parameters_ &); + VariableChange(const Geometry_ &, const VariableChangeParametersBase &); + VariableChange(const Geometry_ &, const eckit::Configuration &); virtual ~VariableChange(); - - void changeVar(const State_ &, State_ &) const override; - void changeVarInverse(const State_ &, State_ &) const override; + VariableChange(const VariableChange &) = delete; + VariableChange(VariableChange &&) = default; + const VariableChange & operator=(const VariableChange &) = delete; + VariableChange & operator=(VariableChange &&) = default; + + /// set input variables for variable transform + void setInputVariables(const Variables & vars) { varin_.reset(new Variables(vars)); } + /// set output variables for variable transform + void setOutputVariables(const Variables & vars) { varout_.reset(new Variables(vars)); } + + /// change variable from state \p xin to \p xout + void changeVar(const State_ & xin, State_ & xout) const; + /// inverse of changeVar, change variables back from \p xout to \p xin + void changeVarInverse(const State_ & xout, State_ & xin) const; + + /// return change of variable \p xin + State_ changeVar(const State_ & xin) const; + /// return inverse of variable change applied to \p xout + State_ changeVarInverse(const State_ & xout) const; private: void print(std::ostream &) const override; - std::unique_ptr chvar_; + std::unique_ptr chvar_; /// pointer to the VariableChange implementation + std::unique_ptr varin_; /// input variables + std::unique_ptr varout_; /// output variables }; // ============================================================================= -template -VariableChange::VariableChange(const Geometry_ & geom, - const Parameters_ & params) - : VariableChangeBase(params), chvar_() +template +VariableChange::VariableChange(const Geometry_ & geom, + const VariableChangeParametersBase & params) + : chvar_() { - Log::trace() << "VariableChange::VariableChange starting" << std::endl; + Log::trace() << "VariableChange::VariableChange starting" << std::endl; util::Timer timer(classname(), "VariableChange"); - chvar_.reset(new CHVAR(geom.geometry(), - parametersOrConfiguration::value>(params))); - Log::trace() << "VariableChange::VariableChange done" << std::endl; + if (params.inputVariables.value() != boost::none) { + varin_.reset(new Variables(*params.inputVariables.value())); + Log::trace() << "VariableChange::VariableChange input variables: " + << *varin_ << std::endl; + } + if (params.outputVariables.value() != boost::none) { + varout_.reset(new Variables(*params.outputVariables.value())); + Log::trace() << "VariableChange::VariableChange output variables: " + << *varout_ << std::endl; + } + chvar_.reset(VariableChangeFactory::create(geom, params)); + Log::trace() << "VariableChange::VariableChange done" << std::endl; } // ----------------------------------------------------------------------------- -template -VariableChange::~VariableChange() { - Log::trace() << "VariableChange::~VariableChange starting" << std::endl; +template +VariableChange::VariableChange(const Geometry_ & geom, const eckit::Configuration & conf) + : VariableChange(geom, + validateAndDeserialize>(conf).variableChangeParameters) +{} + +// ----------------------------------------------------------------------------- + +template +VariableChange::~VariableChange() { + Log::trace() << "VariableChange::~VariableChange starting" << std::endl; util::Timer timer(classname(), "~VariableChange"); chvar_.reset(); - Log::trace() << "VariableChange::~VariableChange done" << std::endl; + Log::trace() << "VariableChange::~VariableChange done" << std::endl; } // ----------------------------------------------------------------------------- -template -void VariableChange::changeVar(const State_ & x1, State_ & x2) const { - Log::trace() << "VariableChange::changeVar starting" << std::endl; +template +void VariableChange::changeVar(const State_ & x1, State_ & x2) const { + Log::trace() << "VariableChange::changeVar starting" << std::endl; util::Timer timer(classname(), "changeVar"); - chvar_->changeVar(x1.state(), x2.state()); - Log::trace() << "VariableChange::changeVar done" << std::endl; + chvar_->changeVar(x1, x2); + Log::trace() << "VariableChange::changeVar done" << std::endl; } // ----------------------------------------------------------------------------- -template -void VariableChange::changeVarInverse(const State_ & x1, State_ & x2) const { - Log::trace() << "VariableChange::changeVarInverse starting" << std::endl; +template +void VariableChange::changeVarInverse(const State_ & x1, State_ & x2) const { + Log::trace() << "VariableChange::changeVarInverse starting" << std::endl; util::Timer timer(classname(), "changeVarInverse"); - chvar_->changeVarInverse(x1.state(), x2.state()); - Log::trace() << "VariableChange::changeVarInverse done" << std::endl; + chvar_->changeVarInverse(x1, x2); + Log::trace() << "VariableChange::changeVarInverse done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +State VariableChange::changeVar(const State_ & xin) const { + Log::trace() << "VariableChange::changeVar starting" << std::endl; + ASSERT(varout_); + State_ xout(xin.geometry(), *varout_, xin.validTime()); + this->changeVar(xin, xout); + Log::trace() << "VariableChange::changeVar done" << std::endl; + return xout; +} + +// ----------------------------------------------------------------------------- + +template +State VariableChange::changeVarInverse(const State_ & xin) const { + Log::trace() << "VariableChange::changeVarInverse starting" << std::endl; + ASSERT(varin_); + State_ xout(xin.geometry(), *varin_, xin.validTime()); + this->changeVarInverse(xin, xout); + Log::trace() << "VariableChange::changeVarInverse done" << std::endl; + return xout; } // ----------------------------------------------------------------------------- -template -void VariableChange::print(std::ostream & os) const { - Log::trace() << "VariableChange::print starting" << std::endl; +template +void VariableChange::print(std::ostream & os) const { + Log::trace() << "VariableChange::print starting" << std::endl; util::Timer timer(classname(), "print"); os << *chvar_; - Log::trace() << "VariableChange::print done" << std::endl; + if (varin_) os << std::endl << "Variable change from: " << *varin_; + if (varout_) os << std::endl << "Variable change to: " << *varout_; + if (varin_ || varout_) os << std::endl; + Log::trace() << "VariableChange::print done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/runs/ConvertState.h b/src/oops/runs/ConvertState.h index 0a180682b..6428c3fc4 100644 --- a/src/oops/runs/ConvertState.h +++ b/src/oops/runs/ConvertState.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2018 UCAR + * (C) Copyright 2018-2021 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -13,10 +13,10 @@ #include #include "eckit/config/LocalConfiguration.h" -#include "oops/base/VariableChangeBase.h" #include "oops/generic/instantiateVariableChangeFactory.h" #include "oops/interface/Geometry.h" #include "oops/interface/State.h" +#include "oops/interface/VariableChange.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" @@ -28,8 +28,7 @@ namespace oops { template class ConvertState : public Application { typedef Geometry Geometry_; typedef State State_; - typedef VariableChangeBase VariableChange_; - typedef VariableChangeFactory VariableChangeFactory_; + typedef VariableChange VariableChange_; public: // ------------------------------------------------------------------------------------------------- @@ -48,13 +47,13 @@ template class ConvertState : public Application { const Geometry_ resol2(outputResolConfig, this->getComm()); // Variable transform(s) - std::vector> chvars; + std::vector chvars; std::vector inverse; std::vector chvarconfs; fullConfig.get("variable changes", chvarconfs); for (size_t cv = 0; cv < chvarconfs.size(); ++cv) { - chvars.emplace_back(VariableChangeFactory_::create(chvarconfs[cv], resol2)); + chvars.emplace_back(resol2, chvarconfs[cv]); inverse.push_back(chvarconfs[cv].getBool("do inverse", false)); } @@ -79,13 +78,14 @@ template class ConvertState : public Application { // Variable transform(s) for (size_t cv = 0; cv < chvars.size(); ++cv) { if (!inverse[cv]) { - State_ xchvarout = chvars[cv]->changeVar(*xx); + State_ xchvarout = chvars[cv].changeVar(*xx); xx.reset(new State_(xchvarout)); } else { - State_ xchvarout = chvars[cv]->changeVarInverse(*xx); + State_ xchvarout = chvars[cv].changeVarInverse(*xx); xx.reset(new State_(xchvarout)); } - Log::test() << "State after " << *chvars[cv] << " transform: " << *xx << std::endl; + Log::test() << "Variable transform: " << chvars[cv] << std::endl; + Log::test() << "State after variable transform: " << *xx << std::endl; } // Write state diff --git a/src/test/interface/VariableChange.h b/src/test/interface/VariableChange.h index 7a127fbdf..8125b93fe 100644 --- a/src/test/interface/VariableChange.h +++ b/src/test/interface/VariableChange.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2018 UCAR + * (C) Copyright 2018-2021 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -20,10 +20,10 @@ #include "eckit/config/Configuration.h" #include "eckit/testing/Test.h" -#include "oops/base/VariableChangeBase.h" #include "oops/generic/instantiateVariableChangeFactory.h" #include "oops/interface/Geometry.h" #include "oops/interface/State.h" +#include "oops/interface/VariableChange.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" #include "oops/util/Expect.h" @@ -70,17 +70,15 @@ template class VariableChangeFixture : private boost::noncopyab template void testVariableChangeInverse() { typedef VariableChangeFixture Test_; - typedef oops::State State_; - typedef oops::VariableChangeBase VariableChange_; - typedef oops::VariableChangeFactory VariableChangeFactory_; + typedef oops::State State_; + typedef oops::VariableChange VariableChange_; // Loop over all variable changes for (std::size_t jj = 0; jj < Test_::confs().size(); ++jj) { // Construct variable change - std::unique_ptr \ - changevar(VariableChangeFactory_::create(Test_::confs()[jj], Test_::resol())); + VariableChange_ changevar(Test_::resol(), Test_::confs()[jj]); - oops::Log::test() << "Testing VariableChange: " << *changevar << std::endl; + oops::Log::test() << "Testing VariableChange: " << changevar << std::endl; // User specified tolerance for pass/fail const double tol = Test_::confs()[jj].getDouble("tolerance inverse"); @@ -99,13 +97,13 @@ template void testVariableChangeInverse() { if (inverseFirst) { oops::Variables varin(Test_::confs()[jj], "input variables"); State_ xin(Test_::resol(), varin, xx.validTime()); - changevar->changeVarInverse(xx, xin); - changevar->changeVar(xin, xx); + changevar.changeVarInverse(xx, xin); + changevar.changeVar(xin, xx); } else { oops::Variables varout(Test_::confs()[jj], "output variables"); State_ xout(Test_::resol(), varout, xx.validTime()); - changevar->changeVar(xx, xout); - changevar->changeVarInverse(xout, xx); + changevar.changeVar(xx, xout); + changevar.changeVarInverse(xout, xx); } // Compute norms of the result and reference From 7f0a73016f5a4b03670e7661b30c5b2ef6ea3f01 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Tue, 9 Feb 2021 11:01:26 -0700 Subject: [PATCH 055/142] improve ObsAuxControl test: add comparison with reference norm (#1058) * add comparison with norm to the ObsAuxControl test * use "is_close_relative" and "relative tolerance" in yaml --- l95/test/testinput/interfaces.yaml | 2 ++ qg/test/testinput/interfaces.yaml | 4 ++++ src/test/interface/ObsAuxControl.h | 9 +++++++++ 3 files changed, 15 insertions(+) diff --git a/l95/test/testinput/interfaces.yaml b/l95/test/testinput/interfaces.yaml index 9a8a93614..865b4f34a 100644 --- a/l95/test/testinput/interfaces.yaml +++ b/l95/test/testinput/interfaces.yaml @@ -109,6 +109,8 @@ observations: tolerance TL: '1.0e-14' obs bias: bias: 0.3 + norm: 0.3 + relative tolerance: 0.0 obs bias error: standard_deviation: 0.5 rms ref: 8.3207407741318846 diff --git a/qg/test/testinput/interfaces.yaml b/qg/test/testinput/interfaces.yaml index 5f6d2116a..eb6473a98 100644 --- a/qg/test/testinput/interfaces.yaml +++ b/qg/test/testinput/interfaces.yaml @@ -76,6 +76,8 @@ observations: tolerance TL: 1.0e-6 obs bias: stream: '-10.0' + norm: 10.0 + relative tolerance: 0.0 obs bias error: stream: '2.0e7' rms ref: 183502589.5028424 @@ -100,6 +102,8 @@ observations: tolerance TL: 1.0e-6 obs bias: uwind: '10.0' + norm: 10.0 + relative tolerance: 0.0 obs bias error: uwind: '15.0' rms ref: 39.644266100943696 diff --git a/src/test/interface/ObsAuxControl.h b/src/test/interface/ObsAuxControl.h index e7eff791f..3dc99d865 100644 --- a/src/test/interface/ObsAuxControl.h +++ b/src/test/interface/ObsAuxControl.h @@ -38,6 +38,15 @@ template void testConstructor() { std::unique_ptr bias(new ObsAux_(Test_::obspace()[jj], Test_::config(jj))); EXPECT(bias.get()); oops::Log::test() << "Testing ObsAuxControl: " << *bias << std::endl; + + // Not all configurations for interface tests specify "obs bias"; need to check + // whether "obs bias" section is available + if (Test_::config(jj).has("obs bias")) { + const double reference = Test_::config(jj).getDouble("obs bias.norm"); + const double tolerance = Test_::config(jj).getDouble("obs bias.relative tolerance"); + EXPECT(oops::is_close_relative(bias->norm(), reference, tolerance)); + } + bias.reset(); EXPECT(!bias.get()); } From 75efdaa73e51a2ea578cbe4e12fb24bfb77f7096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Tue, 9 Feb 2021 20:15:11 +0000 Subject: [PATCH 056/142] ObsOperator test: make it possible to verify an ObsOperator's constructor throws an exception. (#1067) --- src/test/interface/ObsOperator.h | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/test/interface/ObsOperator.h b/src/test/interface/ObsOperator.h index fe863d6ba..386e29fe0 100644 --- a/src/test/interface/ObsOperator.h +++ b/src/test/interface/ObsOperator.h @@ -26,11 +26,14 @@ #include "oops/interface/ObsOperator.h" #include "oops/interface/ObsVector.h" #include "oops/runs/Test.h" +#include "oops/util/Expect.h" #include "test/interface/ObsTestsFixture.h" #include "test/TestEnvironment.h" namespace test { +const char *expectConstructorToThrow = "expect constructor to throw exception with message"; + // ----------------------------------------------------------------------------- /// \brief tests constructor and print method template void testConstructor() { @@ -39,11 +42,19 @@ template void testConstructor() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { eckit::LocalConfiguration obsopconf(Test_::config(jj), "obs operator"); - std::unique_ptr hop(new ObsOperator_(Test_::obspace()[jj], obsopconf)); - EXPECT(hop.get()); - oops::Log::test() << "Testing ObsOperator: " << *hop << std::endl; - hop.reset(); - EXPECT(!hop.get()); + + if (!Test_::config(jj).has(expectConstructorToThrow)) { + std::unique_ptr hop(new ObsOperator_(Test_::obspace()[jj], obsopconf)); + EXPECT(hop.get()); + oops::Log::test() << "Testing ObsOperator: " << *hop << std::endl; + hop.reset(); + EXPECT(!hop.get()); + } else { + // The constructor is expected to throw an exception containing the specified string. + const std::string expectedMessage = Test_::config(jj).getString(expectConstructorToThrow); + EXPECT_THROWS_MSG(ObsOperator_(Test_::obspace()[jj], obsopconf), + expectedMessage.c_str()); + } } } @@ -59,6 +70,9 @@ template void testSimulateObs() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { const eckit::LocalConfiguration & conf = Test_::config(jj); + if (Test_::config(jj).has(expectConstructorToThrow)) + continue; + // initialize observation operator (set variables requested from the model, // variables simulated by the observation operator, other init) eckit::LocalConfiguration obsopconf(conf, "obs operator"); From 63c5307621e557ba21a85e87d34c27721d27ed34 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 10 Feb 2021 06:12:35 -0700 Subject: [PATCH 057/142] pass only obs bias configs to ObsAux classes (#1064) --- l95/src/lorenz95/ObsBias.cc | 11 ++--- l95/src/lorenz95/ObsBiasCorrection.cc | 5 +-- l95/src/lorenz95/ObsBiasCovariance.cc | 17 ++++---- l95/src/lorenz95/ObsBiasCovariance.h | 2 - qg/model/ObsBias.cc | 16 +++---- qg/model/ObsBiasCovariance.cc | 40 ++++++++---------- qg/model/ObsBiasCovariance.h | 2 - qg/model/ObsBiasIncrement.cc | 19 ++------- qg/model/ObsBiasIncrement.h | 2 - src/oops/base/ObsAuxControls.h | 3 +- src/oops/base/ObsAuxCovariances.h | 3 +- src/oops/base/ObsAuxIncrements.h | 17 +------- src/oops/interface/ObsAuxCovariance.h | 2 - src/oops/interface/ObsAuxIncrement.h | 11 ----- src/test/interface/LinearObsOperator.h | 28 ++++++++----- src/test/interface/ObsAuxControl.h | 6 ++- src/test/interface/ObsAuxCovariance.h | 3 +- src/test/interface/ObsAuxIncrement.h | 58 ++++++++++---------------- src/test/interface/ObsOperator.h | 3 +- 19 files changed, 92 insertions(+), 156 deletions(-) diff --git a/l95/src/lorenz95/ObsBias.cc b/l95/src/lorenz95/ObsBias.cc index 604016f4d..9189fdf1f 100644 --- a/l95/src/lorenz95/ObsBias.cc +++ b/l95/src/lorenz95/ObsBias.cc @@ -24,13 +24,10 @@ ObsBias::ObsBias(const ObsTableView &, const eckit::Configuration & conf) : bias_(0.0), active_(false), geovars_(), hdiags_() { oops::Log::trace() << "ObsBias::ObsBias conf is:" << conf << std::endl; - if (conf.has("obs bias")) { - const eckit::LocalConfiguration biasconf(conf, "obs bias"); - if (biasconf.has("bias")) { - bias_ = biasconf.getDouble("bias"); - active_ = true; - oops::Log::info() << "ObsBias::ObsBias created, bias = " << bias_ << std::endl; - } + if (conf.has("bias")) { + bias_ = conf.getDouble("bias"); + active_ = true; + oops::Log::info() << "ObsBias::ObsBias created, bias = " << bias_ << std::endl; } } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObsBiasCorrection.cc b/l95/src/lorenz95/ObsBiasCorrection.cc index a5a5836a5..cf76c56bb 100644 --- a/l95/src/lorenz95/ObsBiasCorrection.cc +++ b/l95/src/lorenz95/ObsBiasCorrection.cc @@ -24,10 +24,7 @@ namespace lorenz95 { ObsBiasCorrection::ObsBiasCorrection(const ObsTableView &, const eckit::Configuration & conf) : bias_(0.0), active_(false) { - if (conf.has("obs bias error")) { - const eckit::LocalConfiguration covconf(conf, "obs bias error"); - active_ = covconf.has("standard_deviation"); - } + active_ = conf.has("standard_deviation"); if (active_) {oops::Log::trace() << "ObsBiasCorrection::ObsBiasCorrection created." << std::endl;} } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObsBiasCovariance.cc b/l95/src/lorenz95/ObsBiasCovariance.cc index c38e05c5d..580855586 100644 --- a/l95/src/lorenz95/ObsBiasCovariance.cc +++ b/l95/src/lorenz95/ObsBiasCovariance.cc @@ -25,17 +25,14 @@ namespace lorenz95 { // ----------------------------------------------------------------------------- ObsBiasCovariance::ObsBiasCovariance(const ObsTableView &, const eckit::Configuration & conf) - : conf_(conf), variance_(0.0), active_(false) + : variance_(0.0), active_(false) { - if (conf_.has("obs bias error")) { - const eckit::LocalConfiguration covconf(conf_, "obs bias error"); - if (covconf.has("standard_deviation")) { - active_ = true; - const double zz = covconf.getDouble("standard_deviation"); - variance_ = zz * zz; - ASSERT(variance_ > 0.0); - oops::Log::info() << "ObsBiasCovariance variance = " << variance_ << std::endl; - } + if (conf.has("standard_deviation")) { + active_ = true; + const double zz = conf.getDouble("standard_deviation"); + variance_ = zz * zz; + ASSERT(variance_ > 0.0); + oops::Log::info() << "ObsBiasCovariance variance = " << variance_ << std::endl; } } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObsBiasCovariance.h b/l95/src/lorenz95/ObsBiasCovariance.h index d68e0a6ff..1b842c10c 100644 --- a/l95/src/lorenz95/ObsBiasCovariance.h +++ b/l95/src/lorenz95/ObsBiasCovariance.h @@ -46,12 +46,10 @@ class ObsBiasCovariance : public util::Printable, void inverseMultiply(const ObsBiasCorrection &, ObsBiasCorrection &) const; void randomize(ObsBiasCorrection &) const; - const eckit::Configuration & config() const {return conf_;} bool active() const {return active_;} private: void print(std::ostream &) const; - const eckit::LocalConfiguration conf_; double variance_; bool active_; }; diff --git a/qg/model/ObsBias.cc b/qg/model/ObsBias.cc index a8e9c69c6..6e8a29cba 100644 --- a/qg/model/ObsBias.cc +++ b/qg/model/ObsBias.cc @@ -26,17 +26,13 @@ namespace qg { ObsBias::ObsBias(const ObsSpaceQG &, const eckit::Configuration & conf) : bias_(ntypes, 0.0), active_(false), geovars_(), hdiags_() { oops::Log::info() << "ObsBias: conf = " << conf << std::endl; - eckit::LocalConfiguration biasconf; - if (conf.has("obs bias")) { - conf.get("obs bias", biasconf); - active_ = biasconf.has("stream") || biasconf.has("uwind") || - biasconf.has("vwind") || biasconf.has("wspeed"); - } + active_ = conf.has("stream") || conf.has("uwind") || + conf.has("vwind") || conf.has("wspeed"); if (active_) { - if (biasconf.has("stream")) bias_[0] = biasconf.getDouble("stream"); - if (biasconf.has("uwind")) bias_[1] = biasconf.getDouble("uwind"); - if (biasconf.has("vwind")) bias_[2] = biasconf.getDouble("vwind"); - if (biasconf.has("wspeed")) bias_[3] = biasconf.getDouble("wspeed"); + if (conf.has("stream")) bias_[0] = conf.getDouble("stream"); + if (conf.has("uwind")) bias_[1] = conf.getDouble("uwind"); + if (conf.has("vwind")) bias_[2] = conf.getDouble("vwind"); + if (conf.has("wspeed")) bias_[3] = conf.getDouble("wspeed"); std::string strn = ""; for (unsigned int jj = 0; jj < ObsBias::ntypes; ++jj) { if (jj > 0) strn += ", "; diff --git a/qg/model/ObsBiasCovariance.cc b/qg/model/ObsBiasCovariance.cc index 2ec592e08..c8d242bb4 100644 --- a/qg/model/ObsBiasCovariance.cc +++ b/qg/model/ObsBiasCovariance.cc @@ -27,31 +27,27 @@ namespace qg { // ----------------------------------------------------------------------------- ObsBiasCovariance::ObsBiasCovariance(const ObsSpaceQG &, const eckit::Configuration & conf) - : conf_(conf), variance_(ObsBias::ntypes, 0.0) + : variance_(ObsBias::ntypes, 0.0) { std::vector zz(4, 0.0); - if (conf.has("obs bias error")) { - const eckit::LocalConfiguration covconf(conf, "obs bias error"); - if (covconf.has("stream")) zz[0] = covconf.getDouble("stream"); - if (covconf.has("uwind")) zz[1] = covconf.getDouble("uwind"); - if (covconf.has("vwind")) zz[2] = covconf.getDouble("vwind"); - if (covconf.has("wspeed")) zz[3] = covconf.getDouble("wspeed"); - } - std::string strn = ""; - for (unsigned int jj = 0; jj < ObsBias::ntypes; ++jj) { - if (jj > 0) strn += ", "; - if (std::abs(zz[jj]) > 1.0e-8) { - variance_[jj] = zz[jj] * zz[jj]; - std::ostringstream strs; - strs << variance_[jj]; - strn += strs.str(); - } else { - variance_[jj] = 0.0; - strn += "0.0"; - } + if (conf.has("stream")) zz[0] = conf.getDouble("stream"); + if (conf.has("uwind")) zz[1] = conf.getDouble("uwind"); + if (conf.has("vwind")) zz[2] = conf.getDouble("vwind"); + if (conf.has("wspeed")) zz[3] = conf.getDouble("wspeed"); + std::string strn = ""; + for (unsigned int jj = 0; jj < ObsBias::ntypes; ++jj) { + if (jj > 0) strn += ", "; + if (std::abs(zz[jj]) > 1.0e-8) { + variance_[jj] = zz[jj] * zz[jj]; + std::ostringstream strs; + strs << variance_[jj]; + strn += strs.str(); + } else { + variance_[jj] = 0.0; + strn += "0.0"; } - oops::Log::info() << "ObsBiasCovariance created, variances = " << strn << std::endl; -// } + } + oops::Log::info() << "ObsBiasCovariance created, variances = " << strn << std::endl; } // ----------------------------------------------------------------------------- void ObsBiasCovariance::multiply(const ObsBiasIncrement & dxin, diff --git a/qg/model/ObsBiasCovariance.h b/qg/model/ObsBiasCovariance.h index 78c9a7e4e..b5597d43b 100644 --- a/qg/model/ObsBiasCovariance.h +++ b/qg/model/ObsBiasCovariance.h @@ -44,12 +44,10 @@ class ObsBiasCovariance : public util::Printable, void inverseMultiply(const ObsBiasIncrement &, ObsBiasIncrement &) const; void randomize(ObsBiasIncrement &) const; - const eckit::Configuration & config() const {return conf_;} bool active(const unsigned int ii) const {return variance_[ii] > 0.0;} private: void print(std::ostream &) const; - const eckit::LocalConfiguration conf_; std::vector variance_; }; diff --git a/qg/model/ObsBiasIncrement.cc b/qg/model/ObsBiasIncrement.cc index a5d8ca223..b355665ee 100644 --- a/qg/model/ObsBiasIncrement.cc +++ b/qg/model/ObsBiasIncrement.cc @@ -26,13 +26,10 @@ namespace qg { ObsBiasIncrement::ObsBiasIncrement(const ObsSpaceQG &, const eckit::Configuration & conf) : bias_(ObsBias::ntypes, 0.0), active_(ObsBias::ntypes, false) { - if (conf.has("obs bias error")) { - const eckit::LocalConfiguration covconf(conf, "obs bias error"); - active_[0] = covconf.has("stream"); - active_[1] = covconf.has("uwind"); - active_[2] = covconf.has("vwind"); - active_[3] = covconf.has("wspeed"); - } + active_[0] = conf.has("stream"); + active_[1] = conf.has("uwind"); + active_[2] = conf.has("vwind"); + active_[3] = conf.has("wspeed"); bool on = false; std::string strn = ""; for (unsigned int jj = 0; jj < ObsBias::ntypes; ++jj) { @@ -57,14 +54,6 @@ ObsBiasIncrement::ObsBiasIncrement(const ObsBiasIncrement & other, this->makePassive(); } // ----------------------------------------------------------------------------- -ObsBiasIncrement::ObsBiasIncrement(const ObsBiasIncrement & other, - const eckit::Configuration &) - : bias_(ObsBias::ntypes, 0.0), active_(other.active_) -{ - for (unsigned int jj = 0; jj < ObsBias::ntypes; ++jj) bias_[jj] = other.bias_[jj]; - this->makePassive(); -} -// ----------------------------------------------------------------------------- void ObsBiasIncrement::makePassive() { for (unsigned int jj = 0; jj < ObsBias::ntypes; ++jj) { if (!active_[jj]) bias_[jj] = 0.0; diff --git a/qg/model/ObsBiasIncrement.h b/qg/model/ObsBiasIncrement.h index db8aef893..5f888a8f5 100644 --- a/qg/model/ObsBiasIncrement.h +++ b/qg/model/ObsBiasIncrement.h @@ -34,8 +34,6 @@ class ObsBiasIncrement : public util::Printable, ObsBiasIncrement(); ObsBiasIncrement(const ObsSpaceQG &, const eckit::Configuration &); ObsBiasIncrement(const ObsBiasIncrement &, const bool copy = true); - ObsBiasIncrement(const ObsBiasIncrement &, const eckit::Configuration &); - ~ObsBiasIncrement() {} /// Linear algebra operators void diff(const ObsBias &, const ObsBias &); diff --git a/src/oops/base/ObsAuxControls.h b/src/oops/base/ObsAuxControls.h index 400563bb2..17d6a6bf2 100644 --- a/src/oops/base/ObsAuxControls.h +++ b/src/oops/base/ObsAuxControls.h @@ -59,8 +59,9 @@ ObsAuxControls::ObsAuxControls(const ObsSpaces_ & odb, const eckit::Configu { std::vector obsconf = conf.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsconf.size(); ++jobs) { + eckit::LocalConfiguration obsauxconf = obsconf[jobs].getSubConfiguration("obs bias"); auxs_.push_back( - std::unique_ptr(new ObsAuxControl_(odb[jobs], obsconf[jobs]))); + std::unique_ptr(new ObsAuxControl_(odb[jobs], obsauxconf))); } } diff --git a/src/oops/base/ObsAuxCovariances.h b/src/oops/base/ObsAuxCovariances.h index 80171adc4..ccaa7e29a 100644 --- a/src/oops/base/ObsAuxCovariances.h +++ b/src/oops/base/ObsAuxCovariances.h @@ -67,8 +67,9 @@ ObsAuxCovariances::ObsAuxCovariances(const ObsSpaces_ & odb, Log::trace() << "ObsAuxCovariances::ObsAuxCovariances starting" << std::endl; std::vector obsconf = conf.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsconf.size(); ++jobs) { + eckit::LocalConfiguration obsauxconf = obsconf[jobs].getSubConfiguration("obs bias error"); cov_.push_back( - std::unique_ptr(new ObsAuxCovariance_(odb[jobs], obsconf[jobs]))); + std::unique_ptr(new ObsAuxCovariance_(odb[jobs], obsauxconf))); } Log::trace() << "ObsAuxCovariances::ObsAuxCovariances done" << std::endl; } diff --git a/src/oops/base/ObsAuxIncrements.h b/src/oops/base/ObsAuxIncrements.h index 3d3a7006d..a828620a3 100644 --- a/src/oops/base/ObsAuxIncrements.h +++ b/src/oops/base/ObsAuxIncrements.h @@ -37,7 +37,6 @@ class ObsAuxIncrements : public util::Printable, /// Constructor, destructor ObsAuxIncrements(const ObsSpaces_ &, const eckit::Configuration &); ObsAuxIncrements(const ObsAuxIncrements &, const bool copy = true); - ObsAuxIncrements(const ObsAuxIncrements &, const eckit::Configuration &); ~ObsAuxIncrements(); /// Access @@ -91,8 +90,9 @@ ObsAuxIncrements::ObsAuxIncrements(const ObsSpaces_ & odb, const eckit::Con { std::vector obsconf = conf.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsconf.size(); ++jobs) { + eckit::LocalConfiguration obsauxconf = obsconf[jobs].getSubConfiguration("obs bias error"); auxs_.push_back( - std::unique_ptr(new ObsAuxIncrement_(odb[jobs], obsconf[jobs]))); + std::unique_ptr(new ObsAuxIncrement_(odb[jobs], obsauxconf))); } } // ----------------------------------------------------------------------------- @@ -109,19 +109,6 @@ ObsAuxIncrements::ObsAuxIncrements(const ObsAuxIncrements & other, const bo } // ----------------------------------------------------------------------------- template -ObsAuxIncrements::ObsAuxIncrements(const ObsAuxIncrements & other, - const eckit::Configuration & conf) : auxs_(other.size()) -{ - Log::trace() << "ObsAuxIncrements::ObsAuxIncrements interpolated starting" << std::endl; - std::vector obsconf; - ASSERT(size() == other.size()); - for (std::size_t jobs = 0; jobs < other.size(); ++jobs) { - auxs_[jobs].reset(new ObsAuxIncrement_(other[jobs])); - } - Log::trace() << "ObsAuxIncrements::ObsAuxIncrements interpolated done" << std::endl; -} -// ----------------------------------------------------------------------------- -template ObsAuxIncrements::~ObsAuxIncrements() { Log::trace() << "ObsAuxIncrements::~ObsAuxIncrements starting" << std::endl; for (std::size_t jobs = 0; jobs < auxs_.size(); ++jobs) auxs_[jobs].reset(); diff --git a/src/oops/interface/ObsAuxCovariance.h b/src/oops/interface/ObsAuxCovariance.h index 49ca56d16..ed3dfd1ff 100644 --- a/src/oops/interface/ObsAuxCovariance.h +++ b/src/oops/interface/ObsAuxCovariance.h @@ -50,8 +50,6 @@ class ObsAuxCovariance : public util::Printable, void inverseMultiply(const ObsAuxIncrement_ &, ObsAuxIncrement_ &) const; void randomize(ObsAuxIncrement_ &) const; - const eckit::Configuration & config() const {return cov_->config();} - private: void print(std::ostream &) const; std::unique_ptr cov_; diff --git a/src/oops/interface/ObsAuxIncrement.h b/src/oops/interface/ObsAuxIncrement.h index ac14ec155..dce7aa53a 100644 --- a/src/oops/interface/ObsAuxIncrement.h +++ b/src/oops/interface/ObsAuxIncrement.h @@ -45,7 +45,6 @@ class ObsAuxIncrement : public util::Printable, /// Constructor, destructor ObsAuxIncrement(const ObsSpace &, const eckit::Configuration &); ObsAuxIncrement(const ObsAuxIncrement &, const bool copy = true); - ObsAuxIncrement(const ObsAuxIncrement &, const eckit::Configuration &); ~ObsAuxIncrement(); /// Interfacing @@ -111,16 +110,6 @@ ObsAuxIncrement::ObsAuxIncrement(const ObsAuxIncrement & other, } // ----------------------------------------------------------------------------- template -ObsAuxIncrement::ObsAuxIncrement(const ObsAuxIncrement & other, - const eckit::Configuration & conf) : aux_() -{ - Log::trace() << "ObsAuxIncrement::ObsAuxIncrement interpolated starting" << std::endl; - util::Timer timer(classname(), "ObsAuxIncrement"); - aux_.reset(new ObsAuxIncrement_(*other.aux_, conf)); - Log::trace() << "ObsAuxIncrement::ObsAuxIncrement interpolated done" << std::endl; -} -// ----------------------------------------------------------------------------- -template ObsAuxIncrement::~ObsAuxIncrement() { Log::trace() << "ObsAuxIncrement::~ObsAuxIncrement starting" << std::endl; util::Timer timer(classname(), "~ObsAuxIncrement"); diff --git a/src/test/interface/LinearObsOperator.h b/src/test/interface/LinearObsOperator.h index b5a5903bf..296425453 100644 --- a/src/test/interface/LinearObsOperator.h +++ b/src/test/interface/LinearObsOperator.h @@ -83,8 +83,10 @@ template void testLinearity() { LinearObsOperator_ hoptl(Test_::obspace()[jj], linobsopconf); // initialize obs bias - const ObsAuxCtrl_ ybias(Test_::obspace()[jj], conf); - ObsAuxIncr_ ybinc(Test_::obspace()[jj], conf); + eckit::LocalConfiguration biasconf = conf.getSubConfiguration("obs bias"); + const ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasconf); + eckit::LocalConfiguration biascovconf = conf.getSubConfiguration("obs bias error"); + ObsAuxIncr_ ybinc(Test_::obspace()[jj], biascovconf); // read geovals from the file const eckit::LocalConfiguration gconf(conf, "geovals"); @@ -93,7 +95,7 @@ template void testLinearity() { const GeoVaLs_ gval(gconf, Test_::obspace()[jj], hopvars); // initialize Obs. Bias Covariance - const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], conf); + const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biascovconf); // set trajectory for TL/AD to be the geovals from the file hoptl.setTrajectory(gval, ybias); @@ -158,12 +160,14 @@ template void testAdjoint() { const double tol = conf.getDouble("linear obs operator test.tolerance AD"); // initialize bias correction - const ObsAuxCtrl_ ybias(Test_::obspace()[jj], conf); - ObsAuxIncr_ ybinc1(Test_::obspace()[jj], conf); // TL - ObsAuxIncr_ ybinc2(Test_::obspace()[jj], conf); // AD + eckit::LocalConfiguration biasconf = conf.getSubConfiguration("obs bias"); + eckit::LocalConfiguration biascovconf = conf.getSubConfiguration("obs bias error"); + const ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasconf); + ObsAuxIncr_ ybinc1(Test_::obspace()[jj], biascovconf); // TL + ObsAuxIncr_ ybinc2(Test_::obspace()[jj], biascovconf); // AD // initialize Obs. Bias Covariance - const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], conf); + const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biascovconf); // read geovals from the file eckit::LocalConfiguration gconf(conf, "geovals"); @@ -239,11 +243,13 @@ template void testTangentLinear() { const int iter = conf.getInt("linear obs operator test.iterations TL", 1); // initialize obs bias from file - const ObsAuxCtrl_ ybias0(Test_::obspace()[jj], conf); - ObsAuxCtrl_ ybias(Test_::obspace()[jj], conf); + eckit::LocalConfiguration biasconf = conf.getSubConfiguration("obs bias"); + eckit::LocalConfiguration biascovconf = conf.getSubConfiguration("obs bias error"); + const ObsAuxCtrl_ ybias0(Test_::obspace()[jj], biasconf); + ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasconf); // initialize Obs. Bias Covariance - const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], conf); + const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biascovconf); // read geovals from the file const eckit::LocalConfiguration gconf(conf, "geovals"); @@ -271,7 +277,7 @@ template void testTangentLinear() { // randomize dx and ybinc GeoVaLs_ dx(gconf, Test_::obspace()[jj], hoptl.requiredVars()); dx.random(); - ObsAuxIncr_ ybinc(Test_::obspace()[jj], conf); + ObsAuxIncr_ ybinc(Test_::obspace()[jj], biascovconf); Bobsbias.randomize(ybinc); // scale dx by x0 diff --git a/src/test/interface/ObsAuxControl.h b/src/test/interface/ObsAuxControl.h index 3dc99d865..1fb9dc150 100644 --- a/src/test/interface/ObsAuxControl.h +++ b/src/test/interface/ObsAuxControl.h @@ -35,7 +35,8 @@ template void testConstructor() { typedef oops::ObsAuxControl ObsAux_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - std::unique_ptr bias(new ObsAux_(Test_::obspace()[jj], Test_::config(jj))); + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); + std::unique_ptr bias(new ObsAux_(Test_::obspace()[jj], biasconf)); EXPECT(bias.get()); oops::Log::test() << "Testing ObsAuxControl: " << *bias << std::endl; @@ -59,7 +60,8 @@ template void testCopyConstructor() { typedef oops::ObsAuxControl ObsAux_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - std::unique_ptr bias(new ObsAux_(Test_::obspace()[jj], Test_::config(jj))); + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); + std::unique_ptr bias(new ObsAux_(Test_::obspace()[jj], biasconf)); std::unique_ptr other(new ObsAux_(*bias)); EXPECT(other.get()); diff --git a/src/test/interface/ObsAuxCovariance.h b/src/test/interface/ObsAuxCovariance.h index 9edd0b5ee..2d990a6c2 100644 --- a/src/test/interface/ObsAuxCovariance.h +++ b/src/test/interface/ObsAuxCovariance.h @@ -34,7 +34,8 @@ template void testConstructor() { typedef oops::ObsAuxCovariance Covariance_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - std::unique_ptr cov(new Covariance_(Test_::obspace()[jj], Test_::config(jj))); + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + std::unique_ptr cov(new Covariance_(Test_::obspace()[jj], biasconf)); EXPECT(cov.get()); oops::Log::test() << "Testing ObsAuxCovariance: " << *cov << std::endl; cov.reset(); diff --git a/src/test/interface/ObsAuxIncrement.h b/src/test/interface/ObsAuxIncrement.h index 38396159f..1b8ac8e2a 100644 --- a/src/test/interface/ObsAuxIncrement.h +++ b/src/test/interface/ObsAuxIncrement.h @@ -51,7 +51,9 @@ template class ObsAuxIncrementFixture : private boost::noncopyabl ObsAuxIncrementFixture() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - std::shared_ptr tmp(new Covariance_(Test_::obspace()[jj], Test_::config(jj))); + eckit::LocalConfiguration biascovconf = + Test_::config(jj).getSubConfiguration("obs bias error"); + std::shared_ptr tmp(new Covariance_(Test_::obspace()[jj], biascovconf)); covar_.push_back(tmp); } } @@ -68,7 +70,8 @@ template void testObsAuxIncrementConstructor() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - AuxIncr_ dx(Test_::obspace()[jj], Test_::config(jj)); + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + AuxIncr_ dx(Test_::obspace()[jj], biasconf); oops::Log::test() << "Printing zero ObsAuxIncrement: " << dx << std::endl; EXPECT(dx.norm() == 0.0); } @@ -81,8 +84,9 @@ template void testObsAuxIncrementCopyConstructor() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - if (Test_::config(jj).has("obs bias")) { - AuxIncr_ dx1(Test_::obspace()[jj], Test_::config(jj)); + if (Test_::config(jj).has("obs bias error")) { + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + AuxIncr_ dx1(Test_::obspace()[jj], biasconf); ObsAuxIncrementFixture::covariance(jj).randomize(dx1); oops::Log::test() << "Printing random ObsAuxIncrement: " << dx1 << std::endl; AuxIncr_ dx2(dx1); @@ -98,39 +102,17 @@ template void testObsAuxIncrementCopyConstructor() { // ----------------------------------------------------------------------------- -template void testObsAuxIncrementChangeRes() { - typedef ObsTestsFixture Test_; - typedef ObsAuxIncrementFixture AuxTest_; - typedef oops::ObsAuxIncrement AuxIncr_; - - for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - if (Test_::config(jj).has("obs bias")) { - AuxIncr_ dx1(Test_::obspace()[jj], Test_::config(jj)); - AuxTest_::covariance(jj).randomize(dx1); - - AuxIncr_ dx2(dx1, Test_::config(jj)); - EXPECT(dx2.norm() > 0.0); - EXPECT(dx2.norm() == dx1.norm()); - -// Check that the copy is equal to the original - dx2 -= dx1; - EXPECT(dx2.norm() == 0.0); - } - } -} - -// ----------------------------------------------------------------------------- - template void testObsAuxIncrementTriangle() { typedef ObsTestsFixture Test_; typedef ObsAuxIncrementFixture AuxTest_; typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - if (Test_::config(jj).has("obs bias")) { - AuxIncr_ dx1(Test_::obspace()[jj], Test_::config(jj)); + if (Test_::config(jj).has("obs bias error")) { + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + AuxIncr_ dx1(Test_::obspace()[jj], biasconf); AuxTest_::covariance(jj).randomize(dx1); - AuxIncr_ dx2(Test_::obspace()[jj], Test_::config(jj)); + AuxIncr_ dx2(Test_::obspace()[jj], biasconf); ObsAuxIncrementFixture::covariance(jj).randomize(dx2); // test triangle inequality @@ -157,7 +139,8 @@ template void testObsAuxIncrementOpPlusEq() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - AuxIncr_ dx1(Test_::obspace()[jj], Test_::config(jj)); + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + AuxIncr_ dx1(Test_::obspace()[jj], biasconf); AuxTest_::covariance(jj).randomize(dx1); AuxIncr_ dx2(dx1); @@ -178,9 +161,10 @@ template void testObsAuxIncrementDotProduct() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - AuxIncr_ dx1(Test_::obspace()[jj], Test_::config(jj)); + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + AuxIncr_ dx1(Test_::obspace()[jj], biasconf); AuxTest_::covariance(jj).randomize(dx1); - AuxIncr_ dx2(Test_::obspace()[jj], Test_::config(jj)); + AuxIncr_ dx2(Test_::obspace()[jj], biasconf); AuxTest_::covariance(jj).randomize(dx2); // test symmetry of dot product @@ -199,7 +183,8 @@ template void testObsAuxIncrementZero() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - AuxIncr_ dx(Test_::obspace()[jj], Test_::config(jj)); + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + AuxIncr_ dx(Test_::obspace()[jj], biasconf); AuxTest_::covariance(jj).randomize(dx); EXPECT(dx.norm() > 0.0); @@ -217,7 +202,8 @@ template void testObsAuxIncrementAxpy() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - AuxIncr_ dx1(Test_::obspace()[jj], Test_::config(jj)); + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + AuxIncr_ dx1(Test_::obspace()[jj], biasconf); AuxTest_::covariance(jj).randomize(dx1); // test axpy @@ -252,8 +238,6 @@ class ObsAuxIncrement : public oops::Test { { testObsAuxIncrementConstructor(); }); ts.emplace_back(CASE("interface/ObsAuxIncrement/testObsAuxIncrementCopyConstructor") { testObsAuxIncrementCopyConstructor(); }); - ts.emplace_back(CASE("interface/ObsAuxIncrement/testObsAuxIncrementChangeRes") - { testObsAuxIncrementChangeRes(); }); ts.emplace_back(CASE("interface/ObsAuxIncrement/testObsAuxIncrementTriangle") { testObsAuxIncrementTriangle(); }); ts.emplace_back(CASE("interface/ObsAuxIncrement/testObsAuxIncrementOpPlusEq") diff --git a/src/test/interface/ObsOperator.h b/src/test/interface/ObsOperator.h index 386e29fe0..5c8c5e8ba 100644 --- a/src/test/interface/ObsOperator.h +++ b/src/test/interface/ObsOperator.h @@ -79,7 +79,8 @@ template void testSimulateObs() { ObsOperator_ hop(Test_::obspace()[jj], obsopconf); // initialize bias correction - const ObsAuxCtrl_ ybias(Test_::obspace()[jj], conf); + eckit::LocalConfiguration biasconf = conf.getSubConfiguration("obs bias"); + const ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasconf); // read geovals from the file eckit::LocalConfiguration gconf(conf, "geovals"); From ecfbbb501cf21313dbfade8a8b7ad95a6b914537 Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Thu, 11 Feb 2021 13:26:07 +0000 Subject: [PATCH 058/142] change virtual to override (#1069) --- src/oops/util/parameters/Parameters.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/oops/util/parameters/Parameters.h b/src/oops/util/parameters/Parameters.h index fde3b2c12..03fa4d5a9 100644 --- a/src/oops/util/parameters/Parameters.h +++ b/src/oops/util/parameters/Parameters.h @@ -73,7 +73,7 @@ namespace oops { className &operator=(const className &) = default; \ className &operator=(className &&) = default; \ private: \ - virtual className* cloneImpl() const { \ + className* cloneImpl() const override { \ return new className(*this); \ } \ public: \ From 0152c766ec5c9f4e0220f8dcaef8f2d7453d8c1d Mon Sep 17 00:00:00 2001 From: "Byoung-Joo (BJ) Jung" <33296274+byoung-joo@users.noreply.github.com> Date: Thu, 11 Feb 2021 13:47:10 -0700 Subject: [PATCH 059/142] make the nonlinear Jb evaluation optional (#1065) * naive implementation On branch feature/optional_nonlinear_jb Changes to be committed: modified: qg/test/testinput/3dvar.yaml modified: src/oops/assimilation/CostFunction.h * clceanup * add a test for demonstration Changes to be committed: modified: qg/test/CMakeLists.txt modified: qg/test/testinput/3dvar.yaml new file: qg/test/testinput/3dvar_hybrid_wo_jb_evaluation.yaml new file: qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test * Move the logic to CostJbTotal modified: qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test modified: src/oops/assimilation/CostFunction.h modified: src/oops/assimilation/CostJbTotal.h * make the logic in a single code line modified: qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test modified: src/oops/assimilation/CostJbTotal.h --- qg/test/CMakeLists.txt | 9 ++ .../3dvar_hybrid_wo_jb_evaluation.yaml | 107 ++++++++++++++++++ .../3dvar_hybrid_wo_jb_evaluation.test | 30 +++++ src/oops/assimilation/CostJbTotal.h | 6 +- 4 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 qg/test/testinput/3dvar_hybrid_wo_jb_evaluation.yaml create mode 100644 qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index e7e0722ae..73224d694 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -3,6 +3,7 @@ list( APPEND qg_testinput testinput/3dvar.yaml testinput/3dvar_change_var.yaml testinput/3dvar_hybrid.yaml + testinput/3dvar_hybrid_wo_jb_evaluation.yaml testinput/3dfgat.yaml testinput/4densvar.yaml testinput/4densvar_hybrid.yaml @@ -84,6 +85,7 @@ list( APPEND qg_testoutput testoutput/3dvar.test testoutput/3dvar_change_var.test testoutput/3dvar_hybrid.test + testoutput/3dvar_hybrid_wo_jb_evaluation.test testoutput/3dfgat.test testoutput/4densvar.test testoutput/4densvar_hybrid.test @@ -595,6 +597,13 @@ oops_add_test( TESTNAME 3dvar_hybrid EXENAME qg_4dvar.x TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_3d ) +oops_add_test( TESTNAME 3dvar_hybrid_wo_jb_evaluation + MODELNAME qg + OMP 2 + YAMLNAME testinput/3dvar_hybrid_wo_jb_evaluation.yaml + EXENAME qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_3d ) + oops_add_test( TESTNAME 3dfgat MODELNAME qg OMP 2 diff --git a/qg/test/testinput/3dvar_hybrid_wo_jb_evaluation.yaml b/qg/test/testinput/3dvar_hybrid_wo_jb_evaluation.yaml new file mode 100644 index 000000000..4dc7d168d --- /dev/null +++ b/qg/test/testinput/3dvar_hybrid_wo_jb_evaluation.yaml @@ -0,0 +1,107 @@ +cost function: + cost type: 3D-Var + window begin: 2010-01-01T09:00:00Z + window length: PT6H + jb evaluation: false + analysis variables: [x] + background: + date: 2010-01-01T12:00:00Z + filename: Data/forecast.fc.2009-12-31T00:00:00Z.P1DT12H.nc + background error: + covariance model: hybrid + components: + - covariance: + covariance model: QgError + horizontal_length_scale: 2.2e6 + maximum_condition_number: 1.0e6 + standard_deviation: 1.8e7 + vertical_length_scale: 15000.0 + weight: + value: 0.5 + - covariance: + covariance model: ensemble + date: 2010-01-01T12:00:00Z + localization: + horizontal_length_scale: 4.0e6 + localization method: QG + maximum_condition_number: 1.0e6 + standard_deviation: 1.0 + vertical_length_scale: 30000.0 + members: + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1DT12H.nc + - date: 2010-01-01T12:00:00Z + filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1DT12H.nc + weight: + value: 0.5 + observations: + - obs operator: + obs type: Stream + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/3dvar_hybrid.obs3d.nc + obs type: Stream + obs error: + covariance model: diagonal + - obs operator: + obs type: Wind + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/3dvar_hybrid.obs3d.nc + obs type: Wind + obs error: + covariance model: diagonal + - obs operator: + obs type: WSpeed + obs space: + obsdatain: + obsfile: Data/truth.obs3d.nc + obsdataout: + obsfile: Data/3dvar_hybrid.obs3d.nc + obs type: WSpeed + obs error: + covariance model: diagonal + geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] +variational: + minimizer: + algorithm: DRIPCG + iterations: + - ninner: 10 + gradient norm reduction: 1.0e-10 + geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + diagnostics: + departures: ombg + test: on + - ninner: 10 + gradient norm reduction: 1.0e-10 + geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + diagnostics: + departures: ombg + test: on +final: + diagnostics: + departures: oman +output: + datadir: Data + exp: 3dvar + frequency: PT6H + type: an diff --git a/qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test b/qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test new file mode 100644 index 000000000..eaec1f279 --- /dev/null +++ b/qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test @@ -0,0 +1,30 @@ +Test : CostJo : Nonlinear Jo(Stream) = 7820.89, nobs = 600, Jo/n = 13.0348, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 1065.61, nobs = 600, Jo/n = 1.77601, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 144.328, nobs = 300, Jo/n = 0.481094, err = 12 +Test : CostFunction: Nonlinear J = 9030.82 +Test : DRIPCGMinimizer: reduction in residual norm = 0.121211 +Test : CostFunction::addIncrement: Analysis: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are activated +Test : Min= -4.4444e+08, Max= 1.0456e+08, RMS= 1.7727e+08 +Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : CostJo : Nonlinear Jo(Stream) = 480.1, nobs = 600, Jo/n = 0.8002, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 202.7, nobs = 600, Jo/n = 0.3379, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 30.36, nobs = 300, Jo/n = 0.1012, err = 12 +Test : CostFunction: Nonlinear J = 713.2 +Test : DRIPCGMinimizer: reduction in residual norm = 0.5399 +Test : CostFunction::addIncrement: Analysis: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are activated +Test : Min= -4.4287e+08, Max= 1.0243e+08, RMS= 1.7719e+08 +Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : CostJo : Nonlinear Jo(Stream) = 425.7, nobs = 600, Jo/n = 0.7095, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 141.4, nobs = 600, Jo/n = 0.2356, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 24.37, nobs = 300, Jo/n = 0.08125, err = 12 +Test : CostFunction: Nonlinear J = 591.4 diff --git a/src/oops/assimilation/CostJbTotal.h b/src/oops/assimilation/CostJbTotal.h index f26b2f347..939ffc07a 100644 --- a/src/oops/assimilation/CostJbTotal.h +++ b/src/oops/assimilation/CostJbTotal.h @@ -110,6 +110,8 @@ template class CostJbTotal { std::unique_ptr resol_; const util::DateTime windowBegin_; const util::DateTime windowEnd_; + + bool jbEvaluation_; }; // ============================================================================= @@ -124,6 +126,7 @@ CostJbTotal::CostJbTotal(const CtrlVar_ & xb, JbState_ * jb, windowBegin_(conf.getString("window begin")), windowEnd_(windowBegin_ + util::Duration(conf.getString("window length"))) { + jbEvaluation_ = conf.getBool("jb evaluation", true); Log::trace() << "CostJbTotal contructed." << std::endl; } @@ -155,7 +158,8 @@ double CostJbTotal::finalize(const CtrlVar_ & mx) const { Log::info() << "CostJb: FG-BG" << dx << std::endl; // Compute Jb value - double zjb = this->evaluate(dx); + double zjb = 0.0; + if (jbEvaluation_) zjb = this->evaluate(dx); Log::trace() << "CostJbTotal::finalize done" << std::endl; return zjb; } From f92213ea256fba35c3174b8312596e71e8359bc9 Mon Sep 17 00:00:00 2001 From: Ryan Honeyager Date: Tue, 16 Feb 2021 17:52:18 -0500 Subject: [PATCH 060/142] Update CMakeLists.txt (#1070) --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 90f420a31..27843723d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,7 +46,7 @@ find_package( Eigen3 REQUIRED NO_MODULE HINTS $ENV{Eigen3_PATH} $ENV{EIGEN3_PATH} $ENV{Eigen_PATH} $ENV{EIGEN_PATH}) find_package( OpenMP COMPONENTS CXX Fortran ) -find_package( MPI REQUIRED COMPONENTS CXX Fortran ) +find_package( MPI REQUIRED COMPONENTS C CXX Fortran ) find_package( NetCDF REQUIRED COMPONENTS Fortran ) find_package( Boost 1.64.0 REQUIRED) find_package( eckit 1.11.6 REQUIRED COMPONENTS MPI ) From af4c85e891f6ae37509144dbd304feba2b17f6cc Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Wed, 17 Feb 2021 12:52:33 +0000 Subject: [PATCH 061/142] virtual -> override (#1073) --- src/oops/util/parameters/Parameters.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/oops/util/parameters/Parameters.h b/src/oops/util/parameters/Parameters.h index 03fa4d5a9..5d1830494 100644 --- a/src/oops/util/parameters/Parameters.h +++ b/src/oops/util/parameters/Parameters.h @@ -35,7 +35,7 @@ namespace oops { className &operator=(const className &) = default; \ className &operator=(className &&) = default; \ private: \ - virtual className* cloneImpl() const = 0; \ + className* cloneImpl() const override = 0; \ public: \ std::unique_ptr clone() const { \ return std::unique_ptr(cloneImpl()); \ From f9229da88d2417aab5f75dc1ba26d53a4633151f Mon Sep 17 00:00:00 2001 From: Marek Wlasak Date: Wed, 17 Feb 2021 19:12:48 +0000 Subject: [PATCH 062/142] Feature/lineargetvalues config (#1044) * added T dx = TUTdx test; * allow all the tests. * making a separate sub test for tutdx=tdx in test inverse. * added log out in test * 100 character long format issue * end of day commit * disabling qg for now and going back on getvalues interface change * failing code with error message that does not make sense! * configuration now compiles - defaults to not including "getvalues" in yaml * removing additional code used fro umjedi test * simple removal of commented code. * change code to make configuration optional. There needs to be some more tidying up. * tidy up * multiple observations types now work * minor refactor * add whitespace * filtering at top level - removal of extra constructor * removal of l95 tests * remove redundant file * responding to Anna's comments * responding to Wojciech's comments * added error trap * using local config in GetValues test * creating GetValuesBase.h, more to follow * comment` * code compiles so far * first implementation of linear get values config * some tidying * code tidy * reverse mistake in git merge * end of day commit * removing factory * update for a single config * cody tidy * responding to some of Anna's comments - not yet finished * reverse change * removing testing. * reverting to a single config argument * code tidy * revert change to vectorizeAndFilter * passing config by reference * tidy config function * remove unnecessary header info. * responding to Anna's comments Co-authored-by: Anna Shlyaeva --- l95/src/lorenz95/GetValuesTLAD.cc | 3 ++- l95/src/lorenz95/GetValuesTLAD.h | 6 ++++- qg/model/GetValuesTLAD.cc | 8 ++++-- qg/model/GetValuesTLAD.h | 5 +++- qg/test/testinput/3dvar.yaml | 6 +++++ qg/test/testinput/eda_3dfgat_1.yaml | 2 ++ qg/test/testinput/hofx.yaml | 6 +++++ src/oops/assimilation/CostJo.h | 2 +- src/oops/assimilation/GETKFSolver.h | 2 +- src/oops/assimilation/LocalEnsembleSolver.h | 2 +- src/oops/base/ObserverTLAD.h | 23 +++++++++++----- src/oops/base/ObserversTLAD.h | 11 +++----- src/oops/interface/LinearGetValues.h | 12 ++++++--- src/oops/runs/HofX.h | 2 +- src/oops/runs/HofXNoModel.h | 2 +- src/oops/util/ConfigFunctions.cc | 29 ++++++++------------- src/oops/util/ConfigFunctions.h | 28 +++++++++----------- src/test/interface/LinearGetValues.h | 20 +++++++++++--- 18 files changed, 105 insertions(+), 64 deletions(-) diff --git a/l95/src/lorenz95/GetValuesTLAD.cc b/l95/src/lorenz95/GetValuesTLAD.cc index df866d373..833761fae 100644 --- a/l95/src/lorenz95/GetValuesTLAD.cc +++ b/l95/src/lorenz95/GetValuesTLAD.cc @@ -25,7 +25,8 @@ namespace lorenz95 { /// Constructor, destructor // ----------------------------------------------------------------------------- GetValuesTLAD::GetValuesTLAD(const Resolution & resol, - const LocsL95 & locs) + const LocsL95 & locs, + const eckit::Configuration & linearGetValuesConf) : resolidx_(locs.size()), times_(locs.times()) { // find indices of gridpoints nearest to all observations (resolidx_) diff --git a/l95/src/lorenz95/GetValuesTLAD.h b/l95/src/lorenz95/GetValuesTLAD.h index 0c741366a..2e3fe0127 100644 --- a/l95/src/lorenz95/GetValuesTLAD.h +++ b/l95/src/lorenz95/GetValuesTLAD.h @@ -16,6 +16,10 @@ #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" +namespace eckit { + class Configuration; +} + namespace lorenz95 { class GomL95; class IncrementL95; @@ -31,7 +35,7 @@ class GetValuesTLAD : public util::Printable, static const std::string classname() {return "lorenz95::GetValuesTLAD";} /// \brief computes indices resolidx_ of nearest gridpoints for all locations \p locs - GetValuesTLAD(const Resolution &, const LocsL95 & locs); + GetValuesTLAD(const Resolution &, const LocsL95 & locs, const eckit::Configuration &); /// \brief fills in \p geovals for all observations in the timeframe (\p t1, \p t2], /// \p geovals are equal to the value of \p state at the nearest gridpoint diff --git a/qg/model/GetValuesTLAD.cc b/qg/model/GetValuesTLAD.cc index f81fd0aa9..dd809fff9 100644 --- a/qg/model/GetValuesTLAD.cc +++ b/qg/model/GetValuesTLAD.cc @@ -22,8 +22,12 @@ namespace qg { // ----------------------------------------------------------------------------- /// Constructor, destructor // ----------------------------------------------------------------------------- -GetValuesTLAD::GetValuesTLAD(const GeometryQG & geom, const LocationsQG & locs) - : locs_(locs) { +GetValuesTLAD::GetValuesTLAD(const GeometryQG & geom, const LocationsQG & locs, + const eckit::Configuration & linearGetValuesConf) + : locs_(locs) +{ + oops::Log::trace() << "GetValuesTLAD create: linearGetValuesConf = " << + linearGetValuesConf << std::endl; } // ----------------------------------------------------------------------------- void GetValuesTLAD::setTrajectory(const StateQG & state, const util::DateTime & t1, diff --git a/qg/model/GetValuesTLAD.h b/qg/model/GetValuesTLAD.h index fe78dd4d0..3fbb9a652 100644 --- a/qg/model/GetValuesTLAD.h +++ b/qg/model/GetValuesTLAD.h @@ -12,6 +12,8 @@ #include #include +#include "eckit/config/Configuration.h" + #include "oops/util/DateTime.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" @@ -33,7 +35,8 @@ class GetValuesTLAD : public util::Printable, static const std::string classname() {return "qg::GetValuesTLAD";} /// \brief saves all locations \p locs to use during filling GeoVaLs - GetValuesTLAD(const GeometryQG &, const LocationsQG & locs); + GetValuesTLAD(const GeometryQG &, const LocationsQG & locs, + const eckit::Configuration &); ~GetValuesTLAD() {} /// \brief fills in \p geovals for all observations in the timeframe (\p t1, \p t2], diff --git a/qg/test/testinput/3dvar.yaml b/qg/test/testinput/3dvar.yaml index d1b404553..aa319473e 100644 --- a/qg/test/testinput/3dvar.yaml +++ b/qg/test/testinput/3dvar.yaml @@ -29,6 +29,8 @@ cost function: obs type: Stream get values: interpolation type: default_1 + linear get values: + interpolation type: default_a - obs error: covariance model: diagonal obs operator: @@ -41,6 +43,8 @@ cost function: obs type: Wind get values: interpolation type: default_2 + linear get values: + interpolation type: default_b - obs error: covariance model: diagonal obs operator: @@ -53,6 +57,8 @@ cost function: obs type: WSpeed get values: interpolation type: default_3 + linear get values: + interpolation type: default_c variational: minimizer: algorithm: DRIPCG diff --git a/qg/test/testinput/eda_3dfgat_1.yaml b/qg/test/testinput/eda_3dfgat_1.yaml index 97e7bf548..820ef2b1a 100644 --- a/qg/test/testinput/eda_3dfgat_1.yaml +++ b/qg/test/testinput/eda_3dfgat_1.yaml @@ -34,6 +34,8 @@ cost function: obs perturbations seed: 1 get values: interpolation type: default_1 + linear get values: + interpolation type: default_a - obs error: covariance model: diagonal random amplitude: 0.5 diff --git a/qg/test/testinput/hofx.yaml b/qg/test/testinput/hofx.yaml index 21cc88711..450b2a59f 100644 --- a/qg/test/testinput/hofx.yaml +++ b/qg/test/testinput/hofx.yaml @@ -22,6 +22,8 @@ observations: obs type: Stream get values: interpolation type: default_1 + linear get values: + interpolation type: default_a - obs space: obsdatain: obsfile: Data/truth.obs4d_12h.nc @@ -32,6 +34,8 @@ observations: obs type: Wind get values: interpolation type: default_2 + linear get values: + interpolation type: default_b - obs space: obsdatain: obsfile: Data/truth.obs4d_12h.nc @@ -42,5 +46,7 @@ observations: obs type: WSpeed get values: interpolation type: default_3 + linear get values: + interpolation type: default_c prints: frequency: PT3H diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index 80efb96e7..d314981f9 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -179,7 +179,7 @@ CostJo::initialize(const CtrlVar_ & xx, const eckit::Configuration & calchofx_.initialize(xx.obsVar(), currentConf_->getInt("iteration")); std::vector getValuesConfig = - util::oopsconfigfunctions::vectoriseAndFilter(obsconf_, "get values"); + util::vectoriseAndFilter(obsconf_, "get values"); getvals_.reset(new GetValuesPost_(obspace_, calchofx_.locations(), calchofx_.requiredVars(), getValuesConfig)); diff --git a/src/oops/assimilation/GETKFSolver.h b/src/oops/assimilation/GETKFSolver.h index 9b2a2ec11..599442050 100644 --- a/src/oops/assimilation/GETKFSolver.h +++ b/src/oops/assimilation/GETKFSolver.h @@ -175,7 +175,7 @@ Observations GETKFSolver::computeHofX(const StateEnsemble4D_ & tmpState += Ztmp[ieig]; std::vector getValuesConfig = - util::oopsconfigfunctions::vectoriseAndFilter(this->obsconf_, "get values"); + util::vectoriseAndFilter(this->obsconf_, "get values"); GetValuesPost_ getvals(this->obspaces_, this->hofx_.locations(), this->hofx_.requiredVars(), getValuesConfig); diff --git a/src/oops/assimilation/LocalEnsembleSolver.h b/src/oops/assimilation/LocalEnsembleSolver.h index b3175cd00..6ba990da9 100644 --- a/src/oops/assimilation/LocalEnsembleSolver.h +++ b/src/oops/assimilation/LocalEnsembleSolver.h @@ -118,7 +118,7 @@ Observations LocalEnsembleSolver::computeHofX(const StateEnsemb hofx_.resetQc(); // fill in geovals std::vector getValuesConfig = - util::oopsconfigfunctions::vectoriseAndFilter(obsconf_, "get values"); + util::vectoriseAndFilter(obsconf_, "get values"); GetValuesPost_ getvals(obspaces_, hofx_.locations(), hofx_.requiredVars(), getValuesConfig); getvals.fill(ens_xx[jj]); diff --git a/src/oops/base/ObserverTLAD.h b/src/oops/base/ObserverTLAD.h index 16b431178..ef4503fba 100644 --- a/src/oops/base/ObserverTLAD.h +++ b/src/oops/base/ObserverTLAD.h @@ -49,7 +49,7 @@ class ObserverTLAD { typedef State State_; public: - ObserverTLAD(const eckit::Configuration &, + ObserverTLAD(const eckit::Configuration & conf, const ObsSpace_ &, const ObsAuxCtrl_ &); ~ObserverTLAD() {} @@ -71,6 +71,7 @@ class ObserverTLAD { // Obs operator ObsOperator_ hop_; LinearObsOperator_ hoptlad_; + eckit::LocalConfiguration linearGetValuesConf_; const ObsAuxCtrl_ & ybias_; Variables geovars_; @@ -85,8 +86,16 @@ template ObserverTLAD::ObserverTLAD(const eckit::Configuration & config, const ObsSpace_ & obsdb, const ObsAuxCtrl_ & ybias) - : obsdb_(obsdb), hop_(obsdb, eckit::LocalConfiguration(config, "obs operator")), - hoptlad_(obsdb, eckit::LocalConfiguration(config, "linear obs operator")), + : obsdb_(obsdb), + hop_(obsdb, eckit::LocalConfiguration(config, "obs operator")), + hoptlad_(obsdb, config.has("linear obs operator") ? + eckit::LocalConfiguration(config, "linear obs operator") : + eckit::LocalConfiguration(config, "obs operator")), + linearGetValuesConf_(config.has("linear get values") ? + eckit::LocalConfiguration(config, "linear get values") : + (config.has("get values") ? + eckit::LocalConfiguration(config, "get values") : + eckit::LocalConfiguration(config, ""))), ybias_(ybias), geovars_(), locs_(hop_.locations()), lingetvals_(), gvals_() { geovars_ += hop_.requiredVars(); @@ -99,13 +108,14 @@ void ObserverTLAD::doInitializeTraj(const State_ & xx, const util::DateTime & winbgn, const util::DateTime & winend) { Log::trace() << "ObserverTLAD::doInitializeTraj start" << std::endl; - lingetvals_.reset(new LinearGetValues_(xx.geometry(), locs_)); + lingetvals_.reset(new LinearGetValues_(xx.geometry(), locs_, linearGetValuesConf_)); gvals_.reset(new GeoVaLs_(locs_, geovars_)); Log::trace() << "ObserverTLAD::doInitializeTraj done" << std::endl; } // ----------------------------------------------------------------------------- template -void ObserverTLAD::doProcessingTraj(const State_ & xx, const util::DateTime & t1, +void ObserverTLAD::doProcessingTraj(const State_ & xx, + const util::DateTime & t1, const util::DateTime & t2) { Log::trace() << "ObserverTLAD::doProcessingTraj start" << std::endl; // Call nonlinear getValues @@ -131,7 +141,8 @@ void ObserverTLAD::doInitializeTL(const Increment_ & dx, } // ----------------------------------------------------------------------------- template -void ObserverTLAD::doProcessingTL(const Increment_ & dx, const util::DateTime & t1, +void ObserverTLAD::doProcessingTL(const Increment_ & dx, + const util::DateTime & t1, const util::DateTime & t2) { Log::trace() << "ObserverTLAD::doProcessingTL start" << std::endl; // Get increment variables at obs locations diff --git a/src/oops/base/ObserversTLAD.h b/src/oops/base/ObserversTLAD.h index 0015c5be1..3d7cd9251 100644 --- a/src/oops/base/ObserversTLAD.h +++ b/src/oops/base/ObserversTLAD.h @@ -88,7 +88,7 @@ class ObserversTLAD : public PostBaseTLAD { // ----------------------------------------------------------------------------- template -ObserversTLAD::ObserversTLAD(const eckit::Configuration & config, +ObserversTLAD::ObserversTLAD(const eckit::Configuration & obsConfig, const ObsSpaces_ & obsdb, const ObsAuxCtrls_ & ybias) : PostBaseTLAD(obsdb.windowStart(), obsdb.windowEnd()), @@ -98,13 +98,10 @@ ObserversTLAD::ObserversTLAD(const eckit::Configuration & config, hslot_(0), hslottraj_(0) { // setup observers - std::vector typeconf = config.getSubConfigurations(); + std::vector typeconf = obsConfig.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsdb.size(); ++jobs) { - // Set LinearObsOperator section to ObsOperator section if not available - if (!typeconf[jobs].has("linear obs operator")) { - typeconf[jobs].set("linear obs operator", typeconf[jobs].getSubConfiguration("obs operator")); - } - std::shared_ptr tmp(new ObserverTLAD_(typeconf[jobs], obsdb[jobs], ybias[jobs])); + std::shared_ptr tmp(new ObserverTLAD_(typeconf[jobs], + obsdb[jobs], ybias[jobs])); observerstlad_.push_back(tmp); } Log::trace() << "ObserversTLAD::ObserversTLAD" << std::endl; diff --git a/src/oops/interface/LinearGetValues.h b/src/oops/interface/LinearGetValues.h index 0566aa48f..df530b03e 100644 --- a/src/oops/interface/LinearGetValues.h +++ b/src/oops/interface/LinearGetValues.h @@ -12,6 +12,8 @@ #include #include +#include "eckit/config/Configuration.h" + #include "oops/interface/Geometry.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/Increment.h" @@ -42,7 +44,7 @@ class LinearGetValues : public util::Printable, static const std::string classname() {return "oops::LinearGetValues";} /// Constructor, destructor - LinearGetValues(const Geometry_ &, const Locations_ &); + LinearGetValues(const Geometry_ &, const Locations_ &, const eckit::Configuration &); virtual ~LinearGetValues(); /// Interfacing @@ -70,11 +72,13 @@ class LinearGetValues : public util::Printable, template LinearGetValues::LinearGetValues(const Geometry_ & resol, - const Locations_ & loc) : lingetvalues_() -{ + const Locations_ & loc, + const eckit::Configuration & linearGetValuesConf) + : lingetvalues_() { Log::trace() << "LinearGetValues::LinearGetValues starting" << std::endl; util::Timer timer(classname(), "LinearGetValues"); - lingetvalues_.reset(new LinearGetValues_(resol.geometry(), loc.locations())); + lingetvalues_.reset(new LinearGetValues_(resol.geometry(), loc.locations(), + linearGetValuesConf)); Log::trace() << "LinearGetValues::LinearGetValues done" << std::endl; } diff --git a/src/oops/runs/HofX.h b/src/oops/runs/HofX.h index 156a35fee..8d0ee539e 100644 --- a/src/oops/runs/HofX.h +++ b/src/oops/runs/HofX.h @@ -112,7 +112,7 @@ template class HofX : public Application { // run the model and compute H(x) std::vector getValuesConfig = - util::oopsconfigfunctions::vectoriseAndFilter(obsConfig, "get values"); + util::vectoriseAndFilter(obsConfig, "get values"); std::shared_ptr getvals(new GetValuesPost_(obspaces, hofx.locations(), diff --git a/src/oops/runs/HofXNoModel.h b/src/oops/runs/HofXNoModel.h index accb8fe87..dfb7762a3 100644 --- a/src/oops/runs/HofXNoModel.h +++ b/src/oops/runs/HofXNoModel.h @@ -98,7 +98,7 @@ template class HofXNoModel : public Application { const VariablesVec_ & vars = hofx.requiredVars(); std::vector getValuesConfig = - util::oopsconfigfunctions::vectoriseAndFilter(obsConfig, "get values"); + util::vectoriseAndFilter(obsConfig, "get values"); // loop over all observation types for (size_t jj = 0; jj < obspaces.size(); ++jj) { diff --git a/src/oops/util/ConfigFunctions.cc b/src/oops/util/ConfigFunctions.cc index 6d0d8155d..bd54d66cb 100644 --- a/src/oops/util/ConfigFunctions.cc +++ b/src/oops/util/ConfigFunctions.cc @@ -10,24 +10,17 @@ namespace util { - namespace oopsconfigfunctions { - - std::vector - vectoriseAndFilter(const eckit::Configuration & config, const std::string & tag) - { - const std::vector - ObsConfigVec(config.getSubConfigurations()); - std::vector filteredConfig; - for (auto conf : ObsConfigVec) { - eckit::LocalConfiguration tempConf; - if (conf.has(tag)) { - tempConf = eckit::LocalConfiguration(conf, tag); - } - filteredConfig.push_back(tempConf); - } - return filteredConfig; + std::vector + vectoriseAndFilter(const eckit::Configuration & config, const std::string & tag) + { + const std::vector + ObsConfigVec(config.getSubConfigurations()); + std::vector filteredConfig; + for (const eckit::LocalConfiguration & conf : ObsConfigVec) { + eckit::LocalConfiguration tempConf = conf.getSubConfiguration(tag); + filteredConfig.push_back(tempConf); } - - } // namespace oopsconfigfunctions + return filteredConfig; + } } // namespace util diff --git a/src/oops/util/ConfigFunctions.h b/src/oops/util/ConfigFunctions.h index 5b9dc13c9..d7046ba72 100644 --- a/src/oops/util/ConfigFunctions.h +++ b/src/oops/util/ConfigFunctions.h @@ -16,21 +16,19 @@ namespace util { - /// Namespace for oopsconfigfunctions - namespace oopsconfigfunctions { - - /// \brief vectoriseAndFilter takes a single configuration object that is - /// internally a list of subConfigurations and breaks them up - /// into a vector of subConfigurations. - /// Then it loops over all subConfigurations and searches - /// for sub(sub)Configurations that are identified by the string tag. - /// If they exist, they are copied into the correct index of a - /// vector of LocalConfigurations. - /// If they don't exist an empty configuration is copied in the vector element. - std::vector - vectoriseAndFilter(const eckit::Configuration &config, const std::string & tag); - - } // namespace oopsconfigfunctions + + /// \brief vectoriseAndFilter takes a single configuration object that is + /// internally a list of subConfigurations and breaks them up + /// into a vector of subConfigurations. + /// Then it loops over all subConfigurations and searches + /// for sub(sub)Configurations that are identified by the string \p tag. + /// If they exist, they are copied into the correct index of a + /// vector of LocalConfigurations. + /// If they don't exist an empty configuration is copied in the vector element. + std::vector + vectoriseAndFilter(const eckit::Configuration &config, const std::string & tag); + + } // namespace util #endif // OOPS_UTIL_CONFIGFUNCTIONS_H_ diff --git a/src/test/interface/LinearGetValues.h b/src/test/interface/LinearGetValues.h index 5ab759535..463f66949 100644 --- a/src/test/interface/LinearGetValues.h +++ b/src/test/interface/LinearGetValues.h @@ -44,7 +44,7 @@ namespace test { // ================================================================================================= template class LinearGetValuesFixture : private boost::noncopyable { - typedef eckit::LocalConfiguration LocalConfig_; + typedef eckit::LocalConfiguration LocalConfig_; typedef oops::GeoVaLs GeoVaLs_; typedef oops::Geometry Geometry_; typedef oops::GetValues GetValues_; @@ -110,7 +110,12 @@ template class LinearGetValuesFixture : private b time_.reset(new DateTime_(state_->validTime())); // LinearGetValues - lineargetvalues_.reset(new LinearGetValues_(*resol_, *locs_)); + LocalConfig_ linearGetValuesConfig; + if (TestEnvironment::config().has("linear get values")) + linearGetValuesConfig = + eckit::LocalConfiguration(TestEnvironment::config(), "linear get values"); + lineargetvalues_.reset(new LinearGetValues_(*resol_, *locs_, + linearGetValuesConfig)); // Set trajectory GeoVaLs_ gvtraj(*locs_, *geovalvars_); @@ -139,8 +144,15 @@ template void testLinearGetValuesConstructor() { typedef LinearGetValuesFixture Test_; typedef oops::LinearGetValues LinearGetValues_; - std::unique_ptr lineargetvalues(new LinearGetValues_(Test_::resol(), - Test_::locs())); + eckit::LocalConfiguration linearGetValuesConfig; + if (TestEnvironment::config().has("linear get values")) + linearGetValuesConfig = + eckit::LocalConfiguration(TestEnvironment::config(), "linear get values"); + + std::unique_ptr + lineargetvalues(new LinearGetValues_(Test_::resol(), + Test_::locs(), + linearGetValuesConfig)); EXPECT(lineargetvalues.get()); oops::Log::test() << "Testing LinearGetValues: " << *lineargetvalues << std::endl; lineargetvalues.reset(); From f5429ce763066f7be90f7fe438e62004102d6f60 Mon Sep 17 00:00:00 2001 From: Mark Miesch Date: Wed, 17 Feb 2021 13:12:02 -0700 Subject: [PATCH 063/142] add issue templates (#1075) Co-authored-by: Anna Shlyaeva --- .github/ISSUE_TEMPLATE/bug-report.md | 28 +++++++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yaml | 8 +++++++ .github/ISSUE_TEMPLATE/general-issue.md | 26 +++++++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yaml create mode 100644 .github/ISSUE_TEMPLATE/general-issue.md diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 000000000..d87615d9c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,28 @@ +--- +name: Bug report +about: Use this template to report bugs +title: "[Bug]" +labels: 'bug' +assignees: '' + +--- + +## Current behavior (describe the bug) +>[Be sure to add a Pipeline, Label, Estimate, Assignees, and Epic](https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest/inside/practices/issues.html) + +## To Reproduce + +> What computer are you running on? + +> What compilers/modules are you using? + +> Steps to reproduce the behavior + +1. +2. +3. +... + +## Expected behavior + +## Additional information (optional) diff --git a/.github/ISSUE_TEMPLATE/config.yaml b/.github/ISSUE_TEMPLATE/config.yaml new file mode 100644 index 000000000..4de2fed63 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yaml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: JCSDA + url: https://jcsda.org/ + about: JCSDA web site + - name: Forums + url: https://forums.jcsda.org/ + about: JCSDA user/developer forums diff --git a/.github/ISSUE_TEMPLATE/general-issue.md b/.github/ISSUE_TEMPLATE/general-issue.md new file mode 100644 index 000000000..dc339e4b1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/general-issue.md @@ -0,0 +1,26 @@ +--- +name: General issue template +about: Use this template for general issues +title: "[New issue]" +labels: '' +assignees: '' + +--- + +## Description +>Provide a detailed description of this issue. +>What problem needs to be fixed? What new capability needs to be added? +>If this is a bug, describe the current behavior (or use the bug template). +>[Be sure to add a Pipeline, Label, Estimate, Assignees, and Epic](https://jointcenterforsatellitedataassimilation-jedi-docs.readthedocs-hosted.com/en/latest/inside/practices/issues.html) + +## Requirements + +>If this is a new feature: What does the new code need to accomplish? Does it require new software dependencies (e.g. new jedi-stack components or new python modules?) +>If this is a bugfix: What is the expected behavior? + +## Acceptance Criteria (Definition of Done) +>What does it mean for this to be finished? + +## Dependencies +>What must be done before this can be done? Add issue dependencies in ZenHub as appropriate +>Does this block progress on other issues? Add this issue as a dependency to other ZenHub issues as appropriate From 1cfb39f38c8087f2f29b9c5ff49ff6c3bd4c9a16 Mon Sep 17 00:00:00 2001 From: Benjamin Menetrier <30638301+benjaminmenetrier@users.noreply.github.com> Date: Thu, 18 Feb 2021 17:03:10 +0100 Subject: [PATCH 064/142] Minor cleaning (#1076) --- src/oops/base/IncrementEnsemble.h | 4 ---- src/oops/base/ModelSpaceCovarianceBase.h | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/oops/base/IncrementEnsemble.h b/src/oops/base/IncrementEnsemble.h index 1f641f2aa..e92e044bf 100644 --- a/src/oops/base/IncrementEnsemble.h +++ b/src/oops/base/IncrementEnsemble.h @@ -108,10 +108,6 @@ IncrementEnsemble::IncrementEnsemble(const eckit::Configuration & conf, const Geometry_ & resol, const Variables & vars) : vars_(vars), ensemblePerturbs_() { - // Get rank from config - std::vector memberConfig; - conf.get("members", memberConfig); - // Check sizes and fill in timeslots util::DateTime tslot = xb.validTime(); diff --git a/src/oops/base/ModelSpaceCovarianceBase.h b/src/oops/base/ModelSpaceCovarianceBase.h index 92d4946e2..7aba9e190 100644 --- a/src/oops/base/ModelSpaceCovarianceBase.h +++ b/src/oops/base/ModelSpaceCovarianceBase.h @@ -400,7 +400,7 @@ void ModelSpaceCovarianceBase::getVariance(Increment_ & variance) const { Increment_ mean(variance); mean.zero(); variance.zero(); - for (int ie = 0; ie < randomizationSize_; ++ie) { + for (size_t ie = 0; ie < randomizationSize_; ++ie) { this->randomize(dx); dx -= mean; dxsq = dx; From 772f4d4aa5b7a0f0f771878eab49f22333f8fecd Mon Sep 17 00:00:00 2001 From: Clementine Gas <43183478+cmgas@users.noreply.github.com> Date: Mon, 22 Feb 2021 13:48:40 -0700 Subject: [PATCH 065/142] Rename block, add tolerance criteria, change ref files (#1077) --- l95/test/testinput/eda.3dvar.block.1.yaml | 2 +- l95/test/testinput/eda.3dvar.block.2.yaml | 2 +- l95/test/testoutput/eda_3dvar_block.test | 44 +++++++++++- qg/test/testinput/eda_3dvar_block_1.yaml | 6 +- qg/test/testinput/eda_3dvar_block_2.yaml | 6 +- qg/test/testinput/eda_3dvar_block_3.yaml | 6 +- qg/test/testinput/eda_3dvar_block_4.yaml | 6 +- qg/test/testoutput/eda_3dvar_block.test | 56 ++++++++++----- src/CMakeLists.txt | 2 +- ...Minimizer.h => DRPBlockLanczosMinimizer.h} | 68 ++++++++++++------- src/oops/assimilation/instantiateMinFactory.h | 6 +- 11 files changed, 144 insertions(+), 60 deletions(-) rename src/oops/assimilation/{BlockBLanczosMinimizer.h => DRPBlockLanczosMinimizer.h} (85%) diff --git a/l95/test/testinput/eda.3dvar.block.1.yaml b/l95/test/testinput/eda.3dvar.block.1.yaml index db45c5618..0e428834c 100644 --- a/l95/test/testinput/eda.3dvar.block.1.yaml +++ b/l95/test/testinput/eda.3dvar.block.1.yaml @@ -25,7 +25,7 @@ cost function: obs operator: {} variational: minimizer: - algorithm: BlockBLanczos + algorithm: DRPBlockLanczos members: 2 iterations: - geometry: diff --git a/l95/test/testinput/eda.3dvar.block.2.yaml b/l95/test/testinput/eda.3dvar.block.2.yaml index 1d2cf201c..3d1216f10 100644 --- a/l95/test/testinput/eda.3dvar.block.2.yaml +++ b/l95/test/testinput/eda.3dvar.block.2.yaml @@ -25,7 +25,7 @@ cost function: obs operator: {} variational: minimizer: - algorithm: BlockBLanczos + algorithm: DRPBlockLanczos members: 2 iterations: - geometry: diff --git a/l95/test/testoutput/eda_3dvar_block.test b/l95/test/testoutput/eda_3dvar_block.test index dc711fe33..74d831748 100644 --- a/l95/test/testoutput/eda_3dvar_block.test +++ b/l95/test/testoutput/eda_3dvar_block.test @@ -1,14 +1,54 @@ Test : CostJb : Nonlinear Jb = 0 Test : CostJo : Nonlinear Jo(Lorenz 95) = 1003.27, nobs = 120, Jo/n = 8.36061, err = 0.4 Test : CostFunction: Nonlinear J = 1003.27 -Test : BlockBLanczosMinimizer: reduction in residual norm = 0.000587397 +Test : Norm reduction all members ( 1) = 0.226112 0.246079 +Test : Quadratic cost function all members: J ( 1) = 157.904 186.615 +Test : Norm reduction all members ( 2) = 0.0711168 0.0801927 +Test : Quadratic cost function all members: J ( 2) = 119.121 129.88 +Test : Norm reduction all members ( 3) = 0.0329696 0.0324763 +Test : Quadratic cost function all members: J ( 3) = 114.134 121.796 +Test : Norm reduction all members ( 4) = 0.0143961 0.0146658 +Test : Quadratic cost function all members: J ( 4) = 113.103 120.416 +Test : Norm reduction all members ( 5) = 0.00801082 0.00603926 +Test : Quadratic cost function all members: J ( 5) = 112.853 120.144 +Test : Norm reduction all members ( 6) = 0.00525854 0.0040795 +Test : Quadratic cost function all members: J ( 6) = 112.769 120.073 +Test : Norm reduction all members ( 7) = 0.00313676 0.00251749 +Test : Quadratic cost function all members: J ( 7) = 112.735 120.046 +Test : Norm reduction all members ( 8) = 0.00178513 0.00129232 +Test : Quadratic cost function all members: J ( 8) = 112.723 120.036 +Test : Norm reduction all members ( 9) = 0.000947141 0.000990084 +Test : Quadratic cost function all members: J ( 9) = 112.719 120.033 +Test : Norm reduction all members (10) = 0.000587397 0.000553939 +Test : Quadratic cost function all members: J (10) = 112.718 120.031 +Test : DRPBlockLanczosMinimizer: reduction in residual norm = 0.000587397 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-02T00:00:00Z Test : Min=6.77364, Max=9.00861, Average=8.0056 Test : CostJb : Nonlinear Jb = 97.3394 Test : CostJo : Nonlinear Jo(Lorenz 95) = 15.3786, nobs = 120, Jo/n = 0.128155, err = 0.4 Test : CostFunction: Nonlinear J = 112.718 -Test : BlockBLanczosMinimizer: reduction in residual norm = 0.00319633 +Test : Norm reduction all members ( 1) = 0.603209 0.531155 +Test : Quadratic cost function all members: J ( 1) = 112.718 120.031 +Test : Norm reduction all members ( 2) = 0.486549 0.413971 +Test : Quadratic cost function all members: J ( 2) = 112.718 120.031 +Test : Norm reduction all members ( 3) = 0.225252 0.225474 +Test : Quadratic cost function all members: J ( 3) = 112.717 120.031 +Test : Norm reduction all members ( 4) = 0.136703 0.150484 +Test : Quadratic cost function all members: J ( 4) = 112.717 120.031 +Test : Norm reduction all members ( 5) = 0.0844788 0.0897435 +Test : Quadratic cost function all members: J ( 5) = 112.717 120.031 +Test : Norm reduction all members ( 6) = 0.0543314 0.0522884 +Test : Quadratic cost function all members: J ( 6) = 112.717 120.031 +Test : Norm reduction all members ( 7) = 0.0325225 0.0286418 +Test : Quadratic cost function all members: J ( 7) = 112.717 120.031 +Test : Norm reduction all members ( 8) = 0.0177981 0.0162206 +Test : Quadratic cost function all members: J ( 8) = 112.717 120.031 +Test : Norm reduction all members ( 9) = 0.00868724 0.00777496 +Test : Quadratic cost function all members: J ( 9) = 112.717 120.031 +Test : Norm reduction all members (10) = 0.00319633 0.00372359 +Test : Quadratic cost function all members: J (10) = 112.717 120.031 +Test : DRPBlockLanczosMinimizer: reduction in residual norm = 0.00319633 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-02T00:00:00Z Test : Min=6.77193, Max=9.00912, Average=8.00572 diff --git a/qg/test/testinput/eda_3dvar_block_1.yaml b/qg/test/testinput/eda_3dvar_block_1.yaml index be50fac61..97c69ecbf 100644 --- a/qg/test/testinput/eda_3dvar_block_1.yaml +++ b/qg/test/testinput/eda_3dvar_block_1.yaml @@ -55,12 +55,12 @@ cost function: obs perturbations seed: 1 variational: minimizer: - algorithm: BlockBLanczos + algorithm: DRPBlockLanczos members: 4 iterations: - diagnostics: departures: ombg - gradient norm reduction: 1.0e-10 + gradient norm reduction: 0.2 ninner: 10 geometry: nx: 20 @@ -70,7 +70,7 @@ variational: obs perturbations: false - diagnostics: departures: ombg - gradient norm reduction: 1.0e-10 + gradient norm reduction: 0.2 ninner: 10 geometry: nx: 40 diff --git a/qg/test/testinput/eda_3dvar_block_2.yaml b/qg/test/testinput/eda_3dvar_block_2.yaml index 9c084f5d2..a6dcabd76 100644 --- a/qg/test/testinput/eda_3dvar_block_2.yaml +++ b/qg/test/testinput/eda_3dvar_block_2.yaml @@ -55,12 +55,12 @@ cost function: obs perturbations seed: 2 variational: minimizer: - algorithm: BlockBLanczos + algorithm: DRPBlockLanczos members: 4 iterations: - diagnostics: departures: ombg - gradient norm reduction: 1.0e-10 + gradient norm reduction: 0.2 ninner: 10 geometry: nx: 20 @@ -70,7 +70,7 @@ variational: obs perturbations: true - diagnostics: departures: ombg - gradient norm reduction: 1.0e-10 + gradient norm reduction: 0.2 ninner: 10 geometry: nx: 40 diff --git a/qg/test/testinput/eda_3dvar_block_3.yaml b/qg/test/testinput/eda_3dvar_block_3.yaml index c0d9c8386..b336c0c48 100644 --- a/qg/test/testinput/eda_3dvar_block_3.yaml +++ b/qg/test/testinput/eda_3dvar_block_3.yaml @@ -55,12 +55,12 @@ cost function: obs perturbations seed: 3 variational: minimizer: - algorithm: BlockBLanczos + algorithm: DRPBlockLanczos members: 4 iterations: - diagnostics: departures: ombg - gradient norm reduction: 1.0e-10 + gradient norm reduction: 0.2 ninner: 10 geometry: nx: 20 @@ -70,7 +70,7 @@ variational: obs perturbations: true - diagnostics: departures: ombg - gradient norm reduction: 1.0e-10 + gradient norm reduction: 0.2 ninner: 10 geometry: nx: 40 diff --git a/qg/test/testinput/eda_3dvar_block_4.yaml b/qg/test/testinput/eda_3dvar_block_4.yaml index a82ed446c..dcf92dbac 100644 --- a/qg/test/testinput/eda_3dvar_block_4.yaml +++ b/qg/test/testinput/eda_3dvar_block_4.yaml @@ -55,12 +55,12 @@ cost function: obs perturbations seed: 4 variational: minimizer: - algorithm: BlockBLanczos + algorithm: DRPBlockLanczos members: 4 iterations: - diagnostics: departures: ombg - gradient norm reduction: 1.0e-10 + gradient norm reduction: 0.2 ninner: 10 geometry: nx: 20 @@ -70,7 +70,7 @@ variational: obs perturbations: true - diagnostics: departures: ombg - gradient norm reduction: 1.0e-10 + gradient norm reduction: 0.2 ninner: 10 geometry: nx: 40 diff --git a/qg/test/testoutput/eda_3dvar_block.test b/qg/test/testoutput/eda_3dvar_block.test index 783becc14..42a417d66 100644 --- a/qg/test/testoutput/eda_3dvar_block.test +++ b/qg/test/testoutput/eda_3dvar_block.test @@ -3,31 +3,55 @@ Test : CostJo : Nonlinear Jo(Stream) = 10879.7, nobs = 600, Jo/n = 18.1329 Test : CostJo : Nonlinear Jo(Wind) = 1663.07, nobs = 600, Jo/n = 2.77178, err = 6 Test : CostJo : Nonlinear Jo(WSpeed) = 163.034, nobs = 300, Jo/n = 0.543446, err = 12 Test : CostFunction: Nonlinear J = 12705.8 -Test : BlockBLanczosMinimizer: reduction in residual norm = 0.0230185 -Test : CostFunction::addIncrement: Analysis: +Test : Norm reduction all members ( 1) = 0.402888 0.342551 0.430302 0.347356 +Test : Quadratic cost function all members: J ( 1) = 5681.52 7492.09 3160.43 5694.31 +Test : Norm reduction all members ( 2) = 0.1949 0.165786 0.260803 0.171396 +Test : Quadratic cost function all members: J ( 2) = 3979.77 5093.07 2507.38 3641.38 +Test : Norm reduction all members ( 3) = 0.158052 0.116268 0.185322 0.0977361 +Test : Quadratic cost function all members: J ( 3) = 3313.65 4119.31 2154.65 2990.07 +Test : DRPBlockLanczosMinimizer: reduction in residual norm = 0.158052 +Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Min= -4.4863e+08, Max= 1.0335e+08, RMS= 1.7694e+08 +Test : Min= -4.4961e+08, Max= 1.0918e+08, RMS= 1.7739e+08 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 151.4 -Test : CostJo : Nonlinear Jo(Stream) = 688.6, nobs = 600, Jo/n = 1.148, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 596.2, nobs = 600, Jo/n = 0.9936, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 91.81, nobs = 300, Jo/n = 0.306, err = 12 -Test : CostFunction: Nonlinear J = 1528 -Test : BlockBLanczosMinimizer: reduction in residual norm = 0.2453 -Test : CostFunction::addIncrement: Analysis: +Test : CostJb : Nonlinear Jb = 31.4 +Test : CostJo : Nonlinear Jo(Stream) = 2150, nobs = 600, Jo/n = 3.584, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 1104, nobs = 600, Jo/n = 1.84, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 83.31, nobs = 300, Jo/n = 0.2777, err = 12 +Test : CostFunction: Nonlinear J = 3369 +Test : Norm reduction all members ( 1) = 0.7216 0.7109 0.7893 0.8465 +Test : Quadratic cost function all members: J ( 1) = 3027 3825 2014 2814 +Test : Norm reduction all members ( 2) = 1.021 0.9841 1.035 1.024 +Test : Quadratic cost function all members: J ( 2) = 2506 3217 1761 2438 +Test : Norm reduction all members ( 3) = 0.7978 0.7637 0.8287 0.8148 +Test : Quadratic cost function all members: J ( 3) = 2136 2681 1581 2157 +Test : Norm reduction all members ( 4) = 0.426 0.5472 0.4854 0.5317 +Test : Quadratic cost function all members: J ( 4) = 1923 2450 1427 1979 +Test : Norm reduction all members ( 5) = 0.4426 0.3654 0.394 0.4996 +Test : Quadratic cost function all members: J ( 5) = 1784 2130 1326 1798 +Test : Norm reduction all members ( 6) = 0.3488 0.2975 0.3185 0.3828 +Test : Quadratic cost function all members: J ( 6) = 1652 1980 1247 1648 +Test : Norm reduction all members ( 7) = 0.2445 0.2136 0.2516 0.3026 +Test : Quadratic cost function all members: J ( 7) = 1550 1898 1189 1576 +Test : Norm reduction all members ( 8) = 0.1635 0.1692 0.1936 0.2002 +Test : Quadratic cost function all members: J ( 8) = 1447 1789 1136 1491 +Test : Norm reduction all members ( 9) = 0.1256 0.1218 0.1586 0.1376 +Test : Quadratic cost function all members: J ( 9) = 1403 1740 1113 1437 +Test : DRPBlockLanczosMinimizer: reduction in residual norm = 0.1256 +Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Min= -4.5389e+08, Max= 1.0534e+08, RMS= 1.7720e+08 +Test : Min= -4.6332e+08, Max= 1.0396e+08, RMS= 1.7698e+08 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 162.8 -Test : CostJo : Nonlinear Jo(Stream) = 620, nobs = 600, Jo/n = 1.033, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 435.2, nobs = 600, Jo/n = 0.7254, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 56.05, nobs = 300, Jo/n = 0.1868, err = 12 -Test : CostFunction: Nonlinear J = 1274 +Test : CostJb : Nonlinear Jb = 162.6 +Test : CostJo : Nonlinear Jo(Stream) = 563.7, nobs = 600, Jo/n = 0.9395, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 490.7, nobs = 600, Jo/n = 0.8179, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 74.86, nobs = 300, Jo/n = 0.2495, err = 12 +Test : CostFunction: Nonlinear J = 1292 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 825e8273d..67694bc89 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,7 +2,6 @@ include_directories( ${CMAKE_CURRENT_SOURCE_DIR} ) list( APPEND oops_src_files -oops/assimilation/BlockBLanczosMinimizer.h oops/assimilation/BMatrix.h oops/assimilation/CalcHofX.h oops/assimilation/ControlIncrement.h @@ -23,6 +22,7 @@ oops/assimilation/CostTermBase.h oops/assimilation/DRGMRESRMinimizer.h oops/assimilation/DRIPCGMinimizer.h oops/assimilation/DRMinimizer.h +oops/assimilation/DRPBlockLanczosMinimizer.h oops/assimilation/DRPCGMinimizer.h oops/assimilation/DRPFOMMinimizer.h oops/assimilation/DRPLanczosMinimizer.h diff --git a/src/oops/assimilation/BlockBLanczosMinimizer.h b/src/oops/assimilation/DRPBlockLanczosMinimizer.h similarity index 85% rename from src/oops/assimilation/BlockBLanczosMinimizer.h rename to src/oops/assimilation/DRPBlockLanczosMinimizer.h index 9e2921d24..e69662062 100644 --- a/src/oops/assimilation/BlockBLanczosMinimizer.h +++ b/src/oops/assimilation/DRPBlockLanczosMinimizer.h @@ -5,8 +5,8 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef OOPS_ASSIMILATION_BLOCKBLANCZOSMINIMIZER_H_ -#define OOPS_ASSIMILATION_BLOCKBLANCZOSMINIMIZER_H_ +#ifndef OOPS_ASSIMILATION_DRPBLOCKLANCZOSMINIMIZER_H_ +#define OOPS_ASSIMILATION_DRPBLOCKLANCZOSMINIMIZER_H_ #include @@ -33,7 +33,7 @@ namespace oops { -template class BlockBLanczosMinimizer : +template class DRPBlockLanczosMinimizer : public DRMinimizer { typedef BMatrix Bmat_; typedef CostFunction CostFct_; @@ -43,9 +43,9 @@ template class BlockBLanczosMinimizer : typedef Eigen::MatrixXd eigenmat_; public: - const std::string classname() const override {return "BlockBLanczosMinimizer";} - BlockBLanczosMinimizer(const eckit::Configuration &, const CostFct_ &); - ~BlockBLanczosMinimizer() {} + const std::string classname() const override {return "DRPBlockLanczosMinimizer";} + DRPBlockLanczosMinimizer(const eckit::Configuration &, const CostFct_ &); + ~DRPBlockLanczosMinimizer() {} private: double solve(CtrlInc_ &, CtrlInc_ &, CtrlInc_ &, const Bmat_ &, const HtRinvH_ &, const double, @@ -76,7 +76,7 @@ template class BlockBLanczosMinimizer : // MPI version (storage of partial Krylov base on each processor) template -BlockBLanczosMinimizer::BlockBLanczosMinimizer(const eckit::Configuration & conf, +DRPBlockLanczosMinimizer::DRPBlockLanczosMinimizer(const eckit::Configuration & conf, const CostFct_ & J) : DRMinimizer(J), members_(conf.getInt("members")), ntasks_(oops::mpi::world().size()), @@ -86,7 +86,7 @@ BlockBLanczosMinimizer::BlockBLanczosMinimizer(const eckit::Configur // ----------------------------------------------------------------------------------------------- template -double BlockBLanczosMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, CtrlInc_ & rr, +double DRPBlockLanczosMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, CtrlInc_ & rr, const Bmat_ & B, const HtRinvH_ & HtRinvH, const double costJ0Jb, const double costJ0JoJc, const int maxiter, const double tolerance) { @@ -110,6 +110,7 @@ double BlockBLanczosMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, C eigenmat_ alpha(members_, members_); eigenmat_ beta = zeromm; eigenmat_ beta0 = zeromm; + eigenmat_ SSLK; std::vector ALPHAS; // store the diagonal blocks of the Arnoldi matrix std::vector BETAS; // store the underdiagonal blocks of the Arnoldi matrix @@ -120,6 +121,10 @@ double BlockBLanczosMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, C double norm_iiter = 0; double norm0 = 0; double normReduction = 1; + double normReductionIter = 1; + + int iterTotal = maxiter; + bool complexValues = false; eigenvec_ norm_red_loc(maxiter); @@ -146,8 +151,8 @@ double BlockBLanczosMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, C Vbase.emplace_back(std::unique_ptr(new CtrlInc_(vv))); Zbase.emplace_back(std::unique_ptr(new CtrlInc_(zz))); - for (int iiter = 0; iiter < maxiter; ++iiter) { - Log::info() << "BlockBLanczos starting iteration " << iiter << " for rank: " << mymember_ + for (int iiter = 0; iiter < maxiter && normReductionIter > tolerance; ++iiter) { + Log::info() << "BlockBLanczos starting iteration " << iiter+1 << " for rank: " << mymember_ << std::endl; // Hessian application: w_i = v_i + HtRinvH * B*v_i = v_i + HtRinvH * z_i @@ -183,26 +188,20 @@ double BlockBLanczosMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, C norm_iiter = sqrt(temp.dot(ss_loc)); normReduction = norm_iiter / norm0; - xh.zero(); - xx.zero(); - const double costJ0 = costJ0Jb + costJ0JoJc; double costJ = costJ0; double costJb = 0; double costJoJc = 0; - eigenmat_ SSLK; for (int ll = 0; ll < iiter+1; ++ll) { SSLK = - (ss.block(ll*members_, 0, members_, members_)); - apply_proj(xh, *Vbase[ll], SSLK, gestag, CommGeo, temp1); - apply_proj(xx, *Zbase[ll], SSLK, gestag, CommGeo, temp1); // Compute the quadratic cost function // J[du_{i}] = J[0] - 0.5 s_{i}^T Z_{i}^T r_{0} // Jb[du_{i}] = 0.5 s_{i}^T V_{i}^T Z_{i} s_{i} temp2.zero(); apply_proj(temp2, *Zbase[ll], SSLK, gestag, CommGeo, temp1); costJ -= 0.5 * dot_product(temp2, rr); - } // this loop can be moved outside the main loop when we don't need the diagnostics anymore + } Log::info() << "BlockBLanczos end of iteration " << iiter+1 << std::endl; printNormReduction(iiter+1, norm_iiter, normReduction); @@ -210,13 +209,34 @@ double BlockBLanczosMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, C norm_red_loc[iiter] = normReduction; costj_loc[iiter] = costJ; + + oops::mpi::world().allReduce(normReduction, normReductionIter, eckit::mpi::max()); + if (normReductionIter < tolerance) { + Log::info() << "DRPBlockLanczos: Achieved required reduction in residual norm." << std::endl; + iterTotal = iiter+1; + } } // main loop iiter oops::mpi::allGather(CommGeo, norm_red_loc, norm_red_all); oops::mpi::allGather(CommGeo, costj_loc, costj_all); - Log::info() << "Norm reduction for all members:\n" << norm_red_all << std::endl; - Log::info() << "Quadratic costJ for all members:\n" << costj_all << std::endl; + xh.zero(); + xx.zero(); + + for (int ll = 0; ll < iterTotal; ++ll) { + SSLK = - (ss.block(ll*members_, 0, members_, members_)); + apply_proj(xh, *Vbase[ll], SSLK, gestag, CommGeo, temp1); + apply_proj(xx, *Zbase[ll], SSLK, gestag, CommGeo, temp1); + + Log::info() << " Norm reduction all members (" << std::setw(2) << ll+1 << ") = " + << norm_red_all.block(ll, 0, 1, members_) << std::endl; + Log::info() << " Quadratic cost function all members: J (" << std::setw(2) << ll+1 << ") = " + << costj_all.block(ll, 0, 1, members_) << std::endl; + Log::test() << " Norm reduction all members (" << std::setw(2) << ll+1 << ") = " + << norm_red_all.block(ll, 0, 1, members_) << std::endl; + Log::test() << " Quadratic cost function all members: J (" << std::setw(2) << ll+1 << ") = " + << costj_all.block(ll, 0, 1, members_) << std::endl; + } eckit::mpi::deleteComm(CommGeoName); return normReduction; @@ -227,7 +247,7 @@ double BlockBLanczosMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, C // ----------------------------------------------------------------------------------------------- template -void BlockBLanczosMinimizer::get_proj(const CtrlInc_ & www, +void DRPBlockLanczosMinimizer::get_proj(const CtrlInc_ & www, const CtrlInc_ & incr_tosend, eigenmat_ & alpha_mat, int & tag, const eckit::mpi::Comm & comm, @@ -264,7 +284,7 @@ void BlockBLanczosMinimizer::get_proj(const CtrlInc_ & www, // ----------------------------------------------------------------------------------------------- template -void BlockBLanczosMinimizer::apply_proj(CtrlInc_ & incr_tochange, +void DRPBlockLanczosMinimizer::apply_proj(CtrlInc_ & incr_tochange, const CtrlInc_ & incr_tosend, const eigenmat_ & alpha_mat, int & tag, const eckit::mpi::Comm & comm, @@ -299,7 +319,7 @@ void BlockBLanczosMinimizer::apply_proj(CtrlInc_ & incr_tochange, // ----------------------------------------------------------------------------------------------- template -void BlockBLanczosMinimizer::mqrgs(CtrlInc_ & zzz, CtrlInc_ & vvv, +void DRPBlockLanczosMinimizer::mqrgs(CtrlInc_ & zzz, CtrlInc_ & vvv, eigenmat_ & beta_mat, const CtrlInc_ & www, int & tag, const eckit::mpi::Comm & comm, @@ -349,7 +369,7 @@ void BlockBLanczosMinimizer::mqrgs(CtrlInc_ & zzz, CtrlInc_ & vvv, // ----------------------------------------------------------------------------------------------- template -void BlockBLanczosMinimizer::HtRinvH0(const CtrlInc_ & z_loc, CtrlInc_ & w_out, +void DRPBlockLanczosMinimizer::HtRinvH0(const CtrlInc_ & z_loc, CtrlInc_ & w_out, const HtRinvH_ & HtRinvH, int & tag, const eckit::mpi::Comm & comm, CtrlInc_ & z_other) { // send z_loc to task 0, process HtRinvH_0 * z_loc and send the result back to original task @@ -381,4 +401,4 @@ void BlockBLanczosMinimizer::HtRinvH0(const CtrlInc_ & z_loc, CtrlIn } // namespace oops -#endif // OOPS_ASSIMILATION_BLOCKBLANCZOSMINIMIZER_H_ +#endif // OOPS_ASSIMILATION_DRPBLOCKLANCZOSMINIMIZER_H_ diff --git a/src/oops/assimilation/instantiateMinFactory.h b/src/oops/assimilation/instantiateMinFactory.h index a6050267b..21f9e05cf 100644 --- a/src/oops/assimilation/instantiateMinFactory.h +++ b/src/oops/assimilation/instantiateMinFactory.h @@ -11,9 +11,9 @@ #ifndef OOPS_ASSIMILATION_INSTANTIATEMINFACTORY_H_ #define OOPS_ASSIMILATION_INSTANTIATEMINFACTORY_H_ -#include "oops/assimilation/BlockBLanczosMinimizer.h" #include "oops/assimilation/DRGMRESRMinimizer.h" #include "oops/assimilation/DRIPCGMinimizer.h" +#include "oops/assimilation/DRPBlockLanczosMinimizer.h" #include "oops/assimilation/DRPCGMinimizer.h" #include "oops/assimilation/DRPFOMMinimizer.h" #include "oops/assimilation/DRPLanczosMinimizer.h" @@ -48,8 +48,8 @@ template void instantiateMinFactory() { static MinMaker > makerRPLanczos_("RPLanczos"); static MinMaker > makerMINRES_("MINRES"); static MinMaker > makerFGMRES_("FGMRES"); - static MinMaker > - makerBlockBLanczos_("BlockBLanczos"); + static MinMaker > + makerBlockBLanczos_("DRPBlockLanczos"); } } // namespace oops From 023b685b2c021456a71db0fc90130523cf16529f Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Mon, 8 Mar 2021 11:14:19 -0700 Subject: [PATCH 066/142] additional check in ObsAuxIncrement maybe-copy-constructor test (#1082) --- src/oops/interface/ObsAuxIncrement.h | 4 +++- src/test/interface/ObsAuxIncrement.h | 12 ++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/oops/interface/ObsAuxIncrement.h b/src/oops/interface/ObsAuxIncrement.h index dce7aa53a..0514193c5 100644 --- a/src/oops/interface/ObsAuxIncrement.h +++ b/src/oops/interface/ObsAuxIncrement.h @@ -44,7 +44,9 @@ class ObsAuxIncrement : public util::Printable, /// Constructor, destructor ObsAuxIncrement(const ObsSpace &, const eckit::Configuration &); - ObsAuxIncrement(const ObsAuxIncrement &, const bool copy = true); + /// Copies \p other if \p copy is true, otherwise creates zero increment + /// of the same size as \p other. + ObsAuxIncrement(const ObsAuxIncrement & other, const bool copy = true); ~ObsAuxIncrement(); /// Interfacing diff --git a/src/test/interface/ObsAuxIncrement.h b/src/test/interface/ObsAuxIncrement.h index 1b8ac8e2a..bc672395b 100644 --- a/src/test/interface/ObsAuxIncrement.h +++ b/src/test/interface/ObsAuxIncrement.h @@ -78,10 +78,10 @@ template void testObsAuxIncrementConstructor() { } // ----------------------------------------------------------------------------- - +/// Tests copy-constructor (with option of allocating, but not copying the data) template void testObsAuxIncrementCopyConstructor() { - typedef ObsTestsFixture Test_; - typedef oops::ObsAuxIncrement AuxIncr_; + typedef ObsTestsFixture Test_; + typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { if (Test_::config(jj).has("obs bias error")) { @@ -89,11 +89,15 @@ template void testObsAuxIncrementCopyConstructor() { AuxIncr_ dx1(Test_::obspace()[jj], biasconf); ObsAuxIncrementFixture::covariance(jj).randomize(dx1); oops::Log::test() << "Printing random ObsAuxIncrement: " << dx1 << std::endl; + /// Test that creating new increment without copying data works + AuxIncr_ dxempty(dx1, false); + EXPECT_EQUAL(dxempty.norm(), 0.0); + /// Test that creating new increment with copying data works AuxIncr_ dx2(dx1); EXPECT(dx2.norm() > 0.0); EXPECT(dx2.norm() == dx1.norm()); -// Check that the copy is equal to the original + /// Test that the copy is equal to the original dx2 -= dx1; EXPECT(dx2.norm() == 0.0); } From 42ec8107f0c8bb3b9e2bbc326bee5ae3e64c05e4 Mon Sep 17 00:00:00 2001 From: Mark J Olah Date: Tue, 9 Mar 2021 13:06:31 -0700 Subject: [PATCH 067/142] Feature/timer update (#1071) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update oops::Timer class. * Add a new oops::LoggingTimer class and output channels * Change Timers to use the CLOCK_MONOTONIC and clock_gettime() * Update timer statistics pretty printing formatting * Fix for floating points errors in simplified l95 tests (#1055) * Modify ref files + data file * Add 4th dimension to simplified tests * Update ref * remove "ObsBias" addition (bias correction now applied inside simulateObs) (#1031) * remove "ObsBias" addition (now done inside simulateObs) * remove third parameter in ObsVector ctor ("fail"); not needed anymore * remove more unused (ta-da!) * Defined macros similar to OOPS_CONCRETE/ABSTRACT_PARAMETERS, but skipping the definitions of the non-copy and non-move constructors. (#1061) Co-authored-by: Anna Shlyaeva * use intel-oneapi for codebuild (#1057) * update codebuild intel; clean up other yamls * bugfix * bugfix * trigger tests * trigger tests * bugfix * qg_4dvar_saddlepoint test hangs; use 15G, 8CPUs for intel * exclude test_qg_4dvar_saddlepoint * refactor VariableChange/VariableChangeBase (#1008) * refactor VariableChange/VariableChangeBase * bugfix for parameters; move comments; missed include * add VarChange base class (one for generic, one for MODEL-specific Varchanges) * add print for varin_,varout_ & remove obsolete comment * improve ObsAuxControl test: add comparison with reference norm (#1058) * add comparison with norm to the ObsAuxControl test * use "is_close_relative" and "relative tolerance" in yaml * ObsOperator test: make it possible to verify an ObsOperator's constructor throws an exception. (#1067) * pass only obs bias configs to ObsAux classes (#1064) * change virtual to override (#1069) * make the nonlinear Jb evaluation optional (#1065) * naive implementation On branch feature/optional_nonlinear_jb Changes to be committed: modified: qg/test/testinput/3dvar.yaml modified: src/oops/assimilation/CostFunction.h * clceanup * add a test for demonstration Changes to be committed: modified: qg/test/CMakeLists.txt modified: qg/test/testinput/3dvar.yaml new file: qg/test/testinput/3dvar_hybrid_wo_jb_evaluation.yaml new file: qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test * Move the logic to CostJbTotal modified: qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test modified: src/oops/assimilation/CostFunction.h modified: src/oops/assimilation/CostJbTotal.h * make the logic in a single code line modified: qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test modified: src/oops/assimilation/CostJbTotal.h * Cleanup cpplint failures * Remove uneeded hdr * Use std::chrono C++ time facilities. Add parameter names to headers. * Address cpplint * Disable build/c++11 cpplint filter Co-authored-by: Clementine Gas <43183478+cmgas@users.noreply.github.com> Co-authored-by: Anna Shlyaeva Co-authored-by: Wojciech Śmigaj Co-authored-by: Maryam Abdi Co-authored-by: Chris Thomas <32307951+ctgh@users.noreply.github.com> Co-authored-by: Byoung-Joo (BJ) Jung <33296274+byoung-joo@users.noreply.github.com> --- CPPLINT.cfg | 2 +- src/oops/util/LibOOPS.cc | 21 ++++++++--- src/oops/util/LibOOPS.h | 11 ++++-- src/oops/util/Logger.h | 1 + src/oops/util/Timer.cc | 71 ++++++++++++++++++++++++++---------- src/oops/util/Timer.h | 34 +++++++++++------ src/oops/util/TimerHelper.cc | 70 ++++++++++++++++++++--------------- 7 files changed, 140 insertions(+), 70 deletions(-) diff --git a/CPPLINT.cfg b/CPPLINT.cfg index 30ec5df67..ab13926ab 100644 --- a/CPPLINT.cfg +++ b/CPPLINT.cfg @@ -1,3 +1,3 @@ set noparent linelength=100 -filter=+build,+legal,+readability,+runtime,+whitespace,-runtime/references,-runtime/printf +filter=+build,-build/c++11,+legal,+readability,+runtime,+whitespace,-runtime/references,-runtime/printf diff --git a/src/oops/util/LibOOPS.cc b/src/oops/util/LibOOPS.cc index 0e807cac0..9e3fdbe36 100644 --- a/src/oops/util/LibOOPS.cc +++ b/src/oops/util/LibOOPS.cc @@ -95,6 +95,7 @@ void LibOOPS::initialise() { do_abortfpe = getEnv("OOPS_ABORTFPE", 1); trap_sigfpe(do_abortfpe); } + enable_timer_channel_ = getEnv("OOPS_TIMER", 0) > 0 && rank_ == 0; #ifdef ENABLE_GPTL do_profile = getEnv("OOPS_PROFILE", 0); @@ -133,14 +134,13 @@ void LibOOPS::finalise(bool finaliseMPI) { } } #endif - // Make sure that these specialised channels that wrap eckit::Log::info() are - // destroyed before eckit::Log::info gets destroyed. - // Just in case someone still tries to log, we reset to empty channels. - infoChannel_.reset(new eckit::Channel()); debugChannel_.reset(new eckit::Channel()); traceChannel_.reset(new eckit::Channel()); statsChannel_.reset(new eckit::Channel()); - testChannel_. reset(new eckit::Channel()); + testChannel_.reset(new eckit::Channel()); + timerChannel_.reset(new eckit::Channel()); + // Destroy info channel last after other channels have flushed all output + infoChannel_.reset(new eckit::Channel()); if (finaliseMPI) eckit::mpi::finaliseAllComms(); @@ -191,6 +191,17 @@ eckit::Channel& LibOOPS::testChannel() const { return *testChannel_; } +eckit::Channel& LibOOPS::timerChannel() const { + if (timerChannel_) {return *timerChannel_;} + if (enable_timer_channel_) { + timerChannel_.reset(new eckit::Channel( + new eckit::PrefixTarget("OOPS_TIMER:", new eckit::OStreamTarget(eckit::Log::info())))); + } else { + timerChannel_.reset(new eckit::Channel()); + } + return *timerChannel_; +} + eckit::Channel& LibOOPS::infoChannel() const { if (rank_ == 0) {return eckit::Log::info();} if (!infoChannel_) infoChannel_.reset(new eckit::Channel()); diff --git a/src/oops/util/LibOOPS.h b/src/oops/util/LibOOPS.h index 3ce08cf8c..9dcf9e82b 100644 --- a/src/oops/util/LibOOPS.h +++ b/src/oops/util/LibOOPS.h @@ -32,12 +32,13 @@ class LibOOPS : public eckit::system::Library { static LibOOPS& instance(); - virtual eckit::Channel& infoChannel() const; + eckit::Channel& infoChannel() const; eckit::Channel& debugChannel() const override; - virtual eckit::Channel& traceChannel() const; - virtual eckit::Channel& statsChannel() const; - virtual eckit::Channel& testChannel() const; + eckit::Channel& traceChannel() const; + eckit::Channel& statsChannel() const; + eckit::Channel& testChannel() const; + eckit::Channel& timerChannel() const; void initialise(); void teeOutput(const std::string &); @@ -56,12 +57,14 @@ class LibOOPS : public eckit::system::Library { mutable std::unique_ptr traceChannel_; mutable std::unique_ptr statsChannel_; mutable std::unique_ptr testChannel_; + mutable std::unique_ptr timerChannel_; size_t rank_; bool debug_; std::string predebug_; bool trace_; std::string pretrace_; + bool enable_timer_channel_; }; // ----------------------------------------------------------------------------- diff --git a/src/oops/util/Logger.h b/src/oops/util/Logger.h index 3d31c888e..5d46fd161 100644 --- a/src/oops/util/Logger.h +++ b/src/oops/util/Logger.h @@ -28,6 +28,7 @@ struct Log { static std::ostream& trace() {return LibOOPS::instance().traceChannel();} // prefix "OOPS_TRACE" static std::ostream& stats() {return LibOOPS::instance().statsChannel();} // prefix "OOPS_STATS" static std::ostream& test() {return LibOOPS::instance().testChannel();} // prefix "Test :" + static std::ostream& timer() {return LibOOPS::instance().timerChannel();} // prefix "OOPS_TIMER:" }; // ----------------------------------------------------------------------------- diff --git a/src/oops/util/Timer.cc b/src/oops/util/Timer.cc index e8550864b..745814ae1 100644 --- a/src/oops/util/Timer.cc +++ b/src/oops/util/Timer.cc @@ -10,37 +10,70 @@ #include "oops/util/Timer.h" -#include -// #include -#include -#include +#include +#include +#include "oops/util/Logger.h" #include "oops/util/TimerHelper.h" namespace util { -// ----------------------------------------------------------------------------- - -Timer::Timer(const std::string & cl, const std::string & met) - : class_(cl), method_(met) { -// start_(std::chrono::high_resolution_clock::now()) { - gettimeofday(&start_, NULL); +namespace { // Local global constants + // Width to print variable names + constexpr int METHOD_PRINT_WIDTH = 60; } +/** InitTime + * + * A class to be static initialized that measures time deltas since program initialization + */ +class InitTime +{ + public: + InitTime() : init_t_(Timer::ClockT::now()) + { } + + double elapsed_sec(const Timer::TimeT &t) { + return std::chrono::duration(t - init_t_).count(); + } + + private: + Timer::TimeT init_t_; +}; + +static InitTime init_time; // static instance representing program static init time + // ----------------------------------------------------------------------------- +Timer::Timer(const std::string & class_name, const std::string & method_name) + : name_(class_name + "::" + method_name), start_(ClockT::now()) +{ } + Timer::~Timer() { -// end = std::chrono::high_resolution_clock::now(); -// std::chrono::duration dt(end-start_); - struct timeval end; - gettimeofday(&end, NULL); + std::chrono::duration dt = ClockT::now() - start_; // elapsed millisecs + TimerHelper::add(name_, dt.count()); +} -// get difference in milliseconds - double dt = (end.tv_sec-start_.tv_sec)*1000.0 - + (end.tv_usec-start_.tv_usec)/1000.0; - std::string name = class_ + "::" + method_; +LoggingTimer::LoggingTimer(const std::string & class_name, const std::string & method_name) + : LoggingTimer(class_name, method_name, oops::Log::timer()) +{ } + +LoggingTimer::LoggingTimer(const std::string & class_name, + const std::string & method_name, std::ostream& log) + : Timer(class_name, method_name), log_(log) +{ + double st = init_time.elapsed_sec(start_); + log_ << std::left << std::setw(METHOD_PRINT_WIDTH) << std::setfill('.') << name_ + << " Start: " << std::fixed << std::setprecision(2) << st << std::endl; +} - TimerHelper::add(name, dt); +LoggingTimer::~LoggingTimer() +{ + double st = init_time.elapsed_sec(start_); + double et = init_time.elapsed_sec(ClockT::now()); + log_ << std::left << std::setw(METHOD_PRINT_WIDTH) << std::setfill('.') << name_ + << ".. End: " << std::fixed << std::setprecision(2) << et + << " Elapsed: " << et-st << " sec" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/util/Timer.h b/src/oops/util/Timer.h index a1afd8bb2..05073c4c5 100644 --- a/src/oops/util/Timer.h +++ b/src/oops/util/Timer.h @@ -11,27 +11,37 @@ #ifndef OOPS_UTIL_TIMER_H_ #define OOPS_UTIL_TIMER_H_ -// #include -#include - +#include +#include #include -#include - namespace util { // ----------------------------------------------------------------------------- -class Timer : private boost::noncopyable { +class Timer { public: - Timer(const std::string &, const std::string &); + using ClockT = std::chrono::steady_clock; // Monotonic clock type + using TimeT = ClockT::time_point; + + Timer(const std::string &class_name, const std::string &method_name); ~Timer(); - private: -// const std::chrono::time_point start_; - struct timeval start_; - const std::string class_; - const std::string method_; + Timer(const Timer&) = delete; // Non-copyable + + protected: + std::string name_; + TimeT start_; +}; + +class LoggingTimer : public Timer { + public: + LoggingTimer(const std::string &class_name, const std::string &method_name); + LoggingTimer(const std::string &class_name, const std::string &method_name, std::ostream &log); + ~LoggingTimer(); + + protected: + std::ostream& log_; }; // ----------------------------------------------------------------------------- diff --git a/src/oops/util/TimerHelper.cc b/src/oops/util/TimerHelper.cc index 7512011ee..729e40ce6 100644 --- a/src/oops/util/TimerHelper.cc +++ b/src/oops/util/TimerHelper.cc @@ -10,7 +10,7 @@ #include "oops/util/TimerHelper.h" -// #include +#include #include #include @@ -51,12 +51,9 @@ void TimerHelper::stop() { // ----------------------------------------------------------------------------- void TimerHelper::add(const std::string & name, const double dt) { -// const std::chrono::duration & dt) { if (getHelper().on_) { getHelper().timers_[name] += dt; getHelper().counts_[name] += 1; -// double secs = dt.count(); -// getHelper().timers_[name] += secs; } } @@ -73,21 +70,31 @@ TimerHelper::~TimerHelper() {} void TimerHelper::print(std::ostream & os) const { typedef std::map::const_iterator cit; -// Local timing statistics - os << " " << std::endl; - os << "---------------------------------------------------------------------" << std::endl; - os << "------------------------- Timing Statistics -------------------------" << std::endl; - os << "---------------------------------------------------------------------" << std::endl; - for (cit jt = timers_.begin(); jt != timers_.end(); ++jt) { - int icount = counts_.at(jt->first); - os << std::setw(52) << std::left << jt->first - << ": " << std::setw(12) << std::right << jt->second << " ms" - << " " << std::setw(6) << icount - << " " << std::setw(12) << std::right << jt->second/icount << " ms/call" - << std::endl; + { + // Local timing statistics + int table_width = 92; + os << " " << std::endl; + os << std::string(table_width, '-') << std::endl; + std::string title = " Timing Statistics "; + float title_half_width = (table_width-title.size())/2.; + os << std::string(std::floor(title_half_width), '-') + << title << std::string(std::ceil(title_half_width), '-') << std::endl + << std::string(table_width, '-') << std::endl + << std::setw(52) << std::left << "Name " << ": " + << std::setw(12) << std::right << "total (ms)" + << std::setw(8) << std::right << "count" + << std::setw(18) << std::right << "time/call (ms)" << std::endl; + for (cit jt = timers_.begin(); jt != timers_.end(); ++jt) { + int icount = counts_.at(jt->first); + os << std::setw(52) << std::left << jt->first + << ": " << std::setw(12) << std::right << std::fixed << std::setprecision(2) << jt->second + << std::setw(8) << icount + << std::setw(18) << std::right << std::fixed << std::setprecision(4) << jt->second/icount + << std::endl; + } + os << std::string(std::floor(title_half_width), '-') + << title << std::string(std::ceil(title_half_width), '-') << std::endl; } - os << "------------------------- Timing Statistics -------------------------" << std::endl; - // For MPI applications, gather and print statistics across tasks size_t ntasks = oops::mpi::world().size(); if (ntasks > 1) { @@ -135,17 +142,21 @@ void TimerHelper::print(std::ostream & os) const { } } // Print global statistics - os << " " << std::endl; - os << "---------------------------------------------------------------------" << std::endl; - os << "------------ Parallel Timing Statistics (" << std::setw(4) << ntasks - << std::setw(24) << " MPI tasks) ------------" << std::endl; - os << "---------------------------------------------------------------------" << std::endl; - os << std::setw(52) << std::left << " Name " << ": " - << std::setw(12) << std::right << "min. (ms)" - << std::setw(12) << std::right << "max. (ms)" - << std::setw(12) << std::right << "avg. (ms)" + int table_width = 114; + std::ostringstream title_s; + title_s << " Parallel Timing Statistics (" << std::setw(4) << ntasks << " MPI tasks) "; + std::string title = title_s.str(); + float title_half_width = (table_width-title.size())/2.; + os << std::endl << std::string(table_width, '-') << std::endl + << std::string(std::floor(title_half_width), '-') + << title << std::string(std::ceil(title_half_width), '-') << std::endl + << std::string(table_width, '-') << std::endl + << std::setw(52) << std::left << "Name " << ": " + << std::setw(12) << std::right << "min (ms)" + << std::setw(12) << std::right << "max (ms)" + << std::setw(12) << std::right << "avg (ms)" << std::setw(12) << std::right << "% total" - << std::setw(12) << std::right << "imbalance" << " (%)" + << std::setw(12) << std::right << "imbal (%)" << std::endl; double total = stats["util::Timers::Total"][2]/ntasks; stats["util::Timers::measured"].fill(0.0); @@ -169,7 +180,8 @@ void TimerHelper::print(std::ostream & os) const { << std::setw(12) << (jt->second[1] - jt->second[0]) / avg * 100.0 << std::endl; } - os << "-------------------- Parallel Timing Statistics ---------------------" << std::endl; + os << std::string(std::floor(title_half_width), '-') + << title << std::string(std::ceil(title_half_width), '-') << std::endl; } } } From ee209d956f806d3faed971d2d2fe5a94affd5b44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Tue, 9 Mar 2021 20:33:08 +0000 Subject: [PATCH 068/142] Parameter extensions required for encapsulation of options taken by the where statement (#1079) * Added a trait checking if a type is present in list of types. * Made it possible to store std::set objects in Parameters. * Added a class that makes it possible for Parameters to store values of types determined only at runtime. * Made it possible to encapsulate YAML entries of type null in Parameters. * Corrected a comment. * Renamed template parameters. * Replaced OptionalParameter by OptionalParameter. * Deleted an obsolete fragment of documentation. Co-authored-by: Ryan Honeyager Co-authored-by: Anna Shlyaeva --- src/CMakeLists.txt | 11 ++ src/oops/util/AnyOf.h | 107 ++++++++++++++++ src/oops/util/TypeTraits.h | 30 +++++ src/oops/util/parameters/OptionalParameter.cc | 58 +++++++++ src/oops/util/parameters/OptionalParameter.h | 56 +++++++++ src/oops/util/parameters/ParameterTraits.cc | 80 ++++++++++++ src/oops/util/parameters/ParameterTraits.h | 24 ++++ .../util/parameters/ParameterTraitsAnyOf.h | 60 +++++++++ src/test/testinput/parameters.yaml | 22 ++++ src/test/util/Parameters.h | 114 ++++++++++++++++++ src/test/util/TypeTraits.cc | 15 +++ src/test/util/TypeTraits.h | 43 +++++++ 12 files changed, 620 insertions(+) create mode 100644 src/oops/util/AnyOf.h create mode 100644 src/oops/util/TypeTraits.h create mode 100644 src/oops/util/parameters/OptionalParameter.cc create mode 100644 src/oops/util/parameters/ParameterTraits.cc create mode 100644 src/oops/util/parameters/ParameterTraitsAnyOf.h create mode 100644 src/test/util/TypeTraits.cc create mode 100644 src/test/util/TypeTraits.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 67694bc89..b09969fc8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -237,6 +237,7 @@ oops/runs/Variational.h oops/util/abor1_cpp.cc oops/util/abor1_cpp.h oops/util/abor1_ftn.F90 +oops/util/AnyOf.h oops/util/AssociativeContainers.h oops/util/CompareNVectors.h oops/util/CompositePath.cc @@ -309,6 +310,7 @@ oops/util/Timer.cc oops/util/Timer.h oops/util/TimerHelper.cc oops/util/TimerHelper.h +oops/util/TypeTraits.h oops/util/utilNamespaceDoc.h oops/util/wildcard.h oops/util/wildcard.cc @@ -323,6 +325,7 @@ oops/util/parameters/NumericConstraints.h oops/util/parameters/ObjectJsonSchema.cc oops/util/parameters/ObjectJsonSchema.h oops/util/parameters/OptionalParameter.h +oops/util/parameters/OptionalParameter.cc oops/util/parameters/OptionalPolymorphicParameter.h oops/util/parameters/Parameter.h oops/util/parameters/ParameterBase.cc @@ -331,7 +334,9 @@ oops/util/parameters/ParameterConstraint.h oops/util/parameters/Parameters.cc oops/util/parameters/Parameters.h oops/util/parameters/ParametersOrConfiguration.h +oops/util/parameters/ParameterTraits.cc oops/util/parameters/ParameterTraits.h +oops/util/parameters/ParameterTraitsAnyOf.h oops/util/parameters/ParameterTraitsScalarOrMap.h oops/util/parameters/PolymorphicParameter.h oops/util/parameters/PolymorphicParameterTraits.h @@ -406,6 +411,7 @@ test/util/MissingValues.h test/util/PropertiesOfNVectors.h test/util/stringFunctions.h test/util/LocalEnvironment.h +test/util/TypeTraits.h ) list (APPEND oops_fheader_files @@ -642,6 +648,11 @@ ecbuild_add_test( TARGET test_util_localenvironment ARGS "test/testinput/empty.yaml" LIBS oops ) +ecbuild_add_test( TARGET test_util_typetraits + SOURCES test/util/TypeTraits.cc + ARGS "test/testinput/empty.yaml" + LIBS oops ) + ecbuild_add_test( TARGET test_util_wildcard SOURCES test/util/wildcard.cc ARGS "test/testinput/empty.yaml" diff --git a/src/oops/util/AnyOf.h b/src/oops/util/AnyOf.h new file mode 100644 index 000000000..b1d7d2b79 --- /dev/null +++ b/src/oops/util/AnyOf.h @@ -0,0 +1,107 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_UTIL_ANYOF_H_ +#define OOPS_UTIL_ANYOF_H_ + +#include +#include + +#include + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/exception/Exceptions.h" +#include "oops/util/CompositePath.h" +#include "oops/util/parameters/ObjectJsonSchema.h" +#include "oops/util/parameters/ParameterTraits.h" +#include "oops/util/TypeTraits.h" + +namespace util { + +/// \brief An object encapsulating the value of a Parameter whose type is unspecified until +/// runtime but must be one of the types listed in the `Types` template parameter. +/// +/// Example: Suppose that a (mandatory) YAML configuration option `values` should be set to a list +/// of integers if certain conditions are met and to a string otherwise. You could then encapsulate +/// it in the following member variable of a Parameters subclass: +/// +/// oops::RequiredParameter>> values{"values", this}; +/// +/// and retrieve the value of this option by calling the `as()` method: +/// +/// if (/*some condition*/) +/// std::vector v = values.value().as>(); +/// else +/// std::string v = values.value().as(); +/// +/// If the value of the YAML option is not in fact an instance of any of the types listed in the +/// `Types` parameter, validation of the YAML document against the JSON schema produced by the +/// Parameters object will fail. If the value of the YAML option is not an instance of the type +/// indicated in the call to `as()`, an exception will be thrown; the exception message will include +/// the path to the offending YAML node. +/// +/// Note: to declare a `(Required|Optional)Parameter `object holding an `AnyOf<...>` value, include +/// not only the `AnyOf.h` header, but also the `ParameterTraitsAnyOf.h` header. +template +class AnyOf { + public: + /// \brief Construct an AnyOf object holding the value \p initialValue. + template + explicit AnyOf(const T &initialValue) : name_("dummy") { + static_assert(util::any_is_same::value, + "T must be one of the types from the Types list"); + oops::ParameterTraits::set(config_, name_, initialValue); + } + + /// \brief Construct an AnyOf object holding the value of the key \p name from the + /// configuration \p config. + /// + /// \p path should be the location of \p config in the full configuration loaded from a YAML file. + AnyOf(const util::CompositePath &path, + const eckit::Configuration &config, const std::string &name) + : path_(path), config_(config), name_(name) + { + ASSERT(config_.has(name_)); + } + + /// \brief Cast the stored value to type T and return it. + /// + /// An exception is thrown if the cast fails. + template + T as() const { + static_assert(util::any_is_same::value, + "T must be one of the types from the Types list"); + util::CompositePath localPath = path_; + boost::optional result; + try { + // get() is expected to throw an exception if config_ has an option name_, but it cannot be + // converted to type T. + result = oops::ParameterTraits::get(localPath, config_, name_); + } catch (eckit::Exception &) { + util::PathComponent component(localPath, name_); + throw eckit::BadValue(localPath.path() + ": unexpected value type"); + } + ASSERT(result != boost::none); + return std::move(*result); + } + + /// \brief Set the \p name key in the configuration \p config to the stored value. + void serialize(eckit::LocalConfiguration &config, const std::string &name) const { + eckit::LocalConfiguration value; + config_.get(name_, value); // read the value from the source Configuration... + config.set(name, value); // and store it in the destination Configuration. + } + + private: + util::CompositePath path_; + eckit::LocalConfiguration config_; + std::string name_; +}; + +} // namespace util + +#endif // OOPS_UTIL_ANYOF_H_ diff --git a/src/oops/util/TypeTraits.h b/src/oops/util/TypeTraits.h new file mode 100644 index 000000000..176cf4a0a --- /dev/null +++ b/src/oops/util/TypeTraits.h @@ -0,0 +1,30 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_UTIL_TYPETRAITS_H_ +#define OOPS_UTIL_TYPETRAITS_H_ + +#include + +namespace util { + +/// A type trait whose `value` member is set to true if and only if the first type in the list of +/// template parameters (`T`) is the same as any of the following types. +template +struct any_is_same { + static const bool value = std::is_same::value || any_is_same::value; +}; + +/// A type trait whose `value` member is set to true if and only if T is the same type as U. +template +struct any_is_same { + static const bool value = std::is_same::value; +}; + +} // namespace util + +#endif // OOPS_UTIL_TYPETRAITS_H_ diff --git a/src/oops/util/parameters/OptionalParameter.cc b/src/oops/util/parameters/OptionalParameter.cc new file mode 100644 index 000000000..d133ef0d5 --- /dev/null +++ b/src/oops/util/parameters/OptionalParameter.cc @@ -0,0 +1,58 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/util/parameters/OptionalParameter.h" + +#include "eckit/value/Value.h" +#include "oops/util/parameters/ObjectJsonSchema.h" + +namespace oops { + +namespace { + +/// Same as eckit::LocalConfiguration, except that it offers an extra public constructor +/// taking an eckit::Value. +/// +/// This is needed to implement the OptionalParameter::serialize() method; it doesn't seem +/// possible to set the value of a Configuration option to NilValue by calling any public method of +/// LocalConfiguration. +/// +/// This is obviously a hack, hence we declare this class here rather than in a separate header +/// so as not to encourage its wider use. +class LocalConfigurationEx : public eckit::LocalConfiguration { + public: + // This class supports constructors taking the same parameters as the public constructors + // of LocalConfiguration... + explicit LocalConfigurationEx(char separator = '.') : LocalConfiguration(separator) {} + explicit LocalConfigurationEx(eckit::Stream& s) : LocalConfiguration(s) {} + explicit LocalConfigurationEx(const eckit::Configuration& other) : LocalConfiguration(other) {} + LocalConfigurationEx(const Configuration& other, const std::string& path) + : LocalConfiguration(other, path) {} + + // and in addition this constructor, which calls a protected constructor of + // LocalConfiguration. + explicit LocalConfigurationEx(const eckit::Value& value, char separator = '.') + : LocalConfiguration(value, separator) {} +}; + +} // namespace + +void OptionalParameter::deserialize(util::CompositePath &/*path*/, + const eckit::Configuration &config) { + value_ = config.has(name_); +} + +void OptionalParameter::serialize(eckit::LocalConfiguration &config) const { + if (value_) + config.set(name_, LocalConfigurationEx(eckit::Value())); +} + +ObjectJsonSchema OptionalParameter::jsonSchema() const { + return ObjectJsonSchema({{name_, {{"type", "\"null\""}}}}); +} + +} // namespace oops diff --git a/src/oops/util/parameters/OptionalParameter.h b/src/oops/util/parameters/OptionalParameter.h index 260d668dc..7824f12d6 100644 --- a/src/oops/util/parameters/OptionalParameter.h +++ b/src/oops/util/parameters/OptionalParameter.h @@ -102,6 +102,62 @@ ObjectJsonSchema OptionalParameter::jsonSchema() const { return schema; } + +/// \brief Specialisation for `void`. +/// +/// OptionalParameter represents YAML options that don't take any values -- only their +/// presence matters. For example, the presence of the following option +/// +/// is_defined: +/// +/// or equivalently +/// +/// is_defined: null +/// +/// can be detected by declaring the following member variable of a Parameters subclass: +/// +/// oops::OptionalParameter isDefined{"is_defined", this}; +/// +/// After deserialization, `isDefined.value()` will return true if this option is present in the +/// YAML file and false otherwise. +template <> +class OptionalParameter : public ParameterBase { + public: + /// \brief Constructor. + /// + /// \param name + /// Name of the key from which this parameter's value will be loaded when parameters are + /// deserialized from a Configuration object. Similarly, name of the key to which this + /// parameter's value will be saved when parameters are serialized to a Configuration object. + /// \param parent + /// Pointer to the Parameters object representing the collection of options located at + /// the same level of the configuration tree as \p name. A call to deserialize() or serialize() + /// on that object will automatically trigger a call to deserialize() or serialize() on this + /// parameter. + OptionalParameter( + const char *name, Parameters *parent) + : ParameterBase(parent), name_(name), value_(false) + {} + + void deserialize(util::CompositePath &path, const eckit::Configuration &config) override; + + void serialize(eckit::LocalConfiguration &config) const override; + + ObjectJsonSchema jsonSchema() const override; + + /// \brief True if this parameter was present in the deserialized Configuration object, false + /// otherwise. + bool value() const { return value_; } + + /// \brief True if this parameter was present in the deserialized Configuration object, false + /// otherwise. + operator const bool &() const { return value_; } + + private: + std::string name_; + bool value_; +}; + } // namespace oops #endif // OOPS_UTIL_PARAMETERS_OPTIONALPARAMETER_H_ diff --git a/src/oops/util/parameters/ParameterTraits.cc b/src/oops/util/parameters/ParameterTraits.cc new file mode 100644 index 000000000..4f26abe1e --- /dev/null +++ b/src/oops/util/parameters/ParameterTraits.cc @@ -0,0 +1,80 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/util/parameters/ParameterTraits.h" + +#include "eckit/types/Types.h" +#include "eckit/value/Value.h" +#include "oops/util/IntSetParser.h" + +namespace oops { + +namespace { + +/// Same as eckit::LocalConfiguration, except that it offers an extra public constructor +/// taking an eckit::Value. +/// +/// This is needed to implement the ParameterTraits::set() method; it doesn't seem +/// possible to set the value of a Configuration option to NilValue by calling any public method of +/// LocalConfiguration. +/// +/// This is obviously a hack, hence we declare this class here rather than in a separate header +/// so as not to encourage its wider use. +class LocalConfigurationEx : public eckit::LocalConfiguration { + public: + // This class supports constructors taking the same parameters as the public constructors + // of LocalConfiguration... + explicit LocalConfigurationEx(char separator = '.') : LocalConfiguration(separator) {} + explicit LocalConfigurationEx(eckit::Stream& s) : LocalConfiguration(s) {} + explicit LocalConfigurationEx(const eckit::Configuration& other) : LocalConfiguration(other) {} + LocalConfigurationEx(const Configuration& other, const std::string& path) + : LocalConfiguration(other, path) {} + + // and in addition this constructor, which calls a protected constructor of + // LocalConfiguration. + explicit LocalConfigurationEx(const eckit::Value& value, char separator = '.') + : LocalConfiguration(value, separator) {} +}; + +} // namespace + +// Specialization for std::set +boost::optional> ParameterTraits, std::false_type>::get( + util::CompositePath &path, const eckit::Configuration &config, const std::string &name) { + boost::optional> result; + + if (!config.has(name)) + return result; + + const std::string valueAsString = config.getString(name); + try { + result = oops::parseIntSet(valueAsString); + } catch (eckit::Exception &) { + throw eckit::Exception(path.path() + ": '" + valueAsString + + "' isn't a list of comma-separated integers or ranges of integers", + Here()); + } + + return result; +} + +void ParameterTraits, std::false_type>::set( + eckit::LocalConfiguration &config, const std::string &name, const std::set &value) { + std::stringstream stream; + stream << value; + std::string valueAsString = stream.str(); + valueAsString.erase(0, 1); // erase the first character (opening bracket) + valueAsString.erase(valueAsString.size() - 1, 1); // erase the last character (closing bracket) + config.set(name, valueAsString); +} + +ObjectJsonSchema ParameterTraits, std::false_type>::jsonSchema( + const std::string &name) { + return ParameterTraits::jsonSchema(name); +} + +} // namespace oops diff --git a/src/oops/util/parameters/ParameterTraits.h b/src/oops/util/parameters/ParameterTraits.h index c39e9ed08..aaf283a6d 100644 --- a/src/oops/util/parameters/ParameterTraits.h +++ b/src/oops/util/parameters/ParameterTraits.h @@ -440,6 +440,30 @@ struct ParameterTraits, std::false_type> { } }; +/// \brief Specialization for set. +/// +/// This specialization handles conversion of std::set to and from string-valued YAML options +/// having the form of comma-separated lists of single integers or ranges of integers. Examples: +/// +/// option_a: 1 +/// option_b: 10-15 +/// option_c: 8,10, 11-15, 1,3 +/// +/// \warning The current implementation can only handle non-negative integers. +template <> +struct ParameterTraits, std::false_type> { + static boost::optional> get(util::CompositePath &path, + const eckit::Configuration &config, + const std::string &name); + + static void set(eckit::LocalConfiguration &config, + const std::string &name, + const std::set &value); + + static ObjectJsonSchema jsonSchema(const std::string &name); +}; + + // Note: to avoid coupling this file too tightly with headers defining a lot of disparate classes, // only specializations of ParameterTraits for commonly used types should be added here. // Specializations for types used less frequently should be defined in separate files (see diff --git a/src/oops/util/parameters/ParameterTraitsAnyOf.h b/src/oops/util/parameters/ParameterTraitsAnyOf.h new file mode 100644 index 000000000..1df424095 --- /dev/null +++ b/src/oops/util/parameters/ParameterTraitsAnyOf.h @@ -0,0 +1,60 @@ +/* + * (C) Copyright 2021 Met Office UK + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_UTIL_PARAMETERS_PARAMETERTRAITSANYOF_H_ +#define OOPS_UTIL_PARAMETERS_PARAMETERTRAITSANYOF_H_ + +#include +#include +#include + +#include + +#include "eckit/config/LocalConfiguration.h" +#include "oops/util/AnyOf.h" +#include "oops/util/parameters/ParameterTraits.h" +#include "oops/util/stringFunctions.h" // for join() + +namespace oops { + +/// \brief Traits dictating how parameters of type AnyOf<...> are transferred to and from +/// eckit::Configuration objects. + +template +struct ParameterTraits, std::false_type> { + static boost::optional> get(util::CompositePath &path, + const eckit::Configuration &config, + const std::string &name) { + boost::optional> result; + + if (!config.has(name)) + return result; + + result = util::AnyOf(path, config, name); + return result; + } + + static void set(eckit::LocalConfiguration &config, + const std::string &name, + const util::AnyOf &value) { + value.serialize(config, name); + } + + static ObjectJsonSchema jsonSchema(const std::string &name) { + const std::vector typeSchemas{ParameterTraits::jsonSchema("")...}; + std::string anyOfSchema = '[' + + util::stringfunctions::join(", ", typeSchemas.begin(), typeSchemas.end(), + [](const ObjectJsonSchema &s) + {return toString(s.properties().at(""));}) + + ']'; + return ObjectJsonSchema({{name, {{"anyOf", std::move(anyOfSchema)}}}}); + } +}; + +} // namespace oops + +#endif // OOPS_UTIL_PARAMETERS_PARAMETERTRAITSANYOF_H_ diff --git a/src/test/testinput/parameters.yaml b/src/test/testinput/parameters.yaml index 2ffb7da89..32ff0332b 100644 --- a/src/test/testinput/parameters.yaml +++ b/src/test/testinput/parameters.yaml @@ -2,6 +2,7 @@ minimal: req_float_parameter: 3 req_duration_parameter: PT1H full: # These parameters must be sorted alphabetically for the serialization test. + any_of_parameter: dog bool_parameter: false channels: 5,6,7 # used by the parameter storing a Variables object embedded_int_parameter: 13 @@ -9,10 +10,12 @@ full: # These parameters must be sorted alphabetically for the serialization tes fruit_parameter: apple int_parameter: 4 int_parameters: [1, 2] + opt_any_of_parameter: [1, 2, 3, 4] opt_date_time_parameter: 2010-02-03T04:05:06Z opt_duration_parameter: PT1H2M3S opt_embedded_date_time_parameter: 2010-03-04T05:06:07Z opt_float_parameter: 5.5 + opt_null_parameter: null range_parameter: max: 8.5 min: 7 @@ -23,6 +26,7 @@ full: # These parameters must be sorted alphabetically for the serialization tes min: 11 req_duration_parameter: PT6H30M req_float_parameter: 6 + set_int_parameter: 2,4,5,6,8 variables_parameter: [u, v] alternative: float_parameter: 13.5 @@ -45,6 +49,10 @@ alternative: max: 112 embedded_int_parameter: 23 opt_embedded_date_time_parameter: 2010-03-14T05:06:17Z + set_int_parameter: 3 + any_of_parameter: [1, 3, 5] + opt_any_of_parameter: cat + opt_null_parameter: misspelled_float: float_parometer: 3.5 misspelled_int: @@ -101,6 +109,12 @@ error_in_range_parameters: range_parameters: ABCDEF req_float_parameter: 3 req_duration_parameter: PT1H +error_in_set_int_parameter: + set_int_parameter: 1- + req_float_parameter: 3 + req_duration_parameter: PT1H +error_in_any_of_parameter: + any_of_parameter: [abc, def] missing_req_float_parameter: req_duration_parameter: PT1H missing_req_duration_parameter: @@ -150,6 +164,14 @@ map_parameter_json_style_unquoted_keys: duration_or_string_to_duration_map_1: { day: PT14H, night: PT10H } float_or_int_to_float_map_2: 3.5 duration_or_string_to_duration_map_2: PT12H +set_int_single_number: + set_int_parameter: 5 +set_int_range: + set_int_parameter: 3-5 +set_int_multiple_numbers_and_ranges: + set_int_parameter: 13,3-5,7, 10-11 +set_int_invalid_range: + set_int_parameter: 13,3-x variables_without_channels: # The filter variables are not ordered alphabetically: this is by design, to test if their # order is preserved during serialization. diff --git a/src/test/util/Parameters.h b/src/test/util/Parameters.h index 0163dd9ac..9be57c00d 100644 --- a/src/test/util/Parameters.h +++ b/src/test/util/Parameters.h @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -24,6 +25,7 @@ #include "oops/base/ParameterTraitsVariables.h" #include "oops/base/Variables.h" #include "oops/runs/Test.h" +#include "oops/util/AnyOf.h" #include "oops/util/Expect.h" #include "oops/util/Logger.h" #include "oops/util/parameters/ConfigurationParameter.h" @@ -35,6 +37,7 @@ #include "oops/util/parameters/Parameter.h" #include "oops/util/parameters/Parameters.h" #include "oops/util/parameters/ParameterTraits.h" +#include "oops/util/parameters/ParameterTraitsAnyOf.h" #include "oops/util/parameters/ParameterTraitsScalarOrMap.h" #include "oops/util/parameters/PolymorphicParameter.h" #include "oops/util/parameters/RequiredParameter.h" @@ -92,6 +95,8 @@ class EmbeddedParameters : public oops::Parameters { class MyParametersBase : public oops::Parameters { OOPS_ABSTRACT_PARAMETERS(MyParametersBase, Parameters) public: + typedef util::AnyOf> AnyOf_; + oops::Parameter floatParameter{"float_parameter", 1.5f, this}; oops::Parameter intParameter{"int_parameter", 2, this}; oops::Parameter boolParameter{"bool_parameter", true, this}; @@ -103,6 +108,11 @@ class MyParametersBase : public oops::Parameters { oops::Parameter> intParameters{"int_parameters", {}, this}; oops::Parameter> rangeParameters{"range_parameters", {}, this}; oops::Parameter variablesParameter{"variables_parameter", {}, this}; + oops::Parameter> setIntParameter{"set_int_parameter", {}, this}; + oops::Parameter anyOfParameter{ + "any_of_parameter", AnyOf_(std::vector({10, 20})), this}; + oops::OptionalParameter optAnyOfParameter{"opt_any_of_parameter", this}; + oops::OptionalParameter optNullParameter{"opt_null_parameter", this}; EmbeddedParameters embeddedParameters{this}; }; @@ -428,6 +438,10 @@ void testDefaultValues() { EXPECT(params.intParameters.value().empty()); EXPECT(params.rangeParameters.value().empty()); EXPECT(params.variablesParameter.value() == oops::Variables()); + EXPECT(params.setIntParameter.value() == std::set()); + EXPECT(params.anyOfParameter.value().as>() == std::vector({10, 20})); + EXPECT(params.optAnyOfParameter.value() == boost::none); + EXPECT_NOT(params.optNullParameter.value()); EXPECT(params.embeddedParameters.intParameter.value() == 3); EXPECT(params.embeddedParameters.optDateTimeParameter.value() == boost::none); @@ -451,6 +465,10 @@ void testDefaultValues() { EXPECT(params.intParameters.value().empty()); EXPECT(params.rangeParameters.value().empty()); EXPECT(params.variablesParameter.value() == oops::Variables()); + EXPECT(params.setIntParameter.value() == std::set()); + EXPECT(params.anyOfParameter.value().as>() == std::vector({10, 20})); + EXPECT(params.optAnyOfParameter.value() == boost::none); + EXPECT_NOT(params.optNullParameter.value()); EXPECT(params.embeddedParameters.intParameter.value() == 3); EXPECT(params.embeddedParameters.optDateTimeParameter.value() == boost::none); } @@ -485,6 +503,12 @@ void testCorrectValues() { EXPECT(params.rangeParameters.value()[1].maxParameter == 12.0f); EXPECT(params.variablesParameter.value() == oops::Variables({"u", "v"}, std::vector({5, 6, 7}))); + EXPECT(params.setIntParameter.value() == std::set({2, 4, 5, 6, 8})); + EXPECT(params.anyOfParameter.value().as() == "dog"); + EXPECT(params.optAnyOfParameter.value() != boost::none); + EXPECT_EQUAL(params.optAnyOfParameter.value()->as>(), + std::vector({1, 2, 3, 4})); + EXPECT(params.optNullParameter.value()); EXPECT(params.embeddedParameters.intParameter.value() == 13); EXPECT(params.embeddedParameters.optDateTimeParameter.value() != boost::none); EXPECT_EQUAL(params.embeddedParameters.optDateTimeParameter.value().get(), @@ -644,6 +668,39 @@ void testIncorrectValueOfRangeParameters() { EXPECT_THROWS_AS(params.deserialize(conf), eckit::Exception); } +void testIncorrectValueOfAnyOfParameter() { + { + // This YAML section sets any_of_parameter to a value that is neither a string nor a list of + // ints. This should be detected at the validation stage. + MyOptionalParameters params; + const eckit::LocalConfiguration conf(TestEnvironment::config(), "error_in_any_of_parameter"); + if (validationSupported) + EXPECT_THROWS_MSG(params.validate(conf), "no subschema has succeeded"); + } + + { + // This YAML section sets any_of_parameter to a string, but the code attempts to load + // it as a vector of ints. This can be detected only in the call to the as() method. + MyOptionalAndRequiredParameters params; + const eckit::LocalConfiguration conf(TestEnvironment::config(), "full"); + params.validate(conf); + params.deserialize(conf); + EXPECT_THROWS_MSG(params.anyOfParameter.value().as>(), + "unexpected value type"); + } + + { + // This YAML section sets any_of_parameter to a vector of ints, but the code attempts to load + // it as a string. This can be detected only in the call to the as() method. + MyOptionalAndRequiredParameters params; + const eckit::LocalConfiguration conf(TestEnvironment::config(), "alternative"); + params.validate(conf); + params.deserialize(conf); + EXPECT_THROWS_MSG(params.anyOfParameter.value().as(), + "unexpected value type"); + } +} + void testMissingRequiredFloatParameter() { MyOptionalAndRequiredParameters params; const eckit::LocalConfiguration conf(TestEnvironment::config(), @@ -759,6 +816,47 @@ void testMapParametersSerialization() { doTestSerialization(conf); } +// Parameters storing std::set objects + +void testSetIntParameters() { + { + MyOptionalParameters params; + const eckit::LocalConfiguration conf(TestEnvironment::config(), + "set_int_single_number"); + EXPECT_NO_THROW(params.validate(conf)); + params.deserialize(conf); + EXPECT(params.setIntParameter.value() == std::set({5})); + } + + { + MyOptionalParameters params; + const eckit::LocalConfiguration conf(TestEnvironment::config(), + "set_int_range"); + EXPECT_NO_THROW(params.validate(conf)); + params.deserialize(conf); + EXPECT(params.setIntParameter.value() == std::set({3, 4, 5})); + } + + { + MyOptionalParameters params; + const eckit::LocalConfiguration conf(TestEnvironment::config(), + "set_int_multiple_numbers_and_ranges"); + EXPECT_NO_THROW(params.validate(conf)); + params.deserialize(conf); + EXPECT(params.setIntParameter.value() == std::set({3, 4, 5, 7, 10, 11, 13})); + } + + { + MyOptionalParameters params; + const eckit::LocalConfiguration conf(TestEnvironment::config(), + "set_int_invalid_range"); + // Note: this syntax error won't be detected at the validation stage, but only at the + // deserialization stage + EXPECT_THROWS_MSG(params.deserialize(conf), + "isn't a list of comma-separated integers or ranges of integers"); + } +} + // Parameters storing Variables objects void testVariablesDeserializationWithoutChannels() { @@ -840,6 +938,11 @@ void expectMatchesFullConf(const MyOptionalAndRequiredParameters ¶ms) { EXPECT_EQUAL(params.floatParameter, 3.5f); EXPECT_EQUAL(params.rangeParameter.value().minParameter, 7.0f); EXPECT_EQUAL(params.rangeParameters.value()[0].minParameter, 9.0f); + EXPECT_EQUAL(params.setIntParameter.value(), std::set({2, 4, 5, 6, 8})); + EXPECT(params.optNullParameter.value()); + EXPECT(params.optAnyOfParameter.value() != boost::none); + EXPECT_EQUAL(params.optAnyOfParameter.value()->as>(), + std::vector({1, 2, 3, 4})); EXPECT(params.embeddedParameters.intParameter.value() == 13); } @@ -847,6 +950,10 @@ void expectMatchesAlternativeConf(const MyOptionalAndRequiredParameters ¶ms) EXPECT_EQUAL(params.floatParameter, 13.5f); EXPECT_EQUAL(params.rangeParameter.value().minParameter, 17.0f); EXPECT_EQUAL(params.rangeParameters.value()[0].minParameter, 19.0f); + EXPECT_EQUAL(params.setIntParameter.value(), std::set({3})); + EXPECT(params.optNullParameter.value()); + EXPECT(params.optAnyOfParameter.value() != boost::none); + EXPECT_EQUAL(params.optAnyOfParameter.value()->as(), "cat"); EXPECT(params.embeddedParameters.intParameter.value() == 23); } @@ -1469,6 +1576,9 @@ class Parameters : public oops::Test { ts.emplace_back(CASE("util/Parameters/testIncorrectValueOfRangeParameters") { testIncorrectValueOfRangeParameters(); }); + ts.emplace_back(CASE("util/Parameters/testIncorrectValueOfAnyOfParameter") { + testIncorrectValueOfAnyOfParameter(); + }); ts.emplace_back(CASE("util/Parameters/testMissingRequiredFloatParameter") { testMissingRequiredFloatParameter(); }); @@ -1515,6 +1625,10 @@ class Parameters : public oops::Test { testMapParametersSerialization(); }); + ts.emplace_back(CASE("util/Parameters/testSetIntParameters") { + testSetIntParameters(); + }); + ts.emplace_back(CASE("util/Parameters/testVariablesDeserializationWithoutChannels") { testVariablesDeserializationWithoutChannels(); }); diff --git a/src/test/util/TypeTraits.cc b/src/test/util/TypeTraits.cc new file mode 100644 index 000000000..26a74b4ed --- /dev/null +++ b/src/test/util/TypeTraits.cc @@ -0,0 +1,15 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/runs/Run.h" +#include "test/util/TypeTraits.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + test::TypeTraits tests; + return run.execute(tests); +} diff --git a/src/test/util/TypeTraits.h b/src/test/util/TypeTraits.h new file mode 100644 index 000000000..de37cb2b4 --- /dev/null +++ b/src/test/util/TypeTraits.h @@ -0,0 +1,43 @@ +/* + * (C) Crown copyright 2021, Met Office + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef TEST_UTIL_TYPETRAITS_H_ +#define TEST_UTIL_TYPETRAITS_H_ + +#include +#include + +#include "eckit/testing/Test.h" +#include "oops/../test/TestEnvironment.h" +#include "oops/runs/Test.h" +#include "oops/util/TypeTraits.h" + +namespace test { + +CASE("util/TypeTraits/any_is_same") { + // The extra parentheses are necessary because EXPECT is a macro and its argument contains commas. + EXPECT((util::any_is_same::value)); + EXPECT((util::any_is_same, std::vector>::value)); + EXPECT((util::any_is_same>::value)); + EXPECT((util::any_is_same, int, std::vector>::value)); + EXPECT((util::any_is_same, int, std::vector, std::vector>::value)); + EXPECT_NOT((util::any_is_same::value)); + EXPECT_NOT((util::any_is_same::value)); + EXPECT_NOT((util::any_is_same, int, std::vector>::value)); +} + +class TypeTraits : public oops::Test { + private: + std::string testid() const override {return "test::TypeTraits";} + + void register_tests() const override {} + void clear() const override {} +}; + +} // namespace test + +#endif // TEST_UTIL_TYPETRAITS_H_ From d3dce9f3442584ae1c6f88d7f46080d92fbbeb3c Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Thu, 11 Mar 2021 12:27:23 -0700 Subject: [PATCH 069/142] changing compare step (#1081) * add compare to liboops, add a dummy test to l95 for testing the new feature * clean ups * better comments * use rank 0 * run compare only for rank 0 * get ref and output filenames from yaml; use avg in tol calc denominator * clean up * abs for both numerator and denominator; fix coding norm * first try * clean up * apply marks comments * clean up * bugfix and add TestReference to cmakelist * add an option to write test output file through yaml * throw exception class * clean up * changes based on Mark O. PR * remove duplicate addStream * fix coding norm error * remove old 4dvar.obsbias.test and update the reference file to the new format. make it backward compatible * add if rank, address JJ comment --- l95/test/CMakeLists.txt | 12 +- l95/test/testinput/4dvar.obsbias.yaml | 7 + l95/test/testoutput/4dvar.obsbias.test | 44 +++--- src/CMakeLists.txt | 2 + src/oops/runs/Run.cc | 13 +- src/oops/runs/Run.h | 8 +- src/oops/util/LibOOPS.cc | 28 +++- src/oops/util/LibOOPS.h | 16 +++ src/oops/util/TestReference.cc | 185 +++++++++++++++++++++++++ src/oops/util/TestReference.h | 83 +++++++++++ 10 files changed, 361 insertions(+), 37 deletions(-) create mode 100644 src/oops/util/TestReference.cc create mode 100644 src/oops/util/TestReference.h diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index 1d26c6409..ea8bc6fc8 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -630,12 +630,12 @@ oops_add_test( TESTNAME 4dsaddlepoint EXENAME l95_4dvar.x TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -oops_add_test( TESTNAME 4dvar.obsbias - MODELNAME l95 - YAMLNAME testinput/4dvar.obsbias.yaml - EXENAME l95_4dvar.x - COMPARE - TEST_DEPENDS test_l95_forecast test_l95_makeobsbias ) +#-------------------------------------------------------------------- +ecbuild_add_test( TARGET test_l95_4dvar.obsbias + COMMAND l95_4dvar.x + ARGS testinput/4dvar.obsbias.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobsbias ) + #-------------------------------------------------------------------- oops_add_test( TESTNAME 4dvar.allbiases diff --git a/l95/test/testinput/4dvar.obsbias.yaml b/l95/test/testinput/4dvar.obsbias.yaml index 2ab7f9cea..73235faac 100644 --- a/l95/test/testinput/4dvar.obsbias.yaml +++ b/l95/test/testinput/4dvar.obsbias.yaml @@ -74,3 +74,10 @@ output: first: PT3H frequency: PT06H type: an +test: + reference filename: testoutput/4dvar.obsbias.test + # Optional: + float relative tolerance: 0.0 + integer tolerance: 0 + log output filename: testoutput/4dvar.obsbias.log.out + test output filename: testoutput/4dvar.obsbias.test.out diff --git a/l95/test/testoutput/4dvar.obsbias.test b/l95/test/testoutput/4dvar.obsbias.test index 0fd7093e2..7e971b7be 100644 --- a/l95/test/testoutput/4dvar.obsbias.test +++ b/l95/test/testoutput/4dvar.obsbias.test @@ -1,22 +1,22 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 294.698, nobs = 160, Jo/n = 1.84186, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.338917 -Test : CostFunction: Nonlinear J = 295.037 -Test : DRIPCGMinimizer: reduction in residual norm = 8.3185e-05 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67871, Max=8.67067, Average=8.01684 -Test : ObsBias = 0.535611 -Test : CostJb : Nonlinear Jb = 2.30305 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 1.04203, nobs = 160, Jo/n = 0.00651266, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.378713 -Test : CostFunction: Nonlinear J = 3.72378 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0182695 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.6865, Max=8.68197, Average=8.01516 -Test : ObsBias = 0.546819 -Test : CostJb : Nonlinear Jb = 2.38623 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.701554, nobs = 160, Jo/n = 0.00438471, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.366753 -Test : CostFunction: Nonlinear J = 3.45454 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 294.698, nobs = 160, Jo/n = 1.84186, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.338917 +CostFunction: Nonlinear J = 295.037 +DRIPCGMinimizer: reduction in residual norm = 8.3185e-05 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67871, Max=8.67067, Average=8.01684 +ObsBias = 0.535611 +CostJb : Nonlinear Jb = 2.30305 +CostJo : Nonlinear Jo(Lorenz 95) = 1.04203, nobs = 160, Jo/n = 0.00651266, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.378713 +CostFunction: Nonlinear J = 3.72378 +DRIPCGMinimizer: reduction in residual norm = 0.0182695 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.6865, Max=8.68197, Average=8.01516 +ObsBias = 0.546819 +CostJb : Nonlinear Jb = 2.38623 +CostJo : Nonlinear Jo(Lorenz 95) = 0.701554, nobs = 160, Jo/n = 0.00438471, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.366753 +CostFunction: Nonlinear J = 3.45454 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b09969fc8..e46589c71 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -306,6 +306,8 @@ oops/util/string_f_c_mod.F90 oops/util/string_utils.F90 oops/util/stringFunctions.cc oops/util/stringFunctions.h +oops/util/TestReference.cc +oops/util/TestReference.h oops/util/Timer.cc oops/util/Timer.h oops/util/TimerHelper.cc diff --git a/src/oops/runs/Run.cc b/src/oops/runs/Run.cc index 5a04d837d..edbb0ac00 100644 --- a/src/oops/runs/Run.cc +++ b/src/oops/runs/Run.cc @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -92,6 +92,13 @@ Run::Run(int argc, char** argv) : eckit::Main(argc, argv, "OOPS_HOME"), config_( // Read configuration config_.reset(new eckit::YAMLConfiguration(configfile)); + +// Read tolerances for testing + if (config_->has("test")) { + auto testConf = config_->getSubConfiguration("test"); + LibOOPS::instance().testReferenceInitialise(testConf); + } + Log::info() << "Configuration input file is: " << configfile << std::endl; Log::info() << "Full configuration is:" << *config_ << std::endl; diff --git a/src/oops/runs/Run.h b/src/oops/runs/Run.h index 14348f2d2..535cdda56 100644 --- a/src/oops/runs/Run.h +++ b/src/oops/runs/Run.h @@ -1,9 +1,9 @@ -/* +/* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ diff --git a/src/oops/util/LibOOPS.cc b/src/oops/util/LibOOPS.cc index 9e3fdbe36..183b9ab4e 100644 --- a/src/oops/util/LibOOPS.cc +++ b/src/oops/util/LibOOPS.cc @@ -13,10 +13,16 @@ /// @date December 2016 #include +#include #include +#include +#include #include #include +#include +#include "eckit/config/LocalConfiguration.h" +#include "eckit/config/YAMLConfiguration.h" #include "eckit/log/Log.h" #include "eckit/log/OStreamTarget.h" #include "eckit/log/PrefixTarget.h" @@ -25,6 +31,7 @@ #include "oops/mpi/mpi.h" #include "oops/util/LibOOPS.h" #include "oops/util/Logger.h" +#include "oops/util/TestReference.h" #ifdef ENABLE_GPTL #include @@ -68,7 +75,7 @@ LibOOPS& LibOOPS::instance() { } /** Initialization of MPI and dependent variables. - * To be called in `main()` by constructor of `oops::Run`. This method initializes MPI and + * To be called in `main()` by constructor of `oops::Run`. This method initializes MPI and * associated variables that must be initialized after static-init time, and only once `eckit::Main` * has been created. */ @@ -95,6 +102,11 @@ void LibOOPS::initialise() { do_abortfpe = getEnv("OOPS_ABORTFPE", 1); trap_sigfpe(do_abortfpe); } + // testStream_ is used by TestReference for comparing test output + // with a reference file + if ( rank_ == 0 ) { + testChannel().addStream(testStream_); + } enable_timer_channel_ = getEnv("OOPS_TIMER", 0) > 0 && rank_ == 0; #ifdef ENABLE_GPTL @@ -115,6 +127,10 @@ void LibOOPS::teeOutput(const std::string & fileprefix) { eckit::Log::addFile(teefile); } +void LibOOPS::testReferenceInitialise(eckit::LocalConfiguration &testConf) { + testReferenceInstance_.Initialise(testConf); +} + /** Clears logs and finalises MPI (unless \p finaliseMPI is false). * To be called in on leaving `main()` by the destructor of `oops::Run`. */ @@ -134,11 +150,20 @@ void LibOOPS::finalise(bool finaliseMPI) { } } #endif + if ( rank_ == 0 ) { + testReferenceInstance_.testReferenceFinalise(testStream_); + } + + // Make sure that these specialised channels that wrap eckit::Log::info() are + // destroyed before eckit::Log::info gets destroyed. + // Just in case someone still tries to log, we reset to empty channels. + debugChannel_.reset(new eckit::Channel()); traceChannel_.reset(new eckit::Channel()); statsChannel_.reset(new eckit::Channel()); testChannel_.reset(new eckit::Channel()); timerChannel_.reset(new eckit::Channel()); + // Destroy info channel last after other channels have flushed all output infoChannel_.reset(new eckit::Channel()); @@ -221,4 +246,3 @@ eckit::Channel& LibOOPS::debugChannel() const { // ----------------------------------------------------------------------------- } // namespace oops - diff --git a/src/oops/util/LibOOPS.h b/src/oops/util/LibOOPS.h index 9dcf9e82b..bbb2b33fd 100644 --- a/src/oops/util/LibOOPS.h +++ b/src/oops/util/LibOOPS.h @@ -16,10 +16,16 @@ #define OOPS_UTIL_LIBOOPS_H_ #include +#include #include + +#include "eckit/config/YAMLConfiguration.h" #include "eckit/system/Library.h" #include "eckit/utils/Translator.h" +#include "oops/util/TestReference.h" + + namespace oops { // ----------------------------------------------------------------------------- @@ -42,6 +48,9 @@ class LibOOPS : public eckit::system::Library { void initialise(); void teeOutput(const std::string &); + + void testReferenceInitialise(eckit::LocalConfiguration &); + void finalise(bool finaliseMPI = true); protected: @@ -51,6 +60,7 @@ class LibOOPS : public eckit::system::Library { std::string gitsha1(unsigned int count) const override; + mutable std::unique_ptr infoChannel_; mutable std::unique_ptr debugChannel_; @@ -64,6 +74,12 @@ class LibOOPS : public eckit::system::Library { std::string predebug_; bool trace_; std::string pretrace_; + + std::stringstream testStream_; + TestReference testReferenceInstance_; + + private: + friend class TestReference; bool enable_timer_channel_; }; diff --git a/src/oops/util/TestReference.cc b/src/oops/util/TestReference.cc new file mode 100644 index 000000000..90a5e924d --- /dev/null +++ b/src/oops/util/TestReference.cc @@ -0,0 +1,185 @@ +/* + * (C) Copyright 2017-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "eckit/config/YAMLConfiguration.h" +#include "eckit/exception/Exceptions.h" + +#include "oops/util/LibOOPS.h" +#include "oops/util/Logger.h" +#include "oops/util/TestReference.h" + + +namespace oops { + +// ----------------------------------------------------------------------------- + +void TestReference::Initialise(eckit::LocalConfiguration &testConf) +{ + initCheck_ = true; + tolFloat_ = testConf.getFloat("float relative tolerance", 0.0); + tolInt_ = testConf.getInt("integer tolerance", 0); + refFileYaml_ = testConf.getString("reference filename"); + if (testConf.has("log output filename")) { + outputFileYaml_ = testConf.getString("log output filename"); + LibOOPS::instance().teeOutput(outputFileYaml_); + if (testConf.has("test output filename")) { + testFileYaml_ = testConf.getString("test output filename"); + oops::Log::info() << "save test output to " << testFileYaml_ << std::endl; + } + } +} + +// ----------------------------------------------------------------------------- +// Reads reference file (refFileYaml_) for tests with comparing step +// refFileYaml_ is read from Yaml using setYaml method +std::ifstream TestReference::readRefFile() +{ + std::ifstream refFile(refFileYaml_); + if (refFile.fail()) { + throw eckit::CantOpenFile(refFileName_ + " does not exist"); + } + return refFile; +} +// ----------------------------------------------------------------------------- +void TestReference::testCompare(std::stringstream & test, + std::ifstream & ref, + float tolFloat, int tolInt) +{ + std::regex regNumber("[-+]?[0-9]*[.]?[0-9]+([eE][-+]?[0-9]+)?"); + std::regex regOnlyFloat("[-+]?[0-9]*[.][0-9]+([eE][-+]?[0-9]+)?"); + int lineCounter = 0; + std::string refLine, testLine; + while (std::getline(ref, refLine)) { + std::getline(test, testLine); + std::string testPrefix = "Test : "; + + // if test prefix is in reference file add it to the testLine + // to match the format + // This can be removed once all reference files are updated and + // test prefix is removed from them. + if (!refLine.find(testPrefix)) { + testLine = testPrefix+testLine; + } + + lineCounter++; + int compareRes = refLine.compare(testLine); + + // if strings don't match investigate more + if (compareRes != 0) { + // find all numbers in the two test and reference strings + std::vector numMatch_refLine( + std::sregex_token_iterator(refLine.begin(), refLine.end(), regNumber), + std::sregex_token_iterator()); + std::vector numMatch_testLine( + std::sregex_token_iterator(testLine.begin(), testLine.end(), regNumber), + std::sregex_token_iterator()); + + // How many numbers are in each string and do they match? + if (numMatch_refLine.size() != numMatch_testLine.size()) { + throw TestReferenceTextMismatchError(lineCounter, testLine, refLine); + } else { + for (auto i = 0; i < numMatch_refLine.size(); i++) { + std::smatch matchCheck; + // separate float and int + std::regex_search(numMatch_refLine[i], matchCheck, regOnlyFloat); + + if (matchCheck.empty()) { + // number is int + int ref_int = stoi(numMatch_refLine[i]); + int test_int = stoi(numMatch_testLine[i]); + int abs_dif = std::abs(ref_int - test_int); + if (abs_dif > tolInt) { + throw TestReferenceIntegerMismatchError(lineCounter, test_int, + ref_int, tolInt, testLine, refLine); + } + } else { + // number is float + double ref_float = stod(numMatch_refLine[i]); + double test_float = stod(numMatch_testLine[i]); + double rel_dir = std::abs((ref_float - test_float)/(0.5 * (ref_float + test_float))); + if (rel_dir > tolFloat) { + throw TestReferenceFloatMismatchError(lineCounter, test_float, + ref_float, tolFloat, testLine, refLine); + } + } + } + } + } + } +} +// ----------------------------------------------------------------------------- +void TestReference::testReferenceFinalise(std::stringstream & testStream_) +{ + if (!initCheck_) { + return; + } + auto refStream = readRefFile(); + if (!testFileYaml_.empty()) { + std::ofstream testFileOut(testFileYaml_); + testFileOut << testStream_.str(); + testFileOut.close(); + } + testCompare(testStream_, refStream, tolFloat_, tolInt_); +} + +// ----------------------------------------------------------------------------- + +TestReferenceTextMismatchError::TestReferenceTextMismatchError(int line_num, + const std::string &test_line, const std::string &ref_line) +{ + std::ostringstream os; + os << "Test reference Text mismatch @ Line:" << line_num + << "\nTest: '" << test_line << "'\nRef: '" << ref_line << "'"; + what_ = os.str(); +} + +// ----------------------------------------------------------------------------- +TestReferenceIntegerMismatchError::TestReferenceIntegerMismatchError(int line_num, + NumT test_val, NumT ref_val, NumT tolerance, + const std::string &test_line, + const std::string &ref_line) +{ + std::ostringstream os; + os << "Test reference Integer mismatch @ Line:" << line_num << "\n" + << "Test Val : " << test_val << "\n" + << "Ref Val : " << ref_val << "\n" + << "Delta : " << std::abs(test_val-ref_val) << "\n" + << "Tolerance: " << tolerance << "\n" + << "Test Line: '" << test_line << "'\n" + << "Ref Line : '" << ref_line << "'"; + what_ = os.str(); +} +// ----------------------------------------------------------------------------- +TestReferenceFloatMismatchError::TestReferenceFloatMismatchError(int line_num, + NumT test_val, NumT ref_val, NumT tolerance, + const std::string &test_line, const std::string &ref_line) +{ + std::ostringstream os; + os << "Test reference Float mismatch @ Line:" << line_num <<"\n" + << std::setprecision(std::numeric_limits::digits10 + 1) + << std::scientific + << "Test Val : " << test_val << "\n" + << "Ref Val : " << ref_val << "\n" + << "Delta : " << std::abs(test_val-ref_val) << "\n" + << "Tolerance: " << tolerance << "\n" + << "Test Line: '" << test_line << "'\n" + << "Ref Line : '" << ref_line << "'"; + what_ = os.str(); +} + + +} // namespace oops diff --git a/src/oops/util/TestReference.h b/src/oops/util/TestReference.h new file mode 100644 index 000000000..ae1b29dfe --- /dev/null +++ b/src/oops/util/TestReference.h @@ -0,0 +1,83 @@ +/* + * (C) Copyright 2017-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_UTIL_TESTREFERENCE_H_ +#define OOPS_UTIL_TESTREFERENCE_H_ + +#include +#include +#include +#include + +#include "eckit/config/LocalConfiguration.h" + + +namespace oops { + +// ----------------------------------------------------------------------------- +class TestReference { + public: + TestReference() = default; // default constructor + + void testReferenceFinalise(std::stringstream &); + void Initialise(eckit::LocalConfiguration &); + + private: + std::ifstream readRefFile(); + void testCompare(std::stringstream &, std::ifstream &, float, int); + bool initCheck_ = false; + + std::string refFileName_; + std::string outputFileYaml_; + std::string refFileYaml_; + std::string testFileYaml_; + double tolFloat_; + int tolInt_; +}; +// ----------------------------------------------------------------------------- +// Base class for all TestReference errors +class TestReferenceError : public std::exception { + public: + const char* what() const noexcept override + { + return what_.c_str(); + } + + protected: + std::string what_; // error message to print +}; + +// Textual mismatch error condition +class TestReferenceTextMismatchError : public TestReferenceError +{ + public: + TestReferenceTextMismatchError(int line_num, + const std::string &test_line, const std::string &ref_line); +}; + +// Integer mismatch error condition +class TestReferenceIntegerMismatchError : public TestReferenceError { + public: + using NumT = int64_t; + TestReferenceIntegerMismatchError(int line_num, + NumT test_val, NumT ref_val, NumT tolerance, + const std::string &test_line, const std::string &ref_line); +}; + +// Floating-point mismatch error condition +class TestReferenceFloatMismatchError : public TestReferenceError { + public: + using NumT = double; + TestReferenceFloatMismatchError(int line_num, + NumT test_val, NumT ref_val, NumT tolerance, + const std::string &test_line, const std::string &ref_line); +}; + + +} // namespace oops + +#endif // OOPS_UTIL_TESTREFERENCE_H_ From f20899cd6439064bf76637ece7ce575aff43e6ea Mon Sep 17 00:00:00 2001 From: JJ Guerrette Date: Thu, 11 Mar 2021 13:59:40 -0700 Subject: [PATCH 070/142] Add explicit reset of uniqe_ptr's for static test fixture instances (#1093) Co-authored-by: Anna Shlyaeva --- src/test/interface/ErrorCovariance.h | 9 ++++++++- src/test/interface/GetValues.h | 12 +++++++++++- src/test/interface/Increment.h | 8 +++++++- src/test/interface/LinearGetValues.h | 16 +++++++++++++++- src/test/interface/LinearVariableChange.h | 7 ++++++- src/test/interface/State.h | 7 ++++++- src/test/interface/VariableChange.h | 6 ++++-- 7 files changed, 57 insertions(+), 8 deletions(-) diff --git a/src/test/interface/ErrorCovariance.h b/src/test/interface/ErrorCovariance.h index ab7f87a2d..d524e7b60 100644 --- a/src/test/interface/ErrorCovariance.h +++ b/src/test/interface/ErrorCovariance.h @@ -51,6 +51,13 @@ template class ErrorCovarianceFixture : private boost::noncopya static const oops::Variables & ctlvars() {return *getInstance().ctlvars_;} static const util::DateTime & time() {return *getInstance().time_;} static const Covariance_ & covariance() {return *getInstance().B_;} + static void reset() { + getInstance().B_.reset(); + getInstance().time_.reset(); + getInstance().ctlvars_.reset(); + getInstance().resol_.reset(); + getInstance().test_.reset(); + } private: static ErrorCovarianceFixture& getInstance() { @@ -212,7 +219,7 @@ template class ErrorCovariance : public oops::Test { public: ErrorCovariance() {} - virtual ~ErrorCovariance() {} + virtual ~ErrorCovariance() {ErrorCovarianceFixture::reset();} private: std::string testid() const override {return "test::ErrorCovariance<" + MODEL::name() + ">";} diff --git a/src/test/interface/GetValues.h b/src/test/interface/GetValues.h index c856c0ee1..4b776132d 100644 --- a/src/test/interface/GetValues.h +++ b/src/test/interface/GetValues.h @@ -60,6 +60,16 @@ template class GetValuesFixture : private boost:: static const LocalConfig_ & testconf() {return *getInstance().testconf_;} static const Locations_ & locs() {return *getInstance().locs_;} static const Variables_ & geovalvars() {return *getInstance().geovalvars_;} + static void reset() { + getInstance().getvalues_.reset(); + getInstance().geovals_.reset(); + getInstance().timeend_.reset(); + getInstance().timebeg_.reset(); + getInstance().locs_.reset(); + getInstance().geovalvars_.reset(); + getInstance().resol_.reset(); + getInstance().testconf_.reset(); + } private: static GetValuesFixture& getInstance() { @@ -232,7 +242,7 @@ template class GetValues : public oops::Test { public: GetValues() {} - virtual ~GetValues() {} + virtual ~GetValues() {GetValuesFixture::reset();} private: std::string testid() const override {return "test::GetValues<" + MODEL::name() + diff --git a/src/test/interface/Increment.h b/src/test/interface/Increment.h index 149255c63..f61689207 100644 --- a/src/test/interface/Increment.h +++ b/src/test/interface/Increment.h @@ -49,6 +49,12 @@ template class IncrementFixture : private boost::noncopyable { static const util::DateTime & time() {return *getInstance().time_;} static const double & tolerance() {return getInstance().tolerance_;} static const eckit::Configuration & test() {return *getInstance().test_;} + static void reset() { + getInstance().time_.reset(); + getInstance().ctlvars_.reset(); + getInstance().test_.reset(); + getInstance().resol_.reset(); + } private: static IncrementFixture& getInstance() { @@ -445,7 +451,7 @@ template class Increment : public oops::Test { public: Increment() {} - virtual ~Increment() {} + virtual ~Increment() {IncrementFixture::reset();} private: std::string testid() const override {return "test::Increment<" + MODEL::name() + ">";} diff --git a/src/test/interface/LinearGetValues.h b/src/test/interface/LinearGetValues.h index 463f66949..8e7fbf60a 100644 --- a/src/test/interface/LinearGetValues.h +++ b/src/test/interface/LinearGetValues.h @@ -67,6 +67,20 @@ template class LinearGetValuesFixture : private b static const State_ & state() {return *getInstance().state_;} static const Variables_ & statevars() {return *getInstance().statevars_;} static const Variables_ & geovalvars() {return *getInstance().geovalvars_;} + static void reset() { + getInstance().lineargetvalues_.reset(); + getInstance().time_.reset(); + getInstance().statevars_.reset(); + getInstance().state_.reset(); + getInstance().getvalues_.reset(); + getInstance().geovals_.reset(); + getInstance().timeend_.reset(); + getInstance().timebeg_.reset(); + getInstance().locs_.reset(); + getInstance().geovalvars_.reset(); + getInstance().resol_.reset(); + getInstance().testconf_.reset(); + } private: static LinearGetValuesFixture& getInstance() { @@ -321,7 +335,7 @@ template class LinearGetValues : public oops::Test { public: LinearGetValues() {} - virtual ~LinearGetValues() {} + virtual ~LinearGetValues() {LinearGetValuesFixture::reset();} private: std::string testid() const override {return "test::LinearGetValues<" + MODEL::name() + ", " diff --git a/src/test/interface/LinearVariableChange.h b/src/test/interface/LinearVariableChange.h index 1da4347bb..313173dfb 100644 --- a/src/test/interface/LinearVariableChange.h +++ b/src/test/interface/LinearVariableChange.h @@ -51,6 +51,11 @@ template class LinearVariableChangeFixture : private boost::non static const State_ & xx() {return *getInstance().xx_;} static const Geometry_ & resol() {return *getInstance().resol_;} static const DateTime_ & time() {return *getInstance().time_;} + static void reset() { + getInstance().time_.reset(); + getInstance().xx_.reset(); + getInstance().resol_.reset(); + } private: static LinearVariableChangeFixture& getInstance() { @@ -280,7 +285,7 @@ template class LinearVariableChange : public oops::Test { public: LinearVariableChange() {} - virtual ~LinearVariableChange() {} + virtual ~LinearVariableChange() {LinearVariableChangeFixture::reset();} private: std::string testid() const override {return "test::LinearVariableChange<" + MODEL::name() + ">";} diff --git a/src/test/interface/State.h b/src/test/interface/State.h index 7a637c3e5..7c4c0a6e8 100644 --- a/src/test/interface/State.h +++ b/src/test/interface/State.h @@ -44,6 +44,10 @@ template class StateFixture : private boost::noncopyable { public: static const eckit::Configuration & test() {return *getInstance().test_;} static const Geometry_ & resol() {return *getInstance().resol_;} + static void reset() { + getInstance().resol_.reset(); + getInstance().test_.reset(); + } private: static StateFixture& getInstance() { @@ -179,6 +183,7 @@ template void testStateAnalyticInitialCondition() { const eckit::LocalConfiguration confgen(Test_::test(), "state generate"); const State_ xx(Test_::resol(), confgen); + const double norm = Test_::test().getDouble("norm generated state"); const double tol = Test_::test().getDouble("tolerance"); @@ -329,7 +334,7 @@ template class State : public oops::Test { public: State() {} - virtual ~State() {} + virtual ~State() {StateFixture::reset();} private: std::string testid() const override {return "test::State<" + MODEL::name() + ">";} diff --git a/src/test/interface/VariableChange.h b/src/test/interface/VariableChange.h index 8125b93fe..44cb4a705 100644 --- a/src/test/interface/VariableChange.h +++ b/src/test/interface/VariableChange.h @@ -40,8 +40,10 @@ template class VariableChangeFixture : private boost::noncopyab public: static std::vector & confs() {return getInstance().confs_;} - static const State_ & xx() {return *getInstance().xx_;} static const Geometry_ & resol() {return *getInstance().resol_;} + static void reset() { + getInstance().resol_.reset(); + } private: static VariableChangeFixture& getInstance() { @@ -160,7 +162,7 @@ template void testVariableChangeFactoryGetMakerNames() { template class VariableChange : public oops::Test { public: VariableChange() {} - virtual ~VariableChange() {} + virtual ~VariableChange() {VariableChangeFixture::reset();} private: std::string testid() const override {return "test::VariableChange<" + MODEL::name() + ">";} From e14c8bebbaa30a181283b897e00911c27dcbb738 Mon Sep 17 00:00:00 2001 From: frolovsa <55715838+frolovsa@users.noreply.github.com> Date: Fri, 12 Mar 2021 15:40:02 -0700 Subject: [PATCH 071/142] Output mean increment; posterior and prior distribution in LocalVolumeSolver (#1084) * added changes to code, yaml, and test output * coding norms * added option to save post mean and ens * fixed a bug * add prior mean output * addressed PR comments --- l95/test/testinput/letkf.yaml | 30 ++++++++ l95/test/testoutput/letkf.test | 9 +++ src/oops/runs/LocalEnsembleDA.h | 121 +++++++++++++++++++++++++------- 3 files changed, 136 insertions(+), 24 deletions(-) diff --git a/l95/test/testinput/letkf.yaml b/l95/test/testinput/letkf.yaml index 693a9717b..51ed5ec16 100644 --- a/l95/test/testinput/letkf.yaml +++ b/l95/test/testinput/letkf.yaml @@ -30,6 +30,11 @@ observations: obs operator: {} driver: + save prior mean: true + save posterior mean: true + save posterior mean increment: true + save prior variance: true + save posterior variance: true local ensemble DA: solver: LETKF @@ -43,3 +48,28 @@ output: date: *date exp: letkf.%{member}% type: an + +output increment: + datadir: Data + date: *date + exp: increment.%{member}% + type: an + +output mean prior: + datadir: Data + date: *date + exp: xbmean.%{member}% + type: an + +output variance prior: + datadir: Data + date: *date + exp: xbvar.%{member}% + type: an + +output variance posterior: + datadir: Data + date: *date + exp: xavar.%{member}% + type: an + diff --git a/l95/test/testoutput/letkf.test b/l95/test/testoutput/letkf.test index ecf5c9560..4c8324c99 100644 --- a/l95/test/testoutput/letkf.test +++ b/l95/test/testoutput/letkf.test @@ -33,6 +33,15 @@ Test : Min=5.25199, Max=9.50208, Average=7.76569 Test : Analysis mean : Test : Valid time: 2010-01-02T00:00:00Z Test : Min=6.73044, Max=9.19002, Average=7.98832 +Test : Analysis mean increment : +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=-1.55316, Max=2.41362, Average=0.222628 +Test : Forecast variance : +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=0.180053, Max=5.95179, Average=2.16311 +Test : Analysis variance : +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=0.216453, Max=8.21437, Average=2.10118 Test : H(x) for member 1: Test : Lorenz 95 nobs= 120 Min=6.31712, Max=10.1291, Average=8.09746 Test : H(x) for member 2: diff --git a/src/oops/runs/LocalEnsembleDA.h b/src/oops/runs/LocalEnsembleDA.h index 44c7c3633..7b9f0c052 100644 --- a/src/oops/runs/LocalEnsembleDA.h +++ b/src/oops/runs/LocalEnsembleDA.h @@ -24,6 +24,7 @@ #include "oops/generic/instantiateObsErrorFactory.h" #include "oops/interface/Geometry.h" #include "oops/interface/GeometryIterator.h" +#include "oops/interface/Increment.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/DateTime.h" @@ -39,6 +40,7 @@ template class LocalEnsembleDA : public Applicati typedef Geometry Geometry_; typedef GeometryIterator GeometryIterator_; typedef IncrementEnsemble4D IncrementEnsemble4D_; + typedef Increment Increment_; typedef LocalEnsembleSolver LocalSolver_; typedef ObsSpaces ObsSpaces_; typedef Observations Observations_; @@ -79,6 +81,8 @@ template class LocalEnsembleDA : public Applicati // Get background configurations const eckit::LocalConfiguration bgConfig(fullConfig, "background"); + // Get driver configuration + const eckit::LocalConfiguration driverConfig(fullConfig, "driver"); // Read all ensemble members StateEnsemble4D_ ens_xx(geometry, bgConfig); @@ -88,13 +92,13 @@ template class LocalEnsembleDA : public Applicati // set up solver std::unique_ptr solver = LocalEnsembleSolverFactory::create(obsdb, geometry, fullConfig, nens); - const eckit::LocalConfiguration driverConfig(fullConfig, "driver"); - - for (size_t jj = 0; jj < nens; ++jj) { - // TODO(Travis) change the way input file name is specified, make - // more similar to how the output ens config is done - Log::test() << "Initial state for member " << jj+1 << ":" << ens_xx[jj] << std::endl; + // test prints for the prior ensemble + bool do_test_prints = driverConfig.getBool("do test prints", true); + if (do_test_prints) { + for (size_t jj = 0; jj < nens; ++jj) { + Log::test() << "Initial state for member " << jj+1 << ":" << ens_xx[jj] << std::endl; + } } // compute H(x) @@ -112,11 +116,12 @@ template class LocalEnsembleDA : public Applicati // calculate background mean State4D_ bkg_mean = ens_xx.mean(); - Log::test() << "Background mean :" << bkg_mean << std::endl; + if (do_test_prints) { + Log::test() << "Background mean :" << bkg_mean << std::endl; + } // calculate background ensemble perturbations IncrementEnsemble4D_ bkg_pert(ens_xx, bkg_mean, statevars); - // TODO(Travis) optionally save the background mean / standard deviation // initialize empty analysis perturbations IncrementEnsemble4D_ ana_pert(geometry, statevars, ens_xx[0].validTimes(), bkg_pert.size()); @@ -134,29 +139,77 @@ template class LocalEnsembleDA : public Applicati ens_xx[jj] += ana_pert[jj]; } - // TODO(Travis) optionally save analysis standard deviation - - // save the analysis mean + // save the posterior mean and ensemble first (since they are needed for the next cycle) + // save the posterior mean State4D_ ana_mean = ens_xx.mean(); // calculate analysis mean - Log::test() << "Analysis mean :" << ana_mean << std::endl; - eckit::LocalConfiguration outConfig(fullConfig, "output"); - outConfig.set("member", 0); - ana_mean.write(outConfig); - - // save the analysis ensemble - size_t mymember; - for (size_t jj=0; jj < nens; ++jj) { - mymember = jj+1; + if (do_test_prints) { + Log::test() << "Analysis mean :" << ana_mean << std::endl; + } + bool save_xamean = driverConfig.getBool("save posterior mean", false); + if (save_xamean) { eckit::LocalConfiguration outConfig(fullConfig, "output"); - outConfig.set("member", mymember); - ens_xx[jj].write(outConfig); + outConfig.set("member", 0); + ana_mean.write(outConfig); + } + + // save the posterior ensemble + bool save_ens = driverConfig.getBool("save posterior ensemble", true); + if (save_ens) { + size_t mymember; + for (size_t jj=0; jj < nens; ++jj) { + mymember = jj+1; + eckit::LocalConfiguration outConfig(fullConfig, "output"); + outConfig.set("member", mymember); + ens_xx[jj].write(outConfig); + } + } + + // below is the diagnostic output ----------------------------- + // save the background mean + bool save_xbmean = driverConfig.getBool("save prior mean", false); + if (save_xbmean) { + eckit::LocalConfiguration outConfig(fullConfig, "output mean prior"); + outConfig.set("member", 0); + bkg_mean.write(outConfig); + } + + // save the analysis increment + bool save_mean_increment = driverConfig.getBool("save posterior mean increment", false); + if (save_mean_increment) { + eckit::LocalConfiguration outConfig(fullConfig, "output increment"); + outConfig.set("member", 0); + for (size_t itime = 0; itime < ana_mean.size(); ++itime) { + Increment_ ana_increment(ana_pert[0][itime], false); + ana_increment.diff(ana_mean[itime], bkg_mean[itime]); + ana_increment.write(outConfig); + if (do_test_prints) { + Log::test() << "Analysis mean increment :" << ana_increment << std::endl; + } + } + } + + // save the prior variance + bool save_variance = driverConfig.getBool("save prior variance", false); + if (save_variance) { + eckit::LocalConfiguration outConfig(fullConfig, "output variance prior"); + outConfig.set("member", 0); + std::string strOut("Forecast variance :"); + saveVariance(outConfig, bkg_pert, do_test_prints, strOut); + } + + // save the posterior variance + save_variance = driverConfig.getBool("save posterior variance", false); + if (save_variance) { + eckit::LocalConfiguration outConfig(fullConfig, "output variance posterior"); + outConfig.set("member", 0); + std::string strOut("Analysis variance :"); + saveVariance(outConfig, ana_pert, do_test_prints, strOut); } // posterior observer // note: if H(X) is read from file, it might have used different time slots for observation - // then LETKF background/analysis perturbations. + // than LETKF background/analysis perturbations. // hence one might not expect that oman and omaf are comparable - // TODO(#926) make explicit separation of background and forecast states in yaml config bool do_posterior_observer = driverConfig.getBool("do posterior observer", true); if (do_posterior_observer) { Observations_ ya_mean = solver->computeHofX(ens_xx, 1, false); @@ -182,6 +235,26 @@ template class LocalEnsembleDA : public Applicati return "oops::LocalEnsembleDA<" + MODEL::name() + ", " + OBS::name() + ">"; } + void saveVariance(const eckit::LocalConfiguration & outConfig, const IncrementEnsemble4D_ & perts, + const bool do_test_prints, const std::string & strOut) const { + // save and optionaly print varaince of an IncrementEnsemble4D_ object + size_t nens = perts.size(); + const double nc = 1.0/(static_cast(nens) - 1.0); + for (size_t itime = 0; itime < perts[0].size(); ++itime) { + Increment_ var(perts[0][itime], false); + for (size_t iens = 0; iens < nens; ++iens) { + Increment_ tmp(perts[iens][itime], true); + tmp.schur_product_with(perts[iens][itime]); + var += tmp; + } + var *= nc; + var.write(outConfig); + if (do_test_prints) { + Log::test() << strOut << var << std::endl; + } + } + } + // ----------------------------------------------------------------------------- }; From 5f36bb5c1b36bad889a06e3f362b49763798e4ac Mon Sep 17 00:00:00 2001 From: Clementine Gas <43183478+cmgas@users.noreply.github.com> Date: Mon, 15 Mar 2021 07:54:08 -0600 Subject: [PATCH 072/142] Add possibility to monitor observations (entire obsspace) (#1088) * Add possibility to monitor observations (no assimilation) and add one test in 3dfgat and change ref file * Remove useless lines in yaml file * Try to not add the Jo terms of monitoring only in NonLinearJ * Monitoring info in the interface/obsspace * Update ref file and add default value for obsspace * Remove passive info from obsspace * Pass the correct configuration Co-authored-by: Anna Shlyaeva --- qg/test/testinput/3dfgat.yaml | 3 ++- qg/test/testoutput/3dfgat.test | 18 +++++++++--------- src/oops/assimilation/CostJo.h | 12 +++++++++--- src/oops/base/ObserversTLAD.h | 25 ++++++++++++++----------- 4 files changed, 34 insertions(+), 24 deletions(-) diff --git a/qg/test/testinput/3dfgat.yaml b/qg/test/testinput/3dfgat.yaml index ec100fe7a..47e96d78f 100644 --- a/qg/test/testinput/3dfgat.yaml +++ b/qg/test/testinput/3dfgat.yaml @@ -20,7 +20,8 @@ cost function: standard_deviation: 1.8e7 vertical_length_scale: 15000.0 observations: - - obs error: + - monitoring only: true + obs error: covariance model: diagonal obs operator: obs type: Stream diff --git a/qg/test/testoutput/3dfgat.test b/qg/test/testoutput/3dfgat.test index c7e388457..6af266123 100644 --- a/qg/test/testoutput/3dfgat.test +++ b/qg/test/testoutput/3dfgat.test @@ -1,19 +1,19 @@ Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 7671.34, nobs = 600, Jo/n = 12.7856, err = 4e+06 +Test : Monitoring only: Nonlinear Jo(Stream) = 7671.34, nobs = 600, Jo/n = 12.7856, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 1038.28, nobs = 600, Jo/n = 1.73047, err = 6 Test : CostJo : Nonlinear Jo(WSpeed) = 139.319, nobs = 300, Jo/n = 0.464395, err = 12 -Test : CostFunction: Nonlinear J = 8848.94 -Test : DRIPCGMinimizer: reduction in residual norm = 0.169464 +Test : CostFunction: Nonlinear J = 1177.6 +Test : DRIPCGMinimizer: reduction in residual norm = 0.186377 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction Test : Boundary conditions are activated -Test : Min= -4.7463e+08, Max= 1.0200e+08, RMS= 1.7921e+08 +Test : Min= -4.6444e+08, Max= 1.0829e+08, RMS= 1.7939e+08 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 77.41 -Test : CostJo : Nonlinear Jo(Stream) = 597.5, nobs = 600, Jo/n = 0.9958, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 330.6, nobs = 600, Jo/n = 0.551, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 43.57, nobs = 300, Jo/n = 0.1452, err = 12 -Test : CostFunction: Nonlinear J = 1049 +Test : CostJb : Nonlinear Jb = 79.13 +Test : Monitoring only: Nonlinear Jo(Stream) = 1596, nobs = 600, Jo/n = 2.66, err = 4e+06 +Test : CostJo : Nonlinear Jo(Wind) = 159.4, nobs = 600, Jo/n = 0.2656, err = 6 +Test : CostJo : Nonlinear Jo(WSpeed) = 34.27, nobs = 300, Jo/n = 0.1142, err = 12 +Test : CostFunction: Nonlinear J = 272.8 diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index d314981f9..5cc141369 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -343,21 +343,27 @@ template double CostJo::printJo(const Departures_ & dy, const Departures_ & grad) const { Log::trace() << "CostJo::printJo start" << std::endl; obspace_.printJo(dy, grad); + std::vector typeconfs = obsconf_.getSubConfigurations(); double zjo = 0.0; for (std::size_t jj = 0; jj < dy.size(); ++jj) { const double zz = 0.5 * dot_product(dy[jj], grad[jj]); const unsigned nobs = grad[jj].nobs(); - if (nobs > 0) { + bool isPassive = typeconfs[jj].getBool("monitoring only", false); + if (nobs > 0 && !isPassive) { Log::test() << "CostJo : Nonlinear Jo(" << obspace_[jj].obsname() << ") = " << zz << ", nobs = " << nobs << ", Jo/n = " << zz/nobs << ", err = " << (*Rmat_)[jj].getRMSE() << std::endl; - } else { + } else if (nobs <= 0 && !isPassive) { Log::test() << "CostJo : Nonlinear Jo(" << obspace_[jj].obsname() << ") = " << zz << " --- No Observations" << std::endl; Log::warning() << "CostJo: No Observations!!!" << std::endl; + } else if (nobs > 0 && isPassive) { + Log::test() << "Monitoring only: Nonlinear Jo(" << obspace_[jj].obsname() << ") = " + << zz << ", nobs = " << nobs << ", Jo/n = " << zz/nobs + << ", err = " << (*Rmat_)[jj].getRMSE() << std::endl; } - zjo += zz; + if (!isPassive) zjo += zz; } Log::trace() << "CostJo::printJo done" << std::endl; diff --git a/src/oops/base/ObserversTLAD.h b/src/oops/base/ObserversTLAD.h index 3d7cd9251..45f338c8f 100644 --- a/src/oops/base/ObserversTLAD.h +++ b/src/oops/base/ObserversTLAD.h @@ -100,8 +100,9 @@ ObserversTLAD::ObserversTLAD(const eckit::Configuration & obsConfig, // setup observers std::vector typeconf = obsConfig.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsdb.size(); ++jobs) { - std::shared_ptr tmp(new ObserverTLAD_(typeconf[jobs], - obsdb[jobs], ybias[jobs])); + std::shared_ptr tmp; + bool passive = typeconf[jobs].getBool("monitoring only", false); + if (!passive) tmp.reset(new ObserverTLAD_(typeconf[jobs], obsdb[jobs], ybias[jobs])); observerstlad_.push_back(tmp); } Log::trace() << "ObserversTLAD::ObserversTLAD" << std::endl; @@ -117,7 +118,7 @@ void ObserversTLAD::doInitializeTraj(const State_ & xx, hslottraj_ = tstep/2; for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - observerstlad_[jj]->doInitializeTraj(xx, winbgn_, winend_); + if (observerstlad_[jj]) observerstlad_[jj]->doInitializeTraj(xx, winbgn_, winend_); } Log::trace() << "ObserversTLAD::doInitializeTraj done" << std::endl; } @@ -129,7 +130,7 @@ void ObserversTLAD::doProcessingTraj(const State_ & xx) { util::DateTime t2 = std::min(xx.validTime()+hslottraj_, winend_); for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - observerstlad_[jj]->doProcessingTraj(xx, t1, t2); + if (observerstlad_[jj]) observerstlad_[jj]->doProcessingTraj(xx, t1, t2); } Log::trace() << "ObserversTLAD::doProcessingTraj done" << std::endl; } @@ -138,7 +139,7 @@ template void ObserversTLAD::doFinalizeTraj(const State_ & xx) { Log::trace() << "ObserversTLAD::doFinalizeTraj start" << std::endl; for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - observerstlad_[jj]->doFinalizeTraj(xx); + if (observerstlad_[jj]) observerstlad_[jj]->doFinalizeTraj(xx); } Log::trace() << "ObserversTLAD::doFinalizeTraj done" << std::endl; } @@ -158,7 +159,7 @@ void ObserversTLAD::doInitializeTL(const Increment_ & dx, Log::trace() << "ObserversTLAD::doInitializeTL start" << std::endl; hslot_ = tstep/2; for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - observerstlad_[jj]->doInitializeTL(dx, winbgn_, winend_); + if (observerstlad_[jj]) observerstlad_[jj]->doInitializeTL(dx, winbgn_, winend_); } Log::trace() << "ObserversTLAD::doInitializeTL done" << std::endl; } @@ -170,7 +171,7 @@ void ObserversTLAD::doProcessingTL(const Increment_ & dx) { util::DateTime t2 = std::min(dx.validTime()+hslot_, winend_); for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - observerstlad_[jj]->doProcessingTL(dx, t1, t2); + if (observerstlad_[jj]) observerstlad_[jj]->doProcessingTL(dx, t1, t2); } Log::trace() << "ObserversTLAD::doProcessingTL done" << std::endl; } @@ -179,7 +180,7 @@ template void ObserversTLAD::doFinalizeTL(const Increment_ & dx) { Log::trace() << "ObserversTLAD::doFinalizeTL start" << std::endl; for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - observerstlad_[jj]->doFinalizeTL(dx, (*ydeptl_)[jj], (*ybiastl_)[jj]); + if (observerstlad_[jj]) observerstlad_[jj]->doFinalizeTL(dx, (*ydeptl_)[jj], (*ybiastl_)[jj]); } Log::trace() << "ObserversTLAD::doFinalizeTL done" << std::endl; } @@ -200,7 +201,9 @@ void ObserversTLAD::doFirstAD(Increment_ & dx, const util::DateTime hslot_ = tstep/2; for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - observerstlad_[jj]->doFirstAD(dx, (*ydepad_)[jj], (*ybiasad_)[jj], winbgn_, winend_); + if (observerstlad_[jj]) { + observerstlad_[jj]->doFirstAD(dx, (*ydepad_)[jj], (*ybiasad_)[jj], winbgn_, winend_); + } } Log::trace() << "ObserversTLAD::doFirstAD done" << std::endl; } @@ -212,7 +215,7 @@ void ObserversTLAD::doProcessingAD(Increment_ & dx) { util::DateTime t2 = std::min(dx.validTime()+hslot_, winend_); for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - observerstlad_[jj]->doProcessingAD(dx, t1, t2); + if (observerstlad_[jj]) observerstlad_[jj]->doProcessingAD(dx, t1, t2); } Log::trace() << "ObserversTLAD::doProcessingAD done" << std::endl; } @@ -221,7 +224,7 @@ template void ObserversTLAD::doLastAD(Increment_ & dx) { Log::trace() << "ObserversTLAD::doLastAD start" << std::endl; for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - observerstlad_[jj]->doLastAD(dx); + if (observerstlad_[jj]) observerstlad_[jj]->doLastAD(dx); } Log::trace() << "ObserversTLAD::doLastAD done" << std::endl; } From ce25006985313ee868db055e93e802ac1c248699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Mon, 15 Mar 2021 15:18:52 -0600 Subject: [PATCH 073/142] Rename HofXNoModel into HofX3D (#1087) * Rename HofXNoModel into HofX3D * Fix for ewok --- ewok/hofx3d.yaml | 4 +- l95/src/executables/CMakeLists.txt | 4 +- l95/src/executables/EnsHofX.cc | 4 +- l95/src/executables/HofX.cc | 4 +- .../executables/{HofXNoModel.cc => HofX3D.cc} | 4 +- l95/test/CMakeLists.txt | 20 ++++---- l95/test/testinput/hofx3d.yaml | 12 +++++ ...tkf_nomodel.yaml => hofx3d_for_getkf.yaml} | 0 l95/test/testinput/hofx_nomodel.yaml | 49 ------------------- l95/test/testoutput/hofx.nomodel.test | 9 ---- l95/test/testoutput/hofx3d.test | 6 +++ ...tkf_nomodel.test => hofx3d_for_getkf.test} | 0 qg/mains/CMakeLists.txt | 4 +- qg/mains/qgEnsHofX.cc | 4 +- qg/mains/qgHofX.cc | 4 +- qg/mains/{qgHofXNoModel.cc => qgHofX3D.cc} | 4 +- qg/test/CMakeLists.txt | 2 +- qg/test/testinput/hofx3d.yaml | 2 +- qg/test/testoutput/hofx3d.test | 10 +--- src/CMakeLists.txt | 4 +- src/oops/runs/{HofXNoModel.h => HofX3D.h} | 46 ++++++----------- src/oops/runs/{HofX.h => HofX4D.h} | 14 +++--- 22 files changed, 71 insertions(+), 139 deletions(-) rename l95/src/executables/{HofXNoModel.cc => HofX3D.cc} (76%) create mode 100644 l95/test/testinput/hofx3d.yaml rename l95/test/testinput/{hofx_for_getkf_nomodel.yaml => hofx3d_for_getkf.yaml} (100%) delete mode 100644 l95/test/testinput/hofx_nomodel.yaml delete mode 100644 l95/test/testoutput/hofx.nomodel.test create mode 100644 l95/test/testoutput/hofx3d.test rename l95/test/testoutput/{hofx_for_getkf_nomodel.test => hofx3d_for_getkf.test} (100%) rename qg/mains/{qgHofXNoModel.cc => qgHofX3D.cc} (78%) rename src/oops/runs/{HofXNoModel.h => HofX3D.h} (69%) rename src/oops/runs/{HofX.h => HofX4D.h} (94%) diff --git a/ewok/hofx3d.yaml b/ewok/hofx3d.yaml index a3eca0826..a160804b0 100644 --- a/ewok/hofx3d.yaml +++ b/ewok/hofx3d.yaml @@ -1,8 +1,6 @@ -forecast length: - $(forecast_length) geometry: $(GEOMETRY) -forecasts: +state: $(BACKGROUND) observations: $(OBSERVATIONS) diff --git a/l95/src/executables/CMakeLists.txt b/l95/src/executables/CMakeLists.txt index 29f197f28..8739891f2 100644 --- a/l95/src/executables/CMakeLists.txt +++ b/l95/src/executables/CMakeLists.txt @@ -30,8 +30,8 @@ ecbuild_add_executable( TARGET l95_hofx.x SOURCES HofX.cc LIBS lorenz95 ) -ecbuild_add_executable( TARGET l95_hofx_nomodel.x - SOURCES HofXNoModel.cc +ecbuild_add_executable( TARGET l95_hofx3d.x + SOURCES HofX3D.cc LIBS lorenz95 ) ecbuild_add_executable( TARGET l95_enshofx.x diff --git a/l95/src/executables/EnsHofX.cc b/l95/src/executables/EnsHofX.cc index ad2b8491a..9a4a0f3be 100644 --- a/l95/src/executables/EnsHofX.cc +++ b/l95/src/executables/EnsHofX.cc @@ -10,11 +10,11 @@ #include "lorenz95/L95Traits.h" #include "oops/runs/EnsembleApplication.h" -#include "oops/runs/HofX.h" +#include "oops/runs/HofX4D.h" #include "oops/runs/Run.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); - oops::EnsembleApplication > enshofx; + oops::EnsembleApplication > enshofx; return run.execute(enshofx); } diff --git a/l95/src/executables/HofX.cc b/l95/src/executables/HofX.cc index a23d5799b..5f9bfb119 100644 --- a/l95/src/executables/HofX.cc +++ b/l95/src/executables/HofX.cc @@ -9,11 +9,11 @@ */ #include "lorenz95/L95Traits.h" -#include "oops/runs/HofX.h" +#include "oops/runs/HofX4D.h" #include "oops/runs/Run.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); - oops::HofX hofx; + oops::HofX4D hofx; return run.execute(hofx); } diff --git a/l95/src/executables/HofXNoModel.cc b/l95/src/executables/HofX3D.cc similarity index 76% rename from l95/src/executables/HofXNoModel.cc rename to l95/src/executables/HofX3D.cc index f31a90ca8..69d56adb0 100644 --- a/l95/src/executables/HofXNoModel.cc +++ b/l95/src/executables/HofX3D.cc @@ -6,11 +6,11 @@ */ #include "lorenz95/L95Traits.h" -#include "oops/runs/HofXNoModel.h" +#include "oops/runs/HofX3D.h" #include "oops/runs/Run.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); - oops::HofXNoModel hofx; + oops::HofX3D hofx; return run.execute(hofx); } diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index ea8bc6fc8..740eabca9 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -62,8 +62,8 @@ list( APPEND l95_test_input testinput/getkf.yaml testinput/getkf_offline_hofx.yaml testinput/hofx.yaml - testinput/hofx_nomodel.yaml - testinput/hofx_for_getkf_nomodel.yaml + testinput/hofx3d.yaml + testinput/hofx3d_for_getkf.yaml testinput/identitymodel.yaml testinput/letkfGSI.yaml testinput/letkf.yaml @@ -148,8 +148,8 @@ list( APPEND l95_testoutput testoutput/getkf.test testoutput/getkf_offline_hofx.test testoutput/hofx.test - testoutput/hofx.nomodel.test - testoutput/hofx_for_getkf_nomodel.test + testoutput/hofx3d.test + testoutput/hofx3d_for_getkf.test testoutput/letkf.test testoutput/letkf_noobs.test testoutput/letkf_qc.test @@ -447,10 +447,10 @@ oops_add_test( TESTNAME hofx EXENAME l95_hofx.x TEST_DEPENDS test_l95_truth test_l95_makeobs4d ) -oops_add_test( TESTNAME hofx.nomodel +oops_add_test( TESTNAME hofx3d MODELNAME l95 - YAMLNAME testinput/hofx_nomodel.yaml - EXENAME l95_hofx_nomodel.x + YAMLNAME testinput/hofx3d.yaml + EXENAME l95_hofx3d.x TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) ##################################################################### @@ -765,9 +765,9 @@ oops_add_test( TESTNAME getkf CTOL 1e-3 TEST_DEPENDS test_l95_makeobs3d test_l95_genenspert ) -oops_add_test( TESTNAME hofx_for_getkf_nomodel +oops_add_test( TESTNAME hofx3d_for_getkf MODELNAME l95 - YAMLNAME testinput/hofx_for_getkf_nomodel.yaml + YAMLNAME testinput/hofx3d_for_getkf.yaml EXENAME l95_letkf.x CTOL 1e-3 TEST_DEPENDS test_l95_letkf ) @@ -777,7 +777,7 @@ oops_add_test( TESTNAME getkf_offline_hofx YAMLNAME testinput/getkf_offline_hofx.yaml EXENAME l95_letkf.x CTOL 1e-3 - TEST_DEPENDS test_l95_hofx_for_getkf_nomodel ) + TEST_DEPENDS test_l95_hofx3d_for_getkf ) ##################################################################### diff --git a/l95/test/testinput/hofx3d.yaml b/l95/test/testinput/hofx3d.yaml new file mode 100644 index 000000000..2f299a8dd --- /dev/null +++ b/l95/test/testinput/hofx3d.yaml @@ -0,0 +1,12 @@ +window begin: 2010-01-01T21:00:00Z +window length: PT6H +geometry: + resol: 40 +state: + date: 2010-01-02T00:00:00Z + filename: Data/test.fc.2010-01-01T00:00:00Z.P1D +observations: +- obs space: + obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt + obsdataout: Data/l95.hofx.2010-01-02T00:00:00Z.obt + obs operator: {} diff --git a/l95/test/testinput/hofx_for_getkf_nomodel.yaml b/l95/test/testinput/hofx3d_for_getkf.yaml similarity index 100% rename from l95/test/testinput/hofx_for_getkf_nomodel.yaml rename to l95/test/testinput/hofx3d_for_getkf.yaml diff --git a/l95/test/testinput/hofx_nomodel.yaml b/l95/test/testinput/hofx_nomodel.yaml deleted file mode 100644 index 0bc4e1e7b..000000000 --- a/l95/test/testinput/hofx_nomodel.yaml +++ /dev/null @@ -1,49 +0,0 @@ -window begin: '2010-01-01T03:00:00Z' # obs window starts 3 hr after the first state -window length: P1D -geometry: - resol: 40 -forecasts: - states: - - date: '2010-01-01T00:00:00Z' - filename: Data/test.an.2010-01-01T00:00:00Z.l95 - - date: '2010-01-01T01:30:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT1H30M - - date: '2010-01-01T03:00:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT3H - - date: '2010-01-01T04:30:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT4H30M - - date: '2010-01-01T06:00:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT6H - - date: '2010-01-01T07:30:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT7H30M - - date: '2010-01-01T09:00:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT9H - - date: '2010-01-01T10:30:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT10H30M - - date: '2010-01-01T12:00:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT12H - - date: '2010-01-01T13:30:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT13H30M - - date: '2010-01-01T15:00:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT15H - - date: '2010-01-01T16:30:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT16H30M - - date: '2010-01-01T18:00:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT18H - - date: '2010-01-01T19:30:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT19H30M - - date: '2010-01-01T21:00:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT21H - - date: '2010-01-01T22:30:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.PT22H30M - - date: '2010-01-02T00:00:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.P1D - - date: '2010-01-02T01:30:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.P1DT1H30M - - date: '2010-01-02T03:00:00Z' - filename: Data/test.fc.2010-01-01T00:00:00Z.P1DT3H -observations: -- obs space: - obsdatain: Data/l95.truth4d.2010-01-02T00:00:00Z.obt - obsdataout: Data/l95.hofx.2010-01-02T00:00:00Z.obt - obs operator: {} diff --git a/l95/test/testoutput/hofx.nomodel.test b/l95/test/testoutput/hofx.nomodel.test deleted file mode 100644 index 42287c05b..000000000 --- a/l95/test/testoutput/hofx.nomodel.test +++ /dev/null @@ -1,9 +0,0 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Min=7, Max=8, Average=7.975 -Test : Final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=6.34504, Max=9.44115, Average=7.96745 -Test : H(x): -Test : Lorenz 95 nobs= 160 Min=6.34504, Max=9.44115, Average=7.97687 -Test : End H(x) diff --git a/l95/test/testoutput/hofx3d.test b/l95/test/testoutput/hofx3d.test new file mode 100644 index 000000000..f9fb5eeb4 --- /dev/null +++ b/l95/test/testoutput/hofx3d.test @@ -0,0 +1,6 @@ +Test : State: +Test : Valid time: 2010-01-02T00:00:00Z +Test : Min=6.65953, Max=9.39191, Average=7.97085 +Test : H(x): +Test : Lorenz 95 nobs= 120 Min=6.65953, Max=9.39191, Average=7.97085 +Test : End H(x) diff --git a/l95/test/testoutput/hofx_for_getkf_nomodel.test b/l95/test/testoutput/hofx3d_for_getkf.test similarity index 100% rename from l95/test/testoutput/hofx_for_getkf_nomodel.test rename to l95/test/testoutput/hofx3d_for_getkf.test diff --git a/qg/mains/CMakeLists.txt b/qg/mains/CMakeLists.txt index c34b08572..a3059b2e8 100644 --- a/qg/mains/CMakeLists.txt +++ b/qg/mains/CMakeLists.txt @@ -33,8 +33,8 @@ ecbuild_add_executable( TARGET qg_hofx.x LIBS qg ) -ecbuild_add_executable( TARGET qg_hofx_nomodel.x - SOURCES qgHofXNoModel.cc +ecbuild_add_executable( TARGET qg_hofx3d.x + SOURCES qgHofX3D.cc LIBS qg ) diff --git a/qg/mains/qgEnsHofX.cc b/qg/mains/qgEnsHofX.cc index 826b7cabb..0de148494 100644 --- a/qg/mains/qgEnsHofX.cc +++ b/qg/mains/qgEnsHofX.cc @@ -10,11 +10,11 @@ #include "model/QgTraits.h" #include "oops/runs/EnsembleApplication.h" -#include "oops/runs/HofX.h" +#include "oops/runs/HofX4D.h" #include "oops/runs/Run.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); - oops::EnsembleApplication< oops::HofX > enshofx; + oops::EnsembleApplication< oops::HofX4D > enshofx; return run.execute(enshofx); } diff --git a/qg/mains/qgHofX.cc b/qg/mains/qgHofX.cc index 015a5527b..44775f63c 100644 --- a/qg/mains/qgHofX.cc +++ b/qg/mains/qgHofX.cc @@ -9,11 +9,11 @@ */ #include "model/QgTraits.h" -#include "oops/runs/HofX.h" +#include "oops/runs/HofX4D.h" #include "oops/runs/Run.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); - oops::HofX hofx; + oops::HofX4D hofx; return run.execute(hofx); } diff --git a/qg/mains/qgHofXNoModel.cc b/qg/mains/qgHofX3D.cc similarity index 78% rename from qg/mains/qgHofXNoModel.cc rename to qg/mains/qgHofX3D.cc index f7a658216..1c9fce5e5 100644 --- a/qg/mains/qgHofXNoModel.cc +++ b/qg/mains/qgHofX3D.cc @@ -6,11 +6,11 @@ */ #include "model/QgTraits.h" -#include "oops/runs/HofXNoModel.h" +#include "oops/runs/HofX3D.h" #include "oops/runs/Run.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); - oops::HofXNoModel hofx; + oops::HofX3D hofx; return run.execute(hofx); } diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index 73224d694..64479c01e 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -413,7 +413,7 @@ oops_add_test( TESTNAME hofx3d MODELNAME qg OMP 2 YAMLNAME testinput/hofx3d.yaml - EXENAME qg_hofx_nomodel.x + EXENAME qg_hofx3d.x TEST_DEPENDS test_qg_make_obs_4d_12h ) diff --git a/qg/test/testinput/hofx3d.yaml b/qg/test/testinput/hofx3d.yaml index b0be54cd9..80e292622 100644 --- a/qg/test/testinput/hofx3d.yaml +++ b/qg/test/testinput/hofx3d.yaml @@ -4,7 +4,7 @@ geometry: nx: 40 ny: 20 depths: [4500.0, 5500.0] -forecasts: +state: date: 2010-01-01T06:00:00Z filename: Data/truth.fc.2009-12-15T00:00:00Z.P17DT6H.nc model: diff --git a/qg/test/testoutput/hofx3d.test b/qg/test/testoutput/hofx3d.test index 7ba180116..80872a33c 100644 --- a/qg/test/testoutput/hofx3d.test +++ b/qg/test/testoutput/hofx3d.test @@ -1,12 +1,4 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.0526e+08, Max= 1.1159e+08, RMS= 1.8719e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Final state: +Test : State: Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 Test : Variable = streamfunction diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e46589c71..5a4883e97 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -224,8 +224,8 @@ oops/runs/EnsVariance.h oops/runs/ExternalDFI.h oops/runs/Forecast.h oops/runs/GenEnsPertB.h -oops/runs/HofX.h -oops/runs/HofXNoModel.h +oops/runs/HofX3D.h +oops/runs/HofX4D.h oops/runs/LocalEnsembleDA.h oops/runs/RTPP.h oops/runs/Run.cc diff --git a/src/oops/runs/HofXNoModel.h b/src/oops/runs/HofX3D.h similarity index 69% rename from src/oops/runs/HofXNoModel.h rename to src/oops/runs/HofX3D.h index dfb7762a3..b63a6e93d 100644 --- a/src/oops/runs/HofXNoModel.h +++ b/src/oops/runs/HofX3D.h @@ -5,8 +5,8 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ -#ifndef OOPS_RUNS_HOFXNOMODEL_H_ -#define OOPS_RUNS_HOFXNOMODEL_H_ +#ifndef OOPS_RUNS_HOFX3D_H_ +#define OOPS_RUNS_HOFX3D_H_ #include #include @@ -15,7 +15,6 @@ #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/CalcHofX.h" -#include "oops/assimilation/State4D.h" #include "oops/base/instantiateObsFilterFactory.h" #include "oops/base/Observations.h" #include "oops/base/ObsSpaces.h" @@ -24,6 +23,7 @@ #include "oops/interface/GeoVaLs.h" #include "oops/interface/GetValues.h" #include "oops/interface/Locations.h" +#include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" #include "oops/util/ConfigFunctions.h" @@ -33,7 +33,7 @@ namespace oops { -template class HofXNoModel : public Application { +template class HofX3D : public Application { typedef CalcHofX CalcHofX_; typedef Geometry Geometry_; typedef GeoVaLs GeoVaLs_; @@ -42,7 +42,7 @@ template class HofXNoModel : public Application { typedef ObsAuxControls ObsAux_; typedef Observations Observations_; typedef ObsSpaces ObsSpaces_; - typedef State4D State4D_; + typedef State State_; typedef std::vector> GeoVaLsVec_; typedef std::vector> LocationsVec_; @@ -51,11 +51,11 @@ template class HofXNoModel : public Application { public: // ----------------------------------------------------------------------------- - explicit HofXNoModel(const eckit::mpi::Comm & comm = oops::mpi::world()) : Application(comm) { + explicit HofX3D(const eckit::mpi::Comm & comm = oops::mpi::world()) : Application(comm) { instantiateObsFilterFactory(); } // ----------------------------------------------------------------------------- - virtual ~HofXNoModel() {} + virtual ~HofX3D() {} // ----------------------------------------------------------------------------- int execute(const eckit::Configuration & fullConfig) const { // Setup observation window @@ -69,10 +69,10 @@ template class HofXNoModel : public Application { const Geometry_ geometry(geometryConfig, this->getComm()); // Setup states for H(x) - const eckit::LocalConfiguration stateConfig(fullConfig, "forecasts"); - Log::info() << "States configuration is:" << stateConfig << std::endl; - State4D_ xx(geometry, stateConfig); - Log::test() << "Initial state: " << xx[0] << std::endl; + const eckit::LocalConfiguration stateConfig(fullConfig, "state"); + Log::info() << "State configuration is:" << stateConfig << std::endl; + State_ xx(geometry, stateConfig); + Log::test() << "State: " << xx << std::endl; // Setup observations const eckit::LocalConfiguration obsConfig(fullConfig, "observations"); @@ -81,17 +81,6 @@ template class HofXNoModel : public Application { CalcHofX_ hofx(obspaces, obsConfig); hofx.initialize(obsaux); - // Setup and check time steps between the states in 4D state - const size_t nstates = xx.size(); - util::Duration tstep = winlen; // for a single state - // if using several states, compute the timestep and check that it's the same for all states - if (nstates > 1) { - tstep = xx[1].validTime() - xx[0].validTime(); - for (size_t ii = 1; ii < nstates; ++ii) { - ASSERT(tstep == (xx[ii].validTime() - xx[ii-1].validTime())); - } - } - // fill in GeoVaLs GeoVaLsVec_ geovals; const LocationsVec_ & locations = hofx.locations(); @@ -103,21 +92,14 @@ template class HofXNoModel : public Application { // loop over all observation types for (size_t jj = 0; jj < obspaces.size(); ++jj) { GetValues_ getvals(geometry, *locations[jj], getValuesConfig[jj]); - // add GeoVaLs for this obs type geovals.emplace_back(new GeoVaLs_(*locations[jj], vars[jj])); - // fill in by looping through 4D state - for (size_t ii = 0; ii < nstates; ++ii) { - util::DateTime t1 = std::max(xx[ii].validTime()-tstep/2, winbgn); - util::DateTime t2 = std::min(xx[ii].validTime()+tstep/2, winend); - getvals.fillGeoVaLs(xx[ii], t1, t2, *geovals[jj]); - } + getvals.fillGeoVaLs(xx, winbgn, winend, *geovals[jj]); } // Compute H(x) on filled in geovals and run the filters Observations_ yobs = hofx.compute(geovals); hofx.saveQcFlags("EffectiveQC"); hofx.saveObsErrors("EffectiveError"); - Log::test() << "Final state: " << xx[xx.size()-1] << std::endl; // Save H(x) Log::test() << "H(x): " << std::endl << yobs << "End H(x)" << std::endl; @@ -128,11 +110,11 @@ template class HofXNoModel : public Application { // ----------------------------------------------------------------------------- private: std::string appname() const { - return "oops::HofXNoModel<" + MODEL::name() + ", " + OBS::name() + ">"; + return "oops::HofX3D<" + MODEL::name() + ", " + OBS::name() + ">"; } // ----------------------------------------------------------------------------- }; } // namespace oops -#endif // OOPS_RUNS_HOFXNOMODEL_H_ +#endif // OOPS_RUNS_HOFX3D_H_ diff --git a/src/oops/runs/HofX.h b/src/oops/runs/HofX4D.h similarity index 94% rename from src/oops/runs/HofX.h rename to src/oops/runs/HofX4D.h index 8d0ee539e..6886ef9bd 100644 --- a/src/oops/runs/HofX.h +++ b/src/oops/runs/HofX4D.h @@ -9,8 +9,8 @@ * does it submit to any jurisdiction. */ -#ifndef OOPS_RUNS_HOFX_H_ -#define OOPS_RUNS_HOFX_H_ +#ifndef OOPS_RUNS_HOFX4D_H_ +#define OOPS_RUNS_HOFX4D_H_ #include #include @@ -44,7 +44,7 @@ namespace oops { /// Application runs model forecast from "initial condition" for the "forecast length" /// and computes H(x) on the run. If "obspert" is specified in the config, the resulting /// H(x) is perturbed. It is saved as "hofx" by default, or as specified "hofx group name" -template class HofX : public Application { +template class HofX4D : public Application { typedef Geometry Geometry_; typedef GetValuesPost GetValuesPost_; typedef Model Model_; @@ -57,12 +57,12 @@ template class HofX : public Application { public: // ----------------------------------------------------------------------------- - explicit HofX(const eckit::mpi::Comm & comm = oops::mpi::world()) : Application(comm) { + explicit HofX4D(const eckit::mpi::Comm & comm = oops::mpi::world()) : Application(comm) { instantiateObsErrorFactory(); instantiateObsFilterFactory(); } // ----------------------------------------------------------------------------- - virtual ~HofX() = default; + virtual ~HofX4D() = default; // ----------------------------------------------------------------------------- int execute(const eckit::Configuration & fullConfig) const { // Setup observation window @@ -149,11 +149,11 @@ template class HofX : public Application { // ----------------------------------------------------------------------------- private: std::string appname() const { - return "oops::HofX<" + MODEL::name() + ", " + OBS::name() + ">"; + return "oops::HofX4D<" + MODEL::name() + ", " + OBS::name() + ">"; } // ----------------------------------------------------------------------------- }; } // namespace oops -#endif // OOPS_RUNS_HOFX_H_ +#endif // OOPS_RUNS_HOFX4D_H_ From 00188df6c2ddbb7a2e2a68e576f9815256978764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Tue, 16 Mar 2021 15:20:51 +0000 Subject: [PATCH 074/142] Pass the contents of the "obs bias" YAML section to constructors of ObsAuxIncrement/Covariance (#1090) * Removed an unused constructor of lorenz95::ObsBiasCorrection. * Pass not only the contents of the "obs bias error" section, but also that of the "obs bias" section to the constructors of ObsAuxIncrement and ObsAuxCovariance. * Revert "Pass not only the contents of the "obs bias error" section, but also that of the "obs bias" section to the constructors of ObsAuxIncrement and ObsAuxCovariance." This reverts commit 0002f17ed62c03be4581a154e59a28ae2f5d4498. * Moved ObsAuxCovariance/Increment options from "obs bias error" to "obs bias/covariance". Co-authored-by: Anna Shlyaeva --- l95/src/lorenz95/ObsBiasCorrection.cc | 10 ++-------- l95/src/lorenz95/ObsBiasCorrection.h | 1 - l95/src/lorenz95/ObsBiasCovariance.cc | 5 +++-- l95/test/testinput/4dvar.allbiases.yaml | 4 ++-- l95/test/testinput/4dvar.obsbias.yaml | 4 ++-- l95/test/testinput/interfaces.yaml | 4 ++-- qg/model/ObsBiasCovariance.cc | 10 ++++++---- qg/model/ObsBiasIncrement.cc | 10 ++++++---- qg/test/testinput/4dvar_obs_biased.yaml | 8 ++++---- qg/test/testinput/interfaces.yaml | 8 ++++---- src/oops/base/ObsAuxCovariances.h | 2 +- src/oops/base/ObsAuxIncrements.h | 2 +- src/test/interface/LinearObsOperator.h | 17 +++++++---------- src/test/interface/ObsAuxCovariance.h | 2 +- src/test/interface/ObsAuxIncrement.h | 24 ++++++++++++------------ 15 files changed, 53 insertions(+), 58 deletions(-) diff --git a/l95/src/lorenz95/ObsBiasCorrection.cc b/l95/src/lorenz95/ObsBiasCorrection.cc index cf76c56bb..f66a52d31 100644 --- a/l95/src/lorenz95/ObsBiasCorrection.cc +++ b/l95/src/lorenz95/ObsBiasCorrection.cc @@ -24,7 +24,8 @@ namespace lorenz95 { ObsBiasCorrection::ObsBiasCorrection(const ObsTableView &, const eckit::Configuration & conf) : bias_(0.0), active_(false) { - active_ = conf.has("standard_deviation"); + const eckit::LocalConfiguration covconf = conf.getSubConfiguration("covariance"); + active_ = covconf.has("standard_deviation"); if (active_) {oops::Log::trace() << "ObsBiasCorrection::ObsBiasCorrection created." << std::endl;} } // ----------------------------------------------------------------------------- @@ -35,13 +36,6 @@ ObsBiasCorrection::ObsBiasCorrection(const ObsBiasCorrection & other, if (active_ && copy) bias_ = other.bias_; } // ----------------------------------------------------------------------------- -ObsBiasCorrection::ObsBiasCorrection(const ObsBiasCorrection & other, - const eckit::Configuration &) - : bias_(0.0), active_(other.active_) -{ - if (active_) bias_ = other.bias_; -} -// ----------------------------------------------------------------------------- void ObsBiasCorrection::diff(const ObsBias & b1, const ObsBias & b2) { if (active_) bias_ = b1.value() - b2.value(); } diff --git a/l95/src/lorenz95/ObsBiasCorrection.h b/l95/src/lorenz95/ObsBiasCorrection.h index f88a63262..754afff38 100644 --- a/l95/src/lorenz95/ObsBiasCorrection.h +++ b/l95/src/lorenz95/ObsBiasCorrection.h @@ -35,7 +35,6 @@ class ObsBiasCorrection : public util::Printable, ObsBiasCorrection(); ObsBiasCorrection(const ObsTableView &, const eckit::Configuration &); ObsBiasCorrection(const ObsBiasCorrection &, const bool copy = true); - ObsBiasCorrection(const ObsBiasCorrection &, const eckit::Configuration &); ~ObsBiasCorrection() {} /// Linear algebra operators diff --git a/l95/src/lorenz95/ObsBiasCovariance.cc b/l95/src/lorenz95/ObsBiasCovariance.cc index 580855586..c47914476 100644 --- a/l95/src/lorenz95/ObsBiasCovariance.cc +++ b/l95/src/lorenz95/ObsBiasCovariance.cc @@ -27,9 +27,10 @@ namespace lorenz95 { ObsBiasCovariance::ObsBiasCovariance(const ObsTableView &, const eckit::Configuration & conf) : variance_(0.0), active_(false) { - if (conf.has("standard_deviation")) { + const eckit::LocalConfiguration covconf = conf.getSubConfiguration("covariance"); + if (covconf.has("standard_deviation")) { active_ = true; - const double zz = conf.getDouble("standard_deviation"); + const double zz = covconf.getDouble("standard_deviation"); variance_ = zz * zz; ASSERT(variance_ > 0.0); oops::Log::info() << "ObsBiasCovariance variance = " << variance_ << std::endl; diff --git a/l95/test/testinput/4dvar.allbiases.yaml b/l95/test/testinput/4dvar.allbiases.yaml index eca7d91f3..e0578bb77 100644 --- a/l95/test/testinput/4dvar.allbiases.yaml +++ b/l95/test/testinput/4dvar.allbiases.yaml @@ -30,8 +30,8 @@ cost function: covariance model: diagonal obs bias: bias: 0.0 - obs bias error: - standard_deviation: 0.8 + covariance: + standard_deviation: 0.8 constraints: - jcdfi: filtered variables: [x] diff --git a/l95/test/testinput/4dvar.obsbias.yaml b/l95/test/testinput/4dvar.obsbias.yaml index 73235faac..8c6257943 100644 --- a/l95/test/testinput/4dvar.obsbias.yaml +++ b/l95/test/testinput/4dvar.obsbias.yaml @@ -26,8 +26,8 @@ cost function: covariance model: diagonal obs bias: bias: 0.0 - obs bias error: - standard_deviation: 0.8 + covariance: + standard_deviation: 0.8 constraints: - jcdfi: filtered variables: [x] diff --git a/l95/test/testinput/interfaces.yaml b/l95/test/testinput/interfaces.yaml index 865b4f34a..5c750ff49 100644 --- a/l95/test/testinput/interfaces.yaml +++ b/l95/test/testinput/interfaces.yaml @@ -111,8 +111,8 @@ observations: bias: 0.3 norm: 0.3 relative tolerance: 0.0 - obs bias error: - standard_deviation: 0.5 + covariance: + standard_deviation: 0.5 rms ref: 8.3207407741318846 tolerance: 1.0e-10 diff --git a/qg/model/ObsBiasCovariance.cc b/qg/model/ObsBiasCovariance.cc index c8d242bb4..b2402ff30 100644 --- a/qg/model/ObsBiasCovariance.cc +++ b/qg/model/ObsBiasCovariance.cc @@ -29,11 +29,13 @@ namespace qg { ObsBiasCovariance::ObsBiasCovariance(const ObsSpaceQG &, const eckit::Configuration & conf) : variance_(ObsBias::ntypes, 0.0) { + const eckit::LocalConfiguration covconf = conf.getSubConfiguration("covariance"); + std::vector zz(4, 0.0); - if (conf.has("stream")) zz[0] = conf.getDouble("stream"); - if (conf.has("uwind")) zz[1] = conf.getDouble("uwind"); - if (conf.has("vwind")) zz[2] = conf.getDouble("vwind"); - if (conf.has("wspeed")) zz[3] = conf.getDouble("wspeed"); + if (covconf.has("stream")) zz[0] = covconf.getDouble("stream"); + if (covconf.has("uwind")) zz[1] = covconf.getDouble("uwind"); + if (covconf.has("vwind")) zz[2] = covconf.getDouble("vwind"); + if (covconf.has("wspeed")) zz[3] = covconf.getDouble("wspeed"); std::string strn = ""; for (unsigned int jj = 0; jj < ObsBias::ntypes; ++jj) { if (jj > 0) strn += ", "; diff --git a/qg/model/ObsBiasIncrement.cc b/qg/model/ObsBiasIncrement.cc index b355665ee..46158f8c6 100644 --- a/qg/model/ObsBiasIncrement.cc +++ b/qg/model/ObsBiasIncrement.cc @@ -26,10 +26,12 @@ namespace qg { ObsBiasIncrement::ObsBiasIncrement(const ObsSpaceQG &, const eckit::Configuration & conf) : bias_(ObsBias::ntypes, 0.0), active_(ObsBias::ntypes, false) { - active_[0] = conf.has("stream"); - active_[1] = conf.has("uwind"); - active_[2] = conf.has("vwind"); - active_[3] = conf.has("wspeed"); + const eckit::LocalConfiguration covconf = conf.getSubConfiguration("covariance"); + + active_[0] = covconf.has("stream"); + active_[1] = covconf.has("uwind"); + active_[2] = covconf.has("vwind"); + active_[3] = covconf.has("wspeed"); bool on = false; std::string strn = ""; for (unsigned int jj = 0; jj < ObsBias::ntypes; ++jj) { diff --git a/qg/test/testinput/4dvar_obs_biased.yaml b/qg/test/testinput/4dvar_obs_biased.yaml index e395fcbda..156b70c7b 100644 --- a/qg/test/testinput/4dvar_obs_biased.yaml +++ b/qg/test/testinput/4dvar_obs_biased.yaml @@ -32,8 +32,8 @@ cost function: obs type: Stream obs bias: stream: 0.0 - obs bias error: - stream: '2.0e7' + covariance: + stream: '2.0e7' - obs error: covariance model: diagonal obs operator: @@ -46,8 +46,8 @@ cost function: obs type: Wind obs bias: uwind: 0.0 - obs bias error: - uwind: '15.0' + covariance: + uwind: '15.0' - obs error: covariance model: diagonal obs operator: diff --git a/qg/test/testinput/interfaces.yaml b/qg/test/testinput/interfaces.yaml index eb6473a98..86d603398 100644 --- a/qg/test/testinput/interfaces.yaml +++ b/qg/test/testinput/interfaces.yaml @@ -78,8 +78,8 @@ observations: stream: '-10.0' norm: 10.0 relative tolerance: 0.0 - obs bias error: - stream: '2.0e7' + covariance: + stream: '2.0e7' rms ref: 183502589.5028424 tolerance: 1.0e-8 - obs error: @@ -104,8 +104,8 @@ observations: uwind: '10.0' norm: 10.0 relative tolerance: 0.0 - obs bias error: - uwind: '15.0' + covariance: + uwind: '15.0' rms ref: 39.644266100943696 tolerance: 1.0e-8 - obs error: diff --git a/src/oops/base/ObsAuxCovariances.h b/src/oops/base/ObsAuxCovariances.h index ccaa7e29a..518450623 100644 --- a/src/oops/base/ObsAuxCovariances.h +++ b/src/oops/base/ObsAuxCovariances.h @@ -67,7 +67,7 @@ ObsAuxCovariances::ObsAuxCovariances(const ObsSpaces_ & odb, Log::trace() << "ObsAuxCovariances::ObsAuxCovariances starting" << std::endl; std::vector obsconf = conf.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsconf.size(); ++jobs) { - eckit::LocalConfiguration obsauxconf = obsconf[jobs].getSubConfiguration("obs bias error"); + eckit::LocalConfiguration obsauxconf = obsconf[jobs].getSubConfiguration("obs bias"); cov_.push_back( std::unique_ptr(new ObsAuxCovariance_(odb[jobs], obsauxconf))); } diff --git a/src/oops/base/ObsAuxIncrements.h b/src/oops/base/ObsAuxIncrements.h index a828620a3..fccc07db2 100644 --- a/src/oops/base/ObsAuxIncrements.h +++ b/src/oops/base/ObsAuxIncrements.h @@ -90,7 +90,7 @@ ObsAuxIncrements::ObsAuxIncrements(const ObsSpaces_ & odb, const eckit::Con { std::vector obsconf = conf.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsconf.size(); ++jobs) { - eckit::LocalConfiguration obsauxconf = obsconf[jobs].getSubConfiguration("obs bias error"); + eckit::LocalConfiguration obsauxconf = obsconf[jobs].getSubConfiguration("obs bias"); auxs_.push_back( std::unique_ptr(new ObsAuxIncrement_(odb[jobs], obsauxconf))); } diff --git a/src/test/interface/LinearObsOperator.h b/src/test/interface/LinearObsOperator.h index 296425453..d9913c511 100644 --- a/src/test/interface/LinearObsOperator.h +++ b/src/test/interface/LinearObsOperator.h @@ -85,8 +85,7 @@ template void testLinearity() { // initialize obs bias eckit::LocalConfiguration biasconf = conf.getSubConfiguration("obs bias"); const ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasconf); - eckit::LocalConfiguration biascovconf = conf.getSubConfiguration("obs bias error"); - ObsAuxIncr_ ybinc(Test_::obspace()[jj], biascovconf); + ObsAuxIncr_ ybinc(Test_::obspace()[jj], biasconf); // read geovals from the file const eckit::LocalConfiguration gconf(conf, "geovals"); @@ -95,7 +94,7 @@ template void testLinearity() { const GeoVaLs_ gval(gconf, Test_::obspace()[jj], hopvars); // initialize Obs. Bias Covariance - const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biascovconf); + const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biasconf); // set trajectory for TL/AD to be the geovals from the file hoptl.setTrajectory(gval, ybias); @@ -161,13 +160,12 @@ template void testAdjoint() { const double tol = conf.getDouble("linear obs operator test.tolerance AD"); // initialize bias correction eckit::LocalConfiguration biasconf = conf.getSubConfiguration("obs bias"); - eckit::LocalConfiguration biascovconf = conf.getSubConfiguration("obs bias error"); const ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasconf); - ObsAuxIncr_ ybinc1(Test_::obspace()[jj], biascovconf); // TL - ObsAuxIncr_ ybinc2(Test_::obspace()[jj], biascovconf); // AD + ObsAuxIncr_ ybinc1(Test_::obspace()[jj], biasconf); // TL + ObsAuxIncr_ ybinc2(Test_::obspace()[jj], biasconf); // AD // initialize Obs. Bias Covariance - const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biascovconf); + const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biasconf); // read geovals from the file eckit::LocalConfiguration gconf(conf, "geovals"); @@ -244,12 +242,11 @@ template void testTangentLinear() { // initialize obs bias from file eckit::LocalConfiguration biasconf = conf.getSubConfiguration("obs bias"); - eckit::LocalConfiguration biascovconf = conf.getSubConfiguration("obs bias error"); const ObsAuxCtrl_ ybias0(Test_::obspace()[jj], biasconf); ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasconf); // initialize Obs. Bias Covariance - const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biascovconf); + const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biasconf); // read geovals from the file const eckit::LocalConfiguration gconf(conf, "geovals"); @@ -277,7 +274,7 @@ template void testTangentLinear() { // randomize dx and ybinc GeoVaLs_ dx(gconf, Test_::obspace()[jj], hoptl.requiredVars()); dx.random(); - ObsAuxIncr_ ybinc(Test_::obspace()[jj], biascovconf); + ObsAuxIncr_ ybinc(Test_::obspace()[jj], biasconf); Bobsbias.randomize(ybinc); // scale dx by x0 diff --git a/src/test/interface/ObsAuxCovariance.h b/src/test/interface/ObsAuxCovariance.h index 2d990a6c2..aa6c9e7ce 100644 --- a/src/test/interface/ObsAuxCovariance.h +++ b/src/test/interface/ObsAuxCovariance.h @@ -34,7 +34,7 @@ template void testConstructor() { typedef oops::ObsAuxCovariance Covariance_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); std::unique_ptr cov(new Covariance_(Test_::obspace()[jj], biasconf)); EXPECT(cov.get()); oops::Log::test() << "Testing ObsAuxCovariance: " << *cov << std::endl; diff --git a/src/test/interface/ObsAuxIncrement.h b/src/test/interface/ObsAuxIncrement.h index bc672395b..996ba6591 100644 --- a/src/test/interface/ObsAuxIncrement.h +++ b/src/test/interface/ObsAuxIncrement.h @@ -51,9 +51,9 @@ template class ObsAuxIncrementFixture : private boost::noncopyabl ObsAuxIncrementFixture() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - eckit::LocalConfiguration biascovconf = - Test_::config(jj).getSubConfiguration("obs bias error"); - std::shared_ptr tmp(new Covariance_(Test_::obspace()[jj], biascovconf)); + eckit::LocalConfiguration biasconf = + Test_::config(jj).getSubConfiguration("obs bias"); + std::shared_ptr tmp(new Covariance_(Test_::obspace()[jj], biasconf)); covar_.push_back(tmp); } } @@ -70,7 +70,7 @@ template void testObsAuxIncrementConstructor() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); AuxIncr_ dx(Test_::obspace()[jj], biasconf); oops::Log::test() << "Printing zero ObsAuxIncrement: " << dx << std::endl; EXPECT(dx.norm() == 0.0); @@ -84,8 +84,8 @@ template void testObsAuxIncrementCopyConstructor() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - if (Test_::config(jj).has("obs bias error")) { - eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + if (Test_::config(jj).has("obs bias")) { + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); AuxIncr_ dx1(Test_::obspace()[jj], biasconf); ObsAuxIncrementFixture::covariance(jj).randomize(dx1); oops::Log::test() << "Printing random ObsAuxIncrement: " << dx1 << std::endl; @@ -112,8 +112,8 @@ template void testObsAuxIncrementTriangle() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - if (Test_::config(jj).has("obs bias error")) { - eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + if (Test_::config(jj).has("obs bias")) { + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); AuxIncr_ dx1(Test_::obspace()[jj], biasconf); AuxTest_::covariance(jj).randomize(dx1); AuxIncr_ dx2(Test_::obspace()[jj], biasconf); @@ -143,7 +143,7 @@ template void testObsAuxIncrementOpPlusEq() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); AuxIncr_ dx1(Test_::obspace()[jj], biasconf); AuxTest_::covariance(jj).randomize(dx1); AuxIncr_ dx2(dx1); @@ -165,7 +165,7 @@ template void testObsAuxIncrementDotProduct() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); AuxIncr_ dx1(Test_::obspace()[jj], biasconf); AuxTest_::covariance(jj).randomize(dx1); AuxIncr_ dx2(Test_::obspace()[jj], biasconf); @@ -187,7 +187,7 @@ template void testObsAuxIncrementZero() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); AuxIncr_ dx(Test_::obspace()[jj], biasconf); AuxTest_::covariance(jj).randomize(dx); EXPECT(dx.norm() > 0.0); @@ -206,7 +206,7 @@ template void testObsAuxIncrementAxpy() { typedef oops::ObsAuxIncrement AuxIncr_; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias error"); + eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); AuxIncr_ dx1(Test_::obspace()[jj], biasconf); AuxTest_::covariance(jj).randomize(dx1); From 62d5185f51ee77f1a0847aed78e5df3689879102 Mon Sep 17 00:00:00 2001 From: Michael Cooke <48374999+mikecooke77@users.noreply.github.com> Date: Tue, 16 Mar 2021 16:25:20 +0000 Subject: [PATCH 075/142] Remove constructors not used by oops (#1102) * Remove generic 1d-var constructors * Changes to make the code compile Co-authored-by: Anna Shlyaeva --- src/oops/assimilation/ControlVariable.h | 14 ----------- src/oops/assimilation/CostFunction.h | 32 ------------------------- src/oops/assimilation/CostJo.h | 18 -------------- src/oops/base/ObsSpaces.h | 9 ------- src/oops/interface/ObsSpace.h | 14 ----------- src/oops/interface/State.h | 13 ---------- src/test/interface/State.h | 5 ---- 7 files changed, 105 deletions(-) diff --git a/src/oops/assimilation/ControlVariable.h b/src/oops/assimilation/ControlVariable.h index bc48eb4f7..a1242cfc2 100644 --- a/src/oops/assimilation/ControlVariable.h +++ b/src/oops/assimilation/ControlVariable.h @@ -55,8 +55,6 @@ class ControlVariable : public util::Printable, /// The arguments define the number of sub-windows and the resolution ControlVariable(const eckit::Configuration &, const Geometry_ &, const ObsSpaces_ &); -/// Constructor added for generic 1d-var under development in ufo - ControlVariable(const eckit::Configuration &, const State_ &, const ObsSpaces_ &); explicit ControlVariable(const ControlVariable &); ~ControlVariable(); @@ -98,18 +96,6 @@ ControlVariable::ControlVariable(const eckit::Configuration & conf, Log::trace() << "ControlVariable contructed" << std::endl; } -// ============================================================================= -/// Constructor added for generic 1d-var under development in ufo -template -ControlVariable::ControlVariable(const eckit::Configuration & conf, - const State_ & statein, const ObsSpaces_ & odb) - : state_(statein), - modbias_(statein.geometry(), conf.getSubConfiguration("model aux control")), - obsbias_(odb, conf.getSubConfiguration("observations")) -{ - Log::trace() << "ControlVariable contructed" << std::endl; -} - // ----------------------------------------------------------------------------- template diff --git a/src/oops/assimilation/CostFunction.h b/src/oops/assimilation/CostFunction.h index 5f5cc17e3..fc75a9a04 100644 --- a/src/oops/assimilation/CostFunction.h +++ b/src/oops/assimilation/CostFunction.h @@ -97,7 +97,6 @@ template class CostFunction : private boost::nonco protected: void setupTerms(const eckit::Configuration &); - void setupTerms(const eckit::Configuration &, const State_ &); // generic 1d-var const CtrlVar_ & background() const {return *xb_;} private: @@ -217,37 +216,6 @@ void CostFunction::setupTerms(const eckit::Configuration & config) { Log::trace() << "CostFunction::setupTerms done" << std::endl; } -// ----------------------------------------------------------------------------- -// This setup terms method has been written for the generic 1d-var which is -// under development in UFO -// ----------------------------------------------------------------------------- -template -void CostFunction::setupTerms(const eckit::Configuration & config, - const State_ & statein) { - Log::trace() << "CostFunction::setupTerms start" << std::endl; - -// Jo - eckit::LocalConfiguration obsconf(config, "observations"); - CostJo * jo = this->newJo(obsconf); - jterms_.push_back(jo); - Log::trace() << "CostFunction::setupTerms Jo added" << std::endl; - -// Jb - xb_.reset(new CtrlVar_(config, statein, jo->obspaces())); - jb_.reset(new JbTotal_(*xb_, this->newJb(config, this->geometry(), *xb_), - config, this->geometry(), jo->obspaces())); - Log::trace() << "CostFunction::setupTerms Jb added" << std::endl; - -// Other constraints - std::vector jcs; - config.get("constraints", jcs); - for (size_t jj = 0; jj < jcs.size(); ++jj) { - CostTermBase * jc = this->newJc(jcs[jj], this->geometry()); - jterms_.push_back(jc); - } - Log::trace() << "CostFunction::setupTerms done" << std::endl; -} - // ----------------------------------------------------------------------------- template diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index 5cc141369..8fb4c68ef 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -75,10 +75,6 @@ template class CostJo : public CostTermBase::CostJo(const eckit::Configuration & joConf, const eckit::mpi Log::trace() << "CostJo::CostJo done" << std::endl; } -// ============================================================================= -/// Constructor added for generic 1d-var under development in ufo -template -CostJo::CostJo(const eckit::Configuration & joConf, - const util::DateTime & winbgn, const util::DateTime & winend, - const ObsSpaces_ & localobs) - : obsconf_(joConf), obspace_(localobs), - yobs_(obspace_, "ObsValue"), - Rmat_(), currentConf_(), gradFG_(), getvals_(), calchofx_(obspace_, obsconf_), - pobstlad_() -{ - Log::trace() << "CostJo::CostJo using local obs spaces done" << std::endl; -} - // ----------------------------------------------------------------------------- template diff --git a/src/oops/base/ObsSpaces.h b/src/oops/base/ObsSpaces.h index c9d58170d..7aa0e5d64 100644 --- a/src/oops/base/ObsSpaces.h +++ b/src/oops/base/ObsSpaces.h @@ -49,8 +49,6 @@ class ObsSpaces : public util::Printable, const util::DateTime &, const util::DateTime &, const eckit::mpi::Comm & time = oops::mpi::myself()); ObsSpaces(const ObsSpaces &, const eckit::geometry::Point2 &, const eckit::Configuration &); -/// Constructor added for generic 1d-var under development in ufo - explicit ObsSpaces(const std::shared_ptr &); ~ObsSpaces(); /// Access @@ -107,13 +105,6 @@ ObsSpaces::ObsSpaces(const ObsSpaces & obss, const eckit::geometry::Po ASSERT(spaces_.size() == obss.size()); } -// ----------------------------------------------------------------------------- -/// Constructor added for generic 1d-var under development in ufo -template -ObsSpaces::ObsSpaces(const std::shared_ptr & obss) - : spaces_(obss), wbgn_(obss->windowStart()), wend_(obss->windowEnd()) -{} - // ----------------------------------------------------------------------------- template diff --git a/src/oops/interface/ObsSpace.h b/src/oops/interface/ObsSpace.h index 804efd34a..6031aac9b 100644 --- a/src/oops/interface/ObsSpace.h +++ b/src/oops/interface/ObsSpace.h @@ -51,9 +51,6 @@ class ObsSpace : public util::Printable, const eckit::mpi::Comm & time = oops::mpi::myself()); ObsSpace(const ObsSpace &, const eckit::geometry::Point2 &, const eckit::Configuration &); -/// Constructor added for generic 1d-var under development in ufo - ObsSpace(const ObsSpace_ &, const eckit::geometry::Point2 &, - const eckit::Configuration &); explicit ObsSpace(const ObsSpace_ &); ~ObsSpace(); @@ -105,17 +102,6 @@ ObsSpace::ObsSpace(const ObsSpace & os, Log::trace() << "ObsSpace::ObsSpace (local) done" << std::endl; } -// ----------------------------------------------------------------------------- -/// Constructor added for generic 1d-var under development in ufo -template -ObsSpace::ObsSpace(const ObsSpace_ & os, const eckit::geometry::Point2 & center, - const eckit::Configuration & conf): obsdb_(), time_(oops::mpi::myself()) { - Log::trace() << "ObsSpace::ObsSpace (local) derived state starting" << std::endl; - util::Timer timer(classname(), "ObsSpace"); - obsdb_.reset(new ObsSpace_(os, center, conf)); - Log::trace() << "ObsSpace::ObsSpace (local) derived state done" << std::endl; -} - // ----------------------------------------------------------------------------- template diff --git a/src/oops/interface/State.h b/src/oops/interface/State.h index 421497551..b9655071a 100644 --- a/src/oops/interface/State.h +++ b/src/oops/interface/State.h @@ -48,7 +48,6 @@ class State : public util::Printable, State(const Geometry_ &, const eckit::Configuration &); State(const Geometry_ &, const State &); State(const State &); - explicit State(const State_ &); ~State(); State & operator=(const State &); // Is that used anywhere? @@ -144,18 +143,6 @@ State::State(const State & other) : state_(), commTime_(other.commTime_) // ----------------------------------------------------------------------------- -template -State::State(const State_ & target) : state_(), commTime_(oops::mpi::myself()) -{ - Log::trace() << "State::State starting copy from derived state" << std::endl; - util::Timer timer(classname(), "State"); - Log::warning() << "State::State creating State from derived state" << std::endl; - state_.reset(new State_(target)); - Log::trace() << "State::State copy from derived state done" << std::endl; -} - -// ----------------------------------------------------------------------------- - template State::~State() { Log::trace() << "State::~State starting" << std::endl; diff --git a/src/test/interface/State.h b/src/test/interface/State.h index 7c4c0a6e8..352c1ba59 100644 --- a/src/test/interface/State.h +++ b/src/test/interface/State.h @@ -115,11 +115,6 @@ template void testStateConstructors() { EXPECT(oops::is_close(xx4.norm(), norm, tol)); EXPECT(xx4.validTime() == vt); EXPECT(xx4.variables() == xx1->variables()); - -// Test explicit State(const State_ &); constructor -// needed for the 1dvar filter - State_ xx5(xx1->state()); - EXPECT(xx5.variables() == xx1->variables()); } // ----------------------------------------------------------------------------- From 31967259f5e6acc53da2e70176aa56328a88477a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Wed, 17 Mar 2021 14:59:23 -0600 Subject: [PATCH 076/142] Observers in CostJo (#1101) * Added Observers * Moved GetValuesPost inside Observers * Moved QC flags inside Observers * Use Observers in 4D H(x) * Review comments --- l95/src/lorenz95/ObsData1D.h | 17 +-- qg/model/GetValuesQG.cc | 20 ++-- qg/model/GetValuesQG.h | 3 +- src/CMakeLists.txt | 1 + src/oops/assimilation/CalcHofX.h | 1 - src/oops/assimilation/CostJo.h | 49 ++++---- src/oops/base/GetValuesPost.h | 16 +-- src/oops/base/ObsFilters.h | 9 +- src/oops/base/Observers.h | 186 +++++++++++++++++++++++++++++++ src/oops/runs/HofX4D.h | 31 +++--- 10 files changed, 260 insertions(+), 73 deletions(-) create mode 100644 src/oops/base/Observers.h diff --git a/l95/src/lorenz95/ObsData1D.h b/l95/src/lorenz95/ObsData1D.h index 99a6112df..be269825e 100644 --- a/l95/src/lorenz95/ObsData1D.h +++ b/l95/src/lorenz95/ObsData1D.h @@ -108,14 +108,17 @@ void ObsData1D::save(const std::string & name) const { // ----------------------------------------------------------------------------- template void ObsData1D::print(std::ostream & os) const { - ASSERT(data_.size() > 0); - DATATYPE zmin = data_.at(0); - DATATYPE zmax = data_.at(0); - for (size_t jj = 0; jj < data_.size(); ++jj) { - if (data_.at(jj) < zmin) zmin = data_.at(jj); - if (data_.at(jj) > zmax) zmax = data_.at(jj); + if (data_.size() > 0) { + DATATYPE zmin = data_.at(0); + DATATYPE zmax = data_.at(0); + for (size_t jj = 0; jj < data_.size(); ++jj) { + if (data_.at(jj) < zmin) zmin = data_.at(jj); + if (data_.at(jj) > zmax) zmax = data_.at(jj); + } + os << "Lorenz 95 nobs= " << data_.size() << " Min=" << zmin << ", Max=" << zmax; + } else { + os << "Lorenz 95 nobs= " << data_.size() << " --- No observations"; } - os << "Lorenz 95 nobs= " << data_.size() << " Min=" << zmin << ", Max=" << zmax; } // ----------------------------------------------------------------------------- } // namespace lorenz95 diff --git a/qg/model/GetValuesQG.cc b/qg/model/GetValuesQG.cc index 03ac0f566..515f68169 100644 --- a/qg/model/GetValuesQG.cc +++ b/qg/model/GetValuesQG.cc @@ -19,41 +19,35 @@ #include "model/LocationsQG.h" #include "model/StateQG.h" - namespace qg { // ----------------------------------------------------------------------------- /// Constructor, destructor // ----------------------------------------------------------------------------- -GetValuesQG::GetValuesQG(const GeometryQG & geom, const LocationsQG & locs, +GetValuesQG::GetValuesQG(const GeometryQG &, const LocationsQG & locs, const eckit::Configuration & conf) : locs_(locs), conf_(conf) { - oops::Log::trace() << "GetValuesQG constructor with config " - << conf_ << std::endl; + oops::Log::trace() << "GetValuesQG constructor with config " << conf_ << std::endl; } - // ----------------------------------------------------------------------------- /// Get state values at observation locations // ----------------------------------------------------------------------------- void GetValuesQG::fillGeoVaLs(const StateQG & state, const util::DateTime & t1, const util::DateTime & t2, GomQG & gom) const { + oops::Log::trace() << "GetValuesQG::fillGeoVaLs start" << std::endl; // the below call is an example if one wanted a different interpolation type const std::string interpType = conf_.getString("interpolation type", "default"); - if (interpType == "default" || - (interpType.compare(0, 8, "default_") == 0)) { - oops::Log::trace() << "GetValuesQG config = " - << conf_ << std::endl; - qg_getvalues_interp_f90(locs_, state.fields().toFortran(), - t1, t2, gom.toFortran()); + if (interpType == "default" || (interpType.compare(0, 8, "default_") == 0)) { + qg_getvalues_interp_f90(locs_, state.fields().toFortran(), t1, t2, gom.toFortran()); } else { - std::string err_message("interpolation type option " + - interpType + " not supported"); + std::string err_message("interpolation type option " + interpType + " not supported"); throw eckit::BadValue(err_message, Here()); } + oops::Log::trace() << "GetValuesQG::fillGeoVaLs done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/qg/model/GetValuesQG.h b/qg/model/GetValuesQG.h index 7da42ac23..a4b121fb5 100644 --- a/qg/model/GetValuesQG.h +++ b/qg/model/GetValuesQG.h @@ -38,8 +38,7 @@ class GetValuesQG : public util::Printable, static const std::string classname() {return "qg::GetValuesQG";} /// \brief saves all locations \p locs to use during filling GeoVaLs - GetValuesQG(const GeometryQG &, const LocationsQG & locs, - const eckit::Configuration &); + GetValuesQG(const GeometryQG &, const LocationsQG & locs, const eckit::Configuration &); ~GetValuesQG() {} /// \brief fills in \p geovals for all observations in the timeframe (\p t1, \p t2], diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5a4883e97..57cf6fab7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -119,6 +119,7 @@ oops/base/ObsEnsemble.h oops/base/ObsErrorBase.h oops/base/ObsErrors.h oops/base/Observations.h +oops/base/Observers.h oops/base/ObserversTLAD.h oops/base/ObserverTLAD.h oops/base/ObsFilterBase.h diff --git a/src/oops/assimilation/CalcHofX.h b/src/oops/assimilation/CalcHofX.h index ff65926bf..fec557723 100644 --- a/src/oops/assimilation/CalcHofX.h +++ b/src/oops/assimilation/CalcHofX.h @@ -144,7 +144,6 @@ void CalcHofX::initialize(const ObsAuxCtrls_ & obsaux, const int iteration) std::vector obsconfs = obsconfig_.getSubConfigurations(); ybias_ = &obsaux; filters_.clear(); - geovars_.clear(); for (size_t jj = 0; jj < obspaces_.size(); ++jj) { CalcHofXParameters observerParams; observerParams.deserialize(obsconfs[jj]); diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index 8fb4c68ef..699f9c0de 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -19,14 +19,13 @@ #include #include "eckit/config/LocalConfiguration.h" -#include "oops/assimilation/CalcHofX.h" #include "oops/assimilation/ControlIncrement.h" #include "oops/assimilation/ControlVariable.h" #include "oops/assimilation/CostTermBase.h" #include "oops/base/Departures.h" -#include "oops/base/GetValuesPost.h" #include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" +#include "oops/base/Observers.h" #include "oops/base/ObserversTLAD.h" #include "oops/base/ObsFilters.h" #include "oops/base/ObsSpaces.h" @@ -34,12 +33,11 @@ #include "oops/base/PostBaseTLAD.h" #include "oops/interface/Geometry.h" #include "oops/interface/Increment.h" +#include "oops/interface/ObsDataVector.h" #include "oops/interface/State.h" #include "oops/mpi/mpi.h" -#include "oops/util/ConfigFunctions.h" #include "oops/util/DateTime.h" #include "oops/util/Logger.h" -#include "oops/util/missingValues.h" namespace oops { @@ -64,10 +62,10 @@ template class CostJo : public CostTermBase Increment_; typedef ObsErrors ObsErrors_; typedef ObsSpaces ObsSpaces_; - typedef GetValuesPost GetValuesPost_; - typedef CalcHofX CalcHofX_; + typedef Observers Observers_; typedef ObserversTLAD ObserversTLAD_; typedef PostBaseTLAD PostBaseTLAD_; + typedef ObsDataVector ObsData_; public: /// Construct \f$ J_o\f$ from \f$ R\f$ and \f$ y_{obs}\f$. @@ -126,11 +124,10 @@ template class CostJo : public CostTermBase gradFG_; - /// Postprocessor passed by \f$ J_o\f$ to the model during integration. - std::shared_ptr getvals_; - /// Used for computing H(x) and running QC filters - CalcHofX_ calchofx_; + Observers_ observers_; + + std::vector > obserrs_; // Obs errors /// Linearized observation operators. std::shared_ptr pobstlad_; @@ -144,9 +141,15 @@ CostJo::CostJo(const eckit::Configuration & joConf, const eckit::mpi const eckit::mpi::Comm & ctime) : obsconf_(joConf), obspace_(obsconf_, comm, winbgn, winend, ctime), yobs_(obspace_, "ObsValue"), - Rmat_(), currentConf_(), gradFG_(), getvals_(), calchofx_(obspace_, obsconf_), + Rmat_(), currentConf_(), gradFG_(), observers_(obspace_, obsconf_), pobstlad_() { + for (size_t jj = 0; jj < obspace_.size(); ++jj) { + /// Allocate and read initial obs error + obserrs_.emplace_back(std::make_shared(obspace_[jj], + obspace_[jj].obsvariables(), "ObsError")); + } + Log::trace() << "CostJo::CostJo done" << std::endl; } @@ -158,15 +161,13 @@ CostJo::initialize(const CtrlVar_ & xx, const eckit::Configuration & Log::trace() << "CostJo::initialize start" << std::endl; currentConf_.reset(new eckit::LocalConfiguration(conf)); - calchofx_.initialize(xx.obsVar(), currentConf_->getInt("iteration")); + int iter = currentConf_->getInt("iteration"); - std::vector getValuesConfig = - util::vectoriseAndFilter(obsconf_, "get values"); + std::shared_ptr > > + getvals(observers_.initialize(xx.obsVar(), obserrs_, iter)); - getvals_.reset(new GetValuesPost_(obspace_, calchofx_.locations(), - calchofx_.requiredVars(), getValuesConfig)); Log::trace() << "CostJo::initialize done" << std::endl; - return getvals_; + return getvals; } // ----------------------------------------------------------------------------- @@ -174,21 +175,20 @@ CostJo::initialize(const CtrlVar_ & xx, const eckit::Configuration & template double CostJo::finalize() { Log::trace() << "CostJo::finalize start" << std::endl; - Observations_ yeqv = calchofx_.compute(getvals_->geovals()); + Observations_ yeqv = observers_.finalize(); Log::info() << "Jo Observation Equivalent:" << std::endl << yeqv << "End Jo Observation Equivalent" << std::endl; const int iterout = currentConf_->getInt("iteration"); // Sace current QC flags and obs error const std::string obsname = "hofx" + std::to_string(iterout); - const std::string qcname = "EffectiveQC" + std::to_string(iterout); - const std::string errname = "EffectiveError" + std::to_string(iterout); yeqv.save(obsname); - calchofx_.maskObsErrors(); - calchofx_.saveQcFlags(qcname); - calchofx_.saveObsErrors(errname); - calchofx_.saveObsErrors("EffectiveError"); // Obs error covariance is looking for that for now + const std::string errname = "EffectiveError" + std::to_string(iterout); + for (size_t jj = 0; jj < obserrs_.size(); ++jj) { + obserrs_[jj]->save(errname); + obserrs_[jj]->save("EffectiveError"); // Obs error covariance is looking for that for now + } // Set observation error covariance Rmat_.reset(new ObsErrors_(obsconf_, obspace_)); @@ -220,7 +220,6 @@ double CostJo::finalize() { ydep.save(depname); } - getvals_.reset(); currentConf_.reset(); Log::trace() << "CostJo::finalize done" << std::endl; return zjo; diff --git a/src/oops/base/GetValuesPost.h b/src/oops/base/GetValuesPost.h index 34d4d17a6..9631b1571 100644 --- a/src/oops/base/GetValuesPost.h +++ b/src/oops/base/GetValuesPost.h @@ -50,12 +50,11 @@ class GetValuesPost : public PostBase> { typedef std::vector> GetValuesVec_; typedef std::vector> GeoVaLsVec_; typedef std::vector> LocationsVec_; - typedef std::vector VariablesVec_; public: /// \brief Saves Locations and Variables to be processed GetValuesPost(const ObsSpaces_ &, - const LocationsVec_ &, const VariablesVec_ &, + const LocationsVec_ &, const std::vector &, const std::vector &); /// \brief Returns geovals filled in during the model run @@ -75,9 +74,8 @@ class GetValuesPost : public PostBase> { util::DateTime winend_; /// End of assimilation window util::Duration hslot_; /// Half time slot - const LocationsVec_ & locations_; /// locations of observations - const VariablesVec_ & geovars_; /// Variables needed from model + const std::vector geovars_; /// Variables needed from model GetValuesVec_ getvals_; /// GetValues used to fill in GeoVaLs GeoVaLsVec_ geovals_; /// GeoVaLs that are filled in const std::vector getvalsconfs_; /// configuration object @@ -88,16 +86,19 @@ class GetValuesPost : public PostBase> { template GetValuesPost::GetValuesPost(const ObsSpaces_ & obsdb, const LocationsVec_ & locations, - const VariablesVec_ & vars, + const std::vector & vars, const std::vector & confs) : PostBase(), winbgn_(obsdb.windowStart()), winend_(obsdb.windowEnd()), hslot_(), - locations_(locations), geovars_(vars), - getvalsconfs_(confs) {} + locations_(locations), geovars_(vars), getvalsconfs_(confs) +{ + Log::trace() << "GetValuesPost::GetValuesPost" << std::endl; +} // ----------------------------------------------------------------------------- template void GetValuesPost::fill(const State4D_ & xx) { + Log::trace() << "GetValuesPost::fill start" << std::endl; const size_t nstates = xx.size(); util::Duration tstep = winend_ - winbgn_; // for a single state // if using several states, compute the timestep and check that it's the same @@ -113,6 +114,7 @@ void GetValuesPost::fill(const State4D_ & xx) { for (size_t ii = 0; ii < nstates; ++ii) { doProcessing(xx[ii]); } + Log::trace() << "GetValuesPost::fill done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/ObsFilters.h b/src/oops/base/ObsFilters.h index 9d048d868..c9a6c51aa 100644 --- a/src/oops/base/ObsFilters.h +++ b/src/oops/base/ObsFilters.h @@ -63,6 +63,8 @@ class ObsFilters : public util::Printable, std::vector filters_; Variables geovars_; Variables diagvars_; + ObsDataPtr_ qcflags_; + ObsDataPtr_ obserr_; }; // ----------------------------------------------------------------------------- @@ -72,7 +74,7 @@ ObsFilters::ObsFilters(const ObsSpace_ & os, const std::vector> & filtersParams, ObsDataPtr_ qcflags, ObsDataPtr_ obserr, const int iteration) - : filters_(), geovars_(), diagvars_() { + : filters_(), geovars_(), diagvars_(), qcflags_(qcflags), obserr_(obserr) { Log::trace() << "ObsFilters::ObsFilters starting:\n"; for (const ObsFilterParametersWrapper &filterParams : filtersParams) Log::trace() << " " << filterParams << std::endl; @@ -81,7 +83,7 @@ ObsFilters::ObsFilters(const ObsSpace_ & os, if (filtersParams.size() > 0) { eckit::LocalConfiguration preconf; preconf.set("filter", "QCmanager"); - filters_.push_back(FilterFactory::create(os, preconf, qcflags, obserr)); + filters_.push_back(FilterFactory::create(os, preconf, qcflags_, obserr_)); } // Create the filters, only at 0-th iteration, or at iterations specified in "apply at iterations" @@ -95,7 +97,7 @@ ObsFilters::ObsFilters(const ObsSpace_ & os, } if (apply) { ObsFilterPtr_ tmp(FilterFactory::create(os, filterParams.filterParameters, - qcflags, obserr)); + qcflags_, obserr_)); geovars_ += tmp->requiredVars(); diagvars_ += tmp->requiredHdiagnostics(); filters_.push_back(tmp); @@ -130,6 +132,7 @@ void ObsFilters::postFilter(const ObsVector_ & hofx, const ObsDiags_ & diag for (const auto & filter : filters_) { filter->postFilter(hofx, diags); } + obserr_->mask(*qcflags_); } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/Observers.h b/src/oops/base/Observers.h new file mode 100644 index 000000000..8fd92d75d --- /dev/null +++ b/src/oops/base/Observers.h @@ -0,0 +1,186 @@ +/* + * (C) Copyright 2020 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ +#ifndef OOPS_BASE_OBSERVERS_H_ +#define OOPS_BASE_OBSERVERS_H_ + +#include +#include +#include + +#include "eckit/config/LocalConfiguration.h" + +#include "oops/base/GetValuesPost.h" +#include "oops/base/ObsAuxControls.h" +#include "oops/base/Observations.h" +#include "oops/base/ObsFilters.h" +#include "oops/base/ObsSpaces.h" +#include "oops/base/Variables.h" +#include "oops/interface/GeoVaLs.h" +#include "oops/interface/Locations.h" +#include "oops/interface/ObsDataVector.h" +#include "oops/interface/ObsDiagnostics.h" +#include "oops/interface/ObsOperator.h" +#include "oops/util/ConfigFunctions.h" +#include "oops/util/Logger.h" +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" + +namespace oops { + +template +class ObserversParameters : public Parameters { + OOPS_CONCRETE_PARAMETERS(ObserversParameters, Parameters) + + public: + oops::RequiredParameter obsOperator{"obs operator", this}; + oops::Parameter>> obsFilters{"obs filters", {}, this}; +}; + +// ----------------------------------------------------------------------------- + +/// \brief Computes observation operator (from GeoVaLs), applies bias correction +/// and runs QC filters +template +class Observers { + typedef GeoVaLs GeoVaLs_; + typedef GetValuesPost GetValuesPost_; + typedef Locations Locations_; + typedef ObsAuxControls ObsAuxCtrls_; + typedef ObsDiagnostics ObsDiags_; + typedef Observations Observations_; + typedef ObsFilters ObsFilters_; + typedef ObsOperator ObsOperator_; + typedef ObsSpaces ObsSpaces_; + template using ObsData_ = ObsDataVector; + template using ObsDataVec_ = std::vector>>; + + typedef std::vector> GeoVaLsVec_; + typedef std::vector> LocationsVec_; + typedef std::vector> ObsFiltersVec_; + typedef std::vector> ObsOperatorVec_; + typedef std::vector VariablesVec_; + + public: +/// \brief Initializes ObsOperators, Locations, and QC data + Observers(const ObsSpaces_ &, const eckit::Configuration &); + +/// \brief Initializes variables, obs bias, obs filters (could be different for +/// different iterations + std::shared_ptr > > initialize(const ObsAuxCtrls_ &, + ObsDataVec_ &, const int iter = 0); + +/// \brief Computes H(x) from the filled in GeoVaLs + Observations_ finalize(); + + private: + eckit::LocalConfiguration obsconfig_; + const ObsSpaces_ & obspaces_; // ObsSpaces used in H(x) + ObsOperatorVec_ obsops_; // Obs operators + LocationsVec_ locations_; // locations (made non local by GetValuesPost) + const ObsAuxCtrls_ * ybias_; // Obs bias + ObsFiltersVec_ filters_; // QC filters + std::shared_ptr getvals_; // Postproc passed to the model during integration. + ObsDataVec_ qcflags_; // QC flags + int iterout_; // Outer iteration + bool initialized_; +}; + +// ----------------------------------------------------------------------------- + +template +Observers::Observers(const ObsSpaces_ & obspaces, + const eckit::Configuration & config) : + obsconfig_(config), obspaces_(obspaces), obsops_(), locations_(), + ybias_(nullptr), filters_(), iterout_(0), initialized_(false) +{ + std::vector obsconfs = config.getSubConfigurations(); + for (size_t jj = 0; jj < obspaces_.size(); ++jj) { + ObserversParameters observerParams; + observerParams.deserialize(obsconfs[jj]); + /// Set up observation operators + obsops_.emplace_back(new ObsOperator_(obspaces_[jj], observerParams.obsOperator)); + locations_.emplace_back(new Locations_(obsops_[jj]->locations())); + qcflags_.emplace_back(new ObsData_(obspaces_[jj], obspaces_[jj].obsvariables())); + } + Log::trace() << "Observers constructed" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +std::shared_ptr > > +Observers::initialize(const ObsAuxCtrls_ & obsaux, ObsDataVec_ & obserrs, + const int iter) { + std::vector obsconfs = obsconfig_.getSubConfigurations(); + iterout_ = iter; + ybias_ = &obsaux; + filters_.clear(); + VariablesVec_ geovars(obspaces_.size()); // variables required from the model + for (size_t jj = 0; jj < obspaces_.size(); ++jj) { + ObserversParameters observerParams; + observerParams.deserialize(obsconfs[jj]); + /// Set up QC filters and run preprocess + filters_.emplace_back(new ObsFilters_(obspaces_[jj], observerParams.obsFilters, + qcflags_[jj], obserrs[jj], iterout_)); + filters_[jj]->preProcess(); + + /// Set up variables requested from the model + geovars[jj] += obsops_[jj]->requiredVars(); + geovars[jj] += (*ybias_)[jj].requiredVars(); + geovars[jj] += filters_[jj]->requiredVars(); + } + + std::vector gvconfs + = util::vectoriseAndFilter(obsconfig_, "get values"); + + getvals_.reset(new GetValuesPost_(obspaces_, locations_, geovars, gvconfs)); + + initialized_ = true; + Log::trace() << "Observers::initialize done" << std::endl; + return getvals_; +} + +// ----------------------------------------------------------------------------- + +template +Observations Observers::finalize() { + oops::Log::trace() << "Observers::finalize start" << std::endl; + ASSERT(initialized_); + + const GeoVaLsVec_ & geovals(getvals_->geovals()); + + Observations yobs(obspaces_); + for (size_t jj = 0; jj < obspaces_.size(); ++jj) { + /// call prior filters + filters_[jj]->priorFilter(*geovals[jj]); + /// compute H(x) + oops::Variables vars; + vars += filters_[jj]->requiredHdiagnostics(); + vars += (*ybias_)[jj].requiredHdiagnostics(); + ObsDiags_ ydiags(obspaces_[jj], *locations_[jj], vars); + obsops_[jj]->simulateObs(*geovals[jj], yobs[jj], (*ybias_)[jj], ydiags); + /// call posterior filters + filters_[jj]->postFilter(yobs[jj], ydiags); + } + initialized_ = false; + +// Save flags for diagnostics + const std::string qcname = "EffectiveQC" + std::to_string(iterout_); + for (const auto & qcflag : qcflags_) { + qcflag->save(qcname); + } + + oops::Log::trace() << "Observers::finalize done" << std::endl; + return yobs; +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_BASE_OBSERVERS_H_ diff --git a/src/oops/runs/HofX4D.h b/src/oops/runs/HofX4D.h index 6886ef9bd..cf40e43a1 100644 --- a/src/oops/runs/HofX4D.h +++ b/src/oops/runs/HofX4D.h @@ -18,12 +18,11 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" -#include "oops/assimilation/CalcHofX.h" -#include "oops/base/GetValuesPost.h" #include "oops/base/instantiateObsFilterFactory.h" #include "oops/base/ObsAuxControls.h" #include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" +#include "oops/base/Observers.h" #include "oops/base/ObsSpaces.h" #include "oops/base/PostProcessor.h" #include "oops/base/StateInfo.h" @@ -31,10 +30,10 @@ #include "oops/interface/Geometry.h" #include "oops/interface/Model.h" #include "oops/interface/ModelAuxControl.h" +#include "oops/interface/ObsDataVector.h" #include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" -#include "oops/util/ConfigFunctions.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" @@ -46,14 +45,15 @@ namespace oops { /// H(x) is perturbed. It is saved as "hofx" by default, or as specified "hofx group name" template class HofX4D : public Application { typedef Geometry Geometry_; - typedef GetValuesPost GetValuesPost_; typedef Model Model_; typedef ModelAuxControl ModelAux_; typedef ObsAuxControls ObsAux_; typedef Observations Observations_; typedef ObsErrors ObsErrors_; + typedef Observers Observers_; typedef ObsSpaces ObsSpaces_; typedef State State_; + typedef ObsDataVector ObsData_; public: // ----------------------------------------------------------------------------- @@ -107,22 +107,23 @@ template class HofX4D : public Application { ObsAux_ obsaux(obspaces, obsConfig); // Setup and run observer - CalcHofX hofx(obspaces, obsConfig); - hofx.initialize(obsaux); + Observers_ hofx(obspaces, obsConfig); -// run the model and compute H(x) - std::vector getValuesConfig = - util::vectoriseAndFilter(obsConfig, "get values"); + std::vector> obserrs; + for (size_t jj = 0; jj < obspaces.size(); ++jj) { + obserrs.emplace_back(std::make_shared(obspaces[jj], + obspaces[jj].obsvariables(), "ObsError")); + } - std::shared_ptr - getvals(new GetValuesPost_(obspaces, hofx.locations(), - hofx.requiredVars(), getValuesConfig)); +// run the model and compute H(x) + std::shared_ptr > > getvals(hofx.initialize(obsaux, obserrs)); post.enrollProcessor(getvals); model.forecast(xx, moderr, flength, post); - Observations_ yobs = hofx.compute(getvals->geovals()); - hofx.saveQcFlags("EffectiveQC"); - hofx.saveObsErrors("EffectiveError"); + Observations_ yobs = hofx.finalize(); + for (size_t jj = 0; jj < obserrs.size(); ++jj) { + obserrs[jj]->save("EffectiveError"); // Obs error covariance is looking for that for now + } Log::test() << "Final state: " << xx << std::endl; Log::test() << "H(x): " << std::endl << yobs << "End H(x)" << std::endl; From 1b3b1d7d4843eb55a8d55a7c6865cae983556c88 Mon Sep 17 00:00:00 2001 From: weihuang-jedi <77079547+weihuang-jedi@users.noreply.github.com> Date: Wed, 17 Mar 2021 15:33:29 -0600 Subject: [PATCH 077/142] Feature/omp log control (#1094) * Sync with less-print. * Sync with less-print, before add openMP directives. * Checking code: src/oops/assimilation/gletkf_mod.f90 Feb. 12, 2021 * Check in src/oops/base/ObsErrors.h, 2/14/2021 * 2/15/2021, OpenMP passed: src/oops/assimilation/LETKFSolver.h * Cleanup OpenMP changes. Remove the log:test() in src/oops/assimilation/LETKFSolver.h. Make that a later PR. * added OMP to letkf testing * Remove a commented out code block. Such block cause code-norm ctest failed. * Turn back to original ObsErrors.h as there is not much to gain in OpenMP, and do not want to intruduce worries. Switch off OpenMP in LETKESolver.h as it caused ctest failure. Did not make much change in gletkf_mod.f90. Some of the loops can use BLAS function, but after check the source code, did not feel much gain of switch over. * Reuse/redefine of "gamma_inv" may cause confusion. Revert this change back. Co-authored-by: Sergey Frolov Co-authored-by: Anna Shlyaeva --- l95/test/CMakeLists.txt | 1 + qg/test/CMakeLists.txt | 1 + src/oops/assimilation/LETKFSolver.h | 3 +++ src/oops/assimilation/gletkf_mod.f90 | 19 +++++++++++++++++++ 4 files changed, 24 insertions(+) diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index 740eabca9..0b8aab869 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -737,6 +737,7 @@ oops_add_test( TESTNAME letkf MODELNAME l95 YAMLNAME testinput/letkf.yaml EXENAME l95_letkf.x + OMP 2 TEST_DEPENDS test_l95_makeobs3d test_l95_genenspert ) oops_add_test( TESTNAME letkf_noobs diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index 64479c01e..3210bbf8b 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -773,4 +773,5 @@ oops_add_test( TESTNAME letkf MODELNAME qg YAMLNAME testinput/letkf.yaml EXENAME qg_letkf.x + OMP 2 TEST_DEPENDS test_qg_make_obs_3d test_qg_gen_ens_pert_B ) diff --git a/src/oops/assimilation/LETKFSolver.h b/src/oops/assimilation/LETKFSolver.h index 58137ed5f..17804a108 100644 --- a/src/oops/assimilation/LETKFSolver.h +++ b/src/oops/assimilation/LETKFSolver.h @@ -170,6 +170,7 @@ void LETKFSolver::computeWeights(const Departures_ & dy_oops, // work = Y^T R^-1 Y + (nens-1)/infl I double infl = inflopt.mult; Eigen::MatrixXd work = Yb*(diagInvR.asDiagonal()*Yb.transpose()); + work.diagonal() += Eigen::VectorXd::Constant(nens_, (nens_-1)/infl); // eigenvalues and eigenvectors of the above matrix @@ -208,6 +209,7 @@ void LETKFSolver::applyWeights(const IncrementEnsemble4D_ & bkg_pert for (size_t itime=0; itime < bkg_pert[0].size(); ++itime) { // make grid point forecast pert ensemble array Eigen::MatrixXd Xb(ngp, nens_); + // #pragma omp parallel for for (size_t iens=0; iens < nens_; ++iens) { LocalIncrement gp = bkg_pert[iens][itime].getLocal(i); std::vector tmp = gp.getVals(); @@ -248,6 +250,7 @@ void LETKFSolver::applyWeights(const IncrementEnsemble4D_ & bkg_pert } // assign Xa to ana_pert + // #pragma omp parallel for private(tmp1) for (size_t iens=0; iens < nens_; ++iens) { for (size_t iv=0; iv < ngp; ++iv) { tmp1[iv] = Xa(iv, iens)+xa(iv); // if Xa = Xb*Wa; diff --git a/src/oops/assimilation/gletkf_mod.f90 b/src/oops/assimilation/gletkf_mod.f90 index 501ed0418..2665d9cc9 100644 --- a/src/oops/assimilation/gletkf_mod.f90 +++ b/src/oops/assimilation/gletkf_mod.f90 @@ -163,10 +163,13 @@ subroutine letkf_core(nobsl,hxens,hxens_orig,dep,& rrloc = sqrt(rrloc) normfact = sqrt(real((nanals/neigv)-1,r_kind)) ! normalize so dot product is covariance + +!$OMP PARALLEL DO PRIVATE(nanal) shared(hxens, nobsl, rrloc, normfact) do nanal=1,nanals hxens(nanal,1:nobsl) = hxens(nanal,1:nobsl) * & rrloc(1:nobsl)/normfact end do +!$OMP END PARALLEL DO ! compute eigenvectors/eigenvalues of HZ^T HZ (left SV) ! (in Bishop paper HZ is nobsl, nanals, here is it nanals, nobsl) @@ -195,6 +198,8 @@ subroutine letkf_core(nobsl,hxens,hxens_orig,dep,& if (ierr .ne. 0) print *,'warning: dsyev* failed, ierr=',ierr deallocate(work1,iwork,work3) ! no longer needed gamma_inv = 0.0_r_kind + +!$OMP PARALLEL DO PRIVATE(nanal) do nanal=1,nanals if (evals(nanal) > eps) then gamma_inv(nanal) = 1./evals(nanal) @@ -202,15 +207,19 @@ subroutine letkf_core(nobsl,hxens,hxens_orig,dep,& evals(nanal) = 0.0_r_kind endif enddo +!$OMP END PARALLEL DO + ! gammapI used in calculation of posterior cov in ensemble space gammapI = evals+1.0 deallocate(evals) ! create HZ^T R**-1/2 allocate(shxens(nanals,nobsl)) +!$OMP PARALLEL DO PRIVATE(nanal) do nanal=1,nanals shxens(nanal,1:nobsl) = hxens(nanal,1:nobsl) * rrloc(1:nobsl) end do +!$OMP END PARALLEL DO deallocate(rrloc) ! compute factor to multiply with model space ensemble perturbations @@ -219,10 +228,12 @@ subroutine letkf_core(nobsl,hxens,hxens_orig,dep,& ! in Bishop paper (eqs 10-12). allocate(swork3(nanals,nanals),swork2(nanals,nanals),pa(nanals,nanals)) +!$OMP PARALLEL DO PRIVATE(nanal) do nanal=1,nanals swork3(nanal,:) = evecs(nanal,:)/gammapI swork2(nanal,:) = evecs(nanal,:) enddo +!$OMP END PARALLEL DO ! pa = C (Gamma + I)**-1 C^T (analysis error cov in ensemble space) !pa = matmul(swork3,transpose(swork2)) @@ -232,14 +243,18 @@ subroutine letkf_core(nobsl,hxens,hxens_orig,dep,& ! (nanals, nobsl) x (nobsl,) = (nanals,) ! in Bishop paper HZ is nobsl, nanals, here is it nanals, nobsl allocate(swork1(nanals)) +!$OMP PARALLEL DO PRIVATE(nanal) do nanal=1,nanals swork1(nanal) = sum(shxens(nanal,:)*dep(:)) end do +!$OMP END PARALLEL DO ! wts_ensmean = C (Gamma + I)**-1 C^T (HZ)^ T R**-1/2 (y - HXmean) ! (nanals, nanals) x (nanals,) = (nanals,) +!$OMP PARALLEL DO PRIVATE(nanal) do nanal=1,nanals wts_ensmean(nanal) = sum(pa(nanal,:)*swork1(:))/normfact end do +!$OMP END PARALLEL DO !if (.not. denkf .and. getkf_inflation) then ! allocate(paens(nanals,nanals)) @@ -262,10 +277,12 @@ subroutine letkf_core(nobsl,hxens,hxens_orig,dep,& pa = 0.5*pa else gammapI = sqrt(1.0/gammapI) +!$OMP PARALLEL DO PRIVATE(nanal) do nanal=1,nanals swork3(nanal,:) = & evecs(nanal,:)*(1.-gammapI(:))*gamma_inv(:) enddo +!$OMP END PARALLEL DO ! swork2 still contains eigenvectors, over-write pa ! pa = C [ (I - (Gamma+I)**-1/2)*Gamma**-1 ] C^T !pa = matmul(swork3,transpose(swork2)) @@ -308,9 +325,11 @@ subroutine letkf_core(nobsl,hxens,hxens_orig,dep,& ! (nanals, nanals) x (nanals, nanals) deallocate(shxens,pa) gammapI = sqrt(1.0/gammapI) +!$OMP PARALLEL DO PRIVATE(nanal) do nanal=1,nanals swork3(nanal,:) = evecs(nanal,:)*gammapI enddo +!$OMP END PARALLEL DO ! swork2 already contains evecs ! wts_ensperts = ! C (Gamma + I)**-1/2 C^T (square root of analysis error cov in ensemble space) From 6d85c7d83b1838d3ba8bd311ca5eabf4887084e6 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 17 Mar 2021 15:55:55 -0600 Subject: [PATCH 078/142] remove lines in CMakeLists that are not used (#1104) --- src/CMakeLists.txt | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 57cf6fab7..f147b5f97 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -447,39 +447,8 @@ file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test/testinput) CREATE_SYMLINK( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ${oops_test_input} ) file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/test/testoutput) -function(generate_fortran_bindings output filename) - - set( options "" ) - set( single_value_args OUTPUT MODULE ) - set( multi_value_args "" ) - cmake_parse_arguments( _PAR "${options}" "${single_value_args}" "${multi_value_args}" ${_FIRST_ARG} ${ARGN} ) - - get_filename_component(base ${filename} NAME_WE) - set(base_abs ${CMAKE_CURRENT_SOURCE_DIR}/${base}) - set(outfile ${CMAKE_CURRENT_BINARY_DIR}/${base}_c_binding.f90) - - if( _PAR_OUTPUT ) - set(outfile ${_PAR_OUTPUT}) - endif() - set(${output} ${${output}} ${outfile} PARENT_SCOPE) - - if( _PAR_MODULE ) - add_custom_command( - OUTPUT ${outfile} - COMMAND python ${PROJECT_SOURCE_DIR}/tools/c2f.py ${CMAKE_CURRENT_SOURCE_DIR}/${filename} -o ${outfile} -m ${_PAR_MODULE} - DEPENDS ${filename} ) - else() - add_custom_command( - OUTPUT ${outfile} - COMMAND python ${PROJECT_SOURCE_DIR}/tools/c2f.py ${CMAKE_CURRENT_SOURCE_DIR}/${filename} -o ${outfile} - DEPENDS ${filename} ) - endif() - set_source_files_properties(${outfile} PROPERTIES GENERATED TRUE) -endfunction() - ecbuild_add_library( TARGET ${PROJECT_NAME} - SOURCES ${FORTRAN_BINDINGS} - ${oops_src_files} + SOURCES ${oops_src_files} INSTALL_HEADERS LISTED HEADER_DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} LINKER_LANGUAGE CXX ) From 8f720605b84a225fc7403fec1187f41af0ff1663 Mon Sep 17 00:00:00 2001 From: Mark Miesch Date: Wed, 17 Mar 2021 16:18:33 -0600 Subject: [PATCH 079/142] remove output file before running test (#1105) Co-authored-by: Anna Shlyaeva --- tools/test_wrapper.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/test_wrapper.sh b/tools/test_wrapper.sh index 85673d081..6ee4d7818 100755 --- a/tools/test_wrapper.sh +++ b/tools/test_wrapper.sh @@ -15,6 +15,9 @@ ftol=$6 idif=$7 mpicmd=$8 +# remove existing run file +rm testoutput/${runfile} + # Run Test cmd="${mpicmd} ${exename} ${yamlname} testoutput/${runfile}" echo ${cmd} From 6563a90aa3d14a7759c769af66a21e5ebc51cdcb Mon Sep 17 00:00:00 2001 From: Clementine Gas <43183478+cmgas@users.noreply.github.com> Date: Wed, 17 Mar 2021 18:14:19 -0600 Subject: [PATCH 080/142] Option to save Krylov basis in Lanczos algorithm (#1103) * Move writeIncrement to function in MinimizerUtils * Add diagnostics to DRPLanczos * Removoe useless print * Rename block, add tolerance criteria, change ref files * To be reversed, static int for outerloop * Remove outerloop counter * Re-add online diagnostics yaml key * Coding norms * Remove outer loop and change ref * Correction wrong config! * Rename writeIncrement to writeCtrlIncrement, add type=base in qg model * Fix for l95 base type * Useless prints * Rename function, update yaml * Revert qg yaml change and add save in block Lanczos * Adding the space after the Analysis in test file * Adapt to comments Co-authored-by: Anna Shlyaeva --- l95/src/lorenz95/IncrementL95.cc | 5 ++ l95/src/lorenz95/StateL95.cc | 5 ++ l95/test/testinput/4dvar.drplanczos.yaml | 7 +++ l95/test/testinput/eda.3dvar.block.1.yaml | 7 +++ l95/test/testinput/eda.3dvar.block.2.yaml | 7 +++ qg/model/qg_tools_mod.F90 | 10 +++- qg/test/testinput/4dvar_drplanczos.yaml | 7 +++ .../assimilation/DRPBlockLanczosMinimizer.h | 14 ++++- src/oops/assimilation/DRPLanczosMinimizer.h | 12 +++- src/oops/assimilation/Minimizer.h | 31 +--------- src/oops/assimilation/MinimizerUtils.h | 59 +++++++++++++++++++ 11 files changed, 128 insertions(+), 36 deletions(-) diff --git a/l95/src/lorenz95/IncrementL95.cc b/l95/src/lorenz95/IncrementL95.cc index bea4c7313..309ee8d85 100644 --- a/l95/src/lorenz95/IncrementL95.cc +++ b/l95/src/lorenz95/IncrementL95.cc @@ -172,6 +172,11 @@ void IncrementL95::write(const eckit::Configuration & config) const { std::string type = config.getString("type"); std::string filename = dir+"/"+exp+"."+type; + if (type == "krylov") { + std::string iter = config.getString("iteration"); + filename += "."+iter; + } + const util::DateTime antime(config.getString("date")); filename += "."+antime.toString(); const util::Duration step = time_ - antime; diff --git a/l95/src/lorenz95/StateL95.cc b/l95/src/lorenz95/StateL95.cc index c5ad9a0c2..9ecda8bc7 100644 --- a/l95/src/lorenz95/StateL95.cc +++ b/l95/src/lorenz95/StateL95.cc @@ -144,6 +144,11 @@ void StateL95::write(const eckit::Configuration & config) const { filename += "."+time_.toString(); } + if (type == "krylov") { + std::string iter = config.getString("iteration"); + filename += "."+iter+"."+time_.toString(); + } + sf::swapNameMember(config, filename); oops::Log::trace() << "StateL95::write opening " << filename << std::endl; diff --git a/l95/test/testinput/4dvar.drplanczos.yaml b/l95/test/testinput/4dvar.drplanczos.yaml index 120442167..17abe80d2 100644 --- a/l95/test/testinput/4dvar.drplanczos.yaml +++ b/l95/test/testinput/4dvar.drplanczos.yaml @@ -32,6 +32,13 @@ cost function: variational: minimizer: algorithm: DRPLanczos + online diagnostics: + write basis: true + krylov basis: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: 4dvar.drplanczos + type: krylov iterations: - ninner: 10 gradient norm reduction: 1.0e-10 diff --git a/l95/test/testinput/eda.3dvar.block.1.yaml b/l95/test/testinput/eda.3dvar.block.1.yaml index 0e428834c..8931bcc49 100644 --- a/l95/test/testinput/eda.3dvar.block.1.yaml +++ b/l95/test/testinput/eda.3dvar.block.1.yaml @@ -27,6 +27,13 @@ variational: minimizer: algorithm: DRPBlockLanczos members: 2 + online diagnostics: + write basis: true + krylov basis: + datadir: Data + date: 2010-01-01T00:00:00Z + exp: 3dvar.block.m1 + type: krylov iterations: - geometry: resol: 40 diff --git a/l95/test/testinput/eda.3dvar.block.2.yaml b/l95/test/testinput/eda.3dvar.block.2.yaml index 3d1216f10..c65cffc3e 100644 --- a/l95/test/testinput/eda.3dvar.block.2.yaml +++ b/l95/test/testinput/eda.3dvar.block.2.yaml @@ -27,6 +27,13 @@ variational: minimizer: algorithm: DRPBlockLanczos members: 2 + online diagnostics: + write basis: true + krylov basis: + datadir: Data + date: 2010-01-01T00:00:00Z + exp: 3dvar.block.m2 + type: krylov iterations: - geometry: resol: 40 diff --git a/qg/model/qg_tools_mod.F90 b/qg/model/qg_tools_mod.F90 index 7142f4f97..f9980e230 100644 --- a/qg/model/qg_tools_mod.F90 +++ b/qg/model/qg_tools_mod.F90 @@ -42,7 +42,7 @@ function genfilename(f_conf,length,vdate) ! Local variables integer :: lenfn -character(len=length) :: fdbdir,expver,typ,validitydate,referencedate,sstep,mmb +character(len=length) :: fdbdir,expver,typ,validitydate,referencedate,sstep,mmb,iter character(len=2*length) :: prefix character(len=:),allocatable :: str @@ -89,6 +89,14 @@ function genfilename(f_conf,length,vdate) genfilename = trim(prefix)//'.'//trim(validitydate)//'.nc' endif +if (typ=='krylov') then + call f_conf%get_or_die("iteration",str) + iter = str + call datetime_to_string(vdate,validitydate) + lenfn = lenfn+1+len_trim(iter)+1+len_trim(validitydate) + genfilename = trim(prefix)//'.'// trim(iter)//'.'//trim(validitydate)//'.nc' +endif + ! Check filename length if (lenfn>length) call abor1_ftn('genfilename: filename too long') diff --git a/qg/test/testinput/4dvar_drplanczos.yaml b/qg/test/testinput/4dvar_drplanczos.yaml index 54a760b1f..1c53fce40 100644 --- a/qg/test/testinput/4dvar_drplanczos.yaml +++ b/qg/test/testinput/4dvar_drplanczos.yaml @@ -61,6 +61,13 @@ variational: algorithm: DRPLanczos preconditioner: maxpairs: 3 + online diagnostics: + write basis: true + krylov basis: + datadir: Data + date: 2010-01-01T12:00:00Z + exp: 4dvar.drplanczos + type: krylov iterations: - ninner: 10 gradient norm reduction: 1.0e-10 diff --git a/src/oops/assimilation/DRPBlockLanczosMinimizer.h b/src/oops/assimilation/DRPBlockLanczosMinimizer.h index e69662062..fe5acc55c 100644 --- a/src/oops/assimilation/DRPBlockLanczosMinimizer.h +++ b/src/oops/assimilation/DRPBlockLanczosMinimizer.h @@ -39,8 +39,8 @@ template class DRPBlockLanczosMinimizer : typedef CostFunction CostFct_; typedef ControlIncrement CtrlInc_; typedef HtRinvHMatrix HtRinvH_; - typedef Eigen::VectorXd eigenvec_; - typedef Eigen::MatrixXd eigenmat_; + typedef Eigen::VectorXd eigenvec_; + typedef Eigen::MatrixXd eigenmat_; public: const std::string classname() const override {return "DRPBlockLanczosMinimizer";} @@ -61,12 +61,17 @@ template class DRPBlockLanczosMinimizer : void HtRinvH0(const CtrlInc_ &, CtrlInc_ &, const HtRinvH_ &, int &, const eckit::mpi::Comm &, CtrlInc_ &); + // For MPI purposes const int members_; const int ntasks_; const int tasks_per_member_; const int global_task_; const int mymember_; const int local_task_; + + // For diagnostics + eckit::LocalConfiguration diagConf_; + int outerLoop_; }; // =============================================================================================== @@ -81,7 +86,8 @@ DRPBlockLanczosMinimizer::DRPBlockLanczosMinimizer(const eckit::Conf : DRMinimizer(J), members_(conf.getInt("members")), ntasks_(oops::mpi::world().size()), tasks_per_member_(ntasks_/members_), global_task_(oops::mpi::world().rank()), - mymember_(global_task_ / tasks_per_member_), local_task_(global_task_%tasks_per_member_) {} + mymember_(global_task_ / tasks_per_member_), local_task_(global_task_%tasks_per_member_), + diagConf_(conf), outerLoop_(0) {} // ----------------------------------------------------------------------------------------------- @@ -227,6 +233,7 @@ double DRPBlockLanczosMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, SSLK = - (ss.block(ll*members_, 0, members_, members_)); apply_proj(xh, *Vbase[ll], SSLK, gestag, CommGeo, temp1); apply_proj(xx, *Zbase[ll], SSLK, gestag, CommGeo, temp1); + if (outerLoop_ == 0) writeKrylovBasis(diagConf_, *Zbase[ll], ll); Log::info() << " Norm reduction all members (" << std::setw(2) << ll+1 << ") = " << norm_red_all.block(ll, 0, 1, members_) << std::endl; @@ -239,6 +246,7 @@ double DRPBlockLanczosMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, } eckit::mpi::deleteComm(CommGeoName); + ++outerLoop_; return normReduction; } diff --git a/src/oops/assimilation/DRPLanczosMinimizer.h b/src/oops/assimilation/DRPLanczosMinimizer.h index f2c103810..2da522f16 100644 --- a/src/oops/assimilation/DRPLanczosMinimizer.h +++ b/src/oops/assimilation/DRPLanczosMinimizer.h @@ -92,15 +92,19 @@ template class DRPLanczosMinimizer : public DRMini std::vector> zvecs_; std::vector alphas_; std::vector betas_; + + // For diagnostics + eckit::LocalConfiguration diagConf_; + int outerLoop_; }; // ============================================================================= template DRPLanczosMinimizer::DRPLanczosMinimizer(const eckit::Configuration & conf, - const CostFct_ & J) - : DRMinimizer(J), lmp_(conf), - hvecs_(), vvecs_(), zvecs_(), alphas_(), betas_() {} + const CostFct_ & J) + : DRMinimizer(J), lmp_(conf), hvecs_(), vvecs_(), zvecs_(), alphas_(), + betas_(), diagConf_(conf), outerLoop_(0) {} // ----------------------------------------------------------------------------- @@ -240,8 +244,10 @@ double DRPLanczosMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, Ctr for (unsigned int jj = 0; jj < ss.size(); ++jj) { dx.axpy(ss[jj], *zvecs_[jj]); dxh.axpy(ss[jj], *hvecs_[jj]); + if (outerLoop_ == 0) writeKrylovBasis(diagConf_, *zvecs_[jj], jj); } + ++outerLoop_; return normReduction; } diff --git a/src/oops/assimilation/Minimizer.h b/src/oops/assimilation/Minimizer.h index 38c6f5115..b74d125a5 100644 --- a/src/oops/assimilation/Minimizer.h +++ b/src/oops/assimilation/Minimizer.h @@ -22,6 +22,7 @@ #include "oops/assimilation/DualVector.h" #include "oops/assimilation/HMatrix.h" #include "oops/assimilation/HtMatrix.h" +#include "oops/assimilation/MinimizerUtils.h" #include "oops/interface/Increment.h" #include "oops/interface/State.h" #include "oops/util/dot_product.h" @@ -59,7 +60,6 @@ template class Minimizer : private boost::noncopya void tlmApproxTest(const H_ &); void tlmTaylorTest(const H_ &); - void writeIncrement(const eckit::Configuration & config, const CtrlInc_ &); void tlmPropagTest(const eckit::Configuration & config, const CtrlInc_ &); const CostFct_ & J_; @@ -81,7 +81,7 @@ Minimizer::minimize(const eckit::Configuration & config) { ControlIncrement * dx = this->doMinimize(config); // Write increment - this->writeIncrement(config, *dx); + writeIncrement(config, *dx, outerIteration_); // TLM propagation test this->tlmPropagTest(config, *dx); @@ -94,33 +94,6 @@ Minimizer::minimize(const eckit::Configuration & config) { // ----------------------------------------------------------------------------- -template -void Minimizer::writeIncrement(const eckit::Configuration & config, - const CtrlInc_ & dx) { -// Write out the increment - - if (config.has("online diagnostics")) { - const eckit::LocalConfiguration onlineDiag(config, "online diagnostics"); - bool writeinc = onlineDiag.getBool("write increment", false); - - if (writeinc) { - // print log - Log::info() << "Write Increment - starting: " << outerIteration_ - << std::endl << std::endl; - - const eckit::LocalConfiguration incConf(config, "increment"); - - // write increment - dx.write(incConf); - - // print log - Log::info() << std::endl << "Write Increment: done." << std::endl; - } - } -} - -// ----------------------------------------------------------------------------- - template void Minimizer::tlmTests(const eckit::Configuration & config) { // Tangent Linear tests diff --git a/src/oops/assimilation/MinimizerUtils.h b/src/oops/assimilation/MinimizerUtils.h index 70e8a8070..36634608a 100644 --- a/src/oops/assimilation/MinimizerUtils.h +++ b/src/oops/assimilation/MinimizerUtils.h @@ -8,6 +8,12 @@ #ifndef OOPS_ASSIMILATION_MINIMIZERUTILS_H_ #define OOPS_ASSIMILATION_MINIMIZERUTILS_H_ +#include + +#include "eckit/config/Configuration.h" + +#include "oops/assimilation/ControlIncrement.h" + namespace oops { /// Prints to Log::info gradient reduction \p grad and normalized gradient reduction \p norm @@ -17,6 +23,59 @@ void printNormReduction(int iteration, const double & grad, const double & norm) /// iteration \p iteration void printQuadraticCostFunction(int iteration, const double & costJ, const double & costJb, const double & costJoJc); + +// ----------------------------------------------------------------------------- + +template +void writeIncrement(const eckit::Configuration & config, + const ControlIncrement & dx, const int & loop) { +// Write out the increment + + if (config.has("online diagnostics")) { + const eckit::LocalConfiguration onlineDiag(config, "online diagnostics"); + bool writeinc = onlineDiag.getBool("write increment", false); + + if (writeinc) { + // print log + Log::info() << "Write Increment - starting: " << loop << std::endl << std::endl; + + const eckit::LocalConfiguration incConf(config, "increment"); + + // write increment + dx.write(incConf); + + // print log + Log::info() << std::endl << "Write Increment: done." << std::endl; + } + } +} + +// ----------------------------------------------------------------------------- + +template +void writeKrylovBasis(const eckit::Configuration & config, + const ControlIncrement & dx, + const int & loop) { +// Write out the increment + if (config.has("online diagnostics")) { + eckit::LocalConfiguration diagConf(config, "online diagnostics"); + bool writeinc = diagConf.getBool("write basis", false); + + if (writeinc) { + // print log + Log::info() << "Write Krylov Basis: starting: " << loop << std::endl; + + eckit::LocalConfiguration basisConf(config, "krylov basis"); + basisConf.set("iteration", loop); + + // write increment + dx.write(basisConf); + + // print log + Log::info() << "Write Krylov Basis: done." << std::endl; + } + } +} // ----------------------------------------------------------------------------- } // namespace oops From 54fe9081120ace4b185a212ebff005c89252634c Mon Sep 17 00:00:00 2001 From: frolovsa <55715838+frolovsa@users.noreply.github.com> Date: Thu, 18 Mar 2021 11:57:27 -0600 Subject: [PATCH 081/142] Feature/halo4 (#1095) * first go at passing geometry to obs space constructor * fixed configuration assignment bug * debug prints, migrating to orion * change default interface to ObsSpaces.h * nebugging, need to update develop * remove debug prints * fixed coding norms * move obs config update from ObsSpaces to LocalEnsembleDA (#1107) Co-authored-by: Anna Shlyaeva --- src/oops/runs/LocalEnsembleDA.h | 60 +++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/src/oops/runs/LocalEnsembleDA.h b/src/oops/runs/LocalEnsembleDA.h index 7b9f0c052..f95a04b56 100644 --- a/src/oops/runs/LocalEnsembleDA.h +++ b/src/oops/runs/LocalEnsembleDA.h @@ -8,8 +8,10 @@ #ifndef OOPS_RUNS_LOCALENSEMBLEDA_H_ #define OOPS_RUNS_LOCALENSEMBLEDA_H_ +#include #include #include +#include #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/instantiateLocalEnsembleSolverFactory.h" @@ -74,9 +76,26 @@ template class LocalEnsembleDA : public Applicati const eckit::LocalConfiguration geometryConfig(fullConfig, "geometry"); const Geometry_ geometry(geometryConfig, this->getComm()); + // if any of the obs. spaces uses Halo distribution it will need to know the geometry + // of the local grid on this PE + eckit::LocalConfiguration obsConfig(fullConfig, "observations"); + std::vector patchCenter(2, 0.0); + double patchRadius = 0.0; + computePatchGeometry(geometry, patchCenter, patchRadius); + + // update observations configs with information on patch center and radius + std::vector obsConfigs = obsConfig.getSubConfigurations(); + for (auto & conf : obsConfigs) { + conf.set("obs space.center", patchCenter); + conf.set("obs space.radius", patchRadius); + } + eckit::LocalConfiguration tmp; + tmp.set("observations", obsConfigs); + obsConfig = tmp.getSubConfiguration("observations"); + // Setup observations - const eckit::LocalConfiguration obsConfig(fullConfig, "observations"); - ObsSpaces_ obsdb(obsConfig, this->getComm(), winbgn, winend); + const eckit::mpi::Comm & time = oops::mpi::myself(); + ObsSpaces_ obsdb(obsConfig, this->getComm(), winbgn, winend, time); Observations_ yobs(obsdb, "ObsValue"); // Get background configurations @@ -92,7 +111,6 @@ template class LocalEnsembleDA : public Applicati // set up solver std::unique_ptr solver = LocalEnsembleSolverFactory::create(obsdb, geometry, fullConfig, nens); - // test prints for the prior ensemble bool do_test_prints = driverConfig.getBool("do test prints", true); if (do_test_prints) { @@ -235,6 +253,42 @@ template class LocalEnsembleDA : public Applicati return "oops::LocalEnsembleDA<" + MODEL::name() + ", " + OBS::name() + ">"; } + void computePatchGeometry(const Geometry_ & geometry, std::vector & patchCenter, + double & patchRadius) const { + // since Halo distribution is only implemented in ioda, we can assume that + // x is lon and y is lat on Earth. then we need to compute average of x on a circle + eckit::geometry::Point2 gptmp; + const double radius_earth = 6.371e6; + const double deg2rad = 3.14159265/180.0; + + // compute patch center + double sinXmean = 0; + double cosXmean = 0; + double ymean = 0; + int n = 0; + for (GeometryIterator_ i = geometry.begin(); i != geometry.end(); ++i) { + gptmp = *i; + cosXmean += cos(gptmp.x()*deg2rad); + sinXmean += sin(gptmp.x()*deg2rad); + ymean += gptmp.y(); + ++n; + } + cosXmean = cosXmean/static_cast(n); + sinXmean = sinXmean/static_cast(n); + patchCenter[0] = atan2(sinXmean, cosXmean)/deg2rad; + patchCenter[1] = ymean/static_cast(n); + + // compute radius + eckit::geometry::Point2 center(patchCenter[0], patchCenter[1]); + patchRadius = 0; + for (GeometryIterator_ i = geometry.begin(); i != geometry.end(); ++i) { + double dist = eckit::geometry::Sphere::distance(radius_earth, center, *i); + patchRadius = fmax(patchRadius, dist); + } + Log::debug() << "patch center=" << patchCenter + << " patch radius=" << patchRadius << std::endl; + } + void saveVariance(const eckit::LocalConfiguration & outConfig, const IncrementEnsemble4D_ & perts, const bool do_test_prints, const std::string & strOut) const { // save and optionaly print varaince of an IncrementEnsemble4D_ object From 972ea7a0eea0a77fe83c3243c395ff2bdb02e3cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Fri, 19 Mar 2021 10:23:46 -0600 Subject: [PATCH 082/142] Improve 3D H(x) efficiency (#1098) * Rename HofXNoModel into HofX3D * Fix for ewok * Rename HofXNoModel into HofX3D * Fix for ewok * Faster H(x) version 1 * Added HofX3Dslow * Added timestamp * QG convert state * Adressing reviews --- l95/src/executables/HofX3D.cc | 2 + l95/src/lorenz95/CMakeLists.txt | 1 + l95/src/lorenz95/ObsBias.cc | 3 +- l95/src/lorenz95/ObservationL95.cc | 2 +- l95/src/lorenz95/ObservationTLAD.cc | 3 +- .../lorenz95/instantiateL95ChangeVarFactory.h | 24 ++++ qg/mains/qgHofX3D.cc | 2 + qg/model/ChangeVarQG.cc | 21 ++- qg/model/ChangeVarQG.h | 4 - qg/model/ChangeVarTLADQG.cc | 4 +- qg/model/QgFortran.h | 5 +- qg/model/instantiateQgChangeVarFactory.h | 1 + qg/model/qg_change_var_interface.F90 | 35 ++++- qg/model/qg_change_var_mod.F90 | 101 ++++++++++++--- qg/model/qg_convert_q_to_x_mod.F90 | 10 +- qg/model/qg_convert_x_to_q_mod.F90 | 10 +- qg/model/qg_convert_x_to_uv_mod.F90 | 4 +- qg/model/qg_fields_mod.F90 | 104 +++++++++++++-- qg/test/testoutput/convertstate.test | 8 +- src/CMakeLists.txt | 2 + src/oops/interface/ChangeVariables.h | 120 ++++++++++++++++++ src/oops/interface/VariableChange.h | 11 +- src/oops/runs/HofX3D.h | 20 ++- src/oops/runs/HofX3Dslow.h | 120 ++++++++++++++++++ src/oops/util/LibOOPS.cc | 9 ++ src/test/interface/VariableChange.h | 6 +- 26 files changed, 550 insertions(+), 82 deletions(-) create mode 100644 l95/src/lorenz95/instantiateL95ChangeVarFactory.h create mode 100644 src/oops/interface/ChangeVariables.h create mode 100644 src/oops/runs/HofX3Dslow.h diff --git a/l95/src/executables/HofX3D.cc b/l95/src/executables/HofX3D.cc index 69d56adb0..22948ec97 100644 --- a/l95/src/executables/HofX3D.cc +++ b/l95/src/executables/HofX3D.cc @@ -5,12 +5,14 @@ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ +#include "lorenz95/instantiateL95ChangeVarFactory.h" #include "lorenz95/L95Traits.h" #include "oops/runs/HofX3D.h" #include "oops/runs/Run.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); + lorenz95::instantiateL95ChangeVarFactory(); oops::HofX3D hofx; return run.execute(hofx); } diff --git a/l95/src/lorenz95/CMakeLists.txt b/l95/src/lorenz95/CMakeLists.txt index 8290f2431..03f858e2f 100644 --- a/l95/src/lorenz95/CMakeLists.txt +++ b/l95/src/lorenz95/CMakeLists.txt @@ -17,6 +17,7 @@ set( _l95_srcs GomL95.h IncrementL95.cc IncrementL95.h + instantiateL95ChangeVarFactory.h instantiateLocalizationFactory.h Iterator.cc Iterator.h diff --git a/l95/src/lorenz95/ObsBias.cc b/l95/src/lorenz95/ObsBias.cc index 9189fdf1f..3c885dd90 100644 --- a/l95/src/lorenz95/ObsBias.cc +++ b/l95/src/lorenz95/ObsBias.cc @@ -12,6 +12,7 @@ #include #include +#include #include "eckit/config/Configuration.h" #include "lorenz95/ObsBiasCorrection.h" @@ -21,7 +22,7 @@ namespace lorenz95 { // ----------------------------------------------------------------------------- ObsBias::ObsBias(const ObsTableView &, const eckit::Configuration & conf) - : bias_(0.0), active_(false), geovars_(), hdiags_() + : bias_(0.0), active_(false), geovars_(std::vector{"x"}), hdiags_() { oops::Log::trace() << "ObsBias::ObsBias conf is:" << conf << std::endl; if (conf.has("bias")) { diff --git a/l95/src/lorenz95/ObservationL95.cc b/l95/src/lorenz95/ObservationL95.cc index 5811bbd9e..9c5bb6b9f 100644 --- a/l95/src/lorenz95/ObservationL95.cc +++ b/l95/src/lorenz95/ObservationL95.cc @@ -26,7 +26,7 @@ namespace lorenz95 { // ----------------------------------------------------------------------------- ObservationL95::ObservationL95(const ObsTableView & ot, const eckit::Configuration &) - : obsdb_(ot), inputs_() + : obsdb_(ot), inputs_(std::vector{"x"}) {} // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObservationTLAD.cc b/l95/src/lorenz95/ObservationTLAD.cc index 7d6e7fe04..8ee67461e 100644 --- a/l95/src/lorenz95/ObservationTLAD.cc +++ b/l95/src/lorenz95/ObservationTLAD.cc @@ -11,6 +11,7 @@ #include "lorenz95/ObservationTLAD.h" #include +#include #include "eckit/config/Configuration.h" #include "lorenz95/GomL95.h" @@ -24,7 +25,7 @@ namespace lorenz95 { // ----------------------------------------------------------------------------- ObservationTLAD::ObservationTLAD(const ObsTableView &, const eckit::Configuration &) - : inputs_() + : inputs_(std::vector{"x"}) {} // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/instantiateL95ChangeVarFactory.h b/l95/src/lorenz95/instantiateL95ChangeVarFactory.h new file mode 100644 index 000000000..c51ae5a61 --- /dev/null +++ b/l95/src/lorenz95/instantiateL95ChangeVarFactory.h @@ -0,0 +1,24 @@ +/* + * (C) Copyright 2017-2020 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef LORENZ95_INSTANTIATEL95CHANGEVARFACTORY_H_ +#define LORENZ95_INSTANTIATEL95CHANGEVARFACTORY_H_ + +#include "lorenz95/L95Traits.h" +#include "oops/base/VariableChangeBase.h" +#include "oops/generic/IdVariableChange.h" + +namespace lorenz95 { + +void instantiateL95ChangeVarFactory() { + static oops::GenericVariableChangeMaker > + makerL95_("default"); +} + +} // namespace lorenz95 + +#endif // LORENZ95_INSTANTIATEL95CHANGEVARFACTORY_H_ diff --git a/qg/mains/qgHofX3D.cc b/qg/mains/qgHofX3D.cc index 1c9fce5e5..2b40fe900 100644 --- a/qg/mains/qgHofX3D.cc +++ b/qg/mains/qgHofX3D.cc @@ -6,11 +6,13 @@ */ #include "model/QgTraits.h" +#include "oops/qg/instantiateQgChangeVarFactory.h" #include "oops/runs/HofX3D.h" #include "oops/runs/Run.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); + qg::instantiateQgChangeVarFactory(); oops::HofX3D hofx; return run.execute(hofx); } diff --git a/qg/model/ChangeVarQG.cc b/qg/model/ChangeVarQG.cc index 74c4b3211..64fc8ad33 100644 --- a/qg/model/ChangeVarQG.cc +++ b/qg/model/ChangeVarQG.cc @@ -10,30 +10,25 @@ #include #include -#include "eckit/config/Configuration.h" -#include "model/GeometryQG.h" -#include "model/StateQG.h" -#include "oops/base/Variables.h" #include "oops/util/Logger.h" +#include "model/QgFortran.h" +#include "model/StateQG.h" + namespace qg { // ----------------------------------------------------------------------------- -ChangeVarQG::ChangeVarQG(const GeometryQG & resol, const eckit::Configuration & conf) { - oops::Log::trace() << "ChangeVarQG::ChangeVarQG start" << std::endl; - const oops::Variables vars_in(conf, "input variables"); - const oops::Variables vars_out(conf, "output variables"); - qg_change_var_setup_f90(keyConfig_, vars_in, vars_out); - oops::Log::trace() << "ChangeVarQG::ChangeVarQG done" << std::endl; -} +ChangeVarQG::ChangeVarQG(const GeometryQG &, const eckit::Configuration &) {} // ----------------------------------------------------------------------------- ChangeVarQG::~ChangeVarQG() {} // ----------------------------------------------------------------------------- void ChangeVarQG::changeVar(const StateQG & xa, StateQG & xm) const { - qg_change_var_f90(keyConfig_, xa.fields().toFortran(), xm.fields().toFortran()); + ASSERT(xa.variables().has("x") || xa.variables().has("q")); + qg_change_var_f90(xa.fields().toFortran(), xm.fields().toFortran()); } // ----------------------------------------------------------------------------- void ChangeVarQG::changeVarInverse(const StateQG & xm, StateQG & xa) const { - qg_change_var_inv_f90(keyConfig_, xm.fields().toFortran(), xa.fields().toFortran()); + ASSERT(xm.variables().has("x") || xm.variables().has("q")); + qg_change_var_f90(xm.fields().toFortran(), xa.fields().toFortran()); } // ----------------------------------------------------------------------------- void ChangeVarQG::print(std::ostream & os) const { diff --git a/qg/model/ChangeVarQG.h b/qg/model/ChangeVarQG.h index 8a632c7ac..67614b1f9 100644 --- a/qg/model/ChangeVarQG.h +++ b/qg/model/ChangeVarQG.h @@ -13,7 +13,6 @@ #include "oops/base/VariableChangeBase.h" -#include "oops/qg/QgFortran.h" #include "oops/qg/QgTraits.h" // Forward declarations @@ -41,9 +40,6 @@ class ChangeVarQG: public oops::VariableChangeBase { private: void print(std::ostream &) const override; - -// Data - F90chvar keyConfig_; }; // ----------------------------------------------------------------------------- diff --git a/qg/model/ChangeVarTLADQG.cc b/qg/model/ChangeVarTLADQG.cc index 49d755932..0832e56b5 100644 --- a/qg/model/ChangeVarTLADQG.cc +++ b/qg/model/ChangeVarTLADQG.cc @@ -31,12 +31,12 @@ ChangeVarTLADQG::ChangeVarTLADQG(const StateQG &, const StateQG &, ChangeVarTLADQG::~ChangeVarTLADQG() {} // ----------------------------------------------------------------------------- void ChangeVarTLADQG::multiply(const IncrementQG & dxa, IncrementQG & dxm) const { - qg_change_var_f90(keyConfig_, dxa.fields().toFortran(), dxm.fields().toFortran()); + qg_change_var_tl_f90(keyConfig_, dxa.fields().toFortran(), dxm.fields().toFortran()); oops::Log::debug() << "ChangeVarTLADQG::multiply" << dxm << std::endl; } // ----------------------------------------------------------------------------- void ChangeVarTLADQG::multiplyInverse(const IncrementQG & dxm, IncrementQG & dxa) const { - qg_change_var_inv_f90(keyConfig_, dxm.fields().toFortran(), dxa.fields().toFortran()); + qg_change_var_inv_tl_f90(keyConfig_, dxm.fields().toFortran(), dxa.fields().toFortran()); oops::Log::debug() << "ChangeVarTLADQG::multiplyInverse" << dxm << std::endl; } // ----------------------------------------------------------------------------- diff --git a/qg/model/QgFortran.h b/qg/model/QgFortran.h index 1a471e233..f7db0cf4b 100644 --- a/qg/model/QgFortran.h +++ b/qg/model/QgFortran.h @@ -69,8 +69,9 @@ extern "C" { // Change of variable // ----------------------------------------------------------------------------- void qg_change_var_setup_f90(F90chvar &, const oops::Variables &, const oops::Variables &); - void qg_change_var_f90(const F90chvar &, const F90flds &, const F90flds &); - void qg_change_var_inv_f90(const F90chvar &, const F90flds &, const F90flds &); + void qg_change_var_f90(const F90flds &, const F90flds &); + void qg_change_var_tl_f90(const F90chvar &, const F90flds &, const F90flds &); + void qg_change_var_inv_tl_f90(const F90chvar &, const F90flds &, const F90flds &); void qg_change_var_ad_f90(const F90chvar &, const F90flds &, const F90flds &); void qg_change_var_inv_ad_f90(const F90chvar &, const F90flds &, const F90flds &); diff --git a/qg/model/instantiateQgChangeVarFactory.h b/qg/model/instantiateQgChangeVarFactory.h index 05a3078e7..5e1bd6d9b 100644 --- a/qg/model/instantiateQgChangeVarFactory.h +++ b/qg/model/instantiateQgChangeVarFactory.h @@ -22,6 +22,7 @@ namespace qg { void instantiateQgChangeVarFactory() { static oops::VariableChangeMaker makerchangevar_("ChVarQG"); + static oops::VariableChangeMaker makerdefchavar_("default"); static oops::LinearVariableChangeMaker > diff --git a/qg/model/qg_change_var_interface.F90 b/qg/model/qg_change_var_interface.F90 index c778fb925..aa8df9679 100644 --- a/qg/model/qg_change_var_interface.F90 +++ b/qg/model/qg_change_var_interface.F90 @@ -58,9 +58,32 @@ subroutine qg_change_var_delete_c(c_key_self) bind (c,name='qg_change_var_delete call qg_change_var_registry%remove(c_key_self) end subroutine qg_change_var_delete_c +! ------------------------------------------------------------------------------ + +!> Change of variable +subroutine qg_change_var_c(c_key_fld_in,c_key_fld_out) bind (c,name='qg_change_var_f90') + +implicit none + +! Passed variables +integer(c_int),intent(in) :: c_key_fld_in !< Input field +integer(c_int),intent(in) :: c_key_fld_out !< Output field + +! Local variables +type(qg_fields),pointer :: fld_in,fld_out + +! Interface +call qg_fields_registry%get(c_key_fld_in,fld_in) +call qg_fields_registry%get(c_key_fld_out,fld_out) + +! Call Fortran +call qg_change_var(fld_in,fld_out) + +end subroutine qg_change_var_c + ! ------------------------------------------------------------------------------ !> Change of variable -subroutine qg_change_var_c(c_key_conf,c_key_fld_in,c_key_fld_out) bind (c,name='qg_change_var_f90') +subroutine qg_change_var_tl_c(c_key_conf,c_key_fld_in,c_key_fld_out) bind (c,name='qg_change_var_tl_f90') implicit none @@ -79,12 +102,12 @@ subroutine qg_change_var_c(c_key_conf,c_key_fld_in,c_key_fld_out) bind (c,name=' call qg_fields_registry%get(c_key_fld_out,fld_out) ! Call Fortran -call qg_change_var(conf,fld_in,fld_out) +call qg_change_var_tl(conf,fld_in,fld_out) -end subroutine qg_change_var_c +end subroutine qg_change_var_tl_c ! ------------------------------------------------------------------------------ !> Change of variable - inverse -subroutine qg_change_var_inv_c(c_key_conf,c_key_fld_in,c_key_fld_out) bind (c,name='qg_change_var_inv_f90') +subroutine qg_change_var_inv_tl_c(c_key_conf,c_key_fld_in,c_key_fld_out) bind (c,name='qg_change_var_inv_tl_f90') implicit none @@ -103,9 +126,9 @@ subroutine qg_change_var_inv_c(c_key_conf,c_key_fld_in,c_key_fld_out) bind (c,na call qg_fields_registry%get(c_key_fld_out,fld_out) ! Call Fortran -call qg_change_var_inv(conf,fld_in,fld_out) +call qg_change_var_inv_tl(conf,fld_in,fld_out) -end subroutine qg_change_var_inv_c +end subroutine qg_change_var_inv_tl_c ! ------------------------------------------------------------------------------ !> Change of variable - adjoint subroutine qg_change_var_ad_c(c_key_conf,c_key_fld_in,c_key_fld_out) bind (c,name='qg_change_var_ad_f90') diff --git a/qg/model/qg_change_var_mod.F90 b/qg/model/qg_change_var_mod.F90 index 72593e821..c12344a70 100644 --- a/qg/model/qg_change_var_mod.F90 +++ b/qg/model/qg_change_var_mod.F90 @@ -8,8 +8,10 @@ module qg_change_var_mod +use kinds use qg_convert_q_to_x_mod use qg_convert_x_to_q_mod +use qg_convert_x_to_uv_mod use qg_fields_mod use oops_variables_mod @@ -17,8 +19,9 @@ module qg_change_var_mod private public :: qg_change_var_config -public :: qg_change_var_registry -public :: qg_change_var_setup,qg_change_var,qg_change_var_inv,qg_change_var_ad,qg_change_var_inv_ad +public :: qg_change_var_registry, qg_change_var_setup +public :: qg_change_var_tl, qg_change_var_inv_tl, qg_change_var_ad, qg_change_var_inv_ad +public :: qg_change_var ! ------------------------------------------------------------------------------ type :: qg_change_var_config character(len=1024) :: varchange !< Variable change name @@ -65,8 +68,74 @@ subroutine qg_change_var_setup(self,vars_in,vars_out) end subroutine qg_change_var_setup ! ------------------------------------------------------------------------------ + !> Change of variable -subroutine qg_change_var(conf,fld_in,fld_out) +subroutine qg_change_var(fld_in,fld_out) +implicit none +type(qg_fields),intent(in) :: fld_in !< Input fields +type(qg_fields),intent(inout) :: fld_out !< Output fields +real(kind_real), allocatable :: tmpu(:,:,:), tmpv(:,:,:), tmpx(:,:,:) + +! Checks +call qg_fields_check_resolution(fld_in,fld_out) +if (.not.allocated(fld_in%streamfct).and..not.allocated(fld_in%pv)) then + call abor1_ftn('qg_change_var: streamfct or pv input required') +endif + +call qg_fields_copy(fld_out,fld_in,.true.) + +! Get streamfunction +if (allocated(fld_out%streamfct)) then + if (allocated(fld_in%streamfct)) then + fld_out%streamfct = fld_in%streamfct + write(*,*)'qg_change_var: copied streamfct' + else + call convert_q_to_x(fld_in%geom, fld_in%pv, fld_in%x_north, fld_in%x_south, fld_out%streamfct) + write(*,*)'qg_change_var: converted pv to streamfct' + endif + fld_out%gfld3d = fld_out%streamfct +endif + +! Get potential vorticity +if (allocated(fld_out%pv)) then + if (allocated(fld_in%pv)) then + fld_out%pv = fld_in%pv + write(*,*)'qg_change_var: copied pv' + else + call convert_x_to_q(fld_in%geom, fld_in%streamfct, fld_in%x_north, fld_in%x_south, fld_out%pv) + write(*,*)'qg_change_var: converted streamfct to pv' + endif + if (fld_out%lq) fld_out%gfld3d = fld_out%pv +endif + +! Get u and v +if (allocated(fld_out%u) .or. allocated(fld_out%v)) then + allocate(tmpu(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz)) + allocate(tmpv(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz)) + if (allocated(fld_in%streamfct)) then + call convert_x_to_uv(fld_in%geom,fld_in%streamfct,fld_in%x_north,fld_in%x_south, tmpu,tmpv) + write(*,*)'qg_change_var: converted streamfct in to uv' + elseif (allocated(fld_out%streamfct)) then + call convert_x_to_uv(fld_out%geom,fld_out%streamfct,fld_out%x_north,fld_out%x_south, tmpu, tmpv) + write(*,*)'qg_change_var: converted streamfct out to uv' + else ! fld_in has q if it did not have x + allocate(tmpx(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz)) + call convert_q_to_x(fld_in%geom, fld_in%pv, fld_in%x_north, fld_in%x_south, tmpx) + call convert_x_to_uv(fld_in%geom, tmpx, fld_in%x_north, fld_in%x_south, tmpu, tmpv) + write(*,*)'qg_change_var: converted pv in to uv' + deallocate(tmpx) + endif + if (allocated(fld_out%u)) fld_out%u = tmpu + if (allocated(fld_out%v)) fld_out%v = tmpv + deallocate(tmpu) + deallocate(tmpv) +endif + +end subroutine qg_change_var + +! ------------------------------------------------------------------------------ +!> Change of variable +subroutine qg_change_var_tl(conf,fld_in,fld_out) implicit none @@ -84,8 +153,8 @@ subroutine qg_change_var(conf,fld_in,fld_out) call qg_fields_copy(fld_out,fld_in) case ('x_to_q') ! Check fields variables - if (fld_in%lq) call abor1_ftn('qg_change_var: wrong input fields variables for '//trim(conf%varchange)) - if (.not.fld_out%lq) call abor1_ftn('qg_change_var: wrong output fields variables for '//trim(conf%varchange)) + if (fld_in%lq) call abor1_ftn('qg_change_var_tl: wrong input fields variables for '//trim(conf%varchange)) + if (.not.fld_out%lq) call abor1_ftn('qg_change_var_tl: wrong output fields variables for '//trim(conf%varchange)) ! Conversion call convert_x_to_q_tl(fld_in%geom,fld_in%gfld3d,fld_out%gfld3d) @@ -94,8 +163,8 @@ subroutine qg_change_var(conf,fld_in,fld_out) call qg_fields_copy(fld_out,fld_in,.true.) case ('q_to_x') ! Check fields variables - if (.not.fld_in%lq) call abor1_ftn('qg_change_var: wrong input fields variables for '//trim(conf%varchange)) - if (fld_out%lq) call abor1_ftn('qg_change_var: wrong output fields variables for '//trim(conf%varchange)) + if (.not.fld_in%lq) call abor1_ftn('qg_change_var_tl: wrong input fields variables for '//trim(conf%varchange)) + if (fld_out%lq) call abor1_ftn('qg_change_var_tl: wrong output fields variables for '//trim(conf%varchange)) ! Conversion call convert_q_to_x_tl(fld_in%geom,fld_in%gfld3d,fld_out%gfld3d) @@ -103,13 +172,13 @@ subroutine qg_change_var(conf,fld_in,fld_out) ! Copy boundary conditions call qg_fields_copy(fld_out,fld_in,.true.) case default - call abor1_ftn('qg_change_var: wrong variable change') + call abor1_ftn('qg_change_var_tl: wrong variable change') end select -end subroutine qg_change_var +end subroutine qg_change_var_tl ! ------------------------------------------------------------------------------ !> Change of variable - inverse -subroutine qg_change_var_inv(conf,fld_in,fld_out) +subroutine qg_change_var_inv_tl(conf,fld_in,fld_out) implicit none @@ -127,8 +196,8 @@ subroutine qg_change_var_inv(conf,fld_in,fld_out) call qg_fields_copy(fld_out,fld_in) case ('x_to_q') ! Check fields variables - if (.not.fld_in%lq) call abor1_ftn('qg_change_var_inv: wrong input fields variables for '//trim(conf%varchange)) - if (fld_out%lq) call abor1_ftn('qg_change_var_inv: wrong output fields variables for '//trim(conf%varchange)) + if (.not.fld_in%lq) call abor1_ftn('qg_change_var_inv_tl: wrong input fields variables for '//trim(conf%varchange)) + if (fld_out%lq) call abor1_ftn('qg_change_var_inv_tl: wrong output fields variables for '//trim(conf%varchange)) ! Conversion call convert_q_to_x_tl(fld_in%geom,fld_in%gfld3d,fld_out%gfld3d) @@ -137,8 +206,8 @@ subroutine qg_change_var_inv(conf,fld_in,fld_out) call qg_fields_copy(fld_out,fld_in,.true.) case ('q_to_x') ! Check fields variables - if (fld_in%lq) call abor1_ftn('qg_change_var_inv: wrong input fields variables for '//trim(conf%varchange)) - if (.not.fld_out%lq) call abor1_ftn('qg_change_var_inv: wrong output fields variables for '//trim(conf%varchange)) + if (fld_in%lq) call abor1_ftn('qg_change_var_inv_tl: wrong input fields variables for '//trim(conf%varchange)) + if (.not.fld_out%lq) call abor1_ftn('qg_change_var_inv_tl: wrong output fields variables for '//trim(conf%varchange)) ! Conversion call convert_x_to_q_tl(fld_in%geom,fld_in%gfld3d,fld_out%gfld3d) @@ -146,10 +215,10 @@ subroutine qg_change_var_inv(conf,fld_in,fld_out) ! Copy boundary conditions call qg_fields_copy(fld_out,fld_in,.true.) case default - call abor1_ftn('qg_change_var_inv: wrong variable change') + call abor1_ftn('qg_change_var_inv_tl: wrong variable change') end select -end subroutine qg_change_var_inv +end subroutine qg_change_var_inv_tl ! ------------------------------------------------------------------------------ !> Change of variable - adjoint subroutine qg_change_var_ad(conf,fld_in,fld_out) diff --git a/qg/model/qg_convert_q_to_x_mod.F90 b/qg/model/qg_convert_q_to_x_mod.F90 index 1b46d5732..e78ab77ed 100644 --- a/qg/model/qg_convert_q_to_x_mod.F90 +++ b/qg/model/qg_convert_q_to_x_mod.F90 @@ -30,11 +30,12 @@ subroutine convert_q_to_x(geom,q,x_north,x_south,x) real(kind_real),intent(in) :: q(geom%nx,geom%ny,geom%nz) !< Potential vorticity real(kind_real),intent(in) :: x_north(geom%nz) !< Streamfunction on northern wall real(kind_real),intent(in) :: x_south(geom%nz) !< Streamfunction on southern wall -real(kind_real),intent(out) :: x(geom%nx,geom%ny,geom%nz) !< Streamfunction +real(kind_real),intent(inout) :: x(geom%nx,geom%ny,geom%nz) !< Streamfunction ! Local variables integer :: ix,iy,iz real(kind_real) :: q_nobc(geom%nx,geom%ny,geom%nz),pinv_q(geom%nx,geom%ny,geom%nz),pinv_x(geom%nx,geom%ny,geom%nz) +real(kind_real) :: zz ! Subtract the beta term and the heating term !$omp parallel do schedule(static) private(iy) @@ -45,10 +46,11 @@ subroutine convert_q_to_x(geom,q,x_north,x_south,x) !$omp end parallel do ! Subtract the contribution from the boundaries +zz = 1.0 / (geom%deltay * geom%deltay) !$omp parallel do schedule(static) private(iz) do iz=1,geom%nz - q_nobc(:,1,iz) = q_nobc(:,1,iz)-x_south(iz)/geom%deltay**2 - q_nobc(:,geom%ny,iz) = q_nobc(:,geom%ny,iz)-x_north(iz)/geom%deltay**2 + q_nobc(:,1,iz) = q_nobc(:,1,iz)-x_south(iz)*zz + q_nobc(:,geom%ny,iz) = q_nobc(:,geom%ny,iz)-x_north(iz)*zz enddo !$omp end parallel do @@ -89,7 +91,7 @@ subroutine convert_q_to_x_tl(geom,q,x) ! Passed variables type(qg_geom),intent(in) :: geom !< Geometry real(kind_real),intent(in) :: q(geom%nx,geom%ny,geom%nz) !< Potential vorticity -real(kind_real),intent(out) :: x(geom%nx,geom%ny,geom%nz) !< Streamfunction +real(kind_real),intent(inout) :: x(geom%nx,geom%ny,geom%nz) !< Streamfunction ! Local variables integer :: ix,iy,iz diff --git a/qg/model/qg_convert_x_to_q_mod.F90 b/qg/model/qg_convert_x_to_q_mod.F90 index 0c96ddfc8..af70ac2e7 100644 --- a/qg/model/qg_convert_x_to_q_mod.F90 +++ b/qg/model/qg_convert_x_to_q_mod.F90 @@ -30,11 +30,12 @@ subroutine convert_x_to_q(geom,x,x_north,x_south,q) real(kind_real),intent(in) :: x(geom%nx,geom%ny,geom%nz) !< Streamfunction real(kind_real),intent(in) :: x_north(geom%nz) !< Streamfunction on northern wall real(kind_real),intent(in) :: x_south(geom%nz) !< Streamfunction on southern wall -real(kind_real),intent(out) :: q(geom%nx,geom%ny,geom%nz) !< Potential vorticity +real(kind_real),intent(inout) :: q(geom%nx,geom%ny,geom%nz) !< Potential vorticity ! Local variables integer :: ix,iy,iz real(kind_real) :: del2x(geom%nx,geom%ny,geom%nz) +real(kind_real) :: zz ! Laplacian of the streamfunction call laplacian_2d(geom,x,del2x) @@ -51,10 +52,11 @@ subroutine convert_x_to_q(geom,x,x_north,x_south,q) !$omp end parallel do ! Add the contribution from the boundaries +zz = 1.0 / (geom%deltay * geom%deltay) !$omp parallel do schedule(static) private(iz) do iz=1,geom%nz - q(:,1,iz) = q(:,1,iz)+x_south(iz)/geom%deltay**2 - q(:,geom%ny,iz) = q(:,geom%ny,iz)+x_north(iz)/geom%deltay**2 + q(:,1,iz) = q(:,1,iz)+x_south(iz)*zz + q(:,geom%ny,iz) = q(:,geom%ny,iz)+x_north(iz)*zz enddo !$omp end parallel do @@ -76,7 +78,7 @@ subroutine convert_x_to_q_tl(geom,x,q) ! Passed variables type(qg_geom),intent(in) :: geom !< Geometry real(kind_real),intent(in) :: x(geom%nx,geom%ny,geom%nz) !< Streamfunction -real(kind_real),intent(out) :: q(geom%nx,geom%ny,geom%nz) !< Potential vorticity +real(kind_real),intent(inout) :: q(geom%nx,geom%ny,geom%nz) !< Potential vorticity ! Local variables integer :: ix,iy,iz diff --git a/qg/model/qg_convert_x_to_uv_mod.F90 b/qg/model/qg_convert_x_to_uv_mod.F90 index f9d778a80..6e5c7bd5d 100644 --- a/qg/model/qg_convert_x_to_uv_mod.F90 +++ b/qg/model/qg_convert_x_to_uv_mod.F90 @@ -28,8 +28,8 @@ subroutine convert_x_to_uv(geom,x,x_north,x_south,u,v) real(kind_real),intent(in) :: x(geom%nx,geom%ny,geom%nz) !< Streamfunction real(kind_real),intent(in) :: x_north(geom%nz) !< Streamfunction on northern wall real(kind_real),intent(in) :: x_south(geom%nz) !< Streamfunction on southern wall -real(kind_real),intent(out) :: u(geom%nx,geom%ny,geom%nz) !< Zonal wind -real(kind_real),intent(out) :: v(geom%nx,geom%ny,geom%nz) !< Meridional wind +real(kind_real),intent(inout) :: u(geom%nx,geom%ny,geom%nz) !< Zonal wind +real(kind_real),intent(inout) :: v(geom%nx,geom%ny,geom%nz) !< Meridional wind ! Local variables integer :: iz diff --git a/qg/model/qg_fields_mod.F90 b/qg/model/qg_fields_mod.F90 index 6d6bc2463..6b6b9cb85 100644 --- a/qg/model/qg_fields_mod.F90 +++ b/qg/model/qg_fields_mod.F90 @@ -52,7 +52,11 @@ module qg_fields_mod type(qg_geom),pointer :: geom !< Geometry logical :: lq !< PV as main variable (streamfunction if false) logical :: lbc !< Boundaries are present - real(kind_real),allocatable :: gfld3d(:,:,:) !< 3d field + real(kind_real),allocatable :: gfld3d(:,:,:) !< TEMPORARY: TO BE REMOVED + real(kind_real),allocatable :: streamfct(:,:,:) !< Streamfunction + real(kind_real),allocatable :: pv(:,:,:) !< PV + real(kind_real),allocatable :: u(:,:,:) !< U wind + real(kind_real),allocatable :: v(:,:,:) !< V wind real(kind_real),allocatable :: x_north(:) !< Streamfunction on northern wall real(kind_real),allocatable :: x_south(:) !< Streamfunction on southern wall real(kind_real),allocatable :: q_north(:,:) !< PV on northern wall @@ -91,25 +95,35 @@ subroutine qg_fields_create(self,geom,vars,lbc) ! Associate geometry self%geom => geom -! Set variables -if (vars%has('x') .and. vars%has('q')) then - call abor1_ftn('qg_fields_create: x and q cannot be set as fields together') -elseif (vars%has('u') .or. vars%has('v')) then - call abor1_ftn('qg_fieldsçcreate: u and v cannot be set as fields') -elseif (vars%has('x')) then +!!! TEMPORARY: TO BE REMOVED !!! +if (vars%has('x')) then self%lq = .false. elseif (vars%has('q')) then self%lq = .true. else - call abor1_ftn('qg_fields_create: x or q should be set as fields') + call abor1_ftn('qg_fields_create: fields must have x or q') endif ! Set boundaries self%lbc = lbc -! Allocate 3d field +!!! TEMPORARY: TO BE REMOVED !!! allocate(self%gfld3d(self%geom%nx,self%geom%ny,self%geom%nz)) +! Allocate 3d fields +if (vars%has('x')) then + allocate(self%streamfct(self%geom%nx,self%geom%ny,self%geom%nz)) +endif +if (vars%has('q')) then + allocate(self%pv(self%geom%nx,self%geom%ny,self%geom%nz)) +endif +if (vars%has('u')) then + allocate(self%u(self%geom%nx,self%geom%ny,self%geom%nz)) +endif +if (vars%has('v')) then + allocate(self%v(self%geom%nx,self%geom%ny,self%geom%nz)) +endif + ! Allocate boundaries if (self%lbc) then ! Allocation @@ -148,6 +162,7 @@ subroutine qg_fields_create_default(self,geom,lbc) ! Allocate 3d field allocate(self%gfld3d(self%geom%nx,self%geom%ny,self%geom%nz)) +allocate(self%streamfct(self%geom%nx,self%geom%ny,self%geom%nz)) ! Allocate boundaries if (self%lbc) then @@ -183,6 +198,20 @@ subroutine qg_fields_create_from_other(self,other,geom) ! Allocate 3d field allocate(self%gfld3d(self%geom%nx,self%geom%ny,self%geom%nz)) +! Allocate 3d fields +if (allocated(other%streamfct)) then + allocate(self%streamfct(self%geom%nx,self%geom%ny,self%geom%nz)) +endif +if (allocated(other%pv)) then + allocate(self%pv(self%geom%nx,self%geom%ny,self%geom%nz)) +endif +if (allocated(other%u)) then + allocate(self%u(self%geom%nx,self%geom%ny,self%geom%nz)) +endif +if (allocated(other%v)) then + allocate(self%v(self%geom%nx,self%geom%ny,self%geom%nz)) +endif + ! Allocate boundaries if (self%lbc) then ! Allocation @@ -207,6 +236,10 @@ subroutine qg_fields_delete(self) ! Release memory if (allocated(self%gfld3d)) deallocate(self%gfld3d) +if (allocated(self%streamfct)) deallocate(self%streamfct) +if (allocated(self%pv)) deallocate(self%pv) +if (allocated(self%u)) deallocate(self%u) +if (allocated(self%v)) deallocate(self%v) if (allocated(self%x_north)) deallocate(self%x_north) if (allocated(self%x_south)) deallocate(self%x_south) if (allocated(self%q_north)) deallocate(self%q_north) @@ -227,6 +260,10 @@ subroutine qg_fields_zero(self) ! Set fields to zero self%gfld3d = 0.0 +if (allocated(self%streamfct)) self%streamfct = 0.0 +if (allocated(self%pv)) self%pv = 0.0 +if (allocated(self%u)) self%u = 0.0 +if (allocated(self%v)) self%v = 0.0 if (self%lbc) then self%x_north = 0.0 self%x_south = 0.0 @@ -249,6 +286,10 @@ subroutine qg_fields_ones(self) ! Set fields to ones self%gfld3d = 1.0 +if (allocated(self%streamfct)) self%streamfct = 1.0 +if (allocated(self%pv)) self%pv = 1.0 +if (allocated(self%u)) self%u = 1.0 +if (allocated(self%v)) self%v = 1.0 if (self%lbc) then self%x_north = 1.0 self%x_south = 1.0 @@ -297,6 +338,7 @@ subroutine qg_fields_dirac(self,f_conf) ! Setup Diracs do idir=1,ndir self%gfld3d(ixdir(idir),iydir(idir),izdir(idir)) = 1.0 + if (allocated(self%streamfct)) self%streamfct(ixdir(idir),iydir(idir),izdir(idir)) = 1.0 end do end subroutine qg_fields_dirac @@ -343,6 +385,10 @@ subroutine qg_fields_copy(self,other,bconly) ! Copy 3D field self%gfld3d = other%gfld3d + if (allocated(self%streamfct).and.allocated(other%streamfct)) self%streamfct = other%streamfct + if (allocated(self%pv).and.allocated(other%pv)) self%pv = other%pv + if (allocated(self%u).and.allocated(other%u)) self%u = other%u + if (allocated(self%v).and.allocated(other%v)) self%v = other%v end if if (self%lbc) then @@ -376,6 +422,10 @@ subroutine qg_fields_self_add(self,rhs) ! Add field self%gfld3d = self%gfld3d+rhs%gfld3d +if (allocated(self%streamfct).and.allocated(rhs%streamfct)) self%streamfct = self%streamfct + rhs%streamfct +if (allocated(self%pv).and.allocated(rhs%pv)) self%pv = self%pv + rhs%pv +if (allocated(self%u).and.allocated(rhs%u)) self%u = self%u + rhs%u +if (allocated(self%v).and.allocated(rhs%v)) self%v = self%v + rhs%v if (self%lbc.and.rhs%lbc) then self%x_north = self%x_north+rhs%x_north self%x_south = self%x_south+rhs%x_south @@ -400,6 +450,10 @@ subroutine qg_fields_self_sub(self,rhs) ! Subtract field self%gfld3d = self%gfld3d-rhs%gfld3d +if (allocated(self%streamfct).and.allocated(rhs%streamfct)) self%streamfct = self%streamfct - rhs%streamfct +if (allocated(self%pv).and.allocated(rhs%pv)) self%pv = self%pv - rhs%pv +if (allocated(self%u).and.allocated(rhs%u)) self%u = self%u - rhs%u +if (allocated(self%v).and.allocated(rhs%v)) self%v = self%v - rhs%v if (self%lbc.and.rhs%lbc) then self%x_north = self%x_north-rhs%x_north self%x_south = self%x_south-rhs%x_south @@ -423,6 +477,10 @@ subroutine qg_fields_self_mul(self,zz) ! Multiply with a scalar self%gfld3d = zz*self%gfld3d +if (allocated(self%streamfct)) self%streamfct = zz * self%streamfct +if (allocated(self%pv)) self%pv = zz * self%pv +if (allocated(self%u)) self%u = zz * self%u +if (allocated(self%v)) self%v = zz * self%v if (self%lbc) then self%x_north = zz*self%x_north self%x_south = zz*self%x_south @@ -448,6 +506,10 @@ subroutine qg_fields_axpy(self,zz,rhs) ! Apply apxy self%gfld3d = self%gfld3d+zz*rhs%gfld3d +if (allocated(self%streamfct).and.allocated(rhs%streamfct)) self%streamfct = self%streamfct + zz * rhs%streamfct +if (allocated(self%pv).and.allocated(rhs%pv)) self%pv = self%pv + zz * rhs%pv +if (allocated(self%u).and.allocated(rhs%u)) self%u = self%u + zz * rhs%u +if (allocated(self%v).and.allocated(rhs%v)) self%v = self%v + zz * rhs%v if (self%lbc.and.rhs%lbc) then self%x_north = self%x_north+zz*rhs%x_north self%x_south = self%x_south+zz*rhs%x_south @@ -472,6 +534,10 @@ subroutine qg_fields_self_schur(self,rhs) ! Schur product self%gfld3d = self%gfld3d*rhs%gfld3d +if (allocated(self%streamfct).and.allocated(rhs%streamfct)) self%streamfct = self%streamfct * rhs%streamfct +if (allocated(self%pv).and.allocated(rhs%pv)) self%pv = self%pv * rhs%pv +if (allocated(self%u).and.allocated(rhs%u)) self%u = self%u * rhs%u +if (allocated(self%v).and.allocated(rhs%v)) self%v = self%v * rhs%v if (self%lbc.and.rhs%lbc) then self%x_north = self%x_north*rhs%x_north self%x_south = self%x_south*rhs%x_south @@ -517,6 +583,10 @@ subroutine qg_fields_add_incr(self,rhs) if ((self%geom%nx==rhs%geom%nx).and.(self%geom%ny==rhs%geom%ny).and.(self%geom%nz==rhs%geom%nz)) then ! Same resolution self%gfld3d = self%gfld3d+rhs%gfld3d + if (allocated(self%streamfct).and.allocated(rhs%streamfct)) self%streamfct = self%streamfct + rhs%streamfct + if (allocated(self%pv).and.allocated(rhs%pv)) self%pv = self%pv + rhs%pv + if (allocated(self%u).and.allocated(rhs%u)) self%u = self%u + rhs%u + if (allocated(self%v).and.allocated(rhs%v)) self%v = self%v + rhs%v else ! Different resolutions call abor1_ftn('qg_fields_add_incr: not coded for low res increment yet') @@ -549,6 +619,10 @@ subroutine qg_fields_diff_incr(lhs,fld1,fld2) if ((fld1%geom%nx==lhs%geom%nx).and.(fld1%geom%ny==lhs%geom%ny).and.(fld1%geom%nz==lhs%geom%nz)) then ! Same resolution lhs%gfld3d = fld1%gfld3d-fld2%gfld3d + if (allocated(lhs%streamfct).and.allocated(fld1%streamfct)) lhs%streamfct = fld1%streamfct - fld2%streamfct + if (allocated(lhs%pv).and.allocated(fld1%pv)) lhs%pv = fld1%pv - fld2%pv + if (allocated(lhs%u).and.allocated(fld1%u)) lhs%u = fld1%u - fld2%u + if (allocated(lhs%v).and.allocated(fld1%v)) lhs%v = fld1%v - fld2%v else ! Different resolutions call abor1_ftn('qg_fields_diff_incr: not coded for low res increment yet') @@ -627,6 +701,12 @@ subroutine qg_fields_change_resol(fld,rhs) call abor1_ftn('qg_fields_change_resol: different variables') endif +if (fld%lq) then + if (allocated(fld%pv)) fld%pv = fld%gfld3d +else + if (allocated(fld%streamfct)) fld%streamfct = fld%gfld3d +endif + end subroutine qg_fields_change_resol ! ------------------------------------------------------------------------------ !> Read fields from file @@ -740,6 +820,12 @@ subroutine qg_fields_read_file(fld,f_conf,vdate) ! Check field call qg_fields_check(fld) +if (fld%lq) then + if (allocated(fld%pv)) fld%pv = fld%gfld3d +else + if (allocated(fld%streamfct)) fld%streamfct = fld%gfld3d +endif + end subroutine qg_fields_read_file ! ------------------------------------------------------------------------------ !> Write fields to file diff --git a/qg/test/testoutput/convertstate.test b/qg/test/testoutput/convertstate.test index d75d8a02a..db769053d 100644 --- a/qg/test/testoutput/convertstate.test +++ b/qg/test/testoutput/convertstate.test @@ -14,7 +14,7 @@ Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = potential vorticity Test : Boundary conditions are activated -Test : Min= -5.8900e-04, Max= 9.6046e-04, RMS= 3.5956e-04 +Test : Min= -5.8027e-04, Max= 5.4508e-04, RMS= 3.2514e-04 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: QG change of variable @@ -36,7 +36,7 @@ Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = potential vorticity Test : Boundary conditions are activated -Test : Min= -5.8900e-04, Max= 9.6046e-04, RMS= 3.5956e-04 +Test : Min= -5.8027e-04, Max= 5.4508e-04, RMS= 3.2514e-04 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: QG change of variable @@ -96,7 +96,7 @@ Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = potential vorticity Test : Boundary conditions are activated -Test : Min= -5.9340e-04, Max= 9.5651e-04, RMS= 3.5978e-04 +Test : Min= -5.7613e-04, Max= 5.4113e-04, RMS= 3.2534e-04 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: QG change of variable @@ -118,7 +118,7 @@ Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 Test : Variable = potential vorticity Test : Boundary conditions are activated -Test : Min= -5.9340e-04, Max= 9.5651e-04, RMS= 3.5978e-04 +Test : Min= -5.7613e-04, Max= 5.4113e-04, RMS= 3.2534e-04 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: QG change of variable diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f147b5f97..15c803f9a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -181,6 +181,7 @@ oops/generic/unstructured_interpolation_mod.F90 oops/generic/VerticalLocEV.h oops/interface/AnalyticInit.h +oops/interface/ChangeVariables.h oops/interface/ErrorCovariance.h oops/interface/Geometry.h oops/interface/GeometryIterator.h @@ -226,6 +227,7 @@ oops/runs/ExternalDFI.h oops/runs/Forecast.h oops/runs/GenEnsPertB.h oops/runs/HofX3D.h +oops/runs/HofX3Dslow.h oops/runs/HofX4D.h oops/runs/LocalEnsembleDA.h oops/runs/RTPP.h diff --git a/src/oops/interface/ChangeVariables.h b/src/oops/interface/ChangeVariables.h new file mode 100644 index 000000000..33d9f759f --- /dev/null +++ b/src/oops/interface/ChangeVariables.h @@ -0,0 +1,120 @@ +/* + * (C) Copyright 2018-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_INTERFACE_CHANGEVARIABLES_H_ +#define OOPS_INTERFACE_CHANGEVARIABLES_H_ + +#include +#include + +#include "eckit/config/LocalConfiguration.h" +#include "oops/base/Variables.h" +#include "oops/interface/Geometry.h" +#include "oops/interface/State.h" +#include "oops/interface/VariableChange.h" +#include "oops/util/Logger.h" +#include "oops/util/ObjectCounter.h" +#include "oops/util/Printable.h" +#include "oops/util/Timer.h" + +namespace oops { + +/// \brief Encapsulates the nonlinear variable change +/// There should not be a factory for ChangeVariable, it should be a trait class. +/// This is a temporary solution to get around it until we can fix the models. +template +class ChangeVariables : public util::Printable, + private util::ObjectCounter > { + typedef Geometry Geometry_; + typedef State State_; + typedef VariableChange VariableChange_; + + public: + static const std::string classname() {return "oops::ChangeVariables";} + + ChangeVariables(const eckit::Configuration &, const Geometry_ &, + const Variables &, const Variables &); + virtual ~ChangeVariables(); + ChangeVariables(const ChangeVariables &) = delete; + ChangeVariables(ChangeVariables &&) = default; + const ChangeVariables & operator=(const ChangeVariables &) = delete; + ChangeVariables & operator=(ChangeVariables &&) = default; + + /// change variable from state \p xin to \p xout + void changeVar(const State_ & xin, State_ & xout) const; + /// inverse of changeVar, change variables back from \p xout to \p xin + void changeVarInverse(const State_ & xout, State_ & xin) const; + + private: + void print(std::ostream &) const override; + + std::unique_ptr chvar_; /// pointer to the VariableChange implementation +}; + +// ============================================================================= + +template +ChangeVariables::ChangeVariables(const eckit::Configuration & conf, const Geometry_ & geom, + const Variables & varin, const Variables & varout) + : chvar_() +{ + Log::trace() << "ChangeVariables::ChangeVariables starting" << std::endl; +// Comment timer for now to avoid double counting +// util::Timer timer(classname(), "ChangeVariables"); + eckit::LocalConfiguration chconf(conf); + if (!chconf.has("variable change")) chconf.set("variable change", "default"); + chconf.set("input variables", varin.variables()); + chconf.set("output variables", varout.variables()); + chvar_.reset(new VariableChange_(geom, chconf)); + Log::trace() << "ChangeVariables::ChangeVariables done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +ChangeVariables::~ChangeVariables() { + Log::trace() << "ChangeVariables::~ChangeVariables starting" << std::endl; +// util::Timer timer(classname(), "~ChangeVariables"); + chvar_.reset(); + Log::trace() << "ChangeVariables::~ChangeVariables done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void ChangeVariables::changeVar(const State_ & x1, State_ & x2) const { + Log::trace() << "ChangeVariables::changeVar starting" << std::endl; +// util::Timer timer(classname(), "changeVar"); + chvar_->changeVar(x1, x2); + Log::trace() << "ChangeVariables::changeVar done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void ChangeVariables::changeVarInverse(const State_ & x1, State_ & x2) const { + Log::trace() << "ChangeVariables::changeVarInverse starting" << std::endl; +// util::Timer timer(classname(), "changeVarInverse"); + chvar_->changeVarInverse(x1, x2); + Log::trace() << "ChangeVariables::changeVarInverse done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void ChangeVariables::print(std::ostream & os) const { + Log::trace() << "ChangeVariables::print starting" << std::endl; +// util::Timer timer(classname(), "print"); + os << *chvar_; + Log::trace() << "ChangeVariables::print done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_INTERFACE_CHANGEVARIABLES_H_ diff --git a/src/oops/interface/VariableChange.h b/src/oops/interface/VariableChange.h index ba92c1ff8..16df20c00 100644 --- a/src/oops/interface/VariableChange.h +++ b/src/oops/interface/VariableChange.h @@ -48,11 +48,6 @@ class VariableChange : public util::Printable, const VariableChange & operator=(const VariableChange &) = delete; VariableChange & operator=(VariableChange &&) = default; - /// set input variables for variable transform - void setInputVariables(const Variables & vars) { varin_.reset(new Variables(vars)); } - /// set output variables for variable transform - void setOutputVariables(const Variables & vars) { varout_.reset(new Variables(vars)); } - /// change variable from state \p xin to \p xout void changeVar(const State_ & xin, State_ & xout) const; /// inverse of changeVar, change variables back from \p xout to \p xin @@ -82,13 +77,11 @@ VariableChange::VariableChange(const Geometry_ & geom, util::Timer timer(classname(), "VariableChange"); if (params.inputVariables.value() != boost::none) { varin_.reset(new Variables(*params.inputVariables.value())); - Log::trace() << "VariableChange::VariableChange input variables: " - << *varin_ << std::endl; + Log::debug() << "VariableChange::VariableChange input variables: " << *varin_ << std::endl; } if (params.outputVariables.value() != boost::none) { varout_.reset(new Variables(*params.outputVariables.value())); - Log::trace() << "VariableChange::VariableChange output variables: " - << *varout_ << std::endl; + Log::debug() << "VariableChange::VariableChange output variables: " << *varout_ << std::endl; } chvar_.reset(VariableChangeFactory::create(geom, params)); Log::trace() << "VariableChange::VariableChange done" << std::endl; diff --git a/src/oops/runs/HofX3D.h b/src/oops/runs/HofX3D.h index b63a6e93d..6b6ac6031 100644 --- a/src/oops/runs/HofX3D.h +++ b/src/oops/runs/HofX3D.h @@ -19,6 +19,8 @@ #include "oops/base/Observations.h" #include "oops/base/ObsSpaces.h" #include "oops/base/Variables.h" +#include "oops/generic/instantiateVariableChangeFactory.h" +#include "oops/interface/ChangeVariables.h" #include "oops/interface/Geometry.h" #include "oops/interface/GeoVaLs.h" #include "oops/interface/GetValues.h" @@ -43,6 +45,7 @@ template class HofX3D : public Application { typedef Observations Observations_; typedef ObsSpaces ObsSpaces_; typedef State State_; + typedef ChangeVariables ChangeVariables_; typedef std::vector> GeoVaLsVec_; typedef std::vector> LocationsVec_; @@ -53,6 +56,7 @@ template class HofX3D : public Application { // ----------------------------------------------------------------------------- explicit HofX3D(const eckit::mpi::Comm & comm = oops::mpi::world()) : Application(comm) { instantiateObsFilterFactory(); + instantiateVariableChangeFactory(); } // ----------------------------------------------------------------------------- virtual ~HofX3D() {} @@ -85,6 +89,20 @@ template class HofX3D : public Application { GeoVaLsVec_ geovals; const LocationsVec_ & locations = hofx.locations(); const VariablesVec_ & vars = hofx.requiredVars(); + Log::debug() << "HofX3D: Required hofx size = " << hofx.requiredVars().size() << std::endl; + + Variables geovars; + Log::debug() << "HofX3D: Required vars size = " << vars.size() << std::endl; + for (size_t jj = 0; jj < vars.size(); ++jj) { + Log::debug() << "HofX3D: Required vars:" << vars[jj] << std::endl; + geovars += vars[jj]; + } + Log::debug() << "HofX3D: Required variables:" << geovars << std::endl; + eckit::LocalConfiguration chvarconf; // empty for now + ChangeVariables_ chvar(chvarconf, geometry, xx.variables(), geovars); + + State_ zz(geometry, geovars, xx.validTime()); + chvar.changeVar(xx, zz); std::vector getValuesConfig = util::vectoriseAndFilter(obsConfig, "get values"); @@ -93,7 +111,7 @@ template class HofX3D : public Application { for (size_t jj = 0; jj < obspaces.size(); ++jj) { GetValues_ getvals(geometry, *locations[jj], getValuesConfig[jj]); geovals.emplace_back(new GeoVaLs_(*locations[jj], vars[jj])); - getvals.fillGeoVaLs(xx, winbgn, winend, *geovals[jj]); + getvals.fillGeoVaLs(zz, winbgn, winend, *geovals[jj]); } // Compute H(x) on filled in geovals and run the filters diff --git a/src/oops/runs/HofX3Dslow.h b/src/oops/runs/HofX3Dslow.h new file mode 100644 index 000000000..b89e08af3 --- /dev/null +++ b/src/oops/runs/HofX3Dslow.h @@ -0,0 +1,120 @@ +/* + * (C) Copyright 2018-2020 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_RUNS_HOFX3DSLOW_H_ +#define OOPS_RUNS_HOFX3DSLOW_H_ + +#include +#include +#include +#include + +#include "eckit/config/LocalConfiguration.h" +#include "oops/assimilation/CalcHofX.h" +#include "oops/base/instantiateObsFilterFactory.h" +#include "oops/base/Observations.h" +#include "oops/base/ObsSpaces.h" +#include "oops/base/Variables.h" +#include "oops/interface/Geometry.h" +#include "oops/interface/GeoVaLs.h" +#include "oops/interface/GetValues.h" +#include "oops/interface/Locations.h" +#include "oops/interface/State.h" +#include "oops/mpi/mpi.h" +#include "oops/runs/Application.h" +#include "oops/util/ConfigFunctions.h" +#include "oops/util/DateTime.h" +#include "oops/util/Duration.h" +#include "oops/util/Logger.h" + +namespace oops { + +template class HofX3Dslow : public Application { + typedef CalcHofX CalcHofX_; + typedef Geometry Geometry_; + typedef GeoVaLs GeoVaLs_; + typedef GetValues GetValues_; + typedef Locations Locations_; + typedef ObsAuxControls ObsAux_; + typedef Observations Observations_; + typedef ObsSpaces ObsSpaces_; + typedef State State_; + + typedef std::vector> GeoVaLsVec_; + typedef std::vector> LocationsVec_; + typedef std::vector VariablesVec_; + + + public: +// ----------------------------------------------------------------------------- + explicit HofX3Dslow(const eckit::mpi::Comm & comm = oops::mpi::world()) : Application(comm) { + instantiateObsFilterFactory(); + } +// ----------------------------------------------------------------------------- + virtual ~HofX3Dslow() {} +// ----------------------------------------------------------------------------- + int execute(const eckit::Configuration & fullConfig) const { + // Setup observation window + const util::Duration winlen(fullConfig.getString("window length")); + const util::DateTime winbgn(fullConfig.getString("window begin")); + const util::DateTime winend(winbgn + winlen); + Log::info() << "Observation window from " << winbgn << " to " << winend << std::endl; + + // Setup geometry + const eckit::LocalConfiguration geometryConfig(fullConfig, "geometry"); + const Geometry_ geometry(geometryConfig, this->getComm()); + + // Setup states for H(x) + const eckit::LocalConfiguration stateConfig(fullConfig, "state"); + Log::info() << "State configuration is:" << stateConfig << std::endl; + State_ xx(geometry, stateConfig); + Log::test() << "State: " << xx << std::endl; + + // Setup observations + const eckit::LocalConfiguration obsConfig(fullConfig, "observations"); + ObsSpaces_ obspaces(obsConfig, this->getComm(), winbgn, winend); + ObsAux_ obsaux(obspaces, obsConfig); + CalcHofX_ hofx(obspaces, obsConfig); + hofx.initialize(obsaux); + + // fill in GeoVaLs + GeoVaLsVec_ geovals; + const LocationsVec_ & locations = hofx.locations(); + const VariablesVec_ & vars = hofx.requiredVars(); + + std::vector getValuesConfig = + util::vectoriseAndFilter(obsConfig, "get values"); + + // loop over all observation types + for (size_t jj = 0; jj < obspaces.size(); ++jj) { + GetValues_ getvals(geometry, *locations[jj], getValuesConfig[jj]); + geovals.emplace_back(new GeoVaLs_(*locations[jj], vars[jj])); + getvals.fillGeoVaLs(xx, winbgn, winend, *geovals[jj]); + } + + // Compute H(x) on filled in geovals and run the filters + Observations_ yobs = hofx.compute(geovals); + hofx.saveQcFlags("EffectiveQC"); + hofx.saveObsErrors("EffectiveError"); + + // Save H(x) + Log::test() << "H(x): " << std::endl << yobs << "End H(x)" << std::endl; + yobs.save("hofx"); + + return 0; + } +// ----------------------------------------------------------------------------- + private: + std::string appname() const { + return "oops::HofX3Dslow<" + MODEL::name() + ", " + OBS::name() + ">"; + } +// ----------------------------------------------------------------------------- +}; + +} // namespace oops + +#endif // OOPS_RUNS_HOFX3DSLOW_H_ diff --git a/src/oops/util/LibOOPS.cc b/src/oops/util/LibOOPS.cc index 183b9ab4e..63af903d2 100644 --- a/src/oops/util/LibOOPS.cc +++ b/src/oops/util/LibOOPS.cc @@ -80,6 +80,10 @@ LibOOPS& LibOOPS::instance() { * has been created. */ void LibOOPS::initialise() { + std::time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + Log::info() << "OOPS Starting " + << std::put_time(std::localtime(&now), "%F %T (UTC%z)") << std::endl; + rank_ = oops::mpi::world().rank(); const int it = getEnv("OOPS_TRACE", 0); @@ -150,10 +154,15 @@ void LibOOPS::finalise(bool finaliseMPI) { } } #endif + if ( rank_ == 0 ) { testReferenceInstance_.testReferenceFinalise(testStream_); } + std::time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + Log::info() << "OOPS Ending " + << std::put_time(std::localtime(&now), "%F %T (UTC%z)") << std::endl; + // Make sure that these specialised channels that wrap eckit::Log::info() are // destroyed before eckit::Log::info gets destroyed. // Just in case someone still tries to log, we reset to empty channels. diff --git a/src/test/interface/VariableChange.h b/src/test/interface/VariableChange.h index 44cb4a705..bbc93e131 100644 --- a/src/test/interface/VariableChange.h +++ b/src/test/interface/VariableChange.h @@ -88,8 +88,7 @@ template void testVariableChangeInverse() { const eckit::LocalConfiguration initialConfig(Test_::confs()[jj], "state"); State_ xx(Test_::resol(), initialConfig); - // Save copy of the initial state - State_ xref(xx); + const double xxnorm_ref = xx.norm(); // Order, inverse first or not (default) // Note: switch input and output variables in configuration if true @@ -100,16 +99,17 @@ template void testVariableChangeInverse() { oops::Variables varin(Test_::confs()[jj], "input variables"); State_ xin(Test_::resol(), varin, xx.validTime()); changevar.changeVarInverse(xx, xin); +// xx.zero(); Test for GEOS fails if uncommented changevar.changeVar(xin, xx); } else { oops::Variables varout(Test_::confs()[jj], "output variables"); State_ xout(Test_::resol(), varout, xx.validTime()); changevar.changeVar(xx, xout); +// xx.zero(); Test for GEOS fails if uncommented changevar.changeVarInverse(xout, xx); } // Compute norms of the result and reference - const double xxnorm_ref = xref.norm(); const double xxnorm_tst = xx.norm(); // Print the input and final state From a9b6a0d2e296bd83bdfc8df8823aa4b43eb664e6 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Tue, 23 Mar 2021 09:16:23 -0600 Subject: [PATCH 083/142] add yaml option for updating obs config with geometry info (#1109) --- src/oops/runs/LocalEnsembleDA.h | 41 ++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/src/oops/runs/LocalEnsembleDA.h b/src/oops/runs/LocalEnsembleDA.h index f95a04b56..da2690772 100644 --- a/src/oops/runs/LocalEnsembleDA.h +++ b/src/oops/runs/LocalEnsembleDA.h @@ -69,29 +69,21 @@ template class LocalEnsembleDA : public Applicati const util::DateTime winbgn(fullConfig.getString("window begin")); const util::Duration winlen(fullConfig.getString("window length")); const util::DateTime winend(winbgn + winlen); - const util::DateTime winhalf = winbgn + winlen/2; Log::info() << "Observation window from " << winbgn << " to " << winend << std::endl; // Setup geometry const eckit::LocalConfiguration geometryConfig(fullConfig, "geometry"); const Geometry_ geometry(geometryConfig, this->getComm()); - // if any of the obs. spaces uses Halo distribution it will need to know the geometry - // of the local grid on this PE + // Get observations configuration eckit::LocalConfiguration obsConfig(fullConfig, "observations"); - std::vector patchCenter(2, 0.0); - double patchRadius = 0.0; - computePatchGeometry(geometry, patchCenter, patchRadius); + // Get driver configuration + const eckit::LocalConfiguration driverConfig(fullConfig, "driver"); - // update observations configs with information on patch center and radius - std::vector obsConfigs = obsConfig.getSubConfigurations(); - for (auto & conf : obsConfigs) { - conf.set("obs space.center", patchCenter); - conf.set("obs space.radius", patchRadius); - } - eckit::LocalConfiguration tmp; - tmp.set("observations", obsConfigs); - obsConfig = tmp.getSubConfiguration("observations"); + // if any of the obs. spaces uses Halo distribution it will need to know the geometry + // of the local grid on this PE + bool update_obsconfig = driverConfig.getBool("update obs config with geometry info", false); + if (update_obsconfig) updateConfigWithPatchGeometry(geometry, obsConfig); // Setup observations const eckit::mpi::Comm & time = oops::mpi::myself(); @@ -100,8 +92,6 @@ template class LocalEnsembleDA : public Applicati // Get background configurations const eckit::LocalConfiguration bgConfig(fullConfig, "background"); - // Get driver configuration - const eckit::LocalConfiguration driverConfig(fullConfig, "driver"); // Read all ensemble members StateEnsemble4D_ ens_xx(geometry, bgConfig); @@ -253,8 +243,11 @@ template class LocalEnsembleDA : public Applicati return "oops::LocalEnsembleDA<" + MODEL::name() + ", " + OBS::name() + ">"; } - void computePatchGeometry(const Geometry_ & geometry, std::vector & patchCenter, - double & patchRadius) const { + void updateConfigWithPatchGeometry(const Geometry_ & geometry, + eckit::LocalConfiguration & obsConfig) const { + std::vector patchCenter(2, 0.0); + double patchRadius = 0.0; + // since Halo distribution is only implemented in ioda, we can assume that // x is lon and y is lat on Earth. then we need to compute average of x on a circle eckit::geometry::Point2 gptmp; @@ -287,6 +280,16 @@ template class LocalEnsembleDA : public Applicati } Log::debug() << "patch center=" << patchCenter << " patch radius=" << patchRadius << std::endl; + + // update observations configs with information on patch center and radius + std::vector obsConfigs = obsConfig.getSubConfigurations(); + for (auto & conf : obsConfigs) { + conf.set("obs space.center", patchCenter); + conf.set("obs space.radius", patchRadius); + } + eckit::LocalConfiguration tmp; + tmp.set("observations", obsConfigs); + obsConfig = tmp.getSubConfiguration("observations"); } void saveVariance(const eckit::LocalConfiguration & outConfig, const IncrementEnsemble4D_ & perts, From ce962f33e0885ffbc9123b86d76ae01cb0a617d9 Mon Sep 17 00:00:00 2001 From: Mark Miesch Date: Tue, 23 Mar 2021 14:15:30 -0600 Subject: [PATCH 084/142] Revert "changing compare step" (#1106) * Revert "changing compare step (#1081)" This reverts commit d3dce9f3442584ae1c6f88d7f46080d92fbbeb3c. * address coding norm error Co-authored-by: Anna Shlyaeva Co-authored-by: Maryam Abdi --- l95/test/CMakeLists.txt | 12 +- l95/test/testinput/4dvar.obsbias.yaml | 7 - l95/test/testoutput/4dvar.obsbias.test | 44 +++--- src/CMakeLists.txt | 2 - src/oops/runs/Run.cc | 13 +- src/oops/runs/Run.h | 8 +- src/oops/util/LibOOPS.cc | 25 +--- src/oops/util/LibOOPS.h | 16 --- src/oops/util/TestReference.cc | 185 ------------------------- src/oops/util/TestReference.h | 83 ----------- 10 files changed, 37 insertions(+), 358 deletions(-) delete mode 100644 src/oops/util/TestReference.cc delete mode 100644 src/oops/util/TestReference.h diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index 0b8aab869..ee94fd396 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -630,12 +630,12 @@ oops_add_test( TESTNAME 4dsaddlepoint EXENAME l95_4dvar.x TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -#-------------------------------------------------------------------- -ecbuild_add_test( TARGET test_l95_4dvar.obsbias - COMMAND l95_4dvar.x - ARGS testinput/4dvar.obsbias.yaml - TEST_DEPENDS test_l95_forecast test_l95_makeobsbias ) - +oops_add_test( TESTNAME 4dvar.obsbias + MODELNAME l95 + YAMLNAME testinput/4dvar.obsbias.yaml + EXENAME l95_4dvar.x + COMPARE + TEST_DEPENDS test_l95_forecast test_l95_makeobsbias ) #-------------------------------------------------------------------- oops_add_test( TESTNAME 4dvar.allbiases diff --git a/l95/test/testinput/4dvar.obsbias.yaml b/l95/test/testinput/4dvar.obsbias.yaml index 8c6257943..e0b969eb0 100644 --- a/l95/test/testinput/4dvar.obsbias.yaml +++ b/l95/test/testinput/4dvar.obsbias.yaml @@ -74,10 +74,3 @@ output: first: PT3H frequency: PT06H type: an -test: - reference filename: testoutput/4dvar.obsbias.test - # Optional: - float relative tolerance: 0.0 - integer tolerance: 0 - log output filename: testoutput/4dvar.obsbias.log.out - test output filename: testoutput/4dvar.obsbias.test.out diff --git a/l95/test/testoutput/4dvar.obsbias.test b/l95/test/testoutput/4dvar.obsbias.test index 7e971b7be..0fd7093e2 100644 --- a/l95/test/testoutput/4dvar.obsbias.test +++ b/l95/test/testoutput/4dvar.obsbias.test @@ -1,22 +1,22 @@ -CostJb : Nonlinear Jb = 0 -CostJo : Nonlinear Jo(Lorenz 95) = 294.698, nobs = 160, Jo/n = 1.84186, err = 0.4 -CostJcDFI: Nonlinear Jc = 0.338917 -CostFunction: Nonlinear J = 295.037 -DRIPCGMinimizer: reduction in residual norm = 8.3185e-05 -CostFunction::addIncrement: Analysis: - Valid time: 2010-01-01T03:00:00Z - Min=7.67871, Max=8.67067, Average=8.01684 -ObsBias = 0.535611 -CostJb : Nonlinear Jb = 2.30305 -CostJo : Nonlinear Jo(Lorenz 95) = 1.04203, nobs = 160, Jo/n = 0.00651266, err = 0.4 -CostJcDFI: Nonlinear Jc = 0.378713 -CostFunction: Nonlinear J = 3.72378 -DRIPCGMinimizer: reduction in residual norm = 0.0182695 -CostFunction::addIncrement: Analysis: - Valid time: 2010-01-01T03:00:00Z - Min=7.6865, Max=8.68197, Average=8.01516 -ObsBias = 0.546819 -CostJb : Nonlinear Jb = 2.38623 -CostJo : Nonlinear Jo(Lorenz 95) = 0.701554, nobs = 160, Jo/n = 0.00438471, err = 0.4 -CostJcDFI: Nonlinear Jc = 0.366753 -CostFunction: Nonlinear J = 3.45454 +Test : CostJb : Nonlinear Jb = 0 +Test : CostJo : Nonlinear Jo(Lorenz 95) = 294.698, nobs = 160, Jo/n = 1.84186, err = 0.4 +Test : CostJcDFI: Nonlinear Jc = 0.338917 +Test : CostFunction: Nonlinear J = 295.037 +Test : DRIPCGMinimizer: reduction in residual norm = 8.3185e-05 +Test : CostFunction::addIncrement: Analysis: +Test : Valid time: 2010-01-01T03:00:00Z +Test : Min=7.67871, Max=8.67067, Average=8.01684 +Test : ObsBias = 0.535611 +Test : CostJb : Nonlinear Jb = 2.30305 +Test : CostJo : Nonlinear Jo(Lorenz 95) = 1.04203, nobs = 160, Jo/n = 0.00651266, err = 0.4 +Test : CostJcDFI: Nonlinear Jc = 0.378713 +Test : CostFunction: Nonlinear J = 3.72378 +Test : DRIPCGMinimizer: reduction in residual norm = 0.0182695 +Test : CostFunction::addIncrement: Analysis: +Test : Valid time: 2010-01-01T03:00:00Z +Test : Min=7.6865, Max=8.68197, Average=8.01516 +Test : ObsBias = 0.546819 +Test : CostJb : Nonlinear Jb = 2.38623 +Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.701554, nobs = 160, Jo/n = 0.00438471, err = 0.4 +Test : CostJcDFI: Nonlinear Jc = 0.366753 +Test : CostFunction: Nonlinear J = 3.45454 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 15c803f9a..fc96291af 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -309,8 +309,6 @@ oops/util/string_f_c_mod.F90 oops/util/string_utils.F90 oops/util/stringFunctions.cc oops/util/stringFunctions.h -oops/util/TestReference.cc -oops/util/TestReference.h oops/util/Timer.cc oops/util/Timer.h oops/util/TimerHelper.cc diff --git a/src/oops/runs/Run.cc b/src/oops/runs/Run.cc index edbb0ac00..5a04d837d 100644 --- a/src/oops/runs/Run.cc +++ b/src/oops/runs/Run.cc @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -92,13 +92,6 @@ Run::Run(int argc, char** argv) : eckit::Main(argc, argv, "OOPS_HOME"), config_( // Read configuration config_.reset(new eckit::YAMLConfiguration(configfile)); - -// Read tolerances for testing - if (config_->has("test")) { - auto testConf = config_->getSubConfiguration("test"); - LibOOPS::instance().testReferenceInitialise(testConf); - } - Log::info() << "Configuration input file is: " << configfile << std::endl; Log::info() << "Full configuration is:" << *config_ << std::endl; diff --git a/src/oops/runs/Run.h b/src/oops/runs/Run.h index 535cdda56..14348f2d2 100644 --- a/src/oops/runs/Run.h +++ b/src/oops/runs/Run.h @@ -1,9 +1,9 @@ -/* +/* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ diff --git a/src/oops/util/LibOOPS.cc b/src/oops/util/LibOOPS.cc index 63af903d2..4fa2c7d72 100644 --- a/src/oops/util/LibOOPS.cc +++ b/src/oops/util/LibOOPS.cc @@ -13,16 +13,10 @@ /// @date December 2016 #include -#include #include -#include -#include #include #include -#include -#include "eckit/config/LocalConfiguration.h" -#include "eckit/config/YAMLConfiguration.h" #include "eckit/log/Log.h" #include "eckit/log/OStreamTarget.h" #include "eckit/log/PrefixTarget.h" @@ -31,7 +25,6 @@ #include "oops/mpi/mpi.h" #include "oops/util/LibOOPS.h" #include "oops/util/Logger.h" -#include "oops/util/TestReference.h" #ifdef ENABLE_GPTL #include @@ -75,7 +68,7 @@ LibOOPS& LibOOPS::instance() { } /** Initialization of MPI and dependent variables. - * To be called in `main()` by constructor of `oops::Run`. This method initializes MPI and + * To be called in `main()` by constructor of `oops::Run`. This method initializes MPI and * associated variables that must be initialized after static-init time, and only once `eckit::Main` * has been created. */ @@ -106,11 +99,6 @@ void LibOOPS::initialise() { do_abortfpe = getEnv("OOPS_ABORTFPE", 1); trap_sigfpe(do_abortfpe); } - // testStream_ is used by TestReference for comparing test output - // with a reference file - if ( rank_ == 0 ) { - testChannel().addStream(testStream_); - } enable_timer_channel_ = getEnv("OOPS_TIMER", 0) > 0 && rank_ == 0; #ifdef ENABLE_GPTL @@ -131,10 +119,6 @@ void LibOOPS::teeOutput(const std::string & fileprefix) { eckit::Log::addFile(teefile); } -void LibOOPS::testReferenceInitialise(eckit::LocalConfiguration &testConf) { - testReferenceInstance_.Initialise(testConf); -} - /** Clears logs and finalises MPI (unless \p finaliseMPI is false). * To be called in on leaving `main()` by the destructor of `oops::Run`. */ @@ -154,11 +138,6 @@ void LibOOPS::finalise(bool finaliseMPI) { } } #endif - - if ( rank_ == 0 ) { - testReferenceInstance_.testReferenceFinalise(testStream_); - } - std::time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); Log::info() << "OOPS Ending " << std::put_time(std::localtime(&now), "%F %T (UTC%z)") << std::endl; @@ -172,7 +151,6 @@ void LibOOPS::finalise(bool finaliseMPI) { statsChannel_.reset(new eckit::Channel()); testChannel_.reset(new eckit::Channel()); timerChannel_.reset(new eckit::Channel()); - // Destroy info channel last after other channels have flushed all output infoChannel_.reset(new eckit::Channel()); @@ -255,3 +233,4 @@ eckit::Channel& LibOOPS::debugChannel() const { // ----------------------------------------------------------------------------- } // namespace oops + diff --git a/src/oops/util/LibOOPS.h b/src/oops/util/LibOOPS.h index bbb2b33fd..9dcf9e82b 100644 --- a/src/oops/util/LibOOPS.h +++ b/src/oops/util/LibOOPS.h @@ -16,16 +16,10 @@ #define OOPS_UTIL_LIBOOPS_H_ #include -#include #include - -#include "eckit/config/YAMLConfiguration.h" #include "eckit/system/Library.h" #include "eckit/utils/Translator.h" -#include "oops/util/TestReference.h" - - namespace oops { // ----------------------------------------------------------------------------- @@ -48,9 +42,6 @@ class LibOOPS : public eckit::system::Library { void initialise(); void teeOutput(const std::string &); - - void testReferenceInitialise(eckit::LocalConfiguration &); - void finalise(bool finaliseMPI = true); protected: @@ -60,7 +51,6 @@ class LibOOPS : public eckit::system::Library { std::string gitsha1(unsigned int count) const override; - mutable std::unique_ptr infoChannel_; mutable std::unique_ptr debugChannel_; @@ -74,12 +64,6 @@ class LibOOPS : public eckit::system::Library { std::string predebug_; bool trace_; std::string pretrace_; - - std::stringstream testStream_; - TestReference testReferenceInstance_; - - private: - friend class TestReference; bool enable_timer_channel_; }; diff --git a/src/oops/util/TestReference.cc b/src/oops/util/TestReference.cc deleted file mode 100644 index 90a5e924d..000000000 --- a/src/oops/util/TestReference.cc +++ /dev/null @@ -1,185 +0,0 @@ -/* - * (C) Copyright 2017-2021 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "eckit/config/YAMLConfiguration.h" -#include "eckit/exception/Exceptions.h" - -#include "oops/util/LibOOPS.h" -#include "oops/util/Logger.h" -#include "oops/util/TestReference.h" - - -namespace oops { - -// ----------------------------------------------------------------------------- - -void TestReference::Initialise(eckit::LocalConfiguration &testConf) -{ - initCheck_ = true; - tolFloat_ = testConf.getFloat("float relative tolerance", 0.0); - tolInt_ = testConf.getInt("integer tolerance", 0); - refFileYaml_ = testConf.getString("reference filename"); - if (testConf.has("log output filename")) { - outputFileYaml_ = testConf.getString("log output filename"); - LibOOPS::instance().teeOutput(outputFileYaml_); - if (testConf.has("test output filename")) { - testFileYaml_ = testConf.getString("test output filename"); - oops::Log::info() << "save test output to " << testFileYaml_ << std::endl; - } - } -} - -// ----------------------------------------------------------------------------- -// Reads reference file (refFileYaml_) for tests with comparing step -// refFileYaml_ is read from Yaml using setYaml method -std::ifstream TestReference::readRefFile() -{ - std::ifstream refFile(refFileYaml_); - if (refFile.fail()) { - throw eckit::CantOpenFile(refFileName_ + " does not exist"); - } - return refFile; -} -// ----------------------------------------------------------------------------- -void TestReference::testCompare(std::stringstream & test, - std::ifstream & ref, - float tolFloat, int tolInt) -{ - std::regex regNumber("[-+]?[0-9]*[.]?[0-9]+([eE][-+]?[0-9]+)?"); - std::regex regOnlyFloat("[-+]?[0-9]*[.][0-9]+([eE][-+]?[0-9]+)?"); - int lineCounter = 0; - std::string refLine, testLine; - while (std::getline(ref, refLine)) { - std::getline(test, testLine); - std::string testPrefix = "Test : "; - - // if test prefix is in reference file add it to the testLine - // to match the format - // This can be removed once all reference files are updated and - // test prefix is removed from them. - if (!refLine.find(testPrefix)) { - testLine = testPrefix+testLine; - } - - lineCounter++; - int compareRes = refLine.compare(testLine); - - // if strings don't match investigate more - if (compareRes != 0) { - // find all numbers in the two test and reference strings - std::vector numMatch_refLine( - std::sregex_token_iterator(refLine.begin(), refLine.end(), regNumber), - std::sregex_token_iterator()); - std::vector numMatch_testLine( - std::sregex_token_iterator(testLine.begin(), testLine.end(), regNumber), - std::sregex_token_iterator()); - - // How many numbers are in each string and do they match? - if (numMatch_refLine.size() != numMatch_testLine.size()) { - throw TestReferenceTextMismatchError(lineCounter, testLine, refLine); - } else { - for (auto i = 0; i < numMatch_refLine.size(); i++) { - std::smatch matchCheck; - // separate float and int - std::regex_search(numMatch_refLine[i], matchCheck, regOnlyFloat); - - if (matchCheck.empty()) { - // number is int - int ref_int = stoi(numMatch_refLine[i]); - int test_int = stoi(numMatch_testLine[i]); - int abs_dif = std::abs(ref_int - test_int); - if (abs_dif > tolInt) { - throw TestReferenceIntegerMismatchError(lineCounter, test_int, - ref_int, tolInt, testLine, refLine); - } - } else { - // number is float - double ref_float = stod(numMatch_refLine[i]); - double test_float = stod(numMatch_testLine[i]); - double rel_dir = std::abs((ref_float - test_float)/(0.5 * (ref_float + test_float))); - if (rel_dir > tolFloat) { - throw TestReferenceFloatMismatchError(lineCounter, test_float, - ref_float, tolFloat, testLine, refLine); - } - } - } - } - } - } -} -// ----------------------------------------------------------------------------- -void TestReference::testReferenceFinalise(std::stringstream & testStream_) -{ - if (!initCheck_) { - return; - } - auto refStream = readRefFile(); - if (!testFileYaml_.empty()) { - std::ofstream testFileOut(testFileYaml_); - testFileOut << testStream_.str(); - testFileOut.close(); - } - testCompare(testStream_, refStream, tolFloat_, tolInt_); -} - -// ----------------------------------------------------------------------------- - -TestReferenceTextMismatchError::TestReferenceTextMismatchError(int line_num, - const std::string &test_line, const std::string &ref_line) -{ - std::ostringstream os; - os << "Test reference Text mismatch @ Line:" << line_num - << "\nTest: '" << test_line << "'\nRef: '" << ref_line << "'"; - what_ = os.str(); -} - -// ----------------------------------------------------------------------------- -TestReferenceIntegerMismatchError::TestReferenceIntegerMismatchError(int line_num, - NumT test_val, NumT ref_val, NumT tolerance, - const std::string &test_line, - const std::string &ref_line) -{ - std::ostringstream os; - os << "Test reference Integer mismatch @ Line:" << line_num << "\n" - << "Test Val : " << test_val << "\n" - << "Ref Val : " << ref_val << "\n" - << "Delta : " << std::abs(test_val-ref_val) << "\n" - << "Tolerance: " << tolerance << "\n" - << "Test Line: '" << test_line << "'\n" - << "Ref Line : '" << ref_line << "'"; - what_ = os.str(); -} -// ----------------------------------------------------------------------------- -TestReferenceFloatMismatchError::TestReferenceFloatMismatchError(int line_num, - NumT test_val, NumT ref_val, NumT tolerance, - const std::string &test_line, const std::string &ref_line) -{ - std::ostringstream os; - os << "Test reference Float mismatch @ Line:" << line_num <<"\n" - << std::setprecision(std::numeric_limits::digits10 + 1) - << std::scientific - << "Test Val : " << test_val << "\n" - << "Ref Val : " << ref_val << "\n" - << "Delta : " << std::abs(test_val-ref_val) << "\n" - << "Tolerance: " << tolerance << "\n" - << "Test Line: '" << test_line << "'\n" - << "Ref Line : '" << ref_line << "'"; - what_ = os.str(); -} - - -} // namespace oops diff --git a/src/oops/util/TestReference.h b/src/oops/util/TestReference.h deleted file mode 100644 index ae1b29dfe..000000000 --- a/src/oops/util/TestReference.h +++ /dev/null @@ -1,83 +0,0 @@ -/* - * (C) Copyright 2017-2021 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef OOPS_UTIL_TESTREFERENCE_H_ -#define OOPS_UTIL_TESTREFERENCE_H_ - -#include -#include -#include -#include - -#include "eckit/config/LocalConfiguration.h" - - -namespace oops { - -// ----------------------------------------------------------------------------- -class TestReference { - public: - TestReference() = default; // default constructor - - void testReferenceFinalise(std::stringstream &); - void Initialise(eckit::LocalConfiguration &); - - private: - std::ifstream readRefFile(); - void testCompare(std::stringstream &, std::ifstream &, float, int); - bool initCheck_ = false; - - std::string refFileName_; - std::string outputFileYaml_; - std::string refFileYaml_; - std::string testFileYaml_; - double tolFloat_; - int tolInt_; -}; -// ----------------------------------------------------------------------------- -// Base class for all TestReference errors -class TestReferenceError : public std::exception { - public: - const char* what() const noexcept override - { - return what_.c_str(); - } - - protected: - std::string what_; // error message to print -}; - -// Textual mismatch error condition -class TestReferenceTextMismatchError : public TestReferenceError -{ - public: - TestReferenceTextMismatchError(int line_num, - const std::string &test_line, const std::string &ref_line); -}; - -// Integer mismatch error condition -class TestReferenceIntegerMismatchError : public TestReferenceError { - public: - using NumT = int64_t; - TestReferenceIntegerMismatchError(int line_num, - NumT test_val, NumT ref_val, NumT tolerance, - const std::string &test_line, const std::string &ref_line); -}; - -// Floating-point mismatch error condition -class TestReferenceFloatMismatchError : public TestReferenceError { - public: - using NumT = double; - TestReferenceFloatMismatchError(int line_num, - NumT test_val, NumT ref_val, NumT tolerance, - const std::string &test_line, const std::string &ref_line); -}; - - -} // namespace oops - -#endif // OOPS_UTIL_TESTREFERENCE_H_ From 6e6deb4025190fa864252db7828e82a4979b6a09 Mon Sep 17 00:00:00 2001 From: Dan Holdaway <27729500+danholdaway@users.noreply.github.com> Date: Wed, 24 Mar 2021 10:59:23 -0400 Subject: [PATCH 085/142] Tools for dealing with ensemble of increments (#1086) * Add some tools needed for dealing with ensembles of perturbations * update increment conversion test output * Clean up * fixes, add extra functionality to ens recenter app * remove change var options in ConvertIncrement test * add descriptor for each of the new increment ensemble constructors Co-authored-by: danholdaway Co-authored-by: Anna Shlyaeva --- qg/mains/CMakeLists.txt | 5 + qg/mains/qgConvertIncrement.cc | 18 +++ qg/test/CMakeLists.txt | 8 ++ qg/test/testinput/convertincrement.yaml | 37 ++++++ qg/test/testoutput/convertincrement.test | 44 +++++++ src/oops/base/IncrementEnsemble.h | 62 ++++++++++ src/oops/generic/IdLinearVariableChange.h | 4 +- src/oops/runs/ConvertIncrement.h | 133 ++++++++++++++++++++++ src/oops/runs/EnsRecenter.h | 13 +++ 9 files changed, 322 insertions(+), 2 deletions(-) create mode 100644 qg/mains/qgConvertIncrement.cc create mode 100644 qg/test/testinput/convertincrement.yaml create mode 100644 qg/test/testoutput/convertincrement.test create mode 100644 src/oops/runs/ConvertIncrement.h diff --git a/qg/mains/CMakeLists.txt b/qg/mains/CMakeLists.txt index a3059b2e8..3d5eb45be 100644 --- a/qg/mains/CMakeLists.txt +++ b/qg/mains/CMakeLists.txt @@ -68,6 +68,11 @@ ecbuild_add_executable( TARGET qg_convertstate.x LIBS qg ) +ecbuild_add_executable( TARGET qg_convertincrement.x + SOURCES qgConvertIncrement.cc + LIBS qg + ) + ecbuild_add_executable( TARGET qg_ens_variance.x SOURCES qgEnsVariance.cc LIBS qg diff --git a/qg/mains/qgConvertIncrement.cc b/qg/mains/qgConvertIncrement.cc new file mode 100644 index 000000000..ab6fc9eaf --- /dev/null +++ b/qg/mains/qgConvertIncrement.cc @@ -0,0 +1,18 @@ +/* + * (C) Copyright 2018-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "model/instantiateQgChangeVarFactory.h" +#include "model/QgTraits.h" +#include "oops/runs/ConvertIncrement.h" +#include "oops/runs/Run.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + qg::instantiateQgChangeVarFactory(); + oops::ConvertIncrement ci; + return run.execute(ci); +} diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index 3210bbf8b..a3a5a93b3 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -21,6 +21,7 @@ list( APPEND qg_testinput testinput/addincrement.yaml testinput/addincrement_scaled.yaml testinput/analytic_forecast.yaml + testinput/convertincrement.yaml testinput/convertstate.yaml testinput/dfi.yaml testinput/diffstates.yaml @@ -101,6 +102,7 @@ list( APPEND qg_testoutput testoutput/4dvar_rpcg.test testoutput/4dvar_saddlepoint.test testoutput/analytic_forecast.test + testoutput/convertincrement.test testoutput/convertstate.test testoutput/addincrement.test testoutput/addincrement_scaled.test @@ -764,6 +766,12 @@ oops_add_test( TESTNAME addincrement_scaled EXENAME qg_addincrement.x TEST_DEPENDS test_qg_diffstates ) +oops_add_test( TESTNAME convertincrement + MODELNAME qg + OMP 2 + YAMLNAME testinput/convertincrement.yaml + EXENAME qg_convertincrement.x + TEST_DEPENDS test_qg_diffstates ) ##################################################################### # LETKF tests diff --git a/qg/test/testinput/convertincrement.yaml b/qg/test/testinput/convertincrement.yaml new file mode 100644 index 000000000..e31590885 --- /dev/null +++ b/qg/test/testinput/convertincrement.yaml @@ -0,0 +1,37 @@ +input geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] +output geometry: + nx: 20 + ny: 10 + depths: [4500.0, 5500.0] +linear variable changes: + - variable change: ChVarQG + input variables: [x] + output variables: [q] + - variable change: ChVarQG + input variables: [q] + output variables: [x] + - variable change: Identity + input variables: [x] + output variables: [x] + - variable change: Identity + input variables: [x] + output variables: [x] + do inverse: true +increments: +- date: '2010-01-01T12:00:00Z' + input variables: [x] + input: + date: '2010-01-01T12:00:00Z' + filename: Data/difst.in.2010-01-01T12:00:00Z.nc + state variables: [x] + output: + date: '2010-01-01T12:00:00Z' + datadir: Data + exp: convertinc + type: in + trajectory: + date: '2010-01-01T12:00:00Z' + filename: Data/mem002.eda_3dvar.an.2010-01-01T12:00:00Z.nc diff --git a/qg/test/testoutput/convertincrement.test b/qg/test/testoutput/convertincrement.test new file mode 100644 index 000000000..9a849ab94 --- /dev/null +++ b/qg/test/testoutput/convertincrement.test @@ -0,0 +1,44 @@ +Test : Input increment: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 +Test : Trajectory state: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 40, 20, 2 +Test : Variable = streamfunction +Test : Boundary conditions are activated +Test : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 +Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Increment after variable transform: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 20, 10, 2 +Test : Variable = potential vorticity +Test : Boundary conditions are not activated +Test : Min= -1.7247e-04, Max= 5.3114e-05, RMS= 2.0353e-05 +Test : Increment after variable transform: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 20, 10, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 +Test : Increment after variable transform: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 20, 10, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 +Test : Increment after variable transform: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 20, 10, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 +Test : Output increment: +Test : Valid time: 2010-01-01T12:00:00Z +Test : Resolution = 20, 10, 2 +Test : Variable = streamfunction +Test : Boundary conditions are not activated +Test : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 diff --git a/src/oops/base/IncrementEnsemble.h b/src/oops/base/IncrementEnsemble.h index e92e044bf..6add14a10 100644 --- a/src/oops/base/IncrementEnsemble.h +++ b/src/oops/base/IncrementEnsemble.h @@ -53,6 +53,12 @@ template class IncrementEnsemble { IncrementEnsemble(const StateEnsemble_ & ens, const State_ & mean, const Variables & vars); IncrementEnsemble(const eckit::Configuration &, const State_ &, const State_ &, const Geometry_ &, const Variables &); + /// \brief construct ensemble of perturbations by reading them from disk + IncrementEnsemble(const Geometry_ &, const Variables &, const eckit::Configuration &); + /// \brief construct ensemble of perturbations by reading two state ensembles (one member at a + // time) and taking the difference of each set of pairs + IncrementEnsemble(const Geometry_ &, const Variables &, const eckit::Configuration &, + const eckit::Configuration &); /// Accessors size_t size() const {return ensemblePerturbs_.size();} @@ -160,6 +166,62 @@ IncrementEnsemble::IncrementEnsemble(const eckit::Configuration & conf, // ----------------------------------------------------------------------------- +template +IncrementEnsemble::IncrementEnsemble(const Geometry_ & resol, const Variables & vars, + const eckit::Configuration & config) + : vars_(vars), ensemblePerturbs_() +{ + std::vector memberConfig; + config.get("members", memberConfig); + + // Datetime for ensemble + util::DateTime tslot = util::DateTime(config.getString("date")); + + // Reserve memory to hold ensemble + ensemblePerturbs_.reserve(memberConfig.size()); + + // Loop over all ensemble members + for (size_t jj = 0; jj < memberConfig.size(); ++jj) { + Increment_ dx(resol, vars_, tslot); + dx.read(memberConfig[jj]); + ensemblePerturbs_.emplace_back(std::move(dx)); + } + Log::trace() << "IncrementEnsemble:contructor (by reading increment ensemble) done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +IncrementEnsemble::IncrementEnsemble(const Geometry_ & resol, const Variables & vars, + const eckit::Configuration & configBase, + const eckit::Configuration & configPert) + : vars_(vars), ensemblePerturbs_() +{ + std::vector memberConfigBase; + configBase.get("members", memberConfigBase); + + std::vector memberConfigPert; + configPert.get("members", memberConfigPert); + + // Ensure input ensembles are of the same size + ASSERT(memberConfigBase.size() == memberConfigPert.size()); + + // Reserve memory to hold ensemble + ensemblePerturbs_.reserve(memberConfigBase.size()); + + // Loop over all ensemble members + for (size_t jj = 0; jj < memberConfigBase.size(); ++jj) { + State_ xBase(resol, memberConfigBase[jj]); + State_ xPert(resol, memberConfigPert[jj]); + Increment_ dx(resol, vars_, xBase.validTime()); + dx.diff(xBase, xPert); + ensemblePerturbs_.emplace_back(std::move(dx)); + } + Log::trace() << "IncrementEnsemble:contructor (by diffing state ensembles) done" << std::endl; +} + +// ----------------------------------------------------------------------------- + template void IncrementEnsemble::releaseMember() { ensemblePerturbs_.erase(ensemblePerturbs_.begin()); diff --git a/src/oops/generic/IdLinearVariableChange.h b/src/oops/generic/IdLinearVariableChange.h index 29ce975b2..e615e9ca3 100644 --- a/src/oops/generic/IdLinearVariableChange.h +++ b/src/oops/generic/IdLinearVariableChange.h @@ -1,6 +1,6 @@ /* * (C) Copyright 2017-2018 UCAR. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ @@ -46,7 +46,7 @@ class IdLinearVariableChange : public LinearVariableChangeBase { void multiplyInverseAD(const Increment_ & dx1, Increment_ & dx2) const override {dx2 = dx1;} private: - void print(std::ostream &) const override {} + void print(std::ostream & os) const override {os << "IdVariableChange";} }; // ----------------------------------------------------------------------------- diff --git a/src/oops/runs/ConvertIncrement.h b/src/oops/runs/ConvertIncrement.h new file mode 100644 index 000000000..60802a0c6 --- /dev/null +++ b/src/oops/runs/ConvertIncrement.h @@ -0,0 +1,133 @@ +/* + * (C) Copyright 2018-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_RUNS_CONVERTINCREMENT_H_ +#define OOPS_RUNS_CONVERTINCREMENT_H_ + +#include +#include +#include + +#include "eckit/config/LocalConfiguration.h" +#include "oops/base/LinearVariableChangeBase.h" +#include "oops/generic/instantiateVariableChangeFactory.h" +#include "oops/interface/Geometry.h" +#include "oops/interface/Increment.h" +#include "oops/interface/State.h" +#include "oops/mpi/mpi.h" +#include "oops/runs/Application.h" +#include "oops/util/DateTime.h" +#include "oops/util/Duration.h" +#include "oops/util/Logger.h" + +namespace oops { + +template class ConvertIncrement : public Application { + typedef Geometry Geometry_; + typedef Increment Increment_; + typedef State State_; + typedef LinearVariableChangeBase LinearVariableChange_; + typedef LinearVariableChangeFactory LinearVariableChangeFactory_; + + public: +// ------------------------------------------------------------------------------------------------- + explicit ConvertIncrement(const eckit::mpi::Comm & comm = oops::mpi::world()) : Application(comm) + { + instantiateVariableChangeFactory(); + } +// ------------------------------------------------------------------------------------------------- + virtual ~ConvertIncrement() {} +// ------------------------------------------------------------------------------------------------- + int execute(const eckit::Configuration & fullConfig) const { +// Setup resolution for intput and output + const eckit::LocalConfiguration inputResolConfig(fullConfig, "input geometry"); + const Geometry_ resol1(inputResolConfig, this->getComm()); + + const eckit::LocalConfiguration outputResolConfig(fullConfig, "output geometry"); + const Geometry_ resol2(outputResolConfig, this->getComm()); + +// Variable transform(s) + std::vector inverse; + std::vector adjoint; + + std::vector chvarconfs; + fullConfig.get("linear variable changes", chvarconfs); + for (size_t cv = 0; cv < chvarconfs.size(); ++cv) { + inverse.push_back(chvarconfs[cv].getBool("do inverse", false)); + adjoint.push_back(chvarconfs[cv].getBool("do adjoint", false)); + } + +// List of input and output increments + std::vector incrementsConf; + fullConfig.get("increments", incrementsConf); + int nincrements = incrementsConf.size(); + +// Loop over increments + for (int jm = 0; jm < nincrements; ++jm) { +// Print output + Log::info() << "Converting increment " << jm+1 << " of " << nincrements << std::endl; + +// Datetime for incrmement + const util::DateTime incdatetime(incrementsConf[jm].getString("date")); + +// Variables for input increment + const Variables incvars(incrementsConf[jm], "input variables"); + +// Read input + const eckit::LocalConfiguration inputConfig(incrementsConf[jm], "input"); + Increment_ dxi(resol1, incvars, incdatetime); + dxi.read(inputConfig); + Log::test() << "Input increment: " << dxi << std::endl; + +// Copy and change resolution + std::unique_ptr dx(new Increment_(resol2, dxi)); // Pointer that can be reset + +// Trajectory state for linear variable transform + std::unique_ptr xtraj; // Pointer that can be reset + +// Variable transform(s) + for (size_t cv = 0; cv < chvarconfs.size(); ++cv) { + // Read trajectory + if (cv == 0) { + const eckit::LocalConfiguration trajConfig(incrementsConf[jm], "trajectory"); + xtraj.reset(new State_(resol1, trajConfig)); + ASSERT(xtraj->validTime() == dx->validTime()); // Check time is consistent + Log::test() << "Trajectory state: " << *xtraj << std::endl; + } + + // Create variable change + std::unique_ptr lvc; + lvc.reset(LinearVariableChangeFactory_::create(*xtraj, *xtraj, resol2, chvarconfs[cv])); + + // Print info + Log::info() << "Variable transform " << cv+1 << " of " << chvarconfs.size() << ": " + << *lvc << std::endl; + + Increment_ xchvarout = lvc->multiply(*dx); + dx.reset(new Increment_(xchvarout)); + + Log::test() << "Increment after variable transform: " << *dx << std::endl; + } + +// Write state + const eckit::LocalConfiguration outputConfig(incrementsConf[jm], "output"); + dx->write(outputConfig); + + Log::test() << "Output increment: " << *dx << std::endl; + } + return 0; + } +// ------------------------------------------------------------------------------------------------- + private: + std::string appname() const { + return "oops::ConvertIncrement<" + MODEL::name() + ">"; + } +// ------------------------------------------------------------------------------------------------- +}; + +} // namespace oops +#endif // OOPS_RUNS_CONVERTINCREMENT_H_ diff --git a/src/oops/runs/EnsRecenter.h b/src/oops/runs/EnsRecenter.h index c5d165c45..c1a834fdb 100644 --- a/src/oops/runs/EnsRecenter.h +++ b/src/oops/runs/EnsRecenter.h @@ -44,6 +44,12 @@ template class EnsRecenter : public Application { const eckit::LocalConfiguration bkgConfig(fullConfig, "center"); State_ x_center(resol, bkgConfig); + // Optionally zero the center + bool zeroCenter = fullConfig.getBool("zero center", false); + if (zeroCenter) { + x_center.zero(); + } + // Get ensemble configuration std::vector ensConfig; fullConfig.get("ensemble", ensConfig); @@ -62,6 +68,12 @@ template class EnsRecenter : public Application { } Log::test() << "Ensemble mean: " << std::endl << ensmean << std::endl; + // Optionally write the mean out + if (fullConfig.has("ensemble mean output")) { + eckit::LocalConfiguration meanout(fullConfig, "ensemble mean output"); + ensmean.write(meanout); + } + // Setup variables const Variables vars(fullConfig, "recenter variables"); @@ -79,6 +91,7 @@ template class EnsRecenter : public Application { x.write(recenterout); Log::test() << "Recentered member " << jj << " : " << x << std::endl; } + return 0; } // ----------------------------------------------------------------------------- From 4fd604f90cebee8326f83ff3fce3eee977821a48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Wed, 24 Mar 2021 10:44:36 -0600 Subject: [PATCH 086/142] put_time not working on hera (#1115) * put_time not working on hera * use std::strftime instead of std::put_time Co-authored-by: Anna Shlyaeva Co-authored-by: Anna Shlyaeva --- src/oops/util/LibOOPS.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/oops/util/LibOOPS.cc b/src/oops/util/LibOOPS.cc index 4fa2c7d72..e2a81bd8b 100644 --- a/src/oops/util/LibOOPS.cc +++ b/src/oops/util/LibOOPS.cc @@ -74,8 +74,9 @@ LibOOPS& LibOOPS::instance() { */ void LibOOPS::initialise() { std::time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); - Log::info() << "OOPS Starting " - << std::put_time(std::localtime(&now), "%F %T (UTC%z)") << std::endl; + char nowstr[100]; + std::strftime(nowstr, sizeof(nowstr), "%F %T (UTC%z)", std::localtime(&now)); + Log::info() << "OOPS Starting " << nowstr << std::endl; rank_ = oops::mpi::world().rank(); @@ -139,8 +140,9 @@ void LibOOPS::finalise(bool finaliseMPI) { } #endif std::time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); - Log::info() << "OOPS Ending " - << std::put_time(std::localtime(&now), "%F %T (UTC%z)") << std::endl; + char nowstr[100]; + std::strftime(nowstr, sizeof(nowstr), "%F %T (UTC%z)", std::localtime(&now)); + Log::info() << "OOPS Ending " << nowstr << std::endl; // Make sure that these specialised channels that wrap eckit::Log::info() are // destroyed before eckit::Log::info gets destroyed. From de6e63d2adf95c89c5537504818a7877aebb76fe Mon Sep 17 00:00:00 2001 From: Mark J Olah Date: Mon, 29 Mar 2021 15:24:55 -0600 Subject: [PATCH 087/142] TestReference rewrite without regex (#1114) * Removed std::regex dependency from TestReference. Addressed additional error conditions. * Add unit testing for TestReference and refactor member names * Fix merging with develop * Cleanup headers and use fwd decalarations where possible * Fix coding norms * Rework parse_numbers algorithm to eliminate bugs * write testChannel to testStream * Modify l95_4dvar_obsbias test to use new TestReference class Co-authored-by: Maryam Abdi-Oskouei --- l95/test/CMakeLists.txt | 12 +- l95/test/testinput/4dvar.obsbias.yaml | 6 + src/CMakeLists.txt | 8 + src/oops/runs/Run.cc | 4 + src/oops/util/LibOOPS.cc | 18 +- src/oops/util/LibOOPS.h | 11 ++ src/oops/util/TestReference.cc | 250 ++++++++++++++++++++++++++ src/oops/util/TestReference.h | 99 ++++++++++ src/test/util/TestReference.cc | 16 ++ src/test/util/TestReference.h | 83 +++++++++ 10 files changed, 499 insertions(+), 8 deletions(-) create mode 100644 src/oops/util/TestReference.cc create mode 100644 src/oops/util/TestReference.h create mode 100644 src/test/util/TestReference.cc create mode 100644 src/test/util/TestReference.h diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index ee94fd396..a31a21028 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -630,12 +630,12 @@ oops_add_test( TESTNAME 4dsaddlepoint EXENAME l95_4dvar.x TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -oops_add_test( TESTNAME 4dvar.obsbias - MODELNAME l95 - YAMLNAME testinput/4dvar.obsbias.yaml - EXENAME l95_4dvar.x - COMPARE - TEST_DEPENDS test_l95_forecast test_l95_makeobsbias ) +# OOPS TestReference unit test example +ecbuild_add_test( TARGET test_l95_4dvar.obsbias + COMMAND l95_4dvar.x + ARGS testinput/4dvar.obsbias.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobsbias ) + #-------------------------------------------------------------------- oops_add_test( TESTNAME 4dvar.allbiases diff --git a/l95/test/testinput/4dvar.obsbias.yaml b/l95/test/testinput/4dvar.obsbias.yaml index e0b969eb0..0af58c0f8 100644 --- a/l95/test/testinput/4dvar.obsbias.yaml +++ b/l95/test/testinput/4dvar.obsbias.yaml @@ -74,3 +74,9 @@ output: first: PT3H frequency: PT06H type: an +test: + reference filename: testoutput/4dvar.obsbias.test + float relative tolerance: 0.0 + integer tolerance: 0 + log output filename: testoutput/4dvar.obsbias.log.out + test output filename: testoutput/4dvar.obsbias.test.out diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fc96291af..466dcfa2e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -309,6 +309,8 @@ oops/util/string_f_c_mod.F90 oops/util/string_utils.F90 oops/util/stringFunctions.cc oops/util/stringFunctions.h +oops/util/TestReference.cc +oops/util/TestReference.h oops/util/Timer.cc oops/util/Timer.h oops/util/TimerHelper.cc @@ -414,6 +416,7 @@ test/util/MissingValues.h test/util/PropertiesOfNVectors.h test/util/stringFunctions.h test/util/LocalEnvironment.h +test/util/TestReference.h test/util/TypeTraits.h ) @@ -579,6 +582,11 @@ ecbuild_add_test( TARGET test_util_stringfunctions ARGS "test/testinput/empty.yaml" LIBS oops ) +ecbuild_add_test( TARGET test_util_testreference + SOURCES test/util/TestReference.cc + ARGS "test/testinput/empty.yaml" + LIBS oops ) + ecbuild_add_test( TARGET test_mpi_mpi MPI 4 SOURCES test/mpi/mpi.cc diff --git a/src/oops/runs/Run.cc b/src/oops/runs/Run.cc index 5a04d837d..38d0b2f6e 100644 --- a/src/oops/runs/Run.cc +++ b/src/oops/runs/Run.cc @@ -92,6 +92,10 @@ Run::Run(int argc, char** argv) : eckit::Main(argc, argv, "OOPS_HOME"), config_( // Read configuration config_.reset(new eckit::YAMLConfiguration(configfile)); + // Configure TestReference with "test:" sub-config + if (config_->has("test")) + LibOOPS::instance().testReferenceInitialise(config_->getSubConfiguration("test")); + Log::info() << "Configuration input file is: " << configfile << std::endl; Log::info() << "Full configuration is:" << *config_ << std::endl; diff --git a/src/oops/util/LibOOPS.cc b/src/oops/util/LibOOPS.cc index e2a81bd8b..66012bd1b 100644 --- a/src/oops/util/LibOOPS.cc +++ b/src/oops/util/LibOOPS.cc @@ -68,7 +68,7 @@ LibOOPS& LibOOPS::instance() { } /** Initialization of MPI and dependent variables. - * To be called in `main()` by constructor of `oops::Run`. This method initializes MPI and + * To be called in `main()` by constructor of `oops::Run`. This method initializes MPI and * associated variables that must be initialized after static-init time, and only once `eckit::Main` * has been created. */ @@ -102,6 +102,12 @@ void LibOOPS::initialise() { } enable_timer_channel_ = getEnv("OOPS_TIMER", 0) > 0 && rank_ == 0; + // testStream_ is used by TestReference for comparing test output + // with a reference file + if ( rank_ == 0 ) { + testChannel().addStream(testStream_); + } + #ifdef ENABLE_GPTL do_profile = getEnv("OOPS_PROFILE", 0); #endif @@ -120,6 +126,10 @@ void LibOOPS::teeOutput(const std::string & fileprefix) { eckit::Log::addFile(teefile); } +void LibOOPS::testReferenceInitialise(const eckit::LocalConfiguration &testConf) { + testReference_.initialise(testConf); +} + /** Clears logs and finalises MPI (unless \p finaliseMPI is false). * To be called in on leaving `main()` by the destructor of `oops::Run`. */ @@ -139,6 +149,11 @@ void LibOOPS::finalise(bool finaliseMPI) { } } #endif + + if ( rank_ == 0 ) { + testReference_.finalise(testStream_.str()); + } + std::time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); char nowstr[100]; std::strftime(nowstr, sizeof(nowstr), "%F %T (UTC%z)", std::localtime(&now)); @@ -235,4 +250,3 @@ eckit::Channel& LibOOPS::debugChannel() const { // ----------------------------------------------------------------------------- } // namespace oops - diff --git a/src/oops/util/LibOOPS.h b/src/oops/util/LibOOPS.h index 9dcf9e82b..e74ef1c82 100644 --- a/src/oops/util/LibOOPS.h +++ b/src/oops/util/LibOOPS.h @@ -16,10 +16,14 @@ #define OOPS_UTIL_LIBOOPS_H_ #include +#include #include + #include "eckit/system/Library.h" #include "eckit/utils/Translator.h" +#include "oops/util/TestReference.h" + namespace oops { // ----------------------------------------------------------------------------- @@ -41,6 +45,7 @@ class LibOOPS : public eckit::system::Library { eckit::Channel& timerChannel() const; void initialise(); + void testReferenceInitialise(const eckit::LocalConfiguration &); void teeOutput(const std::string &); void finalise(bool finaliseMPI = true); @@ -64,6 +69,12 @@ class LibOOPS : public eckit::system::Library { std::string predebug_; bool trace_; std::string pretrace_; + + // TestReferece associated member variables + std::stringstream testStream_; + TestReference testReference_; + + private: bool enable_timer_channel_; }; diff --git a/src/oops/util/TestReference.cc b/src/oops/util/TestReference.cc new file mode 100644 index 000000000..08f3e435b --- /dev/null +++ b/src/oops/util/TestReference.cc @@ -0,0 +1,250 @@ +/* + * (C) Copyright 2017-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/util/TestReference.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/exception/Exceptions.h" + +#include "oops/util/LibOOPS.h" +#include "oops/util/Logger.h" + + +namespace oops { + +/** Local helper routines to parse numbers without using std::regex facilities + */ +namespace { + + /** Determine if string is a valid integer representation + */ + bool is_integer_repr(const std::string &s) + { + auto begin = s.begin(); + // Drop initial +/- char + if (*begin == '-' || *begin == '+') { + begin++; + if (begin == s.end()) return false; + } + // All remaining chars should be digits + return std::all_of(begin, s.end(), [](unsigned char c){return std::isdigit(c);} ); + } + + /** Return a vector of all non-overlapping substrings that can be parsed as valid numbers + */ + std::vector parse_numbers(const std::string &s) + { + std::vector numbers; + std::string init_char = "+-0123456789"; // A number must start with these chars + auto start = s.find_first_of(init_char); + while (start < s.size()) { + // Number must have a digit as first or second char + if (!std::isdigit(s[start])) { + if (start == s.size()-1) break; + if (!std::isdigit(s[start+1])) { + start = s.find_first_of(init_char, start+1); + continue; + } + } + // Sequence of chars will parse as a number, pos is the extent successfully converted + size_t pos = 0; + std::stod(s.substr(start), &pos); + numbers.push_back(s.substr(start, pos)); + start = s.find_first_of(init_char, start+pos); + } + return numbers; + } + +} // anonymous namespace + +// ----------------------------------------------------------------------------- + +void TestReference::initialise(const eckit::LocalConfiguration &conf) +{ + initCheck_ = true; + refFile_ = conf.getString("reference filename"); + oops::Log::info() << "[TestReference] Comparing to reference file: " << refFile_ << std::endl; + tolFloat_ = conf.getFloat("float relative tolerance", 0.0); + tolInt_ = conf.getInt("integer tolerance", 0); + if (conf.has("log output filename")) { + outputFile_ = conf.getString("log output filename"); + LibOOPS::instance().teeOutput(outputFile_); + oops::Log::info() << "[TestReference] Saving Log output to: " << outputFile_ << std::endl; + } + if (conf.has("test output filename")) { + testFile_ = conf.getString("test output filename"); + oops::Log::info() << "[TestReference] Saving Test output to: " << testFile_ << std::endl; + } +} + + +void TestReference::compare(const std::string & test, + const std::string & ref, + FloatT tolFloat, IntT tolInt) +{ + int lineCounter = 0; + std::string testPrefix = "Test : "; // Test prefix string to remove + std::stringstream testStream(test); + std::stringstream refStream(ref); + std::string refLine, testLine; + + while (std::getline(refStream, refLine)) { + lineCounter++; + std::getline(testStream, testLine); + + // Check that a corresponding test line exists + if (testStream.fail()) { + throw TestReferenceMissingTestLineError(lineCounter, refLine); + } + + // Remove test prefix from refLine if present + if (testPrefix == refLine.substr(0, testPrefix.size())) { + refLine.erase(0, testPrefix.size()); + } + + if (refLine == testLine) continue; + + // strings don't match - parse all numbers in the test and reference strings + auto refNums = parse_numbers(refLine); + auto testNums = parse_numbers(testLine); + + if (refNums.empty() || testNums.empty() || refNums.size() != testNums.size()) { + // No numbers parsed or differing numbers parsed in each line. + throw TestReferenceTextMismatchError(lineCounter, testLine, refLine); + } + + // Check if each pair of numbers are within tolerance + for (size_t i = 0; i < refNums.size(); i++) { + if (is_integer_repr(refNums[i]) && is_integer_repr(testNums[i])) { + // both test and reference appear to be integers + auto ref_int = stol(refNums[i]); + auto test_int = stol(testNums[i]); + if (std::abs(ref_int - test_int) > tolInt) + throw TestReferenceIntegerMismatchError(lineCounter, test_int, + ref_int, tolInt, testLine, refLine); + } else { + FloatT ref_float = stod(refNums[i]); + FloatT test_float = stod(testNums[i]); + FloatT rel_diff = std::abs((ref_float - test_float)/(0.5 * (ref_float + test_float))); + if (rel_diff > tolFloat) + throw TestReferenceFloatMismatchError(lineCounter, test_float, + ref_float, tolFloat, testLine, refLine); + } + } + } + + // Check there are no more remaining test lines to process + std::getline(testStream, testLine); + if (!testStream.fail()) { + throw TestReferenceMissingReferenceLineError(lineCounter, testLine); + } +} + + +void TestReference::finalise(const std::string & testStr) +{ + if (!initCheck_) return; + + // Read reference file to string + std::ifstream refFileIn(refFile_); + if (refFileIn.fail()) throw eckit::CantOpenFile(refFile_); + std::string refStr(std::istreambuf_iterator{refFileIn}, std::istreambuf_iterator{}); + refFileIn.close(); + + if (!testFile_.empty()) { + // Write test string to file + std::ofstream testFileOut(testFile_); + if (!testFileOut) throw eckit::CantOpenFile(testFile_); + testFileOut << testStr; + testFileOut.close(); + } + + compare(testStr, refStr, tolFloat_, tolInt_); +} + +// ----------------------------------------------------------------------------- + +TestReferenceMissingReferenceLineError::TestReferenceMissingReferenceLineError(int line_num, + const std::string &test_line) +{ + std::ostringstream os; + os << "TestReference: Missing reference file line corresponding to test output Line#:" << line_num + << "\nTest line: '" << test_line << "'"; + what_ = os.str(); +} + + +TestReferenceMissingTestLineError::TestReferenceMissingTestLineError(int line_num, + const std::string &ref_line) +{ + std::ostringstream os; + os << "TestReference: Missing test output line corresponding to reference file Line#:" << line_num + << "\nRef line: '" << ref_line << "'"; + what_ = os.str(); +} + + +TestReferenceTextMismatchError::TestReferenceTextMismatchError(int line_num, + const std::string &test_line, const std::string &ref_line) +{ + std::ostringstream os; + os << "Test reference Text mismatch @ Line:" << line_num + << "\nTest: '" << test_line << "'\nRef: '" << ref_line << "'"; + what_ = os.str(); +} + + +TestReferenceIntegerMismatchError::TestReferenceIntegerMismatchError(int line_num, + NumT test_val, NumT ref_val, NumT tolerance, + const std::string &test_line, + const std::string &ref_line) +{ + std::ostringstream os; + os << "Test reference Integer mismatch @ Line:" << line_num << "\n" + << "Test Val : " << test_val << "\n" + << "Ref Val : " << ref_val << "\n" + << "Delta : " << std::abs(test_val-ref_val) << "\n" + << "Tolerance: " << tolerance << "\n" + << "Test Line: '" << test_line << "'\n" + << "Ref Line : '" << ref_line << "'"; + what_ = os.str(); +} + + +TestReferenceFloatMismatchError::TestReferenceFloatMismatchError(int line_num, + NumT test_val, NumT ref_val, NumT tolerance, + const std::string &test_line, + const std::string &ref_line) +{ + std::ostringstream os; + os << "Test reference Float mismatch @ Line:" << line_num <<"\n" + << std::setprecision(std::numeric_limits::digits10 + 1) + << std::scientific + << "Test Val : " << test_val << "\n" + << "Ref Val : " << ref_val << "\n" + << "Delta : " << std::abs(test_val-ref_val) << "\n" + << "Tolerance: " << tolerance << "\n" + << "Test Line: '" << test_line << "'\n" + << "Ref Line : '" << ref_line << "'"; + what_ = os.str(); +} + +} // namespace oops diff --git a/src/oops/util/TestReference.h b/src/oops/util/TestReference.h new file mode 100644 index 000000000..89b678afb --- /dev/null +++ b/src/oops/util/TestReference.h @@ -0,0 +1,99 @@ +/* + * (C) Copyright 2017-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_UTIL_TESTREFERENCE_H_ +#define OOPS_UTIL_TESTREFERENCE_H_ + +#include +#include +#include + +namespace eckit { + class LocalConfiguration; +} + +namespace oops { + +class TestReference { + public: + using FloatT = double; + using IntT = int64_t; + + void finalise(const std::string &testStr); + void initialise(const eckit::LocalConfiguration &conf); + + static void compare(const std::string &test, const std::string &ref, + FloatT tolFloat, IntT tolInt); + + private: + bool initCheck_ = false; + + std::string refFile_; + std::string outputFile_; + std::string testFile_; + FloatT tolFloat_; + IntT tolInt_; +}; + +// Base class for all TestReference errors +class TestReferenceError : public std::exception { + public: + const char* what() const noexcept override + { + return what_.c_str(); + } + + protected: + std::string what_; // error message to print +}; + +// Error: One or more reference lines are missing +class TestReferenceMissingReferenceLineError : public TestReferenceError +{ + public: + TestReferenceMissingReferenceLineError(int line_num, + const std::string &test_line); +}; + +// Error: One or more test lines are missing +class TestReferenceMissingTestLineError : public TestReferenceError +{ + public: + TestReferenceMissingTestLineError(int line_num, + const std::string &ref_line); +}; + +// Error: Test and reference lines don't match +class TestReferenceTextMismatchError : public TestReferenceError +{ + public: + TestReferenceTextMismatchError(int line_num, + const std::string &test_line, const std::string &ref_line); +}; + +// Error: Test and reference lines parsed integer representations not within tolerance +class TestReferenceIntegerMismatchError : public TestReferenceError { + public: + using NumT = TestReference::IntT; + TestReferenceIntegerMismatchError(int line_num, + NumT test_val, NumT ref_val, NumT tolerance, + const std::string &test_line, const std::string &ref_line); +}; + +// Error: Test and reference lines parsed floating-point representations not within tolerance +class TestReferenceFloatMismatchError : public TestReferenceError { + public: + using NumT = TestReference::FloatT; + TestReferenceFloatMismatchError(int line_num, + NumT test_val, NumT ref_val, NumT tolerance, + const std::string &test_line, const std::string &ref_line); +}; + + +} // namespace oops + +#endif // OOPS_UTIL_TESTREFERENCE_H_ diff --git a/src/test/util/TestReference.cc b/src/test/util/TestReference.cc new file mode 100644 index 000000000..36ab654e1 --- /dev/null +++ b/src/test/util/TestReference.cc @@ -0,0 +1,16 @@ +/* + * (C) Copyright 2017-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "test/util/TestReference.h" + +#include "oops/runs/Run.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + test::TestReference tests; + return run.execute(tests); +} diff --git a/src/test/util/TestReference.h b/src/test/util/TestReference.h new file mode 100644 index 000000000..7bda5de85 --- /dev/null +++ b/src/test/util/TestReference.h @@ -0,0 +1,83 @@ +/* + * (C) Copyright 2017-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef TEST_UTIL_TESTREFERENCE_H_ +#define TEST_UTIL_TESTREFERENCE_H_ + +#include + +#include "oops/runs/Test.h" +#include "oops/util/TestReference.h" + +namespace test { + +CASE("util/TestReference") { + std::string test1 = "Line 1: +0123456789\n"; + std::string test2 = "Line 1: +0123456789\nLine 2: +99.98764321e-3\n"; + std::string test3 = "-ABC-XYZ-\n"; + std::string test4 = "-99-+EE-++44E0\n"; // Parsed as {"-99", "+44E0"} + + std::string good_ref1 = "Line 1: 123456789\n"; + std::string good_ref2 = "Line 1: 123456789\nLine 2: 9.9987643212e-2\n"; + std::string good_ref4 = "-9.90000E1-+EE-+44\n"; + + std::string bad_ref1 = "Line 1: -123456789\n"; + std::string bad_ref2_1 = "Line 1: 123456789\nLine 2: 9.9987643e-2\n"; + std::string bad_ref2_2 = "Line 1: 123456789\nLine 2: 9.9987643212e-2 42\n"; + std::string bad_ref3 = "-ABC+XYZ-\n"; + std::string bad_ref4_1 = "-99-+EE-+-43\n"; + std::string bad_ref4_2 = "-90-+EE-+-43\n"; + std::string bad_ref4_3 = "-99-+EE-++44EE0\n"; // Parsed as {"-99", "+44", "0"} + + oops::TestReference::IntT iTol = 0; + oops::TestReference::FloatT fTol = 1e-9; + + // Extra parentheses protect EXPECT_* macros arguments which cannot contain commas. + EXPECT_NO_THROW((oops::TestReference::compare(test1, good_ref1, fTol, iTol))); + + EXPECT_NO_THROW((oops::TestReference::compare(test2, good_ref2, fTol, iTol))); + + EXPECT_THROWS_AS((oops::TestReference::compare(test1, good_ref2, fTol, iTol)), + oops::TestReferenceMissingTestLineError); + + EXPECT_THROWS_AS((oops::TestReference::compare(test2, good_ref1, fTol, iTol)), + oops::TestReferenceMissingReferenceLineError); + + EXPECT_THROWS_AS((oops::TestReference::compare(test1, bad_ref1, fTol, iTol)), + oops::TestReferenceIntegerMismatchError); + + EXPECT_THROWS_AS((oops::TestReference::compare(test2, bad_ref2_1, fTol, iTol)), + oops::TestReferenceFloatMismatchError); + + EXPECT_THROWS_AS((oops::TestReference::compare(test2, bad_ref2_2, fTol, iTol)), + oops::TestReferenceTextMismatchError); + + EXPECT_THROWS_AS((oops::TestReference::compare(test3, bad_ref3, fTol, iTol)), + oops::TestReferenceTextMismatchError); + + EXPECT_NO_THROW((oops::TestReference::compare(test4, good_ref4, fTol, iTol))); + + EXPECT_THROWS_AS((oops::TestReference::compare(test4, bad_ref4_1, fTol, iTol)), + oops::TestReferenceFloatMismatchError); + + EXPECT_THROWS_AS((oops::TestReference::compare(test4, bad_ref4_2, fTol, iTol)), + oops::TestReferenceIntegerMismatchError); + + EXPECT_THROWS_AS((oops::TestReference::compare(test4, bad_ref4_3, fTol, iTol)), + oops::TestReferenceTextMismatchError); +} // CASE("util/TestReference") + +class TestReference : public oops::Test { + private: + std::string testid() const override {return "test::TestReference";} + void register_tests() const override {} + void clear() const override {} +}; + +} // namespace test + +#endif // TEST_UTIL_TESTREFERENCE_H_ From 99533c4d0352625c48c546e02bd2dda0225b19f6 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Mon, 29 Mar 2021 16:19:36 -0600 Subject: [PATCH 088/142] add ObsVector::mask test and QG implementation of ObsVector::mask (#1116) * add ObsVector::mask test and update QG to work * add a test for ObsVector::ones * add a test for ObsVector::packEigen after applying a mask * change comment --- l95/src/lorenz95/ObsVec1D.cc | 4 + l95/src/lorenz95/ObsVec1D.h | 3 + l95/test/testinput/interfaces.yaml | 1 + qg/model/ObsDataQG.h | 5 +- qg/model/ObsVecQG.cc | 46 +++++----- qg/model/ObsVecQG.h | 5 +- qg/model/QgFortran.h | 7 +- qg/model/qg_obsvec_interface.F90 | 54 +++++++++-- qg/model/qg_obsvec_mod.F90 | 125 +++++++++++++++++++------ qg/test/testinput/interfaces.yaml | 3 + src/oops/interface/ObsVector.h | 17 +++- src/test/interface/ObsVector.h | 143 ++++++++++++++++++++++++++++- 12 files changed, 351 insertions(+), 62 deletions(-) diff --git a/l95/src/lorenz95/ObsVec1D.cc b/l95/src/lorenz95/ObsVec1D.cc index bfdb96bde..c5b7c4b65 100644 --- a/l95/src/lorenz95/ObsVec1D.cc +++ b/l95/src/lorenz95/ObsVec1D.cc @@ -109,6 +109,10 @@ void ObsVec1D::zero() { for (double & val : data_) val = 0.0; } // ----------------------------------------------------------------------------- +void ObsVec1D::ones() { + for (double & val : data_) val = 1.0; +} +// ----------------------------------------------------------------------------- void ObsVec1D::invert() { for (double & val : data_) { if (val != missing_) val = 1.0/val; diff --git a/l95/src/lorenz95/ObsVec1D.h b/l95/src/lorenz95/ObsVec1D.h index b7e7987df..804f8a6c0 100644 --- a/l95/src/lorenz95/ObsVec1D.h +++ b/l95/src/lorenz95/ObsVec1D.h @@ -52,6 +52,9 @@ class ObsVec1D : public util::Printable, double & operator[](const std::size_t ii) {return data_.at(ii);} void zero(); + /// set all values to ones (for tests) + void ones(); + void axpy(const double &, const ObsVec1D &); void invert(); void random(); diff --git a/l95/test/testinput/interfaces.yaml b/l95/test/testinput/interfaces.yaml index 5c750ff49..060de9617 100644 --- a/l95/test/testinput/interfaces.yaml +++ b/l95/test/testinput/interfaces.yaml @@ -114,6 +114,7 @@ observations: covariance: standard_deviation: 0.5 rms ref: 8.3207407741318846 + reference nobs: 160 tolerance: 1.0e-10 local obs space: diff --git a/qg/model/ObsDataQG.h b/qg/model/ObsDataQG.h index 57013cfdd..b1893f351 100644 --- a/qg/model/ObsDataQG.h +++ b/qg/model/ObsDataQG.h @@ -46,6 +46,8 @@ class ObsDataQG : public util::Printable, void read(const std::string &); void save(const std::string &) const; + const int & toFortran() const {return data_.toFortran();} + private: void print(std::ostream &) const; @@ -74,7 +76,8 @@ void ObsDataQG::zero() { } // ----------------------------------------------------------------------------- template -void ObsDataQG::mask(const ObsDataQG) { +void ObsDataQG::mask(const ObsDataQG mask) { + qg_obsvec_mask_f90(data_.toFortran(), mask.toFortran()); } // ----------------------------------------------------------------------------- template diff --git a/qg/model/ObsVecQG.cc b/qg/model/ObsVecQG.cc index db2ed261c..131cd3487 100644 --- a/qg/model/ObsVecQG.cc +++ b/qg/model/ObsVecQG.cc @@ -1,6 +1,6 @@ /* * (C) Copyright 2009-2016 ECMWF. - * (C) Copyright 2017-2019 UCAR. + * (C) Copyright 2017-2021 UCAR. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -13,6 +13,7 @@ #include "oops/util/Logger.h" +#include "model/ObsDataQG.h" #include "model/ObsSpaceQG.h" #include "model/ObsVecQG.h" #include "model/QgFortran.h" @@ -48,7 +49,6 @@ ObsVecQG::~ObsVecQG() { } // ----------------------------------------------------------------------------- ObsVecQG & ObsVecQG::operator= (const ObsVecQG & rhs) { - ASSERT(nobs() == rhs.nobs()); const int keyOvecRhs = rhs.keyOvec_; qg_obsvec_copy_f90(keyOvec_, keyOvecRhs); return *this; @@ -60,28 +60,24 @@ ObsVecQG & ObsVecQG::operator*= (const double & zz) { } // ----------------------------------------------------------------------------- ObsVecQG & ObsVecQG::operator+= (const ObsVecQG & rhs) { - ASSERT(nobs() == rhs.nobs()); const int keyOvecRhs = rhs.keyOvec_; qg_obsvec_add_f90(keyOvec_, keyOvecRhs); return *this; } // ----------------------------------------------------------------------------- ObsVecQG & ObsVecQG::operator-= (const ObsVecQG & rhs) { - ASSERT(nobs() == rhs.nobs()); const int keyOvecRhs = rhs.keyOvec_; qg_obsvec_sub_f90(keyOvec_, keyOvecRhs); return *this; } // ----------------------------------------------------------------------------- ObsVecQG & ObsVecQG::operator*= (const ObsVecQG & rhs) { - ASSERT(nobs() == rhs.nobs()); const int keyOvecRhs = rhs.keyOvec_; qg_obsvec_mul_f90(keyOvec_, keyOvecRhs); return *this; } // ----------------------------------------------------------------------------- ObsVecQG & ObsVecQG::operator/= (const ObsVecQG & rhs) { - ASSERT(nobs() == rhs.nobs()); const int keyOvecRhs = rhs.keyOvec_; qg_obsvec_div_f90(keyOvec_, keyOvecRhs); return *this; @@ -91,8 +87,11 @@ void ObsVecQG::zero() { qg_obsvec_zero_f90(keyOvec_); } // ----------------------------------------------------------------------------- +void ObsVecQG::ones() { + qg_obsvec_ones_f90(keyOvec_); +} +// ----------------------------------------------------------------------------- void ObsVecQG::axpy(const double & zz, const ObsVecQG & rhs) { - ASSERT(nobs() == rhs.nobs()); const int keyOvecRhs = rhs.keyOvec_; qg_obsvec_axpy_f90(keyOvec_, zz, keyOvecRhs); } @@ -106,7 +105,6 @@ void ObsVecQG::random() { } // ----------------------------------------------------------------------------- double ObsVecQG::dot_product_with(const ObsVecQG & other) const { - ASSERT(nobs() == other.nobs()); const int keyOvecOther = other.keyOvec_; double zz; qg_obsvec_dotprod_f90(keyOvec_, keyOvecOther, zz); @@ -124,17 +122,17 @@ double ObsVecQG::rms() const { return zz; } // ----------------------------------------------------------------------------- +void ObsVecQG::mask(const ObsDataQG & mask) { + qg_obsvec_mask_f90(keyOvec_, mask.toFortran()); +} +// ----------------------------------------------------------------------------- void ObsVecQG::save(const std::string & name) const { obsdb_.putdb(name, keyOvec_); } // ----------------------------------------------------------------------------- Eigen::VectorXd ObsVecQG::packEigen() const { Eigen::VectorXd vec(nobs()); - double val; - for (unsigned int ii = 0; ii < nobs(); ++ii) { - qg_obsvec_getat_f90(keyOvec_, ii, val); - vec(ii) = val; - } + qg_obsvec_get_f90(keyOvec_, vec.data(), vec.size()); return vec; } // ----------------------------------------------------------------------------- @@ -143,15 +141,19 @@ void ObsVecQG::read(const std::string & name) { } // ----------------------------------------------------------------------------- void ObsVecQG::print(std::ostream & os) const { - double zmin, zmax, zavg; - qg_obsvec_stats_f90(keyOvec_, zmin, zmax, zavg); - std::ios_base::fmtflags f(os.flags()); - os << obsdb_.obsname() << " nobs= " << nobs() - << std::scientific << std::setprecision(4) - << " Min=" << std::setw(12) << zmin - << ", Max=" << std::setw(12) << zmax - << ", Average=" << std::setw(12) << zavg; - os.flags(f); + if (nobs() == 0) { + os << obsdb_.obsname() << " no observations."; + } else { + double zmin, zmax, zavg; + qg_obsvec_stats_f90(keyOvec_, zmin, zmax, zavg); + std::ios_base::fmtflags f(os.flags()); + os << obsdb_.obsname() << " nobs= " << nobs() + << std::scientific << std::setprecision(4) + << " Min=" << std::setw(12) << zmin + << ", Max=" << std::setw(12) << zmax + << ", Average=" << std::setw(12) << zavg; + os.flags(f); + } } // ----------------------------------------------------------------------------- unsigned int ObsVecQG::nobs() const { diff --git a/qg/model/ObsVecQG.h b/qg/model/ObsVecQG.h index 3a9393bd2..ed7142500 100644 --- a/qg/model/ObsVecQG.h +++ b/qg/model/ObsVecQG.h @@ -49,13 +49,16 @@ class ObsVecQG : public util::Printable, ObsVecQG & operator/= (const ObsVecQG &); Eigen::VectorXd packEigen() const; + /// set all values to zero void zero(); + /// set all values to one + void ones(); void axpy(const double &, const ObsVecQG &); void invert(); void random(); double dot_product_with(const ObsVecQG &) const; double rms() const; - void mask(const ObsDataQG &) {} + void mask(const ObsDataQG &); unsigned int nobs() const; diff --git a/qg/model/QgFortran.h b/qg/model/QgFortran.h index f7db0cf4b..860357fdc 100644 --- a/qg/model/QgFortran.h +++ b/qg/model/QgFortran.h @@ -225,6 +225,10 @@ extern "C" { void qg_obsvec_copy_f90(const F90ovec &, const F90ovec &); void qg_obsvec_copy_local_f90(const F90ovec &, const F90ovec &, const int &, const int *); void qg_obsvec_zero_f90(const F90ovec &); + void qg_obsvec_ones_f90(const F90ovec &); + /// set ObsVector (with key \p obsvector_key) values to missing values where + /// mask ObsVector (with key \p mask_key) values are set to 1 + void qg_obsvec_mask_f90(const F90ovec & obsvector_key, const F90ovec & mask_key); void qg_obsvec_mul_scal_f90(const F90ovec &, const double &); void qg_obsvec_add_f90(const F90ovec &, const F90ovec &); void qg_obsvec_sub_f90(const F90ovec &, const F90ovec &); @@ -236,7 +240,8 @@ extern "C" { void qg_obsvec_dotprod_f90(const F90ovec &, const F90ovec &, double &); void qg_obsvec_stats_f90(const F90ovec &, double &, double &, double &); void qg_obsvec_nobs_f90(const F90ovec &, int &); - void qg_obsvec_getat_f90(const F90ovec &, const int &, double &); + /// fill \p data (size \p nobs) with all non-masked out (non-missing) values + void qg_obsvec_get_f90(const F90ovec &, double * data, const int & nobs); // ----------------------------------------------------------------------------- // Streamfunction observations diff --git a/qg/model/qg_obsvec_interface.F90 b/qg/model/qg_obsvec_interface.F90 index 79b3508fe..0112c7ac8 100644 --- a/qg/model/qg_obsvec_interface.F90 +++ b/qg/model/qg_obsvec_interface.F90 @@ -1,5 +1,5 @@ ! (C) Copyright 2009-2016 ECMWF. -! (C) Copyright 2017-2019 UCAR. +! (C) Copyright 2017-2021 UCAR. ! ! This software is licensed under the terms of the Apache Licence Version 2.0 ! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -149,6 +149,46 @@ subroutine qg_obsvec_zero_c(c_key_self) bind(c,name='qg_obsvec_zero_f90') end subroutine qg_obsvec_zero_c ! ------------------------------------------------------------------------------ +!> Set observation vector to ones +subroutine qg_obsvec_ones_c(c_key_self) bind(c,name='qg_obsvec_ones_f90') + +implicit none + +! Passed variables +integer(c_int),intent(in) :: c_key_self !< Observation vector + +! Local variables +type(qg_obsvec),pointer :: self + +! Interface +call qg_obsvec_registry%get(c_key_self,self) + +! Call Fortran +call qg_obsvec_ones(self) + +end subroutine qg_obsvec_ones_c +! ------------------------------------------------------------------------------ +!> Mask self observation vector (set values to missing where mask is set) +subroutine qg_obsvec_mask_c(c_key_self,c_key_mask) bind(c,name='qg_obsvec_mask_f90') + +implicit none + +! Passed variables +integer(c_int),intent(in) :: c_key_self !< Observation vector +integer(c_int),intent(in) :: c_key_mask !< Mask + +! Local variables +type(qg_obsvec),pointer :: self,mask + +! Interface +call qg_obsvec_registry%get(c_key_self,self) +call qg_obsvec_registry%get(c_key_mask,mask) + +! Call Fortran +call qg_obsvec_mask(self,mask) + +end subroutine qg_obsvec_mask_c +! ------------------------------------------------------------------------------ !> Multiply observation vector with a scalar subroutine qg_obsvec_mul_scal_c(c_key_self,zz) bind(c,name='qg_obsvec_mul_scal_f90') @@ -379,15 +419,15 @@ subroutine qg_obsvec_nobs_c(c_key_self,kobs) bind(c,name='qg_obsvec_nobs_f90') end subroutine qg_obsvec_nobs_c ! ------------------------------------------------------------------------------ -!> Get observation value at iob location -subroutine qg_obsvec_getat_c(c_key_self,iob,val) bind(c,name='qg_obsvec_getat_f90') +!> Get all non-masked out observation values +subroutine qg_obsvec_get_c(c_key_self,vals,nvals) bind(c,name='qg_obsvec_get_f90') implicit none ! Passed variables integer(c_int),intent(in) :: c_key_self !< Observation vector -integer(c_int),intent(in) :: iob !< Observation index -real(c_double),intent(out):: val !< ob. value +integer(c_int),intent(in) :: nvals !< number of obs +real(c_double),intent(out),dimension(nvals) :: vals !< ob. values ! Local vector type(qg_obsvec),pointer :: self @@ -396,9 +436,9 @@ subroutine qg_obsvec_getat_c(c_key_self,iob,val) bind(c,name='qg_obsvec_getat_f9 call qg_obsvec_registry%get(c_key_self,self) ! Call Fortran -call qg_obsvec_getat(self,iob,val) +call qg_obsvec_get(self,vals,nvals) -end subroutine qg_obsvec_getat_c +end subroutine qg_obsvec_get_c ! ------------------------------------------------------------------------------ end module qg_obsvec_interface diff --git a/qg/model/qg_obsvec_mod.F90 b/qg/model/qg_obsvec_mod.F90 index 7245a3188..55a31fb0d 100644 --- a/qg/model/qg_obsvec_mod.F90 +++ b/qg/model/qg_obsvec_mod.F90 @@ -11,6 +11,7 @@ module qg_obsvec_mod use iso_c_binding use kinds +use missing_values_mod use random_mod implicit none @@ -18,9 +19,10 @@ module qg_obsvec_mod private public :: qg_obsvec public :: qg_obsvec_registry -public :: qg_obsvec_setup,qg_obsvec_clone,qg_obsvec_delete,qg_obsvec_copy,qg_obsvec_zero,qg_obsvec_mul_scal,qg_obsvec_add, & +public :: qg_obsvec_setup,qg_obsvec_clone,qg_obsvec_delete,qg_obsvec_copy,qg_obsvec_zero, & + & qg_obsvec_ones, qg_obsvec_mask, qg_obsvec_mul_scal,qg_obsvec_add, & & qg_obsvec_sub,qg_obsvec_mul,qg_obsvec_div,qg_obsvec_axpy,qg_obsvec_invert,qg_obsvec_random,qg_obsvec_dotprod, & - & qg_obsvec_stats,qg_obsvec_nobs,qg_obsvec_copy_local,qg_obsvec_getat + & qg_obsvec_stats,qg_obsvec_nobs,qg_obsvec_copy_local,qg_obsvec_get ! ------------------------------------------------------------------------------ interface subroutine qg_obsvec_random_i(odb,nn,zz) bind(c,name='qg_obsvec_random_f') @@ -36,6 +38,7 @@ end subroutine qg_obsvec_random_i integer :: nobs = 0 !< Number of observations integer :: nlev = 0 !< Number of levels real(kind_real),allocatable :: values(:,:) !< Values + real(kind_real) :: missing !< Missing value end type qg_obsvec #define LISTED_TYPE qg_obsvec @@ -75,6 +78,7 @@ subroutine qg_obsvec_setup(self,nlev,nobs) ! Initialization self%values = 0.0_kind_real +self%missing = missing_value(self%missing) end subroutine qg_obsvec_setup ! ------------------------------------------------------------------------------ @@ -90,6 +94,7 @@ subroutine qg_obsvec_clone(self,other) ! Set sizes self%nlev = other%nlev self%nobs = other%nobs +self%missing = other%missing ! Allocation allocate(self%values(self%nlev,self%nobs)) @@ -125,6 +130,7 @@ subroutine qg_obsvec_copy(self,other) ! Set sizes self%nlev = other%nlev self%nobs = other%nobs + self%missing = other%missing ! Allocation allocate(self%values(self%nlev,self%nobs)) @@ -179,6 +185,34 @@ subroutine qg_obsvec_zero(self) self%values = 0.0 end subroutine qg_obsvec_zero +! ------------------------------------------------------------------------------ +!> Set observation vector to ones +subroutine qg_obsvec_ones(self) + +implicit none + +! Passed variables +type(qg_obsvec),intent(inout) :: self !< Observation vector + +! Set observation vector to ones +self%values = 1.0 + +end subroutine qg_obsvec_ones +! ------------------------------------------------------------------------------ +!> Mask observation vector (set values to missing values where mask == 1) +subroutine qg_obsvec_mask(self,mask) +implicit none + +! Passed variables +type(qg_obsvec),intent(inout) :: self !< Observation vector +type(qg_obsvec),intent(in) :: mask !< mask + +if ((self%nobs/=mask%nobs).or.(self%nlev/=mask%nlev)) call abor1_ftn('qg_obsvec_mask: inconsistent sizes') + +where(mask%values == 1) self%values = self%missing + +end subroutine qg_obsvec_mask + ! ------------------------------------------------------------------------------ !> Multiply observation vector with a scalar subroutine qg_obsvec_mul_scal(self,zz) @@ -190,7 +224,7 @@ subroutine qg_obsvec_mul_scal(self,zz) real(kind_real),intent(in) :: zz !< Multiplier ! Multiply observation vector with a scalar -self%values = zz*self%values +where(self%values /= self%missing) self%values = zz*self%values end subroutine qg_obsvec_mul_scal ! ------------------------------------------------------------------------------ @@ -203,8 +237,14 @@ subroutine qg_obsvec_add(self,other) type(qg_obsvec),intent(inout) :: self !< Observation vector type(qg_obsvec),intent(in) :: other !< Other observation vector +if ((self%nobs/=other%nobs).or.(self%nlev/=other%nlev)) call abor1_ftn('qg_obsvec_add: inconsistent sizes') + ! Add observation vector -self%values = self%values+other%values +where(self%values /= self%missing .and. other%values /= other%missing) + self%values = self%values+other%values +elsewhere + self%values = self%missing +endwhere end subroutine qg_obsvec_add ! ------------------------------------------------------------------------------ @@ -217,8 +257,14 @@ subroutine qg_obsvec_sub(self,other) type(qg_obsvec),intent(inout) :: self !< Observation vector type(qg_obsvec),intent(in) :: other !< Other observation vector +if ((self%nobs/=other%nobs).or.(self%nlev/=other%nlev)) call abor1_ftn('qg_obsvec_sub: inconsistent sizes') + ! Subtract observation vector -self%values = self%values-other%values +where(self%values /= self%missing .and. other%values /= other%missing) + self%values = self%values-other%values +elsewhere + self%values = self%missing +endwhere end subroutine qg_obsvec_sub ! ------------------------------------------------------------------------------ @@ -231,8 +277,14 @@ subroutine qg_obsvec_mul(self,other) type(qg_obsvec),intent(inout) :: self !< Observation vector type(qg_obsvec),intent(in) :: other !< Other observation vector +if ((self%nobs/=other%nobs).or.(self%nlev/=other%nlev)) call abor1_ftn('qg_obsvec_mul: inconsistent sizes') + ! Multiply observation vector -self%values = self%values*other%values +where(self%values /= self%missing .and. other%values /= other%missing) + self%values = self%values*other%values +elsewhere + self%values = self%missing +endwhere end subroutine qg_obsvec_mul ! ------------------------------------------------------------------------------ @@ -245,8 +297,14 @@ subroutine qg_obsvec_div(self,other) type(qg_obsvec),intent(inout) :: self !< Observation vector type(qg_obsvec),intent(in) :: other !< Other observation vector +if ((self%nobs/=other%nobs).or.(self%nlev/=other%nlev)) call abor1_ftn('qg_obsvec_div: inconsistent sizes') + ! Divide observation vector -self%values = self%values/other%values +where(self%values /= self%missing .and. other%values /= other%missing) + self%values = self%values/other%values +elsewhere + self%values = self%missing +endwhere end subroutine qg_obsvec_div ! ------------------------------------------------------------------------------ @@ -260,8 +318,14 @@ subroutine qg_obsvec_axpy(self,zz,other) real(kind_real),intent(in) :: zz !< Multiplier type(qg_obsvec),intent(in) :: other !< Other observation vector +if ((self%nobs/=other%nobs).or.(self%nlev/=other%nlev)) call abor1_ftn('qg_obsvec_axpy: inconsistent sizes') + ! Apply axpy on observation vector -self%values = self%values+zz*other%values +where(self%values /= self%missing .and. other%values /= other%missing) + self%values = self%values+zz*other%values +elsewhere + self%values = self%missing +endwhere end subroutine qg_obsvec_axpy ! ------------------------------------------------------------------------------ @@ -274,7 +338,7 @@ subroutine qg_obsvec_invert(self) type(qg_obsvec),intent(inout) :: self !< Observation vector ! Invert observation vector -self%values = 1.0/self%values +where(self%values /= self%missing) self%values = 1.0/self%values end subroutine qg_obsvec_invert ! ------------------------------------------------------------------------------ @@ -320,7 +384,9 @@ subroutine qg_obsvec_dotprod(obsvec1,obsvec2,zz) ! Loop over values do jobs=1,obsvec1%nobs do jlev=1,obsvec1%nlev - zz = zz+obsvec1%values(jlev,jobs)*obsvec2%values(jlev,jobs) + if (obsvec1%values(jlev, jobs) /= obsvec1%missing .and. & + obsvec2%values(jlev, jobs) /= obsvec2%missing) & + zz = zz+obsvec1%values(jlev,jobs)*obsvec2%values(jlev,jobs) enddo enddo @@ -340,9 +406,10 @@ subroutine qg_obsvec_stats(self,zmin,zmax,zavg) if (self%nobs*self%nlev>0) then ! Compute statistics if (.not.allocated(self%values)) call abor1_ftn('qg_obsvec_stats: obs vector not allocated') - zmin = minval(self%values) - zmax = maxval(self%values) - zavg = sum(self%values)/real(self%nlev*self%nobs,kind_real) + zmin = minval(self%values, mask = (self%values /= self%missing)) + zmax = maxval(self%values, mask = (self%values /= self%missing)) + zavg = sum(self%values, mask = (self%values /= self%missing)) / & + count(mask = (self%values /= self%missing)) else ! Empty observation vector zmin = 0.0 @@ -362,32 +429,36 @@ subroutine qg_obsvec_nobs(self,kobs) integer,intent(inout) :: kobs !< Observation vector size ! Get observation vector size -kobs = self%nobs*self%nlev +kobs = count(mask = (self%values /= self%missing)) end subroutine qg_obsvec_nobs ! ------------------------------------------------------------------------------ -!> Get value from observation vector at location (iob) -subroutine qg_obsvec_getat(self,iob,val) +!> Get non-missing values from observation vector into vals array +subroutine qg_obsvec_get(self,vals,nvals) implicit none ! Passed variables type(qg_obsvec),intent(in) :: self !< Observation vector -integer,intent(in) :: iob !< index into observation vector -real(kind_real), intent(out) :: val!< returned value - -integer :: i1, i2 - -i1 = iob / self%nobs + 1 -i2 = iob - self%nobs*(i1-1) + 1 -! Retrieve obs. value from vector +integer,intent(in) :: nvals !< Number of non-missing values +real(kind_real), dimension(nvals), intent(out) :: vals!< returned value -if (i1>self%nlev .or. i2>self%nobs) call abor1_ftn ('qg_obsvec_getat: index is out of bounds') +integer :: jobs, jlev, jval -val = self%values(i1,i2) +jval = 1 +! Loop over values +do jobs=1,self%nobs + do jlev=1,self%nlev + if (self%values(jlev, jobs) /= self%missing) then + if (jval > nvals) call abor1_ftn('qg_obsvec_get: inconsistent vector size') + vals(jval) = self%values(jlev, jobs) + jval = jval + 1 + endif + enddo +enddo -end subroutine qg_obsvec_getat +end subroutine qg_obsvec_get ! ------------------------------------------------------------------------------ end module qg_obsvec_mod diff --git a/qg/test/testinput/interfaces.yaml b/qg/test/testinput/interfaces.yaml index 86d603398..62bca6d27 100644 --- a/qg/test/testinput/interfaces.yaml +++ b/qg/test/testinput/interfaces.yaml @@ -81,6 +81,7 @@ observations: covariance: stream: '2.0e7' rms ref: 183502589.5028424 + reference nobs: 800 tolerance: 1.0e-8 - obs error: covariance model: diagonal @@ -107,6 +108,7 @@ observations: covariance: uwind: '15.0' rms ref: 39.644266100943696 + reference nobs: 800 tolerance: 1.0e-8 - obs error: covariance model: diagonal @@ -127,6 +129,7 @@ observations: tolerance AD: 1.0e-10 tolerance TL: 1.0e-6 rms ref: 58.474969231121605 + reference nobs: 400 tolerance: 1.0e-8 background: date: 2009-12-31T00:00:00Z diff --git a/src/oops/interface/ObsVector.h b/src/oops/interface/ObsVector.h index dabe8be94..e9daf010f 100644 --- a/src/oops/interface/ObsVector.h +++ b/src/oops/interface/ObsVector.h @@ -64,22 +64,25 @@ class ObsVector : public util::Printable, ObsVector & operator*= (const ObsVector &); ObsVector & operator/= (const ObsVector &); -/// Pack into an Eigen vector (excluding vector elements that are masked out) + /// Pack into an Eigen vector (excluding vector elements that are masked out) Eigen::VectorXd packEigen() const; void zero(); + /// Set this ObsVector to ones (used in tests) + void ones(); void axpy(const double &, const ObsVector &); void invert(); void random(); double dot_product_with(const ObsVector &) const; double rms() const; -/// Mask out elements of the vector where the passed in flags are > 0 + /// Mask out elements of the vector where the passed in flags are > 0 void mask(const ObsDataVector &); // I/O void save(const std::string &) const; void read(const std::string &); + /// number of non-masked out observations unsigned int nobs() const; private: @@ -209,6 +212,16 @@ void ObsVector::zero() { } // ----------------------------------------------------------------------------- template +void ObsVector::ones() { + Log::trace() << "ObsVector::ones starting" << std::endl; + util::Timer timer(classname(), "ones"); + + data_->ones(); + + Log::trace() << "ObsVector::ones done" << std::endl; +} +// ----------------------------------------------------------------------------- +template void ObsVector::axpy(const double & zz, const ObsVector & rhs) { Log::trace() << "ObsVector::axpy starting" << std::endl; util::Timer timer(classname(), "axpy"); diff --git a/src/test/interface/ObsVector.h b/src/test/interface/ObsVector.h index ca2f250d4..2b7789139 100644 --- a/src/test/interface/ObsVector.h +++ b/src/test/interface/ObsVector.h @@ -19,6 +19,7 @@ #include "eckit/testing/Test.h" #include "oops/base/Variables.h" +#include "oops/interface/ObsDataVector.h" #include "oops/interface/ObsVector.h" #include "oops/runs/Test.h" #include "oops/util/dot_product.h" @@ -70,7 +71,11 @@ template void testCopyConstructor() { } // ----------------------------------------------------------------------------- - +/// Test that: +/// - dot_product of a random vector with self is non-zero +/// - dot_product of a random vector with a zero vector is zero +/// - dot_product of a zero vector with self is zero +/// - dot_product of a vector of ones with self is equal to nobs template void testNotZero() { typedef ObsTestsFixture Test_; typedef oops::ObsVector ObsVector_; @@ -88,6 +93,11 @@ template void testNotZero() { EXPECT(dot_product(ov2, ov1) == zero); EXPECT(dot_product(ov2, ov2) == zero); + + ObsVector_ ov3(ov1); + ov3.ones(); + + EXPECT(dot_product(ov3, ov3) == ov3.nobs()); } } // ----------------------------------------------------------------------------- @@ -152,6 +162,135 @@ template void testReadWrite() { } } // ----------------------------------------------------------------------------- +/// \brief Tests ObsVector::mask method. +/// \details Tests that: +/// - mask of all zeros (nothing to mask) applied to ObsVector doesn't change +/// its size and content; +/// - mask of either all ones (if "mask variable" isn't specified in yaml), or +/// from the file applied to ObsVector changes its size. +/// - linear algebra operations with ObsVector that were masked out produce +/// ObsVectors that have the same number of obs masked out. +template void testMask() { + typedef ObsTestsFixture Test_; + typedef oops::ObsDataVector ObsDataVector_; + typedef oops::ObsSpace ObsSpace_; + typedef oops::ObsVector ObsVector_; + + const double tolerance = 1.0e-8; + for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + const ObsSpace_ & obspace = Test_::obspace()[jj]; + + ObsVector_ reference(obspace); + reference.random(); + oops::Log::test() << "ObsVector before masking: " << reference << std::endl; + + const size_t nobs_all = Test_::config(jj).getInt("reference nobs"); + EXPECT_EQUAL(reference.nobs(), nobs_all); + EXPECT(nobs_all > 0); + + /// apply empty mask, check that vector is the same + ObsDataVector_ unsetmask(obspace, obspace.obsvariables()); + unsetmask.zero(); + ObsVector_ with_unsetmask(reference); + with_unsetmask.mask(unsetmask); + oops::Log::test() << "ObsVector masked with all-zero-mask: " << with_unsetmask << std::endl; + EXPECT_EQUAL(with_unsetmask.nobs(), nobs_all); + with_unsetmask -= reference; + EXPECT_EQUAL(with_unsetmask.rms(), 0.0); + + /// apply non-empty mask, check number of observations + std::string maskvarname; + size_t nobs_after_mask; + /// if mask variable is available, apply mask from file and read reference number of masked obs + if (Test_::config(jj).has("mask variable")) { + maskvarname = Test_::config(jj).getString("mask variable"); + nobs_after_mask = Test_::config(jj).getInt("reference masked nobs"); + EXPECT_NOT_EQUAL(nobs_after_mask, nobs_all); + /// if mask variable is unavailable, apply mask with all ones + } else { + // Hack for mask with ones: use ObsVector set to ones, write to ObsSpace, + // then read as ObsDataVector. + maskvarname = "set_mask"; + nobs_after_mask = 0; + ObsVector_ tmp(obspace); + tmp.ones(); + tmp.save(maskvarname); + } + ObsDataVector_ mask(obspace, obspace.obsvariables(), maskvarname); + ObsVector_ with_mask(reference); + with_mask.mask(mask); + oops::Log::test() << "ObsVector masked with " << maskvarname << " mask: " << + with_mask << std::endl; + EXPECT_EQUAL(with_mask.nobs(), nobs_after_mask); + + /// Test that various linear algebra operations with masked out ObsVector + /// produce masked out ObsVector + /// Test invert() + ObsVector_ test(with_mask); + test.invert(); + EXPECT_EQUAL(test.nobs(), nobs_after_mask); + + /// Test *=(float) + test *= 2.0; + EXPECT_EQUAL(test.nobs(), nobs_after_mask); + + /// Test +=(ObsVector) + test.random(); + EXPECT_EQUAL(test.nobs(), nobs_all); + test += with_mask; + EXPECT_EQUAL(test.nobs(), nobs_after_mask); + test.random(); + EXPECT_EQUAL(test.nobs(), nobs_all); + with_mask += test; + EXPECT_EQUAL(with_mask.nobs(), nobs_after_mask); + + /// Test -=(ObsVector) + EXPECT_EQUAL(test.nobs(), nobs_all); + test -= with_mask; + EXPECT_EQUAL(test.nobs(), nobs_after_mask); + test.random(); + EXPECT_EQUAL(test.nobs(), nobs_all); + with_mask -= test; + EXPECT_EQUAL(with_mask.nobs(), nobs_after_mask); + + /// Test *=(ObsVector) + EXPECT_EQUAL(test.nobs(), nobs_all); + test *= with_mask; + EXPECT_EQUAL(test.nobs(), nobs_after_mask); + test.random(); + EXPECT_EQUAL(test.nobs(), nobs_all); + with_mask *= test; + EXPECT_EQUAL(with_mask.nobs(), nobs_after_mask); + + /// Test /=(ObsVector) + EXPECT_EQUAL(test.nobs(), nobs_all); + test /= with_mask; + EXPECT_EQUAL(test.nobs(), nobs_after_mask); + test.random(); + EXPECT_EQUAL(test.nobs(), nobs_all); + with_mask /= test; + EXPECT_EQUAL(with_mask.nobs(), nobs_after_mask); + + /// Test axpy + EXPECT_EQUAL(test.nobs(), nobs_all); + test.axpy(2.0, with_mask); + EXPECT_EQUAL(test.nobs(), nobs_after_mask); + test.random(); + EXPECT_EQUAL(test.nobs(), nobs_all); + with_mask.axpy(2.0, test); + EXPECT_EQUAL(with_mask.nobs(), nobs_after_mask); + + /// test packEigen + Eigen::VectorXd with_mask_vec = with_mask.packEigen(); + EXPECT(with_mask_vec.size() == with_mask.nobs()); + if (with_mask.nobs() > 0) { + double rms1 = with_mask.rms(); + double rms2 = sqrt(with_mask_vec.squaredNorm() / with_mask_vec.size()); + EXPECT(std::abs(rms1-rms2) < tolerance); + } + } +} +// ----------------------------------------------------------------------------- template void testPackEigen() { typedef ObsTestsFixture Test_; typedef oops::ObsVector ObsVector_; @@ -194,6 +333,8 @@ class ObsVector : public oops::Test { { testLinearAlgebra(); }); ts.emplace_back(CASE("interface/ObsVector/testReadWrite") { testReadWrite(); }); + ts.emplace_back(CASE("interface/ObsVector/testMask") + { testMask(); }); ts.emplace_back(CASE("interface/ObsVector/testPackEigen") { testPackEigen(); }); } From 35c88880dc033fae97c557fad40d328111f73361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Mon, 29 Mar 2021 16:48:42 -0600 Subject: [PATCH 089/142] Fix number of MPI tasks for EDA L95 test (#1121) Co-authored-by: Anna Shlyaeva --- l95/src/lorenz95/Resolution.h | 6 ++++-- l95/test/CMakeLists.txt | 2 +- qg/model/GeometryQG.cc | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/l95/src/lorenz95/Resolution.h b/l95/src/lorenz95/Resolution.h index 7cfe2fc82..a4f51289d 100644 --- a/l95/src/lorenz95/Resolution.h +++ b/l95/src/lorenz95/Resolution.h @@ -44,8 +44,10 @@ class Resolution : public util::Printable { typedef ResolutionParameters Parameters_; Resolution(const ResolutionParameters & parameters, const eckit::mpi::Comm & comm) - : resol_(parameters.resol), comm_(comm) {} - explicit Resolution(const int resol): resol_(resol), comm_(oops::mpi::myself()) {} + : resol_(parameters.resol), comm_(comm) + {ASSERT(comm_.size() == 1);} + explicit Resolution(const int resol): resol_(resol), comm_(oops::mpi::myself()) + {ASSERT(comm_.size() == 1);} int npoints() const {return resol_;} diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index a31a21028..429cbbcb1 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -699,7 +699,7 @@ ecbuild_add_test( TARGET test_l95_eda_4dvar TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) oops_add_test( TESTNAME eda_3dvar_block - MPI 4 + MPI 2 MODELNAME l95 YAMLNAME testinput/eda.3dvar.block.yaml EXENAME l95_eda.x diff --git a/qg/model/GeometryQG.cc b/qg/model/GeometryQG.cc index 3ca705afd..27b8ca544 100644 --- a/qg/model/GeometryQG.cc +++ b/qg/model/GeometryQG.cc @@ -26,6 +26,7 @@ namespace qg { // ----------------------------------------------------------------------------- GeometryQG::GeometryQG(const GeometryQgParameters & params, const eckit::mpi::Comm & comm) : comm_(comm) { + ASSERT(comm_.size() == 1); qg_geom_setup_f90(keyGeom_, params.toConfiguration()); // Set ATLAS lon/lat field @@ -45,6 +46,7 @@ GeometryQG::GeometryQG(const GeometryQgParameters & params, } // ----------------------------------------------------------------------------- GeometryQG::GeometryQG(const GeometryQG & other) : comm_(other.comm_) { + ASSERT(comm_.size() == 1); qg_geom_clone_f90(keyGeom_, other.keyGeom_); // Copy ATLAS function space From b432b814f438e04f78db6332ff2388d53d98d205 Mon Sep 17 00:00:00 2001 From: Carwyn Pelley Date: Wed, 31 Mar 2021 16:51:25 +0100 Subject: [PATCH 090/142] BUG: Fix Partial datetime missing component handling (#1110) * Working state * TEST: Added a formally missing component now valid component * ENH: PartialDateTime parameter trait * TEST: Tested the new PartialDateTime parameter trait. * MAINT: Review changes * MAINT: Exception message change from review Co-authored-by: Anna Shlyaeva --- src/oops/util/PartialDateTime.cc | 87 +++++++++++++++++++--- src/oops/util/PartialDateTime.h | 24 ++++-- src/oops/util/parameters/ParameterTraits.h | 28 +++++++ src/oops/util/parameters/Parameters.cc | 12 ++- src/test/testinput/parameters.yaml | 5 ++ src/test/util/Parameters.h | 21 ++++++ src/test/util/PartialDateTime.cc | 86 ++++++++++++++------- 7 files changed, 222 insertions(+), 41 deletions(-) diff --git a/src/oops/util/PartialDateTime.cc b/src/oops/util/PartialDateTime.cc index d510e90d9..b0fbdbfe0 100644 --- a/src/oops/util/PartialDateTime.cc +++ b/src/oops/util/PartialDateTime.cc @@ -4,9 +4,12 @@ * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ +#include "oops/util/PartialDateTime.h" + +#include +#include "eckit/exception/Exceptions.h" #include "oops/util/dateFunctions.h" -#include "oops/util/PartialDateTime.h" namespace util { @@ -16,19 +19,55 @@ namespace util { PartialDateTime::PartialDateTime(int year, int month, int day, int hour, int minute, int second) : partialdt_({year, month, day, hour, minute, second}) { - populatedvalues_ = {year != unset_, month != unset_, day != unset_, - hour != unset_, minute != unset_, second != unset_}; + + datetime_string_ = padNumber(year, 4); + datetime_string_ += "-" + padNumber(month, 2); + datetime_string_ += "-" + padNumber(day, 2); + datetime_string_ += "T" + padNumber(hour, 2); + datetime_string_ += ":" + padNumber(minute, 2); + datetime_string_ += ":" + padNumber(second, 2) + "Z"; } // ----------------------------------------------------------------------------- PartialDateTime::PartialDateTime(std::string const &datetime_string) { int year, month, day, hour, minute, second; - util::datefunctions::stringToYYYYMMDDhhmmss(datetime_string, year, month, day, + datetime_string_ = datetime_string; + + // Derive an ordinary datetime string + std::string alt_datetime_string = datetime_string; + + // Determine if stringUnset_ is present and for which component(s). + if (datetime_string.size() != 20) + throw eckit::BadParameter("Partial date-time string '" + datetime_string + + "' is of unexpected length"); + + std::array populatedvalues = {true, true, true, true, true, true}; + bool isSet; + for (size_t ind=0; ind < datetime_string.size(); ind++) { + isSet = datetime_string[ind] != stringUnset_; + if (!isSet) alt_datetime_string[ind] = '0'; // Replace * with 0 + if (ind <= 3) { + populatedvalues[0] &= isSet; + } else if (ind >= 5 && ind <= 6) { + populatedvalues[1] &= isSet; + } else if (ind >= 8 && ind <= 9) { + populatedvalues[2] &= isSet; + } else if (ind >= 11 && ind <= 12) { + populatedvalues[3] &= isSet; + } else if (ind >= 14 && ind <= 15) { + populatedvalues[4] &= isSet; + } else if (ind >= 17 && ind <= 18) { + populatedvalues[5] &= isSet; + } + } + util::datefunctions::stringToYYYYMMDDhhmmss(alt_datetime_string, year, month, day, hour, minute, second); + // Replace with unset values partialdt_ = {year, month, day, hour, minute, second}; - populatedvalues_ = {year != unset_, month != unset_, day != unset_, - hour != unset_, minute != unset_, second != unset_}; + for (size_t ind=0; ind < partialdt_.size(); ind++) { + if (!populatedvalues[ind]) partialdt_[ind] = intUnset_; + } } @@ -56,13 +95,43 @@ int PartialDateTime::minute() const {return partialdt_[4];} int PartialDateTime::second() const {return partialdt_[5];} +// ----------------------------------------------------------------------------- +std::string PartialDateTime::padNumber(const int num, const int width) const { + std::ostringstream ss; + if (num == intUnset_) { + ss << std::setw(width) << std::setfill(stringUnset_) << stringUnset_; + } else { + ss << std::setw(width) << std::setfill('0') << num; + } + return ss.str(); +} + + +// ----------------------------------------------------------------------------- +bool PartialDateTime::operator==(const PartialDateTime &other) const { + if (this->year() != other.year()) return false; + if (this->month() != other.month()) return false; + if (this->day() != other.day()) return false; + if (this->hour() != other.hour()) return false; + if (this->minute() != other.minute()) return false; + if (this->second() != other.second()) return false; + return true; +} + + +// ----------------------------------------------------------------------------- +bool PartialDateTime::operator!=(const PartialDateTime &other) const { + return !(*this == other); +} + + // ----------------------------------------------------------------------------- bool PartialDateTime::operator==(const DateTime &other) const { int year, month, day, hour, minute, second; other.toYYYYMMDDhhmmss(year, month, day, hour, minute, second); std::array cmpdt {year, month, day, hour, minute, second}; for (size_t i=0; i < partialdt_.size(); i++) { - if ((populatedvalues_[i]) && (partialdt_[i] != cmpdt[i])) { + if ((partialdt_[i] != intUnset_) && (partialdt_[i] != cmpdt[i])) { return false; } } @@ -76,7 +145,7 @@ bool PartialDateTime::operator<(const DateTime &other) const { other.toYYYYMMDDhhmmss(year, month, day, hour, minute, second); std::array cmpdt {year, month, day, hour, minute, second}; for (size_t i=0; i < partialdt_.size(); i++) { - if ((populatedvalues_[i]) && (partialdt_[i] != cmpdt[i])) { + if ((partialdt_[i] != intUnset_) && (partialdt_[i] != cmpdt[i])) { return (partialdt_[i] < cmpdt[i]); } } @@ -90,7 +159,7 @@ bool PartialDateTime::operator>(const DateTime &other) const { other.toYYYYMMDDhhmmss(year, month, day, hour, minute, second); std::array cmpdt {year, month, day, hour, minute, second}; for (size_t i=0; i < partialdt_.size(); i++) { - if ((populatedvalues_[i]) && (partialdt_[i] != cmpdt[i])) { + if ((partialdt_[i] != intUnset_) && (partialdt_[i] != cmpdt[i])) { return (partialdt_[i] > cmpdt[i]); } } diff --git a/src/oops/util/PartialDateTime.h b/src/oops/util/PartialDateTime.h index df8265869..1925b221b 100644 --- a/src/oops/util/PartialDateTime.h +++ b/src/oops/util/PartialDateTime.h @@ -24,19 +24,27 @@ namespace util { class PartialDateTime { private: - static int const unset_ = 0; + std::string datetime_string_; + static char const stringUnset_ = '*'; + static int const intUnset_ = -1; std::array partialdt_; - std::array populatedvalues_; public: - // sets the date given YYYY,MM,DD,hh,mm,ss - PartialDateTime(int year = unset_, int month = unset_, int day = unset_, - int hour = unset_, int minute = unset_, int second = unset_); + // \brief Sets the date given YYYY,MM,DD,hh,mm,ss + PartialDateTime(int year = intUnset_, int month = intUnset_, int day = intUnset_, + int hour = intUnset_, int minute = intUnset_, int second = intUnset_); - // sets the date given a string + // \brief Sets the date given a string + // \brief String must represent an extended ISO 8601 format, where an asterisk is interpreted + // as those components which are to be ignored by the comparison operators. explicit PartialDateTime(std::string const &datetime_string); + // \brief Generate extended ISO 8601 string representation for this PartialDateTime + std::string toString() const {return datetime_string_;} + // Comparison operators + bool operator==(const PartialDateTime &) const; + bool operator!=(const PartialDateTime &) const; bool operator==(const util::DateTime &) const; bool operator!=(const util::DateTime &) const; bool operator<(const util::DateTime &) const; @@ -56,6 +64,10 @@ class PartialDateTime { int hour() const; int minute() const; int second() const; + + private: + // \brief Used for creating a PartialDateTime string + std::string padNumber(const int num, const int width) const; }; diff --git a/src/oops/util/parameters/ParameterTraits.h b/src/oops/util/parameters/ParameterTraits.h index aaf283a6d..7e3787359 100644 --- a/src/oops/util/parameters/ParameterTraits.h +++ b/src/oops/util/parameters/ParameterTraits.h @@ -30,6 +30,7 @@ #include "oops/util/NamedEnumerator.h" #include "oops/util/parameters/ObjectJsonSchema.h" #include "oops/util/parameters/Parameters.h" +#include "oops/util/PartialDateTime.h" #include "oops/util/stringFunctions.h" // for join() namespace oops { @@ -304,6 +305,33 @@ struct ParameterTraits { } }; + +/// \brief Specialization for PartialDateTime objects. +template <> +struct ParameterTraits { + static boost::optional get(util::CompositePath &path, + const eckit::Configuration &config, + const std::string& name) { + std::string value; + if (config.get(name, value)) { + return util::PartialDateTime(value); + } else { + return boost::none; + } + } + + static void set(eckit::LocalConfiguration &config, + const std::string &name, + const util::PartialDateTime &value) { + config.set(name, value.toString()); + } + + static ObjectJsonSchema jsonSchema(const std::string &name) { + return ObjectJsonSchema({{name, {{"type", "\"string\""}, + {"format", "\"partial-date-time\""}}}}); + } +}; + /// \brief Specialization for vectors. template struct ParameterTraits, std::false_type> { diff --git a/src/oops/util/parameters/Parameters.cc b/src/oops/util/parameters/Parameters.cc index 046ce36c7..f095e6578 100644 --- a/src/oops/util/parameters/Parameters.cc +++ b/src/oops/util/parameters/Parameters.cc @@ -93,11 +93,21 @@ void checkStringFormat(const std::string &format, const std::string &value) { // PT1H30M (a duration of 1 h and 30 min), P1YT1S (a duration of 1 year and 1 s) static const std::regex regex( R"(^P(?!$)(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T(?=\d+[HMS])(\d+H)?(\d+M)?(\d+S)?)?$)"); - std::smatch matches; if (!std::regex_match(value, matches, regex)) { throw std::invalid_argument(value + " is not a duration string."); } + + } else if (format == "partial-date-time") { + // Matches ISO 8601 representations of datetimes but additionally supporting the asterix + // character to denote partial definition. + static const std::regex regex( + "^[0-9*]{4}-[0-9*]{2}-[0-9*]{2}T[0-9*]{2}:[0-9*]{2}:[0-9*]{2}Z$"); + std::smatch matches; + if (!std::regex_match(value, matches, regex)) { + throw std::invalid_argument(value + " is not a partial-date-time string."); + } + } else { nlohmann::json_schema::default_string_format_check(format, value); } diff --git a/src/test/testinput/parameters.yaml b/src/test/testinput/parameters.yaml index 32ff0332b..c9fb2e22d 100644 --- a/src/test/testinput/parameters.yaml +++ b/src/test/testinput/parameters.yaml @@ -16,6 +16,7 @@ full: # These parameters must be sorted alphabetically for the serialization tes opt_embedded_date_time_parameter: 2010-03-04T05:06:07Z opt_float_parameter: 5.5 opt_null_parameter: null + opt_partialDT_parameter: "2010-**-03T04:05:06Z" range_parameter: max: 8.5 min: 7 @@ -97,6 +98,10 @@ error_in_opt_duration_parameter: opt_duration_parameter: ABCDEF req_float_parameter: 3 req_duration_parameter: PT1H +error_in_opt_partialDT_parameter: + opt_partialDT_parameter: ABCDEF + req_float_parameter: 3 + req_duration_parameter: PT1H error_in_fruit_parameter: fruit_parameter: ABCDEF req_float_parameter: 3 diff --git a/src/test/util/Parameters.h b/src/test/util/Parameters.h index 9be57c00d..3ec53380d 100644 --- a/src/test/util/Parameters.h +++ b/src/test/util/Parameters.h @@ -103,6 +103,8 @@ class MyParametersBase : public oops::Parameters { oops::OptionalParameter optFloatParameter{"opt_float_parameter", this}; oops::OptionalParameter optDateTimeParameter{"opt_date_time_parameter", this}; oops::OptionalParameter optDurationParameter{"opt_duration_parameter", this}; + oops::OptionalParameter optPartialDateTimeParameter{ + "opt_partialDT_parameter", this}; oops::Parameter fruitParameter{"fruit_parameter", Fruit::ORANGE, this}; oops::Parameter rangeParameter{"range_parameter", RangeParameters(), this}; oops::Parameter> intParameters{"int_parameters", {}, this}; @@ -430,6 +432,7 @@ void testDefaultValues() { EXPECT(params.optFloatParameter.value() == boost::none); EXPECT(params.optDateTimeParameter.value() == boost::none); EXPECT(params.optDurationParameter.value() == boost::none); + EXPECT(params.optPartialDateTimeParameter.value() == boost::none); EXPECT_THROWS_AS(params.reqFloatParameter.value(), boost::bad_optional_access); EXPECT_THROWS_AS(params.reqDurationParameter.value(), boost::bad_optional_access); EXPECT(params.fruitParameter == Fruit::ORANGE); @@ -456,6 +459,7 @@ void testDefaultValues() { EXPECT(params.optFloatParameter.value() == boost::none); EXPECT(params.optDateTimeParameter.value() == boost::none); EXPECT(params.optDurationParameter.value() == boost::none); + EXPECT(params.optPartialDateTimeParameter.value() == boost::none); EXPECT_EQUAL(params.reqFloatParameter, 3.0f); EXPECT_EQUAL(params.reqFloatParameter.value(), 3.0f); EXPECT_EQUAL(params.reqDurationParameter.value(), util::Duration("PT1H")); @@ -489,6 +493,9 @@ void testCorrectValues() { EXPECT_EQUAL(params.optDateTimeParameter.value().get(), util::DateTime(2010, 2, 3, 4, 5, 6)); EXPECT(params.optDurationParameter.value() != boost::none); EXPECT_EQUAL(params.optDurationParameter.value().get(), util::Duration("PT01H02M03S")); + EXPECT(params.optPartialDateTimeParameter.value() != boost::none); + EXPECT(params.optPartialDateTimeParameter.value().get() == + util::PartialDateTime(2010, -1, 3, 4, 5, 6)); EXPECT_EQUAL(params.reqFloatParameter, 6.0f); EXPECT_EQUAL(params.reqFloatParameter.value(), 6.0f); EXPECT_EQUAL(params.reqDurationParameter.value(), util::Duration("PT06H30M")); @@ -619,6 +626,17 @@ void testIncorrectValueOfOptionalFloatParameter() { EXPECT_THROWS_AS(params.deserialize(conf), eckit::BadParameter); } + +void testIncorrectValueOfOptionalPartialDateTimeParameter() { + MyOptionalAndRequiredParameters params; + const eckit::LocalConfiguration conf(TestEnvironment::config(), + "error_in_opt_partialDT_parameter"); + if (validationSupported) + EXPECT_THROWS_MSG(params.validate(conf), "ABCDEF is not a partial-date-time string"); + EXPECT_THROWS_AS(params.deserialize(conf), eckit::BadParameter); +} + + void testIncorrectValueOfOptionalDateTimeParameter() { MyOptionalAndRequiredParameters params; const eckit::LocalConfiguration conf(TestEnvironment::config(), @@ -1561,6 +1579,9 @@ class Parameters : public oops::Test { ts.emplace_back(CASE("util/Parameters/incorrectValueOfOptionalFloatParameter") { testIncorrectValueOfOptionalFloatParameter(); }); + ts.emplace_back(CASE("util/Parameters/incorrectValueOfOptionalPartialDateTimeParameter") { + testIncorrectValueOfOptionalPartialDateTimeParameter(); + }); ts.emplace_back(CASE("util/Parameters/incorrectValueOfOptionalDateTimeParameter") { testIncorrectValueOfOptionalDateTimeParameter(); }); diff --git a/src/test/util/PartialDateTime.cc b/src/test/util/PartialDateTime.cc index f4765e321..78bf811a3 100644 --- a/src/test/util/PartialDateTime.cc +++ b/src/test/util/PartialDateTime.cc @@ -12,6 +12,22 @@ namespace { + int notset = -1; + + CASE("test_toString_basic") { + util::PartialDateTime pdt(2011, 9, 16, 13, 55, 20); + std::string res = pdt.toString(); + std::string tar = "2011-09-16T13:55:20Z"; + EXPECT_EQUAL(res, tar); + } + + CASE("test_toString_with_missing") { + util::PartialDateTime pdt(2011, 9, 16, 13, -1, 20); + std::string res = pdt.toString(); + std::string tar = "2011-09-16T13:**:20Z"; + EXPECT_EQUAL(res, tar); + } + CASE("test_construction_int_arg") { util::PartialDateTime pdt(2011, 9, 16, 13, 55, 20); EXPECT(pdt.year() == 2011); @@ -25,7 +41,6 @@ namespace { CASE("test_construction_default") { util::PartialDateTime pdt{}; - int notset = 0; EXPECT(pdt.year() == notset); EXPECT(pdt.month() == notset); EXPECT(pdt.day() == notset); @@ -46,16 +61,37 @@ namespace { } + CASE("test_construction_string_arg_unset") { + // Check we properly handle the case where the string contains unset components + // Note that any bit of a component results in it being considered unset. + util::PartialDateTime pdt("2011-09-1*T13:55:00Z"); + EXPECT(pdt.year() == 2011); + EXPECT(pdt.month() == 9); + EXPECT(pdt.day() == notset); + EXPECT(pdt.hour() == 13); + EXPECT(pdt.minute() == 55); + EXPECT(pdt.second() == 0); + + pdt = util::PartialDateTime("2011-09-*6T13:55:00Z"); + EXPECT(pdt.year() == 2011); + EXPECT(pdt.month() == 9); + EXPECT(pdt.day() == notset); + EXPECT(pdt.hour() == 13); + EXPECT(pdt.minute() == 55); + EXPECT(pdt.second() == 0); + } + + CASE("test_gt_operator") { // We also perform an equivelent check with DateTime operator working with a // PartialDateTime. util::DateTime dt1(2011, 9, 16, 13, 55, 20); - util::PartialDateTime pdt1(0, 0, 0, 12); - util::PartialDateTime pdt2(0, 0, 0, 13); - util::PartialDateTime pdt3(0, 0, 0, 14); - util::PartialDateTime pdt4(2011, 0, 0, 14); - util::PartialDateTime pdt5(2010, 0, 0, 14); + util::PartialDateTime pdt1(notset, notset, notset, 12); + util::PartialDateTime pdt2(notset, notset, notset, 13); + util::PartialDateTime pdt3(notset, notset, notset, 14); + util::PartialDateTime pdt4(2011, notset, notset, 14); + util::PartialDateTime pdt5(2010, notset, notset, 14); EXPECT(!(pdt1 > dt1)); EXPECT(!(dt1 < pdt1)); @@ -79,11 +115,11 @@ namespace { // PartialDateTime. util::DateTime dt1(2011, 9, 16, 13, 55, 20); - util::PartialDateTime pdt1(0, 0, 0, 12); - util::PartialDateTime pdt2(0, 0, 0, 13); - util::PartialDateTime pdt3(0, 0, 0, 14); - util::PartialDateTime pdt4(2011, 0, 0, 14); - util::PartialDateTime pdt5(2010, 0, 0, 14); + util::PartialDateTime pdt1(notset, notset, notset, 12); + util::PartialDateTime pdt2(notset, notset, notset, 13); + util::PartialDateTime pdt3(notset, notset, notset, 14); + util::PartialDateTime pdt4(2011, notset, notset, 14); + util::PartialDateTime pdt5(2010, notset, notset, 14); EXPECT(!(pdt1 >= dt1)); EXPECT(!(dt1 <= pdt1)); @@ -107,11 +143,11 @@ namespace { // PartialDateTime. util::DateTime dt1(2011, 9, 16, 13, 55, 20); - util::PartialDateTime pdt1(0, 0, 0, 12); - util::PartialDateTime pdt2(0, 0, 0, 13); - util::PartialDateTime pdt3(0, 0, 0, 14); - util::PartialDateTime pdt4(2011, 0, 0, 14); - util::PartialDateTime pdt5(2010, 0, 0, 14); + util::PartialDateTime pdt1(notset, notset, notset, 12); + util::PartialDateTime pdt2(notset, notset, notset, 13); + util::PartialDateTime pdt3(notset, notset, notset, 14); + util::PartialDateTime pdt4(2011, notset, notset, 14); + util::PartialDateTime pdt5(2010, notset, notset, 14); EXPECT((pdt1 < dt1)); EXPECT((dt1 > pdt1)); @@ -135,11 +171,11 @@ namespace { // PartialDateTime. util::DateTime dt1(2011, 9, 16, 13, 55, 20); - util::PartialDateTime pdt1(0, 0, 0, 12); - util::PartialDateTime pdt2(0, 0, 0, 13); - util::PartialDateTime pdt3(0, 0, 0, 14); - util::PartialDateTime pdt4(2011, 0, 0, 14); - util::PartialDateTime pdt5(2010, 0, 0, 14); + util::PartialDateTime pdt1(notset, notset, notset, 12); + util::PartialDateTime pdt2(notset, notset, notset, 13); + util::PartialDateTime pdt3(notset, notset, notset, 14); + util::PartialDateTime pdt4(2011, notset, notset, 14); + util::PartialDateTime pdt5(2010, notset, notset, 14); EXPECT((pdt1 <= dt1)); EXPECT((dt1 >= pdt1)); @@ -163,8 +199,8 @@ namespace { // PartialDateTime. util::DateTime dt1(2011, 9, 16, 13, 55, 20); - util::PartialDateTime pdt1(0, 0, 0, 13); - util::PartialDateTime pdt2(0, 0, 0, 14); + util::PartialDateTime pdt1(notset, notset, notset, 13); + util::PartialDateTime pdt2(notset, notset, notset, 14); EXPECT(pdt1 == dt1); EXPECT(dt1 == pdt1); @@ -179,8 +215,8 @@ namespace { // PartialDateTime. util::DateTime dt1(2011, 9, 16, 13, 55, 20); - util::PartialDateTime pdt1(0, 0, 0, 13); - util::PartialDateTime pdt2(0, 0, 0, 14); + util::PartialDateTime pdt1(notset, notset, notset, 13); + util::PartialDateTime pdt2(notset, notset, notset, 14); EXPECT(!(pdt1 != dt1)); EXPECT(!(dt1 != pdt1)); From df0ec7746595c171da766301f64b791f28004e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Wed, 31 Mar 2021 10:59:21 -0600 Subject: [PATCH 091/142] Feature/obsvecs (#1119) * ObsVector <-> ObsDataVector conversions * Pass ObsVector to ObsFilters * Added ObsDataVector test * Pass obs errors by reference Co-authored-by: Anna Shlyaeva --- l95/src/lorenz95/ObsData1D.h | 42 ++++++-- l95/src/lorenz95/ObsVec1D.cc | 13 +++ l95/src/lorenz95/ObsVec1D.h | 6 +- l95/test/CMakeLists.txt | 6 ++ l95/test/executables/TestObsDataVector.cc | 17 +++ qg/model/ObsDataQG.h | 6 ++ qg/model/ObsVecQG.cc | 5 + qg/model/ObsVecQG.h | 5 +- qg/test/CMakeLists.txt | 6 ++ qg/test/executables/TestObsDataVector.cc | 17 +++ src/CMakeLists.txt | 2 + src/oops/assimilation/CalcHofX.h | 10 +- src/oops/assimilation/CostJo.h | 9 +- src/oops/assimilation/LocalEnsembleSolver.h | 2 +- src/oops/base/ObsFilters.h | 27 +++-- src/oops/base/Observers.h | 8 +- src/oops/interface/ObsDataVector.h | 49 +++------ src/oops/interface/ObsDataVector_head.h | 66 ++++++++++++ src/oops/interface/ObsVector.h | 12 ++- src/oops/runs/HofX4D.h | 7 +- src/test/interface/ObsDataVector.h | 109 ++++++++++++++++++++ 21 files changed, 345 insertions(+), 79 deletions(-) create mode 100644 l95/test/executables/TestObsDataVector.cc create mode 100644 qg/test/executables/TestObsDataVector.cc create mode 100644 src/oops/interface/ObsDataVector_head.h create mode 100644 src/test/interface/ObsDataVector.h diff --git a/l95/src/lorenz95/ObsData1D.h b/l95/src/lorenz95/ObsData1D.h index be269825e..69aeb7bc1 100644 --- a/l95/src/lorenz95/ObsData1D.h +++ b/l95/src/lorenz95/ObsData1D.h @@ -9,6 +9,7 @@ #define LORENZ95_OBSDATA1D_H_ #include +#include #include #include #include @@ -22,6 +23,7 @@ #include "oops/util/Printable.h" #include "lorenz95/ObsTableView.h" +#include "lorenz95/ObsVec1D.h" namespace lorenz95 { @@ -36,6 +38,7 @@ class ObsData1D : public util::Printable, ObsData1D(const ObsTableView &, const oops::Variables &, const std::string &); ObsData1D(const ObsData1D &); + explicit ObsData1D(const ObsVec1D &); ~ObsData1D() {} ObsData1D & operator= (const ObsData1D &); @@ -75,6 +78,20 @@ ObsData1D::ObsData1D(const ObsData1D & other) {} // ----------------------------------------------------------------------------- template +ObsData1D::ObsData1D(const ObsVec1D & other) + : obsdb_(other.obsdb()), data_(other.size()) { + const DATATYPE missing = util::missingValue(missing); + const double dmiss = util::missingValue(dmiss); + for (size_t jj = 0; jj < data_.size(); ++jj) { + if (other[jj] == dmiss) { + data_.at(jj) = missing; + } else { + data_.at(jj) = static_cast(other[jj]); + } + } +} +// ----------------------------------------------------------------------------- +template ObsData1D & ObsData1D::operator= (const ObsData1D & rhs) { ASSERT(data_.size() == rhs.data_.size()); data_ = rhs.data_; @@ -108,16 +125,25 @@ void ObsData1D::save(const std::string & name) const { // ----------------------------------------------------------------------------- template void ObsData1D::print(std::ostream & os) const { - if (data_.size() > 0) { - DATATYPE zmin = data_.at(0); - DATATYPE zmax = data_.at(0); - for (size_t jj = 0; jj < data_.size(); ++jj) { - if (data_.at(jj) < zmin) zmin = data_.at(jj); - if (data_.at(jj) > zmax) zmax = data_.at(jj); + DATATYPE missing = util::missingValue(missing); + DATATYPE zmin = std::numeric_limits::max(); + DATATYPE zmax = std::numeric_limits::lowest(); + DATATYPE zavg = 0.0; + size_t iobs = 0; + for (const DATATYPE & val : data_) { + if (val != missing) { + if (val < zmin) zmin = val; + if (val > zmax) zmax = val; + zavg += val; + ++iobs; } - os << "Lorenz 95 nobs= " << data_.size() << " Min=" << zmin << ", Max=" << zmax; + } + if (iobs > 0) { + zavg /= static_cast(iobs); + os << "Lorenz 95 nobs= " << iobs << " Min=" << zmin << ", Max=" << zmax + << ", Average=" << zavg; } else { - os << "Lorenz 95 nobs= " << data_.size() << " --- No observations"; + os << "Lorenz 95 : No observations"; } } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObsVec1D.cc b/l95/src/lorenz95/ObsVec1D.cc index c5b7c4b65..e63037263 100644 --- a/l95/src/lorenz95/ObsVec1D.cc +++ b/l95/src/lorenz95/ObsVec1D.cc @@ -16,6 +16,7 @@ #include "eckit/config/Configuration.h" #include "eckit/exception/Exceptions.h" +#include "lorenz95/ObsData1D.h" #include "lorenz95/ObsTableView.h" #include "oops/util/Logger.h" #include "oops/util/missingValues.h" @@ -168,6 +169,18 @@ void ObsVec1D::mask(const ObsData1D & mask) { } } // ----------------------------------------------------------------------------- +ObsVec1D & ObsVec1D::operator=(const ObsData1D & rhs) { + const float fmiss = util::missingValue(fmiss); + for (size_t jj = 0; jj < data_.size(); ++jj) { + if (rhs[jj] == fmiss) { + data_.at(jj) = missing_; + } else { + data_.at(jj) = static_cast(rhs[jj]); + } + } + return *this; +} +// ----------------------------------------------------------------------------- void ObsVec1D::save(const std::string & name) const { obsdb_.putdb(name, data_); } diff --git a/l95/src/lorenz95/ObsVec1D.h b/l95/src/lorenz95/ObsVec1D.h index 804f8a6c0..339406f1e 100644 --- a/l95/src/lorenz95/ObsVec1D.h +++ b/l95/src/lorenz95/ObsVec1D.h @@ -19,10 +19,9 @@ #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" -#include "lorenz95/ObsData1D.h" - namespace lorenz95 { class ObsTableView; + template class ObsData1D; // ----------------------------------------------------------------------------- /// Vector in observation space @@ -48,6 +47,7 @@ class ObsVec1D : public util::Printable, ObsVec1D & operator/= (const ObsVec1D &); Eigen::VectorXd packEigen() const; + size_t size() const {return data_.size();} const double & operator[](const std::size_t ii) const {return data_.at(ii);} double & operator[](const std::size_t ii) {return data_.at(ii);} @@ -61,8 +61,10 @@ class ObsVec1D : public util::Printable, double dot_product_with(const ObsVec1D &) const; double rms() const; void mask(const ObsData1D &); + ObsVec1D & operator= (const ObsData1D &); unsigned int nobs() const; + const ObsTableView & obsdb() const {return obsdb_;} // I/O void save(const std::string &) const; diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index 429cbbcb1..67747bd9c 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -338,6 +338,12 @@ ecbuild_add_test( TARGET test_l95_obsvector LIBS lorenz95 TEST_DEPENDS test_l95_truth ) +ecbuild_add_test( TARGET test_l95_obsdatavector + SOURCES executables/TestObsDataVector.cc + ARGS "testinput/interfaces.yaml" + LIBS lorenz95 + TEST_DEPENDS test_l95_truth ) + ecbuild_add_test( TARGET test_l95_departures_ensemble SOURCES executables/TestDeparturesEnsemble.cc ARGS "testinput/interfaces.yaml" diff --git a/l95/test/executables/TestObsDataVector.cc b/l95/test/executables/TestObsDataVector.cc new file mode 100644 index 000000000..5a471af88 --- /dev/null +++ b/l95/test/executables/TestObsDataVector.cc @@ -0,0 +1,17 @@ +/* + * (C) Copyright 2021-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "lorenz95/L95Traits.h" +#include "oops/runs/Run.h" +#include "test/interface/ObsDataVector.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + test::ObsDataVector tests; + return run.execute(tests); +} + diff --git a/qg/model/ObsDataQG.h b/qg/model/ObsDataQG.h index b1893f351..7c3a40492 100644 --- a/qg/model/ObsDataQG.h +++ b/qg/model/ObsDataQG.h @@ -35,6 +35,7 @@ class ObsDataQG : public util::Printable, ObsDataQG(const ObsSpaceQG &, const oops::Variables &, const std::string &); ObsDataQG(const ObsDataQG &); + explicit ObsDataQG(const ObsVecQG &); ~ObsDataQG() {} ObsDataQG & operator= (const ObsDataQG &); @@ -47,6 +48,7 @@ class ObsDataQG : public util::Printable, void save(const std::string &) const; const int & toFortran() const {return data_.toFortran();} + const ObsVecQG & vect() const {return data_;} private: void print(std::ostream &) const; @@ -65,6 +67,10 @@ ObsDataQG::ObsDataQG(const ObsDataQG & other): data_(other.data_) { } // ----------------------------------------------------------------------------- template +ObsDataQG::ObsDataQG(const ObsVecQG & other): data_(other) { +} +// ----------------------------------------------------------------------------- +template ObsDataQG & ObsDataQG::operator= (const ObsDataQG & rhs) { data_ = rhs.data_; return *this; diff --git a/qg/model/ObsVecQG.cc b/qg/model/ObsVecQG.cc index 131cd3487..d16cb0ae0 100644 --- a/qg/model/ObsVecQG.cc +++ b/qg/model/ObsVecQG.cc @@ -83,6 +83,11 @@ ObsVecQG & ObsVecQG::operator/= (const ObsVecQG & rhs) { return *this; } // ----------------------------------------------------------------------------- +ObsVecQG & ObsVecQG::operator=(const ObsDataQG & rhs) { + *this = rhs.vect(); + return *this; +} +// ----------------------------------------------------------------------------- void ObsVecQG::zero() { qg_obsvec_zero_f90(keyOvec_); } diff --git a/qg/model/ObsVecQG.h b/qg/model/ObsVecQG.h index ed7142500..a9235c23e 100644 --- a/qg/model/ObsVecQG.h +++ b/qg/model/ObsVecQG.h @@ -23,9 +23,7 @@ namespace qg { class ObsSpaceQG; - - template - class ObsDataQG; + template class ObsDataQG; // ----------------------------------------------------------------------------- /// ObsVecQG class to handle vectors in observation space for QG model. @@ -59,6 +57,7 @@ class ObsVecQG : public util::Printable, double dot_product_with(const ObsVecQG &) const; double rms() const; void mask(const ObsDataQG &); + ObsVecQG & operator=(const ObsDataQG &); unsigned int nobs() const; diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index a3a5a93b3..7623747ae 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -281,6 +281,12 @@ ecbuild_add_test( TARGET test_qg_obs_vector LIBS qg TEST_DEPENDS test_qg_make_obs_4d_24h ) +ecbuild_add_test( TARGET test_qg_obsdatavector + SOURCES executables/TestObsDataVector.cc + ARGS "testinput/interfaces.yaml" + LIBS qg + TEST_DEPENDS test_qg_make_obs_4d_24h ) + ecbuild_add_test( TARGET test_qg_departures_ensemble SOURCES executables/TestDeparturesEnsemble.cc ARGS "testinput/interfaces.yaml" diff --git a/qg/test/executables/TestObsDataVector.cc b/qg/test/executables/TestObsDataVector.cc new file mode 100644 index 000000000..b25bdb4e1 --- /dev/null +++ b/qg/test/executables/TestObsDataVector.cc @@ -0,0 +1,17 @@ +/* + * (C) Copyright 2021-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "model/QgTraits.h" +#include "oops/runs/Run.h" +#include "test/interface/ObsDataVector.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + test::ObsDataVector tests; + return run.execute(tests); +} + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 466dcfa2e..1a66dc8c2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -202,6 +202,7 @@ oops/interface/ObsAuxControl.h oops/interface/ObsAuxCovariance.h oops/interface/ObsAuxIncrement.h oops/interface/ObsDataVector.h +oops/interface/ObsDataVector_head.h oops/interface/ObsDiagnostics.h oops/interface/ObsErrorCovariance.h oops/interface/ObsFilter.h @@ -388,6 +389,7 @@ test/interface/ModelAuxIncrement.h test/interface/ObsAuxControl.h test/interface/ObsAuxCovariance.h test/interface/ObsAuxIncrement.h +test/interface/ObsDataVector.h test/interface/ObsOperator.h test/interface/ObsSpace.h test/interface/ObsTestsFixture.h diff --git a/src/oops/assimilation/CalcHofX.h b/src/oops/assimilation/CalcHofX.h index fec557723..4f435c192 100644 --- a/src/oops/assimilation/CalcHofX.h +++ b/src/oops/assimilation/CalcHofX.h @@ -23,6 +23,7 @@ #include "oops/interface/ObsDataVector.h" #include "oops/interface/ObsDiagnostics.h" #include "oops/interface/ObsOperator.h" +#include "oops/interface/ObsVector.h" #include "oops/util/Logger.h" #include "oops/util/parameters/Parameter.h" #include "oops/util/parameters/Parameters.h" @@ -53,6 +54,7 @@ class CalcHofX { typedef ObsFilters ObsFilters_; typedef ObsOperator ObsOperator_; typedef ObsSpaces ObsSpaces_; + typedef ObsVector ObsVector_; template using ObsData_ = ObsDataVector; template using ObsDataVec_ = std::vector>>; @@ -60,6 +62,7 @@ class CalcHofX { typedef std::vector> LocationsVec_; typedef std::vector> ObsFiltersVec_; typedef std::vector> ObsOperatorVec_; + typedef std::vector> ObsVectorVec_; typedef std::vector VariablesVec_; public: @@ -98,7 +101,7 @@ class CalcHofX { LocationsVec_ locations_; // locations const ObsAuxCtrls_ * ybias_; // Obs bias ObsDataVec_ qcflags_; // QC flags - ObsDataVec_ obserrs_; // Obs error variances (used in QC filters) + ObsVectorVec_ obserrs_; // Obs error variances (used in QC filters) ObsFiltersVec_ filters_; // QC filters VariablesVec_ geovars_; // variables required from the model }; @@ -131,8 +134,7 @@ CalcHofX::CalcHofX(const ObsSpaces_ & obspaces, qcflags_.emplace_back(std::make_shared>(obspaces[jj], obspaces[jj].obsvariables())); /// Allocate and read initial obs error - obserrs_.emplace_back(std::make_shared>(obspaces[jj], - obspaces[jj].obsvariables(), "ObsError")); + obserrs_.emplace_back(std::make_shared(obspaces[jj], "ObsError")); } Log::trace() << "CalcHofX constructed" << std::endl; } @@ -149,7 +151,7 @@ void CalcHofX::initialize(const ObsAuxCtrls_ & obsaux, const int iteration) observerParams.deserialize(obsconfs[jj]); /// Set up QC filters and run preprocess filters_.emplace_back(new ObsFilters_(obspaces_[jj], observerParams.obsFilters, - qcflags_[jj], obserrs_[jj], iteration)); + qcflags_[jj], *obserrs_[jj], iteration)); filters_[jj]->preProcess(); /// Set up variables requested from the model diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index 699f9c0de..b8959b287 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -27,7 +27,6 @@ #include "oops/base/Observations.h" #include "oops/base/Observers.h" #include "oops/base/ObserversTLAD.h" -#include "oops/base/ObsFilters.h" #include "oops/base/ObsSpaces.h" #include "oops/base/PostBase.h" #include "oops/base/PostBaseTLAD.h" @@ -56,7 +55,6 @@ template class CostJo : public CostTermBase CtrlInc_; typedef Departures Departures_; typedef Observations Observations_; - typedef ObsFilters ObsFilters_; typedef Geometry Geometry_; typedef State State_; typedef Increment Increment_; @@ -65,7 +63,7 @@ template class CostJo : public CostTermBase Observers_; typedef ObserversTLAD ObserversTLAD_; typedef PostBaseTLAD PostBaseTLAD_; - typedef ObsDataVector ObsData_; + typedef ObsVector ObsVector_; public: /// Construct \f$ J_o\f$ from \f$ R\f$ and \f$ y_{obs}\f$. @@ -127,7 +125,7 @@ template class CostJo : public CostTermBase > obserrs_; // Obs errors + std::vector > obserrs_; // Obs errors /// Linearized observation operators. std::shared_ptr pobstlad_; @@ -146,8 +144,7 @@ CostJo::CostJo(const eckit::Configuration & joConf, const eckit::mpi { for (size_t jj = 0; jj < obspace_.size(); ++jj) { /// Allocate and read initial obs error - obserrs_.emplace_back(std::make_shared(obspace_[jj], - obspace_[jj].obsvariables(), "ObsError")); + obserrs_.emplace_back(std::make_shared(obspace_[jj], "ObsError")); } Log::trace() << "CostJo::CostJo done" << std::endl; diff --git a/src/oops/assimilation/LocalEnsembleSolver.h b/src/oops/assimilation/LocalEnsembleSolver.h index 6ba990da9..9db736582 100644 --- a/src/oops/assimilation/LocalEnsembleSolver.h +++ b/src/oops/assimilation/LocalEnsembleSolver.h @@ -113,9 +113,9 @@ Observations LocalEnsembleSolver::computeHofX(const StateEnsemb } else { // compute and save H(x) Log::debug() << "Computing H(X) online" << std::endl; - hofx_.initialize(obsaux_, iteration); for (size_t jj = 0; jj < nens; ++jj) { hofx_.resetQc(); + hofx_.initialize(obsaux_, iteration); // fill in geovals std::vector getValuesConfig = util::vectoriseAndFilter(obsconf_, "get values"); diff --git a/src/oops/base/ObsFilters.h b/src/oops/base/ObsFilters.h index c9a6c51aa..023627ee1 100644 --- a/src/oops/base/ObsFilters.h +++ b/src/oops/base/ObsFilters.h @@ -39,7 +39,7 @@ class ObsFilters : public util::Printable, typedef ObsSpace ObsSpace_; typedef ObsVector ObsVector_; typedef std::shared_ptr > ObsFilterPtr_; - template using ObsDataPtr_ = std::shared_ptr >; + typedef std::shared_ptr > ObsDataPtr_; public: /// Initialize all filters for \p obspace, from parameters, using @@ -47,8 +47,7 @@ class ObsFilters : public util::Printable, /// \p iteration argument indicates outer loop iteration in the variational /// assimilation ObsFilters(const ObsSpace_ &, const std::vector> &, - ObsDataPtr_ qcflags, ObsDataPtr_ obserr, - const int iteration = 0); + ObsDataPtr_ qcflags, ObsVector_ & obserr, const int iteration = 0); void preProcess() const; void priorFilter(const GeoVaLs_ &) const; @@ -63,8 +62,9 @@ class ObsFilters : public util::Printable, std::vector filters_; Variables geovars_; Variables diagvars_; - ObsDataPtr_ qcflags_; - ObsDataPtr_ obserr_; + ObsDataPtr_ qcflags_; + ObsVector_ & obserr_; + std::shared_ptr > obserrtmp_; }; // ----------------------------------------------------------------------------- @@ -72,9 +72,9 @@ class ObsFilters : public util::Printable, template ObsFilters::ObsFilters(const ObsSpace_ & os, const std::vector> & filtersParams, - ObsDataPtr_ qcflags, ObsDataPtr_ obserr, - const int iteration) - : filters_(), geovars_(), diagvars_(), qcflags_(qcflags), obserr_(obserr) { + ObsDataPtr_ qcflags, ObsVector_ & obserr, const int iteration) + : filters_(), geovars_(), diagvars_(), qcflags_(qcflags), obserr_(obserr), + obserrtmp_(new ObsDataVector(obserr)) { Log::trace() << "ObsFilters::ObsFilters starting:\n"; for (const ObsFilterParametersWrapper &filterParams : filtersParams) Log::trace() << " " << filterParams << std::endl; @@ -83,7 +83,7 @@ ObsFilters::ObsFilters(const ObsSpace_ & os, if (filtersParams.size() > 0) { eckit::LocalConfiguration preconf; preconf.set("filter", "QCmanager"); - filters_.push_back(FilterFactory::create(os, preconf, qcflags_, obserr_)); + filters_.push_back(FilterFactory::create(os, preconf, qcflags_, obserrtmp_)); } // Create the filters, only at 0-th iteration, or at iterations specified in "apply at iterations" @@ -97,7 +97,7 @@ ObsFilters::ObsFilters(const ObsSpace_ & os, } if (apply) { ObsFilterPtr_ tmp(FilterFactory::create(os, filterParams.filterParameters, - qcflags_, obserr_)); + qcflags_, obserrtmp_)); geovars_ += tmp->requiredVars(); diagvars_ += tmp->requiredHdiagnostics(); filters_.push_back(tmp); @@ -114,6 +114,8 @@ void ObsFilters::preProcess() const { for (const auto & filter : filters_) { filter->preProcess(); } + obserrtmp_->mask(*qcflags_); + obserr_ = *obserrtmp_; } // ----------------------------------------------------------------------------- @@ -123,6 +125,8 @@ void ObsFilters::priorFilter(const GeoVaLs_ & gv) const { for (const auto & filter : filters_) { filter->priorFilter(gv); } + obserrtmp_->mask(*qcflags_); + obserr_ = *obserrtmp_; } // ----------------------------------------------------------------------------- @@ -132,7 +136,8 @@ void ObsFilters::postFilter(const ObsVector_ & hofx, const ObsDiags_ & diag for (const auto & filter : filters_) { filter->postFilter(hofx, diags); } - obserr_->mask(*qcflags_); + obserrtmp_->mask(*qcflags_); + obserr_ = *obserrtmp_; } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/Observers.h b/src/oops/base/Observers.h index 8fd92d75d..2dc49f4a0 100644 --- a/src/oops/base/Observers.h +++ b/src/oops/base/Observers.h @@ -24,6 +24,7 @@ #include "oops/interface/ObsDataVector.h" #include "oops/interface/ObsDiagnostics.h" #include "oops/interface/ObsOperator.h" +#include "oops/interface/ObsVector.h" #include "oops/util/ConfigFunctions.h" #include "oops/util/Logger.h" #include "oops/util/parameters/Parameter.h" @@ -56,6 +57,7 @@ class Observers { typedef ObsFilters ObsFilters_; typedef ObsOperator ObsOperator_; typedef ObsSpaces ObsSpaces_; + typedef std::vector>> ObsVectors_; template using ObsData_ = ObsDataVector; template using ObsDataVec_ = std::vector>>; @@ -72,7 +74,7 @@ class Observers { /// \brief Initializes variables, obs bias, obs filters (could be different for /// different iterations std::shared_ptr > > initialize(const ObsAuxCtrls_ &, - ObsDataVec_ &, const int iter = 0); + ObsVectors_ &, const int iter = 0); /// \brief Computes H(x) from the filled in GeoVaLs Observations_ finalize(); @@ -114,7 +116,7 @@ Observers::Observers(const ObsSpaces_ & obspaces, template std::shared_ptr > > -Observers::initialize(const ObsAuxCtrls_ & obsaux, ObsDataVec_ & obserrs, +Observers::initialize(const ObsAuxCtrls_ & obsaux, ObsVectors_ & obserrs, const int iter) { std::vector obsconfs = obsconfig_.getSubConfigurations(); iterout_ = iter; @@ -126,7 +128,7 @@ Observers::initialize(const ObsAuxCtrls_ & obsaux, ObsDataVec_preProcess(); /// Set up variables requested from the model diff --git a/src/oops/interface/ObsDataVector.h b/src/oops/interface/ObsDataVector.h index 0754546e6..90581213d 100644 --- a/src/oops/interface/ObsDataVector.h +++ b/src/oops/interface/ObsDataVector.h @@ -14,53 +14,20 @@ #include "oops/base/Variables.h" #include "oops/interface/ObsSpace.h" +#include "oops/interface/ObsVector.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" #include "oops/util/Timer.h" -namespace oops { - -// ----------------------------------------------------------------------------- - -template -class ObsDataVector : public util::Printable, - private util::ObjectCounter > { - typedef typename OBS::template ObsDataVector ObsDataVec_; - - public: - static const std::string classname() {return "oops::ObsDataVector";} - - ObsDataVector(const ObsSpace &, const Variables &, const std::string name = ""); - explicit ObsDataVector(const ObsDataVector &); - ~ObsDataVector(); - -/// Interfacing - ObsDataVec_ & obsdatavector() {return *data_;} - const ObsDataVec_ & obsdatavector() const {return *data_;} - - std::shared_ptr obsdatavectorptr() {return data_;} - std::shared_ptr obsdatavectorptr() const {return data_;} - - ObsDataVector & operator = (const ObsDataVector &); +#include "oops/interface/ObsDataVector_head.h" - void zero(); - void mask(const ObsDataVector &); - unsigned int nobs() const {return data_->nobs();} - -// I/O - void read(const std::string &); - void save(const std::string &) const; - - private: - void print(std::ostream &) const; - std::shared_ptr data_; -}; +namespace oops { // ----------------------------------------------------------------------------- template ObsDataVector::ObsDataVector(const ObsSpace & os, - const Variables & vars, const std::string name) + const Variables & vars, const std::string name) : data_() { Log::trace() << "ObsDataVector::ObsDataVector starting" << std::endl; @@ -78,6 +45,14 @@ ObsDataVector::ObsDataVector(const ObsDataVector & other): data_( } // ----------------------------------------------------------------------------- template +ObsDataVector::ObsDataVector(ObsVector & other): data_() { + Log::trace() << "ObsDataVector::ObsDataVector starting" << std::endl; + util::Timer timer(classname(), "ObsDataVector"); + data_.reset(new ObsDataVec_(other.obsvector())); + Log::trace() << "ObsDataVector::ObsDataVector done" << std::endl; +} +// ----------------------------------------------------------------------------- +template ObsDataVector::~ObsDataVector() { Log::trace() << "ObsDataVector::~ObsDataVector starting" << std::endl; util::Timer timer(classname(), "~ObsDataVector"); diff --git a/src/oops/interface/ObsDataVector_head.h b/src/oops/interface/ObsDataVector_head.h new file mode 100644 index 000000000..5f2ba6edd --- /dev/null +++ b/src/oops/interface/ObsDataVector_head.h @@ -0,0 +1,66 @@ +/* + * (C) Copyright 2018 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_INTERFACE_OBSDATAVECTOR_HEAD_H_ +#define OOPS_INTERFACE_OBSDATAVECTOR_HEAD_H_ + +#include +#include +#include + +#include "oops/base/Variables.h" +#include "oops/interface/ObsSpace.h" +#include "oops/util/Logger.h" +#include "oops/util/ObjectCounter.h" +#include "oops/util/Printable.h" +#include "oops/util/Timer.h" + +namespace oops { + template class ObsVector; + +// ----------------------------------------------------------------------------- + +template +class ObsDataVector : public util::Printable, + private util::ObjectCounter > { + typedef typename OBS::template ObsDataVector ObsDataVec_; + + public: + static const std::string classname() {return "oops::ObsDataVector";} + + ObsDataVector(const ObsSpace &, const Variables &, const std::string name = ""); + explicit ObsDataVector(const ObsDataVector &); + explicit ObsDataVector(ObsVector &); + ~ObsDataVector(); + +/// Interfacing + ObsDataVec_ & obsdatavector() {return *data_;} + const ObsDataVec_ & obsdatavector() const {return *data_;} + + std::shared_ptr obsdatavectorptr() {return data_;} + std::shared_ptr obsdatavectorptr() const {return data_;} + + ObsDataVector & operator = (const ObsDataVector &); + + void zero(); + void mask(const ObsDataVector &); + unsigned int nobs() const {return data_->nobs();} + +// I/O + void read(const std::string &); + void save(const std::string &) const; + + private: + void print(std::ostream &) const; + std::shared_ptr data_; +}; + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_INTERFACE_OBSDATAVECTOR_HEAD_H_ diff --git a/src/oops/interface/ObsVector.h b/src/oops/interface/ObsVector.h index e9daf010f..ed4f22c60 100644 --- a/src/oops/interface/ObsVector.h +++ b/src/oops/interface/ObsVector.h @@ -17,7 +17,7 @@ #include #include -#include "oops/interface/ObsDataVector.h" +#include "oops/interface/ObsDataVector_head.h" #include "oops/interface/ObsSpace.h" #include "oops/util/gatherPrint.h" #include "oops/util/Logger.h" @@ -77,6 +77,7 @@ class ObsVector : public util::Printable, double rms() const; /// Mask out elements of the vector where the passed in flags are > 0 void mask(const ObsDataVector &); + ObsVector & operator =(const ObsDataVector &); // I/O void save(const std::string &) const; @@ -272,6 +273,15 @@ void ObsVector::mask(const ObsDataVector & qc) { } // ----------------------------------------------------------------------------- template +ObsVector & ObsVector::operator=(const ObsDataVector & rhs) { + Log::trace() << "ObsVector::operator= starting" << std::endl; + util::Timer timer(classname(), "operator="); + *data_ = rhs.obsdatavector(); + Log::trace() << "ObsVector::operator= done" << std::endl; + return *this; +} +// ----------------------------------------------------------------------------- +template double ObsVector::rms() const { Log::trace() << "ObsVector::rms starting" << std::endl; util::Timer timer(classname(), "rms"); diff --git a/src/oops/runs/HofX4D.h b/src/oops/runs/HofX4D.h index cf40e43a1..1b4152331 100644 --- a/src/oops/runs/HofX4D.h +++ b/src/oops/runs/HofX4D.h @@ -31,6 +31,7 @@ #include "oops/interface/Model.h" #include "oops/interface/ModelAuxControl.h" #include "oops/interface/ObsDataVector.h" +#include "oops/interface/ObsVector.h" #include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" @@ -52,6 +53,7 @@ template class HofX4D : public Application { typedef ObsErrors ObsErrors_; typedef Observers Observers_; typedef ObsSpaces ObsSpaces_; + typedef ObsVector ObsVector_; typedef State State_; typedef ObsDataVector ObsData_; @@ -109,10 +111,9 @@ template class HofX4D : public Application { // Setup and run observer Observers_ hofx(obspaces, obsConfig); - std::vector> obserrs; + std::vector> obserrs; for (size_t jj = 0; jj < obspaces.size(); ++jj) { - obserrs.emplace_back(std::make_shared(obspaces[jj], - obspaces[jj].obsvariables(), "ObsError")); + obserrs.emplace_back(std::make_shared(obspaces[jj], "ObsError")); } // run the model and compute H(x) diff --git a/src/test/interface/ObsDataVector.h b/src/test/interface/ObsDataVector.h new file mode 100644 index 000000000..4305b879e --- /dev/null +++ b/src/test/interface/ObsDataVector.h @@ -0,0 +1,109 @@ +/* + * (C) Copyright 2021-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef TEST_INTERFACE_OBSDATAVECTOR_H_ +#define TEST_INTERFACE_OBSDATAVECTOR_H_ + +#include +#include +#include + +#define ECKIT_TESTING_SELF_REGISTER_CASES 0 + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/testing/Test.h" +#include "oops/base/Variables.h" +#include "oops/interface/ObsDataVector.h" +#include "oops/interface/ObsVector.h" +#include "oops/runs/Test.h" +#include "oops/util/dot_product.h" +#include "test/interface/ObsTestsFixture.h" +#include "test/TestEnvironment.h" + +namespace test { + +// ----------------------------------------------------------------------------- +/// \brief tests constructor and print method +template void testConstructors() { + typedef ObsTestsFixture Test_; + typedef oops::ObsDataVector ObsDataVector_; + + for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + oops::Variables vars(Test_::obspace()[jj].obsvariables()); + std::unique_ptr odv1(new ObsDataVector_(Test_::obspace()[jj], vars)); + EXPECT(odv1.get()); + + odv1->zero(); + oops::Log::test() << "Printing zero ObsDataVector: " << *odv1 << std::endl; + + std::unique_ptr odv2(new ObsDataVector_(*odv1)); + EXPECT(odv2.get()); + + odv2.reset(); + EXPECT(!odv2.get()); + EXPECT(odv1.get()); + + odv1.reset(); + EXPECT(!odv1.get()); + } +} + +// ----------------------------------------------------------------------------- +template void testObsVector() { + typedef ObsTestsFixture Test_; + typedef oops::ObsDataVector ObsDataVector_; + + const double tolerance = 1.0e-10; + for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + oops::ObsVector ov1(Test_::obspace()[jj]); + ov1.random(); + + ObsDataVector_ odv(ov1); + oops::Log::test() << "Printing random ObsDataVector: " << odv << std::endl; + + oops::ObsVector ov2(Test_::obspace()[jj]); + ov2 = odv; + + ov1 -= ov2; + oops::Log::test() << "ObsVector - ObsDataVector = " << ov1 << std::endl; + const double diff = dot_product(ov1, ov1); + oops::Log::test() << "ObsVector, ObsDataVector diff = " << diff << std::endl; + EXPECT(diff < tolerance); + } +} +// ----------------------------------------------------------------------------- + +template +class ObsDataVector : public oops::Test { + typedef ObsTestsFixture Test_; + + public: + ObsDataVector() {} + virtual ~ObsDataVector() {} + + private: + std::string testid() const override {return "test::ObsDataVector<" + OBS::name() + ", float>";} + + void register_tests() const override { + std::vector& ts = eckit::testing::specification(); + + ts.emplace_back(CASE("interface/ObsDataVector/testConstructors") + { testConstructors(); }); + ts.emplace_back(CASE("interface/ObsDataVector/testObsVector") + { testObsVector(); }); + } + + void clear() const override { + Test_::reset(); + } +}; + +// ============================================================================= + +} // namespace test + +#endif // TEST_INTERFACE_OBSDATAVECTOR_H_ From 8418858320138797ea32a8ab028511dc01e95b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Wed, 31 Mar 2021 14:01:28 -0600 Subject: [PATCH 092/142] Pass post processors to the terms of the cost function. (#1123) * ObsVector <-> ObsDataVector conversions * Pass ObsVector to ObsFilters * Added ObsDataVector test * Pass obs errors by reference * Pass post processors to cost terms --- src/oops/assimilation/CostFunction.h | 11 ++-- src/oops/assimilation/CostJbTotal.h | 65 +++++++++++++---------- src/oops/assimilation/CostJcDFI.h | 45 ++++++++-------- src/oops/assimilation/CostJo.h | 44 +++++++-------- src/oops/assimilation/CostTermBase.h | 22 ++++---- src/oops/assimilation/HBHtMatrix.h | 7 ++- src/oops/assimilation/HMatrix.h | 5 +- src/oops/assimilation/HessianMatrix.h | 19 +++---- src/oops/assimilation/HtMatrix.h | 2 +- src/oops/assimilation/HtRinvHMatrix.h | 4 +- src/oops/assimilation/JqTermTLAD.h | 4 ++ src/oops/assimilation/LBHessianMatrix.h | 16 +++--- src/oops/assimilation/SaddlePointMatrix.h | 17 +++--- 13 files changed, 129 insertions(+), 132 deletions(-) diff --git a/src/oops/assimilation/CostFunction.h b/src/oops/assimilation/CostFunction.h index fc75a9a04..c927d0bf3 100644 --- a/src/oops/assimilation/CostFunction.h +++ b/src/oops/assimilation/CostFunction.h @@ -227,7 +227,7 @@ double CostFunction::evaluate(const CtrlVar_ & fguess, PostProcessor pp(post); jb_->initialize(fguess); for (unsigned jj = 0; jj < jterms_.size(); ++jj) { - pp.enrollProcessor(jterms_[jj].initialize(fguess, config)); + jterms_[jj].initialize(fguess, config, pp); } // Run NL model @@ -261,10 +261,9 @@ double CostFunction::linearize(const CtrlVar_ & fguess, // Setup trajectory for terms of cost function PostProcessorTLAD pptraj; - JqTermTLAD_ * jq = jb_->initializeTraj(fguess, lowres, innerConf); - pptraj.enrollProcessor(jq); + jb_->initializeTraj(fguess, lowres, innerConf, pptraj); for (unsigned jj = 0; jj < jterms_.size(); ++jj) { - pptraj.enrollProcessor(jterms_[jj].initializeTraj(fguess, lowres, innerConf)); + jterms_[jj].initializeTraj(fguess, lowres, innerConf, pptraj); } // Specific linearization if needed (including TLM) @@ -274,7 +273,7 @@ double CostFunction::linearize(const CtrlVar_ & fguess, double zzz = this->evaluate(fguess, innerConf, post); // Finalize trajectory setup - jb_->finalizeTraj(jq); + jb_->finalizeTraj(); for (unsigned jj = 0; jj < jterms_.size(); ++jj) { jterms_[jj].finalizeTraj(); } @@ -294,7 +293,7 @@ void CostFunction::computeGradientFG(CtrlInc_ & grad) const { for (unsigned jj = 0; jj < jterms_.size(); ++jj) { std::shared_ptr tmp(jterms_[jj].newGradientFG()); - costad.enrollProcessor(jterms_[jj].setupAD(tmp, grad)); + jterms_[jj].setupAD(tmp, grad, costad); } this->runADJ(grad, costad, pp); diff --git a/src/oops/assimilation/CostJbTotal.h b/src/oops/assimilation/CostJbTotal.h index 939ffc07a..e333791e3 100644 --- a/src/oops/assimilation/CostJbTotal.h +++ b/src/oops/assimilation/CostJbTotal.h @@ -20,6 +20,7 @@ #include "oops/assimilation/CostJbState.h" #include "oops/base/ObsAuxCovariances.h" #include "oops/base/ObsSpaces.h" +#include "oops/base/PostProcessorTLAD.h" #include "oops/interface/Geometry.h" #include "oops/interface/ModelAuxCovariance.h" #include "oops/interface/State.h" @@ -42,6 +43,7 @@ template class CostJbTotal { typedef ModelAuxCovariance ModelAuxCovar_; typedef ObsAuxCovariances ObsAuxCovars_; typedef ObsSpaces ObsSpaces_; + typedef PostProcessorTLAD PostProcTLAD_; public: /// Construct \f$ J_b\f$. @@ -53,20 +55,20 @@ template class CostJbTotal { /// Initialize before nonlinear model integration. void initialize(const CtrlVar_ &) const; - JqTermTLAD_ * initializeTraj(const CtrlVar_ &, const Geometry_ &, - const eckit::Configuration &); + void initializeTraj(const CtrlVar_ &, const Geometry_ &, + const eckit::Configuration &, PostProcTLAD_ &); /// Finalize computation after nonlinear model integration. double finalize(const CtrlVar_ &) const; - void finalizeTraj(JqTermTLAD_ *); + void finalizeTraj(); /// Initialize before starting the TL run. - JqTermTLAD_ * initializeTL() const; - void finalizeTL(JqTermTLAD_ *, const CtrlInc_ &, CtrlInc_ &) const; + unsigned int initializeTL(PostProcTLAD_ &) const; + void finalizeTL(const CtrlInc_ &, CtrlInc_ &) const; /// Initialize before starting the AD run. - JqTermTLAD_ * initializeAD(CtrlInc_ &, const CtrlInc_ &) const; - void finalizeAD(JqTermTLAD_ *) const; + void initializeAD(CtrlInc_ &, const CtrlInc_ &, PostProcTLAD_ &) const; + void finalizeAD() const; /// Multiply by covariance matrix and its inverse. void multiplyB(const CtrlInc_ &, CtrlInc_ &) const; @@ -112,6 +114,9 @@ template class CostJbTotal { const util::DateTime windowEnd_; bool jbEvaluation_; + std::shared_ptr jqtraj_; + mutable std::shared_ptr jqtl_; + mutable std::shared_ptr jqad_; }; // ============================================================================= @@ -124,7 +129,8 @@ CostJbTotal::CostJbTotal(const CtrlVar_ & xb, JbState_ * jb, jbModBias_(conf.getSubConfiguration("model aux error"), resol), jbObsBias_(odb, conf.getSubConfiguration("observations")), dxFG_(), resol_(), windowBegin_(conf.getString("window begin")), - windowEnd_(windowBegin_ + util::Duration(conf.getString("window length"))) + windowEnd_(windowBegin_ + util::Duration(conf.getString("window length"))), + jqtraj_() { jbEvaluation_ = conf.getBool("jb evaluation", true); Log::trace() << "CostJbTotal contructed." << std::endl; @@ -167,9 +173,9 @@ double CostJbTotal::finalize(const CtrlVar_ & mx) const { // ----------------------------------------------------------------------------- template -JqTermTLAD * CostJbTotal::initializeTraj(const CtrlVar_ & fg, - const Geometry_ & resol, - const eckit::Configuration & inner) { +void CostJbTotal::initializeTraj(const CtrlVar_ & fg, const Geometry_ & resol, + const eckit::Configuration & inner, + PostProcTLAD_ & pptraj) { Log::trace() << "CostJbTotal::initializeTraj start" << std::endl; fg_ = &fg; resol_.reset(new Geometry_(resol)); @@ -178,16 +184,16 @@ JqTermTLAD * CostJbTotal::initializeTraj(const CtrlVar_ & fg, jbModBias_.linearize(fg.modVar(), *resol_); jbObsBias_.linearize(fg.obsVar(), inner); - JqTermTLAD_ * jqlin = jb_->initializeJqTLAD(); + jqtraj_.reset(jb_->initializeJqTLAD()); + pptraj.enrollProcessor(jqtraj_); Log::trace() << "CostJbTotal::initializeTraj done" << std::endl; - return jqlin; } // ----------------------------------------------------------------------------- template -void CostJbTotal::finalizeTraj(JqTermTLAD_ * jqlin) { +void CostJbTotal::finalizeTraj() { Log::trace() << "CostJbTotal::finalizeTraj start" << std::endl; ASSERT(fg_); // Compute and save first guess increment. @@ -195,7 +201,7 @@ void CostJbTotal::finalizeTraj(JqTermTLAD_ * jqlin) { // Compute x_0 - x_b for Jb (and Jq is present) const State_ * mx = &fg_->state(); - if (jqlin) mx = &jqlin->getMxi(); + if (jqtraj_) mx = &jqtraj_->getMxi(); jb_->computeIncrement(xb_.state(), fg_->state(), *mx, dxFG_->state()); // Model and Obs biases @@ -204,6 +210,7 @@ void CostJbTotal::finalizeTraj(JqTermTLAD_ * jqlin) { // Print increment Log::info() << "CostJb: FG-BG" << *dxFG_ << std::endl; + jqtraj_.reset(); Log::trace() << "CostJbTotal::finalizeTraj done" << std::endl; } @@ -262,42 +269,46 @@ void CostJbTotal::addGradientFG(CtrlInc_ & grad, CtrlInc_ & gradJb) // ----------------------------------------------------------------------------- template -JqTermTLAD * CostJbTotal::initializeTL() const { +unsigned int CostJbTotal::initializeTL(PostProcTLAD_ & pptl) const { Log::trace() << "CostJbTotal::initializeTL start" << std::endl; - JqTermTLAD_ * jqtl = jb_->initializeJqTL(); + jqtl_.reset(jb_->initializeJqTL()); + unsigned int iq = 0; // Hack... + if (jqtl_) iq = 1; // Hack... + pptl.enrollProcessor(jqtl_); Log::trace() << "CostJbTotal::initializeTL done" << std::endl; - return jqtl; + return iq; // Hack... } // ----------------------------------------------------------------------------- template -void CostJbTotal::finalizeTL(JqTermTLAD_ * jqtl, const CtrlInc_ & bgns, - CtrlInc_ & dx) const { +void CostJbTotal::finalizeTL(const CtrlInc_ & bgns, CtrlInc_ & dx) const { Log::trace() << "CostJbTotal::finalizeTL start" << std::endl; dx = bgns; - if (jqtl) jqtl->computeModelErrorTL(dx.state()); + if (jqtl_) jqtl_->computeModelErrorTL(dx.state()); + jqtl_.reset(); Log::trace() << "CostJbTotal::finalizeTL done" << std::endl; } // ----------------------------------------------------------------------------- template -JqTermTLAD * CostJbTotal::initializeAD(CtrlInc_ & bgns, - const CtrlInc_ & dx) const { +void CostJbTotal::initializeAD(CtrlInc_ & bgns, const CtrlInc_ & dx, + PostProcTLAD_ & ppad) const { Log::trace() << "CostJbTotal::initializeAD start" << std::endl; - JqTermTLAD_ * jqad = jb_->initializeJqAD(dx.state()); + jqad_.reset(jb_->initializeJqAD(dx.state())); bgns += dx; + ppad.enrollProcessor(jqad_); Log::trace() << "CostJbTotal::initializeAD done" << std::endl; - return jqad; } // ----------------------------------------------------------------------------- template -void CostJbTotal::finalizeAD(JqTermTLAD_ * jqad) const { +void CostJbTotal::finalizeAD() const { Log::trace() << "CostJbTotal::finalizeAD start" << std::endl; - if (jqad) jqad->clear(); + if (jqad_) jqad_->clear(); + jqad_.reset(); Log::trace() << "CostJbTotal::finalizeAD done" << std::endl; } diff --git a/src/oops/assimilation/CostJcDFI.h b/src/oops/assimilation/CostJcDFI.h index 875c54ce0..5f6b4c38e 100644 --- a/src/oops/assimilation/CostJcDFI.h +++ b/src/oops/assimilation/CostJcDFI.h @@ -19,8 +19,8 @@ #include "oops/assimilation/ControlVariable.h" #include "oops/assimilation/CostTermBase.h" #include "oops/base/DolphChebyshev.h" -#include "oops/base/PostBase.h" -#include "oops/base/PostBaseTLAD.h" +#include "oops/base/PostProcessor.h" +#include "oops/base/PostProcessorTLAD.h" #include "oops/base/Variables.h" #include "oops/base/WeightedDiff.h" #include "oops/base/WeightedDiffTLAD.h" @@ -46,8 +46,9 @@ template class CostJcDFI : public CostTermBase CtrlVar_; typedef Geometry Geometry_; typedef Increment Increment_; - typedef PostBaseTLAD PostBaseTLAD_; typedef State State_; + typedef PostProcessor PostProc_; + typedef PostProcessorTLAD PostProcTLAD_; public: /// Construct \f$ J_c\f$. @@ -58,21 +59,20 @@ template class CostJcDFI : public CostTermBase > initialize(const CtrlVar_ &, - const eckit::Configuration &) override; - std::shared_ptr > initializeTraj(const CtrlVar_ &, const Geometry_ &, - const eckit::Configuration &) override; + void initialize(const CtrlVar_ &, const eckit::Configuration &, PostProc_ &) override; + void initializeTraj(const CtrlVar_ &, const Geometry_ &, + const eckit::Configuration &, PostProcTLAD_ &) override; /// Finalize computation after nonlinear model integration. double finalize() override; void finalizeTraj() override; /// Initialize \f$ J_c\f$ before starting the TL run. - std::shared_ptr setupTL(const CtrlInc_ &) const override; + void setupTL(const CtrlInc_ &, PostProcTLAD_ &) const override; /// Initialize \f$ J_c\f$ before starting the AD run. - std::shared_ptr setupAD( - std::shared_ptr, CtrlInc_ &) const override; + void setupAD(std::shared_ptr, + CtrlInc_ &, PostProcTLAD_ &) const override; /// Multiply by \f$ C\f$ and \f$ C^{-1}\f$. std::unique_ptr @@ -124,11 +124,11 @@ CostJcDFI::CostJcDFI(const eckit::Configuration & conf, const Geomet // ----------------------------------------------------------------------------- template -std::shared_ptr > > -CostJcDFI::initialize(const CtrlVar_ &, const eckit::Configuration &) { +void CostJcDFI::initialize(const CtrlVar_ &, const eckit::Configuration &, + PostProc_ & pp) { filter_.reset(new WeightedDiff(vars_, vt_, span_, tstep_, resol_, *wfct_)); - return filter_; + pp.enrollProcessor(filter_); } // ----------------------------------------------------------------------------- @@ -145,13 +145,13 @@ double CostJcDFI::finalize() { // ----------------------------------------------------------------------------- template -std::shared_ptr > -CostJcDFI::initializeTraj(const CtrlVar_ &, const Geometry_ & tlres, - const eckit::Configuration & innerConf) { +void CostJcDFI::initializeTraj(const CtrlVar_ &, const Geometry_ & tlres, + const eckit::Configuration & innerConf, + PostProcTLAD_ & pptraj) { tlres_.reset(new Geometry_(tlres)); tlstep_ = util::Duration(innerConf.getString("linear model.tstep", tstep_.toString())); ftlad_.reset(new WeightedDiffTLAD(vars_, vt_, span_, tstep_, *tlres_, *wfct_)); - return ftlad_; + pptraj.enrollProcessor(ftlad_); } // ----------------------------------------------------------------------------- @@ -180,21 +180,20 @@ std::unique_ptr CostJcDFI::newGradientFG() co // ----------------------------------------------------------------------------- template -std::shared_ptr > -CostJcDFI::setupTL(const CtrlInc_ &) const { +void CostJcDFI::setupTL(const CtrlInc_ &, PostProcTLAD_ & pptl) const { ftlad_->setupTL(*tlres_); - return ftlad_; + pptl.enrollProcessor(ftlad_); } // ----------------------------------------------------------------------------- template -std::shared_ptr > -CostJcDFI::setupAD(std::shared_ptr pv, CtrlInc_ &) const { +void CostJcDFI::setupAD(std::shared_ptr pv, + CtrlInc_ &, PostProcTLAD_ & ppad) const { std::shared_ptr dx = std::dynamic_pointer_cast(pv); ftlad_->setupAD(dx); - return ftlad_; + ppad.enrollProcessor(ftlad_); } // ----------------------------------------------------------------------------- diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index b8959b287..b9975028c 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -29,7 +29,8 @@ #include "oops/base/ObserversTLAD.h" #include "oops/base/ObsSpaces.h" #include "oops/base/PostBase.h" -#include "oops/base/PostBaseTLAD.h" +#include "oops/base/PostProcessor.h" +#include "oops/base/PostProcessorTLAD.h" #include "oops/interface/Geometry.h" #include "oops/interface/Increment.h" #include "oops/interface/ObsDataVector.h" @@ -62,6 +63,8 @@ template class CostJo : public CostTermBase ObsSpaces_; typedef Observers Observers_; typedef ObserversTLAD ObserversTLAD_; + typedef PostProcessor PostProc_; + typedef PostProcessorTLAD PostProcTLAD_; typedef PostBaseTLAD PostBaseTLAD_; typedef ObsVector ObsVector_; @@ -75,21 +78,19 @@ template class CostJo : public CostTermBase > initialize(const CtrlVar_ &, - const eckit::Configuration &) override; - std::shared_ptr initializeTraj(const CtrlVar_ &, - const Geometry_ &, - const eckit::Configuration &) override; + void initialize(const CtrlVar_ &, const eckit::Configuration &, PostProc_ &) override; + void initializeTraj(const CtrlVar_ &, const Geometry_ &, + const eckit::Configuration &, PostProcTLAD_ &) override; /// Finalize \f$ J_o\f$ after the integration of the model. double finalize() override; void finalizeTraj() override; /// Initialize \f$ J_o\f$ before starting the TL run. - std::shared_ptr setupTL(const CtrlInc_ &) const override; + void setupTL(const CtrlInc_ &, PostProcTLAD_ &) const override; /// Initialize \f$ J_o\f$ before starting the AD run. - std::shared_ptr setupAD( - std::shared_ptr, CtrlInc_ &) const override; + void setupAD(std::shared_ptr, + CtrlInc_ &, PostProcTLAD_ &) const override; /// Multiply by \f$ R\f$ and \f$ R^{-1}\f$. std::unique_ptr @@ -153,8 +154,8 @@ CostJo::CostJo(const eckit::Configuration & joConf, const eckit::mpi // ----------------------------------------------------------------------------- template -std::shared_ptr > > -CostJo::initialize(const CtrlVar_ & xx, const eckit::Configuration & conf) { +void CostJo::initialize(const CtrlVar_ & xx, const eckit::Configuration & conf, + PostProc_ & pp) { Log::trace() << "CostJo::initialize start" << std::endl; currentConf_.reset(new eckit::LocalConfiguration(conf)); @@ -163,8 +164,8 @@ CostJo::initialize(const CtrlVar_ & xx, const eckit::Configuration & std::shared_ptr > > getvals(observers_.initialize(xx.obsVar(), obserrs_, iter)); + pp.enrollProcessor(getvals); Log::trace() << "CostJo::initialize done" << std::endl; - return getvals; } // ----------------------------------------------------------------------------- @@ -225,13 +226,13 @@ double CostJo::finalize() { // ----------------------------------------------------------------------------- template -std::shared_ptr > -CostJo::initializeTraj(const CtrlVar_ & xx, const Geometry_ &, - const eckit::Configuration & conf) { +void CostJo::initializeTraj(const CtrlVar_ & xx, const Geometry_ &, + const eckit::Configuration & conf, + PostProcTLAD_ & pptraj) { Log::trace() << "CostJo::initializeTraj start" << std::endl; pobstlad_.reset(new ObserversTLAD_(obsconf_, obspace_, xx.obsVar())); + pptraj.enrollProcessor(pobstlad_); Log::trace() << "CostJo::initializeTraj done" << std::endl; - return pobstlad_; } // ----------------------------------------------------------------------------- @@ -244,26 +245,25 @@ void CostJo::finalizeTraj() { // ----------------------------------------------------------------------------- template -std::shared_ptr > CostJo::setupTL(const CtrlInc_ & dx) const { +void CostJo::setupTL(const CtrlInc_ & dx, PostProcTLAD_ & pptl) const { Log::trace() << "CostJo::setupTL start" << std::endl; ASSERT(pobstlad_); pobstlad_->setupTL(dx.obsVar()); + pptl.enrollProcessor(pobstlad_); Log::trace() << "CostJo::setupTL done" << std::endl; - return pobstlad_; } // ----------------------------------------------------------------------------- template -std::shared_ptr > CostJo::setupAD( - std::shared_ptr pv, - CtrlInc_ & dx) const { +void CostJo::setupAD(std::shared_ptr pv, + CtrlInc_ & dx, PostProcTLAD_ & ppad) const { Log::trace() << "CostJo::setupAD start" << std::endl; ASSERT(pobstlad_); std::shared_ptr dy = std::dynamic_pointer_cast(pv); pobstlad_->setupAD(dy, dx.obsVar()); + ppad.enrollProcessor(pobstlad_); Log::trace() << "CostJo::setupAD done" << std::endl; - return pobstlad_; } // ----------------------------------------------------------------------------- diff --git a/src/oops/assimilation/CostTermBase.h b/src/oops/assimilation/CostTermBase.h index 9a679e7d9..ebf6bb9ce 100644 --- a/src/oops/assimilation/CostTermBase.h +++ b/src/oops/assimilation/CostTermBase.h @@ -15,8 +15,8 @@ #include "oops/assimilation/ControlIncrement.h" #include "oops/assimilation/ControlVariable.h" -#include "oops/base/PostBase.h" -#include "oops/base/PostBaseTLAD.h" +#include "oops/base/PostProcessor.h" +#include "oops/base/PostProcessorTLAD.h" #include "oops/interface/Geometry.h" #include "oops/interface/Increment.h" #include "oops/interface/State.h" @@ -38,29 +38,29 @@ template class CostTermBase { typedef Geometry Geometry_; typedef State State_; typedef Increment Increment_; - typedef std::shared_ptr > PostPtr_; - typedef std::shared_ptr > PostPtrTLAD_; + typedef PostProcessor PostProc_; + typedef PostProcessorTLAD PostProcTLAD_; public: /// Destructor virtual ~CostTermBase() {} /// Initialize before nonlinear model integration. - virtual PostPtr_ initialize(const ControlVariable &, - const eckit::Configuration &) = 0; - virtual PostPtrTLAD_ initializeTraj(const ControlVariable &, - const Geometry_ &, const eckit::Configuration &) = 0; + virtual void initialize(const ControlVariable &, const eckit::Configuration &, + PostProc_ &) = 0; + virtual void initializeTraj(const ControlVariable &, const Geometry_ &, + const eckit::Configuration &, PostProcTLAD_ &) = 0; /// Finalize computation after nonlinear model integration. virtual double finalize() = 0; virtual void finalizeTraj() = 0; /// Initialize before starting the TL run. - virtual PostPtrTLAD_ setupTL(const ControlIncrement &) const = 0; + virtual void setupTL(const ControlIncrement &, PostProcTLAD_ &) const = 0; /// Initialize before starting the AD run. - virtual PostPtrTLAD_ setupAD(std::shared_ptr, - ControlIncrement &) const = 0; + virtual void setupAD(std::shared_ptr, + ControlIncrement &, PostProcTLAD_ &) const = 0; /// Multiply by covariance (or weight) matrix and its inverse. virtual std::unique_ptr diff --git a/src/oops/assimilation/HBHtMatrix.h b/src/oops/assimilation/HBHtMatrix.h index bd2b9d36b..6ab5a6a23 100644 --- a/src/oops/assimilation/HBHtMatrix.h +++ b/src/oops/assimilation/HBHtMatrix.h @@ -49,8 +49,7 @@ template class HBHtMatrix : private boost::noncopy // ----------------------------------------------------------------------------- template -HBHtMatrix::HBHtMatrix(const CostFct_ & j, - const bool test) +HBHtMatrix::HBHtMatrix(const CostFct_ & j, const bool test) : j_(j), test_(test), iter_(0) {} @@ -66,7 +65,7 @@ void HBHtMatrix::multiply(const Dual_ & dy, Dual_ & dz) const { j_.zeroAD(ww); PostProcessorTLAD costad; for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - costad.enrollProcessor(j_.jterm(jj).setupAD(dy.getv(jj), ww)); + j_.jterm(jj).setupAD(dy.getv(jj), ww, costad); } j_.runADJ(ww, costad); @@ -77,7 +76,7 @@ void HBHtMatrix::multiply(const Dual_ & dy, Dual_ & dz) const { // Run TLM PostProcessorTLAD costtl; for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - costtl.enrollProcessor(j_.jterm(jj).setupTL(zz)); + j_.jterm(jj).setupTL(zz, costtl); } CtrlInc_ mzz(zz); j_.runTLM(mzz, costtl); diff --git a/src/oops/assimilation/HMatrix.h b/src/oops/assimilation/HMatrix.h index 8516314d9..4c3f3e617 100644 --- a/src/oops/assimilation/HMatrix.h +++ b/src/oops/assimilation/HMatrix.h @@ -38,13 +38,12 @@ template class HMatrix : private boost::noncopyabl public: explicit HMatrix(const CostFct_ & j): j_(j) {} - void multiply(CtrlInc_ & dx, DualVector & dy, - const bool idModel = false) const { + void multiply(CtrlInc_ & dx, DualVector & dy, const bool idModel = false) const { PostProcessor post; PostProcessorTLAD cost; for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - cost.enrollProcessor(j_.jterm(jj).setupTL(dx)); + j_.jterm(jj).setupTL(dx, cost); } j_.runTLM(dx, cost, post, idModel); diff --git a/src/oops/assimilation/HessianMatrix.h b/src/oops/assimilation/HessianMatrix.h index be6ba0f16..b50faf549 100644 --- a/src/oops/assimilation/HessianMatrix.h +++ b/src/oops/assimilation/HessianMatrix.h @@ -34,8 +34,7 @@ template class HessianMatrix : private boost::nonc typedef JqTermTLAD JqTermTLAD_; public: - explicit HessianMatrix(const CostFct_ & j, - const bool test = false); + explicit HessianMatrix(const CostFct_ & j, const bool test = false); void multiply(const CtrlInc_ & dx, CtrlInc_ & dz) const; @@ -61,12 +60,9 @@ void HessianMatrix::multiply(const CtrlInc_ & dx, CtrlInc_ & dz) con // Setup TL terms of cost function PostProcessorTLAD costtl; - JqTermTLAD_ * jqtl = j_.jb().initializeTL(); - costtl.enrollProcessor(jqtl); - unsigned iq = 0; - if (jqtl) iq = 1; + unsigned iq = j_.jb().initializeTL(costtl); for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - costtl.enrollProcessor(j_.jterm(jj).setupTL(dx)); + j_.jterm(jj).setupTL(dx, costtl); } // Run TLM @@ -82,10 +78,9 @@ void HessianMatrix::multiply(const CtrlInc_ & dx, CtrlInc_ & dz) con // Jb CtrlInc_ tmp(j_.jb()); - j_.jb().finalizeTL(jqtl, dx, dw); + j_.jb().finalizeTL(dx, dw); j_.jb().multiplyBinv(dw, tmp); - JqTermTLAD_ * jqad = j_.jb().initializeAD(dz, tmp); - costad.enrollProcessor(jqad); + j_.jb().initializeAD(dz, tmp, costad); j_.zeroAD(dw); @@ -96,13 +91,13 @@ void HessianMatrix::multiply(const CtrlInc_ & dx, CtrlInc_ & dz) con for (unsigned jj = 0; jj < j_.nterms(); ++jj) { ww.append(costtl.releaseOutputFromTL(iq+jj)); zz.append(j_.jterm(jj).multiplyCoInv(*ww.getv(jj))); - costad.enrollProcessor(j_.jterm(jj).setupAD(zz.getv(jj), dw)); + j_.jterm(jj).setupAD(zz.getv(jj), dw, costad); } // Run ADJ j_.runADJ(dw, costad); dz += dw; - j_.jb().finalizeAD(jqad); + j_.jb().finalizeAD(); if (test_) { // , where dy = Rinv H dx diff --git a/src/oops/assimilation/HtMatrix.h b/src/oops/assimilation/HtMatrix.h index 554130fd4..edf3cd547 100644 --- a/src/oops/assimilation/HtMatrix.h +++ b/src/oops/assimilation/HtMatrix.h @@ -43,7 +43,7 @@ template class HtMatrix : private boost::noncopyab PostProcessorTLAD cost; // Don't zero out dx here for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - cost.enrollProcessor(j_.jterm(jj).setupAD(dy.getv(jj), dx)); + j_.jterm(jj).setupAD(dy.getv(jj), dx, cost); } j_.runADJ(dx, cost, post, idModel); } diff --git a/src/oops/assimilation/HtRinvHMatrix.h b/src/oops/assimilation/HtRinvHMatrix.h index 15f788cd2..8bf17d817 100644 --- a/src/oops/assimilation/HtRinvHMatrix.h +++ b/src/oops/assimilation/HtRinvHMatrix.h @@ -64,7 +64,7 @@ void HtRinvHMatrix::multiply(const CtrlInc_ & dx, CtrlInc_ & dz) con // Setup TL terms of cost function PostProcessorTLAD costtl; for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - costtl.enrollProcessor(j_.jterm(jj).setupTL(dx)); + j_.jterm(jj).setupTL(dx, costtl); } // Run TLM @@ -81,7 +81,7 @@ void HtRinvHMatrix::multiply(const CtrlInc_ & dx, CtrlInc_ & dz) con for (unsigned jj = 0; jj < j_.nterms(); ++jj) { ww.append(costtl.releaseOutputFromTL(jj)); zz.append(j_.jterm(jj).multiplyCoInv(*ww.getv(jj))); - costad.enrollProcessor(j_.jterm(jj).setupAD(zz.getv(jj), dz)); + j_.jterm(jj).setupAD(zz.getv(jj), dz, costad); } // Run ADJ diff --git a/src/oops/assimilation/JqTermTLAD.h b/src/oops/assimilation/JqTermTLAD.h index 3a7a1455b..5b6215592 100644 --- a/src/oops/assimilation/JqTermTLAD.h +++ b/src/oops/assimilation/JqTermTLAD.h @@ -130,6 +130,8 @@ State & JqTermTLAD::getMxi() const { template void JqTermTLAD::doFinalizeTL(const Increment_ & dx) { Log::trace() << "JqTermTLAD::doFinalizeTL start" << std::endl; + Log::debug() << "JqTermTLAD::doFinalizeTL MPI size " << commTime_.size() << std::endl; + Log::debug() << "JqTermTLAD::doFinalizeTL MPI rank " << commTime_.rank() << std::endl; size_t mytime = commTime_.rank(); if (mytime + 1 < commTime_.size()) oops::mpi::send(commTime_, dx, mytime+1, 2468); Log::trace() << "JqTermTLAD::doFinalizeTL done" << std::endl; @@ -141,6 +143,8 @@ template void JqTermTLAD::computeModelErrorTL(Increment_ & dx) { Log::trace() << "JqTermTLAD::computeModelErrorTL start" << std::endl; // Compute x_i - M(x_{i-1}) + Log::debug() << "JqTermTLAD::computeModelErrorTL MPI size " << commTime_.size() << std::endl; + Log::debug() << "JqTermTLAD::computeModelErrorTL MPI rank " << commTime_.rank() << std::endl; size_t mytime = commTime_.rank(); if (mytime > 0) { Increment_ mxim1(dx, false); diff --git a/src/oops/assimilation/LBHessianMatrix.h b/src/oops/assimilation/LBHessianMatrix.h index 20e57c615..8c43ac515 100644 --- a/src/oops/assimilation/LBHessianMatrix.h +++ b/src/oops/assimilation/LBHessianMatrix.h @@ -41,12 +41,9 @@ template class LBHessianMatrix : private boost::no void multiply(const CtrlInc_ & dx, CtrlInc_ & dz) const { // Setup TL terms of cost function PostProcessorTLAD costtl; - JqTermTLAD_ * jqtl = j_.jb().initializeTL(); - costtl.enrollProcessor(jqtl); - unsigned iq = 0; - if (jqtl) iq = 1; + unsigned iq = j_.jb().initializeTL(costtl); for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - costtl.enrollProcessor(j_.jterm(jj).setupTL(dx)); + j_.jterm(jj).setupTL(dx, costtl); } // Run TLM @@ -62,17 +59,16 @@ template class LBHessianMatrix : private boost::no // Jb CtrlInc_ tmp(j_.jb()); - j_.jb().finalizeTL(jqtl, dx, dw); + j_.jb().finalizeTL(dx, dw); tmp = dw; - JqTermTLAD_ * jqad = j_.jb().initializeAD(dz, tmp); - costad.enrollProcessor(jqad); + j_.jb().initializeAD(dz, tmp, costad); j_.zeroAD(dw); // Jo + Jc for (unsigned jj = 0; jj < j_.nterms(); ++jj) { std::unique_ptr ww(costtl.releaseOutputFromTL(iq+jj)); std::shared_ptr zz(j_.jterm(jj).multiplyCoInv(*ww)); - costad.enrollProcessor(j_.jterm(jj).setupAD(zz, dw)); + j_.jterm(jj).setupAD(zz, dw, costad); } // Run ADJ @@ -83,7 +79,7 @@ template class LBHessianMatrix : private boost::no j_.jb().multiplyB(dw, zz); dz += zz; - j_.jb().finalizeAD(jqad); + j_.jb().finalizeAD(); } private: diff --git a/src/oops/assimilation/SaddlePointMatrix.h b/src/oops/assimilation/SaddlePointMatrix.h index 8d77bca0e..7c5f79315 100644 --- a/src/oops/assimilation/SaddlePointMatrix.h +++ b/src/oops/assimilation/SaddlePointMatrix.h @@ -20,7 +20,6 @@ #include "oops/base/PostProcessorTLAD.h" namespace oops { - template class JqTermTLAD; /// The Saddle-point matrix. /*! @@ -33,7 +32,6 @@ class SaddlePointMatrix : private boost::noncopyable { typedef ControlIncrement CtrlInc_; typedef CostFunction CostFct_; typedef SaddlePointVector SPVector_; - typedef JqTermTLAD JqTermTLAD_; public: explicit SaddlePointMatrix(const CostFct_ & j): j_(j) {} @@ -46,8 +44,7 @@ class SaddlePointMatrix : private boost::noncopyable { // ============================================================================= template -void SaddlePointMatrix::multiply(const SPVector_ & x, - SPVector_ & z) const { +void SaddlePointMatrix::multiply(const SPVector_ & x, SPVector_ & z) const { CtrlInc_ ww(j_.jb()); // The three blocks below could be done in parallel @@ -56,26 +53,24 @@ void SaddlePointMatrix::multiply(const SPVector_ & x, PostProcessorTLAD costad; j_.zeroAD(ww); z.dx(new CtrlInc_(j_.jb())); - JqTermTLAD_ * jqad = j_.jb().initializeAD(z.dx(), x.lambda().dx()); - costad.enrollProcessor(jqad); + j_.jb().initializeAD(z.dx(), x.lambda().dx(), costad); for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - costad.enrollProcessor(j_.jterm(jj).setupAD(x.lambda().getv(jj), ww)); + j_.jterm(jj).setupAD(x.lambda().getv(jj), ww, costad); } j_.runADJ(ww, costad); z.dx() += ww; // TLM block PostProcessorTLAD costtl; - JqTermTLAD_ * jqtl = j_.jb().initializeTL(); - costtl.enrollProcessor(jqtl); + j_.jb().initializeTL(costtl); for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - costtl.enrollProcessor(j_.jterm(jj).setupTL(x.dx())); + j_.jterm(jj).setupTL(x.dx(), costtl); } CtrlInc_ mdx(x.dx()); j_.runTLM(mdx, costtl); z.lambda().clear(); z.lambda().dx(new CtrlInc_(j_.jb())); - j_.jb().finalizeTL(jqtl, x.dx(), z.lambda().dx()); + j_.jb().finalizeTL(x.dx(), z.lambda().dx()); for (unsigned jj = 0; jj < j_.nterms(); ++jj) { z.lambda().append(costtl.releaseOutputFromTL(jj+1)); } From 2fb7425f35165320b935e13d40559a6cedaf316b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Wed, 31 Mar 2021 19:02:36 -0600 Subject: [PATCH 093/142] Split Jo by obs type (#1125) * ObsVector <-> ObsDataVector conversions * Pass ObsVector to ObsFilters * Added ObsDataVector test * Pass obs errors by reference * Pass post processors to cost terms * Jo by obs type * Use Observer in Observers * Cleaning * Fix merge error * Another merge error --- qg/test/testoutput/3dfgat.test | 4 +- src/CMakeLists.txt | 3 + src/oops/assimilation/CostJo.h | 156 +++++++--------------- src/oops/assimilation/CostJoType.h | 208 +++++++++++++++++++++++++++++ src/oops/base/GetValuePost.h | 125 +++++++++++++++++ src/oops/base/Observations.h | 2 +- src/oops/base/Observer.h | 175 ++++++++++++++++++++++++ src/oops/base/Observers.h | 150 +++++---------------- src/oops/runs/HofX4D.h | 26 +--- 9 files changed, 603 insertions(+), 246 deletions(-) create mode 100644 src/oops/assimilation/CostJoType.h create mode 100644 src/oops/base/GetValuePost.h create mode 100644 src/oops/base/Observer.h diff --git a/qg/test/testoutput/3dfgat.test b/qg/test/testoutput/3dfgat.test index 6af266123..fba4eda38 100644 --- a/qg/test/testoutput/3dfgat.test +++ b/qg/test/testoutput/3dfgat.test @@ -1,5 +1,5 @@ Test : CostJb : Nonlinear Jb = 0 -Test : Monitoring only: Nonlinear Jo(Stream) = 7671.34, nobs = 600, Jo/n = 12.7856, err = 4e+06 +Test : CostJo : Nonlinear Jo(Stream) = 7671.34, nobs = 600, Jo/n = 12.7856, err = 4e+06 (Monitoring only) Test : CostJo : Nonlinear Jo(Wind) = 1038.28, nobs = 600, Jo/n = 1.73047, err = 6 Test : CostJo : Nonlinear Jo(WSpeed) = 139.319, nobs = 300, Jo/n = 0.464395, err = 12 Test : CostFunction: Nonlinear J = 1177.6 @@ -13,7 +13,7 @@ Test : Min= -4.6444e+08, Max= 1.0829e+08, RMS= 1.7939e+08 Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 79.13 -Test : Monitoring only: Nonlinear Jo(Stream) = 1596, nobs = 600, Jo/n = 2.66, err = 4e+06 +Test : CostJo : Nonlinear Jo(Stream) = 1596, nobs = 600, Jo/n = 2.66, err = 4e+06 (Monitoring only) Test : CostJo : Nonlinear Jo(Wind) = 159.4, nobs = 600, Jo/n = 0.2656, err = 6 Test : CostJo : Nonlinear Jo(WSpeed) = 34.27, nobs = 300, Jo/n = 0.1142, err = 12 Test : CostFunction: Nonlinear J = 272.8 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1a66dc8c2..9d1019a75 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -18,6 +18,7 @@ oops/assimilation/CostJbState.h oops/assimilation/CostJbTotal.h oops/assimilation/CostJcDFI.h oops/assimilation/CostJo.h +oops/assimilation/CostJoType.h oops/assimilation/CostTermBase.h oops/assimilation/DRGMRESRMinimizer.h oops/assimilation/DRIPCGMinimizer.h @@ -94,6 +95,7 @@ oops/base/DolphChebyshev.h oops/base/EnsembleCovariance.h oops/base/GeneralizedDepartures.h oops/base/GeoVaLsWriter.h +oops/base/GetValuePost.h oops/base/GetValuesPost.h oops/base/LocalIncrement.cc oops/base/LocalIncrement.h @@ -119,6 +121,7 @@ oops/base/ObsEnsemble.h oops/base/ObsErrorBase.h oops/base/ObsErrors.h oops/base/Observations.h +oops/base/Observer.h oops/base/Observers.h oops/base/ObserversTLAD.h oops/base/ObserverTLAD.h diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index b9975028c..4c0b27764 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -21,19 +21,17 @@ #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/ControlIncrement.h" #include "oops/assimilation/ControlVariable.h" +#include "oops/assimilation/CostJoType.h" #include "oops/assimilation/CostTermBase.h" #include "oops/base/Departures.h" #include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" -#include "oops/base/Observers.h" #include "oops/base/ObserversTLAD.h" #include "oops/base/ObsSpaces.h" -#include "oops/base/PostBase.h" #include "oops/base/PostProcessor.h" #include "oops/base/PostProcessorTLAD.h" #include "oops/interface/Geometry.h" #include "oops/interface/Increment.h" -#include "oops/interface/ObsDataVector.h" #include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/util/DateTime.h" @@ -51,9 +49,10 @@ namespace oops { */ template class CostJo : public CostTermBase, - private boost::noncopyable { + private boost::noncopyable { typedef ControlVariable CtrlVar_; typedef ControlIncrement CtrlInc_; + typedef CostJoType JoType_; typedef Departures Departures_; typedef Observations Observations_; typedef Geometry Geometry_; @@ -61,12 +60,9 @@ template class CostJo : public CostTermBase Increment_; typedef ObsErrors ObsErrors_; typedef ObsSpaces ObsSpaces_; - typedef Observers Observers_; typedef ObserversTLAD ObserversTLAD_; typedef PostProcessor PostProc_; typedef PostProcessorTLAD PostProcTLAD_; - typedef PostBaseTLAD PostBaseTLAD_; - typedef ObsVector ObsVector_; public: /// Construct \f$ J_o\f$ from \f$ R\f$ and \f$ y_{obs}\f$. @@ -107,27 +103,19 @@ template class CostJo : public CostTermBase Rmat_; - /// Configuration for current initialize/finalize pair - std::unique_ptr currentConf_; + std::vector> jos_; - /// Gradient at first guess : \f$ R^{-1} (H(x_{fg})-y_{obs}) \f$. + /// Jo Gradient at first guess : \f$ R^{-1} (H(x_{fg})-y_{obs}) \f$. std::unique_ptr gradFG_; - /// Used for computing H(x) and running QC filters - Observers_ observers_; - - std::vector > obserrs_; // Obs errors - /// Linearized observation operators. std::shared_ptr pobstlad_; }; @@ -138,16 +126,14 @@ template CostJo::CostJo(const eckit::Configuration & joConf, const eckit::mpi::Comm & comm, const util::DateTime & winbgn, const util::DateTime & winend, const eckit::mpi::Comm & ctime) - : obsconf_(joConf), obspace_(obsconf_, comm, winbgn, winend, ctime), - yobs_(obspace_, "ObsValue"), - Rmat_(), currentConf_(), gradFG_(), observers_(obspace_, obsconf_), - pobstlad_() + : obsconf_(joConf), obspaces_(obsconf_, comm, winbgn, winend, ctime), + yobs_(obspaces_, "ObsValue"), jos_(), gradFG_() { - for (size_t jj = 0; jj < obspace_.size(); ++jj) { - /// Allocate and read initial obs error - obserrs_.emplace_back(std::make_shared(obspace_[jj], "ObsError")); + Log::trace() << "CostJo::CostJo start" << std::endl; + std::vector confs(obsconf_.getSubConfigurations()); + for (size_t jj = 0; jj < obspaces_.size(); ++jj) { + jos_.emplace_back(new JoType_(obspaces_[jj], confs[jj])); } - Log::trace() << "CostJo::CostJo done" << std::endl; } @@ -157,14 +143,12 @@ template void CostJo::initialize(const CtrlVar_ & xx, const eckit::Configuration & conf, PostProc_ & pp) { Log::trace() << "CostJo::initialize start" << std::endl; + gradFG_.reset(); - currentConf_.reset(new eckit::LocalConfiguration(conf)); - int iter = currentConf_->getInt("iteration"); - - std::shared_ptr > > - getvals(observers_.initialize(xx.obsVar(), obserrs_, iter)); + for (size_t jj = 0; jj < jos_.size(); ++jj) { + pp.enrollProcessor(jos_[jj]->initialize(xx.state().geometry(), xx.obsVar()[jj], conf)); + } - pp.enrollProcessor(getvals); Log::trace() << "CostJo::initialize done" << std::endl; } @@ -173,52 +157,35 @@ void CostJo::initialize(const CtrlVar_ & xx, const eckit::Configurat template double CostJo::finalize() { Log::trace() << "CostJo::finalize start" << std::endl; - Observations_ yeqv = observers_.finalize(); - Log::info() << "Jo Observation Equivalent:" << std::endl << yeqv - << "End Jo Observation Equivalent" << std::endl; - const int iterout = currentConf_->getInt("iteration"); - -// Sace current QC flags and obs error - const std::string obsname = "hofx" + std::to_string(iterout); - yeqv.save(obsname); - - const std::string errname = "EffectiveError" + std::to_string(iterout); - for (size_t jj = 0; jj < obserrs_.size(); ++jj) { - obserrs_[jj]->save(errname); - obserrs_[jj]->save("EffectiveError"); // Obs error covariance is looking for that for now - } -// Set observation error covariance - Rmat_.reset(new ObsErrors_(obsconf_, obspace_)); + // Obs, simulated obs and departures (held here for nice prints and diagnostics) + Observations_ yeqv(obspaces_); + Departures_ ydep(obspaces_); + + // Gradient at first guess (to define inner loop rhs) + gradFG_.reset(new Departures_(obspaces_)); -// Perturb observations according to obs error statistics - bool obspert = currentConf_->getBool("obs perturbations", false); - if (obspert) { - yobs_.perturb(*Rmat_); - Log::info() << "Perturbed observations: " << yobs_ << std::endl; + // Actual Jo computations + for (size_t jj = 0; jj < jos_.size(); ++jj) { + jos_[jj]->finalize(yobs_[jj], yeqv[jj], ydep[jj], (*gradFG_)[jj]); } -// Compute departures - Departures_ ydep(yeqv - yobs_); - Log::info() << "Jo Bias Corrected Departures:" << std::endl << ydep - << "End Jo Bias Corrected Departures" << std::endl; + // Print diagnostics + Log::info() << "Jo Observations:" << std::endl << yobs_ + << "End Jo Observations" << std::endl; -// Compute Jo - if (!gradFG_) { - gradFG_.reset(new Departures_(ydep)); - } else { - *gradFG_ = ydep; - } - Rmat_->inverseMultiply(*gradFG_); + Log::info() << "Jo Observations Equivalent:" << std::endl << yeqv + << "End Jo Observations Equivalent" << std::endl; - double zjo = this->printJo(ydep, *gradFG_); + Log::info() << "Jo Bias Corrected Departures:" << std::endl << ydep + << "End Jo Bias Corrected Departures" << std::endl; - if (currentConf_->has("diagnostics.departures")) { - const std::string depname = currentConf_->getString("diagnostics.departures"); - ydep.save(depname); + // Print Jo table + double zjo = 0.0; + for (size_t jj = 0; jj < jos_.size(); ++jj) { + zjo += jos_[jj]->printJo(ydep[jj], (*gradFG_)[jj]); } - currentConf_.reset(); Log::trace() << "CostJo::finalize done" << std::endl; return zjo; } @@ -230,7 +197,7 @@ void CostJo::initializeTraj(const CtrlVar_ & xx, const Geometry_ &, const eckit::Configuration & conf, PostProcTLAD_ & pptraj) { Log::trace() << "CostJo::initializeTraj start" << std::endl; - pobstlad_.reset(new ObserversTLAD_(obsconf_, obspace_, xx.obsVar())); + pobstlad_.reset(new ObserversTLAD_(obsconf_, obspaces_, xx.obsVar())); pptraj.enrollProcessor(pobstlad_); Log::trace() << "CostJo::initializeTraj done" << std::endl; } @@ -273,7 +240,9 @@ std::unique_ptr CostJo::multiplyCovar(const GeneralizedDepartures & v1) const { Log::trace() << "CostJo::multiplyCovar start" << std::endl; std::unique_ptr y1(new Departures_(dynamic_cast(v1))); - Rmat_->multiply(*y1); + for (size_t jj = 0; jj < jos_.size(); ++jj) { + jos_[jj]->multiplyR((*y1)[jj]); + } return std::move(y1); } @@ -284,7 +253,9 @@ std::unique_ptr CostJo::multiplyCoInv(const GeneralizedDepartures & v1) const { Log::trace() << "CostJo::multiplyCoInv start" << std::endl; std::unique_ptr y1(new Departures_(dynamic_cast(v1))); - Rmat_->inverseMultiply(*y1); + for (size_t jj = 0; jj < jos_.size(); ++jj) { + jos_[jj]->inverseMultiplyR((*y1)[jj]); + } return std::move(y1); } @@ -293,7 +264,7 @@ CostJo::multiplyCoInv(const GeneralizedDepartures & v1) const { template std::unique_ptr CostJo::newDualVector() const { Log::trace() << "CostJo::newDualVector start" << std::endl; - std::unique_ptr ydep(new Departures_(obspace_)); + std::unique_ptr ydep(new Departures_(obspaces_)); ydep->zero(); Log::trace() << "CostJo::newDualVector done" << std::endl; return std::move(ydep); @@ -317,39 +288,6 @@ void CostJo::resetLinearization() { // ----------------------------------------------------------------------------- -template -double CostJo::printJo(const Departures_ & dy, const Departures_ & grad) const { - Log::trace() << "CostJo::printJo start" << std::endl; - obspace_.printJo(dy, grad); - std::vector typeconfs = obsconf_.getSubConfigurations(); - - double zjo = 0.0; - for (std::size_t jj = 0; jj < dy.size(); ++jj) { - const double zz = 0.5 * dot_product(dy[jj], grad[jj]); - const unsigned nobs = grad[jj].nobs(); - bool isPassive = typeconfs[jj].getBool("monitoring only", false); - if (nobs > 0 && !isPassive) { - Log::test() << "CostJo : Nonlinear Jo(" << obspace_[jj].obsname() << ") = " - << zz << ", nobs = " << nobs << ", Jo/n = " << zz/nobs - << ", err = " << (*Rmat_)[jj].getRMSE() << std::endl; - } else if (nobs <= 0 && !isPassive) { - Log::test() << "CostJo : Nonlinear Jo(" << obspace_[jj].obsname() << ") = " - << zz << " --- No Observations" << std::endl; - Log::warning() << "CostJo: No Observations!!!" << std::endl; - } else if (nobs > 0 && isPassive) { - Log::test() << "Monitoring only: Nonlinear Jo(" << obspace_[jj].obsname() << ") = " - << zz << ", nobs = " << nobs << ", Jo/n = " << zz/nobs - << ", err = " << (*Rmat_)[jj].getRMSE() << std::endl; - } - if (!isPassive) zjo += zz; - } - - Log::trace() << "CostJo::printJo done" << std::endl; - return zjo; -} - -// ----------------------------------------------------------------------------- - } // namespace oops #endif // OOPS_ASSIMILATION_COSTJO_H_ diff --git a/src/oops/assimilation/CostJoType.h b/src/oops/assimilation/CostJoType.h new file mode 100644 index 000000000..9ad97f306 --- /dev/null +++ b/src/oops/assimilation/CostJoType.h @@ -0,0 +1,208 @@ +/* + * (C) Copyright 2021-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_ASSIMILATION_COSTJOTYPE_H_ +#define OOPS_ASSIMILATION_COSTJOTYPE_H_ + +#include +#include +#include + +#include + +#include "eckit/config/LocalConfiguration.h" +#include "oops/base/ObsErrorBase.h" +#include "oops/base/Observer.h" +#include "oops/base/PostBase.h" +#include "oops/interface/Geometry.h" +#include "oops/interface/ObsAuxControl.h" +#include "oops/interface/ObsSpace.h" +#include "oops/interface/ObsVector.h" +#include "oops/interface/State.h" +#include "oops/mpi/mpi.h" +#include "oops/util/DateTime.h" +#include "oops/util/Logger.h" + +namespace oops { + +// ----------------------------------------------------------------------------- + +/// Jo Cost Function +/*! + * The CostJoType class encapsulates the Jo term of the cost function for a single obs type. + */ + +template class CostJoType : private boost::noncopyable { + typedef Geometry Geometry_; + typedef ObsAuxControl ObsAuxCtrl_; + typedef ObsErrorBase ObsError_; + typedef Observer Observer_; + typedef ObsSpace ObsSpace_; + typedef ObsVector ObsVector_; + typedef PostBase> PostBase_; + typedef State State_; + + public: + /// Construct \f$ J_o\f$ from \f$ R\f$ and \f$ y_{obs}\f$. + CostJoType(ObsSpace_ &, const eckit::Configuration &); + + /// Destructor + virtual ~CostJoType() {} + + /// Initialize \f$ J_o\f$ before starting the integration of the model. + std::shared_ptr initialize(const Geometry_ &, const ObsAuxCtrl_ &, + const eckit::Configuration &); + + /// Finalize \f$ J_o\f$ after the integration of the model. + void finalize(ObsVector_ &, ObsVector_ &, ObsVector_ &, ObsVector_ &); + + /// Multiply by \f$ R\f$ and \f$ R^{-1}\f$. + void multiplyR(ObsVector_ &) const; + void inverseMultiplyR(ObsVector_ &) const; + + /// Print Jo + double printJo(const ObsVector_ &, const ObsVector_ &) const; + + private: + eckit::LocalConfiguration obsconf_; + ObsSpace_ & obspace_; + ObsVector_ obserr_; // Obs errors (should come from R matrix) + + std::unique_ptr Rmat_; + + /// Configuration for current initialize/finalize pair + std::unique_ptr currentConf_; + int iter_; + + /// Used for computing H(x) and running QC filters + Observer_ observer_; +}; + +// ============================================================================= + +template +CostJoType::CostJoType(ObsSpace_ & obspace, const eckit::Configuration & conf) + : obsconf_(conf), obspace_(obspace), + obserr_(obspace_, "ObsError"), + Rmat_(), currentConf_(), iter_(0), observer_(obspace_, obsconf_) +{ + Log::trace() << "CostJoType::CostJoType done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +std::shared_ptr > > +CostJoType::initialize(const Geometry_ & geom, const ObsAuxCtrl_ & ybias, + const eckit::Configuration & conf) { + Log::trace() << "CostJoType::initialize start" << std::endl; + + currentConf_.reset(new eckit::LocalConfiguration(conf)); + iter_ = currentConf_->getInt("iteration"); + + std::shared_ptr getvals = observer_.initialize(geom, ybias, obserr_, iter_); + + Log::trace() << "CostJoType::initialize done" << std::endl; + return getvals; +} + +// ----------------------------------------------------------------------------- + +template +void CostJoType::finalize(ObsVector_ & yobs, ObsVector_ & yeqv, + ObsVector_ & ydep, ObsVector_ & grad) { + Log::trace() << "CostJoType::finalize start" << std::endl; + +// Get simulated obs values from observer + observer_.finalize(yeqv); + +// Sace current simulated obs, QC flags and obs error + const std::string obsname = "hofx" + std::to_string(iter_); + yeqv.save(obsname); + + const std::string errname = "EffectiveError" + std::to_string(iter_); + obserr_.save(errname); + obserr_.save("EffectiveError"); // Obs error covariance is looking for that for now + +// Set observation error covariance + const eckit::LocalConfiguration rconf(obsconf_, "obs error"); + Rmat_ = ObsErrorFactory::create(rconf, obspace_); + +// Perturb observations according to obs error statistics + bool obspert = currentConf_->getBool("obs perturbations", false); + Log::debug() << "CostJoType::finalize obspert = " << obspert << std::endl; + if (obspert) { + ObsVector_ ypert(obspace_); + Rmat_->randomize(ypert); + yobs += ypert; + Log::info() << "Perturbed observations: " << yobs << std::endl; + } + Log::debug() << "CostJoType::finalize Observations: " << yobs << std::endl; + +// Compute departures and Jo gradient + ydep = yeqv; + ydep -= yobs; + grad = ydep; + Rmat_->inverseMultiply(grad); + +// Save departures for diagnostics if required + if (currentConf_->has("diagnostics.departures")) { + const std::string depname = currentConf_->getString("diagnostics.departures"); + ydep.save(depname); + } + + currentConf_.reset(); + Log::trace() << "CostJoType::finalize done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +double CostJoType::printJo(const ObsVector_ & dy, const ObsVector_ & grad) const { + Log::trace() << "CostJoType::printJo start" << std::endl; + + double zjo = 0.0; + const unsigned nobs = grad.nobs(); + Log::test() << "CostJo : Nonlinear Jo(" << obspace_.obsname() << ") = "; + + if (nobs > 0) { + zjo = 0.5 * dot_product(dy, grad); + const double err = Rmat_->getRMSE(); + Log::test() << zjo << ", nobs = " << nobs << ", Jo/n = " << zjo/nobs << ", err = " << err; + } else { + Log::test() << zjo << " --- No Observations"; + } + + if (obsconf_.getBool("monitoring only", false)) { + Log::test() << " (Monitoring only)"; + zjo = 0.0; + } + Log::test() << std::endl; + + Log::trace() << "CostJoType::printJo done" << std::endl; + return zjo; +} + +// ----------------------------------------------------------------------------- + +template +void CostJoType::multiplyR(ObsVector_ & dy) const { + Rmat_->multiply(dy); +} + +// ----------------------------------------------------------------------------- + +template +void CostJoType::inverseMultiplyR(ObsVector_ & dy) const { + Rmat_->inverseMultiply(dy); +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_ASSIMILATION_COSTJOTYPE_H_ diff --git a/src/oops/base/GetValuePost.h b/src/oops/base/GetValuePost.h new file mode 100644 index 000000000..b9c2b4a24 --- /dev/null +++ b/src/oops/base/GetValuePost.h @@ -0,0 +1,125 @@ +/* + * (C) Copyright 2020-2020 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_BASE_GETVALUEPOST_H_ +#define OOPS_BASE_GETVALUEPOST_H_ + +#include +#include +#include +#include +#include + +#include "oops/base/PostBase.h" +#include "oops/base/Variables.h" +#include "oops/interface/Geometry.h" +#include "oops/interface/GeoVaLs.h" +#include "oops/interface/GetValues.h" +#include "oops/interface/Locations.h" +#include "oops/interface/State.h" +#include "oops/util/DateTime.h" +#include "oops/util/Duration.h" +#include "oops/util/Logger.h" + +namespace eckit { + class Configuration; +} + +namespace oops { + +/// \brief Fills GeoVaLs with requested variables at requested locations during model run +template +class GetValuePost : public PostBase> { + typedef Geometry Geometry_; + typedef GeoVaLs GeoVaLs_; + typedef GetValues GetValues_; + typedef Locations Locations_; + typedef State State_; + + public: +/// \brief Saves Locations and Variables to be processed + GetValuePost(const eckit::Configuration &, const Geometry_ &, + const util::DateTime &, const util::DateTime &, + const Locations_ &, const Variables &); + +/// \brief Returns geovals filled in during the model run + std::unique_ptr releaseGeoVaLs(); + + private: +/// \brief initialization before model run: sets up GetValues and allocate GeoVaLs + void doInitialize(const State_ &, const util::DateTime &, const util::Duration &) override; +/// \brief called at each model step: fill in GeoVaLs for the current time slot + void doProcessing(const State_ &) override; + +// Data + util::DateTime winbgn_; /// Begining of assimilation window + util::DateTime winend_; /// End of assimilation window + util::Duration hslot_; /// Half time slot + + const Locations_ & locations_; /// locations of observations + const Variables geovars_; /// Variables needed from model + GetValues_ getvals_; /// GetValues used to fill in GeoVaLs + std::unique_ptr geovals_; /// GeoVaLs that are filled in + bool initialized_; +}; + +// ----------------------------------------------------------------------------- + +template +GetValuePost::GetValuePost(const eckit::Configuration & conf, const Geometry_ & geom, + const util::DateTime & bgn, const util::DateTime & end, + const Locations_ & locations, const Variables & vars) + : PostBase(), + winbgn_(bgn), winend_(end), hslot_(), locations_(locations), geovars_(vars), + getvals_(geom, locations_, conf), geovals_(), initialized_(false) +{ + Log::trace() << "GetValuePost::GetValuePost" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void GetValuePost::doInitialize(const State_ & xx, const util::DateTime &, + const util::Duration & tstep) { + Log::trace() << "GetValuePost::doInitialize start" << std::endl; + hslot_ = tstep/2; + + geovals_.reset(new GeoVaLs_(locations_, geovars_)); + + initialized_ = true; + Log::trace() << "GetValuePost::doInitialize done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void GetValuePost::doProcessing(const State_ & xx) { + Log::trace() << "GetValuePost::doProcessing start" << std::endl; + ASSERT(initialized_); + util::DateTime t1 = std::max(xx.validTime()-hslot_, winbgn_); + util::DateTime t2 = std::min(xx.validTime()+hslot_, winend_); + +// Get state variables at obs locations + getvals_.fillGeoVaLs(xx, t1, t2, *geovals_); + Log::trace() << "GetValuePost::doProcessing done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +std::unique_ptr> GetValuePost::releaseGeoVaLs() { + Log::trace() << "GetValuePost::releaseGeoVaLs" << std::endl; + initialized_ = false; + // Release ownership of GeoVaLs + return std::move(geovals_); +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_BASE_GETVALUEPOST_H_ diff --git a/src/oops/base/Observations.h b/src/oops/base/Observations.h index 55a004fd3..743faabf6 100644 --- a/src/oops/base/Observations.h +++ b/src/oops/base/Observations.h @@ -70,7 +70,7 @@ template class Observations : public util::Printable { Observations & operator*=(const double); /// Perturbations - void perturb(const ObsErrors_ &); + void perturb(const ObsErrors_ &); // to be removed private: void print(std::ostream &) const; diff --git a/src/oops/base/Observer.h b/src/oops/base/Observer.h new file mode 100644 index 000000000..f62de8388 --- /dev/null +++ b/src/oops/base/Observer.h @@ -0,0 +1,175 @@ +/* + * (C) Copyright 2020 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ +#ifndef OOPS_BASE_OBSERVER_H_ +#define OOPS_BASE_OBSERVER_H_ + +#include +#include +#include + +#include "eckit/config/LocalConfiguration.h" + +#include "oops/base/GetValuePost.h" +#include "oops/base/ObsFilters.h" +#include "oops/base/Variables.h" +#include "oops/interface/Geometry.h" +#include "oops/interface/GeoVaLs.h" +#include "oops/interface/Locations.h" +#include "oops/interface/ObsAuxControl.h" +#include "oops/interface/ObsDataVector.h" +#include "oops/interface/ObsDiagnostics.h" +#include "oops/interface/ObsOperator.h" +#include "oops/interface/ObsSpace.h" +#include "oops/interface/ObsVector.h" +#include "oops/util/Logger.h" +#include "oops/util/parameters/Parameter.h" +#include "oops/util/parameters/Parameters.h" +#include "oops/util/parameters/RequiredParameter.h" + +namespace oops { + +template +class ObserverParameters : public Parameters { + OOPS_CONCRETE_PARAMETERS(ObserverParameters, Parameters) + + public: + oops::RequiredParameter obsOperator{"obs operator", this}; + oops::Parameter>> obsFilters{"obs filters", {}, this}; +}; + +// ----------------------------------------------------------------------------- + +/// \brief Computes observation operator, applying bias correction and QC filters +template +class Observer { + typedef Geometry Geometry_; + typedef GeoVaLs GeoVaLs_; + typedef GetValuePost GetValPost_; + typedef Locations Locations_; + typedef ObsAuxControl ObsAuxCtrl_; + typedef ObsDataVector ObsDataInt_; + typedef ObsDiagnostics ObsDiags_; + typedef ObsFilters ObsFilters_; + typedef ObsOperator ObsOperator_; + typedef ObsSpace ObsSpace_; + typedef ObsVector ObsVector_; + + public: +/// \brief Initializes ObsOperators, Locations, and QC data + Observer(const ObsSpace_ &, const eckit::Configuration &); + +/// \brief Initializes variables, obs bias, obs filters (could be different for +/// different iterations + std::shared_ptr initialize(const Geometry_ &, const ObsAuxCtrl_ &, + ObsVector_ &, const int iter = 0); + +/// \brief Computes H(x) from the filled in GeoVaLs + void finalize(ObsVector_ &); + + private: + eckit::LocalConfiguration obsconfig_; + const ObsSpace_ & obspace_; // ObsSpace used in H(x) + std::unique_ptr obsop_; // Obs operator + std::unique_ptr locations_; // locations + const ObsAuxCtrl_ * ybias_; // Obs bias + std::unique_ptr filters_; // QC filters + std::shared_ptr getvals_; // Postproc passed to the model during integration. + std::shared_ptr qcflags_; // QC flags (should not be a pointer) + int iterout_; // Outer iteration + bool initialized_; +}; + +// ----------------------------------------------------------------------------- + +template +Observer::Observer(const ObsSpace_ & obspace, const eckit::Configuration & config) + : obsconfig_(config), obspace_(obspace), obsop_(), locations_(), + ybias_(nullptr), filters_(), qcflags_(), iterout_(0), initialized_(false) +{ + Log::trace() << "Observer::Observer start" << std::endl; + ObserverParameters observerParams; + observerParams.deserialize(config); + /// Set up observation operators + obsop_.reset(new ObsOperator_(obspace_, observerParams.obsOperator)); + qcflags_.reset(new ObsDataInt_(obspace_, obspace_.obsvariables())); + + Log::trace() << "Observer::Observer done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +std::shared_ptr> +Observer::initialize(const Geometry_ & geom, const ObsAuxCtrl_ & ybias, + ObsVector_ & obserr, const int iter) { +// could pass state (or even geom) and obsbias instead of control var (easier for HofX?) + iterout_ = iter; + ybias_ = &ybias; + + ObserverParameters observerParams; + observerParams.deserialize(obsconfig_); + /// Set up QC filters and run preprocess + filters_.reset(new ObsFilters_(obspace_, observerParams.obsFilters, + qcflags_, obserr, iterout_)); + filters_->preProcess(); + + locations_.reset(new Locations_(obsop_->locations())); + + /// Set up variables that will be requested from the model + Variables geovars; + geovars += obsop_->requiredVars(); + geovars += ybias_->requiredVars(); + geovars += filters_->requiredVars(); + + eckit::LocalConfiguration gvconf = obsconfig_.getSubConfiguration("get values"); + + getvals_.reset(new GetValPost_(gvconf, geom, obspace_.windowStart(), + obspace_.windowEnd(), *locations_, geovars)); + + initialized_ = true; + Log::trace() << "Observer::initialize done" << std::endl; + return getvals_; +} + +// ----------------------------------------------------------------------------- + +template +void Observer::finalize(ObsVector_ & yobsim) { + oops::Log::trace() << "Observer::finalize start" << std::endl; + ASSERT(initialized_); + + // GetValues releases GeoVaLs, Observer takes ownership + std::unique_ptr geovals = getvals_->releaseGeoVaLs(); + + /// Call prior filters + filters_->priorFilter(*geovals); + + /// Setup diagnostics + Variables vars; + vars += filters_->requiredHdiagnostics(); + vars += ybias_->requiredHdiagnostics(); + ObsDiags_ ydiags(obspace_, *locations_, vars); + + /// Compute H(x) + obsop_->simulateObs(*geovals, yobsim, *ybias_, ydiags); + + /// Call posterior filters + filters_->postFilter(yobsim, ydiags); + + /// Save flags (for diagnostics use) + const std::string qcname = "EffectiveQC" + std::to_string(iterout_); + qcflags_->save(qcname); + + initialized_ = false; + oops::Log::trace() << "Observer::finalize done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_BASE_OBSERVER_H_ diff --git a/src/oops/base/Observers.h b/src/oops/base/Observers.h index 2dc49f4a0..dce1bf161 100644 --- a/src/oops/base/Observers.h +++ b/src/oops/base/Observers.h @@ -4,6 +4,7 @@ * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. */ + #ifndef OOPS_BASE_OBSERVERS_H_ #define OOPS_BASE_OBSERVERS_H_ @@ -13,59 +14,32 @@ #include "eckit/config/LocalConfiguration.h" -#include "oops/base/GetValuesPost.h" #include "oops/base/ObsAuxControls.h" #include "oops/base/Observations.h" -#include "oops/base/ObsFilters.h" +#include "oops/base/Observer.h" #include "oops/base/ObsSpaces.h" -#include "oops/base/Variables.h" -#include "oops/interface/GeoVaLs.h" -#include "oops/interface/Locations.h" -#include "oops/interface/ObsDataVector.h" -#include "oops/interface/ObsDiagnostics.h" -#include "oops/interface/ObsOperator.h" +#include "oops/base/PostProcessor.h" +#include "oops/interface/Geometry.h" #include "oops/interface/ObsVector.h" -#include "oops/util/ConfigFunctions.h" +#include "oops/interface/State.h" #include "oops/util/Logger.h" -#include "oops/util/parameters/Parameter.h" -#include "oops/util/parameters/Parameters.h" -#include "oops/util/parameters/RequiredParameter.h" namespace oops { -template -class ObserversParameters : public Parameters { - OOPS_CONCRETE_PARAMETERS(ObserversParameters, Parameters) - - public: - oops::RequiredParameter obsOperator{"obs operator", this}; - oops::Parameter>> obsFilters{"obs filters", {}, this}; -}; - // ----------------------------------------------------------------------------- /// \brief Computes observation operator (from GeoVaLs), applies bias correction /// and runs QC filters template class Observers { - typedef GeoVaLs GeoVaLs_; - typedef GetValuesPost GetValuesPost_; - typedef Locations Locations_; - typedef ObsAuxControls ObsAuxCtrls_; - typedef ObsDiagnostics ObsDiags_; - typedef Observations Observations_; - typedef ObsFilters ObsFilters_; - typedef ObsOperator ObsOperator_; - typedef ObsSpaces ObsSpaces_; - typedef std::vector>> ObsVectors_; - template using ObsData_ = ObsDataVector; - template using ObsDataVec_ = std::vector>>; - - typedef std::vector> GeoVaLsVec_; - typedef std::vector> LocationsVec_; - typedef std::vector> ObsFiltersVec_; - typedef std::vector> ObsOperatorVec_; - typedef std::vector VariablesVec_; + typedef Geometry Geometry_; + typedef ObsAuxControls ObsAuxCtrls_; + typedef Observations Observations_; + typedef Observer Observer_; + typedef ObsSpaces ObsSpaces_; + typedef ObsVector ObsVector_; + typedef State State_; + typedef PostProcessor PostProc_; public: /// \brief Initializes ObsOperators, Locations, and QC data @@ -73,112 +47,60 @@ class Observers { /// \brief Initializes variables, obs bias, obs filters (could be different for /// different iterations - std::shared_ptr > > initialize(const ObsAuxCtrls_ &, - ObsVectors_ &, const int iter = 0); + void initialize(const Geometry_ &, const ObsAuxCtrls_ &, PostProc_ &); /// \brief Computes H(x) from the filled in GeoVaLs - Observations_ finalize(); + void finalize(Observations_ &); private: - eckit::LocalConfiguration obsconfig_; - const ObsSpaces_ & obspaces_; // ObsSpaces used in H(x) - ObsOperatorVec_ obsops_; // Obs operators - LocationsVec_ locations_; // locations (made non local by GetValuesPost) - const ObsAuxCtrls_ * ybias_; // Obs bias - ObsFiltersVec_ filters_; // QC filters - std::shared_ptr getvals_; // Postproc passed to the model during integration. - ObsDataVec_ qcflags_; // QC flags - int iterout_; // Outer iteration - bool initialized_; + const ObsSpaces_ & obspaces_; // ObsSpaces used in H(x) + std::vector> observers_; + std::vector> obserrs_; }; // ----------------------------------------------------------------------------- template -Observers::Observers(const ObsSpaces_ & obspaces, - const eckit::Configuration & config) : - obsconfig_(config), obspaces_(obspaces), obsops_(), locations_(), - ybias_(nullptr), filters_(), iterout_(0), initialized_(false) +Observers::Observers(const ObsSpaces_ & obspaces, const eckit::Configuration & config) + : obspaces_(obspaces) { + Log::trace() << "Observers::Observers start" << std::endl; + std::vector obsconfs = config.getSubConfigurations(); for (size_t jj = 0; jj < obspaces_.size(); ++jj) { - ObserversParameters observerParams; - observerParams.deserialize(obsconfs[jj]); - /// Set up observation operators - obsops_.emplace_back(new ObsOperator_(obspaces_[jj], observerParams.obsOperator)); - locations_.emplace_back(new Locations_(obsops_[jj]->locations())); - qcflags_.emplace_back(new ObsData_(obspaces_[jj], obspaces_[jj].obsvariables())); + observers_.emplace_back(new Observer_(obspaces_[jj], obsconfs[jj])); } - Log::trace() << "Observers constructed" << std::endl; + Log::trace() << "Observers::Observers done" << std::endl; } // ----------------------------------------------------------------------------- template -std::shared_ptr > > -Observers::initialize(const ObsAuxCtrls_ & obsaux, ObsVectors_ & obserrs, - const int iter) { - std::vector obsconfs = obsconfig_.getSubConfigurations(); - iterout_ = iter; - ybias_ = &obsaux; - filters_.clear(); - VariablesVec_ geovars(obspaces_.size()); // variables required from the model - for (size_t jj = 0; jj < obspaces_.size(); ++jj) { - ObserversParameters observerParams; - observerParams.deserialize(obsconfs[jj]); - /// Set up QC filters and run preprocess - filters_.emplace_back(new ObsFilters_(obspaces_[jj], observerParams.obsFilters, - qcflags_[jj], *obserrs[jj], iterout_)); - filters_[jj]->preProcess(); - - /// Set up variables requested from the model - geovars[jj] += obsops_[jj]->requiredVars(); - geovars[jj] += (*ybias_)[jj].requiredVars(); - geovars[jj] += filters_[jj]->requiredVars(); - } - - std::vector gvconfs - = util::vectoriseAndFilter(obsconfig_, "get values"); +void Observers::initialize(const Geometry_ & geom, const ObsAuxCtrls_ & obsaux, + PostProc_ & pp) { + Log::trace() << "Observers::initialize start" << std::endl; - getvals_.reset(new GetValuesPost_(obspaces_, locations_, geovars, gvconfs)); + for (size_t jj = 0; jj < observers_.size(); ++jj) { + obserrs_.emplace_back(new ObsVector_(obspaces_[jj], "ObsError")); + pp.enrollProcessor(observers_[jj]->initialize(geom, obsaux[jj], *obserrs_[jj])); + } - initialized_ = true; Log::trace() << "Observers::initialize done" << std::endl; - return getvals_; } // ----------------------------------------------------------------------------- template -Observations Observers::finalize() { +void Observers::finalize(Observations_ & yobs) { oops::Log::trace() << "Observers::finalize start" << std::endl; - ASSERT(initialized_); - - const GeoVaLsVec_ & geovals(getvals_->geovals()); - - Observations yobs(obspaces_); - for (size_t jj = 0; jj < obspaces_.size(); ++jj) { - /// call prior filters - filters_[jj]->priorFilter(*geovals[jj]); - /// compute H(x) - oops::Variables vars; - vars += filters_[jj]->requiredHdiagnostics(); - vars += (*ybias_)[jj].requiredHdiagnostics(); - ObsDiags_ ydiags(obspaces_[jj], *locations_[jj], vars); - obsops_[jj]->simulateObs(*geovals[jj], yobs[jj], (*ybias_)[jj], ydiags); - /// call posterior filters - filters_[jj]->postFilter(yobs[jj], ydiags); - } - initialized_ = false; -// Save flags for diagnostics - const std::string qcname = "EffectiveQC" + std::to_string(iterout_); - for (const auto & qcflag : qcflags_) { - qcflag->save(qcname); + for (size_t jj = 0; jj < observers_.size(); ++jj) { + observers_[jj]->finalize(yobs[jj]); + obserrs_[jj]->save("EffectiveError"); // Obs error covariance is looking for that for now } + obserrs_.clear(); oops::Log::trace() << "Observers::finalize done" << std::endl; - return yobs; } // ----------------------------------------------------------------------------- diff --git a/src/oops/runs/HofX4D.h b/src/oops/runs/HofX4D.h index 1b4152331..4eec69fb1 100644 --- a/src/oops/runs/HofX4D.h +++ b/src/oops/runs/HofX4D.h @@ -12,9 +12,7 @@ #ifndef OOPS_RUNS_HOFX4D_H_ #define OOPS_RUNS_HOFX4D_H_ -#include #include -#include #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" @@ -30,8 +28,6 @@ #include "oops/interface/Geometry.h" #include "oops/interface/Model.h" #include "oops/interface/ModelAuxControl.h" -#include "oops/interface/ObsDataVector.h" -#include "oops/interface/ObsVector.h" #include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" @@ -53,9 +49,7 @@ template class HofX4D : public Application { typedef ObsErrors ObsErrors_; typedef Observers Observers_; typedef ObsSpaces ObsSpaces_; - typedef ObsVector ObsVector_; typedef State State_; - typedef ObsDataVector ObsData_; public: // ----------------------------------------------------------------------------- @@ -108,25 +102,17 @@ template class HofX4D : public Application { ObsSpaces_ obspaces(obsConfig, this->getComm(), winbgn, winend); ObsAux_ obsaux(obspaces, obsConfig); -// Setup and run observer +// Setup and initialize observer Observers_ hofx(obspaces, obsConfig); - - std::vector> obserrs; - for (size_t jj = 0; jj < obspaces.size(); ++jj) { - obserrs.emplace_back(std::make_shared(obspaces[jj], "ObsError")); - } + hofx.initialize(geometry, obsaux, post); // run the model and compute H(x) - std::shared_ptr > > getvals(hofx.initialize(obsaux, obserrs)); - post.enrollProcessor(getvals); model.forecast(xx, moderr, flength, post); - - Observations_ yobs = hofx.finalize(); - for (size_t jj = 0; jj < obserrs.size(); ++jj) { - obserrs[jj]->save("EffectiveError"); // Obs error covariance is looking for that for now - } - Log::test() << "Final state: " << xx << std::endl; + +// Get observations from observer + Observations_ yobs(obspaces); + hofx.finalize(yobs); Log::test() << "H(x): " << std::endl << yobs << "End H(x)" << std::endl; // Perturb H(x) if needed (can be used for generating obs in OSSE: perturbed H(x) could be saved From 3000219aac2a62ad6055d818fe739bbe44f2cd9a Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Fri, 2 Apr 2021 08:49:20 -0600 Subject: [PATCH 094/142] Change ObsLocalization interface (#1113) * start on refactoring obslocalization problem with QC: pack saves only non-qcd out obs; obsloc does all obs * refactor to fix qc issues * use GeometryIterator instead of Point in ObsLocalization * add a primitive test for ObsLocalization * change ObsLoc interface (closer to what would be needed to move search for local obs into ObsLoc) * improve ObsLoc test: now looping over all of Geometry --- l95/src/lorenz95/ObsLocGC99.cc | 23 ++-- l95/src/lorenz95/ObsLocGC99.h | 25 ++-- l95/test/CMakeLists.txt | 6 + l95/test/executables/TestObsLocalization.cc | 17 +++ l95/test/testinput/getkf.yaml | 8 +- l95/test/testinput/getkf_offline_hofx.yaml | 8 +- l95/test/testinput/interfaces.yaml | 3 + l95/test/testinput/letkf.yaml | 8 +- l95/test/testinput/letkfGSI.yaml | 8 +- l95/test/testinput/letkf_noobs.yaml | 8 +- l95/test/testinput/letkf_qc.yaml | 8 +- l95/test/testoutput/letkf_qc.test | 18 +-- qg/test/testinput/letkf.yaml | 12 +- src/CMakeLists.txt | 3 +- src/oops/assimilation/GETKFSolver.h | 17 ++- src/oops/assimilation/LETKFSolver.h | 18 ++- src/oops/assimilation/LETKFSolverGSI.h | 8 +- src/oops/base/Departures.h | 8 ++ src/oops/base/ObsErrors.h | 32 ++--- src/oops/base/ObsLocalizationBase.h | 87 ++++++------- src/oops/base/ObsLocalizations.h | 84 +++++++++++++ src/oops/base/ObsSpaces.h | 2 +- src/oops/generic/LocalObsErrorDiag.h | 73 ----------- src/oops/generic/instantiateObsErrorFactory.h | 2 - src/oops/interface/ObsLocalization.h | 50 ++++---- src/test/interface/ObsLocalization.h | 114 ++++++++++++++++++ 26 files changed, 407 insertions(+), 243 deletions(-) create mode 100644 l95/test/executables/TestObsLocalization.cc create mode 100644 src/oops/base/ObsLocalizations.h delete mode 100644 src/oops/generic/LocalObsErrorDiag.h create mode 100644 src/test/interface/ObsLocalization.h diff --git a/l95/src/lorenz95/ObsLocGC99.cc b/l95/src/lorenz95/ObsLocGC99.cc index d2d18b0ac..7a97ddcd4 100644 --- a/l95/src/lorenz95/ObsLocGC99.cc +++ b/l95/src/lorenz95/ObsLocGC99.cc @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020 UCAR + * (C) Copyright 2020-2021 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -7,9 +7,11 @@ #include "lorenz95/ObsLocGC99.h" +#include + #include "eckit/config/Configuration.h" -#include "eckit/exception/Exceptions.h" +#include "lorenz95/L95Traits.h" #include "lorenz95/ObsTableView.h" #include "lorenz95/ObsVec1D.h" @@ -19,27 +21,24 @@ // ----------------------------------------------------------------------------- namespace lorenz95 { -static oops::ObsLocalizationMaker> makerGC_("Gaspari-Cohn"); + +static oops::ObsLocalizationMaker> makerGC_("Gaspari-Cohn"); // ----------------------------------------------------------------------------- ObsLocGC99::ObsLocGC99(const eckit::Configuration & config, const ObsTableView & obsdb) - : obsdb_(obsdb), - rscale_(config.getDouble("lengthscale")) + : rscale_(config.getDouble("lengthscale")), obsdb_(obsdb) { } // ----------------------------------------------------------------------------- -void ObsLocGC99::multiply(ObsVec1D & dy) const { +void ObsLocGC99::computeLocalization(const Iterator &, ObsVec1D & result) const { const std::vector & obsdist = obsdb_.obsdist(); double missing = util::missingValue(missing); - for (unsigned int ii=0; ii < dy.nobs(); ++ii) { - if (dy[ii] != missing) { - double gc = oops::gc99(obsdist[ii]/rscale_); - dy[ii] = dy[ii]*gc; - } + for (unsigned int ii=0; ii < obsdb_.nobs(); ++ii) { + result[ii] = oops::gc99(obsdist[ii]/rscale_); } } diff --git a/l95/src/lorenz95/ObsLocGC99.h b/l95/src/lorenz95/ObsLocGC99.h index 3b3cb7777..09d794dbb 100644 --- a/l95/src/lorenz95/ObsLocGC99.h +++ b/l95/src/lorenz95/ObsLocGC99.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020 UCAR + * (C) Copyright 2020-2021 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -9,34 +9,31 @@ #define LORENZ95_OBSLOCGC99_H_ #include -#include -#include -#include #include "eckit/config/Configuration.h" -#include "oops/util/DateTime.h" #include "oops/util/Printable.h" -#include "lorenz95/L95Traits.h" - // Forward declarations namespace lorenz95 { + class Iterator; + class ObsTableView; class ObsVec1D; -/// ObsLocalization matrix for Lorenz 95 model. - -// ----------------------------------------------------------------------------- +/// Observation space localization for Lorenz 95 model (Gaspari-Cohn) class ObsLocGC99: public util::Printable { public: - static const std::string classname() {return "lorenz95::ObsLocGC99";} - ObsLocGC99(const eckit::Configuration &, const ObsTableView &); - void multiply(ObsVec1D &) const; + + /// compute localization and save in \p obsvector + void computeLocalization(const Iterator &, ObsVec1D & obsvector) const; private: void print(std::ostream &) const override; - const ObsTableView & obsdb_; + + /// Gaspari-Cohn localization distance (localization goes to zero at rscale_) const double rscale_; + + const ObsTableView & obsdb_; }; // ----------------------------------------------------------------------------- } // namespace lorenz95 diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index 67747bd9c..668ae6945 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -332,6 +332,12 @@ ecbuild_add_test( TARGET test_l95_localobsspace LIBS lorenz95 TEST_DEPENDS test_l95_truth ) +ecbuild_add_test( TARGET test_l95_obslocalization + SOURCES executables/TestObsLocalization.cc + ARGS "testinput/interfaces.yaml" + LIBS lorenz95 + TEST_DEPENDS test_l95_truth ) + ecbuild_add_test( TARGET test_l95_obsvector SOURCES executables/TestObsVector.cc ARGS "testinput/interfaces.yaml" diff --git a/l95/test/executables/TestObsLocalization.cc b/l95/test/executables/TestObsLocalization.cc new file mode 100644 index 000000000..9dc944582 --- /dev/null +++ b/l95/test/executables/TestObsLocalization.cc @@ -0,0 +1,17 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "lorenz95/L95Traits.h" +#include "oops/runs/Run.h" +#include "test/interface/ObsLocalization.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + test::ObsLocalization tests; + return run.execute(tests); +} + diff --git a/l95/test/testinput/getkf.yaml b/l95/test/testinput/getkf.yaml index cfebf9fdf..3735507f3 100644 --- a/l95/test/testinput/getkf.yaml +++ b/l95/test/testinput/getkf.yaml @@ -20,10 +20,10 @@ background: observations: - obs error: - covariance model: localized diagonal - localization: - localization method: Gaspari-Cohn - lengthscale: .1 + covariance model: diagonal + obs localization: + localization method: Gaspari-Cohn + lengthscale: .1 obs space: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obs operator: {} diff --git a/l95/test/testinput/getkf_offline_hofx.yaml b/l95/test/testinput/getkf_offline_hofx.yaml index 9fbc4e244..844aadd3a 100644 --- a/l95/test/testinput/getkf_offline_hofx.yaml +++ b/l95/test/testinput/getkf_offline_hofx.yaml @@ -20,10 +20,10 @@ background: observations: - obs error: - covariance model: localized diagonal - localization: - localization method: Gaspari-Cohn - lengthscale: .1 + covariance model: diagonal + obs localization: + localization method: Gaspari-Cohn + lengthscale: .1 obs space: obsdatain: Data/l95.getkf.2010-01-02T00:00:00Z.obt obs operator: {} diff --git a/l95/test/testinput/interfaces.yaml b/l95/test/testinput/interfaces.yaml index 060de9617..68d9b929b 100644 --- a/l95/test/testinput/interfaces.yaml +++ b/l95/test/testinput/interfaces.yaml @@ -113,6 +113,9 @@ observations: relative tolerance: 0.0 covariance: standard_deviation: 0.5 + obs localization: + lengthscale: 0.2 + localization method: Gaspari-Cohn rms ref: 8.3207407741318846 reference nobs: 160 tolerance: 1.0e-10 diff --git a/l95/test/testinput/letkf.yaml b/l95/test/testinput/letkf.yaml index 51ed5ec16..15d5a4159 100644 --- a/l95/test/testinput/letkf.yaml +++ b/l95/test/testinput/letkf.yaml @@ -20,10 +20,10 @@ background: observations: - obs error: - covariance model: localized diagonal - localization: - localization method: Gaspari-Cohn - lengthscale: .1 + covariance model: diagonal + obs localization: + localization method: Gaspari-Cohn + lengthscale: .1 obs space: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obsdataout: Data/l95.letkf.2010-01-02T00:00:00Z.obt diff --git a/l95/test/testinput/letkfGSI.yaml b/l95/test/testinput/letkfGSI.yaml index 5358a34dd..598dff73f 100644 --- a/l95/test/testinput/letkfGSI.yaml +++ b/l95/test/testinput/letkfGSI.yaml @@ -20,10 +20,10 @@ background: observations: - obs error: - covariance model: localized diagonal - localization: - localization method: Gaspari-Cohn - lengthscale: .1 + covariance model: diagonal + obs localization: + localization method: Gaspari-Cohn + lengthscale: .1 obs space: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obsdataout: Data/l95.letkf_gsi.2010-01-02T00:00:00Z.obt diff --git a/l95/test/testinput/letkf_noobs.yaml b/l95/test/testinput/letkf_noobs.yaml index fb9646a67..d5bd9a7f2 100644 --- a/l95/test/testinput/letkf_noobs.yaml +++ b/l95/test/testinput/letkf_noobs.yaml @@ -20,10 +20,10 @@ background: observations: - obs error: - covariance model: localized diagonal - localization: - localization method: Gaspari-Cohn - lengthscale: .1 + covariance model: diagonal + obs localization: + localization method: Gaspari-Cohn + lengthscale: .1 obs space: obsdatain: Data/noobs.obt obs operator: {} diff --git a/l95/test/testinput/letkf_qc.yaml b/l95/test/testinput/letkf_qc.yaml index 93675cfce..3331fb492 100644 --- a/l95/test/testinput/letkf_qc.yaml +++ b/l95/test/testinput/letkf_qc.yaml @@ -20,10 +20,10 @@ background: observations: - obs error: - covariance model: localized diagonal - localization: - localization method: Gaspari-Cohn - lengthscale: .1 + covariance model: diagonal + obs localization: + localization method: Gaspari-Cohn + lengthscale: .1 obs space: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obs operator: {} diff --git a/l95/test/testoutput/letkf_qc.test b/l95/test/testoutput/letkf_qc.test index 53c68532e..9fb90345e 100644 --- a/l95/test/testoutput/letkf_qc.test +++ b/l95/test/testoutput/letkf_qc.test @@ -32,20 +32,20 @@ Test : Valid time: 2010-01-02T00:00:00Z Test : Min=5.25199, Max=9.50208, Average=7.76569 Test : Analysis mean : Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.73044, Max=9.19002, Average=7.98343 +Test : Min=6.73044, Max=9.19002, Average=7.98752 Test : H(x) for member 1: -Test : Lorenz 95 nobs= 120 Min=6.31712, Max=10.1291, Average=8.09238 +Test : Lorenz 95 nobs= 120 Min=6.31712, Max=10.1291, Average=8.09272 Test : H(x) for member 2: -Test : Lorenz 95 nobs= 120 Min=5.62763, Max=9.94295, Average=7.96969 +Test : Lorenz 95 nobs= 120 Min=5.62563, Max=9.94295, Average=7.97729 Test : H(x) for member 3: -Test : Lorenz 95 nobs= 120 Min=5.98634, Max=10.5924, Average=8.01273 +Test : Lorenz 95 nobs= 120 Min=5.98634, Max=10.5924, Average=8.01934 Test : H(x) for member 4: -Test : Lorenz 95 nobs= 120 Min=4.57542, Max=11.5295, Average=7.98536 +Test : Lorenz 95 nobs= 120 Min=4.57542, Max=11.5295, Average=7.98859 Test : H(x) for member 5: -Test : Lorenz 95 nobs= 120 Min=4.96477, Max=9.69106, Average=7.857 +Test : Lorenz 95 nobs= 120 Min=4.72238, Max=9.79569, Average=7.85967 Test : H(x) ensemble analysis mean: -Test : Lorenz 95 nobs= 120 Min=6.73044, Max=9.19002, Average=7.98343 +Test : Lorenz 95 nobs= 120 Min=6.73044, Max=9.19002, Average=7.98752 Test : analysis y - H(x): -Test : Lorenz 95 nobs= 120 Min=-0.3032, Max=0.441464, Average=0.0279604 +Test : Lorenz 95 nobs= 120 Min=-0.265703, Max=0.441464, Average=0.0238667 Test : ombg RMS: 0.864096 -Test : oman RMS: 0.149894 +Test : oman RMS: 0.14948 diff --git a/qg/test/testinput/letkf.yaml b/qg/test/testinput/letkf.yaml index 1372eb949..1a5b46e8e 100644 --- a/qg/test/testinput/letkf.yaml +++ b/qg/test/testinput/letkf.yaml @@ -56,8 +56,8 @@ observations: obs type: Stream obs error: covariance model: diagonal - localization: - lengthscale: 5e6 + obs localization: + lengthscale: 5e6 - obs operator: obs type: Wind obs space: @@ -68,8 +68,8 @@ observations: obs type: Wind obs error: covariance model: diagonal - localization: - lengthscale: 5e6 + obs localization: + lengthscale: 5e6 - obs operator: obs type: WSpeed obs space: @@ -80,8 +80,8 @@ observations: obs type: WSpeed obs error: covariance model: diagonal - localization: - lengthscale: 5e6 + obs localization: + lengthscale: 5e6 driver: diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9d1019a75..92f4938f9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -129,6 +129,7 @@ oops/base/ObsFilterBase.h oops/base/ObsFilterParametersBase.h oops/base/ObsFilters.h oops/base/ObsLocalizationBase.h +oops/base/ObsLocalizations.h oops/base/ObsSpaceBase.cc oops/base/ObsSpaceBase.h oops/base/ObsSpaces.h @@ -177,7 +178,6 @@ oops/generic/interpolatorunstrc_interface.F90 oops/generic/InterpolatorUnstructured.h oops/generic/InterpolatorUnstructured.cc oops/generic/LinearModelId.h -oops/generic/LocalObsErrorDiag.h oops/generic/ObsErrorDiag.h oops/generic/PseudoModel.h oops/generic/unstructured_interpolation_mod.F90 @@ -393,6 +393,7 @@ test/interface/ObsAuxControl.h test/interface/ObsAuxCovariance.h test/interface/ObsAuxIncrement.h test/interface/ObsDataVector.h +test/interface/ObsLocalization.h test/interface/ObsOperator.h test/interface/ObsSpace.h test/interface/ObsTestsFixture.h diff --git a/src/oops/assimilation/GETKFSolver.h b/src/oops/assimilation/GETKFSolver.h index 599442050..9461662e9 100644 --- a/src/oops/assimilation/GETKFSolver.h +++ b/src/oops/assimilation/GETKFSolver.h @@ -26,6 +26,7 @@ #include "oops/base/ObsEnsemble.h" #include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" +#include "oops/base/ObsLocalizations.h" #include "oops/base/ObsSpaces.h" #include "oops/base/StateEnsemble4D.h" #include "oops/generic/VerticalLocEV.h" @@ -56,6 +57,7 @@ class GETKFSolver : public LocalEnsembleSolver { typedef ObsEnsemble ObsEnsemble_; typedef ObsErrors ObsErrors_; typedef Observations Observations_; + typedef ObsLocalizations ObsLocalizations_; typedef ObsSpaces ObsSpaces_; typedef State4D State4D_; typedef StateEnsemble4D StateEnsemble4D_; @@ -77,7 +79,7 @@ class GETKFSolver : public LocalEnsembleSolver { private: /// Computes weights void computeWeights(const Departures_ &, const DeparturesEnsemble_ &, - const DeparturesEnsemble_ &, const ObsErrors_ &); + const DeparturesEnsemble_ &, const Departures_ &); /// Applies weights and adds posterior inflation void applyWeights(const IncrementEnsemble4D_ &, IncrementEnsemble4D_ &, @@ -200,7 +202,7 @@ template void GETKFSolver::computeWeights(const Departures_ & dy, const DeparturesEnsemble_ & Yb, const DeparturesEnsemble_ & YbOrig, - const ObsErrors_ & R) { + const Departures_ & R_invvar) { // compute transformation matrix, save in Wa_, wa_ // Yb(nobs,neig*nens), YbOrig(nobs,nens) // uses GSI GETKF code @@ -219,7 +221,7 @@ void GETKFSolver::computeWeights(const Departures_ & dy, Eigen::MatrixXd eYb2 = YbOrig.packEigen(); Eigen::MatrixXf eYb2_f = eYb2.cast(); - Eigen::MatrixXd eR = R.packInverseVarianceEigen(); + Eigen::MatrixXd eR = R_invvar.packEigen(); Eigen::MatrixXf eR_f = eR.cast(); Eigen::MatrixXf Wa_f(this->nanal_, this->nens_); @@ -336,7 +338,14 @@ void GETKFSolver::measurementUpdate(const IncrementEnsemble4D_ & bkg DeparturesEnsemble_ local_HZ(local_obs, HZb_); // create local obs errors ObsErrors_ local_R(this->obsconf_, local_obs); - computeWeights(local_omb, local_HZ, local_Yb, local_R); + Departures_ invVarR = local_R.inverseVariance(); + // and apply localization + ObsLocalizations_ loc(this->obsconf_, local_obs); + Departures_ locvector(local_obs); + locvector.ones(); + loc.computeLocalization(i, locvector); + invVarR *= locvector; + computeWeights(local_omb, local_HZ, local_Yb, invVarR); applyWeights(bkg_pert, ana_pert, i); } } diff --git a/src/oops/assimilation/LETKFSolver.h b/src/oops/assimilation/LETKFSolver.h index 17804a108..50a5e9f46 100644 --- a/src/oops/assimilation/LETKFSolver.h +++ b/src/oops/assimilation/LETKFSolver.h @@ -22,6 +22,7 @@ #include "oops/base/IncrementEnsemble4D.h" #include "oops/base/LocalIncrement.h" #include "oops/base/ObsErrors.h" +#include "oops/base/ObsLocalizations.h" #include "oops/base/ObsSpaces.h" #include "oops/interface/Geometry.h" #include "oops/interface/GeometryIterator.h" @@ -49,6 +50,7 @@ class LETKFSolver : public LocalEnsembleSolver { typedef GeometryIterator GeometryIterator_; typedef IncrementEnsemble4D IncrementEnsemble4D_; typedef ObsErrors ObsErrors_; + typedef ObsLocalizations ObsLocalizations_; typedef ObsSpaces ObsSpaces_; public: @@ -63,7 +65,7 @@ class LETKFSolver : public LocalEnsembleSolver { protected: /// Computes weights virtual void computeWeights(const Departures_ &, const DeparturesEnsemble_ &, - const ObsErrors_ &); + const Departures_ &); /// Applies weights and adds posterior inflation virtual void applyWeights(const IncrementEnsemble4D_ &, IncrementEnsemble4D_ &, @@ -142,7 +144,14 @@ void LETKFSolver::measurementUpdate(const IncrementEnsemble4D_ & bkg DeparturesEnsemble_ local_Yb(local_obs, this->Yb_); // create local obs errors ObsErrors_ local_R(this->obsconf_, local_obs); - computeWeights(local_omb, local_Yb, local_R); + Departures_ invVarR = local_R.inverseVariance(); + // and apply localization + ObsLocalizations_ loc(this->obsconf_, local_obs); + Departures_ locvector(local_obs); + locvector.ones(); + loc.computeLocalization(i, locvector); + invVarR *= locvector; + computeWeights(local_omb, local_Yb, invVarR); applyWeights(bkg_pert, ana_pert, i); } } @@ -152,7 +161,7 @@ void LETKFSolver::measurementUpdate(const IncrementEnsemble4D_ & bkg template void LETKFSolver::computeWeights(const Departures_ & dy_oops, const DeparturesEnsemble_ & Yb_oops, - const ObsErrors_ & R_oops) { + const Departures_ & R_invvar_oops ) { // compute transformation matrix, save in Wa_, wa_ // uses C++ eigen interface // implements LETKF from Hunt et al. 2007 @@ -164,13 +173,12 @@ void LETKFSolver::computeWeights(const Departures_ & dy_oops, Eigen::MatrixXd dy = dy_oops.packEigen(); Eigen::MatrixXd Yb = Yb_oops.packEigen(); - Eigen::MatrixXd diagInvR = R_oops.packInverseVarianceEigen(); + Eigen::VectorXd diagInvR = R_invvar_oops.packEigen(); // fill in the work matrix // work = Y^T R^-1 Y + (nens-1)/infl I double infl = inflopt.mult; Eigen::MatrixXd work = Yb*(diagInvR.asDiagonal()*Yb.transpose()); - work.diagonal() += Eigen::VectorXd::Constant(nens_, (nens_-1)/infl); // eigenvalues and eigenvectors of the above matrix diff --git a/src/oops/assimilation/LETKFSolverGSI.h b/src/oops/assimilation/LETKFSolverGSI.h index a2302ab49..69302fecc 100644 --- a/src/oops/assimilation/LETKFSolverGSI.h +++ b/src/oops/assimilation/LETKFSolverGSI.h @@ -35,7 +35,7 @@ class LETKFSolverGSI : public LETKFSolver { /// Computes weights void computeWeights(const Departures_ &, const DeparturesEnsemble_ &, - const ObsErrors_ &); + const Departures_ &) override; }; // ----------------------------------------------------------------------------- @@ -52,7 +52,7 @@ LETKFSolverGSI::LETKFSolverGSI(ObsSpaces_ & obspaces, const Geometry template void LETKFSolverGSI::computeWeights(const Departures_ & dy, const DeparturesEnsemble_ & Yb, - const ObsErrors_ & R) { + const Departures_ & R_invvar) { // compute transformation matrix, save in Wa_, wa_ // uses GSI GETKF code const int nobsl = dy.nobs(); @@ -65,8 +65,8 @@ void LETKFSolverGSI::computeWeights(const Departures_ & dy, Eigen::MatrixXd eYb = Yb.packEigen(); Eigen::MatrixXf eYb_f = eYb.cast(); - Eigen::MatrixXd eR = R.packInverseVarianceEigen(); - Eigen::MatrixXf eR_f = eR.cast(); + Eigen::VectorXd eR = R_invvar.packEigen(); + Eigen::VectorXf eR_f = eR.cast(); Eigen::MatrixXf Wa_f(this->nens_, this->nens_); Eigen::VectorXf wa_f(this->nens_); diff --git a/src/oops/base/Departures.h b/src/oops/base/Departures.h index 820b50cd4..0e99d1c30 100644 --- a/src/oops/base/Departures.h +++ b/src/oops/base/Departures.h @@ -66,6 +66,7 @@ class Departures : public util::Printable, Departures & operator*=(const Departures &); Departures & operator/=(const Departures &); void zero(); + void ones(); void random(); void invert(); void axpy(const double &, const Departures &); @@ -161,6 +162,13 @@ void Departures::zero() { } // ----------------------------------------------------------------------------- template +void Departures::ones() { + for (auto & dep : dep_) { + dep.ones(); + } +} +// ----------------------------------------------------------------------------- +template void Departures::random() { for (size_t jj = 0; jj < dep_.size(); ++jj) { dep_[jj].random(); diff --git a/src/oops/base/ObsErrors.h b/src/oops/base/ObsErrors.h index 8489bf208..f9b6b3b35 100644 --- a/src/oops/base/ObsErrors.h +++ b/src/oops/base/ObsErrors.h @@ -11,7 +11,6 @@ #ifndef OOPS_BASE_OBSERRORS_H_ #define OOPS_BASE_OBSERRORS_H_ -#include #include #include #include @@ -34,7 +33,6 @@ class ObsErrors : public util::Printable, typedef Departures Departures_; typedef ObsErrorBase ObsError_; typedef ObsSpaces ObsSpaces_; - typedef ObsVector ObsVector_; public: static const std::string classname() {return "oops::ObsErrors";} @@ -53,20 +51,20 @@ class ObsErrors : public util::Printable, /// Generate random perturbation void randomize(Departures_ &) const; -/// Pack inverseVariance into an Eigen vector (excluding observations -/// that are masked out) - Eigen::VectorXd packInverseVarianceEigen() const; + /// returns inverse of observation error variance + Departures_ inverseVariance() const; private: void print(std::ostream &) const; std::vector > err_; + const ObsSpaces_ & os_; }; // ----------------------------------------------------------------------------- template ObsErrors::ObsErrors(const eckit::Configuration & config, - const ObsSpaces_ & os) : err_() { + const ObsSpaces_ & os) : err_(), os_(os) { std::vector obsconf = config.getSubConfigurations(); for (size_t jj = 0; jj < os.size(); ++jj) { eckit::LocalConfiguration conf(obsconf[jj], "obs error"); @@ -104,24 +102,12 @@ void ObsErrors::randomize(Departures_ & dy) const { // ----------------------------------------------------------------------------- template -Eigen::VectorXd ObsErrors::packInverseVarianceEigen() const { - // compute nobs accross all obs errors - unsigned int nobs = 0; - for (size_t iov = 0; iov < err_.size(); ++iov) { - const ObsVector_ & ov = err_[iov]->inverseVariance(); - nobs += ov.nobs(); - } - - // concatinate all inverseVariance into a 1d vector - Eigen::VectorXd vec(nobs); - unsigned int ii = 0; - for (size_t iov = 0; iov < err_.size(); ++iov) { - const ObsVector_ & ov = err_[iov]->inverseVariance(); - vec.segment(ii, ov.nobs()) = ov.packEigen(); - ii += ov.nobs(); +Departures ObsErrors::inverseVariance() const { + Departures_ invvar(os_); + for (size_t jj = 0; jj < err_.size(); ++jj) { + invvar[jj] = err_[jj]->inverseVariance(); } - ASSERT(ii == nobs); - return vec; + return invvar; } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/ObsLocalizationBase.h b/src/oops/base/ObsLocalizationBase.h index 6fc9f2257..b15833a3f 100644 --- a/src/oops/base/ObsLocalizationBase.h +++ b/src/oops/base/ObsLocalizationBase.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2020 UCAR + * (C) Copyright 2020-2021 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -11,67 +11,68 @@ #include #include #include -#include #include #include "eckit/config/Configuration.h" +#include "oops/interface/GeometryIterator.h" #include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" #include "oops/util/Printable.h" namespace oops { -// ----------------------------------------------------------------------------- -/// Base class for generic localizations - -template +/// Base class for observation-space localization +template class ObsLocalizationBase : public util::Printable, private boost::noncopyable { - typedef ObsVector ObsVector_; + typedef GeometryIterator GeometryIterator_; + typedef ObsVector ObsVector_; public: - ObsLocalizationBase() {} - virtual ~ObsLocalizationBase() {} + ObsLocalizationBase() = default; + virtual ~ObsLocalizationBase() = default; - virtual void multiply(ObsVector_ &) const = 0; + /// fill \p obsvector with observation-space localization between + /// observations and \p point in model-space + virtual void computeLocalization(const GeometryIterator_ & point, + ObsVector_ & obsvector) const = 0; }; // ============================================================================= -/// ObsLocalizationFactory Factory -template +/// ObsLocalization Factory +template class ObsLocalizationFactory { - typedef ObsSpace ObsSpace_; + typedef ObsSpace ObsSpace_; public: - static std::unique_ptr> create(const eckit::Configuration &, - const ObsSpace_ &); + static std::unique_ptr> create(const eckit::Configuration &, + const ObsSpace_ &); protected: explicit ObsLocalizationFactory(const std::string &); private: - virtual ObsLocalizationBase * make(const eckit::Configuration &, - const ObsSpace_ &) = 0; - static std::map < std::string, ObsLocalizationFactory * > & getMakers() { - static std::map < std::string, ObsLocalizationFactory * > makers_; + virtual ObsLocalizationBase * make(const eckit::Configuration &, + const ObsSpace_ &) = 0; + static std::map < std::string, ObsLocalizationFactory * > & getMakers() { + static std::map < std::string, ObsLocalizationFactory * > makers_; return makers_; } }; // ----------------------------------------------------------------------------- -template -class ObsLocalizationMaker : public ObsLocalizationFactory { - typedef ObsSpace ObsSpace_; - virtual ObsLocalizationBase * make(const eckit::Configuration & conf, - const ObsSpace_ & obsspace) - { return new T(conf, obsspace); } +template +class ObsLocalizationMaker : public ObsLocalizationFactory { + typedef ObsSpace ObsSpace_; + virtual ObsLocalizationBase * make(const eckit::Configuration & conf, + const ObsSpace_ & obspace) + { return new T(conf, obspace); } public: explicit ObsLocalizationMaker(const std::string & name) : - ObsLocalizationFactory(name) {} + ObsLocalizationFactory(name) {} }; // ----------------------------------------------------------------------------- -template -ObsLocalizationFactory::ObsLocalizationFactory(const std::string & name) { +template +ObsLocalizationFactory::ObsLocalizationFactory(const std::string & name) { if (getMakers().find(name) != getMakers().end()) { throw std::runtime_error(name + " already registered in obs localization factory."); } @@ -80,19 +81,23 @@ ObsLocalizationFactory::ObsLocalizationFactory(const std::string & name) { // ----------------------------------------------------------------------------- -template -std::unique_ptr> ObsLocalizationFactory::create( - const eckit::Configuration & conf, const ObsSpace_ & obsspace) { - Log::trace() << "ObsLocalizationBase::create starting" << std::endl; - const std::string id = conf.getString("localization method"); - typename std::map*>::iterator - jloc = getMakers().find(id); - if (jloc == getMakers().end()) { - throw std::runtime_error(id + " does not exist in obs localization factory."); +template +std::unique_ptr> ObsLocalizationFactory::create( + const eckit::Configuration & conf, const ObsSpace_ & obspace) { + Log::trace() << "ObsLocalizationBase::create starting" << std::endl; + if (conf.has("localization method")) { + const std::string id = conf.getString("localization method"); + typename std::map*>::iterator + jloc = getMakers().find(id); + if (jloc == getMakers().end()) { + throw std::runtime_error(id + " does not exist in obs localization factory."); + } + std::unique_ptr> ptr(jloc->second->make(conf, obspace)); + Log::trace() << "ObsLocalizationBase::create done" << std::endl; + return ptr; + } else { + return nullptr; } - std::unique_ptr> ptr(jloc->second->make(conf, obsspace)); - Log::trace() << "ObsLocalizationBase::create done" << std::endl; - return ptr; } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/ObsLocalizations.h b/src/oops/base/ObsLocalizations.h new file mode 100644 index 000000000..0b572d166 --- /dev/null +++ b/src/oops/base/ObsLocalizations.h @@ -0,0 +1,84 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_BASE_OBSLOCALIZATIONS_H_ +#define OOPS_BASE_OBSLOCALIZATIONS_H_ + +#include +#include +#include + +#include + +#include "oops/base/Departures.h" +#include "oops/base/ObsLocalizationBase.h" +#include "oops/base/ObsSpaces.h" +#include "oops/util/Logger.h" +#include "oops/util/Printable.h" + +namespace oops { + +// ----------------------------------------------------------------------------- +/// \brief Container for ObsLocalizations for all observation types that are used in DA +template +class ObsLocalizations : public util::Printable, + private boost::noncopyable { + typedef GeometryIterator GeometryIterator_; + typedef Departures Observations_; + typedef ObsLocalizationBase ObsLocalization_; + typedef ObsSpaces ObsSpaces_; + + public: + static const std::string classname() {return "oops::ObsLocalizations";} + + ObsLocalizations(const eckit::Configuration &, const ObsSpaces_ &); + + /// fill \p obsvectors with observation-space localizations between + /// observations and \p point in model-space + void computeLocalization(const GeometryIterator_ & point, Observations_ & obsvectors) const; + + private: + void print(std::ostream &) const; + std::vector > local_; +}; + +// ----------------------------------------------------------------------------- + +template +ObsLocalizations::ObsLocalizations(const eckit::Configuration & config, + const ObsSpaces_ & obspaces) { + std::vector obsconf = config.getSubConfigurations(); + for (size_t jj = 0; jj < obsconf.size(); ++jj) { + eckit::LocalConfiguration conf(obsconf[jj], "obs localization"); + local_.emplace_back(ObsLocalizationFactory::create(conf, obspaces[jj])); + } +} + +// ----------------------------------------------------------------------------- + +template +void ObsLocalizations::computeLocalization(const GeometryIterator_ & point, + Observations_ & obsvec) const { + for (size_t jj = 0; jj < local_.size(); ++jj) { + if (local_[jj]) local_[jj]->computeLocalization(point, obsvec[jj]); + } +} + +// ----------------------------------------------------------------------------- + +template +void ObsLocalizations::print(std::ostream & os) const { + for (size_t jj = 0; jj < local_.size(); ++jj) { + if (local_[jj]) os << *local_[jj]; + } +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_BASE_OBSLOCALIZATIONS_H_ diff --git a/src/oops/base/ObsSpaces.h b/src/oops/base/ObsSpaces.h index 7aa0e5d64..d5bba9858 100644 --- a/src/oops/base/ObsSpaces.h +++ b/src/oops/base/ObsSpaces.h @@ -98,7 +98,7 @@ ObsSpaces::ObsSpaces(const ObsSpaces & obss, const eckit::geometry::Po { std::vector typeconfs = conf.getSubConfigurations(); for (std::size_t jj = 0; jj < obss.size(); ++jj) { - eckit::LocalConfiguration locconf(typeconfs[jj], "obs error.localization"); + eckit::LocalConfiguration locconf(typeconfs[jj], "obs localization"); std::shared_ptr tmp(new ObsSpace_(obss[jj], center, locconf)); spaces_.push_back(tmp); } diff --git a/src/oops/generic/LocalObsErrorDiag.h b/src/oops/generic/LocalObsErrorDiag.h deleted file mode 100644 index e9ff44ca0..000000000 --- a/src/oops/generic/LocalObsErrorDiag.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * (C) Copyright 2020 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef OOPS_GENERIC_LOCALOBSERRORDIAG_H_ -#define OOPS_GENERIC_LOCALOBSERRORDIAG_H_ - -#include -#include -#include - -#include "eckit/config/Configuration.h" -#include "oops/base/ObsErrorBase.h" -#include "oops/base/ObsLocalizationBase.h" -#include "oops/generic/ObsErrorDiag.h" -#include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" -#include "oops/util/Logger.h" - -namespace oops { - -// ----------------------------------------------------------------------------- -/// Diagonal observation error covariance matrix with R-localization -// localization (inflating observation error variances) is done in the constructor -// the rest of the methods are not overriden; ObsErrorDiag methods would be used -// instead - -template -class LocalObsErrorDiag : public ObsErrorDiag { - typedef ObsLocalizationBase ObsLocalization_; - typedef ObsSpace ObsSpace_; - - public: -/// Initialize and inflate local R for obs. localization - LocalObsErrorDiag(const eckit::Configuration &, const ObsSpace_ &); - - private: - void print(std::ostream &) const override; -}; - -// ============================================================================= - -template -LocalObsErrorDiag::LocalObsErrorDiag - (const eckit::Configuration & conf, const ObsSpace_ & obsdb) - : ObsErrorDiag(conf, obsdb) -{ -// if Localization section is available; localize covariances - if (conf.has("localization")) { - eckit::LocalConfiguration locconf(conf, "localization"); - std::unique_ptr loc(ObsLocalizationFactory::create(locconf, obsdb)); - loc->multiply(this->inverseVariance_); - } -} - -// ----------------------------------------------------------------------------- - -template -void LocalObsErrorDiag::print(std::ostream & os) const { - os << "Localized diagonal observation error covariance, inverse variances: " - << this->inverseVariance_ << std::endl; -} - - -// ----------------------------------------------------------------------------- - - -} // namespace oops - -#endif // OOPS_GENERIC_LOCALOBSERRORDIAG_H_ diff --git a/src/oops/generic/instantiateObsErrorFactory.h b/src/oops/generic/instantiateObsErrorFactory.h index 80480eb46..b18e6b817 100644 --- a/src/oops/generic/instantiateObsErrorFactory.h +++ b/src/oops/generic/instantiateObsErrorFactory.h @@ -12,14 +12,12 @@ #define OOPS_GENERIC_INSTANTIATEOBSERRORFACTORY_H_ #include "oops/base/ObsErrorBase.h" -#include "oops/generic/LocalObsErrorDiag.h" #include "oops/generic/ObsErrorDiag.h" namespace oops { template void instantiateObsErrorFactory() { static ObsErrorMaker > makerDiag_("diagonal"); - static ObsErrorMaker > makerLocalDiag_("localized diagonal"); } } // namespace oops diff --git a/src/oops/interface/ObsLocalization.h b/src/oops/interface/ObsLocalization.h index ff7192bc4..f33d525ea 100644 --- a/src/oops/interface/ObsLocalization.h +++ b/src/oops/interface/ObsLocalization.h @@ -1,5 +1,5 @@ /* - * (C) Copyright 2017-2020 UCAR + * (C) Copyright 2017-2021 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -12,6 +12,7 @@ #include #include "eckit/config/LocalConfiguration.h" +#include "oops/base/LocalIncrement.h" #include "oops/base/ObsLocalizationBase.h" #include "oops/interface/ObsSpace.h" #include "oops/interface/ObsVector.h" @@ -21,8 +22,9 @@ namespace oops { // ----------------------------------------------------------------------------- -template -class ObsLocalization : public ObsLocalizationBase { +template +class ObsLocalization : public ObsLocalizationBase { + typedef GeometryIterator GeometryIterator_; typedef ObsSpace ObsSpace_; typedef ObsVector ObsVector_; @@ -32,7 +34,7 @@ class ObsLocalization : public ObsLocalizationBase { ObsLocalization(const eckit::Configuration &, const ObsSpace_ &); ~ObsLocalization(); - void multiply(ObsVector_ &) const override; + void computeLocalization(const GeometryIterator_ &, ObsVector_ &) const override; private: void print(std::ostream &) const override; @@ -42,43 +44,43 @@ class ObsLocalization : public ObsLocalizationBase { // ----------------------------------------------------------------------------- -template -ObsLocalization::ObsLocalization(const eckit::Configuration & conf, - const ObsSpace_ & os) +template +ObsLocalization::ObsLocalization(const eckit::Configuration & conf, + const ObsSpace_ & obspace) : obsloc_() { - Log::trace() << "ObsLocalization::ObsLocalization Configuration starting" << - std::endl; + Log::trace() << "ObsLocalization::ObsLocalization starting" << std::endl; util::Timer timer(classname(), "ObsLocalization"); - obsloc_.reset(new LOC(conf, os.obsspace())); - Log::trace() << "ObsLocalization::ObsLocalization Configuration done" << std::endl; + obsloc_.reset(new LOC(conf, obspace.obsspace())); + Log::trace() << "ObsLocalization::ObsLocalization done" << std::endl; } // ----------------------------------------------------------------------------- -template -ObsLocalization::~ObsLocalization() { - Log::trace() << "ObsLocalization::~ObsLocalization starting" << std::endl; +template +ObsLocalization::~ObsLocalization() { + Log::trace() << "ObsLocalization::~ObsLocalization starting" << std::endl; util::Timer timer(classname(), "~ObsLocalization"); obsloc_.reset(); - Log::trace() << "ObsLocalization::~ObsLocalization done" << std::endl; + Log::trace() << "ObsLocalization::~ObsLocalization done" << std::endl; } // ----------------------------------------------------------------------------- -template -void ObsLocalization::multiply(ObsVector_ & dy) const { - Log::trace() << "ObsLocalization:: preProcess starting" << std::endl; - util::Timer timer(classname(), "preProcess"); - obsloc_->multiply(dy.obsvector()); - Log::trace() << "ObsLocalization:: preProcess done" << std::endl; +template +void ObsLocalization::computeLocalization(const GeometryIterator_ & p, + ObsVector_ & obsvector) const { + Log::trace() << "ObsLocalization:: computeLocalization starting" << std::endl; + util::Timer timer(classname(), "computeLocalization"); + obsloc_->computeLocalization(p.geometryiter(), obsvector.obsvector()); + Log::trace() << "ObsLocalization:: computeLocalization done" << std::endl; } // ----------------------------------------------------------------------------- -template -void ObsLocalization::print(std::ostream & os) const { - os << "ObsLocalization " << *obsloc_; +template +void ObsLocalization::print(std::ostream & os) const { + os << *obsloc_; } // ----------------------------------------------------------------------------- diff --git a/src/test/interface/ObsLocalization.h b/src/test/interface/ObsLocalization.h new file mode 100644 index 000000000..9a872694b --- /dev/null +++ b/src/test/interface/ObsLocalization.h @@ -0,0 +1,114 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef TEST_INTERFACE_OBSLOCALIZATION_H_ +#define TEST_INTERFACE_OBSLOCALIZATION_H_ + +#include +#include +#include + +#define ECKIT_TESTING_SELF_REGISTER_CASES 0 + +#include + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/testing/Test.h" +#include "oops/base/ObsLocalizationBase.h" +#include "oops/interface/Geometry.h" +#include "oops/interface/GeometryIterator.h" +#include "oops/interface/ObsVector.h" +#include "oops/mpi/mpi.h" +#include "oops/runs/Test.h" +#include "test/interface/ObsTestsFixture.h" +#include "test/TestEnvironment.h" + +namespace test { + +/// Tests that obs localization applied to a zero vector returns zero. +/// Tests that obs localization applied to a vector of ones makes rms(obsvec) < 1 +template void testObsLocalization() { + typedef ObsTestsFixture Test_; + typedef oops::Geometry Geometry_; + typedef oops::GeometryIterator GeometryIterator_; + typedef oops::ObsLocalizationBase ObsLocalization_; + typedef oops::ObsSpace ObsSpace_; + typedef oops::ObsVector ObsVector_; + + const eckit::LocalConfiguration geometryConfig(TestEnvironment::config(), "geometry"); + Geometry_ geometry(geometryConfig, oops::mpi::world()); + + // loop over all obs spaces + for (size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + eckit::LocalConfiguration locconf(Test_::config(jj), "obs localization"); + // variable used to check if localization is tested at least once below + bool tested = false; + // loop over geometry points + for (GeometryIterator_ ii = geometry.begin(); ii != geometry.end(); ++ii) { + // initialize local observation space and a local obs vector + ObsSpace_ localobs(Test_::obspace()[jj], *ii, locconf); + ObsVector_ obsvector(localobs); + // only test if there are some observations in the local obs space + if (obsvector.nobs() > 0) { + tested = true; + // initialize obs-space localization + std::unique_ptr obsloc = + oops::ObsLocalizationFactory::create(locconf, localobs); + oops::Log::test() << "Testing obs-space localization: " << *obsloc << + " at geometry iterator " << ii << std::endl; + // apply obs localization to a zero vector, check that result is zero + obsvector.zero(); + EXPECT_EQUAL(obsvector.rms(), 0.0); + ObsVector_ locvector(localobs); + obsloc->computeLocalization(ii, locvector); + obsvector *= locvector; + EXPECT_EQUAL(obsvector.rms(), 0.0); + // apply localization to a vector of ones, check that rms(result) < 1 + obsvector.ones(); + EXPECT_EQUAL(obsvector.rms(), 1.0); + locvector.ones(); + obsloc->computeLocalization(ii, locvector); + obsvector *= locvector; + oops::Log::test() << "Localization applied to local ObsVector of ones: " << + obsvector << std::endl; + EXPECT(obsvector.rms() < 1.0); + } + } + EXPECT(tested); + } +} + +// ----------------------------------------------------------------------------- + +template class ObsLocalization : public oops::Test { + typedef ObsTestsFixture Test_; + + public: + ObsLocalization() = default; + virtual ~ObsLocalization() = default; + + private: + std::string testid() const override {return "test::ObsLocalization<" + MODEL::name() + "," + + OBS::name() + ">";} + + void register_tests() const override { + std::vector& ts = eckit::testing::specification(); + + ts.emplace_back(CASE("interface/ObsLocalization/testObsLocalization") + { testObsLocalization(); }); + } + + void clear() const override { + Test_::reset(); + } +}; + +// ----------------------------------------------------------------------------- + +} // namespace test + +#endif // TEST_INTERFACE_OBSLOCALIZATION_H_ From 8942654310b87b3c8f1925010e145baeabdc18b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Fri, 2 Apr 2021 09:36:57 -0600 Subject: [PATCH 095/142] Added GetValuePosts (#1135) * Added GetValuePosts * Fix copyright --- qg/model/ObsDataQG.h | 1 - qg/model/qg_fields_mod.F90 | 6 ++ qg/model/qg_model_mod.F90 | 6 ++ src/CMakeLists.txt | 1 + src/oops/assimilation/CostJo.h | 9 ++- src/oops/assimilation/CostJoType.h | 11 ++-- src/oops/base/GetValuePost.h | 22 ++++---- src/oops/base/GetValuePosts.h | 90 ++++++++++++++++++++++++++++++ src/oops/base/Observers.h | 15 +++-- 9 files changed, 135 insertions(+), 26 deletions(-) create mode 100644 src/oops/base/GetValuePosts.h diff --git a/qg/model/ObsDataQG.h b/qg/model/ObsDataQG.h index 7c3a40492..08ac7a632 100644 --- a/qg/model/ObsDataQG.h +++ b/qg/model/ObsDataQG.h @@ -49,7 +49,6 @@ class ObsDataQG : public util::Printable, const int & toFortran() const {return data_.toFortran();} const ObsVecQG & vect() const {return data_;} - private: void print(std::ostream &) const; diff --git a/qg/model/qg_fields_mod.F90 b/qg/model/qg_fields_mod.F90 index 6b6b9cb85..03d78e6df 100644 --- a/qg/model/qg_fields_mod.F90 +++ b/qg/model/qg_fields_mod.F90 @@ -1054,6 +1054,12 @@ subroutine qg_fields_analytic_init(fld,f_conf,vdate) ! Check field call qg_fields_check(fld) +if (fld%lq) then + if (allocated(fld%pv)) fld%pv = fld%gfld3d +else + if (allocated(fld%streamfct)) fld%streamfct = fld%gfld3d +endif + end subroutine qg_fields_analytic_init ! ------------------------------------------------------------------------------ !> Fields statistics diff --git a/qg/model/qg_model_mod.F90 b/qg/model/qg_model_mod.F90 index 3b63e529d..5e8051ab0 100644 --- a/qg/model/qg_model_mod.F90 +++ b/qg/model/qg_model_mod.F90 @@ -109,6 +109,12 @@ subroutine qg_model_propagate(conf,fld) fld%gfld3d = x endif +if (fld%lq) then + if (allocated(fld%pv)) fld%pv = fld%gfld3d +else + if (allocated(fld%streamfct)) fld%streamfct = fld%gfld3d +endif + end subroutine qg_model_propagate ! ------------------------------------------------------------------------------ !> Perform a timestep of the QG model - tangent linear diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 92f4938f9..2cd1f7515 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -96,6 +96,7 @@ oops/base/EnsembleCovariance.h oops/base/GeneralizedDepartures.h oops/base/GeoVaLsWriter.h oops/base/GetValuePost.h +oops/base/GetValuePosts.h oops/base/GetValuesPost.h oops/base/LocalIncrement.cc oops/base/LocalIncrement.h diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index 4c0b27764..f199865e9 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -24,6 +24,7 @@ #include "oops/assimilation/CostJoType.h" #include "oops/assimilation/CostTermBase.h" #include "oops/base/Departures.h" +#include "oops/base/GetValuePosts.h" #include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" #include "oops/base/ObserversTLAD.h" @@ -56,6 +57,7 @@ template class CostJo : public CostTermBase Departures_; typedef Observations Observations_; typedef Geometry Geometry_; + typedef GetValuePosts GetValuePosts_; typedef State State_; typedef Increment Increment_; typedef ObsErrors ObsErrors_; @@ -103,7 +105,7 @@ template class CostJo : public CostTermBase::CostJo(const eckit::Configuration & joConf, const eckit::mpi yobs_(obspaces_, "ObsValue"), jos_(), gradFG_() { Log::trace() << "CostJo::CostJo start" << std::endl; + jos_.reserve(obspaces_.size()); std::vector confs(obsconf_.getSubConfigurations()); for (size_t jj = 0; jj < obspaces_.size(); ++jj) { jos_.emplace_back(new JoType_(obspaces_[jj], confs[jj])); @@ -145,9 +148,11 @@ void CostJo::initialize(const CtrlVar_ & xx, const eckit::Configurat Log::trace() << "CostJo::initialize start" << std::endl; gradFG_.reset(); + std::shared_ptr getvals(new GetValuePosts_()); for (size_t jj = 0; jj < jos_.size(); ++jj) { - pp.enrollProcessor(jos_[jj]->initialize(xx.state().geometry(), xx.obsVar()[jj], conf)); + getvals->append(jos_[jj]->initialize(xx.state().geometry(), xx.obsVar()[jj], conf)); } + pp.enrollProcessor(getvals); Log::trace() << "CostJo::initialize done" << std::endl; } diff --git a/src/oops/assimilation/CostJoType.h b/src/oops/assimilation/CostJoType.h index 9ad97f306..956aa1d99 100644 --- a/src/oops/assimilation/CostJoType.h +++ b/src/oops/assimilation/CostJoType.h @@ -15,6 +15,7 @@ #include #include "eckit/config/LocalConfiguration.h" +#include "oops/base/GetValuePost.h" #include "oops/base/ObsErrorBase.h" #include "oops/base/Observer.h" #include "oops/base/PostBase.h" @@ -45,6 +46,7 @@ template class CostJoType : private boost::noncopy typedef ObsVector ObsVector_; typedef PostBase> PostBase_; typedef State State_; + typedef std::shared_ptr> GetValuePtr_; public: /// Construct \f$ J_o\f$ from \f$ R\f$ and \f$ y_{obs}\f$. @@ -54,8 +56,7 @@ template class CostJoType : private boost::noncopy virtual ~CostJoType() {} /// Initialize \f$ J_o\f$ before starting the integration of the model. - std::shared_ptr initialize(const Geometry_ &, const ObsAuxCtrl_ &, - const eckit::Configuration &); + GetValuePtr_ initialize(const Geometry_ &, const ObsAuxCtrl_ &, const eckit::Configuration &); /// Finalize \f$ J_o\f$ after the integration of the model. void finalize(ObsVector_ &, ObsVector_ &, ObsVector_ &, ObsVector_ &); @@ -96,7 +97,7 @@ CostJoType::CostJoType(ObsSpace_ & obspace, const eckit::Configurati // ----------------------------------------------------------------------------- template -std::shared_ptr > > +std::shared_ptr> CostJoType::initialize(const Geometry_ & geom, const ObsAuxCtrl_ & ybias, const eckit::Configuration & conf) { Log::trace() << "CostJoType::initialize start" << std::endl; @@ -104,7 +105,7 @@ CostJoType::initialize(const Geometry_ & geom, const ObsAuxCtrl_ & y currentConf_.reset(new eckit::LocalConfiguration(conf)); iter_ = currentConf_->getInt("iteration"); - std::shared_ptr getvals = observer_.initialize(geom, ybias, obserr_, iter_); + GetValuePtr_ getvals = observer_.initialize(geom, ybias, obserr_, iter_); Log::trace() << "CostJoType::initialize done" << std::endl; return getvals; @@ -134,14 +135,12 @@ void CostJoType::finalize(ObsVector_ & yobs, ObsVector_ & yeqv, // Perturb observations according to obs error statistics bool obspert = currentConf_->getBool("obs perturbations", false); - Log::debug() << "CostJoType::finalize obspert = " << obspert << std::endl; if (obspert) { ObsVector_ ypert(obspace_); Rmat_->randomize(ypert); yobs += ypert; Log::info() << "Perturbed observations: " << yobs << std::endl; } - Log::debug() << "CostJoType::finalize Observations: " << yobs << std::endl; // Compute departures and Jo gradient ydep = yeqv; diff --git a/src/oops/base/GetValuePost.h b/src/oops/base/GetValuePost.h index b9c2b4a24..af7cdaa4d 100644 --- a/src/oops/base/GetValuePost.h +++ b/src/oops/base/GetValuePost.h @@ -33,7 +33,7 @@ namespace oops { /// \brief Fills GeoVaLs with requested variables at requested locations during model run template -class GetValuePost : public PostBase> { +class GetValuePost { typedef Geometry Geometry_; typedef GeoVaLs GeoVaLs_; typedef GetValues GetValues_; @@ -49,13 +49,15 @@ class GetValuePost : public PostBase> { /// \brief Returns geovals filled in during the model run std::unique_ptr releaseGeoVaLs(); - private: /// \brief initialization before model run: sets up GetValues and allocate GeoVaLs - void doInitialize(const State_ &, const util::DateTime &, const util::Duration &) override; + void initialize(const util::Duration &); /// \brief called at each model step: fill in GeoVaLs for the current time slot - void doProcessing(const State_ &) override; + void process(const State_ &); + +/// Variables that will be required from the State + const Variables & requiredVariables() const {return geovars_;} -// Data + private: util::DateTime winbgn_; /// Begining of assimilation window util::DateTime winend_; /// End of assimilation window util::Duration hslot_; /// Half time slot @@ -73,8 +75,7 @@ template GetValuePost::GetValuePost(const eckit::Configuration & conf, const Geometry_ & geom, const util::DateTime & bgn, const util::DateTime & end, const Locations_ & locations, const Variables & vars) - : PostBase(), - winbgn_(bgn), winend_(end), hslot_(), locations_(locations), geovars_(vars), + : winbgn_(bgn), winend_(end), hslot_(), locations_(locations), geovars_(vars), getvals_(geom, locations_, conf), geovals_(), initialized_(false) { Log::trace() << "GetValuePost::GetValuePost" << std::endl; @@ -83,13 +84,10 @@ GetValuePost::GetValuePost(const eckit::Configuration & conf, const // ----------------------------------------------------------------------------- template -void GetValuePost::doInitialize(const State_ & xx, const util::DateTime &, - const util::Duration & tstep) { +void GetValuePost::initialize(const util::Duration & tstep) { Log::trace() << "GetValuePost::doInitialize start" << std::endl; hslot_ = tstep/2; - geovals_.reset(new GeoVaLs_(locations_, geovars_)); - initialized_ = true; Log::trace() << "GetValuePost::doInitialize done" << std::endl; } @@ -97,7 +95,7 @@ void GetValuePost::doInitialize(const State_ & xx, const util::DateT // ----------------------------------------------------------------------------- template -void GetValuePost::doProcessing(const State_ & xx) { +void GetValuePost::process(const State_ & xx) { Log::trace() << "GetValuePost::doProcessing start" << std::endl; ASSERT(initialized_); util::DateTime t1 = std::max(xx.validTime()-hslot_, winbgn_); diff --git a/src/oops/base/GetValuePosts.h b/src/oops/base/GetValuePosts.h new file mode 100644 index 000000000..512620a4d --- /dev/null +++ b/src/oops/base/GetValuePosts.h @@ -0,0 +1,90 @@ +/* + * (C) Copyright 2021-2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_BASE_GETVALUEPOSTS_H_ +#define OOPS_BASE_GETVALUEPOSTS_H_ + +#include +#include +#include +#include +#include + +#include "oops/base/GetValuePost.h" +#include "oops/base/PostBase.h" +#include "oops/interface/ChangeVariables.h" +#include "oops/interface/State.h" +#include "oops/util/DateTime.h" +#include "oops/util/Duration.h" +#include "oops/util/Logger.h" + +namespace oops { + +/// \brief Fills GeoVaLs with requested variables at requested locations during model run +template +class GetValuePosts : public PostBase> { + typedef ChangeVariables ChangeVariables_; + typedef GetValues GetValues_; + typedef State State_; + typedef std::shared_ptr> GetValuePtr_; + + public: +/// \brief Saves Locations and Variables to be processed + GetValuePosts(); + + void append(GetValuePtr_); + + private: +/// \brief initialization before model run: sets up GetValues and allocate GeoVaLs + void doInitialize(const State_ &, const util::DateTime &, const util::Duration &) override; +/// \brief called at each model step: fill in GeoVaLs for the current time slot + void doProcessing(const State_ &) override; + +// Data + std::vector getvals_; +}; + +// ----------------------------------------------------------------------------- + +template +GetValuePosts::GetValuePosts() : PostBase(), getvals_() { + Log::trace() << "GetValuePosts::GetValuePosts" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void GetValuePosts::append(GetValuePtr_ getval) { + Log::trace() << "GetValuePosts::append start" << std::endl; + getvals_.push_back(getval); + Log::trace() << "GetValuePosts::append done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void GetValuePosts::doInitialize(const State_ &, const util::DateTime &, + const util::Duration & tstep) { + Log::trace() << "GetValuePosts::doInitialize start" << std::endl; + for (GetValuePtr_ getval : getvals_) getval->initialize(tstep); + Log::trace() << "GetValuePosts::doInitialize done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void GetValuePosts::doProcessing(const State_ & xx) { + Log::trace() << "GetValuePosts::doProcessing start" << std::endl; + for (GetValuePtr_ getval : getvals_) getval->process(xx); + Log::trace() << "GetValuePosts::doProcessing done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_BASE_GETVALUEPOSTS_H_ diff --git a/src/oops/base/Observers.h b/src/oops/base/Observers.h index dce1bf161..4d9274462 100644 --- a/src/oops/base/Observers.h +++ b/src/oops/base/Observers.h @@ -14,6 +14,7 @@ #include "eckit/config/LocalConfiguration.h" +#include "oops/base/GetValuePosts.h" #include "oops/base/ObsAuxControls.h" #include "oops/base/Observations.h" #include "oops/base/Observer.h" @@ -33,6 +34,7 @@ namespace oops { template class Observers { typedef Geometry Geometry_; + typedef GetValuePosts GetValuePosts_; typedef ObsAuxControls ObsAuxCtrls_; typedef Observations Observations_; typedef Observer Observer_; @@ -54,15 +56,15 @@ class Observers { private: const ObsSpaces_ & obspaces_; // ObsSpaces used in H(x) + std::vector obserrs_; std::vector> observers_; - std::vector> obserrs_; }; // ----------------------------------------------------------------------------- template Observers::Observers(const ObsSpaces_ & obspaces, const eckit::Configuration & config) - : obspaces_(obspaces) + : obspaces_(obspaces), obserrs_(), observers_() { Log::trace() << "Observers::Observers start" << std::endl; @@ -79,11 +81,14 @@ template void Observers::initialize(const Geometry_ & geom, const ObsAuxCtrls_ & obsaux, PostProc_ & pp) { Log::trace() << "Observers::initialize start" << std::endl; + obserrs_.reserve(observers_.size()); + std::shared_ptr getvals(new GetValuePosts_()); for (size_t jj = 0; jj < observers_.size(); ++jj) { - obserrs_.emplace_back(new ObsVector_(obspaces_[jj], "ObsError")); - pp.enrollProcessor(observers_[jj]->initialize(geom, obsaux[jj], *obserrs_[jj])); + obserrs_.emplace_back(obspaces_[jj], "ObsError"); + getvals->append(observers_[jj]->initialize(geom, obsaux[jj], obserrs_[jj])); } + pp.enrollProcessor(getvals); Log::trace() << "Observers::initialize done" << std::endl; } @@ -96,7 +101,7 @@ void Observers::finalize(Observations_ & yobs) { for (size_t jj = 0; jj < observers_.size(); ++jj) { observers_[jj]->finalize(yobs[jj]); - obserrs_[jj]->save("EffectiveError"); // Obs error covariance is looking for that for now + obserrs_[jj].save("EffectiveError"); // Obs error covariance is looking for that for now } obserrs_.clear(); From bd81537bb4ccb1bf2ad9c70ba187a3d0d7b97944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Mon, 5 Apr 2021 09:55:13 -0600 Subject: [PATCH 096/142] Very minor tidy-up in toy models (#1138) * Very minor tidy-up in toy models * More cleaning --- l95/src/lorenz95/BackgroundCheck.cc | 6 ++++++ l95/src/lorenz95/GomL95.cc | 32 +++++++++++------------------ l95/src/lorenz95/IncrementL95.h | 2 -- qg/model/IncrementQG.h | 4 +--- 4 files changed, 19 insertions(+), 25 deletions(-) diff --git a/l95/src/lorenz95/BackgroundCheck.cc b/l95/src/lorenz95/BackgroundCheck.cc index 888a6855b..c6c1d6352 100644 --- a/l95/src/lorenz95/BackgroundCheck.cc +++ b/l95/src/lorenz95/BackgroundCheck.cc @@ -29,17 +29,23 @@ BackgroundCheck::BackgroundCheck(const ObsTableView & obsdb, const Parameters_ & void BackgroundCheck::postFilter(const ObsVec1D & hofx, const ObsDiags1D &) const { std::vector yobs; obsdb_.getdb("ObsValue", yobs); + size_t inflate = 0; + size_t ireject = 0; for (size_t jj = 0; jj < yobs.size(); ++jj) { if (std::abs(yobs[jj] - hofx[jj]) > options_.threshold) { // inflate obs error variance if (options_.inflation.value() != boost::none) { (*obserr_)[jj] *= *options_.inflation.value(); + ++inflate; // or reject observation } else { (*qcflags_)[jj] = 1; + ++ireject; } } } + oops::Log::info() << "BackgroundCheck::postFilter rejected = " << ireject + << ", inflated = " << inflate << std::endl; } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/GomL95.cc b/l95/src/lorenz95/GomL95.cc index 0ac19ee5c..6f628adb8 100644 --- a/l95/src/lorenz95/GomL95.cc +++ b/l95/src/lorenz95/GomL95.cc @@ -132,28 +132,20 @@ void GomL95::write(const eckit::Configuration & conf) const { } // ----------------------------------------------------------------------------- void GomL95::print(std::ostream & os) const { - double zmin = locval_[0]; - double zmax = locval_[0]; - size_t jmax = 0; - double zavg = 0.0; - for (size_t jj = 0; jj < size_; ++jj) { - if (locval_[jj] < zmin) zmin = locval_[jj]; - if (locval_[jj] > zmax) { - zmax = locval_[jj]; - jmax = jj; + if (size_ > 0) { + double zmin = locval_[0]; + double zmax = locval_[0]; + double zavg = 0.0; + for (size_t jj = 0; jj < size_; ++jj) { + if (locval_[jj] < zmin) zmin = locval_[jj]; + if (locval_[jj] > zmax) zmax = locval_[jj]; + zavg += locval_[jj]; } - zavg += locval_[jj]; + zavg /= size_; + os << size_ << "values, Min=" << zmin << ", Max=" << zmax << ", Average=" << zavg; + } else { + os << " No observations"; } - zavg /= size_; - os << size_ << "values, Min=" << zmin << ", Max=" << zmax << ", Average=" << zavg; - - // If the min value across all variables is positive, then this may be an - // error measurement. If so, print the location where the maximum occurs - // to the debug stream, for use in debugging - - if (zmin >= 0.0) - oops::Log::debug() << std::endl << "GomL95: Maximum Value = " << std::setprecision(4) - << zmax << " at location = " << jmax << std::endl; } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/IncrementL95.h b/l95/src/lorenz95/IncrementL95.h index 2c6cb4f51..890a7d757 100644 --- a/l95/src/lorenz95/IncrementL95.h +++ b/l95/src/lorenz95/IncrementL95.h @@ -23,7 +23,6 @@ #include "lorenz95/Iterator.h" #include "lorenz95/Resolution.h" -#include "oops/base/GeneralizedDepartures.h" #include "oops/base/LocalIncrement.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" @@ -56,7 +55,6 @@ namespace lorenz95 { // ----------------------------------------------------------------------------- class IncrementL95 : public util::Printable, public util::Serializable, - public oops::GeneralizedDepartures, private util::ObjectCounter { public: static const std::string classname() {return "lorenz95::IncrementL95";} diff --git a/qg/model/IncrementQG.h b/qg/model/IncrementQG.h index d7f625550..3a7c94392 100644 --- a/qg/model/IncrementQG.h +++ b/qg/model/IncrementQG.h @@ -21,7 +21,6 @@ #include "eckit/config/LocalConfiguration.h" -#include "oops/base/GeneralizedDepartures.h" #include "oops/base/LocalIncrement.h" #include "oops/util/DateTime.h" #include "oops/util/dot_product.h" @@ -60,8 +59,7 @@ namespace qg { // ----------------------------------------------------------------------------- -class IncrementQG : public oops::GeneralizedDepartures, - public util::Printable, +class IncrementQG : public util::Printable, public util::Serializable, private util::ObjectCounter { public: From 35923ceb362c9933dbdbf3d0d8f24688cf86f205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Tue, 6 Apr 2021 16:36:21 +0100 Subject: [PATCH 097/142] Allow ObsAuxControl, -Increment and -Covariance objects to be configured with a subclass of Parameters instead of a Configuration (#1129) * Made the constructors of ObsAux* classes take a Parameters object rather than a Configuration. * Replaced the Configuration parameters of the ObsBias* class constructors in the L95 and QG models with GenericParameters. * Moved YAML options used by the ObsAuxControl test but not by the ObsAuxControl class itself from the "obs bias" section to a new "obs bias test" section. * Lorenz95: Parameters subclass encapsulating bias correction options (#1130) * Encapsulated Lorenz95's bias correction options in a Parameters subclass. * Corrected a copyright message. * QG: Parameters subclass encapsulating bias correction options (#1131) * Encapsulated QG's bias correction options in a Parameters subclass. * Removed unnecessary quotes from YAML files. * Removed further unnecessary quotes. * Corrected a copyright message. * Replaced vectors by 4-element arrays in QG's ObsBias and ObsBiasCovariance. * ObsAuxControl::read()/write(): take Parameters_ instead of a Configuration. * trigger CI Co-authored-by: Anna Shlyaeva Co-authored-by: Anna Shlyaeva --- l95/src/lorenz95/CMakeLists.txt | 1 + l95/src/lorenz95/ObsBias.cc | 8 ++-- l95/src/lorenz95/ObsBias.h | 14 +++---- l95/src/lorenz95/ObsBiasCorrection.cc | 8 ++-- l95/src/lorenz95/ObsBiasCorrection.h | 6 ++- l95/src/lorenz95/ObsBiasCovariance.cc | 8 ++-- l95/src/lorenz95/ObsBiasCovariance.h | 6 ++- l95/src/lorenz95/ObsBiasParameters.h | 41 ++++++++++++++++++++ l95/test/testinput/interfaces.yaml | 5 ++- qg/model/CMakeLists.txt | 1 + qg/model/ObsBias.cc | 25 +++++++----- qg/model/ObsBias.h | 17 ++++---- qg/model/ObsBiasCovariance.cc | 23 ++++++----- qg/model/ObsBiasCovariance.h | 13 ++++--- qg/model/ObsBiasIncrement.cc | 15 ++++---- qg/model/ObsBiasIncrement.h | 5 ++- qg/model/ObsBiasParameters.h | 47 +++++++++++++++++++++++ qg/test/testinput/4dvar_obs_biased.yaml | 4 +- qg/test/testinput/interfaces.yaml | 14 ++++--- qg/test/testinput/make_obs_4d_biased.yaml | 4 +- src/oops/base/ObsAuxControls.h | 23 +++++++++-- src/oops/base/ObsAuxCovariances.h | 4 +- src/oops/base/ObsAuxIncrements.h | 4 +- src/oops/interface/ObsAuxControl.h | 26 ++++++------- src/oops/interface/ObsAuxCovariance.h | 14 ++++--- src/oops/interface/ObsAuxIncrement.h | 12 +++--- src/test/interface/LinearObsOperator.h | 28 ++++++++------ src/test/interface/ObsAuxControl.h | 12 ++++-- src/test/interface/ObsAuxCovariance.h | 4 +- src/test/interface/ObsAuxIncrement.h | 36 ++++++++++++----- src/test/interface/ObsOperator.h | 4 +- 31 files changed, 298 insertions(+), 134 deletions(-) create mode 100644 l95/src/lorenz95/ObsBiasParameters.h create mode 100644 qg/model/ObsBiasParameters.h diff --git a/l95/src/lorenz95/CMakeLists.txt b/l95/src/lorenz95/CMakeLists.txt index 03f858e2f..3871770a0 100644 --- a/l95/src/lorenz95/CMakeLists.txt +++ b/l95/src/lorenz95/CMakeLists.txt @@ -40,6 +40,7 @@ set( _l95_srcs ObsBiasCorrection.h ObsBiasCovariance.cc ObsBiasCovariance.h + ObsBiasParameters.h ObsDiags1D.h ObsLocGC99.cc ObsLocGC99.h diff --git a/l95/src/lorenz95/ObsBias.cc b/l95/src/lorenz95/ObsBias.cc index 3c885dd90..b44290049 100644 --- a/l95/src/lorenz95/ObsBias.cc +++ b/l95/src/lorenz95/ObsBias.cc @@ -21,12 +21,12 @@ // ----------------------------------------------------------------------------- namespace lorenz95 { // ----------------------------------------------------------------------------- -ObsBias::ObsBias(const ObsTableView &, const eckit::Configuration & conf) +ObsBias::ObsBias(const ObsTableView &, const Parameters_ & params) : bias_(0.0), active_(false), geovars_(std::vector{"x"}), hdiags_() { - oops::Log::trace() << "ObsBias::ObsBias conf is:" << conf << std::endl; - if (conf.has("bias")) { - bias_ = conf.getDouble("bias"); + oops::Log::trace() << "ObsBias::ObsBias conf is:" << params << std::endl; + if (params.bias.value() != boost::none) { + bias_ = *params.bias.value(); active_ = true; oops::Log::info() << "ObsBias::ObsBias created, bias = " << bias_ << std::endl; } diff --git a/l95/src/lorenz95/ObsBias.h b/l95/src/lorenz95/ObsBias.h index 986eaf9cf..543c43da8 100644 --- a/l95/src/lorenz95/ObsBias.h +++ b/l95/src/lorenz95/ObsBias.h @@ -15,14 +15,12 @@ #include #include +#include "lorenz95/ObsBiasParameters.h" + #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" -namespace eckit { - class Configuration; -} - namespace lorenz95 { class ObsBiasCorrection; class ObsTableView; @@ -35,9 +33,11 @@ class ObsBias : public util::Printable, private boost::noncopyable, private util::ObjectCounter { public: + typedef ObsBiasParameters Parameters_; + static const std::string classname() {return "lorenz95::ObsBias";} - ObsBias(const ObsTableView &, const eckit::Configuration &); + ObsBias(const ObsTableView &, const Parameters_ &); ObsBias(const ObsBias &, const bool); ~ObsBias() {} @@ -48,8 +48,8 @@ class ObsBias : public util::Printable, double & value() {return bias_;} /// I/O and diagnostics - void read(const eckit::Configuration &) {} - void write(const eckit::Configuration &) const {} + void read(const Parameters_ &) {} + void write(const Parameters_ &) const {} double norm() const {return std::abs(bias_);} /// Other diff --git a/l95/src/lorenz95/ObsBiasCorrection.cc b/l95/src/lorenz95/ObsBiasCorrection.cc index f66a52d31..e08f68d3c 100644 --- a/l95/src/lorenz95/ObsBiasCorrection.cc +++ b/l95/src/lorenz95/ObsBiasCorrection.cc @@ -21,11 +21,13 @@ // ----------------------------------------------------------------------------- namespace lorenz95 { // ----------------------------------------------------------------------------- -ObsBiasCorrection::ObsBiasCorrection(const ObsTableView &, const eckit::Configuration & conf) +ObsBiasCorrection::ObsBiasCorrection(const ObsTableView &, const Parameters_ & params) : bias_(0.0), active_(false) { - const eckit::LocalConfiguration covconf = conf.getSubConfiguration("covariance"); - active_ = covconf.has("standard_deviation"); + if (params.covariance.value() != boost::none && + params.covariance.value()->standardDeviation.value() != boost::none) { + active_ = true; + } if (active_) {oops::Log::trace() << "ObsBiasCorrection::ObsBiasCorrection created." << std::endl;} } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObsBiasCorrection.h b/l95/src/lorenz95/ObsBiasCorrection.h index 754afff38..65ffaea56 100644 --- a/l95/src/lorenz95/ObsBiasCorrection.h +++ b/l95/src/lorenz95/ObsBiasCorrection.h @@ -15,6 +15,8 @@ #include #include +#include "lorenz95/ObsBiasParameters.h" + #include "oops/util/Printable.h" #include "oops/util/Serializable.h" @@ -31,9 +33,11 @@ namespace lorenz95 { class ObsBiasCorrection : public util::Printable, public util::Serializable { public: + typedef ObsBiasParameters Parameters_; + /// Constructor, destructor ObsBiasCorrection(); - ObsBiasCorrection(const ObsTableView &, const eckit::Configuration &); + ObsBiasCorrection(const ObsTableView &, const Parameters_ &); ObsBiasCorrection(const ObsBiasCorrection &, const bool copy = true); ~ObsBiasCorrection() {} diff --git a/l95/src/lorenz95/ObsBiasCovariance.cc b/l95/src/lorenz95/ObsBiasCovariance.cc index c47914476..ad98b35b0 100644 --- a/l95/src/lorenz95/ObsBiasCovariance.cc +++ b/l95/src/lorenz95/ObsBiasCovariance.cc @@ -24,13 +24,13 @@ // ----------------------------------------------------------------------------- namespace lorenz95 { // ----------------------------------------------------------------------------- -ObsBiasCovariance::ObsBiasCovariance(const ObsTableView &, const eckit::Configuration & conf) +ObsBiasCovariance::ObsBiasCovariance(const ObsTableView &, const Parameters_ & params) : variance_(0.0), active_(false) { - const eckit::LocalConfiguration covconf = conf.getSubConfiguration("covariance"); - if (covconf.has("standard_deviation")) { + if (params.covariance.value() != boost::none && + params.covariance.value()->standardDeviation.value() != boost::none) { active_ = true; - const double zz = covconf.getDouble("standard_deviation"); + const double zz = *params.covariance.value()->standardDeviation.value(); variance_ = zz * zz; ASSERT(variance_ > 0.0); oops::Log::info() << "ObsBiasCovariance variance = " << variance_ << std::endl; diff --git a/l95/src/lorenz95/ObsBiasCovariance.h b/l95/src/lorenz95/ObsBiasCovariance.h index 1b842c10c..16c199713 100644 --- a/l95/src/lorenz95/ObsBiasCovariance.h +++ b/l95/src/lorenz95/ObsBiasCovariance.h @@ -15,7 +15,7 @@ #include #include -#include "eckit/config/LocalConfiguration.h" +#include "lorenz95/ObsBiasParameters.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" @@ -34,10 +34,12 @@ class ObsBiasCovariance : public util::Printable, private boost::noncopyable, private util::ObjectCounter { public: + typedef ObsBiasParameters Parameters_; + static const std::string classname() {return "lorenz95::ObsBiasCovariance";} /// Constructor, destructor - ObsBiasCovariance(const ObsTableView &, const eckit::Configuration &); + ObsBiasCovariance(const ObsTableView &, const Parameters_ &); ~ObsBiasCovariance() {} /// Linear algebra operators diff --git a/l95/src/lorenz95/ObsBiasParameters.h b/l95/src/lorenz95/ObsBiasParameters.h new file mode 100644 index 000000000..027100441 --- /dev/null +++ b/l95/src/lorenz95/ObsBiasParameters.h @@ -0,0 +1,41 @@ +/* + * (C) Crown copyright 2021, Met Office. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef LORENZ95_OBSBIASPARAMETERS_H_ +#define LORENZ95_OBSBIASPARAMETERS_H_ + +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameters.h" + +namespace lorenz95 { + +/// Parameters taken by the ObsBias, ObsBiasCorrection and ObsBiasCovariance classes. + +// ----------------------------------------------------------------------------- + +class ObsBiasCovarianceParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsBiasCovarianceParameters, Parameters) + + public: + oops::OptionalParameter standardDeviation{"standard_deviation", this}; +}; + +// ----------------------------------------------------------------------------- + +class ObsBiasParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsBiasParameters, Parameters) + + public: + oops::OptionalParameter bias{"bias", this}; + oops::OptionalParameter covariance{"covariance", this}; +}; + +// ----------------------------------------------------------------------------- + +} // namespace lorenz95 + +#endif // LORENZ95_OBSBIASPARAMETERS_H_ diff --git a/l95/test/testinput/interfaces.yaml b/l95/test/testinput/interfaces.yaml index 68d9b929b..0851d665d 100644 --- a/l95/test/testinput/interfaces.yaml +++ b/l95/test/testinput/interfaces.yaml @@ -109,10 +109,11 @@ observations: tolerance TL: '1.0e-14' obs bias: bias: 0.3 - norm: 0.3 - relative tolerance: 0.0 covariance: standard_deviation: 0.5 + obs bias test: + norm: 0.3 + relative tolerance: 0.0 obs localization: lengthscale: 0.2 localization method: Gaspari-Cohn diff --git a/qg/model/CMakeLists.txt b/qg/model/CMakeLists.txt index 61f198e52..3e5908dea 100644 --- a/qg/model/CMakeLists.txt +++ b/qg/model/CMakeLists.txt @@ -32,6 +32,7 @@ ModelQG.cc ModelQG.h ObsBias.cc ObsBias.h +ObsBiasParameters.h ObsBiasCovariance.cc ObsBiasCovariance.h ObsBiasIncrement.cc diff --git a/qg/model/ObsBias.cc b/qg/model/ObsBias.cc index 6e8a29cba..c5de85e15 100644 --- a/qg/model/ObsBias.cc +++ b/qg/model/ObsBias.cc @@ -23,16 +23,19 @@ // ----------------------------------------------------------------------------- namespace qg { // ----------------------------------------------------------------------------- -ObsBias::ObsBias(const ObsSpaceQG &, const eckit::Configuration & conf) - : bias_(ntypes, 0.0), active_(false), geovars_(), hdiags_() { - oops::Log::info() << "ObsBias: conf = " << conf << std::endl; - active_ = conf.has("stream") || conf.has("uwind") || - conf.has("vwind") || conf.has("wspeed"); +ObsBias::ObsBias(const ObsSpaceQG &, const Parameters_ & params) + : active_(false), geovars_(), hdiags_() { + oops::Log::info() << "ObsBias: conf = " << params << std::endl; + bias_.fill(0.0); + active_ = params.stream.value() != boost::none || + params.uwind.value() != boost::none || + params.vwind.value() != boost::none || + params.wspeed.value() != boost::none; if (active_) { - if (conf.has("stream")) bias_[0] = conf.getDouble("stream"); - if (conf.has("uwind")) bias_[1] = conf.getDouble("uwind"); - if (conf.has("vwind")) bias_[2] = conf.getDouble("vwind"); - if (conf.has("wspeed")) bias_[3] = conf.getDouble("wspeed"); + if (params.stream.value() != boost::none) bias_[0] = *params.stream.value(); + if (params.uwind.value() != boost::none) bias_[1] = *params.uwind.value(); + if (params.vwind.value() != boost::none) bias_[2] = *params.vwind.value(); + if (params.wspeed.value() != boost::none) bias_[3] = *params.wspeed.value(); std::string strn = ""; for (unsigned int jj = 0; jj < ObsBias::ntypes; ++jj) { if (jj > 0) strn += ", "; @@ -45,11 +48,13 @@ ObsBias::ObsBias(const ObsSpaceQG &, const eckit::Configuration & conf) } // ----------------------------------------------------------------------------- ObsBias::ObsBias(const ObsBias & other, const bool copy) - : bias_(ntypes, 0.0), active_(other.active_), + : active_(other.active_), geovars_(other.geovars_), hdiags_(other.hdiags_) { if (active_ && copy) { for (unsigned int jj = 0; jj < ntypes; ++jj) bias_[jj] = other.bias_[jj]; + } else { + bias_.fill(0.0); } } // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsBias.h b/qg/model/ObsBias.h index b5ba5f4eb..0f8d6b187 100644 --- a/qg/model/ObsBias.h +++ b/qg/model/ObsBias.h @@ -11,19 +11,16 @@ #ifndef QG_MODEL_OBSBIAS_H_ #define QG_MODEL_OBSBIAS_H_ +#include #include #include -#include #include +#include "model/ObsBiasParameters.h" #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" -namespace eckit { - class Configuration; -} - namespace qg { class ObsBiasIncrement; class ObsSpaceQG; @@ -36,10 +33,12 @@ class ObsBias : public util::Printable, private boost::noncopyable, private util::ObjectCounter { public: + typedef ObsBiasParameters Parameters_; + static const unsigned int ntypes = 4; static const std::string classname() {return "qg::ObsBias";} - ObsBias(const ObsSpaceQG &, const eckit::Configuration &); + ObsBias(const ObsSpaceQG &, const Parameters_ &); ObsBias(const ObsBias &, const bool); ~ObsBias() {} @@ -47,8 +46,8 @@ class ObsBias : public util::Printable, ObsBias & operator=(const ObsBias &); /// I/O and diagnostics - void read(const eckit::Configuration &) {} - void write(const eckit::Configuration &) const {} + void read(const Parameters_ &) {} + void write(const Parameters_ &) const {} double norm() const; const double & operator[](const unsigned int ii) const {return bias_[ii];} @@ -63,7 +62,7 @@ class ObsBias : public util::Printable, private: void print(std::ostream &) const; - std::vector bias_; + std::array bias_; bool active_; const oops::Variables geovars_; const oops::Variables hdiags_; diff --git a/qg/model/ObsBiasCovariance.cc b/qg/model/ObsBiasCovariance.cc index b2402ff30..57168b567 100644 --- a/qg/model/ObsBiasCovariance.cc +++ b/qg/model/ObsBiasCovariance.cc @@ -10,15 +10,13 @@ #include "model/ObsBiasCovariance.h" +#include #include #include #include #include #include -#include -#include "eckit/config/LocalConfiguration.h" -#include "model/ObsBias.h" #include "model/ObsBiasIncrement.h" #include "oops/util/Logger.h" #include "oops/util/Random.h" @@ -26,16 +24,17 @@ // ----------------------------------------------------------------------------- namespace qg { // ----------------------------------------------------------------------------- -ObsBiasCovariance::ObsBiasCovariance(const ObsSpaceQG &, const eckit::Configuration & conf) - : variance_(ObsBias::ntypes, 0.0) +ObsBiasCovariance::ObsBiasCovariance(const ObsSpaceQG &, const Parameters_ & params) { - const eckit::LocalConfiguration covconf = conf.getSubConfiguration("covariance"); - - std::vector zz(4, 0.0); - if (covconf.has("stream")) zz[0] = covconf.getDouble("stream"); - if (covconf.has("uwind")) zz[1] = covconf.getDouble("uwind"); - if (covconf.has("vwind")) zz[2] = covconf.getDouble("vwind"); - if (covconf.has("wspeed")) zz[3] = covconf.getDouble("wspeed"); + std::array zz; + zz.fill(0.0); + if (params.covariance.value() != boost::none) { + const ObsBiasCovarianceParameters& covparams = *params.covariance.value(); + if (covparams.stream.value() != boost::none) zz[0] = *covparams.stream.value(); + if (covparams.uwind.value() != boost::none) zz[1] = *covparams.uwind.value(); + if (covparams.vwind.value() != boost::none) zz[2] = *covparams.vwind.value(); + if (covparams.wspeed.value() != boost::none) zz[3] = *covparams.wspeed.value(); + } std::string strn = ""; for (unsigned int jj = 0; jj < ObsBias::ntypes; ++jj) { if (jj > 0) strn += ", "; diff --git a/qg/model/ObsBiasCovariance.h b/qg/model/ObsBiasCovariance.h index b5597d43b..8565958e4 100644 --- a/qg/model/ObsBiasCovariance.h +++ b/qg/model/ObsBiasCovariance.h @@ -11,14 +11,15 @@ #ifndef QG_MODEL_OBSBIASCOVARIANCE_H_ #define QG_MODEL_OBSBIASCOVARIANCE_H_ +#include #include #include -#include #include -#include "eckit/config/LocalConfiguration.h" - +#include "model/ObsBias.h" +#include "model/ObsBiasParameters.h" #include "oops/util/ObjectCounter.h" +#include "oops/util/parameters/GenericParameters.h" #include "oops/util/Printable.h" namespace qg { @@ -32,10 +33,12 @@ class ObsBiasCovariance : public util::Printable, private boost::noncopyable, private util::ObjectCounter { public: + typedef ObsBiasParameters Parameters_; + static const std::string classname() {return "qg::ObsBiasCovariance";} /// Constructor, destructor - ObsBiasCovariance(const ObsSpaceQG &, const eckit::Configuration &); + ObsBiasCovariance(const ObsSpaceQG &, const Parameters_ &); ~ObsBiasCovariance() {} /// Linear algebra operators @@ -48,7 +51,7 @@ class ObsBiasCovariance : public util::Printable, private: void print(std::ostream &) const; - std::vector variance_; + std::array variance_; }; // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsBiasIncrement.cc b/qg/model/ObsBiasIncrement.cc index 46158f8c6..d79d3b912 100644 --- a/qg/model/ObsBiasIncrement.cc +++ b/qg/model/ObsBiasIncrement.cc @@ -23,15 +23,16 @@ // ----------------------------------------------------------------------------- namespace qg { // ----------------------------------------------------------------------------- -ObsBiasIncrement::ObsBiasIncrement(const ObsSpaceQG &, const eckit::Configuration & conf) +ObsBiasIncrement::ObsBiasIncrement(const ObsSpaceQG &, const Parameters_ & params) : bias_(ObsBias::ntypes, 0.0), active_(ObsBias::ntypes, false) { - const eckit::LocalConfiguration covconf = conf.getSubConfiguration("covariance"); - - active_[0] = covconf.has("stream"); - active_[1] = covconf.has("uwind"); - active_[2] = covconf.has("vwind"); - active_[3] = covconf.has("wspeed"); + if (params.covariance.value() != boost::none) { + const ObsBiasCovarianceParameters& covparams = *params.covariance.value(); + active_[0] = (covparams.stream.value() != boost::none); + active_[1] = (covparams.uwind.value() != boost::none); + active_[2] = (covparams.vwind.value() != boost::none); + active_[3] = (covparams.wspeed.value() != boost::none); + } bool on = false; std::string strn = ""; for (unsigned int jj = 0; jj < ObsBias::ntypes; ++jj) { diff --git a/qg/model/ObsBiasIncrement.h b/qg/model/ObsBiasIncrement.h index 5f888a8f5..dfe152a92 100644 --- a/qg/model/ObsBiasIncrement.h +++ b/qg/model/ObsBiasIncrement.h @@ -14,6 +14,7 @@ #include #include +#include "model/ObsBiasParameters.h" #include "oops/util/Printable.h" #include "oops/util/Serializable.h" @@ -30,9 +31,11 @@ namespace qg { class ObsBiasIncrement : public util::Printable, public util::Serializable { public: + typedef ObsBiasParameters Parameters_; + /// Constructor, destructor ObsBiasIncrement(); - ObsBiasIncrement(const ObsSpaceQG &, const eckit::Configuration &); + ObsBiasIncrement(const ObsSpaceQG &, const Parameters_ &); ObsBiasIncrement(const ObsBiasIncrement &, const bool copy = true); /// Linear algebra operators diff --git a/qg/model/ObsBiasParameters.h b/qg/model/ObsBiasParameters.h new file mode 100644 index 000000000..4a95b07af --- /dev/null +++ b/qg/model/ObsBiasParameters.h @@ -0,0 +1,47 @@ +/* + * (C) Crown copyright 2021, Met Office. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef QG_MODEL_OBSBIASPARAMETERS_H_ +#define QG_MODEL_OBSBIASPARAMETERS_H_ + +#include "oops/util/parameters/OptionalParameter.h" +#include "oops/util/parameters/Parameters.h" + +namespace qg { + +/// Parameters taken by the ObsBias, ObsBiasCorrection and ObsBiasCovariance classes. + +// ----------------------------------------------------------------------------- + +class ObsBiasCovarianceParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsBiasCovarianceParameters, Parameters) + + public: + oops::OptionalParameter stream{"stream", this}; + oops::OptionalParameter uwind{"uwind", this}; + oops::OptionalParameter vwind{"vwind", this}; + oops::OptionalParameter wspeed{"wspeed", this}; +}; + +// ----------------------------------------------------------------------------- + +class ObsBiasParameters : public oops::Parameters { + OOPS_CONCRETE_PARAMETERS(ObsBiasParameters, Parameters) + + public: + oops::OptionalParameter stream{"stream", this}; + oops::OptionalParameter uwind{"uwind", this}; + oops::OptionalParameter vwind{"vwind", this}; + oops::OptionalParameter wspeed{"wspeed", this}; + oops::OptionalParameter covariance{"covariance", this}; +}; + +// ----------------------------------------------------------------------------- + +} // namespace qg + +#endif // QG_MODEL_OBSBIASPARAMETERS_H_ diff --git a/qg/test/testinput/4dvar_obs_biased.yaml b/qg/test/testinput/4dvar_obs_biased.yaml index 156b70c7b..b40b8191e 100644 --- a/qg/test/testinput/4dvar_obs_biased.yaml +++ b/qg/test/testinput/4dvar_obs_biased.yaml @@ -33,7 +33,7 @@ cost function: obs bias: stream: 0.0 covariance: - stream: '2.0e7' + stream: 2.0e7 - obs error: covariance model: diagonal obs operator: @@ -47,7 +47,7 @@ cost function: obs bias: uwind: 0.0 covariance: - uwind: '15.0' + uwind: 15.0 - obs error: covariance model: diagonal obs operator: diff --git a/qg/test/testinput/interfaces.yaml b/qg/test/testinput/interfaces.yaml index 62bca6d27..9c4aa8c33 100644 --- a/qg/test/testinput/interfaces.yaml +++ b/qg/test/testinput/interfaces.yaml @@ -75,11 +75,12 @@ observations: tolerance AD: 1.0e-10 tolerance TL: 1.0e-6 obs bias: - stream: '-10.0' + stream: -10.0 + covariance: + stream: 2.0e7 + obs bias test: norm: 10.0 relative tolerance: 0.0 - covariance: - stream: '2.0e7' rms ref: 183502589.5028424 reference nobs: 800 tolerance: 1.0e-8 @@ -102,11 +103,12 @@ observations: tolerance AD: 1.0e-10 tolerance TL: 1.0e-6 obs bias: - uwind: '10.0' + uwind: 10.0 + covariance: + uwind: 15.0 + obs bias test: norm: 10.0 relative tolerance: 0.0 - covariance: - uwind: '15.0' rms ref: 39.644266100943696 reference nobs: 800 tolerance: 1.0e-8 diff --git a/qg/test/testinput/make_obs_4d_biased.yaml b/qg/test/testinput/make_obs_4d_biased.yaml index bf84b823d..242d085d4 100644 --- a/qg/test/testinput/make_obs_4d_biased.yaml +++ b/qg/test/testinput/make_obs_4d_biased.yaml @@ -25,7 +25,7 @@ observations: obs_error: 4.0e6 obs_period: PT3H obs bias: - stream: '-1.0e7' + stream: -1.0e7 - obs operator: obs type: Wind obs space: @@ -39,7 +39,7 @@ observations: obs_error: 6.0 obs_period: PT6H obs bias: - uwind: '10.0' + uwind: 10.0 - obs operator: obs type: WSpeed obs space: diff --git a/src/oops/base/ObsAuxControls.h b/src/oops/base/ObsAuxControls.h index 17d6a6bf2..a2f90fedd 100644 --- a/src/oops/base/ObsAuxControls.h +++ b/src/oops/base/ObsAuxControls.h @@ -26,6 +26,7 @@ template class ObsAuxControls : public util::Printable { typedef ObsAuxControl ObsAuxControl_; typedef ObsSpaces ObsSpaces_; + typedef typename ObsAuxControl_::Parameters_ Parameters_; public: static const std::string classname() {return "oops::ObsAuxControls";} @@ -60,8 +61,10 @@ ObsAuxControls::ObsAuxControls(const ObsSpaces_ & odb, const eckit::Configu std::vector obsconf = conf.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsconf.size(); ++jobs) { eckit::LocalConfiguration obsauxconf = obsconf[jobs].getSubConfiguration("obs bias"); + Parameters_ obsauxparams; + obsauxparams.validateAndDeserialize(obsauxconf); auxs_.push_back( - std::unique_ptr(new ObsAuxControl_(odb[jobs], obsauxconf))); + std::unique_ptr(new ObsAuxControl_(odb[jobs], obsauxparams))); } } @@ -92,7 +95,14 @@ ObsAuxControls::~ObsAuxControls() { template void ObsAuxControls::read(const eckit::Configuration & conf) { Log::trace() << "ObsAuxControls::read starting" << std::endl; - for (std::size_t jobs = 0; jobs < auxs_.size(); ++jobs) auxs_[jobs]->read(conf); + std::vector obsconfs = conf.getSubConfigurations("observations"); + ASSERT(obsconfs.size() == auxs_.size()); + for (std::size_t jobs = 0; jobs < auxs_.size(); ++jobs) { + eckit::LocalConfiguration obsauxconf = obsconfs[jobs].getSubConfiguration("obs bias"); + Parameters_ params; + params.validateAndDeserialize(obsauxconf); + auxs_[jobs]->read(params); + } Log::trace() << "ObsAuxControls::read done" << std::endl; } @@ -101,7 +111,14 @@ void ObsAuxControls::read(const eckit::Configuration & conf) { template void ObsAuxControls::write(const eckit::Configuration & conf) const { Log::trace() << "ObsAuxControls::write starting" << std::endl; - for (std::size_t jobs = 0; jobs < auxs_.size(); ++jobs) auxs_[jobs]->write(conf); + std::vector obsconfs = conf.getSubConfigurations("observations"); + ASSERT(obsconfs.size() == auxs_.size()); + for (std::size_t jobs = 0; jobs < auxs_.size(); ++jobs) { + eckit::LocalConfiguration obsauxconf = obsconfs[jobs].getSubConfiguration("obs bias"); + Parameters_ params; + params.validateAndDeserialize(obsauxconf); + auxs_[jobs]->write(params); + } Log::trace() << "ObsAuxControls::write done" << std::endl; } diff --git a/src/oops/base/ObsAuxCovariances.h b/src/oops/base/ObsAuxCovariances.h index 518450623..afd36ae5b 100644 --- a/src/oops/base/ObsAuxCovariances.h +++ b/src/oops/base/ObsAuxCovariances.h @@ -68,8 +68,10 @@ ObsAuxCovariances::ObsAuxCovariances(const ObsSpaces_ & odb, std::vector obsconf = conf.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsconf.size(); ++jobs) { eckit::LocalConfiguration obsauxconf = obsconf[jobs].getSubConfiguration("obs bias"); + typename ObsAuxCovariance_::Parameters_ obsauxparams; + obsauxparams.validateAndDeserialize(obsauxconf); cov_.push_back( - std::unique_ptr(new ObsAuxCovariance_(odb[jobs], obsauxconf))); + std::unique_ptr(new ObsAuxCovariance_(odb[jobs], obsauxparams))); } Log::trace() << "ObsAuxCovariances::ObsAuxCovariances done" << std::endl; } diff --git a/src/oops/base/ObsAuxIncrements.h b/src/oops/base/ObsAuxIncrements.h index fccc07db2..126f1ce67 100644 --- a/src/oops/base/ObsAuxIncrements.h +++ b/src/oops/base/ObsAuxIncrements.h @@ -91,8 +91,10 @@ ObsAuxIncrements::ObsAuxIncrements(const ObsSpaces_ & odb, const eckit::Con std::vector obsconf = conf.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsconf.size(); ++jobs) { eckit::LocalConfiguration obsauxconf = obsconf[jobs].getSubConfiguration("obs bias"); + typename ObsAuxIncrement_::Parameters_ obsauxparams; + obsauxparams.validateAndDeserialize(obsauxconf); auxs_.push_back( - std::unique_ptr(new ObsAuxIncrement_(odb[jobs], obsauxconf))); + std::unique_ptr(new ObsAuxIncrement_(odb[jobs], obsauxparams))); } } // ----------------------------------------------------------------------------- diff --git a/src/oops/interface/ObsAuxControl.h b/src/oops/interface/ObsAuxControl.h index 2820ddef8..84d0a038f 100644 --- a/src/oops/interface/ObsAuxControl.h +++ b/src/oops/interface/ObsAuxControl.h @@ -21,10 +21,6 @@ #include "oops/util/Printable.h" #include "oops/util/Timer.h" -namespace eckit { - class Configuration; -} - namespace oops { class Variables; @@ -33,12 +29,14 @@ namespace oops { template class ObsAuxControl : public util::Printable, private util::ObjectCounter > { - typedef typename OBS::ObsAuxControl ObsAuxControl_; + typedef typename OBS::ObsAuxControl ObsAuxControl_; public: + typedef typename ObsAuxControl_::Parameters_ Parameters_; + static const std::string classname() {return "oops::ObsAuxControl";} - ObsAuxControl(const ObsSpace &, const eckit::Configuration &); + ObsAuxControl(const ObsSpace &, const Parameters_ ¶ms); explicit ObsAuxControl(const ObsAuxControl &, const bool copy = true); ~ObsAuxControl(); @@ -47,8 +45,8 @@ class ObsAuxControl : public util::Printable, ObsAuxControl_ & obsauxcontrol() {return *aux_;} /// I/O and diagnostics - void read(const eckit::Configuration &); - void write(const eckit::Configuration &) const; + void read(const Parameters_ &); + void write(const Parameters_ &) const; double norm() const; /// Other @@ -67,11 +65,11 @@ class ObsAuxControl : public util::Printable, template ObsAuxControl::ObsAuxControl(const ObsSpace & os, - const eckit::Configuration & conf) : aux_() + const Parameters_ & params) : aux_() { Log::trace() << "ObsAuxControl::ObsAuxControl starting" << std::endl; util::Timer timer(classname(), "ObsAuxControl"); - aux_.reset(new ObsAuxControl_(os.obsspace(), conf)); + aux_.reset(new ObsAuxControl_(os.obsspace(), params)); Log::trace() << "ObsAuxControl::ObsAuxControl done" << std::endl; } @@ -99,20 +97,20 @@ ObsAuxControl::~ObsAuxControl() { // ----------------------------------------------------------------------------- template -void ObsAuxControl::read(const eckit::Configuration & conf) { +void ObsAuxControl::read(const Parameters_ & params) { Log::trace() << "ObsAuxControl::read starting" << std::endl; util::Timer timer(classname(), "read"); - aux_->read(conf); + aux_->read(params); Log::trace() << "ObsAuxControl::read done" << std::endl; } // ----------------------------------------------------------------------------- template -void ObsAuxControl::write(const eckit::Configuration & conf) const { +void ObsAuxControl::write(const Parameters_ & params) const { Log::trace() << "ObsAuxControl::write starting" << std::endl; util::Timer timer(classname(), "write"); - aux_->write(conf); + aux_->write(params); Log::trace() << "ObsAuxControl::write done" << std::endl; } diff --git a/src/oops/interface/ObsAuxCovariance.h b/src/oops/interface/ObsAuxCovariance.h index ed3dfd1ff..95d238804 100644 --- a/src/oops/interface/ObsAuxCovariance.h +++ b/src/oops/interface/ObsAuxCovariance.h @@ -34,14 +34,16 @@ template class ObsAuxCovariance : public util::Printable, private boost::noncopyable, private util::ObjectCounter > { - typedef typename OBS::ObsAuxCovariance ObsAuxCovariance_; - typedef ObsAuxControl ObsAuxControl_; - typedef ObsAuxIncrement ObsAuxIncrement_; + typedef typename OBS::ObsAuxCovariance ObsAuxCovariance_; + typedef ObsAuxControl ObsAuxControl_; + typedef ObsAuxIncrement ObsAuxIncrement_; public: + typedef typename ObsAuxCovariance_::Parameters_ Parameters_; + static const std::string classname() {return "oops::ObsAuxCovariance";} - ObsAuxCovariance(const ObsSpace &, const eckit::Configuration &); + ObsAuxCovariance(const ObsSpace &, const Parameters_ &); ~ObsAuxCovariance(); /// Operators @@ -59,11 +61,11 @@ class ObsAuxCovariance : public util::Printable, template ObsAuxCovariance::ObsAuxCovariance(const ObsSpace & os, - const eckit::Configuration & conf) : cov_() + const Parameters_ & params) : cov_() { Log::trace() << "ObsAuxCovariance::ObsAuxCovariance starting" << std::endl; util::Timer timer(classname(), "ObsAuxCovariance"); - cov_.reset(new ObsAuxCovariance_(os.obsspace(), conf)); + cov_.reset(new ObsAuxCovariance_(os.obsspace(), params)); Log::trace() << "ObsAuxCovariance::ObsAuxCovariance done" << std::endl; } diff --git a/src/oops/interface/ObsAuxIncrement.h b/src/oops/interface/ObsAuxIncrement.h index 0514193c5..01a364d6a 100644 --- a/src/oops/interface/ObsAuxIncrement.h +++ b/src/oops/interface/ObsAuxIncrement.h @@ -36,14 +36,16 @@ template class ObsAuxIncrement : public util::Printable, public util::Serializable, private util::ObjectCounter > { - typedef typename OBS::ObsAuxIncrement ObsAuxIncrement_; - typedef ObsAuxControl ObsAuxControl_; + typedef typename OBS::ObsAuxIncrement ObsAuxIncrement_; + typedef ObsAuxControl ObsAuxControl_; public: + typedef typename ObsAuxIncrement_::Parameters_ Parameters_; + static const std::string classname() {return "oops::ObsAuxIncrement";} /// Constructor, destructor - ObsAuxIncrement(const ObsSpace &, const eckit::Configuration &); + ObsAuxIncrement(const ObsSpace &, const Parameters_ &); /// Copies \p other if \p copy is true, otherwise creates zero increment /// of the same size as \p other. ObsAuxIncrement(const ObsAuxIncrement & other, const bool copy = true); @@ -93,11 +95,11 @@ ObsAuxControl & operator+=(ObsAuxControl & xx, const ObsAuxIncrement ObsAuxIncrement::ObsAuxIncrement(const ObsSpace & os, - const eckit::Configuration & conf) : aux_() + const Parameters_ & params) : aux_() { Log::trace() << "ObsAuxIncrement::ObsAuxIncrement starting" << std::endl; util::Timer timer(classname(), "ObsAuxIncrement"); - aux_.reset(new ObsAuxIncrement_(os.obsspace(), conf)); + aux_.reset(new ObsAuxIncrement_(os.obsspace(), params)); Log::trace() << "ObsAuxIncrement::ObsAuxIncrement done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/test/interface/LinearObsOperator.h b/src/test/interface/LinearObsOperator.h index d9913c511..42737726b 100644 --- a/src/test/interface/LinearObsOperator.h +++ b/src/test/interface/LinearObsOperator.h @@ -84,8 +84,10 @@ template void testLinearity() { // initialize obs bias eckit::LocalConfiguration biasconf = conf.getSubConfiguration("obs bias"); - const ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasconf); - ObsAuxIncr_ ybinc(Test_::obspace()[jj], biasconf); + typename ObsAuxCtrl_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + const ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasparams); + ObsAuxIncr_ ybinc(Test_::obspace()[jj], biasparams); // read geovals from the file const eckit::LocalConfiguration gconf(conf, "geovals"); @@ -94,7 +96,7 @@ template void testLinearity() { const GeoVaLs_ gval(gconf, Test_::obspace()[jj], hopvars); // initialize Obs. Bias Covariance - const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biasconf); + const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biasparams); // set trajectory for TL/AD to be the geovals from the file hoptl.setTrajectory(gval, ybias); @@ -160,12 +162,14 @@ template void testAdjoint() { const double tol = conf.getDouble("linear obs operator test.tolerance AD"); // initialize bias correction eckit::LocalConfiguration biasconf = conf.getSubConfiguration("obs bias"); - const ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasconf); - ObsAuxIncr_ ybinc1(Test_::obspace()[jj], biasconf); // TL - ObsAuxIncr_ ybinc2(Test_::obspace()[jj], biasconf); // AD + typename ObsAuxCtrl_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + const ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasparams); + ObsAuxIncr_ ybinc1(Test_::obspace()[jj], biasparams); // TL + ObsAuxIncr_ ybinc2(Test_::obspace()[jj], biasparams); // AD // initialize Obs. Bias Covariance - const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biasconf); + const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biasparams); // read geovals from the file eckit::LocalConfiguration gconf(conf, "geovals"); @@ -242,11 +246,13 @@ template void testTangentLinear() { // initialize obs bias from file eckit::LocalConfiguration biasconf = conf.getSubConfiguration("obs bias"); - const ObsAuxCtrl_ ybias0(Test_::obspace()[jj], biasconf); - ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasconf); + typename ObsAuxCtrl_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + const ObsAuxCtrl_ ybias0(Test_::obspace()[jj], biasparams); + ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasparams); // initialize Obs. Bias Covariance - const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biasconf); + const ObsAuxCov_ Bobsbias(Test_::obspace()[jj], biasparams); // read geovals from the file const eckit::LocalConfiguration gconf(conf, "geovals"); @@ -274,7 +280,7 @@ template void testTangentLinear() { // randomize dx and ybinc GeoVaLs_ dx(gconf, Test_::obspace()[jj], hoptl.requiredVars()); dx.random(); - ObsAuxIncr_ ybinc(Test_::obspace()[jj], biasconf); + ObsAuxIncr_ ybinc(Test_::obspace()[jj], biasparams); Bobsbias.randomize(ybinc); // scale dx by x0 diff --git a/src/test/interface/ObsAuxControl.h b/src/test/interface/ObsAuxControl.h index 1fb9dc150..a301880bf 100644 --- a/src/test/interface/ObsAuxControl.h +++ b/src/test/interface/ObsAuxControl.h @@ -36,15 +36,17 @@ template void testConstructor() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); - std::unique_ptr bias(new ObsAux_(Test_::obspace()[jj], biasconf)); + typename ObsAux_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + std::unique_ptr bias(new ObsAux_(Test_::obspace()[jj], biasparams)); EXPECT(bias.get()); oops::Log::test() << "Testing ObsAuxControl: " << *bias << std::endl; // Not all configurations for interface tests specify "obs bias"; need to check // whether "obs bias" section is available if (Test_::config(jj).has("obs bias")) { - const double reference = Test_::config(jj).getDouble("obs bias.norm"); - const double tolerance = Test_::config(jj).getDouble("obs bias.relative tolerance"); + const double reference = Test_::config(jj).getDouble("obs bias test.norm"); + const double tolerance = Test_::config(jj).getDouble("obs bias test.relative tolerance"); EXPECT(oops::is_close_relative(bias->norm(), reference, tolerance)); } @@ -61,7 +63,9 @@ template void testCopyConstructor() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); - std::unique_ptr bias(new ObsAux_(Test_::obspace()[jj], biasconf)); + typename ObsAux_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + std::unique_ptr bias(new ObsAux_(Test_::obspace()[jj], biasparams)); std::unique_ptr other(new ObsAux_(*bias)); EXPECT(other.get()); diff --git a/src/test/interface/ObsAuxCovariance.h b/src/test/interface/ObsAuxCovariance.h index aa6c9e7ce..e412a560a 100644 --- a/src/test/interface/ObsAuxCovariance.h +++ b/src/test/interface/ObsAuxCovariance.h @@ -35,7 +35,9 @@ template void testConstructor() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); - std::unique_ptr cov(new Covariance_(Test_::obspace()[jj], biasconf)); + typename Covariance_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + std::unique_ptr cov(new Covariance_(Test_::obspace()[jj], biasparams)); EXPECT(cov.get()); oops::Log::test() << "Testing ObsAuxCovariance: " << *cov << std::endl; cov.reset(); diff --git a/src/test/interface/ObsAuxIncrement.h b/src/test/interface/ObsAuxIncrement.h index 996ba6591..1700ec255 100644 --- a/src/test/interface/ObsAuxIncrement.h +++ b/src/test/interface/ObsAuxIncrement.h @@ -53,7 +53,9 @@ template class ObsAuxIncrementFixture : private boost::noncopyabl for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); - std::shared_ptr tmp(new Covariance_(Test_::obspace()[jj], biasconf)); + typename Covariance_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + std::shared_ptr tmp(new Covariance_(Test_::obspace()[jj], biasparams)); covar_.push_back(tmp); } } @@ -71,7 +73,9 @@ template void testObsAuxIncrementConstructor() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); - AuxIncr_ dx(Test_::obspace()[jj], biasconf); + typename AuxIncr_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + AuxIncr_ dx(Test_::obspace()[jj], biasparams); oops::Log::test() << "Printing zero ObsAuxIncrement: " << dx << std::endl; EXPECT(dx.norm() == 0.0); } @@ -86,7 +90,9 @@ template void testObsAuxIncrementCopyConstructor() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { if (Test_::config(jj).has("obs bias")) { eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); - AuxIncr_ dx1(Test_::obspace()[jj], biasconf); + typename AuxIncr_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + AuxIncr_ dx1(Test_::obspace()[jj], biasparams); ObsAuxIncrementFixture::covariance(jj).randomize(dx1); oops::Log::test() << "Printing random ObsAuxIncrement: " << dx1 << std::endl; /// Test that creating new increment without copying data works @@ -114,9 +120,11 @@ template void testObsAuxIncrementTriangle() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { if (Test_::config(jj).has("obs bias")) { eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); - AuxIncr_ dx1(Test_::obspace()[jj], biasconf); + typename AuxIncr_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + AuxIncr_ dx1(Test_::obspace()[jj], biasparams); AuxTest_::covariance(jj).randomize(dx1); - AuxIncr_ dx2(Test_::obspace()[jj], biasconf); + AuxIncr_ dx2(Test_::obspace()[jj], biasparams); ObsAuxIncrementFixture::covariance(jj).randomize(dx2); // test triangle inequality @@ -144,7 +152,9 @@ template void testObsAuxIncrementOpPlusEq() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); - AuxIncr_ dx1(Test_::obspace()[jj], biasconf); + typename AuxIncr_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + AuxIncr_ dx1(Test_::obspace()[jj], biasparams); AuxTest_::covariance(jj).randomize(dx1); AuxIncr_ dx2(dx1); @@ -166,9 +176,11 @@ template void testObsAuxIncrementDotProduct() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); - AuxIncr_ dx1(Test_::obspace()[jj], biasconf); + typename AuxIncr_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + AuxIncr_ dx1(Test_::obspace()[jj], biasparams); AuxTest_::covariance(jj).randomize(dx1); - AuxIncr_ dx2(Test_::obspace()[jj], biasconf); + AuxIncr_ dx2(Test_::obspace()[jj], biasparams); AuxTest_::covariance(jj).randomize(dx2); // test symmetry of dot product @@ -188,7 +200,9 @@ template void testObsAuxIncrementZero() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); - AuxIncr_ dx(Test_::obspace()[jj], biasconf); + typename AuxIncr_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + AuxIncr_ dx(Test_::obspace()[jj], biasparams); AuxTest_::covariance(jj).randomize(dx); EXPECT(dx.norm() > 0.0); @@ -207,7 +221,9 @@ template void testObsAuxIncrementAxpy() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { eckit::LocalConfiguration biasconf = Test_::config(jj).getSubConfiguration("obs bias"); - AuxIncr_ dx1(Test_::obspace()[jj], biasconf); + typename AuxIncr_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + AuxIncr_ dx1(Test_::obspace()[jj], biasparams); AuxTest_::covariance(jj).randomize(dx1); // test axpy diff --git a/src/test/interface/ObsOperator.h b/src/test/interface/ObsOperator.h index 5c8c5e8ba..411d20ea7 100644 --- a/src/test/interface/ObsOperator.h +++ b/src/test/interface/ObsOperator.h @@ -80,7 +80,9 @@ template void testSimulateObs() { // initialize bias correction eckit::LocalConfiguration biasconf = conf.getSubConfiguration("obs bias"); - const ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasconf); + typename ObsAuxCtrl_::Parameters_ biasparams; + biasparams.validateAndDeserialize(biasconf); + const ObsAuxCtrl_ ybias(Test_::obspace()[jj], biasparams); // read geovals from the file eckit::LocalConfiguration gconf(conf, "geovals"); From a3411e93371d0ac9eed0018c9134b51ec4e1b25a Mon Sep 17 00:00:00 2001 From: Mark J Olah Date: Thu, 8 Apr 2021 08:34:35 -0600 Subject: [PATCH 098/142] Prefer find modules from jedicmake (#1146) --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 27843723d..d990005de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,8 @@ include( ${PROJECT_NAME}_compiler_flags ) include( GNUInstallDirs ) ## Dependencies +find_package( jedicmake QUIET ) # Prefer find modules from jedi-cmake + if( ENABLE_MKL ) find_package( MKL ) endif() @@ -58,7 +60,6 @@ else() endif() #Optional Dependencies -find_package( jedicmake QUIET ) if( jedicmake_FOUND AND ENABLE_GPTL ) find_package( GPTL ) add_compile_definitions(ENABLE_GPTL) From 28ea6370e9c2fb45a604400a115218a448640bd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Mon, 12 Apr 2021 14:32:50 +0100 Subject: [PATCH 099/142] Make the Variables constructor taking a vector of channels work correctly if that vector is empty (#1147) * Make the Variables constructor taking a vector of channels work correctly if that vector is empty * Added a unit test. --- src/oops/base/Variables.cc | 12 ++++++++---- src/oops/base/Variables.h | 2 +- src/test/base/Variables.h | 23 +++++++++++++++++++---- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/oops/base/Variables.cc b/src/oops/base/Variables.cc index 9cc4879fd..927ff6f99 100644 --- a/src/oops/base/Variables.cc +++ b/src/oops/base/Variables.cc @@ -67,12 +67,16 @@ Variables::Variables(const std::vector & vars, const std::string & // ----------------------------------------------------------------------------- -Variables::Variables(const std::vector & vars, const std::vector channels) +Variables::Variables(const std::vector & vars, const std::vector & channels) : convention_(""), vars_(0), channels_(channels) { Log::trace() << "Variables::Variables start " << vars << " @ " << channels << std::endl; - for (size_t jvar = 0; jvar < vars.size(); ++jvar) { - for (size_t jch = 0; jch < channels_.size(); ++jch) { - vars_.push_back(vars[jvar]+"_"+std::to_string(channels_[jch])); + if (channels.empty()) { + vars_ = vars; + } else { + for (size_t jvar = 0; jvar < vars.size(); ++jvar) { + for (size_t jch = 0; jch < channels_.size(); ++jch) { + vars_.push_back(vars[jvar]+"_"+std::to_string(channels_[jch])); + } } } Log::trace() << "Variables::Variables done" << std::endl; diff --git a/src/oops/base/Variables.h b/src/oops/base/Variables.h index 752b8262e..5d250e236 100644 --- a/src/oops/base/Variables.h +++ b/src/oops/base/Variables.h @@ -27,7 +27,7 @@ class Variables : public util::Printable { Variables(); Variables(const eckit::Configuration &, const std::string &); explicit Variables(const std::vector &, const std::string & conv = ""); - Variables(const std::vector &, const std::vector); + Variables(const std::vector & vars, const std::vector & channels); ~Variables(); diff --git a/src/test/base/Variables.h b/src/test/base/Variables.h index ee5c3e0a6..f9fd2498b 100644 --- a/src/test/base/Variables.h +++ b/src/test/base/Variables.h @@ -32,10 +32,25 @@ void testConstructor() { std::unique_ptr vars(new oops::Variables()); EXPECT(vars.get()); - const std::vector varnames{"bt", "emiss"}; - const std::vector channels{1, 2, 3, 4}; - std::unique_ptr other(new oops::Variables(varnames, channels)); - EXPECT(other.get()); + { + const std::vector varnames{"bt", "emiss"}; + const std::vector channels{1, 2, 3, 4}; + std::unique_ptr other(new oops::Variables(varnames, channels)); + EXPECT(other.get()); + const std::vector expectedVariables{"bt_1", "bt_2", "bt_3", "bt_4", + "emiss_1", "emiss_2", "emiss_3", "emiss_4"}; + EXPECT(other->variables() == expectedVariables); + EXPECT(other->channels() == channels); + } + + { + const std::vector varnames{"bt", "emiss"}; + const std::vector channels{}; + std::unique_ptr other(new oops::Variables(varnames, channels)); + EXPECT(other.get()); + EXPECT(other->variables() == varnames); + EXPECT(other->channels() == channels); + } vars.reset(); EXPECT(!vars.get()); From 95f65668e9e6a6b2693dc43d4f260dc64cb6d69b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Mon, 12 Apr 2021 09:24:58 -0600 Subject: [PATCH 100/142] Hack to get a fast 4D H(x) without breaking all models (#1149) * Hack to get a fast 4D H(x) without breaking all models * Removal of blank spaces Co-authored-by: Fabio L R Diniz <45880035+fabiolrdiniz@users.noreply.github.com> --- ewok/hofx4dhack.yaml | 14 +++ src/CMakeLists.txt | 1 + src/oops/runs/HofX4Dhack.h | 192 +++++++++++++++++++++++++++++++++++++ 3 files changed, 207 insertions(+) create mode 100644 ewok/hofx4dhack.yaml create mode 100644 src/oops/runs/HofX4Dhack.h diff --git a/ewok/hofx4dhack.yaml b/ewok/hofx4dhack.yaml new file mode 100644 index 000000000..09e926cd3 --- /dev/null +++ b/ewok/hofx4dhack.yaml @@ -0,0 +1,14 @@ +geometry: + $(GEOMETRY) +states: + - $(BACKGROUND) + - $(BACKGROUND) + - $(BACKGROUND) + - $(BACKGROUND) + - $(BACKGROUND) + - $(BACKGROUND) + - $(BACKGROUND) +observations: + $(OBSERVATIONS) +window begin: '{{window_begin}}' +window length: $(window_length) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2cd1f7515..9f5adc975 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -234,6 +234,7 @@ oops/runs/GenEnsPertB.h oops/runs/HofX3D.h oops/runs/HofX3Dslow.h oops/runs/HofX4D.h +oops/runs/HofX4Dhack.h oops/runs/LocalEnsembleDA.h oops/runs/RTPP.h oops/runs/Run.cc diff --git a/src/oops/runs/HofX4Dhack.h b/src/oops/runs/HofX4Dhack.h new file mode 100644 index 000000000..080d7e92a --- /dev/null +++ b/src/oops/runs/HofX4Dhack.h @@ -0,0 +1,192 @@ +/* + * (C) Copyright 2018-2020 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_RUNS_HOFX4DHACK_H_ +#define OOPS_RUNS_HOFX4DHACK_H_ + +#include +#include +#include +#include + +#include "eckit/config/LocalConfiguration.h" +#include "eckit/exception/Exceptions.h" + +#include "oops/assimilation/CalcHofX.h" +#include "oops/base/instantiateObsFilterFactory.h" +#include "oops/base/Observations.h" +#include "oops/base/ObsSpaces.h" +#include "oops/base/Variables.h" +#include "oops/generic/instantiateVariableChangeFactory.h" +#include "oops/interface/ChangeVariables.h" +#include "oops/interface/Geometry.h" +#include "oops/interface/GeoVaLs.h" +#include "oops/interface/GetValues.h" +#include "oops/interface/Locations.h" +#include "oops/interface/State.h" +#include "oops/mpi/mpi.h" +#include "oops/runs/Application.h" +#include "oops/util/ConfigFunctions.h" +#include "oops/util/DateTime.h" +#include "oops/util/Duration.h" +#include "oops/util/Logger.h" + +// ######################################################################## +// ######################################################################## +// ### This file is a temporary hack until all models can work with a ### +// ### single variable transform in GetValues. Then it will be removed. ### +// ######################################################################## +// ######################################################################## + +namespace oops { + +template class HofX4Dhack : public Application { + typedef CalcHofX CalcHofX_; + typedef Geometry Geometry_; + typedef GeoVaLs GeoVaLs_; + typedef GetValues GetValues_; + typedef Locations Locations_; + typedef ObsAuxControls ObsAux_; + typedef Observations Observations_; + typedef ObsSpaces ObsSpaces_; + typedef State State_; + typedef ChangeVariables ChangeVariables_; + + typedef std::vector> GeoVaLsVec_; + typedef std::vector> LocationsVec_; + typedef std::vector VariablesVec_; + + public: +// ----------------------------------------------------------------------------- + explicit HofX4Dhack(const eckit::mpi::Comm & comm = oops::mpi::world()) : Application(comm) { + instantiateObsFilterFactory(); + instantiateVariableChangeFactory(); + } +// ----------------------------------------------------------------------------- + virtual ~HofX4Dhack() {} +// ----------------------------------------------------------------------------- + int execute(const eckit::Configuration & fullConfig) const { + // Setup observation window + const util::Duration winlen(fullConfig.getString("window length")); + const util::DateTime winbgn(fullConfig.getString("window begin")); + const util::DateTime winend(winbgn + winlen); + Log::info() << "Observation window from " << winbgn << " to " << winend << std::endl; + + // Get information about states + std::vector statesConf; + if (fullConfig.has("states")) { + statesConf = fullConfig.getSubConfigurations("states"); + } else { + if (fullConfig.has("state")) { + statesConf[0] = eckit::LocalConfiguration(fullConfig, "state"); + } else { + eckit::BadParameter("Error in background state(s) configuration"); + } + } + size_t nsubwin = statesConf.size(); + + size_t ntasks = this->getComm().size(); + size_t myrank = this->getComm().rank(); + ASSERT(ntasks % nsubwin == 0); + size_t ntaskpslot = ntasks / nsubwin; + size_t mysubwin = myrank / ntaskpslot; + Log::debug() << "ntasks = " << ntasks << " nsubwin = " << nsubwin << std::endl; + + // Define local sub-window + util::Duration subWinLength = winlen; + if (nsubwin > 1) subWinLength = winlen / (nsubwin - 1); + Log::debug() << "Task " << mysubwin << " subWinLength " << subWinLength << std::endl; + util::DateTime subWinTime = winbgn + mysubwin * subWinLength; + util::DateTime subWinBegin = subWinTime - subWinLength/2; + util::DateTime subWinEnd = subWinTime + subWinLength/2; + if (mysubwin == 0) subWinBegin = subWinTime; + if (mysubwin == nsubwin - 1) subWinEnd = subWinTime; + ASSERT(subWinBegin >= winbgn); + ASSERT(subWinEnd <= winend); + Log::debug() << "Task " << mysubwin << " Obs times " << subWinTime + << ", " << subWinBegin << ", " << subWinEnd << std::endl; + // Create a communicator for same sub-window, to be used for communications in space + std::string sgeom = "comm_geom_" + std::to_string(mysubwin); + char const *geomName = sgeom.c_str(); + eckit::mpi::Comm * commSpace = &this->getComm().split(mysubwin, geomName); + // Create a communicator for same local area, to be used for communications in time + size_t myarea = commSpace->rank(); + std::string stime = "comm_time_" + std::to_string(myarea); + char const *timeName = stime.c_str(); + eckit::mpi::Comm * commTime = &this->getComm().split(myarea, timeName); + ASSERT(commTime->size() == nsubwin); + + // Setup geometry + const eckit::LocalConfiguration geometryConfig(fullConfig, "geometry"); + const Geometry_ geometry(geometryConfig, *commSpace); + + // Setup states for H(x) + Log::info() << "State configuration is:" << statesConf[mysubwin] << std::endl; + State_ xx(geometry, statesConf[mysubwin]); + Log::test() << "State: " << xx << std::endl; + Log::debug() << "Task " << mysubwin << " State time " << xx.validTime() << std::endl; + ASSERT(xx.validTime() == subWinTime); + + // Setup observations + const eckit::LocalConfiguration obsConfig(fullConfig, "observations"); + ObsSpaces_ obspaces(obsConfig, *commSpace, subWinBegin, subWinEnd, *commTime); + ObsAux_ obsaux(obspaces, obsConfig); + CalcHofX_ hofx(obspaces, obsConfig); + hofx.initialize(obsaux); + + // fill in GeoVaLs + GeoVaLsVec_ geovals; + const LocationsVec_ & locations = hofx.locations(); + const VariablesVec_ & vars = hofx.requiredVars(); + Log::debug() << "HofX4Dhack: Required hofx size = " << hofx.requiredVars().size() << std::endl; + + Variables geovars; + Log::debug() << "HofX4Dhack: Required vars size = " << vars.size() << std::endl; + for (size_t jj = 0; jj < vars.size(); ++jj) { + Log::debug() << "HofX4Dhack: Required vars:" << vars[jj] << std::endl; + geovars += vars[jj]; + } + Log::debug() << "HofX4Dhack: Required variables:" << geovars << std::endl; + eckit::LocalConfiguration chvarconf; // empty for now + ChangeVariables_ chvar(chvarconf, geometry, xx.variables(), geovars); + + State_ zz(geometry, geovars, xx.validTime()); + chvar.changeVar(xx, zz); + + std::vector getValuesConfig = + util::vectoriseAndFilter(obsConfig, "get values"); + + // loop over all observation types + for (size_t jj = 0; jj < obspaces.size(); ++jj) { + GetValues_ getvals(geometry, *locations[jj], getValuesConfig[jj]); + geovals.emplace_back(new GeoVaLs_(*locations[jj], vars[jj])); + getvals.fillGeoVaLs(zz, subWinBegin, subWinEnd, *geovals[jj]); + } + + // Compute H(x) on filled in geovals and run the filters + Observations_ yobs = hofx.compute(geovals); + hofx.saveQcFlags("EffectiveQC"); + hofx.saveObsErrors("EffectiveError"); + + // Save H(x) + Log::test() << "H(x): " << std::endl << yobs << "End H(x)" << std::endl; + Log::info() << "H(x): " << std::endl << yobs << "End H(x)" << std::endl; + yobs.save("hofx"); + + return 0; + } +// ----------------------------------------------------------------------------- + private: + std::string appname() const { + return "oops::HofX4Dhack<" + MODEL::name() + ", " + OBS::name() + ">"; + } +// ----------------------------------------------------------------------------- +}; + +} // namespace oops + +#endif // OOPS_RUNS_HOFX4DHACK_H_ From 32c4c7d82f296dd45711ee48fc64cb7a9219fa6d Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Mon, 12 Apr 2021 15:54:39 -0600 Subject: [PATCH 101/142] change computeWeights take Eigen objects on input (#1151) * make computeWeights take Eigen on input * rename some vars and add to comments --- src/oops/assimilation/GETKFSolver.h | 51 ++++++++++++++------------ src/oops/assimilation/LETKFSolver.h | 29 ++++++++------- src/oops/assimilation/LETKFSolverGSI.h | 35 ++++++++---------- 3 files changed, 59 insertions(+), 56 deletions(-) diff --git a/src/oops/assimilation/GETKFSolver.h b/src/oops/assimilation/GETKFSolver.h index 9461662e9..c7760be91 100644 --- a/src/oops/assimilation/GETKFSolver.h +++ b/src/oops/assimilation/GETKFSolver.h @@ -77,9 +77,14 @@ class GETKFSolver : public LocalEnsembleSolver { IncrementEnsemble4D_ &) override; private: - /// Computes weights - void computeWeights(const Departures_ &, const DeparturesEnsemble_ &, - const DeparturesEnsemble_ &, const Departures_ &); + /// Computes weights for ensemble update with local observations + /// \param[in] omb Observation departures (nlocalobs) + /// \param[in] Yb Ensemble perturbations for all the background memebers + /// (nens*neig, nlocalobs) + /// \param[in] YbOrig Ensemble perturbations for the members to be updated (nens, nlocalobs) + /// \param[in] invvarR Inverse of observation error variances (nlocalobs) + void computeWeights(const Eigen::VectorXd & omb, const Eigen::MatrixXd & Yb, + const Eigen::MatrixXd & YbOrig, const Eigen::VectorXd & invvarR); /// Applies weights and adds posterior inflation void applyWeights(const IncrementEnsemble4D_ &, IncrementEnsemble4D_ &, @@ -199,30 +204,22 @@ Observations GETKFSolver::computeHofX(const StateEnsemble4D_ & // ----------------------------------------------------------------------------- template -void GETKFSolver::computeWeights(const Departures_ & dy, - const DeparturesEnsemble_ & Yb, - const DeparturesEnsemble_ & YbOrig, - const Departures_ & R_invvar) { +void GETKFSolver::computeWeights(const Eigen::VectorXd & dy, + const Eigen::MatrixXd & Yb, + const Eigen::MatrixXd & YbOrig, + const Eigen::VectorXd & R_invvar) { // compute transformation matrix, save in Wa_, wa_ // Yb(nobs,neig*nens), YbOrig(nobs,nens) // uses GSI GETKF code util::Timer timer(classname(), "computeWeights"); - const int nobsl = dy.nobs(); + const int nobsl = dy.size(); - // cast oops objects to eigen obejects - // then cast eigen to eigen - Eigen::MatrixXd edy = dy.packEigen(); - Eigen::MatrixXf edy_f = edy.cast(); - - Eigen::MatrixXd eYb = Yb.packEigen(); - Eigen::MatrixXf eYb_f = eYb.cast(); - - Eigen::MatrixXd eYb2 = YbOrig.packEigen(); - Eigen::MatrixXf eYb2_f = eYb2.cast(); - - Eigen::MatrixXd eR = R_invvar.packEigen(); - Eigen::MatrixXf eR_f = eR.cast(); + // cast eigen to eigen + Eigen::MatrixXf dy_f = dy.cast(); + Eigen::MatrixXf Yb_f = Yb.cast(); + Eigen::MatrixXf YbOrig_f = YbOrig.cast(); + Eigen::MatrixXf R_invvar_f = R_invvar.cast(); Eigen::MatrixXf Wa_f(this->nanal_, this->nens_); Eigen::VectorXf wa_f(this->nanal_); @@ -231,9 +228,9 @@ void GETKFSolver::computeWeights(const Departures_ & dy, const int getkf_inflation = 0; const int denkf = 0; const int getkf = 1; - letkf_core_f90(nobsl, eYb_f.data(), eYb2_f.data(), edy_f.data(), + letkf_core_f90(nobsl, Yb_f.data(), YbOrig_f.data(), dy_f.data(), wa_f.data(), Wa_f.data(), - eR_f.data(), nanal_, neig_, + R_invvar_f.data(), nanal_, neig_, getkf_inflation, denkf, getkf); this->Wa_ = Wa_f.cast(); this->wa_ = wa_f.cast(); @@ -334,8 +331,12 @@ void GETKFSolver::measurementUpdate(const IncrementEnsemble4D_ & bkg this->copyLocalIncrement(bkg_pert, i, ana_pert); } else { // if obs are present do normal KF update + Eigen::VectorXd local_omb_vec = local_omb.packEigen(); + // get local Yb & HZ DeparturesEnsemble_ local_Yb(local_obs, this->Yb_); + Eigen::MatrixXd local_Yb_mat = local_Yb.packEigen(); DeparturesEnsemble_ local_HZ(local_obs, HZb_); + Eigen::MatrixXd local_HZ_mat = local_HZ.packEigen(); // create local obs errors ObsErrors_ local_R(this->obsconf_, local_obs); Departures_ invVarR = local_R.inverseVariance(); @@ -345,7 +346,9 @@ void GETKFSolver::measurementUpdate(const IncrementEnsemble4D_ & bkg locvector.ones(); loc.computeLocalization(i, locvector); invVarR *= locvector; - computeWeights(local_omb, local_HZ, local_Yb, invVarR); + Eigen::VectorXd invVarR_vec = invVarR.packEigen(); + // compute and apply weights + computeWeights(local_omb_vec, local_HZ_mat, local_Yb_mat, invVarR_vec); applyWeights(bkg_pert, ana_pert, i); } } diff --git a/src/oops/assimilation/LETKFSolver.h b/src/oops/assimilation/LETKFSolver.h index 50a5e9f46..6f52e567b 100644 --- a/src/oops/assimilation/LETKFSolver.h +++ b/src/oops/assimilation/LETKFSolver.h @@ -63,9 +63,12 @@ class LETKFSolver : public LocalEnsembleSolver { const GeometryIterator_ &, IncrementEnsemble4D_ &) override; protected: - /// Computes weights - virtual void computeWeights(const Departures_ &, const DeparturesEnsemble_ &, - const Departures_ &); + /// Computes weights for ensemble update with local observations + /// \param[in] omb Observation departures (nlocalobs) + /// \param[in] Yb Ensemble perturbations (nens, nlocalobs) + /// \param[in] invvarR Inverse of observation error variances (nlocalobs) + virtual void computeWeights(const Eigen::VectorXd & omb, const Eigen::MatrixXd & Yb, + const Eigen::VectorXd & invvarR); /// Applies weights and adds posterior inflation virtual void applyWeights(const IncrementEnsemble4D_ &, IncrementEnsemble4D_ &, @@ -141,7 +144,10 @@ void LETKFSolver::measurementUpdate(const IncrementEnsemble4D_ & bkg this->copyLocalIncrement(bkg_pert, i, ana_pert); } else { // if obs are present do normal KF update + Eigen::VectorXd local_omb_vec = local_omb.packEigen(); + // create local Yb DeparturesEnsemble_ local_Yb(local_obs, this->Yb_); + Eigen::MatrixXd local_Yb_mat = local_Yb.packEigen(); // create local obs errors ObsErrors_ local_R(this->obsconf_, local_obs); Departures_ invVarR = local_R.inverseVariance(); @@ -151,7 +157,10 @@ void LETKFSolver::measurementUpdate(const IncrementEnsemble4D_ & bkg locvector.ones(); loc.computeLocalization(i, locvector); invVarR *= locvector; - computeWeights(local_omb, local_Yb, invVarR); + Eigen::VectorXd local_R_vec = invVarR.packEigen(); + + // compute and apply weights + computeWeights(local_omb_vec, local_Yb_mat, local_R_vec); applyWeights(bkg_pert, ana_pert, i); } } @@ -159,9 +168,9 @@ void LETKFSolver::measurementUpdate(const IncrementEnsemble4D_ & bkg // ----------------------------------------------------------------------------- template -void LETKFSolver::computeWeights(const Departures_ & dy_oops, - const DeparturesEnsemble_ & Yb_oops, - const Departures_ & R_invvar_oops ) { +void LETKFSolver::computeWeights(const Eigen::VectorXd & dy, + const Eigen::MatrixXd & Yb, + const Eigen::VectorXd & diagInvR ) { // compute transformation matrix, save in Wa_, wa_ // uses C++ eigen interface // implements LETKF from Hunt et al. 2007 @@ -169,12 +178,6 @@ void LETKFSolver::computeWeights(const Departures_ & dy_oops, const LETKFInflationParameters & inflopt = options_.infl; - // cast oops objects to eigen - Eigen::MatrixXd dy = dy_oops.packEigen(); - - Eigen::MatrixXd Yb = Yb_oops.packEigen(); - - Eigen::VectorXd diagInvR = R_invvar_oops.packEigen(); // fill in the work matrix // work = Y^T R^-1 Y + (nens-1)/infl I double infl = inflopt.mult; diff --git a/src/oops/assimilation/LETKFSolverGSI.h b/src/oops/assimilation/LETKFSolverGSI.h index 69302fecc..c18d574df 100644 --- a/src/oops/assimilation/LETKFSolverGSI.h +++ b/src/oops/assimilation/LETKFSolverGSI.h @@ -33,9 +33,12 @@ class LETKFSolverGSI : public LETKFSolver { public: LETKFSolverGSI(ObsSpaces_ &, const Geometry_ &, const eckit::Configuration &, size_t); - /// Computes weights - void computeWeights(const Departures_ &, const DeparturesEnsemble_ &, - const Departures_ &) override; + /// Computes weights for ensemble update with local observations + /// \param[in] omb Observation departures (nlocalobs) + /// \param[in] Yb Ensemble perturbations (nens, nlocalobs) + /// \param[in] invvarR Inverse of observation error variances (nlocalobs) + virtual void computeWeights(const Eigen::VectorXd & omb, const Eigen::MatrixXd & Yb, + const Eigen::VectorXd & invvarR); }; // ----------------------------------------------------------------------------- @@ -50,23 +53,17 @@ LETKFSolverGSI::LETKFSolverGSI(ObsSpaces_ & obspaces, const Geometry // ----------------------------------------------------------------------------- template -void LETKFSolverGSI::computeWeights(const Departures_ & dy, - const DeparturesEnsemble_ & Yb, - const Departures_ & R_invvar) { +void LETKFSolverGSI::computeWeights(const Eigen::VectorXd & dy, + const Eigen::MatrixXd & Yb, + const Eigen::VectorXd & R_invvar) { // compute transformation matrix, save in Wa_, wa_ // uses GSI GETKF code - const int nobsl = dy.nobs(); + const int nobsl = dy.size(); - // cast oops objects to eigen obejects - // then cast eigen to eigen - Eigen::MatrixXd edy = dy.packEigen(); - Eigen::MatrixXf edy_f = edy.cast(); - - Eigen::MatrixXd eYb = Yb.packEigen(); - Eigen::MatrixXf eYb_f = eYb.cast(); - - Eigen::VectorXd eR = R_invvar.packEigen(); - Eigen::VectorXf eR_f = eR.cast(); + // cast eigen to eigen + Eigen::VectorXf dy_f = dy.cast(); + Eigen::MatrixXf Yb_f = Yb.cast(); + Eigen::VectorXf R_invvar_f = R_invvar.cast(); Eigen::MatrixXf Wa_f(this->nens_, this->nens_); Eigen::VectorXf wa_f(this->nens_); @@ -76,9 +73,9 @@ void LETKFSolverGSI::computeWeights(const Departures_ & dy, const int getkf_inflation = 0; const int denkf = 0; const int getkf = 0; - letkf_core_f90(nobsl, eYb_f.data(), eYb_f.data(), edy_f.data(), + letkf_core_f90(nobsl, Yb_f.data(), Yb_f.data(), dy_f.data(), wa_f.data(), Wa_f.data(), - eR_f.data(), this->nens_, neigv, + R_invvar_f.data(), this->nens_, neigv, getkf_inflation, denkf, getkf); this->Wa_ = Wa_f.cast(); this->wa_ = wa_f.cast(); From bfa880b26d990a54b2490db8f4e7786608f82b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Tue, 13 Apr 2021 10:32:49 -0600 Subject: [PATCH 102/142] Observers and GetValuePosts (#1143) * All tests pass with l95, QG, sw and fv3 * Observers in Jo * Clean-up * Some renaming * Review comments Co-authored-by: Anna Shlyaeva --- src/CMakeLists.txt | 3 +- src/oops/assimilation/CostFunction.h | 33 +-- src/oops/assimilation/CostJbTotal.h | 7 +- src/oops/assimilation/CostJcDFI.h | 90 ++++--- src/oops/assimilation/CostJo.h | 192 ++++++++------ src/oops/assimilation/CostJoType.h | 207 --------------- src/oops/assimilation/CostTermBase.h | 44 ++-- src/oops/assimilation/HBHtMatrix.h | 34 ++- src/oops/assimilation/HMatrix.h | 9 +- src/oops/assimilation/HessianMatrix.h | 32 ++- src/oops/assimilation/HtMatrix.h | 5 +- src/oops/assimilation/HtRinvHMatrix.h | 32 ++- .../assimilation/IncrementalAssimilation.h | 2 +- src/oops/assimilation/JqTermTLAD.h | 1 - src/oops/assimilation/LBHessianMatrix.h | 11 +- src/oops/assimilation/SaddlePointMatrix.h | 14 +- src/oops/base/Departures.h | 4 +- src/oops/base/GeneralizedDepartures.h | 6 +- src/oops/base/GetValuePost.h | 1 - src/oops/base/GetValuePosts.h | 2 +- src/oops/base/GetValueTLAD.h | 202 +++++++++++++++ src/oops/base/GetValueTLADs.h | 120 +++++++++ src/oops/base/Observer.h | 7 +- src/oops/base/ObserverTLAD.h | 199 +++++++-------- src/oops/base/Observers.h | 12 +- src/oops/base/ObserversTLAD.h | 236 ++++++------------ src/oops/base/PostBaseTLAD.h | 3 - src/oops/base/PostProcessorTLAD.h | 5 - src/oops/base/WeightedDiffTLAD.h | 9 +- src/oops/interface/Increment.h | 2 - 30 files changed, 819 insertions(+), 705 deletions(-) delete mode 100644 src/oops/assimilation/CostJoType.h create mode 100644 src/oops/base/GetValueTLAD.h create mode 100644 src/oops/base/GetValueTLADs.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9f5adc975..edc169a69 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -18,7 +18,6 @@ oops/assimilation/CostJbState.h oops/assimilation/CostJbTotal.h oops/assimilation/CostJcDFI.h oops/assimilation/CostJo.h -oops/assimilation/CostJoType.h oops/assimilation/CostTermBase.h oops/assimilation/DRGMRESRMinimizer.h oops/assimilation/DRIPCGMinimizer.h @@ -98,6 +97,8 @@ oops/base/GeoVaLsWriter.h oops/base/GetValuePost.h oops/base/GetValuePosts.h oops/base/GetValuesPost.h +oops/base/GetValueTLAD.h +oops/base/GetValueTLADs.h oops/base/LocalIncrement.cc oops/base/LocalIncrement.h oops/base/HybridCovariance.h diff --git a/src/oops/assimilation/CostFunction.h b/src/oops/assimilation/CostFunction.h index c927d0bf3..843d70f36 100644 --- a/src/oops/assimilation/CostFunction.h +++ b/src/oops/assimilation/CostFunction.h @@ -90,8 +90,8 @@ template class CostFunction : private boost::nonco /// Access \f$ J_b\f$ const JbTotal_ & jb() const {return *jb_;} /// Access terms of the cost function other than \f$ J_b\f$ - const CostBase_ & jterm(const unsigned ii) const {return jterms_[ii];} - unsigned nterms() const {return jterms_.size();} + const CostBase_ & jterm(const size_t ii) const {return jterms_[ii];} + size_t nterms() const {return jterms_.size();} double getCostJb() const {return costJb_;} double getCostJoJc() const {return costJoJc_;} @@ -224,10 +224,10 @@ double CostFunction::evaluate(const CtrlVar_ & fguess, PostProcessor post) { Log::trace() << "CostFunction::evaluate start" << std::endl; // Setup terms of cost function - PostProcessor pp(post); jb_->initialize(fguess); - for (unsigned jj = 0; jj < jterms_.size(); ++jj) { - jterms_[jj].initialize(fguess, config, pp); + PostProcessor pp(post); + for (size_t jj = 0; jj < jterms_.size(); ++jj) { + jterms_[jj].setPostProc(fguess, config, pp); } // Run NL model @@ -239,8 +239,8 @@ double CostFunction::evaluate(const CtrlVar_ & fguess, costJb_ = jb_->finalize(mfguess); zzz += costJb_; costJoJc_ = 0.0; - for (unsigned jj = 0; jj < jterms_.size(); ++jj) { - costJoJc_ += jterms_[jj].finalize(); + for (size_t jj = 0; jj < jterms_.size(); ++jj) { + costJoJc_ += jterms_[jj].computeCost(); } zzz += costJoJc_; Log::test() << "CostFunction: Nonlinear J = " << zzz << std::endl; @@ -262,8 +262,8 @@ double CostFunction::linearize(const CtrlVar_ & fguess, // Setup trajectory for terms of cost function PostProcessorTLAD pptraj; jb_->initializeTraj(fguess, lowres, innerConf, pptraj); - for (unsigned jj = 0; jj < jterms_.size(); ++jj) { - jterms_[jj].initializeTraj(fguess, lowres, innerConf, pptraj); + for (size_t jj = 0; jj < jterms_.size(); ++jj) { + jterms_[jj].setPostProcTraj(fguess, innerConf, lowres, pptraj); } // Specific linearization if needed (including TLM) @@ -274,8 +274,8 @@ double CostFunction::linearize(const CtrlVar_ & fguess, // Finalize trajectory setup jb_->finalizeTraj(); - for (unsigned jj = 0; jj < jterms_.size(); ++jj) { - jterms_[jj].finalizeTraj(); + for (size_t jj = 0; jj < jterms_.size(); ++jj) { + jterms_[jj].computeCostTraj(); } Log::trace() << "CostFunction::linearize done" << std::endl; @@ -291,12 +291,17 @@ void CostFunction::computeGradientFG(CtrlInc_ & grad) const { PostProcessorTLAD costad; this->zeroAD(grad); - for (unsigned jj = 0; jj < jterms_.size(); ++jj) { + for (size_t jj = 0; jj < jterms_.size(); ++jj) { std::shared_ptr tmp(jterms_[jj].newGradientFG()); - jterms_[jj].setupAD(tmp, grad, costad); + jterms_[jj].computeCostAD(tmp, grad, costad); } this->runADJ(grad, costad, pp); + + for (size_t jj = 0; jj < jterms_.size(); ++jj) { + jterms_[jj].setPostProcAD(); + } + Log::info() << "CostFunction::computeGradientFG: gradient:" << grad << std::endl; Log::trace() << "CostFunction::computeGradientFG done" << std::endl; } @@ -324,7 +329,7 @@ void CostFunction::addIncrement(CtrlVar_ & xx, const CtrlInc_ & dx, template void CostFunction::resetLinearization() { Log::trace() << "CostFunction::resetLinearization start" << std::endl; - for (unsigned jj = 0; jj < jterms_.size(); ++jj) { + for (size_t jj = 0; jj < jterms_.size(); ++jj) { jterms_[jj].resetLinearization(); } Log::trace() << "CostFunction::resetLinearization done" << std::endl; diff --git a/src/oops/assimilation/CostJbTotal.h b/src/oops/assimilation/CostJbTotal.h index e333791e3..0827b51a7 100644 --- a/src/oops/assimilation/CostJbTotal.h +++ b/src/oops/assimilation/CostJbTotal.h @@ -63,7 +63,7 @@ template class CostJbTotal { void finalizeTraj(); /// Initialize before starting the TL run. - unsigned int initializeTL(PostProcTLAD_ &) const; + void initializeTL(PostProcTLAD_ &) const; void finalizeTL(const CtrlInc_ &, CtrlInc_ &) const; /// Initialize before starting the AD run. @@ -269,14 +269,11 @@ void CostJbTotal::addGradientFG(CtrlInc_ & grad, CtrlInc_ & gradJb) // ----------------------------------------------------------------------------- template -unsigned int CostJbTotal::initializeTL(PostProcTLAD_ & pptl) const { +void CostJbTotal::initializeTL(PostProcTLAD_ & pptl) const { Log::trace() << "CostJbTotal::initializeTL start" << std::endl; jqtl_.reset(jb_->initializeJqTL()); - unsigned int iq = 0; // Hack... - if (jqtl_) iq = 1; // Hack... pptl.enrollProcessor(jqtl_); Log::trace() << "CostJbTotal::initializeTL done" << std::endl; - return iq; // Hack... } // ----------------------------------------------------------------------------- diff --git a/src/oops/assimilation/CostJcDFI.h b/src/oops/assimilation/CostJcDFI.h index 5f6b4c38e..70eae1e58 100644 --- a/src/oops/assimilation/CostJcDFI.h +++ b/src/oops/assimilation/CostJcDFI.h @@ -58,21 +58,23 @@ template class CostJcDFI : public CostTermBase, - CtrlInc_ &, PostProcTLAD_ &) const override; +/// Adjoint Jc DFI computation + void computeCostAD(std::shared_ptr, + CtrlInc_ &, PostProcTLAD_ &) const override; + void setPostProcAD() const override {} /// Multiply by \f$ C\f$ and \f$ C^{-1}\f$. std::unique_ptr @@ -108,8 +110,8 @@ template class CostJcDFI : public CostTermBase CostJcDFI::CostJcDFI(const eckit::Configuration & conf, const Geometry_ & resol, - const util::DateTime & vt, const util::Duration & span, - const util::Duration & tstep) + const util::DateTime & vt, const util::Duration & span, + const util::Duration & tstep) : vt_(vt), span_(span), alpha_(0), wfct_(), gradFG_(), resol_(resol), tstep_(tstep), tlres_(), tlstep_(), filter_(), vars_(conf, "filtered variables") { @@ -124,8 +126,8 @@ CostJcDFI::CostJcDFI(const eckit::Configuration & conf, const Geomet // ----------------------------------------------------------------------------- template -void CostJcDFI::initialize(const CtrlVar_ &, const eckit::Configuration &, - PostProc_ & pp) { +void CostJcDFI::setPostProc(const CtrlVar_ &, const eckit::Configuration &, + PostProc_ & pp) { filter_.reset(new WeightedDiff(vars_, vt_, span_, tstep_, resol_, *wfct_)); pp.enrollProcessor(filter_); @@ -134,7 +136,7 @@ void CostJcDFI::initialize(const CtrlVar_ &, const eckit::Configurat // ----------------------------------------------------------------------------- template -double CostJcDFI::finalize() { +double CostJcDFI::computeCost() { double zz = 0.5 * alpha_; std::unique_ptr dx(filter_->releaseDiff()); zz *= dot_product(*dx, *dx); @@ -145,11 +147,10 @@ double CostJcDFI::finalize() { // ----------------------------------------------------------------------------- template -void CostJcDFI::initializeTraj(const CtrlVar_ &, const Geometry_ & tlres, - const eckit::Configuration & innerConf, - PostProcTLAD_ & pptraj) { +void CostJcDFI::setPostProcTraj(const CtrlVar_ &, const eckit::Configuration & conf, + const Geometry_ & tlres, PostProcTLAD_ & pptraj) { tlres_.reset(new Geometry_(tlres)); - tlstep_ = util::Duration(innerConf.getString("linear model.tstep", tstep_.toString())); + tlstep_ = util::Duration(conf.getString("linear model.tstep", tstep_.toString())); ftlad_.reset(new WeightedDiffTLAD(vars_, vt_, span_, tstep_, *tlres_, *wfct_)); pptraj.enrollProcessor(ftlad_); } @@ -157,7 +158,7 @@ void CostJcDFI::initializeTraj(const CtrlVar_ &, const Geometry_ & t // ----------------------------------------------------------------------------- template -void CostJcDFI::finalizeTraj() { +void CostJcDFI::computeCostTraj() { gradFG_.reset(ftlad_->releaseDiff()); *gradFG_ *= alpha_; } @@ -165,35 +166,31 @@ void CostJcDFI::finalizeTraj() { // ----------------------------------------------------------------------------- template -std::unique_ptr CostJcDFI::newDualVector() const { - std::unique_ptr dx(new Increment_(*tlres_, vars_, vt_)); - return std::move(dx); -} - -// ----------------------------------------------------------------------------- - -template -std::unique_ptr CostJcDFI::newGradientFG() const { - return std::unique_ptr(new Increment_(*gradFG_)); +void CostJcDFI::setPostProcTL(const CtrlInc_ &, PostProcTLAD_ & pptl) const { + ftlad_->setupTL(*tlres_); + pptl.enrollProcessor(ftlad_); } // ----------------------------------------------------------------------------- template -void CostJcDFI::setupTL(const CtrlInc_ &, PostProcTLAD_ & pptl) const { - ftlad_->setupTL(*tlres_); - pptl.enrollProcessor(ftlad_); +void CostJcDFI::computeCostTL(const CtrlInc_ & dx, GeneralizedDepartures & gdep) const { + Log::trace() << "CostJcDFI::computeCostTL start" << std::endl; + Increment_ & ydep = dynamic_cast(gdep); + ftlad_->finalTL(ydep); + Log::trace() << "CostJcDFI::computeCostTL done" << std::endl; } // ----------------------------------------------------------------------------- template -void CostJcDFI::setupAD(std::shared_ptr pv, - CtrlInc_ &, PostProcTLAD_ & ppad) const { - std::shared_ptr - dx = std::dynamic_pointer_cast(pv); +void CostJcDFI::computeCostAD(std::shared_ptr pv, + CtrlInc_ &, PostProcTLAD_ & ppad) const { + Log::trace() << "CostJcDFI::computeCostAD start" << std::endl; + std::shared_ptr dx = std::dynamic_pointer_cast(pv); ftlad_->setupAD(dx); ppad.enrollProcessor(ftlad_); + Log::trace() << "CostJcDFI::computeCostAD done" << std::endl; } // ----------------------------------------------------------------------------- @@ -221,6 +218,21 @@ CostJcDFI::multiplyCoInv(const GeneralizedDepartures & dv1) const { // ----------------------------------------------------------------------------- +template +std::unique_ptr CostJcDFI::newDualVector() const { + std::unique_ptr dx(new Increment_(*tlres_, vars_, vt_)); + return std::move(dx); +} + +// ----------------------------------------------------------------------------- + +template +std::unique_ptr CostJcDFI::newGradientFG() const { + return std::unique_ptr(new Increment_(*gradFG_)); +} + +// ----------------------------------------------------------------------------- + template void CostJcDFI::resetLinearization() { gradFG_.reset(); diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index f199865e9..2a18fbae3 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -21,12 +21,12 @@ #include "eckit/config/LocalConfiguration.h" #include "oops/assimilation/ControlIncrement.h" #include "oops/assimilation/ControlVariable.h" -#include "oops/assimilation/CostJoType.h" #include "oops/assimilation/CostTermBase.h" #include "oops/base/Departures.h" #include "oops/base/GetValuePosts.h" #include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" +#include "oops/base/Observers.h" #include "oops/base/ObserversTLAD.h" #include "oops/base/ObsSpaces.h" #include "oops/base/PostProcessor.h" @@ -53,7 +53,6 @@ template class CostJo : public CostTermBase CtrlVar_; typedef ControlIncrement CtrlInc_; - typedef CostJoType JoType_; typedef Departures Departures_; typedef Observations Observations_; typedef Geometry Geometry_; @@ -62,6 +61,7 @@ template class CostJo : public CostTermBase Increment_; typedef ObsErrors ObsErrors_; typedef ObsSpaces ObsSpaces_; + typedef Observers Observers_; typedef ObserversTLAD ObserversTLAD_; typedef PostProcessor PostProc_; typedef PostProcessorTLAD PostProcTLAD_; @@ -76,19 +76,23 @@ template class CostJo : public CostTermBase, - CtrlInc_ &, PostProcTLAD_ &) const override; + void computeCostAD(std::shared_ptr, + CtrlInc_ &, PostProcTLAD_ &) const override; + void setPostProcAD() const override; /// Multiply by \f$ R\f$ and \f$ R^{-1}\f$. std::unique_ptr @@ -112,14 +116,18 @@ template class CostJo : public CostTermBase Rmat_; - std::vector> jos_; + Observers_ observers_; /// Jo Gradient at first guess : \f$ R^{-1} (H(x_{fg})-y_{obs}) \f$. std::unique_ptr gradFG_; /// Linearized observation operators. - std::shared_ptr pobstlad_; + std::shared_ptr obstlad_; + + /// Configuration for current initialize/finalize pair + std::unique_ptr currentConf_; }; // ============================================================================= @@ -129,113 +137,157 @@ CostJo::CostJo(const eckit::Configuration & joConf, const eckit::mpi const util::DateTime & winbgn, const util::DateTime & winend, const eckit::mpi::Comm & ctime) : obsconf_(joConf), obspaces_(obsconf_, comm, winbgn, winend, ctime), - yobs_(obspaces_, "ObsValue"), jos_(), gradFG_() + yobs_(obspaces_, "ObsValue"), Rmat_(), observers_(obspaces_, joConf), gradFG_(), + obstlad_(), currentConf_() { - Log::trace() << "CostJo::CostJo start" << std::endl; - jos_.reserve(obspaces_.size()); - std::vector confs(obsconf_.getSubConfigurations()); - for (size_t jj = 0; jj < obspaces_.size(); ++jj) { - jos_.emplace_back(new JoType_(obspaces_[jj], confs[jj])); - } - Log::trace() << "CostJo::CostJo done" << std::endl; + Log::trace() << "CostJo::CostJo" << std::endl; } // ----------------------------------------------------------------------------- template -void CostJo::initialize(const CtrlVar_ & xx, const eckit::Configuration & conf, - PostProc_ & pp) { - Log::trace() << "CostJo::initialize start" << std::endl; +void CostJo::setPostProc(const CtrlVar_ & xx, const eckit::Configuration & conf, + PostProc_ & pp) { + Log::trace() << "CostJo::setPostProc start" << std::endl; gradFG_.reset(); - std::shared_ptr getvals(new GetValuePosts_()); - for (size_t jj = 0; jj < jos_.size(); ++jj) { - getvals->append(jos_[jj]->initialize(xx.state().geometry(), xx.obsVar()[jj], conf)); - } - pp.enrollProcessor(getvals); + currentConf_.reset(new eckit::LocalConfiguration(conf)); + const int iterout = currentConf_->getInt("iteration"); + + observers_.initialize(xx.state().geometry(), xx.obsVar(), pp, iterout); - Log::trace() << "CostJo::initialize done" << std::endl; + Log::trace() << "CostJo::setPostProc done" << std::endl; } // ----------------------------------------------------------------------------- template -double CostJo::finalize() { - Log::trace() << "CostJo::finalize start" << std::endl; +double CostJo::computeCost() { + Log::trace() << "CostJo::computeCost start" << std::endl; // Obs, simulated obs and departures (held here for nice prints and diagnostics) Observations_ yeqv(obspaces_); - Departures_ ydep(obspaces_); + observers_.finalize(yeqv); - // Gradient at first guess (to define inner loop rhs) - gradFG_.reset(new Departures_(obspaces_)); + const int iterout = currentConf_->getInt("iteration"); + + // Sace current obs estimates + const std::string obsname = "hofx" + std::to_string(iterout); + yeqv.save(obsname); + + // Set observation error covariance + Rmat_.reset(new ObsErrors_(obsconf_, obspaces_)); - // Actual Jo computations - for (size_t jj = 0; jj < jos_.size(); ++jj) { - jos_[jj]->finalize(yobs_[jj], yeqv[jj], ydep[jj], (*gradFG_)[jj]); + // Perturb observations according to obs error statistics + bool obspert = currentConf_->getBool("obs perturbations", false); + if (obspert) { + yobs_.perturb(*Rmat_); + Log::info() << "Perturbed observations: " << yobs_ << std::endl; } + // Gradient at first guess (to define inner loop rhs) + Departures_ ydep(yeqv - yobs_); + gradFG_.reset(new Departures_(ydep)); + Rmat_->inverseMultiply(*gradFG_); + // Print diagnostics Log::info() << "Jo Observations:" << std::endl << yobs_ - << "End Jo Observations" << std::endl; + << "End Jo Observations" << std::endl; Log::info() << "Jo Observations Equivalent:" << std::endl << yeqv - << "End Jo Observations Equivalent" << std::endl; + << "End Jo Observations Equivalent" << std::endl; Log::info() << "Jo Bias Corrected Departures:" << std::endl << ydep - << "End Jo Bias Corrected Departures" << std::endl; + << "End Jo Bias Corrected Departures" << std::endl; // Print Jo table double zjo = 0.0; - for (size_t jj = 0; jj < jos_.size(); ++jj) { - zjo += jos_[jj]->printJo(ydep[jj], (*gradFG_)[jj]); + std::vector typeconfs = obsconf_.getSubConfigurations(); + for (size_t jj = 0; jj < obspaces_.size(); ++jj) { + double zz = 0.0; + const unsigned nobs = (*gradFG_)[jj].nobs(); + Log::test() << "CostJo : Nonlinear Jo(" << obspaces_[jj].obsname() << ") = "; + + if (nobs > 0) { + zz = 0.5 * dot_product(ydep[jj], (*gradFG_)[jj]); + const double err = (*Rmat_)[jj].getRMSE(); + Log::test() << zz << ", nobs = " << nobs << ", Jo/n = " << zz/nobs << ", err = " << err; + } else { + Log::test() << zz << " --- No Observations"; + } + + if (typeconfs[jj].getBool("monitoring only", false)) { + Log::test() << " (Monitoring only)"; + } else { + zjo += zz; + } + Log::test() << std::endl; } - Log::trace() << "CostJo::finalize done" << std::endl; + Log::trace() << "CostJo::computeCost done" << std::endl; return zjo; } // ----------------------------------------------------------------------------- template -void CostJo::initializeTraj(const CtrlVar_ & xx, const Geometry_ &, - const eckit::Configuration & conf, - PostProcTLAD_ & pptraj) { - Log::trace() << "CostJo::initializeTraj start" << std::endl; - pobstlad_.reset(new ObserversTLAD_(obsconf_, obspaces_, xx.obsVar())); - pptraj.enrollProcessor(pobstlad_); - Log::trace() << "CostJo::initializeTraj done" << std::endl; +void CostJo::setPostProcTraj(const CtrlVar_ & xx, const eckit::Configuration & conf, + const Geometry_ & lowres, PostProcTLAD_ & pptraj) { + Log::trace() << "CostJo::setPostProcTraj start" << std::endl; + obstlad_.reset(new ObserversTLAD_(obspaces_, obsconf_)); + obstlad_->initializeTraj(lowres, xx.obsVar(), pptraj); + Log::trace() << "CostJo::setPostProcTraj done" << std::endl; } // ----------------------------------------------------------------------------- template -void CostJo::finalizeTraj() { - Log::trace() << "CostJo::finalizeTraj done" << std::endl; +void CostJo::computeCostTraj() { + obstlad_->finalizeTraj(); + Log::trace() << "CostJo::computeCostTraj done" << std::endl; } // ----------------------------------------------------------------------------- template -void CostJo::setupTL(const CtrlInc_ & dx, PostProcTLAD_ & pptl) const { - Log::trace() << "CostJo::setupTL start" << std::endl; - ASSERT(pobstlad_); - pobstlad_->setupTL(dx.obsVar()); - pptl.enrollProcessor(pobstlad_); - Log::trace() << "CostJo::setupTL done" << std::endl; +void CostJo::setPostProcTL(const CtrlInc_ & dx, PostProcTLAD_ & pptl) const { + Log::trace() << "CostJo::setPostProcTL start" << std::endl; + ASSERT(obstlad_); + obstlad_->initializeTL(pptl); + Log::trace() << "CostJo::setPostProcTL done" << std::endl; } // ----------------------------------------------------------------------------- template -void CostJo::setupAD(std::shared_ptr pv, - CtrlInc_ & dx, PostProcTLAD_ & ppad) const { - Log::trace() << "CostJo::setupAD start" << std::endl; - ASSERT(pobstlad_); +void CostJo::computeCostTL(const CtrlInc_ & dx, GeneralizedDepartures & gdep) const { + Log::trace() << "CostJo::computeCostTL start" << std::endl; + ASSERT(obstlad_); + Departures_ & ydep = dynamic_cast(gdep); + obstlad_->finalizeTL(dx.obsVar(), ydep); + Log::trace() << "CostJo::computeCostTL done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void CostJo::computeCostAD(std::shared_ptr pv, + CtrlInc_ & dx, PostProcTLAD_ & ppad) const { + Log::trace() << "CostJo::computeCostAD start" << std::endl; + ASSERT(obstlad_); std::shared_ptr dy = std::dynamic_pointer_cast(pv); - pobstlad_->setupAD(dy, dx.obsVar()); - ppad.enrollProcessor(pobstlad_); - Log::trace() << "CostJo::setupAD done" << std::endl; + obstlad_->initializeAD(*dy, dx.obsVar(), ppad); + Log::trace() << "CostJo::computeCostAD done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void CostJo::setPostProcAD() const { + Log::trace() << "CostJo::setPostProcAD start" << std::endl; + ASSERT(obstlad_); + obstlad_->finalizeAD(); + Log::trace() << "CostJo::setPostProcAD done" << std::endl; } // ----------------------------------------------------------------------------- @@ -245,9 +297,7 @@ std::unique_ptr CostJo::multiplyCovar(const GeneralizedDepartures & v1) const { Log::trace() << "CostJo::multiplyCovar start" << std::endl; std::unique_ptr y1(new Departures_(dynamic_cast(v1))); - for (size_t jj = 0; jj < jos_.size(); ++jj) { - jos_[jj]->multiplyR((*y1)[jj]); - } + Rmat_->multiply(*y1); return std::move(y1); } @@ -258,9 +308,7 @@ std::unique_ptr CostJo::multiplyCoInv(const GeneralizedDepartures & v1) const { Log::trace() << "CostJo::multiplyCoInv start" << std::endl; std::unique_ptr y1(new Departures_(dynamic_cast(v1))); - for (size_t jj = 0; jj < jos_.size(); ++jj) { - jos_[jj]->inverseMultiplyR((*y1)[jj]); - } + Rmat_->inverseMultiply(*y1); return std::move(y1); } @@ -287,7 +335,7 @@ std::unique_ptr CostJo::newGradientFG() const template void CostJo::resetLinearization() { Log::trace() << "CostJo::resetLinearization start" << std::endl; - pobstlad_.reset(); + obstlad_.reset(); Log::trace() << "CostJo::resetLinearization done" << std::endl; } diff --git a/src/oops/assimilation/CostJoType.h b/src/oops/assimilation/CostJoType.h deleted file mode 100644 index 956aa1d99..000000000 --- a/src/oops/assimilation/CostJoType.h +++ /dev/null @@ -1,207 +0,0 @@ -/* - * (C) Copyright 2021-2021 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef OOPS_ASSIMILATION_COSTJOTYPE_H_ -#define OOPS_ASSIMILATION_COSTJOTYPE_H_ - -#include -#include -#include - -#include - -#include "eckit/config/LocalConfiguration.h" -#include "oops/base/GetValuePost.h" -#include "oops/base/ObsErrorBase.h" -#include "oops/base/Observer.h" -#include "oops/base/PostBase.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/ObsAuxControl.h" -#include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" -#include "oops/interface/State.h" -#include "oops/mpi/mpi.h" -#include "oops/util/DateTime.h" -#include "oops/util/Logger.h" - -namespace oops { - -// ----------------------------------------------------------------------------- - -/// Jo Cost Function -/*! - * The CostJoType class encapsulates the Jo term of the cost function for a single obs type. - */ - -template class CostJoType : private boost::noncopyable { - typedef Geometry Geometry_; - typedef ObsAuxControl ObsAuxCtrl_; - typedef ObsErrorBase ObsError_; - typedef Observer Observer_; - typedef ObsSpace ObsSpace_; - typedef ObsVector ObsVector_; - typedef PostBase> PostBase_; - typedef State State_; - typedef std::shared_ptr> GetValuePtr_; - - public: - /// Construct \f$ J_o\f$ from \f$ R\f$ and \f$ y_{obs}\f$. - CostJoType(ObsSpace_ &, const eckit::Configuration &); - - /// Destructor - virtual ~CostJoType() {} - - /// Initialize \f$ J_o\f$ before starting the integration of the model. - GetValuePtr_ initialize(const Geometry_ &, const ObsAuxCtrl_ &, const eckit::Configuration &); - - /// Finalize \f$ J_o\f$ after the integration of the model. - void finalize(ObsVector_ &, ObsVector_ &, ObsVector_ &, ObsVector_ &); - - /// Multiply by \f$ R\f$ and \f$ R^{-1}\f$. - void multiplyR(ObsVector_ &) const; - void inverseMultiplyR(ObsVector_ &) const; - - /// Print Jo - double printJo(const ObsVector_ &, const ObsVector_ &) const; - - private: - eckit::LocalConfiguration obsconf_; - ObsSpace_ & obspace_; - ObsVector_ obserr_; // Obs errors (should come from R matrix) - - std::unique_ptr Rmat_; - - /// Configuration for current initialize/finalize pair - std::unique_ptr currentConf_; - int iter_; - - /// Used for computing H(x) and running QC filters - Observer_ observer_; -}; - -// ============================================================================= - -template -CostJoType::CostJoType(ObsSpace_ & obspace, const eckit::Configuration & conf) - : obsconf_(conf), obspace_(obspace), - obserr_(obspace_, "ObsError"), - Rmat_(), currentConf_(), iter_(0), observer_(obspace_, obsconf_) -{ - Log::trace() << "CostJoType::CostJoType done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -std::shared_ptr> -CostJoType::initialize(const Geometry_ & geom, const ObsAuxCtrl_ & ybias, - const eckit::Configuration & conf) { - Log::trace() << "CostJoType::initialize start" << std::endl; - - currentConf_.reset(new eckit::LocalConfiguration(conf)); - iter_ = currentConf_->getInt("iteration"); - - GetValuePtr_ getvals = observer_.initialize(geom, ybias, obserr_, iter_); - - Log::trace() << "CostJoType::initialize done" << std::endl; - return getvals; -} - -// ----------------------------------------------------------------------------- - -template -void CostJoType::finalize(ObsVector_ & yobs, ObsVector_ & yeqv, - ObsVector_ & ydep, ObsVector_ & grad) { - Log::trace() << "CostJoType::finalize start" << std::endl; - -// Get simulated obs values from observer - observer_.finalize(yeqv); - -// Sace current simulated obs, QC flags and obs error - const std::string obsname = "hofx" + std::to_string(iter_); - yeqv.save(obsname); - - const std::string errname = "EffectiveError" + std::to_string(iter_); - obserr_.save(errname); - obserr_.save("EffectiveError"); // Obs error covariance is looking for that for now - -// Set observation error covariance - const eckit::LocalConfiguration rconf(obsconf_, "obs error"); - Rmat_ = ObsErrorFactory::create(rconf, obspace_); - -// Perturb observations according to obs error statistics - bool obspert = currentConf_->getBool("obs perturbations", false); - if (obspert) { - ObsVector_ ypert(obspace_); - Rmat_->randomize(ypert); - yobs += ypert; - Log::info() << "Perturbed observations: " << yobs << std::endl; - } - -// Compute departures and Jo gradient - ydep = yeqv; - ydep -= yobs; - grad = ydep; - Rmat_->inverseMultiply(grad); - -// Save departures for diagnostics if required - if (currentConf_->has("diagnostics.departures")) { - const std::string depname = currentConf_->getString("diagnostics.departures"); - ydep.save(depname); - } - - currentConf_.reset(); - Log::trace() << "CostJoType::finalize done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -double CostJoType::printJo(const ObsVector_ & dy, const ObsVector_ & grad) const { - Log::trace() << "CostJoType::printJo start" << std::endl; - - double zjo = 0.0; - const unsigned nobs = grad.nobs(); - Log::test() << "CostJo : Nonlinear Jo(" << obspace_.obsname() << ") = "; - - if (nobs > 0) { - zjo = 0.5 * dot_product(dy, grad); - const double err = Rmat_->getRMSE(); - Log::test() << zjo << ", nobs = " << nobs << ", Jo/n = " << zjo/nobs << ", err = " << err; - } else { - Log::test() << zjo << " --- No Observations"; - } - - if (obsconf_.getBool("monitoring only", false)) { - Log::test() << " (Monitoring only)"; - zjo = 0.0; - } - Log::test() << std::endl; - - Log::trace() << "CostJoType::printJo done" << std::endl; - return zjo; -} - -// ----------------------------------------------------------------------------- - -template -void CostJoType::multiplyR(ObsVector_ & dy) const { - Rmat_->multiply(dy); -} - -// ----------------------------------------------------------------------------- - -template -void CostJoType::inverseMultiplyR(ObsVector_ & dy) const { - Rmat_->inverseMultiply(dy); -} - -// ----------------------------------------------------------------------------- - -} // namespace oops - -#endif // OOPS_ASSIMILATION_COSTJOTYPE_H_ diff --git a/src/oops/assimilation/CostTermBase.h b/src/oops/assimilation/CostTermBase.h index ebf6bb9ce..53463472f 100644 --- a/src/oops/assimilation/CostTermBase.h +++ b/src/oops/assimilation/CostTermBase.h @@ -31,7 +31,7 @@ namespace oops { /// Base Class for Cost Function Terms /*! - * Abstract base class for the terms of the cost function. + * Abstract base class for the terms of the cost function (other than Jb). */ template class CostTermBase { @@ -45,22 +45,32 @@ template class CostTermBase { /// Destructor virtual ~CostTermBase() {} -/// Initialize before nonlinear model integration. - virtual void initialize(const ControlVariable &, const eckit::Configuration &, - PostProc_ &) = 0; - virtual void initializeTraj(const ControlVariable &, const Geometry_ &, - const eckit::Configuration &, PostProcTLAD_ &) = 0; - -/// Finalize computation after nonlinear model integration. - virtual double finalize() = 0; - virtual void finalizeTraj() = 0; - -/// Initialize before starting the TL run. - virtual void setupTL(const ControlIncrement &, PostProcTLAD_ &) const = 0; - -/// Initialize before starting the AD run. - virtual void setupAD(std::shared_ptr, - ControlIncrement &, PostProcTLAD_ &) const = 0; +/// Initialize and set post-processors to collect data during nonlinear model integration + virtual void setPostProc(const ControlVariable &, const eckit::Configuration &, + PostProc_ &) = 0; +/// Finish computation of cost function term after nonlinear model integration + virtual double computeCost() = 0; + +/// Set post-processors for nonlinear model integration and save linearisation trajectory + virtual void setPostProcTraj(const ControlVariable &, const eckit::Configuration &, + const Geometry_ &, PostProcTLAD_ &) = 0; +/// Finish cost computation and trajectory handling after nonlinear model integration + virtual void computeCostTraj() = 0; + +/// Initialize and set TL post-processors to collect data during TL model integration + virtual void setPostProcTL(const ControlIncrement &, PostProcTLAD_ &) const = 0; +/// Finish cost computation after TL model integration + virtual void computeCostTL(const ControlIncrement &, + GeneralizedDepartures &) const = 0; + +/// Adjoint of computeCostTL (initialize and set post-processors adjoint to force AD model) +// Going by the book, computeCostAD should have the same arguments as computeCostTL (with +// swapped constness). PostProcTLAD is added because it is the way forcing is passed to the +// model (adjoint operations are called in reverse order so computeCostAD will come first). + virtual void computeCostAD(std::shared_ptr, + ControlIncrement &, PostProcTLAD_ &) const = 0; +/// Adjoint ot setPostProcTL (clean-up) + virtual void setPostProcAD() const = 0; /// Multiply by covariance (or weight) matrix and its inverse. virtual std::unique_ptr diff --git a/src/oops/assimilation/HBHtMatrix.h b/src/oops/assimilation/HBHtMatrix.h index 6ab5a6a23..23c71c849 100644 --- a/src/oops/assimilation/HBHtMatrix.h +++ b/src/oops/assimilation/HBHtMatrix.h @@ -11,6 +11,9 @@ #ifndef OOPS_ASSIMILATION_HBHTMATRIX_H_ #define OOPS_ASSIMILATION_HBHTMATRIX_H_ +#include +#include + #include #include "oops/assimilation/ControlIncrement.h" @@ -35,8 +38,7 @@ template class HBHtMatrix : private boost::noncopy typedef DualVector Dual_; public: - explicit HBHtMatrix(const CostFct_ & j, - const bool test = false); + explicit HBHtMatrix(const CostFct_ & j, const bool test = false); void multiply(const Dual_ & dy, Dual_ & dz) const; @@ -65,9 +67,12 @@ void HBHtMatrix::multiply(const Dual_ & dy, Dual_ & dz) const { j_.zeroAD(ww); PostProcessorTLAD costad; for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - j_.jterm(jj).setupAD(dy.getv(jj), ww, costad); + j_.jterm(jj).computeCostAD(dy.getv(jj), ww, costad); } j_.runADJ(ww, costad); + for (unsigned jj = 0; jj < j_.nterms(); ++jj) { + j_.jterm(jj).setPostProcAD(); + } // Multiply by B CtrlInc_ zz(j_.jb()); @@ -76,26 +81,29 @@ void HBHtMatrix::multiply(const Dual_ & dy, Dual_ & dz) const { // Run TLM PostProcessorTLAD costtl; for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - j_.jterm(jj).setupTL(zz, costtl); + j_.jterm(jj).setPostProcTL(zz, costtl); } + CtrlInc_ mzz(zz); j_.runTLM(mzz, costtl); // Get TLM outputs dz.clear(); for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - dz.append(costtl.releaseOutputFromTL(jj)); + std::unique_ptr ztmp = j_.jterm(jj).newDualVector(); + j_.jterm(jj).computeCostTL(zz, *ztmp); + dz.append(std::move(ztmp)); } +// Tests if (test_) { - // , where dx = B Gt dy - double adj_tst_fwd = dot_product(dz, dy); - // < dx, Gt dy>, where dx = B Gt dy - double adj_tst_bwd = dot_product(zz, ww); - - Log::info() << "Online adjoint test, iteration: " << iter_ << std::endl - << util::PrintAdjTest(adj_tst_fwd, adj_tst_bwd, "G") - << std::endl; + // , where dx = B Gt dy + double adj_tst_fwd = dot_product(dz, dy); + // < dx, Gt dy>, where dx = B Gt dy + double adj_tst_bwd = dot_product(zz, ww); + + Log::info() << "Online adjoint test, iteration: " << iter_ << std::endl + << util::PrintAdjTest(adj_tst_fwd, adj_tst_bwd, "G") << std::endl; } } diff --git a/src/oops/assimilation/HMatrix.h b/src/oops/assimilation/HMatrix.h index 4c3f3e617..ddcf6ce38 100644 --- a/src/oops/assimilation/HMatrix.h +++ b/src/oops/assimilation/HMatrix.h @@ -11,6 +11,9 @@ #ifndef OOPS_ASSIMILATION_HMATRIX_H_ #define OOPS_ASSIMILATION_HMATRIX_H_ +#include +#include + #include #include "oops/assimilation/ControlIncrement.h" @@ -43,14 +46,16 @@ template class HMatrix : private boost::noncopyabl PostProcessorTLAD cost; for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - j_.jterm(jj).setupTL(dx, cost); + j_.jterm(jj).setPostProcTL(dx, cost); } j_.runTLM(dx, cost, post, idModel); dy.clear(); for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - dy.append(cost.releaseOutputFromTL(jj)); + std::unique_ptr dytmp(j_.jterm(jj).newDualVector()); + j_.jterm(jj).computeCostTL(dx, *dytmp); + dy.append(std::move(dytmp)); } } diff --git a/src/oops/assimilation/HessianMatrix.h b/src/oops/assimilation/HessianMatrix.h index b50faf549..4ee478ba9 100644 --- a/src/oops/assimilation/HessianMatrix.h +++ b/src/oops/assimilation/HessianMatrix.h @@ -11,6 +11,9 @@ #ifndef OOPS_ASSIMILATION_HESSIANMATRIX_H_ #define OOPS_ASSIMILATION_HESSIANMATRIX_H_ +#include +#include + #include #include "oops/assimilation/ControlIncrement.h" @@ -60,9 +63,8 @@ void HessianMatrix::multiply(const CtrlInc_ & dx, CtrlInc_ & dz) con // Setup TL terms of cost function PostProcessorTLAD costtl; - unsigned iq = j_.jb().initializeTL(costtl); for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - j_.jterm(jj).setupTL(dx, costtl); + j_.jterm(jj).setPostProcTL(dx, costtl); } // Run TLM @@ -89,25 +91,29 @@ void HessianMatrix::multiply(const CtrlInc_ & dx, CtrlInc_ & dz) con // Jo + Jc for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - ww.append(costtl.releaseOutputFromTL(iq+jj)); - zz.append(j_.jterm(jj).multiplyCoInv(*ww.getv(jj))); - j_.jterm(jj).setupAD(zz.getv(jj), dw, costad); + std::unique_ptr wtmp = j_.jterm(jj).newDualVector(); + j_.jterm(jj).computeCostTL(dx, *wtmp); + zz.append(j_.jterm(jj).multiplyCoInv(*wtmp)); + j_.jterm(jj).computeCostAD(zz.getv(jj), dw, costad); + if (test_) ww.append(std::move(wtmp)); } // Run ADJ j_.runADJ(dw, costad); dz += dw; j_.jb().finalizeAD(); + for (unsigned jj = 0; jj < j_.nterms(); ++jj) { + j_.jterm(jj).setPostProcAD(); + } if (test_) { - // , where dy = Rinv H dx - double adj_tst_fwd = dot_product(ww, zz); - // , where dy = Rinv H dx - double adj_tst_bwd = dot_product(dx, dw); - - Log::info() << "Online adjoint test, iteration: " << iter_ << std::endl - << util::PrintAdjTest(adj_tst_fwd, adj_tst_bwd, "G") - << std::endl; + // , where dy = Rinv H dx + double adj_tst_fwd = dot_product(ww, zz); + // , where dy = Rinv H dx + double adj_tst_bwd = dot_product(dx, dw); + + Log::info() << "Online adjoint test, iteration: " << iter_ << std::endl + << util::PrintAdjTest(adj_tst_fwd, adj_tst_bwd, "G") << std::endl; } } diff --git a/src/oops/assimilation/HtMatrix.h b/src/oops/assimilation/HtMatrix.h index edf3cd547..e31b7bc11 100644 --- a/src/oops/assimilation/HtMatrix.h +++ b/src/oops/assimilation/HtMatrix.h @@ -43,9 +43,12 @@ template class HtMatrix : private boost::noncopyab PostProcessorTLAD cost; // Don't zero out dx here for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - j_.jterm(jj).setupAD(dy.getv(jj), dx, cost); + j_.jterm(jj).computeCostAD(dy.getv(jj), dx, cost); } j_.runADJ(dx, cost, post, idModel); + for (unsigned jj = 0; jj < j_.nterms(); ++jj) { + j_.jterm(jj).setPostProcAD(); + } } private: diff --git a/src/oops/assimilation/HtRinvHMatrix.h b/src/oops/assimilation/HtRinvHMatrix.h index 8bf17d817..fc5043f98 100644 --- a/src/oops/assimilation/HtRinvHMatrix.h +++ b/src/oops/assimilation/HtRinvHMatrix.h @@ -11,6 +11,9 @@ #ifndef OOPS_ASSIMILATION_HTRINVHMATRIX_H_ #define OOPS_ASSIMILATION_HTRINVHMATRIX_H_ +#include +#include + #include #include "oops/assimilation/ControlIncrement.h" @@ -64,7 +67,7 @@ void HtRinvHMatrix::multiply(const CtrlInc_ & dx, CtrlInc_ & dz) con // Setup TL terms of cost function PostProcessorTLAD costtl; for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - j_.jterm(jj).setupTL(dx, costtl); + j_.jterm(jj).setPostProcTL(dx, costtl); } // Run TLM @@ -79,23 +82,28 @@ void HtRinvHMatrix::multiply(const CtrlInc_ & dx, CtrlInc_ & dz) con DualVector zz; for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - ww.append(costtl.releaseOutputFromTL(jj)); - zz.append(j_.jterm(jj).multiplyCoInv(*ww.getv(jj))); - j_.jterm(jj).setupAD(zz.getv(jj), dz, costad); + std::unique_ptr wtmp(j_.jterm(jj).newDualVector()); + j_.jterm(jj).computeCostTL(dx, *wtmp); + zz.append(j_.jterm(jj).multiplyCoInv(*wtmp)); + j_.jterm(jj).computeCostAD(zz.getv(jj), dz, costad); + if (test_) ww.append(std::move(wtmp)); } // Run ADJ j_.runADJ(dz, costad); + for (unsigned jj = 0; jj < j_.nterms(); ++jj) { + j_.jterm(jj).setPostProcAD(); + } + if (test_) { - // , where dy = Rinv H dx - double adj_tst_fwd = dot_product(ww, zz); - // , where dy = Rinv H dx - double adj_tst_bwd = dot_product(dx, dz); - - Log::info() << "Online adjoint test, iteration: " << iter_ << std::endl - << util::PrintAdjTest(adj_tst_fwd, adj_tst_bwd, "G") - << std::endl; + // , where dy = Rinv H dx + double adj_tst_fwd = dot_product(ww, zz); + // , where dy = Rinv H dx + double adj_tst_bwd = dot_product(dx, dz); + + Log::info() << "Online adjoint test, iteration: " << iter_ << std::endl + << util::PrintAdjTest(adj_tst_fwd, adj_tst_bwd, "G") << std::endl; } } diff --git a/src/oops/assimilation/IncrementalAssimilation.h b/src/oops/assimilation/IncrementalAssimilation.h index fbf31db77..740069a32 100644 --- a/src/oops/assimilation/IncrementalAssimilation.h +++ b/src/oops/assimilation/IncrementalAssimilation.h @@ -49,7 +49,7 @@ int IncrementalAssimilation(ControlVariable & xx, CostFunction(jouter)); // Get configuration for current outer iteration Log::info() << "IncrementalAssimilation: Configuration for outer iteration " - << jouter << ":\n" << iterconfs[jouter]; + << jouter << ":" << std::endl << iterconfs[jouter] << std::endl; // Setup for the trajectory run PostProcessor post; diff --git a/src/oops/assimilation/JqTermTLAD.h b/src/oops/assimilation/JqTermTLAD.h index 5b6215592..11d565216 100644 --- a/src/oops/assimilation/JqTermTLAD.h +++ b/src/oops/assimilation/JqTermTLAD.h @@ -39,7 +39,6 @@ class JqTermTLAD : public PostBaseTLAD { State_ & getMxi() const; void computeModelErrorTL(Increment_ &); - std::unique_ptr releaseOutputFromTL() override {return nullptr;} void setupAD(const Increment_ & dx); private: diff --git a/src/oops/assimilation/LBHessianMatrix.h b/src/oops/assimilation/LBHessianMatrix.h index 8c43ac515..2622de335 100644 --- a/src/oops/assimilation/LBHessianMatrix.h +++ b/src/oops/assimilation/LBHessianMatrix.h @@ -41,9 +41,8 @@ template class LBHessianMatrix : private boost::no void multiply(const CtrlInc_ & dx, CtrlInc_ & dz) const { // Setup TL terms of cost function PostProcessorTLAD costtl; - unsigned iq = j_.jb().initializeTL(costtl); for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - j_.jterm(jj).setupTL(dx, costtl); + j_.jterm(jj).setPostProcTL(dx, costtl); } // Run TLM @@ -66,9 +65,10 @@ template class LBHessianMatrix : private boost::no j_.zeroAD(dw); // Jo + Jc for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - std::unique_ptr ww(costtl.releaseOutputFromTL(iq+jj)); + std::unique_ptr ww = j_.jterm(jj).newDualVector(); + j_.jterm(jj).computeCostTL(dx, *ww); std::shared_ptr zz(j_.jterm(jj).multiplyCoInv(*ww)); - j_.jterm(jj).setupAD(zz, dw, costad); + j_.jterm(jj).computeCostAD(zz, dw, costad); } // Run ADJ @@ -80,6 +80,9 @@ template class LBHessianMatrix : private boost::no dz += zz; j_.jb().finalizeAD(); + for (unsigned jj = 0; jj < j_.nterms(); ++jj) { + j_.jterm(jj).setPostProcAD(); + } } private: diff --git a/src/oops/assimilation/SaddlePointMatrix.h b/src/oops/assimilation/SaddlePointMatrix.h index 7c5f79315..f258d22d1 100644 --- a/src/oops/assimilation/SaddlePointMatrix.h +++ b/src/oops/assimilation/SaddlePointMatrix.h @@ -11,6 +11,9 @@ #ifndef OOPS_ASSIMILATION_SADDLEPOINTMATRIX_H_ #define OOPS_ASSIMILATION_SADDLEPOINTMATRIX_H_ +#include +#include + #include #include "oops/assimilation/ControlIncrement.h" @@ -55,16 +58,19 @@ void SaddlePointMatrix::multiply(const SPVector_ & x, SPVector_ & z) z.dx(new CtrlInc_(j_.jb())); j_.jb().initializeAD(z.dx(), x.lambda().dx(), costad); for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - j_.jterm(jj).setupAD(x.lambda().getv(jj), ww, costad); + j_.jterm(jj).computeCostAD(x.lambda().getv(jj), ww, costad); } j_.runADJ(ww, costad); z.dx() += ww; + for (unsigned jj = 0; jj < j_.nterms(); ++jj) { + j_.jterm(jj).setPostProcAD(); + } // TLM block PostProcessorTLAD costtl; j_.jb().initializeTL(costtl); for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - j_.jterm(jj).setupTL(x.dx(), costtl); + j_.jterm(jj).setPostProcTL(x.dx(), costtl); } CtrlInc_ mdx(x.dx()); j_.runTLM(mdx, costtl); @@ -72,7 +78,9 @@ void SaddlePointMatrix::multiply(const SPVector_ & x, SPVector_ & z) z.lambda().dx(new CtrlInc_(j_.jb())); j_.jb().finalizeTL(x.dx(), z.lambda().dx()); for (unsigned jj = 0; jj < j_.nterms(); ++jj) { - z.lambda().append(costtl.releaseOutputFromTL(jj+1)); + std::unique_ptr ztmp(j_.jterm(jj).newDualVector()); + j_.jterm(jj).computeCostTL(x.dx(), *ztmp); + z.lambda().append(std::move(ztmp)); } // Diagonal block diff --git a/src/oops/base/Departures.h b/src/oops/base/Departures.h index 0e99d1c30..e06e3c961 100644 --- a/src/oops/base/Departures.h +++ b/src/oops/base/Departures.h @@ -24,7 +24,6 @@ #include "oops/interface/ObsVector.h" #include "oops/util/dot_product.h" #include "oops/util/Logger.h" -#include "oops/util/Printable.h" namespace oops { @@ -41,8 +40,7 @@ template class Observations; // ----------------------------------------------------------------------------- template -class Departures : public util::Printable, - public GeneralizedDepartures { +class Departures : public GeneralizedDepartures { typedef ObsSpaces ObsSpaces_; typedef ObsVector ObsVector_; template using ObsData_ = ObsDataVector; diff --git a/src/oops/base/GeneralizedDepartures.h b/src/oops/base/GeneralizedDepartures.h index d3d7e7c57..a19f61d11 100644 --- a/src/oops/base/GeneralizedDepartures.h +++ b/src/oops/base/GeneralizedDepartures.h @@ -11,6 +11,8 @@ #ifndef OOPS_BASE_GENERALIZEDDEPARTURES_H_ #define OOPS_BASE_GENERALIZEDDEPARTURES_H_ +#include "oops/util/Printable.h" + namespace oops { /// Abstract base class for quantities @@ -19,10 +21,12 @@ namespace oops { * measured by the cost function to gather them in a DualVector object. */ -class GeneralizedDepartures { +class GeneralizedDepartures : public util::Printable { public: GeneralizedDepartures() {} virtual ~GeneralizedDepartures() {} + private: + void print(std::ostream &) const = 0; }; } // namespace oops diff --git a/src/oops/base/GetValuePost.h b/src/oops/base/GetValuePost.h index af7cdaa4d..bdf38bd0c 100644 --- a/src/oops/base/GetValuePost.h +++ b/src/oops/base/GetValuePost.h @@ -14,7 +14,6 @@ #include #include -#include "oops/base/PostBase.h" #include "oops/base/Variables.h" #include "oops/interface/Geometry.h" #include "oops/interface/GeoVaLs.h" diff --git a/src/oops/base/GetValuePosts.h b/src/oops/base/GetValuePosts.h index 512620a4d..d36ef7c30 100644 --- a/src/oops/base/GetValuePosts.h +++ b/src/oops/base/GetValuePosts.h @@ -28,7 +28,6 @@ namespace oops { template class GetValuePosts : public PostBase> { typedef ChangeVariables ChangeVariables_; - typedef GetValues GetValues_; typedef State State_; typedef std::shared_ptr> GetValuePtr_; @@ -79,6 +78,7 @@ void GetValuePosts::doInitialize(const State_ &, const util::DateTim template void GetValuePosts::doProcessing(const State_ & xx) { Log::trace() << "GetValuePosts::doProcessing start" << std::endl; +// Change of variables will go here for (GetValuePtr_ getval : getvals_) getval->process(xx); Log::trace() << "GetValuePosts::doProcessing done" << std::endl; } diff --git a/src/oops/base/GetValueTLAD.h b/src/oops/base/GetValueTLAD.h new file mode 100644 index 000000000..fcc455965 --- /dev/null +++ b/src/oops/base/GetValueTLAD.h @@ -0,0 +1,202 @@ +/* + * (C) Copyright 2020-2020 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_BASE_GETVALUETLAD_H_ +#define OOPS_BASE_GETVALUETLAD_H_ + +#include +#include +#include +#include +#include + +#include "oops/base/Variables.h" +#include "oops/interface/Geometry.h" +#include "oops/interface/GeoVaLs.h" +#include "oops/interface/GetValues.h" +#include "oops/interface/Increment.h" +#include "oops/interface/LinearGetValues.h" +#include "oops/interface/Locations.h" +#include "oops/interface/State.h" +#include "oops/util/DateTime.h" +#include "oops/util/Duration.h" +#include "oops/util/Logger.h" + +namespace eckit { + class Configuration; +} + +namespace oops { + +// GetValueTLAD is its own class for now. Once the change of variables is moved out it +// should be a subclass of GetValuePost that adds setGeoVaLs, processTL and processAD +// The other methods are the same. + +/// \brief TLAD of filling GeoVaLs with requested variables at requested locations during model run +template +class GetValueTLAD { + typedef Geometry Geometry_; + typedef GeoVaLs GeoVaLs_; + typedef LinearGetValues GetValues_; + typedef Increment Increment_; + typedef Locations Locations_; + typedef State State_; + + public: +/// \brief Saves Locations and Variables to be processed + GetValueTLAD(const eckit::Configuration &, const Geometry_ &, + const util::DateTime &, const util::DateTime &, + const Locations_ &, const Variables &, const Variables &); + +/// Same finalize is used for traj and TL + std::unique_ptr finalize(); + +/// Linearization trajectory + void initializeTraj(const util::Duration &); + void processTraj(const State_ &); + +/// TL + void initializeTL(const util::Duration &); + void processTL(const Increment_ &); + +/// AD + void setAD(std::unique_ptr &); + void initializeAD(const util::Duration &); + void processAD(Increment_ &); + void finalizeAD(); + +/// Variables that will be required from the State + const Variables & requiredVariables() const {return geovars_;} + + private: + util::DateTime winbgn_; /// Begining of assimilation window + util::DateTime winend_; /// End of assimilation window + util::Duration hslot_; /// Half time slot + + const Locations_ & locations_; /// locations of observations + const Variables geovars_; /// Variables needed from model + const Variables linvars_; /// Variables needed from linear model + GetValues_ getvals_; /// GetValues used to fill in GeoVaLs + std::unique_ptr geovals_; /// GeoVaLs that are filled in + std::unique_ptr gvalsad_; /// Input GeoVaLs for adjoint forcing +}; + +// ----------------------------------------------------------------------------- + +template +GetValueTLAD::GetValueTLAD(const eckit::Configuration & conf, const Geometry_ & geom, + const util::DateTime & bgn, const util::DateTime & end, + const Locations_ & locations, + const Variables & vars, const Variables & varl) + : winbgn_(bgn), winend_(end), hslot_(), locations_(locations), geovars_(vars), linvars_(varl), + getvals_(geom, locations_, conf), geovals_(), gvalsad_(nullptr) +{ + Log::trace() << "GetValueTLAD::GetValueTLAD" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void GetValueTLAD::initializeTraj(const util::Duration & tstep) { + Log::trace() << "GetValueTLAD::initializeTraj start" << std::endl; + hslot_ = tstep/2; + geovals_.reset(new GeoVaLs_(locations_, geovars_)); + Log::trace() << "GetValueTLAD::initializeTraj done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void GetValueTLAD::processTraj(const State_ & xx) { + Log::trace() << "GetValueTLAD::processTraj start" << std::endl; + ASSERT(geovals_); + util::DateTime t1 = std::max(xx.validTime()-hslot_, winbgn_); + util::DateTime t2 = std::min(xx.validTime()+hslot_, winend_); + +// Get state variables at obs locations + getvals_.setTrajectory(xx, t1, t2, *geovals_); + Log::trace() << "GetValueTLAD::processTraj done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void GetValueTLAD::initializeTL(const util::Duration & tstep) { + Log::trace() << "GetValueTLAD::initializeTL start" << std::endl; + hslot_ = tstep/2; + geovals_.reset(new GeoVaLs_(locations_, linvars_)); + Log::trace() << "GetValueTLAD::initializeTL done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void GetValueTLAD::processTL(const Increment_ & dx) { + Log::trace() << "GetValueTLAD::processTL start" << std::endl; + ASSERT(geovals_); + util::DateTime t1 = std::max(dx.validTime()-hslot_, winbgn_); + util::DateTime t2 = std::min(dx.validTime()+hslot_, winend_); + +// Get state variables at obs locations + getvals_.fillGeoVaLsTL(dx, t1, t2, *geovals_); + Log::trace() << "GetValueTLAD::processTL done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +std::unique_ptr> GetValueTLAD::finalize() { + Log::trace() << "GetValueTLAD::finalize" << std::endl; + // Release ownership of GeoVaLs + return std::move(geovals_); +} + +// ----------------------------------------------------------------------------- + +template +void GetValueTLAD::setAD(std::unique_ptr & geovals) { + // Take ownership of GeoVaLs + gvalsad_ = std::move(geovals); + Log::trace() << "GetValueTLAD::setAD" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void GetValueTLAD::initializeAD(const util::Duration & tstep) { + hslot_ = tstep/2; + Log::trace() << "GetValueTLAD::initializeAD" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void GetValueTLAD::processAD(Increment_ & dx) { + Log::trace() << "GetValueTLAD::processAD start" << std::endl; + ASSERT(gvalsad_); + util::DateTime t1 = std::max(dx.validTime()-hslot_, winbgn_); + util::DateTime t2 = std::min(dx.validTime()+hslot_, winend_); + +// Get state variables at obs locations + getvals_.fillGeoVaLsAD(dx, t1, t2, *gvalsad_); + + Log::trace() << "GetValueTLAD::processAD done" << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void GetValueTLAD::finalizeAD() { + ASSERT(gvalsad_); + gvalsad_.reset(); +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_BASE_GETVALUETLAD_H_ diff --git a/src/oops/base/GetValueTLADs.h b/src/oops/base/GetValueTLADs.h new file mode 100644 index 000000000..53c281f32 --- /dev/null +++ b/src/oops/base/GetValueTLADs.h @@ -0,0 +1,120 @@ +/* + * (C) Copyright 2021-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_BASE_GETVALUETLADS_H_ +#define OOPS_BASE_GETVALUETLADS_H_ + +#include +#include + +#include "oops/base/GetValueTLAD.h" +#include "oops/base/PostBaseTLAD.h" +#include "oops/interface/Increment.h" +#include "oops/interface/State.h" +#include "oops/util/DateTime.h" +#include "oops/util/Duration.h" + +namespace oops { + +/// Computes observation equivalent TL and AD to/from increments. + +template +class GetValueTLADs : public PostBaseTLAD { + typedef Increment Increment_; + typedef State State_; + typedef std::shared_ptr> GetValPtr_; + + public: + GetValueTLADs(const util::DateTime &, const util::DateTime &); + + void append(GetValPtr_); + + private: +// Methods + void doInitializeTraj(const State_ &, const util::DateTime &, const util::Duration &) override; + void doProcessingTraj(const State_ &) override; + void doFinalizeTraj(const State_ &) override {} + + void doInitializeTL(const Increment_ &, const util::DateTime &, const util::Duration &) override; + void doProcessingTL(const Increment_ &) override; + void doFinalizeTL(const Increment_ &) override {} + + void doFirstAD(Increment_ &, const util::DateTime &, const util::Duration &) override; + void doProcessingAD(Increment_ &) override; + void doLastAD(Increment_ &) override {} + +// Data + std::vector getvals_; +}; + +// ----------------------------------------------------------------------------- +template +GetValueTLADs::GetValueTLADs(const util::DateTime & bgn, const util::DateTime & end) + : PostBaseTLAD(bgn, end), getvals_() +{ + Log::trace() << "GetValueTLADs::GetValueTLADs" << std::endl; +} +// ----------------------------------------------------------------------------- +template +void GetValueTLADs::append(GetValPtr_ getval) { + Log::trace() << "GetValuePosts::append start" << std::endl; + getvals_.push_back(getval); + Log::trace() << "GetValuePosts::append done" << std::endl; +} +// ----------------------------------------------------------------------------- +template +void GetValueTLADs::doInitializeTraj(const State_ &, const util::DateTime &, + const util::Duration & tstep) { + Log::trace() << "GetValueTLADs::doInitializeTraj start" << std::endl; + for (GetValPtr_ getval : getvals_) getval->initializeTraj(tstep); + Log::trace() << "GetValueTLADs::doInitializeTraj done" << std::endl; +} +// ----------------------------------------------------------------------------- +template +void GetValueTLADs::doProcessingTraj(const State_ & xx) { + Log::trace() << "GetValueTLADs::doProcessingTraj start" << std::endl; +// Change of variables traj will go here + for (GetValPtr_ getval : getvals_) getval->processTraj(xx); + Log::trace() << "GetValueTLADs::doProcessingTraj done" << std::endl; +} +// ----------------------------------------------------------------------------- +template +void GetValueTLADs::doInitializeTL(const Increment_ &, const util::DateTime &, + const util::Duration & tstep) { + Log::trace() << "GetValueTLADs::doInitializeTL start" << std::endl; + for (GetValPtr_ getval : getvals_) getval->initializeTL(tstep); + Log::trace() << "GetValueTLADs::doInitializeTL done" << std::endl; +} +// ----------------------------------------------------------------------------- +template +void GetValueTLADs::doProcessingTL(const Increment_ & dx) { + Log::trace() << "GetValueTLADs::doProcessingTL start" << std::endl; +// TL change of variables will go here + for (GetValPtr_ getval : getvals_) getval->processTL(dx); + Log::trace() << "GetValueTLADs::doProcessingTL done" << std::endl; +} +// ----------------------------------------------------------------------------- +template +void GetValueTLADs::doFirstAD(Increment_ &, const util::DateTime &, + const util::Duration & tstep) { + Log::trace() << "GetValueTLADs::doFirstAD start" << std::endl; + for (GetValPtr_ getval : getvals_) getval->initializeAD(tstep); + Log::trace() << "GetValueTLADs::doFirstAD done" << std::endl; +} +// ----------------------------------------------------------------------------- +template +void GetValueTLADs::doProcessingAD(Increment_ & dx) { + Log::trace() << "GetValueTLADs::doProcessingAD start" << std::endl; +// AD change of variables will go here + for (GetValPtr_ getval : getvals_) getval->processAD(dx); + Log::trace() << "GetValueTLADs::doProcessingAD done" << std::endl; +} +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_BASE_GETVALUETLADS_H_ diff --git a/src/oops/base/Observer.h b/src/oops/base/Observer.h index f62de8388..39ff6d975 100644 --- a/src/oops/base/Observer.h +++ b/src/oops/base/Observer.h @@ -65,7 +65,7 @@ class Observer { /// \brief Initializes variables, obs bias, obs filters (could be different for /// different iterations std::shared_ptr initialize(const Geometry_ &, const ObsAuxCtrl_ &, - ObsVector_ &, const int iter = 0); + ObsVector_ &, const int iter); /// \brief Computes H(x) from the filled in GeoVaLs void finalize(ObsVector_ &); @@ -106,7 +106,7 @@ template std::shared_ptr> Observer::initialize(const Geometry_ & geom, const ObsAuxCtrl_ & ybias, ObsVector_ & obserr, const int iter) { -// could pass state (or even geom) and obsbias instead of control var (easier for HofX?) + Log::trace() << "Observer::initialize start" << std::endl; iterout_ = iter; ybias_ = &ybias; @@ -163,9 +163,10 @@ void Observer::finalize(ObsVector_ & yobsim) { /// Save flags (for diagnostics use) const std::string qcname = "EffectiveQC" + std::to_string(iterout_); qcflags_->save(qcname); + Log::info() << "Observer::finalize QC = " << *qcflags_ << std::endl; initialized_ = false; - oops::Log::trace() << "Observer::finalize done" << std::endl; + Log::trace() << "Observer::finalize done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/ObserverTLAD.h b/src/oops/base/ObserverTLAD.h index ef4503fba..39f7768e0 100644 --- a/src/oops/base/ObserverTLAD.h +++ b/src/oops/base/ObserverTLAD.h @@ -15,18 +15,16 @@ #include #include "eckit/config/Configuration.h" +#include "oops/base/GetValueTLAD.h" +#include "oops/interface/Geometry.h" #include "oops/interface/GeoVaLs.h" -#include "oops/interface/Increment.h" -#include "oops/interface/LinearGetValues.h" #include "oops/interface/LinearObsOperator.h" #include "oops/interface/Locations.h" #include "oops/interface/ObsAuxControl.h" #include "oops/interface/ObsAuxIncrement.h" -#include "oops/interface/ObsDiagnostics.h" #include "oops/interface/ObsOperator.h" #include "oops/interface/ObsSpace.h" #include "oops/interface/ObsVector.h" -#include "oops/interface/State.h" #include "oops/util/DateTime.h" namespace oops { @@ -35,155 +33,130 @@ namespace oops { template class ObserverTLAD { + typedef Geometry Geometry_; typedef GeoVaLs GeoVaLs_; - typedef Increment Increment_; - typedef LinearGetValues LinearGetValues_; + typedef GetValueTLAD GetValTLAD_; typedef LinearObsOperator LinearObsOperator_; typedef Locations Locations_; typedef ObsAuxControl ObsAuxCtrl_; typedef ObsAuxIncrement ObsAuxIncr_; - typedef ObsDiagnostics ObsDiags_; typedef ObsOperator ObsOperator_; typedef ObsSpace ObsSpace_; typedef ObsVector ObsVector_; - typedef State State_; public: - ObserverTLAD(const eckit::Configuration & conf, - const ObsSpace_ &, const ObsAuxCtrl_ &); + ObserverTLAD(const ObsSpace_ &, const eckit::Configuration &); ~ObserverTLAD() {} - void doInitializeTraj(const State_ &, const util::DateTime &, const util::DateTime &); - void doProcessingTraj(const State_ &, const util::DateTime &, const util::DateTime &); - void doFinalizeTraj(const State_ &); + std::shared_ptr initializeTraj(const Geometry_ &); + void finalizeTraj(const ObsAuxCtrl_ &); - void doInitializeTL(const Increment_ &, const util::DateTime &, const util::DateTime &); - void doProcessingTL(const Increment_ &, const util::DateTime &, const util::DateTime &); - void doFinalizeTL(const Increment_ &, ObsVector_ &, const ObsAuxIncr_ &); + std::shared_ptr initializeTL(); + void finalizeTL(const ObsAuxIncr_ &, ObsVector_ &); - void doFirstAD(Increment_ &, const ObsVector_ &, ObsAuxIncr_ &, - const util::DateTime &, const util::DateTime &); - void doProcessingAD(Increment_ &, const util::DateTime &, const util::DateTime &); - void doLastAD(Increment_ &); + std::shared_ptr initializeAD(const ObsVector_ &, ObsAuxIncr_ &); + void finalizeAD(); private: - const ObsSpace_ & obsdb_; -// Obs operator - ObsOperator_ hop_; - LinearObsOperator_ hoptlad_; - eckit::LocalConfiguration linearGetValuesConf_; - - const ObsAuxCtrl_ & ybias_; - Variables geovars_; - Locations_ locs_; - - std::unique_ptr lingetvals_; - std::shared_ptr gvals_; + eckit::LocalConfiguration obsconfig_; + const ObsSpace_ & obspace_; // ObsSpace used in H(x) + LinearObsOperator_ hoptlad_; // Linear obs operator + std::shared_ptr getvals_; // Postproc passed to the model during integration + std::unique_ptr locations_; // locations + util::DateTime winbgn_; // Begining of assimilation window + util::DateTime winend_; // End of assimilation window + bool init_; }; // ----------------------------------------------------------------------------- template -ObserverTLAD::ObserverTLAD(const eckit::Configuration & config, - const ObsSpace_ & obsdb, - const ObsAuxCtrl_ & ybias) - : obsdb_(obsdb), - hop_(obsdb, eckit::LocalConfiguration(config, "obs operator")), - hoptlad_(obsdb, config.has("linear obs operator") ? - eckit::LocalConfiguration(config, "linear obs operator") : - eckit::LocalConfiguration(config, "obs operator")), - linearGetValuesConf_(config.has("linear get values") ? - eckit::LocalConfiguration(config, "linear get values") : - (config.has("get values") ? - eckit::LocalConfiguration(config, "get values") : - eckit::LocalConfiguration(config, ""))), - ybias_(ybias), geovars_(), locs_(hop_.locations()), lingetvals_(), gvals_() +ObserverTLAD::ObserverTLAD(const ObsSpace_ & obsdb, const eckit::Configuration & conf) + : obsconfig_(conf), obspace_(obsdb), + hoptlad_(obspace_, conf.has("linear obs operator") ? + eckit::LocalConfiguration(conf, "linear obs operator") : + eckit::LocalConfiguration(conf, "obs operator")), + getvals_(), locations_(), winbgn_(obsdb.windowStart()), winend_(obsdb.windowEnd()), init_(false) { - geovars_ += hop_.requiredVars(); - geovars_ += ybias_.requiredVars(); Log::trace() << "ObserverTLAD::ObserverTLAD" << std::endl; } // ----------------------------------------------------------------------------- template -void ObserverTLAD::doInitializeTraj(const State_ & xx, - const util::DateTime & winbgn, - const util::DateTime & winend) { - Log::trace() << "ObserverTLAD::doInitializeTraj start" << std::endl; - lingetvals_.reset(new LinearGetValues_(xx.geometry(), locs_, linearGetValuesConf_)); - gvals_.reset(new GeoVaLs_(locs_, geovars_)); - Log::trace() << "ObserverTLAD::doInitializeTraj done" << std::endl; +std::shared_ptr> +ObserverTLAD::initializeTraj(const Geometry_ & geom) { + Log::trace() << "ObserverTLAD::initializeTraj start" << std::endl; + +// hop is only needed to get locations + ObsOperator_ hop(obspace_, eckit::LocalConfiguration(obsconfig_, "obs operator")); + locations_.reset(new Locations_(hop.locations())); + + eckit::LocalConfiguration gvconf(obsconfig_.has("linear get values") ? + eckit::LocalConfiguration(obsconfig_, "linear get values") : + (obsconfig_.has("get values") ? + eckit::LocalConfiguration(obsconfig_, "get values") : + eckit::LocalConfiguration(obsconfig_, ""))); + + getvals_.reset(new GetValTLAD_(gvconf, geom, winbgn_, winend_, + *locations_, hop.requiredVars(), hoptlad_.requiredVars())); + + init_ = true; + Log::trace() << "ObserverTLAD::initializeTraj done" << std::endl; + return getvals_; } // ----------------------------------------------------------------------------- template -void ObserverTLAD::doProcessingTraj(const State_ & xx, - const util::DateTime & t1, - const util::DateTime & t2) { - Log::trace() << "ObserverTLAD::doProcessingTraj start" << std::endl; -// Call nonlinear getValues - lingetvals_->setTrajectory(xx, t1, t2, *gvals_); - Log::trace() << "ObserverTLAD::doProcessingTraj done" << std::endl; -} -// ----------------------------------------------------------------------------- -template -void ObserverTLAD::doFinalizeTraj(const State_ & xx) { - Log::trace() << "ObserverTLAD::doFinalizeTraj start" << std::endl; - hoptlad_.setTrajectory(*gvals_, ybias_); - gvals_.reset(); - Log::trace() << "ObserverTLAD::doFinalizeTraj done" << std::endl; -} -// ----------------------------------------------------------------------------- -template -void ObserverTLAD::doInitializeTL(const Increment_ & dx, - const util::DateTime & winbgn, - const util::DateTime & winend) { - Log::trace() << "ObserverTLAD::doInitializeTL start" << std::endl; - gvals_.reset(new GeoVaLs_(locs_, hoptlad_.requiredVars())); - Log::trace() << "ObserverTLAD::doInitializeTL done" << std::endl; -} -// ----------------------------------------------------------------------------- -template -void ObserverTLAD::doProcessingTL(const Increment_ & dx, - const util::DateTime & t1, - const util::DateTime & t2) { - Log::trace() << "ObserverTLAD::doProcessingTL start" << std::endl; -// Get increment variables at obs locations - lingetvals_->fillGeoVaLsTL(dx, t1, t2, *gvals_); - Log::trace() << "ObserverTLAD::doProcessingTL done" << std::endl; +void ObserverTLAD::finalizeTraj(const ObsAuxCtrl_ & ybias) { + Log::trace() << "ObserverTLAD::finalizeTraj start" << std::endl; + ASSERT(init_); + + // GetValues releases GeoVaLs, Observer takes ownership + std::unique_ptr geovals = getvals_->finalize(); + + /// Set linearization trajectory for H(x) + hoptlad_.setTrajectory(*geovals, ybias); + + init_ = false; + Log::trace() << "ObserverTLAD::finalizeTraj done" << std::endl; } // ----------------------------------------------------------------------------- template -void ObserverTLAD::doFinalizeTL(const Increment_ &, ObsVector_ & ydeptl, - const ObsAuxIncr_ & ybiastl) { - Log::trace() << "ObserverTLAD::doFinalizeTL start" << std::endl; - hoptlad_.simulateObsTL(*gvals_, ydeptl, ybiastl); - gvals_.reset(); - Log::trace() << "ObserverTLAD::doFinalizeTL done" << std::endl; +std::shared_ptr> ObserverTLAD::initializeTL() { + Log::trace() << "ObserverTLAD::initializeTL" << std::endl; + return getvals_; } // ----------------------------------------------------------------------------- template -void ObserverTLAD::doFirstAD(Increment_ & dx, const ObsVector_ & ydepad, - ObsAuxIncr_ & ybiasad, - const util::DateTime & winbgn, - const util::DateTime & winend) { - Log::trace() << "ObserverTLAD::doFirstAD start" << std::endl; - gvals_.reset(new GeoVaLs_(locs_, hoptlad_.requiredVars())); - hoptlad_.simulateObsAD(*gvals_, ydepad, ybiasad); - Log::trace() << "ObserverTLAD::doFirstAD done" << std::endl; +void ObserverTLAD::finalizeTL(const ObsAuxIncr_ & ybiastl, ObsVector_ & ydeptl) { + Log::trace() << "ObserverTLAD::finalizeTL start" << std::endl; + + // GetValues releases GeoVaLs, Observer takes ownership + std::unique_ptr geovals = getvals_->finalize(); + + // Compute linear H(x) + hoptlad_.simulateObsTL(*geovals, ydeptl, ybiastl); + + Log::trace() << "ObserverTLAD::finalizeTL done" << std::endl; } // ----------------------------------------------------------------------------- template -void ObserverTLAD::doProcessingAD(Increment_ & dx, const util::DateTime & t1, - const util::DateTime & t2) { - Log::trace() << "ObserverTLAD::doProcessingAD start" << std::endl; -// Adjoint of get increment variables at obs locations - lingetvals_->fillGeoVaLsAD(dx, t1, t2, *gvals_); - Log::trace() << "ObserverTLAD::doProcessingAD done" << std::endl; +std::shared_ptr> +ObserverTLAD::initializeAD(const ObsVector_ & ydepad, ObsAuxIncr_ & ybiasad) { + Log::trace() << "ObserverTLAD::initializeAD start" << std::endl; + + // Compute adjoint of H(x) + std::unique_ptr geovals(new GeoVaLs_(*locations_, hoptlad_.requiredVars())); + hoptlad_.simulateObsAD(*geovals, ydepad, ybiasad); + + // GetValues get GeoVaLs and takes ownership + getvals_->setAD(geovals); + + Log::trace() << "ObserverTLAD::initializeAD done" << std::endl; + return getvals_; } // ----------------------------------------------------------------------------- template -void ObserverTLAD::doLastAD(Increment_ &) { - Log::trace() << "ObserverTLAD::doLastAD start" << std::endl; - gvals_.reset(); - Log::trace() << "ObserverTLAD::doLastAD done" << std::endl; +void ObserverTLAD::finalizeAD() { + getvals_->finalizeAD(); + Log::trace() << "ObserverTLAD::finalizeAD done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/Observers.h b/src/oops/base/Observers.h index 4d9274462..c5d95986e 100644 --- a/src/oops/base/Observers.h +++ b/src/oops/base/Observers.h @@ -49,7 +49,7 @@ class Observers { /// \brief Initializes variables, obs bias, obs filters (could be different for /// different iterations - void initialize(const Geometry_ &, const ObsAuxCtrls_ &, PostProc_ &); + void initialize(const Geometry_ &, const ObsAuxCtrls_ &, PostProc_ &, const int iter = 0); /// \brief Computes H(x) from the filled in GeoVaLs void finalize(Observations_ &); @@ -72,6 +72,7 @@ Observers::Observers(const ObsSpaces_ & obspaces, const eckit::Confi for (size_t jj = 0; jj < obspaces_.size(); ++jj) { observers_.emplace_back(new Observer_(obspaces_[jj], obsconfs[jj])); } + Log::trace() << "Observers::Observers done" << std::endl; } @@ -79,14 +80,16 @@ Observers::Observers(const ObsSpaces_ & obspaces, const eckit::Confi template void Observers::initialize(const Geometry_ & geom, const ObsAuxCtrls_ & obsaux, - PostProc_ & pp) { + PostProc_ & pp, const int iter) { Log::trace() << "Observers::initialize start" << std::endl; obserrs_.reserve(observers_.size()); + std::string errname = "ObsError"; + if (iter > 0) errname = "EffectiveError"; std::shared_ptr getvals(new GetValuePosts_()); for (size_t jj = 0; jj < observers_.size(); ++jj) { - obserrs_.emplace_back(obspaces_[jj], "ObsError"); - getvals->append(observers_[jj]->initialize(geom, obsaux[jj], obserrs_[jj])); + obserrs_.emplace_back(obspaces_[jj], errname); + getvals->append(observers_[jj]->initialize(geom, obsaux[jj], obserrs_[jj], iter)); } pp.enrollProcessor(getvals); @@ -102,6 +105,7 @@ void Observers::finalize(Observations_ & yobs) { for (size_t jj = 0; jj < observers_.size(); ++jj) { observers_[jj]->finalize(yobs[jj]); obserrs_[jj].save("EffectiveError"); // Obs error covariance is looking for that for now + Log::info() << "Observers::finalize obs err = " << obserrs_[jj] << std::endl; } obserrs_.clear(); diff --git a/src/oops/base/ObserversTLAD.h b/src/oops/base/ObserversTLAD.h index 45f338c8f..6beb7d8b4 100644 --- a/src/oops/base/ObserversTLAD.h +++ b/src/oops/base/ObserversTLAD.h @@ -19,14 +19,13 @@ #include "eckit/config/Configuration.h" #include "oops/base/Departures.h" #include "oops/base/GeneralizedDepartures.h" +#include "oops/base/GetValueTLADs.h" #include "oops/base/ObsAuxControls.h" #include "oops/base/ObsAuxIncrements.h" -#include "oops/base/Observations.h" #include "oops/base/ObserverTLAD.h" #include "oops/base/ObsSpaces.h" -#include "oops/base/PostBaseTLAD.h" -#include "oops/interface/Increment.h" -#include "oops/interface/State.h" +#include "oops/base/PostProcessorTLAD.h" +#include "oops/interface/Geometry.h" #include "oops/util/DateTime.h" #include "oops/util/Duration.h" @@ -35,198 +34,115 @@ namespace oops { /// Computes observation equivalent TL and AD to/from increments. template -class ObserversTLAD : public PostBaseTLAD { - typedef Departures Departures_; - typedef GeoVaLs GeoVaLs_; - typedef Increment Increment_; - typedef Observations Observations_; - typedef ObsAuxControls ObsAuxCtrls_; - typedef ObsAuxIncrements ObsAuxIncrs_; - typedef ObserverTLAD ObserverTLAD_; - typedef ObsSpaces ObsSpaces_; - typedef State State_; +class ObserversTLAD { + typedef Departures Departures_; + typedef Geometry Geometry_; + typedef GeoVaLs GeoVaLs_; + typedef GetValueTLADs GetValueTLADs_; + typedef Observations Observations_; + typedef ObsAuxControls ObsAuxCtrls_; + typedef ObsAuxIncrements ObsAuxIncrs_; + typedef ObserverTLAD ObserverTLAD_; + typedef ObsSpaces ObsSpaces_; + typedef PostProcessorTLAD PostProcTLAD_; public: - ObserversTLAD(const eckit::Configuration &, - const ObsSpaces_ &, const ObsAuxCtrls_ &); - ~ObserversTLAD() {} + ObserversTLAD(const ObsSpaces_ &, const eckit::Configuration &); - std::unique_ptr releaseOutputFromTL() override {return std::move(ydeptl_);} - void setupTL(const ObsAuxIncrs_ &); - void setupAD(std::shared_ptr, ObsAuxIncrs_ &); + void initializeTraj(const Geometry_ &, const ObsAuxCtrls_ &, PostProcTLAD_ &); + void finalizeTraj(); - private: -// Methods - void doInitializeTraj(const State_ &, - const util::DateTime &, const util::Duration &) override; - void doProcessingTraj(const State_ &) override; - void doFinalizeTraj(const State_ &) override; - - void doInitializeTL(const Increment_ &, - const util::DateTime &, const util::Duration &) override; - void doProcessingTL(const Increment_ &) override; - void doFinalizeTL(const Increment_ &) override; - - void doFirstAD(Increment_ &, const util::DateTime &, const util::Duration &) override; - void doProcessingAD(Increment_ &) override; - void doLastAD(Increment_ &) override; - -// Obs operator - std::vector> observerstlad_; + void initializeTL(PostProcTLAD_ &); + void finalizeTL(const ObsAuxIncrs_ &, Departures_ &); -// Data - ObsSpaces_ obspace_; - std::unique_ptr ydeptl_; - const ObsAuxIncrs_ * ybiastl_; - std::shared_ptr ydepad_; - ObsAuxIncrs_ * ybiasad_; + void initializeAD(const Departures_ &, ObsAuxIncrs_ &, PostProcTLAD_ &); + void finalizeAD(); - util::DateTime winbgn_; //!< Begining of assimilation window - util::DateTime winend_; //!< End of assimilation window - util::Duration hslot_, hslottraj_; //!< Half time slot + private: + std::vector> observers_; + const ObsAuxCtrls_ * ybias_; + util::DateTime winbgn_; + util::DateTime winend_; }; // ----------------------------------------------------------------------------- template -ObserversTLAD::ObserversTLAD(const eckit::Configuration & obsConfig, - const ObsSpaces_ & obsdb, - const ObsAuxCtrls_ & ybias) - : PostBaseTLAD(obsdb.windowStart(), obsdb.windowEnd()), - observerstlad_(), obspace_(obsdb), - ydeptl_(), ybiastl_(), ydepad_(), ybiasad_(), - winbgn_(obsdb.windowStart()), winend_(obsdb.windowEnd()), - hslot_(0), hslottraj_(0) +ObserversTLAD::ObserversTLAD(const ObsSpaces_ & obspaces, + const eckit::Configuration & obsConfig) + : observers_(), ybias_(nullptr), winbgn_(obspaces.windowStart()), winend_(obspaces.windowEnd()) { - // setup observers - std::vector typeconf = obsConfig.getSubConfigurations(); - for (std::size_t jobs = 0; jobs < obsdb.size(); ++jobs) { - std::shared_ptr tmp; - bool passive = typeconf[jobs].getBool("monitoring only", false); - if (!passive) tmp.reset(new ObserverTLAD_(typeconf[jobs], obsdb[jobs], ybias[jobs])); - observerstlad_.push_back(tmp); + Log::trace() << "ObserversTLAD::ObserversTLAD start" << std::endl; + std::vector obsconfs = obsConfig.getSubConfigurations(); + for (size_t jj = 0; jj < obspaces.size(); ++jj) { + bool passive = obsconfs[jj].getBool("monitoring only", false); + std::unique_ptr tmp; + if (!passive) tmp.reset(new ObserverTLAD_(obspaces[jj], obsconfs[jj])); + observers_.push_back(std::move(tmp)); } - Log::trace() << "ObserversTLAD::ObserversTLAD" << std::endl; + Log::trace() << "ObserversTLAD::ObserversTLAD done" << std::endl; } // ----------------------------------------------------------------------------- template -void ObserversTLAD::doInitializeTraj(const State_ & xx, - const util::DateTime & end, - const util::Duration & tstep) { - Log::trace() << "ObserversTLAD::doInitializeTraj start" << std::endl; -// Create full trajectory object - - hslottraj_ = tstep/2; - - for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - if (observerstlad_[jj]) observerstlad_[jj]->doInitializeTraj(xx, winbgn_, winend_); +void ObserversTLAD::initializeTraj(const Geometry_ & geom, const ObsAuxCtrls_ & ybias, + PostProcTLAD_ & pp) { + Log::trace() << "ObserversTLAD::initializeTraj start" << std::endl; + ybias_ = &ybias; + std::shared_ptr getvals(new GetValueTLADs_(winbgn_, winend_)); + for (size_t jj = 0; jj < observers_.size(); ++jj) { + if (observers_[jj]) getvals->append(observers_[jj]->initializeTraj(geom)); } - Log::trace() << "ObserversTLAD::doInitializeTraj done" << std::endl; + pp.enrollProcessor(getvals); + Log::trace() << "ObserversTLAD::initializeTraj done" << std::endl; } // ----------------------------------------------------------------------------- template -void ObserversTLAD::doProcessingTraj(const State_ & xx) { - Log::trace() << "ObserversTLAD::doProcessingTraj start" << std::endl; - util::DateTime t1 = std::max(xx.validTime()-hslottraj_, winbgn_); - util::DateTime t2 = std::min(xx.validTime()+hslottraj_, winend_); - - for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - if (observerstlad_[jj]) observerstlad_[jj]->doProcessingTraj(xx, t1, t2); +void ObserversTLAD::finalizeTraj() { + Log::trace() << "ObserversTLAD::finalizeTraj start" << std::endl; + for (size_t jj = 0; jj < observers_.size(); ++jj) { + if (observers_[jj]) observers_[jj]->finalizeTraj((*ybias_)[jj]); } - Log::trace() << "ObserversTLAD::doProcessingTraj done" << std::endl; + Log::trace() << "ObserversTLAD::finalizeTraj done" << std::endl; } // ----------------------------------------------------------------------------- template -void ObserversTLAD::doFinalizeTraj(const State_ & xx) { - Log::trace() << "ObserversTLAD::doFinalizeTraj start" << std::endl; - for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - if (observerstlad_[jj]) observerstlad_[jj]->doFinalizeTraj(xx); +void ObserversTLAD::initializeTL(PostProcTLAD_ & pp) { + Log::trace() << "ObserversTLAD::initializeTL start" << std::endl; + std::shared_ptr getvals(new GetValueTLADs_(winbgn_, winend_)); + for (size_t jj = 0; jj < observers_.size(); ++jj) { + if (observers_[jj]) getvals->append(observers_[jj]->initializeTL()); } - Log::trace() << "ObserversTLAD::doFinalizeTraj done" << std::endl; + pp.enrollProcessor(getvals); + Log::trace() << "ObserversTLAD::initializeTL done" << std::endl; } // ----------------------------------------------------------------------------- template -void ObserversTLAD::setupTL(const ObsAuxIncrs_ & ybiastl) { - Log::trace() << "ObserversTLAD::setupTL start" << std::endl; - ydeptl_.reset(new Departures_(obspace_)); - ybiastl_ = &ybiastl; - Log::trace() << "ObserversTLAD::setupTL done" << std::endl; -} -// ----------------------------------------------------------------------------- -template -void ObserversTLAD::doInitializeTL(const Increment_ & dx, - const util::DateTime & end, - const util::Duration & tstep) { - Log::trace() << "ObserversTLAD::doInitializeTL start" << std::endl; - hslot_ = tstep/2; - for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - if (observerstlad_[jj]) observerstlad_[jj]->doInitializeTL(dx, winbgn_, winend_); +void ObserversTLAD::finalizeTL(const ObsAuxIncrs_ & ybias, Departures_ & dy) { + Log::trace() << "ObserversTLAD::finalizeTL start" << std::endl; + for (size_t jj = 0; jj < observers_.size(); ++jj) { + if (observers_[jj]) observers_[jj]->finalizeTL(ybias[jj], dy[jj]); } - Log::trace() << "ObserversTLAD::doInitializeTL done" << std::endl; + Log::trace() << "ObserversTLAD::finalizeTL done" << std::endl; } // ----------------------------------------------------------------------------- template -void ObserversTLAD::doProcessingTL(const Increment_ & dx) { - Log::trace() << "ObserversTLAD::doProcessingTL start" << std::endl; - util::DateTime t1 = std::max(dx.validTime()-hslot_, winbgn_); - util::DateTime t2 = std::min(dx.validTime()+hslot_, winend_); - - for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - if (observerstlad_[jj]) observerstlad_[jj]->doProcessingTL(dx, t1, t2); - } - Log::trace() << "ObserversTLAD::doProcessingTL done" << std::endl; -} -// ----------------------------------------------------------------------------- -template -void ObserversTLAD::doFinalizeTL(const Increment_ & dx) { - Log::trace() << "ObserversTLAD::doFinalizeTL start" << std::endl; - for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - if (observerstlad_[jj]) observerstlad_[jj]->doFinalizeTL(dx, (*ydeptl_)[jj], (*ybiastl_)[jj]); - } - Log::trace() << "ObserversTLAD::doFinalizeTL done" << std::endl; -} -// ----------------------------------------------------------------------------- -template -void ObserversTLAD::setupAD(std::shared_ptr ydepad, - ObsAuxIncrs_ & ybiasad) { - Log::trace() << "ObserversTLAD::setupAD start" << std::endl; - ydepad_ = ydepad; - ybiasad_ = &ybiasad; - Log::trace() << "ObserversTLAD::setupAD done" << std::endl; -} -// ----------------------------------------------------------------------------- -template -void ObserversTLAD::doFirstAD(Increment_ & dx, const util::DateTime & bgn, - const util::Duration & tstep) { - Log::trace() << "ObserversTLAD::doFirstAD start" << std::endl; - hslot_ = tstep/2; - - for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - if (observerstlad_[jj]) { - observerstlad_[jj]->doFirstAD(dx, (*ydepad_)[jj], (*ybiasad_)[jj], winbgn_, winend_); - } - } - Log::trace() << "ObserversTLAD::doFirstAD done" << std::endl; -} -// ----------------------------------------------------------------------------- -template -void ObserversTLAD::doProcessingAD(Increment_ & dx) { - Log::trace() << "ObserversTLAD::doProcessingAD start" << std::endl; - util::DateTime t1 = std::max(dx.validTime()-hslot_, winbgn_); - util::DateTime t2 = std::min(dx.validTime()+hslot_, winend_); - - for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - if (observerstlad_[jj]) observerstlad_[jj]->doProcessingAD(dx, t1, t2); +void ObserversTLAD::initializeAD(const Departures_ & dy, ObsAuxIncrs_ & ybias, + PostProcTLAD_ & pp) { + Log::trace() << "ObserversTLAD::initializeAD start" << std::endl; + std::shared_ptr getvals(new GetValueTLADs_(winbgn_, winend_)); + for (size_t jj = 0; jj < observers_.size(); ++jj) { + if (observers_[jj]) getvals->append(observers_[jj]->initializeAD(dy[jj], ybias[jj])); } - Log::trace() << "ObserversTLAD::doProcessingAD done" << std::endl; + pp.enrollProcessor(getvals); + Log::trace() << "ObserversTLAD::initializeAD done" << std::endl; } // ----------------------------------------------------------------------------- template -void ObserversTLAD::doLastAD(Increment_ & dx) { - Log::trace() << "ObserversTLAD::doLastAD start" << std::endl; - for (std::size_t jj = 0; jj < observerstlad_.size(); ++jj) { - if (observerstlad_[jj]) observerstlad_[jj]->doLastAD(dx); +void ObserversTLAD::finalizeAD() { + Log::trace() << "ObserversTLAD::finalizeAD start" << std::endl; + for (size_t jj = 0; jj < observers_.size(); ++jj) { + if (observers_[jj]) observers_[jj]->finalizeAD(); } - Log::trace() << "ObserversTLAD::doLastAD done" << std::endl; + Log::trace() << "ObserversTLAD::finalizeAD done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/PostBaseTLAD.h b/src/oops/base/PostBaseTLAD.h index 7207b014d..e4cc17c53 100644 --- a/src/oops/base/PostBaseTLAD.h +++ b/src/oops/base/PostBaseTLAD.h @@ -79,9 +79,6 @@ class PostBaseTLAD : private boost::noncopyable { this->doFinalizeTL(dx); } -/// Return TL dual space output - virtual std::unique_ptr releaseOutputFromTL() = 0; - /// Adjoint methods void initializeAD(Increment_ & dx, const util::DateTime & bgn, const util::Duration & step) { diff --git a/src/oops/base/PostProcessorTLAD.h b/src/oops/base/PostProcessorTLAD.h index dc6ba35e9..8d27b2946 100644 --- a/src/oops/base/PostProcessorTLAD.h +++ b/src/oops/base/PostProcessorTLAD.h @@ -91,11 +91,6 @@ class PostProcessorTLAD { } } -/// Get TL dual space output - std::unique_ptr releaseOutputFromTL(unsigned int ii) { - return processors_[ii]->releaseOutputFromTL(); - } - /// Adjoint methods void initializeAD(Increment_ & dx, const util::DateTime & bgn, const util::Duration & step) { diff --git a/src/oops/base/WeightedDiffTLAD.h b/src/oops/base/WeightedDiffTLAD.h index e2b19987e..44c068734 100644 --- a/src/oops/base/WeightedDiffTLAD.h +++ b/src/oops/base/WeightedDiffTLAD.h @@ -54,8 +54,8 @@ class WeightedDiffTLAD : public PostBaseTLAD { virtual ~WeightedDiffTLAD() {} Increment_ * releaseDiff() {return wdiff_.releaseDiff();} - std::unique_ptr releaseOutputFromTL() override; void setupTL(const Geometry_ &); + void finalTL(Increment_ &); void setupAD(std::shared_ptr); private: @@ -185,11 +185,12 @@ void WeightedDiffTLAD::doProcessingTL(const Increment_ & xx) { // ----------------------------------------------------------------------------- template -std::unique_ptr WeightedDiffTLAD::releaseOutputFromTL() { - Log::trace() << "WeightedDiffTLAD::releaseOutputFromTL" << std::endl; +void WeightedDiffTLAD::finalTL(Increment_ & out) { + Log::trace() << "WeightedDiffTLAD::finalTL start" << std::endl; ASSERT(linit_); ASSERT(std::abs(sum_) < 1.0e-8); - return std::move(avg_); + out = *avg_; + Log::trace() << "WeightedDiffTLAD::finalTL done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/interface/Increment.h b/src/oops/interface/Increment.h index 886e2f537..d52443ffc 100644 --- a/src/oops/interface/Increment.h +++ b/src/oops/interface/Increment.h @@ -29,7 +29,6 @@ #include "oops/util/Duration.h" #include "oops/util/gatherPrint.h" #include "oops/util/ObjectCounter.h" -#include "oops/util/Printable.h" #include "oops/util/Serializable.h" #include "oops/util/Timer.h" @@ -41,7 +40,6 @@ namespace interface { /// Some fields that are present in a State may not be present in an Increment. template class Increment : public oops::GeneralizedDepartures, - public util::Printable, public util::Serializable, private util::ObjectCounter > { typedef typename MODEL::Increment Increment_; From c4495632e965bba00937ac90a1779a2e5c3b5c31 Mon Sep 17 00:00:00 2001 From: Benjamin Menetrier <30638301+benjaminmenetrier@users.noreply.github.com> Date: Tue, 13 Apr 2021 19:54:52 +0200 Subject: [PATCH 103/142] QG fields and change of variables refactoring (#1144) * Truth working * Bugfixes * Bugfixes and updated test files * Final bugfix for 3d and 4dvar_change_var * Change of variable simplification, fields print cleaning * Update GOM arrays, add pointer destructor for oops_variables * Remove useless names * Reset qg_gom_create * Reset 3dvar_change_var Co-authored-by: Anna Shlyaeva --- qg/model/CMakeLists.txt | 3 +- qg/model/ChangeVarQG.cc | 2 - qg/model/ChangeVarTLADQG.cc | 16 +- qg/model/ChangeVarTLADQG.h | 3 - qg/model/FieldsQG.cc | 83 +- qg/model/GomQG.cc | 24 +- qg/model/GomQG.h | 3 + qg/model/ModelQG.h | 8 +- qg/model/QgFortran.h | 21 +- qg/model/TlmQG.cc | 26 +- qg/model/qg_change_var_interface.F90 | 105 +- qg/model/qg_change_var_mod.F90 | 426 +++---- qg/model/qg_convert_x_to_u_mod.F90 | 90 ++ ...o_uv_mod.F90 => qg_convert_x_to_v_mod.F90} | 61 +- qg/model/qg_error_covariance_interface.F90 | 21 +- qg/model/qg_error_covariance_mod.F90 | 161 +-- qg/model/qg_fields_interface.F90 | 44 +- qg/model/qg_fields_mod.F90 | 1118 +++++++++-------- qg/model/qg_getvalues_mod.F90 | 148 +-- qg/model/qg_gom_interface.F90 | 79 +- qg/model/qg_gom_mod.F90 | 438 ++++--- qg/model/qg_interp_mod.F90 | 52 +- qg/model/qg_model_mod.F90 | 173 ++- qg/model/qg_stream_mod.F90 | 10 +- qg/model/qg_wind_mod.F90 | 6 +- qg/model/qg_wspeed_interface.F90 | 11 +- qg/model/qg_wspeed_mod.F90 | 26 +- qg/test/CMakeLists.txt | 16 +- qg/test/testinput/4dvar_change_var.yaml | 111 -- qg/test/testinput/dirac_cov.yaml | 1 + qg/test/testinput/dirac_hyb_field.yaml | 1 + qg/test/testinput/dirac_hyb_value.yaml | 1 + qg/test/testinput/dirac_loc_3d.yaml | 1 + qg/test/testinput/dirac_loc_4d.yaml | 1 + qg/test/testinput/dirac_no_loc.yaml | 1 + qg/test/testoutput/3densvar.test | 16 +- qg/test/testoutput/3dfgat.test | 8 +- qg/test/testoutput/3dvar.test | 16 +- qg/test/testoutput/3dvar_change_var.test | 16 +- qg/test/testoutput/3dvar_hybrid.test | 16 +- .../3dvar_hybrid_wo_jb_evaluation.test | 16 +- qg/test/testoutput/4densvar.test | 112 +- qg/test/testoutput/4densvar_hybrid.test | 112 +- qg/test/testoutput/4dvar_change_var.test | 36 - qg/test/testoutput/4dvar_dripcg.test | 16 +- qg/test/testoutput/4dvar_drpcg_lmp.test | 16 +- qg/test/testoutput/4dvar_drpfom.test | 16 +- qg/test/testoutput/4dvar_drplanczos.test | 16 +- .../testoutput/4dvar_drplanczos_hybrid.test | 16 +- qg/test/testoutput/4dvar_ipcg.test | 16 +- qg/test/testoutput/4dvar_obs_biased.test | 16 +- qg/test/testoutput/4dvar_rpcg.test | 16 +- qg/test/testoutput/4dvar_saddlepoint.test | 32 +- qg/test/testoutput/addincrement.test | 20 +- qg/test/testoutput/addincrement_scaled.test | 24 +- qg/test/testoutput/analytic_forecast.test | 16 +- qg/test/testoutput/convertincrement.test | 32 +- qg/test/testoutput/convertstate.test | 128 +- qg/test/testoutput/dfi.test | 24 +- qg/test/testoutput/diffstates.test | 20 +- qg/test/testoutput/dirac_cov.test | 12 +- qg/test/testoutput/dirac_hyb_field.test | 12 +- qg/test/testoutput/dirac_hyb_value.test | 16 +- qg/test/testoutput/dirac_loc_3d.test | 16 +- qg/test/testoutput/dirac_loc_4d.test | 208 +-- qg/test/testoutput/dirac_no_loc.test | 12 +- qg/test/testoutput/eda_3dvar_block.test | 20 +- qg/test/testoutput/ens_forecast.test | 16 +- qg/test/testoutput/ens_hofx.test | 16 +- qg/test/testoutput/ens_recenter.test | 88 +- qg/test/testoutput/ens_variance.test | 4 +- .../ens_variance_inflation_field.test | 4 +- .../ens_variance_inflation_value.test | 4 +- qg/test/testoutput/forecast.test | 16 +- qg/test/testoutput/gen_ens_pert_B.test | 88 +- qg/test/testoutput/hofx.test | 16 +- qg/test/testoutput/hofx3d.test | 8 +- qg/test/testoutput/letkf.test | 168 +-- qg/test/testoutput/make_obs_3d.test | 16 +- qg/test/testoutput/make_obs_4d_12h.test | 16 +- qg/test/testoutput/make_obs_4d_24h.test | 16 +- qg/test/testoutput/make_obs_4d_biased.test | 16 +- qg/test/testoutput/rtpp.test | 32 +- qg/test/testoutput/static_b_init.test | 4 +- qg/test/testoutput/truth.test | 16 +- qg/test/testoutput/uniform_field_hybrid.test | 16 +- .../testoutput/uniform_field_inflation.test | 16 +- src/oops/base/variables_f.cc | 12 + src/oops/base/variables_f.h | 2 + src/oops/base/variables_interface.f | 16 + src/oops/base/variables_mod.F90 | 30 +- 91 files changed, 2223 insertions(+), 2772 deletions(-) create mode 100644 qg/model/qg_convert_x_to_u_mod.F90 rename qg/model/{qg_convert_x_to_uv_mod.F90 => qg_convert_x_to_v_mod.F90} (57%) delete mode 100644 qg/test/testinput/4dvar_change_var.yaml delete mode 100644 qg/test/testoutput/4dvar_change_var.test diff --git a/qg/model/CMakeLists.txt b/qg/model/CMakeLists.txt index 3e5908dea..b4c20b066 100644 --- a/qg/model/CMakeLists.txt +++ b/qg/model/CMakeLists.txt @@ -84,7 +84,8 @@ qg_change_var_mod.F90 qg_constants_mod.F90 qg_convert_q_to_x_mod.F90 qg_convert_x_to_q_mod.F90 -qg_convert_x_to_uv_mod.F90 +qg_convert_x_to_u_mod.F90 +qg_convert_x_to_v_mod.F90 qg_differential_solver_mod.F90 qg_error_covariance_interface.F90 qg_error_covariance_mod.F90 diff --git a/qg/model/ChangeVarQG.cc b/qg/model/ChangeVarQG.cc index 64fc8ad33..e2f9e1d54 100644 --- a/qg/model/ChangeVarQG.cc +++ b/qg/model/ChangeVarQG.cc @@ -22,12 +22,10 @@ ChangeVarQG::ChangeVarQG(const GeometryQG &, const eckit::Configuration &) {} ChangeVarQG::~ChangeVarQG() {} // ----------------------------------------------------------------------------- void ChangeVarQG::changeVar(const StateQG & xa, StateQG & xm) const { - ASSERT(xa.variables().has("x") || xa.variables().has("q")); qg_change_var_f90(xa.fields().toFortran(), xm.fields().toFortran()); } // ----------------------------------------------------------------------------- void ChangeVarQG::changeVarInverse(const StateQG & xm, StateQG & xa) const { - ASSERT(xm.variables().has("x") || xm.variables().has("q")); qg_change_var_f90(xm.fields().toFortran(), xa.fields().toFortran()); } // ----------------------------------------------------------------------------- diff --git a/qg/model/ChangeVarTLADQG.cc b/qg/model/ChangeVarTLADQG.cc index 0832e56b5..2222355ce 100644 --- a/qg/model/ChangeVarTLADQG.cc +++ b/qg/model/ChangeVarTLADQG.cc @@ -20,33 +20,27 @@ namespace qg { // ----------------------------------------------------------------------------- ChangeVarTLADQG::ChangeVarTLADQG(const StateQG &, const StateQG &, - const GeometryQG & resol, const eckit::Configuration & conf) { - oops::Log::trace() << "ChangeVarTLADQG::ChangeVarTLADQG start" << std::endl; - const oops::Variables vars_in(conf, "input variables"); - const oops::Variables vars_out(conf, "output variables"); - qg_change_var_setup_f90(keyConfig_, vars_in, vars_out); - oops::Log::trace() << "ChangeVarTLADQG::ChangeVarTLADQG done" << std::endl; -} + const GeometryQG & resol, const eckit::Configuration & conf) {} // ----------------------------------------------------------------------------- ChangeVarTLADQG::~ChangeVarTLADQG() {} // ----------------------------------------------------------------------------- void ChangeVarTLADQG::multiply(const IncrementQG & dxa, IncrementQG & dxm) const { - qg_change_var_tl_f90(keyConfig_, dxa.fields().toFortran(), dxm.fields().toFortran()); + qg_change_var_tl_f90(dxa.fields().toFortran(), dxm.fields().toFortran()); oops::Log::debug() << "ChangeVarTLADQG::multiply" << dxm << std::endl; } // ----------------------------------------------------------------------------- void ChangeVarTLADQG::multiplyInverse(const IncrementQG & dxm, IncrementQG & dxa) const { - qg_change_var_inv_tl_f90(keyConfig_, dxm.fields().toFortran(), dxa.fields().toFortran()); + qg_change_var_tl_f90(dxm.fields().toFortran(), dxa.fields().toFortran()); oops::Log::debug() << "ChangeVarTLADQG::multiplyInverse" << dxm << std::endl; } // ----------------------------------------------------------------------------- void ChangeVarTLADQG::multiplyAD(const IncrementQG & dxm, IncrementQG & dxa) const { - qg_change_var_ad_f90(keyConfig_, dxm.fields().toFortran(), dxa.fields().toFortran()); + qg_change_var_ad_f90(dxm.fields().toFortran(), dxa.fields().toFortran()); oops::Log::debug() << "ChangeVarTLADQG::multiplyAD" << dxm << std::endl; } // ----------------------------------------------------------------------------- void ChangeVarTLADQG::multiplyInverseAD(const IncrementQG & dxa, IncrementQG & dxm) const { - qg_change_var_inv_ad_f90(keyConfig_, dxa.fields().toFortran(), dxm.fields().toFortran()); + qg_change_var_ad_f90(dxa.fields().toFortran(), dxm.fields().toFortran()); oops::Log::debug() << "ChangeVarTLADQG::multiplyInverseAD" << dxm << std::endl; } // ----------------------------------------------------------------------------- diff --git a/qg/model/ChangeVarTLADQG.h b/qg/model/ChangeVarTLADQG.h index 6075181b2..574313783 100644 --- a/qg/model/ChangeVarTLADQG.h +++ b/qg/model/ChangeVarTLADQG.h @@ -44,9 +44,6 @@ class ChangeVarTLADQG: public util::Printable { private: void print(std::ostream &) const override; - -// Data - F90chvar keyConfig_; }; // ----------------------------------------------------------------------------- diff --git a/qg/model/FieldsQG.cc b/qg/model/FieldsQG.cc index 74f033e1e..37652dc4f 100644 --- a/qg/model/FieldsQG.cc +++ b/qg/model/FieldsQG.cc @@ -10,9 +10,11 @@ #include "model/FieldsQG.h" +#include #include #include #include +#include #include #include @@ -124,7 +126,7 @@ void FieldsQG::schur_product_with(const FieldsQG & dx) { } // ----------------------------------------------------------------------------- void FieldsQG::random() { - qg_fields_random_f90(keyFlds_); + qg_fields_random_f90(keyFlds_, vars_); } // ----------------------------------------------------------------------------- void FieldsQG::dirac(const eckit::Configuration & config) { @@ -177,46 +179,55 @@ double FieldsQG::norm() const { } // ----------------------------------------------------------------------------- void FieldsQG::print(std::ostream & os) const { - int nx, ny, nz, nb, lq, lbc; - qg_fields_sizes_f90(keyFlds_, nx, ny, nz, nb); - qg_fields_vars_f90(keyFlds_, lq, lbc); + // Resolution + int nx, ny, nz; + qg_fields_sizes_f90(keyFlds_, nx, ny, nz); os << std::endl << " Resolution = " << nx << ", " << ny << ", " << nz; - if (lq == 1) { - os << std::endl << " Variable = potential vorticity"; - } else { - os << std::endl << " Variable = streamfunction"; - } - if (lbc == 1) { - os << std::endl << " Boundary conditions are activated"; - } else { - os << std::endl << " Boundary conditions are not activated"; - } - std::vector zstat(3*(1+nb)); - qg_fields_gpnorm_f90(keyFlds_, nb, zstat[0]); - for (int jj = 0; jj < 1+nb; ++jj) { - std::ios_base::fmtflags f(os.flags()); - os << std::endl << std::scientific << std::setprecision(4) - << " Min=" << std::setw(12) << zstat[3*jj] - << ", Max=" << std::setw(12) << zstat[3*jj+1] - << ", RMS=" << std::setw(12) << zstat[3*jj+2]; - os.flags(f); + + // Min, max, RMS of fields + std::vector var{"Streamfunction :", + "Potential vorticity :", + "U-wind :", + "V-wind :", + "Streamfunction LBC :", + "Potential vorticity LBC:"}; + std::vector vpresent(6); + std::vector vmin(6); + std::vector vmax(6); + std::vector vrms(6); + qg_fields_gpnorm_f90(keyFlds_, vpresent.data(), vmin.data(), vmax.data(), vrms.data()); + for (int jj = 0; jj < 6; ++jj) { + if (vpresent[jj] == 1) { + std::ios_base::fmtflags f(os.flags()); + os << std::endl << " " << var[jj] << std::scientific << std::setprecision(4) + << " Min=" << std::setw(12) << vmin[jj] + << ", Max=" << std::setw(12) << vmax[jj] + << ", RMS=" << std::setw(12) << vrms[jj]; + os.flags(f); + } } } // ----------------------------------------------------------------------------- bool FieldsQG::isForModel(const bool & nonlinear) const { - int nx, ny, nz, nb; - qg_fields_sizes_f90(keyFlds_, nx, ny, nz, nb); bool ok = true; - if (nonlinear) ok = (nb == 2); + if (nonlinear) { + int lbc; + qg_fields_lbc_f90(keyFlds_, lbc); + ok = (lbc == 1); + } return ok; } // ----------------------------------------------------------------------------- oops::LocalIncrement FieldsQG::getLocal(const GeometryQGIterator & iter) const { - int nx, ny, nz, nb; - qg_fields_sizes_f90(keyFlds_, nx, ny, nz, nb); - std::vector varlens(1, nz); - std::vector values(nz); - qg_fields_getpoint_f90(keyFlds_, iter.toFortran(), nz, values[0]); + int nx, ny, nz; + qg_fields_sizes_f90(keyFlds_, nx, ny, nz); + std::vector varlens(vars_.size()); + for (unsigned int ii = 0; ii < vars_.size(); ii++) { + varlens[ii] = nz; + } + int lenvalues = std::accumulate(varlens.begin(), varlens.end(), 0); + std::vector values(lenvalues); + qg_fields_getpoint_f90(keyFlds_, iter.toFortran(), values.size(), values[0]); return oops::LocalIncrement(vars_, values, varlens); } // ----------------------------------------------------------------------------- @@ -227,9 +238,13 @@ void FieldsQG::setLocal(const oops::LocalIncrement & x, const GeometryQGIterator // ----------------------------------------------------------------------------- size_t FieldsQG::serialSize() const { size_t nn = 0; - int nx, ny, nz, nb; - qg_fields_sizes_f90(keyFlds_, nx, ny, nz, nb); - nn += nx * ny * nz + nb * (nx + 1) * nz; + int nx, ny, nz, lbc; + qg_fields_sizes_f90(keyFlds_, nx, ny, nz); + qg_fields_lbc_f90(keyFlds_, lbc); + nn += nx * ny * nz; + if (lbc == 1) { + nn += + 2 * (nx + 1) * nz; + } nn += time_.serialSize(); return nn; } diff --git a/qg/model/GomQG.cc b/qg/model/GomQG.cc index b0195c1ff..b4532f719 100644 --- a/qg/model/GomQG.cc +++ b/qg/model/GomQG.cc @@ -22,24 +22,29 @@ namespace qg { // ----------------------------------------------------------------------------- -GomQG::GomQG(const LocationsQG & locs, const oops::Variables & var) { +GomQG::GomQG(const LocationsQG & locs, const oops::Variables & vars): + vars_(vars) +{ // gom_setup just creates and allocates the GeoVaLs object without filling // in values - qg_gom_setup_f90(keyGom_, locs, var); + qg_gom_setup_f90(keyGom_, locs, vars_); } // ----------------------------------------------------------------------------- /*! QG GeoVaLs Constructor with Config */ GomQG::GomQG(const eckit::Configuration & config, - const ObsSpaceQG & ospace, const oops::Variables &) + const ObsSpaceQG & ospace, const oops::Variables & vars): + vars_(vars) { - qg_gom_create_f90(keyGom_); + qg_gom_create_f90(keyGom_, vars_); qg_gom_read_file_f90(keyGom_, config); } // ----------------------------------------------------------------------------- // Copy constructor -GomQG::GomQG(const GomQG & other) { - qg_gom_create_f90(keyGom_); +GomQG::GomQG(const GomQG & other): + vars_(other.vars_) +{ + qg_gom_create_f90(keyGom_, vars_); qg_gom_copy_f90(keyGom_, other.keyGom_); } // ----------------------------------------------------------------------------- @@ -124,13 +129,14 @@ void GomQG::print(std::ostream & os) const { if (zmin >= 0.0) { double mxval; - int iloc, ivar; + int iloc; + oops::Variables maxvar; - qg_gom_maxloc_f90(keyGom_, mxval, iloc, ivar); + qg_gom_maxloc_f90(keyGom_, mxval, iloc, maxvar); oops::Log::debug() << "GomQG: Maximum Value = " << std::setprecision(4) << mxval << " at location = " << iloc - << " and variable = " << ivar << std::endl; + << " and variable = " << maxvar << std::endl; } } // ----------------------------------------------------------------------------- diff --git a/qg/model/GomQG.h b/qg/model/GomQG.h index 08a2e0345..650e3cce5 100644 --- a/qg/model/GomQG.h +++ b/qg/model/GomQG.h @@ -15,6 +15,8 @@ #include #include +#include "oops/base/Variables.h" + #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" @@ -62,6 +64,7 @@ class GomQG : public util::Printable, private: void print(std::ostream &) const; F90gom keyGom_; + oops::Variables vars_; }; } // namespace qg diff --git a/qg/model/ModelQG.h b/qg/model/ModelQG.h index 73bc0e728..92cba8fdd 100644 --- a/qg/model/ModelQG.h +++ b/qg/model/ModelQG.h @@ -41,17 +41,11 @@ class ModelQgParameters : public oops::ModelParametersBase { OOPS_CONCRETE_PARAMETERS(ModelQgParameters, ModelParametersBase) public: - /// Model option: using stream function or potential vorticity as variable - oops::Parameter use_vorticity{"use potential vorticity", false, this}; /// Model time step oops::RequiredParameter tstep{"tstep", this}; oops::Variables variables() const { - if (use_vorticity) { - return oops::Variables({"q"}); - } else { - return oops::Variables({"x"}); - } + return oops::Variables({"x"}); } }; diff --git a/qg/model/QgFortran.h b/qg/model/QgFortran.h index 860357fdc..af18e7ea1 100644 --- a/qg/model/QgFortran.h +++ b/qg/model/QgFortran.h @@ -37,8 +37,6 @@ namespace qg { class LocationsQG; class ObsSpaceQG; -// Change of variable key type -typedef int F90chvar; // Geometry key type typedef int F90geom; // Geometry iterator key type @@ -68,12 +66,9 @@ extern "C" { // ----------------------------------------------------------------------------- // Change of variable // ----------------------------------------------------------------------------- - void qg_change_var_setup_f90(F90chvar &, const oops::Variables &, const oops::Variables &); void qg_change_var_f90(const F90flds &, const F90flds &); - void qg_change_var_tl_f90(const F90chvar &, const F90flds &, const F90flds &); - void qg_change_var_inv_tl_f90(const F90chvar &, const F90flds &, const F90flds &); - void qg_change_var_ad_f90(const F90chvar &, const F90flds &, const F90flds &); - void qg_change_var_inv_ad_f90(const F90chvar &, const F90flds &, const F90flds &); + void qg_change_var_tl_f90(const F90flds &, const F90flds &); + void qg_change_var_ad_f90(const F90flds &, const F90flds &); // ----------------------------------------------------------------------------- // Error covariance @@ -95,7 +90,7 @@ extern "C" { void qg_fields_zero_f90(const F90flds &); void qg_fields_ones_f90(const F90flds &); void qg_fields_dirac_f90(const F90flds &, const eckit::Configuration &); - void qg_fields_random_f90(const F90flds &); + void qg_fields_random_f90(const F90flds &, const oops::Variables &); void qg_fields_copy_f90(const F90flds &, const F90flds &); void qg_fields_self_add_f90(const F90flds &, const F90flds &); void qg_fields_self_sub_f90(const F90flds &, const F90flds &); @@ -112,10 +107,10 @@ extern "C" { const util::DateTime &); void qg_fields_analytic_init_f90(const F90flds &, const eckit::Configuration &, util::DateTime &); - void qg_fields_gpnorm_f90(const F90flds &, const int &, double &); + void qg_fields_gpnorm_f90(const F90flds &, int[], double[], double[], double[]); void qg_fields_rms_f90(const F90flds &, double &); - void qg_fields_sizes_f90(const F90flds &, int &, int &, int &, int &); - void qg_fields_vars_f90(const F90flds &, int &, int &); + void qg_fields_sizes_f90(const F90flds &, int &, int &, int &); + void qg_fields_lbc_f90(const F90flds &, int &); void qg_fields_set_atlas_f90(const F90flds &, const oops::Variables &, atlas::field::FieldSetImpl *); void qg_fields_to_atlas_f90(const F90flds &, const oops::Variables &, @@ -167,7 +162,7 @@ extern "C" { // Local Values (GOM) // ----------------------------------------------------------------------------- void qg_gom_setup_f90(F90gom &, const LocationsQG &, const oops::Variables &); - void qg_gom_create_f90(F90gom &); + void qg_gom_create_f90(F90gom &, const oops::Variables &); void qg_gom_delete_f90(F90gom &); void qg_gom_copy_f90(const F90gom &, const F90gom &); void qg_gom_zero_f90(const F90gom &); @@ -181,7 +176,7 @@ extern "C" { void qg_gom_rms_f90(const F90gom &, double &); void qg_gom_dotprod_f90(const F90gom &, const F90gom &, double &); void qg_gom_stats_f90(const F90gom &, int &, double &, double &, double &); - void qg_gom_maxloc_f90(const F90gom &, double &, int &, int &); + void qg_gom_maxloc_f90(const F90gom &, double &, int &, const oops::Variables &); void qg_gom_read_file_f90(const F90gom &, const eckit::Configuration &); void qg_gom_write_file_f90(const F90gom &, const eckit::Configuration &); void qg_gom_analytic_init_f90(const F90gom &, const LocationsQG &, diff --git a/qg/model/TlmQG.cc b/qg/model/TlmQG.cc index 08a0a4b5a..389013d36 100644 --- a/qg/model/TlmQG.cc +++ b/qg/model/TlmQG.cc @@ -111,31 +111,9 @@ void TlmQG::print(std::ostream & os) const { if (traj_.size() > 0) { os << "QG TLM Trajectory: times are:"; for (trajICst jtra = traj_.begin(); jtra != traj_.end(); ++jtra) { + // Time os << " " << jtra->first; - int nx, ny, nz, nb, lq, lbc; - qg_fields_sizes_f90(jtra->second, nx, ny, nz, nb); - qg_fields_vars_f90(jtra->second, lq, lbc); - os << std::endl << " Resolution = " << nx << ", " << ny << ", " << nz; - if (lq == 1) { - os << std::endl << " Variable = potential vorticity"; - } else { - os << std::endl << " Variable = streamfunction"; - } - if (lbc == 1) { - os << std::endl << " Boundary conditions are activated"; - } else { - os << std::endl << " Boundary conditions are not activated"; - } - std::vector zstat(3*(1+nb)); - qg_fields_gpnorm_f90(jtra->second, nb, zstat[0]); - for (int jj = 0; jj < 1+nb; ++jj) { - std::ios_base::fmtflags f(os.flags()); - os << std::endl << std::scientific << std::setprecision(4) - << " Min=" << std::setw(12) << zstat[3*jj] - << ", Max=" << std::setw(12) << zstat[3*jj+1] - << ", RMS=" << std::setw(12) << zstat[3*jj+2]; - os.flags(f); - } + os << " " << jtra->second; } } } diff --git a/qg/model/qg_change_var_interface.F90 b/qg/model/qg_change_var_interface.F90 index aa8df9679..47166c8ed 100644 --- a/qg/model/qg_change_var_interface.F90 +++ b/qg/model/qg_change_var_interface.F90 @@ -20,46 +20,6 @@ module qg_change_var_interface ! ------------------------------------------------------------------------------ contains ! ------------------------------------------------------------------------------ -!> Setup change of variable -subroutine qg_change_var_setup_c(c_key_self,c_vars_in,c_vars_out) bind (c,name='qg_change_var_setup_f90') - -implicit none - -! Passed variables -integer(c_int),intent(inout) :: c_key_self !< Variable change -type(c_ptr),value,intent(in) :: c_vars_in !< Input variables -type(c_ptr),value,intent(in) :: c_vars_out !< Output variables - -! Local variable -type(qg_change_var_config),pointer :: self -type(oops_variables) :: vars_in,vars_out - -! Interface -call qg_change_var_registry%init() -call qg_change_var_registry%add(c_key_self) -call qg_change_var_registry%get(c_key_self,self) -vars_in = oops_variables(c_vars_in) -vars_out = oops_variables(c_vars_out) - -! Call Fortran -call qg_change_var_setup(self,vars_in,vars_out) - -end subroutine qg_change_var_setup_c -! ------------------------------------------------------------------------------ -!> Delete error covariance matrix -subroutine qg_change_var_delete_c(c_key_self) bind (c,name='qg_change_var_delete_f90') - -implicit none - -! Passed variables -integer(c_int),intent(inout) :: c_key_self !< Error covariance configuration - -! Clear interface -call qg_change_var_registry%remove(c_key_self) - -end subroutine qg_change_var_delete_c -! ------------------------------------------------------------------------------ - !> Change of variable subroutine qg_change_var_c(c_key_fld_in,c_key_fld_out) bind (c,name='qg_change_var_f90') @@ -80,102 +40,47 @@ subroutine qg_change_var_c(c_key_fld_in,c_key_fld_out) bind (c,name='qg_change_v call qg_change_var(fld_in,fld_out) end subroutine qg_change_var_c - ! ------------------------------------------------------------------------------ -!> Change of variable -subroutine qg_change_var_tl_c(c_key_conf,c_key_fld_in,c_key_fld_out) bind (c,name='qg_change_var_tl_f90') +!> Change of variable - tangent linear +subroutine qg_change_var_tl_c(c_key_fld_in,c_key_fld_out) bind (c,name='qg_change_var_tl_f90') implicit none ! Passed variables -integer(c_int),intent(in) :: c_key_conf !< Variable change integer(c_int),intent(in) :: c_key_fld_in !< Input field integer(c_int),intent(in) :: c_key_fld_out !< Output field ! Local variables -type(qg_change_var_config),pointer :: conf type(qg_fields),pointer :: fld_in,fld_out ! Interface -call qg_change_var_registry%get(c_key_conf,conf) call qg_fields_registry%get(c_key_fld_in,fld_in) call qg_fields_registry%get(c_key_fld_out,fld_out) ! Call Fortran -call qg_change_var_tl(conf,fld_in,fld_out) +call qg_change_var_tl(fld_in,fld_out) end subroutine qg_change_var_tl_c ! ------------------------------------------------------------------------------ -!> Change of variable - inverse -subroutine qg_change_var_inv_tl_c(c_key_conf,c_key_fld_in,c_key_fld_out) bind (c,name='qg_change_var_inv_tl_f90') - -implicit none - -! Passed variables -integer(c_int),intent(in) :: c_key_conf !< Configuration -integer(c_int),intent(in) :: c_key_fld_in !< Input field -integer(c_int),intent(in) :: c_key_fld_out !< Output field - -! Local variables -type(qg_change_var_config),pointer :: conf -type(qg_fields),pointer :: fld_in,fld_out - -! Interface -call qg_change_var_registry%get(c_key_conf,conf) -call qg_fields_registry%get(c_key_fld_in,fld_in) -call qg_fields_registry%get(c_key_fld_out,fld_out) - -! Call Fortran -call qg_change_var_inv_tl(conf,fld_in,fld_out) - -end subroutine qg_change_var_inv_tl_c -! ------------------------------------------------------------------------------ !> Change of variable - adjoint -subroutine qg_change_var_ad_c(c_key_conf,c_key_fld_in,c_key_fld_out) bind (c,name='qg_change_var_ad_f90') +subroutine qg_change_var_ad_c(c_key_fld_in,c_key_fld_out) bind (c,name='qg_change_var_ad_f90') implicit none ! Passed variables -integer(c_int),intent(in) :: c_key_conf !< Configuration integer(c_int),intent(in) :: c_key_fld_in !< Input field integer(c_int),intent(in) :: c_key_fld_out !< Output field ! Local variables -type(qg_change_var_config),pointer :: conf type(qg_fields),pointer :: fld_in,fld_out ! Interface -call qg_change_var_registry%get(c_key_conf,conf) call qg_fields_registry%get(c_key_fld_in,fld_in) call qg_fields_registry%get(c_key_fld_out,fld_out) ! Call Fortran -call qg_change_var_ad(conf,fld_in,fld_out) +call qg_change_var_ad(fld_in,fld_out) end subroutine qg_change_var_ad_c ! ------------------------------------------------------------------------------ -!> Change of variable - inverse adjoint -subroutine qg_change_var_inv_ad_c(c_key_conf,c_key_fld_in,c_key_fld_out) bind (c,name='qg_change_var_inv_ad_f90') - -implicit none - -! Passed variables -integer(c_int),intent(in) :: c_key_conf !< Configuration -integer(c_int),intent(in) :: c_key_fld_in !< Input field -integer(c_int),intent(in) :: c_key_fld_out !< Output field - -! Local variables -type(qg_change_var_config),pointer :: conf -type(qg_fields),pointer :: fld_in,fld_out - -! Interface -call qg_change_var_registry%get(c_key_conf,conf) -call qg_fields_registry%get(c_key_fld_in,fld_in) -call qg_fields_registry%get(c_key_fld_out,fld_out) - -! Call Fortran -call qg_change_var_inv_ad(conf,fld_in,fld_out) - -end subroutine qg_change_var_inv_ad_c -! ------------------------------------------------------------------------------ end module qg_change_var_interface diff --git a/qg/model/qg_change_var_mod.F90 b/qg/model/qg_change_var_mod.F90 index c12344a70..329f2ea5b 100644 --- a/qg/model/qg_change_var_mod.F90 +++ b/qg/model/qg_change_var_mod.F90 @@ -11,20 +11,36 @@ module qg_change_var_mod use kinds use qg_convert_q_to_x_mod use qg_convert_x_to_q_mod -use qg_convert_x_to_uv_mod +use qg_convert_x_to_u_mod +use qg_convert_x_to_v_mod use qg_fields_mod use oops_variables_mod implicit none private -public :: qg_change_var_config -public :: qg_change_var_registry, qg_change_var_setup -public :: qg_change_var_tl, qg_change_var_inv_tl, qg_change_var_ad, qg_change_var_inv_ad +public :: qg_change_var_registry public :: qg_change_var +public :: qg_change_var_tl,qg_change_var_ad ! ------------------------------------------------------------------------------ type :: qg_change_var_config - character(len=1024) :: varchange !< Variable change name + ! Input variables flags + logical :: x_in + logical :: q_in + logical :: u_in + logical :: v_in + + ! Output variables flags + logical :: x_out + logical :: q_out + logical :: u_out + logical :: v_out + + ! Conversions + logical :: x_to_q + logical :: q_to_x + logical :: x_to_u + logical :: x_to_v end type qg_change_var_config #define LISTED_TYPE qg_change_var_config @@ -42,268 +58,244 @@ module qg_change_var_mod !> Linked list implementation #include "oops/util/linkedList_c.f" ! ------------------------------------------------------------------------------ -!> Setup change of variable -subroutine qg_change_var_setup(self,vars_in,vars_out) +!> Setup change of variables setup +subroutine qg_change_var_setup(conf,fld_in,fld_out,ad) implicit none ! Passed variables -type(qg_change_var_config),intent(inout) :: self !< Variable change -type(oops_variables),intent(in) :: vars_in !< Input variables -type(oops_variables),intent(in) :: vars_out !< Output variables +type(qg_change_var_config),intent(inout) :: conf !< Variable change +type(qg_fields),intent(in) :: fld_in !< Input field +type(qg_fields),intent(in) :: fld_out !< Output field +logical,intent(in),optional :: ad !< Adjoint flag + +! Local variables +logical :: lad + +! Local flag +lad = .false. +if (present(ad)) lad = ad + +! Check what is allocated in fields +if (lad) then + ! Adjoint case + conf%x_in = allocated(fld_out%x) + conf%q_in = allocated(fld_out%q) + conf%u_in = allocated(fld_out%u) + conf%v_in = allocated(fld_out%v) + conf%x_out = allocated(fld_in%x) + conf%q_out = allocated(fld_in%q) + conf%u_out = allocated(fld_in%u) + conf%v_out = allocated(fld_in%v) +else + ! Normal case + conf%x_in = allocated(fld_in%x) + conf%q_in = allocated(fld_in%q) + conf%u_in = allocated(fld_in%u) + conf%v_in = allocated(fld_in%v) + conf%x_out = allocated(fld_out%x) + conf%q_out = allocated(fld_out%q) + conf%u_out = allocated(fld_out%u) + conf%v_out = allocated(fld_out%v) +endif -! Check -if ((vars_in%nvars() /= 1) .or. (vars_out%nvars() /= 1)) then - call abor1_ftn('qg_change_var_setup: wrong change of variable') +! Check input/output consistency +if (conf%x_out.and.(.not.(conf%x_in.or.conf%q_in))) call abor1_ftn('qg_change_var_setup: x or q required to compute x') +if (conf%q_out.and.(.not.(conf%x_in.or.conf%q_in))) call abor1_ftn('qg_change_var_setup: x or q required to compute q') +if (conf%u_out.and.(.not.(conf%x_in.or.conf%q_in.or.conf%u_in))) & + & call abor1_ftn('qg_change_var_setup: x, q or u required to compute u') +if (conf%v_out.and.(.not.(conf%x_in.or.conf%q_in.or.conf%v_in))) & + & call abor1_ftn('qg_change_var_setup: x, q or v required to compute v') + +! Initialize required conversions +conf%q_to_x = .false. +conf%x_to_q = .false. +conf%x_to_u = .false. +conf%x_to_v = .false. + +! Define required conversions +if (conf%x_out) conf%q_to_x = (.not.conf%x_in) +if (conf%q_out) conf%x_to_q = (.not.conf%q_in) +if (conf%u_out) then + if (.not.conf%u_in) then + conf%x_to_u = .true. + conf%q_to_x = (.not.conf%x_in) + endif endif -if (vars_in%variable(1) == vars_out%variable(1)) then - self%varchange = 'identity' -elseif ((vars_in%variable(1) == 'x') .and. (vars_out%variable(1) == 'q')) then - self%varchange = 'x_to_q' -elseif ((vars_in%variable(1) == 'q') .and. (vars_out%variable(1) == 'x')) then - self%varchange = 'q_to_x' -else - call abor1_ftn('qg_change_var_setup: wrong change of variable') +if (conf%v_out) then + if (.not.conf%v_in) then + conf%x_to_v = .true. + conf%q_to_x = (.not.conf%x_in) + endif endif end subroutine qg_change_var_setup ! ------------------------------------------------------------------------------ +!> Get variables +subroutine qg_change_var_get(conf,fld,x,q,u,v) -!> Change of variable -subroutine qg_change_var(fld_in,fld_out) implicit none -type(qg_fields),intent(in) :: fld_in !< Input fields -type(qg_fields),intent(inout) :: fld_out !< Output fields -real(kind_real), allocatable :: tmpu(:,:,:), tmpv(:,:,:), tmpx(:,:,:) -! Checks -call qg_fields_check_resolution(fld_in,fld_out) -if (.not.allocated(fld_in%streamfct).and..not.allocated(fld_in%pv)) then - call abor1_ftn('qg_change_var: streamfct or pv input required') +! Passed variables +type(qg_change_var_config),intent(in) :: conf !< Variable change +type(qg_fields),intent(in) :: fld !< Fields +real(kind_real),intent(out) :: x(fld%geom%nx,fld%geom%ny,fld%geom%nz) !< Streamfunction +real(kind_real),intent(out) :: q(fld%geom%nx,fld%geom%ny,fld%geom%nz) !< Potential vorticity +real(kind_real),intent(out) :: u(fld%geom%nx,fld%geom%ny,fld%geom%nz) !< Zonal wind +real(kind_real),intent(out) :: v(fld%geom%nx,fld%geom%ny,fld%geom%nz) !< Meridional wind + +if (allocated(fld%x)) then + x = fld%x +else + x = 0.0_kind_real endif - -call qg_fields_copy(fld_out,fld_in,.true.) - -! Get streamfunction -if (allocated(fld_out%streamfct)) then - if (allocated(fld_in%streamfct)) then - fld_out%streamfct = fld_in%streamfct - write(*,*)'qg_change_var: copied streamfct' - else - call convert_q_to_x(fld_in%geom, fld_in%pv, fld_in%x_north, fld_in%x_south, fld_out%streamfct) - write(*,*)'qg_change_var: converted pv to streamfct' - endif - fld_out%gfld3d = fld_out%streamfct +if (allocated(fld%q)) then + q = fld%q +else + q = 0.0_kind_real endif - -! Get potential vorticity -if (allocated(fld_out%pv)) then - if (allocated(fld_in%pv)) then - fld_out%pv = fld_in%pv - write(*,*)'qg_change_var: copied pv' - else - call convert_x_to_q(fld_in%geom, fld_in%streamfct, fld_in%x_north, fld_in%x_south, fld_out%pv) - write(*,*)'qg_change_var: converted streamfct to pv' - endif - if (fld_out%lq) fld_out%gfld3d = fld_out%pv +if (allocated(fld%u)) then + u = fld%u +else + u = 0.0_kind_real endif - -! Get u and v -if (allocated(fld_out%u) .or. allocated(fld_out%v)) then - allocate(tmpu(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz)) - allocate(tmpv(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz)) - if (allocated(fld_in%streamfct)) then - call convert_x_to_uv(fld_in%geom,fld_in%streamfct,fld_in%x_north,fld_in%x_south, tmpu,tmpv) - write(*,*)'qg_change_var: converted streamfct in to uv' - elseif (allocated(fld_out%streamfct)) then - call convert_x_to_uv(fld_out%geom,fld_out%streamfct,fld_out%x_north,fld_out%x_south, tmpu, tmpv) - write(*,*)'qg_change_var: converted streamfct out to uv' - else ! fld_in has q if it did not have x - allocate(tmpx(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz)) - call convert_q_to_x(fld_in%geom, fld_in%pv, fld_in%x_north, fld_in%x_south, tmpx) - call convert_x_to_uv(fld_in%geom, tmpx, fld_in%x_north, fld_in%x_south, tmpu, tmpv) - write(*,*)'qg_change_var: converted pv in to uv' - deallocate(tmpx) - endif - if (allocated(fld_out%u)) fld_out%u = tmpu - if (allocated(fld_out%v)) fld_out%v = tmpv - deallocate(tmpu) - deallocate(tmpv) +if (allocated(fld%v)) then + v = fld%v +else + v = 0.0_kind_real endif -end subroutine qg_change_var +end subroutine qg_change_var_get +! ------------------------------------------------------------------------------ +!> Set variables +subroutine qg_change_var_set(conf,fld,x,q,u,v) + +implicit none +! Passed variables +type(qg_change_var_config),intent(in) :: conf !< Variable change +type(qg_fields),intent(inout) :: fld !< Fields +real(kind_real),intent(in) :: x(fld%geom%nx,fld%geom%ny,fld%geom%nz) !< Streamfunction +real(kind_real),intent(in) :: q(fld%geom%nx,fld%geom%ny,fld%geom%nz) !< Potential vorticity +real(kind_real),intent(in) :: u(fld%geom%nx,fld%geom%ny,fld%geom%nz) !< Zonal wind +real(kind_real),intent(in) :: v(fld%geom%nx,fld%geom%ny,fld%geom%nz) !< Meridional wind + +if (allocated(fld%x)) fld%x = x +if (allocated(fld%q)) fld%q = q +if (allocated(fld%u)) fld%u = u +if (allocated(fld%v)) fld%v = v + +end subroutine qg_change_var_set ! ------------------------------------------------------------------------------ !> Change of variable -subroutine qg_change_var_tl(conf,fld_in,fld_out) +subroutine qg_change_var(fld_in,fld_out) implicit none ! Passed variables -type(qg_change_var_config),intent(in) :: conf !< Variable change -type(qg_fields),intent(in) :: fld_in !< Input fields -type(qg_fields),intent(inout) :: fld_out !< Output fields +type(qg_fields),intent(in) :: fld_in !< Input fields +type(qg_fields),intent(inout) :: fld_out !< Output fields -! Check fields resolution +! Local variables +real(kind_real) :: x(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz),q(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz) +real(kind_real) :: u(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz),v(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz) +type(qg_change_var_config) :: conf + +! Check resolution call qg_fields_check_resolution(fld_in,fld_out) -select case (trim(conf%varchange)) -case ('identity') - ! Copy fields - call qg_fields_copy(fld_out,fld_in) -case ('x_to_q') - ! Check fields variables - if (fld_in%lq) call abor1_ftn('qg_change_var_tl: wrong input fields variables for '//trim(conf%varchange)) - if (.not.fld_out%lq) call abor1_ftn('qg_change_var_tl: wrong output fields variables for '//trim(conf%varchange)) - - ! Conversion - call convert_x_to_q_tl(fld_in%geom,fld_in%gfld3d,fld_out%gfld3d) - - ! Copy boundary conditions - call qg_fields_copy(fld_out,fld_in,.true.) -case ('q_to_x') - ! Check fields variables - if (.not.fld_in%lq) call abor1_ftn('qg_change_var_tl: wrong input fields variables for '//trim(conf%varchange)) - if (fld_out%lq) call abor1_ftn('qg_change_var_tl: wrong output fields variables for '//trim(conf%varchange)) - - ! Conversion - call convert_q_to_x_tl(fld_in%geom,fld_in%gfld3d,fld_out%gfld3d) - - ! Copy boundary conditions - call qg_fields_copy(fld_out,fld_in,.true.) -case default - call abor1_ftn('qg_change_var_tl: wrong variable change') -end select +! Copy boundary conditions +call qg_fields_copy_lbc(fld_out,fld_in) -end subroutine qg_change_var_tl +! Define change of variable configuration +call qg_change_var_setup(conf,fld_in,fld_out) + +! Get x, q, u and v +call qg_change_var_get(conf,fld_in,x,q,u,v) + +! Conversions +if (conf%x_to_q) call convert_x_to_q(fld_in%geom,x,fld_in%x_north,fld_in%x_south,q) +if (conf%q_to_x) call convert_q_to_x(fld_in%geom,q,fld_in%x_north,fld_in%x_south,x) +if (conf%x_to_u) call convert_x_to_u(fld_in%geom,x,fld_in%x_north,fld_in%x_south,u) +if (conf%x_to_v) call convert_x_to_v(fld_in%geom,x,v) + +! Set x, q, u and v +call qg_change_var_set(conf,fld_out,x,q,u,v) + +end subroutine qg_change_var ! ------------------------------------------------------------------------------ -!> Change of variable - inverse -subroutine qg_change_var_inv_tl(conf,fld_in,fld_out) +!> Change of variable +subroutine qg_change_var_tl(fld_in,fld_out) implicit none ! Passed variables -type(qg_change_var_config),intent(in) :: conf !< Variable change -type(qg_fields),intent(in) :: fld_in !< Input fields -type(qg_fields),intent(inout) :: fld_out !< Output fields +type(qg_fields),intent(in) :: fld_in !< Input fields +type(qg_fields),intent(inout) :: fld_out !< Output fields -! Check fields resolution +! Local variables +real(kind_real) :: x(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz),q(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz) +real(kind_real) :: u(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz),v(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz) +type(qg_change_var_config) :: conf + +! Check resolution call qg_fields_check_resolution(fld_in,fld_out) -select case (trim(conf%varchange)) -case ('identity') - ! Copy fields - call qg_fields_copy(fld_out,fld_in) -case ('x_to_q') - ! Check fields variables - if (.not.fld_in%lq) call abor1_ftn('qg_change_var_inv_tl: wrong input fields variables for '//trim(conf%varchange)) - if (fld_out%lq) call abor1_ftn('qg_change_var_inv_tl: wrong output fields variables for '//trim(conf%varchange)) - - ! Conversion - call convert_q_to_x_tl(fld_in%geom,fld_in%gfld3d,fld_out%gfld3d) - - ! Copy boundary conditions - call qg_fields_copy(fld_out,fld_in,.true.) -case ('q_to_x') - ! Check fields variables - if (fld_in%lq) call abor1_ftn('qg_change_var_inv_tl: wrong input fields variables for '//trim(conf%varchange)) - if (.not.fld_out%lq) call abor1_ftn('qg_change_var_inv_tl: wrong output fields variables for '//trim(conf%varchange)) - - ! Conversion - call convert_x_to_q_tl(fld_in%geom,fld_in%gfld3d,fld_out%gfld3d) - - ! Copy boundary conditions - call qg_fields_copy(fld_out,fld_in,.true.) -case default - call abor1_ftn('qg_change_var_inv_tl: wrong variable change') -end select - -end subroutine qg_change_var_inv_tl +! Copy boundary conditions +call qg_fields_copy_lbc(fld_out,fld_in) + +! Define change of variable configuration +call qg_change_var_setup(conf,fld_in,fld_out) + +! Get x, q, u and v +call qg_change_var_get(conf,fld_in,x,q,u,v) + +! Conversions +if (conf%x_to_q) call convert_x_to_q_tl(fld_in%geom,x,q) +if (conf%q_to_x) call convert_q_to_x_tl(fld_in%geom,q,x) +if (conf%x_to_u) call convert_x_to_u_tl(fld_in%geom,x,u) +if (conf%x_to_v) call convert_x_to_v_tl(fld_in%geom,x,v) + +! Set x, q, u and v +call qg_change_var_set(conf,fld_out,x,q,u,v) + +end subroutine qg_change_var_tl ! ------------------------------------------------------------------------------ !> Change of variable - adjoint -subroutine qg_change_var_ad(conf,fld_in,fld_out) +subroutine qg_change_var_ad(fld_in,fld_out) implicit none ! Passed variables -type(qg_change_var_config),intent(in) :: conf !< Variable change -type(qg_fields),intent(in) :: fld_in !< Input fields -type(qg_fields),intent(inout) :: fld_out !< Output fields +type(qg_fields),intent(in) :: fld_in !< Input fields +type(qg_fields),intent(inout) :: fld_out !< Output fields + +! Local variables +real(kind_real) :: x(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz),q(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz) +real(kind_real) :: u(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz),v(fld_in%geom%nx,fld_in%geom%ny,fld_in%geom%nz) +type(qg_change_var_config) :: conf -! Check fields resolution +! Checks call qg_fields_check_resolution(fld_in,fld_out) -select case (trim(conf%varchange)) -case ('identity') - ! Copy fields - call qg_fields_copy(fld_out,fld_in) -case ('x_to_q') - ! Check fields variables - if (.not.fld_in%lq) call abor1_ftn('qg_change_var_ad: wrong input fields variables for '//trim(conf%varchange)) - if (fld_out%lq) call abor1_ftn('qg_change_var_ad: wrong output fields variables for '//trim(conf%varchange)) - - ! Conversion - call convert_x_to_q_ad(fld_in%geom,fld_in%gfld3d,fld_out%gfld3d) - - ! Copy boundary conditions - call qg_fields_copy(fld_out,fld_in,.true.) -case ('q_to_x') - ! Check fields variables - if (fld_in%lq) call abor1_ftn('qg_change_var_ad: wrong input fields variables for '//trim(conf%varchange)) - if (.not.fld_out%lq) call abor1_ftn('qg_change_var_ad: wrong output fields variables for '//trim(conf%varchange)) - - ! Conversion - call convert_q_to_x_ad(fld_in%geom,fld_in%gfld3d,fld_out%gfld3d) - - ! Copy boundary conditions - call qg_fields_copy(fld_out,fld_in,.true.) -case default - call abor1_ftn('qg_change_var_ad: wrong variable change') -end select +! Copy boundary conditions +call qg_fields_copy_lbc(fld_out,fld_in) -end subroutine qg_change_var_ad -! ------------------------------------------------------------------------------ -!> Change of variable - inverse adjoint -subroutine qg_change_var_inv_ad(conf,fld_in,fld_out) +! Define change of variable configuration +call qg_change_var_setup(conf,fld_in,fld_out,.true.) -implicit none +! Get x, q, u and v +call qg_change_var_get(conf,fld_in,x,q,u,v) -! Passed variables -type(qg_change_var_config),intent(in) :: conf !< Variable change -type(qg_fields),intent(in) :: fld_in !< Input fields -type(qg_fields),intent(inout) :: fld_out !< Output fields +! Conversions +if (conf%x_to_v) call convert_x_to_v_ad(fld_in%geom,v,x) +if (conf%x_to_u) call convert_x_to_u_ad(fld_in%geom,u,x) +if (conf%q_to_x) call convert_q_to_x_ad(fld_in%geom,x,q) +if (conf%x_to_q) call convert_x_to_q_ad(fld_in%geom,q,x) -! Check fields resolution -call qg_fields_check_resolution(fld_in,fld_out) +! Set x, q, u and v +call qg_change_var_set(conf,fld_out,x,q,u,v) -select case (trim(conf%varchange)) -case ('identity') - ! Copy fields - call qg_fields_copy(fld_out,fld_in) -case ('x_to_q') - ! Check fields variables - if (fld_in%lq) call abor1_ftn('qg_change_var_inv_ad: wrong input fields variables for '//trim(conf%varchange)) - if (.not.fld_out%lq) call abor1_ftn('qg_change_var_inv_ad: wrong output fields variables for '//trim(conf%varchange)) - - ! Conversion - call convert_q_to_x_ad(fld_in%geom,fld_in%gfld3d,fld_out%gfld3d) - - ! Copy boundary conditions - call qg_fields_copy(fld_out,fld_in,.true.) -case ('q_to_x') - ! Check fields variables - if (.not.fld_in%lq) call abor1_ftn('qg_change_var_inv_ad: wrong input fields variables for '//trim(conf%varchange)) - if (fld_out%lq) call abor1_ftn('qg_change_var_inv_ad: wrong output fields variables for '//trim(conf%varchange)) - - ! Conversion - call convert_x_to_q_ad(fld_in%geom,fld_in%gfld3d,fld_out%gfld3d) - - ! Copy boundary conditions - call qg_fields_copy(fld_out,fld_in,.true.) -case default - call abor1_ftn('qg_change_var_inv_ad: wrong variable change') -end select - -end subroutine qg_change_var_inv_ad +end subroutine qg_change_var_ad ! ------------------------------------------------------------------------------ end module qg_change_var_mod diff --git a/qg/model/qg_convert_x_to_u_mod.F90 b/qg/model/qg_convert_x_to_u_mod.F90 new file mode 100644 index 000000000..0ecf49d86 --- /dev/null +++ b/qg/model/qg_convert_x_to_u_mod.F90 @@ -0,0 +1,90 @@ +! (C) Copyright 2009-2016 ECMWF. +! +! This software is licensed under the terms of the Apache Licence Version 2.0 +! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +! In applying this licence, ECMWF does not waive the privileges and immunities +! granted to it by virtue of its status as an intergovernmental organisation nor +! does it submit to ageom%ny jurisdiction. + +module qg_convert_x_to_u_mod + +use kinds +use qg_geom_mod + +implicit none + +private +public :: convert_x_to_u,convert_x_to_u_tl,convert_x_to_u_ad +! ------------------------------------------------------------------------------ +contains +! ------------------------------------------------------------------------------ +!> Convert streafunction to zonal wind +subroutine convert_x_to_u(geom,x,x_north,x_south,u) + +implicit none + +! Passed variables +type(qg_geom),intent(in) :: geom !< Geometry +real(kind_real),intent(in) :: x(geom%nx,geom%ny,geom%nz) !< Streamfunction +real(kind_real),intent(in) :: x_north(geom%nz) !< Streamfunction on northern wall +real(kind_real),intent(in) :: x_south(geom%nz) !< Streamfunction on southern wall +real(kind_real),intent(inout) :: u(geom%nx,geom%ny,geom%nz) !< Zonal wind + +! Local variables +integer :: iz + +!$omp parallel do schedule(static) private(iz) +do iz=1,geom%nz + u(:,2:geom%ny,iz) = 0.5*x(:,1:geom%ny-1,iz)/geom%deltay + u(:,1,iz) = 0.5*x_south(iz)/geom%deltay + u(:,1:geom%ny-1,iz) = u(:,1:geom%ny-1,iz)-0.5*x(:,2:geom%ny,iz)/geom%deltay + u(:,geom%ny,iz) = u(:,geom%ny,iz)-0.5*x_north(iz)/geom%deltay +enddo +!$omp end parallel do + +end subroutine convert_x_to_u +! ------------------------------------------------------------------------------ +!> Convert streafunction to zonal wind - tangent Linear +subroutine convert_x_to_u_tl(geom,x,u) + +implicit none + +! Passed variables +type(qg_geom),intent(in) :: geom !< Geometry +real(kind_real),intent(in) :: x(geom%nx,geom%ny,geom%nz) !< Streamfunction +real(kind_real),intent(out) :: u(geom%nx,geom%ny,geom%nz) !< Zonal wind + +! Local variables +integer :: iz + +!$omp parallel do schedule(static) private(iz) +do iz=1,geom%nz + u(:,2:geom%ny,iz) = 0.5*x(:,1:geom%ny-1,iz)/geom%deltay + u(:,1,iz) = 0.0 + u(:,1:geom%ny-1,iz) = u(:,1:geom%ny-1,iz)-0.5*x(:,2:geom%ny,iz)/geom%deltay +enddo +!$omp end parallel do + +end subroutine convert_x_to_u_tl +! ------------------------------------------------------------------------------ +!> Convert streafunction to zonal wind - adjoint +subroutine convert_x_to_u_ad(geom,u,x) + +implicit none + +! Passed variables +type(qg_geom),intent(in) :: geom !< Geometry +real(kind_real),intent(in) :: u(geom%nx,geom%ny,geom%nz) !< Zonal wind +real(kind_real),intent(inout) :: x(geom%nx,geom%ny,geom%nz) !< Streamfunction + +! Local variables +integer :: iz + +do iz=1,geom%nz + x(:,2:geom%ny,iz) = x(:,2:geom%ny,iz)-0.5/geom%deltay*u(:,1:geom%ny-1,iz) + x(:,1:geom%ny-1,iz) = x(:,1:geom%ny-1,iz)+0.5/geom%deltay*u(:,2:geom%ny,iz) +enddo + +end subroutine convert_x_to_u_ad +! ------------------------------------------------------------------------------ +end module qg_convert_x_to_u_mod diff --git a/qg/model/qg_convert_x_to_uv_mod.F90 b/qg/model/qg_convert_x_to_v_mod.F90 similarity index 57% rename from qg/model/qg_convert_x_to_uv_mod.F90 rename to qg/model/qg_convert_x_to_v_mod.F90 index 6e5c7bd5d..e4be2ebd2 100644 --- a/qg/model/qg_convert_x_to_uv_mod.F90 +++ b/qg/model/qg_convert_x_to_v_mod.F90 @@ -6,7 +6,7 @@ ! granted to it by virtue of its status as an intergovernmental organisation nor ! does it submit to ageom%ny jurisdiction. -module qg_convert_x_to_uv_mod +module qg_convert_x_to_v_mod use kinds use qg_geom_mod @@ -14,37 +14,23 @@ module qg_convert_x_to_uv_mod implicit none private -public :: convert_x_to_uv,convert_x_to_uv_tl,convert_x_to_uv_ad +public :: convert_x_to_v,convert_x_to_v_tl,convert_x_to_v_ad ! ------------------------------------------------------------------------------ contains ! ------------------------------------------------------------------------------ -!> Convert streafunction to wind components -subroutine convert_x_to_uv(geom,x,x_north,x_south,u,v) +!> Convert streafunction to meridional wind +subroutine convert_x_to_v(geom,x,v) implicit none ! Passed variables -type(qg_geom),intent(in) :: geom !< Geometry -real(kind_real),intent(in) :: x(geom%nx,geom%ny,geom%nz) !< Streamfunction -real(kind_real),intent(in) :: x_north(geom%nz) !< Streamfunction on northern wall -real(kind_real),intent(in) :: x_south(geom%nz) !< Streamfunction on southern wall -real(kind_real),intent(inout) :: u(geom%nx,geom%ny,geom%nz) !< Zonal wind +type(qg_geom),intent(in) :: geom !< Geometry +real(kind_real),intent(in) :: x(geom%nx,geom%ny,geom%nz) !< Streamfunction real(kind_real),intent(inout) :: v(geom%nx,geom%ny,geom%nz) !< Meridional wind ! Local variables integer :: iz -! Zonal wind -!$omp parallel do schedule(static) private(iz) -do iz=1,geom%nz - u(:,2:geom%ny,iz) = 0.5*x(:,1:geom%ny-1,iz)/geom%deltay - u(:,1,iz) = 0.5*x_south(iz)/geom%deltay - u(:,1:geom%ny-1,iz) = u(:,1:geom%ny-1,iz)-0.5*x(:,2:geom%ny,iz)/geom%deltay - u(:,geom%ny,iz) = u(:,geom%ny,iz)-0.5*x_north(iz)/geom%deltay -enddo -!$omp end parallel do - -! Meridional wind !$omp parallel do schedule(static) private(iz) do iz=1,geom%nz v(1:geom%nx-1,:,iz) = 0.5*x(2:geom%nx,:,iz)/geom%deltax @@ -54,32 +40,21 @@ subroutine convert_x_to_uv(geom,x,x_north,x_south,u,v) enddo !$omp end parallel do -end subroutine convert_x_to_uv +end subroutine convert_x_to_v ! ------------------------------------------------------------------------------ -!> Convert streafunction to wind components - tangent Linear -subroutine convert_x_to_uv_tl(geom,x,u,v) +!> Convert streafunction to meridional wind - tangent Linear +subroutine convert_x_to_v_tl(geom,x,v) implicit none ! Passed variables type(qg_geom),intent(in) :: geom !< Geometry real(kind_real),intent(in) :: x(geom%nx,geom%ny,geom%nz) !< Streamfunction -real(kind_real),intent(out) :: u(geom%nx,geom%ny,geom%nz) !< Zonal wind real(kind_real),intent(out) :: v(geom%nx,geom%ny,geom%nz) !< Meridional wind ! Local variables integer :: iz -! Zonal wind -!$omp parallel do schedule(static) private(iz) -do iz=1,geom%nz - u(:,2:geom%ny,iz) = 0.5*x(:,1:geom%ny-1,iz)/geom%deltay - u(:,1,iz) = 0.0 - u(:,1:geom%ny-1,iz) = u(:,1:geom%ny-1,iz)-0.5*x(:,2:geom%ny,iz)/geom%deltay -enddo -!$omp end parallel do - -! Meridional wind !$omp parallel do schedule(static) private(iz) do iz=1,geom%nz v(1:geom%nx-1,:,iz) = 0.5*x(2:geom%nx,:,iz)/geom%deltax @@ -89,29 +64,21 @@ subroutine convert_x_to_uv_tl(geom,x,u,v) enddo !$omp end parallel do -end subroutine convert_x_to_uv_tl +end subroutine convert_x_to_v_tl ! ------------------------------------------------------------------------------ -!> Convert streafunction to wind components - adjoint -subroutine convert_x_to_uv_ad(geom,u,v,x) +!> Convert streafunction to meridional wind - adjoint +subroutine convert_x_to_v_ad(geom,v,x) implicit none ! Passed variables type(qg_geom),intent(in) :: geom !< Geometry -real(kind_real),intent(in) :: u(geom%nx,geom%ny,geom%nz) !< Zonal wind real(kind_real),intent(in) :: v(geom%nx,geom%ny,geom%nz) !< Meridional wind real(kind_real),intent(inout) :: x(geom%nx,geom%ny,geom%nz) !< Streamfunction ! Local variables integer :: iz -! Zonal wind -do iz=1,geom%nz - x(:,2:geom%ny,iz) = x(:,2:geom%ny,iz)-0.5/geom%deltay*u(:,1:geom%ny-1,iz) - x(:,1:geom%ny-1,iz) = x(:,1:geom%ny-1,iz)+0.5/geom%deltay*u(:,2:geom%ny,iz) -enddo - -! Meridional wind do iz=1,geom%nz x(geom%nx,:,iz) = x(geom%nx,:,iz)-0.5/geom%deltax*v(1,:,iz) x(1:geom%nx-1,:,iz) = x(1:geom%nx-1,:,iz)-0.5/geom%deltax*v(2:geom%nx,:,iz) @@ -119,6 +86,6 @@ subroutine convert_x_to_uv_ad(geom,u,v,x) x(2:geom%nx,:,iz) = x(2:geom%nx,:,iz)+0.5/geom%deltax*v(1:geom%nx-1,:,iz) enddo -end subroutine convert_x_to_uv_ad +end subroutine convert_x_to_v_ad ! ------------------------------------------------------------------------------ -end module qg_convert_x_to_uv_mod +end module qg_convert_x_to_v_mod diff --git a/qg/model/qg_error_covariance_interface.F90 b/qg/model/qg_error_covariance_interface.F90 index b8ec59c5d..87dae13f2 100644 --- a/qg/model/qg_error_covariance_interface.F90 +++ b/qg/model/qg_error_covariance_interface.F90 @@ -15,6 +15,7 @@ module qg_error_covariance_interface use qg_error_covariance_mod use qg_fields_mod use qg_geom_mod +use oops_variables_mod implicit none @@ -72,48 +73,48 @@ subroutine qg_error_covariance_delete_c(c_key_self) bind (c,name='qg_error_covar end subroutine qg_error_covariance_delete_c ! ------------------------------------------------------------------------------ !> Multiply by error covariance matrix -subroutine qg_error_covariance_mult_c(c_key_conf,c_key_in,c_key_out) bind(c,name='qg_error_covariance_mult_f90') +subroutine qg_error_covariance_mult_c(c_key_self,c_key_in,c_key_out) bind(c,name='qg_error_covariance_mult_f90') implicit none ! Passed variables -integer(c_int),intent(in) :: c_key_conf !< Error covariance configuration +integer(c_int),intent(in) :: c_key_self !< Error covariance configuration integer(c_int),intent(in) :: c_key_in !< Input field integer(c_int),intent(in) :: c_key_out !< Output field ! Local variables -type(qg_error_covariance_config),pointer :: conf +type(qg_error_covariance_config),pointer :: self type(qg_fields),pointer :: fld_in,fld_out ! Interface -call qg_error_covariance_registry%get(c_key_conf,conf) +call qg_error_covariance_registry%get(c_key_self,self) call qg_fields_registry%get(c_key_in,fld_in) call qg_fields_registry%get(c_key_out,fld_out) ! Call Fortran -call qg_error_covariance_mult(conf,fld_in,fld_out) +call qg_error_covariance_mult(self,fld_in,fld_out) end subroutine qg_error_covariance_mult_c ! ------------------------------------------------------------------------------ !> Randomize error covariance -subroutine qg_error_covariance_randomize_c(c_key_conf,c_key_out) bind(c,name='qg_error_covariance_randomize_f90') +subroutine qg_error_covariance_randomize_c(c_key_self,c_key_out) bind(c,name='qg_error_covariance_randomize_f90') implicit none ! Passed variables -integer(c_int),intent(in) :: c_key_conf !< Error covariance configuration +integer(c_int),intent(in) :: c_key_self !< Error covariance configuration integer(c_int),intent(in) :: c_key_out !< Output field ! Local variables -type(qg_error_covariance_config),pointer :: conf +type(qg_error_covariance_config),pointer :: self type(qg_fields),pointer :: fld_out ! Interface -call qg_error_covariance_registry%get(c_key_conf,conf) +call qg_error_covariance_registry%get(c_key_self,self) call qg_fields_registry%get(c_key_out,fld_out) ! Call Fortran -call qg_error_covariance_randomize(conf,fld_out) +call qg_error_covariance_randomize(self,fld_out) end subroutine qg_error_covariance_randomize_c ! ------------------------------------------------------------------------------ diff --git a/qg/model/qg_error_covariance_mod.F90 b/qg/model/qg_error_covariance_mod.F90 index dcc04ffaa..2ee029e71 100644 --- a/qg/model/qg_error_covariance_mod.F90 +++ b/qg/model/qg_error_covariance_mod.F90 @@ -14,10 +14,10 @@ module qg_error_covariance_mod use iso_c_binding use kinds !$ use omp_lib +use oops_variables_mod use qg_constants_mod use qg_fields_mod use qg_geom_mod -use oops_variables_mod use random_mod implicit none @@ -29,6 +29,9 @@ module qg_error_covariance_mod & qg_error_covariance_randomize ! ------------------------------------------------------------------------------ type :: qg_error_covariance_config + integer :: nx !< Number of points in the zonal direction + integer :: ny !< Number of points in the meridional direction + integer :: nz !< Number of vertical levels real(kind_real) :: sigma !< Standard deviation real(kind_real),allocatable :: sqrt_zonal(:) !< Spectral weights for the spectral of the zonal correlation matrix real(kind_real),allocatable :: sqrt_merid(:,:) !< Square-root of the meridional correlation matrix @@ -73,6 +76,7 @@ subroutine qg_error_covariance_setup(self,f_conf,geom) real(kind_real),allocatable :: norm(:,:) character(len=160) :: record type(qg_fields) :: fld_in,fld_out +type(oops_variables) :: vars ! Get parameters call f_conf%get_or_die("standard_deviation",self%sigma) @@ -87,6 +91,11 @@ subroutine qg_error_covariance_setup(self,f_conf,geom) call abor1_ftn('qg_error_covariance_setup: odd number of zonal grid points') endif +! Copy grid size +self%nx = geom%nx +self%ny = geom%ny +self%nz = geom%nz + ! Allocation allocate(self%sqrt_merid(geom%ny,geom%ny)) allocate(self%sqrt_vert(geom%nz,geom%nz)) @@ -189,15 +198,17 @@ subroutine qg_error_covariance_setup(self,f_conf,geom) enddo ! Compute normalization factor -call qg_fields_create_default(fld_in,geom,.false.) -call qg_fields_create_default(fld_out,geom,.false.) +vars = oops_variables() +call vars%push_back('x') +call qg_fields_create(fld_in,geom,vars,.false.) +call qg_fields_create(fld_out,geom,vars,.false.) self%norm = 1.0 do iz=1,geom%nz do iy=1,geom%ny call qg_fields_zero(fld_in) - fld_in%gfld3d(1,iy,iz) = 1.0 + fld_in%x(1,iy,iz) = 1.0 call qg_error_covariance_mult(self,fld_in,fld_out) - norm(iy,iz) = 1.0/sqrt(fld_out%gfld3d(1,iy,iz)) + norm(iy,iz) = 1.0/sqrt(fld_out%x(1,iy,iz)) end do end do self%norm = norm*self%sigma @@ -213,6 +224,7 @@ subroutine qg_error_covariance_setup(self,f_conf,geom) deallocate(revalsz) deallocate(evectsz) deallocate(evalsz) +call vars%destruct() call qg_fields_delete(fld_in) call qg_fields_delete(fld_out) @@ -235,12 +247,12 @@ subroutine qg_error_covariance_delete(self) end subroutine qg_error_covariance_delete ! ------------------------------------------------------------------------------ !> Multiply by error covariance matrix -subroutine qg_error_covariance_mult(conf,fld_in,fld_out) +subroutine qg_error_covariance_mult(self,fld_in,fld_out) implicit none ! Passed variables -type(qg_error_covariance_config),intent(in) :: conf !< Error covariance configuration +type(qg_error_covariance_config),intent(in) :: self !< Error covariance configuration type(qg_fields),intent(in) :: fld_in !< Input field type(qg_fields),intent(inout) :: fld_out !< Output field @@ -252,19 +264,19 @@ subroutine qg_error_covariance_mult(conf,fld_in,fld_out) call qg_fields_copy(fld_tmp,fld_in) ! Apply covariance matrix -call qg_error_covariance_sqrt_mult_ad(conf,fld_tmp,fld_out) +call qg_error_covariance_sqrt_mult_ad(self,fld_tmp,fld_out) call qg_fields_copy(fld_tmp,fld_out) -call qg_error_covariance_sqrt_mult(conf,fld_tmp,fld_out) +call qg_error_covariance_sqrt_mult(self,fld_tmp,fld_out) end subroutine qg_error_covariance_mult ! ------------------------------------------------------------------------------ !> Randomize error covariance -subroutine qg_error_covariance_randomize(conf,fld_out) +subroutine qg_error_covariance_randomize(self,fld_out) implicit none ! Passed variables -type(qg_error_covariance_config),intent(in) :: conf !< Error covariance configuration +type(qg_error_covariance_config),intent(in) :: self !< Error covariance configuration type(qg_fields),intent(inout) :: fld_out !< Output field ! Local variables @@ -272,74 +284,72 @@ subroutine qg_error_covariance_randomize(conf,fld_out) ! Initialize temporary field call qg_fields_create_from_other(fld_tmp,fld_out,fld_out%geom) -call qg_fields_random(fld_tmp) +call qg_fields_random(fld_tmp,'x') ! Apply square-root of the covariance matrix -call qg_error_covariance_sqrt_mult(conf,fld_tmp,fld_out) +call qg_error_covariance_sqrt_mult(self,fld_tmp,fld_out) end subroutine qg_error_covariance_randomize ! ------------------------------------------------------------------------------ ! Private ! ------------------------------------------------------------------------------ !> Multiply by error covariance matrix square-root, zonal part -subroutine qg_error_covariance_sqrt_mult_zonal(conf,fld_in,fld_out) +subroutine qg_error_covariance_sqrt_mult_zonal(self,fld) implicit none ! Passed variables -type(qg_error_covariance_config),intent(in) :: conf !< Error covariance configuration -type(qg_fields),intent(in) :: fld_in !< Input field -type(qg_fields),intent(inout) :: fld_out !< Output field +type(qg_error_covariance_config),intent(in) :: self !< Error covariance configuration +real(kind_real),intent(inout) :: fld(self%nx,self%ny,self%nz) !< Field ! Local variables integer :: iy,iz,m,iri -real(kind_real) :: zfour(fld_in%geom%nx+2) +real(kind_real) :: zfour(self%nx+2) -do iz=1,fld_in%geom%nz - do iy=1,fld_in%geom%ny - call fft_fwd(fld_in%geom%nx,fld_in%gfld3d(:,iy,iz),zfour) +do iz=1,self%nz + do iy=1,self%ny + call fft_fwd(self%nx,fld(:,iy,iz),zfour) !$omp parallel do schedule(static) private(m,iri) - do m=0,fld_in%geom%nx/2 + do m=0,self%nx/2 do iri=1,2 - zfour(2*m+iri) = zfour(2*m+iri)*conf%sqrt_zonal(m) + zfour(2*m+iri) = zfour(2*m+iri)*self%sqrt_zonal(m) enddo enddo !$omp end parallel do - call fft_inv(fld_in%geom%nx,zfour,fld_out%gfld3d(:,iy,iz)) + call fft_inv(self%nx,zfour,fld(:,iy,iz)) enddo enddo end subroutine qg_error_covariance_sqrt_mult_zonal ! ------------------------------------------------------------------------------ !> Multiply by error covariance matrix square-root - meridional part -subroutine qg_error_covariance_sqrt_mult_meridional(conf,fld_in,fld_out) +subroutine qg_error_covariance_sqrt_mult_meridional(self,fld) implicit none ! Passed variables -type(qg_error_covariance_config),intent(in) :: conf !< Error covariance configuration -type(qg_fields),intent(in) :: fld_in !< Input field -type(qg_fields),intent(inout) :: fld_out !< Output field +type(qg_error_covariance_config),intent(in) :: self !< Error covariance configuration +real(kind_real),intent(inout) :: fld(self%nx,self%ny,self%nz) !< Field ! Local variables integer :: ix,iz real(kind_real),allocatable :: arr_in(:),arr_out(:) !$omp parallel do schedule(static) private(iz,ix) firstprivate(arr_in,arr_out) -do iz=1,fld_in%geom%nz - do ix=1,fld_in%geom%nx +do iz=1,self%nz + do ix=1,self%nx ! Allocation - allocate(arr_in(fld_in%geom%ny)) - allocate(arr_out(fld_in%geom%ny)) + allocate(arr_in(self%ny)) + allocate(arr_out(self%ny)) ! Initialize - arr_in = fld_in%gfld3d(ix,:,iz) + arr_in = fld(ix,:,iz) ! Apply transform - call dsymv('L',fld_in%geom%ny,1.0_kind_real,conf%sqrt_merid,fld_in%geom%ny,arr_in,1,0.0_kind_real,arr_out,1) + call dsymv('L',self%ny,1.0_kind_real,self%sqrt_merid,self%ny,arr_in,1,0.0_kind_real,arr_out,1) ! Copy - fld_out%gfld3d(ix,:,iz) = arr_out + fld(ix,:,iz) = arr_out ! Release memory deallocate(arr_in) @@ -351,34 +361,33 @@ subroutine qg_error_covariance_sqrt_mult_meridional(conf,fld_in,fld_out) end subroutine qg_error_covariance_sqrt_mult_meridional ! ------------------------------------------------------------------------------ !> Multiply by error covariance matrix square-root - vertical part -subroutine qg_error_covariance_sqrt_mult_vertical(conf,fld_in,fld_out) +subroutine qg_error_covariance_sqrt_mult_vertical(self,fld) implicit none ! Passed variables -type(qg_error_covariance_config),intent(in) :: conf !< Error covariance configuration -type(qg_fields),intent(in) :: fld_in !< Input field -type(qg_fields),intent(inout) :: fld_out !< Output field +type(qg_error_covariance_config),intent(in) :: self !< Error covariance configuration +real(kind_real),intent(inout) :: fld(self%nx,self%ny,self%nz) !< Field ! Local variables integer :: ix,iy real(kind_real),allocatable :: arr_in(:),arr_out(:) !$omp parallel do schedule(static) private(iy,ix) firstprivate(arr_in,arr_out) -do iy=1,fld_in%geom%ny - do ix=1,fld_in%geom%nx +do iy=1,self%ny + do ix=1,self%nx ! Allocation - allocate(arr_in(fld_in%geom%nz)) - allocate(arr_out(fld_in%geom%nz)) + allocate(arr_in(self%nz)) + allocate(arr_out(self%nz)) ! Initialize - arr_in = fld_in%gfld3d(ix,iy,:) + arr_in = fld(ix,iy,:) ! Apply transform - call dsymv('L',fld_in%geom%nz,1.0_kind_real,conf%sqrt_vert,fld_in%geom%nz,arr_in,1,0.0_kind_real,arr_out,1) + call dsymv('L',self%nz,1.0_kind_real,self%sqrt_vert,self%nz,arr_in,1,0.0_kind_real,arr_out,1) ! Copy - fld_out%gfld3d(ix,iy,:) = arr_out + fld(ix,iy,:) = arr_out ! Release memory deallocate(arr_in) @@ -390,88 +399,84 @@ subroutine qg_error_covariance_sqrt_mult_vertical(conf,fld_in,fld_out) end subroutine qg_error_covariance_sqrt_mult_vertical ! ------------------------------------------------------------------------------ !> Multiply by error covariance matrix square-root -subroutine qg_error_covariance_sqrt_mult(conf,fld_in,fld_out) +subroutine qg_error_covariance_sqrt_mult(self,fld_in,fld_out) implicit none ! Passed variables -type(qg_error_covariance_config),intent(in) :: conf !< Error covariance configuration +type(qg_error_covariance_config),intent(in) :: self !< Error covariance configuration type(qg_fields),intent(in) :: fld_in !< Input field type(qg_fields),intent(inout) :: fld_out !< Output field ! Local variables integer :: ix -type(qg_fields) :: fld_tmp -! Initialization -call qg_fields_create_from_other(fld_tmp,fld_in,fld_in%geom) -call qg_fields_copy(fld_tmp,fld_in) +! Check input/output +if (.not.allocated(fld_in%x)) call abor1_ftn("qg_error_covariance_sqrt_mult: x required as input") +if (.not.allocated(fld_out%x)) call abor1_ftn("qg_error_covariance_sqrt_mult: x required as output") + +! Copy field +call qg_fields_copy(fld_out,fld_in) ! Multiply by symmetric square-root of vertical correlation matrix -call qg_error_covariance_sqrt_mult_vertical(conf,fld_tmp,fld_out) -call qg_fields_copy(fld_tmp,fld_out) +call qg_error_covariance_sqrt_mult_vertical(self,fld_out%x) ! Multiply by square-root of meridional correlation matrix -call qg_error_covariance_sqrt_mult_meridional(conf,fld_tmp,fld_out) -call qg_fields_copy(fld_tmp,fld_out) +call qg_error_covariance_sqrt_mult_meridional(self,fld_out%x) ! Multiply by square-root of zonal correlation matrix -call qg_error_covariance_sqrt_mult_zonal(conf,fld_tmp,fld_out) -call qg_fields_copy(fld_tmp,fld_out) +call qg_error_covariance_sqrt_mult_zonal(self,fld_out%x) ! Multiply by normalization factor !$omp parallel do schedule(static) private(ix) -do ix=1,fld_in%geom%nx - fld_out%gfld3d(ix,:,:) = fld_tmp%gfld3d(ix,:,:)*conf%norm +do ix=1,fld_out%geom%nx + fld_out%x(ix,:,:) = fld_out%x(ix,:,:)*self%norm end do !$omp end parallel do -call qg_fields_copy(fld_tmp,fld_out) ! Multiply by standard deviation -fld_out%gfld3d = fld_tmp%gfld3d*conf%sigma +fld_out%x = fld_out%x*self%sigma end subroutine qg_error_covariance_sqrt_mult ! ------------------------------------------------------------------------------ !> Multiply by error covariance matrix square-root - adjoint -subroutine qg_error_covariance_sqrt_mult_ad(conf,fld_in,fld_out) +subroutine qg_error_covariance_sqrt_mult_ad(self,fld_in,fld_out) implicit none ! Passed variables -type(qg_error_covariance_config),intent(in) :: conf !< Error covariance configuration +type(qg_error_covariance_config),intent(in) :: self !< Error covariance configuration type(qg_fields),intent(in) :: fld_in !< Input field type(qg_fields),intent(inout) :: fld_out !< Output field ! Local variables integer :: ix -type(qg_fields) :: fld_tmp -! Initialization -call qg_fields_create_from_other(fld_tmp,fld_in,fld_in%geom) -call qg_fields_copy(fld_tmp,fld_in) +! Check input/output +if (.not.allocated(fld_in%x)) call abor1_ftn("qg_error_covariance_sqrt_mult: x required as input") +if (.not.allocated(fld_out%x)) call abor1_ftn("qg_error_covariance_sqrt_mult: x required as output") + +! Copy field +call qg_fields_copy(fld_out,fld_in) ! Multiply by standard deviation -fld_out%gfld3d = fld_tmp%gfld3d*conf%sigma -call qg_fields_copy(fld_tmp,fld_out) +fld_out%x = fld_out%x*self%sigma ! Multiply by normalization factor !$omp parallel do schedule(static) private(ix) -do ix=1,fld_in%geom%nx - fld_out%gfld3d(ix,:,:) = fld_tmp%gfld3d(ix,:,:)*conf%norm +do ix=1,fld_out%geom%nx + fld_out%x(ix,:,:) = fld_out%x(ix,:,:)*self%norm end do !$omp end parallel do -call qg_fields_copy(fld_tmp,fld_out) ! Multiply by square-root of zonal correlation matrix -call qg_error_covariance_sqrt_mult_zonal(conf,fld_tmp,fld_out) -call qg_fields_copy(fld_tmp,fld_out) +call qg_error_covariance_sqrt_mult_zonal(self,fld_out%x) ! Multiply by square-root of meridional correlation matrix -call qg_error_covariance_sqrt_mult_meridional(conf,fld_tmp,fld_out) -call qg_fields_copy(fld_tmp,fld_out) +call qg_error_covariance_sqrt_mult_meridional(self,fld_out%x) ! Multiply by symmetric square-root of vertical correlation matrix -call qg_error_covariance_sqrt_mult_vertical(conf,fld_tmp,fld_out) +call qg_error_covariance_sqrt_mult_vertical(self,fld_out%x) end subroutine qg_error_covariance_sqrt_mult_ad ! ------------------------------------------------------------------------------ diff --git a/qg/model/qg_fields_interface.F90 b/qg/model/qg_fields_interface.F90 index 594f519b9..5b975ec31 100644 --- a/qg/model/qg_fields_interface.F90 +++ b/qg/model/qg_fields_interface.F90 @@ -13,12 +13,11 @@ module qg_fields_interface use fckit_configuration_module, only: fckit_configuration use iso_c_binding use kinds +use oops_variables_mod use qg_fields_mod use qg_geom_mod use qg_geom_iter_mod -use qg_gom_mod use qg_locs_mod -use oops_variables_mod implicit none @@ -166,21 +165,30 @@ subroutine qg_fields_dirac_c(c_key_self,c_conf) bind(c,name='qg_fields_dirac_f90 end subroutine qg_fields_dirac_c ! ------------------------------------------------------------------------------ !> Generate random fields -subroutine qg_fields_random_c(c_key_self) bind(c,name='qg_fields_random_f90') +subroutine qg_fields_random_c(c_key_self,c_vars) bind(c,name='qg_fields_random_f90') implicit none ! Passed variables integer(c_int),intent(in) :: c_key_self !< Fields +type(c_ptr),value,intent(in) :: c_vars !< List of variables ! Local variables type(qg_fields),pointer :: self +type(oops_variables) :: vars ! Interface call qg_fields_registry%get(c_key_self,self) +vars = oops_variables(c_vars) ! Call Fortran -call qg_fields_random(self) +if (vars%has('x')) then + call qg_fields_random(self,'x') +elseif (vars%has('q')) then + call qg_fields_random(self,'q') +else + call abor1_ftn('qg_fields_random_c: x or q required in output field') +endif end subroutine qg_fields_random_c ! ------------------------------------------------------------------------------ @@ -481,14 +489,16 @@ subroutine qg_fields_analytic_init_c(c_key_fld,c_conf,c_dt) bind(c,name='qg_fiel end subroutine qg_fields_analytic_init_c ! ------------------------------------------------------------------------------ !> Fields statistics -subroutine qg_fields_gpnorm_c(c_key_fld,nb,pstat) bind(c,name='qg_fields_gpnorm_f90') +subroutine qg_fields_gpnorm_c(c_key_fld,vpresent,vmin,vmax,vrms) bind(c,name='qg_fields_gpnorm_f90') implicit none ! Passed variables -integer(c_int),intent(in) :: c_key_fld !< Fields -integer(c_int),intent(in) :: nb !< Number of boundaries -real(c_double),intent(inout) :: pstat(3*(1+nb)) !< Statistics +integer(c_int),intent(in) :: c_key_fld !< Fields +integer(c_int),intent(inout) :: vpresent(6) !< Variables presence flag +real(c_double),intent(inout) :: vmin(6) !< Variables minimum +real(c_double),intent(inout) :: vmax(6) !< Variables maximum +real(c_double),intent(inout) :: vrms(6) !< Variables RMS ! Local variables type(qg_fields),pointer :: fld @@ -497,7 +507,7 @@ subroutine qg_fields_gpnorm_c(c_key_fld,nb,pstat) bind(c,name='qg_fields_gpnorm_ call qg_fields_registry%get(c_key_fld,fld) ! Call Fortran -call qg_fields_gpnorm(fld,nb,pstat) +call qg_fields_gpnorm(fld,vpresent,vmin,vmax,vrms) end subroutine qg_fields_gpnorm_c ! ------------------------------------------------------------------------------ @@ -522,7 +532,7 @@ subroutine qg_fields_rms_c(c_key_fld,prms) bind(c,name='qg_fields_rms_f90') end subroutine qg_fields_rms_c ! ------------------------------------------------------------------------------ !> Get fields geometry -subroutine qg_fields_sizes_c(c_key_fld,c_nx,c_ny,c_nz,c_nb) bind(c,name='qg_fields_sizes_f90') +subroutine qg_fields_sizes_c(c_key_fld,c_nx,c_ny,c_nz) bind(c,name='qg_fields_sizes_f90') implicit none @@ -531,7 +541,6 @@ subroutine qg_fields_sizes_c(c_key_fld,c_nx,c_ny,c_nz,c_nb) bind(c,name='qg_fiel integer(c_int),intent(inout) :: c_nx !< X size integer(c_int),intent(inout) :: c_ny !< Y size integer(c_int),intent(inout) :: c_nz !< Z size -integer(c_int),intent(inout) :: c_nb !< Number of boundaries ! Local variables type(qg_fields),pointer :: fld @@ -540,19 +549,18 @@ subroutine qg_fields_sizes_c(c_key_fld,c_nx,c_ny,c_nz,c_nb) bind(c,name='qg_fiel call qg_fields_registry%get(c_key_fld,fld) ! Call Fortran -call qg_fields_sizes(fld,c_nx,c_ny,c_nz,c_nb) +call qg_fields_sizes(fld,c_nx,c_ny,c_nz) end subroutine qg_fields_sizes_c ! ------------------------------------------------------------------------------ -!> Get fields variables -subroutine qg_fields_vars_c(c_key_fld,c_lq,c_lbc) bind(c,name='qg_fields_vars_f90') +!> Get fields geometry +subroutine qg_fields_lbc_c(c_key_fld,c_lbc) bind(c,name='qg_fields_lbc_f90') implicit none ! Passed variables integer(c_int),intent(in) :: c_key_fld !< Fields -integer(c_int),intent(inout) :: c_lq !< Potential vorticity flag -integer(c_int),intent(inout) :: c_lbc !< Boundaries flag +integer(c_int),intent(inout) :: c_lbc !< LBC presence ! Local variables type(qg_fields),pointer :: fld @@ -561,9 +569,9 @@ subroutine qg_fields_vars_c(c_key_fld,c_lq,c_lbc) bind(c,name='qg_fields_vars_f9 call qg_fields_registry%get(c_key_fld,fld) ! Call Fortran -call qg_fields_vars(fld,c_lq,c_lbc) +call qg_fields_lbc(fld,c_lbc) -end subroutine qg_fields_vars_c +end subroutine qg_fields_lbc_c ! ------------------------------------------------------------------------------ !> Create ATLAS fields subroutine qg_fields_set_atlas_c(c_key_fld,c_vars,c_afieldset) bind (c,name='qg_fields_set_atlas_f90') diff --git a/qg/model/qg_fields_mod.F90 b/qg/model/qg_fields_mod.F90 index 03d78e6df..5fe06980a 100644 --- a/qg/model/qg_fields_mod.F90 +++ b/qg/model/qg_fields_mod.F90 @@ -22,14 +22,13 @@ module qg_fields_mod use qg_constants_mod use qg_convert_q_to_x_mod use qg_convert_x_to_q_mod -use qg_convert_x_to_uv_mod +use qg_convert_x_to_u_mod +use qg_convert_x_to_v_mod use qg_geom_mod use qg_geom_iter_mod -use qg_gom_mod use qg_interp_mod use qg_locs_mod use qg_tools_mod -use oops_variables_mod use random_mod implicit none @@ -37,30 +36,28 @@ module qg_fields_mod private public :: qg_fields public :: qg_fields_registry -public :: qg_fields_create,qg_fields_create_default,qg_fields_create_from_other,qg_fields_delete, & +public :: qg_fields_create,qg_fields_create_from_other,qg_fields_delete, & & qg_fields_zero,qg_fields_ones,qg_fields_dirac,qg_fields_random, & - & qg_fields_copy,qg_fields_self_add,qg_fields_self_sub,qg_fields_self_mul,qg_fields_axpy,qg_fields_self_schur, & - & qg_fields_dot_prod,qg_fields_add_incr,qg_fields_diff_incr,qg_fields_change_resol,qg_fields_read_file, & - & qg_fields_write_file,qg_fields_analytic_init,qg_fields_gpnorm,qg_fields_rms,qg_fields_sizes,qg_fields_vars, & - & qg_fields_set_atlas,qg_fields_to_atlas, qg_fields_from_atlas, & - & qg_fields_getpoint,qg_fields_setpoint,qg_fields_serialize,qg_fields_deserialize, qg_fields_check, & - & qg_fields_check_resolution,qg_fields_check_variables + & qg_fields_copy,qg_fields_copy_lbc,qg_fields_self_add,qg_fields_self_sub,qg_fields_self_mul,qg_fields_axpy, & + & qg_fields_self_schur,qg_fields_dot_prod,qg_fields_add_incr,qg_fields_diff_incr,qg_fields_change_resol, & + & qg_fields_read_file,qg_fields_write_file,qg_fields_analytic_init,qg_fields_gpnorm,qg_fields_rms,qg_fields_sizes, & + & qg_fields_lbc,qg_fields_set_atlas,qg_fields_to_atlas,qg_fields_from_atlas, & + & qg_fields_getpoint,qg_fields_setpoint,qg_fields_serialize,qg_fields_deserialize, & + & qg_fields_complete,qg_fields_check,qg_fields_check_resolution ! ------------------------------------------------------------------------------ integer,parameter :: rseed = 7 !< Random seed (for reproducibility) type :: qg_fields type(qg_geom),pointer :: geom !< Geometry - logical :: lq !< PV as main variable (streamfunction if false) logical :: lbc !< Boundaries are present - real(kind_real),allocatable :: gfld3d(:,:,:) !< TEMPORARY: TO BE REMOVED - real(kind_real),allocatable :: streamfct(:,:,:) !< Streamfunction - real(kind_real),allocatable :: pv(:,:,:) !< PV + real(kind_real),allocatable :: x(:,:,:) !< Streamfunction + real(kind_real),allocatable :: q(:,:,:) !< Potential vorticity real(kind_real),allocatable :: u(:,:,:) !< U wind real(kind_real),allocatable :: v(:,:,:) !< V wind real(kind_real),allocatable :: x_north(:) !< Streamfunction on northern wall real(kind_real),allocatable :: x_south(:) !< Streamfunction on southern wall - real(kind_real),allocatable :: q_north(:,:) !< PV on northern wall - real(kind_real),allocatable :: q_south(:,:) !< PV on southern wall + real(kind_real),allocatable :: q_north(:,:) !< q on northern wall + real(kind_real),allocatable :: q_south(:,:) !< q on southern wall end type qg_fields #define LISTED_TYPE qg_fields @@ -86,7 +83,7 @@ subroutine qg_fields_create(self,geom,vars,lbc) ! Passed variables type(qg_fields),intent(inout) :: self !< Fields type(qg_geom),target,intent(in) :: geom !< Geometry -type(oops_variables),intent(in) :: vars !< Variables +type(oops_variables),intent(in) :: vars !< List of variables logical,intent(in) :: lbc !< Boundaries flag ! Local variables @@ -95,34 +92,14 @@ subroutine qg_fields_create(self,geom,vars,lbc) ! Associate geometry self%geom => geom -!!! TEMPORARY: TO BE REMOVED !!! -if (vars%has('x')) then - self%lq = .false. -elseif (vars%has('q')) then - self%lq = .true. -else - call abor1_ftn('qg_fields_create: fields must have x or q') -endif - ! Set boundaries self%lbc = lbc -!!! TEMPORARY: TO BE REMOVED !!! -allocate(self%gfld3d(self%geom%nx,self%geom%ny,self%geom%nz)) - ! Allocate 3d fields -if (vars%has('x')) then - allocate(self%streamfct(self%geom%nx,self%geom%ny,self%geom%nz)) -endif -if (vars%has('q')) then - allocate(self%pv(self%geom%nx,self%geom%ny,self%geom%nz)) -endif -if (vars%has('u')) then - allocate(self%u(self%geom%nx,self%geom%ny,self%geom%nz)) -endif -if (vars%has('v')) then - allocate(self%v(self%geom%nx,self%geom%ny,self%geom%nz)) -endif +if (vars%has('x')) allocate(self%x(self%geom%nx,self%geom%ny,self%geom%nz)) +if (vars%has('q')) allocate(self%q(self%geom%nx,self%geom%ny,self%geom%nz)) +if (vars%has('u')) allocate(self%u(self%geom%nx,self%geom%ny,self%geom%nz)) +if (vars%has('v')) allocate(self%v(self%geom%nx,self%geom%ny,self%geom%nz)) ! Allocate boundaries if (self%lbc) then @@ -138,46 +115,6 @@ subroutine qg_fields_create(self,geom,vars,lbc) end subroutine qg_fields_create ! ------------------------------------------------------------------------------ -!> Create fields from geometry (x) -subroutine qg_fields_create_default(self,geom,lbc) - -implicit none - -! Passed variables -type(qg_fields),intent(inout) :: self !< Fields -type(qg_geom),target,intent(in) :: geom !< Geometry -logical,intent(in) :: lbc !< Boundaries flag - -! Local variables -character(len=1024) :: record - -! Associate geometry -self%geom => geom - -! Set variables -self%lq = .false. - -! Set boundaries -self%lbc = lbc - -! Allocate 3d field -allocate(self%gfld3d(self%geom%nx,self%geom%ny,self%geom%nz)) -allocate(self%streamfct(self%geom%nx,self%geom%ny,self%geom%nz)) - -! Allocate boundaries -if (self%lbc) then - ! Allocation - allocate(self%x_north(self%geom%nz)) - allocate(self%x_south(self%geom%nz)) - allocate(self%q_north(self%geom%nx,self%geom%nz)) - allocate(self%q_south(self%geom%nx,self%geom%nz)) -endif - -! Initialize -call qg_fields_zero(self) - -end subroutine qg_fields_create_default -! ------------------------------------------------------------------------------ !> Create fields from another one subroutine qg_fields_create_from_other(self,other,geom) @@ -192,25 +129,13 @@ subroutine qg_fields_create_from_other(self,other,geom) self%geom => geom ! Copy attributes -self%lq = other%lq self%lbc = other%lbc -! Allocate 3d field -allocate(self%gfld3d(self%geom%nx,self%geom%ny,self%geom%nz)) - ! Allocate 3d fields -if (allocated(other%streamfct)) then - allocate(self%streamfct(self%geom%nx,self%geom%ny,self%geom%nz)) -endif -if (allocated(other%pv)) then - allocate(self%pv(self%geom%nx,self%geom%ny,self%geom%nz)) -endif -if (allocated(other%u)) then - allocate(self%u(self%geom%nx,self%geom%ny,self%geom%nz)) -endif -if (allocated(other%v)) then - allocate(self%v(self%geom%nx,self%geom%ny,self%geom%nz)) -endif +if (allocated(other%x)) allocate(self%x(self%geom%nx,self%geom%ny,self%geom%nz)) +if (allocated(other%q)) allocate(self%q(self%geom%nx,self%geom%ny,self%geom%nz)) +if (allocated(other%u)) allocate(self%u(self%geom%nx,self%geom%ny,self%geom%nz)) +if (allocated(other%v)) allocate(self%v(self%geom%nx,self%geom%ny,self%geom%nz)) ! Allocate boundaries if (self%lbc) then @@ -235,9 +160,8 @@ subroutine qg_fields_delete(self) type(qg_fields),intent(inout) :: self !< Fields ! Release memory -if (allocated(self%gfld3d)) deallocate(self%gfld3d) -if (allocated(self%streamfct)) deallocate(self%streamfct) -if (allocated(self%pv)) deallocate(self%pv) +if (allocated(self%x)) deallocate(self%x) +if (allocated(self%q)) deallocate(self%q) if (allocated(self%u)) deallocate(self%u) if (allocated(self%v)) deallocate(self%v) if (allocated(self%x_north)) deallocate(self%x_north) @@ -259,16 +183,15 @@ subroutine qg_fields_zero(self) call qg_fields_check(self) ! Set fields to zero -self%gfld3d = 0.0 -if (allocated(self%streamfct)) self%streamfct = 0.0 -if (allocated(self%pv)) self%pv = 0.0 -if (allocated(self%u)) self%u = 0.0 -if (allocated(self%v)) self%v = 0.0 +if (allocated(self%x)) self%x = 0.0_kind_real +if (allocated(self%q)) self%q = 0.0_kind_real +if (allocated(self%u)) self%u = 0.0_kind_real +if (allocated(self%v)) self%v = 0.0_kind_real if (self%lbc) then - self%x_north = 0.0 - self%x_south = 0.0 - self%q_north = 0.0 - self%q_south = 0.0 + self%x_north = 0.0_kind_real + self%x_south = 0.0_kind_real + self%q_north = 0.0_kind_real + self%q_south = 0.0_kind_real endif end subroutine qg_fields_zero @@ -285,9 +208,8 @@ subroutine qg_fields_ones(self) call qg_fields_check(self) ! Set fields to ones -self%gfld3d = 1.0 -if (allocated(self%streamfct)) self%streamfct = 1.0 -if (allocated(self%pv)) self%pv = 1.0 +if (allocated(self%x)) self%x = 1.0 +if (allocated(self%q)) self%q = 1.0 if (allocated(self%u)) self%u = 1.0 if (allocated(self%v)) self%v = 1.0 if (self%lbc) then @@ -311,6 +233,8 @@ subroutine qg_fields_dirac(self,f_conf) ! Local variables integer :: ndir,idir integer,allocatable :: ixdir(:),iydir(:),izdir(:) +character(len=1) :: var +character(len=:),allocatable :: str ! Check field call qg_fields_check(self) @@ -329,6 +253,8 @@ subroutine qg_fields_dirac(self,f_conf) call f_conf%get_or_die("ixdir",ixdir) call f_conf%get_or_die("iydir",iydir) call f_conf%get_or_die("izdir",izdir) +call f_conf%get_or_die("var",str) +var = str ! Check Diracs positions if (any(ixdir<1).or.any(ixdir>self%geom%nx)) call abor1_ftn('qg_fields_dirac: invalid ixdir') @@ -336,60 +262,88 @@ subroutine qg_fields_dirac(self,f_conf) if (any(izdir<1).or.any(izdir>self%geom%nz)) call abor1_ftn('qg_fields_dirac: invalid izdir') ! Setup Diracs +call qg_fields_zero(self) do idir=1,ndir - self%gfld3d(ixdir(idir),iydir(idir),izdir(idir)) = 1.0 - if (allocated(self%streamfct)) self%streamfct(ixdir(idir),iydir(idir),izdir(idir)) = 1.0 -end do + select case (var) + case ('x') + if (.not.allocated(self%x)) call abor1_ftn('qg_fields_dirac: x should be allocated') + self%x(ixdir(idir),iydir(idir),izdir(idir)) = 1.0 + case ('q') + if (.not.allocated(self%q)) call abor1_ftn('qg_fields_dirac: q should be allocated') + self%q(ixdir(idir),iydir(idir),izdir(idir)) = 1.0 + case default + call abor1_ftn('qg_fields_dirac: wrong variable') + endselect +enddo + +! Complete other fields +call qg_fields_complete(self,var) end subroutine qg_fields_dirac ! ------------------------------------------------------------------------------ !> Generate random fields -subroutine qg_fields_random(self) +subroutine qg_fields_random(self,var) implicit none ! Passed variables type(qg_fields),intent(inout) :: self !< Fields +character(len=1),intent(in) :: var !< Variable to randomize ('x' or 'q') ! Check field call qg_fields_check(self) ! Set at random value -call normal_distribution(self%gfld3d,0.0_kind_real,1.0_kind_real,rseed) +select case (var) +case ('x') + if (.not.allocated(self%x)) call abor1_ftn('qg_fields_random: x should be allocated') + call normal_distribution(self%x,0.0_kind_real,1.0_kind_real,rseed) +case ('q') + if (.not.allocated(self%q)) call abor1_ftn('qg_fields_random: q should be allocated') + call normal_distribution(self%q,0.0_kind_real,1.0_kind_real,rseed) +case default + call abor1_ftn('qg_fields_random: wrong variable') +endselect + +! Complete other fields +call qg_fields_complete(self,var) end subroutine qg_fields_random ! ------------------------------------------------------------------------------ !> Copy fields -subroutine qg_fields_copy(self,other,bconly) +subroutine qg_fields_copy(self,other) implicit none ! Passed variables type(qg_fields),intent(inout) :: self !< Fields type(qg_fields),intent(in) :: other !< Other fields -logical,intent(in),optional :: bconly !< Boundary condition only flag - -! Local variables -logical :: lbconly - -! Local flag -lbconly = .false. -if (present(bconly)) lbconly = bconly ! Check resolution call qg_fields_check_resolution(self,other) -if (.not.lbconly) then - ! Check variables - call qg_fields_check_variables(self,other) +! Copy 3D field +if (allocated(self%x).and.allocated(other%x)) self%x = other%x +if (allocated(self%q).and.allocated(other%q)) self%q = other%q +if (allocated(self%u).and.allocated(other%u)) self%u = other%u +if (allocated(self%v).and.allocated(other%v)) self%v = other%v + +! Copy LBC +call qg_fields_copy_lbc(self,other) + +end subroutine qg_fields_copy +! ------------------------------------------------------------------------------ +!> Copy fields LBC +subroutine qg_fields_copy_lbc(self,other) + +implicit none + +! Passed variables +type(qg_fields),intent(inout) :: self !< Fields +type(qg_fields),intent(in) :: other !< Other fields - ! Copy 3D field - self%gfld3d = other%gfld3d - if (allocated(self%streamfct).and.allocated(other%streamfct)) self%streamfct = other%streamfct - if (allocated(self%pv).and.allocated(other%pv)) self%pv = other%pv - if (allocated(self%u).and.allocated(other%u)) self%u = other%u - if (allocated(self%v).and.allocated(other%v)) self%v = other%v -end if +! Check resolution +call qg_fields_check_resolution(self,other) if (self%lbc) then if (other%lbc) then @@ -398,14 +352,14 @@ subroutine qg_fields_copy(self,other,bconly) self%q_north = other%q_north self%q_south = other%q_south else - self%x_north = 0.0 - self%x_south = 0.0 - self%q_north = 0.0 - self%q_south = 0.0 + self%x_north = 0.0_kind_real + self%x_south = 0.0_kind_real + self%q_north = 0.0_kind_real + self%q_south = 0.0_kind_real endif endif -end subroutine qg_fields_copy +end subroutine qg_fields_copy_lbc ! ------------------------------------------------------------------------------ !> Add fields subroutine qg_fields_self_add(self,rhs) @@ -418,12 +372,10 @@ subroutine qg_fields_self_add(self,rhs) ! Check resolution call qg_fields_check_resolution(self,rhs) -call qg_fields_check_variables(self,rhs) ! Add field -self%gfld3d = self%gfld3d+rhs%gfld3d -if (allocated(self%streamfct).and.allocated(rhs%streamfct)) self%streamfct = self%streamfct + rhs%streamfct -if (allocated(self%pv).and.allocated(rhs%pv)) self%pv = self%pv + rhs%pv +if (allocated(self%x).and.allocated(rhs%x)) self%x = self%x + rhs%x +if (allocated(self%q).and.allocated(rhs%q)) self%q = self%q + rhs%q if (allocated(self%u).and.allocated(rhs%u)) self%u = self%u + rhs%u if (allocated(self%v).and.allocated(rhs%v)) self%v = self%v + rhs%v if (self%lbc.and.rhs%lbc) then @@ -446,12 +398,10 @@ subroutine qg_fields_self_sub(self,rhs) ! Check resolution call qg_fields_check_resolution(self,rhs) -call qg_fields_check_variables(self,rhs) ! Subtract field -self%gfld3d = self%gfld3d-rhs%gfld3d -if (allocated(self%streamfct).and.allocated(rhs%streamfct)) self%streamfct = self%streamfct - rhs%streamfct -if (allocated(self%pv).and.allocated(rhs%pv)) self%pv = self%pv - rhs%pv +if (allocated(self%x).and.allocated(rhs%x)) self%x = self%x - rhs%x +if (allocated(self%q).and.allocated(rhs%q)) self%q = self%q - rhs%q if (allocated(self%u).and.allocated(rhs%u)) self%u = self%u - rhs%u if (allocated(self%v).and.allocated(rhs%v)) self%v = self%v - rhs%v if (self%lbc.and.rhs%lbc) then @@ -476,9 +426,8 @@ subroutine qg_fields_self_mul(self,zz) call qg_fields_check(self) ! Multiply with a scalar -self%gfld3d = zz*self%gfld3d -if (allocated(self%streamfct)) self%streamfct = zz * self%streamfct -if (allocated(self%pv)) self%pv = zz * self%pv +if (allocated(self%x)) self%x = zz * self%x +if (allocated(self%q)) self%q = zz * self%q if (allocated(self%u)) self%u = zz * self%u if (allocated(self%v)) self%v = zz * self%v if (self%lbc) then @@ -502,12 +451,10 @@ subroutine qg_fields_axpy(self,zz,rhs) ! Check resolution call qg_fields_check_resolution(self,rhs) -call qg_fields_check_variables(self,rhs) ! Apply apxy -self%gfld3d = self%gfld3d+zz*rhs%gfld3d -if (allocated(self%streamfct).and.allocated(rhs%streamfct)) self%streamfct = self%streamfct + zz * rhs%streamfct -if (allocated(self%pv).and.allocated(rhs%pv)) self%pv = self%pv + zz * rhs%pv +if (allocated(self%x).and.allocated(rhs%x)) self%x = self%x + zz * rhs%x +if (allocated(self%q).and.allocated(rhs%q)) self%q = self%q + zz * rhs%q if (allocated(self%u).and.allocated(rhs%u)) self%u = self%u + zz * rhs%u if (allocated(self%v).and.allocated(rhs%v)) self%v = self%v + zz * rhs%v if (self%lbc.and.rhs%lbc) then @@ -530,12 +477,10 @@ subroutine qg_fields_self_schur(self,rhs) ! Check resolution call qg_fields_check_resolution(self,rhs) -call qg_fields_check_variables(self,rhs) ! Schur product -self%gfld3d = self%gfld3d*rhs%gfld3d -if (allocated(self%streamfct).and.allocated(rhs%streamfct)) self%streamfct = self%streamfct * rhs%streamfct -if (allocated(self%pv).and.allocated(rhs%pv)) self%pv = self%pv * rhs%pv +if (allocated(self%x).and.allocated(rhs%x)) self%x = self%x * rhs%x +if (allocated(self%q).and.allocated(rhs%q)) self%q = self%q * rhs%q if (allocated(self%u).and.allocated(rhs%u)) self%u = self%u * rhs%u if (allocated(self%v).and.allocated(rhs%v)) self%v = self%v * rhs%v if (self%lbc.and.rhs%lbc) then @@ -559,10 +504,13 @@ subroutine qg_fields_dot_prod(fld1,fld2,zprod) ! Check resolution call qg_fields_check_resolution(fld1,fld2) -call qg_fields_check_variables(fld1,fld2) ! Compute dot product -zprod = sum(fld1%gfld3d*fld2%gfld3d) +zprod = 0.0_kind_real +if (allocated(fld1%x).and.allocated(fld2%x)) zprod = zprod+sum(fld1%x*fld2%x) +if (allocated(fld1%q).and.allocated(fld2%q)) zprod = zprod+sum(fld1%q*fld2%q) +if (allocated(fld1%u).and.allocated(fld2%u)) zprod = zprod+sum(fld1%u*fld2%u) +if (allocated(fld1%v).and.allocated(fld2%v)) zprod = zprod+sum(fld1%v*fld2%v) end subroutine qg_fields_dot_prod ! ------------------------------------------------------------------------------ @@ -579,20 +527,15 @@ subroutine qg_fields_add_incr(self,rhs) call qg_fields_check(self) call qg_fields_check(rhs) -if (self%lq.eqv.rhs%lq) then - if ((self%geom%nx==rhs%geom%nx).and.(self%geom%ny==rhs%geom%ny).and.(self%geom%nz==rhs%geom%nz)) then - ! Same resolution - self%gfld3d = self%gfld3d+rhs%gfld3d - if (allocated(self%streamfct).and.allocated(rhs%streamfct)) self%streamfct = self%streamfct + rhs%streamfct - if (allocated(self%pv).and.allocated(rhs%pv)) self%pv = self%pv + rhs%pv - if (allocated(self%u).and.allocated(rhs%u)) self%u = self%u + rhs%u - if (allocated(self%v).and.allocated(rhs%v)) self%v = self%v + rhs%v - else - ! Different resolutions - call abor1_ftn('qg_fields_add_incr: not coded for low res increment yet') - endif +if ((self%geom%nx==rhs%geom%nx).and.(self%geom%ny==rhs%geom%ny).and.(self%geom%nz==rhs%geom%nz)) then + ! Same resolution + if (allocated(self%x).and.allocated(rhs%x)) self%x = self%x + rhs%x + if (allocated(self%q).and.allocated(rhs%q)) self%q = self%q + rhs%q + if (allocated(self%u).and.allocated(rhs%u)) self%u = self%u + rhs%u + if (allocated(self%v).and.allocated(rhs%v)) self%v = self%v + rhs%v else - call abor1_ftn('qg_fields_add_incr: different variables') + ! Different resolutions + call abor1_ftn('qg_fields_add_incr: not coded for low res increment yet') endif end subroutine qg_fields_add_incr @@ -609,26 +552,20 @@ subroutine qg_fields_diff_incr(lhs,fld1,fld2) ! Check resolution call qg_fields_check_resolution(fld1,fld2) -call qg_fields_check_variables(fld1,fld2) call qg_fields_check(lhs) ! Initialization call qg_fields_zero(lhs) -if (lhs%lq.eqv.fld1%lq) then - if ((fld1%geom%nx==lhs%geom%nx).and.(fld1%geom%ny==lhs%geom%ny).and.(fld1%geom%nz==lhs%geom%nz)) then - ! Same resolution - lhs%gfld3d = fld1%gfld3d-fld2%gfld3d - if (allocated(lhs%streamfct).and.allocated(fld1%streamfct)) lhs%streamfct = fld1%streamfct - fld2%streamfct - if (allocated(lhs%pv).and.allocated(fld1%pv)) lhs%pv = fld1%pv - fld2%pv - if (allocated(lhs%u).and.allocated(fld1%u)) lhs%u = fld1%u - fld2%u - if (allocated(lhs%v).and.allocated(fld1%v)) lhs%v = fld1%v - fld2%v - else - ! Different resolutions - call abor1_ftn('qg_fields_diff_incr: not coded for low res increment yet') - endif +if ((fld1%geom%nx==lhs%geom%nx).and.(fld1%geom%ny==lhs%geom%ny).and.(fld1%geom%nz==lhs%geom%nz)) then + ! Same resolution + if (allocated(lhs%x).and.allocated(fld1%x)) lhs%x = fld1%x - fld2%x + if (allocated(lhs%q).and.allocated(fld1%q)) lhs%q = fld1%q - fld2%q + if (allocated(lhs%u).and.allocated(fld1%u)) lhs%u = fld1%u - fld2%u + if (allocated(lhs%v).and.allocated(fld1%v)) lhs%v = fld1%v - fld2%v else - call abor1_ftn('qg_fields_diff_incr: different variables') + ! Different resolutions + call abor1_ftn('qg_fields_diff_incr: not coded for low res increment yet') endif end subroutine qg_fields_diff_incr @@ -645,66 +582,67 @@ subroutine qg_fields_change_resol(fld,rhs) integer :: ix,iy,iz real(kind_real), allocatable, dimension(:,:,:) :: q1, q2 -! Check fields -call qg_fields_check(fld) -call qg_fields_check(rhs) - -if (fld%lq.eqv.rhs%lq) then - if ((fld%geom%nx==rhs%geom%nx).and.(fld%geom%ny==rhs%geom%ny).and.(fld%geom%nz==rhs%geom%nz)) then - ! Same resolution - call qg_fields_copy(fld,rhs) - else - do ix = 1,fld%geom%nx - do iy = 1,fld%geom%ny - do iz = 1,fld%geom%nz - call qg_interp_trilinear( rhs%geom,fld%geom%lon(ix,iy),fld%geom%lat(ix,iy),fld%geom%z(iz), & - rhs%gfld3d,fld%gfld3d(ix,iy,iz) ) - enddo +if ((fld%geom%nx==rhs%geom%nx).and.(fld%geom%ny==rhs%geom%ny).and.(fld%geom%nz==rhs%geom%nz)) then + ! Same resolution + call qg_fields_copy(fld,rhs) +else + ! Trilinear interpolation + do ix=1,fld%geom%nx + do iy=1,fld%geom%ny + do iz=1,fld%geom%nz + if (allocated(rhs%x).and.allocated(fld%x)) then + call qg_interp_trilinear(rhs%geom,fld%geom%lon(ix,iy),fld%geom%lat(ix,iy),fld%geom%z(iz), & + rhs%x,fld%x(ix,iy,iz)) + endif + if (allocated(rhs%q).and.allocated(fld%q)) then + call qg_interp_trilinear(rhs%geom,fld%geom%lon(ix,iy),fld%geom%lat(ix,iy),fld%geom%z(iz), & + rhs%q,fld%q(ix,iy,iz)) + endif + if (allocated(rhs%u).and.allocated(fld%u)) then + call qg_interp_trilinear(rhs%geom,fld%geom%lon(ix,iy),fld%geom%lat(ix,iy),fld%geom%z(iz), & + rhs%u,fld%u(ix,iy,iz)) + endif + if (allocated(rhs%v).and.allocated(fld%v)) then + call qg_interp_trilinear(rhs%geom,fld%geom%lon(ix,iy),fld%geom%lat(ix,iy),fld%geom%z(iz), & + rhs%v,fld%v(ix,iy,iz)) + endif enddo enddo - if (fld%lbc) then - if (rhs%lbc) then - allocate(q1(rhs%geom%nx,rhs%geom%ny,rhs%geom%nz)) - allocate(q2(fld%geom%nx,fld%geom%ny,fld%geom%nz)) - do iy = 1,rhs%geom%ny - q1(:,iy,:) = rhs%q_south - enddo - do ix = 1,fld%geom%nx - do iz = 1,fld%geom%nz - call qg_interp_trilinear( rhs%geom,fld%geom%lon(ix,1),fld%geom%lat(ix,1),fld%geom%z(iz), & - q1,q2(ix,1,iz) ) - enddo - enddo - fld%q_south = q2(:,1,:) - do iy = 1,rhs%geom%ny - q1(:,iy,:) = rhs%q_north + enddo + + ! Deal with boundary conditions + if (fld%lbc) then + if (rhs%lbc) then + allocate(q1(rhs%geom%nx,rhs%geom%ny,rhs%geom%nz)) + allocate(q2(fld%geom%nx,fld%geom%ny,fld%geom%nz)) + do iy=1,rhs%geom%ny + q1(:,iy,:) = rhs%q_south + enddo + do ix=1,fld%geom%nx + do iz=1,fld%geom%nz + call qg_interp_trilinear(rhs%geom,fld%geom%lon(ix,1),fld%geom%lat(ix,1),fld%geom%z(iz),q1,q2(ix,1,iz) ) enddo - do ix = 1,fld%geom%nx - do iz = 1,fld%geom%nz - call qg_interp_trilinear( rhs%geom,fld%geom%lon(ix,1),fld%geom%lat(ix,1),fld%geom%z(iz), & - q1,q2(ix,1,iz) ) - enddo + enddo + fld%q_south = q2(:,1,:) + do iy=1,rhs%geom%ny + q1(:,iy,:) = rhs%q_north + enddo + do ix=1,fld%geom%nx + do iz=1,fld%geom%nz + call qg_interp_trilinear(rhs%geom,fld%geom%lon(ix,1),fld%geom%lat(ix,1),fld%geom%z(iz),q1,q2(ix,1,iz)) enddo - fld%q_north = q2(:,1,:) - deallocate(q1,q2) - fld%x_north = rhs%x_north - fld%x_south = rhs%x_south - else - fld%x_north = 0.0 - fld%x_south = 0.0 - fld%q_north = 0.0 - fld%q_south = 0.0 - endif + enddo + fld%q_north = q2(:,1,:) + deallocate(q1,q2) + fld%x_north = rhs%x_north + fld%x_south = rhs%x_south + else + fld%x_north = 0.0_kind_real + fld%x_south = 0.0_kind_real + fld%q_north = 0.0_kind_real + fld%q_south = 0.0_kind_real endif endif -else - call abor1_ftn('qg_fields_change_resol: different variables') -endif - -if (fld%lq) then - if (allocated(fld%pv)) fld%pv = fld%gfld3d -else - if (allocated(fld%streamfct)) fld%streamfct = fld%gfld3d endif end subroutine qg_fields_change_resol @@ -722,7 +660,7 @@ subroutine qg_fields_read_file(fld,f_conf,vdate) ! Local variables integer :: iread,nx,ny,nz,bc -integer :: ncid,nx_id,ny_id,nz_id,gfld3d_id,x_north_id,x_south_id,q_north_id,q_south_id +integer :: ncid,nx_id,ny_id,nz_id,x_id,q_id,u_id,v_id,x_north_id,x_south_id,q_north_id,q_south_id logical :: lbc character(len=20) :: sdate character(len=1024) :: record,filename @@ -781,18 +719,17 @@ subroutine qg_fields_read_file(fld,f_conf,vdate) lbc = .true. else call abor1_ftn('qg_fields_read_file: wrong bc value') - end if + endif call ncerr(nf90_get_att(ncid,nf90_global,'sdate',sdate)) ! Test attributes consistency with the field if ((.not.lbc).and.fld%lbc) call abor1_ftn('qg_fields_read_file: LBC are missing in NetCDF file') ! Get variables ids - if (fld%lq) then - call ncerr(nf90_inq_varid(ncid,'q',gfld3d_id)) - else - call ncerr(nf90_inq_varid(ncid,'x',gfld3d_id)) - endif + if (allocated(fld%x)) call ncerr(nf90_inq_varid(ncid,'x',x_id)) + if (allocated(fld%q)) call ncerr(nf90_inq_varid(ncid,'q',q_id)) + if (allocated(fld%u)) call ncerr(nf90_inq_varid(ncid,'u',u_id)) + if (allocated(fld%v)) call ncerr(nf90_inq_varid(ncid,'v',v_id)) if (fld%lbc) then call ncerr(nf90_inq_varid(ncid,'x_north',x_north_id)) call ncerr(nf90_inq_varid(ncid,'x_south',x_south_id)) @@ -801,7 +738,10 @@ subroutine qg_fields_read_file(fld,f_conf,vdate) endif ! Get variables - call ncerr(nf90_get_var(ncid,gfld3d_id,fld%gfld3d)) + if (allocated(fld%x)) call ncerr(nf90_get_var(ncid,x_id,fld%x)) + if (allocated(fld%q)) call ncerr(nf90_get_var(ncid,q_id,fld%q)) + if (allocated(fld%u)) call ncerr(nf90_get_var(ncid,u_id,fld%u)) + if (allocated(fld%v)) call ncerr(nf90_get_var(ncid,v_id,fld%v)) if (fld%lbc) then call ncerr(nf90_get_var(ncid,x_north_id,fld%x_north)) call ncerr(nf90_get_var(ncid,x_south_id,fld%x_south)) @@ -815,17 +755,11 @@ subroutine qg_fields_read_file(fld,f_conf,vdate) ! Set date call fckit_log%info('qg_fields_read_file: validity date is '//sdate) call datetime_set(sdate,vdate) -end if +endif ! Check field call qg_fields_check(fld) -if (fld%lq) then - if (allocated(fld%pv)) fld%pv = fld%gfld3d -else - if (allocated(fld%streamfct)) fld%streamfct = fld%gfld3d -endif - end subroutine qg_fields_read_file ! ------------------------------------------------------------------------------ !> Write fields to file @@ -842,42 +776,30 @@ subroutine qg_fields_write_file(fld,f_conf,vdate) integer :: ncid,nx_id,ny_id,nz_id,lon_id,lat_id,z_id,area_id,heat_id,x_id,q_id,u_id,v_id integer :: x_north_id,x_south_id,q_north_id,q_south_id integer :: info -real(kind_real) :: x(fld%geom%nx,fld%geom%ny,fld%geom%nz),q(fld%geom%nx,fld%geom%ny,fld%geom%nz) -real(kind_real) :: u(fld%geom%nx,fld%geom%ny,fld%geom%nz),v(fld%geom%nx,fld%geom%ny,fld%geom%nz) -logical :: lwx,lwq,lwuv,ismpi,mainpe character(len=20) :: sdate character(len=1024) :: filename +type(oops_variables) :: vars +type(qg_fields) :: fld_io ! Check field call qg_fields_check(fld) -! Compute streamfunction and potential vorticity -if (fld%lq) then - q = fld%gfld3d - lwq = .true. - if (fld%lbc) then - call convert_q_to_x(fld%geom,q,fld%x_north,fld%x_south,x) - lwx = .true. - else - lwx = .false. - endif +! Get all variables +vars = oops_variables() +call vars%push_back('x') +call vars%push_back('q') +call vars%push_back('u') +call vars%push_back('v') +call qg_fields_create(fld_io,fld%geom,vars,.true.) +call qg_fields_copy_lbc(fld_io,fld) +if (allocated(fld%x)) then + fld_io%x = fld%x + call qg_fields_complete(fld_io,'x') +elseif (allocated(fld%q)) then + fld_io%q = fld%q + call qg_fields_complete(fld_io,'q') else - x = fld%gfld3d - lwx = .true. - if (fld%lbc) then - call convert_x_to_q(fld%geom,x,fld%x_north,fld%x_south,q) - lwq = .true. - else - lwq = .false. - endif -endif - -! Compute wind -if (fld%lbc) then - call convert_x_to_uv(fld%geom,x,fld%x_north,fld%x_south,u,v) - lwuv = .true. -else - lwuv = .false. + call abor1_ftn('qg_fields_write_file: x or q required') endif ! Set filename @@ -900,7 +822,7 @@ subroutine qg_fields_write_file(fld,f_conf,vdate) call ncerr(nf90_put_att(ncid,nf90_global,'bc',1)) else call ncerr(nf90_put_att(ncid,nf90_global,'bc',0)) -end if +endif call ncerr(nf90_put_att(ncid,nf90_global,'sdate',sdate)) ! Define variables @@ -909,20 +831,14 @@ subroutine qg_fields_write_file(fld,f_conf,vdate) call ncerr(nf90_def_var(ncid,'z',nf90_double,(/nz_id/),z_id)) call ncerr(nf90_def_var(ncid,'area',nf90_double,(/nx_id,ny_id/),area_id)) call ncerr(nf90_def_var(ncid,'heat',nf90_double,(/nx_id,ny_id/),heat_id)) -if (lwx) then - call ncerr(nf90_def_var(ncid,'x',nf90_double,(/nx_id,ny_id,nz_id/),x_id)) - call ncerr(nf90_put_att(ncid,x_id,'_FillValue',missing_value(1.0_kind_real))) -endif -if (lwq) then - call ncerr(nf90_def_var(ncid,'q',nf90_double,(/nx_id,ny_id,nz_id/),q_id)) - call ncerr(nf90_put_att(ncid,q_id,'_FillValue',missing_value(1.0_kind_real))) -endif -if (lwuv) then - call ncerr(nf90_def_var(ncid,'u',nf90_double,(/nx_id,ny_id,nz_id/),u_id)) - call ncerr(nf90_put_att(ncid,u_id,'_FillValue',missing_value(1.0_kind_real))) - call ncerr(nf90_def_var(ncid,'v',nf90_double,(/nx_id,ny_id,nz_id/),v_id)) - call ncerr(nf90_put_att(ncid,v_id,'_FillValue',missing_value(1.0_kind_real))) -endif +call ncerr(nf90_def_var(ncid,'x',nf90_double,(/nx_id,ny_id,nz_id/),x_id)) +call ncerr(nf90_put_att(ncid,x_id,'_FillValue',missing_value(1.0_kind_real))) +call ncerr(nf90_def_var(ncid,'q',nf90_double,(/nx_id,ny_id,nz_id/),q_id)) +call ncerr(nf90_put_att(ncid,q_id,'_FillValue',missing_value(1.0_kind_real))) +call ncerr(nf90_def_var(ncid,'u',nf90_double,(/nx_id,ny_id,nz_id/),u_id)) +call ncerr(nf90_put_att(ncid,u_id,'_FillValue',missing_value(1.0_kind_real))) +call ncerr(nf90_def_var(ncid,'v',nf90_double,(/nx_id,ny_id,nz_id/),v_id)) +call ncerr(nf90_put_att(ncid,v_id,'_FillValue',missing_value(1.0_kind_real))) if (fld%lbc) then call ncerr(nf90_def_var(ncid,'x_north',nf90_double,(/nz_id/),x_north_id)) call ncerr(nf90_put_att(ncid,x_north_id,'_FillValue',missing_value(1.0_kind_real))) @@ -932,7 +848,7 @@ subroutine qg_fields_write_file(fld,f_conf,vdate) call ncerr(nf90_put_att(ncid,q_north_id,'_FillValue',missing_value(1.0_kind_real))) call ncerr(nf90_def_var(ncid,'q_south',nf90_double,(/nx_id,nz_id/),q_south_id)) call ncerr(nf90_put_att(ncid,q_south_id,'_FillValue',missing_value(1.0_kind_real))) -end if +endif ! End definitions call ncerr(nf90_enddef(ncid)) @@ -943,12 +859,10 @@ subroutine qg_fields_write_file(fld,f_conf,vdate) call ncerr(nf90_put_var(ncid,z_id,fld%geom%z)) call ncerr(nf90_put_var(ncid,area_id,fld%geom%area)) call ncerr(nf90_put_var(ncid,heat_id,fld%geom%heat)) -if (lwx) call ncerr(nf90_put_var(ncid,x_id,x)) -if (lwq) call ncerr(nf90_put_var(ncid,q_id,q)) -if (lwuv) then - call ncerr(nf90_put_var(ncid,u_id,u)) - call ncerr(nf90_put_var(ncid,v_id,v)) -endif +call ncerr(nf90_put_var(ncid,x_id,fld_io%x)) +call ncerr(nf90_put_var(ncid,q_id,fld_io%q)) +call ncerr(nf90_put_var(ncid,u_id,fld_io%u)) +call ncerr(nf90_put_var(ncid,v_id,fld_io%v)) if (fld%lbc) then call ncerr(nf90_put_var(ncid,x_north_id,fld%x_north)) call ncerr(nf90_put_var(ncid,x_south_id,fld%x_south)) @@ -959,6 +873,9 @@ subroutine qg_fields_write_file(fld,f_conf,vdate) ! Close NetCDF file call ncerr(nf90_close(ncid)) +! Release memory +call vars%destruct() + end subroutine qg_fields_write_file ! ------------------------------------------------------------------------------ !> Analytic initialization of fields @@ -1013,7 +930,7 @@ subroutine qg_fields_analytic_init(fld,f_conf,vdate) enddo call baroclinic_instability(0.0_kind_real,domain_meridional,fld%geom%z(iz),'x',fld%x_north(iz)) call baroclinic_instability(0.0_kind_real,0.0_kind_real,fld%geom%z(iz),'x',fld%x_south(iz)) - end do + enddo case ('large-vortices') ! Large vortices do iz=1,fld%geom%nz @@ -1030,10 +947,10 @@ subroutine qg_fields_analytic_init(fld,f_conf,vdate) call f_conf%get_or_die("uval",uval) x = uval case default - call abor1_ftn ('qg_fields_analytic_init: unknown initialization') -end select + call abor1_ftn('qg_fields_analytic_init: unknown initialization') +endselect -! Compute PV +! Compute q call convert_x_to_q(fld%geom,x,fld%x_north,fld%x_south,q) do iz=1,fld%geom%nz do ix=1,fld%geom%nx @@ -1042,67 +959,86 @@ subroutine qg_fields_analytic_init(fld,f_conf,vdate) do ix=1,fld%geom%nx fld%q_north(ix,iz) = 2.0*q(ix,fld%geom%ny,iz)-q(ix,fld%geom%ny-1,iz) enddo -end do +enddo -! Copy 3d field -if (fld%lq) then - fld%gfld3d = q +! Copy 3d field and ensure consistency +if (allocated(x)) then + fld%x = x + call qg_fields_complete(fld,'x') +elseif (allocated(q)) then + fld%q = q + call qg_fields_complete(fld,'q') else - fld%gfld3d = x + call abor1_ftn('qg_fields_analytic_init: x or q required') endif ! Check field call qg_fields_check(fld) -if (fld%lq) then - if (allocated(fld%pv)) fld%pv = fld%gfld3d -else - if (allocated(fld%streamfct)) fld%streamfct = fld%gfld3d -endif - end subroutine qg_fields_analytic_init ! ------------------------------------------------------------------------------ !> Fields statistics -subroutine qg_fields_gpnorm(fld,nb,pstat) +subroutine qg_fields_gpnorm(fld,vpresent,vmin,vmax,vrms) implicit none ! Passed variables -type(qg_fields),intent(in) :: fld !< Fields -integer,intent(in) :: nb !< Number of boundaries -real(kind_real),intent(inout) :: pstat(3*(1+nb)) !< Statistics - -! Local variables -integer :: jj,js,jvb -real(kind_real) :: stat(3,1+nb) +type(qg_fields),intent(in) :: fld !< Fields +integer,intent(inout) :: vpresent(6) !< Variables presence flag +real(kind_real),intent(inout) :: vmin(6) !< Variables minimum +real(kind_real),intent(inout) :: vmax(6) !< Variables maximum +real(kind_real),intent(inout) :: vrms(6) !< Variables RMS ! Check field call qg_fields_check(fld) -! Check number of stats -if ((fld%lbc.and.(nb/=2)).or.((.not.fld%lbc).and.(nb>0))) call abor1_ftn('qg_fields_gpnorm: error number of fields') - -! 3d field -stat(1,1) = minval(fld%gfld3d) -stat(2,1) = maxval(fld%gfld3d) -stat(3,1) = sqrt(sum(fld%gfld3d**2)/real(fld%geom%nx*fld%geom%ny*fld%geom%nz,kind_real)) +! Initialization +vpresent = 0 +vmin = 0.0_kind_real +vmax = 0.0_kind_real +vrms = 0.0_kind_real + +! 3d fields +if (allocated(fld%x)) then + vpresent(1) = 1 + vmin(1) = minval(fld%x) + vmax(1) = maxval(fld%x) + vrms(1) = sqrt(sum(fld%x**2)/real(fld%geom%nx*fld%geom%ny*fld%geom%nz,kind_real)) +endif +if (allocated(fld%q)) then + vpresent(2) = 1 + vmin(2) = minval(fld%q) + vmax(2) = maxval(fld%q) + vrms(2) = sqrt(sum(fld%q**2)/real(fld%geom%nx*fld%geom%ny*fld%geom%nz,kind_real)) +endif +if (allocated(fld%u)) then + vpresent(3) = 1 + vmin(3) = minval(fld%u) + vmax(3) = maxval(fld%u) + vrms(3) = sqrt(sum(fld%u**2)/real(fld%geom%nx*fld%geom%ny*fld%geom%nz,kind_real)) +endif +if (allocated(fld%v)) then + vpresent(4) = 1 + vmin(4) = minval(fld%v) + vmax(4) = maxval(fld%v) + vrms(4) = sqrt(sum(fld%v**2)/real(fld%geom%nx*fld%geom%ny*fld%geom%nz,kind_real)) +endif ! Boundaries -if (nb==2) then +if (fld%lbc) then ! Streamfunction - stat(1,2) = min(minval(fld%x_north),minval(fld%x_south)) - stat(2,2) = max(maxval(fld%x_north),maxval(fld%x_south)) - stat(3,2) = sqrt(sum(fld%x_north**2+fld%x_south**2)/real(2*fld%geom%nz,kind_real)) + vpresent(5) = 1 + vmin(5) = min(minval(fld%x_north),minval(fld%x_south)) + vmax(5) = max(maxval(fld%x_north),maxval(fld%x_south)) + vrms(5) = sqrt(sum(fld%x_north**2+fld%x_south**2)/real(2*fld%geom%nz,kind_real)) ! Potential vorticity - stat(1,3) = min(minval(fld%q_north),minval(fld%q_south)) - stat(2,3) = max(maxval(fld%q_north),maxval(fld%q_south)) - stat(3,3) = sqrt(sum(fld%q_north**2+fld%q_south**2)/real(2*fld%geom%nx*fld%geom%nz,kind_real)) + vpresent(6) = 1 + vmin(6) = min(minval(fld%q_north),minval(fld%q_south)) + vmax(6) = max(maxval(fld%q_north),maxval(fld%q_south)) + vrms(6) = sqrt(sum(fld%q_north**2+fld%q_south**2)/real(2*fld%geom%nx*fld%geom%nz,kind_real)) endif -! Pack -pstat = reshape(stat,(/3*(1+nb)/)) - end subroutine qg_fields_gpnorm ! ------------------------------------------------------------------------------ !> Fields RMS @@ -1116,28 +1052,45 @@ subroutine qg_fields_rms(fld,prms) ! Local variables integer :: norm -real(kind_real) :: zz ! Check field call qg_fields_check(fld) -! 3d field -zz = sum(fld%gfld3d**2) -norm = fld%geom%nx*fld%geom%ny*fld%geom%nz +! Initialization +prms = 0.0_kind_real +norm = 0.0_kind_real + +! 3d fields +if (allocated(fld%x)) then + prms = prms+sum(fld%x**2) + norm = norm+real(fld%geom%nx*fld%geom%ny*fld%geom%nz,kind_real) +endif +if (allocated(fld%q)) then + prms = prms+sum(fld%q**2) + norm = norm+real(fld%geom%nx*fld%geom%ny*fld%geom%nz,kind_real) +endif +if (allocated(fld%u)) then + prms = prms+sum(fld%u**2) + norm = norm+real(fld%geom%nx*fld%geom%ny*fld%geom%nz,kind_real) +endif +if (allocated(fld%v)) then + prms = prms+sum(fld%v**2) + norm = norm+real(fld%geom%nx*fld%geom%ny*fld%geom%nz,kind_real) +endif ! Boundaries if (fld%lbc) then - zz = zz+sum(fld%x_north**2+fld%x_south**2)+sum(fld%q_north**2+fld%q_south**2) - norm = norm+2*(1+fld%geom%nx)*fld%geom%nz -end if + prms = prms+sum(fld%x_north**2+fld%x_south**2)+sum(fld%q_north**2+fld%q_south**2) + norm = norm+real(2*(1+fld%geom%nx)*fld%geom%nz,kind_real) +endif -! Normalize -prms = sqrt(zz/real(norm,kind_real)) +! Normalize and square-root +prms = sqrt(prms/norm) end subroutine qg_fields_rms ! ------------------------------------------------------------------------------ !> Get fields geometry -subroutine qg_fields_sizes(fld,nx,ny,nz,nb) +subroutine qg_fields_sizes(fld,nx,ny,nz) implicit none @@ -1146,47 +1099,33 @@ subroutine qg_fields_sizes(fld,nx,ny,nz,nb) integer,intent(out) :: nx !< X size integer,intent(out) :: ny !< Y size integer,intent(out) :: nz !< Z size -integer,intent(out) :: nb !< Number of boundaries ! Copy sizes nx = fld%geom%nx ny = fld%geom%ny nz = fld%geom%nz -if (fld%lbc) then - ! North and South boundaries - nb = 2 -else - ! No boundaries - nb = 0 -endif end subroutine qg_fields_sizes ! ------------------------------------------------------------------------------ -!> Get fields variables -subroutine qg_fields_vars(fld,lq,lbc) +!> Get LBC presence +subroutine qg_fields_lbc(fld,lbc) implicit none ! Passed variables type(qg_fields),intent(in) :: fld !< Fields -integer,intent(out) :: lq !< Potential vorticity flag -integer,intent(out) :: lbc !< Boundaries flag +integer,intent(out) :: lbc !< LBC presence -! Potential vorticity flag -if (fld%lq) then - lq = 1 -else - lq = 0 -endif +! Check field +call qg_fields_check(fld) -! Boundaries flag if (fld%lbc) then lbc = 1 else lbc = 0 endif -end subroutine qg_fields_vars +end subroutine qg_fields_lbc ! ------------------------------------------------------------------------------ !> Set ATLAS field subroutine qg_fields_set_atlas(self,vars,afieldset) @@ -1195,29 +1134,31 @@ subroutine qg_fields_set_atlas(self,vars,afieldset) ! Passed variables type(qg_fields),intent(in) :: self !< Fields -type(oops_variables),intent(in) :: vars !< Variables +type(oops_variables),intent(in) :: vars !< List of variables type(atlas_fieldset),intent(inout) :: afieldset !< ATLAS fieldset ! Local variables +integer :: jvar character(len=1024) :: fieldname type(atlas_field) :: afield ! Get or create field -if (vars%has('x')) fieldname = 'x' -if (vars%has('q')) fieldname = 'q' -if (afieldset%has_field(trim(fieldname))) then - ! Get afield - afield = afieldset%field(trim(fieldname)) -else - ! Create field - afield = self%geom%afunctionspace%create_field(name=trim(fieldname),kind=atlas_real(kind_real),levels=self%geom%nz) - - ! Add field - call afieldset%add(afield) -end if - -! Release pointer -call afield%final() +do jvar=1,vars%nvars() + fieldname = vars%variable(jvar) + if (afieldset%has_field(trim(fieldname))) then + ! Get afield + afield = afieldset%field(trim(fieldname)) + else + ! Create field + afield = self%geom%afunctionspace%create_field(name=trim(fieldname),kind=atlas_real(kind_real),levels=self%geom%nz) + + ! Add field + call afieldset%add(afield) + endif + + ! Release pointer + call afield%final() +enddo end subroutine qg_fields_set_atlas ! ------------------------------------------------------------------------------ @@ -1228,62 +1169,56 @@ subroutine qg_fields_to_atlas(self,vars,afieldset) ! Passed variables type(qg_fields),intent(in) :: self !< Fields -type(oops_variables),intent(in) :: vars !< Variables +type(oops_variables),intent(in) :: vars !< List of variables type(atlas_fieldset),intent(inout) :: afieldset !< ATLAS fieldset ! Local variables -integer :: iv,ix,iy,iz,inode -integer(kind_int),pointer :: int_ptr_1(:),int_ptr_2(:,:) -real(kind_real) :: gfld3d(self%geom%nx,self%geom%ny,self%geom%nz) -real(kind_real),pointer :: real_ptr_1(:),real_ptr_2(:,:) +integer :: jvar,ix,iy,iz,inode +real(kind_real),pointer :: ptr(:,:) character(len=1024) :: fieldname type(atlas_field) :: afield ! Get variable -if (vars%has('x')) then - if (self%lq) then - call convert_q_to_x(self%geom,self%gfld3d,self%x_north,self%x_south,gfld3d) - else - gfld3d = self%gfld3d - end if -end if -if (vars%has('q')) then - if (self%lq) then - gfld3d = self%gfld3d - else - call convert_x_to_q(self%geom,self%gfld3d,self%x_north,self%x_south,gfld3d) - end if -end if - -! Get or create field -if (vars%has('x')) fieldname = 'x' -if (vars%has('q')) fieldname = 'q' -if (afieldset%has_field(trim(fieldname))) then - ! Get afield - afield = afieldset%field(trim(fieldname)) -else - ! Create field - afield = self%geom%afunctionspace%create_field(name=trim(fieldname),kind=atlas_real(kind_real),levels=self%geom%nz) - - ! Add field - call afieldset%add(afield) -end if - -! Copy field -call afield%data(real_ptr_2) -do iz=1,self%geom%nz - inode = 0 - do iy=1,self%geom%ny - do ix=1,self%geom%nx - inode = inode+1 - real_ptr_2(iz,inode) = gfld3d(ix,iy,iz) - enddo - enddo +do jvar=1,vars%nvars() + fieldname = vars%variable(jvar) + if (afieldset%has_field(trim(fieldname))) then + ! Get afield + afield = afieldset%field(trim(fieldname)) + else + ! Create field + afield = self%geom%afunctionspace%create_field(name=trim(fieldname),kind=atlas_real(kind_real),levels=self%geom%nz) + + ! Add field + call afieldset%add(afield) + endif + + ! Copy field + call afield%data(ptr) + do iz=1,self%geom%nz + inode = 0 + do iy=1,self%geom%ny + do ix=1,self%geom%nx + inode = inode+1 + select case (trim(fieldname)) + case ('x') + ptr(iz,inode) = self%x(ix,iy,iz) + case ('q') + ptr(iz,inode) = self%q(ix,iy,iz) + case ('u') + ptr(iz,inode) = self%u(ix,iy,iz) + case ('v') + ptr(iz,inode) = self%v(ix,iy,iz) + case default + call abor1_ftn('qg_fields_to_atlas: wrong variable') + endselect + enddo + enddo + enddo + + ! Release pointer + call afield%final() enddo -! Release pointer -call afield%final() - end subroutine qg_fields_to_atlas ! ------------------------------------------------------------------------------ !> Get fields from ATLAS @@ -1293,52 +1228,47 @@ subroutine qg_fields_from_atlas(self,vars,afieldset) ! Passed variables type(qg_fields),intent(inout) :: self !< Fields -type(oops_variables),intent(in) :: vars !< Variables +type(oops_variables),intent(in) :: vars !< List of variables type(atlas_fieldset),intent(inout) :: afieldset !< ATLAS fieldset ! Local variables -integer :: ix,iy,iz,inode -real(kind_real) :: gfld3d(self%geom%nx,self%geom%ny,self%geom%nz) -real(kind_real),pointer :: real_ptr_1(:),real_ptr_2(:,:) -character(len=1) :: cgrid +integer :: jvar,ix,iy,iz,inode +real(kind_real),pointer :: ptr(:,:) character(len=1024) :: fieldname type(atlas_field) :: afield -! Get field -if (vars%has('x')) fieldname = 'x' -if (vars%has('q')) fieldname = 'q' -afield = afieldset%field(trim(fieldname)) - -! Copy field -call afield%data(real_ptr_2) -do iz=1,self%geom%nz - inode = 0 - do iy=1,self%geom%ny - do ix=1,self%geom%nx - inode = inode+1 - gfld3d(ix,iy,iz) = real_ptr_2(iz,inode) - enddo - enddo -enddo - ! Get variable -if (vars%has('x')) then - if (self%lq) then - call convert_x_to_q(self%geom,gfld3d,self%x_north,self%x_south,self%gfld3d) - else - self%gfld3d = gfld3d - end if -end if -if (vars%has('q')) then - if (self%lq) then - self%gfld3d = gfld3d - else - call convert_x_to_q(self%geom,gfld3d,self%x_north,self%x_south,self%gfld3d) - end if -end if - -! Release pointer -call afield%final() +do jvar=1,vars%nvars() + ! Get afield + fieldname = vars%variable(jvar) + afield = afieldset%field(trim(fieldname)) + + ! Copy field + call afield%data(ptr) + do iz=1,self%geom%nz + inode = 0 + do iy=1,self%geom%ny + do ix=1,self%geom%nx + inode = inode+1 + select case (trim(fieldname)) + case ('x') + self%x(ix,iy,iz) = ptr(iz,inode) + case ('q') + self%q(ix,iy,iz) = ptr(iz,inode) + case ('u') + self%u(ix,iy,iz) = ptr(iz,inode) + case ('v') + self%v(ix,iy,iz) = ptr(iz,inode) + case default + call abor1_ftn('qg_fields_to_atlas: wrong variable') + endselect + enddo + enddo + enddo + + ! Release pointer + call afield%final() +enddo end subroutine qg_fields_from_atlas ! ------------------------------------------------------------------------------ @@ -1354,6 +1284,7 @@ subroutine qg_fields_getpoint(fld,iter,nval,vals) real(kind_real),intent(inout) :: vals(nval) !< Values ! Local variables +integer :: ii character(len=1024) :: record ! Check @@ -1367,8 +1298,26 @@ subroutine qg_fields_getpoint(fld,iter,nval,vals) call abor1_ftn(record) endif +! Initialization +ii = 0 + ! Get values -vals = fld%gfld3d(iter%ilon,iter%ilat,:) +if (allocated(fld%x)) then + vals(ii+1:ii+fld%geom%nz) = fld%x(iter%ilon,iter%ilat,:) + ii = ii+fld%geom%nz +endif +if (allocated(fld%q)) then + vals(ii+1:ii+fld%geom%nz) = fld%q(iter%ilon,iter%ilat,:) + ii = ii+fld%geom%nz +endif +if (allocated(fld%u)) then + vals(ii+1:ii+fld%geom%nz) = fld%u(iter%ilon,iter%ilat,:) + ii = ii+fld%geom%nz +endif +if (allocated(fld%v)) then + vals(ii+1:ii+fld%geom%nz) = fld%v(iter%ilon,iter%ilat,:) + ii = ii+fld%geom%nz +endif end subroutine qg_fields_getpoint ! ------------------------------------------------------------------------------ @@ -1384,6 +1333,7 @@ subroutine qg_fields_setpoint(fld,iter,nval,vals) real(kind_real),intent(in) :: vals(nval) !< Values ! Local variables +integer :: ii character(len=1024) :: record ! Check @@ -1397,8 +1347,26 @@ subroutine qg_fields_setpoint(fld,iter,nval,vals) call abor1_ftn(record) endif -! Set values -fld%gfld3d(iter%ilon,iter%ilat,:) = vals +! Initialization +ii = 0 + +! Get values +if (allocated(fld%x)) then + fld%x(iter%ilon,iter%ilat,:) = vals(ii+1:ii+fld%geom%nz) + ii = ii+fld%geom%nz +endif +if (allocated(fld%q)) then + fld%q(iter%ilon,iter%ilat,:) = vals(ii+1:ii+fld%geom%nz) + ii = ii+fld%geom%nz +endif +if (allocated(fld%u)) then + fld%u(iter%ilon,iter%ilat,:) = vals(ii+1:ii+fld%geom%nz) + ii = ii+fld%geom%nz +endif +if (allocated(fld%v)) then + fld%v(iter%ilon,iter%ilat,:) = vals(ii+1:ii+fld%geom%nz) + ii = ii+fld%geom%nz +endif end subroutine qg_fields_setpoint ! ------------------------------------------------------------------------------ @@ -1419,11 +1387,25 @@ subroutine qg_fields_serialize(fld,vsize,vect_fld) ind = 0 ! Copy -do iz = 1,fld%geom%nz - do iy = 1,fld%geom%ny - do ix = 1,fld%geom%nx - ind = ind + 1 - vect_fld(ind) = fld%gfld3d(ix,iy,iz) +do iz=1,fld%geom%nz + do iy=1,fld%geom%ny + do ix=1,fld%geom%nx + if (allocated(fld%x)) then + ind = ind + 1 + vect_fld(ind) = fld%x(ix,iy,iz) + endif + if (allocated(fld%q)) then + ind = ind + 1 + vect_fld(ind) = fld%q(ix,iy,iz) + endif + if (allocated(fld%u)) then + ind = ind + 1 + vect_fld(ind) = fld%u(ix,iy,iz) + endif + if (allocated(fld%v)) then + ind = ind + 1 + vect_fld(ind) = fld%v(ix,iy,iz) + endif enddo enddo enddo @@ -1461,11 +1443,25 @@ subroutine qg_fields_deserialize(self,vsize,vect_fld,index) ! 3d field index = 1 + index -do iz = 1,self%geom%nz - do iy = 1,self%geom%ny - do ix = 1,self%geom%nx - self%gfld3d(ix,iy,iz) = vect_fld(index) - index = index+1 +do iz=1,self%geom%nz + do iy=1,self%geom%ny + do ix=1,self%geom%nx + if (allocated(self%x)) then + self%x(ix,iy,iz) = vect_fld(index) + index = index+1 + endif + if (allocated(self%q)) then + self%x(ix,iy,iz) = vect_fld(index) + index = index+1 + endif + if (allocated(self%u)) then + self%x(ix,iy,iz) = vect_fld(index) + index = index+1 + endif + if (allocated(self%v)) then + self%x(ix,iy,iz) = vect_fld(index) + index = index+1 + endif enddo enddo enddo @@ -1490,6 +1486,73 @@ subroutine qg_fields_deserialize(self,vsize,vect_fld,index) end subroutine qg_fields_deserialize ! ------------------------------------------------------------------------------ +!> Complete missing fields consistently +subroutine qg_fields_complete(self,var) + +implicit none + +! Passed variables +type(qg_fields),intent(inout) :: self !< Fields +character(len=1),intent(in) :: var !< Reference variable ('x' or 'q') + +! Local variables +real(kind_real) :: x(self%geom%nx,self%geom%ny,self%geom%nz) +real(kind_real) :: q(self%geom%nx,self%geom%ny,self%geom%nz) +real(kind_real) :: u(self%geom%nx,self%geom%ny,self%geom%nz) +real(kind_real) :: v(self%geom%nx,self%geom%ny,self%geom%nz) + +select case (var) +case ('x') + if (allocated(self%q)) then + if (self%lbc) then + call convert_x_to_q(self%geom,self%x,self%x_north,self%x_south,self%q) + else + call convert_x_to_q_tl(self%geom,self%x,self%q) + endif + endif + if (allocated(self%u)) then + if (self%lbc) then + call convert_x_to_u(self%geom,self%x,self%x_north,self%x_south,self%u) + else + call convert_x_to_u_tl(self%geom,self%x,self%u) + endif + endif + if (allocated(self%v)) then + if (self%lbc) then + call convert_x_to_v(self%geom,self%x,self%v) + else + call convert_x_to_v_tl(self%geom,self%x,self%v) + endif + endif +case ('q') + if (allocated(self%x).or.allocated(self%u).or.allocated(self%v)) then + if (self%lbc) then + call convert_q_to_x(self%geom,self%q,self%x_north,self%x_south,x) + else + call convert_q_to_x_tl(self%geom,self%q,x) + endif + if (allocated(self%x)) self%x = x + if (allocated(self%u)) then + if (self%lbc) then + call convert_x_to_u(self%geom,self%x,self%x_north,self%x_south,self%u) + else + call convert_x_to_u_tl(self%geom,self%x,self%u) + endif + endif + if (allocated(self%v)) then + if (self%lbc) then + call convert_x_to_v(self%geom,self%x,self%v) + else + call convert_x_to_v_tl(self%geom,self%x,self%v) + endif + endif + endif +case default + call abor1_ftn('qg_fields_complete: wrong variable') +end select + +end subroutine qg_fields_complete +! ------------------------------------------------------------------------------ !> Check fields subroutine qg_fields_check(self) @@ -1506,10 +1569,27 @@ subroutine qg_fields_check(self) bad = .false. ! Check 3d field -bad = bad.or.(.not.allocated(self%gfld3d)) -bad = bad.or.(size(self%gfld3d,1)/=self%geom%nx) -bad = bad.or.(size(self%gfld3d,2)/=self%geom%ny) -bad = bad.or.(size(self%gfld3d,3)/=self%geom%nz) +bad = bad.or.(.not.(allocated(self%x).or.allocated(self%q).or.allocated(self%u).or.allocated(self%v))) +if (allocated(self%x)) then + bad = bad.or.(size(self%x,1)/=self%geom%nx) + bad = bad.or.(size(self%x,2)/=self%geom%ny) + bad = bad.or.(size(self%x,3)/=self%geom%nz) +endif +if (allocated(self%q)) then + bad = bad.or.(size(self%q,1)/=self%geom%nx) + bad = bad.or.(size(self%q,2)/=self%geom%ny) + bad = bad.or.(size(self%q,3)/=self%geom%nz) +endif +if (allocated(self%u)) then + bad = bad.or.(size(self%u,1)/=self%geom%nx) + bad = bad.or.(size(self%u,2)/=self%geom%ny) + bad = bad.or.(size(self%u,3)/=self%geom%nz) +endif +if (allocated(self%v)) then + bad = bad.or.(size(self%v,1)/=self%geom%nx) + bad = bad.or.(size(self%v,2)/=self%geom%ny) + bad = bad.or.(size(self%v,3)/=self%geom%nz) +endif ! Check boundaries if (self%lbc) then @@ -1534,8 +1614,22 @@ subroutine qg_fields_check(self) call fckit_log%info('qg_fields_check: field not consistent') write(record,*) ' nx,ny,nz,lbc = ',self%geom%nx,self%geom%ny,self%geom%nz,self%lbc call fckit_log%info(record) - write(record,*) ' shape(gfld3d) = ',shape(self%gfld3d) - call fckit_log%info(record) + if (allocated(self%x)) then + write(record,*) ' shape(x) = ',shape(self%x) + call fckit_log%info(record) + endif + if (allocated(self%q)) then + write(record,*) ' shape(q) = ',shape(self%q) + call fckit_log%info(record) + endif + if (allocated(self%u)) then + write(record,*) ' shape(u) = ',shape(self%u) + call fckit_log%info(record) + endif + if (allocated(self%v)) then + write(record,*) ' shape(v) = ',shape(self%v) + call fckit_log%info(record) + endif if (self%lbc) then write(record,*) ' shape(x_north) = ',shape(self%x_north) call fckit_log%info(record) @@ -1576,28 +1670,4 @@ subroutine qg_fields_check_resolution(fld1,fld2) end subroutine qg_fields_check_resolution ! ------------------------------------------------------------------------------ -!> Check fields variables -subroutine qg_fields_check_variables(fld1,fld2) - -implicit none - -! Passed variables -type(qg_fields),intent(in) :: fld1 !< First fields -type(qg_fields),intent(in) :: fld2 !< Second fields - -! Local variables -character(len=1024) :: record - -! Check fields consistency -if (fld1%lq.neqv.fld2%lq) then - write(record,*) 'qg_fields_check_variables: variables inconsistency, ',fld1%lq,' and ',fld2%lq - call abor1_ftn(record) -endif - -! Check fields independently -call qg_fields_check(fld1) -call qg_fields_check(fld2) - -end subroutine qg_fields_check_variables -! ------------------------------------------------------------------------------ end module qg_fields_mod diff --git a/qg/model/qg_getvalues_mod.F90 b/qg/model/qg_getvalues_mod.F90 index 4c420209c..f5804f912 100644 --- a/qg/model/qg_getvalues_mod.F90 +++ b/qg/model/qg_getvalues_mod.F90 @@ -11,9 +11,8 @@ module qg_getvalues_mod use iso_c_binding use kinds !$ use omp_lib -use qg_convert_q_to_x_mod -use qg_convert_x_to_q_mod -use qg_convert_x_to_uv_mod +use oops_variables_mod +use qg_change_var_mod use qg_fields_mod use qg_geom_mod use qg_gom_mod @@ -43,57 +42,44 @@ subroutine qg_getvalues_interp(locs,fld,t1,t2,gom) ! Local variables integer :: jloc -real(kind_real),allocatable :: x(:,:,:),q(:,:,:),u(:,:,:),v(:,:,:) real(kind_real), pointer :: lonlat(:,:), z(:) type(atlas_field) :: lonlat_field, z_field +type(qg_fields) :: fld_gom -! get locations +! Get locations lonlat_field = locs%lonlat() call lonlat_field%data(lonlat) - z_field = locs%altitude() call z_field%data(z) ! Check field call qg_fields_check(fld) -! Allocation -allocate(x(fld%geom%nx,fld%geom%ny,fld%geom%nz)) -allocate(q(fld%geom%nx,fld%geom%ny,fld%geom%nz)) -allocate(u(fld%geom%nx,fld%geom%ny,fld%geom%nz)) -allocate(v(fld%geom%nx,fld%geom%ny,fld%geom%nz)) - -! Get variables -if (fld%lq) then - q = fld%gfld3d -else - x = fld%gfld3d -endif -if (gom%ix /= 0.or.gom%iu /= 0.or.gom%iv /= 0) then - if (fld%lq) call convert_q_to_x(fld%geom,q,fld%x_north,fld%x_south,x) -endif -if (gom%iq /= 0) then - if (.not.fld%lq) call convert_x_to_q(fld%geom,x,fld%x_north,fld%x_south,q) -endif -if (gom%iu /= 0.or.gom%iv /= 0) call convert_x_to_uv(fld%geom,x,fld%x_north,fld%x_south,u,v) +! Create field with GOM variables +call qg_fields_create(fld_gom,fld%geom,gom%vars,.true.) +call qg_fields_copy_lbc(fld_gom,fld) + +! Apply change of variables +call qg_change_var(fld,fld_gom) !$omp parallel do schedule(static) private(jloc) do jloc=1,locs%nlocs() ! Check if current obs is in this time frame (t1,t2] if (t1 < locs%times(jloc) .and. locs%times(jloc) <= t2) then ! Interpolate variables - if (gom%ix /= 0) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & - & z(jloc),x,gom%values(gom%ix,jloc)) - if (gom%iq /= 0) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & - & z(jloc),q,gom%values(gom%iq,jloc)) - if (gom%iu /= 0) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & - & z(jloc),u,gom%values(gom%iu,jloc)) - if (gom%iv /= 0) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & - & z(jloc),v,gom%values(gom%iv,jloc)) + if (gom%vars%has('x')) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & + & z(jloc),fld_gom%x,gom%x(jloc)) + if (gom%vars%has('q')) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & + & z(jloc),fld_gom%q,gom%q(jloc)) + if (gom%vars%has('u')) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & + & z(jloc),fld_gom%u,gom%u(jloc)) + if (gom%vars%has('v')) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & + & z(jloc),fld_gom%v,gom%v(jloc)) endif enddo !$omp end parallel do +! Release memory call lonlat_field%final() call z_field%final() @@ -112,57 +98,43 @@ subroutine qg_getvalues_interp_tl(locs,fld,t1,t2,gom) ! Local variables integer :: jloc -real(kind_real),allocatable :: x(:,:,:),q(:,:,:),u(:,:,:),v(:,:,:) real(kind_real), pointer :: lonlat(:,:), z(:) type(atlas_field) :: lonlat_field, z_field +type(qg_fields) :: fld_gom -! get locations +! Get locations lonlat_field = locs%lonlat() call lonlat_field%data(lonlat) - z_field = locs%altitude() call z_field%data(z) ! Check field call qg_fields_check(fld) -! Allocation -allocate(x(fld%geom%nx,fld%geom%ny,fld%geom%nz)) -allocate(q(fld%geom%nx,fld%geom%ny,fld%geom%nz)) -allocate(u(fld%geom%nx,fld%geom%ny,fld%geom%nz)) -allocate(v(fld%geom%nx,fld%geom%ny,fld%geom%nz)) - -! Get variables -if (fld%lq) then - q = fld%gfld3d -else - x = fld%gfld3d -endif -if (gom%ix /= 0.or.gom%iu /= 0.or.gom%iv /= 0) then - if (fld%lq) call convert_q_to_x_tl(fld%geom,q,x) -endif -if (gom%iq /= 0) then - if (.not.fld%lq) call convert_x_to_q_tl(fld%geom,x,q) -endif -if (gom%iu /= 0.or.gom%iv /= 0) call convert_x_to_uv_tl(fld%geom,x,u,v) +! Create field with GOM variables +call qg_fields_create(fld_gom,fld%geom,gom%vars,.false.) + +! Apply change of variables +call qg_change_var_tl(fld,fld_gom) !$omp parallel do schedule(static) private(jloc) do jloc=1,locs%nlocs() ! Check if current obs is in this time frame (t1,t2] if (t1 < locs%times(jloc) .and. locs%times(jloc) <= t2) then ! Interpolate variables - if (gom%ix /= 0) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & - & z(jloc),x,gom%values(gom%ix,jloc)) - if (gom%iq /= 0) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & - & z(jloc),q,gom%values(gom%iq,jloc)) - if (gom%iu /= 0) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & - & z(jloc),u,gom%values(gom%iu,jloc)) - if (gom%iv /= 0) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & - & z(jloc),v,gom%values(gom%iv,jloc)) + if (gom%vars%has('x')) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & + & z(jloc),fld_gom%x,gom%x(jloc)) + if (gom%vars%has('q')) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & + & z(jloc),fld_gom%q,gom%q(jloc)) + if (gom%vars%has('u')) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & + & z(jloc),fld_gom%u,gom%u(jloc)) + if (gom%vars%has('v')) call qg_interp_trilinear(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & + & z(jloc),fld_gom%v,gom%v(jloc)) endif enddo !$omp end parallel do +! Release memory call lonlat_field%final() call z_field%final() @@ -184,58 +156,44 @@ subroutine qg_getvalues_interp_ad(locs,fld,t1,t2,gom) real(kind_real),allocatable :: x(:,:,:),q(:,:,:),u(:,:,:),v(:,:,:) real(kind_real), pointer :: lonlat(:,:), z(:) type(atlas_field) :: lonlat_field, z_field +type(qg_fields) :: fld_gom,fld_tmp -! get locations +! Get locations lonlat_field = locs%lonlat() call lonlat_field%data(lonlat) - z_field = locs%altitude() call z_field%data(z) ! Check field call qg_fields_check(fld) -! Allocation -allocate(x(fld%geom%nx,fld%geom%ny,fld%geom%nz)) -allocate(q(fld%geom%nx,fld%geom%ny,fld%geom%nz)) -allocate(u(fld%geom%nx,fld%geom%ny,fld%geom%nz)) -allocate(v(fld%geom%nx,fld%geom%ny,fld%geom%nz)) +! Create field with GOM variables +call qg_fields_create(fld_gom,fld%geom,gom%vars,.false.) ! Initialization -x = 0.0 -q = 0.0 -u = 0.0 -v = 0.0 +call qg_fields_zero(fld_gom) do jloc=locs%nlocs(),1,-1 ! Check if current obs is in this time frame (t1,t2] if (t1 < locs%times(jloc) .and. locs%times(jloc) <= t2) then ! Interpolate variables - if (gom%ix /= 0) call qg_interp_trilinear_ad(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & - & z(jloc),gom%values(gom%ix,jloc),x) - if (gom%iq /= 0) call qg_interp_trilinear_ad(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & - & z(jloc),gom%values(gom%iq,jloc),q) - if (gom%iu /= 0) call qg_interp_trilinear_ad(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & - & z(jloc),gom%values(gom%iu,jloc),u) - if (gom%iv /= 0) call qg_interp_trilinear_ad(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & - & z(jloc),gom%values(gom%iv,jloc),v) + if (gom%vars%has('x')) call qg_interp_trilinear_ad(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & + & z(jloc),gom%x(jloc),fld_gom%x) + if (gom%vars%has('q')) call qg_interp_trilinear_ad(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & + & z(jloc),gom%q(jloc),fld_gom%q) + if (gom%vars%has('u')) call qg_interp_trilinear_ad(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & + & z(jloc),gom%u(jloc),fld_gom%u) + if (gom%vars%has('v')) call qg_interp_trilinear_ad(fld%geom,lonlat(1,jloc),lonlat(2,jloc), & + & z(jloc),gom%v(jloc),fld_gom%v) endif enddo -! Get variables -if (gom%iu /= 0.or.gom%iv /= 0) call convert_x_to_uv_ad(fld%geom,u,v,x) -if (gom%iq /= 0) then - if (.not.fld%lq) call convert_x_to_q_ad(fld%geom,q,x) -endif -if (gom%ix /= 0.or.gom%iu /= 0.or.gom%iv /= 0) then - if (fld%lq) call convert_q_to_x_ad(fld%geom,x,q) -endif -if (fld%lq) then - fld%gfld3d = fld%gfld3d+q -else - fld%gfld3d = fld%gfld3d+x -endif +! Apply change of variables +call qg_fields_create_from_other(fld_tmp,fld,fld%geom) +call qg_change_var_ad(fld_gom,fld_tmp) +call qg_fields_self_add(fld,fld_tmp) +! Release memory call lonlat_field%final() call z_field%final() diff --git a/qg/model/qg_gom_interface.F90 b/qg/model/qg_gom_interface.F90 index b5b8e4c30..4c7f513ca 100644 --- a/qg/model/qg_gom_interface.F90 +++ b/qg/model/qg_gom_interface.F90 @@ -28,34 +28,34 @@ subroutine qg_gom_setup_c(c_key_self,c_locs,c_vars) bind(c,name='qg_gom_setup_f9 implicit none ! Passed variables -integer(c_int),intent(inout) :: c_key_self !< GOM -type(c_ptr),value,intent(in) :: c_locs !< Locations -type(c_ptr),value,intent(in) :: c_vars !< Variables +integer(c_int),intent(inout) :: c_key_self !< GOM +type(c_ptr),value,intent(in) :: c_locs !< Locations +type(c_ptr),value,intent(in) :: c_vars !< Variables ! Local variables type(qg_gom),pointer :: self type(qg_locs) :: locs -type(oops_variables) :: vars ! Interface call qg_gom_registry%init() call qg_gom_registry%add(c_key_self) call qg_gom_registry%get(c_key_self,self) locs = qg_locs(c_locs) -vars = oops_variables(c_vars) +self%vars = oops_variables(c_vars) ! Call Fortran -call qg_gom_setup(self,locs%nlocs(),vars) +call qg_gom_setup(self,locs%nlocs()) end subroutine qg_gom_setup_c ! ------------------------------------------------------------------------------ -!> Create GOM -subroutine qg_gom_create_c(c_key_self) bind(c,name='qg_gom_create_f90') +!> Create GOM and do nothing +subroutine qg_gom_create_c(c_key_self,c_vars) bind(c,name='qg_gom_create_f90') implicit none ! Passed variables integer(c_int),intent(inout) :: c_key_self !< GOM +type(c_ptr),value,intent(in) :: c_vars !< Variables ! Local variables type(qg_gom),pointer :: self @@ -64,9 +64,7 @@ subroutine qg_gom_create_c(c_key_self) bind(c,name='qg_gom_create_f90') call qg_gom_registry%init() call qg_gom_registry%add(c_key_self) call qg_gom_registry%get(c_key_self,self) - -! Call Fortran -call qg_gom_create(self) +self%vars = oops_variables(c_vars) end subroutine qg_gom_create_c ! ------------------------------------------------------------------------------ @@ -81,7 +79,6 @@ subroutine qg_gom_delete_c(c_key_self) bind(c,name='qg_gom_delete_f90') ! Local variables type(qg_gom),pointer :: self - ! Interface call qg_gom_registry%get(c_key_self,self) @@ -99,8 +96,8 @@ subroutine qg_gom_copy_c(c_key_self,c_key_other) bind(c,name='qg_gom_copy_f90') implicit none ! Passed variables -integer(c_int),intent(in) :: c_key_self !< GOM -integer(c_int),intent(in) :: c_key_other !< Other GOM +integer(c_int),intent(inout) :: c_key_self !< GOM +integer(c_int),intent(in) :: c_key_other !< Other GOM ! Local variables type(qg_gom),pointer :: self @@ -173,13 +170,13 @@ subroutine qg_gom_random_c(c_key_self) bind(c,name='qg_gom_random_f90') end subroutine qg_gom_random_c ! ------------------------------------------------------------------------------ !> Multiply GOM with a scalar -subroutine qg_gom_mult_c(c_key_self,zz) bind(c,name='qg_gom_mult_f90') +subroutine qg_gom_mult_c(c_key_self,c_zz) bind(c,name='qg_gom_mult_f90') implicit none ! Passed variables integer(c_int),intent(in) :: c_key_self !< GOM -real(c_double),intent(in) :: zz !< Multiplier +real(c_double),intent(in) :: c_zz !< Multiplier ! Local variables type(qg_gom),pointer :: self @@ -189,11 +186,7 @@ subroutine qg_gom_mult_c(c_key_self,zz) bind(c,name='qg_gom_mult_f90') call qg_gom_registry%get(c_key_self,self) ! Call Fortran -do jo=1,self%nobs - do jv=1,self%nv - self%values(jv,jo) = zz * self%values(jv,jo) - enddo -enddo +call qg_gom_mult(self,c_zz) end subroutine qg_gom_mult_c ! ------------------------------------------------------------------------------ @@ -286,13 +279,13 @@ subroutine qg_gom_divide_c(c_key_self,c_key_other) bind(c,name='qg_gom_divide_f9 end subroutine qg_gom_divide_c ! ------------------------------------------------------------------------------ !> Compute GOM RMS -subroutine qg_gom_rms_c(c_key_self,rms) bind(c,name='qg_gom_rms_f90') +subroutine qg_gom_rms_c(c_key_self,c_rms) bind(c,name='qg_gom_rms_f90') implicit none ! Passed variables integer(c_int),intent(in) :: c_key_self !< GOM -real(c_double),intent(inout) :: rms !< RMS +real(c_double),intent(inout) :: c_rms !< RMS ! Local variables type(qg_gom),pointer :: self @@ -301,19 +294,19 @@ subroutine qg_gom_rms_c(c_key_self,rms) bind(c,name='qg_gom_rms_f90') call qg_gom_registry%get(c_key_self,self) ! Call Fortran -call qg_gom_rms(self,rms) +call qg_gom_rms(self,c_rms) end subroutine qg_gom_rms_c ! ------------------------------------------------------------------------------ !> GOM dot product -subroutine qg_gom_dotprod_c(c_key_gom1,c_key_gom2,prod) bind(c,name='qg_gom_dotprod_f90') +subroutine qg_gom_dotprod_c(c_key_gom1,c_key_gom2,c_prod) bind(c,name='qg_gom_dotprod_f90') implicit none ! Passed variables integer(c_int),intent(in) :: c_key_gom1 !< GOM 1 integer(c_int),intent(in) :: c_key_gom2 !< GOM 2 -real(c_double),intent(inout) :: prod !< Dot product +real(c_double),intent(inout) :: c_prod !< Dot product ! Local variables type(qg_gom),pointer :: gom1,gom2 @@ -323,21 +316,21 @@ subroutine qg_gom_dotprod_c(c_key_gom1,c_key_gom2,prod) bind(c,name='qg_gom_dotp call qg_gom_registry%get(c_key_gom2,gom2) ! Call Fortran -call qg_gom_dotprod(gom1,gom2,prod) +call qg_gom_dotprod(gom1,gom2,c_prod) end subroutine qg_gom_dotprod_c ! ------------------------------------------------------------------------------ !> Compute GOM statistics -subroutine qg_gom_stats_c(c_key_self,kobs,pmin,pmax,prms) bind(c,name='qg_gom_stats_f90') +subroutine qg_gom_stats_c(c_key_self,c_kobs,c_pmin,c_pmax,c_prms) bind(c,name='qg_gom_stats_f90') implicit none ! Passed variables integer(c_int),intent(in) :: c_key_self !< GOM -integer(c_int),intent(inout) :: kobs !< Number of observations -real(c_double),intent(inout) :: pmin !< Minimum value -real(c_double),intent(inout) :: pmax !< Maximum value -real(c_double),intent(inout) :: prms !< RMS +integer(c_int),intent(inout) :: c_kobs !< Number of observations +real(c_double),intent(inout) :: c_pmin !< Minimum value +real(c_double),intent(inout) :: c_pmax !< Maximum value +real(c_double),intent(inout) :: c_prms !< RMS ! Local variables type(qg_gom),pointer :: self @@ -346,29 +339,31 @@ subroutine qg_gom_stats_c(c_key_self,kobs,pmin,pmax,prms) bind(c,name='qg_gom_st call qg_gom_registry%get(c_key_self,self) ! Call Fortran -call qg_gom_stats(self,kobs,pmin,pmax,prms) +call qg_gom_stats(self,c_kobs,c_pmin,c_pmax,c_prms) end subroutine qg_gom_stats_c ! ------------------------------------------------------------------------------ !> Find and locate GOM max. value -subroutine qg_gom_maxloc_c(c_key_self,mxval,iloc,ivar) bind(c,name='qg_gom_maxloc_f90') +subroutine qg_gom_maxloc_c(c_key_self,c_mxval,c_mxloc,c_mxvar) bind(c,name='qg_gom_maxloc_f90') implicit none ! Passed variables integer(c_int),intent(in) :: c_key_self !< GOM -real(c_double),intent(inout) :: mxval !< Maximum value -integer(c_int),intent(inout) :: iloc !< Location of maximum value -integer(c_int),intent(inout) :: ivar !< Variable with maximum value +real(c_double),intent(inout) :: c_mxval !< Maximum value +integer(c_int),intent(inout) :: c_mxloc !< Location of maximum value +type(c_ptr),value,intent(in) :: c_mxvar !< Variable of maximum value ! Local variables type(qg_gom),pointer :: self +type(oops_variables) :: mxvar ! Interface call qg_gom_registry%get(c_key_self,self) +mxvar = oops_variables(c_mxvar) ! Call Fortran -call qg_gom_maxloc(self,mxval,iloc,ivar) +call qg_gom_maxloc(self,c_mxval,c_mxloc,mxvar) end subroutine qg_gom_maxloc_c ! ------------------------------------------------------------------------------ @@ -378,16 +373,16 @@ subroutine qg_gom_read_file_c(c_key_self,c_conf) bind(c,name='qg_gom_read_file_f implicit none ! Passed variables -integer(c_int),intent(in) :: c_key_self !< GOM -type(c_ptr),value,intent(in) :: c_conf !< Configuration +integer(c_int),intent(inout) :: c_key_self !< GOM +type(c_ptr),value,intent(in) :: c_conf !< Configuration ! Local variables type(fckit_configuration) :: f_conf type(qg_gom),pointer :: self ! Interface -f_conf = fckit_configuration(c_conf) call qg_gom_registry%get(c_key_self,self) +f_conf = fckit_configuration(c_conf) ! Call Fortran call qg_gom_read_file(self,f_conf) @@ -423,7 +418,7 @@ subroutine qg_gom_analytic_init_c(c_key_self,c_locs,c_conf) bind(c,name='qg_gom_ ! Passed variables integer(c_int),intent(in) :: c_key_self !< GOM -type(c_ptr),value,intent(in) :: c_locs !< Locations +type(c_ptr),value,intent(in) :: c_locs !< Locations type(c_ptr),value,intent(in) :: c_conf !< Configuration ! Local variables diff --git a/qg/model/qg_gom_mod.F90 b/qg/model/qg_gom_mod.F90 index 4833ff549..8d4e7f6fb 100644 --- a/qg/model/qg_gom_mod.F90 +++ b/qg/model/qg_gom_mod.F90 @@ -26,20 +26,18 @@ module qg_gom_mod private public :: qg_gom public :: qg_gom_registry -public :: qg_gom_setup,qg_gom_create,qg_gom_delete,qg_gom_copy,qg_gom_zero,qg_gom_abs,qg_gom_random,qg_gom_mult, & +public :: qg_gom_setup,qg_gom_delete,qg_gom_copy,qg_gom_zero,qg_gom_abs,qg_gom_random,qg_gom_mult, & & qg_gom_add,qg_gom_diff,qg_gom_schurmult,qg_gom_divide,qg_gom_rms,qg_gom_dotprod,qg_gom_stats,qg_gom_maxloc, & & qg_gom_read_file, qg_gom_write_file,qg_gom_analytic_init ! ------------------------------------------------------------------------------ type :: qg_gom - integer :: nobs !< Number of observations - integer :: used !< Index of used observation - integer :: ix !< Streamfunction index - integer :: iq !< Potential vorticity index - integer :: iu !< Zonal wind index - integer :: iv !< Meridian wind index - integer :: nv !< Number of variables - real(kind_real), allocatable :: values(:,:) !< Observations values - logical :: lalloc !< Allocation flag + integer :: nobs !< Number of observations + real(kind_real), allocatable :: x(:) !< Streamfunction observations values + real(kind_real), allocatable :: q(:) !< Potential vorticity observations values + real(kind_real), allocatable :: u(:) !< Zonal wind observations values + real(kind_real), allocatable :: v(:) !< Meridian wind observations values + logical :: lalloc = .false. !< Allocation flag + type(oops_variables) :: vars !< Variables end type qg_gom #define LISTED_TYPE qg_gom @@ -58,61 +56,26 @@ module qg_gom_mod #include "oops/util/linkedList_c.f" ! ------------------------------------------------------------------------------ !> Setup GOM -subroutine qg_gom_setup(self,nobs,vars) +subroutine qg_gom_setup(self,nobs) implicit none ! Passed variables -type(qg_gom),intent(inout) :: self !< GOM -integer, intent(in) :: nobs !< Number of observations -type(oops_variables),intent(in) :: vars !< Variables - -! Local variables -integer :: ivar +type(qg_gom),intent(inout) :: self !< GOM +integer, intent(in) :: nobs !< Number of observations ! Set attributes self%nobs = nobs -self%used = 0 -self%nv = 0 -self%ix = 0; self%iq = 0; self%iu = 0; self%iv = 0 -do ivar = 1, vars%nvars() - if (vars%variable(ivar) == 'x') then - self%nv = self%nv+1 - self%ix = self%nv - endif - if (vars%variable(ivar) == 'q') then - self%nv = self%nv+1 - self%iq = self%nv - endif - if (vars%variable(ivar) == 'u') then - self%nv = self%nv+1 - self%iu = self%nv - endif - if (vars%variable(ivar) == 'v') then - self%nv = self%nv+1 - self%iv = self%nv - endif -enddo ! Allocation -allocate(self%values(self%nv,self%nobs)) +if (self%vars%has('x')) allocate(self%x(self%nobs)) +if (self%vars%has('q')) allocate(self%q(self%nobs)) +if (self%vars%has('u')) allocate(self%u(self%nobs)) +if (self%vars%has('v')) allocate(self%v(self%nobs)) self%lalloc = .true. end subroutine qg_gom_setup ! ------------------------------------------------------------------------------ -!> Create GOM -subroutine qg_gom_create(self) - -implicit none - -! Passed variables -type(qg_gom),intent(inout) :: self !< GOM - -! Set allocation flag -self%lalloc = .false. - -end subroutine qg_gom_create -! ------------------------------------------------------------------------------ !> Delete GOM subroutine qg_gom_delete(self) @@ -122,9 +85,11 @@ subroutine qg_gom_delete(self) type(qg_gom),intent(inout) :: self !< GOM ! Release memory -if (self%lalloc) then - deallocate(self%values) -endif +if (allocated(self%x)) deallocate(self%x) +if (allocated(self%q)) deallocate(self%q) +if (allocated(self%u)) deallocate(self%u) +if (allocated(self%v)) deallocate(self%v) +self%lalloc = .false. end subroutine qg_gom_delete ! ------------------------------------------------------------------------------ @@ -134,25 +99,26 @@ subroutine qg_gom_copy(self,other) implicit none ! Passed variables -type(qg_gom),intent(inout) :: self !< GOM -type(qg_gom),intent(in) :: other !< Other GOM +type(qg_gom),intent(inout) :: self !< GOM +type(qg_gom),intent(in) :: other !< Other GOM -! Copy attribues +! Copy attributes self%nobs = other%nobs -self%ix = other%ix -self%iq = other%iq -self%iu = other%iu -self%iv = other%iv -self%nv = other%nv -self%used = other%used + ! Allocation if (.not.self%lalloc) then - allocate(self%values(self%nv,self%nobs)) - self%lalloc = .true. + if (self%vars%has('x')) allocate(self%x(self%nobs)) + if (self%vars%has('q')) allocate(self%q(self%nobs)) + if (self%vars%has('u')) allocate(self%u(self%nobs)) + if (self%vars%has('v')) allocate(self%v(self%nobs)) + self%lalloc = .true. endif ! Copy -self%values = other%values +if (self%vars%has('x')) self%x = other%x +if (self%vars%has('q')) self%q = other%q +if (self%vars%has('u')) self%u = other%u +if (self%vars%has('v')) self%v = other%v end subroutine qg_gom_copy ! ------------------------------------------------------------------------------ @@ -165,7 +131,10 @@ subroutine qg_gom_zero(self) type(qg_gom),intent(inout) :: self !< GOM ! Set to zero -self%values = 0.0 +if (self%vars%has('x')) self%x = 0.0 +if (self%vars%has('q')) self%q = 0.0 +if (self%vars%has('u')) self%u = 0.0 +if (self%vars%has('v')) self%v = 0.0 end subroutine qg_gom_zero ! ------------------------------------------------------------------------------ @@ -178,7 +147,10 @@ subroutine qg_gom_abs(self) type(qg_gom),intent(inout) :: self !< GOM ! Get absolute value -self%values = abs(self%values) +if (self%vars%has('x')) self%x = abs(self%x) +if (self%vars%has('q')) self%q = abs(self%q) +if (self%vars%has('u')) self%u = abs(self%u) +if (self%vars%has('v')) self%v = abs(self%v) end subroutine qg_gom_abs ! ------------------------------------------------------------------------------ @@ -190,8 +162,39 @@ subroutine qg_gom_random(self) ! Passed variables type(qg_gom),intent(inout) :: self !< GOM +! Local variables +integer :: nv +real(kind_real),allocatable :: values(:,:) + +! TODO(Benjamin): change that in a following PR +nv = 0 +if (self%vars%has('x')) nv = nv+1 +if (self%vars%has('q')) nv = nv+1 +if (self%vars%has('u')) nv = nv+1 +if (self%vars%has('v')) nv = nv+1 +allocate(values(nv,self%nobs)) + ! Generate random GOM values -call normal_distribution(self%values,0.0_kind_real,1.0_kind_real) +call normal_distribution(values,0.0_kind_real,1.0_kind_real) + +! Split random values +nv = 0 +if (self%vars%has('x')) then + nv = nv+1 + self%x = values(nv,:) +endif +if (self%vars%has('q')) then + nv = nv+1 + self%q = values(nv,:) +endif +if (self%vars%has('u')) then + nv = nv+1 + self%u = values(nv,:) +endif +if (self%vars%has('v')) then + nv = nv+1 + self%v = values(nv,:) +endif end subroutine qg_gom_random ! ------------------------------------------------------------------------------ @@ -204,15 +207,11 @@ subroutine qg_gom_mult(self,zz) type(qg_gom),intent(inout) :: self !< GOM real(kind_real),intent(in) :: zz !< Multiplier -! Local variables -integer :: jo,jv - ! Multiply GOM with a scalar -do jo=1,self%nobs - do jv=1,self%nv - self%values(jv,jo) = zz*self%values(jv,jo) - enddo -enddo +if (self%vars%has('x')) self%x = zz*self%x +if (self%vars%has('q')) self%q = zz*self%q +if (self%vars%has('u')) self%u = zz*self%u +if (self%vars%has('v')) self%v = zz*self%v end subroutine qg_gom_mult ! ------------------------------------------------------------------------------ @@ -225,15 +224,11 @@ subroutine qg_gom_add(self,other) type(qg_gom),intent(inout) :: self !< GOM type(qg_gom),intent(in) :: other !< Other GOM -! Local variables -integer :: jo,jv - ! Add GOM -do jo=1,self%nobs - do jv=1,self%nv - self%values(jv,jo) = self%values(jv,jo)+other%values(jv,jo) - enddo -enddo +if (self%vars%has('x')) self%x = self%x+other%x +if (self%vars%has('q')) self%q = self%q+other%q +if (self%vars%has('u')) self%u = self%u+other%u +if (self%vars%has('v')) self%v = self%v+other%v end subroutine qg_gom_add ! ------------------------------------------------------------------------------ @@ -246,15 +241,11 @@ subroutine qg_gom_diff(self,other) type(qg_gom),intent(inout) :: self !< GOM type(qg_gom),intent(in) :: other !< Other GOM -! Local variables -integer :: jo,jv - ! Subtract GOM -do jo=1,self%nobs - do jv=1,self%nv - self%values(jv,jo) = self%values(jv,jo)-other%values(jv,jo) - enddo -enddo +if (self%vars%has('x')) self%x = self%x-other%x +if (self%vars%has('q')) self%q = self%q-other%q +if (self%vars%has('u')) self%u = self%u-other%u +if (self%vars%has('v')) self%v = self%v-other%v end subroutine qg_gom_diff ! ------------------------------------------------------------------------------ @@ -267,15 +258,11 @@ subroutine qg_gom_schurmult(self,other) type(qg_gom),intent(inout) :: self !< GOM type(qg_gom),intent(in) :: other !< Other GOM -! Local variables -integer :: jo,jv - -! Add GOM -do jo=1,self%nobs - do jv=1,self%nv - self%values(jv,jo) = self%values(jv,jo)*other%values(jv,jo) - enddo -enddo +! Multiply GOM +if (self%vars%has('x')) self%x = self%x*other%x +if (self%vars%has('q')) self%q = self%q*other%q +if (self%vars%has('u')) self%u = self%u*other%u +if (self%vars%has('v')) self%v = self%v*other%v end subroutine qg_gom_schurmult ! ------------------------------------------------------------------------------ @@ -290,20 +277,41 @@ subroutine qg_gom_divide(self,other) ! Local variables real(kind_real) :: tol -integer :: jloc,jvar +integer :: jloc ! Set tolerance tol = epsilon(tol) ! Conditional division -do jvar=1,self%nv - do jloc=1,self%nobs - if (abs(other%values(jvar,jloc))>tol) then - self%values(jvar,jloc) = self%values(jvar,jloc)/other%values(jvar,jloc) +do jloc=1,self%nobs + if (self%vars%has('x')) then + if (abs(other%x(jloc))>tol) then + self%x(jloc) = self%x(jloc)/other%x(jloc) + else + self%x(jloc) = 0.0 + endif + endif + if (self%vars%has('q')) then + if (abs(other%q(jloc))>tol) then + self%q(jloc) = self%q(jloc)/other%q(jloc) + else + self%q(jloc) = 0.0 + endif + endif + if (self%vars%has('u')) then + if (abs(other%u(jloc))>tol) then + self%u(jloc) = self%u(jloc)/other%u(jloc) + else + self%u(jloc) = 0.0 + endif + endif + if (self%vars%has('v')) then + if (abs(other%v(jloc))>tol) then + self%v(jloc) = self%v(jloc)/other%v(jloc) else - self%values(jvar,jloc) = 0.0 + self%v(jloc) = 0.0 endif - enddo + endif enddo end subroutine qg_gom_divide @@ -318,20 +326,32 @@ subroutine qg_gom_rms(self,rms) real(kind_real),intent(inout) :: rms !< RMS ! Local variables -integer :: jo,jv +integer :: nv ! Initialization rms = 0.0 +nv = 0 ! Loop over values -do jo=1,self%nobs - do jv=1,self%nv - rms = rms+self%values(jv,jo)**2 - enddo -enddo +if (self%vars%has('x')) then + rms = rms+sum(self%x**2) + nv = nv+1 +endif +if (self%vars%has('q')) then + rms = rms+sum(self%q**2) + nv = nv+1 +endif +if (self%vars%has('u')) then + rms = rms+sum(self%u**2) + nv = nv+1 +endif +if (self%vars%has('v')) then + rms = rms+sum(self%v**2) + nv = nv+1 +endif ! Normalize and take square-root -rms = sqrt(rms/(self%nobs*self%nv)) +rms = sqrt(rms/real(self%nobs*nv,kind_real)) end subroutine qg_gom_rms ! ------------------------------------------------------------------------------ @@ -349,17 +369,16 @@ subroutine qg_gom_dotprod(gom1,gom2,prod) integer :: jo,jv ! Check -if ((gom1%nv/=gom2%nv).or.(gom1%nobs/=gom2%nobs)) call abor1_ftn('qg_gom_dotprod: inconsistent GOM sizes') +if (gom1%nobs/=gom2%nobs) call abor1_ftn('qg_gom_dotprod: inconsistent GOM sizes') ! Initialization prod = 0.0 -! Loop over values -do jo=1,gom1%nobs - do jv=1,gom1%nv - prod = prod+gom1%values(jv,jo)*gom2%values(jv,jo) - enddo -enddo +! Dot product +if (gom1%vars%has('x').and.gom2%vars%has('x')) prod = prod+sum(gom1%x*gom2%x) +if (gom1%vars%has('q').and.gom2%vars%has('q')) prod = prod+sum(gom1%q*gom2%q) +if (gom1%vars%has('u').and.gom2%vars%has('u')) prod = prod+sum(gom1%u*gom2%u) +if (gom1%vars%has('v').and.gom2%vars%has('v')) prod = prod+sum(gom1%v*gom2%v) end subroutine qg_gom_dotprod ! ------------------------------------------------------------------------------ @@ -375,12 +394,41 @@ subroutine qg_gom_stats(self,kobs,pmin,pmax,prms) real(kind_real),intent(inout) :: pmax !< Maximum value real(kind_real),intent(inout) :: prms !< RMS +! Local variables +integer :: nv + ! Compute GOM stats kobs = self%nobs -if (self%nobs*self%nv>0) then - pmin = minval(self%values) - pmax = maxval(self%values) - prms = sqrt(sum(self%values**2)/real(self%nobs*self%nv,kind_real)) +if (self%nobs>0) then + pmin = huge(1.0) + pmax = -huge(1.0) + prms = 0.0 + nv = 0 + if (self%vars%has('x')) then + pmin = min(pmin,minval(self%x)) + pmax = max(pmax,maxval(self%x)) + prms = prms+sum(self%x**2) + nv = nv+1 + endif + if (self%vars%has('q')) then + pmin = min(pmin,minval(self%q)) + pmax = max(pmax,maxval(self%q)) + prms = prms+sum(self%q**2) + nv = nv+1 + endif + if (self%vars%has('u')) then + pmin = min(pmin,minval(self%u)) + pmax = max(pmax,maxval(self%u)) + prms = prms+sum(self%u**2) + nv = nv+1 + endif + if (self%vars%has('v')) then + pmin = min(pmin,minval(self%v)) + pmax = max(pmax,maxval(self%v)) + prms = prms+sum(self%v**2) + nv = nv+1 + endif + prms = sqrt(prms/real(self%nobs*nv,kind_real)) else pmin = 0.0 pmax = 0.0 @@ -390,26 +438,62 @@ subroutine qg_gom_stats(self,kobs,pmin,pmax,prms) end subroutine qg_gom_stats ! ------------------------------------------------------------------------------ !> Find and locate GOM max. value -subroutine qg_gom_maxloc(self,mxval,iloc,ivar) +subroutine qg_gom_maxloc(self,mxval,mxloc,mxvar) implicit none ! Passed variables -type(qg_gom),intent(inout) :: self !< GOM -real(kind_real),intent(inout) :: mxval !< Maximum value -integer,intent(inout) :: iloc !< Location of maximum value -integer,intent(inout) :: ivar !< Variable with maximum value +type(qg_gom),intent(inout) :: self !< GOM +real(kind_real),intent(inout) :: mxval !< Maximum value +integer,intent(inout) :: mxloc !< Location of maximum value +type(oops_variables),intent(inout) :: mxvar !< Variable of maximum value ! Local variables -integer :: mxloc(2) +integer :: mxloc_arr(1),mxval_tmp +character(len=1) :: var + +! Initialization +mxval = -huge(1.0) ! Find GOM max. value -mxval = maxval(self%values) -mxloc = maxloc(self%values) +if (self%vars%has('x')) then + mxval_tmp = maxval(self%x) + if (mxval_tmp>mxval) then + mxval = mxval + mxloc_arr = maxloc(self%x) + var = 'x' + endif +endif +if (self%vars%has('q')) then + mxval_tmp = maxval(self%q) + if (mxval_tmp>mxval) then + mxval = mxval + mxloc_arr = maxloc(self%q) + var = 'q' + endif +endif +if (self%vars%has('u')) then + mxval_tmp = maxval(self%u) + if (mxval_tmp>mxval) then + mxval = mxval + mxloc_arr = maxloc(self%u) + var = 'u' + endif +endif +if (self%vars%has('v')) then + mxval_tmp = maxval(self%v) + if (mxval_tmp>mxval) then + mxval = mxval + mxloc_arr = maxloc(self%v) + var = 'v' + endif +endif ! Locate GOM max. value -ivar = mxloc(1) -iloc = mxloc(2) +mxloc = mxloc_arr(1) + +! Set GOM max. variable +call mxvar%push_back(var) end subroutine qg_gom_maxloc ! ------------------------------------------------------------------------------ @@ -423,13 +507,10 @@ subroutine qg_gom_read_file(self,f_conf) type(fckit_configuration),intent(in) :: f_conf !< FCKIT configuration ! Local variables -integer :: ncid,nobs_id,nv_id,values_id +integer :: ncid,nobs_id,nobs,x_id,q_id,u_id,v_id character(len=1024) :: filename character(len=:),allocatable :: str -! Check allocation -if (self%lalloc) call abor1_ftn('qg_gom_read_file: gom alredy allocated') - ! Get filename call f_conf%get_or_die("filename",str) filename = str @@ -438,34 +519,26 @@ subroutine qg_gom_read_file(self,f_conf) ! Open NetCDF file call ncerr(nf90_open(trim(filename)//'.nc',nf90_nowrite,ncid)) -! Get dimensions ids +! Get dimension id call ncerr(nf90_inq_dimid(ncid,'nobs',nobs_id)) -call ncerr(nf90_inq_dimid(ncid,'nv',nv_id)) -! Get dimensions -call ncerr(nf90_inquire_dimension(ncid,nobs_id,len=self%nobs)) -call ncerr(nf90_inquire_dimension(ncid,nv_id,len=self%nv)) +! Get dimension +call ncerr(nf90_inquire_dimension(ncid,nobs_id,len=nobs)) -! Get attributes -call ncerr(nf90_get_att(ncid,nf90_global,'used',self%used)) -call ncerr(nf90_get_att(ncid,nf90_global,'ix',self%ix)) -call ncerr(nf90_get_att(ncid,nf90_global,'iq',self%iq)) -call ncerr(nf90_get_att(ncid,nf90_global,'iu',self%iu)) -call ncerr(nf90_get_att(ncid,nf90_global,'iv',self%iv)) -call ncerr(nf90_get_att(ncid,nf90_global,'nv',self%nv)) - -! Allocation -allocate(self%values(self%nv,self%nobs)) -self%lalloc = .true. - -! Initialization -call qg_gom_zero(self) +! GOM setup +call qg_gom_setup(self,nobs) ! Get variables ids -call ncerr(nf90_inq_varid(ncid,'values',values_id)) +if (self%vars%has('x')) call ncerr(nf90_inq_varid(ncid,'x',x_id)) +if (self%vars%has('q')) call ncerr(nf90_inq_varid(ncid,'q',q_id)) +if (self%vars%has('u')) call ncerr(nf90_inq_varid(ncid,'u',u_id)) +if (self%vars%has('v')) call ncerr(nf90_inq_varid(ncid,'v',v_id)) ! Get variables -call ncerr(nf90_get_var(ncid,values_id,self%values)) +if (self%vars%has('x')) call ncerr(nf90_get_var(ncid,x_id,self%x)) +if (self%vars%has('q')) call ncerr(nf90_get_var(ncid,q_id,self%q)) +if (self%vars%has('u')) call ncerr(nf90_get_var(ncid,u_id,self%u)) +if (self%vars%has('v')) call ncerr(nf90_get_var(ncid,v_id,self%v)) ! Close NetCDF file call ncerr(nf90_close(ncid)) @@ -482,7 +555,7 @@ subroutine qg_gom_write_file(self,f_conf) type(fckit_configuration),intent(in) :: f_conf !< FCKIT configuration ! Local variables -integer :: ncid,nobs_id,nv_id,values_id +integer :: ncid,nobs_id,x_id,q_id,u_id,v_id character(len=1024) :: filename character(len=:),allocatable :: str @@ -499,24 +572,21 @@ subroutine qg_gom_write_file(self,f_conf) ! Define dimensions call ncerr(nf90_def_dim(ncid,'nobs',self%nobs,nobs_id)) -call ncerr(nf90_def_dim(ncid,'nv',self%nv,nv_id)) - -! Define attributes -call ncerr(nf90_put_att(ncid,nf90_global,'used',self%used)) -call ncerr(nf90_put_att(ncid,nf90_global,'ix',self%ix)) -call ncerr(nf90_put_att(ncid,nf90_global,'iq',self%iq)) -call ncerr(nf90_put_att(ncid,nf90_global,'iu',self%iu)) -call ncerr(nf90_put_att(ncid,nf90_global,'iv',self%iv)) -call ncerr(nf90_put_att(ncid,nf90_global,'nv',self%nv)) ! Define variables -call ncerr(nf90_def_var(ncid,'values',nf90_double,(/nv_id,nobs_id/),values_id)) +if (self%vars%has('x')) call ncerr(nf90_def_var(ncid,'x',nf90_double,(/nobs_id/),x_id)) +if (self%vars%has('q')) call ncerr(nf90_def_var(ncid,'q',nf90_double,(/nobs_id/),q_id)) +if (self%vars%has('u')) call ncerr(nf90_def_var(ncid,'u',nf90_double,(/nobs_id/),u_id)) +if (self%vars%has('v')) call ncerr(nf90_def_var(ncid,'v',nf90_double,(/nobs_id/),v_id)) ! End definitions call ncerr(nf90_enddef(ncid)) ! Put variables -call ncerr(nf90_put_var(ncid,values_id,self%values)) +if (self%vars%has('x')) call ncerr(nf90_put_var(ncid,x_id,self%x)) +if (self%vars%has('q')) call ncerr(nf90_put_var(ncid,q_id,self%q)) +if (self%vars%has('u')) call ncerr(nf90_put_var(ncid,u_id,self%u)) +if (self%vars%has('v')) call ncerr(nf90_put_var(ncid,v_id,self%v)) ! Close NetCDF file call ncerr(nf90_close(ncid)) @@ -549,7 +619,7 @@ subroutine qg_gom_analytic_init(self,locs,f_conf) call z_field%data(z) ! Check allocation -if (.not. self%lalloc) call abor1_ftn('qg_gom_analytic init: gom not allocated') +if (.not.self%lalloc) call abor1_ftn('qg_gom_analytic init: gom not allocated') ! Get analytic configuration call f_conf%get_or_die("analytic_init",str) @@ -562,19 +632,19 @@ subroutine qg_gom_analytic_init(self,locs,f_conf) call lonlat_to_xy(lonlat(1,iloc),lonlat(2,iloc),x,y) ! Compute values for baroclinic instability - if (self%ix>0) call baroclinic_instability(x,y,z(iloc),'x',self%values(self%ix,iloc)) - if (self%iq>0) call baroclinic_instability(x,y,z(iloc),'q',self%values(self%iq,iloc)) - if (self%iu>0) call baroclinic_instability(x,y,z(iloc),'u',self%values(self%iu,iloc)) - if (self%iv>0) call baroclinic_instability(x,y,z(iloc),'v',self%values(self%iv,iloc)) + if (self%vars%has('x')) call baroclinic_instability(x,y,z(iloc),'x',self%x(iloc)) + if (self%vars%has('q')) call baroclinic_instability(x,y,z(iloc),'q',self%q(iloc)) + if (self%vars%has('u')) call baroclinic_instability(x,y,z(iloc),'u',self%u(iloc)) + if (self%vars%has('v')) call baroclinic_instability(x,y,z(iloc),'v',self%v(iloc)) case ('large-vortices') ! Go to cartesian coordinates call lonlat_to_xy(lonlat(1,iloc),lonlat(2,iloc),x,y) ! Compute values for large vortices - if (self%ix>0) call large_vortices(x,y,z(iloc),'x',self%values(self%ix,iloc)) - if (self%iq>0) call large_vortices(x,y,z(iloc),'q',self%values(self%iq,iloc)) - if (self%iu>0) call large_vortices(x,y,z(iloc),'u',self%values(self%iu,iloc)) - if (self%iv>0) call large_vortices(x,y,z(iloc),'v',self%values(self%iv,iloc)) + if (self%vars%has('x')) call large_vortices(x,y,z(iloc),'x',self%x(iloc)) + if (self%vars%has('q')) call large_vortices(x,y,z(iloc),'q',self%q(iloc)) + if (self%vars%has('u')) call large_vortices(x,y,z(iloc),'u',self%u(iloc)) + if (self%vars%has('v')) call large_vortices(x,y,z(iloc),'v',self%v(iloc)) case default call abor1_ftn('qg_gom_analytic_init: unknown initialization') endselect diff --git a/qg/model/qg_interp_mod.F90 b/qg/model/qg_interp_mod.F90 index 63044b8b2..e1f6f2e6f 100644 --- a/qg/model/qg_interp_mod.F90 +++ b/qg/model/qg_interp_mod.F90 @@ -25,15 +25,15 @@ module qg_interp_mod contains ! ------------------------------------------------------------------------------ !> Trilinear interpolation -subroutine qg_interp_trilinear(geom,lon,lat,z,gfld3d,val) +subroutine qg_interp_trilinear(geom,lon,lat,z,field,val) ! Passed variables -type(qg_geom),intent(in) :: geom !< Geometry -real(kind_real),intent(in) :: lon !< Longitude -real(kind_real),intent(in) :: lat !< Latitude -real(kind_real),intent(in) :: z !< Altitude -real(kind_real),intent(in) :: gfld3d(geom%nx,geom%ny,geom%nz) !< 3D Field -real(kind_real),intent(out) :: val !< Value +type(qg_geom),intent(in) :: geom !< Geometry +real(kind_real),intent(in) :: lon !< Longitude +real(kind_real),intent(in) :: lat !< Latitude +real(kind_real),intent(in) :: z !< Altitude +real(kind_real),intent(in) :: field(geom%nx,geom%ny,geom%nz) !< Field +real(kind_real),intent(out) :: val !< Value ! Local variables integer :: jxm1,jxo,jxp1,jxp2 @@ -64,10 +64,10 @@ subroutine qg_interp_trilinear(geom,lon,lat,z,gfld3d,val) endif ! Interpolate along x -oo = (1.0-ax)*gfld3d(jxo,jyo,jzo)+ax*gfld3d(jxp1,jyo,jzo) -op1 = (1.0-ax)*gfld3d(jxo,jyo,jzp1)+ax*gfld3d(jxp1,jyo,jzp1) -p1o = (1.0-ax)*gfld3d(jxo,jyp1,jzo)+ax*gfld3d(jxp1,jyp1,jzo) -p1p1 = (1.0-ax)*gfld3d(jxo,jyp1,jzp1)+ax*gfld3d(jxp1,jyp1,jzp1) +oo = (1.0-ax)*field(jxo,jyo,jzo)+ax*field(jxp1,jyo,jzo) +op1 = (1.0-ax)*field(jxo,jyo,jzp1)+ax*field(jxp1,jyo,jzp1) +p1o = (1.0-ax)*field(jxo,jyp1,jzo)+ax*field(jxp1,jyp1,jzo) +p1p1 = (1.0-ax)*field(jxo,jyp1,jzp1)+ax*field(jxp1,jyp1,jzp1) ! Interpolate along y o = (1.0-ay)*oo+ay*p1o @@ -79,15 +79,15 @@ subroutine qg_interp_trilinear(geom,lon,lat,z,gfld3d,val) end subroutine qg_interp_trilinear ! ------------------------------------------------------------------------------ !> Interpolation - adjoint -subroutine qg_interp_trilinear_ad(geom,lon,lat,z,val,gfld3d) +subroutine qg_interp_trilinear_ad(geom,lon,lat,z,val,field) ! Passed variables -type(qg_geom),intent(in) :: geom !< Geometry -real(kind_real),intent(in) :: lon !< Longitude -real(kind_real),intent(in) :: lat !< Latitude -real(kind_real),intent(in) :: z !< Altitude -real(kind_real),intent(in) :: val !< Value -real(kind_real),intent(inout) :: gfld3d(geom%nx,geom%ny,geom%nz) !< 3D field +type(qg_geom),intent(in) :: geom !< Geometry +real(kind_real),intent(in) :: lon !< Longitude +real(kind_real),intent(in) :: lat !< Latitude +real(kind_real),intent(in) :: z !< Altitude +real(kind_real),intent(in) :: val !< Value +real(kind_real),intent(inout) :: field(geom%nx,geom%ny,geom%nz) !< Field ! Local variables integer :: jxm1,jxo,jxp1,jxp2 @@ -128,14 +128,14 @@ subroutine qg_interp_trilinear_ad(geom,lon,lat,z,val,gfld3d) p1p1 = ay*p1 ! Interpolate along x -gfld3d(jxo,jyo,jzo) = gfld3d(jxo,jyo,jzo)+(1.0-ax)*oo -gfld3d(jxp1,jyo,jzo) = gfld3d(jxp1,jyo,jzo)+ax*oo -gfld3d(jxo,jyo,jzp1) = gfld3d(jxo,jyo,jzp1)+(1.0-ax)*op1 -gfld3d(jxp1,jyo,jzp1) = gfld3d(jxp1,jyo,jzp1)+ax*op1 -gfld3d(jxo,jyp1,jzo) = gfld3d(jxo,jyp1,jzo)+(1.0-ax)*p1o -gfld3d(jxp1,jyp1,jzo) = gfld3d(jxp1,jyp1,jzo)+ax*p1o -gfld3d(jxo,jyp1,jzp1) = gfld3d(jxo,jyp1,jzp1)+(1.0-ax)*p1p1 -gfld3d(jxp1,jyp1,jzp1) = gfld3d(jxp1,jyp1,jzp1)+ax*p1p1 +field(jxo,jyo,jzo) = field(jxo,jyo,jzo)+(1.0-ax)*oo +field(jxp1,jyo,jzo) = field(jxp1,jyo,jzo)+ax*oo +field(jxo,jyo,jzp1) = field(jxo,jyo,jzp1)+(1.0-ax)*op1 +field(jxp1,jyo,jzp1) = field(jxp1,jyo,jzp1)+ax*op1 +field(jxo,jyp1,jzo) = field(jxo,jyp1,jzo)+(1.0-ax)*p1o +field(jxp1,jyp1,jzo) = field(jxp1,jyp1,jzo)+ax*p1o +field(jxo,jyp1,jzp1) = field(jxo,jyp1,jzp1)+(1.0-ax)*p1p1 +field(jxp1,jyp1,jzp1) = field(jxp1,jyp1,jzp1)+ax*p1p1 end subroutine qg_interp_trilinear_ad ! ------------------------------------------------------------------------------ diff --git a/qg/model/qg_model_mod.F90 b/qg/model/qg_model_mod.F90 index 5e8051ab0..9323d9940 100644 --- a/qg/model/qg_model_mod.F90 +++ b/qg/model/qg_model_mod.F90 @@ -17,7 +17,8 @@ module qg_model_mod use qg_constants_mod use qg_convert_q_to_x_mod use qg_convert_x_to_q_mod -use qg_convert_x_to_uv_mod +use qg_convert_x_to_u_mod +use qg_convert_x_to_v_mod use qg_fields_mod use random_mod @@ -82,38 +83,31 @@ subroutine qg_model_propagate(conf,fld) type(qg_fields),intent(inout) :: fld !< State fields ! Local variables -real(kind_real) :: x(fld%geom%nx,fld%geom%ny,fld%geom%nz),q(fld%geom%nx,fld%geom%ny,fld%geom%nz) +real(kind_real) :: q(fld%geom%nx,fld%geom%ny,fld%geom%nz),qnew(fld%geom%nx,fld%geom%ny,fld%geom%nz) real(kind_real) :: u(fld%geom%nx,fld%geom%ny,fld%geom%nz),v(fld%geom%nx,fld%geom%ny,fld%geom%nz) -real(kind_real) :: qnew(fld%geom%nx,fld%geom%ny,fld%geom%nz) -! Initialize streamfunction and potential vorticity -if (fld%lq) then - q = fld%gfld3d - call convert_q_to_x(fld%geom,q,fld%x_north,fld%x_south,x) -else - x = fld%gfld3d - call convert_x_to_q(fld%geom,x,fld%x_north,fld%x_south,q) -endif +! Check input +if (.not.allocated(fld%x)) call abor1_ftn('qg_model_propagate: x required') +if (.not.allocated(fld%x_north)) call abor1_ftn('qg_model_propagate: x_north required') +if (.not.allocated(fld%x_south)) call abor1_ftn('qg_model_propagate: x_south required') +if (.not.allocated(fld%q_north)) call abor1_ftn('qg_model_propagate: q_north required') +if (.not.allocated(fld%q_south)) call abor1_ftn('qg_model_propagate: q_south required') -! Initialize wind -call convert_x_to_uv(fld%geom,x,fld%x_north,fld%x_south,u,v) +! Compute potential vorticity +call convert_x_to_q(fld%geom,fld%x,fld%x_north,fld%x_south,q) + +! Compute wind +call convert_x_to_u(fld%geom,fld%x,fld%x_north,fld%x_south,u) +call convert_x_to_v(fld%geom,fld%x,v) ! Advect potential vorticity call advect_q(fld%geom,conf%dt,u,v,q,fld%q_north,fld%q_south,qnew) -! Save streamfunction or potential vorticity -if (fld%lq) then - fld%gfld3d = qnew -else - call convert_q_to_x(fld%geom,qnew,fld%x_north,fld%x_south,x) - fld%gfld3d = x -endif +! Compute streamfunction +call convert_q_to_x(fld%geom,qnew,fld%x_north,fld%x_south,fld%x) -if (fld%lq) then - if (allocated(fld%pv)) fld%pv = fld%gfld3d -else - if (allocated(fld%streamfct)) fld%streamfct = fld%gfld3d -endif +! Complete other fields +call qg_fields_complete(fld,'x') end subroutine qg_model_propagate ! ------------------------------------------------------------------------------ @@ -128,50 +122,47 @@ subroutine qg_model_propagate_tl(conf,traj,fld) type(qg_fields),intent(inout) :: fld !< Increment fields ! Local variables -real(kind_real) :: x_traj(fld%geom%nx,fld%geom%ny,fld%geom%nz),q_traj(fld%geom%nx,fld%geom%ny,fld%geom%nz) +real(kind_real) :: q_traj(fld%geom%nx,fld%geom%ny,fld%geom%nz) real(kind_real) :: u_traj(fld%geom%nx,fld%geom%ny,fld%geom%nz),v_traj(fld%geom%nx,fld%geom%ny,fld%geom%nz) -real(kind_real) :: x(fld%geom%nx,fld%geom%ny,fld%geom%nz),q(fld%geom%nx,fld%geom%ny,fld%geom%nz) +real(kind_real) :: q(fld%geom%nx,fld%geom%ny,fld%geom%nz),qnew(fld%geom%nx,fld%geom%ny,fld%geom%nz) real(kind_real) :: u(fld%geom%nx,fld%geom%ny,fld%geom%nz),v(fld%geom%nx,fld%geom%ny,fld%geom%nz) -real(kind_real) :: qnew(fld%geom%nx,fld%geom%ny,fld%geom%nz) ! Trajectory -! Initialize streamfunction and potential vorticity -if (traj%lq) then - q_traj = traj%gfld3d - call convert_q_to_x(fld%geom,q_traj,traj%x_north,traj%x_south,x_traj) -else - x_traj = traj%gfld3d - call convert_x_to_q(fld%geom,x_traj,traj%x_north,traj%x_south,q_traj) -endif +! Check input +if (.not.allocated(traj%x)) call abor1_ftn('qg_model_propagate_tl: x trajectory required') +if (.not.allocated(traj%x_north)) call abor1_ftn('qg_model_propagate_tl: x_north required') +if (.not.allocated(traj%x_south)) call abor1_ftn('qg_model_propagate_tl: x_south required') +if (.not.allocated(traj%q_north)) call abor1_ftn('qg_model_propagate_tl: q_north required') +if (.not.allocated(traj%q_south)) call abor1_ftn('qg_model_propagate_tl: q_south required') + +! Compute potential vorticity +call convert_x_to_q(traj%geom,traj%x,traj%x_north,traj%x_south,q_traj) ! Compute wind -call convert_x_to_uv(fld%geom,x_traj,traj%x_north,traj%x_south,u_traj,v_traj) +call convert_x_to_u(traj%geom,traj%x,traj%x_north,traj%x_south,u_traj) +call convert_x_to_v(traj%geom,traj%x,v_traj) ! Perturbation -! Initialize streamfunction and potential vorticity -if (fld%lq) then - q = fld%gfld3d - call convert_q_to_x_tl(fld%geom,q,x) -else - x = fld%gfld3d - call convert_x_to_q_tl(fld%geom,x,q) -endif +! Check input +if (.not.allocated(fld%x)) call abor1_ftn('qg_model_propagate_tl: x perturbation required') + +! Compute potential vorticity +call convert_x_to_q_tl(fld%geom,fld%x,q) ! Compute wind -call convert_x_to_uv_tl(fld%geom,x,u,v) +call convert_x_to_u_tl(fld%geom,fld%x,u) +call convert_x_to_v_tl(fld%geom,fld%x,v) -! Advect PV +! Advect potential vorticity call advect_q_tl(fld%geom,conf%dt,u_traj,v_traj,q_traj,traj%q_north,traj%q_south,u,v,q,qnew) -! Save streamfunction or potential vorticity -if (fld%lq) then - fld%gfld3d = qnew -else - call convert_q_to_x_tl(fld%geom,qnew,x) - fld%gfld3d = x -endif +! Compute streamfunction +call convert_q_to_x_tl(fld%geom,qnew,fld%x) + +! Complete other fields +call qg_fields_complete(fld,'x') end subroutine qg_model_propagate_tl ! ------------------------------------------------------------------------------ @@ -186,58 +177,56 @@ subroutine qg_model_propagate_ad(conf,traj,fld) type(qg_fields),intent(inout) :: fld !< Increment fields ! Local variables -real(kind_real) :: x_traj(fld%geom%nx,fld%geom%ny,fld%geom%nz),q_traj(fld%geom%nx,fld%geom%ny,fld%geom%nz) +real(kind_real) :: q_traj(fld%geom%nx,fld%geom%ny,fld%geom%nz) real(kind_real) :: u_traj(fld%geom%nx,fld%geom%ny,fld%geom%nz),v_traj(fld%geom%nx,fld%geom%ny,fld%geom%nz) -real(kind_real) :: x(fld%geom%nx,fld%geom%ny,fld%geom%nz),q(fld%geom%nx,fld%geom%ny,fld%geom%nz) +real(kind_real) :: q(fld%geom%nx,fld%geom%ny,fld%geom%nz),qnew(fld%geom%nx,fld%geom%ny,fld%geom%nz) real(kind_real) :: u(fld%geom%nx,fld%geom%ny,fld%geom%nz),v(fld%geom%nx,fld%geom%ny,fld%geom%nz) -real(kind_real) :: qnew(fld%geom%nx,fld%geom%ny,fld%geom%nz) ! Trajectory -! Initialize streamfunction and potential vorticity -if (traj%lq) then - q_traj = traj%gfld3d - call convert_q_to_x(fld%geom,q_traj,traj%x_north,traj%x_south,x_traj) -else - x_traj = traj%gfld3d - call convert_x_to_q(fld%geom,x_traj,traj%x_north,traj%x_south,q_traj) -endif +! Check input +if (.not.allocated(traj%x)) call abor1_ftn('qg_model_propagate_tl: x trajectory required') +if (.not.allocated(traj%x_north)) call abor1_ftn('qg_model_propagate_tl: x_north required') +if (.not.allocated(traj%x_south)) call abor1_ftn('qg_model_propagate_tl: x_south required') +if (.not.allocated(traj%q_north)) call abor1_ftn('qg_model_propagate_tl: q_north required') +if (.not.allocated(traj%q_south)) call abor1_ftn('qg_model_propagate_tl: q_south required') + +! Compute potential vorticity +call convert_x_to_q(traj%geom,traj%x,traj%x_north,traj%x_south,q_traj) ! Compute wind -call convert_x_to_uv(fld%geom,x_traj,traj%x_north,traj%x_south,u_traj,v_traj) +call convert_x_to_u(traj%geom,traj%x,traj%x_north,traj%x_south,u_traj) +call convert_x_to_v(traj%geom,traj%x,v_traj) ! Perturbation +! Check input +if (.not.allocated(fld%x)) call abor1_ftn('qg_model_propagate_tl: x perturbation required') + ! Initialization -x = 0.0 -q = 0.0 -u = 0.0 -v = 0.0 -qnew = 0.0 - -! Save streamfunction or potential vorticity -if (fld%lq) then - qnew = fld%gfld3d -else - x = fld%gfld3d - call convert_q_to_x_ad(fld%geom,x,qnew) -endif - -! Advect PV +u = 0.0_kind_real +v = 0.0_kind_real +q = 0.0_kind_real +qnew = 0.0_kind_real + +! Compute streamfunction +call convert_q_to_x_ad(fld%geom,fld%x,qnew) + +! Advect potential vorticity call advect_q_ad(fld%geom,conf%dt,u_traj,v_traj,q_traj,traj%q_north,traj%q_south,qnew,u,v,q) +! Initialize x +fld%x = 0.0_kind_real + ! Compute wind -x = 0.0 -call convert_x_to_uv_ad(fld%geom,u,v,x) - -! Initialize streamfunction and potential vorticity -if (fld%lq) then - call convert_q_to_x_ad(fld%geom,x,q) - fld%gfld3d = q -else - call convert_x_to_q_ad(fld%geom,q,x) - fld%gfld3d = x -endif +call convert_x_to_v_ad(fld%geom,v,fld%x) +call convert_x_to_u_ad(fld%geom,u,fld%x) + +! Compute potential vorticity +call convert_x_to_q_ad(fld%geom,q,fld%x) + +! Complete other fields +call qg_fields_complete(fld,'x') end subroutine qg_model_propagate_ad ! ------------------------------------------------------------------------------ diff --git a/qg/model/qg_stream_mod.F90 b/qg/model/qg_stream_mod.F90 index 73219d77e..f812f815b 100644 --- a/qg/model/qg_stream_mod.F90 +++ b/qg/model/qg_stream_mod.F90 @@ -36,7 +36,7 @@ subroutine qg_stream_equiv(gom,hofx,bias) ! Loop over observations do iobs=1,gom%nobs - hofx%values(1,iobs) = gom%values(1,iobs)+bias + hofx%values(1,iobs) = gom%x(iobs)+bias enddo end subroutine qg_stream_equiv @@ -47,16 +47,16 @@ subroutine qg_stream_equiv_ad(gom,hofx,bias) implicit none ! Passed variables -type(qg_gom),intent(inout) :: gom !< GOM -type(qg_obsvec),intent(in) :: hofx !< Observation vector -real(kind_real),intent(inout) :: bias !< Bias +type(qg_gom),intent(inout) :: gom !< GOM +type(qg_obsvec),intent(in) :: hofx !< Observation vector +real(kind_real),intent(inout) :: bias !< Bias ! Local variables integer :: iobs ! Loop over observations do iobs=1,gom%nobs - gom%values(1,iobs) = hofx%values(1,iobs) + gom%x(iobs) = hofx%values(1,iobs) bias = bias+hofx%values(1,iobs) enddo diff --git a/qg/model/qg_wind_mod.F90 b/qg/model/qg_wind_mod.F90 index f18f1aea3..cbebdba35 100644 --- a/qg/model/qg_wind_mod.F90 +++ b/qg/model/qg_wind_mod.F90 @@ -37,7 +37,8 @@ subroutine qg_wind_equiv(gom,hofx,bias) ! Loop over observations do iobs=1,gom%nobs - hofx%values(1:2,iobs) = gom%values(1:2,iobs)+bias + hofx%values(1,iobs) = gom%u(iobs)+bias(1) + hofx%values(2,iobs) = gom%v(iobs)+bias(2) enddo end subroutine qg_wind_equiv @@ -57,7 +58,8 @@ subroutine qg_wind_equiv_ad(gom,hofx,bias) ! Loop over observations do iobs=1,gom%nobs - gom%values(1:2,iobs) = hofx%values(1:2,iobs) + gom%u(iobs) = hofx%values(1,iobs) + gom%v(iobs) = hofx%values(2,iobs) bias(1) = bias(1)+hofx%values(1,iobs) bias(2) = bias(2)+hofx%values(2,iobs) enddo diff --git a/qg/model/qg_wspeed_interface.F90 b/qg/model/qg_wspeed_interface.F90 index be516a67c..8d536bb2e 100644 --- a/qg/model/qg_wspeed_interface.F90 +++ b/qg/model/qg_wspeed_interface.F90 @@ -100,22 +100,21 @@ subroutine qg_wspeed_gettraj_c(c_nobs,c_vars,c_key_traj) bind(c,name='qg_wspeed_ implicit none ! Passed variables -integer(c_int),intent(in) :: c_nobs !< Number of observations -type(c_ptr),value,intent(in) :: c_vars !< Variables -integer(c_int),intent(inout) :: c_key_traj !< GOM trajectory +integer(c_int),intent(in) :: c_nobs !< Number of observations +type(c_ptr),value,intent(in) :: c_vars !< Variables +integer(c_int),intent(inout) :: c_key_traj !< GOM trajectory ! Local variables -type(oops_variables) :: vars type(qg_gom),pointer :: traj ! Interface call qg_gom_registry%init() call qg_gom_registry%add(c_key_traj) call qg_gom_registry%get(c_key_traj,traj) -vars = oops_variables(c_vars) +traj%vars = oops_variables(c_vars) ! Call Fortran -call qg_gom_setup(traj,c_nobs,vars) +call qg_gom_setup(traj,c_nobs) end subroutine qg_wspeed_gettraj_c ! ------------------------------------------------------------------------------ diff --git a/qg/model/qg_wspeed_mod.F90 b/qg/model/qg_wspeed_mod.F90 index 000c5ff04..9325349a4 100644 --- a/qg/model/qg_wspeed_mod.F90 +++ b/qg/model/qg_wspeed_mod.F90 @@ -42,7 +42,7 @@ subroutine qg_wspeed_equiv(gom,hofx,bias) ! Loop over observations do iobs=1,gom%nobs - hofx%values(1,iobs) = sqrt(gom%values(1,iobs)*gom%values(1,iobs)+gom%values(2,iobs)*gom%values(2,iobs)) + hofx%values(1,iobs) = sqrt(gom%u(iobs)*gom%u(iobs)+gom%v(iobs)*gom%v(iobs)) enddo end subroutine qg_wspeed_equiv @@ -64,11 +64,11 @@ subroutine qg_wspeed_equiv_tl(gom,hofx,traj,bias) ! Loop over observations do iobs=1,gom%nobs - zu = traj%values(1,iobs) - zv = traj%values(2,iobs) + zu = traj%u(iobs) + zv = traj%v(iobs) zt = sqrt(zu**2+zv**2) if (zt>epsilon(zt)) then - hofx%values(1,iobs) = (zu*gom%values(1,iobs)+zv*gom%values(2,iobs))/zt + hofx%values(1,iobs) = (zu*gom%u(iobs)+zv*gom%v(iobs))/zt else hofx%values(1,iobs) = 0.0 endif @@ -93,15 +93,15 @@ subroutine qg_wspeed_equiv_ad(gom,hofx,traj,bias) ! Loop over observations do iobs=1,gom%nobs - zu = traj%values(1,iobs) - zv = traj%values(2,iobs) + zu = traj%u(iobs) + zv = traj%v(iobs) zt = sqrt(zu**2+zv**2) if (zt>epsilon(zt)) then - gom%values(1,iobs) = zu*hofx%values(1,iobs)/zt - gom%values(2,iobs) = zv*hofx%values(1,iobs)/zt + gom%u(iobs) = zu*hofx%values(1,iobs)/zt + gom%v(iobs) = zv*hofx%values(1,iobs)/zt else - gom%values(1,iobs) = 0.0 - gom%values(2,iobs) = 0.0 + gom%u(iobs) = 0.0 + gom%v(iobs) = 0.0 endif enddo @@ -121,9 +121,9 @@ subroutine qg_wspeed_settraj(gom,traj) ! Loop over observations do iobs=1,gom%nobs - traj%values(1,iobs) = gom%values(1,iobs) - traj%values(2,iobs) = gom%values(2,iobs) -enddo + traj%u(iobs) = gom%u(iobs) + traj%v(iobs) = gom%v(iobs) +end do end subroutine qg_wspeed_settraj ! ------------------------------------------------------------------------------ diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index 7623747ae..f606acd91 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -7,7 +7,6 @@ list( APPEND qg_testinput testinput/3dfgat.yaml testinput/4densvar.yaml testinput/4densvar_hybrid.yaml - testinput/4dvar_change_var.yaml testinput/4dvar_dripcg.yaml testinput/4dvar_drpcg_lmp.yaml testinput/4dvar_drpfom.yaml @@ -90,7 +89,6 @@ list( APPEND qg_testoutput testoutput/3dfgat.test testoutput/4densvar.test testoutput/4densvar_hybrid.test - testoutput/4dvar_change_var.test testoutput/4dvar_dripcg.test testoutput/4dvar_drpcg_lmp.test testoutput/4dvar_drpfom.test @@ -588,15 +586,14 @@ oops_add_test( TESTNAME 3dvar EXENAME qg_4dvar.x TEST_DEPENDS test_qg_forecast test_qg_make_obs_3d ) +#-------------------------------------------------------------------- + oops_add_test( TESTNAME 3dvar_change_var MODELNAME qg OMP 2 YAMLNAME testinput/3dvar_change_var.yaml EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_3d - COMPARE) - -#-------------------------------------------------------------------- + TEST_DEPENDS test_qg_forecast test_qg_make_obs_3d ) oops_add_test( TESTNAME 3dvar_hybrid MODELNAME qg @@ -637,13 +634,6 @@ oops_add_test( TESTNAME 4densvar_hybrid EXENAME qg_4dvar.x TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_4d_12h ) -oops_add_test( TESTNAME 4dvar_change_var - MODELNAME qg - OMP 2 - YAMLNAME testinput/4dvar_change_var.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) - oops_add_test( TESTNAME 4dvar_dripcg MODELNAME qg OMP 2 diff --git a/qg/test/testinput/4dvar_change_var.yaml b/qg/test/testinput/4dvar_change_var.yaml deleted file mode 100644 index 70210eddd..000000000 --- a/qg/test/testinput/4dvar_change_var.yaml +++ /dev/null @@ -1,111 +0,0 @@ -cost function: - cost type: 4D-Var - window begin: 2010-01-01T00:00:00Z - window length: PT24H - analysis variables: [q] - background: - date: 2010-01-01T00:00:00Z - filename: Data/forecast.fc.2009-12-31T00:00:00Z.P1D.nc - state variables: [q] - background error: - covariance model: QgError - horizontal_length_scale: 2.2e6 - maximum_condition_number: 1.0e6 - standard_deviation: 1.8e7 - variable changes: - - variable change: ChVarQG - input variables: [x] - output variables: [q] - vertical_length_scale: 15000.0 - observations: - - obs operator: - obs type: Stream - obs space: - obsdatain: - obsfile: Data/truth.obs4d_24h.nc - obsdataout: - obsfile: Data/4dvar_change_var.obs4d_24h.nc - obs type: Stream - obs error: - covariance model: diagonal - - obs operator: - obs type: Wind - obs space: - obsdatain: - obsfile: Data/truth.obs4d_24h.nc - obsdataout: - obsfile: Data/4dvar_change_var.obs4d_24h.nc - obs type: Wind - obs error: - covariance model: diagonal - - obs operator: - obs type: WSpeed - obs space: - obsdatain: - obsfile: Data/truth.obs4d_24h.nc - obsdataout: - obsfile: Data/4dvar_change_var.obs4d_24h.nc - obs type: WSpeed - obs error: - covariance model: diagonal - constraints: - - jcdfi: - alpha: 5.0e9 - cutoff: PT3H - type: DolphChebyshev - filtered variables: ["q"] - geometry: - nx: 40 - ny: 20 - depths: [4500.0, 5500.0] - model: - name: QG - use potential vorticity: true - tstep: PT1H -variational: - minimizer: - algorithm: DRIPCG - iterations: - - ninner: 10 - gradient norm reduction: 1.0e-10 - geometry: - nx: 40 - ny: 20 - depths: [4500.0, 5500.0] - linear model: - name: QgTLM - trajectory: - tstep: PT1H - tstep: PT1H - variable change: Identity - tlm variables: ["q"] - diagnostics: - departures: ombg - test: on - - ninner: 10 - gradient norm reduction: 1.0e-10 - geometry: - nx: 40 - ny: 20 - depths: [4500.0, 5500.0] - linear model: - name: QgTLM - trajectory: - tstep: PT1H - tstep: PT1H - variable change: Identity - tlm variables: ["q"] - diagnostics: - departures: ombg - test: on -final: - diagnostics: - departures: oman - prints: - frequency: PT1H -output: - datadir: Data - exp: 4dvar_change_var - first: PT0S - frequency: PT6H - type: an diff --git a/qg/test/testinput/dirac_cov.yaml b/qg/test/testinput/dirac_cov.yaml index 32be6d4a0..ad1a22d48 100644 --- a/qg/test/testinput/dirac_cov.yaml +++ b/qg/test/testinput/dirac_cov.yaml @@ -10,6 +10,7 @@ dirac: ixdir: [20] iydir: [10] izdir: [1] + var: x geometry: nx: 40 ny: 20 diff --git a/qg/test/testinput/dirac_hyb_field.yaml b/qg/test/testinput/dirac_hyb_field.yaml index 87d86adc2..9c98f6682 100644 --- a/qg/test/testinput/dirac_hyb_field.yaml +++ b/qg/test/testinput/dirac_hyb_field.yaml @@ -47,6 +47,7 @@ dirac: ixdir: [20] iydir: [10] izdir: [1] + var: x geometry: nx: 40 ny: 20 diff --git a/qg/test/testinput/dirac_hyb_value.yaml b/qg/test/testinput/dirac_hyb_value.yaml index fe7371b57..fde8268a3 100644 --- a/qg/test/testinput/dirac_hyb_value.yaml +++ b/qg/test/testinput/dirac_hyb_value.yaml @@ -45,6 +45,7 @@ dirac: ixdir: [20] iydir: [10] izdir: [1] + var: x geometry: nx: 40 ny: 20 diff --git a/qg/test/testinput/dirac_loc_3d.yaml b/qg/test/testinput/dirac_loc_3d.yaml index 5147512d4..28cfdc7c9 100644 --- a/qg/test/testinput/dirac_loc_3d.yaml +++ b/qg/test/testinput/dirac_loc_3d.yaml @@ -32,6 +32,7 @@ dirac: ixdir: [20] iydir: [10] izdir: [1] + var: x geometry: nx: 40 ny: 20 diff --git a/qg/test/testinput/dirac_loc_4d.yaml b/qg/test/testinput/dirac_loc_4d.yaml index bda56e2e3..7bfc527c2 100644 --- a/qg/test/testinput/dirac_loc_4d.yaml +++ b/qg/test/testinput/dirac_loc_4d.yaml @@ -282,6 +282,7 @@ dirac: ixdir: [20] iydir: [10] izdir: [1] + var: x geometry: nx: 40 ny: 20 diff --git a/qg/test/testinput/dirac_no_loc.yaml b/qg/test/testinput/dirac_no_loc.yaml index bab5ad23e..54e01be2c 100644 --- a/qg/test/testinput/dirac_no_loc.yaml +++ b/qg/test/testinput/dirac_no_loc.yaml @@ -26,6 +26,7 @@ dirac: ixdir: [20] iydir: [10] izdir: [1] + var: x geometry: nx: 40 ny: 20 diff --git a/qg/test/testoutput/3densvar.test b/qg/test/testoutput/3densvar.test index 9010c7554..27e6ba906 100644 --- a/qg/test/testoutput/3densvar.test +++ b/qg/test/testoutput/3densvar.test @@ -7,11 +7,9 @@ Test : DRIPCGMinimizer: reduction in residual norm = 0.0780493 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4489e+08, Max= 9.7006e+07, RMS= 1.7680e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4489e+08, Max= 9.7006e+07, RMS= 1.7680e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 50.19 Test : CostJo : Nonlinear Jo(Stream) = 439.3, nobs = 600, Jo/n = 0.7321, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 117.3, nobs = 600, Jo/n = 0.1955, err = 6 @@ -21,11 +19,9 @@ Test : DRIPCGMinimizer: reduction in residual norm = 0.3926 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4245e+08, Max= 9.8041e+07, RMS= 1.7680e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4245e+08, Max= 9.8041e+07, RMS= 1.7680e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 59.3 Test : CostJo : Nonlinear Jo(Stream) = 407.4, nobs = 600, Jo/n = 0.679, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 75.74, nobs = 600, Jo/n = 0.1262, err = 6 diff --git a/qg/test/testoutput/3dfgat.test b/qg/test/testoutput/3dfgat.test index fba4eda38..f05065ac0 100644 --- a/qg/test/testoutput/3dfgat.test +++ b/qg/test/testoutput/3dfgat.test @@ -7,11 +7,9 @@ Test : DRIPCGMinimizer: reduction in residual norm = 0.186377 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6444e+08, Max= 1.0829e+08, RMS= 1.7939e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6444e+08, Max= 1.0829e+08, RMS= 1.7939e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 79.13 Test : CostJo : Nonlinear Jo(Stream) = 1596, nobs = 600, Jo/n = 2.66, err = 4e+06 (Monitoring only) Test : CostJo : Nonlinear Jo(Wind) = 159.4, nobs = 600, Jo/n = 0.2656, err = 6 diff --git a/qg/test/testoutput/3dvar.test b/qg/test/testoutput/3dvar.test index 55259b5b8..4956a39a7 100644 --- a/qg/test/testoutput/3dvar.test +++ b/qg/test/testoutput/3dvar.test @@ -7,11 +7,9 @@ Test : DRIPCGMinimizer: reduction in residual norm = 0.166244 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4870e+08, Max= 1.0374e+08, RMS= 1.7699e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4870e+08, Max= 1.0374e+08, RMS= 1.7699e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 62.81 Test : CostJo : Nonlinear Jo(Stream) = 666.7, nobs = 600, Jo/n = 1.111, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 459.7, nobs = 600, Jo/n = 0.7661, err = 6 @@ -21,11 +19,9 @@ Test : DRIPCGMinimizer: reduction in residual norm = 0.523 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4549e+08, Max= 1.0365e+08, RMS= 1.7647e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4549e+08, Max= 1.0365e+08, RMS= 1.7647e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 97.9 Test : CostJo : Nonlinear Jo(Stream) = 493.1, nobs = 600, Jo/n = 0.8218, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 302.1, nobs = 600, Jo/n = 0.5035, err = 6 diff --git a/qg/test/testoutput/3dvar_change_var.test b/qg/test/testoutput/3dvar_change_var.test index 52162c239..6a3df60b1 100644 --- a/qg/test/testoutput/3dvar_change_var.test +++ b/qg/test/testoutput/3dvar_change_var.test @@ -7,11 +7,9 @@ Test : DRIPCGMinimizer: reduction in residual norm = 0.0295408 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = potential vorticity -Test : Boundary conditions are activated -Test : Min= -7.1570e-04, Max= 6.3930e-04, RMS= 3.3441e-04 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Potential vorticity : Min= -7.1570e-04, Max= 6.3930e-04, RMS= 3.3441e-04 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 56.44 Test : CostJo : Nonlinear Jo(Stream) = 754.3, nobs = 600, Jo/n = 1.257, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 464.9, nobs = 600, Jo/n = 0.7748, err = 6 @@ -21,11 +19,9 @@ Test : DRIPCGMinimizer: reduction in residual norm = 0.2537 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = potential vorticity -Test : Boundary conditions are activated -Test : Min= -7.3088e-04, Max= 6.4855e-04, RMS= 3.3092e-04 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Potential vorticity : Min= -7.3088e-04, Max= 6.4855e-04, RMS= 3.3092e-04 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 87.9 Test : CostJo : Nonlinear Jo(Stream) = 514.7, nobs = 600, Jo/n = 0.8579, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 316.9, nobs = 600, Jo/n = 0.5281, err = 6 diff --git a/qg/test/testoutput/3dvar_hybrid.test b/qg/test/testoutput/3dvar_hybrid.test index 2a53bdc62..9089a6604 100644 --- a/qg/test/testoutput/3dvar_hybrid.test +++ b/qg/test/testoutput/3dvar_hybrid.test @@ -7,11 +7,9 @@ Test : DRIPCGMinimizer: reduction in residual norm = 0.121211 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4444e+08, Max= 1.0456e+08, RMS= 1.7727e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4444e+08, Max= 1.0456e+08, RMS= 1.7727e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 33.64 Test : CostJo : Nonlinear Jo(Stream) = 480.1, nobs = 600, Jo/n = 0.8002, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 202.7, nobs = 600, Jo/n = 0.3379, err = 6 @@ -21,11 +19,9 @@ Test : DRIPCGMinimizer: reduction in residual norm = 0.5399 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4287e+08, Max= 1.0243e+08, RMS= 1.7719e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4287e+08, Max= 1.0243e+08, RMS= 1.7719e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 41.85 Test : CostJo : Nonlinear Jo(Stream) = 425.7, nobs = 600, Jo/n = 0.7095, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 141.4, nobs = 600, Jo/n = 0.2356, err = 6 diff --git a/qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test b/qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test index eaec1f279..05ad37a72 100644 --- a/qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test +++ b/qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test @@ -6,11 +6,9 @@ Test : DRIPCGMinimizer: reduction in residual norm = 0.121211 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4444e+08, Max= 1.0456e+08, RMS= 1.7727e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4444e+08, Max= 1.0456e+08, RMS= 1.7727e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJo : Nonlinear Jo(Stream) = 480.1, nobs = 600, Jo/n = 0.8002, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 202.7, nobs = 600, Jo/n = 0.3379, err = 6 Test : CostJo : Nonlinear Jo(WSpeed) = 30.36, nobs = 300, Jo/n = 0.1012, err = 12 @@ -19,11 +17,9 @@ Test : DRIPCGMinimizer: reduction in residual norm = 0.5399 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4287e+08, Max= 1.0243e+08, RMS= 1.7719e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4287e+08, Max= 1.0243e+08, RMS= 1.7719e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJo : Nonlinear Jo(Stream) = 425.7, nobs = 600, Jo/n = 0.7095, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 141.4, nobs = 600, Jo/n = 0.2356, err = 6 Test : CostJo : Nonlinear Jo(WSpeed) = 24.37, nobs = 300, Jo/n = 0.08125, err = 12 diff --git a/qg/test/testoutput/4densvar.test b/qg/test/testoutput/4densvar.test index 53820ef3d..4bf452bfb 100644 --- a/qg/test/testoutput/4densvar.test +++ b/qg/test/testoutput/4densvar.test @@ -7,53 +7,39 @@ Test : DRPLanczosMinimizer: reduction in residual norm = 0.0322775 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.9807e+08, Max= 1.0008e+08, RMS= 1.8725e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.9807e+08, Max= 1.0008e+08, RMS= 1.8725e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T01:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.9492e+08, Max= 1.0017e+08, RMS= 1.8633e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.9492e+08, Max= 1.0017e+08, RMS= 1.8633e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T02:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.9188e+08, Max= 1.0020e+08, RMS= 1.8548e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.9188e+08, Max= 1.0020e+08, RMS= 1.8548e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T03:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8887e+08, Max= 1.0018e+08, RMS= 1.8467e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8887e+08, Max= 1.0018e+08, RMS= 1.8467e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T04:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8585e+08, Max= 1.0012e+08, RMS= 1.8392e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8585e+08, Max= 1.0012e+08, RMS= 1.8392e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T05:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8277e+08, Max= 1.0003e+08, RMS= 1.8321e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8277e+08, Max= 1.0003e+08, RMS= 1.8321e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8127e+08, Max= 9.9929e+07, RMS= 1.8252e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8127e+08, Max= 9.9929e+07, RMS= 1.8252e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 46.5093 Test : CostJo : Nonlinear Jo(Stream) = 14.6029, nobs = 150, Jo/n = 0.0973528, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 14.1107, nobs = 80, Jo/n = 0.176384, err = 6 @@ -63,53 +49,39 @@ Test : DRPLanczosMinimizer: reduction in residual norm = 0.235447 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.9882e+08, Max= 1.0111e+08, RMS= 1.8745e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.9882e+08, Max= 1.0111e+08, RMS= 1.8745e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T01:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.9551e+08, Max= 1.0117e+08, RMS= 1.8651e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.9551e+08, Max= 1.0117e+08, RMS= 1.8651e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T02:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.9229e+08, Max= 1.0118e+08, RMS= 1.8562e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.9229e+08, Max= 1.0118e+08, RMS= 1.8562e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T03:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8907e+08, Max= 1.0114e+08, RMS= 1.8479e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8907e+08, Max= 1.0114e+08, RMS= 1.8479e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T04:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8581e+08, Max= 1.0105e+08, RMS= 1.8401e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8581e+08, Max= 1.0105e+08, RMS= 1.8401e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T05:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8243e+08, Max= 1.0094e+08, RMS= 1.8328e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8243e+08, Max= 1.0094e+08, RMS= 1.8328e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8056e+08, Max= 1.0080e+08, RMS= 1.8256e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8056e+08, Max= 1.0080e+08, RMS= 1.8256e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 50.6449 Test : CostJo : Nonlinear Jo(Stream) = 10.8546, nobs = 150, Jo/n = 0.0723643, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 7.84779, nobs = 80, Jo/n = 0.0980974, err = 6 diff --git a/qg/test/testoutput/4densvar_hybrid.test b/qg/test/testoutput/4densvar_hybrid.test index b00c557f1..36f04d831 100644 --- a/qg/test/testoutput/4densvar_hybrid.test +++ b/qg/test/testoutput/4densvar_hybrid.test @@ -7,53 +7,39 @@ Test : DRPLanczosMinimizer: reduction in residual norm = 0.0171331 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8004e+08, Max= 9.7012e+07, RMS= 1.8293e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8004e+08, Max= 9.7012e+07, RMS= 1.8293e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T01:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.9089e+08, Max= 1.0599e+08, RMS= 1.8624e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.9089e+08, Max= 1.0599e+08, RMS= 1.8624e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T02:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.9400e+08, Max= 9.9891e+07, RMS= 1.8726e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.9400e+08, Max= 9.9891e+07, RMS= 1.8726e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T03:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.7220e+08, Max= 9.7340e+07, RMS= 1.8096e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.7220e+08, Max= 9.7340e+07, RMS= 1.8096e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T04:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8569e+08, Max= 1.0071e+08, RMS= 1.8495e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8569e+08, Max= 1.0071e+08, RMS= 1.8495e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T05:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6799e+08, Max= 9.7587e+07, RMS= 1.7900e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6799e+08, Max= 9.7587e+07, RMS= 1.7900e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.7783e+08, Max= 1.0136e+08, RMS= 1.8282e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.7783e+08, Max= 1.0136e+08, RMS= 1.8282e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 46.203 Test : CostJo : Nonlinear Jo(Stream) = 3.32558, nobs = 150, Jo/n = 0.0221705, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 6.39078, nobs = 80, Jo/n = 0.0798848, err = 6 @@ -63,53 +49,39 @@ Test : DRPLanczosMinimizer: reduction in residual norm = 0.139746 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8138e+08, Max= 9.6984e+07, RMS= 1.8296e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8138e+08, Max= 9.6984e+07, RMS= 1.8296e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T01:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.9362e+08, Max= 1.0496e+08, RMS= 1.8642e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.9362e+08, Max= 1.0496e+08, RMS= 1.8642e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T02:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.9436e+08, Max= 9.9889e+07, RMS= 1.8715e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.9436e+08, Max= 9.9889e+07, RMS= 1.8715e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T03:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.7373e+08, Max= 9.8372e+07, RMS= 1.8125e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.7373e+08, Max= 9.8372e+07, RMS= 1.8125e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T04:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8526e+08, Max= 1.0074e+08, RMS= 1.8485e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8526e+08, Max= 1.0074e+08, RMS= 1.8485e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T05:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6955e+08, Max= 9.8656e+07, RMS= 1.7930e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6955e+08, Max= 9.8656e+07, RMS= 1.7930e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.7744e+08, Max= 1.0139e+08, RMS= 1.8274e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.7744e+08, Max= 1.0139e+08, RMS= 1.8274e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 48.2511 Test : CostJo : Nonlinear Jo(Stream) = 2.37944, nobs = 150, Jo/n = 0.0158629, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 4.56144, nobs = 80, Jo/n = 0.0570181, err = 6 diff --git a/qg/test/testoutput/4dvar_change_var.test b/qg/test/testoutput/4dvar_change_var.test deleted file mode 100644 index 79ad4cb9a..000000000 --- a/qg/test/testoutput/4dvar_change_var.test +++ /dev/null @@ -1,36 +0,0 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 -Test : CostJcDFI: Nonlinear Jc = 48.6879 -Test : CostFunction: Nonlinear J = 12647.5 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0348733 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Variable = potential vorticity -Test : Boundary conditions are activated -Test : Min= -7.0828e-04, Max= 6.2704e-04, RMS= 3.3117e-04 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 28.15 -Test : CostJo : Nonlinear Jo(Stream) = 561.6, nobs = 800, Jo/n = 0.702, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 218.7, nobs = 800, Jo/n = 0.2734, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 39.43, nobs = 400, Jo/n = 0.09857, err = 12 -Test : CostJcDFI: Nonlinear Jc = 54.8 -Test : CostFunction: Nonlinear J = 902.7 -Test : DRIPCGMinimizer: reduction in residual norm = 0.1147 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Variable = potential vorticity -Test : Boundary conditions are activated -Test : Min= -6.7969e-04, Max= 6.5276e-04, RMS= 3.2813e-04 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 31.57 -Test : CostJo : Nonlinear Jo(Stream) = 169.9, nobs = 800, Jo/n = 0.2124, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 114.5, nobs = 800, Jo/n = 0.1431, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 18.66, nobs = 400, Jo/n = 0.04665, err = 12 -Test : CostJcDFI: Nonlinear Jc = 59.45 -Test : CostFunction: Nonlinear J = 394.1 diff --git a/qg/test/testoutput/4dvar_dripcg.test b/qg/test/testoutput/4dvar_dripcg.test index 4a88a83c5..cfbce7056 100644 --- a/qg/test/testoutput/4dvar_dripcg.test +++ b/qg/test/testoutput/4dvar_dripcg.test @@ -8,11 +8,9 @@ Test : DRIPCGMinimizer: reduction in residual norm = 0.0903626 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.1952e+08, Max= 1.0284e+08, RMS= 1.8679e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.1952e+08, Max= 1.0284e+08, RMS= 1.8679e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 27.31 Test : CostJo : Nonlinear Jo(Stream) = 1158, nobs = 800, Jo/n = 1.447, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 272.1, nobs = 800, Jo/n = 0.3402, err = 6 @@ -23,11 +21,9 @@ Test : DRIPCGMinimizer: reduction in residual norm = 0.1754 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.1828e+08, Max= 1.0887e+08, RMS= 1.8760e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.1828e+08, Max= 1.0887e+08, RMS= 1.8760e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 56.87 Test : CostJo : Nonlinear Jo(Stream) = 768.3, nobs = 800, Jo/n = 0.9604, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 233.1, nobs = 800, Jo/n = 0.2914, err = 6 diff --git a/qg/test/testoutput/4dvar_drpcg_lmp.test b/qg/test/testoutput/4dvar_drpcg_lmp.test index 102b884de..5c3b91a46 100644 --- a/qg/test/testoutput/4dvar_drpcg_lmp.test +++ b/qg/test/testoutput/4dvar_drpcg_lmp.test @@ -8,11 +8,9 @@ Test : DRPCGMinimizer: reduction in residual norm = 0.0336644 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.2116e+08, Max= 1.0307e+08, RMS= 1.8911e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.2116e+08, Max= 1.0307e+08, RMS= 1.8911e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 28.45 Test : CostJo : Nonlinear Jo(Stream) = 558.1, nobs = 800, Jo/n = 0.6977, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 217.8, nobs = 800, Jo/n = 0.2722, err = 6 @@ -23,11 +21,9 @@ Test : DRPCGMinimizer: reduction in residual norm = 0.1334 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.2249e+08, Max= 1.0481e+08, RMS= 1.9120e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.2249e+08, Max= 1.0481e+08, RMS= 1.9120e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 47.48 Test : CostJo : Nonlinear Jo(Stream) = 211.4, nobs = 800, Jo/n = 0.2643, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 113.8, nobs = 800, Jo/n = 0.1423, err = 6 diff --git a/qg/test/testoutput/4dvar_drpfom.test b/qg/test/testoutput/4dvar_drpfom.test index 68a07f52b..fa7e4d479 100644 --- a/qg/test/testoutput/4dvar_drpfom.test +++ b/qg/test/testoutput/4dvar_drpfom.test @@ -8,11 +8,9 @@ Test : DRPFOMMinimizer: reduction in residual norm = 0.0336644 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.2116e+08, Max= 1.0307e+08, RMS= 1.8911e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.2116e+08, Max= 1.0307e+08, RMS= 1.8911e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 28.45 Test : CostJo : Nonlinear Jo(Stream) = 558.1, nobs = 800, Jo/n = 0.6977, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 217.8, nobs = 800, Jo/n = 0.2722, err = 6 @@ -23,11 +21,9 @@ Test : DRPFOMMinimizer: reduction in residual norm = 0.1269 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.2053e+08, Max= 1.0439e+08, RMS= 1.9129e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.2053e+08, Max= 1.0439e+08, RMS= 1.9129e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 31.72 Test : CostJo : Nonlinear Jo(Stream) = 168.6, nobs = 800, Jo/n = 0.2107, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 114.4, nobs = 800, Jo/n = 0.143, err = 6 diff --git a/qg/test/testoutput/4dvar_drplanczos.test b/qg/test/testoutput/4dvar_drplanczos.test index f4db784d9..460c569ae 100644 --- a/qg/test/testoutput/4dvar_drplanczos.test +++ b/qg/test/testoutput/4dvar_drplanczos.test @@ -8,11 +8,9 @@ Test : DRPLanczosMinimizer: reduction in residual norm = 0.0336644 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.2116e+08, Max= 1.0307e+08, RMS= 1.8911e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.2116e+08, Max= 1.0307e+08, RMS= 1.8911e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 28.45 Test : CostJo : Nonlinear Jo(Stream) = 558.1, nobs = 800, Jo/n = 0.6977, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 217.8, nobs = 800, Jo/n = 0.2722, err = 6 @@ -23,11 +21,9 @@ Test : DRPLanczosMinimizer: reduction in residual norm = 0.1265 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.1935e+08, Max= 1.0477e+08, RMS= 1.9089e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.1935e+08, Max= 1.0477e+08, RMS= 1.9089e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 34.18 Test : CostJo : Nonlinear Jo(Stream) = 153.5, nobs = 800, Jo/n = 0.1919, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 104.6, nobs = 800, Jo/n = 0.1308, err = 6 diff --git a/qg/test/testoutput/4dvar_drplanczos_hybrid.test b/qg/test/testoutput/4dvar_drplanczos_hybrid.test index e2117ce11..fe0924a1d 100644 --- a/qg/test/testoutput/4dvar_drplanczos_hybrid.test +++ b/qg/test/testoutput/4dvar_drplanczos_hybrid.test @@ -8,11 +8,9 @@ Test : DRPLanczosMinimizer: reduction in residual norm = 0.0142818 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.1710e+08, Max= 1.0488e+08, RMS= 1.8894e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.1710e+08, Max= 1.0488e+08, RMS= 1.8894e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 41.74 Test : CostJo : Nonlinear Jo(Stream) = 542.1, nobs = 800, Jo/n = 0.6776, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 144.3, nobs = 800, Jo/n = 0.1804, err = 6 @@ -23,11 +21,9 @@ Test : DRPLanczosMinimizer: reduction in residual norm = 0.1142 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.2175e+08, Max= 1.0535e+08, RMS= 1.9034e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.2175e+08, Max= 1.0535e+08, RMS= 1.9034e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 39.05 Test : CostJo : Nonlinear Jo(Stream) = 86.94, nobs = 800, Jo/n = 0.1087, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 62.92, nobs = 800, Jo/n = 0.07865, err = 6 diff --git a/qg/test/testoutput/4dvar_ipcg.test b/qg/test/testoutput/4dvar_ipcg.test index e77e7dd2f..2cd35680d 100644 --- a/qg/test/testoutput/4dvar_ipcg.test +++ b/qg/test/testoutput/4dvar_ipcg.test @@ -8,11 +8,9 @@ Test : IPCGMinimizer: reduction in residual norm = 0.0328 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.1660e+08, Max= 1.0367e+08, RMS= 1.8975e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.1660e+08, Max= 1.0367e+08, RMS= 1.8975e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 68.07 Test : CostJo : Nonlinear Jo(Stream) = 393.1, nobs = 800, Jo/n = 0.4913, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 134.1, nobs = 800, Jo/n = 0.1676, err = 6 @@ -23,11 +21,9 @@ Test : IPCGMinimizer: reduction in residual norm = 0.2145 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.2297e+08, Max= 1.0515e+08, RMS= 1.9148e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.2297e+08, Max= 1.0515e+08, RMS= 1.9148e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 63.98 Test : CostJo : Nonlinear Jo(Stream) = 121.9, nobs = 800, Jo/n = 0.1524, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 84.01, nobs = 800, Jo/n = 0.105, err = 6 diff --git a/qg/test/testoutput/4dvar_obs_biased.test b/qg/test/testoutput/4dvar_obs_biased.test index d00f76dac..05cbb2262 100644 --- a/qg/test/testoutput/4dvar_obs_biased.test +++ b/qg/test/testoutput/4dvar_obs_biased.test @@ -8,11 +8,9 @@ Test : DRPCGMinimizer: reduction in residual norm = 0.00387793 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.1520e+08, Max= 1.0618e+08, RMS= 1.8848e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.1520e+08, Max= 1.0618e+08, RMS= 1.8848e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : ObsBias = -1.16824e+07, 0, 0, 0 Test : ObsBias = 0, 9.67042, 0, 0 Test : CostJb : Nonlinear Jb = 60.42 @@ -25,11 +23,9 @@ Test : DRPCGMinimizer: reduction in residual norm = 0.1576 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.2306e+08, Max= 1.0568e+08, RMS= 1.9156e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.2306e+08, Max= 1.0568e+08, RMS= 1.9156e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : ObsBias = -9.70392e+06, 0, 0, 0 Test : ObsBias = 0, 9.83281, 0, 0 Test : CostJb : Nonlinear Jb = 57.71 diff --git a/qg/test/testoutput/4dvar_rpcg.test b/qg/test/testoutput/4dvar_rpcg.test index 1ed0d07d1..cdce0a44a 100644 --- a/qg/test/testoutput/4dvar_rpcg.test +++ b/qg/test/testoutput/4dvar_rpcg.test @@ -7,11 +7,9 @@ Test : RPCGMinimizer: reduction in residual norm = 0.0339189 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.2145e+08, Max= 1.0309e+08, RMS= 1.8913e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.2145e+08, Max= 1.0309e+08, RMS= 1.8913e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 28.47 Test : CostJo : Nonlinear Jo(Stream) = 562.2, nobs = 800, Jo/n = 0.7027, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 218.5, nobs = 800, Jo/n = 0.2731, err = 6 @@ -21,11 +19,9 @@ Test : RPCGMinimizer: reduction in residual norm = 0.1177 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.2031e+08, Max= 1.0434e+08, RMS= 1.9125e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.2031e+08, Max= 1.0434e+08, RMS= 1.9125e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 31.66 Test : CostJo : Nonlinear Jo(Stream) = 168.2, nobs = 800, Jo/n = 0.2102, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 113.8, nobs = 800, Jo/n = 0.1422, err = 6 diff --git a/qg/test/testoutput/4dvar_saddlepoint.test b/qg/test/testoutput/4dvar_saddlepoint.test index 8547f1319..ffde6af3d 100644 --- a/qg/test/testoutput/4dvar_saddlepoint.test +++ b/qg/test/testoutput/4dvar_saddlepoint.test @@ -9,18 +9,14 @@ Test : SaddlePointMinimizer: reduction in residual norm = 2.295e-13 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6781e+08, Max= 9.6458e+07, RMS= 1.7904e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6781e+08, Max= 9.6458e+07, RMS= 1.7904e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3337e+08, Max= 9.3777e+07, RMS= 1.6813e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3337e+08, Max= 9.3777e+07, RMS= 1.6813e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 0 Test : CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 @@ -32,18 +28,14 @@ Test : SaddlePointMinimizer: reduction in residual norm = 2.295e-13 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6781e+08, Max= 9.6458e+07, RMS= 1.7904e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6781e+08, Max= 9.6458e+07, RMS= 1.7904e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3337e+08, Max= 9.3777e+07, RMS= 1.6813e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3337e+08, Max= 9.3777e+07, RMS= 1.6813e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 0 Test : CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 diff --git a/qg/test/testoutput/addincrement.test b/qg/test/testoutput/addincrement.test index c3fbc53c6..0d47ef8ef 100644 --- a/qg/test/testoutput/addincrement.test +++ b/qg/test/testoutput/addincrement.test @@ -1,22 +1,16 @@ Test : State: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 +Test : Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 Test : State plus increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6263e+08, Max= 1.0660e+08, RMS= 1.7472e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6263e+08, Max= 1.0660e+08, RMS= 1.7472e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/addincrement_scaled.test b/qg/test/testoutput/addincrement_scaled.test index baa35aaf0..c128933ee 100644 --- a/qg/test/testoutput/addincrement_scaled.test +++ b/qg/test/testoutput/addincrement_scaled.test @@ -1,28 +1,20 @@ Test : State: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 +Test : Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 Test : Scaled the increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -9.2140e+06, Max= 2.8742e+07, RMS= 4.2153e+06 +Test : Streamfunction : Min= -9.2140e+06, Max= 2.8742e+07, RMS= 4.2153e+06 Test : State plus increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.5411e+08, Max= 1.0285e+08, RMS= 1.7552e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.5411e+08, Max= 1.0285e+08, RMS= 1.7552e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/analytic_forecast.test b/qg/test/testoutput/analytic_forecast.test index bace94360..533e434bb 100644 --- a/qg/test/testoutput/analytic_forecast.test +++ b/qg/test/testoutput/analytic_forecast.test @@ -1,16 +1,12 @@ Test : Initial state: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.1289e+08, Max= 4.1289e+08, RMS= 1.5614e+08 -Test : Min= -3.6309e+01, Max= 0.0000e+00, RMS= 1.8839e+01 -Test : Min= -7.3653e-05, Max= 8.0801e-05, RMS= 7.5159e-05 +Test : Streamfunction : Min= -4.1289e+08, Max= 4.1289e+08, RMS= 1.5614e+08 +Test : Streamfunction LBC : Min= -3.6309e+01, Max= 0.0000e+00, RMS= 1.8839e+01 +Test : Potential vorticity LBC: Min= -7.3653e-05, Max= 8.0801e-05, RMS= 7.5159e-05 Test : Final state: Test : Valid time: 2010-01-05T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.2065e+08, Max= 4.2065e+08, RMS= 1.8324e+08 -Test : Min= -3.6309e+01, Max= 0.0000e+00, RMS= 1.8839e+01 -Test : Min= -7.3653e-05, Max= 8.0801e-05, RMS= 7.5159e-05 +Test : Streamfunction : Min= -4.2065e+08, Max= 4.2065e+08, RMS= 1.8324e+08 +Test : Streamfunction LBC : Min= -3.6309e+01, Max= 0.0000e+00, RMS= 1.8839e+01 +Test : Potential vorticity LBC: Min= -7.3653e-05, Max= 8.0801e-05, RMS= 7.5159e-05 diff --git a/qg/test/testoutput/convertincrement.test b/qg/test/testoutput/convertincrement.test index 9a849ab94..3ab67edc1 100644 --- a/qg/test/testoutput/convertincrement.test +++ b/qg/test/testoutput/convertincrement.test @@ -1,44 +1,30 @@ Test : Input increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 +Test : Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 Test : Trajectory state: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Increment after variable transform: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = potential vorticity -Test : Boundary conditions are not activated -Test : Min= -1.7247e-04, Max= 5.3114e-05, RMS= 2.0353e-05 +Test : Potential vorticity : Min= -1.7247e-04, Max= 5.3114e-05, RMS= 2.0353e-05 Test : Increment after variable transform: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 +Test : Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 Test : Increment after variable transform: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 +Test : Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 Test : Increment after variable transform: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 +Test : Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 Test : Output increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 +Test : Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 diff --git a/qg/test/testoutput/convertstate.test b/qg/test/testoutput/convertstate.test index db769053d..614438b95 100644 --- a/qg/test/testoutput/convertstate.test +++ b/qg/test/testoutput/convertstate.test @@ -1,164 +1,132 @@ Test : Input state: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4304e+08, Max= 9.3153e+07, RMS= 1.7024e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4304e+08, Max= 9.3153e+07, RMS= 1.7024e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: QG change of variable Test : Variable change from: 1 variables: x Test : Variable change to: 1 variables: q Test : State after variable transform: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = potential vorticity -Test : Boundary conditions are activated -Test : Min= -5.8027e-04, Max= 5.4508e-04, RMS= 3.2514e-04 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Potential vorticity : Min= -5.8027e-04, Max= 5.4508e-04, RMS= 3.2514e-04 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: QG change of variable Test : Variable change from: 1 variables: q Test : Variable change to: 1 variables: x Test : State after variable transform: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: QG change of variable Test : Variable change from: 1 variables: q Test : Variable change to: 1 variables: x Test : State after variable transform: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = potential vorticity -Test : Boundary conditions are activated -Test : Min= -5.8027e-04, Max= 5.4508e-04, RMS= 3.2514e-04 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Potential vorticity : Min= -5.8027e-04, Max= 5.4508e-04, RMS= 3.2514e-04 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: QG change of variable Test : Variable change from: 1 variables: x Test : Variable change to: 1 variables: q Test : State after variable transform: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: IdVariableChange Test : Variable change from: 1 variables: x Test : Variable change to: 1 variables: x Test : State after variable transform: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: IdVariableChange Test : Variable change from: 1 variables: x Test : Variable change to: 1 variables: x Test : State after variable transform: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Output state: Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Input state: Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3926e+08, Max= 9.3338e+07, RMS= 1.6950e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3926e+08, Max= 9.3338e+07, RMS= 1.6950e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: QG change of variable Test : Variable change from: 1 variables: x Test : Variable change to: 1 variables: q Test : State after variable transform: Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = potential vorticity -Test : Boundary conditions are activated -Test : Min= -5.7613e-04, Max= 5.4113e-04, RMS= 3.2534e-04 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Potential vorticity : Min= -5.7613e-04, Max= 5.4113e-04, RMS= 3.2534e-04 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: QG change of variable Test : Variable change from: 1 variables: q Test : Variable change to: 1 variables: x Test : State after variable transform: Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: QG change of variable Test : Variable change from: 1 variables: q Test : Variable change to: 1 variables: x Test : State after variable transform: Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = potential vorticity -Test : Boundary conditions are activated -Test : Min= -5.7613e-04, Max= 5.4113e-04, RMS= 3.2534e-04 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Potential vorticity : Min= -5.7613e-04, Max= 5.4113e-04, RMS= 3.2534e-04 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: QG change of variable Test : Variable change from: 1 variables: x Test : Variable change to: 1 variables: q Test : State after variable transform: Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: IdVariableChange Test : Variable change from: 1 variables: x Test : Variable change to: 1 variables: x Test : State after variable transform: Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Variable transform: IdVariableChange Test : Variable change from: 1 variables: x Test : Variable change to: 1 variables: x Test : State after variable transform: Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Output state: Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 20, 10, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/dfi.test b/qg/test/testoutput/dfi.test index 4d9cbac17..bed6e6c7e 100644 --- a/qg/test/testoutput/dfi.test +++ b/qg/test/testoutput/dfi.test @@ -1,24 +1,18 @@ Test : Initial state: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6781e+08, Max= 9.6458e+07, RMS= 1.7904e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6781e+08, Max= 9.6458e+07, RMS= 1.7904e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Filtered state: Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.5184e+08, Max= 9.3196e+07, RMS= 1.7271e+08 -Test : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.5184e+08, Max= 9.3196e+07, RMS= 1.7271e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Final state: Test : Valid time: 2010-01-08T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -3.9403e+08, Max= 1.7285e+08, RMS= 1.5325e+08 -Test : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -3.9403e+08, Max= 1.7285e+08, RMS= 1.5325e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/diffstates.test b/qg/test/testoutput/diffstates.test index 804cb4f25..9c540be7b 100644 --- a/qg/test/testoutput/diffstates.test +++ b/qg/test/testoutput/diffstates.test @@ -1,22 +1,16 @@ Test : Input state 1: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Input state 2: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.5934e+08, Max= 1.0390e+08, RMS= 1.7850e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.5934e+08, Max= 1.0390e+08, RMS= 1.7850e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Output increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 +Test : Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 diff --git a/qg/test/testoutput/dirac_cov.test b/qg/test/testoutput/dirac_cov.test index ca1bcd579..abf99e3cc 100644 --- a/qg/test/testoutput/dirac_cov.test +++ b/qg/test/testoutput/dirac_cov.test @@ -1,18 +1,12 @@ Test : Input Dirac increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 +Test : Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 Test : B * Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -7.2330e+05, Max= 3.2400e+14, RMS= 6.2881e+13 +Test : Streamfunction : Min= -7.2330e+05, Max= 3.2400e+14, RMS= 6.2881e+13 Test : Randomized variance: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 2.8165e+14, Max= 3.6315e+14, RMS= 3.2388e+14 +Test : Streamfunction : Min= 2.8165e+14, Max= 3.6315e+14, RMS= 3.2388e+14 diff --git a/qg/test/testoutput/dirac_hyb_field.test b/qg/test/testoutput/dirac_hyb_field.test index 4b9905ea6..c426ce866 100644 --- a/qg/test/testoutput/dirac_hyb_field.test +++ b/qg/test/testoutput/dirac_hyb_field.test @@ -1,18 +1,12 @@ Test : Input Dirac increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 +Test : Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 Test : B * Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.3348e+13, Max= 3.1207e+14, RMS= 5.8758e+13 +Test : Streamfunction : Min= -1.3348e+13, Max= 3.1207e+14, RMS= 5.8758e+13 Test : Localized Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 2.1340e-06, Max= 1.0000e+00, RMS= 3.4333e-01 +Test : Streamfunction : Min= 2.1340e-06, Max= 1.0000e+00, RMS= 3.4333e-01 diff --git a/qg/test/testoutput/dirac_hyb_value.test b/qg/test/testoutput/dirac_hyb_value.test index b22c1b291..031aae710 100644 --- a/qg/test/testoutput/dirac_hyb_value.test +++ b/qg/test/testoutput/dirac_hyb_value.test @@ -1,24 +1,16 @@ Test : Input Dirac increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 +Test : Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 Test : B * Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.3348e+13, Max= 3.1207e+14, RMS= 5.8758e+13 +Test : Streamfunction : Min= -1.3348e+13, Max= 3.1207e+14, RMS= 5.8758e+13 Test : Localized Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 2.1340e-06, Max= 1.0000e+00, RMS= 3.4333e-01 +Test : Streamfunction : Min= 2.1340e-06, Max= 1.0000e+00, RMS= 3.4333e-01 Test : Randomized variance: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 1.0521e+14, Max= 9.2106e+14, RMS= 3.1869e+14 +Test : Streamfunction : Min= 1.0521e+14, Max= 9.2106e+14, RMS= 3.1869e+14 diff --git a/qg/test/testoutput/dirac_loc_3d.test b/qg/test/testoutput/dirac_loc_3d.test index af8491abf..8be9a04af 100644 --- a/qg/test/testoutput/dirac_loc_3d.test +++ b/qg/test/testoutput/dirac_loc_3d.test @@ -1,24 +1,16 @@ Test : Input Dirac increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 +Test : Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 Test : B * Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -6.5158e+13, Max= 3.7255e+14, RMS= 5.8877e+13 +Test : Streamfunction : Min= -6.5158e+13, Max= 3.7255e+14, RMS= 5.8877e+13 Test : Localized Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 2.1340e-06, Max= 1.0000e+00, RMS= 3.4333e-01 +Test : Streamfunction : Min= 2.1340e-06, Max= 1.0000e+00, RMS= 3.4333e-01 Test : Randomized variance: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 1.3256e+12, Max= 1.4252e+15, RMS= 3.6277e+14 +Test : Streamfunction : Min= 1.3256e+12, Max= 1.4252e+15, RMS= 3.6277e+14 diff --git a/qg/test/testoutput/dirac_loc_4d.test b/qg/test/testoutput/dirac_loc_4d.test index f5a26d2fd..2d7666a3c 100644 --- a/qg/test/testoutput/dirac_loc_4d.test +++ b/qg/test/testoutput/dirac_loc_4d.test @@ -1,264 +1,160 @@ Test : Input Dirac increment: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 Test : Valid time: 2010-01-01T01:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 Test : Valid time: 2010-01-01T02:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 Test : Valid time: 2010-01-01T03:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 Test : Valid time: 2010-01-01T04:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 Test : Valid time: 2010-01-01T05:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 +Test : Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 Test : Valid time: 2010-01-01T07:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 Test : Valid time: 2010-01-01T08:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 Test : Valid time: 2010-01-01T11:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 Test : B * Increment: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -4.0432e+13, Max= 1.9692e+14, RMS= 2.2098e+13 +Test : Streamfunction : Min= -4.0432e+13, Max= 1.9692e+14, RMS= 2.2098e+13 Test : Valid time: 2010-01-01T01:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -4.0552e+13, Max= 2.0894e+14, RMS= 2.2747e+13 +Test : Streamfunction : Min= -4.0552e+13, Max= 2.0894e+14, RMS= 2.2747e+13 Test : Valid time: 2010-01-01T02:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -3.9769e+13, Max= 2.1965e+14, RMS= 2.3431e+13 +Test : Streamfunction : Min= -3.9769e+13, Max= 2.1965e+14, RMS= 2.3431e+13 Test : Valid time: 2010-01-01T03:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -3.8287e+13, Max= 2.2877e+14, RMS= 2.4126e+13 +Test : Streamfunction : Min= -3.8287e+13, Max= 2.2877e+14, RMS= 2.4126e+13 Test : Valid time: 2010-01-01T04:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -3.6244e+13, Max= 2.3634e+14, RMS= 2.4826e+13 +Test : Streamfunction : Min= -3.6244e+13, Max= 2.3634e+14, RMS= 2.4826e+13 Test : Valid time: 2010-01-01T05:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -3.3760e+13, Max= 2.4250e+14, RMS= 2.5516e+13 +Test : Streamfunction : Min= -3.3760e+13, Max= 2.4250e+14, RMS= 2.5516e+13 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -3.0971e+13, Max= 2.4747e+14, RMS= 2.6180e+13 +Test : Streamfunction : Min= -3.0971e+13, Max= 2.4747e+14, RMS= 2.6180e+13 Test : Valid time: 2010-01-01T07:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -2.8061e+13, Max= 2.5136e+14, RMS= 2.6788e+13 +Test : Streamfunction : Min= -2.8061e+13, Max= 2.5136e+14, RMS= 2.6788e+13 Test : Valid time: 2010-01-01T08:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -2.5144e+13, Max= 2.5423e+14, RMS= 2.7311e+13 +Test : Streamfunction : Min= -2.5144e+13, Max= 2.5423e+14, RMS= 2.7311e+13 Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -2.3979e+13, Max= 2.5615e+14, RMS= 2.7721e+13 +Test : Streamfunction : Min= -2.3979e+13, Max= 2.5615e+14, RMS= 2.7721e+13 Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -2.3448e+13, Max= 2.5735e+14, RMS= 2.8021e+13 +Test : Streamfunction : Min= -2.3448e+13, Max= 2.5735e+14, RMS= 2.8021e+13 Test : Valid time: 2010-01-01T11:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -2.2850e+13, Max= 2.5807e+14, RMS= 2.8223e+13 +Test : Streamfunction : Min= -2.2850e+13, Max= 2.5807e+14, RMS= 2.8223e+13 Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -2.2238e+13, Max= 2.5841e+14, RMS= 2.8326e+13 +Test : Streamfunction : Min= -2.2238e+13, Max= 2.5841e+14, RMS= 2.8326e+13 Test : Localized Increment: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 +Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 Test : Valid time: 2010-01-01T01:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 +Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 Test : Valid time: 2010-01-01T02:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 +Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 Test : Valid time: 2010-01-01T03:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 +Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 Test : Valid time: 2010-01-01T04:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 +Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 Test : Valid time: 2010-01-01T05:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 +Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 +Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 Test : Valid time: 2010-01-01T07:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 +Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 Test : Valid time: 2010-01-01T08:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 +Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 +Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 +Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 Test : Valid time: 2010-01-01T11:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 +Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 +Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 Test : Randomized variance: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 3.7303e+12, Max= 1.0538e+15, RMS= 2.4663e+14 +Test : Streamfunction : Min= 3.7303e+12, Max= 1.0538e+15, RMS= 2.4663e+14 Test : Valid time: 2010-01-01T01:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 3.9097e+12, Max= 1.1299e+15, RMS= 2.5694e+14 +Test : Streamfunction : Min= 3.9097e+12, Max= 1.1299e+15, RMS= 2.5694e+14 Test : Valid time: 2010-01-01T02:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 2.4068e+12, Max= 1.2020e+15, RMS= 2.6754e+14 +Test : Streamfunction : Min= 2.4068e+12, Max= 1.2020e+15, RMS= 2.6754e+14 Test : Valid time: 2010-01-01T03:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 1.8149e+12, Max= 1.2679e+15, RMS= 2.7848e+14 +Test : Streamfunction : Min= 1.8149e+12, Max= 1.2679e+15, RMS= 2.7848e+14 Test : Valid time: 2010-01-01T04:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 1.9945e+12, Max= 1.3266e+15, RMS= 2.8969e+14 +Test : Streamfunction : Min= 1.9945e+12, Max= 1.3266e+15, RMS= 2.8969e+14 Test : Valid time: 2010-01-01T05:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 2.4492e+12, Max= 1.3767e+15, RMS= 3.0102e+14 +Test : Streamfunction : Min= 2.4492e+12, Max= 1.3767e+15, RMS= 3.0102e+14 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 2.3155e+12, Max= 1.4163e+15, RMS= 3.1227e+14 +Test : Streamfunction : Min= 2.3155e+12, Max= 1.4163e+15, RMS= 3.1227e+14 Test : Valid time: 2010-01-01T07:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 2.2409e+12, Max= 1.4568e+15, RMS= 3.2326e+14 +Test : Streamfunction : Min= 2.2409e+12, Max= 1.4568e+15, RMS= 3.2326e+14 Test : Valid time: 2010-01-01T08:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 2.0067e+12, Max= 1.5122e+15, RMS= 3.3397e+14 +Test : Streamfunction : Min= 2.0067e+12, Max= 1.5122e+15, RMS= 3.3397e+14 Test : Valid time: 2010-01-01T09:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 1.7114e+12, Max= 1.5718e+15, RMS= 3.4435e+14 +Test : Streamfunction : Min= 1.7114e+12, Max= 1.5718e+15, RMS= 3.4435e+14 Test : Valid time: 2010-01-01T10:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 1.4554e+12, Max= 1.6333e+15, RMS= 3.5419e+14 +Test : Streamfunction : Min= 1.4554e+12, Max= 1.6333e+15, RMS= 3.5419e+14 Test : Valid time: 2010-01-01T11:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 1.2259e+12, Max= 1.6944e+15, RMS= 3.6352e+14 +Test : Streamfunction : Min= 1.2259e+12, Max= 1.6944e+15, RMS= 3.6352e+14 Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 1.0259e+12, Max= 1.7698e+15, RMS= 3.7255e+14 +Test : Streamfunction : Min= 1.0259e+12, Max= 1.7698e+15, RMS= 3.7255e+14 diff --git a/qg/test/testoutput/dirac_no_loc.test b/qg/test/testoutput/dirac_no_loc.test index 2e346891c..6c48c95b8 100644 --- a/qg/test/testoutput/dirac_no_loc.test +++ b/qg/test/testoutput/dirac_no_loc.test @@ -1,18 +1,12 @@ Test : Input Dirac increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 +Test : Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 Test : B * Increment: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -3.0667e+14, Max= 4.4252e+14, RMS= 1.0739e+14 +Test : Streamfunction : Min= -3.0667e+14, Max= 4.4252e+14, RMS= 1.0739e+14 Test : Randomized variance: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 1.1897e+12, Max= 1.7159e+15, RMS= 3.9892e+14 +Test : Streamfunction : Min= 1.1897e+12, Max= 1.7159e+15, RMS= 3.9892e+14 diff --git a/qg/test/testoutput/eda_3dvar_block.test b/qg/test/testoutput/eda_3dvar_block.test index 42a417d66..3ca6ddcd6 100644 --- a/qg/test/testoutput/eda_3dvar_block.test +++ b/qg/test/testoutput/eda_3dvar_block.test @@ -10,14 +10,12 @@ Test : Quadratic cost function all members: J ( 2) = 3979.77 5093.07 2507 Test : Norm reduction all members ( 3) = 0.158052 0.116268 0.185322 0.0977361 Test : Quadratic cost function all members: J ( 3) = 3313.65 4119.31 2154.65 2990.07 Test : DRPBlockLanczosMinimizer: reduction in residual norm = 0.158052 -Test : CostFunction::addIncrement: Analysis: +Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4961e+08, Max= 1.0918e+08, RMS= 1.7739e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4961e+08, Max= 1.0918e+08, RMS= 1.7739e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 31.4 Test : CostJo : Nonlinear Jo(Stream) = 2150, nobs = 600, Jo/n = 3.584, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 1104, nobs = 600, Jo/n = 1.84, err = 6 @@ -42,14 +40,12 @@ Test : Quadratic cost function all members: J ( 8) = 1447 1789 1136 1491 Test : Norm reduction all members ( 9) = 0.1256 0.1218 0.1586 0.1376 Test : Quadratic cost function all members: J ( 9) = 1403 1740 1113 1437 Test : DRPBlockLanczosMinimizer: reduction in residual norm = 0.1256 -Test : CostFunction::addIncrement: Analysis: +Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6332e+08, Max= 1.0396e+08, RMS= 1.7698e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6332e+08, Max= 1.0396e+08, RMS= 1.7698e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : CostJb : Nonlinear Jb = 162.6 Test : CostJo : Nonlinear Jo(Stream) = 563.7, nobs = 600, Jo/n = 0.9395, err = 4e+06 Test : CostJo : Nonlinear Jo(Wind) = 490.7, nobs = 600, Jo/n = 0.8179, err = 6 diff --git a/qg/test/testoutput/ens_forecast.test b/qg/test/testoutput/ens_forecast.test index 8a7fbfb71..17bd52b83 100644 --- a/qg/test/testoutput/ens_forecast.test +++ b/qg/test/testoutput/ens_forecast.test @@ -1,16 +1,12 @@ Test : Initial state: Test : Valid time: 2009-12-31T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8796e+08, Max= 1.0889e+08, RMS= 2.0493e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8796e+08, Max= 1.0889e+08, RMS= 2.0493e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Final state: Test : Valid time: 2010-01-02T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.1708e+08, Max= 1.0232e+08, RMS= 1.6357e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.1708e+08, Max= 1.0232e+08, RMS= 1.6357e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/ens_hofx.test b/qg/test/testoutput/ens_hofx.test index 3096e2c80..7fdc1e504 100644 --- a/qg/test/testoutput/ens_hofx.test +++ b/qg/test/testoutput/ens_hofx.test @@ -1,19 +1,15 @@ Test : Initial state: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Final state: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4224e+08, Max= 1.2418e+08, RMS= 1.7584e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4224e+08, Max= 1.2418e+08, RMS= 1.7584e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : H(x): Test : Stream nobs= 300 Min= -5.3856e+08, Max= 8.6950e+07, Average= -1.2072e+08 Test : Wind nobs= 160 Min= -1.2440e+02, Max= 9.4257e+01, Average= 1.1768e+01 diff --git a/qg/test/testoutput/ens_recenter.test b/qg/test/testoutput/ens_recenter.test index 2427acf73..b61758f76 100644 --- a/qg/test/testoutput/ens_recenter.test +++ b/qg/test/testoutput/ens_recenter.test @@ -1,88 +1,66 @@ Test : Original member 0 : Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Original member 1 : Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6931e+08, Max= 9.9327e+07, RMS= 1.7976e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6931e+08, Max= 9.9327e+07, RMS= 1.7976e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Original member 2 : Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.9074e+08, Max= 1.0042e+08, RMS= 1.8589e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.9074e+08, Max= 1.0042e+08, RMS= 1.8589e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Original member 3 : Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.7276e+08, Max= 9.4808e+07, RMS= 1.7978e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.7276e+08, Max= 9.4808e+07, RMS= 1.7978e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Original member 4 : Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.7215e+08, Max= 9.9052e+07, RMS= 1.8304e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.7215e+08, Max= 9.9052e+07, RMS= 1.8304e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Ensemble mean: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.7486e+08, Max= 9.6575e+07, RMS= 1.8216e+08 -Test : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.7486e+08, Max= 9.6575e+07, RMS= 1.8216e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Recentered member 0 : Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.7348e+08, Max= 1.5063e+08, RMS= 1.8806e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.7348e+08, Max= 1.5063e+08, RMS= 1.8806e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Recentered member 1 : Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6614e+08, Max= 9.2017e+07, RMS= 1.8151e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6614e+08, Max= 9.2017e+07, RMS= 1.8151e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Recentered member 2 : Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8521e+08, Max= 9.5093e+07, RMS= 1.8805e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8521e+08, Max= 9.5093e+07, RMS= 1.8805e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Recentered member 3 : Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6890e+08, Max= 8.9656e+07, RMS= 1.8163e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6890e+08, Max= 8.9656e+07, RMS= 1.8163e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Recentered member 4 : Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6887e+08, Max= 9.3721e+07, RMS= 1.8456e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6887e+08, Max= 9.3721e+07, RMS= 1.8456e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/ens_variance.test b/qg/test/testoutput/ens_variance.test index 759f91a54..aadfc0b72 100644 --- a/qg/test/testoutput/ens_variance.test +++ b/qg/test/testoutput/ens_variance.test @@ -1,6 +1,4 @@ Test : Variance: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 4.7139e+11, Max= 1.5009e+15, RMS= 2.8630e+14 +Test : Streamfunction : Min= 4.7139e+11, Max= 1.5009e+15, RMS= 2.8630e+14 diff --git a/qg/test/testoutput/ens_variance_inflation_field.test b/qg/test/testoutput/ens_variance_inflation_field.test index b2009e882..822269926 100644 --- a/qg/test/testoutput/ens_variance_inflation_field.test +++ b/qg/test/testoutput/ens_variance_inflation_field.test @@ -1,6 +1,4 @@ Test : Variance: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 1.8856e+12, Max= 6.0037e+15, RMS= 1.1452e+15 +Test : Streamfunction : Min= 1.8856e+12, Max= 6.0037e+15, RMS= 1.1452e+15 diff --git a/qg/test/testoutput/ens_variance_inflation_value.test b/qg/test/testoutput/ens_variance_inflation_value.test index b2009e882..822269926 100644 --- a/qg/test/testoutput/ens_variance_inflation_value.test +++ b/qg/test/testoutput/ens_variance_inflation_value.test @@ -1,6 +1,4 @@ Test : Variance: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= 1.8856e+12, Max= 6.0037e+15, RMS= 1.1452e+15 +Test : Streamfunction : Min= 1.8856e+12, Max= 6.0037e+15, RMS= 1.1452e+15 diff --git a/qg/test/testoutput/forecast.test b/qg/test/testoutput/forecast.test index 8a7fbfb71..17bd52b83 100644 --- a/qg/test/testoutput/forecast.test +++ b/qg/test/testoutput/forecast.test @@ -1,16 +1,12 @@ Test : Initial state: Test : Valid time: 2009-12-31T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8796e+08, Max= 1.0889e+08, RMS= 2.0493e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8796e+08, Max= 1.0889e+08, RMS= 2.0493e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Final state: Test : Valid time: 2010-01-02T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.1708e+08, Max= 1.0232e+08, RMS= 1.6357e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.1708e+08, Max= 1.0232e+08, RMS= 1.6357e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/gen_ens_pert_B.test b/qg/test/testoutput/gen_ens_pert_B.test index 38d762e34..4dd5a10c7 100644 --- a/qg/test/testoutput/gen_ens_pert_B.test +++ b/qg/test/testoutput/gen_ens_pert_B.test @@ -1,88 +1,66 @@ Test : Initial state: Test : Valid time: 2009-12-31T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.8796e+08, Max= 1.0889e+08, RMS= 2.0493e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.8796e+08, Max= 1.0889e+08, RMS= 2.0493e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Member 0 final state: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4224e+08, Max= 1.2418e+08, RMS= 1.7584e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4224e+08, Max= 1.2418e+08, RMS= 1.7584e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Member 1 final state: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3426e+08, Max= 9.8104e+07, RMS= 1.6992e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3426e+08, Max= 9.8104e+07, RMS= 1.6992e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Member 2 final state: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4601e+08, Max= 1.0169e+08, RMS= 1.7665e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4601e+08, Max= 1.0169e+08, RMS= 1.7665e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Member 3 final state: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3446e+08, Max= 1.0277e+08, RMS= 1.6981e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3446e+08, Max= 1.0277e+08, RMS= 1.6981e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Member 4 final state: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3100e+08, Max= 9.5883e+07, RMS= 1.6904e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3100e+08, Max= 9.5883e+07, RMS= 1.6904e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Member 5 final state: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3282e+08, Max= 1.0319e+08, RMS= 1.7295e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3282e+08, Max= 1.0319e+08, RMS= 1.7295e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Member 6 final state: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3246e+08, Max= 9.6905e+07, RMS= 1.7096e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3246e+08, Max= 9.6905e+07, RMS= 1.7096e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Member 7 final state: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.2419e+08, Max= 1.2494e+08, RMS= 1.6860e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.2419e+08, Max= 1.2494e+08, RMS= 1.6860e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Member 8 final state: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3878e+08, Max= 9.3644e+07, RMS= 1.7330e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3878e+08, Max= 9.3644e+07, RMS= 1.7330e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Member 9 final state: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3011e+08, Max= 1.0896e+08, RMS= 1.6876e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3011e+08, Max= 1.0896e+08, RMS= 1.6876e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/hofx.test b/qg/test/testoutput/hofx.test index f2d05723a..803f972a2 100644 --- a/qg/test/testoutput/hofx.test +++ b/qg/test/testoutput/hofx.test @@ -1,19 +1,15 @@ Test : Initial state: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Final state: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.5859e+08, Max= 1.0654e+08, RMS= 1.7877e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.5859e+08, Max= 1.0654e+08, RMS= 1.7877e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : H(x): Test : Stream nobs= 300 Min= -5.6944e+08, Max= 9.0622e+07, Average= -1.3204e+08 Test : Wind nobs= 160 Min= -8.3586e+01, Max= 1.0673e+02, Average= 1.0558e+01 diff --git a/qg/test/testoutput/hofx3d.test b/qg/test/testoutput/hofx3d.test index 80872a33c..2df8e55f1 100644 --- a/qg/test/testoutput/hofx3d.test +++ b/qg/test/testoutput/hofx3d.test @@ -1,11 +1,9 @@ Test : State: Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.0526e+08, Max= 1.1159e+08, RMS= 1.8719e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.0526e+08, Max= 1.1159e+08, RMS= 1.8719e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : H(x): Test : Stream nobs= 300 Min= -5.8159e+08, Max= 8.9514e+07, Average= -1.3681e+08 Test : Wind nobs= 160 Min= -8.3304e+01, Max= 1.0663e+02, Average= 1.0113e+01 diff --git a/qg/test/testoutput/letkf.test b/qg/test/testoutput/letkf.test index 4ba733ee8..b4434c856 100644 --- a/qg/test/testoutput/letkf.test +++ b/qg/test/testoutput/letkf.test @@ -1,113 +1,83 @@ Test : Initial state for member 1: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.5492e+08, Max= 1.1027e+08, RMS= 1.7951e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.5492e+08, Max= 1.1027e+08, RMS= 1.7951e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4224e+08, Max= 1.2418e+08, RMS= 1.7584e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4224e+08, Max= 1.2418e+08, RMS= 1.7584e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Initial state for member 2: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6931e+08, Max= 9.9327e+07, RMS= 1.7976e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6931e+08, Max= 9.9327e+07, RMS= 1.7976e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.5020e+08, Max= 9.8531e+07, RMS= 1.7427e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.5020e+08, Max= 9.8531e+07, RMS= 1.7427e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3426e+08, Max= 9.8104e+07, RMS= 1.6992e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3426e+08, Max= 9.8104e+07, RMS= 1.6992e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Initial state for member 3: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.9074e+08, Max= 1.0042e+08, RMS= 1.8589e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.9074e+08, Max= 1.0042e+08, RMS= 1.8589e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6561e+08, Max= 1.0048e+08, RMS= 1.8093e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6561e+08, Max= 1.0048e+08, RMS= 1.8093e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4601e+08, Max= 1.0169e+08, RMS= 1.7665e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4601e+08, Max= 1.0169e+08, RMS= 1.7665e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Initial state for member 4: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.7276e+08, Max= 9.4808e+07, RMS= 1.7978e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.7276e+08, Max= 9.4808e+07, RMS= 1.7978e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.5080e+08, Max= 9.8565e+07, RMS= 1.7375e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.5080e+08, Max= 9.8565e+07, RMS= 1.7375e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3446e+08, Max= 1.0277e+08, RMS= 1.6981e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3446e+08, Max= 1.0277e+08, RMS= 1.6981e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Initial state for member 5: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.7215e+08, Max= 9.9052e+07, RMS= 1.8304e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.7215e+08, Max= 9.9052e+07, RMS= 1.8304e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4705e+08, Max= 9.6271e+07, RMS= 1.7467e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4705e+08, Max= 9.6271e+07, RMS= 1.7467e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3100e+08, Max= 9.5883e+07, RMS= 1.6904e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3100e+08, Max= 9.5883e+07, RMS= 1.6904e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : H(x) for member 1: Test : Stream nobs= 300 Min= -5.4020e+08, Max= 8.6950e+07, Average= -1.2080e+08 Test : Wind nobs= 160 Min= -1.2372e+02, Max= 9.5123e+01, Average= 1.1752e+01 @@ -139,47 +109,35 @@ Test : WSpeed nobs= 300 Min= -2.7821e+01, Max= 3.2298e+01, Average= 4.182 Test : Background mean : Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.7486e+08, Max= 9.6575e+07, RMS= 1.8216e+08 -Test : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.7486e+08, Max= 9.6575e+07, RMS= 1.8216e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.5279e+08, Max= 9.3048e+07, RMS= 1.7609e+08 -Test : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.5279e+08, Max= 9.3048e+07, RMS= 1.7609e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.3566e+08, Max= 9.5381e+07, RMS= 1.7157e+08 -Test : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.3566e+08, Max= 9.5381e+07, RMS= 1.7157e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Analysis mean : Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.9026e+08, Max= 9.5856e+07, RMS= 1.8792e+08 -Test : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.9026e+08, Max= 9.5856e+07, RMS= 1.8792e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.6092e+08, Max= 9.1362e+07, RMS= 1.8251e+08 -Test : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.6092e+08, Max= 9.1362e+07, RMS= 1.8251e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4655e+08, Max= 9.6366e+07, RMS= 1.7841e+08 -Test : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4655e+08, Max= 9.6366e+07, RMS= 1.7841e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : H(x) for member 1: Test : Stream nobs= 300 Min= -5.4486e+08, Max= 9.1832e+07, Average= -1.2800e+08 Test : Wind nobs= 160 Min= -9.6559e+01, Max= 1.0881e+02, Average= 1.1563e+01 diff --git a/qg/test/testoutput/make_obs_3d.test b/qg/test/testoutput/make_obs_3d.test index 14019a7b7..f5bfd1c47 100644 --- a/qg/test/testoutput/make_obs_3d.test +++ b/qg/test/testoutput/make_obs_3d.test @@ -1,19 +1,15 @@ Test : Initial state: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Final state: Test : Valid time: 2010-01-01T15:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4390e+08, Max= 1.0635e+08, RMS= 1.7643e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4390e+08, Max= 1.0635e+08, RMS= 1.7643e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : H(x): Test : Stream nobs= 600 Min= -5.7803e+08, Max= 1.2841e+08, Average= -1.1552e+08 Test : Wind nobs= 600 Min= -7.1086e+01, Max= 1.1626e+02, Average= 1.3189e+01 diff --git a/qg/test/testoutput/make_obs_4d_12h.test b/qg/test/testoutput/make_obs_4d_12h.test index f2d05723a..803f972a2 100644 --- a/qg/test/testoutput/make_obs_4d_12h.test +++ b/qg/test/testoutput/make_obs_4d_12h.test @@ -1,19 +1,15 @@ Test : Initial state: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Final state: Test : Valid time: 2010-01-01T12:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.5859e+08, Max= 1.0654e+08, RMS= 1.7877e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.5859e+08, Max= 1.0654e+08, RMS= 1.7877e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : H(x): Test : Stream nobs= 300 Min= -5.6944e+08, Max= 9.0622e+07, Average= -1.3204e+08 Test : Wind nobs= 160 Min= -8.3586e+01, Max= 1.0673e+02, Average= 1.0558e+01 diff --git a/qg/test/testoutput/make_obs_4d_24h.test b/qg/test/testoutput/make_obs_4d_24h.test index 03f2db3af..be6aada2a 100644 --- a/qg/test/testoutput/make_obs_4d_24h.test +++ b/qg/test/testoutput/make_obs_4d_24h.test @@ -1,19 +1,15 @@ Test : Initial state: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Final state: Test : Valid time: 2010-01-02T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4100e+08, Max= 1.1396e+08, RMS= 1.7043e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4100e+08, Max= 1.1396e+08, RMS= 1.7043e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : H(x): Test : Stream nobs= 800 Min= -6.1451e+08, Max= 1.4118e+08, Average= -1.1514e+08 Test : Wind nobs= 800 Min= -7.0969e+01, Max= 1.1687e+02, Average= 1.3047e+01 diff --git a/qg/test/testoutput/make_obs_4d_biased.test b/qg/test/testoutput/make_obs_4d_biased.test index 9ceaea187..3308845df 100644 --- a/qg/test/testoutput/make_obs_4d_biased.test +++ b/qg/test/testoutput/make_obs_4d_biased.test @@ -1,19 +1,15 @@ Test : Initial state: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Final state: Test : Valid time: 2010-01-02T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4100e+08, Max= 1.1396e+08, RMS= 1.7043e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4100e+08, Max= 1.1396e+08, RMS= 1.7043e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : H(x): Test : Stream nobs= 800 Min= -6.2451e+08, Max= 1.3118e+08, Average= -1.2514e+08 Test : Wind nobs= 800 Min= -6.9902e+01, Max= 1.2687e+02, Average= 1.8047e+01 diff --git a/qg/test/testoutput/rtpp.test b/qg/test/testoutput/rtpp.test index ce6729552..70b372e67 100644 --- a/qg/test/testoutput/rtpp.test +++ b/qg/test/testoutput/rtpp.test @@ -1,32 +1,24 @@ Test : Background member 1: Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.5492e+08, Max= 1.1027e+08, RMS= 1.7951e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.5492e+08, Max= 1.1027e+08, RMS= 1.7951e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Analysis member 1: Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.5020e+08, Max= 9.8531e+07, RMS= 1.7427e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.5020e+08, Max= 9.8531e+07, RMS= 1.7427e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Updated Analysis member 1: Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.5256e+08, Max= 9.3779e+07, RMS= 1.7629e+08 -Test : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.5256e+08, Max= 9.3779e+07, RMS= 1.7629e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Analysis mean: Test : Valid time: 2010-01-01T06:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.5256e+08, Max= 9.3779e+07, RMS= 1.7629e+08 -Test : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.5256e+08, Max= 9.3779e+07, RMS= 1.7629e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/static_b_init.test b/qg/test/testoutput/static_b_init.test index c324396c7..69182978b 100644 --- a/qg/test/testoutput/static_b_init.test +++ b/qg/test/testoutput/static_b_init.test @@ -1,5 +1,3 @@ Test : Valid time: 2009-12-31T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are not activated -Test : Min= -4.5233e+07, Max= 3.6083e+07, RMS= 1.5609e+07 +Test : Streamfunction : Min= -4.5233e+07, Max= 3.6083e+07, RMS= 1.5609e+07 diff --git a/qg/test/testoutput/truth.test b/qg/test/testoutput/truth.test index df0972b98..1c8ffe43c 100644 --- a/qg/test/testoutput/truth.test +++ b/qg/test/testoutput/truth.test @@ -1,16 +1,12 @@ Test : Initial state: Test : Valid time: 2009-12-15T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -3.8125e+08, Max= -4.7657e+06, RMS= 1.6644e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -3.8125e+08, Max= -4.7657e+06, RMS= 1.6644e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 Test : Final state: Test : Valid time: 2010-01-02T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= -4.4486e+08, Max= 1.2473e+08, RMS= 1.7599e+08 -Test : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Test : Streamfunction : Min= -4.4486e+08, Max= 1.2473e+08, RMS= 1.7599e+08 +Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 +Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/uniform_field_hybrid.test b/qg/test/testoutput/uniform_field_hybrid.test index f5f96d825..47818d51e 100644 --- a/qg/test/testoutput/uniform_field_hybrid.test +++ b/qg/test/testoutput/uniform_field_hybrid.test @@ -1,16 +1,12 @@ Test : Initial state: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= 5.0000e-01, Max= 5.0000e-01, RMS= 5.0000e-01 -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 +Test : Streamfunction : Min= 5.0000e-01, Max= 5.0000e-01, RMS= 5.0000e-01 +Test : Streamfunction LBC : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Potential vorticity LBC: Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 Test : Final state: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= 5.0000e-01, Max= 5.0000e-01, RMS= 5.0000e-01 -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 +Test : Streamfunction : Min= 5.0000e-01, Max= 5.0000e-01, RMS= 5.0000e-01 +Test : Streamfunction LBC : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Potential vorticity LBC: Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 diff --git a/qg/test/testoutput/uniform_field_inflation.test b/qg/test/testoutput/uniform_field_inflation.test index d28320b90..9b0959826 100644 --- a/qg/test/testoutput/uniform_field_inflation.test +++ b/qg/test/testoutput/uniform_field_inflation.test @@ -1,16 +1,12 @@ Test : Initial state: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= 2.0000e+00, Max= 2.0000e+00, RMS= 2.0000e+00 -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 +Test : Streamfunction : Min= 2.0000e+00, Max= 2.0000e+00, RMS= 2.0000e+00 +Test : Streamfunction LBC : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Potential vorticity LBC: Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 Test : Final state: Test : Valid time: 2010-01-01T00:00:00Z Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Min= 2.0000e+00, Max= 2.0000e+00, RMS= 2.0000e+00 -Test : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 +Test : Streamfunction : Min= 2.0000e+00, Max= 2.0000e+00, RMS= 2.0000e+00 +Test : Streamfunction LBC : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +Test : Potential vorticity LBC: Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 diff --git a/src/oops/base/variables_f.cc b/src/oops/base/variables_f.cc index 74a796736..99dbddb40 100644 --- a/src/oops/base/variables_f.cc +++ b/src/oops/base/variables_f.cc @@ -16,6 +16,18 @@ namespace oops { +// ----------------------------------------------------------------------------- +oops::Variables* variables_empty_ctor_f() { + oops::Variables* vars = new oops::Variables(); + return vars; +} + +// ----------------------------------------------------------------------------- +void variables_destruct_f(oops::Variables * vars) { + ASSERT(vars != nullptr); + delete vars; +} + // ----------------------------------------------------------------------------- void variables_push_back_f(oops::Variables & vars, const char * vname) { vars.push_back(std::string(vname)); diff --git a/src/oops/base/variables_f.h b/src/oops/base/variables_f.h index 04626e475..2c0c0865f 100644 --- a/src/oops/base/variables_f.h +++ b/src/oops/base/variables_f.h @@ -19,6 +19,8 @@ namespace oops { extern "C" { + oops::Variables* variables_empty_ctor_f(); + void variables_destruct_f(oops::Variables *); void variables_push_back_f(oops::Variables &, const char *); size_t variables_size_f(const oops::Variables &); void variables_getvariablelength_f(const oops::Variables &, const size_t &, size_t &); diff --git a/src/oops/base/variables_interface.f b/src/oops/base/variables_interface.f index 9e4b0fff8..9391c1b03 100644 --- a/src/oops/base/variables_interface.f +++ b/src/oops/base/variables_interface.f @@ -10,6 +10,22 @@ interface !------------------------------------------------------------------------------- +type(c_ptr) function c_variables_empty_ctor() bind(C, name='variables_empty_ctor_f') + use, intrinsic :: iso_c_binding + implicit none +end function c_variables_empty_ctor + +!------------------------------------------------------------------------------- + +subroutine c_variables_destruct(vars) bind(C, name='variables_destruct_f') + use, intrinsic :: iso_c_binding + implicit none + + type(c_ptr), value :: vars +end subroutine c_variables_destruct + +!------------------------------------------------------------------------------- + subroutine c_variables_push_back(vars, str) bind(C, name='variables_push_back_f') use, intrinsic :: iso_c_binding implicit none diff --git a/src/oops/base/variables_mod.F90 b/src/oops/base/variables_mod.F90 index f3fe41ec0..6e8a3e7ac 100644 --- a/src/oops/base/variables_mod.F90 +++ b/src/oops/base/variables_mod.F90 @@ -17,6 +17,8 @@ module oops_variables_mod private type(c_ptr) :: ptr contains + procedure, public :: destruct + procedure, private :: push_back_string procedure, private :: push_back_vector @@ -31,6 +33,7 @@ module oops_variables_mod interface oops_variables module procedure ctor_from_ptr + module procedure empty_ctor end interface private @@ -42,6 +45,8 @@ module oops_variables_mod !------------------------------------------------------------------------------- function ctor_from_ptr(ptr) result(this) + use iso_c_binding, only: c_ptr + implicit none type(oops_variables) :: this type(c_ptr), intent(in) :: ptr @@ -50,8 +55,27 @@ end function ctor_from_ptr !------------------------------------------------------------------------------- +function empty_ctor() result(this) + type(oops_variables) :: this + + this%ptr = c_variables_empty_ctor() +end function empty_ctor + +!------------------------------------------------------------------------------- + +subroutine destruct(this) + use iso_c_binding, only: c_null_ptr + implicit none + class(oops_variables), intent(inout) :: this + + call c_variables_destruct(this%ptr) + this%ptr = c_null_ptr +end subroutine destruct + +!------------------------------------------------------------------------------- + subroutine push_back_string(this, varname) - use iso_c_binding, only: c_ptr, c_char + use iso_c_binding, only: c_char use string_f_c_mod implicit none class(oops_variables), intent(in) :: this @@ -68,7 +92,7 @@ end subroutine push_back_string !------------------------------------------------------------------------------- subroutine push_back_vector(this, varnames) - use iso_c_binding, only: c_ptr, c_char + use iso_c_binding, only: c_char use string_f_c_mod implicit none class(oops_variables), intent(in) :: this @@ -97,7 +121,7 @@ end function nvars !------------------------------------------------------------------------------- function variable(this, jj) result(varname) - use iso_c_binding, only: c_ptr, c_char, c_size_t + use iso_c_binding, only: c_char, c_size_t use string_f_c_mod implicit none From dc766277ccdaf2c5556446ef605939d4d3b46e1c Mon Sep 17 00:00:00 2001 From: Clementine Gas <43183478+cmgas@users.noreply.github.com> Date: Tue, 13 Apr 2021 16:56:43 -0600 Subject: [PATCH 104/142] Print output of block Lanczos at higher precision (#1153) * Change print precision for cost J * Full precision * Update tests * Update to new compare * Reverse new compare script * less precision for test Co-authored-by: Clementine Gas Co-authored-by: Anna Shlyaeva --- l95/test/testoutput/eda_3dvar_block.test | 84 +++++++++---------- qg/test/testoutput/eda_3dvar_block.test | 48 +++++------ .../assimilation/DRPBlockLanczosMinimizer.h | 15 +++- 3 files changed, 77 insertions(+), 70 deletions(-) diff --git a/l95/test/testoutput/eda_3dvar_block.test b/l95/test/testoutput/eda_3dvar_block.test index 74d831748..e51f9793a 100644 --- a/l95/test/testoutput/eda_3dvar_block.test +++ b/l95/test/testoutput/eda_3dvar_block.test @@ -1,55 +1,55 @@ Test : CostJb : Nonlinear Jb = 0 Test : CostJo : Nonlinear Jo(Lorenz 95) = 1003.27, nobs = 120, Jo/n = 8.36061, err = 0.4 Test : CostFunction: Nonlinear J = 1003.27 -Test : Norm reduction all members ( 1) = 0.226112 0.246079 -Test : Quadratic cost function all members: J ( 1) = 157.904 186.615 -Test : Norm reduction all members ( 2) = 0.0711168 0.0801927 -Test : Quadratic cost function all members: J ( 2) = 119.121 129.88 -Test : Norm reduction all members ( 3) = 0.0329696 0.0324763 -Test : Quadratic cost function all members: J ( 3) = 114.134 121.796 -Test : Norm reduction all members ( 4) = 0.0143961 0.0146658 -Test : Quadratic cost function all members: J ( 4) = 113.103 120.416 -Test : Norm reduction all members ( 5) = 0.00801082 0.00603926 -Test : Quadratic cost function all members: J ( 5) = 112.853 120.144 -Test : Norm reduction all members ( 6) = 0.00525854 0.0040795 -Test : Quadratic cost function all members: J ( 6) = 112.769 120.073 -Test : Norm reduction all members ( 7) = 0.00313676 0.00251749 -Test : Quadratic cost function all members: J ( 7) = 112.735 120.046 -Test : Norm reduction all members ( 8) = 0.00178513 0.00129232 -Test : Quadratic cost function all members: J ( 8) = 112.723 120.036 -Test : Norm reduction all members ( 9) = 0.000947141 0.000990084 -Test : Quadratic cost function all members: J ( 9) = 112.719 120.033 -Test : Norm reduction all members (10) = 0.000587397 0.000553939 -Test : Quadratic cost function all members: J (10) = 112.718 120.031 +Test : Norm reduction all members ( 1) = 0.226112, 0.246079 +Test : Quadratic cost function all members: J ( 1) = 157.904, 186.615 +Test : Norm reduction all members ( 2) = 0.0711168, 0.0801927 +Test : Quadratic cost function all members: J ( 2) = 119.121, 129.88 +Test : Norm reduction all members ( 3) = 0.0329696, 0.0324763 +Test : Quadratic cost function all members: J ( 3) = 114.134, 121.796 +Test : Norm reduction all members ( 4) = 0.0143961, 0.0146658 +Test : Quadratic cost function all members: J ( 4) = 113.103, 120.416 +Test : Norm reduction all members ( 5) = 0.00801082, 0.00603926 +Test : Quadratic cost function all members: J ( 5) = 112.853, 120.144 +Test : Norm reduction all members ( 6) = 0.00525854, 0.0040795 +Test : Quadratic cost function all members: J ( 6) = 112.769, 120.073 +Test : Norm reduction all members ( 7) = 0.00313676, 0.00251749 +Test : Quadratic cost function all members: J ( 7) = 112.735, 120.046 +Test : Norm reduction all members ( 8) = 0.00178513, 0.00129232 +Test : Quadratic cost function all members: J ( 8) = 112.723, 120.036 +Test : Norm reduction all members ( 9) = 0.000947141, 0.000990084 +Test : Quadratic cost function all members: J ( 9) = 112.719, 120.033 +Test : Norm reduction all members (10) = 0.000587397, 0.000553939 +Test : Quadratic cost function all members: J (10) = 112.718, 120.031 Test : DRPBlockLanczosMinimizer: reduction in residual norm = 0.000587397 -Test : CostFunction::addIncrement: Analysis: +Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-02T00:00:00Z Test : Min=6.77364, Max=9.00861, Average=8.0056 Test : CostJb : Nonlinear Jb = 97.3394 Test : CostJo : Nonlinear Jo(Lorenz 95) = 15.3786, nobs = 120, Jo/n = 0.128155, err = 0.4 Test : CostFunction: Nonlinear J = 112.718 -Test : Norm reduction all members ( 1) = 0.603209 0.531155 -Test : Quadratic cost function all members: J ( 1) = 112.718 120.031 -Test : Norm reduction all members ( 2) = 0.486549 0.413971 -Test : Quadratic cost function all members: J ( 2) = 112.718 120.031 -Test : Norm reduction all members ( 3) = 0.225252 0.225474 -Test : Quadratic cost function all members: J ( 3) = 112.717 120.031 -Test : Norm reduction all members ( 4) = 0.136703 0.150484 -Test : Quadratic cost function all members: J ( 4) = 112.717 120.031 -Test : Norm reduction all members ( 5) = 0.0844788 0.0897435 -Test : Quadratic cost function all members: J ( 5) = 112.717 120.031 -Test : Norm reduction all members ( 6) = 0.0543314 0.0522884 -Test : Quadratic cost function all members: J ( 6) = 112.717 120.031 -Test : Norm reduction all members ( 7) = 0.0325225 0.0286418 -Test : Quadratic cost function all members: J ( 7) = 112.717 120.031 -Test : Norm reduction all members ( 8) = 0.0177981 0.0162206 -Test : Quadratic cost function all members: J ( 8) = 112.717 120.031 -Test : Norm reduction all members ( 9) = 0.00868724 0.00777496 -Test : Quadratic cost function all members: J ( 9) = 112.717 120.031 -Test : Norm reduction all members (10) = 0.00319633 0.00372359 -Test : Quadratic cost function all members: J (10) = 112.717 120.031 +Test : Norm reduction all members ( 1) = 0.603209, 0.531155 +Test : Quadratic cost function all members: J ( 1) = 112.718, 120.031 +Test : Norm reduction all members ( 2) = 0.486549, 0.413971 +Test : Quadratic cost function all members: J ( 2) = 112.718, 120.031 +Test : Norm reduction all members ( 3) = 0.225252, 0.225474 +Test : Quadratic cost function all members: J ( 3) = 112.717, 120.031 +Test : Norm reduction all members ( 4) = 0.136703, 0.150484 +Test : Quadratic cost function all members: J ( 4) = 112.717, 120.031 +Test : Norm reduction all members ( 5) = 0.0844788, 0.0897435 +Test : Quadratic cost function all members: J ( 5) = 112.717, 120.031 +Test : Norm reduction all members ( 6) = 0.0543314, 0.0522884 +Test : Quadratic cost function all members: J ( 6) = 112.717, 120.031 +Test : Norm reduction all members ( 7) = 0.0325225, 0.0286418 +Test : Quadratic cost function all members: J ( 7) = 112.717, 120.031 +Test : Norm reduction all members ( 8) = 0.0177981, 0.0162206 +Test : Quadratic cost function all members: J ( 8) = 112.717, 120.031 +Test : Norm reduction all members ( 9) = 0.00868724, 0.00777496 +Test : Quadratic cost function all members: J ( 9) = 112.717, 120.031 +Test : Norm reduction all members (10) = 0.00319633, 0.00372359 +Test : Quadratic cost function all members: J (10) = 112.717, 120.031 Test : DRPBlockLanczosMinimizer: reduction in residual norm = 0.00319633 -Test : CostFunction::addIncrement: Analysis: +Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-02T00:00:00Z Test : Min=6.77193, Max=9.00912, Average=8.00572 Test : CostJb : Nonlinear Jb = 97.3419 diff --git a/qg/test/testoutput/eda_3dvar_block.test b/qg/test/testoutput/eda_3dvar_block.test index 3ca6ddcd6..2be2d0215 100644 --- a/qg/test/testoutput/eda_3dvar_block.test +++ b/qg/test/testoutput/eda_3dvar_block.test @@ -3,12 +3,12 @@ Test : CostJo : Nonlinear Jo(Stream) = 10879.7, nobs = 600, Jo/n = 18.1329 Test : CostJo : Nonlinear Jo(Wind) = 1663.07, nobs = 600, Jo/n = 2.77178, err = 6 Test : CostJo : Nonlinear Jo(WSpeed) = 163.034, nobs = 300, Jo/n = 0.543446, err = 12 Test : CostFunction: Nonlinear J = 12705.8 -Test : Norm reduction all members ( 1) = 0.402888 0.342551 0.430302 0.347356 -Test : Quadratic cost function all members: J ( 1) = 5681.52 7492.09 3160.43 5694.31 -Test : Norm reduction all members ( 2) = 0.1949 0.165786 0.260803 0.171396 -Test : Quadratic cost function all members: J ( 2) = 3979.77 5093.07 2507.38 3641.38 -Test : Norm reduction all members ( 3) = 0.158052 0.116268 0.185322 0.0977361 -Test : Quadratic cost function all members: J ( 3) = 3313.65 4119.31 2154.65 2990.07 +Test : Norm reduction all members ( 1) = 0.402888, 0.342551, 0.430302, 0.347356 +Test : Quadratic cost function all members: J ( 1) = 5681.52, 7492.09, 3160.43, 5694.31 +Test : Norm reduction all members ( 2) = 0.1949, 0.165786, 0.260803, 0.171396 +Test : Quadratic cost function all members: J ( 2) = 3979.77, 5093.07, 2507.38, 3641.38 +Test : Norm reduction all members ( 3) = 0.158052, 0.116268, 0.185322, 0.0977361 +Test : Quadratic cost function all members: J ( 3) = 3313.65, 4119.31, 2154.65, 2990.07 Test : DRPBlockLanczosMinimizer: reduction in residual norm = 0.158052 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z @@ -21,24 +21,24 @@ Test : CostJo : Nonlinear Jo(Stream) = 2150, nobs = 600, Jo/n = 3.584, err Test : CostJo : Nonlinear Jo(Wind) = 1104, nobs = 600, Jo/n = 1.84, err = 6 Test : CostJo : Nonlinear Jo(WSpeed) = 83.31, nobs = 300, Jo/n = 0.2777, err = 12 Test : CostFunction: Nonlinear J = 3369 -Test : Norm reduction all members ( 1) = 0.7216 0.7109 0.7893 0.8465 -Test : Quadratic cost function all members: J ( 1) = 3027 3825 2014 2814 -Test : Norm reduction all members ( 2) = 1.021 0.9841 1.035 1.024 -Test : Quadratic cost function all members: J ( 2) = 2506 3217 1761 2438 -Test : Norm reduction all members ( 3) = 0.7978 0.7637 0.8287 0.8148 -Test : Quadratic cost function all members: J ( 3) = 2136 2681 1581 2157 -Test : Norm reduction all members ( 4) = 0.426 0.5472 0.4854 0.5317 -Test : Quadratic cost function all members: J ( 4) = 1923 2450 1427 1979 -Test : Norm reduction all members ( 5) = 0.4426 0.3654 0.394 0.4996 -Test : Quadratic cost function all members: J ( 5) = 1784 2130 1326 1798 -Test : Norm reduction all members ( 6) = 0.3488 0.2975 0.3185 0.3828 -Test : Quadratic cost function all members: J ( 6) = 1652 1980 1247 1648 -Test : Norm reduction all members ( 7) = 0.2445 0.2136 0.2516 0.3026 -Test : Quadratic cost function all members: J ( 7) = 1550 1898 1189 1576 -Test : Norm reduction all members ( 8) = 0.1635 0.1692 0.1936 0.2002 -Test : Quadratic cost function all members: J ( 8) = 1447 1789 1136 1491 -Test : Norm reduction all members ( 9) = 0.1256 0.1218 0.1586 0.1376 -Test : Quadratic cost function all members: J ( 9) = 1403 1740 1113 1437 +Test : Norm reduction all members ( 1) = 0.7216, 0.7109, 0.7893, 0.8465 +Test : Quadratic cost function all members: J ( 1) = 3027, 3825, 2014, 2814 +Test : Norm reduction all members ( 2) = 1.021, 0.9841, 1.035, 1.024 +Test : Quadratic cost function all members: J ( 2) = 2506, 3217, 1761, 2438 +Test : Norm reduction all members ( 3) = 0.7978, 0.7637, 0.8287, 0.8148 +Test : Quadratic cost function all members: J ( 3) = 2136, 2681, 1581, 2157 +Test : Norm reduction all members ( 4) = 0.426, 0.5472, 0.4854, 0.5317 +Test : Quadratic cost function all members: J ( 4) = 1923, 2450, 1427, 1979 +Test : Norm reduction all members ( 5) = 0.4426, 0.3654, 0.394, 0.4996 +Test : Quadratic cost function all members: J ( 5) = 1784, 2130, 1326, 1798 +Test : Norm reduction all members ( 6) = 0.3488, 0.2975, 0.3185, 0.3828 +Test : Quadratic cost function all members: J ( 6) = 1652, 1980, 1247, 1648 +Test : Norm reduction all members ( 7) = 0.2445, 0.2136, 0.2516, 0.3026 +Test : Quadratic cost function all members: J ( 7) = 1550, 1898, 1189, 1576 +Test : Norm reduction all members ( 8) = 0.1635, 0.1692, 0.1936, 0.2002 +Test : Quadratic cost function all members: J ( 8) = 1447, 1789, 1136, 1491 +Test : Norm reduction all members ( 9) = 0.1256, 0.1218, 0.1586, 0.1376 +Test : Quadratic cost function all members: J ( 9) = 1403, 1740, 1113, 1437 Test : DRPBlockLanczosMinimizer: reduction in residual norm = 0.1256 Test : CostFunction::addIncrement: Analysis: Test : Valid time: 2010-01-01T12:00:00Z diff --git a/src/oops/assimilation/DRPBlockLanczosMinimizer.h b/src/oops/assimilation/DRPBlockLanczosMinimizer.h index fe5acc55c..744b6a6e2 100644 --- a/src/oops/assimilation/DRPBlockLanczosMinimizer.h +++ b/src/oops/assimilation/DRPBlockLanczosMinimizer.h @@ -228,21 +228,28 @@ double DRPBlockLanczosMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, xh.zero(); xx.zero(); + eigenmat_ tmp_norm; + eigenmat_ tmp_costj; + Eigen::IOFormat HeavyFmt(-2, 0, ", ", ";\n"); + Eigen::IOFormat TestFmt(-1, 0, ", ", ";\n"); + for (int ll = 0; ll < iterTotal; ++ll) { SSLK = - (ss.block(ll*members_, 0, members_, members_)); apply_proj(xh, *Vbase[ll], SSLK, gestag, CommGeo, temp1); apply_proj(xx, *Zbase[ll], SSLK, gestag, CommGeo, temp1); if (outerLoop_ == 0) writeKrylovBasis(diagConf_, *Zbase[ll], ll); + tmp_norm = norm_red_all.block(ll, 0, 1, members_); + tmp_costj = costj_all.block(ll, 0, 1, members_); Log::info() << " Norm reduction all members (" << std::setw(2) << ll+1 << ") = " - << norm_red_all.block(ll, 0, 1, members_) << std::endl; + << tmp_norm.format(HeavyFmt) << std::endl; Log::info() << " Quadratic cost function all members: J (" << std::setw(2) << ll+1 << ") = " - << costj_all.block(ll, 0, 1, members_) << std::endl; + << tmp_costj.format(HeavyFmt) << std::endl; Log::test() << " Norm reduction all members (" << std::setw(2) << ll+1 << ") = " - << norm_red_all.block(ll, 0, 1, members_) << std::endl; + << tmp_norm.format(TestFmt) << std::endl; Log::test() << " Quadratic cost function all members: J (" << std::setw(2) << ll+1 << ") = " - << costj_all.block(ll, 0, 1, members_) << std::endl; + << tmp_costj.format(TestFmt) << std::endl; } eckit::mpi::deleteComm(CommGeoName); From 586767a92e90935f254e76ad7d75032f7c4ca9ba Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 14 Apr 2021 09:52:18 -0600 Subject: [PATCH 105/142] add packEigenSize and optimize Departures::packEigen (#1154) --- l95/src/lorenz95/ObsVec1D.h | 2 ++ qg/model/ObsVecQG.h | 1 + src/oops/base/Departures.h | 25 +++++++++++++++++++++---- src/oops/base/DeparturesEnsemble.h | 2 +- src/oops/interface/ObsVector.h | 11 +++++++---- src/test/base/Departures.h | 3 ++- src/test/base/DeparturesEnsemble.h | 2 +- src/test/interface/ObsVector.h | 4 ++-- 8 files changed, 37 insertions(+), 13 deletions(-) diff --git a/l95/src/lorenz95/ObsVec1D.h b/l95/src/lorenz95/ObsVec1D.h index 339406f1e..0d8593a12 100644 --- a/l95/src/lorenz95/ObsVec1D.h +++ b/l95/src/lorenz95/ObsVec1D.h @@ -47,6 +47,8 @@ class ObsVec1D : public util::Printable, ObsVec1D & operator/= (const ObsVec1D &); Eigen::VectorXd packEigen() const; + size_t packEigenSize() const {return nobs();} + size_t size() const {return data_.size();} const double & operator[](const std::size_t ii) const {return data_.at(ii);} double & operator[](const std::size_t ii) {return data_.at(ii);} diff --git a/qg/model/ObsVecQG.h b/qg/model/ObsVecQG.h index a9235c23e..55ff1d0f8 100644 --- a/qg/model/ObsVecQG.h +++ b/qg/model/ObsVecQG.h @@ -46,6 +46,7 @@ class ObsVecQG : public util::Printable, ObsVecQG & operator*= (const ObsVecQG &); ObsVecQG & operator/= (const ObsVecQG &); Eigen::VectorXd packEigen() const; + size_t packEigenSize() const {return nobs();} /// set all values to zero void zero(); diff --git a/src/oops/base/Departures.h b/src/oops/base/Departures.h index e06e3c961..e197f8857 100644 --- a/src/oops/base/Departures.h +++ b/src/oops/base/Departures.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -78,6 +79,8 @@ class Departures : public GeneralizedDepartures { /// Pack departures in an Eigen vector (excluding departures that are masked out) Eigen::VectorXd packEigen() const; +/// Size of departures packed into an Eigen vector + size_t packEigenSize() const; /// Save departures values void save(const std::string &) const; @@ -221,17 +224,31 @@ void Departures::mask(ObsDataVec_ qcflags) { // ----------------------------------------------------------------------------- template Eigen::VectorXd Departures::packEigen() const { - Eigen::VectorXd vec(nobs()); + std::vector len(dep_.size()); + for (size_t idep = 0; idep < dep_.size(); ++idep) { + len[idep] = dep_[idep].packEigenSize(); + } + size_t all_len = std::accumulate(len.begin(), len.end(), 0); + + Eigen::VectorXd vec(all_len); size_t ii = 0; for (size_t idep = 0; idep < dep_.size(); ++idep) { - vec.segment(ii, dep_[idep].nobs()) = dep_[idep].packEigen(); - ii += dep_[idep].nobs(); + vec.segment(ii, len[idep]) = dep_[idep].packEigen(); + ii += len[idep]; } - ASSERT(ii == nobs()); return vec; } // ----------------------------------------------------------------------------- template +size_t Departures::packEigenSize() const { + size_t len = 0; + for (size_t idep = 0; idep < dep_.size(); ++idep) { + len += dep_[idep].packEigenSize(); + } + return len; +} +// ----------------------------------------------------------------------------- +template void Departures::save(const std::string & name) const { for (size_t jj = 0; jj < dep_.size(); ++jj) { dep_[jj].save(name); diff --git a/src/oops/base/DeparturesEnsemble.h b/src/oops/base/DeparturesEnsemble.h index 2b1c47d4b..cc697c29e 100644 --- a/src/oops/base/DeparturesEnsemble.h +++ b/src/oops/base/DeparturesEnsemble.h @@ -71,7 +71,7 @@ DeparturesEnsemble::DeparturesEnsemble(const ObsSpaces_ & local, template Eigen::MatrixXd DeparturesEnsemble::packEigen() const { - std::size_t myNobs = ensemblePerturbs_[0].nobs(); + std::size_t myNobs = ensemblePerturbs_[0].packEigenSize(); std::size_t myNens = ensemblePerturbs_.size(); Eigen::MatrixXd depEns(myNens, myNobs); diff --git a/src/oops/interface/ObsVector.h b/src/oops/interface/ObsVector.h index ed4f22c60..aca5a8513 100644 --- a/src/oops/interface/ObsVector.h +++ b/src/oops/interface/ObsVector.h @@ -48,7 +48,7 @@ class ObsVector : public util::Printable, /// Creates vector from \p obsspace. If \p name is specified, reads the /// specified \p name variable from \p obsspace. Otherwise, zero vector is created. explicit ObsVector(const ObsSpace & obsspace, const std::string name = ""); - explicit ObsVector(const ObsVector &); + ObsVector(const ObsVector &); ObsVector(const ObsSpace &, const ObsVector &); ~ObsVector(); @@ -64,8 +64,12 @@ class ObsVector : public util::Printable, ObsVector & operator*= (const ObsVector &); ObsVector & operator/= (const ObsVector &); - /// Pack into an Eigen vector (excluding vector elements that are masked out) + /// Pack observations local to this MPI task into an Eigen vector + /// (excluding vector elements that are masked out) Eigen::VectorXd packEigen() const; + /// Number of non-masked out observations local to this MPI task + /// (size of an Eigen vector returned by `packEigen` + size_t packEigenSize() const {return data_->packEigenSize();} void zero(); /// Set this ObsVector to ones (used in tests) @@ -83,7 +87,7 @@ class ObsVector : public util::Printable, void save(const std::string &) const; void read(const std::string &); - /// number of non-masked out observations + /// number of non-masked out observations (across all MPI tasks) unsigned int nobs() const; private: @@ -335,7 +339,6 @@ Eigen::VectorXd ObsVector::packEigen() const { util::Timer timer(classname(), "packEigen"); Eigen::VectorXd vec = data_->packEigen(); - ASSERT(vec.size() == nobs()); Log::trace() << "ObsVector::packEigen done" << std::endl; return vec; diff --git a/src/test/base/Departures.h b/src/test/base/Departures.h index ed240a82c..cbf963b6d 100644 --- a/src/test/base/Departures.h +++ b/src/test/base/Departures.h @@ -50,7 +50,8 @@ template void testDepartures() { // check pack operator Eigen::VectorXd ypack = y.packEigen(); - oops::Log::info() << "ypack: " << ypack << std::endl; + EXPECT_EQUAL(ypack.size(), y.packEigenSize()); + oops::Log::test() << "ypack: " << ypack << std::endl; double rms = ypack.norm() / sqrt(ypack.size()); oops::Log::test() << "rms(ypack)=" << rms << std::endl; oops::Log::test() << "y.rms()=" << ref_rms << std::endl; diff --git a/src/test/base/DeparturesEnsemble.h b/src/test/base/DeparturesEnsemble.h index a6f6f194c..3f648e2f8 100644 --- a/src/test/base/DeparturesEnsemble.h +++ b/src/test/base/DeparturesEnsemble.h @@ -38,7 +38,7 @@ template void testDeparturesEnsemble() { // rms for each ensemble member computed using depatures.rms() method size_t myNens = 5; DeparturesEnsemble_ yens(Test_::obspace(), myNens); - size_t myNobs = yens[0].nobs(); + size_t myNobs = yens[0].packEigenSize(); std::vector rms1(myNens); for (size_t ii=0; ii < myNens; ++ii) { diff --git a/src/test/interface/ObsVector.h b/src/test/interface/ObsVector.h index 2b7789139..5b898a81f 100644 --- a/src/test/interface/ObsVector.h +++ b/src/test/interface/ObsVector.h @@ -301,9 +301,9 @@ template void testPackEigen() { double rms1 = ov1.rms(); Eigen::VectorXd vec = ov1.packEigen(); - EXPECT(vec.size() == ov1.nobs()); + EXPECT(vec.size() == ov1.packEigenSize()); - double rms2 = sqrt(vec.squaredNorm() / ov1.nobs()); + double rms2 = sqrt(vec.squaredNorm() / vec.size()); EXPECT(std::abs(rms1-rms2) < tolerance); } } From cdadb951d6ce5abe55316dfcc39f8d103acc19dc Mon Sep 17 00:00:00 2001 From: frolovsa <55715838+frolovsa@users.noreply.github.com> Date: Wed, 14 Apr 2021 11:45:10 -0600 Subject: [PATCH 106/142] add soar function to src/oops/generic (#1156) * added soar function and test * fixed coding norms * moved comments arround * change copyright date Co-authored-by: Anna Shlyaeva --- src/CMakeLists.txt | 6 +++++ src/oops/generic/soar.cc | 22 +++++++++++++++++ src/oops/generic/soar.h | 18 ++++++++++++++ src/test/generic/soar.cc | 51 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+) create mode 100644 src/oops/generic/soar.cc create mode 100644 src/oops/generic/soar.h create mode 100644 src/test/generic/soar.cc diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index edc169a69..6a875336d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -182,6 +182,8 @@ oops/generic/InterpolatorUnstructured.cc oops/generic/LinearModelId.h oops/generic/ObsErrorDiag.h oops/generic/PseudoModel.h +oops/generic/soar.h +oops/generic/soar.cc oops/generic/unstructured_interpolation_mod.F90 oops/generic/VerticalLocEV.h @@ -550,6 +552,10 @@ ecbuild_add_test( TARGET test_generic_gc99 SOURCES test/generic/gc99.cc LIBS oops eckit ) +ecbuild_add_test( TARGET test_generic_soar + SOURCES test/generic/soar.cc + LIBS oops eckit ) + ecbuild_add_test( TARGET test_util_isanypointinvolumeinterior SOURCES test/util/IsAnyPointInVolumeInterior.cc ARGS "test/testinput/empty.yaml" diff --git a/src/oops/generic/soar.cc b/src/oops/generic/soar.cc new file mode 100644 index 000000000..318858238 --- /dev/null +++ b/src/oops/generic/soar.cc @@ -0,0 +1,22 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include +#include + +#include "oops/generic/soar.h" + +namespace oops { + +double soar(const double & distnorm) { + // distnorm - normalized distance c*r, where c is the decay prameter and r is distance + + double soarvalue = (1 + distnorm)*exp(-distnorm); + return soarvalue; +} +} // namespace oops + diff --git a/src/oops/generic/soar.h b/src/oops/generic/soar.h new file mode 100644 index 000000000..efe4d006e --- /dev/null +++ b/src/oops/generic/soar.h @@ -0,0 +1,18 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_GENERIC_SOAR_H_ +#define OOPS_GENERIC_SOAR_H_ + +namespace oops { + +/// computes second order autoregressive function +double soar(const double & distnorm); + +} // namespace oops +#endif // OOPS_GENERIC_SOAR_H_ + diff --git a/src/test/generic/soar.cc b/src/test/generic/soar.cc new file mode 100644 index 000000000..bbc11893a --- /dev/null +++ b/src/test/generic/soar.cc @@ -0,0 +1,51 @@ +/* + * (C) Copyright 2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include +#include + +#include "oops/generic/soar.h" +#include "oops/util/Logger.h" + +#include "eckit/testing/Test.h" + +namespace { + + +// ----------------------------------------------------------------------------- + + CASE("test_soar") { + double t; + t = oops::soar(0.0); + oops::Log::info() << "soar(0.0)=" << t << std::endl; + EXPECT(std::abs(t-1) < 2*DBL_EPSILON); + + t = oops::soar(1.0); + oops::Log::info() << "soar(1.0)=" << t << std::endl; + EXPECT(std::abs(t-0.735758882342885) < 1e-6); + + t = oops::soar(2.0); + oops::Log::info() << "soar(2.0)=" << t << std::endl; + EXPECT(std::abs(t-0.406005849709838) < 1e-6); + + t = oops::soar(4.0); + oops::Log::info() << "soar(4.0)=" << t << std::endl; + EXPECT(std::abs(t-0.091578194443671) < 1e-6); + + t = oops::soar(8.0); + oops::Log::info() << "soar(8.0)=" << t << std::endl; + EXPECT(std::abs(t-0.003019163651123) < 1e-6); + } + +// ----------------------------------------------------------------------------- + +} // anonymous namespace + +int main(int argc, char **argv) +{ + return eckit::testing::run_tests ( argc, argv ); +} From 74d4640c6826840d57806dbe3e37dcc4e76375af Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Wed, 14 Apr 2021 15:12:33 -0600 Subject: [PATCH 107/142] remove unused ObsSpace::printJo (#1159) --- l95/src/lorenz95/ObsTable.cc | 7 ------- l95/src/lorenz95/ObsTable.h | 2 -- l95/src/lorenz95/ObsTableView.cc | 6 ------ l95/src/lorenz95/ObsTableView.h | 3 --- qg/model/ObsSpaceQG.cc | 8 -------- qg/model/ObsSpaceQG.h | 3 --- src/oops/base/ObsSpaces.h | 16 ---------------- src/oops/interface/ObsSpace.h | 11 ----------- 8 files changed, 56 deletions(-) diff --git a/l95/src/lorenz95/ObsTable.cc b/l95/src/lorenz95/ObsTable.cc index 56780f7d6..72e1ebf1c 100644 --- a/l95/src/lorenz95/ObsTable.cc +++ b/l95/src/lorenz95/ObsTable.cc @@ -24,7 +24,6 @@ #include "eckit/exception/Exceptions.h" #include "lorenz95/LocsL95.h" -#include "lorenz95/ObsVec1D.h" #include "oops/mpi/mpi.h" #include "oops/util/abor1_cpp.h" #include "oops/util/DateTime.h" @@ -231,12 +230,6 @@ void ObsTable::random(std::vector & data) const { for (size_t jj = 0; jj < data.size(); ++jj) data[jj] = x[jj]; } -// ----------------------------------------------------------------------------- - -void ObsTable::printJo(const ObsVec1D & ydep, const ObsVec1D & grad) { - oops::Log::info() << "ObsTable::printJo not implemented" << std::endl; -} - // ----------------------------------------------------------------------------- // ObsTable Private Methods // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObsTable.h b/l95/src/lorenz95/ObsTable.h index cfe7a851f..81c56d16c 100644 --- a/l95/src/lorenz95/ObsTable.h +++ b/l95/src/lorenz95/ObsTable.h @@ -29,7 +29,6 @@ namespace eckit { namespace lorenz95 { class LocsL95; - class ObsVec1D; /// A Simple Observation Data Handler /*! @@ -57,7 +56,6 @@ class ObsTable : public oops::ObsSpaceBase, bool has(const std::string & col) const; void generateDistribution(const eckit::Configuration &); void random(std::vector &) const; - void printJo(const ObsVec1D &, const ObsVec1D &); unsigned int nobs() const {return times_.size();} const std::vector locations() const { return locations_; } const std::vector times() const { return times_; } diff --git a/l95/src/lorenz95/ObsTableView.cc b/l95/src/lorenz95/ObsTableView.cc index 8e442cd6a..25e62ee73 100644 --- a/l95/src/lorenz95/ObsTableView.cc +++ b/l95/src/lorenz95/ObsTableView.cc @@ -183,12 +183,6 @@ std::unique_ptr ObsTableView::locations() const { // ----------------------------------------------------------------------------- -void ObsTableView::printJo(const ObsVec1D & x1, const ObsVec1D & x2) { - oops::Log::info() << "ObsTableView::printJo not implemented" << std::endl; -} - -// ----------------------------------------------------------------------------- - void ObsTableView::print(std::ostream & os) const { os << *obstable_ << std::endl; os << "Local observation indices: " << localobs_; diff --git a/l95/src/lorenz95/ObsTableView.h b/l95/src/lorenz95/ObsTableView.h index 9a4e9683f..4ff36d56d 100644 --- a/l95/src/lorenz95/ObsTableView.h +++ b/l95/src/lorenz95/ObsTableView.h @@ -25,8 +25,6 @@ namespace lorenz95 { -class ObsVec1D; - /// A Simple Observation Data Handler /*! * ObsTableView defines a simple observation handler @@ -56,7 +54,6 @@ class ObsTableView : public util::Printable, unsigned int nobs() const; void generateDistribution(const eckit::Configuration &); std::unique_ptr locations() const; - void printJo(const ObsVec1D &, const ObsVec1D &); size_t index(const size_t ii) const {return localobs_[ii];} const std::string & obsname() const {return obstable_->obsname();} diff --git a/qg/model/ObsSpaceQG.cc b/qg/model/ObsSpaceQG.cc index 184dff46e..1095527e4 100644 --- a/qg/model/ObsSpaceQG.cc +++ b/qg/model/ObsSpaceQG.cc @@ -26,8 +26,6 @@ #include "oops/util/Duration.h" #include "oops/util/Logger.h" -#include "model/ObsVecQG.h" - using atlas::array::make_view; namespace qg { @@ -181,12 +179,6 @@ std::unique_ptr ObsSpaceQG::locations() const { // ----------------------------------------------------------------------------- -void ObsSpaceQG::printJo(const ObsVecQG & dy, const ObsVecQG & grad) const { - oops::Log::info() << "ObsSpaceQG::printJo not implemented" << std::endl; -} - -// ----------------------------------------------------------------------------- - int ObsSpaceQG::nobs() const { if ( isLocal_ ) { return localobs_.size(); diff --git a/qg/model/ObsSpaceQG.h b/qg/model/ObsSpaceQG.h index 2a18bf415..06bef05d0 100644 --- a/qg/model/ObsSpaceQG.h +++ b/qg/model/ObsSpaceQG.h @@ -33,7 +33,6 @@ namespace eckit { } namespace qg { - class ObsVecQG; /// \brief ObsSpace for QG model // \details ObsSpaceQG is created for each obs type. The underlying Fortran @@ -59,8 +58,6 @@ class ObsSpaceQG : public oops::ObsSpaceBase { /// create locations for the whole time window std::unique_ptr locations() const; - void printJo(const ObsVecQG &, const ObsVecQG &) const; - /// return number of observations (unique locations) int nobs() const; diff --git a/src/oops/base/ObsSpaces.h b/src/oops/base/ObsSpaces.h index d5bba9858..ffc1c0b2a 100644 --- a/src/oops/base/ObsSpaces.h +++ b/src/oops/base/ObsSpaces.h @@ -31,15 +31,11 @@ #include "oops/util/Timer.h" namespace oops { - template - class Departures; // ----------------------------------------------------------------------------- - template class ObsSpaces : public util::Printable, private util::ObjectCounter > { - typedef Departures Departures_; typedef ObsSpace ObsSpace_; public: @@ -60,9 +56,6 @@ class ObsSpaces : public util::Printable, const util::DateTime & windowStart() const {return wbgn_;} const util::DateTime & windowEnd() const {return wend_;} -/// Other - void printJo(const Departures_ &, const Departures_ &) const; // To be changed - private: void print(std::ostream &) const; @@ -121,15 +114,6 @@ void ObsSpaces::print(std::ostream & os) const { // ----------------------------------------------------------------------------- -template -void ObsSpaces::printJo(const Departures_ & dy, const Departures_ & grad) const { - for (std::size_t jj = 0; jj < spaces_.size(); ++jj) { - spaces_[jj]->printJo(dy[jj], grad[jj]); - } -} - -// ----------------------------------------------------------------------------- - } // namespace oops #endif // OOPS_BASE_OBSSPACES_H_ diff --git a/src/oops/interface/ObsSpace.h b/src/oops/interface/ObsSpace.h index 6031aac9b..3baa21faa 100644 --- a/src/oops/interface/ObsSpace.h +++ b/src/oops/interface/ObsSpace.h @@ -64,7 +64,6 @@ class ObsSpace : public util::Printable, const Variables & obsvariables() const; // Other - void printJo(const ObsVector_ &, const ObsVector_ &) const; const std::string & obsname() const {return obsdb_->obsname();} const eckit::mpi::Comm & timeComm() const {return time_;} @@ -143,16 +142,6 @@ const Variables & ObsSpace::obsvariables() const { // ----------------------------------------------------------------------------- -template -void ObsSpace::printJo(const ObsVector_ & dy, const ObsVector_ & grad) const { - Log::trace() << "ObsSpace::printJo starting" << std::endl; - util::Timer timer(classname(), "printJo"); - obsdb_->printJo(dy.obsvector(), grad.obsvector()); - Log::trace() << "ObsSpace::printJo done" << std::endl; -} - -// ----------------------------------------------------------------------------- - } // namespace oops #endif // OOPS_INTERFACE_OBSSPACE_H_ From f35ccd6f7860d0a6109eed84789d27b8ce99cb44 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Thu, 15 Apr 2021 08:47:35 -0600 Subject: [PATCH 108/142] add ObsIterator for ObsSpace, test + l95 implementation (#1157) * add ObsIterator for ObsSpace, test + l95 implementation * correct comments, missing override, optimization --- l95/src/lorenz95/CMakeLists.txt | 2 + l95/src/lorenz95/Iterator.cc | 4 - l95/src/lorenz95/Iterator.h | 5 +- l95/src/lorenz95/L95Traits.h | 2 + l95/src/lorenz95/ObsIterator.cc | 42 +++++++++ l95/src/lorenz95/ObsIterator.h | 51 +++++++++++ l95/src/lorenz95/ObsTable.h | 4 +- l95/src/lorenz95/ObsTableView.cc | 31 ++++++- l95/src/lorenz95/ObsTableView.h | 7 ++ l95/test/CMakeLists.txt | 6 ++ l95/test/executables/TestObsIterator.cc | 17 ++++ l95/test/testinput/interfaces.yaml | 6 ++ src/CMakeLists.txt | 1 + src/oops/interface/Geometry.h | 2 +- src/oops/interface/GeometryIterator.h | 70 +++++++-------- src/oops/interface/ObsSpace.h | 27 ++++++ src/test/interface/ObsIterator.h | 112 ++++++++++++++++++++++++ 17 files changed, 341 insertions(+), 48 deletions(-) create mode 100644 l95/src/lorenz95/ObsIterator.cc create mode 100644 l95/src/lorenz95/ObsIterator.h create mode 100644 l95/test/executables/TestObsIterator.cc create mode 100644 src/test/interface/ObsIterator.h diff --git a/l95/src/lorenz95/CMakeLists.txt b/l95/src/lorenz95/CMakeLists.txt index 3871770a0..a7be7d4f4 100644 --- a/l95/src/lorenz95/CMakeLists.txt +++ b/l95/src/lorenz95/CMakeLists.txt @@ -42,6 +42,8 @@ set( _l95_srcs ObsBiasCovariance.h ObsBiasParameters.h ObsDiags1D.h + ObsIterator.cc + ObsIterator.h ObsLocGC99.cc ObsLocGC99.h ObsTable.cc diff --git a/l95/src/lorenz95/Iterator.cc b/l95/src/lorenz95/Iterator.cc index 43a0d25e9..fec05f708 100644 --- a/l95/src/lorenz95/Iterator.cc +++ b/l95/src/lorenz95/Iterator.cc @@ -15,10 +15,6 @@ namespace lorenz95 { Iterator::Iterator(const Resolution & res, const int & index): res_(res.npoints()), index_(index) { } -// ----------------------------------------------------------------------------- -Iterator::~Iterator() { -} - // ----------------------------------------------------------------------------- bool Iterator::operator==(const Iterator & other) const { return ((res_ == other.res_) && (index_ == other.index_)); diff --git a/l95/src/lorenz95/Iterator.h b/l95/src/lorenz95/Iterator.h index e304ceac5..dda713fa0 100644 --- a/l95/src/lorenz95/Iterator.h +++ b/l95/src/lorenz95/Iterator.h @@ -31,8 +31,7 @@ class Iterator: public std::iterator using ObsDataVector = lorenz95::ObsData1D; + typedef lorenz95::ObsIterator GeometryIterator; typedef lorenz95::ObservationL95 ObsOperator; typedef lorenz95::ObservationTLAD LinearObsOperator; diff --git a/l95/src/lorenz95/ObsIterator.cc b/l95/src/lorenz95/ObsIterator.cc new file mode 100644 index 000000000..09c0e2ad3 --- /dev/null +++ b/l95/src/lorenz95/ObsIterator.cc @@ -0,0 +1,42 @@ +/* + * (C) Copyright 2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "lorenz95/ObsIterator.h" +#include + +// ----------------------------------------------------------------------------- +namespace lorenz95 { + +// ----------------------------------------------------------------------------- +ObsIterator::ObsIterator(const std::vector & locations, const int & index): + locations_(locations), index_(index) { +} + +// ----------------------------------------------------------------------------- +bool ObsIterator::operator==(const ObsIterator & other) const { + return ((index_ == other.index_) && (locations_ == other.locations_)); +} + +// ----------------------------------------------------------------------------- +bool ObsIterator::operator!=(const ObsIterator & other) const { + return ((index_ != other.index_) || (locations_ != other.locations_)); +} + +// ----------------------------------------------------------------------------- +eckit::geometry::Point2 ObsIterator::operator*() const { + return eckit::geometry::Point2(locations_[index_], 0.0); +} + +// ----------------------------------------------------------------------------- +ObsIterator& ObsIterator::operator++() { + index_++; + return *this; +} + +// ----------------------------------------------------------------------------- + +} // namespace lorenz95 diff --git a/l95/src/lorenz95/ObsIterator.h b/l95/src/lorenz95/ObsIterator.h new file mode 100644 index 000000000..6743489bf --- /dev/null +++ b/l95/src/lorenz95/ObsIterator.h @@ -0,0 +1,51 @@ +/* + * (C) Copyright 2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef LORENZ95_OBSITERATOR_H_ +#define LORENZ95_OBSITERATOR_H_ + +#include +#include +#include + +#include "eckit/geometry/Point2.h" + +#include "oops/util/ObjectCounter.h" +#include "oops/util/Printable.h" + +namespace lorenz95 { + +/// Iterator over all observations +class ObsIterator: public std::iterator, + public util::Printable, + private util::ObjectCounter { + public: + static const std::string classname() {return "lorenz95::ObsIterator";} + + ObsIterator(const std::vector & locations, const int & index); + + bool operator==(const ObsIterator &) const; + bool operator!=(const ObsIterator &) const; + + /// return location of current observation + eckit::geometry::Point2 operator*() const; + + ObsIterator& operator++(); + + private: + void print(std::ostream & os) const override {os << index_;} + + /// locations (in 1D) of all observations + const std::vector locations_; + /// index of a current observation + int index_; +}; + +} // namespace lorenz95 + +#endif // LORENZ95_OBSITERATOR_H_ diff --git a/l95/src/lorenz95/ObsTable.h b/l95/src/lorenz95/ObsTable.h index 81c56d16c..7613142e7 100644 --- a/l95/src/lorenz95/ObsTable.h +++ b/l95/src/lorenz95/ObsTable.h @@ -57,8 +57,8 @@ class ObsTable : public oops::ObsSpaceBase, void generateDistribution(const eckit::Configuration &); void random(std::vector &) const; unsigned int nobs() const {return times_.size();} - const std::vector locations() const { return locations_; } - const std::vector times() const { return times_; } + const std::vector & locations() const { return locations_; } + const std::vector & times() const { return times_; } const oops::Variables & obsvariables() const { return obsvars_; } const std::string & obsname() const {return obsname_;} diff --git a/l95/src/lorenz95/ObsTableView.cc b/l95/src/lorenz95/ObsTableView.cc index 25e62ee73..ab915e932 100644 --- a/l95/src/lorenz95/ObsTableView.cc +++ b/l95/src/lorenz95/ObsTableView.cc @@ -16,6 +16,8 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/types/Types.h" +#include "lorenz95/ObsIterator.h" + #include "oops/util/Logger.h" #include "oops/util/missingValues.h" @@ -39,7 +41,7 @@ ObsTableView::ObsTableView(const ObsTableView & obstable, const eckit::Configuration & conf) : obstable_(obstable.obstable_), localobs_(), obsdist_() { - std::vector locations = obstable.obstable_->locations(); + const std::vector & locations = obstable.obstable_->locations(); const double dist = conf.getDouble("lengthscale"); for (unsigned int jj = 0; jj < obstable.nobs(); ++jj) { double curdist = std::abs(center[0] - locations[jj]); @@ -167,8 +169,8 @@ void ObsTableView::generateDistribution(const eckit::Configuration & conf) { std::unique_ptr ObsTableView::locations() const { // get times and locations from the obsspace - std::vector all_times = obstable_->times(); - std::vector all_locs = obstable_->locations(); + const std::vector & all_times = obstable_->times(); + const std::vector & all_locs = obstable_->locations(); // set up locations const unsigned int nobs = localobs_.size(); std::vector locs(nobs); @@ -181,6 +183,29 @@ std::unique_ptr ObsTableView::locations() const { return std::unique_ptr(new LocsL95(locs, times)); } +// ----------------------------------------------------------------------------- +ObsIterator ObsTableView::begin() const { + const std::vector & all_locs = obstable_->locations(); + // set up locations + const unsigned int nobs = localobs_.size(); + std::vector locs(nobs); + for (unsigned int i = 0; i < nobs; i++) { + locs[i] = all_locs[localobs_[i]]; + } + return ObsIterator(locs, 0); +} +// ----------------------------------------------------------------------------- +ObsIterator ObsTableView::end() const { + const std::vector & all_locs = obstable_->locations(); + // set up locations + const unsigned int nobs = localobs_.size(); + std::vector locs(nobs); + for (unsigned int i = 0; i < nobs; i++) { + locs[i] = all_locs[localobs_[i]]; + } + return ObsIterator(locs, locs.size()); +} + // ----------------------------------------------------------------------------- void ObsTableView::print(std::ostream & os) const { diff --git a/l95/src/lorenz95/ObsTableView.h b/l95/src/lorenz95/ObsTableView.h index 4ff36d56d..da5100002 100644 --- a/l95/src/lorenz95/ObsTableView.h +++ b/l95/src/lorenz95/ObsTableView.h @@ -25,6 +25,8 @@ namespace lorenz95 { +class ObsIterator; + /// A Simple Observation Data Handler /*! * ObsTableView defines a simple observation handler @@ -58,6 +60,11 @@ class ObsTableView : public util::Printable, size_t index(const size_t ii) const {return localobs_[ii];} const std::string & obsname() const {return obstable_->obsname();} + /// iterator to the first observation + ObsIterator begin() const; + /// iterator to the last observation + ObsIterator end() const; + const util::DateTime & windowStart() const {return obstable_->windowStart();} const util::DateTime & windowEnd() const {return obstable_->windowEnd();} const oops::Variables & obsvariables() const { return obstable_->obsvariables(); } diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index 668ae6945..2690fa9cc 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -332,6 +332,12 @@ ecbuild_add_test( TARGET test_l95_localobsspace LIBS lorenz95 TEST_DEPENDS test_l95_truth ) +ecbuild_add_test( TARGET test_l95_obs_iterator + SOURCES executables/TestObsIterator.cc + ARGS "testinput/interfaces.yaml" + LIBS lorenz95 + TEST_DEPENDS test_l95_truth ) + ecbuild_add_test( TARGET test_l95_obslocalization SOURCES executables/TestObsLocalization.cc ARGS "testinput/interfaces.yaml" diff --git a/l95/test/executables/TestObsIterator.cc b/l95/test/executables/TestObsIterator.cc new file mode 100644 index 000000000..69d6edc57 --- /dev/null +++ b/l95/test/executables/TestObsIterator.cc @@ -0,0 +1,17 @@ +/* + * (C) Copyright 2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "lorenz95/L95Traits.h" +#include "oops/runs/Run.h" +#include "test/interface/ObsIterator.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + test::ObsIterator tests; + return run.execute(tests); +} + diff --git a/l95/test/testinput/interfaces.yaml b/l95/test/testinput/interfaces.yaml index 0851d665d..ac811e5cf 100644 --- a/l95/test/testinput/interfaces.yaml +++ b/l95/test/testinput/interfaces.yaml @@ -120,6 +120,12 @@ observations: rms ref: 8.3207407741318846 reference nobs: 160 tolerance: 1.0e-10 + obs iterator test: + reference nlocs: 160 + lon1: 0.0 + lat1: 0.0 + lon2: 0.05 + lat2: 0.0 local obs space: location: diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6a875336d..f4b76b639 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -398,6 +398,7 @@ test/interface/ObsAuxControl.h test/interface/ObsAuxCovariance.h test/interface/ObsAuxIncrement.h test/interface/ObsDataVector.h +test/interface/ObsIterator.h test/interface/ObsLocalization.h test/interface/ObsOperator.h test/interface/ObsSpace.h diff --git a/src/oops/interface/Geometry.h b/src/oops/interface/Geometry.h index 605c68621..424f1b092 100644 --- a/src/oops/interface/Geometry.h +++ b/src/oops/interface/Geometry.h @@ -78,7 +78,7 @@ class Geometry : public util::Printable, /// Iterator to the first gridpoint of Geometry (only used in LocalEnsembleDA) GeometryIterator_ begin() const; - /// Iterator to the last gridpoint fo Geometry (only used in LocalEnsembleDA) + /// Iterator to the past-the-end gridpoint of Geometry (only used in LocalEnsembleDA) GeometryIterator_ end() const; /// Values of vertical coordinate in units specified by string (only used in GETKF) std::vector verticalCoord(std::string &) const; diff --git a/src/oops/interface/GeometryIterator.h b/src/oops/interface/GeometryIterator.h index 14b23c25d..ef06e3c57 100644 --- a/src/oops/interface/GeometryIterator.h +++ b/src/oops/interface/GeometryIterator.h @@ -25,12 +25,12 @@ namespace oops { // ----------------------------------------------------------------------------- -template +template class GeometryIterator: public std::iterator, public util::Printable, - private util::ObjectCounter> { - typedef typename MODEL::GeometryIterator GeometryIterator_; + private util::ObjectCounter> { + typedef typename TRAIT::GeometryIterator GeometryIterator_; public: static const std::string classname() {return "oops::GeometryIterator";} @@ -55,88 +55,88 @@ class GeometryIterator: public std::iterator -GeometryIterator::GeometryIterator(const GeometryIterator& other) { - Log::trace() << "GeometryIterator::GeometryIterator starting" << std::endl; +template +GeometryIterator::GeometryIterator(const GeometryIterator& other) { + Log::trace() << "GeometryIterator::GeometryIterator starting" << std::endl; util::Timer timer(classname(), "GeometryIterator"); geometryiter_.reset(new GeometryIterator_(other.geometryiter())); - Log::trace() << "GeometryIterator::GeometryIterator done" << std::endl; + Log::trace() << "GeometryIterator::GeometryIterator done" << std::endl; } // ----------------------------------------------------------------------------- -template -GeometryIterator::GeometryIterator(const GeometryIterator_& iter) { - Log::trace() << "GeometryIterator::GeometryIterator starting" << std::endl; +template +GeometryIterator::GeometryIterator(const GeometryIterator_& iter) { + Log::trace() << "GeometryIterator::GeometryIterator starting" << std::endl; util::Timer timer(classname(), "GeometryIterator"); geometryiter_.reset(new GeometryIterator_(iter)); - Log::trace() << "GeometryIterator::GeometryIterator done" << std::endl; + Log::trace() << "GeometryIterator::GeometryIterator done" << std::endl; } // ----------------------------------------------------------------------------- -template -GeometryIterator::~GeometryIterator() { - Log::trace() << "GeometryIterator::~GeometryIterator starting" << std::endl; +template +GeometryIterator::~GeometryIterator() { + Log::trace() << "GeometryIterator::~GeometryIterator starting" << std::endl; util::Timer timer(classname(), "~GeometryIterator"); geometryiter_.reset(); - Log::trace() << "GeometryIterator::~GeometryIterator done" << std::endl; + Log::trace() << "GeometryIterator::~GeometryIterator done" << std::endl; } // ----------------------------------------------------------------------------- -template -bool GeometryIterator::operator==(const GeometryIterator& other) { - Log::trace() << "GeometryIterator::operator== starting" << std::endl; +template +bool GeometryIterator::operator==(const GeometryIterator& other) { + Log::trace() << "GeometryIterator::operator== starting" << std::endl; util::Timer timer(classname(), "operator=="); bool equals = (*geometryiter_ == other.geometryiter()); - Log::trace() << "GeometryIterator::operator== done" << std::endl; + Log::trace() << "GeometryIterator::operator== done" << std::endl; return equals; } // ----------------------------------------------------------------------------- -template -bool GeometryIterator::operator!=(const GeometryIterator& other) { - Log::trace() << "GeometryIterator::operator!= starting" << std::endl; +template +bool GeometryIterator::operator!=(const GeometryIterator& other) { + Log::trace() << "GeometryIterator::operator!= starting" << std::endl; util::Timer timer(classname(), "operator!="); bool notequals = (*geometryiter_ != other.geometryiter()); - Log::trace() << "GeometryIterator::operator!= done" << std::endl; + Log::trace() << "GeometryIterator::operator!= done" << std::endl; return notequals; } // ----------------------------------------------------------------------------- -template -eckit::geometry::Point2 GeometryIterator::operator*() const { - Log::trace() << "GeometryIterator::operator* starting" << std::endl; +template +eckit::geometry::Point2 GeometryIterator::operator*() const { + Log::trace() << "GeometryIterator::operator* starting" << std::endl; util::Timer timer(classname(), "operator*"); eckit::geometry::Point2 loc = *(*geometryiter_); - Log::trace() << "GeometryIterator::operator* done" << std::endl; + Log::trace() << "GeometryIterator::operator* done" << std::endl; return loc; } // ----------------------------------------------------------------------------- -template -GeometryIterator GeometryIterator::operator++() { - Log::trace() << "GeometryIterator::operator++ starting" << std::endl; +template +GeometryIterator GeometryIterator::operator++() { + Log::trace() << "GeometryIterator::operator++ starting" << std::endl; util::Timer timer(classname(), "operator++"); ++(*geometryiter_); - Log::trace() << "GeometryIterator::operator++ done" << std::endl; + Log::trace() << "GeometryIterator::operator++ done" << std::endl; return *this; } // ----------------------------------------------------------------------------- -template -void GeometryIterator::print(std::ostream & os) const { - Log::trace() << "GeometryIterator::print starting" << std::endl; +template +void GeometryIterator::print(std::ostream & os) const { + Log::trace() << "GeometryIterator::print starting" << std::endl; util::Timer timer(classname(), "print"); os << *geometryiter_; - Log::trace() << "GeometryIterator::print done" << std::endl; + Log::trace() << "GeometryIterator::print done" << std::endl; } diff --git a/src/oops/interface/ObsSpace.h b/src/oops/interface/ObsSpace.h index 3baa21faa..2266c41ed 100644 --- a/src/oops/interface/ObsSpace.h +++ b/src/oops/interface/ObsSpace.h @@ -18,6 +18,7 @@ #include "eckit/geometry/Point2.h" #include "oops/base/Variables.h" +#include "oops/interface/GeometryIterator.h" #include "oops/mpi/mpi.h" #include "oops/util/Logger.h" #include "oops/util/ObjectCounter.h" @@ -42,6 +43,7 @@ class ObsSpace : public util::Printable, private util::ObjectCounter > { typedef typename OBS::ObsSpace ObsSpace_; typedef ObsVector ObsVector_; + typedef GeometryIterator ObsIterator_; public: static const std::string classname() {return "oops::ObsSpace";} @@ -66,6 +68,11 @@ class ObsSpace : public util::Printable, // Other const std::string & obsname() const {return obsdb_->obsname();} + /// Iterator to the first observation + ObsIterator_ begin() const; + /// Iterator to the past-the-end observation (after last) + ObsIterator_ end() const; + const eckit::mpi::Comm & timeComm() const {return time_;} private: @@ -142,6 +149,26 @@ const Variables & ObsSpace::obsvariables() const { // ----------------------------------------------------------------------------- +template +GeometryIterator ObsSpace::begin() const { + Log::trace() << "ObsSpace::begin starting" << std::endl; + util::Timer timer(classname(), "begin"); + Log::trace() << "ObsSpace::begin done" << std::endl; + return ObsIterator_(obsdb_->begin()); +} + +// ----------------------------------------------------------------------------- + +template +GeometryIterator ObsSpace::end() const { + Log::trace() << "ObsSpace::end starting" << std::endl; + util::Timer timer(classname(), "end"); + Log::trace() << "ObsSpace::end done" << std::endl; + return ObsIterator_(obsdb_->end()); +} + +// ----------------------------------------------------------------------------- + } // namespace oops #endif // OOPS_INTERFACE_OBSSPACE_H_ diff --git a/src/test/interface/ObsIterator.h b/src/test/interface/ObsIterator.h new file mode 100644 index 000000000..28be12bc5 --- /dev/null +++ b/src/test/interface/ObsIterator.h @@ -0,0 +1,112 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef TEST_INTERFACE_OBSITERATOR_H_ +#define TEST_INTERFACE_OBSITERATOR_H_ + +#include +#include + +#define ECKIT_TESTING_SELF_REGISTER_CASES 0 + +#include "eckit/config/Configuration.h" +#include "eckit/testing/Test.h" + +#include "oops/interface/GeometryIterator.h" +#include "oops/interface/ObsSpace.h" +#include "oops/runs/Test.h" + +#include "test/interface/ObsTestsFixture.h" +#include "test/TestEnvironment.h" + +namespace test { + +// ----------------------------------------------------------------------------- +/*! \brief Tests of ObsSpace::begin/end; ObsIterator ctor, ==/!=/++ operators + * and dereferencing operator. + * + * \details testBasic tests the following: + * + * 1. Initialize ObsIterator to ObsSpace::begin() and check equality + * 2. Initialize ObsIterator to ObsSpace::end() and check equality + * 3. Check inequality of the two iterators + * 4. Print out the begin iterator, to "test" print method + * 5. Loop over iterator (test operator++), compute number of iterations, compare + * with reference + * 6. Compare the first two Points with reference. + */ +template void testBasic() { + typedef oops::GeometryIterator ObsIterator_; + typedef ObsTestsFixture Test_; + + for (size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + // Initialize two iterators (begin and end), test equality + ObsIterator_ iter1 = Test_::obspace()[jj].begin(); + EXPECT(iter1 == Test_::obspace()[jj].begin()); + + ObsIterator_ iter2 = Test_::obspace()[jj].end(); + EXPECT(iter2 == Test_::obspace()[jj].end()); + + // For all current use cases begin() shouldn't be the same as end(); test it + EXPECT(iter1 != iter2); + + // At least test that nothing fails on print + oops::Log::test() << "ObsSpace::begin " << iter1 << std::endl; + + // loop over all points in iterator, compute number of those points, and compare + // with reference + size_t nlocs = 0; + for (ObsIterator_ ii = Test_::obspace()[jj].begin(); ii != Test_::obspace()[jj].end(); ++ii) { + nlocs++; + } + size_t nlocs_ref = Test_::config(jj).getUnsigned("obs iterator test.reference nlocs"); + EXPECT_EQUAL(nlocs, nlocs_ref); + + // test that the begin() point is the same as reference + double lon1 = Test_::config(jj).getDouble("obs iterator test.lon1"); + double lat1 = Test_::config(jj).getDouble("obs iterator test.lat1"); + const eckit::geometry::Point2 point1(lon1, lat1); + EXPECT_EQUAL(*iter1, point1); + + // test that the point after begin() is the same as reference + ++iter1; + double lon2 = Test_::config(jj).getDouble("obs iterator test.lon2"); + double lat2 = Test_::config(jj).getDouble("obs iterator test.lat2"); + const eckit::geometry::Point2 point2(lon2, lat2); + EXPECT_EQUAL(*iter1, point2); + } +} + + +// ----------------------------------------------------------------------------- + +template class ObsIterator : public oops::Test { + typedef ObsTestsFixture Test_; + public: + ObsIterator() = default; + virtual ~ObsIterator() = default; + + private: + std::string testid() const override {return "test::ObsIterator<" + OBS::name() + ">";} + + void register_tests() const override { + std::vector& ts = eckit::testing::specification(); + + ts.emplace_back(CASE("interface/ObsIterator/testBasic") + { testBasic(); }); + } + + void clear() const override { + Test_::reset(); + } +}; + +// ============================================================================= + +} // namespace test + +#endif // TEST_INTERFACE_OBSITERATOR_H_ From 4990562107bfc8846346e7380b1daa29b2781555 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Thu, 15 Apr 2021 17:01:52 +0100 Subject: [PATCH 109/142] Fix failure of test_util_parameters with eckit 1.16.0 or later (#1162) * Reordered options in a test file to fix parameter serialisation tests. * Updated a comment. * Restored the old parameters.yaml file under a different name for compatibility with older versions of eckit. * Fixed a typo in a comment. Co-authored-by: August Weinbren Co-authored-by: August Weinbren Co-authored-by: mmiesch --- src/CMakeLists.txt | 17 +- src/test/testinput/parameters.yaml | 74 ++-- .../testinput/parameters_older_eckit.yaml | 341 ++++++++++++++++++ src/test/util/Parameters.h | 7 +- 4 files changed, 399 insertions(+), 40 deletions(-) create mode 100644 src/test/testinput/parameters_older_eckit.yaml diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f4b76b639..84c8bb074 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -454,6 +454,7 @@ list( APPEND oops_test_input test/testinput/unstructured_interpolation.yaml test/testinput/variables.yaml test/testinput/parameters.yaml + test/testinput/parameters_older_eckit.yaml test/testinput/empty.yaml test/testinput/mpi.yaml test/testinput/spectrallmp.yaml @@ -544,10 +545,18 @@ ecbuild_add_test( TARGET test_util_pushstringvector ARGS "test/testinput/pushstringvector.yaml" LIBS oops ) -ecbuild_add_test( TARGET test_util_parameters - SOURCES test/base/Parameters.cc - ARGS "test/testinput/parameters.yaml" - LIBS oops ) +if (eckit_VERSION VERSION_GREATER_EQUAL 1.16) + ecbuild_add_test( TARGET test_util_parameters + SOURCES test/base/Parameters.cc + ARGS "test/testinput/parameters.yaml" + LIBS oops ) +else() + # Due to changes in eckit 1.16 a different input file is required. + ecbuild_add_test( TARGET test_util_parameters + SOURCES test/base/Parameters.cc + ARGS "test/testinput/parameters_older_eckit.yaml" + LIBS oops ) +endif() ecbuild_add_test( TARGET test_generic_gc99 SOURCES test/generic/gc99.cc diff --git a/src/test/testinput/parameters.yaml b/src/test/testinput/parameters.yaml index c9fb2e22d..e04703641 100644 --- a/src/test/testinput/parameters.yaml +++ b/src/test/testinput/parameters.yaml @@ -1,34 +1,38 @@ minimal: req_float_parameter: 3 req_duration_parameter: PT1H -full: # These parameters must be sorted alphabetically for the serialization test. - any_of_parameter: dog - bool_parameter: false - channels: 5,6,7 # used by the parameter storing a Variables object - embedded_int_parameter: 13 +full: + # These parameters must be specified in the order the corresponding + # Parameter objects are declared in the C++ file; otherwise the + # serialization test, which isn't very sophisticated and simply relies + # on a string comparison, will fail. float_parameter: 3.5 - fruit_parameter: apple int_parameter: 4 - int_parameters: [1, 2] - opt_any_of_parameter: [1, 2, 3, 4] + bool_parameter: false + opt_float_parameter: 5.5 opt_date_time_parameter: 2010-02-03T04:05:06Z opt_duration_parameter: PT1H2M3S - opt_embedded_date_time_parameter: 2010-03-04T05:06:07Z - opt_float_parameter: 5.5 - opt_null_parameter: null opt_partialDT_parameter: "2010-**-03T04:05:06Z" + fruit_parameter: apple range_parameter: - max: 8.5 min: 7 + max: 8.5 + int_parameters: [1, 2] range_parameters: - - max: 10 - min: 9 - - max: 12 - min: 11 - req_duration_parameter: PT6H30M - req_float_parameter: 6 - set_int_parameter: 2,4,5,6,8 + - min: 9 + max: 10 + - min: 11 + max: 12 variables_parameter: [u, v] + channels: 5,6,7 # used by the parameter storing a Variables object + set_int_parameter: 2,4,5,6,8 + any_of_parameter: dog + opt_any_of_parameter: [1, 2, 3, 4] + opt_null_parameter: null + embedded_int_parameter: 13 + opt_embedded_date_time_parameter: 2010-03-04T05:06:07Z + req_float_parameter: 6 + req_duration_parameter: PT6H30M alternative: float_parameter: 13.5 int_parameter: 14 @@ -155,13 +159,17 @@ map_parameter_yaml_style_unquoted_keys: night: PT10H float_or_int_to_float_map_2: 3.5 duration_or_string_to_duration_map_2: PT12H -map_parameter_json_style_quoted_keys: # These parameters must be sorted alphabetically for the serialization test. - duration_or_string_to_duration_map_1: { "day": PT14H, "night": PT10H } - duration_or_string_to_duration_map_2: PT12H - float_or_int_to_float_map_1: { "6": 2.5, "8": 4 } - float_or_int_to_float_map_2: 3.5 +map_parameter_json_style_quoted_keys: + # These parameters must be specified in the order the corresponding + # Parameter objects are declared in the C++ file; otherwise the + # serialization test, which isn't very sophisticated and simply relies + # on a string comparison, will fail. int_to_float_map: { "5": 1.5, "7": 3 } string_to_duration_map: { "day": PT16H, "night": PT8H } + float_or_int_to_float_map_1: { "6": 2.5, "8": 4 } + duration_or_string_to_duration_map_1: { "day": PT14H, "night": PT10H } + float_or_int_to_float_map_2: 3.5 + duration_or_string_to_duration_map_2: PT12H map_parameter_json_style_unquoted_keys: int_to_float_map: { 5: 1.5, 7: 3 } string_to_duration_map: { day: PT16H, night: PT8H } @@ -183,19 +191,23 @@ variables_without_channels: filter_variables: [air_temperature, air_pressure] operator_variables: [relative_humidity] variables_with_channels: - channels: 5,6,7 filter_variables: [air_temperature, air_pressure] + channels: 5,6,7 operator_variables: [relative_humidity] -device: # These parameters must be sorted alphabetically for the serialization test. +device: + # These parameters must be specified in the order the corresponding + # Parameter objects are declared in the C++ file; otherwise the + # serialization test, which isn't very sophisticated and simply relies + # on a string comparison, will fail. + required_device: + type: screen + diameter: 30 device_with_default: - paper_format: A3 type: printer + paper_format: A3 optional_device: - paper_format: Letter type: printer - required_device: - diameter: 30 - type: screen + paper_format: Letter alternative_device: optional_device: type: screen diff --git a/src/test/testinput/parameters_older_eckit.yaml b/src/test/testinput/parameters_older_eckit.yaml new file mode 100644 index 000000000..c9fb2e22d --- /dev/null +++ b/src/test/testinput/parameters_older_eckit.yaml @@ -0,0 +1,341 @@ +minimal: + req_float_parameter: 3 + req_duration_parameter: PT1H +full: # These parameters must be sorted alphabetically for the serialization test. + any_of_parameter: dog + bool_parameter: false + channels: 5,6,7 # used by the parameter storing a Variables object + embedded_int_parameter: 13 + float_parameter: 3.5 + fruit_parameter: apple + int_parameter: 4 + int_parameters: [1, 2] + opt_any_of_parameter: [1, 2, 3, 4] + opt_date_time_parameter: 2010-02-03T04:05:06Z + opt_duration_parameter: PT1H2M3S + opt_embedded_date_time_parameter: 2010-03-04T05:06:07Z + opt_float_parameter: 5.5 + opt_null_parameter: null + opt_partialDT_parameter: "2010-**-03T04:05:06Z" + range_parameter: + max: 8.5 + min: 7 + range_parameters: + - max: 10 + min: 9 + - max: 12 + min: 11 + req_duration_parameter: PT6H30M + req_float_parameter: 6 + set_int_parameter: 2,4,5,6,8 + variables_parameter: [u, v] +alternative: + float_parameter: 13.5 + int_parameter: 14 + bool_parameter: true + opt_float_parameter: 15.5 + opt_date_time_parameter: 2010-02-13T04:05:06Z + opt_duration_parameter: PT11H02M03S + req_float_parameter: 16 + req_duration_parameter: PT16H30M + fruit_parameter: orange + range_parameter: + min: 17 + max: 18.5 + int_parameters: [11, 12] + range_parameters: + - min: 19 + max: 110 + - min: 111 + max: 112 + embedded_int_parameter: 23 + opt_embedded_date_time_parameter: 2010-03-14T05:06:17Z + set_int_parameter: 3 + any_of_parameter: [1, 3, 5] + opt_any_of_parameter: cat + opt_null_parameter: +misspelled_float: + float_parometer: 3.5 +misspelled_int: + int_parometer: 4 +misspelled_bool: + bool_parometer: false +misspelled_dt: + opt_date_time_parometer: 2010-02-03T04:05:06Z +misspelled_dur: + opt_duration_parometer: PT01H02M03S +misspelled_fruit: + fruit_parometer: apple +misspelled_ints: + int_parometers: [1, 2] +misspelled_nested_param: + range_parameter: + min: 7 + mox: 8.5 +misspelled_nested_params: + range_parameters: + - mon: 9 + max: 10 + - min: 11 + max: 12 +misspelled_nesting_param: + ronge_parameter: + min: 7 + max: 8.5 +error_in_float_parameter: + float_parameter: ABCDEF + req_float_parameter: 3 + req_duration_parameter: PT1H +error_in_opt_float_parameter: + opt_float_parameter: ABCDEF + req_float_parameter: 3 + req_duration_parameter: PT1H +error_in_opt_date_time_parameter: + opt_date_time_parameter: ABCDEF + req_float_parameter: 3 + req_duration_parameter: PT1H +error_in_opt_duration_parameter: + opt_duration_parameter: ABCDEF + req_float_parameter: 3 + req_duration_parameter: PT1H +error_in_opt_partialDT_parameter: + opt_partialDT_parameter: ABCDEF + req_float_parameter: 3 + req_duration_parameter: PT1H +error_in_fruit_parameter: + fruit_parameter: ABCDEF + req_float_parameter: 3 + req_duration_parameter: PT1H +error_in_int_parameters: + int_parameters: ABCDEF + req_float_parameter: 3 + req_duration_parameter: PT1H +error_in_range_parameters: + range_parameters: ABCDEF + req_float_parameter: 3 + req_duration_parameter: PT1H +error_in_set_int_parameter: + set_int_parameter: 1- + req_float_parameter: 3 + req_duration_parameter: PT1H +error_in_any_of_parameter: + any_of_parameter: [abc, def] +missing_req_float_parameter: + req_duration_parameter: PT1H +missing_req_duration_parameter: + req_float_parameter: 3 +# Commented because eckit fails to parse quoted key names +#map_parameter_yaml_style_quoted_keys: +# int_to_float_map: +# "5": 1.5 +# "7": 3 +# string_to_duration_map: +# "day": PT16H +# "night": PT8H +# float_or_int_to_float_map_1: +# "6": 2.5 +# "8": 4 +# duration_or_string_to_duration_map_1: +# "day": PT14H +# "night": PT10H +# float_or_int_to_float_map_2: 3.5 +# duration_or_string_to_duration_map_2: PT12H +map_parameter_yaml_style_unquoted_keys: + int_to_float_map: + 5: 1.5 + 7: 3 + string_to_duration_map: + day: PT16H + night: PT8H + float_or_int_to_float_map_1: + 6: 2.5 + 8: 4 + duration_or_string_to_duration_map_1: + day: PT14H + night: PT10H + float_or_int_to_float_map_2: 3.5 + duration_or_string_to_duration_map_2: PT12H +map_parameter_json_style_quoted_keys: # These parameters must be sorted alphabetically for the serialization test. + duration_or_string_to_duration_map_1: { "day": PT14H, "night": PT10H } + duration_or_string_to_duration_map_2: PT12H + float_or_int_to_float_map_1: { "6": 2.5, "8": 4 } + float_or_int_to_float_map_2: 3.5 + int_to_float_map: { "5": 1.5, "7": 3 } + string_to_duration_map: { "day": PT16H, "night": PT8H } +map_parameter_json_style_unquoted_keys: + int_to_float_map: { 5: 1.5, 7: 3 } + string_to_duration_map: { day: PT16H, night: PT8H } + float_or_int_to_float_map_1: { 6: 2.5, 8: 4 } + duration_or_string_to_duration_map_1: { day: PT14H, night: PT10H } + float_or_int_to_float_map_2: 3.5 + duration_or_string_to_duration_map_2: PT12H +set_int_single_number: + set_int_parameter: 5 +set_int_range: + set_int_parameter: 3-5 +set_int_multiple_numbers_and_ranges: + set_int_parameter: 13,3-5,7, 10-11 +set_int_invalid_range: + set_int_parameter: 13,3-x +variables_without_channels: + # The filter variables are not ordered alphabetically: this is by design, to test if their + # order is preserved during serialization. + filter_variables: [air_temperature, air_pressure] + operator_variables: [relative_humidity] +variables_with_channels: + channels: 5,6,7 + filter_variables: [air_temperature, air_pressure] + operator_variables: [relative_humidity] +device: # These parameters must be sorted alphabetically for the serialization test. + device_with_default: + paper_format: A3 + type: printer + optional_device: + paper_format: Letter + type: printer + required_device: + diameter: 30 + type: screen +alternative_device: + optional_device: + type: screen + diameter: 40 + required_device: + type: printer + paper_format: A5 + device_with_default: + type: screen + diameter: 20 +device_with_default_not_set: + device_with_default: {} + optional_device: {} + required_device: + diameter: 30 + type: screen +optional_device_not_set: + device_with_default: {} + optional_device: {} + required_device: + diameter: 30 + type: screen +required_device_not_set: + device_with_default: {} + optional_device: {} + required_device: {} +invalid_type_of_device_with_default: + device_with_default: + diameter: 30 + type: ABCDEF + required_device: + diameter: 30 + type: screen +invalid_type_of_optional_device: + optional_device: + diameter: 30 + type: ABCDEF + required_device: + diameter: 30 + type: screen +invalid_type_of_required_device: + required_device: + diameter: 30 + type: ABCDEF +misspelled_diameter_of_optional_device: + optional_device: + diamester: 30 + type: screen + required_device: + diameter: 30 + type: screen +misspelled_diameter_of_device_with_default: + device_with_default: + diamester: 30 + type: screen + required_device: + diameter: 30 + type: screen +misspelled_diameter_of_required_device: + required_device: + diamester: 30 + type: screen +constraints_met: + int_with_min: 5 + int_with_exclusive_min: 6 + int_with_max: 5 + int_with_exclusive_max: 4 + float_with_min: 5.5 + float_with_exclusive_min: 6 + float_with_max: 5.5 + float_with_exclusive_max: 5 +int_min_constraint_not_met: + int_with_min: 4 # violates constraint + int_with_exclusive_min: 6 + int_with_max: 5 + int_with_exclusive_max: 4 + float_with_min: 5.5 + float_with_exclusive_min: 6 + float_with_max: 5.5 + float_with_exclusive_max: 5 +float_min_constraint_not_met: + float_with_min: 5 # violates constraint + int_with_min: 5 + int_with_exclusive_min: 6 + int_with_max: 5 + int_with_exclusive_max: 4 + float_with_exclusive_min: 6 + float_with_max: 5.5 + float_with_exclusive_max: 5 +int_exclusive_min_constraint_not_met: + int_with_exclusive_min: 5 # violates constraint + int_with_min: 5 + int_with_max: 5 + int_with_exclusive_max: 4 + float_with_min: 5.5 + float_with_exclusive_min: 6 + float_with_max: 5.5 + float_with_exclusive_max: 5 +float_exclusive_min_constraint_not_met: + float_with_exclusive_min: 5.5 # violates constraint + int_with_min: 5 + int_with_exclusive_min: 6 + int_with_max: 5 + int_with_exclusive_max: 4 + float_with_min: 5.5 + float_with_max: 5.5 + float_with_exclusive_max: 5 +int_max_constraint_not_met: + int_with_max: 6 # violates constraint + int_with_min: 5 + int_with_exclusive_min: 6 + int_with_exclusive_max: 4 + float_with_min: 5.5 + float_with_exclusive_min: 6 + float_with_max: 5.5 + float_with_exclusive_max: 5 +float_max_constraint_not_met: + float_with_max: 6 # violates constraint + int_with_min: 5 + int_with_exclusive_min: 6 + int_with_max: 5 + int_with_exclusive_max: 4 + float_with_min: 5.5 + float_with_exclusive_min: 6 + float_with_exclusive_max: 5 +int_exclusive_max_constraint_not_met: + int_with_exclusive_max: 5 # violates constraint + int_with_min: 5 + int_with_exclusive_min: 6 + int_with_max: 5 + float_with_min: 5.5 + float_with_exclusive_min: 6 + float_with_max: 5.5 + float_with_exclusive_max: 5 +float_exclusive_max_constraint_not_met: + float_with_exclusive_max: 5.5 # violates constraint + int_with_min: 5 + int_with_exclusive_min: 6 + int_with_max: 5 + int_with_exclusive_max: 4 + float_with_min: 5.5 + float_with_exclusive_min: 6 + float_with_max: 5.5 diff --git a/src/test/util/Parameters.h b/src/test/util/Parameters.h index 3ec53380d..35ace81ac 100644 --- a/src/test/util/Parameters.h +++ b/src/test/util/Parameters.h @@ -396,11 +396,8 @@ void doTestSerialization(const eckit::Configuration &config) { // back into a configuration. The test verifies that the configuration objects produce the same // output when printed. // - // For this to work, parameter names in the YAML file must be ordered alphabetically; that's - // because the YAML parser creates configurations storing keys and values OrderedMapContent - // objects (preserving the order in which individual options were specified in the YAML file), - // but the LocalConfiguration::set() method stores keys and values in MapContent objects (with - // keys ordered alphabetically). + // For this to work, parameter names in the YAML file must be specified in the same order in which + // the corresponding Parameter objects are declared in the C++ class of which they are members. ParametersType params; EXPECT_NO_THROW(params.validate(config)); From 0ff2d64fc101958340ab359ecbbaf882e883ec13 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Thu, 15 Apr 2021 16:42:01 -0600 Subject: [PATCH 110/142] move local observation search from ObsSpace to ObsLocalization (#1141) * start on refactoring obslocalization problem with QC: pack saves only non-qcd out obs; obsloc does all obs * refactor to fix qc issues * use GeometryIterator instead of Point in ObsLocalization * add a primitive test for ObsLocalization * start on refactoring obslocalization problem with QC: pack saves only non-qcd out obs; obsloc does all obs * refactor to fix qc issues * use GeometryIterator instead of Point in ObsLocalization * add a primitive test for ObsLocalization * change ObsLoc interface (closer to what would be needed to move search for local obs into ObsLoc) * remove a bunch of code (use ObsLoc instead of second ObsSpace ctor) * work on QG update and clean up of L95 (QG not working yet) * add ObsVector::mask test and update QG to work * add a test for ObsVector::ones * update obsLocalization test * bugfix: errors have to be created after H(x) is run for now * add packEigenSize (different from nobs() for mpi distributed data!) * Change for #1141: ObsLocalization test to only test at specified gridpoints (#1158) * change ObsLocalization test to only test at specified gridpoints * update the test (run EXPECT after MPI comm, otherwise test hangs when fails) * change packEigen to take a mask on input --- l95/src/lorenz95/BackgroundCheck.cc | 2 +- l95/src/lorenz95/BackgroundCheck.h | 6 +- l95/src/lorenz95/CMakeLists.txt | 2 - l95/src/lorenz95/GomL95.cc | 4 +- l95/src/lorenz95/GomL95.h | 4 +- l95/src/lorenz95/L95Traits.h | 4 +- l95/src/lorenz95/ObsBias.cc | 2 +- l95/src/lorenz95/ObsBias.h | 4 +- l95/src/lorenz95/ObsBiasCorrection.cc | 2 +- l95/src/lorenz95/ObsBiasCorrection.h | 4 +- l95/src/lorenz95/ObsBiasCovariance.cc | 2 +- l95/src/lorenz95/ObsBiasCovariance.h | 4 +- l95/src/lorenz95/ObsData1D.h | 8 +- l95/src/lorenz95/ObsDiags1D.h | 4 +- l95/src/lorenz95/ObsLocGC99.cc | 25 +- l95/src/lorenz95/ObsLocGC99.h | 16 +- l95/src/lorenz95/ObsTable.cc | 10 +- l95/src/lorenz95/ObsTable.h | 7 +- l95/src/lorenz95/ObsTableView.cc | 218 ------------------ l95/src/lorenz95/ObsTableView.h | 81 ------- l95/src/lorenz95/ObsVec1D.cc | 32 +-- l95/src/lorenz95/ObsVec1D.h | 13 +- l95/src/lorenz95/ObservationL95.cc | 5 +- l95/src/lorenz95/ObservationL95.h | 8 +- l95/src/lorenz95/ObservationTLAD.cc | 2 +- l95/src/lorenz95/ObservationTLAD.h | 4 +- l95/src/lorenz95/QCmanager.h | 4 +- l95/test/CMakeLists.txt | 18 -- l95/test/executables/TestDepartures.cc | 17 -- .../executables/TestDeparturesEnsemble.cc | 17 -- l95/test/executables/TestLocalObsSpace.cc | 17 -- l95/test/testinput/hofx3d_for_getkf.yaml | 8 +- l95/test/testinput/interfaces.yaml | 16 +- qg/model/CMakeLists.txt | 2 + qg/model/ObsDataQG.h | 15 ++ qg/model/ObsLocQG.cc | 66 ++++++ qg/model/ObsLocQG.h | 44 ++++ qg/model/ObsSpaceQG.cc | 62 +---- qg/model/ObsSpaceQG.h | 9 - qg/model/ObsVecQG.cc | 24 +- qg/model/ObsVecQG.h | 8 +- qg/model/QgFortran.h | 10 +- qg/model/qg_obsdb_interface.F90 | 34 +-- qg/model/qg_obsdb_mod.F90 | 51 +--- qg/model/qg_obsvec_interface.F90 | 59 +++-- qg/model/qg_obsvec_mod.F90 | 65 +++--- qg/test/CMakeLists.txt | 24 +- qg/test/executables/TestDepartures.cc | 17 -- qg/test/executables/TestDeparturesEnsemble.cc | 17 -- ...ocalObsSpace.cc => TestObsLocalization.cc} | 6 +- qg/test/testinput/interfaces.yaml | 33 ++- qg/test/testinput/letkf.yaml | 3 + src/CMakeLists.txt | 1 - src/oops/assimilation/GETKFSolver.h | 38 +-- src/oops/assimilation/LETKFSolver.h | 36 +-- src/oops/assimilation/LocalEnsembleSolver.h | 17 +- src/oops/base/Departures.h | 26 +-- src/oops/base/DeparturesEnsemble.h | 29 +-- src/oops/base/ObsLocalizationBase.h | 30 +-- src/oops/base/ObsLocalizations.h | 14 +- src/oops/base/ObsSpaces.h | 17 -- src/oops/base/Observations.h | 12 - src/oops/interface/ObsLocalization.h | 9 +- src/oops/interface/ObsSpace.h | 25 -- src/oops/interface/ObsVector.h | 36 ++- src/test/base/Departures.h | 86 ------- src/test/base/DeparturesEnsemble.h | 82 ------- src/test/interface/LocalObsSpace.h | 137 ----------- src/test/interface/ObsLocalization.h | 107 ++++++--- src/test/interface/ObsTestsFixture.h | 6 +- src/test/interface/ObsVector.h | 25 +- 71 files changed, 593 insertions(+), 1259 deletions(-) delete mode 100644 l95/src/lorenz95/ObsTableView.cc delete mode 100644 l95/src/lorenz95/ObsTableView.h delete mode 100644 l95/test/executables/TestDepartures.cc delete mode 100644 l95/test/executables/TestDeparturesEnsemble.cc delete mode 100644 l95/test/executables/TestLocalObsSpace.cc create mode 100644 qg/model/ObsLocQG.cc create mode 100644 qg/model/ObsLocQG.h delete mode 100644 qg/test/executables/TestDepartures.cc delete mode 100644 qg/test/executables/TestDeparturesEnsemble.cc rename qg/test/executables/{TestLocalObsSpace.cc => TestObsLocalization.cc} (70%) delete mode 100644 src/test/base/Departures.h delete mode 100644 src/test/base/DeparturesEnsemble.h delete mode 100644 src/test/interface/LocalObsSpace.h diff --git a/l95/src/lorenz95/BackgroundCheck.cc b/l95/src/lorenz95/BackgroundCheck.cc index c6c1d6352..1705d257a 100644 --- a/l95/src/lorenz95/BackgroundCheck.cc +++ b/l95/src/lorenz95/BackgroundCheck.cc @@ -19,7 +19,7 @@ static oops::FilterMaker > makerBackgroundCheck_("Background Check"); // ----------------------------------------------------------------------------- -BackgroundCheck::BackgroundCheck(const ObsTableView & obsdb, const Parameters_ & parameters, +BackgroundCheck::BackgroundCheck(const ObsTable & obsdb, const Parameters_ & parameters, std::shared_ptr > qcflags, std::shared_ptr > obserr) : obsdb_(obsdb), options_(parameters), qcflags_(qcflags), obserr_(obserr), novars_() { diff --git a/l95/src/lorenz95/BackgroundCheck.h b/l95/src/lorenz95/BackgroundCheck.h index 44fce1e68..5c05b80a6 100644 --- a/l95/src/lorenz95/BackgroundCheck.h +++ b/l95/src/lorenz95/BackgroundCheck.h @@ -20,7 +20,7 @@ namespace lorenz95 { class GomL95; template class ObsData1D; - class ObsTableView; + class ObsTable; class ObsDiags1D; class ObsVec1D; @@ -44,7 +44,7 @@ class BackgroundCheck : public util::Printable { public: typedef BackgroundCheckParameters Parameters_; - BackgroundCheck(const ObsTableView &, const Parameters_ &, + BackgroundCheck(const ObsTable &, const Parameters_ &, std::shared_ptr >, std::shared_ptr >); void preProcess() const {} @@ -57,7 +57,7 @@ class BackgroundCheck : public util::Printable { private: void print(std::ostream & os) const; - const ObsTableView & obsdb_; + const ObsTable & obsdb_; Parameters_ options_; std::shared_ptr > qcflags_; // QC flags std::shared_ptr > obserr_; // obs error stddev diff --git a/l95/src/lorenz95/CMakeLists.txt b/l95/src/lorenz95/CMakeLists.txt index a7be7d4f4..73bfb56ce 100644 --- a/l95/src/lorenz95/CMakeLists.txt +++ b/l95/src/lorenz95/CMakeLists.txt @@ -48,8 +48,6 @@ set( _l95_srcs ObsLocGC99.h ObsTable.cc ObsTable.h - ObsTableView.cc - ObsTableView.h ObsVec1D.cc ObsVec1D.h ObsData1D.h diff --git a/l95/src/lorenz95/GomL95.cc b/l95/src/lorenz95/GomL95.cc index 6f628adb8..95a7cb621 100644 --- a/l95/src/lorenz95/GomL95.cc +++ b/l95/src/lorenz95/GomL95.cc @@ -18,7 +18,7 @@ #include "eckit/config/Configuration.h" #include "lorenz95/LocsL95.h" -#include "lorenz95/ObsTableView.h" +#include "lorenz95/ObsTable.h" #include "oops/util/abor1_cpp.h" #include "oops/util/Logger.h" #include "oops/util/Random.h" @@ -39,7 +39,7 @@ GomL95::GomL95(const LocsL95 & locs, const oops::Variables &) // ----------------------------------------------------------------------------- /*! Constructor with Configuration */ GomL95::GomL95(const eckit::Configuration & conf, - const ObsTableView &, const oops::Variables &) + const ObsTable &, const oops::Variables &) : size_(0), locval_() { this->read(conf); diff --git a/l95/src/lorenz95/GomL95.h b/l95/src/lorenz95/GomL95.h index 6d677ded0..058298322 100644 --- a/l95/src/lorenz95/GomL95.h +++ b/l95/src/lorenz95/GomL95.h @@ -25,7 +25,7 @@ namespace oops { namespace lorenz95 { class LocsL95; - class ObsTableView; + class ObsTable; /// GomL95 class to handle locations for L95 model. @@ -35,7 +35,7 @@ class GomL95 : public util::Printable, static const std::string classname() {return "lorenz95::GomL95";} GomL95(const LocsL95 &, const oops::Variables &); - GomL95(const eckit::Configuration &, const ObsTableView &, + GomL95(const eckit::Configuration &, const ObsTable &, const oops::Variables &); void zero(); diff --git a/l95/src/lorenz95/L95Traits.h b/l95/src/lorenz95/L95Traits.h index b7252709e..87a61530f 100644 --- a/l95/src/lorenz95/L95Traits.h +++ b/l95/src/lorenz95/L95Traits.h @@ -32,7 +32,7 @@ #include "lorenz95/ObservationL95.h" #include "lorenz95/ObservationTLAD.h" #include "lorenz95/ObsIterator.h" -#include "lorenz95/ObsTableView.h" +#include "lorenz95/ObsTable.h" #include "lorenz95/ObsVec1D.h" #include "lorenz95/Resolution.h" #include "lorenz95/StateL95.h" @@ -62,7 +62,7 @@ struct L95Traits { struct L95ObsTraits { static std::string name() {return "Lorenz 95 Obs";} - typedef lorenz95::ObsTableView ObsSpace; + typedef lorenz95::ObsTable ObsSpace; typedef lorenz95::ObsVec1D ObsVector; template using ObsDataVector = lorenz95::ObsData1D; typedef lorenz95::ObsIterator GeometryIterator; diff --git a/l95/src/lorenz95/ObsBias.cc b/l95/src/lorenz95/ObsBias.cc index b44290049..95a7e306e 100644 --- a/l95/src/lorenz95/ObsBias.cc +++ b/l95/src/lorenz95/ObsBias.cc @@ -21,7 +21,7 @@ // ----------------------------------------------------------------------------- namespace lorenz95 { // ----------------------------------------------------------------------------- -ObsBias::ObsBias(const ObsTableView &, const Parameters_ & params) +ObsBias::ObsBias(const ObsTable &, const Parameters_ & params) : bias_(0.0), active_(false), geovars_(std::vector{"x"}), hdiags_() { oops::Log::trace() << "ObsBias::ObsBias conf is:" << params << std::endl; diff --git a/l95/src/lorenz95/ObsBias.h b/l95/src/lorenz95/ObsBias.h index 543c43da8..e98ec6043 100644 --- a/l95/src/lorenz95/ObsBias.h +++ b/l95/src/lorenz95/ObsBias.h @@ -23,7 +23,7 @@ namespace lorenz95 { class ObsBiasCorrection; - class ObsTableView; + class ObsTable; /// Class to handle observation bias parameters. @@ -37,7 +37,7 @@ class ObsBias : public util::Printable, static const std::string classname() {return "lorenz95::ObsBias";} - ObsBias(const ObsTableView &, const Parameters_ &); + ObsBias(const ObsTable &, const Parameters_ &); ObsBias(const ObsBias &, const bool); ~ObsBias() {} diff --git a/l95/src/lorenz95/ObsBiasCorrection.cc b/l95/src/lorenz95/ObsBiasCorrection.cc index e08f68d3c..edf68267f 100644 --- a/l95/src/lorenz95/ObsBiasCorrection.cc +++ b/l95/src/lorenz95/ObsBiasCorrection.cc @@ -21,7 +21,7 @@ // ----------------------------------------------------------------------------- namespace lorenz95 { // ----------------------------------------------------------------------------- -ObsBiasCorrection::ObsBiasCorrection(const ObsTableView &, const Parameters_ & params) +ObsBiasCorrection::ObsBiasCorrection(const ObsTable &, const Parameters_ & params) : bias_(0.0), active_(false) { if (params.covariance.value() != boost::none && diff --git a/l95/src/lorenz95/ObsBiasCorrection.h b/l95/src/lorenz95/ObsBiasCorrection.h index 65ffaea56..02912456f 100644 --- a/l95/src/lorenz95/ObsBiasCorrection.h +++ b/l95/src/lorenz95/ObsBiasCorrection.h @@ -26,7 +26,7 @@ namespace eckit { namespace lorenz95 { class ObsBias; - class ObsTableView; + class ObsTable; // ----------------------------------------------------------------------------- @@ -37,7 +37,7 @@ class ObsBiasCorrection : public util::Printable, /// Constructor, destructor ObsBiasCorrection(); - ObsBiasCorrection(const ObsTableView &, const Parameters_ &); + ObsBiasCorrection(const ObsTable &, const Parameters_ &); ObsBiasCorrection(const ObsBiasCorrection &, const bool copy = true); ~ObsBiasCorrection() {} diff --git a/l95/src/lorenz95/ObsBiasCovariance.cc b/l95/src/lorenz95/ObsBiasCovariance.cc index ad98b35b0..ba50c58ac 100644 --- a/l95/src/lorenz95/ObsBiasCovariance.cc +++ b/l95/src/lorenz95/ObsBiasCovariance.cc @@ -24,7 +24,7 @@ // ----------------------------------------------------------------------------- namespace lorenz95 { // ----------------------------------------------------------------------------- -ObsBiasCovariance::ObsBiasCovariance(const ObsTableView &, const Parameters_ & params) +ObsBiasCovariance::ObsBiasCovariance(const ObsTable &, const Parameters_ & params) : variance_(0.0), active_(false) { if (params.covariance.value() != boost::none && diff --git a/l95/src/lorenz95/ObsBiasCovariance.h b/l95/src/lorenz95/ObsBiasCovariance.h index 16c199713..70afa3263 100644 --- a/l95/src/lorenz95/ObsBiasCovariance.h +++ b/l95/src/lorenz95/ObsBiasCovariance.h @@ -26,7 +26,7 @@ namespace eckit { namespace lorenz95 { class ObsBias; class ObsBiasCorrection; - class ObsTableView; + class ObsTable; // ----------------------------------------------------------------------------- @@ -39,7 +39,7 @@ class ObsBiasCovariance : public util::Printable, static const std::string classname() {return "lorenz95::ObsBiasCovariance";} /// Constructor, destructor - ObsBiasCovariance(const ObsTableView &, const Parameters_ &); + ObsBiasCovariance(const ObsTable &, const Parameters_ &); ~ObsBiasCovariance() {} /// Linear algebra operators diff --git a/l95/src/lorenz95/ObsData1D.h b/l95/src/lorenz95/ObsData1D.h index 69aeb7bc1..316de71fa 100644 --- a/l95/src/lorenz95/ObsData1D.h +++ b/l95/src/lorenz95/ObsData1D.h @@ -22,7 +22,7 @@ #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" -#include "lorenz95/ObsTableView.h" +#include "lorenz95/ObsTable.h" #include "lorenz95/ObsVec1D.h" namespace lorenz95 { @@ -36,7 +36,7 @@ class ObsData1D : public util::Printable, public: static const std::string classname() {return "lorenz95::ObsData1D";} - ObsData1D(const ObsTableView &, const oops::Variables &, const std::string &); + ObsData1D(const ObsTable &, const oops::Variables &, const std::string &); ObsData1D(const ObsData1D &); explicit ObsData1D(const ObsVec1D &); ~ObsData1D() {} @@ -57,14 +57,14 @@ class ObsData1D : public util::Printable, private: void print(std::ostream &) const; - const ObsTableView & obsdb_; + const ObsTable & obsdb_; std::vector data_; }; //----------------------------------------------------------------------------- template -ObsData1D::ObsData1D(const ObsTableView & ot, const oops::Variables &, +ObsData1D::ObsData1D(const ObsTable & ot, const oops::Variables &, const std::string & name) : obsdb_(ot), data_(ot.nobs()) { diff --git a/l95/src/lorenz95/ObsDiags1D.h b/l95/src/lorenz95/ObsDiags1D.h index eca20077e..79854df23 100644 --- a/l95/src/lorenz95/ObsDiags1D.h +++ b/l95/src/lorenz95/ObsDiags1D.h @@ -14,7 +14,7 @@ #include "oops/base/Variables.h" #include "oops/util/Printable.h" -#include "lorenz95/ObsTableView.h" +#include "lorenz95/ObsTable.h" namespace lorenz95 { class LocsL95; @@ -23,7 +23,7 @@ namespace lorenz95 { class ObsDiags1D : public util::Printable { public: - ObsDiags1D(const ObsTableView &, const LocsL95 &, const oops::Variables &) {} + ObsDiags1D(const ObsTable &, const LocsL95 &, const oops::Variables &) {} ~ObsDiags1D() {} // I/O diff --git a/l95/src/lorenz95/ObsLocGC99.cc b/l95/src/lorenz95/ObsLocGC99.cc index 7a97ddcd4..f7ecf8eb1 100644 --- a/l95/src/lorenz95/ObsLocGC99.cc +++ b/l95/src/lorenz95/ObsLocGC99.cc @@ -7,17 +7,20 @@ #include "lorenz95/ObsLocGC99.h" +#include +#include #include #include "eckit/config/Configuration.h" +#include "eckit/geometry/Point2.h" +#include "lorenz95/Iterator.h" #include "lorenz95/L95Traits.h" -#include "lorenz95/ObsTableView.h" +#include "lorenz95/ObsTable.h" #include "lorenz95/ObsVec1D.h" #include "oops/generic/gc99.h" #include "oops/interface/ObsLocalization.h" -#include "oops/util/missingValues.h" // ----------------------------------------------------------------------------- namespace lorenz95 { @@ -27,18 +30,26 @@ static oops::ObsLocalizationMaker & obsdist = obsdb_.obsdist(); - double missing = util::missingValue(missing); +void ObsLocGC99::computeLocalization(const Iterator & iterator, ObsData1D & outside, + ObsVec1D & result) const { + std::vector locations = obsdb_.locations(); + eckit::geometry::Point2 center = *iterator; for (unsigned int ii=0; ii < obsdb_.nobs(); ++ii) { - result[ii] = oops::gc99(obsdist[ii]/rscale_); + double curdist = std::abs(center[0] - locations[ii]); + curdist = std::min(curdist, 1.-curdist); + if (curdist >= rscale_) { + outside[ii] = 1; + } else { + outside[ii] = 0; + result[ii] = oops::gc99(curdist/rscale_); + } } } diff --git a/l95/src/lorenz95/ObsLocGC99.h b/l95/src/lorenz95/ObsLocGC99.h index 09d794dbb..f6b634cfb 100644 --- a/l95/src/lorenz95/ObsLocGC99.h +++ b/l95/src/lorenz95/ObsLocGC99.h @@ -13,19 +13,24 @@ #include "eckit/config/Configuration.h" #include "oops/util/Printable.h" +#include "lorenz95/ObsData1D.h" + // Forward declarations namespace lorenz95 { class Iterator; - class ObsTableView; + class ObsTable; class ObsVec1D; /// Observation space localization for Lorenz 95 model (Gaspari-Cohn) class ObsLocGC99: public util::Printable { public: - ObsLocGC99(const eckit::Configuration &, const ObsTableView &); + ObsLocGC99(const eckit::Configuration &, const ObsTable &); - /// compute localization and save in \p obsvector - void computeLocalization(const Iterator &, ObsVec1D & obsvector) const; + /// compute localization and save localization values in \p obsvector and + /// localization flags (1: outside of localization; 0: inside localization area) + /// in \p outside + void computeLocalization(const Iterator &, ObsData1D & outside, + ObsVec1D & obsvector) const; private: void print(std::ostream &) const override; @@ -33,7 +38,8 @@ class ObsLocGC99: public util::Printable { /// Gaspari-Cohn localization distance (localization goes to zero at rscale_) const double rscale_; - const ObsTableView & obsdb_; + /// ObsSpace associated with the observations + const ObsTable & obsdb_; }; // ----------------------------------------------------------------------------- } // namespace lorenz95 diff --git a/l95/src/lorenz95/ObsTable.cc b/l95/src/lorenz95/ObsTable.cc index 72e1ebf1c..24be3b216 100644 --- a/l95/src/lorenz95/ObsTable.cc +++ b/l95/src/lorenz95/ObsTable.cc @@ -23,7 +23,7 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" -#include "lorenz95/LocsL95.h" +#include "lorenz95/ObsIterator.h" #include "oops/mpi/mpi.h" #include "oops/util/abor1_cpp.h" #include "oops/util/DateTime.h" @@ -340,6 +340,14 @@ void ObsTable::otWrite(const std::string & filename) const { oops::Log::trace() << "ObsTable::otWrite done" << std::endl; } +// ----------------------------------------------------------------------------- +ObsIterator ObsTable::begin() const { + return ObsIterator(locations_, 0); +} +// ----------------------------------------------------------------------------- +ObsIterator ObsTable::end() const { + return ObsIterator(locations_, locations_.size()); +} // ----------------------------------------------------------------------------- void ObsTable::print(std::ostream & os) const { diff --git a/l95/src/lorenz95/ObsTable.h b/l95/src/lorenz95/ObsTable.h index 7613142e7..f4db40911 100644 --- a/l95/src/lorenz95/ObsTable.h +++ b/l95/src/lorenz95/ObsTable.h @@ -28,7 +28,7 @@ namespace eckit { } namespace lorenz95 { - class LocsL95; + class ObsIterator; /// A Simple Observation Data Handler /*! @@ -62,6 +62,11 @@ class ObsTable : public oops::ObsSpaceBase, const oops::Variables & obsvariables() const { return obsvars_; } const std::string & obsname() const {return obsname_;} + /// iterator to the first observation + ObsIterator begin() const; + /// iterator to the observation past-the-last + ObsIterator end() const; + private: void print(std::ostream &) const; void otOpen(const std::string &); diff --git a/l95/src/lorenz95/ObsTableView.cc b/l95/src/lorenz95/ObsTableView.cc deleted file mode 100644 index ab915e932..000000000 --- a/l95/src/lorenz95/ObsTableView.cc +++ /dev/null @@ -1,218 +0,0 @@ -/* - * (C) Copyright 2018 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "lorenz95/ObsTableView.h" - -#include -#include -#include -#include -#include - -#include "eckit/config/LocalConfiguration.h" -#include "eckit/types/Types.h" - -#include "lorenz95/ObsIterator.h" - -#include "oops/util/Logger.h" -#include "oops/util/missingValues.h" - -// ----------------------------------------------------------------------------- -namespace lorenz95 { - -ObsTableView::ObsTableView(const eckit::Configuration & config, const eckit::mpi::Comm & comm, - const util::DateTime & bgn, const util::DateTime & end, - const eckit::mpi::Comm & time) - : obstable_(new ObsTable(config, comm, bgn, end, time)), - localobs_(obstable_->nobs()), obsdist_(obstable_->nobs(), 0.0) -{ - std::iota(localobs_.begin(), localobs_.end(), 0); - oops::Log::trace() << "ObsTableView::ObsTableView created nobs = " << nobs() << std::endl; -} - -// ----------------------------------------------------------------------------- - -ObsTableView::ObsTableView(const ObsTableView & obstable, - const eckit::geometry::Point2 & center, - const eckit::Configuration & conf) - : obstable_(obstable.obstable_), localobs_(), obsdist_() -{ - const std::vector & locations = obstable.obstable_->locations(); - const double dist = conf.getDouble("lengthscale"); - for (unsigned int jj = 0; jj < obstable.nobs(); ++jj) { - double curdist = std::abs(center[0] - locations[jj]); - curdist = std::min(curdist, 1.-curdist); - if ( curdist < dist ) { - localobs_.push_back(jj); - obsdist_.push_back(curdist); - } - } - oops::Log::trace() << "ObsTableView::ObsTableView created" << std::endl; -} - -// ----------------------------------------------------------------------------- - -ObsTableView::~ObsTableView() { - oops::Log::trace() << "ObsTableView::ObsTableView destructed" << std::endl; -} - -// ----------------------------------------------------------------------------- - -void ObsTableView::putdb(const std::string & col, const std::vector & vec) const { - int missing; - std::vector fullvec(obstable_->nobs(), util::missingValue(missing)); - if (obstable_->has(col)) { - obstable_->getdb(col, fullvec); - } - for (unsigned int i = 0; i < nobs(); i++) { - fullvec[localobs_[i]] = vec[i]; - } - obstable_->putdb(col, fullvec); - oops::Log::trace() << "ObsTableView::putdb done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -void ObsTableView::putdb(const std::string & col, const std::vector & vec) const { - float missing; - std::vector fullvec(obstable_->nobs(), util::missingValue(missing)); - if (obstable_->has(col)) { - obstable_->getdb(col, fullvec); - } - for (unsigned int i = 0; i < nobs(); i++) { - fullvec[localobs_[i]] = vec[i]; - } - obstable_->putdb(col, fullvec); - oops::Log::trace() << "ObsTableView::putdb done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -void ObsTableView::putdb(const std::string & col, const std::vector & vec) const { - double missing; - std::vector fullvec(obstable_->nobs(), util::missingValue(missing)); - if (obstable_->has(col)) { - obstable_->getdb(col, fullvec); - } - for (unsigned int i = 0; i < nobs(); i++) { - fullvec[localobs_[i]] = vec[i]; - } - obstable_->putdb(col, fullvec); - oops::Log::trace() << "ObsTableView::putdb done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -void ObsTableView::getdb(const std::string & col, std::vector & vec) const { - std::vector fullvec; - obstable_->getdb(col, fullvec); - vec.resize(nobs()); - for (unsigned int i = 0; i < nobs(); i++) { - vec[i] = fullvec[localobs_[i]]; - } - oops::Log::trace() << "ObsTableView::getdb done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -void ObsTableView::getdb(const std::string & col, std::vector & vec) const { - std::vector fullvec; - obstable_->getdb(col, fullvec); - vec.resize(nobs()); - for (unsigned int i = 0; i < nobs(); i++) { - vec[i] = fullvec[localobs_[i]]; - } - oops::Log::trace() << "ObsTableView::getdb done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -void ObsTableView::getdb(const std::string & col, std::vector & vec) const { - std::vector fullvec; - obstable_->getdb(col, fullvec); - vec.resize(nobs()); - for (unsigned int i = 0; i < nobs(); i++) { - vec[i] = fullvec[localobs_[i]]; - } - oops::Log::trace() << "ObsTableView::getdb done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -void ObsTableView::random(std::vector & v) const { - obstable_->random(v); - oops::Log::trace() << "ObsTableView::random done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -unsigned int ObsTableView::nobs() const { - return localobs_.size(); - oops::Log::trace() << "ObsTableView::nobs done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -void ObsTableView::generateDistribution(const eckit::Configuration & conf) { - obstable_->generateDistribution(conf); - int nobs = obstable_->nobs(); - for (int i = 0; i < nobs; i++) - localobs_.push_back(i); - oops::Log::trace() << "ObsTableView::generateDistribution done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -std::unique_ptr ObsTableView::locations() const { - // get times and locations from the obsspace - const std::vector & all_times = obstable_->times(); - const std::vector & all_locs = obstable_->locations(); - // set up locations - const unsigned int nobs = localobs_.size(); - std::vector locs(nobs); - std::vector times(nobs); - for (unsigned int i = 0; i < nobs; i++) { - locs[i] = all_locs[localobs_[i]]; - times[i] = all_times[localobs_[i]]; - } - oops::Log::trace() << "ObsTableView::locations done" << std::endl; - return std::unique_ptr(new LocsL95(locs, times)); -} - -// ----------------------------------------------------------------------------- -ObsIterator ObsTableView::begin() const { - const std::vector & all_locs = obstable_->locations(); - // set up locations - const unsigned int nobs = localobs_.size(); - std::vector locs(nobs); - for (unsigned int i = 0; i < nobs; i++) { - locs[i] = all_locs[localobs_[i]]; - } - return ObsIterator(locs, 0); -} -// ----------------------------------------------------------------------------- -ObsIterator ObsTableView::end() const { - const std::vector & all_locs = obstable_->locations(); - // set up locations - const unsigned int nobs = localobs_.size(); - std::vector locs(nobs); - for (unsigned int i = 0; i < nobs; i++) { - locs[i] = all_locs[localobs_[i]]; - } - return ObsIterator(locs, locs.size()); -} - -// ----------------------------------------------------------------------------- - -void ObsTableView::print(std::ostream & os) const { - os << *obstable_ << std::endl; - os << "Local observation indices: " << localobs_; -} - -// ----------------------------------------------------------------------------- - -} // namespace lorenz95 diff --git a/l95/src/lorenz95/ObsTableView.h b/l95/src/lorenz95/ObsTableView.h deleted file mode 100644 index da5100002..000000000 --- a/l95/src/lorenz95/ObsTableView.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - * (C) Copyright 2018 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef LORENZ95_OBSTABLEVIEW_H_ -#define LORENZ95_OBSTABLEVIEW_H_ - -#include -#include -#include -#include - -#include "eckit/geometry/Point2.h" -#include "eckit/mpi/Comm.h" - -#include "oops/base/Variables.h" -#include "oops/util/DateTime.h" -#include "oops/util/ObjectCounter.h" - -#include "lorenz95/LocsL95.h" -#include "lorenz95/ObsTable.h" - -namespace lorenz95 { - -class ObsIterator; - -/// A Simple Observation Data Handler -/*! - * ObsTableView defines a simple observation handler - * that mimicks the interfaces required from ODB. - */ - -// ----------------------------------------------------------------------------- -class ObsTableView : public util::Printable, - private util::ObjectCounter { - public: - static const std::string classname() {return "lorenz95::ObsTableView";} - - ObsTableView(const eckit::Configuration &, const eckit::mpi::Comm &, - const util::DateTime &, const util::DateTime &, const eckit::mpi::Comm &); - ObsTableView(const ObsTableView &, const eckit::geometry::Point2 &, - const eckit::Configuration &); - ~ObsTableView(); - - void putdb(const std::string &, const std::vector &) const; - void putdb(const std::string &, const std::vector &) const; - void putdb(const std::string &, const std::vector &) const; - void getdb(const std::string &, std::vector &) const; - void getdb(const std::string &, std::vector &) const; - void getdb(const std::string &, std::vector &) const; - - void random(std::vector &) const; - unsigned int nobs() const; - void generateDistribution(const eckit::Configuration &); - std::unique_ptr locations() const; - - size_t index(const size_t ii) const {return localobs_[ii];} - const std::string & obsname() const {return obstable_->obsname();} - - /// iterator to the first observation - ObsIterator begin() const; - /// iterator to the last observation - ObsIterator end() const; - - const util::DateTime & windowStart() const {return obstable_->windowStart();} - const util::DateTime & windowEnd() const {return obstable_->windowEnd();} - const oops::Variables & obsvariables() const { return obstable_->obsvariables(); } - const std::vector & obsdist() const {return obsdist_;} - private: - void print(std::ostream &) const; - std::shared_ptr obstable_; - std::vector localobs_; - std::vector obsdist_; -}; -// ----------------------------------------------------------------------------- -} // namespace lorenz95 - -#endif // LORENZ95_OBSTABLEVIEW_H_ diff --git a/l95/src/lorenz95/ObsVec1D.cc b/l95/src/lorenz95/ObsVec1D.cc index e63037263..76446d2f0 100644 --- a/l95/src/lorenz95/ObsVec1D.cc +++ b/l95/src/lorenz95/ObsVec1D.cc @@ -17,13 +17,13 @@ #include "eckit/config/Configuration.h" #include "eckit/exception/Exceptions.h" #include "lorenz95/ObsData1D.h" -#include "lorenz95/ObsTableView.h" +#include "lorenz95/ObsTable.h" #include "oops/util/Logger.h" #include "oops/util/missingValues.h" namespace lorenz95 { // ----------------------------------------------------------------------------- -ObsVec1D::ObsVec1D(const ObsTableView & ot, +ObsVec1D::ObsVec1D(const ObsTable & ot, const std::string & name) : obsdb_(ot), data_(ot.nobs()), missing_(util::missingValue(missing_)) { @@ -43,13 +43,6 @@ ObsVec1D & ObsVec1D::operator= (const ObsVec1D & rhs) { return *this; } -// ----------------------------------------------------------------------------- -ObsVec1D::ObsVec1D(const ObsTableView & ot, const ObsVec1D & other) - : obsdb_(ot), data_(ot.nobs()), missing_(util::missingValue(missing_)) { - for (size_t ii = 0; ii < ot.nobs(); ++ii) { - data_[ii] = other[ot.index(ii)]; - } -} // ----------------------------------------------------------------------------- ObsVec1D & ObsVec1D::operator*= (const double & zz) { for (double & val : data_) { @@ -185,18 +178,27 @@ void ObsVec1D::save(const std::string & name) const { obsdb_.putdb(name, data_); } // ----------------------------------------------------------------------------- -Eigen::VectorXd ObsVec1D::packEigen() const { - Eigen::VectorXd vec(nobs()); +Eigen::VectorXd ObsVec1D::packEigen(const ObsData1D & mask) const { + Eigen::VectorXd vec(packEigenSize(mask)); size_t ii = 0; - for (const double & val : data_) { - if (val != missing_) { - vec(ii++) = val; + for (size_t jj = 0; jj < data_.size(); ++jj) { + if ((data_[jj] != missing_) && (mask[jj] == 0)) { + vec(ii++) = data_[jj]; } } - ASSERT(ii == nobs()); return vec; } // ----------------------------------------------------------------------------- +size_t ObsVec1D::packEigenSize(const ObsData1D & mask) const { + size_t ii = 0; + for (size_t jj = 0; jj < data_.size(); ++jj) { + if ((data_[jj] != missing_) && (mask[jj] == 0)) { + ii++; + } + } + return ii; +} +// ----------------------------------------------------------------------------- void ObsVec1D::read(const std::string & name) { obsdb_.getdb(name, data_); } diff --git a/l95/src/lorenz95/ObsVec1D.h b/l95/src/lorenz95/ObsVec1D.h index 0d8593a12..c9c3cbae9 100644 --- a/l95/src/lorenz95/ObsVec1D.h +++ b/l95/src/lorenz95/ObsVec1D.h @@ -20,7 +20,7 @@ #include "oops/util/Printable.h" namespace lorenz95 { - class ObsTableView; + class ObsTable; template class ObsData1D; // ----------------------------------------------------------------------------- @@ -34,9 +34,8 @@ class ObsVec1D : public util::Printable, public: static const std::string classname() {return "lorenz95::ObsVec1D";} - explicit ObsVec1D(const ObsTableView &, const std::string & name = ""); + explicit ObsVec1D(const ObsTable &, const std::string & name = ""); ObsVec1D(const ObsVec1D &); - ObsVec1D(const ObsTableView &, const ObsVec1D &); ~ObsVec1D() = default; ObsVec1D & operator= (const ObsVec1D &); @@ -46,8 +45,8 @@ class ObsVec1D : public util::Printable, ObsVec1D & operator*= (const ObsVec1D &); ObsVec1D & operator/= (const ObsVec1D &); - Eigen::VectorXd packEigen() const; - size_t packEigenSize() const {return nobs();} + Eigen::VectorXd packEigen(const ObsData1D &) const; + size_t packEigenSize(const ObsData1D &) const; size_t size() const {return data_.size();} const double & operator[](const std::size_t ii) const {return data_.at(ii);} @@ -66,7 +65,7 @@ class ObsVec1D : public util::Printable, ObsVec1D & operator= (const ObsData1D &); unsigned int nobs() const; - const ObsTableView & obsdb() const {return obsdb_;} + const ObsTable & obsdb() const {return obsdb_;} // I/O void save(const std::string &) const; @@ -75,7 +74,7 @@ class ObsVec1D : public util::Printable, private: void print(std::ostream &) const; - const ObsTableView & obsdb_; + const ObsTable & obsdb_; std::vector data_; const double missing_; }; diff --git a/l95/src/lorenz95/ObservationL95.cc b/l95/src/lorenz95/ObservationL95.cc index 9c5bb6b9f..449a6d2d1 100644 --- a/l95/src/lorenz95/ObservationL95.cc +++ b/l95/src/lorenz95/ObservationL95.cc @@ -15,6 +15,7 @@ #include "eckit/config/Configuration.h" #include "lorenz95/GomL95.h" +#include "lorenz95/LocsL95.h" #include "lorenz95/ObsBias.h" #include "lorenz95/ObsDiags1D.h" #include "lorenz95/ObsVec1D.h" @@ -25,7 +26,7 @@ namespace lorenz95 { // ----------------------------------------------------------------------------- -ObservationL95::ObservationL95(const ObsTableView & ot, const eckit::Configuration &) +ObservationL95::ObservationL95(const ObsTable & ot, const eckit::Configuration &) : obsdb_(ot), inputs_(std::vector{"x"}) {} @@ -45,7 +46,7 @@ void ObservationL95::simulateObs(const GomL95 & gom, ObsVec1D & ovec, // ----------------------------------------------------------------------------- std::unique_ptr ObservationL95::locations() const { - return obsdb_.locations(); + return std::unique_ptr(new LocsL95(obsdb_.locations(), obsdb_.times())); } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObservationL95.h b/l95/src/lorenz95/ObservationL95.h index 45148bfcf..f4757dde7 100644 --- a/l95/src/lorenz95/ObservationL95.h +++ b/l95/src/lorenz95/ObservationL95.h @@ -18,7 +18,7 @@ #include #include "lorenz95/ObservationTLAD.h" -#include "lorenz95/ObsTableView.h" +#include "lorenz95/ObsTable.h" #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" @@ -49,7 +49,7 @@ class ObservationL95 : public util::Printable, public: static const std::string classname() {return "lorenz95::ObservationL95";} - ObservationL95(const ObsTableView &, const eckit::Configuration &); + ObservationL95(const ObsTable &, const eckit::Configuration &); ~ObservationL95(); // Obs Operators @@ -59,11 +59,11 @@ class ObservationL95 : public util::Printable, const oops::Variables & requiredVars() const {return inputs_;} std::unique_ptr locations() const; - const ObsTableView & table() const {return obsdb_;} + const ObsTable & table() const {return obsdb_;} private: void print(std::ostream &) const; - const ObsTableView & obsdb_; + const ObsTable & obsdb_; const oops::Variables inputs_; }; diff --git a/l95/src/lorenz95/ObservationTLAD.cc b/l95/src/lorenz95/ObservationTLAD.cc index 8ee67461e..d0d235046 100644 --- a/l95/src/lorenz95/ObservationTLAD.cc +++ b/l95/src/lorenz95/ObservationTLAD.cc @@ -24,7 +24,7 @@ namespace lorenz95 { // ----------------------------------------------------------------------------- -ObservationTLAD::ObservationTLAD(const ObsTableView &, const eckit::Configuration &) +ObservationTLAD::ObservationTLAD(const ObsTable &, const eckit::Configuration &) : inputs_(std::vector{"x"}) {} diff --git a/l95/src/lorenz95/ObservationTLAD.h b/l95/src/lorenz95/ObservationTLAD.h index 94b3ae5f5..c3ba0b72f 100644 --- a/l95/src/lorenz95/ObservationTLAD.h +++ b/l95/src/lorenz95/ObservationTLAD.h @@ -16,7 +16,7 @@ #include -#include "lorenz95/ObsTableView.h" +#include "lorenz95/ObsTable.h" #include "oops/base/Variables.h" #include "oops/util/ObjectCounter.h" @@ -46,7 +46,7 @@ class ObservationTLAD : public util::Printable, public: static const std::string classname() {return "lorenz95::ObservationTLAD";} - ObservationTLAD(const ObsTableView &, const eckit::Configuration &); + ObservationTLAD(const ObsTable &, const eckit::Configuration &); // Obs Operators void setTrajectory(const GomL95 &, const ObsBias &); diff --git a/l95/src/lorenz95/QCmanager.h b/l95/src/lorenz95/QCmanager.h index 5eb79a33c..6fafa33b5 100644 --- a/l95/src/lorenz95/QCmanager.h +++ b/l95/src/lorenz95/QCmanager.h @@ -19,7 +19,7 @@ namespace lorenz95 { class GomL95; template class ObsData1D; - class ObsTableView; + class ObsTable; class ObsDiags1D; class ObsVec1D; @@ -27,7 +27,7 @@ namespace lorenz95 { class QCmanager : public util::Printable { public: - QCmanager(const ObsTableView &, const eckit::Configuration &, + QCmanager(const ObsTable &, const eckit::Configuration &, std::shared_ptr >, std::shared_ptr >): novars_() {} ~QCmanager() {} diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index 2690fa9cc..8947c856e 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -326,12 +326,6 @@ ecbuild_add_test( TARGET test_l95_obsspace LIBS lorenz95 TEST_DEPENDS test_l95_truth ) -ecbuild_add_test( TARGET test_l95_localobsspace - SOURCES executables/TestLocalObsSpace.cc - ARGS "testinput/interfaces.yaml" - LIBS lorenz95 - TEST_DEPENDS test_l95_truth ) - ecbuild_add_test( TARGET test_l95_obs_iterator SOURCES executables/TestObsIterator.cc ARGS "testinput/interfaces.yaml" @@ -356,18 +350,6 @@ ecbuild_add_test( TARGET test_l95_obsdatavector LIBS lorenz95 TEST_DEPENDS test_l95_truth ) -ecbuild_add_test( TARGET test_l95_departures_ensemble - SOURCES executables/TestDeparturesEnsemble.cc - ARGS "testinput/interfaces.yaml" - LIBS lorenz95 - TEST_DEPENDS test_l95_truth ) - -ecbuild_add_test( TARGET test_l95_departures - SOURCES executables/TestDepartures.cc - ARGS "testinput/interfaces.yaml" - LIBS lorenz95 - TEST_DEPENDS test_l95_truth ) - ecbuild_add_test( TARGET test_l95_obsoperator SOURCES executables/TestObsOperator.cc ARGS "testinput/interfaces.yaml" diff --git a/l95/test/executables/TestDepartures.cc b/l95/test/executables/TestDepartures.cc deleted file mode 100644 index 17092dd51..000000000 --- a/l95/test/executables/TestDepartures.cc +++ /dev/null @@ -1,17 +0,0 @@ -/* - * (C) Copyright 2020-2020 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "lorenz95/L95Traits.h" -#include "oops/runs/Run.h" -#include "test/base/Departures.h" - -int main(int argc, char ** argv) { - oops::Run run(argc, argv); - test::Departures tests; - return run.execute(tests); -} - diff --git a/l95/test/executables/TestDeparturesEnsemble.cc b/l95/test/executables/TestDeparturesEnsemble.cc deleted file mode 100644 index 6a4a34e77..000000000 --- a/l95/test/executables/TestDeparturesEnsemble.cc +++ /dev/null @@ -1,17 +0,0 @@ -/* - * (C) Copyright 2020-2020 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "lorenz95/L95Traits.h" -#include "oops/runs/Run.h" -#include "test/base/DeparturesEnsemble.h" - -int main(int argc, char ** argv) { - oops::Run run(argc, argv); - test::DeparturesEnsemble tests; - return run.execute(tests); -} - diff --git a/l95/test/executables/TestLocalObsSpace.cc b/l95/test/executables/TestLocalObsSpace.cc deleted file mode 100644 index 81892ba62..000000000 --- a/l95/test/executables/TestLocalObsSpace.cc +++ /dev/null @@ -1,17 +0,0 @@ -/* - * (C) Copyright 2018-2019 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "lorenz95/L95Traits.h" -#include "oops/runs/Run.h" -#include "test/interface/LocalObsSpace.h" - -int main(int argc, char ** argv) { - oops::Run run(argc, argv); - test::LocalObsSpace tests; - return run.execute(tests); -} - diff --git a/l95/test/testinput/hofx3d_for_getkf.yaml b/l95/test/testinput/hofx3d_for_getkf.yaml index 5e87271f1..3bd3bea4a 100644 --- a/l95/test/testinput/hofx3d_for_getkf.yaml +++ b/l95/test/testinput/hofx3d_for_getkf.yaml @@ -20,10 +20,10 @@ background: observations: - obs error: - covariance model: localized diagonal - localization: - localization method: Gaspari-Cohn - lengthscale: .1 + covariance model: diagonal + obs localization: + localization method: Gaspari-Cohn + lengthscale: .1 obs space: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obsdataout: Data/l95.getkf.2010-01-02T00:00:00Z.obt diff --git a/l95/test/testinput/interfaces.yaml b/l95/test/testinput/interfaces.yaml index ac811e5cf..80cd40705 100644 --- a/l95/test/testinput/interfaces.yaml +++ b/l95/test/testinput/interfaces.yaml @@ -115,8 +115,13 @@ observations: norm: 0.3 relative tolerance: 0.0 obs localization: - lengthscale: 0.2 + lengthscale: 0.19 localization method: Gaspari-Cohn + reference local nobs: [56, 64, 56, 64, 56, 64, 56, 64, 56, 64] + reference gridpoints: + lons: [ 0, 0.025, 0.05, 0.075, 0.1, 0.875, 0.9, 0.925, 0.95, 0.975 ] + lats: [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + localization reduces values: true rms ref: 8.3207407741318846 reference nobs: 160 tolerance: 1.0e-10 @@ -126,12 +131,3 @@ observations: lat1: 0.0 lon2: 0.05 lat2: 0.0 - -local obs space: - location: - lon: 0.5 - lat: 0 - localization: - lengthscale: 0.2 - variable name: ObsValue - reference nobs: 64 diff --git a/qg/model/CMakeLists.txt b/qg/model/CMakeLists.txt index b4c20b066..f7580c928 100644 --- a/qg/model/CMakeLists.txt +++ b/qg/model/CMakeLists.txt @@ -39,6 +39,8 @@ ObsBiasIncrement.cc ObsBiasIncrement.h ObsDataQG.h ObsDiagsQG.h +ObsLocQG.cc +ObsLocQG.h ObsOpBaseQG.cc ObsOpBaseQG.h ObsOpBaseTLAD.cc diff --git a/qg/model/ObsDataQG.h b/qg/model/ObsDataQG.h index 08ac7a632..ed04179af 100644 --- a/qg/model/ObsDataQG.h +++ b/qg/model/ObsDataQG.h @@ -40,7 +40,12 @@ class ObsDataQG : public util::Printable, ObsDataQG & operator= (const ObsDataQG &); + /// set all values to zero void zero(); + /// set \p i-th value to zero + void zero(int i); + /// set all values to one + void ones(); void mask(const ObsDataQG); // I/O @@ -81,6 +86,16 @@ void ObsDataQG::zero() { } // ----------------------------------------------------------------------------- template +void ObsDataQG::zero(int i) { + data_.zero(i); +} +// ----------------------------------------------------------------------------- +template +void ObsDataQG::ones() { + data_.ones(); +} +// ----------------------------------------------------------------------------- +template void ObsDataQG::mask(const ObsDataQG mask) { qg_obsvec_mask_f90(data_.toFortran(), mask.toFortran()); } diff --git a/qg/model/ObsLocQG.cc b/qg/model/ObsLocQG.cc new file mode 100644 index 000000000..4acecae96 --- /dev/null +++ b/qg/model/ObsLocQG.cc @@ -0,0 +1,66 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "model/ObsLocQG.h" + +#include + +#include "atlas/array.h" +#include "eckit/config/Configuration.h" +#include "eckit/geometry/Point2.h" +#include "eckit/geometry/Sphere.h" + +#include "oops/interface/ObsLocalization.h" +#include "oops/util/Logger.h" + +#include "model/GeometryQGIterator.h" +#include "model/LocationsQG.h" +#include "model/ObsSpaceQG.h" +#include "model/QgTraits.h" + +using atlas::array::make_view; + +namespace qg { + +static oops::ObsLocalizationMaker> makerObsLoc_("Heaviside"); + +// ----------------------------------------------------------------------------- + +ObsLocQG::ObsLocQG(const eckit::Configuration & conf, const ObsSpaceQG & obsdb) + : lengthscale_(conf.getDouble("lengthscale")), obsdb_(obsdb) +{ +} + +// ----------------------------------------------------------------------------- + +void ObsLocQG::computeLocalization(const GeometryQGIterator & p, + ObsDataQG & outside, ObsVecQG &) const { + std::unique_ptr locs = obsdb_.locations(); + atlas::Field field_lonlat = locs->lonlat(); + auto lonlat = make_view(field_lonlat); + eckit::geometry::Point2 refPoint = *p; + + outside.ones(); + for (int jj = 0; jj < locs->size(); ++jj) { + eckit::geometry::Point2 obsPoint(lonlat(jj, 0), lonlat(jj, 1)); + double localDist = eckit::geometry::Sphere::distance(6.371e6, refPoint, obsPoint); + if (localDist < lengthscale_) { + outside.zero(jj); + } + } +} + +// ----------------------------------------------------------------------------- + +void ObsLocQG::print(std::ostream & os) const { + os << "Observation space localization: Heaviside with lengthscale = " << lengthscale_; +} + +// ----------------------------------------------------------------------------- + +} // namespace qg diff --git a/qg/model/ObsLocQG.h b/qg/model/ObsLocQG.h new file mode 100644 index 000000000..f49b238f5 --- /dev/null +++ b/qg/model/ObsLocQG.h @@ -0,0 +1,44 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef QG_MODEL_OBSLOCQG_H_ +#define QG_MODEL_OBSLOCQG_H_ + +#include + +#include "atlas/field.h" + +#include "oops/util/Printable.h" + +#include "oops/qg/ObsDataQG.h" + +namespace eckit { + class Configuration; +} + +namespace qg { + class GeometryQGIterator; + class ObsSpaceQG; + class ObsVecQG; + +/// \brief Observation-space localization for QG model (Heaviside function +/// with prescribed lengthscale). +class ObsLocQG : public util::Printable { + public: + ObsLocQG(const eckit::Configuration &, const ObsSpaceQG &); + + void computeLocalization(const GeometryQGIterator &, ObsDataQG &, ObsVecQG &) const; + + private: + void print(std::ostream &) const override; + const double lengthscale_; + const ObsSpaceQG & obsdb_; +}; + +} // namespace qg + +#endif // QG_MODEL_OBSLOCQG_H_ diff --git a/qg/model/ObsSpaceQG.cc b/qg/model/ObsSpaceQG.cc index 1095527e4..f1500be5b 100644 --- a/qg/model/ObsSpaceQG.cc +++ b/qg/model/ObsSpaceQG.cc @@ -40,7 +40,7 @@ ObsSpaceQG::ObsSpaceQG(const eckit::Configuration & config, const eckit::mpi::Co const util::DateTime & bgn, const util::DateTime & end, const eckit::mpi::Comm & timeComm) : oops::ObsSpaceBase(config, comm, bgn, end), obsname_(config.getString("obs type")), - winbgn_(bgn), winend_(end), obsvars_(), isLocal_(false), comm_(comm) + winbgn_(bgn), winend_(end), obsvars_() { typedef std::map< std::string, F90odb >::iterator otiter; @@ -109,62 +109,24 @@ ObsSpaceQG::ObsSpaceQG(const eckit::Configuration & config, const eckit::mpi::Co // ----------------------------------------------------------------------------- -ObsSpaceQG::ObsSpaceQG(const ObsSpaceQG & obsdb, - const eckit::geometry::Point2 & refPoint, - const eckit::Configuration & conf) - : oops::ObsSpaceBase(eckit::LocalConfiguration(), obsdb.comm_, - obsdb.windowStart(), obsdb.windowEnd()), - key_(obsdb.key_), obsname_(obsdb.obsname_), - winbgn_(obsdb.winbgn_), winend_(obsdb.winend_), obsvars_(obsdb.obsvars_), - localobs_(), isLocal_(true), comm_(obsdb.comm_) -{ - oops::Log::trace() << "ObsSpaceQG for LocalObs starting" << std::endl; - const double dist = conf.getDouble("lengthscale"); - - // get locations of all obs - std::unique_ptr locs = locations(); - - atlas::Field field_lonlat = locs->lonlat(); - auto lonlat = make_view(field_lonlat); - - for (int jj = 0; jj < locs->size(); ++jj) { - eckit::geometry::Point2 obsPoint(lonlat(jj, 0), lonlat(jj, 1)); - double localDist = eckit::geometry::Sphere::distance(6.371e6, refPoint, obsPoint); - if (localDist < dist) localobs_.push_back(jj); - } - - oops::Log::trace() << "ObsSpaceQG for LocalObs done" << std::endl; -} - -// ----------------------------------------------------------------------------- - ObsSpaceQG::~ObsSpaceQG() { - if ( !isLocal_ ) { - ASSERT(theObsFileCount_ > 0); - theObsFileCount_--; - if (theObsFileCount_ == 0) { - theObsFileRegister_.clear(); - qg_obsdb_delete_f90(key_); - } + ASSERT(theObsFileCount_ > 0); + theObsFileCount_--; + if (theObsFileCount_ == 0) { + theObsFileRegister_.clear(); + qg_obsdb_delete_f90(key_); } } // ----------------------------------------------------------------------------- void ObsSpaceQG::getdb(const std::string & col, int & keyData) const { - if ( isLocal_ ) { - qg_obsdb_get_local_f90(key_, obsname_.size(), obsname_.c_str(), col.size(), - col.c_str(), localobs_.size(), localobs_.data(), keyData); - } else { - qg_obsdb_get_f90(key_, obsname_.size(), obsname_.c_str(), col.size(), col.c_str(), keyData); - } + qg_obsdb_get_f90(key_, obsname_.size(), obsname_.c_str(), col.size(), col.c_str(), keyData); } // ----------------------------------------------------------------------------- void ObsSpaceQG::putdb(const std::string & col, const int & keyData) const { - // not implemented for local ObsSpace - ASSERT(isLocal_ == false); qg_obsdb_put_f90(key_, obsname_.size(), obsname_.c_str(), col.size(), col.c_str(), keyData); } @@ -180,13 +142,9 @@ std::unique_ptr ObsSpaceQG::locations() const { // ----------------------------------------------------------------------------- int ObsSpaceQG::nobs() const { - if ( isLocal_ ) { - return localobs_.size(); - } else { - int iobs; - qg_obsdb_nobs_f90(key_, obsname_.size(), obsname_.c_str(), iobs); - return iobs; - } + int iobs; + qg_obsdb_nobs_f90(key_, obsname_.size(), obsname_.c_str(), iobs); + return iobs; } // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsSpaceQG.h b/qg/model/ObsSpaceQG.h index 06bef05d0..0730175a4 100644 --- a/qg/model/ObsSpaceQG.h +++ b/qg/model/ObsSpaceQG.h @@ -45,9 +45,6 @@ class ObsSpaceQG : public oops::ObsSpaceBase { /// create full ObsSpace (read or generate data) ObsSpaceQG(const eckit::Configuration &, const eckit::mpi::Comm &, const util::DateTime &, const util::DateTime &, const eckit::mpi::Comm &); - /// create local ObsSpace - ObsSpaceQG(const ObsSpaceQG &, const eckit::geometry::Point2 &, - const eckit::Configuration &); ~ObsSpaceQG(); /// read data or metadata @@ -67,9 +64,6 @@ class ObsSpaceQG : public oops::ObsSpaceBase { /// observation type const std::string & obsname() const {return obsname_;} - /// local observations indices - const std::vector & localobs() const { return localobs_;} - /// interface with Fortran const F90odb & toFortran() const {return key_;} @@ -81,9 +75,6 @@ class ObsSpaceQG : public oops::ObsSpaceBase { const util::DateTime winbgn_; // window for the observations const util::DateTime winend_; oops::Variables obsvars_; // variables simulated by ObsOperators - std::vector localobs_; // indices of local observations - bool isLocal_; // true if it's a local subset - const eckit::mpi::Comm & comm_; // MPI communicator associated with ObsSpace // defines mapping for Fortran structures static std::map < std::string, F90odb > theObsFileRegister_; diff --git a/qg/model/ObsVecQG.cc b/qg/model/ObsVecQG.cc index d16cb0ae0..97d4b7374 100644 --- a/qg/model/ObsVecQG.cc +++ b/qg/model/ObsVecQG.cc @@ -36,14 +36,6 @@ ObsVecQG::ObsVecQG(const ObsVecQG & other) qg_obsvec_copy_f90(keyOvec_, other.keyOvec_); } // ----------------------------------------------------------------------------- -ObsVecQG::ObsVecQG(const ObsSpaceQG & obsdb, const ObsVecQG & other) - : obsdb_(obsdb), keyOvec_(0) -{ - qg_obsvec_setup_f90(keyOvec_, obsdb.obsvariables().size(), obsdb.nobs()); - qg_obsvec_copy_local_f90(keyOvec_, other.keyOvec_, obsdb.localobs().size(), - obsdb.localobs().data()); -} -// ----------------------------------------------------------------------------- ObsVecQG::~ObsVecQG() { qg_obsvec_delete_f90(keyOvec_); } @@ -92,6 +84,10 @@ void ObsVecQG::zero() { qg_obsvec_zero_f90(keyOvec_); } // ----------------------------------------------------------------------------- +void ObsVecQG::zero(int ii) { + qg_obsvec_zero_ith_f90(keyOvec_, ii); +} +// ----------------------------------------------------------------------------- void ObsVecQG::ones() { qg_obsvec_ones_f90(keyOvec_); } @@ -135,12 +131,18 @@ void ObsVecQG::save(const std::string & name) const { obsdb_.putdb(name, keyOvec_); } // ----------------------------------------------------------------------------- -Eigen::VectorXd ObsVecQG::packEigen() const { - Eigen::VectorXd vec(nobs()); - qg_obsvec_get_f90(keyOvec_, vec.data(), vec.size()); +Eigen::VectorXd ObsVecQG::packEigen(const ObsDataQG & mask) const { + Eigen::VectorXd vec(packEigenSize(mask)); + qg_obsvec_get_withmask_f90(keyOvec_, mask.toFortran(), vec.data(), vec.size()); return vec; } // ----------------------------------------------------------------------------- +size_t ObsVecQG::packEigenSize(const ObsDataQG & mask) const { + int nobs; + qg_obsvec_nobs_withmask_f90(keyOvec_, mask.toFortran(), nobs); + return nobs; +} +// ----------------------------------------------------------------------------- void ObsVecQG::read(const std::string & name) { obsdb_.getdb(name, keyOvec_); } diff --git a/qg/model/ObsVecQG.h b/qg/model/ObsVecQG.h index 55ff1d0f8..a21281152 100644 --- a/qg/model/ObsVecQG.h +++ b/qg/model/ObsVecQG.h @@ -36,7 +36,6 @@ class ObsVecQG : public util::Printable, ObsVecQG(const ObsSpaceQG &, const std::string & name = ""); ObsVecQG(const ObsVecQG &); - ObsVecQG(const ObsSpaceQG &, const ObsVecQG &); ~ObsVecQG(); ObsVecQG & operator = (const ObsVecQG &); @@ -45,11 +44,14 @@ class ObsVecQG : public util::Printable, ObsVecQG & operator-= (const ObsVecQG &); ObsVecQG & operator*= (const ObsVecQG &); ObsVecQG & operator/= (const ObsVecQG &); - Eigen::VectorXd packEigen() const; - size_t packEigenSize() const {return nobs();} + + Eigen::VectorXd packEigen(const ObsDataQG &) const; + size_t packEigenSize(const ObsDataQG &) const; /// set all values to zero void zero(); + /// set \p i-th value to zero + void zero(int i); /// set all values to one void ones(); void axpy(const double &, const ObsVecQG &); diff --git a/qg/model/QgFortran.h b/qg/model/QgFortran.h index af18e7ea1..375a021dd 100644 --- a/qg/model/QgFortran.h +++ b/qg/model/QgFortran.h @@ -199,9 +199,6 @@ extern "C" { void qg_obsdb_delete_f90(F90odb &); void qg_obsdb_get_f90(const F90odb &, const int &, const char *, const int &, const char *, const F90ovec &); - void qg_obsdb_get_local_f90(const F90odb &, const int &, const char *, - const int &, const char *, const int &, const int *, - const F90ovec &); void qg_obsdb_put_f90(const F90odb &, const int &, const char *, const int &, const char *, const F90ovec &); void qg_obsdb_locations_f90(const F90odb &, const int &, const char *, @@ -218,8 +215,8 @@ extern "C" { void qg_obsvec_clone_f90(F90ovec &, const F90ovec &); void qg_obsvec_delete_f90(F90ovec &); void qg_obsvec_copy_f90(const F90ovec &, const F90ovec &); - void qg_obsvec_copy_local_f90(const F90ovec &, const F90ovec &, const int &, const int *); void qg_obsvec_zero_f90(const F90ovec &); + void qg_obsvec_zero_ith_f90(const F90ovec &, const int &); void qg_obsvec_ones_f90(const F90ovec &); /// set ObsVector (with key \p obsvector_key) values to missing values where /// mask ObsVector (with key \p mask_key) values are set to 1 @@ -236,7 +233,10 @@ extern "C" { void qg_obsvec_stats_f90(const F90ovec &, double &, double &, double &); void qg_obsvec_nobs_f90(const F90ovec &, int &); /// fill \p data (size \p nobs) with all non-masked out (non-missing) values - void qg_obsvec_get_f90(const F90ovec &, double * data, const int & nobs); + void qg_obsvec_get_withmask_f90(const F90ovec &, const F90ovec & mask_key, + double * data, const int & nobs); + void qg_obsvec_nobs_withmask_f90(const F90ovec &, const F90ovec & mask_key, int &); + // ----------------------------------------------------------------------------- // Streamfunction observations diff --git a/qg/model/qg_obsdb_interface.F90 b/qg/model/qg_obsdb_interface.F90 index 29cbe5210..544cd9c81 100644 --- a/qg/model/qg_obsdb_interface.F90 +++ b/qg/model/qg_obsdb_interface.F90 @@ -103,42 +103,10 @@ subroutine qg_obsdb_get_c(c_key_self,lgrp,c_grp,lcol,c_col,c_key_ovec) bind(c,na call qg_obsvec_registry%get(c_key_ovec,ovec) ! Call Fortran -call qg_obsdb_get(self,trim(grp),trim(col),ovec,.false.) +call qg_obsdb_get(self,trim(grp),trim(col),ovec) end subroutine qg_obsdb_get_c ! ------------------------------------------------------------------------------ -!> Get observations data for a local subset -subroutine qg_obsdb_get_local_c(c_key_self,lgrp,c_grp,lcol,c_col,c_idxsize,c_idx,& - c_key_ovec) bind(c,name='qg_obsdb_get_local_f90') -implicit none - -! Passed variables -integer(c_int),intent(in) :: c_key_self !< Observation data -integer(c_int),intent(in) :: lgrp !< Group size -character(kind=c_char,len=1),intent(in) :: c_grp(lgrp+1) !< Group name -integer(c_int),intent(in) :: lcol !< Column size -character(kind=c_char,len=1),intent(in) :: c_col(lcol+1) !< Column name -integer(c_int),intent(in) :: c_key_ovec !< Observation vector -integer(c_int),intent(in) :: c_idxsize !< size of local obs index vector -integer(c_int),intent(in) :: c_idx(c_idxsize) !< Index vector for local obs - -! Local variables -type(qg_obsdb),pointer :: self -type(qg_obsvec),pointer :: ovec -character(len=lgrp) :: grp -character(len=lcol) :: col - -! Interface -call qg_obsdb_registry%get(c_key_self,self) -call c_f_string(c_grp,grp) -call c_f_string(c_col,col) -call qg_obsvec_registry%get(c_key_ovec,ovec) - -! Call Fortran -call qg_obsdb_get(self,trim(grp),trim(col),ovec,.true.,c_idx) - -end subroutine qg_obsdb_get_local_c -! ------------------------------------------------------------------------------ !> Put observation data subroutine qg_obsdb_put_c(c_key_self,lgrp,c_grp,lcol,c_col,c_key_ovec) bind(c,name='qg_obsdb_put_f90') diff --git a/qg/model/qg_obsdb_mod.F90 b/qg/model/qg_obsdb_mod.F90 index 0a4f58ab5..ce4d587e5 100644 --- a/qg/model/qg_obsdb_mod.F90 +++ b/qg/model/qg_obsdb_mod.F90 @@ -153,7 +153,7 @@ subroutine qg_obsdb_delete(self) end subroutine qg_obsdb_delete ! ------------------------------------------------------------------------------ !> Get observation data -subroutine qg_obsdb_get(self,grp,col,ovec,local,idx) +subroutine qg_obsdb_get(self,grp,col,ovec) implicit none @@ -162,8 +162,6 @@ subroutine qg_obsdb_get(self,grp,col,ovec,local,idx) character(len=*),intent(in) :: grp !< Group character(len=*),intent(in) :: col !< Column type(qg_obsvec),intent(inout) :: ovec !< Observation vector -logical,intent(in) :: local !< whether local subsetting is to be done -integer,intent(in),optional :: idx(:) !< subset of indices to get ! Local variables type(group_data),pointer :: jgrp @@ -193,31 +191,14 @@ subroutine qg_obsdb_get(self,grp,col,ovec,local,idx) ! Get observation data if (allocated(ovec%values)) deallocate(ovec%values) ovec%nlev = jcol%nlev -if (local) then - ! if local subset, but desired subset is zero length, return empty vector. - if ( .not. present(idx)) then - ovec%nobs = 0 - else - ovec%nobs = size(idx) - endif - - ! get only a subset of the obs - allocate(ovec%values(ovec%nlev,ovec%nobs)) - do jobs=1,ovec%nobs - do jlev=1,jcol%nlev - ovec%values(jlev,jobs) = jcol%values(jlev,idx(jobs)+1) - enddo - enddo -else - ! get all the obs - ovec%nobs = jgrp%nobs - allocate(ovec%values(ovec%nlev,ovec%nobs)) - do jobs=1,jgrp%nobs - do jlev=1,jcol%nlev - ovec%values(jlev,jobs) = jcol%values(jlev,jobs) - enddo +! get all the obs +ovec%nobs = jgrp%nobs +allocate(ovec%values(ovec%nlev,ovec%nobs)) +do jobs=1,jgrp%nobs + do jlev=1,jcol%nlev + ovec%values(jlev,jobs) = jcol%values(jlev,jobs) enddo -end if +enddo end subroutine qg_obsdb_get ! ------------------------------------------------------------------------------ @@ -458,8 +439,6 @@ subroutine qg_obsdb_read(self,winbgn,winend) ! Get dimensions call ncerr(nf90_inquire_dimension(ncid,nobs_id,len=nobsfile)) call ncerr(nf90_inquire_dimension(ncid,ncol_id,len=ncol)) - write(record,*) 'qg_obsdb_read: reading ',nobsfile,' ',jgrp%grpname,' observations' - call fckit_log%info(record) ! Get variables ids call ncerr(nf90_inq_varid(ncid,'times_'//igrpchar,times_id)) @@ -485,8 +464,6 @@ subroutine qg_obsdb_read(self,winbgn,winend) inwindow(iobs) = .false. endif end do - write(record,*) 'qg_obsdb_read: keeping ',jgrp%nobs,' ',jgrp%grpname,' observations' - call fckit_log%info(record) allocate(jgrp%times(jgrp%nobs)) jobs=0 @@ -497,8 +474,6 @@ subroutine qg_obsdb_read(self,winbgn,winend) endif end do deallocate(alltimes) - write(record,*) 'qg_obsdb_read: ',jgrp%grpname,' after times ',jgrp%nobs - call fckit_log%info(record) ! Loop over columns do icol=1,ncol @@ -514,8 +489,6 @@ subroutine qg_obsdb_read(self,winbgn,winend) ! Get variables call ncerr(nf90_get_var(ncid,nlev_id,jcol%nlev,(/icol/))) call ncerr(nf90_get_var(ncid,colname_id,jcol%colname,(/1,icol/),(/50,1/))) - write(record,*) 'qg_obsdb_read: ',jgrp%grpname,' after get var ',jgrp%nobs - call fckit_log%info(record) ! Allocation allocate(readbuf(jcol%nlev,nobsfile)) @@ -523,8 +496,6 @@ subroutine qg_obsdb_read(self,winbgn,winend) ! Get values call ncerr(nf90_get_var(ncid,values_id,readbuf(1:jcol%nlev,:),(/1,icol,1/),(/jcol%nlev,1,nobsfile/))) - write(record,*) 'qg_obsdb_read: ',jgrp%grpname,' after get values ',jgrp%nobs - call fckit_log%info(record) jobs = 0 do iobs=1,nobsfile @@ -534,16 +505,10 @@ subroutine qg_obsdb_read(self,winbgn,winend) endif enddo deallocate(readbuf) - write(record,*) 'qg_obsdb_read: ',jgrp%grpname,' after readbuf ',jgrp%nobs, jobs - call fckit_log%info(record) enddo deallocate(inwindow) - write(record,*) 'qg_obsdb_read: ',jgrp%grpname,' done ',jgrp%nobs - call fckit_log%info(record) enddo - write(record,*) 'qg_obsdb_read: closing file' - call fckit_log%info(record) ! Close NetCDF file call ncerr(nf90_close(ncid)) diff --git a/qg/model/qg_obsvec_interface.F90 b/qg/model/qg_obsvec_interface.F90 index 0112c7ac8..9ec95fc23 100644 --- a/qg/model/qg_obsvec_interface.F90 +++ b/qg/model/qg_obsvec_interface.F90 @@ -107,37 +107,33 @@ subroutine qg_obsvec_copy_c(c_key_self,c_key_other) bind(c,name='qg_obsvec_copy_ end subroutine qg_obsvec_copy_c ! ------------------------------------------------------------------------------ -!> Copy a local subset of the observation vector -subroutine qg_obsvec_copy_local_c(c_key_self,c_key_other, c_idxsize,c_idx) bind(c,name='qg_obsvec_copy_local_f90') +!> Set observation vector to zero +subroutine qg_obsvec_zero_c(c_key_self) bind(c,name='qg_obsvec_zero_f90') implicit none ! Passed variables -integer(c_int),intent(in) :: c_key_self !< Observation vector -integer(c_int),intent(in) :: c_key_other !< Other observation vector -integer(c_int),intent(in) :: c_idxsize -integer(c_int),intent(in) :: c_idx(c_idxsize) +integer(c_int),intent(in) :: c_key_self !< Observation vector ! Local variables -type(qg_obsvec),pointer :: self,other +type(qg_obsvec),pointer :: self ! Interface call qg_obsvec_registry%get(c_key_self,self) -call qg_obsvec_registry%get(c_key_other,other) ! Call Fortran -call qg_obsvec_copy_local(self,other,c_idx) +call qg_obsvec_zero(self) -end subroutine qg_obsvec_copy_local_c +end subroutine qg_obsvec_zero_c ! ------------------------------------------------------------------------------ -!> Set observation vector to zero -subroutine qg_obsvec_zero_c(c_key_self) bind(c,name='qg_obsvec_zero_f90') +!> Set i-th value of the observation vector to zero +subroutine qg_obsvec_zero_ith_c(c_key_self, i) bind(c,name='qg_obsvec_zero_ith_f90') implicit none ! Passed variables integer(c_int),intent(in) :: c_key_self !< Observation vector - +integer(c_int),intent(in) :: i !< index of value to be set to zero ! Local variables type(qg_obsvec),pointer :: self @@ -145,9 +141,10 @@ subroutine qg_obsvec_zero_c(c_key_self) bind(c,name='qg_obsvec_zero_f90') call qg_obsvec_registry%get(c_key_self,self) ! Call Fortran -call qg_obsvec_zero(self) +! increase index by 1 (C indices start with 0; Fortran indices start with 1) +call qg_obsvec_zero_ith(self, i+1) -end subroutine qg_obsvec_zero_c +end subroutine qg_obsvec_zero_ith_c ! ------------------------------------------------------------------------------ !> Set observation vector to ones subroutine qg_obsvec_ones_c(c_key_self) bind(c,name='qg_obsvec_ones_f90') @@ -417,28 +414,52 @@ subroutine qg_obsvec_nobs_c(c_key_self,kobs) bind(c,name='qg_obsvec_nobs_f90') call qg_obsvec_nobs(self,kobs) end subroutine qg_obsvec_nobs_c +! ------------------------------------------------------------------------------ +!> Get observation vector size (only non-masked observations) +subroutine qg_obsvec_nobs_withmask_c(c_key_self,c_key_mask,kobs) bind(c,name='qg_obsvec_nobs_withmask_f90') + +implicit none + +! Passed variables +integer(c_int),intent(in) :: c_key_self !< Observation vector +integer(c_int),intent(in) :: c_key_mask !< Mask +integer(c_int),intent(inout) :: kobs !< Observation vector size + +! Local vector +type(qg_obsvec),pointer :: self, mask + +! Interface +call qg_obsvec_registry%get(c_key_self,self) +call qg_obsvec_registry%get(c_key_mask,mask) + +! Call Fortran +call qg_obsvec_nobs_withmask(self,mask,kobs) + +end subroutine qg_obsvec_nobs_withmask_c ! ------------------------------------------------------------------------------ !> Get all non-masked out observation values -subroutine qg_obsvec_get_c(c_key_self,vals,nvals) bind(c,name='qg_obsvec_get_f90') +subroutine qg_obsvec_get_withmask_c(c_key_self,c_key_mask,vals,nvals) bind(c,name='qg_obsvec_get_withmask_f90') implicit none ! Passed variables integer(c_int),intent(in) :: c_key_self !< Observation vector +integer(c_int),intent(in) :: c_key_mask !< Mask integer(c_int),intent(in) :: nvals !< number of obs real(c_double),intent(out),dimension(nvals) :: vals !< ob. values ! Local vector -type(qg_obsvec),pointer :: self +type(qg_obsvec),pointer :: self, mask ! Interface call qg_obsvec_registry%get(c_key_self,self) +call qg_obsvec_registry%get(c_key_mask,mask) ! Call Fortran -call qg_obsvec_get(self,vals,nvals) +call qg_obsvec_get_withmask(self,mask,vals,nvals) -end subroutine qg_obsvec_get_c +end subroutine qg_obsvec_get_withmask_c ! ------------------------------------------------------------------------------ end module qg_obsvec_interface diff --git a/qg/model/qg_obsvec_mod.F90 b/qg/model/qg_obsvec_mod.F90 index 55a31fb0d..40f400f46 100644 --- a/qg/model/qg_obsvec_mod.F90 +++ b/qg/model/qg_obsvec_mod.F90 @@ -20,9 +20,9 @@ module qg_obsvec_mod public :: qg_obsvec public :: qg_obsvec_registry public :: qg_obsvec_setup,qg_obsvec_clone,qg_obsvec_delete,qg_obsvec_copy,qg_obsvec_zero, & - & qg_obsvec_ones, qg_obsvec_mask, qg_obsvec_mul_scal,qg_obsvec_add, & + & qg_obsvec_zero_ith, qg_obsvec_ones, qg_obsvec_mask, qg_obsvec_mul_scal,qg_obsvec_add, & & qg_obsvec_sub,qg_obsvec_mul,qg_obsvec_div,qg_obsvec_axpy,qg_obsvec_invert,qg_obsvec_random,qg_obsvec_dotprod, & - & qg_obsvec_stats,qg_obsvec_nobs,qg_obsvec_copy_local,qg_obsvec_get + & qg_obsvec_stats,qg_obsvec_nobs,qg_obsvec_nobs_withmask,qg_obsvec_get_withmask ! ------------------------------------------------------------------------------ interface subroutine qg_obsvec_random_i(odb,nn,zz) bind(c,name='qg_obsvec_random_f') @@ -141,50 +141,32 @@ subroutine qg_obsvec_copy(self,other) end subroutine qg_obsvec_copy ! ------------------------------------------------------------------------------ -!> Copy a local subset of the observation vector -subroutine qg_obsvec_copy_local(self,other,idx) +!> Set observation vector to zero +subroutine qg_obsvec_zero(self) implicit none ! Passed variables type(qg_obsvec),intent(inout) :: self !< Observation vector -type(qg_obsvec),intent(in) :: other !< Other observation vector -integer,intent(in) :: idx(:) - -! local variables -integer :: i - -if ((other%nlev/=self%nlev).or.(size(idx)/=self%nobs)) then - ! Release memory - deallocate(self%values) - - ! Set sizes - self%nlev = other%nlev - self%nobs = size(idx) - - ! Allocation - allocate(self%values(self%nlev,self%nobs)) -endif -! Copy data -do i = 1,self%nobs - self%values(:,i) = other%values(:,idx(i)+1) -enddo +! Set observation vector to zero +self%values = 0.0 -end subroutine qg_obsvec_copy_local +end subroutine qg_obsvec_zero ! ------------------------------------------------------------------------------ -!> Set observation vector to zero -subroutine qg_obsvec_zero(self) +!> Set i-th value of observation vector to zero +subroutine qg_obsvec_zero_ith(self, i) implicit none ! Passed variables type(qg_obsvec),intent(inout) :: self !< Observation vector +integer, intent(in) :: i ! Set observation vector to zero -self%values = 0.0 +self%values(:,i) = 0.0 -end subroutine qg_obsvec_zero +end subroutine qg_obsvec_zero_ith ! ------------------------------------------------------------------------------ !> Set observation vector to ones subroutine qg_obsvec_ones(self) @@ -433,14 +415,31 @@ subroutine qg_obsvec_nobs(self,kobs) end subroutine qg_obsvec_nobs +! ------------------------------------------------------------------------------ +!> Get observation vector size (only non-masked observations) +subroutine qg_obsvec_nobs_withmask(self,obsmask,kobs) + +implicit none + +! Passed variables +type(qg_obsvec),intent(in) :: self !< Observation vector +type(qg_obsvec),intent(in) :: obsmask !< mask +integer,intent(inout) :: kobs !< Observation vector size + +! Get observation vector size +kobs = count(mask = (self%values /= self%missing) .and. (obsmask%values == 0)) + +end subroutine qg_obsvec_nobs_withmask + ! ------------------------------------------------------------------------------ !> Get non-missing values from observation vector into vals array -subroutine qg_obsvec_get(self,vals,nvals) +subroutine qg_obsvec_get_withmask(self,obsmask,vals,nvals) implicit none ! Passed variables type(qg_obsvec),intent(in) :: self !< Observation vector +type(qg_obsvec),intent(in) :: obsmask !< mask integer,intent(in) :: nvals !< Number of non-missing values real(kind_real), dimension(nvals), intent(out) :: vals!< returned value @@ -450,7 +449,7 @@ subroutine qg_obsvec_get(self,vals,nvals) ! Loop over values do jobs=1,self%nobs do jlev=1,self%nlev - if (self%values(jlev, jobs) /= self%missing) then + if ((self%values(jlev, jobs) /= self%missing) .and. (obsmask%values(jlev, jobs) == 0)) then if (jval > nvals) call abor1_ftn('qg_obsvec_get: inconsistent vector size') vals(jval) = self%values(jlev, jobs) jval = jval + 1 @@ -458,7 +457,7 @@ subroutine qg_obsvec_get(self,vals,nvals) enddo enddo -end subroutine qg_obsvec_get +end subroutine qg_obsvec_get_withmask ! ------------------------------------------------------------------------------ end module qg_obsvec_mod diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index f606acd91..70628c21b 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -267,12 +267,6 @@ ecbuild_add_test( TARGET test_qg_obsspace LIBS qg TEST_DEPENDS test_qg_make_obs_4d_24h ) -ecbuild_add_test( TARGET test_qg_localobsspace - SOURCES executables/TestLocalObsSpace.cc - ARGS "testinput/interfaces.yaml" - LIBS qg - TEST_DEPENDS test_qg_make_obs_4d_24h ) - ecbuild_add_test( TARGET test_qg_obs_vector SOURCES executables/TestObsVector.cc ARGS "testinput/interfaces.yaml" @@ -285,18 +279,6 @@ ecbuild_add_test( TARGET test_qg_obsdatavector LIBS qg TEST_DEPENDS test_qg_make_obs_4d_24h ) -ecbuild_add_test( TARGET test_qg_departures_ensemble - SOURCES executables/TestDeparturesEnsemble.cc - ARGS "testinput/interfaces.yaml" - LIBS qg - TEST_DEPENDS test_qg_make_obs_4d_24h ) - -ecbuild_add_test( TARGET test_qg_departures - SOURCES executables/TestDepartures.cc - ARGS "testinput/interfaces.yaml" - LIBS qg - TEST_DEPENDS test_qg_make_obs_4d_24h ) - ecbuild_add_test( TARGET test_qg_obs_operator SOURCES executables/TestObsOperator.cc ARGS "testinput/interfaces.yaml" @@ -333,6 +315,12 @@ ecbuild_add_test( TARGET test_qg_obs_aux_covariance LIBS qg TEST_DEPENDS test_qg_make_obs_4d_24h ) +ecbuild_add_test( TARGET test_qg_obs_localization + SOURCES executables/TestObsLocalization.cc + ARGS "testinput/interfaces.yaml" + LIBS qg + TEST_DEPENDS test_qg_make_obs_4d_24h ) + ecbuild_add_test( TARGET test_qg_localization SOURCES executables/TestLocalization.cc ARGS "testinput/interfaces.yaml" diff --git a/qg/test/executables/TestDepartures.cc b/qg/test/executables/TestDepartures.cc deleted file mode 100644 index 1a527dde2..000000000 --- a/qg/test/executables/TestDepartures.cc +++ /dev/null @@ -1,17 +0,0 @@ -/* - * (C) Copyright 2020-2020 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "model/QgTraits.h" -#include "oops/runs/Run.h" -#include "test/base/Departures.h" - -int main(int argc, char ** argv) { - oops::Run run(argc, argv); - test::Departures tests; - return run.execute(tests); -} - diff --git a/qg/test/executables/TestDeparturesEnsemble.cc b/qg/test/executables/TestDeparturesEnsemble.cc deleted file mode 100644 index da0152e50..000000000 --- a/qg/test/executables/TestDeparturesEnsemble.cc +++ /dev/null @@ -1,17 +0,0 @@ -/* - * (C) Copyright 2020-2020 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "model/QgTraits.h" -#include "oops/runs/Run.h" -#include "test/base/DeparturesEnsemble.h" - -int main(int argc, char ** argv) { - oops::Run run(argc, argv); - test::DeparturesEnsemble tests; - return run.execute(tests); -} - diff --git a/qg/test/executables/TestLocalObsSpace.cc b/qg/test/executables/TestObsLocalization.cc similarity index 70% rename from qg/test/executables/TestLocalObsSpace.cc rename to qg/test/executables/TestObsLocalization.cc index 73d7340e6..0ef7e19bc 100644 --- a/qg/test/executables/TestLocalObsSpace.cc +++ b/qg/test/executables/TestObsLocalization.cc @@ -1,5 +1,5 @@ /* - * (C) Copyright 2019 UCAR + * (C) Copyright 2021 UCAR * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. @@ -7,10 +7,10 @@ #include "model/QgTraits.h" #include "oops/runs/Run.h" -#include "test/interface/LocalObsSpace.h" +#include "test/interface/ObsLocalization.h" int main(int argc, char ** argv) { oops::Run run(argc, argv); - test::LocalObsSpace tests; + test::ObsLocalization tests; return run.execute(tests); } diff --git a/qg/test/testinput/interfaces.yaml b/qg/test/testinput/interfaces.yaml index 9c4aa8c33..6f736f053 100644 --- a/qg/test/testinput/interfaces.yaml +++ b/qg/test/testinput/interfaces.yaml @@ -10,14 +10,6 @@ geometry: nx: 40 ny: 20 depths: [4500.0, 5500.0] -local obs space: - location: - lon: -10 - lat: 10 - localization: - lengthscale: 4e6 - reference nobs: 172 - variable name: ObsValue linear model: trajectory: tstep: PT1H @@ -84,6 +76,14 @@ observations: rms ref: 183502589.5028424 reference nobs: 800 tolerance: 1.0e-8 + obs localization: + localization method: Heaviside + lengthscale: 0.0 + reference local nobs: [0] + reference gridpoints: + lons: [-175.5] + lats: [5.623] + localization reduces values: false - obs error: covariance model: diagonal geovals: @@ -112,6 +112,14 @@ observations: rms ref: 39.644266100943696 reference nobs: 800 tolerance: 1.0e-8 + obs localization: + localization method: Heaviside + lengthscale: 1.e8 + reference local nobs: [800] + reference gridpoints: + lons: [-175.5] + lats: [5.623] + localization reduces values: false - obs error: covariance model: diagonal geovals: @@ -133,6 +141,15 @@ observations: rms ref: 58.474969231121605 reference nobs: 400 tolerance: 1.0e-8 + obs localization: + localization method: Heaviside + lengthscale: 5.e6 + reference local nobs: [48] + reference gridpoints: + lons: [-175.5] + lats: [5.623] + localization reduces values: false + background: date: 2009-12-31T00:00:00Z filename: Data/truth.fc.2009-12-15T00:00:00Z.P16D.nc diff --git a/qg/test/testinput/letkf.yaml b/qg/test/testinput/letkf.yaml index 1a5b46e8e..a47b5291f 100644 --- a/qg/test/testinput/letkf.yaml +++ b/qg/test/testinput/letkf.yaml @@ -57,6 +57,7 @@ observations: obs error: covariance model: diagonal obs localization: + localization method: Heaviside lengthscale: 5e6 - obs operator: obs type: Wind @@ -69,6 +70,7 @@ observations: obs error: covariance model: diagonal obs localization: + localization method: Heaviside lengthscale: 5e6 - obs operator: obs type: WSpeed @@ -81,6 +83,7 @@ observations: obs error: covariance model: diagonal obs localization: + localization method: Heaviside lengthscale: 5e6 driver: diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 84c8bb074..5917bca9b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -388,7 +388,6 @@ test/interface/LinearGetValues.h test/interface/LinearModel.h test/interface/LinearObsOperator.h test/interface/LinearVariableChange.h -test/interface/LocalObsSpace.h test/interface/Locations.h test/interface/Model.h test/interface/ModelAuxControl.h diff --git a/src/oops/assimilation/GETKFSolver.h b/src/oops/assimilation/GETKFSolver.h index c7760be91..5159d9481 100644 --- a/src/oops/assimilation/GETKFSolver.h +++ b/src/oops/assimilation/GETKFSolver.h @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -32,6 +33,7 @@ #include "oops/generic/VerticalLocEV.h" #include "oops/interface/Geometry.h" #include "oops/interface/GeometryIterator.h" +#include "oops/interface/ObsDataVector.h" #include "oops/util/ConfigFunctions.h" #include "oops/util/Logger.h" #include "oops/util/Timer.h" @@ -54,6 +56,7 @@ class GETKFSolver : public LocalEnsembleSolver { typedef GeometryIterator GeometryIterator_; typedef GetValuesPost GetValuesPost_; typedef IncrementEnsemble4D IncrementEnsemble4D_; + typedef ObsDataVector ObsDataVector_; typedef ObsEnsemble ObsEnsemble_; typedef ObsErrors ObsErrors_; typedef Observations Observations_; @@ -322,33 +325,32 @@ void GETKFSolver::measurementUpdate(const IncrementEnsemble4D_ & bkg util::Timer timer(classname(), "measurementUpdate"); // create the local subset of observations - ObsSpaces_ local_obs(this->obspaces_, *i, this->obsconf_); - Departures_ local_omb(local_obs, this->omb_); + Departures_ locvector(this->obspaces_); + std::vector> outside; + for (size_t jj = 0; jj < this->obspaces_.size(); ++jj) { + outside.push_back(std::make_shared(this->obspaces_[jj], + this->obspaces_[jj].obsvariables())); + } + locvector.ones(); + this->obsloc_.computeLocalization(i, outside, locvector); + locvector.mask(this->hofx_.qcflags()); + Eigen::VectorXd local_omb_vec = this->omb_.packEigen(outside); - if (local_omb.nobs() == 0) { + if (local_omb_vec.size() == 0) { // no obs. so no need to update Wa_ and wa_ // ana_pert[i]=bkg_pert[i] this->copyLocalIncrement(bkg_pert, i, ana_pert); } else { // if obs are present do normal KF update - Eigen::VectorXd local_omb_vec = local_omb.packEigen(); // get local Yb & HZ - DeparturesEnsemble_ local_Yb(local_obs, this->Yb_); - Eigen::MatrixXd local_Yb_mat = local_Yb.packEigen(); - DeparturesEnsemble_ local_HZ(local_obs, HZb_); - Eigen::MatrixXd local_HZ_mat = local_HZ.packEigen(); + Eigen::MatrixXd local_Yb_mat = this->Yb_.packEigen(outside); + Eigen::MatrixXd local_HZ_mat = this->HZb_.packEigen(outside); // create local obs errors - ObsErrors_ local_R(this->obsconf_, local_obs); - Departures_ invVarR = local_R.inverseVariance(); + Eigen::VectorXd local_invVarR_vec = this->invVarR_->packEigen(outside); // and apply localization - ObsLocalizations_ loc(this->obsconf_, local_obs); - Departures_ locvector(local_obs); - locvector.ones(); - loc.computeLocalization(i, locvector); - invVarR *= locvector; - Eigen::VectorXd invVarR_vec = invVarR.packEigen(); - // compute and apply weights - computeWeights(local_omb_vec, local_HZ_mat, local_Yb_mat, invVarR_vec); + Eigen::VectorXd localization = locvector.packEigen(outside); + local_invVarR_vec.array() *= localization.array(); + computeWeights(local_omb_vec, local_HZ_mat, local_Yb_mat, local_invVarR_vec); applyWeights(bkg_pert, ana_pert, i); } } diff --git a/src/oops/assimilation/LETKFSolver.h b/src/oops/assimilation/LETKFSolver.h index 6f52e567b..3b0d6fe5f 100644 --- a/src/oops/assimilation/LETKFSolver.h +++ b/src/oops/assimilation/LETKFSolver.h @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -26,6 +27,7 @@ #include "oops/base/ObsSpaces.h" #include "oops/interface/Geometry.h" #include "oops/interface/GeometryIterator.h" +#include "oops/interface/ObsDataVector.h" #include "oops/util/Logger.h" #include "oops/util/Timer.h" @@ -50,6 +52,7 @@ class LETKFSolver : public LocalEnsembleSolver { typedef GeometryIterator GeometryIterator_; typedef IncrementEnsemble4D IncrementEnsemble4D_; typedef ObsErrors ObsErrors_; + typedef ObsDataVector ObsDataVector_; typedef ObsLocalizations ObsLocalizations_; typedef ObsSpaces ObsSpaces_; @@ -135,32 +138,31 @@ void LETKFSolver::measurementUpdate(const IncrementEnsemble4D_ & bkg util::Timer timer(classname(), "measurementUpdate"); // create the local subset of observations - ObsSpaces_ local_obs(this->obspaces_, *i, this->obsconf_); - Departures_ local_omb(local_obs, this->omb_); + Departures_ locvector(this->obspaces_); + std::vector> outside; + for (size_t jj = 0; jj < this->obspaces_.size(); ++jj) { + outside.push_back(std::make_shared(this->obspaces_[jj], + this->obspaces_[jj].obsvariables())); + } + locvector.ones(); + this->obsloc_.computeLocalization(i, outside, locvector); + locvector.mask(this->hofx_.qcflags()); + Eigen::VectorXd local_omb_vec = this->omb_.packEigen(outside); - if (local_omb.nobs() == 0) { + if (local_omb_vec.size() == 0) { // no obs. so no need to update Wa_ and wa_ // ana_pert[i]=bkg_pert[i] this->copyLocalIncrement(bkg_pert, i, ana_pert); } else { // if obs are present do normal KF update - Eigen::VectorXd local_omb_vec = local_omb.packEigen(); // create local Yb - DeparturesEnsemble_ local_Yb(local_obs, this->Yb_); - Eigen::MatrixXd local_Yb_mat = local_Yb.packEigen(); + Eigen::MatrixXd local_Yb_mat = this->Yb_.packEigen(outside); // create local obs errors - ObsErrors_ local_R(this->obsconf_, local_obs); - Departures_ invVarR = local_R.inverseVariance(); + Eigen::VectorXd local_invVarR_vec = this->invVarR_->packEigen(outside); // and apply localization - ObsLocalizations_ loc(this->obsconf_, local_obs); - Departures_ locvector(local_obs); - locvector.ones(); - loc.computeLocalization(i, locvector); - invVarR *= locvector; - Eigen::VectorXd local_R_vec = invVarR.packEigen(); - - // compute and apply weights - computeWeights(local_omb_vec, local_Yb_mat, local_R_vec); + Eigen::VectorXd localization = locvector.packEigen(outside); + local_invVarR_vec.array() *= localization.array(); + computeWeights(local_omb_vec, local_Yb_mat, local_invVarR_vec); applyWeights(bkg_pert, ana_pert, i); } } diff --git a/src/oops/assimilation/LocalEnsembleSolver.h b/src/oops/assimilation/LocalEnsembleSolver.h index 9db736582..e0ba1067f 100644 --- a/src/oops/assimilation/LocalEnsembleSolver.h +++ b/src/oops/assimilation/LocalEnsembleSolver.h @@ -22,7 +22,9 @@ #include "oops/base/IncrementEnsemble4D.h" #include "oops/base/ObsAuxControls.h" #include "oops/base/ObsEnsemble.h" +#include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" +#include "oops/base/ObsLocalizations.h" #include "oops/base/ObsSpaces.h" #include "oops/base/StateEnsemble4D.h" #include "oops/interface/Geometry.h" @@ -45,7 +47,9 @@ class LocalEnsembleSolver { typedef IncrementEnsemble4D IncrementEnsemble4D_; typedef ObsAuxControls ObsAuxControls_; typedef ObsEnsemble ObsEnsemble_; + typedef ObsErrors ObsErrors_; typedef Observations Observations_; + typedef ObsLocalizations ObsLocalizations_; typedef ObsSpaces ObsSpaces_; typedef StateEnsemble4D StateEnsemble4D_; @@ -76,6 +80,9 @@ class LocalEnsembleSolver { CalcHofX_ hofx_; // observer Departures_ omb_; // obs - mean(H(x)) DeparturesEnsemble_ Yb_; // ensemble perturbations in the observation space + std::unique_ptr R_; ///< observation errors + std::unique_ptr invVarR_; ///< inverse observation error variance + ObsLocalizations_ obsloc_; ///< observation space localization }; // ----------------------------------------------------------------------------- @@ -84,9 +91,9 @@ template LocalEnsembleSolver::LocalEnsembleSolver(ObsSpaces_ & obspaces, const Geometry_ & geometry, const eckit::Configuration & config, size_t nens) - : obsconf_(config, "observations"), - obspaces_(obspaces), obsaux_(obspaces_, obsconf_), - hofx_(obspaces, obsconf_), omb_(obspaces_), Yb_(obspaces_, nens) + : obsconf_(config, "observations"), obspaces_(obspaces), obsaux_(obspaces_, obsconf_), + hofx_(obspaces, obsconf_), omb_(obspaces_), Yb_(obspaces_, nens), + obsloc_(obsconf_, obspaces_) { } @@ -133,6 +140,10 @@ Observations LocalEnsembleSolver::computeHofX(const StateEnsemb hofx_.maskObsErrors(); hofx_.saveObsErrors("EffectiveError"); } + R_.reset(new ObsErrors_(obsconf_, obspaces_)); + invVarR_.reset(new Departures_(R_->inverseVariance())); + + invVarR_->mask(hofx_.qcflags()); // calculate H(x) ensemble mean Observations_ yb_mean(obsens.mean()); diff --git a/src/oops/base/Departures.h b/src/oops/base/Departures.h index e197f8857..665b59508 100644 --- a/src/oops/base/Departures.h +++ b/src/oops/base/Departures.h @@ -50,8 +50,6 @@ class Departures : public GeneralizedDepartures { public: /// \brief create Departures for all obs (read from ObsSpace if \p name is specified) explicit Departures(const ObsSpaces_ &, const std::string & name = ""); -/// \brief create local Departures - Departures(const ObsSpaces_ &, const Departures &); /// Access size_t size() const {return dep_.size();} @@ -78,9 +76,9 @@ class Departures : public GeneralizedDepartures { void mask(ObsDataVec_); /// Pack departures in an Eigen vector (excluding departures that are masked out) - Eigen::VectorXd packEigen() const; + Eigen::VectorXd packEigen(const ObsDataVec_ &) const; /// Size of departures packed into an Eigen vector - size_t packEigenSize() const; + size_t packEigenSize(const ObsDataVec_ &) const; /// Save departures values void save(const std::string &) const; @@ -106,16 +104,6 @@ Departures::Departures(const ObsSpaces_ & obsdb, } // ----------------------------------------------------------------------------- template -Departures::Departures(const ObsSpaces_ & obsdb, - const Departures & other): dep_() { - dep_.reserve(obsdb.size()); - for (size_t jj = 0; jj < other.dep_.size(); ++jj) { - dep_.emplace_back(obsdb[jj], other[jj]); - } - Log::trace() << "Local Departures created" << std::endl; -} -// ----------------------------------------------------------------------------- -template Departures & Departures::operator+=(const Departures & rhs) { for (size_t jj = 0; jj < dep_.size(); ++jj) { dep_[jj] += rhs[jj]; @@ -223,27 +211,27 @@ void Departures::mask(ObsDataVec_ qcflags) { } // ----------------------------------------------------------------------------- template -Eigen::VectorXd Departures::packEigen() const { +Eigen::VectorXd Departures::packEigen(const ObsDataVec_ & mask) const { std::vector len(dep_.size()); for (size_t idep = 0; idep < dep_.size(); ++idep) { - len[idep] = dep_[idep].packEigenSize(); + len[idep] = dep_[idep].packEigenSize(*mask[idep]); } size_t all_len = std::accumulate(len.begin(), len.end(), 0); Eigen::VectorXd vec(all_len); size_t ii = 0; for (size_t idep = 0; idep < dep_.size(); ++idep) { - vec.segment(ii, len[idep]) = dep_[idep].packEigen(); + vec.segment(ii, len[idep]) = dep_[idep].packEigen(*mask[idep]); ii += len[idep]; } return vec; } // ----------------------------------------------------------------------------- template -size_t Departures::packEigenSize() const { +size_t Departures::packEigenSize(const ObsDataVec_ & mask) const { size_t len = 0; for (size_t idep = 0; idep < dep_.size(); ++idep) { - len += dep_[idep].packEigenSize(); + len += dep_[idep].packEigenSize(*mask[idep]); } return len; } diff --git a/src/oops/base/DeparturesEnsemble.h b/src/oops/base/DeparturesEnsemble.h index cc697c29e..12fe0a120 100644 --- a/src/oops/base/DeparturesEnsemble.h +++ b/src/oops/base/DeparturesEnsemble.h @@ -9,10 +9,12 @@ #define OOPS_BASE_DEPARTURESENSEMBLE_H_ #include +#include #include #include "oops/base/Departures.h" #include "oops/base/ObsSpaces.h" +#include "oops/interface/ObsDataVector.h" #include "oops/util/Logger.h" namespace oops { @@ -23,12 +25,12 @@ namespace oops { template class DeparturesEnsemble { typedef Departures Departures_; typedef ObsSpaces ObsSpaces_; + template using ObsData_ = ObsDataVector; + template using ObsDataVec_ = std::vector>>; + public: /// Creates ensemble of empty Departures size \p nens DeparturesEnsemble(const ObsSpaces_ &, const size_t nens); - /// Creates ensemble of local Departures from full Departures \p other based on local - /// observations \p local - DeparturesEnsemble(const ObsSpaces_ & local, const DeparturesEnsemble & other); /// Accessors and size size_t size() const {return ensemblePerturbs_.size();} @@ -36,7 +38,7 @@ template class DeparturesEnsemble { const Departures_ & operator[](const size_t ii) const {return ensemblePerturbs_[ii];} /// pack ensemble of dep. as contiguous block of memory - Eigen::MatrixXd packEigen() const; + Eigen::MatrixXd packEigen(const ObsDataVec_ &) const; private: std::vector ensemblePerturbs_; // ensemble perturbations @@ -57,26 +59,13 @@ DeparturesEnsemble::DeparturesEnsemble(const ObsSpaces_ & obsdb, const size // ----------------------------------------------------------------------------- template -DeparturesEnsemble::DeparturesEnsemble(const ObsSpaces_ & local, - const DeparturesEnsemble & other) - : ensemblePerturbs_() { - ensemblePerturbs_.reserve(other.size()); - for (const auto & dep : other.ensemblePerturbs_) { - ensemblePerturbs_.emplace_back(local, dep); - } - Log::trace() << "Local DeparturesEnsemble created" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -Eigen::MatrixXd DeparturesEnsemble::packEigen() const { - std::size_t myNobs = ensemblePerturbs_[0].packEigenSize(); +Eigen::MatrixXd DeparturesEnsemble::packEigen(const ObsDataVec_ & mask) const { + std::size_t myNobs = ensemblePerturbs_[0].packEigenSize(mask); std::size_t myNens = ensemblePerturbs_.size(); Eigen::MatrixXd depEns(myNens, myNobs); for (std::size_t iens = 0; iens < myNens; ++iens) { - depEns.row(iens) = ensemblePerturbs_[iens].packEigen(); + depEns.row(iens) = ensemblePerturbs_[iens].packEigen(mask); } Log::trace() << "DeparturesEnsemble::packEigen() completed" << std::endl; return depEns; diff --git a/src/oops/base/ObsLocalizationBase.h b/src/oops/base/ObsLocalizationBase.h index b15833a3f..cfb6e5be1 100644 --- a/src/oops/base/ObsLocalizationBase.h +++ b/src/oops/base/ObsLocalizationBase.h @@ -15,6 +15,7 @@ #include "eckit/config/Configuration.h" #include "oops/interface/GeometryIterator.h" +#include "oops/interface/ObsDataVector.h" #include "oops/interface/ObsSpace.h" #include "oops/util/Printable.h" @@ -25,15 +26,18 @@ template class ObsLocalizationBase : public util::Printable, private boost::noncopyable { typedef GeometryIterator GeometryIterator_; + typedef ObsDataVector ObsDataVector_; typedef ObsVector ObsVector_; public: ObsLocalizationBase() = default; virtual ~ObsLocalizationBase() = default; - /// fill \p obsvector with observation-space localization between - /// observations and \p point in model-space + /// compute obs-space localization: fill \p obsvector with observation-space + /// localization values between observations and \p point in model-space, and + /// fill \p outside with flags on whether obs is local or not (1: outside of + /// localization, 0: inside of localization, local) virtual void computeLocalization(const GeometryIterator_ & point, - ObsVector_ & obsvector) const = 0; + ObsDataVector_ & flags, ObsVector_ & obsvector) const = 0; }; // ============================================================================= @@ -85,19 +89,15 @@ template std::unique_ptr> ObsLocalizationFactory::create( const eckit::Configuration & conf, const ObsSpace_ & obspace) { Log::trace() << "ObsLocalizationBase::create starting" << std::endl; - if (conf.has("localization method")) { - const std::string id = conf.getString("localization method"); - typename std::map*>::iterator - jloc = getMakers().find(id); - if (jloc == getMakers().end()) { - throw std::runtime_error(id + " does not exist in obs localization factory."); - } - std::unique_ptr> ptr(jloc->second->make(conf, obspace)); - Log::trace() << "ObsLocalizationBase::create done" << std::endl; - return ptr; - } else { - return nullptr; + const std::string id = conf.getString("localization method"); + typename std::map*>::iterator + jloc = getMakers().find(id); + if (jloc == getMakers().end()) { + throw std::runtime_error(id + " does not exist in obs localization factory."); } + std::unique_ptr> ptr(jloc->second->make(conf, obspace)); + Log::trace() << "ObsLocalizationBase::create done" << std::endl; + return ptr; } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/ObsLocalizations.h b/src/oops/base/ObsLocalizations.h index 0b572d166..e5a4b85c1 100644 --- a/src/oops/base/ObsLocalizations.h +++ b/src/oops/base/ObsLocalizations.h @@ -31,15 +31,17 @@ class ObsLocalizations : public util::Printable, typedef Departures Observations_; typedef ObsLocalizationBase ObsLocalization_; typedef ObsSpaces ObsSpaces_; + typedef std::vector>> ObsDataVectors_; public: static const std::string classname() {return "oops::ObsLocalizations";} ObsLocalizations(const eckit::Configuration &, const ObsSpaces_ &); - /// fill \p obsvectors with observation-space localizations between - /// observations and \p point in model-space - void computeLocalization(const GeometryIterator_ & point, Observations_ & obsvectors) const; + /// schur-multiply \p obsvectors with observation-space localizations between + /// observations in \p obsspaces and \p point in model-space + void computeLocalization(const GeometryIterator_ & point, + ObsDataVectors_ & local, Observations_ & obsvectors) const; private: void print(std::ostream &) const; @@ -62,9 +64,9 @@ ObsLocalizations::ObsLocalizations(const eckit::Configuration & conf template void ObsLocalizations::computeLocalization(const GeometryIterator_ & point, - Observations_ & obsvec) const { - for (size_t jj = 0; jj < local_.size(); ++jj) { - if (local_[jj]) local_[jj]->computeLocalization(point, obsvec[jj]); + ObsDataVectors_ & local, Observations_ & obsvec) const { + for (size_t jj = 0; jj < obsvec.size(); ++jj) { + if (local_[jj]) local_[jj]->computeLocalization(point, *local[jj], obsvec[jj]); } } diff --git a/src/oops/base/ObsSpaces.h b/src/oops/base/ObsSpaces.h index ffc1c0b2a..006707b27 100644 --- a/src/oops/base/ObsSpaces.h +++ b/src/oops/base/ObsSpaces.h @@ -44,7 +44,6 @@ class ObsSpaces : public util::Printable, ObsSpaces(const eckit::Configuration &, const eckit::mpi::Comm &, const util::DateTime &, const util::DateTime &, const eckit::mpi::Comm & time = oops::mpi::myself()); - ObsSpaces(const ObsSpaces &, const eckit::geometry::Point2 &, const eckit::Configuration &); ~ObsSpaces(); /// Access @@ -84,22 +83,6 @@ ObsSpaces::ObsSpaces(const eckit::Configuration & conf, const eckit::mpi::C // ----------------------------------------------------------------------------- -template -ObsSpaces::ObsSpaces(const ObsSpaces & obss, const eckit::geometry::Point2 & center, - const eckit::Configuration & conf) - : spaces_(0), wbgn_(obss.wbgn_), wend_(obss.wend_) -{ - std::vector typeconfs = conf.getSubConfigurations(); - for (std::size_t jj = 0; jj < obss.size(); ++jj) { - eckit::LocalConfiguration locconf(typeconfs[jj], "obs localization"); - std::shared_ptr tmp(new ObsSpace_(obss[jj], center, locconf)); - spaces_.push_back(tmp); - } - ASSERT(spaces_.size() == obss.size()); -} - -// ----------------------------------------------------------------------------- - template ObsSpaces::~ObsSpaces() {} diff --git a/src/oops/base/Observations.h b/src/oops/base/Observations.h index 743faabf6..bdc18e2e9 100644 --- a/src/oops/base/Observations.h +++ b/src/oops/base/Observations.h @@ -41,8 +41,6 @@ template class Observations : public util::Printable { public: /// \brief create Observations for all obs (read from ObsSpace if name is specified) explicit Observations(const ObsSpaces_ &, const std::string & name = ""); -/// \brief create local Observations - Observations(const ObsSpaces_ &, const Observations &); /// destructor and copy/move constructor/assignments ~Observations() = default; @@ -95,16 +93,6 @@ Observations::Observations(const ObsSpaces_ & obsdb, } // ----------------------------------------------------------------------------- template -Observations::Observations(const ObsSpaces_ & obsdb, - const Observations & other): obsdb_(obsdb), obs_() { - obs_.reserve(obsdb.size()); - for (std::size_t jj = 0; jj < other.size(); ++jj) { - obs_.emplace_back(obsdb[jj], other[jj]); - } - Log::trace() << "Local observations created" << std::endl; -} -// ----------------------------------------------------------------------------- -template Observations::Observations(const Observations & other) : obsdb_(other.obsdb_), obs_(other.obs_) { } diff --git a/src/oops/interface/ObsLocalization.h b/src/oops/interface/ObsLocalization.h index f33d525ea..bad655656 100644 --- a/src/oops/interface/ObsLocalization.h +++ b/src/oops/interface/ObsLocalization.h @@ -14,6 +14,7 @@ #include "eckit/config/LocalConfiguration.h" #include "oops/base/LocalIncrement.h" #include "oops/base/ObsLocalizationBase.h" +#include "oops/interface/ObsDataVector.h" #include "oops/interface/ObsSpace.h" #include "oops/interface/ObsVector.h" #include "oops/util/Logger.h" @@ -26,6 +27,7 @@ template class ObsLocalization : public ObsLocalizationBase { typedef GeometryIterator GeometryIterator_; typedef ObsSpace ObsSpace_; + typedef ObsDataVector ObsDataVector_; typedef ObsVector ObsVector_; public: @@ -34,7 +36,8 @@ class ObsLocalization : public ObsLocalizationBase { ObsLocalization(const eckit::Configuration &, const ObsSpace_ &); ~ObsLocalization(); - void computeLocalization(const GeometryIterator_ &, ObsVector_ &) const override; + void computeLocalization(const GeometryIterator_ &, + ObsDataVector_ &, ObsVector_ &) const override; private: void print(std::ostream &) const override; @@ -69,10 +72,10 @@ ObsLocalization::~ObsLocalization() { template void ObsLocalization::computeLocalization(const GeometryIterator_ & p, - ObsVector_ & obsvector) const { + ObsDataVector_ & local, ObsVector_ & obsvector) const { Log::trace() << "ObsLocalization:: computeLocalization starting" << std::endl; util::Timer timer(classname(), "computeLocalization"); - obsloc_->computeLocalization(p.geometryiter(), obsvector.obsvector()); + obsloc_->computeLocalization(p.geometryiter(), local.obsdatavector(), obsvector.obsvector()); Log::trace() << "ObsLocalization:: computeLocalization done" << std::endl; } diff --git a/src/oops/interface/ObsSpace.h b/src/oops/interface/ObsSpace.h index 2266c41ed..08910429f 100644 --- a/src/oops/interface/ObsSpace.h +++ b/src/oops/interface/ObsSpace.h @@ -51,9 +51,6 @@ class ObsSpace : public util::Printable, ObsSpace(const eckit::Configuration &, const eckit::mpi::Comm &, const util::DateTime &, const util::DateTime &, const eckit::mpi::Comm & time = oops::mpi::myself()); - ObsSpace(const ObsSpace &, const eckit::geometry::Point2 &, - const eckit::Configuration &); - explicit ObsSpace(const ObsSpace_ &); ~ObsSpace(); /// Interfacing @@ -98,28 +95,6 @@ ObsSpace::ObsSpace(const eckit::Configuration & conf, // ----------------------------------------------------------------------------- -template -ObsSpace::ObsSpace(const ObsSpace & os, - const eckit::geometry::Point2 & center, - const eckit::Configuration & conf) : obsdb_(), time_(oops::mpi::myself()) { - Log::trace() << "ObsSpace::ObsSpace (local) starting" << std::endl; - util::Timer timer(classname(), "ObsSpace"); - obsdb_.reset(new ObsSpace_(os.obsspace(), center, conf)); - Log::trace() << "ObsSpace::ObsSpace (local) done" << std::endl; -} - -// ----------------------------------------------------------------------------- - -template -ObsSpace::ObsSpace(const ObsSpace_ & other) : obsdb_(), time_(other.time_) { - Log::trace() << "ObsSpace::ObsSpace starting" << std::endl; - util::Timer timer(classname(), "ObsSpace"); - obsdb_ = other.obsdb_; - Log::trace() << "ObsSpace::ObsSpace done" << std::endl; -} - -// ----------------------------------------------------------------------------- - template ObsSpace::~ObsSpace() { Log::trace() << "ObsSpace::~ObsSpace starting" << std::endl; diff --git a/src/oops/interface/ObsVector.h b/src/oops/interface/ObsVector.h index aca5a8513..8f0e40734 100644 --- a/src/oops/interface/ObsVector.h +++ b/src/oops/interface/ObsVector.h @@ -49,7 +49,6 @@ class ObsVector : public util::Printable, /// specified \p name variable from \p obsspace. Otherwise, zero vector is created. explicit ObsVector(const ObsSpace & obsspace, const std::string name = ""); ObsVector(const ObsVector &); - ObsVector(const ObsSpace &, const ObsVector &); ~ObsVector(); /// Interfacing @@ -65,11 +64,11 @@ class ObsVector : public util::Printable, ObsVector & operator/= (const ObsVector &); /// Pack observations local to this MPI task into an Eigen vector - /// (excluding vector elements that are masked out) - Eigen::VectorXd packEigen() const; + /// (excluding vector elements that are masked out and where \p mask != 0) + Eigen::VectorXd packEigen(const ObsDataVector & mask) const; /// Number of non-masked out observations local to this MPI task - /// (size of an Eigen vector returned by `packEigen` - size_t packEigenSize() const {return data_->packEigenSize();} + /// (size of an Eigen vector returned by `packEigen`) + size_t packEigenSize(const ObsDataVector & mask) const; void zero(); /// Set this ObsVector to ones (used in tests) @@ -119,18 +118,6 @@ ObsVector::ObsVector(const ObsVector & other): data_(), commTime_(other.com } // ----------------------------------------------------------------------------- template -ObsVector::ObsVector(const ObsSpace & os, const ObsVector & other) - : data_(), commTime_(os.timeComm()) -{ - Log::trace() << "ObsVector::ObsVector starting" << std::endl; - util::Timer timer(classname(), "ObsVector"); - - data_.reset(new ObsVector_(os.obsspace(), other.obsvector())); - - Log::trace() << "ObsVector::ObsVector done" << std::endl; -} -// ----------------------------------------------------------------------------- -template ObsVector::~ObsVector() { Log::trace() << "ObsVector::~ObsVector starting" << std::endl; util::Timer timer(classname(), "~ObsVector"); @@ -334,17 +321,28 @@ void ObsVector::save(const std::string & name) const { } // ----------------------------------------------------------------------------- template -Eigen::VectorXd ObsVector::packEigen() const { +Eigen::VectorXd ObsVector::packEigen(const ObsDataVector & mask) const { Log::trace() << "ObsVector::packEigen starting " << std::endl; util::Timer timer(classname(), "packEigen"); - Eigen::VectorXd vec = data_->packEigen(); + Eigen::VectorXd vec = data_->packEigen(mask.obsdatavector()); Log::trace() << "ObsVector::packEigen done" << std::endl; return vec; } // ----------------------------------------------------------------------------- template +size_t ObsVector::packEigenSize(const ObsDataVector & mask) const { + Log::trace() << "ObsVector::packEigenSize starting " << std::endl; + util::Timer timer(classname(), "packEigenSize"); + + size_t len = data_->packEigenSize(mask.obsdatavector()); + + Log::trace() << "ObsVector::packEigen done" << std::endl; + return len; +} +// ----------------------------------------------------------------------------- +template void ObsVector::read(const std::string & name) { Log::trace() << "ObsVector::read starting " << name << std::endl; util::Timer timer(classname(), "read"); diff --git a/src/test/base/Departures.h b/src/test/base/Departures.h deleted file mode 100644 index cbf963b6d..000000000 --- a/src/test/base/Departures.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * (C) Copyright 2020-2020 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef TEST_BASE_DEPARTURES_H_ -#define TEST_BASE_DEPARTURES_H_ - -#include -#include -#include - -#define ECKIT_TESTING_SELF_REGISTER_CASES 0 - -#include "eckit/config/LocalConfiguration.h" -#include "eckit/testing/Test.h" -#include "oops/base/Departures.h" -#include "oops/runs/Test.h" -#include "oops/util/Logger.h" -#include "test/interface/ObsTestsFixture.h" -#include "test/TestEnvironment.h" - -namespace test { - -// ----------------------------------------------------------------------------- - -template void testDepartures() { - typedef ObsTestsFixture Test_; - typedef oops::Departures Departures_; - - Departures_ y(Test_::obspace()); - Departures_ y2(Test_::obspace()); - Departures_ ydiff(Test_::obspace()); - - // check assign and "-=" operator - y.random(); - double ref_rms = y.rms(); - oops::Log::test() << "yrandom.rms=" << ref_rms << std::endl; - - y2.zero(); - oops::Log::test() << "yzero.rms=" << y2.rms() << std::endl; - - ydiff = y; - ydiff -= y2; - oops::Log::test() << "rms(yrandom-yzero)=" << ydiff.rms() << std::endl; - - EXPECT(ydiff.rms() != 0.0); - - // check pack operator - Eigen::VectorXd ypack = y.packEigen(); - EXPECT_EQUAL(ypack.size(), y.packEigenSize()); - oops::Log::test() << "ypack: " << ypack << std::endl; - double rms = ypack.norm() / sqrt(ypack.size()); - oops::Log::test() << "rms(ypack)=" << rms << std::endl; - oops::Log::test() << "y.rms()=" << ref_rms << std::endl; - oops::Log::test() << "y.rms()-rms(ypack)" << ref_rms - rms << std::endl; - - EXPECT(oops::is_close(ref_rms, rms, 1.e-14)); -} - -template class Departures : public oops::Test { - typedef ObsTestsFixture Test_; - public: - Departures() {} - virtual ~Departures() {} - private: - std::string testid() const override {return "test::Departures<" + OBS::name() + ">";} - - void register_tests() const override { - std::vector& ts = eckit::testing::specification(); - ts.emplace_back(CASE("base/Departures/testDepartures") - { testDepartures(); }); - } - - void clear() const override { - Test_::reset(); - } -}; - -// ============================================================================= - -} // namespace test - -#endif // TEST_BASE_DEPARTURES_H_ diff --git a/src/test/base/DeparturesEnsemble.h b/src/test/base/DeparturesEnsemble.h deleted file mode 100644 index 3f648e2f8..000000000 --- a/src/test/base/DeparturesEnsemble.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * (C) Copyright 2020-2020 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef TEST_BASE_DEPARTURESENSEMBLE_H_ -#define TEST_BASE_DEPARTURESENSEMBLE_H_ - -#include -#include -#include - -#define ECKIT_TESTING_SELF_REGISTER_CASES 0 - -#include "eckit/config/LocalConfiguration.h" -#include "eckit/testing/Test.h" -#include "oops/base/Departures.h" -#include "oops/base/DeparturesEnsemble.h" -#include "oops/base/ObsEnsemble.h" -#include "oops/generic/instantiateObsErrorFactory.h" -#include "oops/runs/Test.h" -#include "oops/util/Logger.h" -#include "test/interface/ObsTestsFixture.h" -#include "test/TestEnvironment.h" - -namespace test { - -// ----------------------------------------------------------------------------- - -template void testDeparturesEnsemble() { - typedef ObsTestsFixture Test_; - typedef oops::DeparturesEnsemble DeparturesEnsemble_; - - // tests depaturesEnsemle.packEigen() - // test verifies that computing rms by-hand on the packed array agrees with - // rms for each ensemble member computed using depatures.rms() method - size_t myNens = 5; - DeparturesEnsemble_ yens(Test_::obspace(), myNens); - size_t myNobs = yens[0].packEigenSize(); - - std::vector rms1(myNens); - for (size_t ii=0; ii < myNens; ++ii) { - yens[ii].random(); - rms1[ii] = yens[ii].rms(); - } - - Eigen::MatrixXd yEnsPack = yens.packEigen(); - Eigen::VectorXd rms2 = yEnsPack.rowwise().norm() / sqrt(myNobs); - oops::Log::info() << "norm of eigen stuff: " << rms2 << std::endl; - for (size_t iens = 0; iens < myNens; ++iens) { - oops::Log::test() << "ii=" << iens << " yens[ii].rms=" << rms1[iens] - << " rms(ypack[ii,:])=" << rms2[iens] << std::endl; - EXPECT(oops::is_close(rms2[iens], rms1[iens], 1.e-14)); - } -} - -template class DeparturesEnsemble : public oops::Test { - typedef ObsTestsFixture Test_; - public: - DeparturesEnsemble() {} - virtual ~DeparturesEnsemble() {} - private: - std::string testid() const override {return "test::DeparturesEnsemble<" + OBS::name() + ">";} - - void register_tests() const override { - std::vector& ts = eckit::testing::specification(); - ts.emplace_back(CASE("base/DeparturesEnsemble/testDeparturesEnsemble") - { testDeparturesEnsemble(); }); - } - - void clear() const override { - Test_::reset(); - } -}; - -// ============================================================================= - -} // namespace test - -#endif // TEST_BASE_DEPARTURESENSEMBLE_H_ diff --git a/src/test/interface/LocalObsSpace.h b/src/test/interface/LocalObsSpace.h deleted file mode 100644 index 56cf8d95f..000000000 --- a/src/test/interface/LocalObsSpace.h +++ /dev/null @@ -1,137 +0,0 @@ -/* - * (C) Copyright 2017-2020 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef TEST_INTERFACE_LOCALOBSSPACE_H_ -#define TEST_INTERFACE_LOCALOBSSPACE_H_ - -#include -#include - -#define ECKIT_TESTING_SELF_REGISTER_CASES 0 - -#include "eckit/config/LocalConfiguration.h" -#include "eckit/geometry/Point2.h" -#include "eckit/testing/Test.h" -#include "oops/interface/ObsSpace.h" -#include "oops/interface/ObsVector.h" -#include "oops/runs/Test.h" -#include "oops/util/dot_product.h" -#include "test/interface/ObsTestsFixture.h" -#include "test/TestEnvironment.h" - -namespace test { - -// ----------------------------------------------------------------------------- -/// Tests that number of local observations in LocalObsSpace is the same as in reference -template void testLocalObsSpace() { - typedef ObsTestsFixture Test_; - typedef oops::ObsSpace LocalObsSpace_; - typedef oops::ObsVector ObsVector_; - - const eckit::LocalConfiguration localconf(TestEnvironment::config(), "local obs space"); - - // get center (for localization) from yaml - eckit::LocalConfiguration geolocconf(localconf, "location"); - double lon = geolocconf.getDouble("lon"); - double lat = geolocconf.getDouble("lat"); - const eckit::geometry::Point2 center(lon, lat); - - // get localization test from yaml - eckit::LocalConfiguration locconf(localconf, "localization"); - - // count local nobs for all obs types - int totalNobs = 0; - for (size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - // initialize local observation space - LocalObsSpace_ localobs(Test_::obspace()[jj], center, locconf); - oops::Log::test() << "Local obs within " << locconf << " from " << center << - ": " << localobs << std::endl; - - // count local nobs - ObsVector_ localvec(localobs); - totalNobs += localvec.nobs(); - } - - // test that local nobs is equal to the reference value - const int ref_nobs = localconf.getInt("reference nobs"); - EXPECT(totalNobs == ref_nobs); -} - -// ----------------------------------------------------------------------------- -/// Tests that constructing local ObsVector from local ObsSpace by reading -/// (ObsVector(const ObsSpace &, const std::string &) ctor), and by subsetting -/// full ObsVector (ObsVector(const ObsSpace &, const ObsVector &) ctor) -/// give the same results -template void testLocalObsVector() { - typedef ObsTestsFixture Test_; - typedef oops::ObsSpace LocalObsSpace_; - typedef oops::ObsVector ObsVector_; - - const eckit::LocalConfiguration localconf(TestEnvironment::config(), "local obs space"); - // get center (for localization) from yaml - eckit::LocalConfiguration geolocconf(localconf, "location"); - double lon = geolocconf.getDouble("lon"); - double lat = geolocconf.getDouble("lat"); - const eckit::geometry::Point2 center(lon, lat); - - // get localization test from yaml - eckit::LocalConfiguration locconf(localconf, "localization"); - const std::string varname = localconf.getString("variable name"); - - for (size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - // initialize full ObsVector for a specified variable - ObsVector_ fullvec(Test_::obspace()[jj], varname); - oops::Log::test() << "Full Obsvector: " << fullvec << std::endl; - - // initialize local observation space - LocalObsSpace_ localobs(Test_::obspace()[jj], center, locconf); - oops::Log::test() << "Local obs within " << locconf << " from " << center << - ": " << localobs << std::endl; - - // intialize local obsvector by reading specified variable from local obsspace - ObsVector_ localvec1(localobs, varname); - oops::Log::test() << "Local Obsvector from Local Obsspace: " << localvec1 << std::endl; - - // initialize local obsvector from full obsvector using local obsspace - ObsVector_ localvec2(localobs, fullvec); - oops::Log::test() << "Local ObsVector from full ObsVector: " << localvec2 << std::endl; - // check that the two are equal - EXPECT(localvec1.nobs() == localvec2.nobs()); - localvec2 -= localvec1; - const double rms = dot_product(localvec2, localvec2); - EXPECT(rms == 0); - } -} - -// ----------------------------------------------------------------------------- - -template class LocalObsSpace : public oops::Test { - typedef ObsTestsFixture Test_; - public: - LocalObsSpace() {} - virtual ~LocalObsSpace() {} - private: - std::string testid() const override {return "test::LocalObsSpace<" + OBS::name() + ">";} - - void register_tests() const override { - std::vector& ts = eckit::testing::specification(); - ts.emplace_back(CASE("interface/LocalObsSpace/testLocalObsSpace") - { testLocalObsSpace(); }); - ts.emplace_back(CASE("interface/LocalObsSpace/testLocalObsVector") - { testLocalObsVector(); }); - } - - void clear() const override { - Test_::reset(); - } -}; - -// ============================================================================= - -} // namespace test - -#endif // TEST_INTERFACE_LOCALOBSSPACE_H_ diff --git a/src/test/interface/ObsLocalization.h b/src/test/interface/ObsLocalization.h index 9a872694b..1221c7603 100644 --- a/src/test/interface/ObsLocalization.h +++ b/src/test/interface/ObsLocalization.h @@ -21,6 +21,7 @@ #include "oops/base/ObsLocalizationBase.h" #include "oops/interface/Geometry.h" #include "oops/interface/GeometryIterator.h" +#include "oops/interface/ObsDataVector.h" #include "oops/interface/ObsVector.h" #include "oops/mpi/mpi.h" #include "oops/runs/Test.h" @@ -29,12 +30,19 @@ namespace test { -/// Tests that obs localization applied to a zero vector returns zero. -/// Tests that obs localization applied to a vector of ones makes rms(obsvec) < 1 +/// \brief Tests ObsLocalization::computeLocalization method. +/// \details Tests that for obs localization around specified in yaml Geometry points: +/// 1. number of local obs matches reference ("reference local nobs") +/// 2. obs localization applied to a vector of ones makes rms(obsvec) < 1 or +/// doesn't change the vector (depending on the "localization reduces values" option) +/// Reference gridpoints are specified in yaml as "reference gridpoints.lons" and +/// "reference gridpoints.lats". They don't have to be exactly equal to the lon/lat +/// of the Geometry gridpoints, but should be no further than 1.e-5 distance away. template void testObsLocalization() { typedef ObsTestsFixture Test_; typedef oops::Geometry Geometry_; typedef oops::GeometryIterator GeometryIterator_; + typedef oops::ObsDataVector ObsDataVector_; typedef oops::ObsLocalizationBase ObsLocalization_; typedef oops::ObsSpace ObsSpace_; typedef oops::ObsVector ObsVector_; @@ -44,41 +52,84 @@ template void testObsLocalization() { // loop over all obs spaces for (size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + const ObsSpace_ & obspace = Test_::obspace()[jj]; + // initialize obs-space localization eckit::LocalConfiguration locconf(Test_::config(jj), "obs localization"); - // variable used to check if localization is tested at least once below - bool tested = false; + + // read reference local nobs values and reference gridpoints + const std::vector lons = locconf.getDoubleVector("reference gridpoints.lons"); + const std::vector lats = locconf.getDoubleVector("reference gridpoints.lats"); + const std::vector nobs_local_ref = locconf.getUnsignedVector("reference local nobs"); + ASSERT(lons.size() == lats.size()); + ASSERT(lons.size() == nobs_local_ref.size()); + ASSERT(lons.size() > 0); + std::vector reference_points; + for (size_t jpoint = 0; jpoint < lons.size(); ++jpoint) { + reference_points.emplace_back(lons[jpoint], lats[jpoint]); + } + std::unique_ptr obsloc = + oops::ObsLocalizationFactory::create(locconf, obspace); + oops::Log::test() << "Testing obs-space localization: " << *obsloc << std::endl; + + ObsVector_ locvector(obspace); + ObsVector_ obsvector(obspace); + ObsDataVector_ outside(obspace, obspace.obsvariables()); + + size_t total_tested = 0; + std::vector nobs_local(nobs_local_ref.size(), 0); + std::vector locvector_rms(nobs_local_ref.size(), 0); // loop over geometry points for (GeometryIterator_ ii = geometry.begin(); ii != geometry.end(); ++ii) { - // initialize local observation space and a local obs vector - ObsSpace_ localobs(Test_::obspace()[jj], *ii, locconf); - ObsVector_ obsvector(localobs); - // only test if there are some observations in the local obs space - if (obsvector.nobs() > 0) { - tested = true; - // initialize obs-space localization - std::unique_ptr obsloc = - oops::ObsLocalizationFactory::create(locconf, localobs); - oops::Log::test() << "Testing obs-space localization: " << *obsloc << - " at geometry iterator " << ii << std::endl; - // apply obs localization to a zero vector, check that result is zero - obsvector.zero(); - EXPECT_EQUAL(obsvector.rms(), 0.0); - ObsVector_ locvector(localobs); - obsloc->computeLocalization(ii, locvector); - obsvector *= locvector; - EXPECT_EQUAL(obsvector.rms(), 0.0); - // apply localization to a vector of ones, check that rms(result) < 1 - obsvector.ones(); - EXPECT_EQUAL(obsvector.rms(), 1.0); + // debug print to help decide which points to specify for reference + // set OOPS_DEBUG environment variable to -1 to see prints from all MPI tasks + oops::Log::debug() << "Iterating over " << ii << ": " << *ii << std::endl; + // check if we need to test at this location (if there are any points in the + // reference point list within 1e-5 of this locationn) + const auto & it = std::find_if(reference_points.begin(), reference_points.end(), + [ii] (const eckit::geometry::Point2 & point) {return point.distance(*ii) < 1e-5;}); + if (it != reference_points.end()) { + total_tested++; + size_t index = it - reference_points.begin(); locvector.ones(); - obsloc->computeLocalization(ii, locvector); + obsloc->computeLocalization(ii, outside, locvector); + oops::Log::test() << "Obs localization with geometry iterator: " << ii << ": " + << *ii << std::endl; + oops::Log::test() << "Mask for obs outside of localization: " << outside << std::endl; + oops::Log::test() << "Localization values: " << locvector << std::endl; + locvector.mask(outside); + oops::Log::test() << "Local vector nobs and reference: " << locvector.nobs() << ", " + << nobs_local[index] << std::endl; + // save number of local obs to be tested later + nobs_local[index] = locvector.nobs(); + + // apply localization to a vector of ones + obsvector.ones(); obsvector *= locvector; oops::Log::test() << "Localization applied to local ObsVector of ones: " << obsvector << std::endl; - EXPECT(obsvector.rms() < 1.0); + // save localized vector rms to be tested later + locvector_rms[index] = obsvector.rms(); + } + } + // gather number of tested observations, nobs_local and locvector_rms + // (only one MPI task owns one gridpoint) + Test_::comm().allReduceInPlace(total_tested, eckit::mpi::sum()); + Test_::comm().allReduceInPlace(nobs_local.begin(), nobs_local.end(), eckit::mpi::sum()); + Test_::comm().allReduceInPlace(locvector_rms.begin(), locvector_rms.end(), eckit::mpi::sum()); + // check that we tested all gridpoints + EXPECT_EQUAL(total_tested, lons.size()); + // Test that computed number of local obs is the same as reference + EXPECT_EQUAL(nobs_local_ref, nobs_local); + // check whether localization is expected to reduce obsvector.rms() + if (locconf.getBool("localization reduces values")) { + for (size_t jpoint = 0; jpoint < nobs_local.size(); ++jpoint) { + EXPECT(locvector_rms[jpoint] < 1.0); + } + } else { + for (size_t jpoint = 0; jpoint < nobs_local.size(); ++jpoint) { + EXPECT((nobs_local[jpoint] == 0) || (locvector_rms[jpoint] == 1.0)); } } - EXPECT(tested); } } diff --git a/src/test/interface/ObsTestsFixture.h b/src/test/interface/ObsTestsFixture.h index a00bfe570..d29878bed 100644 --- a/src/test/interface/ObsTestsFixture.h +++ b/src/test/interface/ObsTestsFixture.h @@ -38,6 +38,7 @@ class ObsTestsFixture : private boost::noncopyable { static eckit::LocalConfiguration & config(size_t jj) {return getInstance().configs_.at(jj);} /// accessor to a all obs spaces static ObsSpaces_ & obspace() {return *getInstance().ospaces_;} + static const eckit::mpi::Comm & comm() {return getInstance().comm_;} static void reset() { getInstance().ospaces_.reset(); @@ -46,13 +47,13 @@ class ObsTestsFixture : private boost::noncopyable { } private: - ObsTestsFixture(): tbgn_(), tend_(), ospaces_() { + ObsTestsFixture(): comm_(oops::mpi::world()), tbgn_(), tend_(), ospaces_() { tbgn_.reset(new util::DateTime(TestEnvironment::config().getString("window begin"))); tend_.reset(new util::DateTime(TestEnvironment::config().getString("window end"))); configs_ = TestEnvironment::config().getSubConfigurations("observations"); eckit::LocalConfiguration obsconfig = TestEnvironment::config().getSubConfiguration("observations"); - ospaces_.reset(new ObsSpaces_(obsconfig, oops::mpi::world(), *tbgn_, *tend_)); + ospaces_.reset(new ObsSpaces_(obsconfig, comm_, *tbgn_, *tend_)); } ~ObsTestsFixture() {} @@ -62,6 +63,7 @@ class ObsTestsFixture : private boost::noncopyable { return theObsTestsFixture; } + const eckit::mpi::Comm & comm_; std::unique_ptr tbgn_; std::unique_ptr tend_; std::vector configs_; diff --git a/src/test/interface/ObsVector.h b/src/test/interface/ObsVector.h index 5b898a81f..9ab9c93f1 100644 --- a/src/test/interface/ObsVector.h +++ b/src/test/interface/ObsVector.h @@ -281,9 +281,9 @@ template void testMask() { EXPECT_EQUAL(with_mask.nobs(), nobs_after_mask); /// test packEigen - Eigen::VectorXd with_mask_vec = with_mask.packEigen(); - EXPECT(with_mask_vec.size() == with_mask.nobs()); - if (with_mask.nobs() > 0) { + Eigen::VectorXd with_mask_vec = test.packEigen(mask); + EXPECT(with_mask_vec.size() == test.packEigenSize(mask)); + if (with_mask_vec.size() > 0) { double rms1 = with_mask.rms(); double rms2 = sqrt(with_mask_vec.squaredNorm() / with_mask_vec.size()); EXPECT(std::abs(rms1-rms2) < tolerance); @@ -291,23 +291,6 @@ template void testMask() { } } // ----------------------------------------------------------------------------- -template void testPackEigen() { - typedef ObsTestsFixture Test_; - typedef oops::ObsVector ObsVector_; - const double tolerance = 1.0e-8; - for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { - ObsVector_ ov1(Test_::obspace()[jj]); - ov1.random(); - double rms1 = ov1.rms(); - - Eigen::VectorXd vec = ov1.packEigen(); - EXPECT(vec.size() == ov1.packEigenSize()); - - double rms2 = sqrt(vec.squaredNorm() / vec.size()); - EXPECT(std::abs(rms1-rms2) < tolerance); - } -} -// ----------------------------------------------------------------------------- template class ObsVector : public oops::Test { @@ -335,8 +318,6 @@ class ObsVector : public oops::Test { { testReadWrite(); }); ts.emplace_back(CASE("interface/ObsVector/testMask") { testMask(); }); - ts.emplace_back(CASE("interface/ObsVector/testPackEigen") - { testPackEigen(); }); } void clear() const override { From d6a45a58d2e3e39e90f9c2b39a05ee1afac7a533 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Mon, 19 Apr 2021 08:51:59 -0600 Subject: [PATCH 111/142] Pass R to observers for update by filters (#1160) * Pass R to observers for update by filters * Fix iteration for filters * Coding norms * Code reviews --- src/oops/assimilation/CostJo.h | 31 +++++++----------- src/oops/assimilation/LocalEnsembleSolver.h | 2 +- src/oops/base/ObsErrorBase.h | 17 ++++++++-- src/oops/base/ObsErrors.h | 5 +-- src/oops/base/Observer.h | 34 +++++++++++++++----- src/oops/base/Observers.h | 24 ++++++-------- src/oops/generic/ObsErrorDiag.h | 35 ++++++++++++++++++--- src/oops/runs/HofX4D.h | 14 +++------ 8 files changed, 100 insertions(+), 62 deletions(-) diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index 2a18fbae3..b53b852c6 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -116,8 +116,7 @@ template class CostJo : public CostTermBase Rmat_; - + ObsErrors_ Rmat_; Observers_ observers_; /// Jo Gradient at first guess : \f$ R^{-1} (H(x_{fg})-y_{obs}) \f$. @@ -137,8 +136,8 @@ CostJo::CostJo(const eckit::Configuration & joConf, const eckit::mpi const util::DateTime & winbgn, const util::DateTime & winend, const eckit::mpi::Comm & ctime) : obsconf_(joConf), obspaces_(obsconf_, comm, winbgn, winend, ctime), - yobs_(obspaces_, "ObsValue"), Rmat_(), observers_(obspaces_, joConf), gradFG_(), - obstlad_(), currentConf_() + yobs_(obspaces_, "ObsValue"), Rmat_(obsconf_, obspaces_), observers_(obspaces_, joConf), + gradFG_(), obstlad_(), currentConf_() { Log::trace() << "CostJo::CostJo" << std::endl; } @@ -154,7 +153,7 @@ void CostJo::setPostProc(const CtrlVar_ & xx, const eckit::Configura currentConf_.reset(new eckit::LocalConfiguration(conf)); const int iterout = currentConf_->getInt("iteration"); - observers_.initialize(xx.state().geometry(), xx.obsVar(), pp, iterout); + observers_.initialize(xx.state().geometry(), xx.obsVar(), Rmat_, pp, iterout); Log::trace() << "CostJo::setPostProc done" << std::endl; } @@ -169,26 +168,17 @@ double CostJo::computeCost() { Observations_ yeqv(obspaces_); observers_.finalize(yeqv); - const int iterout = currentConf_->getInt("iteration"); - - // Sace current obs estimates - const std::string obsname = "hofx" + std::to_string(iterout); - yeqv.save(obsname); - - // Set observation error covariance - Rmat_.reset(new ObsErrors_(obsconf_, obspaces_)); - // Perturb observations according to obs error statistics bool obspert = currentConf_->getBool("obs perturbations", false); if (obspert) { - yobs_.perturb(*Rmat_); + yobs_.perturb(Rmat_); Log::info() << "Perturbed observations: " << yobs_ << std::endl; } // Gradient at first guess (to define inner loop rhs) Departures_ ydep(yeqv - yobs_); gradFG_.reset(new Departures_(ydep)); - Rmat_->inverseMultiply(*gradFG_); + Rmat_.inverseMultiply(*gradFG_); // Print diagnostics Log::info() << "Jo Observations:" << std::endl << yobs_ @@ -200,6 +190,9 @@ double CostJo::computeCost() { Log::info() << "Jo Bias Corrected Departures:" << std::endl << ydep << "End Jo Bias Corrected Departures" << std::endl; + Log::info() << "Jo Observations Errors:" << std::endl << Rmat_ + << "End Jo Observations Errors" << std::endl; + // Print Jo table double zjo = 0.0; std::vector typeconfs = obsconf_.getSubConfigurations(); @@ -210,7 +203,7 @@ double CostJo::computeCost() { if (nobs > 0) { zz = 0.5 * dot_product(ydep[jj], (*gradFG_)[jj]); - const double err = (*Rmat_)[jj].getRMSE(); + const double err = Rmat_[jj].getRMSE(); Log::test() << zz << ", nobs = " << nobs << ", Jo/n = " << zz/nobs << ", err = " << err; } else { Log::test() << zz << " --- No Observations"; @@ -297,7 +290,7 @@ std::unique_ptr CostJo::multiplyCovar(const GeneralizedDepartures & v1) const { Log::trace() << "CostJo::multiplyCovar start" << std::endl; std::unique_ptr y1(new Departures_(dynamic_cast(v1))); - Rmat_->multiply(*y1); + Rmat_.multiply(*y1); return std::move(y1); } @@ -308,7 +301,7 @@ std::unique_ptr CostJo::multiplyCoInv(const GeneralizedDepartures & v1) const { Log::trace() << "CostJo::multiplyCoInv start" << std::endl; std::unique_ptr y1(new Departures_(dynamic_cast(v1))); - Rmat_->inverseMultiply(*y1); + Rmat_.inverseMultiply(*y1); return std::move(y1); } diff --git a/src/oops/assimilation/LocalEnsembleSolver.h b/src/oops/assimilation/LocalEnsembleSolver.h index e0ba1067f..3e8532bef 100644 --- a/src/oops/assimilation/LocalEnsembleSolver.h +++ b/src/oops/assimilation/LocalEnsembleSolver.h @@ -138,7 +138,7 @@ Observations LocalEnsembleSolver::computeHofX(const StateEnsemb // TODO(someone) combine qc flags from all ensemble members hofx_.saveQcFlags("EffectiveQC"); hofx_.maskObsErrors(); - hofx_.saveObsErrors("EffectiveError"); + hofx_.saveObsErrors("ObsError"); } R_.reset(new ObsErrors_(obsconf_, obspaces_)); invVarR_.reset(new Departures_(R_->inverseVariance())); diff --git a/src/oops/base/ObsErrorBase.h b/src/oops/base/ObsErrorBase.h index b807f7560..d63417619 100644 --- a/src/oops/base/ObsErrorBase.h +++ b/src/oops/base/ObsErrorBase.h @@ -45,6 +45,18 @@ class ObsErrorBase : public util::Printable, /// Generate random perturbation in \p dy virtual void randomize(ObsVector_ & dy) const = 0; +/// Save obs errors + virtual void save(const std::string &) const = 0; + +/// Return obs error std. dev. The non-const method means caller can modify values, +/// this is used by obs filters. In this case update() should be called to ensure +/// the matrix stays consistent. + virtual ObsVector_ & obserrors() = 0; + virtual const ObsVector_ & obserrors() const = 0; + +/// Update when obs errors standard deviations have been modified + virtual void update() = 0; + /// Return inverseVariance virtual const ObsVector_ & inverseVariance() const = 0; @@ -77,8 +89,7 @@ class ObsErrorFactory { template class ObsErrorMaker : public ObsErrorFactory { typedef ObsSpace ObsSpace_; - virtual ObsErrorBase * make(const eckit::Configuration & conf, - const ObsSpace_ & obs) + virtual ObsErrorBase * make(const eckit::Configuration & conf, const ObsSpace_ & obs) { return new T(conf, obs); } public: explicit ObsErrorMaker(const std::string & name) : ObsErrorFactory(name) {} @@ -100,7 +111,7 @@ template std::unique_ptr> ObsErrorFactory::create(const eckit::Configuration & conf, const ObsSpace_ & obs) { Log::trace() << "ObsErrorBase::create starting" << std::endl; - const std::string id = conf.getString("covariance model"); + const std::string id = conf.getString("covariance model", "diagonal"); typename std::map*>::iterator jerr = getMakers().find(id); if (jerr == getMakers().end()) { diff --git a/src/oops/base/ObsErrors.h b/src/oops/base/ObsErrors.h index f9b6b3b35..2ee7e950a 100644 --- a/src/oops/base/ObsErrors.h +++ b/src/oops/base/ObsErrors.h @@ -41,6 +41,7 @@ class ObsErrors : public util::Printable, /// Accessor and size size_t size() const {return err_.size();} + ObsError_ & operator[](const size_t ii) {return *err_.at(ii);} const ObsError_ & operator[](const size_t ii) const {return *err_.at(ii);} /// Multiply a Departure by \f$R\f$ @@ -67,7 +68,7 @@ ObsErrors::ObsErrors(const eckit::Configuration & config, const ObsSpaces_ & os) : err_(), os_(os) { std::vector obsconf = config.getSubConfigurations(); for (size_t jj = 0; jj < os.size(); ++jj) { - eckit::LocalConfiguration conf(obsconf[jj], "obs error"); + eckit::LocalConfiguration conf = obsconf[jj].getSubConfiguration("obs error"); err_.emplace_back(ObsErrorFactory::create(conf, os[jj])); } } @@ -114,7 +115,7 @@ Departures ObsErrors::inverseVariance() const { template void ObsErrors::print(std::ostream & os) const { - for (size_t jj = 0; jj < err_.size(); ++jj) os << *err_[jj]; + for (size_t jj = 0; jj < err_.size(); ++jj) os << *err_[jj] << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/Observer.h b/src/oops/base/Observer.h index 39ff6d975..35832ffeb 100644 --- a/src/oops/base/Observer.h +++ b/src/oops/base/Observer.h @@ -7,6 +7,7 @@ #ifndef OOPS_BASE_OBSERVER_H_ #define OOPS_BASE_OBSERVER_H_ +#include #include #include #include @@ -14,6 +15,7 @@ #include "eckit/config/LocalConfiguration.h" #include "oops/base/GetValuePost.h" +#include "oops/base/ObsErrorBase.h" #include "oops/base/ObsFilters.h" #include "oops/base/Variables.h" #include "oops/interface/Geometry.h" @@ -53,6 +55,7 @@ class Observer { typedef ObsAuxControl ObsAuxCtrl_; typedef ObsDataVector ObsDataInt_; typedef ObsDiagnostics ObsDiags_; + typedef ObsErrorBase ObsError_; typedef ObsFilters ObsFilters_; typedef ObsOperator ObsOperator_; typedef ObsSpace ObsSpace_; @@ -65,7 +68,7 @@ class Observer { /// \brief Initializes variables, obs bias, obs filters (could be different for /// different iterations std::shared_ptr initialize(const Geometry_ &, const ObsAuxCtrl_ &, - ObsVector_ &, const int iter); + ObsError_ &, const int iter); /// \brief Computes H(x) from the filled in GeoVaLs void finalize(ObsVector_ &); @@ -76,6 +79,7 @@ class Observer { std::unique_ptr obsop_; // Obs operator std::unique_ptr locations_; // locations const ObsAuxCtrl_ * ybias_; // Obs bias + ObsError_ * Rmat_; // Obs error covariance std::unique_ptr filters_; // QC filters std::shared_ptr getvals_; // Postproc passed to the model during integration. std::shared_ptr qcflags_; // QC flags (should not be a pointer) @@ -88,7 +92,7 @@ class Observer { template Observer::Observer(const ObsSpace_ & obspace, const eckit::Configuration & config) : obsconfig_(config), obspace_(obspace), obsop_(), locations_(), - ybias_(nullptr), filters_(), qcflags_(), iterout_(0), initialized_(false) + ybias_(nullptr), filters_(), qcflags_(), iterout_(-1), initialized_(false) { Log::trace() << "Observer::Observer start" << std::endl; ObserverParameters observerParams; @@ -105,21 +109,24 @@ Observer::Observer(const ObsSpace_ & obspace, const eckit::Configura template std::shared_ptr> Observer::initialize(const Geometry_ & geom, const ObsAuxCtrl_ & ybias, - ObsVector_ & obserr, const int iter) { + ObsError_ & R, const int iter) { Log::trace() << "Observer::initialize start" << std::endl; +// Save information for finalize iterout_ = iter; ybias_ = &ybias; + Rmat_ = &R; +// Set up QC filters and run preprocess + int iterfilt = std::max(iter, 0); ObserverParameters observerParams; observerParams.deserialize(obsconfig_); - /// Set up QC filters and run preprocess filters_.reset(new ObsFilters_(obspace_, observerParams.obsFilters, - qcflags_, obserr, iterout_)); + qcflags_, Rmat_->obserrors(), iterfilt)); filters_->preProcess(); locations_.reset(new Locations_(obsop_->locations())); - /// Set up variables that will be requested from the model +// Set up variables that will be requested from the model Variables geovars; geovars += obsop_->requiredVars(); geovars += ybias_->requiredVars(); @@ -127,6 +134,7 @@ Observer::initialize(const Geometry_ & geom, const ObsAuxCtrl_ & ybi eckit::LocalConfiguration gvconf = obsconfig_.getSubConfiguration("get values"); +// Set up GetValues getvals_.reset(new GetValPost_(gvconf, geom, obspace_.windowStart(), obspace_.windowEnd(), *locations_, geovars)); @@ -160,9 +168,19 @@ void Observer::finalize(ObsVector_ & yobsim) { /// Call posterior filters filters_->postFilter(yobsim, ydiags); - /// Save flags (for diagnostics use) - const std::string qcname = "EffectiveQC" + std::to_string(iterout_); + // Update R with obs errors that filters might have updated + Rmat_->update(); + + // Save current obs, obs error estimates and QC flags (for diagnostics use only) + std::string siter = ""; + if (iterout_ >= 0) siter = std::to_string(iterout_); + const std::string qcname = "EffectiveQC" + siter; qcflags_->save(qcname); + const std::string obsname = "hofx" + siter; + yobsim.save(obsname); + const std::string errname = "EffectiveError" + siter; + Rmat_->save(errname); + Log::info() << "Observer::finalize QC = " << *qcflags_ << std::endl; initialized_ = false; diff --git a/src/oops/base/Observers.h b/src/oops/base/Observers.h index c5d95986e..9f418470a 100644 --- a/src/oops/base/Observers.h +++ b/src/oops/base/Observers.h @@ -16,6 +16,7 @@ #include "oops/base/GetValuePosts.h" #include "oops/base/ObsAuxControls.h" +#include "oops/base/ObsErrors.h" #include "oops/base/Observations.h" #include "oops/base/Observer.h" #include "oops/base/ObsSpaces.h" @@ -36,6 +37,7 @@ class Observers { typedef Geometry Geometry_; typedef GetValuePosts GetValuePosts_; typedef ObsAuxControls ObsAuxCtrls_; + typedef ObsErrors ObsErrors_; typedef Observations Observations_; typedef Observer Observer_; typedef ObsSpaces ObsSpaces_; @@ -49,14 +51,13 @@ class Observers { /// \brief Initializes variables, obs bias, obs filters (could be different for /// different iterations - void initialize(const Geometry_ &, const ObsAuxCtrls_ &, PostProc_ &, const int iter = 0); + void initialize(const Geometry_ &, const ObsAuxCtrls_ &, ObsErrors_ &, + PostProc_ &, const int iter = -1); /// \brief Computes H(x) from the filled in GeoVaLs void finalize(Observations_ &); private: - const ObsSpaces_ & obspaces_; // ObsSpaces used in H(x) - std::vector obserrs_; std::vector> observers_; }; @@ -64,13 +65,13 @@ class Observers { template Observers::Observers(const ObsSpaces_ & obspaces, const eckit::Configuration & config) - : obspaces_(obspaces), obserrs_(), observers_() + : observers_() { Log::trace() << "Observers::Observers start" << std::endl; std::vector obsconfs = config.getSubConfigurations(); - for (size_t jj = 0; jj < obspaces_.size(); ++jj) { - observers_.emplace_back(new Observer_(obspaces_[jj], obsconfs[jj])); + for (size_t jj = 0; jj < obspaces.size(); ++jj) { + observers_.emplace_back(new Observer_(obspaces[jj], obsconfs[jj])); } Log::trace() << "Observers::Observers done" << std::endl; @@ -80,16 +81,12 @@ Observers::Observers(const ObsSpaces_ & obspaces, const eckit::Confi template void Observers::initialize(const Geometry_ & geom, const ObsAuxCtrls_ & obsaux, - PostProc_ & pp, const int iter) { + ObsErrors_ & Rmat, PostProc_ & pp, const int iter) { Log::trace() << "Observers::initialize start" << std::endl; - obserrs_.reserve(observers_.size()); - std::string errname = "ObsError"; - if (iter > 0) errname = "EffectiveError"; std::shared_ptr getvals(new GetValuePosts_()); for (size_t jj = 0; jj < observers_.size(); ++jj) { - obserrs_.emplace_back(obspaces_[jj], errname); - getvals->append(observers_[jj]->initialize(geom, obsaux[jj], obserrs_[jj], iter)); + getvals->append(observers_[jj]->initialize(geom, obsaux[jj], Rmat[jj], iter)); } pp.enrollProcessor(getvals); @@ -104,10 +101,7 @@ void Observers::finalize(Observations_ & yobs) { for (size_t jj = 0; jj < observers_.size(); ++jj) { observers_[jj]->finalize(yobs[jj]); - obserrs_[jj].save("EffectiveError"); // Obs error covariance is looking for that for now - Log::info() << "Observers::finalize obs err = " << obserrs_[jj] << std::endl; } - obserrs_.clear(); oops::Log::trace() << "Observers::finalize done" << std::endl; } diff --git a/src/oops/generic/ObsErrorDiag.h b/src/oops/generic/ObsErrorDiag.h index 7d27bc818..c767263b5 100644 --- a/src/oops/generic/ObsErrorDiag.h +++ b/src/oops/generic/ObsErrorDiag.h @@ -12,6 +12,7 @@ #define OOPS_GENERIC_OBSERRORDIAG_H_ #include +#include #include "eckit/config/Configuration.h" #include "oops/base/ObsErrorBase.h" @@ -41,6 +42,9 @@ class ObsErrorDiag : public ObsErrorBase { public: ObsErrorDiag(const eckit::Configuration &, const ObsSpace_ &); +/// Update after obs errors potentially changed + void update() override; + /// Multiply a Departure by \f$R\f$ void multiply(ObsVector_ &) const override; @@ -50,9 +54,16 @@ class ObsErrorDiag : public ObsErrorBase { /// Generate random perturbation void randomize(ObsVector_ &) const override; -/// Get mean error for Jo table +/// Save obs errors + void save(const std::string &) const override; + +/// Get mean std deviation of errors for Jo table double getRMSE() const override {return stddev_.rms();} +/// Get obs errors std deviation + ObsVector_ & obserrors() override {return stddev_;} + const ObsVector_ & obserrors() const override {return stddev_;} + /// Return inverseVariance const ObsVector_ & inverseVariance() const override {return inverseVariance_;} @@ -69,13 +80,21 @@ class ObsErrorDiag : public ObsErrorBase { template ObsErrorDiag::ObsErrorDiag(const eckit::Configuration & conf, const ObsSpace_ & obsgeom) - : stddev_(obsgeom, "EffectiveError"), inverseVariance_(obsgeom) + : stddev_(obsgeom, "ObsError"), inverseVariance_(obsgeom) { options_.deserialize(conf); + this->update(); + Log::trace() << "ObsErrorDiag:ObsErrorDiag constructed nobs = " << stddev_.nobs() << std::endl; +} + +// ----------------------------------------------------------------------------- + +template +void ObsErrorDiag::update() { inverseVariance_ = stddev_; inverseVariance_ *= stddev_; inverseVariance_.invert(); - Log::trace() << "ObsErrorDiag:ObsErrorDiag constructed nobs = " << stddev_.nobs() << std::endl; + Log::info() << "ObsErrorDiag covariance updated " << stddev_.nobs() << std::endl; } // ----------------------------------------------------------------------------- @@ -103,10 +122,16 @@ void ObsErrorDiag::randomize(ObsVector_ & dy) const { // ----------------------------------------------------------------------------- +template +void ObsErrorDiag::save(const std::string & name) const { + stddev_.save(name); +} + +// ----------------------------------------------------------------------------- + template void ObsErrorDiag::print(std::ostream & os) const { - os << "Diagonal observation error covariance, inverse variances: " - << inverseVariance_ << std::endl; + os << "Diagonal observation error covariance" << std::endl << stddev_; } // ----------------------------------------------------------------------------- diff --git a/src/oops/runs/HofX4D.h b/src/oops/runs/HofX4D.h index 4eec69fb1..88db8412f 100644 --- a/src/oops/runs/HofX4D.h +++ b/src/oops/runs/HofX4D.h @@ -101,10 +101,11 @@ template class HofX4D : public Application { const eckit::LocalConfiguration obsConfig(fullConfig, "observations"); ObsSpaces_ obspaces(obsConfig, this->getComm(), winbgn, winend); ObsAux_ obsaux(obspaces, obsConfig); + ObsErrors_ Rmat(obsConfig, obspaces); // Setup and initialize observer Observers_ hofx(obspaces, obsConfig); - hofx.initialize(geometry, obsaux, post); + hofx.initialize(geometry, obsaux, Rmat, post); // run the model and compute H(x) model.forecast(xx, moderr, flength, post); @@ -119,18 +120,13 @@ template class HofX4D : public Application { // as ObsValue if "hofx group name" == ObsValue. bool obspert = fullConfig.getBool("obs perturbations", false); if (obspert) { - ObsErrors_ matR(obsConfig, obspaces); - yobs.perturb(matR); + yobs.perturb(Rmat); Log::test() << "Perturbed H(x): " << std::endl << yobs << "End Perturbed H(x)" << std::endl; } -// Save H(x) either as observations (if "make obs" == true) or as "hofx" +// Save H(x) as observations (if "make obs" == true) const bool makeobs = fullConfig.getBool("make obs", false); - if (makeobs) { - yobs.save("ObsValue"); - } else { - yobs.save("hofx"); - } + if (makeobs) yobs.save("ObsValue"); return 0; } From e5123c4635b4b6acfc42fb24042bfd312e8f34d1 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Thu, 22 Apr 2021 18:40:47 -0600 Subject: [PATCH 112/142] Bring back option to save departures (#1173) * save departures optionally * fix a copy-pasted comment --- src/oops/assimilation/CostJo.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index b53b852c6..bc332c7fe 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -175,8 +175,14 @@ double CostJo::computeCost() { Log::info() << "Perturbed observations: " << yobs_ << std::endl; } - // Gradient at first guess (to define inner loop rhs) + // Compute observations departures and save to output file Departures_ ydep(yeqv - yobs_); + if (currentConf_->has("diagnostics.departures")) { + const std::string depname = currentConf_->getString("diagnostics.departures"); + ydep.save(depname); + } + + // Gradient at first guess (to define inner loop rhs) gradFG_.reset(new Departures_(ydep)); Rmat_.inverseMultiply(*gradFG_); From 496d0221d0ece940d3ca760e3aa3bd8caf0875ef Mon Sep 17 00:00:00 2001 From: Carwyn Pelley Date: Tue, 27 Apr 2021 17:52:04 +0100 Subject: [PATCH 113/142] MAINT: Dissalowing partial missing component in where clause date times (#1177) * ENH: PartialDateTime check string format construction * ENH: Updated parameter traits using pattern * MAINT: char instantiate syntax update * TEST: Updated parameter traits tests --- src/oops/util/PartialDateTime.cc | 45 +++---- src/oops/util/parameters/ParameterTraits.h | 4 +- src/oops/util/parameters/Parameters.cc | 11 -- src/test/util/Parameters.h | 2 +- src/test/util/PartialDateTime.cc | 147 +++++++++++---------- 5 files changed, 103 insertions(+), 106 deletions(-) diff --git a/src/oops/util/PartialDateTime.cc b/src/oops/util/PartialDateTime.cc index b0fbdbfe0..f185a2239 100644 --- a/src/oops/util/PartialDateTime.cc +++ b/src/oops/util/PartialDateTime.cc @@ -6,7 +6,9 @@ */ #include "oops/util/PartialDateTime.h" +#include // std::replace #include +#include #include "eckit/exception/Exceptions.h" #include "oops/util/dateFunctions.h" @@ -37,37 +39,30 @@ PartialDateTime::PartialDateTime(std::string const &datetime_string) { // Derive an ordinary datetime string std::string alt_datetime_string = datetime_string; - // Determine if stringUnset_ is present and for which component(s). - if (datetime_string.size() != 20) - throw eckit::BadParameter("Partial date-time string '" + datetime_string + - "' is of unexpected length"); - + // Determine suitability of string format + std::string expression = R"(^([0-9]{4}|\*{4})-([0-9]{2}|\*{2})-([0-9]{2}|\*{2}))" + R"(T([0-9]{2}|\*{2}):([0-9]{2}|\*{2}):([0-9]{2}|\*{2})Z$)"; + static const std::regex regex(expression); + std::smatch matches; + if (!std::regex_match(datetime_string, matches, regex)) + throw eckit::BadParameter("Partial date-time string '" + datetime_string + + "' is not of the expected format '" + expression + "'"); + + // Determine which components are missing. + std::array component_index = {0, 5, 8, 11, 14, 17}; std::array populatedvalues = {true, true, true, true, true, true}; - bool isSet; - for (size_t ind=0; ind < datetime_string.size(); ind++) { - isSet = datetime_string[ind] != stringUnset_; - if (!isSet) alt_datetime_string[ind] = '0'; // Replace * with 0 - if (ind <= 3) { - populatedvalues[0] &= isSet; - } else if (ind >= 5 && ind <= 6) { - populatedvalues[1] &= isSet; - } else if (ind >= 8 && ind <= 9) { - populatedvalues[2] &= isSet; - } else if (ind >= 11 && ind <= 12) { - populatedvalues[3] &= isSet; - } else if (ind >= 14 && ind <= 15) { - populatedvalues[4] &= isSet; - } else if (ind >= 17 && ind <= 18) { - populatedvalues[5] &= isSet; - } - } + for (size_t ind=0; ind < component_index.size(); ind++) + populatedvalues[ind] = datetime_string[component_index[ind]] != stringUnset_; + + // Replace these with '0' so that we can determine the year, month, etc. + std::replace(alt_datetime_string.begin(), alt_datetime_string.end(), char{stringUnset_}, '0'); + util::datefunctions::stringToYYYYMMDDhhmmss(alt_datetime_string, year, month, day, hour, minute, second); // Replace with unset values partialdt_ = {year, month, day, hour, minute, second}; - for (size_t ind=0; ind < partialdt_.size(); ind++) { + for (size_t ind=0; ind < partialdt_.size(); ind++) if (!populatedvalues[ind]) partialdt_[ind] = intUnset_; - } } diff --git a/src/oops/util/parameters/ParameterTraits.h b/src/oops/util/parameters/ParameterTraits.h index 7e3787359..78019166c 100644 --- a/src/oops/util/parameters/ParameterTraits.h +++ b/src/oops/util/parameters/ParameterTraits.h @@ -327,8 +327,10 @@ struct ParameterTraits { } static ObjectJsonSchema jsonSchema(const std::string &name) { + std::string expression = R"("^([0-9]{4}|\\*{4})-([0-9]{2}|\\*{2})-([0-9]{2}|\\*{2}))" + R"(T([0-9]{2}|\\*{2}):([0-9]{2}|\\*{2}):([0-9]{2}|\\*{2})Z$")"; return ObjectJsonSchema({{name, {{"type", "\"string\""}, - {"format", "\"partial-date-time\""}}}}); + {"pattern", expression}}}}); } }; diff --git a/src/oops/util/parameters/Parameters.cc b/src/oops/util/parameters/Parameters.cc index f095e6578..0cd6ab340 100644 --- a/src/oops/util/parameters/Parameters.cc +++ b/src/oops/util/parameters/Parameters.cc @@ -97,17 +97,6 @@ void checkStringFormat(const std::string &format, const std::string &value) { if (!std::regex_match(value, matches, regex)) { throw std::invalid_argument(value + " is not a duration string."); } - - } else if (format == "partial-date-time") { - // Matches ISO 8601 representations of datetimes but additionally supporting the asterix - // character to denote partial definition. - static const std::regex regex( - "^[0-9*]{4}-[0-9*]{2}-[0-9*]{2}T[0-9*]{2}:[0-9*]{2}:[0-9*]{2}Z$"); - std::smatch matches; - if (!std::regex_match(value, matches, regex)) { - throw std::invalid_argument(value + " is not a partial-date-time string."); - } - } else { nlohmann::json_schema::default_string_format_check(format, value); } diff --git a/src/test/util/Parameters.h b/src/test/util/Parameters.h index 35ace81ac..cffa6bf4f 100644 --- a/src/test/util/Parameters.h +++ b/src/test/util/Parameters.h @@ -629,7 +629,7 @@ void testIncorrectValueOfOptionalPartialDateTimeParameter() { const eckit::LocalConfiguration conf(TestEnvironment::config(), "error_in_opt_partialDT_parameter"); if (validationSupported) - EXPECT_THROWS_MSG(params.validate(conf), "ABCDEF is not a partial-date-time string"); + EXPECT_THROWS_MSG(params.validate(conf), "YAML validation failed."); EXPECT_THROWS_AS(params.deserialize(conf), eckit::BadParameter); } diff --git a/src/test/util/PartialDateTime.cc b/src/test/util/PartialDateTime.cc index 78bf811a3..59faca37a 100644 --- a/src/test/util/PartialDateTime.cc +++ b/src/test/util/PartialDateTime.cc @@ -12,74 +12,85 @@ namespace { - int notset = -1; - - CASE("test_toString_basic") { - util::PartialDateTime pdt(2011, 9, 16, 13, 55, 20); - std::string res = pdt.toString(); - std::string tar = "2011-09-16T13:55:20Z"; - EXPECT_EQUAL(res, tar); - } - - CASE("test_toString_with_missing") { - util::PartialDateTime pdt(2011, 9, 16, 13, -1, 20); - std::string res = pdt.toString(); - std::string tar = "2011-09-16T13:**:20Z"; - EXPECT_EQUAL(res, tar); - } - - CASE("test_construction_int_arg") { - util::PartialDateTime pdt(2011, 9, 16, 13, 55, 20); - EXPECT(pdt.year() == 2011); - EXPECT(pdt.month() == 9); - EXPECT(pdt.day() == 16); - EXPECT(pdt.hour() == 13); - EXPECT(pdt.minute() == 55); - EXPECT(pdt.second() == 20); - } - - - CASE("test_construction_default") { - util::PartialDateTime pdt{}; - EXPECT(pdt.year() == notset); - EXPECT(pdt.month() == notset); - EXPECT(pdt.day() == notset); - EXPECT(pdt.hour() == notset); - EXPECT(pdt.minute() == notset); - EXPECT(pdt.second() == notset); - } - - - CASE("test_construction_string_arg") { - util::PartialDateTime pdt("2011-09-16T13:55:20Z"); - EXPECT(pdt.year() == 2011); - EXPECT(pdt.month() == 9); - EXPECT(pdt.day() == 16); - EXPECT(pdt.hour() == 13); - EXPECT(pdt.minute() == 55); - EXPECT(pdt.second() == 20); - } - - - CASE("test_construction_string_arg_unset") { - // Check we properly handle the case where the string contains unset components - // Note that any bit of a component results in it being considered unset. - util::PartialDateTime pdt("2011-09-1*T13:55:00Z"); - EXPECT(pdt.year() == 2011); - EXPECT(pdt.month() == 9); - EXPECT(pdt.day() == notset); - EXPECT(pdt.hour() == 13); - EXPECT(pdt.minute() == 55); - EXPECT(pdt.second() == 0); - - pdt = util::PartialDateTime("2011-09-*6T13:55:00Z"); - EXPECT(pdt.year() == 2011); - EXPECT(pdt.month() == 9); - EXPECT(pdt.day() == notset); - EXPECT(pdt.hour() == 13); - EXPECT(pdt.minute() == 55); - EXPECT(pdt.second() == 0); - } + int notset = -1; + + CASE("test_toString_basic") { + util::PartialDateTime pdt(2011, 9, 16, 13, 55, 20); + std::string res = pdt.toString(); + std::string tar = "2011-09-16T13:55:20Z"; + EXPECT_EQUAL(res, tar); + } + + CASE("test_toString_with_missing") { + util::PartialDateTime pdt(2011, 9, 16, 13, -1, 20); + std::string res = pdt.toString(); + std::string tar = "2011-09-16T13:**:20Z"; + EXPECT_EQUAL(res, tar); + } + + CASE("test_construction_int_arg") { + util::PartialDateTime pdt(2011, 9, 16, 13, 55, 20); + EXPECT(pdt.year() == 2011); + EXPECT(pdt.month() == 9); + EXPECT(pdt.day() == 16); + EXPECT(pdt.hour() == 13); + EXPECT(pdt.minute() == 55); + EXPECT(pdt.second() == 20); + } + + + CASE("test_construction_default") { + util::PartialDateTime pdt{}; + EXPECT(pdt.year() == notset); + EXPECT(pdt.month() == notset); + EXPECT(pdt.day() == notset); + EXPECT(pdt.hour() == notset); + EXPECT(pdt.minute() == notset); + EXPECT(pdt.second() == notset); + } + + + CASE("test_construction_string_arg") { + util::PartialDateTime pdt("2011-09-16T13:55:20Z"); + EXPECT(pdt.year() == 2011); + EXPECT(pdt.month() == 9); + EXPECT(pdt.day() == 16); + EXPECT(pdt.hour() == 13); + EXPECT(pdt.minute() == 55); + EXPECT(pdt.second() == 20); + } + + + CASE("test_construction_string_arg_unset") { + // Check we properly handle the case where the string contains unset components + // Note that any bit of a component results in it being considered unset. + util::PartialDateTime pdt("2011-09-**T13:55:00Z"); + EXPECT(pdt.year() == 2011); + EXPECT(pdt.month() == 9); + EXPECT(pdt.day() == notset); + EXPECT(pdt.hour() == 13); + EXPECT(pdt.minute() == 55); + EXPECT(pdt.second() == 0); + + pdt = util::PartialDateTime("2011-09-**T13:55:00Z"); + EXPECT(pdt.year() == 2011); + EXPECT(pdt.month() == 9); + EXPECT(pdt.day() == notset); + EXPECT(pdt.hour() == 13); + EXPECT(pdt.minute() == 55); + EXPECT(pdt.second() == 0); + } + + + CASE("test_construction_string_bad_format") { + // Check that we sanity check badly formatted strings + + // Defining a partially missing component + EXPECT_THROWS(util::PartialDateTime("2011-09-*0T13:55:00Z")); + + // Wrong length + EXPECT_THROWS(util::PartialDateTime("2011-09-01T13:55:00Z2")); + } CASE("test_gt_operator") { From 3c88afce67e81e7e27278f4c9bb5fab3d70363a0 Mon Sep 17 00:00:00 2001 From: Clementine Gas <43183478+cmgas@users.noreply.github.com> Date: Tue, 27 Apr 2021 12:16:20 -0600 Subject: [PATCH 114/142] Add ObsIterator+tests in qg model (#1172) * Add ObsIterator to QG - compiles but I still need to implement * operator * Test pass and operator* is added * Fix the test * tests all pass * Change locations_ to hold a pointer instead of a location object * Coding norms qg * extra line * Remove accessors from LocationsQG - I don't need them anymore * Address some comments but the unique_ptr is still copied and not moved - needs change * zero tolerance for l95 test, remove comment * move ptr in ObsIterator constructor * Coding norms... * Comment * Address more comments * last comment --- l95/src/lorenz95/ObsIterator.cc | 6 +-- l95/src/lorenz95/ObsIterator.h | 2 +- l95/test/testinput/interfaces.yaml | 1 + qg/model/CMakeLists.txt | 2 + qg/model/ObsIteratorQG.cc | 50 ++++++++++++++++++++++++ qg/model/ObsIteratorQG.h | 54 ++++++++++++++++++++++++++ qg/model/ObsSpaceQG.cc | 10 +++++ qg/model/ObsSpaceQG.h | 7 ++++ qg/model/QgTraits.h | 8 ++-- qg/test/CMakeLists.txt | 6 +++ qg/test/executables/TestObsIterator.cc | 16 ++++++++ qg/test/testinput/interfaces.yaml | 22 ++++++++++- src/test/interface/ObsIterator.h | 6 ++- 13 files changed, 180 insertions(+), 10 deletions(-) create mode 100644 qg/model/ObsIteratorQG.cc create mode 100644 qg/model/ObsIteratorQG.h create mode 100644 qg/test/executables/TestObsIterator.cc diff --git a/l95/src/lorenz95/ObsIterator.cc b/l95/src/lorenz95/ObsIterator.cc index 09c0e2ad3..f113f1511 100644 --- a/l95/src/lorenz95/ObsIterator.cc +++ b/l95/src/lorenz95/ObsIterator.cc @@ -12,18 +12,18 @@ namespace lorenz95 { // ----------------------------------------------------------------------------- -ObsIterator::ObsIterator(const std::vector & locations, const int & index): +ObsIterator::ObsIterator(const std::vector & locations, int index): locations_(locations), index_(index) { } // ----------------------------------------------------------------------------- bool ObsIterator::operator==(const ObsIterator & other) const { - return ((index_ == other.index_) && (locations_ == other.locations_)); + return (index_ == other.index_); } // ----------------------------------------------------------------------------- bool ObsIterator::operator!=(const ObsIterator & other) const { - return ((index_ != other.index_) || (locations_ != other.locations_)); + return (index_ != other.index_); } // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObsIterator.h b/l95/src/lorenz95/ObsIterator.h index 6743489bf..7dca5236f 100644 --- a/l95/src/lorenz95/ObsIterator.h +++ b/l95/src/lorenz95/ObsIterator.h @@ -27,7 +27,7 @@ class ObsIterator: public std::iterator & locations, const int & index); + ObsIterator(const std::vector & locations, int index); bool operator==(const ObsIterator &) const; bool operator!=(const ObsIterator &) const; diff --git a/l95/test/testinput/interfaces.yaml b/l95/test/testinput/interfaces.yaml index 80cd40705..73696e20c 100644 --- a/l95/test/testinput/interfaces.yaml +++ b/l95/test/testinput/interfaces.yaml @@ -126,6 +126,7 @@ observations: reference nobs: 160 tolerance: 1.0e-10 obs iterator test: + tolerance: 0.0 reference nlocs: 160 lon1: 0.0 lat1: 0.0 diff --git a/qg/model/CMakeLists.txt b/qg/model/CMakeLists.txt index f7580c928..8d8cf3878 100644 --- a/qg/model/CMakeLists.txt +++ b/qg/model/CMakeLists.txt @@ -39,6 +39,8 @@ ObsBiasIncrement.cc ObsBiasIncrement.h ObsDataQG.h ObsDiagsQG.h +ObsIteratorQG.cc +ObsIteratorQG.h ObsLocQG.cc ObsLocQG.h ObsOpBaseQG.cc diff --git a/qg/model/ObsIteratorQG.cc b/qg/model/ObsIteratorQG.cc new file mode 100644 index 000000000..aa10132ba --- /dev/null +++ b/qg/model/ObsIteratorQG.cc @@ -0,0 +1,50 @@ +/* + * (C) Copyright 2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include +#include + +#include "atlas/array.h" +#include "atlas/field.h" + +#include "model/ObsIteratorQG.h" + +// ----------------------------------------------------------------------------- +namespace qg { + +ObsIteratorQG::ObsIteratorQG(const ObsIteratorQG & other): index_(other.index_), + locslonlat_(other.locslonlat_) {} + +// ----------------------------------------------------------------------------- +ObsIteratorQG::ObsIteratorQG(const LocationsQG & locations, int index): + index_(index), locslonlat_(locations.lonlat()) {} + +// ----------------------------------------------------------------------------- +bool ObsIteratorQG::operator==(const ObsIteratorQG & other) const { + return (index_ == other.index_); +} + +// ----------------------------------------------------------------------------- +bool ObsIteratorQG::operator!=(const ObsIteratorQG & other) const { + return (index_!= other.index_); +} + +// ----------------------------------------------------------------------------- +eckit::geometry::Point2 ObsIteratorQG::operator*() const { + auto lonlat = atlas::array::make_view(locslonlat_); + return eckit::geometry::Point2(lonlat(index_, 0), lonlat(index_, 1)); +} + +// ----------------------------------------------------------------------------- +ObsIteratorQG& ObsIteratorQG::operator++() { + index_++; + return *this; +} + +// ----------------------------------------------------------------------------- + +} // namespace qg diff --git a/qg/model/ObsIteratorQG.h b/qg/model/ObsIteratorQG.h new file mode 100644 index 000000000..9b360f8c9 --- /dev/null +++ b/qg/model/ObsIteratorQG.h @@ -0,0 +1,54 @@ +/* + * (C) Copyright 2021 UCAR. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef QG_MODEL_OBSITERATORQG_H_ +#define QG_MODEL_OBSITERATORQG_H_ + +#include +#include +#include + +#include "eckit/geometry/Point2.h" + +#include "model/LocationsQG.h" + +#include "oops/util/ObjectCounter.h" +#include "oops/util/Printable.h" + +namespace qg { + +/// Iterator over all observations +class ObsIteratorQG: public std::iterator, + public util::Printable, + private util::ObjectCounter { + public: + static const std::string classname() {return "qg::ObsIteratorQG";} + + ObsIteratorQG(const ObsIteratorQG &); + ObsIteratorQG(const LocationsQG &, int); + + bool operator==(const ObsIteratorQG &) const; + bool operator!=(const ObsIteratorQG &) const; + + /// return location of current observation + eckit::geometry::Point2 operator*() const; + + ObsIteratorQG& operator++(); + + private: + void print(std::ostream & os) const override {os << index_;} + + /// index of a current observation + int index_; + /// atlas field of the lons and lats + atlas::Field locslonlat_; +}; + +} // namespace qg + +#endif // QG_MODEL_OBSITERATORQG_H_ diff --git a/qg/model/ObsSpaceQG.cc b/qg/model/ObsSpaceQG.cc index f1500be5b..a21616c3d 100644 --- a/qg/model/ObsSpaceQG.cc +++ b/qg/model/ObsSpaceQG.cc @@ -148,6 +148,16 @@ int ObsSpaceQG::nobs() const { } // ----------------------------------------------------------------------------- +// ----------------------------------------------------------------------------- +ObsIteratorQG ObsSpaceQG::begin() const { + return ObsIteratorQG(*this->locations(), 0); +} +// ----------------------------------------------------------------------------- +ObsIteratorQG ObsSpaceQG::end() const { + return ObsIteratorQG(*this->locations(), this->nobs()); +} +// ----------------------------------------------------------------------------- + void ObsSpaceQG::print(std::ostream & os) const { os << "ObsSpace for " << obsname_ << ", " << this->nobs() << " obs"; } diff --git a/qg/model/ObsSpaceQG.h b/qg/model/ObsSpaceQG.h index 0730175a4..90f8343b1 100644 --- a/qg/model/ObsSpaceQG.h +++ b/qg/model/ObsSpaceQG.h @@ -26,6 +26,7 @@ #include "oops/util/DateTime.h" #include "oops/qg/LocationsQG.h" +#include "oops/qg/ObsIteratorQG.h" #include "oops/qg/QgFortran.h" namespace eckit { @@ -33,6 +34,7 @@ namespace eckit { } namespace qg { + class ObsIteratorQG; /// \brief ObsSpace for QG model // \details ObsSpaceQG is created for each obs type. The underlying Fortran @@ -64,6 +66,11 @@ class ObsSpaceQG : public oops::ObsSpaceBase { /// observation type const std::string & obsname() const {return obsname_;} + /// iterator to the first observation + ObsIteratorQG begin() const; + /// iterator to the observation past-the-last + ObsIteratorQG end() const; + /// interface with Fortran const F90odb & toFortran() const {return key_;} diff --git a/qg/model/QgTraits.h b/qg/model/QgTraits.h index 3800dafc9..b7f90a0e9 100644 --- a/qg/model/QgTraits.h +++ b/qg/model/QgTraits.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -30,6 +30,7 @@ #include "oops/qg/ObsBiasIncrement.h" #include "oops/qg/ObsDataQG.h" #include "oops/qg/ObsDiagsQG.h" +#include "oops/qg/ObsIteratorQG.h" #include "oops/qg/ObsOperatorQG.h" #include "oops/qg/ObsOperatorTLAD.h" #include "oops/qg/ObsSpaceQG.h" @@ -67,6 +68,7 @@ struct QgObsTraits { typedef qg::ObsOperatorQG ObsOperator; typedef qg::ObsOperatorTLAD LinearObsOperator; template using ObsDataVector = qg::ObsDataQG; + typedef qg::ObsIteratorQG GeometryIterator; typedef qg::ObsBias ObsAuxControl; typedef qg::ObsBiasIncrement ObsAuxIncrement; diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index 70628c21b..8da6279c4 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -315,6 +315,12 @@ ecbuild_add_test( TARGET test_qg_obs_aux_covariance LIBS qg TEST_DEPENDS test_qg_make_obs_4d_24h ) +ecbuild_add_test( TARGET test_qg_obs_iterator + SOURCES executables/TestObsIterator.cc + ARGS "testinput/interfaces.yaml" + LIBS qg + TEST_DEPENDS test_qg_make_obs_4d_24h ) + ecbuild_add_test( TARGET test_qg_obs_localization SOURCES executables/TestObsLocalization.cc ARGS "testinput/interfaces.yaml" diff --git a/qg/test/executables/TestObsIterator.cc b/qg/test/executables/TestObsIterator.cc new file mode 100644 index 000000000..97f3576b2 --- /dev/null +++ b/qg/test/executables/TestObsIterator.cc @@ -0,0 +1,16 @@ +/* + * (C) Copyright 2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "model/QgTraits.h" +#include "oops/runs/Run.h" +#include "test/interface/ObsIterator.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + test::ObsIterator tests; + return run.execute(tests); +} diff --git a/qg/test/testinput/interfaces.yaml b/qg/test/testinput/interfaces.yaml index 6f736f053..684ea2074 100644 --- a/qg/test/testinput/interfaces.yaml +++ b/qg/test/testinput/interfaces.yaml @@ -84,6 +84,13 @@ observations: lons: [-175.5] lats: [5.623] localization reduces values: false + obs iterator test: + tolerance: 1.0e-6 + reference nlocs: 800 + lon1: -29.87208056 + lat1: 3.637673423 + lon2: 178.9865309 + lat2: 8.23197272 - obs error: covariance model: diagonal geovals: @@ -120,6 +127,13 @@ observations: lons: [-175.5] lats: [5.623] localization reduces values: false + obs iterator test: + tolerance: 1.0e-6 + reference nlocs: 400 + lon1: -154.0892581 + lat1: 80.79695176 + lon2: -74.03059012 + lat2: 78.99264774 - obs error: covariance model: diagonal geovals: @@ -149,7 +163,13 @@ observations: lons: [-175.5] lats: [5.623] localization reduces values: false - + obs iterator test: + tolerance: 1.0e-6 + reference nlocs: 400 + lon1: 112.2691295 + lat1: 66.91093417 + lon2: 131.4322691 + lat2: 82.16827599 background: date: 2009-12-31T00:00:00Z filename: Data/truth.fc.2009-12-15T00:00:00Z.P16D.nc diff --git a/src/test/interface/ObsIterator.h b/src/test/interface/ObsIterator.h index 28be12bc5..060d4b4d6 100644 --- a/src/test/interface/ObsIterator.h +++ b/src/test/interface/ObsIterator.h @@ -44,6 +44,8 @@ template void testBasic() { typedef ObsTestsFixture Test_; for (size_t jj = 0; jj < Test_::obspace().size(); ++jj) { + const double tol = Test_::config(jj).getDouble("obs iterator test.tolerance"); + // Initialize two iterators (begin and end), test equality ObsIterator_ iter1 = Test_::obspace()[jj].begin(); EXPECT(iter1 == Test_::obspace()[jj].begin()); @@ -70,14 +72,14 @@ template void testBasic() { double lon1 = Test_::config(jj).getDouble("obs iterator test.lon1"); double lat1 = Test_::config(jj).getDouble("obs iterator test.lat1"); const eckit::geometry::Point2 point1(lon1, lat1); - EXPECT_EQUAL(*iter1, point1); + EXPECT((*iter1).distance(point1) <= tol); // test that the point after begin() is the same as reference ++iter1; double lon2 = Test_::config(jj).getDouble("obs iterator test.lon2"); double lat2 = Test_::config(jj).getDouble("obs iterator test.lat2"); const eckit::geometry::Point2 point2(lon2, lat2); - EXPECT_EQUAL(*iter1, point2); + EXPECT((*iter1).distance(point2) <= tol); } } From ab026ecdde19134eae94638eb38cb5245ddd73ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Wed, 28 Apr 2021 18:57:14 +0100 Subject: [PATCH 115/142] Add functions performing the allGatherv operation on vectors (#1180) * Added functions performing the allGatherv operation on vectors. * Removed unnecessary template specialisations. --- src/oops/mpi/mpi.cc | 88 +++++++++++++++++++++++++++++++++++++ src/oops/mpi/mpi.h | 59 +++++++++++++++++++++++++ src/test/mpi/mpi.h | 63 +++++++++++++++++++++++--- src/test/testinput/mpi.yaml | 22 +++++++--- 4 files changed, 222 insertions(+), 10 deletions(-) diff --git a/src/oops/mpi/mpi.cc b/src/oops/mpi/mpi.cc index d5c46a4bd..448cc0f07 100644 --- a/src/oops/mpi/mpi.cc +++ b/src/oops/mpi/mpi.cc @@ -11,8 +11,69 @@ #include "oops/mpi/mpi.h" #include +#include #include "eckit/exception/Exceptions.h" +#include "oops/util/DateTime.h" + +namespace { + +// Helper functions used by the implementation of the specialization of allGatherv for a vectors +// of strings + +/// \brief Join strings into a single character array before MPI transfer. +/// +/// \param strings +/// Strings to join. +/// +/// \returns A pair of two vectors. The first is a concatenation of all input strings +/// (without any separating null characters). The second is the list of lengths of these strings. +std::pair, std::vector> encodeStrings( + const std::vector &strings) { + std::pair, std::vector> result; + std::vector &charArray = result.first; + std::vector &lengths = result.second; + + size_t totalLength = 0; + lengths.reserve(strings.size()); + for (const std::string &s : strings) { + lengths.push_back(s.size()); + totalLength += s.size(); + } + + charArray.reserve(totalLength); + for (const std::string &s : strings) { + charArray.insert(charArray.end(), s.begin(), s.end()); + } + + return result; +} + +/// \brief Split a character array into multiple strings. +/// +/// \param charArray +/// A character array storing a number of concatenated strings (without separating null +/// characters). +/// +/// \param lengths +/// The list of lengths of the strings stored in \p charArray. +/// +/// \returns A vector of strings extracted from \p charArray. +std::vector decodeStrings(const std::vector &charArray, + const std::vector &lengths) { + std::vector strings; + strings.reserve(lengths.size()); + + std::vector::const_iterator nextStringBegin = charArray.begin(); + for (size_t length : lengths) { + strings.emplace_back(nextStringBegin, nextStringBegin + length); + nextStringBegin += length; + } + + return strings; +} + +} // namespace namespace oops { namespace mpi { @@ -83,5 +144,32 @@ void allGather(const eckit::mpi::Comm & comm, // ------------------------------------------------------------------------------------------------ +void allGatherv(const eckit::mpi::Comm & comm, std::vector &x) { + size_t globalSize = x.size(); + comm.allReduceInPlace(globalSize, eckit::mpi::sum()); + std::vector globalX(globalSize); + oops::mpi::allGathervUsingSerialize(comm, x.begin(), x.end(), globalX.begin()); + x = std::move(globalX); +} + +// ------------------------------------------------------------------------------------------------ + +void allGatherv(const eckit::mpi::Comm & comm, std::vector &x) { + std::pair, std::vector> encodedX = encodeStrings(x); + + // Gather all character arrays + eckit::mpi::Buffer charBuffer(comm.size()); + comm.allGatherv(encodedX.first.begin(), encodedX.first.end(), charBuffer); + + // Gather all string lengths + eckit::mpi::Buffer lengthBuffer(comm.size()); + comm.allGatherv(encodedX.second.begin(), encodedX.second.end(), lengthBuffer); + + // Free memory + encodedX = {}; + + x = decodeStrings(charBuffer.buffer, lengthBuffer.buffer); +} + } // namespace mpi } // namespace oops diff --git a/src/oops/mpi/mpi.h b/src/oops/mpi/mpi.h index 0650a3e4e..2899e0e03 100644 --- a/src/oops/mpi/mpi.h +++ b/src/oops/mpi/mpi.h @@ -12,6 +12,8 @@ #include +#include +#include #include #include "eckit/exception/Exceptions.h" @@ -19,6 +21,10 @@ #include "oops/util/Timer.h" +namespace util { +class DateTime; +} // namespace util + namespace oops { namespace mpi { @@ -90,6 +96,8 @@ void gather(const eckit::mpi::Comm & comm, const std::vector & sen void allGather(const eckit::mpi::Comm & comm, const Eigen::VectorXd &, Eigen::MatrixXd &); +// ------------------------------------------------------------------------------------------------ + /// \brief A wrapper around the MPI *all gather* operation for serializable types. /// /// The *all gather* operation gathers data from all tasks and delivers the combined data to all @@ -124,5 +132,56 @@ void allGathervUsingSerialize(const eckit::mpi::Comm &comm, CIter first, CIter l // ------------------------------------------------------------------------------------------------ +// The following functions simplify the allGatherv operation on vectors (reducing it to a single +// function call). + +// ------------------------------------------------------------------------------------------------ + +/// \brief Perform the MPI *all gather* operation on a vector of "plain old data". +/// +/// This operation gathers data from all tasks and delivers the combined data to all tasks. +/// +/// \tparam T must be a type for which there exists a specialization of eckit::mpi::Data::Type. +/// +/// \param[in] comm +/// Communicator. +/// \param[inout] x +/// On input, data owned by this task that need to be delivered to all other tasks. On output, +/// combined data received from all tasks (concatenated in the order of increasing task ranks). +template +void allGatherv(const eckit::mpi::Comm & comm, std::vector &x) { + eckit::mpi::Buffer buffer(comm.size()); + comm.allGatherv(x.begin(), x.end(), buffer); + x = std::move(buffer.buffer); +} + +// ------------------------------------------------------------------------------------------------ + +/// \brief Perform the MPI *all gather* operation on a vector of DateTime objects. +/// +/// This operation gathers data from all tasks and delivers the combined data to all tasks. +/// +/// \param[in] comm +/// Communicator. +/// \param[inout] x +/// On input, data owned by this task that need to be delivered to all other tasks. On output, +/// combined data received from all tasks (concatenated in the order of increasing task ranks). +void allGatherv(const eckit::mpi::Comm & comm, std::vector &x); + +// ------------------------------------------------------------------------------------------------ + +/// \brief Perform the MPI *all gather* operation on a vector of DateTime objects. +/// +/// This operation gathers data from all tasks and delivers the combined data to all tasks. +/// +/// \param[in] comm +/// Communicator. +/// \param[inout] x +/// On input, data owned by this task that need to be delivered to all other tasks. On output, +/// combined data received from all tasks (concatenated in the order of increasing task ranks). +void allGatherv(const eckit::mpi::Comm & comm, std::vector &x); + +// ------------------------------------------------------------------------------------------------ + } // namespace mpi } // namespace oops diff --git a/src/test/mpi/mpi.h b/src/test/mpi/mpi.h index ed9e73c8d..db2203491 100644 --- a/src/test/mpi/mpi.h +++ b/src/test/mpi/mpi.h @@ -34,12 +34,53 @@ namespace eckit namespace test { +// ----------------------------------------------------------------------------------------------- class TestParameters : public oops::Parameters { OOPS_CONCRETE_PARAMETERS(TestParameters, Parameters) public: - oops::RequiredParameter> values{"values", this}; + oops::RequiredParameter> datetime{"datetime", this}; + oops::RequiredParameter> int_{"int", this}; + oops::RequiredParameter> string{"string", this}; }; +// ----------------------------------------------------------------------------------------------- +template +const std::vector & getTestData(const TestParameters ¶ms); + +template <> +const std::vector & getTestData(const TestParameters ¶ms) { + return params.datetime; +} + +template <> +const std::vector & getTestData(const TestParameters ¶ms) { + return params.int_; +} + +template <> +const std::vector & getTestData(const TestParameters ¶ms) { + return params.string; +} + +// ----------------------------------------------------------------------------------------------- +template +void testAllGatherv() { + const eckit::Configuration &conf = TestEnvironment::config(); + const eckit::mpi::Comm &comm = oops::mpi::world(); + + TestParameters localParams; + const size_t rank = comm.rank(); + localParams.deserialize(conf.getSubConfiguration("local" + std::to_string(rank))); + std::vector values = getTestData(localParams); + + TestParameters globalParams; + globalParams.deserialize(conf.getSubConfiguration("global")); + const std::vector &expectedResult = getTestData(globalParams); + + oops::mpi::allGatherv(comm, values); + EXPECT_EQUAL(values, expectedResult); +} + // ----------------------------------------------------------------------------------------------- CASE("mpi/mpi/defaultCommunicators") { const eckit::mpi::Comm & world = oops::mpi::world(); @@ -58,11 +99,11 @@ CASE("mpi/mpi/allGathervUsingSerialize") { TestParameters localParams; const size_t rank = comm.rank(); localParams.deserialize(conf.getSubConfiguration("local" + std::to_string(rank))); - const std::vector &localValues = localParams.values; + const std::vector &localValues = localParams.datetime; TestParameters globalParams; globalParams.deserialize(conf.getSubConfiguration("global")); - const std::vector &expectedGlobalValues = globalParams.values; + const std::vector &expectedGlobalValues = globalParams.datetime; size_t numGlobalValues; comm.allReduce(localValues.size(), numGlobalValues, eckit::mpi::Operation::SUM); @@ -73,6 +114,18 @@ CASE("mpi/mpi/allGathervUsingSerialize") { EXPECT_EQUAL(globalValues, expectedGlobalValues); } // ----------------------------------------------------------------------------------------------- +CASE("mpi/mpi/allGathervInt") { + testAllGatherv(); +} +// ----------------------------------------------------------------------------------------------- +CASE("mpi/mpi/allGathervDateTime") { + testAllGatherv(); +} +// ----------------------------------------------------------------------------------------------- +CASE("mpi/mpi/allGathervInt") { + testAllGatherv(); +} +// ----------------------------------------------------------------------------------------------- CASE("mpi/mpi/SendReceive") { const eckit::Configuration &conf = TestEnvironment::config(); const eckit::mpi::Comm &comm = oops::mpi::world(); @@ -103,11 +156,11 @@ CASE("mpi/mpi/gatherSerializable") { TestParameters localParams; const size_t rank = comm.rank(); localParams.deserialize(conf.getSubConfiguration("local" + std::to_string(rank))); - const std::vector &localValues = localParams.values; + const std::vector &localValues = localParams.datetime; TestParameters globalParams; globalParams.deserialize(conf.getSubConfiguration("global")); - const std::vector &expectedGlobalValues = globalParams.values; + const std::vector &expectedGlobalValues = globalParams.datetime; size_t numGlobalValues; comm.allReduce(localValues.size(), numGlobalValues, eckit::mpi::Operation::SUM); diff --git a/src/test/testinput/mpi.yaml b/src/test/testinput/mpi.yaml index 1b228549f..da1a17601 100644 --- a/src/test/testinput/mpi.yaml +++ b/src/test/testinput/mpi.yaml @@ -1,15 +1,27 @@ local0: - values: ['2018-04-14T23:41:28Z', '2018-04-14T22:07:30Z', '2018-04-14T23:36:33Z'] + datetime: ['2018-04-14T23:41:28Z', '2018-04-14T22:07:30Z', '2018-04-14T23:36:33Z'] + int: [10, 20, 30] + string: ["border collie", "chow-chow", "dachshound"] local1: - values: [] + datetime: [] + int: [] + string: [] local2: - values: ['2018-04-14T23:31:51Z', '2018-04-14T23:21:28Z', '2018-04-15T00:56:32Z', '2018-04-14T23:11:11Z'] + datetime: ['2018-04-14T23:31:51Z', '2018-04-14T23:21:28Z', '2018-04-15T00:56:32Z', '2018-04-14T23:11:11Z'] + int: [1, 2, 3, 4] + string: ["great Dane", "Irish setter", "German shepherd", "Dalmatian"] local3: - values: ['2018-04-15T02:46:51Z', '2018-04-15T02:30:56Z'] + datetime: ['2018-04-15T02:46:51Z', '2018-04-15T02:30:56Z'] + int: [100, 200] + string: ["1234", "!?.%"] global: - values: ['2018-04-14T23:41:28Z', '2018-04-14T22:07:30Z', '2018-04-14T23:36:33Z', + datetime: ['2018-04-14T23:41:28Z', '2018-04-14T22:07:30Z', '2018-04-14T23:36:33Z', '2018-04-14T23:31:51Z', '2018-04-14T23:21:28Z', '2018-04-15T00:56:32Z', '2018-04-14T23:11:11Z', '2018-04-15T02:46:51Z', '2018-04-15T02:30:56Z'] + int: [10, 20, 30, 1, 2, 3, 4, 100, 200] + string: ["border collie", "chow-chow", "dachshound", + "great Dane", "Irish setter", "German shepherd", "Dalmatian", + "1234", "!?.%"] send0: '2018-04-14T23:41:28Z' send1: '2019-05-15T02:46:51Z' From 37879a3c42a25ef19d56f345cc54acab92992e71 Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Thu, 29 Apr 2021 12:40:35 -0600 Subject: [PATCH 116/142] update compare tests in l95 and qg (#1178) * update yaml files * update reference files * update CMakelists.txt * clean up * updating qg * clean up * remove oops_add_test --- cmake/oops_add_test.cmake | 40 - l95/test/CMakeLists.txt | 712 ++++++++---------- l95/test/testinput/3dfgat.yaml | 3 + l95/test/testinput/3dvar.yaml | 3 + l95/test/testinput/3dvar_noobs.yaml | 3 + l95/test/testinput/3dvar_qc.yaml | 3 + l95/test/testinput/3dvar_qc_iterations.yaml | 3 + l95/test/testinput/3dvar_qc_obserr.yaml | 3 + l95/test/testinput/4densvar.hybrid.yaml | 3 + l95/test/testinput/4densvar.yaml | 3 + l95/test/testinput/4dforcing.yaml | 3 + l95/test/testinput/4dsaddlepoint.yaml | 3 + l95/test/testinput/4dvar.allbiases.yaml | 3 + l95/test/testinput/4dvar.alpha.yaml | 3 + l95/test/testinput/4dvar.drgmresr.yaml | 3 + l95/test/testinput/4dvar.dripcg.yaml | 3 + l95/test/testinput/4dvar.dripcgqn.yaml | 3 + l95/test/testinput/4dvar.drpcg.yaml | 3 + l95/test/testinput/4dvar.drpcgqn.yaml | 3 + l95/test/testinput/4dvar.drplanclmp.yaml | 3 + l95/test/testinput/4dvar.drplanczos.yaml | 3 + l95/test/testinput/4dvar.fgmres.yaml | 3 + l95/test/testinput/4dvar.gmresr.yaml | 3 + l95/test/testinput/4dvar.hybrid.yaml | 3 + l95/test/testinput/4dvar.ipcg.yaml | 3 + l95/test/testinput/4dvar.lbgmresr.yaml | 3 + l95/test/testinput/4dvar.minres.yaml | 3 + l95/test/testinput/4dvar.modbias.yaml | 3 + l95/test/testinput/4dvar.pcg.yaml | 3 + l95/test/testinput/4dvar.planczos.yaml | 3 + l95/test/testinput/4dvar.rpcg.yaml | 3 + l95/test/testinput/4dvar.rplanczos.yaml | 3 + l95/test/testinput/addincrement.yaml | 3 + l95/test/testinput/addincrement_scaled.yaml | 3 + l95/test/testinput/diffstates.yaml | 3 + l95/test/testinput/eda.3dvar.block.yaml | 3 + l95/test/testinput/forecast.yaml | 3 + .../testinput/forecast_identitymodel.yaml | 3 + l95/test/testinput/forecast_pseudomodel.yaml | 3 + l95/test/testinput/genenspert.yaml | 3 + l95/test/testinput/getkf.yaml | 6 +- l95/test/testinput/getkf_offline_hofx.yaml | 6 +- l95/test/testinput/hofx.yaml | 3 + l95/test/testinput/hofx3d.yaml | 3 + l95/test/testinput/hofx3d_for_getkf.yaml | 6 +- l95/test/testinput/letkf.yaml | 2 + l95/test/testinput/letkfGSI.yaml | 5 + l95/test/testinput/letkf_noobs.yaml | 3 + l95/test/testinput/letkf_qc.yaml | 3 + l95/test/testinput/makeobs3d.yaml | 3 + l95/test/testinput/makeobs4d.yaml | 3 + l95/test/testinput/makeobs4d12h.yaml | 3 + l95/test/testinput/makeobsbias.yaml | 3 + l95/test/testinput/makeobspert.yaml | 3 + .../testinput/simplifiedl95_DRGMRESR.yaml | 4 + l95/test/testinput/simplifiedl95_DRIPCG.yaml | 4 + l95/test/testinput/simplifiedl95_DRPCG.yaml | 4 + l95/test/testinput/simplifiedl95_DRPFOM.yaml | 4 + .../testinput/simplifiedl95_DRPLanczos.yaml | 4 + l95/test/testinput/simplifiedl95_FGMRES.yaml | 4 + l95/test/testinput/simplifiedl95_GMRESR.yaml | 4 + l95/test/testinput/simplifiedl95_IPCG.yaml | 4 + .../testinput/simplifiedl95_LBGMRESR.yaml | 4 + l95/test/testinput/simplifiedl95_MINRES.yaml | 4 + l95/test/testinput/simplifiedl95_PCG.yaml | 4 + .../testinput/simplifiedl95_PLanczos.yaml | 4 + l95/test/testinput/simplifiedl95_RPCG.yaml | 4 + .../testinput/simplifiedl95_RPLanczos.yaml | 4 + l95/test/testinput/truth.yaml | 3 + l95/test/testoutput/3dfgat.test | 20 +- l95/test/testoutput/3dvar.test | 34 +- l95/test/testoutput/3dvar_qc.test | 34 +- l95/test/testoutput/3dvar_qc_iterations.test | 34 +- l95/test/testoutput/3dvar_qc_obserr.test | 34 +- l95/test/testoutput/4densvar.test | 98 +-- l95/test/testoutput/4dsaddlepoint.test | 42 +- l95/test/testoutput/4dvar.allbiases.test | 48 +- l95/test/testoutput/4dvar.alpha.test | 34 +- l95/test/testoutput/4dvar.drgmresr.test | 40 +- l95/test/testoutput/4dvar.dripcg.test | 40 +- l95/test/testoutput/4dvar.dripcgqn.test | 40 +- l95/test/testoutput/4dvar.drpcg.test | 40 +- l95/test/testoutput/4dvar.drpcgqn.test | 40 +- l95/test/testoutput/4dvar.drplanclmp.test | 40 +- l95/test/testoutput/4dvar.drplanczos.test | 40 +- l95/test/testoutput/4dvar.fgmres.test | 40 +- l95/test/testoutput/4dvar.gmresr.test | 40 +- l95/test/testoutput/4dvar.hybrid.test | 34 +- l95/test/testoutput/4dvar.ipcg.test | 40 +- l95/test/testoutput/4dvar.lbgmresr.test | 36 +- l95/test/testoutput/4dvar.minres.test | 40 +- l95/test/testoutput/4dvar.modbias.test | 44 +- l95/test/testoutput/4dvar.obsbias.test | 44 +- l95/test/testoutput/4dvar.pcg.test | 40 +- l95/test/testoutput/4dvar.planczos.test | 40 +- l95/test/testoutput/4dvar.rpcg.test | 40 +- l95/test/testoutput/4dvar.rplanczos.test | 40 +- l95/test/testoutput/addincrement.test | 18 +- l95/test/testoutput/addincrement_scaled.test | 24 +- l95/test/testoutput/diffstates.test | 18 +- l95/test/testoutput/eda_3dvar_block.test | 114 +-- l95/test/testoutput/forecast.test | 12 +- .../testoutput/forecast_identitymodel.test | 12 +- l95/test/testoutput/forecast_pseudomodel.test | 12 +- l95/test/testoutput/genenspert.test | 66 +- l95/test/testoutput/getkf.test | 173 +++-- l95/test/testoutput/getkf_offline_hofx.test | 109 +-- l95/test/testoutput/hofx.test | 18 +- l95/test/testoutput/hofx3d.test | 12 +- l95/test/testoutput/hofx3d_for_getkf.test | 95 ++- l95/test/testoutput/letkf.test | 141 ++-- l95/test/testoutput/letkf_gsi.test | 123 +-- l95/test/testoutput/letkf_noobs.test | 123 +-- l95/test/testoutput/letkf_qc.test | 123 +-- l95/test/testoutput/makeobs3d.test | 18 +- l95/test/testoutput/makeobs4d.test | 18 +- l95/test/testoutput/makeobs4d12h.test | 18 +- l95/test/testoutput/makeobsbias.test | 18 +- l95/test/testoutput/makeobspert.test | 24 +- .../testoutput/simplifiedl95_DRGMRESR.test | 20 +- l95/test/testoutput/simplifiedl95_DRIPCG.test | 20 +- l95/test/testoutput/simplifiedl95_DRPFOM.test | 20 +- .../testoutput/simplifiedl95_DRPLanczos.test | 20 +- l95/test/testoutput/simplifiedl95_FGMRES.test | 20 +- l95/test/testoutput/simplifiedl95_GMRESR.test | 20 +- l95/test/testoutput/simplifiedl95_IPCG.test | 20 +- .../testoutput/simplifiedl95_LBGMRESR.test | 18 +- l95/test/testoutput/simplifiedl95_MINRES.test | 20 +- l95/test/testoutput/simplifiedl95_PCG.test | 20 +- .../testoutput/simplifiedl95_PLanczos.test | 20 +- l95/test/testoutput/simplifiedl95_RPCG.test | 20 +- .../testoutput/simplifiedl95_RPLanczos.test | 20 +- l95/test/testoutput/truth.test | 12 +- qg/test/CMakeLists.txt | 610 +++++++-------- qg/test/testinput/3densvar.yaml | 9 +- qg/test/testinput/3dfgat.yaml | 3 + qg/test/testinput/3dvar.yaml | 3 + qg/test/testinput/3dvar_change_var.yaml | 3 + qg/test/testinput/3dvar_hybrid.yaml | 3 + .../3dvar_hybrid_wo_jb_evaluation.yaml | 3 + qg/test/testinput/4densvar.yaml | 3 + qg/test/testinput/4densvar_hybrid.yaml | 3 + qg/test/testinput/4dvar_dripcg.yaml | 9 +- qg/test/testinput/4dvar_drpcg_lmp.yaml | 3 + qg/test/testinput/4dvar_drpfom.yaml | 9 +- qg/test/testinput/4dvar_drplanczos.yaml | 3 + .../testinput/4dvar_drplanczos_hybrid.yaml | 9 +- qg/test/testinput/4dvar_forcing.yaml | 9 +- qg/test/testinput/4dvar_ipcg.yaml | 3 + qg/test/testinput/4dvar_obs_biased.yaml | 3 + qg/test/testinput/4dvar_rpcg.yaml | 3 + qg/test/testinput/4dvar_saddlepoint.yaml | 10 +- qg/test/testinput/addincrement.yaml | 3 + qg/test/testinput/addincrement_scaled.yaml | 3 + qg/test/testinput/analytic_forecast.yaml | 3 + qg/test/testinput/convertincrement.yaml | 3 + qg/test/testinput/convertstate.yaml | 3 + qg/test/testinput/dfi.yaml | 3 + qg/test/testinput/diffstates.yaml | 3 + qg/test/testinput/dirac_cov.yaml | 3 + qg/test/testinput/dirac_hyb_field.yaml | 5 +- qg/test/testinput/dirac_hyb_value.yaml | 5 +- qg/test/testinput/dirac_loc_3d.yaml | 3 + qg/test/testinput/dirac_loc_4d.yaml | 3 + qg/test/testinput/dirac_no_loc.yaml | 3 + qg/test/testinput/eda_3dvar_block.yaml | 3 + qg/test/testinput/eda_4dvar.yaml | 3 + qg/test/testinput/ens_forecast.yaml | 3 + qg/test/testinput/ens_hofx.yaml | 3 + qg/test/testinput/ens_recenter.yaml | 3 + qg/test/testinput/ens_variance.yaml | 3 + .../ens_variance_inflation_field.yaml | 3 + .../ens_variance_inflation_value.yaml | 3 + qg/test/testinput/forecast.yaml | 3 + qg/test/testinput/gen_ens_pert_B.yaml | 3 + qg/test/testinput/hofx.yaml | 3 + qg/test/testinput/hofx3d.yaml | 3 + qg/test/testinput/letkf.yaml | 3 + qg/test/testinput/make_obs_3d.yaml | 3 + qg/test/testinput/make_obs_4d_12h.yaml | 3 + qg/test/testinput/make_obs_4d_24h.yaml | 3 + qg/test/testinput/make_obs_4d_biased.yaml | 3 + qg/test/testinput/rtpp.yaml | 3 + qg/test/testinput/static_b_init.yaml | 3 + qg/test/testinput/truth.yaml | 3 + qg/test/testinput/uniform_field_hybrid.yaml | 3 + .../testinput/uniform_field_inflation.yaml | 3 + qg/test/testoutput/3densvar.test | 58 +- qg/test/testoutput/3dfgat.test | 34 +- qg/test/testoutput/3dvar.test | 58 +- qg/test/testoutput/3dvar_change_var.test | 58 +- qg/test/testoutput/3dvar_hybrid.test | 58 +- .../3dvar_hybrid_wo_jb_evaluation.test | 52 +- qg/test/testoutput/4densvar.test | 178 ++--- qg/test/testoutput/4densvar_hybrid.test | 178 ++--- qg/test/testoutput/4dvar_dripcg.test | 64 +- qg/test/testoutput/4dvar_drpcg_lmp.test | 64 +- qg/test/testoutput/4dvar_drpfom.test | 64 +- qg/test/testoutput/4dvar_drplanczos.test | 64 +- .../testoutput/4dvar_drplanczos_hybrid.test | 64 +- qg/test/testoutput/4dvar_forcing.test | 0 qg/test/testoutput/4dvar_ipcg.test | 64 +- qg/test/testoutput/4dvar_obs_biased.test | 72 +- qg/test/testoutput/4dvar_rpcg.test | 58 +- qg/test/testoutput/4dvar_saddlepoint.test | 90 +-- qg/test/testoutput/addincrement.test | 32 +- qg/test/testoutput/addincrement_scaled.test | 40 +- qg/test/testoutput/analytic_forecast.test | 24 +- qg/test/testoutput/convertincrement.test | 60 +- qg/test/testoutput/convertstate.test | 276 +++---- qg/test/testoutput/dfi.test | 36 +- qg/test/testoutput/diffstates.test | 32 +- qg/test/testoutput/dirac_cov.test | 24 +- qg/test/testoutput/dirac_hyb_field.test | 24 +- qg/test/testoutput/dirac_hyb_value.test | 32 +- qg/test/testoutput/dirac_loc_3d.test | 32 +- qg/test/testoutput/dirac_loc_4d.test | 320 ++++---- qg/test/testoutput/dirac_no_loc.test | 24 +- qg/test/testoutput/eda_3dvar_block.test | 106 +-- qg/test/testoutput/eda_4dvar.test | 68 +- qg/test/testoutput/ens_hofx.test | 34 +- qg/test/testoutput/ens_recenter.test | 133 ++-- qg/test/testoutput/ens_variance.test | 9 +- .../ens_variance_inflation_field.test | 9 +- .../ens_variance_inflation_value.test | 9 +- qg/test/testoutput/forecast.test | 24 +- qg/test/testoutput/gen_ens_pert_B.test | 132 ++-- qg/test/testoutput/hofx.test | 34 +- qg/test/testoutput/hofx3d.test | 22 +- qg/test/testoutput/letkf.test | 375 ++++----- qg/test/testoutput/make_obs_3d.test | 34 +- qg/test/testoutput/make_obs_4d_12h.test | 34 +- qg/test/testoutput/make_obs_4d_24h.test | 34 +- qg/test/testoutput/make_obs_4d_biased.test | 34 +- qg/test/testoutput/rtpp.test | 48 +- qg/test/testoutput/static_b_init.test | 7 +- qg/test/testoutput/truth.test | 24 +- qg/test/testoutput/uniform_field_hybrid.test | 24 +- .../testoutput/uniform_field_inflation.test | 24 +- 239 files changed, 4163 insertions(+), 3771 deletions(-) delete mode 100644 cmake/oops_add_test.cmake delete mode 100644 qg/test/testoutput/4dvar_forcing.test diff --git a/cmake/oops_add_test.cmake b/cmake/oops_add_test.cmake deleted file mode 100644 index 1805ddfcc..000000000 --- a/cmake/oops_add_test.cmake +++ /dev/null @@ -1,40 +0,0 @@ -function(oops_add_test) - set( single_value_args TESTNAME MODELNAME YAMLNAME EXENAME MPI OMP CTOL IDIF RUN_FILE REF_FILE ) - set( multi_value_args DEPENDS TEST_DEPENDS) - cmake_parse_arguments( _p "${options}" "${single_value_args}" "${multi_value_args}" ${_FIRST_ARG} ${ARGN} ) - - if (_p_MPI) - set(MPI_cmd "${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${_p_MPI}") - endif() - - if (NOT _p_RUN_FILE) - set( _p_RUN_FILE ${_p_TESTNAME}.log.out) - endif() - - if (NOT _p_REF_FILE) - set( _p_REF_FILE ${_p_TESTNAME}.test) - endif() - - if (NOT _p_CTOL) - set( _p_CTOL 0.0) # Max relative diff - endif() - - if (NOT _p_IDIF) - set( _p_IDIF 0) # Max diff - endif() - - ecbuild_add_test( TARGET test_${_p_MODELNAME}_${_p_TESTNAME} - TYPE SCRIPT - COMMAND ${CMAKE_BINARY_DIR}/bin/${PROJECT_NAME}_test_wrapper.sh - ARGS ${CMAKE_BINARY_DIR}/bin/${_p_EXENAME} - ${_p_YAMLNAME} - ${CMAKE_BINARY_DIR}/bin/${PROJECT_NAME}_compare.py - ${_p_RUN_FILE} - ${_p_REF_FILE} - ${_p_CTOL} - ${_p_IDIF} - ${MPI_cmd} - OMP ${_p_OMP} - DEPENDS ${_p_DEPENDS} - TEST_DEPENDS ${_p_TEST_DEPENDS} ) -endfunction() diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index 8947c856e..ebc94a0aa 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -1,4 +1,3 @@ -include ( oops_add_test ) list( APPEND l95_test_input testinput/3dvar.yaml @@ -214,16 +213,14 @@ ecbuild_add_resources( TARGET l95_test_scripts # truth and makeobs4d are required for interface tests ##################################################################### -oops_add_test( TESTNAME truth - MODELNAME l95 - YAMLNAME testinput/truth.yaml - EXENAME l95_forecast.x ) +ecbuild_add_test( TARGET test_l95_truth + COMMAND l95_forecast.x + ARGS testinput/truth.yaml ) -oops_add_test( TESTNAME makeobs4d - MODELNAME l95 - YAMLNAME testinput/makeobs4d.yaml - EXENAME l95_hofx.x - TEST_DEPENDS test_l95_truth ) +ecbuild_add_test( TARGET test_l95_makeobs4d + COMMAND l95_hofx.x + ARGS testinput/makeobs4d.yaml + TEST_DEPENDS test_l95_truth ) ##################################################################### @@ -392,80 +389,68 @@ ecbuild_add_test( TARGET test_l95_localization LIBS lorenz95 TEST_DEPENDS test_l95_truth ) - ##################################################################### # forecast-related tests ##################################################################### -oops_add_test( TESTNAME forecast - MODELNAME l95 - YAMLNAME testinput/forecast.yaml - EXENAME l95_forecast.x ) +ecbuild_add_test( TARGET test_l95_forecast + COMMAND l95_forecast.x + ARGS testinput/forecast.yaml ) -oops_add_test( TESTNAME forecast_pseudomodel - MODELNAME l95 - YAMLNAME testinput/forecast_pseudomodel.yaml - EXENAME l95_forecast.x ) - -oops_add_test( TESTNAME forecast_identitymodel - MODELNAME l95 - YAMLNAME testinput/forecast_identitymodel.yaml - EXENAME l95_forecast.x ) +ecbuild_add_test( TARGET test_l95_forecast_pseudomodel + COMMAND l95_forecast.x + ARGS testinput/forecast_pseudomodel.yaml ) +ecbuild_add_test( TARGET test_l95_forecast_identitymodel + COMMAND l95_forecast.x + ARGS testinput/forecast_identitymodel.yaml ) ##################################################################### # obs-related tests ##################################################################### -oops_add_test( TESTNAME makeobs3d - MODELNAME l95 - YAMLNAME testinput/makeobs3d.yaml - EXENAME l95_hofx.x - TEST_DEPENDS test_l95_truth ) - -oops_add_test( TESTNAME makeobsbias - MODELNAME l95 - YAMLNAME testinput/makeobsbias.yaml - EXENAME l95_hofx.x - TEST_DEPENDS test_l95_truth ) - -oops_add_test( TESTNAME makeobs4d12h - MODELNAME l95 - YAMLNAME testinput/makeobs4d12h.yaml - EXENAME l95_hofx.x - TEST_DEPENDS test_l95_truth ) - -oops_add_test( TESTNAME makeobspert - MODELNAME l95 - YAMLNAME testinput/makeobspert.yaml - EXENAME l95_hofx.x - TEST_DEPENDS test_l95_truth ) - -oops_add_test( TESTNAME hofx - MODELNAME l95 - YAMLNAME testinput/hofx.yaml - EXENAME l95_hofx.x - TEST_DEPENDS test_l95_truth test_l95_makeobs4d ) - -oops_add_test( TESTNAME hofx3d - MODELNAME l95 - YAMLNAME testinput/hofx3d.yaml - EXENAME l95_hofx3d.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) +ecbuild_add_test( TARGET test_l95_makeobs3d + COMMAND l95_hofx.x + ARGS testinput/makeobs3d.yaml + TEST_DEPENDS test_l95_truth ) + +ecbuild_add_test( TARGET test_l95_makeobsbias + COMMAND l95_hofx.x + ARGS testinput/makeobsbias.yaml + TEST_DEPENDS test_l95_truth ) + +ecbuild_add_test( TARGET test_l95_makeobs4d12h + COMMAND l95_hofx.x + ARGS testinput/makeobs4d12h.yaml + TEST_DEPENDS test_l95_truth ) + +ecbuild_add_test( TARGET test_l95_makeobspert + COMMAND l95_hofx.x + ARGS testinput/makeobspert.yaml + TEST_DEPENDS test_l95_truth ) + +ecbuild_add_test( TARGET test_l95_hofx + COMMAND l95_hofx.x + ARGS testinput/hofx.yaml + TEST_DEPENDS test_l95_truth test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_hofx3d + COMMAND l95_hofx3d.x + ARGS testinput/hofx3d.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) ##################################################################### # ensemble-related tests ##################################################################### -oops_add_test( TESTNAME genenspert - MODELNAME l95 - YAMLNAME testinput/genenspert.yaml - EXENAME l95_genpert.x ) +ecbuild_add_test( TARGET test_l95_genenspert + COMMAND l95_genpert.x + ARGS testinput/genenspert.yaml ) ecbuild_add_test( TARGET test_l95_enshofx MPI 4 - COMMAND ${CMAKE_BINARY_DIR}/bin/l95_enshofx.x - ARGS testinput/enshofx.yaml testoutput/enshofx.log.out + COMMAND l95_enshofx.x + ARGS testinput/enshofx.yaml DEPENDS l95_enshofx.x TEST_DEPENDS test_l95_genenspert test_l95_makeobs4d ) @@ -474,163 +459,137 @@ ecbuild_add_test( TARGET test_l95_enshofx # 3d variational tests ##################################################################### -oops_add_test( TESTNAME 3dvar - MODELNAME l95 - YAMLNAME testinput/3dvar.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) - -#oops_add_test( TESTNAME 3dvar_noobs -# MODELNAME l95 -# YAMLNAME testinput/3dvar_noobs.yaml -# EXENAME l95_4dvar.x -# TEST_DEPENDS test_l95_forecast ) - -oops_add_test( TESTNAME 3dvar_qc - MODELNAME l95 - YAMLNAME testinput/3dvar_qc.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) - -oops_add_test( TESTNAME 3dvar_qc_obserr - MODELNAME l95 - YAMLNAME testinput/3dvar_qc_obserr.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) - -oops_add_test( TESTNAME 3dvar_qc_iterations - MODELNAME l95 - YAMLNAME testinput/3dvar_qc_iterations.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) - -oops_add_test( TESTNAME 3dfgat - MODELNAME l95 - YAMLNAME testinput/3dfgat.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) +ecbuild_add_test( TARGET test_l95_3dvar + COMMAND l95_4dvar.x + ARGS testinput/3dvar.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) + +#ecbuild_add_test( TESTNAME 3dvar_noobs +# COMMAND l95_4dvar.x +# ARGS testinput/3dvar_noobs.yaml +# TEST_DEPENDS test_l95_forecast ) + +ecbuild_add_test( TARGET test_l95_3dvar_qc + COMMAND l95_4dvar.x + ARGS testinput/3dvar_qc.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) + +ecbuild_add_test( TARGET test_l95_3dvar_qc_obserr + COMMAND l95_4dvar.x + ARGS testinput/3dvar_qc_obserr.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) + +ecbuild_add_test( TARGET test_l95_3dvar_qc_iterations + COMMAND l95_4dvar.x + ARGS testinput/3dvar_qc_iterations.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) + +ecbuild_add_test( TARGET test_l95_3dfgat + COMMAND l95_4dvar.x + ARGS testinput/3dfgat.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) ##################################################################### # 4d variational tests ##################################################################### -oops_add_test( TESTNAME 4dvar.drgmresr - MODELNAME l95 - YAMLNAME testinput/4dvar.drgmresr.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.dripcg - MODELNAME l95 - YAMLNAME testinput/4dvar.dripcg.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.dripcgqn - MODELNAME l95 - YAMLNAME testinput/4dvar.dripcgqn.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.drpcg - MODELNAME l95 - YAMLNAME testinput/4dvar.drpcg.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.drpcgqn - MODELNAME l95 - YAMLNAME testinput/4dvar.drpcgqn.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.drplanczos - MODELNAME l95 - YAMLNAME testinput/4dvar.drplanczos.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.drplanclmp - MODELNAME l95 - YAMLNAME testinput/4dvar.drplanclmp.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.fgmres - MODELNAME l95 - YAMLNAME testinput/4dvar.fgmres.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.gmresr - MODELNAME l95 - YAMLNAME testinput/4dvar.gmresr.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.ipcg - MODELNAME l95 - YAMLNAME testinput/4dvar.ipcg.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.lbgmresr - MODELNAME l95 - YAMLNAME testinput/4dvar.lbgmresr.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.pcg - MODELNAME l95 - YAMLNAME testinput/4dvar.pcg.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.planczos - MODELNAME l95 - YAMLNAME testinput/4dvar.planczos.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.rpcg - MODELNAME l95 - YAMLNAME testinput/4dvar.rpcg.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.rplanczos - MODELNAME l95 - YAMLNAME testinput/4dvar.rplanczos.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.minres - MODELNAME l95 - YAMLNAME testinput/4dvar.minres.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dvar.modbias - MODELNAME l95 - YAMLNAME testinput/4dvar.modbias.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -#oops_add_test( TESTNAME 4dforcing -# MODELNAME l95 -# YAMLNAME testinput/4dforcing.yaml -# EXENAME l95_4dvar.x -# TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -oops_add_test( TESTNAME 4dsaddlepoint - MPI 2 - MODELNAME l95 - YAMLNAME testinput/4dsaddlepoint.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) - -# OOPS TestReference unit test example +ecbuild_add_test( TARGET test_l95_4dvar.drgmresr + COMMAND l95_4dvar.x + ARGS testinput/4dvar.drgmresr.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.dripcg + COMMAND l95_4dvar.x + ARGS testinput/4dvar.dripcg.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.dripcgqn + COMMAND l95_4dvar.x + ARGS testinput/4dvar.dripcgqn.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.drpcg + COMMAND l95_4dvar.x + ARGS testinput/4dvar.drpcg.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.drpcgqn + COMMAND l95_4dvar.x + ARGS testinput/4dvar.drpcgqn.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.drplanczos + COMMAND l95_4dvar.x + ARGS testinput/4dvar.drplanczos.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.drplanclmp + COMMAND l95_4dvar.x + ARGS testinput/4dvar.drplanclmp.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.fgmres + COMMAND l95_4dvar.x + ARGS testinput/4dvar.fgmres.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.gmresr + COMMAND l95_4dvar.x + ARGS testinput/4dvar.gmresr.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.ipcg + COMMAND l95_4dvar.x + ARGS testinput/4dvar.ipcg.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.lbgmresr + COMMAND l95_4dvar.x + ARGS testinput/4dvar.lbgmresr.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.pcg + COMMAND l95_4dvar.x + ARGS testinput/4dvar.pcg.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.planczos + COMMAND l95_4dvar.x + ARGS testinput/4dvar.planczos.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.rpcg + COMMAND l95_4dvar.x + ARGS testinput/4dvar.rpcg.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.rplanczos + COMMAND l95_4dvar.x + ARGS testinput/4dvar.rplanczos.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.minres + COMMAND l95_4dvar.x + ARGS testinput/4dvar.minres.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dvar.modbias + COMMAND l95_4dvar.x + ARGS testinput/4dvar.modbias.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +#ecbuild_add_test( TARGET test_l95_4dforcing +# COMMAND l95_4dvar.x +# ARGS testinput/4dforcing.yaml +# TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + +ecbuild_add_test( TARGET test_l95_4dsaddlepoint + COMMAND l95_4dvar.x + MPI 2 + ARGS testinput/4dsaddlepoint.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) + ecbuild_add_test( TARGET test_l95_4dvar.obsbias COMMAND l95_4dvar.x ARGS testinput/4dvar.obsbias.yaml @@ -638,40 +597,33 @@ ecbuild_add_test( TARGET test_l95_4dvar.obsbias #-------------------------------------------------------------------- -oops_add_test( TESTNAME 4dvar.allbiases - MODELNAME l95 - YAMLNAME testinput/4dvar.allbiases.yaml - EXENAME l95_4dvar.x - COMPARE - TEST_DEPENDS test_l95_forecast test_l95_makeobsbias ) +ecbuild_add_test( TARGET test_l95_4dvar.allbiases + COMMAND l95_4dvar.x + ARGS testinput/4dvar.allbiases.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobsbias ) #-------------------------------------------------------------------- -oops_add_test( TESTNAME 4dvar.alpha - MODELNAME l95 - YAMLNAME testinput/4dvar.alpha.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d test_l95_genenspert ) - -oops_add_test( TESTNAME 4dvar.hybrid - MODELNAME l95 - YAMLNAME testinput/4dvar.hybrid.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d test_l95_genenspert ) - -oops_add_test( TESTNAME 4densvar - MPI 9 - MODELNAME l95 - YAMLNAME testinput/4densvar.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d12h test_l95_genenspert ) - -oops_add_test( TESTNAME 4densvar.hybrid - MPI 9 - MODELNAME l95 - YAMLNAME testinput/4densvar.hybrid.yaml - EXENAME l95_4dvar.x - TEST_DEPENDS test_l95_forecast test_l95_makeobs4d test_l95_genenspert ) +ecbuild_add_test( TARGET test_l95_4dvar.alpha + COMMAND l95_4dvar.x + ARGS testinput/4dvar.alpha.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d test_l95_genenspert ) + +ecbuild_add_test( TARGET test_l95_4dvar.hybrid + COMMAND l95_4dvar.x + ARGS testinput/4dvar.hybrid.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d test_l95_genenspert ) + +ecbuild_add_test( TARGET test_l95_4densvar + COMMAND l95_4dvar.x + MPI 9 + ARGS testinput/4densvar.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d12h test_l95_genenspert ) +ecbuild_add_test( TARGET test_l95_4densvar.hybrid + COMMAND l95_4dvar.x + MPI 9 + ARGS testinput/4densvar.hybrid.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs4d test_l95_genenspert ) ##################################################################### # EDA tests @@ -679,106 +631,89 @@ oops_add_test( TESTNAME 4densvar.hybrid ecbuild_add_test( TARGET test_l95_eda_3dfgat MPI 4 - COMMAND ${CMAKE_BINARY_DIR}/bin/l95_eda.x + COMMAND l95_eda.x ARGS testinput/eda.3dfgat.yaml DEPENDS l95_eda.x TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) ecbuild_add_test( TARGET test_l95_eda_3dvar MPI 4 - COMMAND ${CMAKE_BINARY_DIR}/bin/l95_eda.x + COMMAND l95_eda.x ARGS testinput/eda.3dvar.yaml DEPENDS l95_eda.x TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) ecbuild_add_test( TARGET test_l95_eda_4dvar MPI 1 - COMMAND ${CMAKE_BINARY_DIR}/bin/l95_eda.x + COMMAND l95_eda.x ARGS testinput/eda.4dvar.yaml DEPENDS l95_eda.x TEST_DEPENDS test_l95_forecast test_l95_makeobs4d ) -oops_add_test( TESTNAME eda_3dvar_block - MPI 2 - MODELNAME l95 - YAMLNAME testinput/eda.3dvar.block.yaml - EXENAME l95_eda.x - COMPARE - TEST_DEPENDS test_l95_genenspert test_l95_makeobs3d ) - +ecbuild_add_test( TARGET test_l95_eda_3dvar_block + MPI 2 + COMMAND l95_eda.x + ARGS testinput/eda.3dvar.block.yaml + TEST_DEPENDS test_l95_genenspert test_l95_makeobs3d ) ##################################################################### # state-related tests ##################################################################### -oops_add_test( TESTNAME diffstates - MODELNAME l95 - YAMLNAME testinput/diffstates.yaml - EXENAME l95_diffstates.x - TEST_DEPENDS test_l95_eda_3dvar test_l95_eda_4dvar ) +ecbuild_add_test( TARGET test_l95_diffstates + COMMAND l95_diffstates.x + ARGS testinput/diffstates.yaml + TEST_DEPENDS test_l95_eda_3dvar test_l95_eda_4dvar ) -oops_add_test( TESTNAME addincrement - MODELNAME l95 - YAMLNAME testinput/addincrement.yaml - EXENAME l95_addincrement.x - TEST_DEPENDS test_l95_diffstates ) +ecbuild_add_test( TARGET test_l95_addincrement + COMMAND l95_addincrement.x + ARGS testinput/addincrement.yaml + TEST_DEPENDS test_l95_diffstates ) -oops_add_test( TESTNAME addincrement_scaled - MODELNAME l95 - YAMLNAME testinput/addincrement_scaled.yaml - EXENAME l95_addincrement.x - TEST_DEPENDS test_l95_diffstates ) +ecbuild_add_test( TARGET test_l95_addincrement_scaled + COMMAND l95_addincrement.x + ARGS testinput/addincrement_scaled.yaml + TEST_DEPENDS test_l95_diffstates ) ##################################################################### # LETKF tests ##################################################################### -oops_add_test( TESTNAME letkf - MODELNAME l95 - YAMLNAME testinput/letkf.yaml - EXENAME l95_letkf.x - OMP 2 - TEST_DEPENDS test_l95_makeobs3d test_l95_genenspert ) - -oops_add_test( TESTNAME letkf_noobs - MODELNAME l95 - YAMLNAME testinput/letkf_noobs.yaml - EXENAME l95_letkf.x - TEST_DEPENDS test_l95_forecast test_l95_genenspert ) - -oops_add_test( TESTNAME letkf_qc - MODELNAME l95 - YAMLNAME testinput/letkf_qc.yaml - EXENAME l95_letkf.x - TEST_DEPENDS test_l95_makeobs3d test_l95_genenspert ) - -oops_add_test( TESTNAME letkf_gsi - MODELNAME l95 - YAMLNAME testinput/letkfGSI.yaml - EXENAME l95_letkf.x - CTOL 1e-4 - TEST_DEPENDS test_l95_makeobs3d test_l95_genenspert ) - -oops_add_test( TESTNAME getkf - MODELNAME l95 - YAMLNAME testinput/getkf.yaml - EXENAME l95_letkf.x - CTOL 1e-3 - TEST_DEPENDS test_l95_makeobs3d test_l95_genenspert ) - -oops_add_test( TESTNAME hofx3d_for_getkf - MODELNAME l95 - YAMLNAME testinput/hofx3d_for_getkf.yaml - EXENAME l95_letkf.x - CTOL 1e-3 - TEST_DEPENDS test_l95_letkf ) - -oops_add_test( TESTNAME getkf_offline_hofx - MODELNAME l95 - YAMLNAME testinput/getkf_offline_hofx.yaml - EXENAME l95_letkf.x - CTOL 1e-3 - TEST_DEPENDS test_l95_hofx3d_for_getkf ) +ecbuild_add_test( TARGET test_l95_letkf + COMMAND l95_letkf.x + ARGS testinput/letkf.yaml + OMP 2 + TEST_DEPENDS test_l95_makeobs3d test_l95_genenspert ) + +ecbuild_add_test( TARGET test_l95_letkf_noobs + COMMAND l95_letkf.x + ARGS testinput/letkf_noobs.yaml + TEST_DEPENDS test_l95_forecast test_l95_genenspert ) + +ecbuild_add_test( TARGET test_l95_letkf_qc + COMMAND l95_letkf.x + ARGS testinput/letkf_qc.yaml + TEST_DEPENDS test_l95_makeobs3d test_l95_genenspert ) + +ecbuild_add_test( TARGET test_l95_letkf_gsi + COMMAND l95_letkf.x + ARGS testinput/letkfGSI.yaml + TEST_DEPENDS test_l95_makeobs3d test_l95_genenspert ) + +ecbuild_add_test( TARGET test_l95_getkf + COMMAND l95_letkf.x + ARGS testinput/getkf.yaml + TEST_DEPENDS test_l95_makeobs3d test_l95_genenspert ) + +ecbuild_add_test( TARGET test_l95_hofx3d_for_getkf + COMMAND l95_letkf.x + ARGS testinput/hofx3d_for_getkf.yaml + TEST_DEPENDS test_l95_letkf ) + +ecbuild_add_test( TARGET test_l95_getkf_offline_hofx + COMMAND l95_letkf.x + ARGS testinput/getkf_offline_hofx.yaml + TEST_DEPENDS test_l95_hofx3d_for_getkf ) ##################################################################### @@ -794,86 +729,59 @@ ecbuild_add_test( TARGET test_l95_linearmodelfactory # tests with simplified L95 ##################################################################### -oops_add_test( TESTNAME simplifiedl95_DRGMRESR - MODELNAME l95 - YAMLNAME testinput/simplifiedl95_DRGMRESR.yaml - CTOL 1e-8 - EXENAME l95_4dvar.x ) - -oops_add_test( TESTNAME simplifiedl95_DRIPCG - MODELNAME l95 - YAMLNAME testinput/simplifiedl95_DRIPCG.yaml - CTOL 1e-8 - EXENAME l95_4dvar.x ) - -oops_add_test( TESTNAME simplifiedl95_DRPCG - MODELNAME l95 - YAMLNAME testinput/simplifiedl95_DRPCG.yaml - CTOL 1e-8 - EXENAME l95_4dvar.x ) - -oops_add_test( TESTNAME simplifiedl95_DRPFOM - MODELNAME l95 - YAMLNAME testinput/simplifiedl95_DRPFOM.yaml - CTOL 1e-8 - EXENAME l95_4dvar.x ) - -oops_add_test( TESTNAME simplifiedl95_DRPLanczos - MODELNAME l95 - YAMLNAME testinput/simplifiedl95_DRPLanczos.yaml - CTOL 1e-8 - EXENAME l95_4dvar.x ) - -oops_add_test( TESTNAME simplifiedl95_FGMRES - MODELNAME l95 - YAMLNAME testinput/simplifiedl95_FGMRES.yaml - CTOL 1e-8 - EXENAME l95_4dvar.x ) - -oops_add_test( TESTNAME simplifiedl95_GMRESR - MODELNAME l95 - YAMLNAME testinput/simplifiedl95_GMRESR.yaml - CTOL 1e-8 - EXENAME l95_4dvar.x ) - -oops_add_test( TESTNAME simplifiedl95_IPCG - MODELNAME l95 - YAMLNAME testinput/simplifiedl95_IPCG.yaml - CTOL 1e-8 - EXENAME l95_4dvar.x ) - -oops_add_test( TESTNAME simplifiedl95_LBGMRESR - MODELNAME l95 - YAMLNAME testinput/simplifiedl95_LBGMRESR.yaml - CTOL 1e-8 - EXENAME l95_4dvar.x ) - -oops_add_test( TESTNAME simplifiedl95_MINRES - MODELNAME l95 - YAMLNAME testinput/simplifiedl95_MINRES.yaml - CTOL 1e-8 - EXENAME l95_4dvar.x ) - -oops_add_test( TESTNAME simplifiedl95_PCG - MODELNAME l95 - YAMLNAME testinput/simplifiedl95_PCG.yaml - CTOL 1e-8 - EXENAME l95_4dvar.x ) - -oops_add_test( TESTNAME simplifiedl95_PLanczos - MODELNAME l95 - YAMLNAME testinput/simplifiedl95_PLanczos.yaml - CTOL 1e-8 - EXENAME l95_4dvar.x ) - -oops_add_test( TESTNAME simplifiedl95_RPCG - MODELNAME l95 - YAMLNAME testinput/simplifiedl95_RPCG.yaml - CTOL 1e-8 - EXENAME l95_4dvar.x ) - -oops_add_test( TESTNAME simplifiedl95_RPLanczos - MODELNAME l95 - YAMLNAME testinput/simplifiedl95_RPLanczos.yaml - CTOL 1e-8 - EXENAME l95_4dvar.x ) +ecbuild_add_test( TARGET test_l95_simplifiedl95_DRGMRESR + COMMAND l95_4dvar.x + ARGS testinput/simplifiedl95_DRGMRESR.yaml ) + +ecbuild_add_test( TARGET test_l95_simplifiedl95_DRIPCG + COMMAND l95_4dvar.x + ARGS testinput/simplifiedl95_DRIPCG.yaml ) + + +ecbuild_add_test( TARGET test_l95_simplifiedl95_DRPCG + COMMAND l95_4dvar.x + ARGS testinput/simplifiedl95_DRPCG.yaml ) + +ecbuild_add_test( TARGET test_l95_simplifiedl95_DRPFOM + COMMAND l95_4dvar.x + ARGS testinput/simplifiedl95_DRPFOM.yaml ) + +ecbuild_add_test( TARGET simplifiedl95_DRPLanczos + COMMAND l95_4dvar.x + ARGS testinput/simplifiedl95_DRPLanczos.yaml ) + +ecbuild_add_test( TARGET simplifiedl95_FGMRES + COMMAND l95_4dvar.x + ARGS testinput/simplifiedl95_FGMRES.yaml ) + +ecbuild_add_test( TARGET simplifiedl95_GMRESR + COMMAND l95_4dvar.x + ARGS testinput/simplifiedl95_GMRESR.yaml ) + +ecbuild_add_test( TARGET simplifiedl95_IPCG + COMMAND l95_4dvar.x + ARGS testinput/simplifiedl95_IPCG.yaml ) + +ecbuild_add_test( TARGET simplifiedl95_LBGMRESR + COMMAND l95_4dvar.x + ARGS testinput/simplifiedl95_LBGMRESR.yaml ) + +ecbuild_add_test( TARGET simplifiedl95_MINRES + COMMAND l95_4dvar.x + ARGS testinput/simplifiedl95_MINRES.yaml ) + +ecbuild_add_test( TARGET simplifiedl95_PCG + COMMAND l95_4dvar.x + ARGS testinput/simplifiedl95_PCG.yaml ) + +ecbuild_add_test( TARGET simplifiedl95_PLanczos + COMMAND l95_4dvar.x + ARGS testinput/simplifiedl95_PLanczos.yaml ) + +ecbuild_add_test( TARGET simplifiedl95_RPCG + COMMAND l95_4dvar.x + ARGS testinput/simplifiedl95_RPCG.yaml ) + +ecbuild_add_test( TARGET simplifiedl95_RPLanczos + COMMAND l95_4dvar.x + ARGS testinput/simplifiedl95_RPLanczos.yaml ) diff --git a/l95/test/testinput/3dfgat.yaml b/l95/test/testinput/3dfgat.yaml index 7fb5ab686..5e3c8ec0d 100644 --- a/l95/test/testinput/3dfgat.yaml +++ b/l95/test/testinput/3dfgat.yaml @@ -48,3 +48,6 @@ output: first: PT3H frequency: PT6H type: an + +test: + reference filename: testoutput/3dfgat.test diff --git a/l95/test/testinput/3dvar.yaml b/l95/test/testinput/3dvar.yaml index 63a382223..80bdd36dc 100644 --- a/l95/test/testinput/3dvar.yaml +++ b/l95/test/testinput/3dvar.yaml @@ -58,3 +58,6 @@ output: exp: test frequency: PT6H type: an + +test: + reference filename: testoutput/3dvar.test diff --git a/l95/test/testinput/3dvar_noobs.yaml b/l95/test/testinput/3dvar_noobs.yaml index 288f0ce07..4a21a8046 100644 --- a/l95/test/testinput/3dvar_noobs.yaml +++ b/l95/test/testinput/3dvar_noobs.yaml @@ -43,3 +43,6 @@ output: exp: test frequency: PT6H type: an + +test: + reference filename: testoutput/3dvar_noobs.test diff --git a/l95/test/testinput/3dvar_qc.yaml b/l95/test/testinput/3dvar_qc.yaml index 6d03e79d2..dd2ba2450 100644 --- a/l95/test/testinput/3dvar_qc.yaml +++ b/l95/test/testinput/3dvar_qc.yaml @@ -48,3 +48,6 @@ output: exp: test frequency: PT6H type: an + +test: + reference filename: testoutput/3dvar_qc.test diff --git a/l95/test/testinput/3dvar_qc_iterations.yaml b/l95/test/testinput/3dvar_qc_iterations.yaml index 4749ef255..699ba87b3 100644 --- a/l95/test/testinput/3dvar_qc_iterations.yaml +++ b/l95/test/testinput/3dvar_qc_iterations.yaml @@ -49,3 +49,6 @@ output: exp: test frequency: PT6H type: an + +test: + reference filename: testoutput/3dvar_qc_iterations.test diff --git a/l95/test/testinput/3dvar_qc_obserr.yaml b/l95/test/testinput/3dvar_qc_obserr.yaml index 0cd548816..15186552d 100644 --- a/l95/test/testinput/3dvar_qc_obserr.yaml +++ b/l95/test/testinput/3dvar_qc_obserr.yaml @@ -62,3 +62,6 @@ output: exp: test frequency: PT6H type: an + +test: + reference filename: testoutput/3dvar_qc_obserr.test diff --git a/l95/test/testinput/4densvar.hybrid.yaml b/l95/test/testinput/4densvar.hybrid.yaml index d311ef26d..10970ecaf 100644 --- a/l95/test/testinput/4densvar.hybrid.yaml +++ b/l95/test/testinput/4densvar.hybrid.yaml @@ -273,3 +273,6 @@ output: first: PT3H frequency: PT6H type: an + +test: + reference filename: testoutput/4densvar.hybrid.test diff --git a/l95/test/testinput/4densvar.yaml b/l95/test/testinput/4densvar.yaml index c6331c4a0..4047b3000 100644 --- a/l95/test/testinput/4densvar.yaml +++ b/l95/test/testinput/4densvar.yaml @@ -261,3 +261,6 @@ output: first: PT3H frequency: PT6H type: an + +test: + reference filename: testoutput/4densvar.test diff --git a/l95/test/testinput/4dforcing.yaml b/l95/test/testinput/4dforcing.yaml index f34ae11ad..53af3a259 100644 --- a/l95/test/testinput/4dforcing.yaml +++ b/l95/test/testinput/4dforcing.yaml @@ -72,3 +72,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dforcing.test diff --git a/l95/test/testinput/4dsaddlepoint.yaml b/l95/test/testinput/4dsaddlepoint.yaml index 9bb6767ae..40a8b276f 100644 --- a/l95/test/testinput/4dsaddlepoint.yaml +++ b/l95/test/testinput/4dsaddlepoint.yaml @@ -77,3 +77,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dsaddlepoint.test diff --git a/l95/test/testinput/4dvar.allbiases.yaml b/l95/test/testinput/4dvar.allbiases.yaml index e0578bb77..56e9a68f9 100644 --- a/l95/test/testinput/4dvar.allbiases.yaml +++ b/l95/test/testinput/4dvar.allbiases.yaml @@ -78,3 +78,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.allbiases.test diff --git a/l95/test/testinput/4dvar.alpha.yaml b/l95/test/testinput/4dvar.alpha.yaml index 39f08946e..6bb212c62 100644 --- a/l95/test/testinput/4dvar.alpha.yaml +++ b/l95/test/testinput/4dvar.alpha.yaml @@ -87,3 +87,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.alpha.test diff --git a/l95/test/testinput/4dvar.drgmresr.yaml b/l95/test/testinput/4dvar.drgmresr.yaml index a21a46abe..b0efc8b89 100644 --- a/l95/test/testinput/4dvar.drgmresr.yaml +++ b/l95/test/testinput/4dvar.drgmresr.yaml @@ -70,3 +70,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.drgmresr.test diff --git a/l95/test/testinput/4dvar.dripcg.yaml b/l95/test/testinput/4dvar.dripcg.yaml index 7ce8dccba..cd0f53f39 100644 --- a/l95/test/testinput/4dvar.dripcg.yaml +++ b/l95/test/testinput/4dvar.dripcg.yaml @@ -70,3 +70,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.dripcg.test diff --git a/l95/test/testinput/4dvar.dripcgqn.yaml b/l95/test/testinput/4dvar.dripcgqn.yaml index 582a64426..bbabd3b2e 100644 --- a/l95/test/testinput/4dvar.dripcgqn.yaml +++ b/l95/test/testinput/4dvar.dripcgqn.yaml @@ -73,3 +73,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.dripcgqn.test diff --git a/l95/test/testinput/4dvar.drpcg.yaml b/l95/test/testinput/4dvar.drpcg.yaml index 90ecab6aa..e2860c40f 100644 --- a/l95/test/testinput/4dvar.drpcg.yaml +++ b/l95/test/testinput/4dvar.drpcg.yaml @@ -70,3 +70,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.drpcg.test diff --git a/l95/test/testinput/4dvar.drpcgqn.yaml b/l95/test/testinput/4dvar.drpcgqn.yaml index 9bf13e86a..332d328bc 100644 --- a/l95/test/testinput/4dvar.drpcgqn.yaml +++ b/l95/test/testinput/4dvar.drpcgqn.yaml @@ -74,3 +74,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.drpcgqn.test diff --git a/l95/test/testinput/4dvar.drplanclmp.yaml b/l95/test/testinput/4dvar.drplanclmp.yaml index fb205a176..5bdc4d7d5 100644 --- a/l95/test/testinput/4dvar.drplanclmp.yaml +++ b/l95/test/testinput/4dvar.drplanclmp.yaml @@ -72,3 +72,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.drplanclmp.test diff --git a/l95/test/testinput/4dvar.drplanczos.yaml b/l95/test/testinput/4dvar.drplanczos.yaml index 17abe80d2..f9fe1698a 100644 --- a/l95/test/testinput/4dvar.drplanczos.yaml +++ b/l95/test/testinput/4dvar.drplanczos.yaml @@ -77,3 +77,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.drplanczos.test diff --git a/l95/test/testinput/4dvar.fgmres.yaml b/l95/test/testinput/4dvar.fgmres.yaml index 5d135e25c..fb1d2ebc9 100644 --- a/l95/test/testinput/4dvar.fgmres.yaml +++ b/l95/test/testinput/4dvar.fgmres.yaml @@ -70,3 +70,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.fgmres.test diff --git a/l95/test/testinput/4dvar.gmresr.yaml b/l95/test/testinput/4dvar.gmresr.yaml index 767175dc3..266009004 100644 --- a/l95/test/testinput/4dvar.gmresr.yaml +++ b/l95/test/testinput/4dvar.gmresr.yaml @@ -70,3 +70,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.gmresr.test diff --git a/l95/test/testinput/4dvar.hybrid.yaml b/l95/test/testinput/4dvar.hybrid.yaml index 4eabc5c6f..e20c9c459 100644 --- a/l95/test/testinput/4dvar.hybrid.yaml +++ b/l95/test/testinput/4dvar.hybrid.yaml @@ -99,3 +99,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.hybrid.test diff --git a/l95/test/testinput/4dvar.ipcg.yaml b/l95/test/testinput/4dvar.ipcg.yaml index d932d104d..7fbb5ed6c 100644 --- a/l95/test/testinput/4dvar.ipcg.yaml +++ b/l95/test/testinput/4dvar.ipcg.yaml @@ -70,3 +70,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.ipcg.test diff --git a/l95/test/testinput/4dvar.lbgmresr.yaml b/l95/test/testinput/4dvar.lbgmresr.yaml index 4cfc9b710..27bf54b71 100644 --- a/l95/test/testinput/4dvar.lbgmresr.yaml +++ b/l95/test/testinput/4dvar.lbgmresr.yaml @@ -70,3 +70,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.lbgmresr.test diff --git a/l95/test/testinput/4dvar.minres.yaml b/l95/test/testinput/4dvar.minres.yaml index 9cfb01881..84f1b939a 100644 --- a/l95/test/testinput/4dvar.minres.yaml +++ b/l95/test/testinput/4dvar.minres.yaml @@ -70,3 +70,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.minres.test diff --git a/l95/test/testinput/4dvar.modbias.yaml b/l95/test/testinput/4dvar.modbias.yaml index 5e19a791e..939c8b106 100644 --- a/l95/test/testinput/4dvar.modbias.yaml +++ b/l95/test/testinput/4dvar.modbias.yaml @@ -74,3 +74,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.modbias.test diff --git a/l95/test/testinput/4dvar.pcg.yaml b/l95/test/testinput/4dvar.pcg.yaml index a3765afa7..91104568f 100644 --- a/l95/test/testinput/4dvar.pcg.yaml +++ b/l95/test/testinput/4dvar.pcg.yaml @@ -70,3 +70,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.pcg.test diff --git a/l95/test/testinput/4dvar.planczos.yaml b/l95/test/testinput/4dvar.planczos.yaml index 08f105d22..811fe53db 100644 --- a/l95/test/testinput/4dvar.planczos.yaml +++ b/l95/test/testinput/4dvar.planczos.yaml @@ -70,3 +70,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.planczos.test diff --git a/l95/test/testinput/4dvar.rpcg.yaml b/l95/test/testinput/4dvar.rpcg.yaml index 380a6fff1..49dba5bec 100644 --- a/l95/test/testinput/4dvar.rpcg.yaml +++ b/l95/test/testinput/4dvar.rpcg.yaml @@ -70,3 +70,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.rpcg.test diff --git a/l95/test/testinput/4dvar.rplanczos.yaml b/l95/test/testinput/4dvar.rplanczos.yaml index 9d9a9cdf1..cfbc630c9 100644 --- a/l95/test/testinput/4dvar.rplanczos.yaml +++ b/l95/test/testinput/4dvar.rplanczos.yaml @@ -70,3 +70,6 @@ output: first: PT3H frequency: PT06H type: an + +test: + reference filename: testoutput/4dvar.rplanczos.test diff --git a/l95/test/testinput/addincrement.yaml b/l95/test/testinput/addincrement.yaml index 9b2286980..65f98e651 100644 --- a/l95/test/testinput/addincrement.yaml +++ b/l95/test/testinput/addincrement.yaml @@ -14,3 +14,6 @@ output: date: '2010-01-02T00:00:00Z' exp: addinc type: an + +test: + reference filename: testoutput/addincrement.test diff --git a/l95/test/testinput/addincrement_scaled.yaml b/l95/test/testinput/addincrement_scaled.yaml index 2bfb43403..ac889dc7c 100644 --- a/l95/test/testinput/addincrement_scaled.yaml +++ b/l95/test/testinput/addincrement_scaled.yaml @@ -15,3 +15,6 @@ output: date: '2010-01-02T00:00:00Z' exp: addinc_scaled type: an + +test: + reference filename: testoutput/addincrement_scaled.test diff --git a/l95/test/testinput/diffstates.yaml b/l95/test/testinput/diffstates.yaml index 03c470210..0d0557131 100644 --- a/l95/test/testinput/diffstates.yaml +++ b/l95/test/testinput/diffstates.yaml @@ -13,3 +13,6 @@ output: date: '2010-01-02T00:00:00Z' exp: difst type: in + +test: + reference filename: testoutput/diffstates.test diff --git a/l95/test/testinput/eda.3dvar.block.yaml b/l95/test/testinput/eda.3dvar.block.yaml index 869049f7c..1b346f052 100644 --- a/l95/test/testinput/eda.3dvar.block.yaml +++ b/l95/test/testinput/eda.3dvar.block.yaml @@ -1,3 +1,6 @@ files: - "testinput/eda.3dvar.block.1.yaml" - "testinput/eda.3dvar.block.2.yaml" + +test: + reference filename: testoutput/eda_3dvar_block.test diff --git a/l95/test/testinput/forecast.yaml b/l95/test/testinput/forecast.yaml index d9d831388..651510675 100644 --- a/l95/test/testinput/forecast.yaml +++ b/l95/test/testinput/forecast.yaml @@ -14,3 +14,6 @@ output: exp: test frequency: PT1H30M type: fc + +test: + reference filename: testoutput/forecast.test diff --git a/l95/test/testinput/forecast_identitymodel.yaml b/l95/test/testinput/forecast_identitymodel.yaml index a46637154..93efdbf03 100644 --- a/l95/test/testinput/forecast_identitymodel.yaml +++ b/l95/test/testinput/forecast_identitymodel.yaml @@ -14,3 +14,6 @@ output: type: idfc geometry: resol: 40 + +test: + reference filename: testoutput/forecast_identitymodel.test diff --git a/l95/test/testinput/forecast_pseudomodel.yaml b/l95/test/testinput/forecast_pseudomodel.yaml index 83c1be2f6..37e912e48 100644 --- a/l95/test/testinput/forecast_pseudomodel.yaml +++ b/l95/test/testinput/forecast_pseudomodel.yaml @@ -27,3 +27,6 @@ output: type: pseudofc geometry: resol: 40 + +test: + reference filename: testoutput/forecast_pseudomodel.test diff --git a/l95/test/testinput/genenspert.yaml b/l95/test/testinput/genenspert.yaml index b9861dec6..2a06cfead 100644 --- a/l95/test/testinput/genenspert.yaml +++ b/l95/test/testinput/genenspert.yaml @@ -21,3 +21,6 @@ output: exp: test frequency: PT1H30M type: ens + +test: + reference filename: testoutput/genenspert.test diff --git a/l95/test/testinput/getkf.yaml b/l95/test/testinput/getkf.yaml index 3735507f3..d8e35dd91 100644 --- a/l95/test/testinput/getkf.yaml +++ b/l95/test/testinput/getkf.yaml @@ -28,7 +28,7 @@ observations: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obs operator: {} -driver: +driver: local ensemble DA: solver: GETKF @@ -46,3 +46,7 @@ output: date: *date exp: getkf.%{member}% type: an + +test: + reference filename: testoutput/getkf.test + float relative tolerance: 1e-3 diff --git a/l95/test/testinput/getkf_offline_hofx.yaml b/l95/test/testinput/getkf_offline_hofx.yaml index 844aadd3a..ccbd903c6 100644 --- a/l95/test/testinput/getkf_offline_hofx.yaml +++ b/l95/test/testinput/getkf_offline_hofx.yaml @@ -28,7 +28,7 @@ observations: obsdatain: Data/l95.getkf.2010-01-02T00:00:00Z.obt obs operator: {} -driver: +driver: read HX from disk: true do posterior observer: false @@ -48,3 +48,7 @@ output: date: *date exp: getkf.%{member}% type: an + +test: + reference filename: testoutput/getkf_offline_hofx.test + float relative tolerance: 1e-3 diff --git a/l95/test/testinput/hofx.yaml b/l95/test/testinput/hofx.yaml index 66eecf999..809b7eef8 100644 --- a/l95/test/testinput/hofx.yaml +++ b/l95/test/testinput/hofx.yaml @@ -15,3 +15,6 @@ observations: obsdatain: Data/l95.truth4d.2010-01-02T00:00:00Z.obt obsdataout: Data/l95.hofx.2010-01-02T00:00:00Z.obt obs operator: {} + +test: + reference filename: testoutput/hofx.test diff --git a/l95/test/testinput/hofx3d.yaml b/l95/test/testinput/hofx3d.yaml index 2f299a8dd..72ae4bda1 100644 --- a/l95/test/testinput/hofx3d.yaml +++ b/l95/test/testinput/hofx3d.yaml @@ -10,3 +10,6 @@ observations: obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt obsdataout: Data/l95.hofx.2010-01-02T00:00:00Z.obt obs operator: {} + +test: + reference filename: testoutput/hofx3d.test diff --git a/l95/test/testinput/hofx3d_for_getkf.yaml b/l95/test/testinput/hofx3d_for_getkf.yaml index 3bd3bea4a..a3b8665b1 100644 --- a/l95/test/testinput/hofx3d_for_getkf.yaml +++ b/l95/test/testinput/hofx3d_for_getkf.yaml @@ -29,7 +29,7 @@ observations: obsdataout: Data/l95.getkf.2010-01-02T00:00:00Z.obt obs operator: {} -driver: +driver: run as observer only: true local ensemble DA: @@ -48,3 +48,7 @@ output: date: *date exp: getkf.%{member}% type: an + +test: + reference filename: testoutput/hofx3d_for_getkf.test + float relative tolerance: 1e-3 diff --git a/l95/test/testinput/letkf.yaml b/l95/test/testinput/letkf.yaml index 15d5a4159..8196be328 100644 --- a/l95/test/testinput/letkf.yaml +++ b/l95/test/testinput/letkf.yaml @@ -73,3 +73,5 @@ output variance posterior: exp: xavar.%{member}% type: an +test: + reference filename: testoutput/letkf.test diff --git a/l95/test/testinput/letkfGSI.yaml b/l95/test/testinput/letkfGSI.yaml index 598dff73f..c643a3e41 100644 --- a/l95/test/testinput/letkfGSI.yaml +++ b/l95/test/testinput/letkfGSI.yaml @@ -43,3 +43,8 @@ output: date: *date exp: letkf_gsi.%{member}% type: an + + +test: + reference filename: testoutput/letkf_gsi.test + float relative tolerance: 1e-4 diff --git a/l95/test/testinput/letkf_noobs.yaml b/l95/test/testinput/letkf_noobs.yaml index d5bd9a7f2..bf29e4cf0 100644 --- a/l95/test/testinput/letkf_noobs.yaml +++ b/l95/test/testinput/letkf_noobs.yaml @@ -42,3 +42,6 @@ output: date: *date exp: letkf.%{member}% type: an + +test: + reference filename: testoutput/letkf_noobs.test diff --git a/l95/test/testinput/letkf_qc.yaml b/l95/test/testinput/letkf_qc.yaml index 3331fb492..b85645ae8 100644 --- a/l95/test/testinput/letkf_qc.yaml +++ b/l95/test/testinput/letkf_qc.yaml @@ -45,3 +45,6 @@ output: date: *date exp: letkf.%{member}% type: an + +test: + reference filename: testoutput/letkf_qc.test diff --git a/l95/test/testinput/makeobs3d.yaml b/l95/test/testinput/makeobs3d.yaml index c64783786..2993d2d53 100644 --- a/l95/test/testinput/makeobs3d.yaml +++ b/l95/test/testinput/makeobs3d.yaml @@ -21,3 +21,6 @@ observations: obs_frequency: PT1H30M obsdataout: Data/l95.truth3d.2010-01-02T00:00:00Z.obt make obs: true + +test: + reference filename: testoutput/makeobs3d.test diff --git a/l95/test/testinput/makeobs4d.yaml b/l95/test/testinput/makeobs4d.yaml index 51e680045..ed850c973 100644 --- a/l95/test/testinput/makeobs4d.yaml +++ b/l95/test/testinput/makeobs4d.yaml @@ -23,3 +23,6 @@ observations: - filter: GOMsaver filename: Data/l95.gom make obs: true + +test: + reference filename: testoutput/makeobs4d.test diff --git a/l95/test/testinput/makeobs4d12h.yaml b/l95/test/testinput/makeobs4d12h.yaml index d14898ad8..d88f287f0 100644 --- a/l95/test/testinput/makeobs4d12h.yaml +++ b/l95/test/testinput/makeobs4d12h.yaml @@ -20,3 +20,6 @@ observations: obs_frequency: PT1H30M obs operator: {} make obs: true + +test: + reference filename: testoutput/makeobs4d12h.test diff --git a/l95/test/testinput/makeobsbias.yaml b/l95/test/testinput/makeobsbias.yaml index 190c3f5e1..4ca2486ff 100644 --- a/l95/test/testinput/makeobsbias.yaml +++ b/l95/test/testinput/makeobsbias.yaml @@ -22,3 +22,6 @@ observations: obs bias: bias: 0.6 make obs: true + +test: + reference filename: testoutput/makeobsbias.test diff --git a/l95/test/testinput/makeobspert.yaml b/l95/test/testinput/makeobspert.yaml index 2e4a11994..cf04f39d1 100644 --- a/l95/test/testinput/makeobspert.yaml +++ b/l95/test/testinput/makeobspert.yaml @@ -26,3 +26,6 @@ observations: filename: Data/l95.gom obs perturbations: true make obs: true + +test: + reference filename: testoutput/makeobspert.test diff --git a/l95/test/testinput/simplifiedl95_DRGMRESR.yaml b/l95/test/testinput/simplifiedl95_DRGMRESR.yaml index f705391b0..30a628d94 100644 --- a/l95/test/testinput/simplifiedl95_DRGMRESR.yaml +++ b/l95/test/testinput/simplifiedl95_DRGMRESR.yaml @@ -44,3 +44,7 @@ output: exp: simplifiedl95 frequency: PT1H type: an + +test: + reference filename: testoutput/simplifiedl95_DRGMRESR.test + float relative tolerance: 1e-8 diff --git a/l95/test/testinput/simplifiedl95_DRIPCG.yaml b/l95/test/testinput/simplifiedl95_DRIPCG.yaml index 8834d0e61..606068c5e 100644 --- a/l95/test/testinput/simplifiedl95_DRIPCG.yaml +++ b/l95/test/testinput/simplifiedl95_DRIPCG.yaml @@ -44,3 +44,7 @@ output: exp: simplifiedl95 frequency: PT1H type: an + +test: + reference filename: testoutput/simplifiedl95_DRIPCG.test + float relative tolerance: 1e-8 diff --git a/l95/test/testinput/simplifiedl95_DRPCG.yaml b/l95/test/testinput/simplifiedl95_DRPCG.yaml index 92ba58c00..42809a453 100644 --- a/l95/test/testinput/simplifiedl95_DRPCG.yaml +++ b/l95/test/testinput/simplifiedl95_DRPCG.yaml @@ -44,3 +44,7 @@ output: exp: simplifiedl95 frequency: PT1H type: an + + test: + reference filename: testoutput/simplifiedl95_DRPCG.test + float relative tolerance: 1e-8 diff --git a/l95/test/testinput/simplifiedl95_DRPFOM.yaml b/l95/test/testinput/simplifiedl95_DRPFOM.yaml index 1b9ac572f..286c6f62b 100644 --- a/l95/test/testinput/simplifiedl95_DRPFOM.yaml +++ b/l95/test/testinput/simplifiedl95_DRPFOM.yaml @@ -44,3 +44,7 @@ output: exp: simplifiedl95 frequency: PT1H type: an + +test: + reference filename: testoutput/simplifiedl95_DRPFOM.test + float relative tolerance: 1e-8 diff --git a/l95/test/testinput/simplifiedl95_DRPLanczos.yaml b/l95/test/testinput/simplifiedl95_DRPLanczos.yaml index 080cf4e3b..a11276af4 100644 --- a/l95/test/testinput/simplifiedl95_DRPLanczos.yaml +++ b/l95/test/testinput/simplifiedl95_DRPLanczos.yaml @@ -44,3 +44,7 @@ output: exp: simplifiedl95 frequency: PT1H type: an + +test: + reference filename: testoutput/simplifiedl95_DRPLanczos.test + float relative tolerance: 1e-8 diff --git a/l95/test/testinput/simplifiedl95_FGMRES.yaml b/l95/test/testinput/simplifiedl95_FGMRES.yaml index 7a8eefe3b..e8023d6f8 100644 --- a/l95/test/testinput/simplifiedl95_FGMRES.yaml +++ b/l95/test/testinput/simplifiedl95_FGMRES.yaml @@ -44,3 +44,7 @@ output: exp: simplifiedl95 frequency: PT1H type: an + +test: + reference filename: testoutput/simplifiedl95_FGMRES.test + float relative tolerance: 1e-8 diff --git a/l95/test/testinput/simplifiedl95_GMRESR.yaml b/l95/test/testinput/simplifiedl95_GMRESR.yaml index e8fa9d82e..58e172ad7 100644 --- a/l95/test/testinput/simplifiedl95_GMRESR.yaml +++ b/l95/test/testinput/simplifiedl95_GMRESR.yaml @@ -44,3 +44,7 @@ output: exp: simplifiedl95 frequency: PT1H type: an + +test: + reference filename: testoutput/simplifiedl95_GMRESR.test + float relative tolerance: 1e-8 diff --git a/l95/test/testinput/simplifiedl95_IPCG.yaml b/l95/test/testinput/simplifiedl95_IPCG.yaml index 7b9748a6e..19c975543 100644 --- a/l95/test/testinput/simplifiedl95_IPCG.yaml +++ b/l95/test/testinput/simplifiedl95_IPCG.yaml @@ -44,3 +44,7 @@ output: exp: simplifiedl95 frequency: PT1H type: an + +test: + reference filename: testoutput/simplifiedl95_IPCG.test + float relative tolerance: 1e-8 diff --git a/l95/test/testinput/simplifiedl95_LBGMRESR.yaml b/l95/test/testinput/simplifiedl95_LBGMRESR.yaml index fd4337d0d..11d7d8840 100644 --- a/l95/test/testinput/simplifiedl95_LBGMRESR.yaml +++ b/l95/test/testinput/simplifiedl95_LBGMRESR.yaml @@ -44,3 +44,7 @@ output: exp: simplifiedl95 frequency: PT1H type: an + +test: + reference filename: testoutput/simplifiedl95_LBGMRESR.test + float relative tolerance: 1e-8 diff --git a/l95/test/testinput/simplifiedl95_MINRES.yaml b/l95/test/testinput/simplifiedl95_MINRES.yaml index a6a4ce921..a6e1aeb52 100644 --- a/l95/test/testinput/simplifiedl95_MINRES.yaml +++ b/l95/test/testinput/simplifiedl95_MINRES.yaml @@ -44,3 +44,7 @@ output: exp: simplifiedl95 frequency: PT1H type: an + +test: + reference filename: testoutput/simplifiedl95_MINRES.test + float relative tolerance: 1e-8 diff --git a/l95/test/testinput/simplifiedl95_PCG.yaml b/l95/test/testinput/simplifiedl95_PCG.yaml index ea983581b..fe0c15317 100644 --- a/l95/test/testinput/simplifiedl95_PCG.yaml +++ b/l95/test/testinput/simplifiedl95_PCG.yaml @@ -44,3 +44,7 @@ output: exp: simplifiedl95 frequency: PT1H type: an + +test: + reference filename: testoutput/simplifiedl95_PCG.test + float relative tolerance: 1e-8 diff --git a/l95/test/testinput/simplifiedl95_PLanczos.yaml b/l95/test/testinput/simplifiedl95_PLanczos.yaml index 2f2eb6910..e27fa6d61 100644 --- a/l95/test/testinput/simplifiedl95_PLanczos.yaml +++ b/l95/test/testinput/simplifiedl95_PLanczos.yaml @@ -44,3 +44,7 @@ output: exp: simplifiedl95 frequency: PT1H type: an + +test: + reference filename: testoutput/simplifiedl95_PLanczos.test + float relative tolerance: 1e-8 diff --git a/l95/test/testinput/simplifiedl95_RPCG.yaml b/l95/test/testinput/simplifiedl95_RPCG.yaml index a678fea74..b7ab16f23 100644 --- a/l95/test/testinput/simplifiedl95_RPCG.yaml +++ b/l95/test/testinput/simplifiedl95_RPCG.yaml @@ -44,3 +44,7 @@ output: exp: simplifiedl95 frequency: PT1H type: an + +test: + reference filename: testoutput/simplifiedl95_RPCG.test + float relative tolerance: 1e-8 diff --git a/l95/test/testinput/simplifiedl95_RPLanczos.yaml b/l95/test/testinput/simplifiedl95_RPLanczos.yaml index 0ee17486e..f82bcf6bd 100644 --- a/l95/test/testinput/simplifiedl95_RPLanczos.yaml +++ b/l95/test/testinput/simplifiedl95_RPLanczos.yaml @@ -44,3 +44,7 @@ output: exp: simplifiedl95 frequency: PT1H type: an + +test: + reference filename: testoutput/simplifiedl95_RPLanczos.test + float relative tolerance: 1e-8 diff --git a/l95/test/testinput/truth.yaml b/l95/test/testinput/truth.yaml index b271bd951..f28e97f90 100644 --- a/l95/test/testinput/truth.yaml +++ b/l95/test/testinput/truth.yaml @@ -14,3 +14,6 @@ output: exp: truth frequency: PT3H type: fc + +test: + reference filename: testoutput/truth.test diff --git a/l95/test/testoutput/3dfgat.test b/l95/test/testoutput/3dfgat.test index d878f1a14..8caed4868 100644 --- a/l95/test/testoutput/3dfgat.test +++ b/l95/test/testoutput/3dfgat.test @@ -1,10 +1,10 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 115.159, nobs = 120, Jo/n = 0.959659, err = 0.4 -Test : CostFunction: Nonlinear J = 115.159 -Test : DRPCGMinimizer: reduction in residual norm = 0.000520492 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T21:00:00Z -Test : Min=6.75324, Max=9.06972, Average=8.01139 -Test : CostJb : Nonlinear Jb = 12.2572 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 11.0963, nobs = 120, Jo/n = 0.0924693, err = 0.4 -Test : CostFunction: Nonlinear J = 23.3535 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 115.159, nobs = 120, Jo/n = 0.959659, err = 0.4 +CostFunction: Nonlinear J = 115.159 +DRPCGMinimizer: reduction in residual norm = 0.000520492 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T21:00:00Z + Min=6.75324, Max=9.06972, Average=8.01139 +CostJb : Nonlinear Jb = 12.2572 +CostJo : Nonlinear Jo(Lorenz 95) = 11.0963, nobs = 120, Jo/n = 0.0924693, err = 0.4 +CostFunction: Nonlinear J = 23.3535 diff --git a/l95/test/testoutput/3dvar.test b/l95/test/testoutput/3dvar.test index 7d3580f3b..5c638c8f1 100644 --- a/l95/test/testoutput/3dvar.test +++ b/l95/test/testoutput/3dvar.test @@ -1,17 +1,17 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114.535, nobs = 120, Jo/n = 0.954461, err = 0.4 -Test : CostFunction: Nonlinear J = 114.535 -Test : DRIPCGMinimizer: reduction in residual norm = 0.000829984 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.75312, Max=9.06712, Average=8.00907 -Test : CostJb : Nonlinear Jb = 12.2981 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 3.17248, nobs = 120, Jo/n = 0.0264374, err = 0.4 -Test : CostFunction: Nonlinear J = 15.4706 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0054729 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.75305, Max=9.06719, Average=8.00913 -Test : CostJb : Nonlinear Jb = 12.2983 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 3.1723, nobs = 120, Jo/n = 0.0264358, err = 0.4 -Test : CostFunction: Nonlinear J = 15.4706 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114.535, nobs = 120, Jo/n = 0.954461, err = 0.4 +CostFunction: Nonlinear J = 114.535 +DRIPCGMinimizer: reduction in residual norm = 0.000829984 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.75312, Max=9.06712, Average=8.00907 +CostJb : Nonlinear Jb = 12.2981 +CostJo : Nonlinear Jo(Lorenz 95) = 3.17248, nobs = 120, Jo/n = 0.0264374, err = 0.4 +CostFunction: Nonlinear J = 15.4706 +DRIPCGMinimizer: reduction in residual norm = 0.0054729 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.75305, Max=9.06719, Average=8.00913 +CostJb : Nonlinear Jb = 12.2983 +CostJo : Nonlinear Jo(Lorenz 95) = 3.1723, nobs = 120, Jo/n = 0.0264358, err = 0.4 +CostFunction: Nonlinear J = 15.4706 diff --git a/l95/test/testoutput/3dvar_qc.test b/l95/test/testoutput/3dvar_qc.test index f1c1acaba..887d1693a 100644 --- a/l95/test/testoutput/3dvar_qc.test +++ b/l95/test/testoutput/3dvar_qc.test @@ -1,17 +1,17 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 54.752, nobs = 110, Jo/n = 0.497746, err = 0.4 -Test : CostFunction: Nonlinear J = 54.752 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0183271 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.61921, Max=8.88478, Average=7.98431 -Test : CostJb : Nonlinear Jb = 8.0214 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 3.56075, nobs = 110, Jo/n = 0.0323705, err = 0.4 -Test : CostFunction: Nonlinear J = 11.5821 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0664484 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.79284, Max=8.89503, Average=7.9997 -Test : CostJb : Nonlinear Jb = 7.39243 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 3.04734, nobs = 110, Jo/n = 0.0277031, err = 0.4 -Test : CostFunction: Nonlinear J = 10.4398 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 54.752, nobs = 110, Jo/n = 0.497746, err = 0.4 +CostFunction: Nonlinear J = 54.752 +DRIPCGMinimizer: reduction in residual norm = 0.0183271 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.61921, Max=8.88478, Average=7.98431 +CostJb : Nonlinear Jb = 8.0214 +CostJo : Nonlinear Jo(Lorenz 95) = 3.56075, nobs = 110, Jo/n = 0.0323705, err = 0.4 +CostFunction: Nonlinear J = 11.5821 +DRIPCGMinimizer: reduction in residual norm = 0.0664484 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.79284, Max=8.89503, Average=7.9997 +CostJb : Nonlinear Jb = 7.39243 +CostJo : Nonlinear Jo(Lorenz 95) = 3.04734, nobs = 110, Jo/n = 0.0277031, err = 0.4 +CostFunction: Nonlinear J = 10.4398 diff --git a/l95/test/testoutput/3dvar_qc_iterations.test b/l95/test/testoutput/3dvar_qc_iterations.test index 6cdf32cc1..86acffb99 100644 --- a/l95/test/testoutput/3dvar_qc_iterations.test +++ b/l95/test/testoutput/3dvar_qc_iterations.test @@ -1,17 +1,17 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114.535, nobs = 120, Jo/n = 0.954461, err = 0.4 -Test : CostFunction: Nonlinear J = 114.535 -Test : DRIPCGMinimizer: reduction in residual norm = 0.000829984 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.75312, Max=9.06712, Average=8.00907 -Test : CostJb : Nonlinear Jb = 12.2981 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 2.27545, nobs = 117, Jo/n = 0.0194483, err = 0.4 -Test : CostFunction: Nonlinear J = 14.5736 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0367195 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.78189, Max=8.91019, Average=8.00776 -Test : CostJb : Nonlinear Jb = 11.443 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 2.86913, nobs = 117, Jo/n = 0.0245225, err = 0.4 -Test : CostFunction: Nonlinear J = 14.3121 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114.535, nobs = 120, Jo/n = 0.954461, err = 0.4 +CostFunction: Nonlinear J = 114.535 +DRIPCGMinimizer: reduction in residual norm = 0.000829984 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.75312, Max=9.06712, Average=8.00907 +CostJb : Nonlinear Jb = 12.2981 +CostJo : Nonlinear Jo(Lorenz 95) = 2.27545, nobs = 117, Jo/n = 0.0194483, err = 0.4 +CostFunction: Nonlinear J = 14.5736 +DRIPCGMinimizer: reduction in residual norm = 0.0367195 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.78189, Max=8.91019, Average=8.00776 +CostJb : Nonlinear Jb = 11.443 +CostJo : Nonlinear Jo(Lorenz 95) = 2.86913, nobs = 117, Jo/n = 0.0245225, err = 0.4 +CostFunction: Nonlinear J = 14.3121 diff --git a/l95/test/testoutput/3dvar_qc_obserr.test b/l95/test/testoutput/3dvar_qc_obserr.test index 9b043860c..2fd539bc7 100644 --- a/l95/test/testoutput/3dvar_qc_obserr.test +++ b/l95/test/testoutput/3dvar_qc_obserr.test @@ -1,17 +1,17 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 69.6978, nobs = 120, Jo/n = 0.580815, err = 0.447214 -Test : CostFunction: Nonlinear J = 69.6978 -Test : DRIPCGMinimizer: reduction in residual norm = 0.00254674 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.98087, Max=8.96294, Average=8.01308 -Test : CostJb : Nonlinear Jb = 9.16112 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 3.84127, nobs = 120, Jo/n = 0.0320105, err = 0.447214 -Test : CostFunction: Nonlinear J = 13.0024 -Test : DRIPCGMinimizer: reduction in residual norm = 0.00698369 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.97981, Max=8.96287, Average=8.01311 -Test : CostJb : Nonlinear Jb = 9.16182 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 3.84039, nobs = 120, Jo/n = 0.0320032, err = 0.447214 -Test : CostFunction: Nonlinear J = 13.0022 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 69.6978, nobs = 120, Jo/n = 0.580815, err = 0.447214 +CostFunction: Nonlinear J = 69.6978 +DRIPCGMinimizer: reduction in residual norm = 0.00254674 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.98087, Max=8.96294, Average=8.01308 +CostJb : Nonlinear Jb = 9.16112 +CostJo : Nonlinear Jo(Lorenz 95) = 3.84127, nobs = 120, Jo/n = 0.0320105, err = 0.447214 +CostFunction: Nonlinear J = 13.0024 +DRIPCGMinimizer: reduction in residual norm = 0.00698369 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.97981, Max=8.96287, Average=8.01311 +CostJb : Nonlinear Jb = 9.16182 +CostJo : Nonlinear Jo(Lorenz 95) = 3.84039, nobs = 120, Jo/n = 0.0320032, err = 0.447214 +CostFunction: Nonlinear J = 13.0022 diff --git a/l95/test/testoutput/4densvar.test b/l95/test/testoutput/4densvar.test index a7f5b2eb6..5c1995635 100644 --- a/l95/test/testoutput/4densvar.test +++ b/l95/test/testoutput/4densvar.test @@ -1,49 +1,49 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 14.0524, nobs = 80, Jo/n = 0.175655, err = 0.4 -Test : CostFunction: Nonlinear J = 14.0524 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0135197 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.05961, Max=8.303, Average=7.9761 -Test : Valid time: 2010-01-01T04:30:00Z -Test : Min=7.079, Max=8.31622, Average=7.97792 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Min=7.10202, Max=8.36578, Average=7.97965 -Test : Valid time: 2010-01-01T07:30:00Z -Test : Min=7.12996, Max=8.42958, Average=7.98125 -Test : Valid time: 2010-01-01T09:00:00Z -Test : Min=7.1639, Max=8.49377, Average=7.98271 -Test : Valid time: 2010-01-01T10:30:00Z -Test : Min=7.2047, Max=8.55704, Average=7.98398 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Min=7.25301, Max=8.6179, Average=7.98503 -Test : Valid time: 2010-01-01T13:30:00Z -Test : Min=7.3093, Max=8.6747, Average=7.98586 -Test : Valid time: 2010-01-01T15:00:00Z -Test : Min=7.29016, Max=8.72562, Average=7.98644 -Test : CostJb : Nonlinear Jb = 0.704756 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.186593, nobs = 80, Jo/n = 0.00233242, err = 0.4 -Test : CostFunction: Nonlinear J = 0.89135 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0509964 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.06017, Max=8.30253, Average=7.97607 -Test : Valid time: 2010-01-01T04:30:00Z -Test : Min=7.07966, Max=8.3157, Average=7.97788 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Min=7.10277, Max=8.36569, Average=7.97959 -Test : Valid time: 2010-01-01T07:30:00Z -Test : Min=7.13078, Max=8.42965, Average=7.98118 -Test : Valid time: 2010-01-01T09:00:00Z -Test : Min=7.16476, Max=8.49393, Average=7.98261 -Test : Valid time: 2010-01-01T10:30:00Z -Test : Min=7.20558, Max=8.55723, Average=7.98386 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Min=7.25388, Max=8.61804, Average=7.98489 -Test : Valid time: 2010-01-01T13:30:00Z -Test : Min=7.31012, Max=8.67468, Average=7.98569 -Test : Valid time: 2010-01-01T15:00:00Z -Test : Min=7.29043, Max=8.72532, Average=7.98624 -Test : CostJb : Nonlinear Jb = 0.705033 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.185862, nobs = 80, Jo/n = 0.00232328, err = 0.4 -Test : CostFunction: Nonlinear J = 0.890895 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 14.0524, nobs = 80, Jo/n = 0.175655, err = 0.4 +CostFunction: Nonlinear J = 14.0524 +DRIPCGMinimizer: reduction in residual norm = 0.0135197 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.05961, Max=8.303, Average=7.9761 + Valid time: 2010-01-01T04:30:00Z + Min=7.079, Max=8.31622, Average=7.97792 + Valid time: 2010-01-01T06:00:00Z + Min=7.10202, Max=8.36578, Average=7.97965 + Valid time: 2010-01-01T07:30:00Z + Min=7.12996, Max=8.42958, Average=7.98125 + Valid time: 2010-01-01T09:00:00Z + Min=7.1639, Max=8.49377, Average=7.98271 + Valid time: 2010-01-01T10:30:00Z + Min=7.2047, Max=8.55704, Average=7.98398 + Valid time: 2010-01-01T12:00:00Z + Min=7.25301, Max=8.6179, Average=7.98503 + Valid time: 2010-01-01T13:30:00Z + Min=7.3093, Max=8.6747, Average=7.98586 + Valid time: 2010-01-01T15:00:00Z + Min=7.29016, Max=8.72562, Average=7.98644 +CostJb : Nonlinear Jb = 0.704756 +CostJo : Nonlinear Jo(Lorenz 95) = 0.186593, nobs = 80, Jo/n = 0.00233242, err = 0.4 +CostFunction: Nonlinear J = 0.89135 +DRIPCGMinimizer: reduction in residual norm = 0.0509964 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.06017, Max=8.30253, Average=7.97607 + Valid time: 2010-01-01T04:30:00Z + Min=7.07966, Max=8.3157, Average=7.97788 + Valid time: 2010-01-01T06:00:00Z + Min=7.10277, Max=8.36569, Average=7.97959 + Valid time: 2010-01-01T07:30:00Z + Min=7.13078, Max=8.42965, Average=7.98118 + Valid time: 2010-01-01T09:00:00Z + Min=7.16476, Max=8.49393, Average=7.98261 + Valid time: 2010-01-01T10:30:00Z + Min=7.20558, Max=8.55723, Average=7.98386 + Valid time: 2010-01-01T12:00:00Z + Min=7.25388, Max=8.61804, Average=7.98489 + Valid time: 2010-01-01T13:30:00Z + Min=7.31012, Max=8.67468, Average=7.98569 + Valid time: 2010-01-01T15:00:00Z + Min=7.29043, Max=8.72532, Average=7.98624 +CostJb : Nonlinear Jb = 0.705033 +CostJo : Nonlinear Jo(Lorenz 95) = 0.185862, nobs = 80, Jo/n = 0.00232328, err = 0.4 +CostFunction: Nonlinear J = 0.890895 diff --git a/l95/test/testoutput/4dsaddlepoint.test b/l95/test/testoutput/4dsaddlepoint.test index 2c7004b8f..39b1f1667 100644 --- a/l95/test/testoutput/4dsaddlepoint.test +++ b/l95/test/testoutput/4dsaddlepoint.test @@ -1,21 +1,21 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostFunction: Nonlinear J = 123.856 -Test : SaddlePointMinimizer: reduction in residual norm = 0.008318 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.64279, Max=8.61955, Average=8.0027 -Test : Valid time: 2010-01-01T15:00:00Z -Test : Min=7.14059, Max=8.7605, Average=8.00521 -Test : CostJb : Nonlinear Jb = 2.33597 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.552166, nobs = 160, Jo/n = 0.00345104, err = 0.4 -Test : CostFunction: Nonlinear J = 2.88814 -Test : SaddlePointMinimizer: reduction in residual norm = 0.1199 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.6677, Max=8.64256, Average=8.01168 -Test : Valid time: 2010-01-01T15:00:00Z -Test : Min=7.14396, Max=8.77199, Average=8.01306 -Test : CostJb : Nonlinear Jb = 2.09932 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.38713, nobs = 160, Jo/n = 0.00241957, err = 0.4 -Test : CostFunction: Nonlinear J = 2.48645 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostFunction: Nonlinear J = 123.856 +SaddlePointMinimizer: reduction in residual norm = 0.008318 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.64279, Max=8.61955, Average=8.0027 + Valid time: 2010-01-01T15:00:00Z + Min=7.14059, Max=8.7605, Average=8.00521 +CostJb : Nonlinear Jb = 2.33597 +CostJo : Nonlinear Jo(Lorenz 95) = 0.552166, nobs = 160, Jo/n = 0.00345104, err = 0.4 +CostFunction: Nonlinear J = 2.88814 +SaddlePointMinimizer: reduction in residual norm = 0.1199 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.6677, Max=8.64256, Average=8.01168 + Valid time: 2010-01-01T15:00:00Z + Min=7.14396, Max=8.77199, Average=8.01306 +CostJb : Nonlinear Jb = 2.09932 +CostJo : Nonlinear Jo(Lorenz 95) = 0.38713, nobs = 160, Jo/n = 0.00241957, err = 0.4 +CostFunction: Nonlinear J = 2.48645 diff --git a/l95/test/testoutput/4dvar.allbiases.test b/l95/test/testoutput/4dvar.allbiases.test index a82ead419..525903822 100644 --- a/l95/test/testoutput/4dvar.allbiases.test +++ b/l95/test/testoutput/4dvar.allbiases.test @@ -1,24 +1,24 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 294.698, nobs = 160, Jo/n = 1.84186, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.338917 -Test : CostFunction: Nonlinear J = 295.037 -Test : DRPLanczosMinimizer: reduction in residual norm = 9.41823e-05 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.66146, Max=8.65431, Average=7.99961 -Test : ModelBias = 0.351503 -Test : ObsBias = 0.5881 -Test : CostJb : Nonlinear Jb = 2.37634 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.820597, nobs = 160, Jo/n = 0.00512873, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.365469 -Test : CostFunction: Nonlinear J = 3.5624 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.012503 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.66624, Max=8.66691, Average=7.99865 -Test : ModelBias = 0.276465 -Test : ObsBias = 0.590795 -Test : CostJb : Nonlinear Jb = 2.44253 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.525164, nobs = 160, Jo/n = 0.00328227, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.360309 -Test : CostFunction: Nonlinear J = 3.328 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 294.698, nobs = 160, Jo/n = 1.84186, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.338917 +CostFunction: Nonlinear J = 295.037 +DRPLanczosMinimizer: reduction in residual norm = 9.41823e-05 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.66146, Max=8.65431, Average=7.99961 +ModelBias = 0.351503 +ObsBias = 0.5881 +CostJb : Nonlinear Jb = 2.37634 +CostJo : Nonlinear Jo(Lorenz 95) = 0.820597, nobs = 160, Jo/n = 0.00512873, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.365469 +CostFunction: Nonlinear J = 3.5624 +DRPLanczosMinimizer: reduction in residual norm = 0.012503 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.66624, Max=8.66691, Average=7.99865 +ModelBias = 0.276465 +ObsBias = 0.590795 +CostJb : Nonlinear Jb = 2.44253 +CostJo : Nonlinear Jo(Lorenz 95) = 0.525164, nobs = 160, Jo/n = 0.00328227, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.360309 +CostFunction: Nonlinear J = 3.328 diff --git a/l95/test/testoutput/4dvar.alpha.test b/l95/test/testoutput/4dvar.alpha.test index 98a54a347..841af3a02 100644 --- a/l95/test/testoutput/4dvar.alpha.test +++ b/l95/test/testoutput/4dvar.alpha.test @@ -1,17 +1,17 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostFunction: Nonlinear J = 123.856 -Test : DRIPCGMinimizer: reduction in residual norm = 0.00691328 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.60159, Max=8.75709, Average=7.99918 -Test : CostJb : Nonlinear Jb = 2.56648 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.854775, nobs = 160, Jo/n = 0.00534235, err = 0.4 -Test : CostFunction: Nonlinear J = 3.42126 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0786184 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.64689, Max=8.79318, Average=8.00802 -Test : CostJb : Nonlinear Jb = 2.66159 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.483128, nobs = 160, Jo/n = 0.00301955, err = 0.4 -Test : CostFunction: Nonlinear J = 3.14471 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostFunction: Nonlinear J = 123.856 +DRIPCGMinimizer: reduction in residual norm = 0.00691328 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.60159, Max=8.75709, Average=7.99918 +CostJb : Nonlinear Jb = 2.56648 +CostJo : Nonlinear Jo(Lorenz 95) = 0.854775, nobs = 160, Jo/n = 0.00534235, err = 0.4 +CostFunction: Nonlinear J = 3.42126 +DRIPCGMinimizer: reduction in residual norm = 0.0786184 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.64689, Max=8.79318, Average=8.00802 +CostJb : Nonlinear Jb = 2.66159 +CostJo : Nonlinear Jo(Lorenz 95) = 0.483128, nobs = 160, Jo/n = 0.00301955, err = 0.4 +CostFunction: Nonlinear J = 3.14471 diff --git a/l95/test/testoutput/4dvar.drgmresr.test b/l95/test/testoutput/4dvar.drgmresr.test index 7bdcf74ad..ee9ed08c5 100644 --- a/l95/test/testoutput/4dvar.drgmresr.test +++ b/l95/test/testoutput/4dvar.drgmresr.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : DRGMRESRMinimizer: reduction in residual norm = 0.00485941 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65705, Max=8.64505, Average=8.00243 -Test : CostJb : Nonlinear Jb = 2.00001 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.93048, nobs = 160, Jo/n = 0.0058155, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.368687 -Test : CostFunction: Nonlinear J = 3.29917 -Test : DRGMRESRMinimizer: reduction in residual norm = 0.0489159 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67719, Max=8.67932, Average=8.01299 -Test : CostJb : Nonlinear Jb = 2.15374 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.519326, nobs = 160, Jo/n = 0.00324579, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.358326 -Test : CostFunction: Nonlinear J = 3.03139 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +DRGMRESRMinimizer: reduction in residual norm = 0.00485941 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65705, Max=8.64505, Average=8.00243 +CostJb : Nonlinear Jb = 2.00001 +CostJo : Nonlinear Jo(Lorenz 95) = 0.93048, nobs = 160, Jo/n = 0.0058155, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.368687 +CostFunction: Nonlinear J = 3.29917 +DRGMRESRMinimizer: reduction in residual norm = 0.0489159 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67719, Max=8.67932, Average=8.01299 +CostJb : Nonlinear Jb = 2.15374 +CostJo : Nonlinear Jo(Lorenz 95) = 0.519326, nobs = 160, Jo/n = 0.00324579, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.358326 +CostFunction: Nonlinear J = 3.03139 diff --git a/l95/test/testoutput/4dvar.dripcg.test b/l95/test/testoutput/4dvar.dripcg.test index aa0e6912f..7a3fc7739 100644 --- a/l95/test/testoutput/4dvar.dripcg.test +++ b/l95/test/testoutput/4dvar.dripcg.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : DRIPCGMinimizer: reduction in residual norm = 0.00514968 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65828, Max=8.64885, Average=8.00311 -Test : CostJb : Nonlinear Jb = 2.01229 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.369308 -Test : CostFunction: Nonlinear J = 3.28354 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0466805 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67765, Max=8.6797, Average=8.01314 -Test : CostJb : Nonlinear Jb = 2.15432 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.358483 -Test : CostFunction: Nonlinear J = 3.0311 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +DRIPCGMinimizer: reduction in residual norm = 0.00514968 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65828, Max=8.64885, Average=8.00311 +CostJb : Nonlinear Jb = 2.01229 +CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.369308 +CostFunction: Nonlinear J = 3.28354 +DRIPCGMinimizer: reduction in residual norm = 0.0466805 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67765, Max=8.6797, Average=8.01314 +CostJb : Nonlinear Jb = 2.15432 +CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.358483 +CostFunction: Nonlinear J = 3.0311 diff --git a/l95/test/testoutput/4dvar.dripcgqn.test b/l95/test/testoutput/4dvar.dripcgqn.test index dffd51d2b..af0bb0f5d 100644 --- a/l95/test/testoutput/4dvar.dripcgqn.test +++ b/l95/test/testoutput/4dvar.dripcgqn.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : DRIPCGMinimizer: reduction in residual norm = 0.00514968 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65828, Max=8.64885, Average=8.00311 -Test : CostJb : Nonlinear Jb = 2.01229 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.369308 -Test : CostFunction: Nonlinear J = 3.28354 -Test : DRIPCGMinimizer: reduction in residual norm = 0.312398 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.6717, Max=8.66983, Average=8.01049 -Test : CostJb : Nonlinear Jb = 2.12251 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.581871, nobs = 160, Jo/n = 0.00363669, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.356093 -Test : CostFunction: Nonlinear J = 3.06048 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +DRIPCGMinimizer: reduction in residual norm = 0.00514968 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65828, Max=8.64885, Average=8.00311 +CostJb : Nonlinear Jb = 2.01229 +CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.369308 +CostFunction: Nonlinear J = 3.28354 +DRIPCGMinimizer: reduction in residual norm = 0.312398 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.6717, Max=8.66983, Average=8.01049 +CostJb : Nonlinear Jb = 2.12251 +CostJo : Nonlinear Jo(Lorenz 95) = 0.581871, nobs = 160, Jo/n = 0.00363669, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.356093 +CostFunction: Nonlinear J = 3.06048 diff --git a/l95/test/testoutput/4dvar.drpcg.test b/l95/test/testoutput/4dvar.drpcg.test index f5566ddf1..0e8b6bc03 100644 --- a/l95/test/testoutput/4dvar.drpcg.test +++ b/l95/test/testoutput/4dvar.drpcg.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : DRPCGMinimizer: reduction in residual norm = 0.00364813 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65828, Max=8.64885, Average=8.00311 -Test : CostJb : Nonlinear Jb = 2.01229 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.369308 -Test : CostFunction: Nonlinear J = 3.28354 -Test : DRPCGMinimizer: reduction in residual norm = 0.0215466 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67765, Max=8.6797, Average=8.01314 -Test : CostJb : Nonlinear Jb = 2.15432 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.358483 -Test : CostFunction: Nonlinear J = 3.0311 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +DRPCGMinimizer: reduction in residual norm = 0.00364813 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65828, Max=8.64885, Average=8.00311 +CostJb : Nonlinear Jb = 2.01229 +CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.369308 +CostFunction: Nonlinear J = 3.28354 +DRPCGMinimizer: reduction in residual norm = 0.0215466 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67765, Max=8.6797, Average=8.01314 +CostJb : Nonlinear Jb = 2.15432 +CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.358483 +CostFunction: Nonlinear J = 3.0311 diff --git a/l95/test/testoutput/4dvar.drpcgqn.test b/l95/test/testoutput/4dvar.drpcgqn.test index 81e159878..a45187222 100644 --- a/l95/test/testoutput/4dvar.drpcgqn.test +++ b/l95/test/testoutput/4dvar.drpcgqn.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : DRPCGMinimizer: reduction in residual norm = 0.00364813 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65828, Max=8.64885, Average=8.00311 -Test : CostJb : Nonlinear Jb = 2.01229 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.369308 -Test : CostFunction: Nonlinear J = 3.28354 -Test : DRPCGMinimizer: reduction in residual norm = 0.0180937 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.6717, Max=8.66983, Average=8.01049 -Test : CostJb : Nonlinear Jb = 2.12253 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.581857, nobs = 160, Jo/n = 0.00363661, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.356092 -Test : CostFunction: Nonlinear J = 3.06048 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +DRPCGMinimizer: reduction in residual norm = 0.00364813 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65828, Max=8.64885, Average=8.00311 +CostJb : Nonlinear Jb = 2.01229 +CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.369308 +CostFunction: Nonlinear J = 3.28354 +DRPCGMinimizer: reduction in residual norm = 0.0180937 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.6717, Max=8.66983, Average=8.01049 +CostJb : Nonlinear Jb = 2.12253 +CostJo : Nonlinear Jo(Lorenz 95) = 0.581857, nobs = 160, Jo/n = 0.00363661, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.356092 +CostFunction: Nonlinear J = 3.06048 diff --git a/l95/test/testoutput/4dvar.drplanclmp.test b/l95/test/testoutput/4dvar.drplanclmp.test index 90cfb2716..5e4609a41 100644 --- a/l95/test/testoutput/4dvar.drplanclmp.test +++ b/l95/test/testoutput/4dvar.drplanclmp.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.00364813 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65828, Max=8.64885, Average=8.00311 -Test : CostJb : Nonlinear Jb = 2.01229 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.369308 -Test : CostFunction: Nonlinear J = 3.28354 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.0215466 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67765, Max=8.6797, Average=8.01314 -Test : CostJb : Nonlinear Jb = 2.15432 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.358483 -Test : CostFunction: Nonlinear J = 3.0311 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +DRPLanczosMinimizer: reduction in residual norm = 0.00364813 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65828, Max=8.64885, Average=8.00311 +CostJb : Nonlinear Jb = 2.01229 +CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.369308 +CostFunction: Nonlinear J = 3.28354 +DRPLanczosMinimizer: reduction in residual norm = 0.0215466 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67765, Max=8.6797, Average=8.01314 +CostJb : Nonlinear Jb = 2.15432 +CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.358483 +CostFunction: Nonlinear J = 3.0311 diff --git a/l95/test/testoutput/4dvar.drplanczos.test b/l95/test/testoutput/4dvar.drplanczos.test index 90cfb2716..5e4609a41 100644 --- a/l95/test/testoutput/4dvar.drplanczos.test +++ b/l95/test/testoutput/4dvar.drplanczos.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.00364813 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65828, Max=8.64885, Average=8.00311 -Test : CostJb : Nonlinear Jb = 2.01229 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.369308 -Test : CostFunction: Nonlinear J = 3.28354 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.0215466 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67765, Max=8.6797, Average=8.01314 -Test : CostJb : Nonlinear Jb = 2.15432 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.358483 -Test : CostFunction: Nonlinear J = 3.0311 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +DRPLanczosMinimizer: reduction in residual norm = 0.00364813 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65828, Max=8.64885, Average=8.00311 +CostJb : Nonlinear Jb = 2.01229 +CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.369308 +CostFunction: Nonlinear J = 3.28354 +DRPLanczosMinimizer: reduction in residual norm = 0.0215466 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67765, Max=8.6797, Average=8.01314 +CostJb : Nonlinear Jb = 2.15432 +CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.358483 +CostFunction: Nonlinear J = 3.0311 diff --git a/l95/test/testoutput/4dvar.fgmres.test b/l95/test/testoutput/4dvar.fgmres.test index e2b2ec7db..fdd3f6b3f 100644 --- a/l95/test/testoutput/4dvar.fgmres.test +++ b/l95/test/testoutput/4dvar.fgmres.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : FGMRESMinimizer: reduction in residual norm = 0.00485941 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65705, Max=8.64505, Average=8.00243 -Test : CostJb : Nonlinear Jb = 2.00001 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.93048, nobs = 160, Jo/n = 0.0058155, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.368687 -Test : CostFunction: Nonlinear J = 3.29917 -Test : FGMRESMinimizer: reduction in residual norm = 0.0489159 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67719, Max=8.67932, Average=8.01299 -Test : CostJb : Nonlinear Jb = 2.15374 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.519326, nobs = 160, Jo/n = 0.00324579, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.358326 -Test : CostFunction: Nonlinear J = 3.03139 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +FGMRESMinimizer: reduction in residual norm = 0.00485941 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65705, Max=8.64505, Average=8.00243 +CostJb : Nonlinear Jb = 2.00001 +CostJo : Nonlinear Jo(Lorenz 95) = 0.93048, nobs = 160, Jo/n = 0.0058155, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.368687 +CostFunction: Nonlinear J = 3.29917 +FGMRESMinimizer: reduction in residual norm = 0.0489159 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67719, Max=8.67932, Average=8.01299 +CostJb : Nonlinear Jb = 2.15374 +CostJo : Nonlinear Jo(Lorenz 95) = 0.519326, nobs = 160, Jo/n = 0.00324579, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.358326 +CostFunction: Nonlinear J = 3.03139 diff --git a/l95/test/testoutput/4dvar.gmresr.test b/l95/test/testoutput/4dvar.gmresr.test index 10a8b2db2..e6da77b3f 100644 --- a/l95/test/testoutput/4dvar.gmresr.test +++ b/l95/test/testoutput/4dvar.gmresr.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : GMRESRMinimizer: reduction in residual norm = 0.00485941 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65705, Max=8.64505, Average=8.00243 -Test : CostJb : Nonlinear Jb = 2.00001 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.93048, nobs = 160, Jo/n = 0.0058155, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.368687 -Test : CostFunction: Nonlinear J = 3.29917 -Test : GMRESRMinimizer: reduction in residual norm = 0.0489159 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67719, Max=8.67932, Average=8.01299 -Test : CostJb : Nonlinear Jb = 2.15374 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.519326, nobs = 160, Jo/n = 0.00324579, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.358326 -Test : CostFunction: Nonlinear J = 3.03139 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +GMRESRMinimizer: reduction in residual norm = 0.00485941 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65705, Max=8.64505, Average=8.00243 +CostJb : Nonlinear Jb = 2.00001 +CostJo : Nonlinear Jo(Lorenz 95) = 0.93048, nobs = 160, Jo/n = 0.0058155, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.368687 +CostFunction: Nonlinear J = 3.29917 +GMRESRMinimizer: reduction in residual norm = 0.0489159 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67719, Max=8.67932, Average=8.01299 +CostJb : Nonlinear Jb = 2.15374 +CostJo : Nonlinear Jo(Lorenz 95) = 0.519326, nobs = 160, Jo/n = 0.00324579, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.358326 +CostFunction: Nonlinear J = 3.03139 diff --git a/l95/test/testoutput/4dvar.hybrid.test b/l95/test/testoutput/4dvar.hybrid.test index e32854a69..c5814b5ef 100644 --- a/l95/test/testoutput/4dvar.hybrid.test +++ b/l95/test/testoutput/4dvar.hybrid.test @@ -1,17 +1,17 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostFunction: Nonlinear J = 123.856 -Test : DRIPCGMinimizer: reduction in residual norm = 0.00563633 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.64536, Max=8.69567, Average=8.00091 -Test : CostJb : Nonlinear Jb = 1.99321 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.827207, nobs = 160, Jo/n = 0.00517004, err = 0.4 -Test : CostFunction: Nonlinear J = 2.82041 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0738768 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67616, Max=8.73757, Average=8.01229 -Test : CostJb : Nonlinear Jb = 2.11184 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.410155, nobs = 160, Jo/n = 0.00256347, err = 0.4 -Test : CostFunction: Nonlinear J = 2.522 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostFunction: Nonlinear J = 123.856 +DRIPCGMinimizer: reduction in residual norm = 0.00563633 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.64536, Max=8.69567, Average=8.00091 +CostJb : Nonlinear Jb = 1.99321 +CostJo : Nonlinear Jo(Lorenz 95) = 0.827207, nobs = 160, Jo/n = 0.00517004, err = 0.4 +CostFunction: Nonlinear J = 2.82041 +DRIPCGMinimizer: reduction in residual norm = 0.0738768 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67616, Max=8.73757, Average=8.01229 +CostJb : Nonlinear Jb = 2.11184 +CostJo : Nonlinear Jo(Lorenz 95) = 0.410155, nobs = 160, Jo/n = 0.00256347, err = 0.4 +CostFunction: Nonlinear J = 2.522 diff --git a/l95/test/testoutput/4dvar.ipcg.test b/l95/test/testoutput/4dvar.ipcg.test index 9dedea5cb..d27749a30 100644 --- a/l95/test/testoutput/4dvar.ipcg.test +++ b/l95/test/testoutput/4dvar.ipcg.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : IPCGMinimizer: reduction in residual norm = 0.00515 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65828, Max=8.64885, Average=8.00311 -Test : CostJb : Nonlinear Jb = 2.01229 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.369308 -Test : CostFunction: Nonlinear J = 3.28354 -Test : IPCGMinimizer: reduction in residual norm = 0.04668 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67765, Max=8.6797, Average=8.01314 -Test : CostJb : Nonlinear Jb = 2.15432 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.358483 -Test : CostFunction: Nonlinear J = 3.0311 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +IPCGMinimizer: reduction in residual norm = 0.00515 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65828, Max=8.64885, Average=8.00311 +CostJb : Nonlinear Jb = 2.01229 +CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.369308 +CostFunction: Nonlinear J = 3.28354 +IPCGMinimizer: reduction in residual norm = 0.04668 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67765, Max=8.6797, Average=8.01314 +CostJb : Nonlinear Jb = 2.15432 +CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.358483 +CostFunction: Nonlinear J = 3.0311 diff --git a/l95/test/testoutput/4dvar.lbgmresr.test b/l95/test/testoutput/4dvar.lbgmresr.test index 8d5051141..408dffc17 100644 --- a/l95/test/testoutput/4dvar.lbgmresr.test +++ b/l95/test/testoutput/4dvar.lbgmresr.test @@ -1,18 +1,18 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.66169, Max=8.63937, Average=8.00288 -Test : CostJb : Nonlinear Jb = 1.95648 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.999028, nobs = 160, Jo/n = 0.00624392, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.371408 -Test : CostFunction: Nonlinear J = 3.32692 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67781, Max=8.6761, Average=8.01307 -Test : CostJb : Nonlinear Jb = 2.1336 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.541345, nobs = 160, Jo/n = 0.00338341, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.358376 -Test : CostFunction: Nonlinear J = 3.03332 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.66169, Max=8.63937, Average=8.00288 +CostJb : Nonlinear Jb = 1.95648 +CostJo : Nonlinear Jo(Lorenz 95) = 0.999028, nobs = 160, Jo/n = 0.00624392, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.371408 +CostFunction: Nonlinear J = 3.32692 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67781, Max=8.6761, Average=8.01307 +CostJb : Nonlinear Jb = 2.1336 +CostJo : Nonlinear Jo(Lorenz 95) = 0.541345, nobs = 160, Jo/n = 0.00338341, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.358376 +CostFunction: Nonlinear J = 3.03332 diff --git a/l95/test/testoutput/4dvar.minres.test b/l95/test/testoutput/4dvar.minres.test index 0a7436606..808628621 100644 --- a/l95/test/testoutput/4dvar.minres.test +++ b/l95/test/testoutput/4dvar.minres.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : MINRESMinimizer: reduction in residual norm = 0.00424441 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65934, Max=8.64257, Average=8.00267 -Test : CostJb : Nonlinear Jb = 1.98013 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.959576, nobs = 160, Jo/n = 0.00599735, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.369885 -Test : CostFunction: Nonlinear J = 3.30959 -Test : MINRESMinimizer: reduction in residual norm = 0.0273814 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67765, Max=8.6776, Average=8.01307 -Test : CostJb : Nonlinear Jb = 2.14261 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.531135, nobs = 160, Jo/n = 0.00331959, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.358352 -Test : CostFunction: Nonlinear J = 3.0321 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +MINRESMinimizer: reduction in residual norm = 0.00424441 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65934, Max=8.64257, Average=8.00267 +CostJb : Nonlinear Jb = 1.98013 +CostJo : Nonlinear Jo(Lorenz 95) = 0.959576, nobs = 160, Jo/n = 0.00599735, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.369885 +CostFunction: Nonlinear J = 3.30959 +MINRESMinimizer: reduction in residual norm = 0.0273814 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67765, Max=8.6776, Average=8.01307 +CostJb : Nonlinear Jb = 2.14261 +CostJo : Nonlinear Jo(Lorenz 95) = 0.531135, nobs = 160, Jo/n = 0.00331959, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.358352 +CostFunction: Nonlinear J = 3.0321 diff --git a/l95/test/testoutput/4dvar.modbias.test b/l95/test/testoutput/4dvar.modbias.test index 10f6db08d..f9c8742a2 100644 --- a/l95/test/testoutput/4dvar.modbias.test +++ b/l95/test/testoutput/4dvar.modbias.test @@ -1,22 +1,22 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 124.356, nobs = 160, Jo/n = 0.777224, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.338917 -Test : CostFunction: Nonlinear J = 124.695 -Test : DRIPCGMinimizer: reduction in residual norm = 0.00543932 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.64822, Max=8.63435, Average=7.98855 -Test : ModelBias = 0.364914 -Test : CostJb : Nonlinear Jb = 2.06408 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.856124, nobs = 160, Jo/n = 0.00535078, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.362824 -Test : CostFunction: Nonlinear J = 3.28303 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0524001 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65313, Max=8.65627, Average=7.98991 -Test : ModelBias = 0.286588 -Test : CostJb : Nonlinear Jb = 2.16432 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.525909, nobs = 160, Jo/n = 0.00328693, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.360505 -Test : CostFunction: Nonlinear J = 3.05074 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 124.356, nobs = 160, Jo/n = 0.777224, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.338917 +CostFunction: Nonlinear J = 124.695 +DRIPCGMinimizer: reduction in residual norm = 0.00543932 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.64822, Max=8.63435, Average=7.98855 +ModelBias = 0.364914 +CostJb : Nonlinear Jb = 2.06408 +CostJo : Nonlinear Jo(Lorenz 95) = 0.856124, nobs = 160, Jo/n = 0.00535078, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.362824 +CostFunction: Nonlinear J = 3.28303 +DRIPCGMinimizer: reduction in residual norm = 0.0524001 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65313, Max=8.65627, Average=7.98991 +ModelBias = 0.286588 +CostJb : Nonlinear Jb = 2.16432 +CostJo : Nonlinear Jo(Lorenz 95) = 0.525909, nobs = 160, Jo/n = 0.00328693, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.360505 +CostFunction: Nonlinear J = 3.05074 diff --git a/l95/test/testoutput/4dvar.obsbias.test b/l95/test/testoutput/4dvar.obsbias.test index 0fd7093e2..7e971b7be 100644 --- a/l95/test/testoutput/4dvar.obsbias.test +++ b/l95/test/testoutput/4dvar.obsbias.test @@ -1,22 +1,22 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 294.698, nobs = 160, Jo/n = 1.84186, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.338917 -Test : CostFunction: Nonlinear J = 295.037 -Test : DRIPCGMinimizer: reduction in residual norm = 8.3185e-05 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67871, Max=8.67067, Average=8.01684 -Test : ObsBias = 0.535611 -Test : CostJb : Nonlinear Jb = 2.30305 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 1.04203, nobs = 160, Jo/n = 0.00651266, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.378713 -Test : CostFunction: Nonlinear J = 3.72378 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0182695 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.6865, Max=8.68197, Average=8.01516 -Test : ObsBias = 0.546819 -Test : CostJb : Nonlinear Jb = 2.38623 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.701554, nobs = 160, Jo/n = 0.00438471, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.366753 -Test : CostFunction: Nonlinear J = 3.45454 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 294.698, nobs = 160, Jo/n = 1.84186, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.338917 +CostFunction: Nonlinear J = 295.037 +DRIPCGMinimizer: reduction in residual norm = 8.3185e-05 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67871, Max=8.67067, Average=8.01684 +ObsBias = 0.535611 +CostJb : Nonlinear Jb = 2.30305 +CostJo : Nonlinear Jo(Lorenz 95) = 1.04203, nobs = 160, Jo/n = 0.00651266, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.378713 +CostFunction: Nonlinear J = 3.72378 +DRIPCGMinimizer: reduction in residual norm = 0.0182695 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.6865, Max=8.68197, Average=8.01516 +ObsBias = 0.546819 +CostJb : Nonlinear Jb = 2.38623 +CostJo : Nonlinear Jo(Lorenz 95) = 0.701554, nobs = 160, Jo/n = 0.00438471, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.366753 +CostFunction: Nonlinear J = 3.45454 diff --git a/l95/test/testoutput/4dvar.pcg.test b/l95/test/testoutput/4dvar.pcg.test index dd79d4ade..1fc7f775c 100644 --- a/l95/test/testoutput/4dvar.pcg.test +++ b/l95/test/testoutput/4dvar.pcg.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : PCGMinimizer: reduction in residual norm = 0.00364813 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65828, Max=8.64885, Average=8.00311 -Test : CostJb : Nonlinear Jb = 2.01229 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.369308 -Test : CostFunction: Nonlinear J = 3.28354 -Test : PCGMinimizer: reduction in residual norm = 0.0215466 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67765, Max=8.6797, Average=8.01314 -Test : CostJb : Nonlinear Jb = 2.15432 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.358483 -Test : CostFunction: Nonlinear J = 3.0311 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +PCGMinimizer: reduction in residual norm = 0.00364813 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65828, Max=8.64885, Average=8.00311 +CostJb : Nonlinear Jb = 2.01229 +CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.369308 +CostFunction: Nonlinear J = 3.28354 +PCGMinimizer: reduction in residual norm = 0.0215466 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67765, Max=8.6797, Average=8.01314 +CostJb : Nonlinear Jb = 2.15432 +CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.358483 +CostFunction: Nonlinear J = 3.0311 diff --git a/l95/test/testoutput/4dvar.planczos.test b/l95/test/testoutput/4dvar.planczos.test index 25367850a..cc69e3567 100644 --- a/l95/test/testoutput/4dvar.planczos.test +++ b/l95/test/testoutput/4dvar.planczos.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : PLanczosMinimizer: reduction in residual norm = 0.00364813 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65828, Max=8.64885, Average=8.00311 -Test : CostJb : Nonlinear Jb = 2.01229 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.369308 -Test : CostFunction: Nonlinear J = 3.28354 -Test : PLanczosMinimizer: reduction in residual norm = 0.0215466 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67765, Max=8.6797, Average=8.01314 -Test : CostJb : Nonlinear Jb = 2.15432 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.358483 -Test : CostFunction: Nonlinear J = 3.0311 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +PLanczosMinimizer: reduction in residual norm = 0.00364813 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65828, Max=8.64885, Average=8.00311 +CostJb : Nonlinear Jb = 2.01229 +CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.369308 +CostFunction: Nonlinear J = 3.28354 +PLanczosMinimizer: reduction in residual norm = 0.0215466 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67765, Max=8.6797, Average=8.01314 +CostJb : Nonlinear Jb = 2.15432 +CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.358483 +CostFunction: Nonlinear J = 3.0311 diff --git a/l95/test/testoutput/4dvar.rpcg.test b/l95/test/testoutput/4dvar.rpcg.test index db4fbe93d..0d81fc623 100644 --- a/l95/test/testoutput/4dvar.rpcg.test +++ b/l95/test/testoutput/4dvar.rpcg.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : RPCGMinimizer: reduction in residual norm = 0.00364813 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65828, Max=8.64885, Average=8.00311 -Test : CostJb : Nonlinear Jb = 2.01229 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.369308 -Test : CostFunction: Nonlinear J = 3.28354 -Test : RPCGMinimizer: reduction in residual norm = 0.0215466 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67765, Max=8.6797, Average=8.01314 -Test : CostJb : Nonlinear Jb = 2.15432 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.358483 -Test : CostFunction: Nonlinear J = 3.0311 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +RPCGMinimizer: reduction in residual norm = 0.00364813 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65828, Max=8.64885, Average=8.00311 +CostJb : Nonlinear Jb = 2.01229 +CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.369308 +CostFunction: Nonlinear J = 3.28354 +RPCGMinimizer: reduction in residual norm = 0.0215466 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67765, Max=8.6797, Average=8.01314 +CostJb : Nonlinear Jb = 2.15432 +CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.358483 +CostFunction: Nonlinear J = 3.0311 diff --git a/l95/test/testoutput/4dvar.rplanczos.test b/l95/test/testoutput/4dvar.rplanczos.test index bbf930270..36321e5fe 100644 --- a/l95/test/testoutput/4dvar.rplanczos.test +++ b/l95/test/testoutput/4dvar.rplanczos.test @@ -1,20 +1,20 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.327606 -Test : CostFunction: Nonlinear J = 124.184 -Test : RPLanczosMinimizer: reduction in residual norm = 0.00364813 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.65828, Max=8.64885, Average=8.00311 -Test : CostJb : Nonlinear Jb = 2.01229 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.369308 -Test : CostFunction: Nonlinear J = 3.28354 -Test : RPLanczosMinimizer: reduction in residual norm = 0.0215466 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.67765, Max=8.6797, Average=8.01314 -Test : CostJb : Nonlinear Jb = 2.15432 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 -Test : CostJcDFI: Nonlinear Jc = 0.358483 -Test : CostFunction: Nonlinear J = 3.0311 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 123.856, nobs = 160, Jo/n = 0.774102, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.327606 +CostFunction: Nonlinear J = 124.184 +RPLanczosMinimizer: reduction in residual norm = 0.00364813 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.65828, Max=8.64885, Average=8.00311 +CostJb : Nonlinear Jb = 2.01229 +CostJo : Nonlinear Jo(Lorenz 95) = 0.901937, nobs = 160, Jo/n = 0.0056371, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.369308 +CostFunction: Nonlinear J = 3.28354 +RPLanczosMinimizer: reduction in residual norm = 0.0215466 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T03:00:00Z + Min=7.67765, Max=8.6797, Average=8.01314 +CostJb : Nonlinear Jb = 2.15432 +CostJo : Nonlinear Jo(Lorenz 95) = 0.518296, nobs = 160, Jo/n = 0.00323935, err = 0.4 +CostJcDFI: Nonlinear Jc = 0.358483 +CostFunction: Nonlinear J = 3.0311 diff --git a/l95/test/testoutput/addincrement.test b/l95/test/testoutput/addincrement.test index a8afe2b12..47d3f3f87 100644 --- a/l95/test/testoutput/addincrement.test +++ b/l95/test/testoutput/addincrement.test @@ -1,9 +1,9 @@ -Test : State: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.5867, Max=9.28852, Average=8.00062 -Test : Increment: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=-0.226841, Max=0.17469, Average=0.0151963 -Test : State plus increment: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.75469, Max=9.06168, Average=8.01582 +State: + Valid time: 2010-01-02T00:00:00Z + Min=6.5867, Max=9.28852, Average=8.00062 +Increment: + Valid time: 2010-01-02T00:00:00Z + Min=-0.226841, Max=0.17469, Average=0.0151963 +State plus increment: + Valid time: 2010-01-02T00:00:00Z + Min=6.75469, Max=9.06168, Average=8.01582 diff --git a/l95/test/testoutput/addincrement_scaled.test b/l95/test/testoutput/addincrement_scaled.test index 0a6f20d6a..6f482b3f4 100644 --- a/l95/test/testoutput/addincrement_scaled.test +++ b/l95/test/testoutput/addincrement_scaled.test @@ -1,12 +1,12 @@ -Test : State: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.5867, Max=9.28852, Average=8.00062 -Test : Increment: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=-0.226841, Max=0.17469, Average=0.0151963 -Test : Scaled the increment: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=-0.113421, Max=0.0873452, Average=0.00759817 -Test : State plus increment: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.6707, Max=9.1751, Average=8.00822 +State: + Valid time: 2010-01-02T00:00:00Z + Min=6.5867, Max=9.28852, Average=8.00062 +Increment: + Valid time: 2010-01-02T00:00:00Z + Min=-0.226841, Max=0.17469, Average=0.0151963 +Scaled the increment: + Valid time: 2010-01-02T00:00:00Z + Min=-0.113421, Max=0.0873452, Average=0.00759817 +State plus increment: + Valid time: 2010-01-02T00:00:00Z + Min=6.6707, Max=9.1751, Average=8.00822 diff --git a/l95/test/testoutput/diffstates.test b/l95/test/testoutput/diffstates.test index 637ca8abe..98941f212 100644 --- a/l95/test/testoutput/diffstates.test +++ b/l95/test/testoutput/diffstates.test @@ -1,9 +1,9 @@ -Test : Input state 1: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.75469, Max=9.06168, Average=8.01582 -Test : Input state 2: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.5867, Max=9.28852, Average=8.00062 -Test : Output increment: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=-0.226841, Max=0.17469, Average=0.0151963 +Input state 1: + Valid time: 2010-01-02T00:00:00Z + Min=6.75469, Max=9.06168, Average=8.01582 +Input state 2: + Valid time: 2010-01-02T00:00:00Z + Min=6.5867, Max=9.28852, Average=8.00062 +Output increment: + Valid time: 2010-01-02T00:00:00Z + Min=-0.226841, Max=0.17469, Average=0.0151963 diff --git a/l95/test/testoutput/eda_3dvar_block.test b/l95/test/testoutput/eda_3dvar_block.test index e51f9793a..6c0dcdad0 100644 --- a/l95/test/testoutput/eda_3dvar_block.test +++ b/l95/test/testoutput/eda_3dvar_block.test @@ -1,57 +1,57 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 1003.27, nobs = 120, Jo/n = 8.36061, err = 0.4 -Test : CostFunction: Nonlinear J = 1003.27 -Test : Norm reduction all members ( 1) = 0.226112, 0.246079 -Test : Quadratic cost function all members: J ( 1) = 157.904, 186.615 -Test : Norm reduction all members ( 2) = 0.0711168, 0.0801927 -Test : Quadratic cost function all members: J ( 2) = 119.121, 129.88 -Test : Norm reduction all members ( 3) = 0.0329696, 0.0324763 -Test : Quadratic cost function all members: J ( 3) = 114.134, 121.796 -Test : Norm reduction all members ( 4) = 0.0143961, 0.0146658 -Test : Quadratic cost function all members: J ( 4) = 113.103, 120.416 -Test : Norm reduction all members ( 5) = 0.00801082, 0.00603926 -Test : Quadratic cost function all members: J ( 5) = 112.853, 120.144 -Test : Norm reduction all members ( 6) = 0.00525854, 0.0040795 -Test : Quadratic cost function all members: J ( 6) = 112.769, 120.073 -Test : Norm reduction all members ( 7) = 0.00313676, 0.00251749 -Test : Quadratic cost function all members: J ( 7) = 112.735, 120.046 -Test : Norm reduction all members ( 8) = 0.00178513, 0.00129232 -Test : Quadratic cost function all members: J ( 8) = 112.723, 120.036 -Test : Norm reduction all members ( 9) = 0.000947141, 0.000990084 -Test : Quadratic cost function all members: J ( 9) = 112.719, 120.033 -Test : Norm reduction all members (10) = 0.000587397, 0.000553939 -Test : Quadratic cost function all members: J (10) = 112.718, 120.031 -Test : DRPBlockLanczosMinimizer: reduction in residual norm = 0.000587397 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.77364, Max=9.00861, Average=8.0056 -Test : CostJb : Nonlinear Jb = 97.3394 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 15.3786, nobs = 120, Jo/n = 0.128155, err = 0.4 -Test : CostFunction: Nonlinear J = 112.718 -Test : Norm reduction all members ( 1) = 0.603209, 0.531155 -Test : Quadratic cost function all members: J ( 1) = 112.718, 120.031 -Test : Norm reduction all members ( 2) = 0.486549, 0.413971 -Test : Quadratic cost function all members: J ( 2) = 112.718, 120.031 -Test : Norm reduction all members ( 3) = 0.225252, 0.225474 -Test : Quadratic cost function all members: J ( 3) = 112.717, 120.031 -Test : Norm reduction all members ( 4) = 0.136703, 0.150484 -Test : Quadratic cost function all members: J ( 4) = 112.717, 120.031 -Test : Norm reduction all members ( 5) = 0.0844788, 0.0897435 -Test : Quadratic cost function all members: J ( 5) = 112.717, 120.031 -Test : Norm reduction all members ( 6) = 0.0543314, 0.0522884 -Test : Quadratic cost function all members: J ( 6) = 112.717, 120.031 -Test : Norm reduction all members ( 7) = 0.0325225, 0.0286418 -Test : Quadratic cost function all members: J ( 7) = 112.717, 120.031 -Test : Norm reduction all members ( 8) = 0.0177981, 0.0162206 -Test : Quadratic cost function all members: J ( 8) = 112.717, 120.031 -Test : Norm reduction all members ( 9) = 0.00868724, 0.00777496 -Test : Quadratic cost function all members: J ( 9) = 112.717, 120.031 -Test : Norm reduction all members (10) = 0.00319633, 0.00372359 -Test : Quadratic cost function all members: J (10) = 112.717, 120.031 -Test : DRPBlockLanczosMinimizer: reduction in residual norm = 0.00319633 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.77193, Max=9.00912, Average=8.00572 -Test : CostJb : Nonlinear Jb = 97.3419 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 15.3755, nobs = 120, Jo/n = 0.128129, err = 0.4 -Test : CostFunction: Nonlinear J = 112.717 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 1003.27, nobs = 120, Jo/n = 8.36061, err = 0.4 +CostFunction: Nonlinear J = 1003.27 + Norm reduction all members ( 1) = 0.226112, 0.246079 + Quadratic cost function all members: J ( 1) = 157.904, 186.615 + Norm reduction all members ( 2) = 0.0711168, 0.0801927 + Quadratic cost function all members: J ( 2) = 119.121, 129.88 + Norm reduction all members ( 3) = 0.0329696, 0.0324763 + Quadratic cost function all members: J ( 3) = 114.134, 121.796 + Norm reduction all members ( 4) = 0.0143961, 0.0146658 + Quadratic cost function all members: J ( 4) = 113.103, 120.416 + Norm reduction all members ( 5) = 0.00801082, 0.00603926 + Quadratic cost function all members: J ( 5) = 112.853, 120.144 + Norm reduction all members ( 6) = 0.00525854, 0.0040795 + Quadratic cost function all members: J ( 6) = 112.769, 120.073 + Norm reduction all members ( 7) = 0.00313676, 0.00251749 + Quadratic cost function all members: J ( 7) = 112.735, 120.046 + Norm reduction all members ( 8) = 0.00178513, 0.00129232 + Quadratic cost function all members: J ( 8) = 112.723, 120.036 + Norm reduction all members ( 9) = 0.000947141, 0.000990084 + Quadratic cost function all members: J ( 9) = 112.719, 120.033 + Norm reduction all members (10) = 0.000587397, 0.000553939 + Quadratic cost function all members: J (10) = 112.718, 120.031 +DRPBlockLanczosMinimizer: reduction in residual norm = 0.000587397 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.77364, Max=9.00861, Average=8.0056 +CostJb : Nonlinear Jb = 97.3394 +CostJo : Nonlinear Jo(Lorenz 95) = 15.3786, nobs = 120, Jo/n = 0.128155, err = 0.4 +CostFunction: Nonlinear J = 112.718 + Norm reduction all members ( 1) = 0.603209, 0.531155 + Quadratic cost function all members: J ( 1) = 112.718, 120.031 + Norm reduction all members ( 2) = 0.486549, 0.413971 + Quadratic cost function all members: J ( 2) = 112.718, 120.031 + Norm reduction all members ( 3) = 0.225252, 0.225474 + Quadratic cost function all members: J ( 3) = 112.717, 120.031 + Norm reduction all members ( 4) = 0.136703, 0.150484 + Quadratic cost function all members: J ( 4) = 112.717, 120.031 + Norm reduction all members ( 5) = 0.0844788, 0.0897435 + Quadratic cost function all members: J ( 5) = 112.717, 120.031 + Norm reduction all members ( 6) = 0.0543314, 0.0522884 + Quadratic cost function all members: J ( 6) = 112.717, 120.031 + Norm reduction all members ( 7) = 0.0325225, 0.0286418 + Quadratic cost function all members: J ( 7) = 112.717, 120.031 + Norm reduction all members ( 8) = 0.0177981, 0.0162206 + Quadratic cost function all members: J ( 8) = 112.717, 120.031 + Norm reduction all members ( 9) = 0.00868724, 0.00777496 + Quadratic cost function all members: J ( 9) = 112.717, 120.031 + Norm reduction all members (10) = 0.00319633, 0.00372359 + Quadratic cost function all members: J (10) = 112.717, 120.031 +DRPBlockLanczosMinimizer: reduction in residual norm = 0.00319633 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.77193, Max=9.00912, Average=8.00572 +CostJb : Nonlinear Jb = 97.3419 +CostJo : Nonlinear Jo(Lorenz 95) = 15.3755, nobs = 120, Jo/n = 0.128129, err = 0.4 +CostFunction: Nonlinear J = 112.717 diff --git a/l95/test/testoutput/forecast.test b/l95/test/testoutput/forecast.test index 84b520cab..8c283e825 100644 --- a/l95/test/testoutput/forecast.test +++ b/l95/test/testoutput/forecast.test @@ -1,6 +1,6 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Min=7, Max=8, Average=7.975 -Test : Final state: -Test : Valid time: 2010-01-04T00:00:00Z -Test : Min=-5.23104, Max=15.2764, Average=6.45631 +Initial state: + Valid time: 2010-01-01T00:00:00Z + Min=7, Max=8, Average=7.975 +Final state: + Valid time: 2010-01-04T00:00:00Z + Min=-5.23104, Max=15.2764, Average=6.45631 diff --git a/l95/test/testoutput/forecast_identitymodel.test b/l95/test/testoutput/forecast_identitymodel.test index 707c20441..89d16c186 100644 --- a/l95/test/testoutput/forecast_identitymodel.test +++ b/l95/test/testoutput/forecast_identitymodel.test @@ -1,6 +1,6 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Min=7, Max=8, Average=7.975 -Test : Final state: -Test : Valid time: 2010-01-04T00:00:00Z -Test : Min=7, Max=8, Average=7.975 +Initial state: + Valid time: 2010-01-01T00:00:00Z + Min=7, Max=8, Average=7.975 +Final state: + Valid time: 2010-01-04T00:00:00Z + Min=7, Max=8, Average=7.975 diff --git a/l95/test/testoutput/forecast_pseudomodel.test b/l95/test/testoutput/forecast_pseudomodel.test index 84b520cab..8c283e825 100644 --- a/l95/test/testoutput/forecast_pseudomodel.test +++ b/l95/test/testoutput/forecast_pseudomodel.test @@ -1,6 +1,6 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Min=7, Max=8, Average=7.975 -Test : Final state: -Test : Valid time: 2010-01-04T00:00:00Z -Test : Min=-5.23104, Max=15.2764, Average=6.45631 +Initial state: + Valid time: 2010-01-01T00:00:00Z + Min=7, Max=8, Average=7.975 +Final state: + Valid time: 2010-01-04T00:00:00Z + Min=-5.23104, Max=15.2764, Average=6.45631 diff --git a/l95/test/testoutput/genenspert.test b/l95/test/testoutput/genenspert.test index 053dcf5c3..95f3fd974 100644 --- a/l95/test/testoutput/genenspert.test +++ b/l95/test/testoutput/genenspert.test @@ -1,33 +1,33 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Min=7, Max=8, Average=7.975 -Test : Member 0 final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=3.21752, Max=11.6627, Average=7.84602 -Test : Member 1 final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=3.37329, Max=11.3831, Average=7.67679 -Test : Member 2 final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=4.80264, Max=11.5457, Average=7.77423 -Test : Member 3 final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=3.15896, Max=11.4182, Average=7.71317 -Test : Member 4 final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=2.58571, Max=11.1087, Average=7.52551 -Test : Member 5 final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=2.47211, Max=11.9171, Average=7.73606 -Test : Member 6 final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=4.38182, Max=11.3824, Average=7.7142 -Test : Member 7 final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=4.1669, Max=11.0728, Average=7.62218 -Test : Member 8 final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=3.12835, Max=11.8686, Average=7.71032 -Test : Member 9 final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=4.08623, Max=11.7555, Average=7.82686 +Initial state: + Valid time: 2010-01-01T00:00:00Z + Min=7, Max=8, Average=7.975 +Member 0 final state: + Valid time: 2010-01-02T03:00:00Z + Min=3.21752, Max=11.6627, Average=7.84602 +Member 1 final state: + Valid time: 2010-01-02T03:00:00Z + Min=3.37329, Max=11.3831, Average=7.67679 +Member 2 final state: + Valid time: 2010-01-02T03:00:00Z + Min=4.80264, Max=11.5457, Average=7.77423 +Member 3 final state: + Valid time: 2010-01-02T03:00:00Z + Min=3.15896, Max=11.4182, Average=7.71317 +Member 4 final state: + Valid time: 2010-01-02T03:00:00Z + Min=2.58571, Max=11.1087, Average=7.52551 +Member 5 final state: + Valid time: 2010-01-02T03:00:00Z + Min=2.47211, Max=11.9171, Average=7.73606 +Member 6 final state: + Valid time: 2010-01-02T03:00:00Z + Min=4.38182, Max=11.3824, Average=7.7142 +Member 7 final state: + Valid time: 2010-01-02T03:00:00Z + Min=4.1669, Max=11.0728, Average=7.62218 +Member 8 final state: + Valid time: 2010-01-02T03:00:00Z + Min=3.12835, Max=11.8686, Average=7.71032 +Member 9 final state: + Valid time: 2010-01-02T03:00:00Z + Min=4.08623, Max=11.7555, Average=7.82686 diff --git a/l95/test/testoutput/getkf.test b/l95/test/testoutput/getkf.test index 7c4f417d3..220204cce 100644 --- a/l95/test/testoutput/getkf.test +++ b/l95/test/testoutput/getkf.test @@ -1,71 +1,102 @@ -Test : Initial state for member 1: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.53609, Max=11.1974, Average=7.90978 -Test : Initial state for member 2: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.77507, Max=11.1844, Average=7.75047 -Test : Initial state for member 3: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=5.36882, Max=10.844, Average=7.80576 -Test : Initial state for member 4: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=4.14639, Max=11.6734, Average=7.76001 -Test : Initial state for member 5: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.17563, Max=10.8183, Average=7.60243 -Test : H(x) for member 1: -Test : Lorenz 95 nobs= 120 Min=3.53609, Max=11.1974, Average=7.90978 -Test : H(x) for member 2: -Test : Lorenz 95 nobs= 120 Min=3.77507, Max=11.1844, Average=7.75047 -Test : H(x) for member 3: -Test : Lorenz 95 nobs= 120 Min=5.36882, Max=10.844, Average=7.80576 -Test : H(x) for member 4: -Test : Lorenz 95 nobs= 120 Min=4.14639, Max=11.6734, Average=7.76001 -Test : H(x) for member 5: -Test : Lorenz 95 nobs= 120 Min=3.17563, Max=10.8183, Average=7.60243 -Test : H(Zx) - ymean for member 1 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-1.98234, Max=2.72035, Average=0.144087 -Test : H(Zx) - ymean for member 2 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-2.99197, Max=2.48196, Average=-0.0152161 -Test : H(Zx) - ymean for member 3 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-2.26382, Max=3.22819, Average=0.0400705 -Test : H(Zx) - ymean for member 4 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-3.42628, Max=3.04792, Average=-0.00567841 -Test : H(Zx) - ymean for member 5 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-3.79871, Max=2.34903, Average=-0.163263 -Test : H(x) ensemble background mean: -Test : Lorenz 95 nobs= 120 Min=5.25199, Max=9.50208, Average=7.76569 -Test : background y - H(x): -Test : Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 -Test : Background mean : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=5.25199, Max=9.50208, Average=7.76569 -Test : Analysis mean : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.73405, Max=9.18815, Average=7.98704 -Test : H(x) for member 1: -Test : Lorenz 95 nobs= 120 Min=6.77434, Max=9.33536, Average=8.03454 -Test : H(x) for member 2: -Test : Lorenz 95 nobs= 120 Min=6.03404, Max=9.36204, Average=7.98875 -Test : H(x) for member 3: -Test : Lorenz 95 nobs= 120 Min=6.25955, Max=9.88003, Average=8.00061 -Test : H(x) for member 4: -Test : Lorenz 95 nobs= 120 Min=5.4671, Max=10.7046, Average=7.99636 -Test : H(x) for member 5: -Test : Lorenz 95 nobs= 120 Min=5.91405, Max=9.12249, Average=7.91495 -Test : H(Zx) - ymean for member 1 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-0.897846, Max=1.3052, Average=0.0475003 -Test : H(Zx) - ymean for member 2 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-1.76882, Max=1.36093, Average=0.00170744 -Test : H(Zx) - ymean for member 3 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-1.23029, Max=1.80876, Average=0.0135681 -Test : H(Zx) - ymean for member 4 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-1.76957, Max=2.06616, Average=0.00931763 -Test : H(Zx) - ymean for member 5 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-2.0385, Max=1.22627, Average=-0.0720935 -Test : H(x) ensemble analysis mean: -Test : Lorenz 95 nobs= 120 Min=6.73405, Max=9.18815, Average=7.98704 -Test : analysis y - H(x): -Test : Lorenz 95 nobs= 120 Min=-0.272405, Max=0.44415, Average=0.0243479 -Test : ombg RMS: 0.864096 -Test : oman RMS: 0.149029 +Initial state for member 1: + Valid time: 2010-01-02T00:00:00Z + Min=3.53609, Max=11.1974, Average=7.90978 + +Initial state for member 2: + Valid time: 2010-01-02T00:00:00Z + Min=3.77507, Max=11.1844, Average=7.75047 + +Initial state for member 3: + Valid time: 2010-01-02T00:00:00Z + Min=5.36882, Max=10.844, Average=7.80576 + +Initial state for member 4: + Valid time: 2010-01-02T00:00:00Z + Min=4.14639, Max=11.6734, Average=7.76001 + +Initial state for member 5: + Valid time: 2010-01-02T00:00:00Z + Min=3.17563, Max=10.8183, Average=7.60243 + +H(x) for member 1: +Lorenz 95 nobs= 120 Min=3.53609, Max=11.1974, Average=7.90978 + +H(x) for member 2: +Lorenz 95 nobs= 120 Min=3.77507, Max=11.1844, Average=7.75047 + +H(x) for member 3: +Lorenz 95 nobs= 120 Min=5.36882, Max=10.844, Average=7.80576 + +H(x) for member 4: +Lorenz 95 nobs= 120 Min=4.14639, Max=11.6734, Average=7.76001 + +H(x) for member 5: +Lorenz 95 nobs= 120 Min=3.17563, Max=10.8183, Average=7.60243 + +H(Zx) - ymean for member 1 eig 1 : +Lorenz 95 nobs= 120 Min=-1.98234, Max=2.72035, Average=0.144087 + +H(Zx) - ymean for member 2 eig 1 : +Lorenz 95 nobs= 120 Min=-2.99197, Max=2.48196, Average=-0.0152161 + +H(Zx) - ymean for member 3 eig 1 : +Lorenz 95 nobs= 120 Min=-2.26382, Max=3.22819, Average=0.0400705 + +H(Zx) - ymean for member 4 eig 1 : +Lorenz 95 nobs= 120 Min=-3.42628, Max=3.04792, Average=-0.00567841 + +H(Zx) - ymean for member 5 eig 1 : +Lorenz 95 nobs= 120 Min=-3.79871, Max=2.34903, Average=-0.163263 + +H(x) ensemble background mean: +Lorenz 95 nobs= 120 Min=5.25199, Max=9.50208, Average=7.76569 + +background y - H(x): +Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 + +Background mean : + Valid time: 2010-01-02T00:00:00Z + Min=5.25199, Max=9.50208, Average=7.76569 + +Analysis mean : + Valid time: 2010-01-02T00:00:00Z + Min=6.73405, Max=9.18815, Average=7.98704 + +H(x) for member 1: +Lorenz 95 nobs= 120 Min=6.77434, Max=9.33537, Average=8.03454 + +H(x) for member 2: +Lorenz 95 nobs= 120 Min=6.03404, Max=9.36204, Average=7.98875 + +H(x) for member 3: +Lorenz 95 nobs= 120 Min=6.25955, Max=9.88003, Average=8.00061 + +H(x) for member 4: +Lorenz 95 nobs= 120 Min=5.4671, Max=10.7046, Average=7.99636 + +H(x) for member 5: +Lorenz 95 nobs= 120 Min=5.91405, Max=9.12249, Average=7.91495 + +H(Zx) - ymean for member 1 eig 1 : +Lorenz 95 nobs= 120 Min=-0.89785, Max=1.3052, Average=0.0475003 + +H(Zx) - ymean for member 2 eig 1 : +Lorenz 95 nobs= 120 Min=-1.76882, Max=1.36094, Average=0.00170603 + +H(Zx) - ymean for member 3 eig 1 : +Lorenz 95 nobs= 120 Min=-1.23029, Max=1.80876, Average=0.0135676 + +H(Zx) - ymean for member 4 eig 1 : +Lorenz 95 nobs= 120 Min=-1.76957, Max=2.06616, Average=0.00931818 + +H(Zx) - ymean for member 5 eig 1 : +Lorenz 95 nobs= 120 Min=-2.0385, Max=1.22627, Average=-0.0720921 + +H(x) ensemble analysis mean: +Lorenz 95 nobs= 120 Min=6.73405, Max=9.18815, Average=7.98704 + +analysis y - H(x): +Lorenz 95 nobs= 120 Min=-0.272405, Max=0.444146, Average=0.0243478 + +ombg RMS: 0.864096 +oman RMS: 0.149028 diff --git a/l95/test/testoutput/getkf_offline_hofx.test b/l95/test/testoutput/getkf_offline_hofx.test index 82f81b96c..ad38d35bf 100644 --- a/l95/test/testoutput/getkf_offline_hofx.test +++ b/l95/test/testoutput/getkf_offline_hofx.test @@ -1,45 +1,64 @@ -Test : Initial state for member 1: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.53609, Max=11.1974, Average=7.90978 -Test : Initial state for member 2: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.77507, Max=11.1844, Average=7.75047 -Test : Initial state for member 3: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=5.36882, Max=10.844, Average=7.80576 -Test : Initial state for member 4: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=4.14639, Max=11.6734, Average=7.76001 -Test : Initial state for member 5: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.17563, Max=10.8183, Average=7.60243 -Test : H(x) for member 1: -Test : Lorenz 95 nobs= 120 Min=3.53609, Max=11.1974, Average=7.90978 -Test : H(x) for member 2: -Test : Lorenz 95 nobs= 120 Min=3.77507, Max=11.1844, Average=7.75048 -Test : H(x) for member 3: -Test : Lorenz 95 nobs= 120 Min=5.36882, Max=10.844, Average=7.80576 -Test : H(x) for member 4: -Test : Lorenz 95 nobs= 120 Min=4.14639, Max=11.6734, Average=7.76001 -Test : H(x) for member 5: -Test : Lorenz 95 nobs= 120 Min=3.17563, Max=10.8183, Average=7.60243 -Test : H(Zx) - ymean for member 1 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-1.98233, Max=2.72039, Average=0.144086 -Test : H(Zx) - ymean for member 2 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-2.99197, Max=2.48195, Average=-0.0152143 -Test : H(Zx) - ymean for member 3 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-2.26382, Max=3.22817, Average=0.040069 -Test : H(Zx) - ymean for member 4 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-3.42627, Max=3.04791, Average=-0.0056785 -Test : H(Zx) - ymean for member 5 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-3.79871, Max=2.34904, Average=-0.163263 -Test : H(x) ensemble background mean: -Test : Lorenz 95 nobs= 120 Min=5.25199, Max=9.50207, Average=7.76569 -Test : background y - H(x): -Test : Lorenz 95 nobs= 120 Min=-1.48378, Max=2.74405, Average=0.2457 -Test : Background mean : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=5.25199, Max=9.50208, Average=7.76569 -Test : Analysis mean : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.73405, Max=9.18814, Average=7.98704 +Initial state for member 1: + Valid time: 2010-01-02T00:00:00Z + Min=3.53609, Max=11.1974, Average=7.90978 + +Initial state for member 2: + Valid time: 2010-01-02T00:00:00Z + Min=3.77507, Max=11.1844, Average=7.75047 + +Initial state for member 3: + Valid time: 2010-01-02T00:00:00Z + Min=5.36882, Max=10.844, Average=7.80576 + +Initial state for member 4: + Valid time: 2010-01-02T00:00:00Z + Min=4.14639, Max=11.6734, Average=7.76001 + +Initial state for member 5: + Valid time: 2010-01-02T00:00:00Z + Min=3.17563, Max=10.8183, Average=7.60243 + +H(x) for member 1: +Lorenz 95 nobs= 120 Min=3.53609, Max=11.1974, Average=7.90978 + +H(x) for member 2: +Lorenz 95 nobs= 120 Min=3.77507, Max=11.1844, Average=7.75048 + +H(x) for member 3: +Lorenz 95 nobs= 120 Min=5.36882, Max=10.844, Average=7.80576 + +H(x) for member 4: +Lorenz 95 nobs= 120 Min=4.14639, Max=11.6734, Average=7.76001 + +H(x) for member 5: +Lorenz 95 nobs= 120 Min=3.17563, Max=10.8183, Average=7.60243 + +H(Zx) - ymean for member 1 eig 1 : +Lorenz 95 nobs= 120 Min=-1.98233, Max=2.72039, Average=0.144086 + +H(Zx) - ymean for member 2 eig 1 : +Lorenz 95 nobs= 120 Min=-2.99197, Max=2.48195, Average=-0.0152143 + +H(Zx) - ymean for member 3 eig 1 : +Lorenz 95 nobs= 120 Min=-2.26382, Max=3.22817, Average=0.040069 + +H(Zx) - ymean for member 4 eig 1 : +Lorenz 95 nobs= 120 Min=-3.42627, Max=3.04791, Average=-0.0056785 + +H(Zx) - ymean for member 5 eig 1 : +Lorenz 95 nobs= 120 Min=-3.79871, Max=2.34904, Average=-0.163263 + +H(x) ensemble background mean: +Lorenz 95 nobs= 120 Min=5.25199, Max=9.50207, Average=7.76569 + +background y - H(x): +Lorenz 95 nobs= 120 Min=-1.48378, Max=2.74405, Average=0.2457 + +Background mean : + Valid time: 2010-01-02T00:00:00Z + Min=5.25199, Max=9.50208, Average=7.76569 + +Analysis mean : + Valid time: 2010-01-02T00:00:00Z + Min=6.73405, Max=9.18814, Average=7.98704 + diff --git a/l95/test/testoutput/hofx.test b/l95/test/testoutput/hofx.test index a9081c852..bf45c809f 100644 --- a/l95/test/testoutput/hofx.test +++ b/l95/test/testoutput/hofx.test @@ -1,9 +1,9 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Min=7, Max=8, Average=7.975 -Test : Final state: -Test : Valid time: 2010-01-03T00:00:00Z -Test : Min=3.26487, Max=12.3145, Average=7.82855 -Test : H(x): -Test : Lorenz 95 nobs= 160 Min=6.34504, Max=9.44115, Average=7.97687 -Test : End H(x) +Initial state: + Valid time: 2010-01-01T00:00:00Z + Min=7, Max=8, Average=7.975 +Final state: + Valid time: 2010-01-03T00:00:00Z + Min=3.26487, Max=12.3145, Average=7.82855 +H(x): +Lorenz 95 nobs= 160 Min=6.34504, Max=9.44115, Average=7.97687 +End H(x) diff --git a/l95/test/testoutput/hofx3d.test b/l95/test/testoutput/hofx3d.test index f9fb5eeb4..808815f50 100644 --- a/l95/test/testoutput/hofx3d.test +++ b/l95/test/testoutput/hofx3d.test @@ -1,6 +1,6 @@ -Test : State: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.65953, Max=9.39191, Average=7.97085 -Test : H(x): -Test : Lorenz 95 nobs= 120 Min=6.65953, Max=9.39191, Average=7.97085 -Test : End H(x) +State: + Valid time: 2010-01-02T00:00:00Z + Min=6.65953, Max=9.39191, Average=7.97085 +H(x): +Lorenz 95 nobs= 120 Min=6.65953, Max=9.39191, Average=7.97085 +End H(x) diff --git a/l95/test/testoutput/hofx3d_for_getkf.test b/l95/test/testoutput/hofx3d_for_getkf.test index 184676ffb..4a867267d 100644 --- a/l95/test/testoutput/hofx3d_for_getkf.test +++ b/l95/test/testoutput/hofx3d_for_getkf.test @@ -1,39 +1,56 @@ -Test : Initial state for member 1: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.53609, Max=11.1974, Average=7.90978 -Test : Initial state for member 2: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.77507, Max=11.1844, Average=7.75047 -Test : Initial state for member 3: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=5.36882, Max=10.844, Average=7.80576 -Test : Initial state for member 4: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=4.14639, Max=11.6734, Average=7.76001 -Test : Initial state for member 5: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.17563, Max=10.8183, Average=7.60243 -Test : H(x) for member 1: -Test : Lorenz 95 nobs= 120 Min=3.53609, Max=11.1974, Average=7.90978 -Test : H(x) for member 2: -Test : Lorenz 95 nobs= 120 Min=3.77507, Max=11.1844, Average=7.75047 -Test : H(x) for member 3: -Test : Lorenz 95 nobs= 120 Min=5.36882, Max=10.844, Average=7.80576 -Test : H(x) for member 4: -Test : Lorenz 95 nobs= 120 Min=4.14639, Max=11.6734, Average=7.76001 -Test : H(x) for member 5: -Test : Lorenz 95 nobs= 120 Min=3.17563, Max=10.8183, Average=7.60243 -Test : H(Zx) - ymean for member 1 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-1.98234, Max=2.72035, Average=0.144087 -Test : H(Zx) - ymean for member 2 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-2.99197, Max=2.48196, Average=-0.0152161 -Test : H(Zx) - ymean for member 3 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-2.26382, Max=3.22819, Average=0.0400705 -Test : H(Zx) - ymean for member 4 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-3.42628, Max=3.04792, Average=-0.00567841 -Test : H(Zx) - ymean for member 5 eig 1 : -Test : Lorenz 95 nobs= 120 Min=-3.79871, Max=2.34903, Average=-0.163263 -Test : H(x) ensemble background mean: -Test : Lorenz 95 nobs= 120 Min=5.25199, Max=9.50208, Average=7.76569 -Test : background y - H(x): -Test : Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 +Initial state for member 1: + Valid time: 2010-01-02T00:00:00Z + Min=3.53609, Max=11.1974, Average=7.90978 + +Initial state for member 2: + Valid time: 2010-01-02T00:00:00Z + Min=3.77507, Max=11.1844, Average=7.75047 + +Initial state for member 3: + Valid time: 2010-01-02T00:00:00Z + Min=5.36882, Max=10.844, Average=7.80576 + +Initial state for member 4: + Valid time: 2010-01-02T00:00:00Z + Min=4.14639, Max=11.6734, Average=7.76001 + +Initial state for member 5: + Valid time: 2010-01-02T00:00:00Z + Min=3.17563, Max=10.8183, Average=7.60243 + +H(x) for member 1: +Lorenz 95 nobs= 120 Min=3.53609, Max=11.1974, Average=7.90978 + +H(x) for member 2: +Lorenz 95 nobs= 120 Min=3.77507, Max=11.1844, Average=7.75047 + +H(x) for member 3: +Lorenz 95 nobs= 120 Min=5.36882, Max=10.844, Average=7.80576 + +H(x) for member 4: +Lorenz 95 nobs= 120 Min=4.14639, Max=11.6734, Average=7.76001 + +H(x) for member 5: +Lorenz 95 nobs= 120 Min=3.17563, Max=10.8183, Average=7.60243 + +H(Zx) - ymean for member 1 eig 1 : +Lorenz 95 nobs= 120 Min=-1.98234, Max=2.72035, Average=0.144087 + +H(Zx) - ymean for member 2 eig 1 : +Lorenz 95 nobs= 120 Min=-2.99197, Max=2.48196, Average=-0.0152161 + +H(Zx) - ymean for member 3 eig 1 : +Lorenz 95 nobs= 120 Min=-2.26382, Max=3.22819, Average=0.0400705 + +H(Zx) - ymean for member 4 eig 1 : +Lorenz 95 nobs= 120 Min=-3.42628, Max=3.04792, Average=-0.00567841 + +H(Zx) - ymean for member 5 eig 1 : +Lorenz 95 nobs= 120 Min=-3.79871, Max=2.34903, Average=-0.163263 + +H(x) ensemble background mean: +Lorenz 95 nobs= 120 Min=5.25199, Max=9.50208, Average=7.76569 + +background y - H(x): +Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 + diff --git a/l95/test/testoutput/letkf.test b/l95/test/testoutput/letkf.test index 4c8324c99..d7095acef 100644 --- a/l95/test/testoutput/letkf.test +++ b/l95/test/testoutput/letkf.test @@ -1,60 +1,81 @@ -Test : Initial state for member 1: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.53609, Max=11.1974, Average=7.90978 -Test : Initial state for member 2: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.77507, Max=11.1844, Average=7.75047 -Test : Initial state for member 3: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=5.36882, Max=10.844, Average=7.80576 -Test : Initial state for member 4: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=4.14639, Max=11.6734, Average=7.76001 -Test : Initial state for member 5: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.17563, Max=10.8183, Average=7.60243 -Test : H(x) for member 1: -Test : Lorenz 95 nobs= 120 Min=3.53609, Max=11.1974, Average=7.90978 -Test : H(x) for member 2: -Test : Lorenz 95 nobs= 120 Min=3.77507, Max=11.1844, Average=7.75047 -Test : H(x) for member 3: -Test : Lorenz 95 nobs= 120 Min=5.36882, Max=10.844, Average=7.80576 -Test : H(x) for member 4: -Test : Lorenz 95 nobs= 120 Min=4.14639, Max=11.6734, Average=7.76001 -Test : H(x) for member 5: -Test : Lorenz 95 nobs= 120 Min=3.17563, Max=10.8183, Average=7.60243 -Test : H(x) ensemble background mean: -Test : Lorenz 95 nobs= 120 Min=5.25199, Max=9.50208, Average=7.76569 -Test : background y - H(x): -Test : Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 -Test : Background mean : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=5.25199, Max=9.50208, Average=7.76569 -Test : Analysis mean : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.73044, Max=9.19002, Average=7.98832 -Test : Analysis mean increment : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=-1.55316, Max=2.41362, Average=0.222628 -Test : Forecast variance : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=0.180053, Max=5.95179, Average=2.16311 -Test : Analysis variance : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=0.216453, Max=8.21437, Average=2.10118 -Test : H(x) for member 1: -Test : Lorenz 95 nobs= 120 Min=6.31712, Max=10.1291, Average=8.09746 -Test : H(x) for member 2: -Test : Lorenz 95 nobs= 120 Min=5.53728, Max=9.94295, Average=7.97522 -Test : H(x) for member 3: -Test : Lorenz 95 nobs= 120 Min=5.98634, Max=10.5924, Average=8.01708 -Test : H(x) for member 4: -Test : Lorenz 95 nobs= 120 Min=4.57542, Max=11.5295, Average=7.98557 -Test : H(x) for member 5: -Test : Lorenz 95 nobs= 120 Min=5.01152, Max=9.81763, Average=7.86627 -Test : H(x) ensemble analysis mean: -Test : Lorenz 95 nobs= 120 Min=6.73044, Max=9.19002, Average=7.98832 -Test : analysis y - H(x): -Test : Lorenz 95 nobs= 120 Min=-0.265703, Max=0.441464, Average=0.0230717 -Test : ombg RMS: 0.864096 -Test : oman RMS: 0.146322 +Initial state for member 1: + Valid time: 2010-01-02T00:00:00Z + Min=3.53609, Max=11.1974, Average=7.90978 + +Initial state for member 2: + Valid time: 2010-01-02T00:00:00Z + Min=3.77507, Max=11.1844, Average=7.75047 + +Initial state for member 3: + Valid time: 2010-01-02T00:00:00Z + Min=5.36882, Max=10.844, Average=7.80576 + +Initial state for member 4: + Valid time: 2010-01-02T00:00:00Z + Min=4.14639, Max=11.6734, Average=7.76001 + +Initial state for member 5: + Valid time: 2010-01-02T00:00:00Z + Min=3.17563, Max=10.8183, Average=7.60243 + +H(x) for member 1: +Lorenz 95 nobs= 120 Min=3.53609, Max=11.1974, Average=7.90978 + +H(x) for member 2: +Lorenz 95 nobs= 120 Min=3.77507, Max=11.1844, Average=7.75047 + +H(x) for member 3: +Lorenz 95 nobs= 120 Min=5.36882, Max=10.844, Average=7.80576 + +H(x) for member 4: +Lorenz 95 nobs= 120 Min=4.14639, Max=11.6734, Average=7.76001 + +H(x) for member 5: +Lorenz 95 nobs= 120 Min=3.17563, Max=10.8183, Average=7.60243 + +H(x) ensemble background mean: +Lorenz 95 nobs= 120 Min=5.25199, Max=9.50208, Average=7.76569 + +background y - H(x): +Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 + +Background mean : + Valid time: 2010-01-02T00:00:00Z + Min=5.25199, Max=9.50208, Average=7.76569 + +Analysis mean : + Valid time: 2010-01-02T00:00:00Z + Min=6.73044, Max=9.19002, Average=7.98832 + +Analysis mean increment : + Valid time: 2010-01-02T00:00:00Z + Min=-1.55316, Max=2.41362, Average=0.222628 +Forecast variance : + Valid time: 2010-01-02T00:00:00Z + Min=0.180053, Max=5.95179, Average=2.16311 +Analysis variance : + Valid time: 2010-01-02T00:00:00Z + Min=0.216453, Max=8.21437, Average=2.10118 +H(x) for member 1: +Lorenz 95 nobs= 120 Min=6.31712, Max=10.1291, Average=8.09746 + +H(x) for member 2: +Lorenz 95 nobs= 120 Min=5.53728, Max=9.94295, Average=7.97522 + +H(x) for member 3: +Lorenz 95 nobs= 120 Min=5.98634, Max=10.5924, Average=8.01708 + +H(x) for member 4: +Lorenz 95 nobs= 120 Min=4.57542, Max=11.5295, Average=7.98557 + +H(x) for member 5: +Lorenz 95 nobs= 120 Min=5.01152, Max=9.81763, Average=7.86627 + +H(x) ensemble analysis mean: +Lorenz 95 nobs= 120 Min=6.73044, Max=9.19002, Average=7.98832 + +analysis y - H(x): +Lorenz 95 nobs= 120 Min=-0.265703, Max=0.441464, Average=0.0230717 + +ombg RMS: 0.864096 +oman RMS: 0.146322 diff --git a/l95/test/testoutput/letkf_gsi.test b/l95/test/testoutput/letkf_gsi.test index f8ef4eaa2..cb6c12b8c 100644 --- a/l95/test/testoutput/letkf_gsi.test +++ b/l95/test/testoutput/letkf_gsi.test @@ -1,51 +1,72 @@ -Test : Initial state for member 1: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.53609, Max=11.1974, Average=7.90978 -Test : Initial state for member 2: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.77507, Max=11.1844, Average=7.75047 -Test : Initial state for member 3: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=5.36882, Max=10.844, Average=7.80576 -Test : Initial state for member 4: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=4.14639, Max=11.6734, Average=7.76001 -Test : Initial state for member 5: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.17563, Max=10.8183, Average=7.60243 -Test : H(x) for member 1: -Test : Lorenz 95 nobs= 120 Min=3.53609, Max=11.1974, Average=7.90978 -Test : H(x) for member 2: -Test : Lorenz 95 nobs= 120 Min=3.77507, Max=11.1844, Average=7.75047 -Test : H(x) for member 3: -Test : Lorenz 95 nobs= 120 Min=5.36882, Max=10.844, Average=7.80576 -Test : H(x) for member 4: -Test : Lorenz 95 nobs= 120 Min=4.14639, Max=11.6734, Average=7.76001 -Test : H(x) for member 5: -Test : Lorenz 95 nobs= 120 Min=3.17563, Max=10.8183, Average=7.60243 -Test : H(x) ensemble background mean: -Test : Lorenz 95 nobs= 120 Min=5.25199, Max=9.50208, Average=7.76569 -Test : background y - H(x): -Test : Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 -Test : Background mean : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=5.25199, Max=9.50208, Average=7.76569 -Test : Analysis mean : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.73405, Max=9.18815, Average=7.98704 -Test : H(x) for member 1: -Test : Lorenz 95 nobs= 120 Min=6.30807, Max=10.1297, Average=8.0962 -Test : H(x) for member 2: -Test : Lorenz 95 nobs= 120 Min=5.53705, Max=9.94181, Average=7.97398 -Test : H(x) for member 3: -Test : Lorenz 95 nobs= 120 Min=5.98678, Max=10.5913, Average=8.01576 -Test : H(x) for member 4: -Test : Lorenz 95 nobs= 120 Min=4.57671, Max=11.5279, Average=7.98432 -Test : H(x) for member 5: -Test : Lorenz 95 nobs= 120 Min=5.01015, Max=9.817, Average=7.86495 -Test : H(x) ensemble analysis mean: -Test : Lorenz 95 nobs= 120 Min=6.73405, Max=9.18815, Average=7.98704 -Test : analysis y - H(x): -Test : Lorenz 95 nobs= 120 Min=-0.272405, Max=0.444147, Average=0.0243478 -Test : ombg RMS: 0.864096 -Test : oman RMS: 0.149028 +Initial state for member 1: + Valid time: 2010-01-02T00:00:00Z + Min=3.53609, Max=11.1974, Average=7.90978 + +Initial state for member 2: + Valid time: 2010-01-02T00:00:00Z + Min=3.77507, Max=11.1844, Average=7.75047 + +Initial state for member 3: + Valid time: 2010-01-02T00:00:00Z + Min=5.36882, Max=10.844, Average=7.80576 + +Initial state for member 4: + Valid time: 2010-01-02T00:00:00Z + Min=4.14639, Max=11.6734, Average=7.76001 + +Initial state for member 5: + Valid time: 2010-01-02T00:00:00Z + Min=3.17563, Max=10.8183, Average=7.60243 + +H(x) for member 1: +Lorenz 95 nobs= 120 Min=3.53609, Max=11.1974, Average=7.90978 + +H(x) for member 2: +Lorenz 95 nobs= 120 Min=3.77507, Max=11.1844, Average=7.75047 + +H(x) for member 3: +Lorenz 95 nobs= 120 Min=5.36882, Max=10.844, Average=7.80576 + +H(x) for member 4: +Lorenz 95 nobs= 120 Min=4.14639, Max=11.6734, Average=7.76001 + +H(x) for member 5: +Lorenz 95 nobs= 120 Min=3.17563, Max=10.8183, Average=7.60243 + +H(x) ensemble background mean: +Lorenz 95 nobs= 120 Min=5.25199, Max=9.50208, Average=7.76569 + +background y - H(x): +Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 + +Background mean : + Valid time: 2010-01-02T00:00:00Z + Min=5.25199, Max=9.50208, Average=7.76569 + +Analysis mean : + Valid time: 2010-01-02T00:00:00Z + Min=6.73405, Max=9.18815, Average=7.98704 + +H(x) for member 1: +Lorenz 95 nobs= 120 Min=6.30807, Max=10.1297, Average=8.0962 + +H(x) for member 2: +Lorenz 95 nobs= 120 Min=5.53705, Max=9.94181, Average=7.97398 + +H(x) for member 3: +Lorenz 95 nobs= 120 Min=5.98678, Max=10.5913, Average=8.01576 + +H(x) for member 4: +Lorenz 95 nobs= 120 Min=4.57671, Max=11.5279, Average=7.98432 + +H(x) for member 5: +Lorenz 95 nobs= 120 Min=5.01015, Max=9.817, Average=7.86495 + +H(x) ensemble analysis mean: +Lorenz 95 nobs= 120 Min=6.73405, Max=9.18815, Average=7.98704 + +analysis y - H(x): +Lorenz 95 nobs= 120 Min=-0.272405, Max=0.444147, Average=0.0243478 + +ombg RMS: 0.864096 +oman RMS: 0.149028 diff --git a/l95/test/testoutput/letkf_noobs.test b/l95/test/testoutput/letkf_noobs.test index a578482e7..faffcd7bc 100644 --- a/l95/test/testoutput/letkf_noobs.test +++ b/l95/test/testoutput/letkf_noobs.test @@ -1,51 +1,72 @@ -Test : Initial state for member 1: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.53609, Max=11.1974, Average=7.90978 -Test : Initial state for member 2: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.77507, Max=11.1844, Average=7.75047 -Test : Initial state for member 3: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=5.36882, Max=10.844, Average=7.80576 -Test : Initial state for member 4: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=4.14639, Max=11.6734, Average=7.76001 -Test : Initial state for member 5: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.17563, Max=10.8183, Average=7.60243 -Test : H(x) for member 1: -Test : Lorenz 95 : No observations -Test : H(x) for member 2: -Test : Lorenz 95 : No observations -Test : H(x) for member 3: -Test : Lorenz 95 : No observations -Test : H(x) for member 4: -Test : Lorenz 95 : No observations -Test : H(x) for member 5: -Test : Lorenz 95 : No observations -Test : H(x) ensemble background mean: -Test : Lorenz 95 : No observations -Test : background y - H(x): -Test : Lorenz 95 : No observations -Test : Background mean : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=5.25199, Max=9.50208, Average=7.76569 -Test : Analysis mean : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=5.25199, Max=9.50208, Average=7.76569 -Test : H(x) for member 1: -Test : Lorenz 95 : No observations -Test : H(x) for member 2: -Test : Lorenz 95 : No observations -Test : H(x) for member 3: -Test : Lorenz 95 : No observations -Test : H(x) for member 4: -Test : Lorenz 95 : No observations -Test : H(x) for member 5: -Test : Lorenz 95 : No observations -Test : H(x) ensemble analysis mean: -Test : Lorenz 95 : No observations -Test : analysis y - H(x): -Test : Lorenz 95 : No observations -Test : ombg RMS: 0 -Test : oman RMS: 0 +Initial state for member 1: + Valid time: 2010-01-02T00:00:00Z + Min=3.53609, Max=11.1974, Average=7.90978 + +Initial state for member 2: + Valid time: 2010-01-02T00:00:00Z + Min=3.77507, Max=11.1844, Average=7.75047 + +Initial state for member 3: + Valid time: 2010-01-02T00:00:00Z + Min=5.36882, Max=10.844, Average=7.80576 + +Initial state for member 4: + Valid time: 2010-01-02T00:00:00Z + Min=4.14639, Max=11.6734, Average=7.76001 + +Initial state for member 5: + Valid time: 2010-01-02T00:00:00Z + Min=3.17563, Max=10.8183, Average=7.60243 + +H(x) for member 1: +Lorenz 95 : No observations + +H(x) for member 2: +Lorenz 95 : No observations + +H(x) for member 3: +Lorenz 95 : No observations + +H(x) for member 4: +Lorenz 95 : No observations + +H(x) for member 5: +Lorenz 95 : No observations + +H(x) ensemble background mean: +Lorenz 95 : No observations + +background y - H(x): +Lorenz 95 : No observations + +Background mean : + Valid time: 2010-01-02T00:00:00Z + Min=5.25199, Max=9.50208, Average=7.76569 + +Analysis mean : + Valid time: 2010-01-02T00:00:00Z + Min=5.25199, Max=9.50208, Average=7.76569 + +H(x) for member 1: +Lorenz 95 : No observations + +H(x) for member 2: +Lorenz 95 : No observations + +H(x) for member 3: +Lorenz 95 : No observations + +H(x) for member 4: +Lorenz 95 : No observations + +H(x) for member 5: +Lorenz 95 : No observations + +H(x) ensemble analysis mean: +Lorenz 95 : No observations + +analysis y - H(x): +Lorenz 95 : No observations + +ombg RMS: 0 +oman RMS: 0 diff --git a/l95/test/testoutput/letkf_qc.test b/l95/test/testoutput/letkf_qc.test index 9fb90345e..561a11a04 100644 --- a/l95/test/testoutput/letkf_qc.test +++ b/l95/test/testoutput/letkf_qc.test @@ -1,51 +1,72 @@ -Test : Initial state for member 1: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.53609, Max=11.1974, Average=7.90978 -Test : Initial state for member 2: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.77507, Max=11.1844, Average=7.75047 -Test : Initial state for member 3: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=5.36882, Max=10.844, Average=7.80576 -Test : Initial state for member 4: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=4.14639, Max=11.6734, Average=7.76001 -Test : Initial state for member 5: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=3.17563, Max=10.8183, Average=7.60243 -Test : H(x) for member 1: -Test : Lorenz 95 nobs= 120 Min=3.53609, Max=11.1974, Average=7.90978 -Test : H(x) for member 2: -Test : Lorenz 95 nobs= 120 Min=3.77507, Max=11.1844, Average=7.75047 -Test : H(x) for member 3: -Test : Lorenz 95 nobs= 120 Min=5.36882, Max=10.844, Average=7.80576 -Test : H(x) for member 4: -Test : Lorenz 95 nobs= 120 Min=4.14639, Max=11.6734, Average=7.76001 -Test : H(x) for member 5: -Test : Lorenz 95 nobs= 120 Min=3.17563, Max=10.8183, Average=7.60243 -Test : H(x) ensemble background mean: -Test : Lorenz 95 nobs= 120 Min=5.25199, Max=9.50208, Average=7.76569 -Test : background y - H(x): -Test : Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 -Test : Background mean : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=5.25199, Max=9.50208, Average=7.76569 -Test : Analysis mean : -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.73044, Max=9.19002, Average=7.98752 -Test : H(x) for member 1: -Test : Lorenz 95 nobs= 120 Min=6.31712, Max=10.1291, Average=8.09272 -Test : H(x) for member 2: -Test : Lorenz 95 nobs= 120 Min=5.62563, Max=9.94295, Average=7.97729 -Test : H(x) for member 3: -Test : Lorenz 95 nobs= 120 Min=5.98634, Max=10.5924, Average=8.01934 -Test : H(x) for member 4: -Test : Lorenz 95 nobs= 120 Min=4.57542, Max=11.5295, Average=7.98859 -Test : H(x) for member 5: -Test : Lorenz 95 nobs= 120 Min=4.72238, Max=9.79569, Average=7.85967 -Test : H(x) ensemble analysis mean: -Test : Lorenz 95 nobs= 120 Min=6.73044, Max=9.19002, Average=7.98752 -Test : analysis y - H(x): -Test : Lorenz 95 nobs= 120 Min=-0.265703, Max=0.441464, Average=0.0238667 -Test : ombg RMS: 0.864096 -Test : oman RMS: 0.14948 +Initial state for member 1: + Valid time: 2010-01-02T00:00:00Z + Min=3.53609, Max=11.1974, Average=7.90978 + +Initial state for member 2: + Valid time: 2010-01-02T00:00:00Z + Min=3.77507, Max=11.1844, Average=7.75047 + +Initial state for member 3: + Valid time: 2010-01-02T00:00:00Z + Min=5.36882, Max=10.844, Average=7.80576 + +Initial state for member 4: + Valid time: 2010-01-02T00:00:00Z + Min=4.14639, Max=11.6734, Average=7.76001 + +Initial state for member 5: + Valid time: 2010-01-02T00:00:00Z + Min=3.17563, Max=10.8183, Average=7.60243 + +H(x) for member 1: +Lorenz 95 nobs= 120 Min=3.53609, Max=11.1974, Average=7.90978 + +H(x) for member 2: +Lorenz 95 nobs= 120 Min=3.77507, Max=11.1844, Average=7.75047 + +H(x) for member 3: +Lorenz 95 nobs= 120 Min=5.36882, Max=10.844, Average=7.80576 + +H(x) for member 4: +Lorenz 95 nobs= 120 Min=4.14639, Max=11.6734, Average=7.76001 + +H(x) for member 5: +Lorenz 95 nobs= 120 Min=3.17563, Max=10.8183, Average=7.60243 + +H(x) ensemble background mean: +Lorenz 95 nobs= 120 Min=5.25199, Max=9.50208, Average=7.76569 + +background y - H(x): +Lorenz 95 nobs= 120 Min=-1.48379, Max=2.74405, Average=0.2457 + +Background mean : + Valid time: 2010-01-02T00:00:00Z + Min=5.25199, Max=9.50208, Average=7.76569 + +Analysis mean : + Valid time: 2010-01-02T00:00:00Z + Min=6.73044, Max=9.19002, Average=7.98752 + +H(x) for member 1: +Lorenz 95 nobs= 120 Min=6.31712, Max=10.1291, Average=8.09272 + +H(x) for member 2: +Lorenz 95 nobs= 120 Min=5.62563, Max=9.94295, Average=7.97729 + +H(x) for member 3: +Lorenz 95 nobs= 120 Min=5.98634, Max=10.5924, Average=8.01934 + +H(x) for member 4: +Lorenz 95 nobs= 120 Min=4.57542, Max=11.5295, Average=7.98859 + +H(x) for member 5: +Lorenz 95 nobs= 120 Min=4.72238, Max=9.79569, Average=7.85967 + +H(x) ensemble analysis mean: +Lorenz 95 nobs= 120 Min=6.73044, Max=9.19002, Average=7.98752 + +analysis y - H(x): +Lorenz 95 nobs= 120 Min=-0.265703, Max=0.441464, Average=0.0238667 + +ombg RMS: 0.864096 +oman RMS: 0.14948 diff --git a/l95/test/testoutput/makeobs3d.test b/l95/test/testoutput/makeobs3d.test index b63ebdc0b..7b562bbcc 100644 --- a/l95/test/testoutput/makeobs3d.test +++ b/l95/test/testoutput/makeobs3d.test @@ -1,9 +1,9 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T21:00:00Z -Test : Min=6.67249, Max=8.95616, Average=8.01488 -Test : Final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=6.56213, Max=9.58572, Average=8.00692 -Test : H(x): -Test : Lorenz 95 nobs= 120 Min=6.54448, Max=9.43695, Average=8.01139 -Test : End H(x) +Initial state: + Valid time: 2010-01-01T21:00:00Z + Min=6.67249, Max=8.95616, Average=8.01488 +Final state: + Valid time: 2010-01-02T03:00:00Z + Min=6.56213, Max=9.58572, Average=8.00692 +H(x): +Lorenz 95 nobs= 120 Min=6.54448, Max=9.43695, Average=8.01139 +End H(x) diff --git a/l95/test/testoutput/makeobs4d.test b/l95/test/testoutput/makeobs4d.test index 64c20072c..f6bb28971 100644 --- a/l95/test/testoutput/makeobs4d.test +++ b/l95/test/testoutput/makeobs4d.test @@ -1,9 +1,9 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.80572, Max=8.97101, Average=8.02427 -Test : Final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=6.56213, Max=9.58572, Average=8.00692 -Test : H(x): -Test : Lorenz 95 nobs= 160 Min=6.55895, Max=9.58572, Average=8.01323 -Test : End H(x) +Initial state: + Valid time: 2010-01-01T03:00:00Z + Min=7.80572, Max=8.97101, Average=8.02427 +Final state: + Valid time: 2010-01-02T03:00:00Z + Min=6.56213, Max=9.58572, Average=8.00692 +H(x): +Lorenz 95 nobs= 160 Min=6.55895, Max=9.58572, Average=8.01323 +End H(x) diff --git a/l95/test/testoutput/makeobs4d12h.test b/l95/test/testoutput/makeobs4d12h.test index 35faae935..4991dc728 100644 --- a/l95/test/testoutput/makeobs4d12h.test +++ b/l95/test/testoutput/makeobs4d12h.test @@ -1,9 +1,9 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.80572, Max=8.97101, Average=8.02427 -Test : Final state: -Test : Valid time: 2010-01-01T15:00:00Z -Test : Min=7.12604, Max=8.73405, Average=8.01944 -Test : H(x): -Test : Lorenz 95 nobs= 80 Min=8, Max=8.73405, Average=8.07237 -Test : End H(x) +Initial state: + Valid time: 2010-01-01T03:00:00Z + Min=7.80572, Max=8.97101, Average=8.02427 +Final state: + Valid time: 2010-01-01T15:00:00Z + Min=7.12604, Max=8.73405, Average=8.01944 +H(x): +Lorenz 95 nobs= 80 Min=8, Max=8.73405, Average=8.07237 +End H(x) diff --git a/l95/test/testoutput/makeobsbias.test b/l95/test/testoutput/makeobsbias.test index 79974b183..c3744005e 100644 --- a/l95/test/testoutput/makeobsbias.test +++ b/l95/test/testoutput/makeobsbias.test @@ -1,9 +1,9 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.80572, Max=8.97101, Average=8.02427 -Test : Final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=6.56213, Max=9.58572, Average=8.00692 -Test : H(x): -Test : Lorenz 95 nobs= 160 Min=7.15895, Max=10.1857, Average=8.61323 -Test : End H(x) +Initial state: + Valid time: 2010-01-01T03:00:00Z + Min=7.80572, Max=8.97101, Average=8.02427 +Final state: + Valid time: 2010-01-02T03:00:00Z + Min=6.56213, Max=9.58572, Average=8.00692 +H(x): +Lorenz 95 nobs= 160 Min=7.15895, Max=10.1857, Average=8.61323 +End H(x) diff --git a/l95/test/testoutput/makeobspert.test b/l95/test/testoutput/makeobspert.test index c5c8553aa..f30c678f3 100644 --- a/l95/test/testoutput/makeobspert.test +++ b/l95/test/testoutput/makeobspert.test @@ -1,12 +1,12 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T03:00:00Z -Test : Min=7.80572, Max=8.97101, Average=8.02427 -Test : Final state: -Test : Valid time: 2010-01-02T03:00:00Z -Test : Min=6.56213, Max=9.58572, Average=8.00692 -Test : H(x): -Test : Lorenz 95 nobs= 160 Min=6.55895, Max=9.58572, Average=8.01323 -Test : End H(x) -Test : Perturbed H(x): -Test : Lorenz 95 nobs= 160 Min=6.26715, Max=9.41343, Average=8.04793 -Test : End Perturbed H(x) +Initial state: + Valid time: 2010-01-01T03:00:00Z + Min=7.80572, Max=8.97101, Average=8.02427 +Final state: + Valid time: 2010-01-02T03:00:00Z + Min=6.56213, Max=9.58572, Average=8.00692 +H(x): +Lorenz 95 nobs= 160 Min=6.55895, Max=9.58572, Average=8.01323 +End H(x) +Perturbed H(x): +Lorenz 95 nobs= 160 Min=6.26715, Max=9.41343, Average=8.04793 +End Perturbed H(x) diff --git a/l95/test/testoutput/simplifiedl95_DRGMRESR.test b/l95/test/testoutput/simplifiedl95_DRGMRESR.test index 64b0b34fe..a4d54e97a 100644 --- a/l95/test/testoutput/simplifiedl95_DRGMRESR.test +++ b/l95/test/testoutput/simplifiedl95_DRGMRESR.test @@ -1,10 +1,10 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 -Test : CostFunction: Nonlinear J = 114 -Test : DRGMRESRMinimizer: reduction in residual norm = 1.25642e-16 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:30:00Z -Test : Min=-3, Max=5, Average=1.25 -Test : CostJb : Nonlinear Jb = 28.5 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 -Test : CostFunction: Nonlinear J = 57 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 +CostFunction: Nonlinear J = 114 +DRGMRESRMinimizer: reduction in residual norm = 1.25642e-16 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:30:00Z + Min=-3, Max=5, Average=1.25 +CostJb : Nonlinear Jb = 28.5 +CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 +CostFunction: Nonlinear J = 57 diff --git a/l95/test/testoutput/simplifiedl95_DRIPCG.test b/l95/test/testoutput/simplifiedl95_DRIPCG.test index 6b2f9bdac..61b872f14 100644 --- a/l95/test/testoutput/simplifiedl95_DRIPCG.test +++ b/l95/test/testoutput/simplifiedl95_DRIPCG.test @@ -1,10 +1,10 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 -Test : CostFunction: Nonlinear J = 114 -Test : DRIPCGMinimizer: reduction in residual norm = 5.88211e-17 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:30:00Z -Test : Min=-3, Max=5, Average=1.25 -Test : CostJb : Nonlinear Jb = 28.5 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 -Test : CostFunction: Nonlinear J = 57 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 +CostFunction: Nonlinear J = 114 +DRIPCGMinimizer: reduction in residual norm = 5.88211e-17 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:30:00Z + Min=-3, Max=5, Average=1.25 +CostJb : Nonlinear Jb = 28.5 +CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 +CostFunction: Nonlinear J = 57 diff --git a/l95/test/testoutput/simplifiedl95_DRPFOM.test b/l95/test/testoutput/simplifiedl95_DRPFOM.test index 06c503249..07b63c006 100644 --- a/l95/test/testoutput/simplifiedl95_DRPFOM.test +++ b/l95/test/testoutput/simplifiedl95_DRPFOM.test @@ -1,10 +1,10 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 -Test : CostFunction: Nonlinear J = 114 -Test : DRPFOMMinimizer: reduction in residual norm = 1.68831e-16 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:30:00Z -Test : Min=-3, Max=5, Average=1.25 -Test : CostJb : Nonlinear Jb = 28.5 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 -Test : CostFunction: Nonlinear J = 57 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 +CostFunction: Nonlinear J = 114 +DRPFOMMinimizer: reduction in residual norm = 1.68831e-16 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:30:00Z + Min=-3, Max=5, Average=1.25 +CostJb : Nonlinear Jb = 28.5 +CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 +CostFunction: Nonlinear J = 57 diff --git a/l95/test/testoutput/simplifiedl95_DRPLanczos.test b/l95/test/testoutput/simplifiedl95_DRPLanczos.test index 06e4a38ce..99b5e118b 100644 --- a/l95/test/testoutput/simplifiedl95_DRPLanczos.test +++ b/l95/test/testoutput/simplifiedl95_DRPLanczos.test @@ -1,10 +1,10 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 -Test : CostFunction: Nonlinear J = 114 -Test : DRPLanczosMinimizer: reduction in residual norm = 1.68831e-16 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:30:00Z -Test : Min=-3, Max=5, Average=1.25 -Test : CostJb : Nonlinear Jb = 28.5 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 -Test : CostFunction: Nonlinear J = 57 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 +CostFunction: Nonlinear J = 114 +DRPLanczosMinimizer: reduction in residual norm = 1.68831e-16 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:30:00Z + Min=-3, Max=5, Average=1.25 +CostJb : Nonlinear Jb = 28.5 +CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 +CostFunction: Nonlinear J = 57 diff --git a/l95/test/testoutput/simplifiedl95_FGMRES.test b/l95/test/testoutput/simplifiedl95_FGMRES.test index ea35eae79..480374265 100644 --- a/l95/test/testoutput/simplifiedl95_FGMRES.test +++ b/l95/test/testoutput/simplifiedl95_FGMRES.test @@ -1,10 +1,10 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 -Test : CostFunction: Nonlinear J = 114 -Test : FGMRESMinimizer: reduction in residual norm = 2.80751e-16 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:30:00Z -Test : Min=-3, Max=5, Average=1.25 -Test : CostJb : Nonlinear Jb = 28.5 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 -Test : CostFunction: Nonlinear J = 57 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 +CostFunction: Nonlinear J = 114 +FGMRESMinimizer: reduction in residual norm = 2.80751e-16 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:30:00Z + Min=-3, Max=5, Average=1.25 +CostJb : Nonlinear Jb = 28.5 +CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 +CostFunction: Nonlinear J = 57 diff --git a/l95/test/testoutput/simplifiedl95_GMRESR.test b/l95/test/testoutput/simplifiedl95_GMRESR.test index 61119d470..974cb7d76 100644 --- a/l95/test/testoutput/simplifiedl95_GMRESR.test +++ b/l95/test/testoutput/simplifiedl95_GMRESR.test @@ -1,10 +1,10 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 -Test : CostFunction: Nonlinear J = 114 -Test : GMRESRMinimizer: reduction in residual norm = 1.86589e-16 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:30:00Z -Test : Min=-3, Max=5, Average=1.25 -Test : CostJb : Nonlinear Jb = 28.5 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 -Test : CostFunction: Nonlinear J = 57 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 +CostFunction: Nonlinear J = 114 +GMRESRMinimizer: reduction in residual norm = 1.86589e-16 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:30:00Z + Min=-3, Max=5, Average=1.25 +CostJb : Nonlinear Jb = 28.5 +CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 +CostFunction: Nonlinear J = 57 diff --git a/l95/test/testoutput/simplifiedl95_IPCG.test b/l95/test/testoutput/simplifiedl95_IPCG.test index 6cc0c95f3..8a308a245 100644 --- a/l95/test/testoutput/simplifiedl95_IPCG.test +++ b/l95/test/testoutput/simplifiedl95_IPCG.test @@ -1,10 +1,10 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 -Test : CostFunction: Nonlinear J = 114 -Test : IPCGMinimizer: reduction in residual norm = 0 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:30:00Z -Test : Min=-3, Max=5, Average=1.25 -Test : CostJb : Nonlinear Jb = 28.5 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 -Test : CostFunction: Nonlinear J = 57 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 +CostFunction: Nonlinear J = 114 +IPCGMinimizer: reduction in residual norm = 0 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:30:00Z + Min=-3, Max=5, Average=1.25 +CostJb : Nonlinear Jb = 28.5 +CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 +CostFunction: Nonlinear J = 57 diff --git a/l95/test/testoutput/simplifiedl95_LBGMRESR.test b/l95/test/testoutput/simplifiedl95_LBGMRESR.test index 1eca620ae..e69e87ea2 100644 --- a/l95/test/testoutput/simplifiedl95_LBGMRESR.test +++ b/l95/test/testoutput/simplifiedl95_LBGMRESR.test @@ -1,9 +1,9 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 -Test : CostFunction: Nonlinear J = 114 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:30:00Z -Test : Min=-3, Max=5, Average=1.25 -Test : CostJb : Nonlinear Jb = 28.5 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 -Test : CostFunction: Nonlinear J = 57 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 +CostFunction: Nonlinear J = 114 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:30:00Z + Min=-3, Max=5, Average=1.25 +CostJb : Nonlinear Jb = 28.5 +CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 +CostFunction: Nonlinear J = 57 diff --git a/l95/test/testoutput/simplifiedl95_MINRES.test b/l95/test/testoutput/simplifiedl95_MINRES.test index 30be63439..d6c196773 100644 --- a/l95/test/testoutput/simplifiedl95_MINRES.test +++ b/l95/test/testoutput/simplifiedl95_MINRES.test @@ -1,10 +1,10 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 -Test : CostFunction: Nonlinear J = 114 -Test : MINRESMinimizer: reduction in residual norm = 2.28878e-16 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:30:00Z -Test : Min=-3, Max=5, Average=1.25 -Test : CostJb : Nonlinear Jb = 28.5 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 -Test : CostFunction: Nonlinear J = 57 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 +CostFunction: Nonlinear J = 114 +MINRESMinimizer: reduction in residual norm = 2.28878e-16 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:30:00Z + Min=-3, Max=5, Average=1.25 +CostJb : Nonlinear Jb = 28.5 +CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 +CostFunction: Nonlinear J = 57 diff --git a/l95/test/testoutput/simplifiedl95_PCG.test b/l95/test/testoutput/simplifiedl95_PCG.test index 63f766f87..40ba49749 100644 --- a/l95/test/testoutput/simplifiedl95_PCG.test +++ b/l95/test/testoutput/simplifiedl95_PCG.test @@ -1,10 +1,10 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 -Test : CostFunction: Nonlinear J = 114 -Test : PCGMinimizer: reduction in residual norm = 2.29704e-16 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:30:00Z -Test : Min=-3, Max=5, Average=1.25 -Test : CostJb : Nonlinear Jb = 28.5 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 -Test : CostFunction: Nonlinear J = 57 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 +CostFunction: Nonlinear J = 114 +PCGMinimizer: reduction in residual norm = 2.29704e-16 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:30:00Z + Min=-3, Max=5, Average=1.25 +CostJb : Nonlinear Jb = 28.5 +CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 +CostFunction: Nonlinear J = 57 diff --git a/l95/test/testoutput/simplifiedl95_PLanczos.test b/l95/test/testoutput/simplifiedl95_PLanczos.test index 262b34a49..8ccb169f4 100644 --- a/l95/test/testoutput/simplifiedl95_PLanczos.test +++ b/l95/test/testoutput/simplifiedl95_PLanczos.test @@ -1,10 +1,10 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 -Test : CostFunction: Nonlinear J = 114 -Test : PLanczosMinimizer: reduction in residual norm = 2.28878e-16 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:30:00Z -Test : Min=-3, Max=5, Average=1.25 -Test : CostJb : Nonlinear Jb = 28.5 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 -Test : CostFunction: Nonlinear J = 57 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 +CostFunction: Nonlinear J = 114 +PLanczosMinimizer: reduction in residual norm = 2.28878e-16 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:30:00Z + Min=-3, Max=5, Average=1.25 +CostJb : Nonlinear Jb = 28.5 +CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 +CostFunction: Nonlinear J = 57 diff --git a/l95/test/testoutput/simplifiedl95_RPCG.test b/l95/test/testoutput/simplifiedl95_RPCG.test index f88a758ef..f22d1fac7 100644 --- a/l95/test/testoutput/simplifiedl95_RPCG.test +++ b/l95/test/testoutput/simplifiedl95_RPCG.test @@ -1,10 +1,10 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 -Test : CostFunction: Nonlinear J = 114 -Test : RPCGMinimizer: reduction in residual norm = 5.88211e-17 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:30:00Z -Test : Min=-3, Max=5, Average=1.25 -Test : CostJb : Nonlinear Jb = 28.5 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 -Test : CostFunction: Nonlinear J = 57 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 +CostFunction: Nonlinear J = 114 +RPCGMinimizer: reduction in residual norm = 5.88211e-17 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:30:00Z + Min=-3, Max=5, Average=1.25 +CostJb : Nonlinear Jb = 28.5 +CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 +CostFunction: Nonlinear J = 57 diff --git a/l95/test/testoutput/simplifiedl95_RPLanczos.test b/l95/test/testoutput/simplifiedl95_RPLanczos.test index d0974eaa0..7dcc62a5e 100644 --- a/l95/test/testoutput/simplifiedl95_RPLanczos.test +++ b/l95/test/testoutput/simplifiedl95_RPLanczos.test @@ -1,10 +1,10 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 -Test : CostFunction: Nonlinear J = 114 -Test : RPLanczosMinimizer: reduction in residual norm = 1.68831e-16 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:30:00Z -Test : Min=-3, Max=5, Average=1.25 -Test : CostJb : Nonlinear Jb = 28.5 -Test : CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 -Test : CostFunction: Nonlinear J = 57 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114, nobs = 4, Jo/n = 28.5, err = 1 +CostFunction: Nonlinear J = 114 +RPLanczosMinimizer: reduction in residual norm = 1.68831e-16 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:30:00Z + Min=-3, Max=5, Average=1.25 +CostJb : Nonlinear Jb = 28.5 +CostJo : Nonlinear Jo(Lorenz 95) = 28.5, nobs = 4, Jo/n = 7.125, err = 1 +CostFunction: Nonlinear J = 57 diff --git a/l95/test/testoutput/truth.test b/l95/test/testoutput/truth.test index 61715981b..8d1a5a300 100644 --- a/l95/test/testoutput/truth.test +++ b/l95/test/testoutput/truth.test @@ -1,6 +1,6 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Min=8, Max=9, Average=8.025 -Test : Final state: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Min=6.55895, Max=9.27812, Average=8.01149 +Initial state: + Valid time: 2010-01-01T00:00:00Z + Min=8, Max=9, Average=8.025 +Final state: + Valid time: 2010-01-02T00:00:00Z + Min=6.55895, Max=9.27812, Average=8.01149 diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index 8da6279c4..e1485055e 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -112,7 +112,7 @@ list( APPEND qg_testoutput testoutput/dirac_loc_3d.test testoutput/dirac_loc_4d.test testoutput/dirac_no_loc.test -# testoutput/eda_4dvar.test + testoutput/eda_4dvar.test testoutput/eda_3dvar_block.test testoutput/ens_forecast.test testoutput/ens_hofx.test @@ -164,18 +164,16 @@ file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/Data) # truth and make_obs_4d_24h are required for interface tests ##################################################################### -oops_add_test( TESTNAME truth - MODELNAME qg - OMP 2 - YAMLNAME testinput/truth.yaml - EXENAME qg_forecast.x ) +ecbuild_add_test( TARGET test_qg_truth + OMP 2 + ARGS testinput/truth.yaml + COMMAND qg_forecast.x ) -oops_add_test( TESTNAME make_obs_4d_24h - MODELNAME qg - OMP 2 - YAMLNAME testinput/make_obs_4d_24h.yaml - EXENAME qg_hofx.x - TEST_DEPENDS test_qg_truth ) +ecbuild_add_test( TARGET test_qg_make_obs_4d_24h + OMP 2 + ARGS testinput/make_obs_4d_24h.yaml + COMMAND qg_hofx.x + TEST_DEPENDS test_qg_truth ) ##################################################################### @@ -363,70 +361,61 @@ ecbuild_add_test( TARGET test_qg_lineargetvalues # forecast-related tests ##################################################################### -oops_add_test( TESTNAME analytic_forecast - MODELNAME qg - OMP 2 - YAMLNAME testinput/analytic_forecast.yaml - EXENAME qg_forecast.x ) +ecbuild_add_test( TARGET test_qg_analytic_forecast + OMP 2 + ARGS testinput/analytic_forecast.yaml + COMMAND qg_forecast.x ) -oops_add_test( TESTNAME forecast - MODELNAME qg - OMP 2 - YAMLNAME testinput/forecast.yaml - EXENAME qg_forecast.x - TEST_DEPENDS test_qg_truth ) +ecbuild_add_test( TARGET test_qg_forecast + OMP 2 + ARGS testinput/forecast.yaml + COMMAND qg_forecast.x + TEST_DEPENDS test_qg_truth ) ##################################################################### # obs-related tests ##################################################################### -oops_add_test( TESTNAME make_obs_3d - MODELNAME qg - OMP 2 - YAMLNAME testinput/make_obs_3d.yaml - EXENAME qg_hofx.x - TEST_DEPENDS test_qg_truth ) - -oops_add_test( TESTNAME make_obs_4d_12h - MODELNAME qg - OMP 2 - YAMLNAME testinput/make_obs_4d_12h.yaml - EXENAME qg_hofx.x - TEST_DEPENDS test_qg_truth ) - -oops_add_test( TESTNAME make_obs_4d_biased - MODELNAME qg - OMP 2 - YAMLNAME testinput/make_obs_4d_biased.yaml - EXENAME qg_hofx.x - TEST_DEPENDS test_qg_truth ) - -oops_add_test( TESTNAME hofx - MODELNAME qg - OMP 2 - YAMLNAME testinput/hofx.yaml - EXENAME qg_hofx.x - TEST_DEPENDS test_qg_make_obs_4d_12h ) - -oops_add_test( TESTNAME hofx3d - MODELNAME qg - OMP 2 - YAMLNAME testinput/hofx3d.yaml - EXENAME qg_hofx3d.x - TEST_DEPENDS test_qg_make_obs_4d_12h ) +ecbuild_add_test( TARGET test_qg_make_obs_3d + OMP 2 + ARGS testinput/make_obs_3d.yaml + COMMAND qg_hofx.x + TEST_DEPENDS test_qg_truth ) + +ecbuild_add_test( TARGET test_qg_make_obs_4d_12h + OMP 2 + ARGS testinput/make_obs_4d_12h.yaml + COMMAND qg_hofx.x + TEST_DEPENDS test_qg_truth ) + +ecbuild_add_test( TARGET test_qg_make_obs_4d_biased + OMP 2 + ARGS testinput/make_obs_4d_biased.yaml + COMMAND qg_hofx.x + TEST_DEPENDS test_qg_truth ) + +ecbuild_add_test( TARGET test_qg_hofx + OMP 2 + ARGS testinput/hofx.yaml + COMMAND qg_hofx.x + TEST_DEPENDS test_qg_make_obs_4d_12h ) +ecbuild_add_test( TARGET test_qg_hofx3d + OMP 2 + ARGS testinput/hofx3d.yaml + COMMAND qg_hofx3d.x + TEST_DEPENDS test_qg_make_obs_4d_12h ) ##################################################################### # ensemble-related tests ##################################################################### -oops_add_test( TESTNAME gen_ens_pert_B - MODELNAME qg - OMP 2 - YAMLNAME testinput/gen_ens_pert_B.yaml - EXENAME qg_gen_ens_pert_B.x - TEST_DEPENDS test_qg_truth ) +ecbuild_add_test( TARGET test_qg_gen_ens_pert_B + OMP 2 + ARGS testinput/gen_ens_pert_B.yaml + COMMAND qg_gen_ens_pert_B.x + TEST_DEPENDS test_qg_truth ) ecbuild_add_test( TARGET test_qg_ens_forecast MPI 2 @@ -442,263 +431,227 @@ ecbuild_add_test( TARGET test_qg_ens_hofx DEPENDS qg_ens_hofx.x TEST_DEPENDS test_qg_gen_ens_pert_B test_qg_make_obs_4d_12h ) -oops_add_test( TESTNAME ens_variance - MODELNAME qg - OMP 2 - YAMLNAME testinput/ens_variance.yaml - EXENAME qg_ens_variance.x - TEST_DEPENDS test_qg_gen_ens_pert_B ) - -oops_add_test( TESTNAME ens_recenter - MODELNAME qg - OMP 2 - YAMLNAME testinput/ens_recenter.yaml - EXENAME qg_ens_recenter.x - TEST_DEPENDS test_qg_gen_ens_pert_B ) - -oops_add_test( TESTNAME uniform_field_inflation - MODELNAME qg - OMP 2 - YAMLNAME testinput/uniform_field_inflation.yaml - EXENAME qg_forecast.x ) - -oops_add_test( TESTNAME ens_variance_inflation_field - MODELNAME qg - OMP 2 - YAMLNAME testinput/ens_variance_inflation_field.yaml - EXENAME qg_ens_variance.x - TEST_DEPENDS test_qg_gen_ens_pert_B test_qg_uniform_field_inflation ) - -oops_add_test( TESTNAME ens_variance_inflation_value - MODELNAME qg - OMP 2 - YAMLNAME testinput/ens_variance_inflation_value.yaml - EXENAME qg_ens_variance.x - TEST_DEPENDS test_qg_gen_ens_pert_B ) - -oops_add_test( TESTNAME rtpp - MODELNAME qg - YAMLNAME testinput/rtpp.yaml - EXENAME qg_rtpp.x - TEST_DEPENDS test_qg_gen_ens_pert_B ) +ecbuild_add_test( TARGET test_qg_ens_variance + OMP 2 + ARGS testinput/ens_variance.yaml + COMMAND qg_ens_variance.x + TEST_DEPENDS test_qg_gen_ens_pert_B ) + +ecbuild_add_test( TARGET test_qg_ens_recenter + OMP 2 + ARGS testinput/ens_recenter.yaml + COMMAND qg_ens_recenter.x + TEST_DEPENDS test_qg_gen_ens_pert_B ) + +ecbuild_add_test( TARGET test_qg_uniform_field_inflation + OMP 2 + ARGS testinput/uniform_field_inflation.yaml + COMMAND qg_forecast.x ) + +ecbuild_add_test( TARGET test_qg_ens_variance_inflation_field + OMP 2 + ARGS testinput/ens_variance_inflation_field.yaml + COMMAND qg_ens_variance.x + TEST_DEPENDS test_qg_gen_ens_pert_B test_qg_uniform_field_inflation ) + +ecbuild_add_test( TARGET test_qg_ens_variance_inflation_value + OMP 2 + ARGS testinput/ens_variance_inflation_value.yaml + COMMAND qg_ens_variance.x + TEST_DEPENDS test_qg_gen_ens_pert_B ) + +ecbuild_add_test( TARGET test_qg_rtpp + ARGS testinput/rtpp.yaml + COMMAND qg_rtpp.x + TEST_DEPENDS test_qg_gen_ens_pert_B ) ##################################################################### # other tests ##################################################################### -oops_add_test( TESTNAME static_b_init - MODELNAME qg - MPI 1 - OMP 2 - YAMLNAME testinput/static_b_init.yaml - EXENAME qg_staticbinit.x - TEST_DEPENDS test_qg_truth ) - -oops_add_test( TESTNAME dfi - MODELNAME qg - OMP 2 - YAMLNAME testinput/dfi.yaml - EXENAME qg_dfi.x - TEST_DEPENDS test_qg_forecast ) - -oops_add_test( TESTNAME convertstate - MODELNAME qg - OMP 2 - YAMLNAME testinput/convertstate.yaml - EXENAME qg_convertstate.x - TEST_DEPENDS test_qg_forecast ) +ecbuild_add_test( TARGET test_qg_static_b_init + MPI 1 + OMP 2 + ARGS testinput/static_b_init.yaml + COMMAND qg_staticbinit.x + TEST_DEPENDS test_qg_truth ) + +ecbuild_add_test( TARGET test_qg_dfi + OMP 2 + ARGS testinput/dfi.yaml + COMMAND qg_dfi.x + TEST_DEPENDS test_qg_forecast ) + +ecbuild_add_test( TARGET test_qg_convertstate + OMP 2 + ARGS testinput/convertstate.yaml + COMMAND qg_convertstate.x + TEST_DEPENDS test_qg_forecast ) ##################################################################### # QG dirac tests ##################################################################### -oops_add_test( TESTNAME dirac_cov - MODELNAME qg - OMP 2 - YAMLNAME testinput/dirac_cov.yaml - EXENAME qg_dirac.x - TEST_DEPENDS test_qg_forecast ) - -oops_add_test( TESTNAME dirac_hyb_value - MODELNAME qg - OMP 2 - YAMLNAME testinput/dirac_hyb_value.yaml - EXENAME qg_dirac.x - TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B ) - -oops_add_test( TESTNAME uniform_field_hybrid - MODELNAME qg - OMP 2 - YAMLNAME testinput/uniform_field_hybrid.yaml - EXENAME qg_forecast.x ) - -oops_add_test( TESTNAME dirac_hyb_field - MODELNAME qg - OMP 2 - YAMLNAME testinput/dirac_hyb_field.yaml - EXENAME qg_dirac.x - TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_uniform_field_hybrid ) - -oops_add_test( TESTNAME dirac_loc_3d - MODELNAME qg - OMP 2 - YAMLNAME testinput/dirac_loc_3d.yaml - EXENAME qg_dirac.x - TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B ) - -oops_add_test( TESTNAME dirac_loc_4d - MODELNAME qg - OMP 2 - MPI 13 - YAMLNAME testinput/dirac_loc_4d.yaml - EXENAME qg_dirac.x - TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B ) - -oops_add_test( TESTNAME dirac_no_loc - MODELNAME qg - OMP 2 - YAMLNAME testinput/dirac_no_loc.yaml - EXENAME qg_dirac.x - TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B ) +ecbuild_add_test( TARGET test_qg_dirac_cov + OMP 2 + ARGS testinput/dirac_cov.yaml + COMMAND qg_dirac.x + TEST_DEPENDS test_qg_forecast ) + +ecbuild_add_test( TARGET test_qg_dirac_hyb_value + OMP 2 + ARGS testinput/dirac_hyb_value.yaml + COMMAND qg_dirac.x + TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B ) + +ecbuild_add_test( TARGET test_qg_uniform_field_hybrid + OMP 2 + ARGS testinput/uniform_field_hybrid.yaml + COMMAND qg_forecast.x ) + +ecbuild_add_test( TARGET test_qg_dirac_hyb_field + OMP 2 + ARGS testinput/dirac_hyb_field.yaml + COMMAND qg_dirac.x + TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_uniform_field_hybrid ) + +ecbuild_add_test( TARGET test_qg_dirac_loc_3d + OMP 2 + ARGS testinput/dirac_loc_3d.yaml + COMMAND qg_dirac.x + TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B ) + +ecbuild_add_test( TARGET test_qg_dirac_loc_4d + OMP 2 + MPI 13 + ARGS testinput/dirac_loc_4d.yaml + COMMAND qg_dirac.x + TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B ) + +ecbuild_add_test( TARGET test_qg_dirac_no_loc + OMP 2 + ARGS testinput/dirac_no_loc.yaml + COMMAND qg_dirac.x + TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B ) ##################################################################### # 3d variational tests ##################################################################### -oops_add_test( TESTNAME 3densvar - MODELNAME qg - OMP 2 - YAMLNAME testinput/3densvar.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_3d ) +ecbuild_add_test( TARGET test_qg_3densvar + OMP 2 + ARGS testinput/3densvar.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_3d ) -oops_add_test( TESTNAME 3dvar - MODELNAME qg - OMP 2 - YAMLNAME testinput/3dvar.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_make_obs_3d ) +ecbuild_add_test( TARGET test_qg_3dvar + OMP 2 + ARGS testinput/3dvar.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_make_obs_3d ) #-------------------------------------------------------------------- -oops_add_test( TESTNAME 3dvar_change_var - MODELNAME qg - OMP 2 - YAMLNAME testinput/3dvar_change_var.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_make_obs_3d ) - -oops_add_test( TESTNAME 3dvar_hybrid - MODELNAME qg - OMP 2 - YAMLNAME testinput/3dvar_hybrid.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_3d ) - -oops_add_test( TESTNAME 3dvar_hybrid_wo_jb_evaluation - MODELNAME qg - OMP 2 - YAMLNAME testinput/3dvar_hybrid_wo_jb_evaluation.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_3d ) - -oops_add_test( TESTNAME 3dfgat - MODELNAME qg - OMP 2 - YAMLNAME testinput/3dfgat.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_make_obs_3d ) +ecbuild_add_test( TARGET test_qg_3dvar_change_var + OMP 2 + ARGS testinput/3dvar_change_var.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_make_obs_3d ) + +ecbuild_add_test( TARGET test_qg_3dvar_hybrid + OMP 2 + ARGS testinput/3dvar_hybrid.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_3d ) + +ecbuild_add_test( TARGET test_qg_3dvar_hybrid_wo_jb_evaluation + OMP 2 + ARGS testinput/3dvar_hybrid_wo_jb_evaluation.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_3d ) + +ecbuild_add_test( TARGET test_qg_3dfgat + OMP 2 + ARGS testinput/3dfgat.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_make_obs_3d ) ##################################################################### # 4d variational tests ##################################################################### -oops_add_test( TESTNAME 4densvar - MODELNAME qg - MPI 7 - YAMLNAME testinput/4densvar.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_4d_12h ) - -oops_add_test( TESTNAME 4densvar_hybrid - MODELNAME qg - MPI 7 - YAMLNAME testinput/4densvar_hybrid.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_4d_12h ) - -oops_add_test( TESTNAME 4dvar_dripcg - MODELNAME qg - OMP 2 - YAMLNAME testinput/4dvar_dripcg.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) - -oops_add_test( TESTNAME 4dvar_drpcg_lmp - MODELNAME qg - OMP 2 - YAMLNAME testinput/4dvar_drpcg_lmp.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) - -oops_add_test( TESTNAME 4dvar_drpfom - MODELNAME qg - OMP 2 - YAMLNAME testinput/4dvar_drpfom.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) - -oops_add_test( TESTNAME 4dvar_drplanczos - MODELNAME qg - OMP 2 - YAMLNAME testinput/4dvar_drplanczos.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) - -oops_add_test( TESTNAME 4dvar_drplanczos_hybrid - MODELNAME qg - OMP 2 - YAMLNAME testinput/4dvar_drplanczos_hybrid.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_4d_24h ) - -#oops_add_test( TESTNAME 4dvar_forcing -# MODELNAME qg -# OMP 2 -# YAMLNAME testinput/4dvar_forcing.yaml -# EXENAME qg_4dvar.x -# TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) - -oops_add_test( TESTNAME 4dvar_ipcg - MODELNAME qg - OMP 2 - YAMLNAME testinput/4dvar_ipcg.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) - -oops_add_test( TESTNAME 4dvar_obs_biased - MODELNAME qg - OMP 2 - YAMLNAME testinput/4dvar_obs_biased.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_biased ) - -oops_add_test( TESTNAME 4dvar_rpcg - MODELNAME qg - OMP 2 - YAMLNAME testinput/4dvar_rpcg.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) - -oops_add_test( TESTNAME 4dvar_saddlepoint - MODELNAME qg - MPI 2 - YAMLNAME testinput/4dvar_saddlepoint.yaml - EXENAME qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h - CTOL 0.0000000001 ) +ecbuild_add_test( TARGET test_qg_4densvar + MPI 7 + ARGS testinput/4densvar.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_4d_12h ) + +ecbuild_add_test( TARGET test_qg_4densvar_hybrid + MPI 7 + ARGS testinput/4densvar_hybrid.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_4d_12h ) + +ecbuild_add_test( TARGET test_qg_4dvar_dripcg + OMP 2 + ARGS testinput/4dvar_dripcg.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) + +ecbuild_add_test( TARGET test_qg_4dvar_drpcg_lmp + OMP 2 + ARGS testinput/4dvar_drpcg_lmp.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) + +ecbuild_add_test( TARGET test_qg_4dvar_drpfom + OMP 2 + ARGS testinput/4dvar_drpfom.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) + +ecbuild_add_test( TARGET test_qg_4dvar_drplanczos + OMP 2 + ARGS testinput/4dvar_drplanczos.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) + +ecbuild_add_test( TARGET test_qg_4dvar_drplanczos_hybrid + OMP 2 + ARGS testinput/4dvar_drplanczos_hybrid.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_gen_ens_pert_B test_qg_make_obs_4d_24h ) + +#ecbuild_add_test( TARGET test_qg_4dvar_forcing +# OMP 2 +# ARGS testinput/4dvar_forcing.yaml +# COMMAND qg_4dvar.x +# TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) + +ecbuild_add_test( TARGET test_qg_4dvar_ipcg + OMP 2 + ARGS testinput/4dvar_ipcg.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) + +ecbuild_add_test( TARGET test_qg_4dvar_obs_biased + OMP 2 + ARGS testinput/4dvar_obs_biased.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_biased ) +ecbuild_add_test( TARGET test_qg_4dvar_rpcg + OMP 2 + ARGS testinput/4dvar_rpcg.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) + +ecbuild_add_test( TARGET test_qg_4dvar_saddlepoint + MPI 2 + ARGS testinput/4dvar_saddlepoint.yaml + COMMAND qg_4dvar.x + TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) ##################################################################### # EDA tests @@ -725,51 +678,44 @@ ecbuild_add_test( TARGET test_qg_eda_4dvar DEPENDS qg_eda.x TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) -oops_add_test( TESTNAME eda_3dvar_block - MODELNAME qg - MPI 4 - YAMLNAME testinput/eda_3dvar_block.yaml - EXENAME qg_eda.x - COMPARE - TEST_DEPENDS test_qg_make_obs_3d test_qg_gen_ens_pert_B ) +ecbuild_add_test( TARGET test_qg_eda_3dvar_block + MPI 4 + ARGS testinput/eda_3dvar_block.yaml + COMMAND qg_eda.x + TEST_DEPENDS test_qg_make_obs_3d test_qg_gen_ens_pert_B ) ##################################################################### # state-related tests ##################################################################### -oops_add_test( TESTNAME diffstates - MODELNAME qg - YAMLNAME testinput/diffstates.yaml - EXENAME qg_diffstates.x - TEST_DEPENDS test_qg_eda_3dvar test_qg_eda_4dvar - DEPENDS qg_eda.x ) - -oops_add_test( TESTNAME addincrement - MODELNAME qg - YAMLNAME testinput/addincrement.yaml - EXENAME qg_addincrement.x - TEST_DEPENDS test_qg_diffstates ) - -oops_add_test( TESTNAME addincrement_scaled - MODELNAME qg - YAMLNAME testinput/addincrement_scaled.yaml - EXENAME qg_addincrement.x - TEST_DEPENDS test_qg_diffstates ) - -oops_add_test( TESTNAME convertincrement - MODELNAME qg - OMP 2 - YAMLNAME testinput/convertincrement.yaml - EXENAME qg_convertincrement.x - TEST_DEPENDS test_qg_diffstates ) +ecbuild_add_test( TARGET test_qg_diffstates + ARGS testinput/diffstates.yaml + COMMAND qg_diffstates.x + TEST_DEPENDS test_qg_eda_3dvar test_qg_eda_4dvar + DEPENDS qg_eda.x ) + +ecbuild_add_test( TARGET test_qg_addincrement + ARGS testinput/addincrement.yaml + COMMAND qg_addincrement.x + TEST_DEPENDS test_qg_diffstates ) + +ecbuild_add_test( TARGET test_qg_addincrement_scaled + ARGS testinput/addincrement_scaled.yaml + COMMAND qg_addincrement.x + TEST_DEPENDS test_qg_diffstates ) + +ecbuild_add_test( TARGET test_qg_convertincrement + OMP 2 + ARGS testinput/convertincrement.yaml + COMMAND qg_convertincrement.x + TEST_DEPENDS test_qg_diffstates ) ##################################################################### # LETKF tests ##################################################################### -oops_add_test( TESTNAME letkf - MODELNAME qg - YAMLNAME testinput/letkf.yaml - EXENAME qg_letkf.x - OMP 2 - TEST_DEPENDS test_qg_make_obs_3d test_qg_gen_ens_pert_B ) +ecbuild_add_test( TARGET test_qg_letkf + ARGS testinput/letkf.yaml + COMMAND qg_letkf.x + OMP 2 + TEST_DEPENDS test_qg_make_obs_3d test_qg_gen_ens_pert_B ) diff --git a/qg/test/testinput/3densvar.yaml b/qg/test/testinput/3densvar.yaml index ff9b65de5..79c7c2ad6 100644 --- a/qg/test/testinput/3densvar.yaml +++ b/qg/test/testinput/3densvar.yaml @@ -44,7 +44,7 @@ cost function: - obs error: covariance model: diagonal obs operator: - obs type: Stream + obs type: Stream obs space: obsdatain: obsfile: Data/truth.obs3d.nc @@ -54,7 +54,7 @@ cost function: - obs error: covariance model: diagonal obs operator: - obs type: Wind + obs type: Wind obs space: obsdatain: obsfile: Data/truth.obs3d.nc @@ -64,7 +64,7 @@ cost function: - obs error: covariance model: diagonal obs operator: - obs type: WSpeed + obs type: WSpeed obs space: obsdatain: obsfile: Data/truth.obs3d.nc @@ -101,3 +101,6 @@ output: exp: 3densvar frequency: PT6H type: an + +test: + reference filename: testoutput/3densvar.test diff --git a/qg/test/testinput/3dfgat.yaml b/qg/test/testinput/3dfgat.yaml index 47e96d78f..d3703188f 100644 --- a/qg/test/testinput/3dfgat.yaml +++ b/qg/test/testinput/3dfgat.yaml @@ -80,3 +80,6 @@ output: first: PT3H frequency: PT6H type: an + +test: + reference filename: testoutput/3dfgat.test diff --git a/qg/test/testinput/3dvar.yaml b/qg/test/testinput/3dvar.yaml index aa319473e..d066f5e2c 100644 --- a/qg/test/testinput/3dvar.yaml +++ b/qg/test/testinput/3dvar.yaml @@ -104,3 +104,6 @@ output: exp: 3dvar frequency: PT6H type: an + +test: + reference filename: testoutput/3dvar.test diff --git a/qg/test/testinput/3dvar_change_var.yaml b/qg/test/testinput/3dvar_change_var.yaml index f98b1b829..1ace1c2fc 100644 --- a/qg/test/testinput/3dvar_change_var.yaml +++ b/qg/test/testinput/3dvar_change_var.yaml @@ -82,3 +82,6 @@ output: exp: 3dvar frequency: PT6H type: an + +test: + reference filename: testoutput/3dvar_change_var.test diff --git a/qg/test/testinput/3dvar_hybrid.yaml b/qg/test/testinput/3dvar_hybrid.yaml index 51c6b2fb7..f23f496bf 100644 --- a/qg/test/testinput/3dvar_hybrid.yaml +++ b/qg/test/testinput/3dvar_hybrid.yaml @@ -104,3 +104,6 @@ output: exp: 3dvar frequency: PT6H type: an + +test: + reference filename: testoutput/3dvar_hybrid.test diff --git a/qg/test/testinput/3dvar_hybrid_wo_jb_evaluation.yaml b/qg/test/testinput/3dvar_hybrid_wo_jb_evaluation.yaml index 4dc7d168d..564c78253 100644 --- a/qg/test/testinput/3dvar_hybrid_wo_jb_evaluation.yaml +++ b/qg/test/testinput/3dvar_hybrid_wo_jb_evaluation.yaml @@ -105,3 +105,6 @@ output: exp: 3dvar frequency: PT6H type: an + +test: + reference filename: testoutput/3dvar_hybrid_wo_jb_evaluation.test diff --git a/qg/test/testinput/4densvar.yaml b/qg/test/testinput/4densvar.yaml index a3b564563..246b514a9 100644 --- a/qg/test/testinput/4densvar.yaml +++ b/qg/test/testinput/4densvar.yaml @@ -251,3 +251,6 @@ output: first: PT0S frequency: PT6H type: an + +test: + reference filename: testoutput/4densvar.test diff --git a/qg/test/testinput/4densvar_hybrid.yaml b/qg/test/testinput/4densvar_hybrid.yaml index 1155abbb5..f5af595b5 100644 --- a/qg/test/testinput/4densvar_hybrid.yaml +++ b/qg/test/testinput/4densvar_hybrid.yaml @@ -258,3 +258,6 @@ output: first: PT0S frequency: PT6H type: an + +test: + reference filename: testoutput/4densvar_hybrid.test diff --git a/qg/test/testinput/4dvar_dripcg.yaml b/qg/test/testinput/4dvar_dripcg.yaml index 037d29d2c..99d35804c 100644 --- a/qg/test/testinput/4dvar_dripcg.yaml +++ b/qg/test/testinput/4dvar_dripcg.yaml @@ -23,7 +23,7 @@ cost function: - obs error: covariance model: diagonal obs operator: - obs type: Stream + obs type: Stream obs space: obsdatain: obsfile: Data/truth.obs4d_24h.nc @@ -33,7 +33,7 @@ cost function: - obs error: covariance model: diagonal obs operator: - obs type: Wind + obs type: Wind obs space: obsdatain: obsfile: Data/truth.obs4d_24h.nc @@ -43,7 +43,7 @@ cost function: - obs error: covariance model: diagonal obs operator: - obs type: WSpeed + obs type: WSpeed obs space: obsdatain: obsfile: Data/truth.obs4d_24h.nc @@ -99,3 +99,6 @@ output: first: PT0S frequency: PT6H type: an + +test: + reference filename: testoutput/4dvar_dripcg.test diff --git a/qg/test/testinput/4dvar_drpcg_lmp.yaml b/qg/test/testinput/4dvar_drpcg_lmp.yaml index 82157dd35..ea9819a19 100644 --- a/qg/test/testinput/4dvar_drpcg_lmp.yaml +++ b/qg/test/testinput/4dvar_drpcg_lmp.yaml @@ -110,3 +110,6 @@ output: first: PT0S frequency: PT6H type: an + +test: + reference filename: testoutput/4dvar_drpcg_lmp.test diff --git a/qg/test/testinput/4dvar_drpfom.yaml b/qg/test/testinput/4dvar_drpfom.yaml index ee657188f..0156b6864 100644 --- a/qg/test/testinput/4dvar_drpfom.yaml +++ b/qg/test/testinput/4dvar_drpfom.yaml @@ -23,7 +23,7 @@ cost function: - obs error: covariance model: diagonal obs operator: - obs type: Stream + obs type: Stream obs space: obsdatain: obsfile: Data/truth.obs4d_24h.nc @@ -33,7 +33,7 @@ cost function: - obs error: covariance model: diagonal obs operator: - obs type: Wind + obs type: Wind obs space: obsdatain: obsfile: Data/truth.obs4d_24h.nc @@ -49,7 +49,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/4dvar_drpfom.obs4d_24h.nc - obs type: WSpeed + obs type: WSpeed constraints: - jcdfi: filtered variables: [x] @@ -99,3 +99,6 @@ output: first: PT0S frequency: PT6H type: an + +test: + reference filename: testoutput/4dvar_drpfom.test diff --git a/qg/test/testinput/4dvar_drplanczos.yaml b/qg/test/testinput/4dvar_drplanczos.yaml index 1c53fce40..5462da35c 100644 --- a/qg/test/testinput/4dvar_drplanczos.yaml +++ b/qg/test/testinput/4dvar_drplanczos.yaml @@ -110,3 +110,6 @@ output: first: PT0S frequency: PT6H type: an + +test: + reference filename: testoutput/4dvar_drplanczos.test diff --git a/qg/test/testinput/4dvar_drplanczos_hybrid.yaml b/qg/test/testinput/4dvar_drplanczos_hybrid.yaml index 4bee9b610..9af4e9e06 100644 --- a/qg/test/testinput/4dvar_drplanczos_hybrid.yaml +++ b/qg/test/testinput/4dvar_drplanczos_hybrid.yaml @@ -59,7 +59,7 @@ cost function: - obs error: covariance model: diagonal obs operator: - obs type: Stream + obs type: Stream obs space: obsdatain: obsfile: Data/truth.obs4d_24h.nc @@ -69,7 +69,7 @@ cost function: - obs error: covariance model: diagonal obs operator: - obs type: Wind + obs type: Wind obs space: obsdatain: obsfile: Data/truth.obs4d_24h.nc @@ -85,7 +85,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/4dvar_drplanczos_hybrid.obs4d_24h.nc - obs type: WSpeed + obs type: WSpeed constraints: - jcdfi: filtered variables: [x] @@ -137,3 +137,6 @@ output: first: PT0S frequency: PT6H type: an + +test: + reference filename: testoutput/4dvar_drplanczos_hybrid.test diff --git a/qg/test/testinput/4dvar_forcing.yaml b/qg/test/testinput/4dvar_forcing.yaml index 32ebbe465..459907f70 100644 --- a/qg/test/testinput/4dvar_forcing.yaml +++ b/qg/test/testinput/4dvar_forcing.yaml @@ -33,7 +33,7 @@ cost function: - obs error: covariance model: diagonal obs operator: - obs type: Stream + obs type: Stream obs space: obsdatain: obsfile: Data/truth.obs4d_24h.nc @@ -43,7 +43,7 @@ cost function: - obs error: covariance model: diagonal obs operator: - obs type: Wind + obs type: Wind obs space: obsdatain: obsfile: Data/truth.obs4d_24h.nc @@ -59,7 +59,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/4dvar_forcing.obs4d_24h.nc - obs type: WSpeed + obs type: WSpeed constraints: - jcdfi: filtered variables: [x] @@ -106,3 +106,6 @@ output: exp: 4dvar_forcing frequency: PT6H type: an + +test: + reference filename: testoutput/4dvar_forcing.test diff --git a/qg/test/testinput/4dvar_ipcg.yaml b/qg/test/testinput/4dvar_ipcg.yaml index 7858985ff..d292f3b8f 100644 --- a/qg/test/testinput/4dvar_ipcg.yaml +++ b/qg/test/testinput/4dvar_ipcg.yaml @@ -106,3 +106,6 @@ output: first: PT0S frequency: PT6H type: an + +test: + reference filename: testoutput/4dvar_ipcg.test diff --git a/qg/test/testinput/4dvar_obs_biased.yaml b/qg/test/testinput/4dvar_obs_biased.yaml index b40b8191e..3e01c2b26 100644 --- a/qg/test/testinput/4dvar_obs_biased.yaml +++ b/qg/test/testinput/4dvar_obs_biased.yaml @@ -107,3 +107,6 @@ output: first: PT0S frequency: PT6H type: an + +test: + reference filename: testoutput/4dvar_obs_biased.test diff --git a/qg/test/testinput/4dvar_rpcg.yaml b/qg/test/testinput/4dvar_rpcg.yaml index 9937e3b95..19bfeb3c6 100644 --- a/qg/test/testinput/4dvar_rpcg.yaml +++ b/qg/test/testinput/4dvar_rpcg.yaml @@ -100,3 +100,6 @@ output: first: PT0S frequency: PT6H type: an + +test: + reference filename: testoutput/4dvar_rpcg.test diff --git a/qg/test/testinput/4dvar_saddlepoint.yaml b/qg/test/testinput/4dvar_saddlepoint.yaml index daaa0e626..171fb4408 100644 --- a/qg/test/testinput/4dvar_saddlepoint.yaml +++ b/qg/test/testinput/4dvar_saddlepoint.yaml @@ -33,7 +33,7 @@ cost function: - obs error: covariance model: diagonal obs operator: - obs type: Stream + obs type: Stream obs space: obsdatain: obsfile: Data/truth.obs4d_24h.nc @@ -43,7 +43,7 @@ cost function: - obs error: covariance model: diagonal obs operator: - obs type: Wind + obs type: Wind obs space: obsdatain: obsfile: Data/truth.obs4d_24h.nc @@ -59,7 +59,7 @@ cost function: obsfile: Data/truth.obs4d_24h.nc obsdataout: obsfile: Data/4dvar_saddlepoint.obs4d_24h.nc - obs type: WSpeed + obs type: WSpeed constraints: - jcdfi: filtered variables: [x] @@ -115,3 +115,7 @@ output: exp: 4dvar_saddlepoint frequency: PT6H type: an + +test: + reference filename: testoutput/4dvar_saddlepoint.test + float relative tolerance: 0.0000000001 diff --git a/qg/test/testinput/addincrement.yaml b/qg/test/testinput/addincrement.yaml index 6e54142e1..8d644cafb 100644 --- a/qg/test/testinput/addincrement.yaml +++ b/qg/test/testinput/addincrement.yaml @@ -18,3 +18,6 @@ output: date: '2010-01-01T12:00:00Z' exp: addinc type: an + +test: + reference filename: testoutput/addincrement.test diff --git a/qg/test/testinput/addincrement_scaled.yaml b/qg/test/testinput/addincrement_scaled.yaml index 23f67ae49..04a3104e2 100644 --- a/qg/test/testinput/addincrement_scaled.yaml +++ b/qg/test/testinput/addincrement_scaled.yaml @@ -19,3 +19,6 @@ output: date: '2010-01-01T12:00:00Z' exp: addinc_scaled type: an + +test: + reference filename: testoutput/addincrement_scaled.test diff --git a/qg/test/testinput/analytic_forecast.yaml b/qg/test/testinput/analytic_forecast.yaml index 574fe5f26..b7beab927 100644 --- a/qg/test/testinput/analytic_forecast.yaml +++ b/qg/test/testinput/analytic_forecast.yaml @@ -18,3 +18,6 @@ geometry: ny: 20 depths: [4500.0, 5500.0] heating: false + +test: + reference filename: testoutput/analytic_forecast.test diff --git a/qg/test/testinput/convertincrement.yaml b/qg/test/testinput/convertincrement.yaml index e31590885..da8e0195d 100644 --- a/qg/test/testinput/convertincrement.yaml +++ b/qg/test/testinput/convertincrement.yaml @@ -35,3 +35,6 @@ increments: trajectory: date: '2010-01-01T12:00:00Z' filename: Data/mem002.eda_3dvar.an.2010-01-01T12:00:00Z.nc + +test: + reference filename: testoutput/convertincrement.test diff --git a/qg/test/testinput/convertstate.yaml b/qg/test/testinput/convertstate.yaml index bdecb4efa..664a75e5e 100644 --- a/qg/test/testinput/convertstate.yaml +++ b/qg/test/testinput/convertstate.yaml @@ -47,3 +47,6 @@ states: datadir: Data exp: convert type: fc + +test: + reference filename: testoutput/convertstate.test diff --git a/qg/test/testinput/dfi.yaml b/qg/test/testinput/dfi.yaml index f9b432d6d..458c5c7fe 100644 --- a/qg/test/testinput/dfi.yaml +++ b/qg/test/testinput/dfi.yaml @@ -22,3 +22,6 @@ geometry: nx: 40 ny: 20 depths: [4500.0, 5500.0] + +test: + reference filename: testoutput/dfi.test diff --git a/qg/test/testinput/diffstates.yaml b/qg/test/testinput/diffstates.yaml index 37da5f540..d9b944a44 100644 --- a/qg/test/testinput/diffstates.yaml +++ b/qg/test/testinput/diffstates.yaml @@ -17,3 +17,6 @@ output: date: '2010-01-01T12:00:00Z' exp: difst type: in + +test: + reference filename: testoutput/diffstates.test diff --git a/qg/test/testinput/dirac_cov.yaml b/qg/test/testinput/dirac_cov.yaml index ad1a22d48..ebc5c5dd1 100644 --- a/qg/test/testinput/dirac_cov.yaml +++ b/qg/test/testinput/dirac_cov.yaml @@ -28,3 +28,6 @@ output variance: datadir: Data exp: dirac_cov_var type: an + +test: + reference filename: testoutput/dirac_cov.test diff --git a/qg/test/testinput/dirac_hyb_field.yaml b/qg/test/testinput/dirac_hyb_field.yaml index 9c98f6682..baae7d20c 100644 --- a/qg/test/testinput/dirac_hyb_field.yaml +++ b/qg/test/testinput/dirac_hyb_field.yaml @@ -33,7 +33,7 @@ background error: weight: date: 2010-01-01T00:00:00Z filename: Data/uniform_field_hybrid.fc.2010-01-01T00:00:00Z.PT0S.nc - - covariance: + - covariance: covariance model: QgError horizontal_length_scale: 2.2e6 maximum_condition_number: 1.0e6 @@ -67,3 +67,6 @@ output localization: date: 2010-01-01T12:00:00Z exp: dirac_qg_hyb_field_localization type: an + +test: + reference filename: testoutput/dirac_hyb_field.test diff --git a/qg/test/testinput/dirac_hyb_value.yaml b/qg/test/testinput/dirac_hyb_value.yaml index fde8268a3..6b1451d34 100644 --- a/qg/test/testinput/dirac_hyb_value.yaml +++ b/qg/test/testinput/dirac_hyb_value.yaml @@ -32,7 +32,7 @@ background error: filename: Data/forecast.ens.10.2009-12-31T00:00:00Z.P1DT12H.nc weight: value: 0.5 - - covariance: + - covariance: covariance model: QgError horizontal_length_scale: 2.2e6 maximum_condition_number: 1.0e6 @@ -69,3 +69,6 @@ output variance: datadir: Data exp: dirac_hyb_value_var type: an + +test: + reference filename: testoutput/dirac_hyb_value.test diff --git a/qg/test/testinput/dirac_loc_3d.yaml b/qg/test/testinput/dirac_loc_3d.yaml index 28cfdc7c9..ae3d9c20d 100644 --- a/qg/test/testinput/dirac_loc_3d.yaml +++ b/qg/test/testinput/dirac_loc_3d.yaml @@ -60,3 +60,6 @@ output variance: datadir: Data exp: dirac_loc_3d_var type: an + +test: + reference filename: testoutput/dirac_loc_3d.test diff --git a/qg/test/testinput/dirac_loc_4d.yaml b/qg/test/testinput/dirac_loc_4d.yaml index 7bfc527c2..284392462 100644 --- a/qg/test/testinput/dirac_loc_4d.yaml +++ b/qg/test/testinput/dirac_loc_4d.yaml @@ -331,3 +331,6 @@ output variance: datadir: Data exp: dirac_loc_4d_var type: an + +test: + reference filename: testoutput/dirac_loc_4d.test diff --git a/qg/test/testinput/dirac_no_loc.yaml b/qg/test/testinput/dirac_no_loc.yaml index 54e01be2c..ebe54ce02 100644 --- a/qg/test/testinput/dirac_no_loc.yaml +++ b/qg/test/testinput/dirac_no_loc.yaml @@ -45,3 +45,6 @@ output variance: datadir: Data exp: dirac_no_loc_var type: an + +test: + reference filename: testoutput/dirac_no_loc.test diff --git a/qg/test/testinput/eda_3dvar_block.yaml b/qg/test/testinput/eda_3dvar_block.yaml index 0dd142b19..b6f421e5a 100644 --- a/qg/test/testinput/eda_3dvar_block.yaml +++ b/qg/test/testinput/eda_3dvar_block.yaml @@ -3,3 +3,6 @@ files: - "testinput/eda_3dvar_block_2.yaml" - "testinput/eda_3dvar_block_3.yaml" - "testinput/eda_3dvar_block_4.yaml" + +test: + reference filename: testoutput/eda_3dvar_block.test diff --git a/qg/test/testinput/eda_4dvar.yaml b/qg/test/testinput/eda_4dvar.yaml index 7ecc7a306..7e5567c2d 100644 --- a/qg/test/testinput/eda_4dvar.yaml +++ b/qg/test/testinput/eda_4dvar.yaml @@ -3,3 +3,6 @@ files: - "testinput/eda_4dvar_2.yaml" - "testinput/eda_4dvar_3.yaml" - "testinput/eda_4dvar_4.yaml" + +test: + reference filename: testoutput/eda_4dvar.test diff --git a/qg/test/testinput/ens_forecast.yaml b/qg/test/testinput/ens_forecast.yaml index dc4a85aa0..9268f1690 100644 --- a/qg/test/testinput/ens_forecast.yaml +++ b/qg/test/testinput/ens_forecast.yaml @@ -1,3 +1,6 @@ files: - "testinput/ens_forecast_1.yaml" - "testinput/ens_forecast_2.yaml" + +test: + reference filename: testoutput/ens_forecast.test diff --git a/qg/test/testinput/ens_hofx.yaml b/qg/test/testinput/ens_hofx.yaml index c6fa7ce83..72b083ed8 100644 --- a/qg/test/testinput/ens_hofx.yaml +++ b/qg/test/testinput/ens_hofx.yaml @@ -4,3 +4,6 @@ files: - "testinput/ens_hofx_3.yaml" - "testinput/ens_hofx_4.yaml" - "testinput/ens_hofx_5.yaml" + +test: + reference filename: testoutput/ens_hofx.test diff --git a/qg/test/testinput/ens_recenter.yaml b/qg/test/testinput/ens_recenter.yaml index 3a6a6f65f..8189fe4f9 100644 --- a/qg/test/testinput/ens_recenter.yaml +++ b/qg/test/testinput/ens_recenter.yaml @@ -26,3 +26,6 @@ recentered output: exp: recenter type: ens date: 2010-01-01T00:00:00Z + +test: + reference filename: testoutput/ens_recenter.test diff --git a/qg/test/testinput/ens_variance.yaml b/qg/test/testinput/ens_variance.yaml index 12968a2b5..86da739ca 100644 --- a/qg/test/testinput/ens_variance.yaml +++ b/qg/test/testinput/ens_variance.yaml @@ -23,3 +23,6 @@ variance output: exp: variance type: fc date: 2010-01-01T00:00:00Z + +test: + reference filename: testoutput/ens_variance.test diff --git a/qg/test/testinput/ens_variance_inflation_field.yaml b/qg/test/testinput/ens_variance_inflation_field.yaml index 415e77d29..b8bbf65dc 100644 --- a/qg/test/testinput/ens_variance_inflation_field.yaml +++ b/qg/test/testinput/ens_variance_inflation_field.yaml @@ -26,3 +26,6 @@ variance output: exp: variance_inflation_field type: fc date: 2010-01-01T00:00:00Z + +test: + reference filename: testoutput/ens_variance_inflation_field.test diff --git a/qg/test/testinput/ens_variance_inflation_value.yaml b/qg/test/testinput/ens_variance_inflation_value.yaml index 9c0fb6415..71e696558 100644 --- a/qg/test/testinput/ens_variance_inflation_value.yaml +++ b/qg/test/testinput/ens_variance_inflation_value.yaml @@ -24,3 +24,6 @@ variance output: exp: variance_inflation_value type: fc date: 2010-01-01T00:00:00Z + +test: + reference filename: testoutput/ens_variance_inflation_value.test diff --git a/qg/test/testinput/forecast.yaml b/qg/test/testinput/forecast.yaml index ca64d38ab..b044af4d6 100644 --- a/qg/test/testinput/forecast.yaml +++ b/qg/test/testinput/forecast.yaml @@ -17,3 +17,6 @@ output: type: fc prints: frequency: PT3H + +test: + reference filename: testoutput/forecast.test diff --git a/qg/test/testinput/gen_ens_pert_B.yaml b/qg/test/testinput/gen_ens_pert_B.yaml index b705b7876..0d88fb868 100644 --- a/qg/test/testinput/gen_ens_pert_B.yaml +++ b/qg/test/testinput/gen_ens_pert_B.yaml @@ -24,3 +24,6 @@ geometry: ny: 20 depths: [4500.0, 5500.0] perturbed variables: [x] + +test: + reference filename: testoutput/gen_ens_pert_B.test diff --git a/qg/test/testinput/hofx.yaml b/qg/test/testinput/hofx.yaml index 450b2a59f..86c7cea6c 100644 --- a/qg/test/testinput/hofx.yaml +++ b/qg/test/testinput/hofx.yaml @@ -50,3 +50,6 @@ observations: interpolation type: default_c prints: frequency: PT3H + +test: + reference filename: testoutput/hofx.test diff --git a/qg/test/testinput/hofx3d.yaml b/qg/test/testinput/hofx3d.yaml index 80e292622..9a040cc9f 100644 --- a/qg/test/testinput/hofx3d.yaml +++ b/qg/test/testinput/hofx3d.yaml @@ -37,3 +37,6 @@ observations: obs type: WSpeed prints: frequency: PT3H + +test: + reference filename: testoutput/hofx3d.test diff --git a/qg/test/testinput/letkf.yaml b/qg/test/testinput/letkf.yaml index a47b5291f..efdcdc05b 100644 --- a/qg/test/testinput/letkf.yaml +++ b/qg/test/testinput/letkf.yaml @@ -108,3 +108,6 @@ output: date: *date_end exp: letkf.end.%{member}% type: an + +test: + reference filename: testoutput/letkf.test diff --git a/qg/test/testinput/make_obs_3d.yaml b/qg/test/testinput/make_obs_3d.yaml index 6be1a0508..3528f28aa 100644 --- a/qg/test/testinput/make_obs_3d.yaml +++ b/qg/test/testinput/make_obs_3d.yaml @@ -49,3 +49,6 @@ observations: obs_error: 12.0 obs_period: PT2H make obs: true + +test: + reference filename: testoutput/make_obs_3d.test diff --git a/qg/test/testinput/make_obs_4d_12h.yaml b/qg/test/testinput/make_obs_4d_12h.yaml index 150b959c8..80aeb4841 100644 --- a/qg/test/testinput/make_obs_4d_12h.yaml +++ b/qg/test/testinput/make_obs_4d_12h.yaml @@ -51,3 +51,6 @@ observations: make obs: true prints: frequency: PT3H + +test: + reference filename: testoutput/make_obs_4d_12h.test diff --git a/qg/test/testinput/make_obs_4d_24h.yaml b/qg/test/testinput/make_obs_4d_24h.yaml index c72c9dd8a..8df29ccc0 100644 --- a/qg/test/testinput/make_obs_4d_24h.yaml +++ b/qg/test/testinput/make_obs_4d_24h.yaml @@ -60,3 +60,6 @@ observations: make obs: true prints: frequency: PT3H + +test: + reference filename: testoutput/make_obs_4d_24h.test diff --git a/qg/test/testinput/make_obs_4d_biased.yaml b/qg/test/testinput/make_obs_4d_biased.yaml index 242d085d4..84cfd2011 100644 --- a/qg/test/testinput/make_obs_4d_biased.yaml +++ b/qg/test/testinput/make_obs_4d_biased.yaml @@ -55,3 +55,6 @@ observations: make obs: true prints: frequency: PT3H + +test: + reference filename: testoutput/make_obs_4d_biased.test diff --git a/qg/test/testinput/rtpp.yaml b/qg/test/testinput/rtpp.yaml index cea62c76a..38f793223 100644 --- a/qg/test/testinput/rtpp.yaml +++ b/qg/test/testinput/rtpp.yaml @@ -21,3 +21,6 @@ output: exp: rtpp.%{member}% type: an factor: 0.5 + +test: + reference filename: testoutput/rtpp.test diff --git a/qg/test/testinput/static_b_init.yaml b/qg/test/testinput/static_b_init.yaml index 96423cea1..f739f7956 100644 --- a/qg/test/testinput/static_b_init.yaml +++ b/qg/test/testinput/static_b_init.yaml @@ -12,3 +12,6 @@ background: filename: Data/truth.fc.2009-12-15T00:00:00Z.P16D.nc date: 2009-12-31T00:00:00Z analysis variables: [x] + +test: + reference filename: testoutput/static_b_init.test diff --git a/qg/test/testinput/truth.yaml b/qg/test/testinput/truth.yaml index ab71f39fb..ca87f0316 100644 --- a/qg/test/testinput/truth.yaml +++ b/qg/test/testinput/truth.yaml @@ -17,3 +17,6 @@ output: type: fc prints: frequency: PT3H + +test: + reference filename: testoutput/truth.test diff --git a/qg/test/testinput/uniform_field_hybrid.yaml b/qg/test/testinput/uniform_field_hybrid.yaml index d60105ddf..63afaef59 100644 --- a/qg/test/testinput/uniform_field_hybrid.yaml +++ b/qg/test/testinput/uniform_field_hybrid.yaml @@ -19,3 +19,6 @@ geometry: ny: 20 depths: [4500.0, 5500.0] heating: false + +test: + reference filename: testoutput/uniform_field_hybrid.test diff --git a/qg/test/testinput/uniform_field_inflation.yaml b/qg/test/testinput/uniform_field_inflation.yaml index 23db796e7..4558275dd 100644 --- a/qg/test/testinput/uniform_field_inflation.yaml +++ b/qg/test/testinput/uniform_field_inflation.yaml @@ -19,3 +19,6 @@ geometry: ny: 20 depths: [4500.0, 5500.0] heating: false + +test: + reference filename: testoutput/uniform_field_inflation.test diff --git a/qg/test/testoutput/3densvar.test b/qg/test/testoutput/3densvar.test index 27e6ba906..6154dcd04 100644 --- a/qg/test/testoutput/3densvar.test +++ b/qg/test/testoutput/3densvar.test @@ -1,29 +1,29 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 7820.89, nobs = 600, Jo/n = 13.0348, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1065.61, nobs = 600, Jo/n = 1.77601, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 144.328, nobs = 300, Jo/n = 0.481094, err = 12 -Test : CostFunction: Nonlinear J = 9030.82 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0780493 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4489e+08, Max= 9.7006e+07, RMS= 1.7680e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 50.19 -Test : CostJo : Nonlinear Jo(Stream) = 439.3, nobs = 600, Jo/n = 0.7321, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 117.3, nobs = 600, Jo/n = 0.1955, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 19.82, nobs = 300, Jo/n = 0.06605, err = 12 -Test : CostFunction: Nonlinear J = 626.6 -Test : DRIPCGMinimizer: reduction in residual norm = 0.3926 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4245e+08, Max= 9.8041e+07, RMS= 1.7680e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 59.3 -Test : CostJo : Nonlinear Jo(Stream) = 407.4, nobs = 600, Jo/n = 0.679, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 75.74, nobs = 600, Jo/n = 0.1262, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 14.42, nobs = 300, Jo/n = 0.04806, err = 12 -Test : CostFunction: Nonlinear J = 556.8 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 7820.89, nobs = 600, Jo/n = 13.0348, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1065.61, nobs = 600, Jo/n = 1.77601, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 144.328, nobs = 300, Jo/n = 0.481094, err = 12 +CostFunction: Nonlinear J = 9030.82 +DRIPCGMinimizer: reduction in residual norm = 0.0780493 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4489e+08, Max= 9.7006e+07, RMS= 1.7680e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 50.19 +CostJo : Nonlinear Jo(Stream) = 439.3, nobs = 600, Jo/n = 0.7321, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 117.3, nobs = 600, Jo/n = 0.1955, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 19.82, nobs = 300, Jo/n = 0.06605, err = 12 +CostFunction: Nonlinear J = 626.6 +DRIPCGMinimizer: reduction in residual norm = 0.3926 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4245e+08, Max= 9.8041e+07, RMS= 1.7680e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 59.3 +CostJo : Nonlinear Jo(Stream) = 407.4, nobs = 600, Jo/n = 0.679, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 75.74, nobs = 600, Jo/n = 0.1262, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 14.42, nobs = 300, Jo/n = 0.04806, err = 12 +CostFunction: Nonlinear J = 556.8 diff --git a/qg/test/testoutput/3dfgat.test b/qg/test/testoutput/3dfgat.test index f05065ac0..87b3f7ff4 100644 --- a/qg/test/testoutput/3dfgat.test +++ b/qg/test/testoutput/3dfgat.test @@ -1,17 +1,17 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 7671.34, nobs = 600, Jo/n = 12.7856, err = 4e+06 (Monitoring only) -Test : CostJo : Nonlinear Jo(Wind) = 1038.28, nobs = 600, Jo/n = 1.73047, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 139.319, nobs = 300, Jo/n = 0.464395, err = 12 -Test : CostFunction: Nonlinear J = 1177.6 -Test : DRIPCGMinimizer: reduction in residual norm = 0.186377 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T09:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6444e+08, Max= 1.0829e+08, RMS= 1.7939e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 79.13 -Test : CostJo : Nonlinear Jo(Stream) = 1596, nobs = 600, Jo/n = 2.66, err = 4e+06 (Monitoring only) -Test : CostJo : Nonlinear Jo(Wind) = 159.4, nobs = 600, Jo/n = 0.2656, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 34.27, nobs = 300, Jo/n = 0.1142, err = 12 -Test : CostFunction: Nonlinear J = 272.8 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 7671.34, nobs = 600, Jo/n = 12.7856, err = 4e+06 (Monitoring only) +CostJo : Nonlinear Jo(Wind) = 1038.28, nobs = 600, Jo/n = 1.73047, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 139.319, nobs = 300, Jo/n = 0.464395, err = 12 +CostFunction: Nonlinear J = 1177.6 +DRIPCGMinimizer: reduction in residual norm = 0.186377 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T09:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6444e+08, Max= 1.0829e+08, RMS= 1.7939e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 79.13 +CostJo : Nonlinear Jo(Stream) = 1596, nobs = 600, Jo/n = 2.66, err = 4e+06 (Monitoring only) +CostJo : Nonlinear Jo(Wind) = 159.4, nobs = 600, Jo/n = 0.2656, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 34.27, nobs = 300, Jo/n = 0.1142, err = 12 +CostFunction: Nonlinear J = 272.8 diff --git a/qg/test/testoutput/3dvar.test b/qg/test/testoutput/3dvar.test index 4956a39a7..81a5510cf 100644 --- a/qg/test/testoutput/3dvar.test +++ b/qg/test/testoutput/3dvar.test @@ -1,29 +1,29 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 7820.89, nobs = 600, Jo/n = 13.0348, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1065.61, nobs = 600, Jo/n = 1.77601, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 144.328, nobs = 300, Jo/n = 0.481094, err = 12 -Test : CostFunction: Nonlinear J = 9030.82 -Test : DRIPCGMinimizer: reduction in residual norm = 0.166244 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4870e+08, Max= 1.0374e+08, RMS= 1.7699e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 62.81 -Test : CostJo : Nonlinear Jo(Stream) = 666.7, nobs = 600, Jo/n = 1.111, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 459.7, nobs = 600, Jo/n = 0.7661, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 64.52, nobs = 300, Jo/n = 0.2151, err = 12 -Test : CostFunction: Nonlinear J = 1254 -Test : DRIPCGMinimizer: reduction in residual norm = 0.523 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4549e+08, Max= 1.0365e+08, RMS= 1.7647e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 97.9 -Test : CostJo : Nonlinear Jo(Stream) = 493.1, nobs = 600, Jo/n = 0.8218, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 302.1, nobs = 600, Jo/n = 0.5035, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 45.86, nobs = 300, Jo/n = 0.1529, err = 12 -Test : CostFunction: Nonlinear J = 938.9 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 7820.89, nobs = 600, Jo/n = 13.0348, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1065.61, nobs = 600, Jo/n = 1.77601, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 144.328, nobs = 300, Jo/n = 0.481094, err = 12 +CostFunction: Nonlinear J = 9030.82 +DRIPCGMinimizer: reduction in residual norm = 0.166244 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4870e+08, Max= 1.0374e+08, RMS= 1.7699e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 62.81 +CostJo : Nonlinear Jo(Stream) = 666.7, nobs = 600, Jo/n = 1.111, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 459.7, nobs = 600, Jo/n = 0.7661, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 64.52, nobs = 300, Jo/n = 0.2151, err = 12 +CostFunction: Nonlinear J = 1254 +DRIPCGMinimizer: reduction in residual norm = 0.523 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4549e+08, Max= 1.0365e+08, RMS= 1.7647e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 97.9 +CostJo : Nonlinear Jo(Stream) = 493.1, nobs = 600, Jo/n = 0.8218, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 302.1, nobs = 600, Jo/n = 0.5035, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 45.86, nobs = 300, Jo/n = 0.1529, err = 12 +CostFunction: Nonlinear J = 938.9 diff --git a/qg/test/testoutput/3dvar_change_var.test b/qg/test/testoutput/3dvar_change_var.test index 6a3df60b1..302eb8c52 100644 --- a/qg/test/testoutput/3dvar_change_var.test +++ b/qg/test/testoutput/3dvar_change_var.test @@ -1,29 +1,29 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 7820.89, nobs = 600, Jo/n = 13.0348, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1065.61, nobs = 600, Jo/n = 1.77601, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 144.328, nobs = 300, Jo/n = 0.481094, err = 12 -Test : CostFunction: Nonlinear J = 9030.82 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0295408 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Potential vorticity : Min= -7.1570e-04, Max= 6.3930e-04, RMS= 3.3441e-04 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 56.44 -Test : CostJo : Nonlinear Jo(Stream) = 754.3, nobs = 600, Jo/n = 1.257, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 464.9, nobs = 600, Jo/n = 0.7748, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 61.71, nobs = 300, Jo/n = 0.2057, err = 12 -Test : CostFunction: Nonlinear J = 1337 -Test : DRIPCGMinimizer: reduction in residual norm = 0.2537 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Potential vorticity : Min= -7.3088e-04, Max= 6.4855e-04, RMS= 3.3092e-04 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 87.9 -Test : CostJo : Nonlinear Jo(Stream) = 514.7, nobs = 600, Jo/n = 0.8579, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 316.9, nobs = 600, Jo/n = 0.5281, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 43.59, nobs = 300, Jo/n = 0.1453, err = 12 -Test : CostFunction: Nonlinear J = 963.1 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 7820.89, nobs = 600, Jo/n = 13.0348, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1065.61, nobs = 600, Jo/n = 1.77601, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 144.328, nobs = 300, Jo/n = 0.481094, err = 12 +CostFunction: Nonlinear J = 9030.82 +DRIPCGMinimizer: reduction in residual norm = 0.0295408 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Potential vorticity : Min= -7.1570e-04, Max= 6.3930e-04, RMS= 3.3441e-04 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 56.44 +CostJo : Nonlinear Jo(Stream) = 754.3, nobs = 600, Jo/n = 1.257, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 464.9, nobs = 600, Jo/n = 0.7748, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 61.71, nobs = 300, Jo/n = 0.2057, err = 12 +CostFunction: Nonlinear J = 1337 +DRIPCGMinimizer: reduction in residual norm = 0.2537 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Potential vorticity : Min= -7.3088e-04, Max= 6.4855e-04, RMS= 3.3092e-04 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 87.9 +CostJo : Nonlinear Jo(Stream) = 514.7, nobs = 600, Jo/n = 0.8579, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 316.9, nobs = 600, Jo/n = 0.5281, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 43.59, nobs = 300, Jo/n = 0.1453, err = 12 +CostFunction: Nonlinear J = 963.1 diff --git a/qg/test/testoutput/3dvar_hybrid.test b/qg/test/testoutput/3dvar_hybrid.test index 9089a6604..70e193e17 100644 --- a/qg/test/testoutput/3dvar_hybrid.test +++ b/qg/test/testoutput/3dvar_hybrid.test @@ -1,29 +1,29 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 7820.89, nobs = 600, Jo/n = 13.0348, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1065.61, nobs = 600, Jo/n = 1.77601, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 144.328, nobs = 300, Jo/n = 0.481094, err = 12 -Test : CostFunction: Nonlinear J = 9030.82 -Test : DRIPCGMinimizer: reduction in residual norm = 0.121211 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4444e+08, Max= 1.0456e+08, RMS= 1.7727e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 33.64 -Test : CostJo : Nonlinear Jo(Stream) = 480.1, nobs = 600, Jo/n = 0.8002, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 202.7, nobs = 600, Jo/n = 0.3379, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 30.36, nobs = 300, Jo/n = 0.1012, err = 12 -Test : CostFunction: Nonlinear J = 746.8 -Test : DRIPCGMinimizer: reduction in residual norm = 0.5399 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4287e+08, Max= 1.0243e+08, RMS= 1.7719e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 41.85 -Test : CostJo : Nonlinear Jo(Stream) = 425.7, nobs = 600, Jo/n = 0.7095, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 141.4, nobs = 600, Jo/n = 0.2356, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 24.37, nobs = 300, Jo/n = 0.08125, err = 12 -Test : CostFunction: Nonlinear J = 633.3 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 7820.89, nobs = 600, Jo/n = 13.0348, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1065.61, nobs = 600, Jo/n = 1.77601, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 144.328, nobs = 300, Jo/n = 0.481094, err = 12 +CostFunction: Nonlinear J = 9030.82 +DRIPCGMinimizer: reduction in residual norm = 0.121211 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4444e+08, Max= 1.0456e+08, RMS= 1.7727e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 33.64 +CostJo : Nonlinear Jo(Stream) = 480.1, nobs = 600, Jo/n = 0.8002, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 202.7, nobs = 600, Jo/n = 0.3379, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 30.36, nobs = 300, Jo/n = 0.1012, err = 12 +CostFunction: Nonlinear J = 746.8 +DRIPCGMinimizer: reduction in residual norm = 0.5399 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4287e+08, Max= 1.0243e+08, RMS= 1.7719e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 41.85 +CostJo : Nonlinear Jo(Stream) = 425.7, nobs = 600, Jo/n = 0.7095, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 141.4, nobs = 600, Jo/n = 0.2356, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 24.37, nobs = 300, Jo/n = 0.08125, err = 12 +CostFunction: Nonlinear J = 633.3 diff --git a/qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test b/qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test index 05ad37a72..73b92acff 100644 --- a/qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test +++ b/qg/test/testoutput/3dvar_hybrid_wo_jb_evaluation.test @@ -1,26 +1,26 @@ -Test : CostJo : Nonlinear Jo(Stream) = 7820.89, nobs = 600, Jo/n = 13.0348, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1065.61, nobs = 600, Jo/n = 1.77601, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 144.328, nobs = 300, Jo/n = 0.481094, err = 12 -Test : CostFunction: Nonlinear J = 9030.82 -Test : DRIPCGMinimizer: reduction in residual norm = 0.121211 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4444e+08, Max= 1.0456e+08, RMS= 1.7727e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJo : Nonlinear Jo(Stream) = 480.1, nobs = 600, Jo/n = 0.8002, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 202.7, nobs = 600, Jo/n = 0.3379, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 30.36, nobs = 300, Jo/n = 0.1012, err = 12 -Test : CostFunction: Nonlinear J = 713.2 -Test : DRIPCGMinimizer: reduction in residual norm = 0.5399 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4287e+08, Max= 1.0243e+08, RMS= 1.7719e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJo : Nonlinear Jo(Stream) = 425.7, nobs = 600, Jo/n = 0.7095, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 141.4, nobs = 600, Jo/n = 0.2356, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 24.37, nobs = 300, Jo/n = 0.08125, err = 12 -Test : CostFunction: Nonlinear J = 591.4 +CostJo : Nonlinear Jo(Stream) = 7820.89, nobs = 600, Jo/n = 13.0348, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1065.61, nobs = 600, Jo/n = 1.77601, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 144.328, nobs = 300, Jo/n = 0.481094, err = 12 +CostFunction: Nonlinear J = 9030.82 +DRIPCGMinimizer: reduction in residual norm = 0.121211 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4444e+08, Max= 1.0456e+08, RMS= 1.7727e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJo : Nonlinear Jo(Stream) = 480.1, nobs = 600, Jo/n = 0.8002, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 202.7, nobs = 600, Jo/n = 0.3379, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 30.36, nobs = 300, Jo/n = 0.1012, err = 12 +CostFunction: Nonlinear J = 713.2 +DRIPCGMinimizer: reduction in residual norm = 0.5399 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4287e+08, Max= 1.0243e+08, RMS= 1.7719e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJo : Nonlinear Jo(Stream) = 425.7, nobs = 600, Jo/n = 0.7095, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 141.4, nobs = 600, Jo/n = 0.2356, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 24.37, nobs = 300, Jo/n = 0.08125, err = 12 +CostFunction: Nonlinear J = 591.4 diff --git a/qg/test/testoutput/4densvar.test b/qg/test/testoutput/4densvar.test index 4bf452bfb..8ff527d5e 100644 --- a/qg/test/testoutput/4densvar.test +++ b/qg/test/testoutput/4densvar.test @@ -1,89 +1,89 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 1613.47, nobs = 150, Jo/n = 10.7565, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 76.701, nobs = 80, Jo/n = 0.958762, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 44.0485, nobs = 150, Jo/n = 0.293656, err = 12 -Test : CostFunction: Nonlinear J = 1734.22 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.0322775 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.9807e+08, Max= 1.0008e+08, RMS= 1.8725e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T01:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.9492e+08, Max= 1.0017e+08, RMS= 1.8633e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T02:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.9188e+08, Max= 1.0020e+08, RMS= 1.8548e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T03:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8887e+08, Max= 1.0018e+08, RMS= 1.8467e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T04:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8585e+08, Max= 1.0012e+08, RMS= 1.8392e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T05:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8277e+08, Max= 1.0003e+08, RMS= 1.8321e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8127e+08, Max= 9.9929e+07, RMS= 1.8252e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 46.5093 -Test : CostJo : Nonlinear Jo(Stream) = 14.6029, nobs = 150, Jo/n = 0.0973528, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 14.1107, nobs = 80, Jo/n = 0.176384, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 8.04152, nobs = 150, Jo/n = 0.0536101, err = 12 -Test : CostFunction: Nonlinear J = 83.2644 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.235447 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.9882e+08, Max= 1.0111e+08, RMS= 1.8745e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T01:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.9551e+08, Max= 1.0117e+08, RMS= 1.8651e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T02:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.9229e+08, Max= 1.0118e+08, RMS= 1.8562e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T03:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8907e+08, Max= 1.0114e+08, RMS= 1.8479e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T04:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8581e+08, Max= 1.0105e+08, RMS= 1.8401e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T05:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8243e+08, Max= 1.0094e+08, RMS= 1.8328e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8056e+08, Max= 1.0080e+08, RMS= 1.8256e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 50.6449 -Test : CostJo : Nonlinear Jo(Stream) = 10.8546, nobs = 150, Jo/n = 0.0723643, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 7.84779, nobs = 80, Jo/n = 0.0980974, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 5.05768, nobs = 150, Jo/n = 0.0337179, err = 12 -Test : CostFunction: Nonlinear J = 74.405 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 1613.47, nobs = 150, Jo/n = 10.7565, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 76.701, nobs = 80, Jo/n = 0.958762, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 44.0485, nobs = 150, Jo/n = 0.293656, err = 12 +CostFunction: Nonlinear J = 1734.22 +DRPLanczosMinimizer: reduction in residual norm = 0.0322775 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9807e+08, Max= 1.0008e+08, RMS= 1.8725e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T01:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9492e+08, Max= 1.0017e+08, RMS= 1.8633e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T02:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9188e+08, Max= 1.0020e+08, RMS= 1.8548e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T03:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8887e+08, Max= 1.0018e+08, RMS= 1.8467e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T04:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8585e+08, Max= 1.0012e+08, RMS= 1.8392e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T05:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8277e+08, Max= 1.0003e+08, RMS= 1.8321e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8127e+08, Max= 9.9929e+07, RMS= 1.8252e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 46.5093 +CostJo : Nonlinear Jo(Stream) = 14.6029, nobs = 150, Jo/n = 0.0973528, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 14.1107, nobs = 80, Jo/n = 0.176384, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 8.04152, nobs = 150, Jo/n = 0.0536101, err = 12 +CostFunction: Nonlinear J = 83.2644 +DRPLanczosMinimizer: reduction in residual norm = 0.235447 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9882e+08, Max= 1.0111e+08, RMS= 1.8745e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T01:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9551e+08, Max= 1.0117e+08, RMS= 1.8651e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T02:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9229e+08, Max= 1.0118e+08, RMS= 1.8562e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T03:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8907e+08, Max= 1.0114e+08, RMS= 1.8479e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T04:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8581e+08, Max= 1.0105e+08, RMS= 1.8401e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T05:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8243e+08, Max= 1.0094e+08, RMS= 1.8328e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8056e+08, Max= 1.0080e+08, RMS= 1.8256e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 50.6449 +CostJo : Nonlinear Jo(Stream) = 10.8546, nobs = 150, Jo/n = 0.0723643, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 7.84779, nobs = 80, Jo/n = 0.0980974, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 5.05768, nobs = 150, Jo/n = 0.0337179, err = 12 +CostFunction: Nonlinear J = 74.405 diff --git a/qg/test/testoutput/4densvar_hybrid.test b/qg/test/testoutput/4densvar_hybrid.test index 36f04d831..25042aab3 100644 --- a/qg/test/testoutput/4densvar_hybrid.test +++ b/qg/test/testoutput/4densvar_hybrid.test @@ -1,89 +1,89 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 1613.47, nobs = 150, Jo/n = 10.7565, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 76.701, nobs = 80, Jo/n = 0.958762, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 44.0485, nobs = 150, Jo/n = 0.293656, err = 12 -Test : CostFunction: Nonlinear J = 1734.22 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.0171331 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8004e+08, Max= 9.7012e+07, RMS= 1.8293e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T01:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.9089e+08, Max= 1.0599e+08, RMS= 1.8624e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T02:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.9400e+08, Max= 9.9891e+07, RMS= 1.8726e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T03:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.7220e+08, Max= 9.7340e+07, RMS= 1.8096e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T04:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8569e+08, Max= 1.0071e+08, RMS= 1.8495e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T05:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6799e+08, Max= 9.7587e+07, RMS= 1.7900e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.7783e+08, Max= 1.0136e+08, RMS= 1.8282e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 46.203 -Test : CostJo : Nonlinear Jo(Stream) = 3.32558, nobs = 150, Jo/n = 0.0221705, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 6.39078, nobs = 80, Jo/n = 0.0798848, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 7.99854, nobs = 150, Jo/n = 0.0533236, err = 12 -Test : CostFunction: Nonlinear J = 63.9179 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.139746 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8138e+08, Max= 9.6984e+07, RMS= 1.8296e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T01:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.9362e+08, Max= 1.0496e+08, RMS= 1.8642e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T02:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.9436e+08, Max= 9.9889e+07, RMS= 1.8715e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T03:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.7373e+08, Max= 9.8372e+07, RMS= 1.8125e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T04:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8526e+08, Max= 1.0074e+08, RMS= 1.8485e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T05:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6955e+08, Max= 9.8656e+07, RMS= 1.7930e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.7744e+08, Max= 1.0139e+08, RMS= 1.8274e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 48.2511 -Test : CostJo : Nonlinear Jo(Stream) = 2.37944, nobs = 150, Jo/n = 0.0158629, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 4.56144, nobs = 80, Jo/n = 0.0570181, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 5.81766, nobs = 150, Jo/n = 0.0387844, err = 12 -Test : CostFunction: Nonlinear J = 61.0096 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 1613.47, nobs = 150, Jo/n = 10.7565, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 76.701, nobs = 80, Jo/n = 0.958762, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 44.0485, nobs = 150, Jo/n = 0.293656, err = 12 +CostFunction: Nonlinear J = 1734.22 +DRPLanczosMinimizer: reduction in residual norm = 0.0171331 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8004e+08, Max= 9.7012e+07, RMS= 1.8293e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T01:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9089e+08, Max= 1.0599e+08, RMS= 1.8624e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T02:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9400e+08, Max= 9.9891e+07, RMS= 1.8726e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T03:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7220e+08, Max= 9.7340e+07, RMS= 1.8096e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T04:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8569e+08, Max= 1.0071e+08, RMS= 1.8495e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T05:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6799e+08, Max= 9.7587e+07, RMS= 1.7900e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7783e+08, Max= 1.0136e+08, RMS= 1.8282e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 46.203 +CostJo : Nonlinear Jo(Stream) = 3.32558, nobs = 150, Jo/n = 0.0221705, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 6.39078, nobs = 80, Jo/n = 0.0798848, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 7.99854, nobs = 150, Jo/n = 0.0533236, err = 12 +CostFunction: Nonlinear J = 63.9179 +DRPLanczosMinimizer: reduction in residual norm = 0.139746 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8138e+08, Max= 9.6984e+07, RMS= 1.8296e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T01:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9362e+08, Max= 1.0496e+08, RMS= 1.8642e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T02:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9436e+08, Max= 9.9889e+07, RMS= 1.8715e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T03:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7373e+08, Max= 9.8372e+07, RMS= 1.8125e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T04:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8526e+08, Max= 1.0074e+08, RMS= 1.8485e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T05:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6955e+08, Max= 9.8656e+07, RMS= 1.7930e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7744e+08, Max= 1.0139e+08, RMS= 1.8274e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 48.2511 +CostJo : Nonlinear Jo(Stream) = 2.37944, nobs = 150, Jo/n = 0.0158629, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 4.56144, nobs = 80, Jo/n = 0.0570181, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 5.81766, nobs = 150, Jo/n = 0.0387844, err = 12 +CostFunction: Nonlinear J = 61.0096 diff --git a/qg/test/testoutput/4dvar_dripcg.test b/qg/test/testoutput/4dvar_dripcg.test index cfbce7056..d9aeb9c1d 100644 --- a/qg/test/testoutput/4dvar_dripcg.test +++ b/qg/test/testoutput/4dvar_dripcg.test @@ -1,32 +1,32 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 10237.1, nobs = 800, Jo/n = 12.7964, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1372.12, nobs = 800, Jo/n = 1.71515, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 161.904, nobs = 400, Jo/n = 0.404759, err = 12 -Test : CostJcDFI: Nonlinear Jc = 39.9199 -Test : CostFunction: Nonlinear J = 11811 -Test : DRIPCGMinimizer: reduction in residual norm = 0.0903626 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.1952e+08, Max= 1.0284e+08, RMS= 1.8679e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 27.31 -Test : CostJo : Nonlinear Jo(Stream) = 1158, nobs = 800, Jo/n = 1.447, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 272.1, nobs = 800, Jo/n = 0.3402, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 50.83, nobs = 400, Jo/n = 0.1271, err = 12 -Test : CostJcDFI: Nonlinear Jc = 44.87 -Test : CostFunction: Nonlinear J = 1553 -Test : DRIPCGMinimizer: reduction in residual norm = 0.1754 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.1828e+08, Max= 1.0887e+08, RMS= 1.8760e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 56.87 -Test : CostJo : Nonlinear Jo(Stream) = 768.3, nobs = 800, Jo/n = 0.9604, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 233.1, nobs = 800, Jo/n = 0.2914, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 45.67, nobs = 400, Jo/n = 0.1142, err = 12 -Test : CostJcDFI: Nonlinear Jc = 45.22 -Test : CostFunction: Nonlinear J = 1149 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 10237.1, nobs = 800, Jo/n = 12.7964, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1372.12, nobs = 800, Jo/n = 1.71515, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 161.904, nobs = 400, Jo/n = 0.404759, err = 12 +CostJcDFI: Nonlinear Jc = 39.9199 +CostFunction: Nonlinear J = 11811 +DRIPCGMinimizer: reduction in residual norm = 0.0903626 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.1952e+08, Max= 1.0284e+08, RMS= 1.8679e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 27.31 +CostJo : Nonlinear Jo(Stream) = 1158, nobs = 800, Jo/n = 1.447, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 272.1, nobs = 800, Jo/n = 0.3402, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 50.83, nobs = 400, Jo/n = 0.1271, err = 12 +CostJcDFI: Nonlinear Jc = 44.87 +CostFunction: Nonlinear J = 1553 +DRIPCGMinimizer: reduction in residual norm = 0.1754 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.1828e+08, Max= 1.0887e+08, RMS= 1.8760e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 56.87 +CostJo : Nonlinear Jo(Stream) = 768.3, nobs = 800, Jo/n = 0.9604, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 233.1, nobs = 800, Jo/n = 0.2914, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 45.67, nobs = 400, Jo/n = 0.1142, err = 12 +CostJcDFI: Nonlinear Jc = 45.22 +CostFunction: Nonlinear J = 1149 diff --git a/qg/test/testoutput/4dvar_drpcg_lmp.test b/qg/test/testoutput/4dvar_drpcg_lmp.test index 5c3b91a46..53f06a373 100644 --- a/qg/test/testoutput/4dvar_drpcg_lmp.test +++ b/qg/test/testoutput/4dvar_drpcg_lmp.test @@ -1,32 +1,32 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 -Test : CostJcDFI: Nonlinear Jc = 57.9613 -Test : CostFunction: Nonlinear J = 12656.8 -Test : DRPCGMinimizer: reduction in residual norm = 0.0336644 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.2116e+08, Max= 1.0307e+08, RMS= 1.8911e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 28.45 -Test : CostJo : Nonlinear Jo(Stream) = 558.1, nobs = 800, Jo/n = 0.6977, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 217.8, nobs = 800, Jo/n = 0.2722, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 39.41, nobs = 400, Jo/n = 0.09852, err = 12 -Test : CostJcDFI: Nonlinear Jc = 60.36 -Test : CostFunction: Nonlinear J = 904.1 -Test : DRPCGMinimizer: reduction in residual norm = 0.1334 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.2249e+08, Max= 1.0481e+08, RMS= 1.9120e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 47.48 -Test : CostJo : Nonlinear Jo(Stream) = 211.4, nobs = 800, Jo/n = 0.2643, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 113.8, nobs = 800, Jo/n = 0.1423, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 23.93, nobs = 400, Jo/n = 0.05983, err = 12 -Test : CostJcDFI: Nonlinear Jc = 65.59 -Test : CostFunction: Nonlinear J = 462.3 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 +CostJcDFI: Nonlinear Jc = 57.9613 +CostFunction: Nonlinear J = 12656.8 +DRPCGMinimizer: reduction in residual norm = 0.0336644 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.2116e+08, Max= 1.0307e+08, RMS= 1.8911e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 28.45 +CostJo : Nonlinear Jo(Stream) = 558.1, nobs = 800, Jo/n = 0.6977, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 217.8, nobs = 800, Jo/n = 0.2722, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 39.41, nobs = 400, Jo/n = 0.09852, err = 12 +CostJcDFI: Nonlinear Jc = 60.36 +CostFunction: Nonlinear J = 904.1 +DRPCGMinimizer: reduction in residual norm = 0.1334 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.2249e+08, Max= 1.0481e+08, RMS= 1.9120e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 47.48 +CostJo : Nonlinear Jo(Stream) = 211.4, nobs = 800, Jo/n = 0.2643, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 113.8, nobs = 800, Jo/n = 0.1423, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 23.93, nobs = 400, Jo/n = 0.05983, err = 12 +CostJcDFI: Nonlinear Jc = 65.59 +CostFunction: Nonlinear J = 462.3 diff --git a/qg/test/testoutput/4dvar_drpfom.test b/qg/test/testoutput/4dvar_drpfom.test index fa7e4d479..daf0843f8 100644 --- a/qg/test/testoutput/4dvar_drpfom.test +++ b/qg/test/testoutput/4dvar_drpfom.test @@ -1,32 +1,32 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 -Test : CostJcDFI: Nonlinear Jc = 57.9613 -Test : CostFunction: Nonlinear J = 12656.8 -Test : DRPFOMMinimizer: reduction in residual norm = 0.0336644 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.2116e+08, Max= 1.0307e+08, RMS= 1.8911e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 28.45 -Test : CostJo : Nonlinear Jo(Stream) = 558.1, nobs = 800, Jo/n = 0.6977, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 217.8, nobs = 800, Jo/n = 0.2722, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 39.41, nobs = 400, Jo/n = 0.09852, err = 12 -Test : CostJcDFI: Nonlinear Jc = 60.36 -Test : CostFunction: Nonlinear J = 904.1 -Test : DRPFOMMinimizer: reduction in residual norm = 0.1269 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.2053e+08, Max= 1.0439e+08, RMS= 1.9129e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 31.72 -Test : CostJo : Nonlinear Jo(Stream) = 168.6, nobs = 800, Jo/n = 0.2107, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 114.4, nobs = 800, Jo/n = 0.143, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 18.41, nobs = 400, Jo/n = 0.04601, err = 12 -Test : CostJcDFI: Nonlinear Jc = 67.93 -Test : CostFunction: Nonlinear J = 401 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 +CostJcDFI: Nonlinear Jc = 57.9613 +CostFunction: Nonlinear J = 12656.8 +DRPFOMMinimizer: reduction in residual norm = 0.0336644 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.2116e+08, Max= 1.0307e+08, RMS= 1.8911e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 28.45 +CostJo : Nonlinear Jo(Stream) = 558.1, nobs = 800, Jo/n = 0.6977, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 217.8, nobs = 800, Jo/n = 0.2722, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 39.41, nobs = 400, Jo/n = 0.09852, err = 12 +CostJcDFI: Nonlinear Jc = 60.36 +CostFunction: Nonlinear J = 904.1 +DRPFOMMinimizer: reduction in residual norm = 0.1269 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.2053e+08, Max= 1.0439e+08, RMS= 1.9129e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 31.72 +CostJo : Nonlinear Jo(Stream) = 168.6, nobs = 800, Jo/n = 0.2107, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 114.4, nobs = 800, Jo/n = 0.143, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 18.41, nobs = 400, Jo/n = 0.04601, err = 12 +CostJcDFI: Nonlinear Jc = 67.93 +CostFunction: Nonlinear J = 401 diff --git a/qg/test/testoutput/4dvar_drplanczos.test b/qg/test/testoutput/4dvar_drplanczos.test index 460c569ae..4716c7794 100644 --- a/qg/test/testoutput/4dvar_drplanczos.test +++ b/qg/test/testoutput/4dvar_drplanczos.test @@ -1,32 +1,32 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 -Test : CostJcDFI: Nonlinear Jc = 57.9613 -Test : CostFunction: Nonlinear J = 12656.8 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.0336644 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.2116e+08, Max= 1.0307e+08, RMS= 1.8911e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 28.45 -Test : CostJo : Nonlinear Jo(Stream) = 558.1, nobs = 800, Jo/n = 0.6977, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 217.8, nobs = 800, Jo/n = 0.2722, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 39.41, nobs = 400, Jo/n = 0.09852, err = 12 -Test : CostJcDFI: Nonlinear Jc = 60.36 -Test : CostFunction: Nonlinear J = 904.1 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.1265 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.1935e+08, Max= 1.0477e+08, RMS= 1.9089e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 34.18 -Test : CostJo : Nonlinear Jo(Stream) = 153.5, nobs = 800, Jo/n = 0.1919, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 104.6, nobs = 800, Jo/n = 0.1308, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 17.99, nobs = 400, Jo/n = 0.04499, err = 12 -Test : CostJcDFI: Nonlinear Jc = 65.93 -Test : CostFunction: Nonlinear J = 376.2 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 +CostJcDFI: Nonlinear Jc = 57.9613 +CostFunction: Nonlinear J = 12656.8 +DRPLanczosMinimizer: reduction in residual norm = 0.0336644 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.2116e+08, Max= 1.0307e+08, RMS= 1.8911e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 28.45 +CostJo : Nonlinear Jo(Stream) = 558.1, nobs = 800, Jo/n = 0.6977, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 217.8, nobs = 800, Jo/n = 0.2722, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 39.41, nobs = 400, Jo/n = 0.09852, err = 12 +CostJcDFI: Nonlinear Jc = 60.36 +CostFunction: Nonlinear J = 904.1 +DRPLanczosMinimizer: reduction in residual norm = 0.1265 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.1935e+08, Max= 1.0477e+08, RMS= 1.9089e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 34.18 +CostJo : Nonlinear Jo(Stream) = 153.5, nobs = 800, Jo/n = 0.1919, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 104.6, nobs = 800, Jo/n = 0.1308, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 17.99, nobs = 400, Jo/n = 0.04499, err = 12 +CostJcDFI: Nonlinear Jc = 65.93 +CostFunction: Nonlinear J = 376.2 diff --git a/qg/test/testoutput/4dvar_drplanczos_hybrid.test b/qg/test/testoutput/4dvar_drplanczos_hybrid.test index fe0924a1d..b905da9a3 100644 --- a/qg/test/testoutput/4dvar_drplanczos_hybrid.test +++ b/qg/test/testoutput/4dvar_drplanczos_hybrid.test @@ -1,32 +1,32 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 -Test : CostJcDFI: Nonlinear Jc = 57.9613 -Test : CostFunction: Nonlinear J = 12656.8 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.0142818 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.1710e+08, Max= 1.0488e+08, RMS= 1.8894e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 41.74 -Test : CostJo : Nonlinear Jo(Stream) = 542.1, nobs = 800, Jo/n = 0.6776, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 144.3, nobs = 800, Jo/n = 0.1804, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 27.98, nobs = 400, Jo/n = 0.06996, err = 12 -Test : CostJcDFI: Nonlinear Jc = 64.52 -Test : CostFunction: Nonlinear J = 820.7 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.1142 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.2175e+08, Max= 1.0535e+08, RMS= 1.9034e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 39.05 -Test : CostJo : Nonlinear Jo(Stream) = 86.94, nobs = 800, Jo/n = 0.1087, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 62.92, nobs = 800, Jo/n = 0.07865, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 10.27, nobs = 400, Jo/n = 0.02567, err = 12 -Test : CostJcDFI: Nonlinear Jc = 67.21 -Test : CostFunction: Nonlinear J = 266.4 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 +CostJcDFI: Nonlinear Jc = 57.9613 +CostFunction: Nonlinear J = 12656.8 +DRPLanczosMinimizer: reduction in residual norm = 0.0142818 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.1710e+08, Max= 1.0488e+08, RMS= 1.8894e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 41.74 +CostJo : Nonlinear Jo(Stream) = 542.1, nobs = 800, Jo/n = 0.6776, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 144.3, nobs = 800, Jo/n = 0.1804, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 27.98, nobs = 400, Jo/n = 0.06996, err = 12 +CostJcDFI: Nonlinear Jc = 64.52 +CostFunction: Nonlinear J = 820.7 +DRPLanczosMinimizer: reduction in residual norm = 0.1142 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.2175e+08, Max= 1.0535e+08, RMS= 1.9034e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 39.05 +CostJo : Nonlinear Jo(Stream) = 86.94, nobs = 800, Jo/n = 0.1087, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 62.92, nobs = 800, Jo/n = 0.07865, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 10.27, nobs = 400, Jo/n = 0.02567, err = 12 +CostJcDFI: Nonlinear Jc = 67.21 +CostFunction: Nonlinear J = 266.4 diff --git a/qg/test/testoutput/4dvar_forcing.test b/qg/test/testoutput/4dvar_forcing.test deleted file mode 100644 index e69de29bb..000000000 diff --git a/qg/test/testoutput/4dvar_ipcg.test b/qg/test/testoutput/4dvar_ipcg.test index 2cd35680d..d23c32226 100644 --- a/qg/test/testoutput/4dvar_ipcg.test +++ b/qg/test/testoutput/4dvar_ipcg.test @@ -1,32 +1,32 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 -Test : CostJcDFI: Nonlinear Jc = 57.9613 -Test : CostFunction: Nonlinear J = 12656.8 -Test : IPCGMinimizer: reduction in residual norm = 0.0328 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.1660e+08, Max= 1.0367e+08, RMS= 1.8975e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 68.07 -Test : CostJo : Nonlinear Jo(Stream) = 393.1, nobs = 800, Jo/n = 0.4913, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 134.1, nobs = 800, Jo/n = 0.1676, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 28.7, nobs = 400, Jo/n = 0.07175, err = 12 -Test : CostJcDFI: Nonlinear Jc = 58.23 -Test : CostFunction: Nonlinear J = 682.1 -Test : IPCGMinimizer: reduction in residual norm = 0.2145 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.2297e+08, Max= 1.0515e+08, RMS= 1.9148e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 63.98 -Test : CostJo : Nonlinear Jo(Stream) = 121.9, nobs = 800, Jo/n = 0.1524, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 84.01, nobs = 800, Jo/n = 0.105, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 17.65, nobs = 400, Jo/n = 0.04411, err = 12 -Test : CostJcDFI: Nonlinear Jc = 67.09 -Test : CostFunction: Nonlinear J = 354.6 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 +CostJcDFI: Nonlinear Jc = 57.9613 +CostFunction: Nonlinear J = 12656.8 +IPCGMinimizer: reduction in residual norm = 0.0328 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.1660e+08, Max= 1.0367e+08, RMS= 1.8975e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 68.07 +CostJo : Nonlinear Jo(Stream) = 393.1, nobs = 800, Jo/n = 0.4913, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 134.1, nobs = 800, Jo/n = 0.1676, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 28.7, nobs = 400, Jo/n = 0.07175, err = 12 +CostJcDFI: Nonlinear Jc = 58.23 +CostFunction: Nonlinear J = 682.1 +IPCGMinimizer: reduction in residual norm = 0.2145 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.2297e+08, Max= 1.0515e+08, RMS= 1.9148e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 63.98 +CostJo : Nonlinear Jo(Stream) = 121.9, nobs = 800, Jo/n = 0.1524, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 84.01, nobs = 800, Jo/n = 0.105, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 17.65, nobs = 400, Jo/n = 0.04411, err = 12 +CostJcDFI: Nonlinear Jc = 67.09 +CostFunction: Nonlinear J = 354.6 diff --git a/qg/test/testoutput/4dvar_obs_biased.test b/qg/test/testoutput/4dvar_obs_biased.test index 05cbb2262..efff71db8 100644 --- a/qg/test/testoutput/4dvar_obs_biased.test +++ b/qg/test/testoutput/4dvar_obs_biased.test @@ -1,36 +1,36 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 17715.2, nobs = 800, Jo/n = 22.144, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1784.27, nobs = 800, Jo/n = 2.23034, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 -Test : CostJcDFI: Nonlinear Jc = 57.9613 -Test : CostFunction: Nonlinear J = 19737.2 -Test : DRPCGMinimizer: reduction in residual norm = 0.00387793 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.1520e+08, Max= 1.0618e+08, RMS= 1.8848e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : ObsBias = -1.16824e+07, 0, 0, 0 -Test : ObsBias = 0, 9.67042, 0, 0 -Test : CostJb : Nonlinear Jb = 60.42 -Test : CostJo : Nonlinear Jo(Stream) = 366, nobs = 800, Jo/n = 0.4575, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 147.6, nobs = 800, Jo/n = 0.1846, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 33.38, nobs = 400, Jo/n = 0.08346, err = 12 -Test : CostJcDFI: Nonlinear Jc = 59.25 -Test : CostFunction: Nonlinear J = 666.7 -Test : DRPCGMinimizer: reduction in residual norm = 0.1576 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.2306e+08, Max= 1.0568e+08, RMS= 1.9156e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : ObsBias = -9.70392e+06, 0, 0, 0 -Test : ObsBias = 0, 9.83281, 0, 0 -Test : CostJb : Nonlinear Jb = 57.71 -Test : CostJo : Nonlinear Jo(Stream) = 131.3, nobs = 800, Jo/n = 0.1641, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 84.24, nobs = 800, Jo/n = 0.1053, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 17.63, nobs = 400, Jo/n = 0.04408, err = 12 -Test : CostJcDFI: Nonlinear Jc = 67.67 -Test : CostFunction: Nonlinear J = 358.5 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 17715.2, nobs = 800, Jo/n = 22.144, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1784.27, nobs = 800, Jo/n = 2.23034, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 +CostJcDFI: Nonlinear Jc = 57.9613 +CostFunction: Nonlinear J = 19737.2 +DRPCGMinimizer: reduction in residual norm = 0.00387793 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.1520e+08, Max= 1.0618e+08, RMS= 1.8848e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +ObsBias = -1.16824e+07, 0, 0, 0 +ObsBias = 0, 9.67042, 0, 0 +CostJb : Nonlinear Jb = 60.42 +CostJo : Nonlinear Jo(Stream) = 366, nobs = 800, Jo/n = 0.4575, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 147.6, nobs = 800, Jo/n = 0.1846, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 33.38, nobs = 400, Jo/n = 0.08346, err = 12 +CostJcDFI: Nonlinear Jc = 59.25 +CostFunction: Nonlinear J = 666.7 +DRPCGMinimizer: reduction in residual norm = 0.1576 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.2306e+08, Max= 1.0568e+08, RMS= 1.9156e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +ObsBias = -9.70392e+06, 0, 0, 0 +ObsBias = 0, 9.83281, 0, 0 +CostJb : Nonlinear Jb = 57.71 +CostJo : Nonlinear Jo(Stream) = 131.3, nobs = 800, Jo/n = 0.1641, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 84.24, nobs = 800, Jo/n = 0.1053, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 17.63, nobs = 400, Jo/n = 0.04408, err = 12 +CostJcDFI: Nonlinear Jc = 67.67 +CostFunction: Nonlinear J = 358.5 diff --git a/qg/test/testoutput/4dvar_rpcg.test b/qg/test/testoutput/4dvar_rpcg.test index cdce0a44a..78a190354 100644 --- a/qg/test/testoutput/4dvar_rpcg.test +++ b/qg/test/testoutput/4dvar_rpcg.test @@ -1,29 +1,29 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 -Test : CostFunction: Nonlinear J = 12598.8 -Test : RPCGMinimizer: reduction in residual norm = 0.0339189 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.2145e+08, Max= 1.0309e+08, RMS= 1.8913e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 28.47 -Test : CostJo : Nonlinear Jo(Stream) = 562.2, nobs = 800, Jo/n = 0.7027, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 218.5, nobs = 800, Jo/n = 0.2731, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 39.61, nobs = 400, Jo/n = 0.09902, err = 12 -Test : CostFunction: Nonlinear J = 848.7 -Test : RPCGMinimizer: reduction in residual norm = 0.1177 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.2031e+08, Max= 1.0434e+08, RMS= 1.9125e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 31.66 -Test : CostJo : Nonlinear Jo(Stream) = 168.2, nobs = 800, Jo/n = 0.2102, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 113.8, nobs = 800, Jo/n = 0.1422, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 18.42, nobs = 400, Jo/n = 0.04605, err = 12 -Test : CostFunction: Nonlinear J = 332 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 +CostFunction: Nonlinear J = 12598.8 +RPCGMinimizer: reduction in residual norm = 0.0339189 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.2145e+08, Max= 1.0309e+08, RMS= 1.8913e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 28.47 +CostJo : Nonlinear Jo(Stream) = 562.2, nobs = 800, Jo/n = 0.7027, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 218.5, nobs = 800, Jo/n = 0.2731, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 39.61, nobs = 400, Jo/n = 0.09902, err = 12 +CostFunction: Nonlinear J = 848.7 +RPCGMinimizer: reduction in residual norm = 0.1177 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.2031e+08, Max= 1.0434e+08, RMS= 1.9125e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 31.66 +CostJo : Nonlinear Jo(Stream) = 168.2, nobs = 800, Jo/n = 0.2102, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 113.8, nobs = 800, Jo/n = 0.1422, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 18.42, nobs = 400, Jo/n = 0.04605, err = 12 +CostFunction: Nonlinear J = 332 diff --git a/qg/test/testoutput/4dvar_saddlepoint.test b/qg/test/testoutput/4dvar_saddlepoint.test index ffde6af3d..ef64b6f98 100644 --- a/qg/test/testoutput/4dvar_saddlepoint.test +++ b/qg/test/testoutput/4dvar_saddlepoint.test @@ -1,45 +1,45 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 -Test : CostJcDFI: Nonlinear Jc = 16.3749 -Test : CostJcDFI: Nonlinear Jc = 13.0699 -Test : CostFunction: Nonlinear J = 12628.2 -Test : SaddlePointMinimizer: reduction in residual norm = 2.295e-13 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6781e+08, Max= 9.6458e+07, RMS= 1.7904e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.3337e+08, Max= 9.3777e+07, RMS= 1.6813e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 -Test : CostJcDFI: Nonlinear Jc = 16.3749 -Test : CostJcDFI: Nonlinear Jc = 13.0699 -Test : CostFunction: Nonlinear J = 12628.2 -Test : SaddlePointMinimizer: reduction in residual norm = 2.295e-13 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6781e+08, Max= 9.6458e+07, RMS= 1.7904e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.3337e+08, Max= 9.3777e+07, RMS= 1.6813e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 -Test : CostJcDFI: Nonlinear Jc = 16.3749 -Test : CostJcDFI: Nonlinear Jc = 13.0699 -Test : CostFunction: Nonlinear J = 12628.2 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 +CostJcDFI: Nonlinear Jc = 16.3749 +CostJcDFI: Nonlinear Jc = 13.0699 +CostFunction: Nonlinear J = 12628.2 +SaddlePointMinimizer: reduction in residual norm = 2.295e-13 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6781e+08, Max= 9.6458e+07, RMS= 1.7904e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.3337e+08, Max= 9.3777e+07, RMS= 1.6813e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 +CostJcDFI: Nonlinear Jc = 16.3749 +CostJcDFI: Nonlinear Jc = 13.0699 +CostFunction: Nonlinear J = 12628.2 +SaddlePointMinimizer: reduction in residual norm = 2.295e-13 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6781e+08, Max= 9.6458e+07, RMS= 1.7904e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.3337e+08, Max= 9.3777e+07, RMS= 1.6813e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 +CostJcDFI: Nonlinear Jc = 16.3749 +CostJcDFI: Nonlinear Jc = 13.0699 +CostFunction: Nonlinear J = 12628.2 diff --git a/qg/test/testoutput/addincrement.test b/qg/test/testoutput/addincrement.test index 0d47ef8ef..682399afd 100644 --- a/qg/test/testoutput/addincrement.test +++ b/qg/test/testoutput/addincrement.test @@ -1,16 +1,16 @@ -Test : State: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 -Test : State plus increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6263e+08, Max= 1.0660e+08, RMS= 1.7472e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +State: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 +State plus increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6263e+08, Max= 1.0660e+08, RMS= 1.7472e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/addincrement_scaled.test b/qg/test/testoutput/addincrement_scaled.test index c128933ee..d880052bf 100644 --- a/qg/test/testoutput/addincrement_scaled.test +++ b/qg/test/testoutput/addincrement_scaled.test @@ -1,20 +1,20 @@ -Test : State: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 -Test : Scaled the increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -9.2140e+06, Max= 2.8742e+07, RMS= 4.2153e+06 -Test : State plus increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.5411e+08, Max= 1.0285e+08, RMS= 1.7552e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +State: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 +Scaled the increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -9.2140e+06, Max= 2.8742e+07, RMS= 4.2153e+06 +State plus increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.5411e+08, Max= 1.0285e+08, RMS= 1.7552e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/analytic_forecast.test b/qg/test/testoutput/analytic_forecast.test index 533e434bb..309bd2e2c 100644 --- a/qg/test/testoutput/analytic_forecast.test +++ b/qg/test/testoutput/analytic_forecast.test @@ -1,12 +1,12 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.1289e+08, Max= 4.1289e+08, RMS= 1.5614e+08 -Test : Streamfunction LBC : Min= -3.6309e+01, Max= 0.0000e+00, RMS= 1.8839e+01 -Test : Potential vorticity LBC: Min= -7.3653e-05, Max= 8.0801e-05, RMS= 7.5159e-05 -Test : Final state: -Test : Valid time: 2010-01-05T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.2065e+08, Max= 4.2065e+08, RMS= 1.8324e+08 -Test : Streamfunction LBC : Min= -3.6309e+01, Max= 0.0000e+00, RMS= 1.8839e+01 -Test : Potential vorticity LBC: Min= -7.3653e-05, Max= 8.0801e-05, RMS= 7.5159e-05 +Initial state: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.1289e+08, Max= 4.1289e+08, RMS= 1.5614e+08 + Streamfunction LBC : Min= -3.6309e+01, Max= 0.0000e+00, RMS= 1.8839e+01 + Potential vorticity LBC: Min= -7.3653e-05, Max= 8.0801e-05, RMS= 7.5159e-05 +Final state: + Valid time: 2010-01-05T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.2065e+08, Max= 4.2065e+08, RMS= 1.8324e+08 + Streamfunction LBC : Min= -3.6309e+01, Max= 0.0000e+00, RMS= 1.8839e+01 + Potential vorticity LBC: Min= -7.3653e-05, Max= 8.0801e-05, RMS= 7.5159e-05 diff --git a/qg/test/testoutput/convertincrement.test b/qg/test/testoutput/convertincrement.test index 3ab67edc1..131c580f4 100644 --- a/qg/test/testoutput/convertincrement.test +++ b/qg/test/testoutput/convertincrement.test @@ -1,30 +1,30 @@ -Test : Input increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 -Test : Trajectory state: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Increment after variable transform: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 20, 10, 2 -Test : Potential vorticity : Min= -1.7247e-04, Max= 5.3114e-05, RMS= 2.0353e-05 -Test : Increment after variable transform: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 20, 10, 2 -Test : Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 -Test : Increment after variable transform: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 20, 10, 2 -Test : Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 -Test : Increment after variable transform: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 20, 10, 2 -Test : Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 -Test : Output increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 20, 10, 2 -Test : Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 +Input increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 +Trajectory state: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Increment after variable transform: + Valid time: 2010-01-01T12:00:00Z + Resolution = 20, 10, 2 + Potential vorticity : Min= -1.7247e-04, Max= 5.3114e-05, RMS= 2.0353e-05 +Increment after variable transform: + Valid time: 2010-01-01T12:00:00Z + Resolution = 20, 10, 2 + Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 +Increment after variable transform: + Valid time: 2010-01-01T12:00:00Z + Resolution = 20, 10, 2 + Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 +Increment after variable transform: + Valid time: 2010-01-01T12:00:00Z + Resolution = 20, 10, 2 + Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 +Output increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 20, 10, 2 + Streamfunction : Min= -1.4944e+07, Max= 5.5077e+07, RMS= 7.4251e+06 diff --git a/qg/test/testoutput/convertstate.test b/qg/test/testoutput/convertstate.test index 614438b95..50643423d 100644 --- a/qg/test/testoutput/convertstate.test +++ b/qg/test/testoutput/convertstate.test @@ -1,132 +1,144 @@ -Test : Input state: -Test : Valid time: 2010-01-01T09:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4304e+08, Max= 9.3153e+07, RMS= 1.7024e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Variable transform: QG change of variable -Test : Variable change from: 1 variables: x -Test : Variable change to: 1 variables: q -Test : State after variable transform: -Test : Valid time: 2010-01-01T09:00:00Z -Test : Resolution = 20, 10, 2 -Test : Potential vorticity : Min= -5.8027e-04, Max= 5.4508e-04, RMS= 3.2514e-04 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Variable transform: QG change of variable -Test : Variable change from: 1 variables: q -Test : Variable change to: 1 variables: x -Test : State after variable transform: -Test : Valid time: 2010-01-01T09:00:00Z -Test : Resolution = 20, 10, 2 -Test : Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Variable transform: QG change of variable -Test : Variable change from: 1 variables: q -Test : Variable change to: 1 variables: x -Test : State after variable transform: -Test : Valid time: 2010-01-01T09:00:00Z -Test : Resolution = 20, 10, 2 -Test : Potential vorticity : Min= -5.8027e-04, Max= 5.4508e-04, RMS= 3.2514e-04 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Variable transform: QG change of variable -Test : Variable change from: 1 variables: x -Test : Variable change to: 1 variables: q -Test : State after variable transform: -Test : Valid time: 2010-01-01T09:00:00Z -Test : Resolution = 20, 10, 2 -Test : Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Variable transform: IdVariableChange -Test : Variable change from: 1 variables: x -Test : Variable change to: 1 variables: x -Test : State after variable transform: -Test : Valid time: 2010-01-01T09:00:00Z -Test : Resolution = 20, 10, 2 -Test : Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Variable transform: IdVariableChange -Test : Variable change from: 1 variables: x -Test : Variable change to: 1 variables: x -Test : State after variable transform: -Test : Valid time: 2010-01-01T09:00:00Z -Test : Resolution = 20, 10, 2 -Test : Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Output state: -Test : Valid time: 2010-01-01T09:00:00Z -Test : Resolution = 20, 10, 2 -Test : Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Input state: -Test : Valid time: 2010-01-01T10:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.3926e+08, Max= 9.3338e+07, RMS= 1.6950e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Variable transform: QG change of variable -Test : Variable change from: 1 variables: x -Test : Variable change to: 1 variables: q -Test : State after variable transform: -Test : Valid time: 2010-01-01T10:00:00Z -Test : Resolution = 20, 10, 2 -Test : Potential vorticity : Min= -5.7613e-04, Max= 5.4113e-04, RMS= 3.2534e-04 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Variable transform: QG change of variable -Test : Variable change from: 1 variables: q -Test : Variable change to: 1 variables: x -Test : State after variable transform: -Test : Valid time: 2010-01-01T10:00:00Z -Test : Resolution = 20, 10, 2 -Test : Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Variable transform: QG change of variable -Test : Variable change from: 1 variables: q -Test : Variable change to: 1 variables: x -Test : State after variable transform: -Test : Valid time: 2010-01-01T10:00:00Z -Test : Resolution = 20, 10, 2 -Test : Potential vorticity : Min= -5.7613e-04, Max= 5.4113e-04, RMS= 3.2534e-04 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Variable transform: QG change of variable -Test : Variable change from: 1 variables: x -Test : Variable change to: 1 variables: q -Test : State after variable transform: -Test : Valid time: 2010-01-01T10:00:00Z -Test : Resolution = 20, 10, 2 -Test : Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Variable transform: IdVariableChange -Test : Variable change from: 1 variables: x -Test : Variable change to: 1 variables: x -Test : State after variable transform: -Test : Valid time: 2010-01-01T10:00:00Z -Test : Resolution = 20, 10, 2 -Test : Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Variable transform: IdVariableChange -Test : Variable change from: 1 variables: x -Test : Variable change to: 1 variables: x -Test : State after variable transform: -Test : Valid time: 2010-01-01T10:00:00Z -Test : Resolution = 20, 10, 2 -Test : Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Output state: -Test : Valid time: 2010-01-01T10:00:00Z -Test : Resolution = 20, 10, 2 -Test : Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Input state: + Valid time: 2010-01-01T09:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4304e+08, Max= 9.3153e+07, RMS= 1.7024e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Variable transform: QG change of variable +Variable change from: 1 variables: x +Variable change to: 1 variables: q + +State after variable transform: + Valid time: 2010-01-01T09:00:00Z + Resolution = 20, 10, 2 + Potential vorticity : Min= -5.8027e-04, Max= 5.4508e-04, RMS= 3.2514e-04 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Variable transform: QG change of variable +Variable change from: 1 variables: q +Variable change to: 1 variables: x + +State after variable transform: + Valid time: 2010-01-01T09:00:00Z + Resolution = 20, 10, 2 + Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Variable transform: QG change of variable +Variable change from: 1 variables: q +Variable change to: 1 variables: x + +State after variable transform: + Valid time: 2010-01-01T09:00:00Z + Resolution = 20, 10, 2 + Potential vorticity : Min= -5.8027e-04, Max= 5.4508e-04, RMS= 3.2514e-04 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Variable transform: QG change of variable +Variable change from: 1 variables: x +Variable change to: 1 variables: q + +State after variable transform: + Valid time: 2010-01-01T09:00:00Z + Resolution = 20, 10, 2 + Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Variable transform: IdVariableChange +Variable change from: 1 variables: x +Variable change to: 1 variables: x + +State after variable transform: + Valid time: 2010-01-01T09:00:00Z + Resolution = 20, 10, 2 + Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Variable transform: IdVariableChange +Variable change from: 1 variables: x +Variable change to: 1 variables: x + +State after variable transform: + Valid time: 2010-01-01T09:00:00Z + Resolution = 20, 10, 2 + Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Output state: + Valid time: 2010-01-01T09:00:00Z + Resolution = 20, 10, 2 + Streamfunction : Min= -4.3180e+08, Max= 7.9749e+07, RMS= 1.6804e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Input state: + Valid time: 2010-01-01T10:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.3926e+08, Max= 9.3338e+07, RMS= 1.6950e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Variable transform: QG change of variable +Variable change from: 1 variables: x +Variable change to: 1 variables: q + +State after variable transform: + Valid time: 2010-01-01T10:00:00Z + Resolution = 20, 10, 2 + Potential vorticity : Min= -5.7613e-04, Max= 5.4113e-04, RMS= 3.2534e-04 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Variable transform: QG change of variable +Variable change from: 1 variables: q +Variable change to: 1 variables: x + +State after variable transform: + Valid time: 2010-01-01T10:00:00Z + Resolution = 20, 10, 2 + Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Variable transform: QG change of variable +Variable change from: 1 variables: q +Variable change to: 1 variables: x + +State after variable transform: + Valid time: 2010-01-01T10:00:00Z + Resolution = 20, 10, 2 + Potential vorticity : Min= -5.7613e-04, Max= 5.4113e-04, RMS= 3.2534e-04 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Variable transform: QG change of variable +Variable change from: 1 variables: x +Variable change to: 1 variables: q + +State after variable transform: + Valid time: 2010-01-01T10:00:00Z + Resolution = 20, 10, 2 + Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Variable transform: IdVariableChange +Variable change from: 1 variables: x +Variable change to: 1 variables: x + +State after variable transform: + Valid time: 2010-01-01T10:00:00Z + Resolution = 20, 10, 2 + Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Variable transform: IdVariableChange +Variable change from: 1 variables: x +Variable change to: 1 variables: x + +State after variable transform: + Valid time: 2010-01-01T10:00:00Z + Resolution = 20, 10, 2 + Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Output state: + Valid time: 2010-01-01T10:00:00Z + Resolution = 20, 10, 2 + Streamfunction : Min= -4.2766e+08, Max= 8.0020e+07, RMS= 1.6726e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7263e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/dfi.test b/qg/test/testoutput/dfi.test index bed6e6c7e..8320865b4 100644 --- a/qg/test/testoutput/dfi.test +++ b/qg/test/testoutput/dfi.test @@ -1,18 +1,18 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6781e+08, Max= 9.6458e+07, RMS= 1.7904e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Filtered state: -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.5184e+08, Max= 9.3196e+07, RMS= 1.7271e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Final state: -Test : Valid time: 2010-01-08T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -3.9403e+08, Max= 1.7285e+08, RMS= 1.5325e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Initial state: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6781e+08, Max= 9.6458e+07, RMS= 1.7904e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Filtered state: + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.5184e+08, Max= 9.3196e+07, RMS= 1.7271e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Final state: + Valid time: 2010-01-08T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -3.9403e+08, Max= 1.7285e+08, RMS= 1.5325e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/diffstates.test b/qg/test/testoutput/diffstates.test index 9c540be7b..6d813aba0 100644 --- a/qg/test/testoutput/diffstates.test +++ b/qg/test/testoutput/diffstates.test @@ -1,16 +1,16 @@ -Test : Input state 1: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Input state 2: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.5934e+08, Max= 1.0390e+08, RMS= 1.7850e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Output increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 +Input state 1: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4593e+08, Max= 1.0320e+08, RMS= 1.7642e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Input state 2: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.5934e+08, Max= 1.0390e+08, RMS= 1.7850e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Output increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.8428e+07, Max= 5.7484e+07, RMS= 8.4307e+06 diff --git a/qg/test/testoutput/dirac_cov.test b/qg/test/testoutput/dirac_cov.test index abf99e3cc..076260229 100644 --- a/qg/test/testoutput/dirac_cov.test +++ b/qg/test/testoutput/dirac_cov.test @@ -1,12 +1,12 @@ -Test : Input Dirac increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 -Test : B * Increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -7.2330e+05, Max= 3.2400e+14, RMS= 6.2881e+13 -Test : Randomized variance: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 2.8165e+14, Max= 3.6315e+14, RMS= 3.2388e+14 +Input Dirac increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 +B * Increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -7.2330e+05, Max= 3.2400e+14, RMS= 6.2881e+13 +Randomized variance: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 2.8165e+14, Max= 3.6315e+14, RMS= 3.2388e+14 diff --git a/qg/test/testoutput/dirac_hyb_field.test b/qg/test/testoutput/dirac_hyb_field.test index c426ce866..b21a045a8 100644 --- a/qg/test/testoutput/dirac_hyb_field.test +++ b/qg/test/testoutput/dirac_hyb_field.test @@ -1,12 +1,12 @@ -Test : Input Dirac increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 -Test : B * Increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.3348e+13, Max= 3.1207e+14, RMS= 5.8758e+13 -Test : Localized Increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 2.1340e-06, Max= 1.0000e+00, RMS= 3.4333e-01 +Input Dirac increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 +B * Increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.3348e+13, Max= 3.1207e+14, RMS= 5.8758e+13 +Localized Increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 2.1340e-06, Max= 1.0000e+00, RMS= 3.4333e-01 diff --git a/qg/test/testoutput/dirac_hyb_value.test b/qg/test/testoutput/dirac_hyb_value.test index 031aae710..7b6d36db9 100644 --- a/qg/test/testoutput/dirac_hyb_value.test +++ b/qg/test/testoutput/dirac_hyb_value.test @@ -1,16 +1,16 @@ -Test : Input Dirac increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 -Test : B * Increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.3348e+13, Max= 3.1207e+14, RMS= 5.8758e+13 -Test : Localized Increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 2.1340e-06, Max= 1.0000e+00, RMS= 3.4333e-01 -Test : Randomized variance: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 1.0521e+14, Max= 9.2106e+14, RMS= 3.1869e+14 +Input Dirac increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 +B * Increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.3348e+13, Max= 3.1207e+14, RMS= 5.8758e+13 +Localized Increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 2.1340e-06, Max= 1.0000e+00, RMS= 3.4333e-01 +Randomized variance: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 1.0521e+14, Max= 9.2106e+14, RMS= 3.1869e+14 diff --git a/qg/test/testoutput/dirac_loc_3d.test b/qg/test/testoutput/dirac_loc_3d.test index 8be9a04af..402d84e69 100644 --- a/qg/test/testoutput/dirac_loc_3d.test +++ b/qg/test/testoutput/dirac_loc_3d.test @@ -1,16 +1,16 @@ -Test : Input Dirac increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 -Test : B * Increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -6.5158e+13, Max= 3.7255e+14, RMS= 5.8877e+13 -Test : Localized Increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 2.1340e-06, Max= 1.0000e+00, RMS= 3.4333e-01 -Test : Randomized variance: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 1.3256e+12, Max= 1.4252e+15, RMS= 3.6277e+14 +Input Dirac increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 +B * Increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -6.5158e+13, Max= 3.7255e+14, RMS= 5.8877e+13 +Localized Increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 2.1340e-06, Max= 1.0000e+00, RMS= 3.4333e-01 +Randomized variance: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 1.3256e+12, Max= 1.4252e+15, RMS= 3.6277e+14 diff --git a/qg/test/testoutput/dirac_loc_4d.test b/qg/test/testoutput/dirac_loc_4d.test index 2d7666a3c..b8edd3e34 100644 --- a/qg/test/testoutput/dirac_loc_4d.test +++ b/qg/test/testoutput/dirac_loc_4d.test @@ -1,160 +1,160 @@ -Test : Input Dirac increment: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Valid time: 2010-01-01T01:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Valid time: 2010-01-01T02:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Valid time: 2010-01-01T03:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Valid time: 2010-01-01T04:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Valid time: 2010-01-01T05:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 -Test : Valid time: 2010-01-01T07:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Valid time: 2010-01-01T08:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Valid time: 2010-01-01T09:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Valid time: 2010-01-01T10:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Valid time: 2010-01-01T11:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : B * Increment: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.0432e+13, Max= 1.9692e+14, RMS= 2.2098e+13 -Test : Valid time: 2010-01-01T01:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.0552e+13, Max= 2.0894e+14, RMS= 2.2747e+13 -Test : Valid time: 2010-01-01T02:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -3.9769e+13, Max= 2.1965e+14, RMS= 2.3431e+13 -Test : Valid time: 2010-01-01T03:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -3.8287e+13, Max= 2.2877e+14, RMS= 2.4126e+13 -Test : Valid time: 2010-01-01T04:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -3.6244e+13, Max= 2.3634e+14, RMS= 2.4826e+13 -Test : Valid time: 2010-01-01T05:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -3.3760e+13, Max= 2.4250e+14, RMS= 2.5516e+13 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -3.0971e+13, Max= 2.4747e+14, RMS= 2.6180e+13 -Test : Valid time: 2010-01-01T07:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -2.8061e+13, Max= 2.5136e+14, RMS= 2.6788e+13 -Test : Valid time: 2010-01-01T08:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -2.5144e+13, Max= 2.5423e+14, RMS= 2.7311e+13 -Test : Valid time: 2010-01-01T09:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -2.3979e+13, Max= 2.5615e+14, RMS= 2.7721e+13 -Test : Valid time: 2010-01-01T10:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -2.3448e+13, Max= 2.5735e+14, RMS= 2.8021e+13 -Test : Valid time: 2010-01-01T11:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -2.2850e+13, Max= 2.5807e+14, RMS= 2.8223e+13 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -2.2238e+13, Max= 2.5841e+14, RMS= 2.8326e+13 -Test : Localized Increment: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 -Test : Valid time: 2010-01-01T01:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 -Test : Valid time: 2010-01-01T02:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 -Test : Valid time: 2010-01-01T03:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 -Test : Valid time: 2010-01-01T04:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 -Test : Valid time: 2010-01-01T05:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 -Test : Valid time: 2010-01-01T07:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 -Test : Valid time: 2010-01-01T08:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 -Test : Valid time: 2010-01-01T09:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 -Test : Valid time: 2010-01-01T10:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 -Test : Valid time: 2010-01-01T11:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 -Test : Randomized variance: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 3.7303e+12, Max= 1.0538e+15, RMS= 2.4663e+14 -Test : Valid time: 2010-01-01T01:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 3.9097e+12, Max= 1.1299e+15, RMS= 2.5694e+14 -Test : Valid time: 2010-01-01T02:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 2.4068e+12, Max= 1.2020e+15, RMS= 2.6754e+14 -Test : Valid time: 2010-01-01T03:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 1.8149e+12, Max= 1.2679e+15, RMS= 2.7848e+14 -Test : Valid time: 2010-01-01T04:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 1.9945e+12, Max= 1.3266e+15, RMS= 2.8969e+14 -Test : Valid time: 2010-01-01T05:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 2.4492e+12, Max= 1.3767e+15, RMS= 3.0102e+14 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 2.3155e+12, Max= 1.4163e+15, RMS= 3.1227e+14 -Test : Valid time: 2010-01-01T07:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 2.2409e+12, Max= 1.4568e+15, RMS= 3.2326e+14 -Test : Valid time: 2010-01-01T08:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 2.0067e+12, Max= 1.5122e+15, RMS= 3.3397e+14 -Test : Valid time: 2010-01-01T09:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 1.7114e+12, Max= 1.5718e+15, RMS= 3.4435e+14 -Test : Valid time: 2010-01-01T10:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 1.4554e+12, Max= 1.6333e+15, RMS= 3.5419e+14 -Test : Valid time: 2010-01-01T11:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 1.2259e+12, Max= 1.6944e+15, RMS= 3.6352e+14 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 1.0259e+12, Max= 1.7698e+15, RMS= 3.7255e+14 +Input Dirac increment: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Valid time: 2010-01-01T01:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Valid time: 2010-01-01T02:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Valid time: 2010-01-01T03:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Valid time: 2010-01-01T04:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Valid time: 2010-01-01T05:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 + Valid time: 2010-01-01T07:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Valid time: 2010-01-01T08:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Valid time: 2010-01-01T09:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Valid time: 2010-01-01T10:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Valid time: 2010-01-01T11:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 +B * Increment: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.0432e+13, Max= 1.9692e+14, RMS= 2.2098e+13 + Valid time: 2010-01-01T01:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.0552e+13, Max= 2.0894e+14, RMS= 2.2747e+13 + Valid time: 2010-01-01T02:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -3.9769e+13, Max= 2.1965e+14, RMS= 2.3431e+13 + Valid time: 2010-01-01T03:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -3.8287e+13, Max= 2.2877e+14, RMS= 2.4126e+13 + Valid time: 2010-01-01T04:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -3.6244e+13, Max= 2.3634e+14, RMS= 2.4826e+13 + Valid time: 2010-01-01T05:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -3.3760e+13, Max= 2.4250e+14, RMS= 2.5516e+13 + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -3.0971e+13, Max= 2.4747e+14, RMS= 2.6180e+13 + Valid time: 2010-01-01T07:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -2.8061e+13, Max= 2.5136e+14, RMS= 2.6788e+13 + Valid time: 2010-01-01T08:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -2.5144e+13, Max= 2.5423e+14, RMS= 2.7311e+13 + Valid time: 2010-01-01T09:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -2.3979e+13, Max= 2.5615e+14, RMS= 2.7721e+13 + Valid time: 2010-01-01T10:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -2.3448e+13, Max= 2.5735e+14, RMS= 2.8021e+13 + Valid time: 2010-01-01T11:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -2.2850e+13, Max= 2.5807e+14, RMS= 2.8223e+13 + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -2.2238e+13, Max= 2.5841e+14, RMS= 2.8326e+13 +Localized Increment: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 + Valid time: 2010-01-01T01:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 + Valid time: 2010-01-01T02:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 + Valid time: 2010-01-01T03:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 + Valid time: 2010-01-01T04:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 + Valid time: 2010-01-01T05:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 + Valid time: 2010-01-01T07:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 + Valid time: 2010-01-01T08:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 + Valid time: 2010-01-01T09:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 + Valid time: 2010-01-01T10:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 + Valid time: 2010-01-01T11:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -1.4154e-09, Max= 1.0000e+00, RMS= 1.3816e-01 +Randomized variance: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 3.7303e+12, Max= 1.0538e+15, RMS= 2.4663e+14 + Valid time: 2010-01-01T01:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 3.9097e+12, Max= 1.1299e+15, RMS= 2.5694e+14 + Valid time: 2010-01-01T02:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 2.4068e+12, Max= 1.2020e+15, RMS= 2.6754e+14 + Valid time: 2010-01-01T03:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 1.8149e+12, Max= 1.2679e+15, RMS= 2.7848e+14 + Valid time: 2010-01-01T04:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 1.9945e+12, Max= 1.3266e+15, RMS= 2.8969e+14 + Valid time: 2010-01-01T05:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 2.4492e+12, Max= 1.3767e+15, RMS= 3.0102e+14 + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 2.3155e+12, Max= 1.4163e+15, RMS= 3.1227e+14 + Valid time: 2010-01-01T07:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 2.2409e+12, Max= 1.4568e+15, RMS= 3.2326e+14 + Valid time: 2010-01-01T08:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 2.0067e+12, Max= 1.5122e+15, RMS= 3.3397e+14 + Valid time: 2010-01-01T09:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 1.7114e+12, Max= 1.5718e+15, RMS= 3.4435e+14 + Valid time: 2010-01-01T10:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 1.4554e+12, Max= 1.6333e+15, RMS= 3.5419e+14 + Valid time: 2010-01-01T11:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 1.2259e+12, Max= 1.6944e+15, RMS= 3.6352e+14 + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 1.0259e+12, Max= 1.7698e+15, RMS= 3.7255e+14 diff --git a/qg/test/testoutput/dirac_no_loc.test b/qg/test/testoutput/dirac_no_loc.test index 6c48c95b8..43aa66249 100644 --- a/qg/test/testoutput/dirac_no_loc.test +++ b/qg/test/testoutput/dirac_no_loc.test @@ -1,12 +1,12 @@ -Test : Input Dirac increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 -Test : B * Increment: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -3.0667e+14, Max= 4.4252e+14, RMS= 1.0739e+14 -Test : Randomized variance: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 1.1897e+12, Max= 1.7159e+15, RMS= 3.9892e+14 +Input Dirac increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 0.0000e+00, Max= 1.0000e+00, RMS= 2.5000e-02 +B * Increment: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -3.0667e+14, Max= 4.4252e+14, RMS= 1.0739e+14 +Randomized variance: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 1.1897e+12, Max= 1.7159e+15, RMS= 3.9892e+14 diff --git a/qg/test/testoutput/eda_3dvar_block.test b/qg/test/testoutput/eda_3dvar_block.test index 2be2d0215..33c42d827 100644 --- a/qg/test/testoutput/eda_3dvar_block.test +++ b/qg/test/testoutput/eda_3dvar_block.test @@ -1,53 +1,53 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 10879.7, nobs = 600, Jo/n = 18.1329, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1663.07, nobs = 600, Jo/n = 2.77178, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 163.034, nobs = 300, Jo/n = 0.543446, err = 12 -Test : CostFunction: Nonlinear J = 12705.8 -Test : Norm reduction all members ( 1) = 0.402888, 0.342551, 0.430302, 0.347356 -Test : Quadratic cost function all members: J ( 1) = 5681.52, 7492.09, 3160.43, 5694.31 -Test : Norm reduction all members ( 2) = 0.1949, 0.165786, 0.260803, 0.171396 -Test : Quadratic cost function all members: J ( 2) = 3979.77, 5093.07, 2507.38, 3641.38 -Test : Norm reduction all members ( 3) = 0.158052, 0.116268, 0.185322, 0.0977361 -Test : Quadratic cost function all members: J ( 3) = 3313.65, 4119.31, 2154.65, 2990.07 -Test : DRPBlockLanczosMinimizer: reduction in residual norm = 0.158052 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4961e+08, Max= 1.0918e+08, RMS= 1.7739e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 31.4 -Test : CostJo : Nonlinear Jo(Stream) = 2150, nobs = 600, Jo/n = 3.584, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 1104, nobs = 600, Jo/n = 1.84, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 83.31, nobs = 300, Jo/n = 0.2777, err = 12 -Test : CostFunction: Nonlinear J = 3369 -Test : Norm reduction all members ( 1) = 0.7216, 0.7109, 0.7893, 0.8465 -Test : Quadratic cost function all members: J ( 1) = 3027, 3825, 2014, 2814 -Test : Norm reduction all members ( 2) = 1.021, 0.9841, 1.035, 1.024 -Test : Quadratic cost function all members: J ( 2) = 2506, 3217, 1761, 2438 -Test : Norm reduction all members ( 3) = 0.7978, 0.7637, 0.8287, 0.8148 -Test : Quadratic cost function all members: J ( 3) = 2136, 2681, 1581, 2157 -Test : Norm reduction all members ( 4) = 0.426, 0.5472, 0.4854, 0.5317 -Test : Quadratic cost function all members: J ( 4) = 1923, 2450, 1427, 1979 -Test : Norm reduction all members ( 5) = 0.4426, 0.3654, 0.394, 0.4996 -Test : Quadratic cost function all members: J ( 5) = 1784, 2130, 1326, 1798 -Test : Norm reduction all members ( 6) = 0.3488, 0.2975, 0.3185, 0.3828 -Test : Quadratic cost function all members: J ( 6) = 1652, 1980, 1247, 1648 -Test : Norm reduction all members ( 7) = 0.2445, 0.2136, 0.2516, 0.3026 -Test : Quadratic cost function all members: J ( 7) = 1550, 1898, 1189, 1576 -Test : Norm reduction all members ( 8) = 0.1635, 0.1692, 0.1936, 0.2002 -Test : Quadratic cost function all members: J ( 8) = 1447, 1789, 1136, 1491 -Test : Norm reduction all members ( 9) = 0.1256, 0.1218, 0.1586, 0.1376 -Test : Quadratic cost function all members: J ( 9) = 1403, 1740, 1113, 1437 -Test : DRPBlockLanczosMinimizer: reduction in residual norm = 0.1256 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6332e+08, Max= 1.0396e+08, RMS= 1.7698e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : CostJb : Nonlinear Jb = 162.6 -Test : CostJo : Nonlinear Jo(Stream) = 563.7, nobs = 600, Jo/n = 0.9395, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 490.7, nobs = 600, Jo/n = 0.8179, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 74.86, nobs = 300, Jo/n = 0.2495, err = 12 -Test : CostFunction: Nonlinear J = 1292 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 10879.7, nobs = 600, Jo/n = 18.1329, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1663.07, nobs = 600, Jo/n = 2.77178, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 163.034, nobs = 300, Jo/n = 0.543446, err = 12 +CostFunction: Nonlinear J = 12705.8 + Norm reduction all members ( 1) = 0.402888, 0.342551, 0.430302, 0.347356 + Quadratic cost function all members: J ( 1) = 5681.52, 7492.09, 3160.43, 5694.31 + Norm reduction all members ( 2) = 0.1949, 0.165786, 0.260803, 0.171396 + Quadratic cost function all members: J ( 2) = 3979.77, 5093.07, 2507.38, 3641.38 + Norm reduction all members ( 3) = 0.158052, 0.116268, 0.185322, 0.0977361 + Quadratic cost function all members: J ( 3) = 3313.65, 4119.31, 2154.65, 2990.07 +DRPBlockLanczosMinimizer: reduction in residual norm = 0.158052 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4961e+08, Max= 1.0918e+08, RMS= 1.7739e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 31.4 +CostJo : Nonlinear Jo(Stream) = 2150, nobs = 600, Jo/n = 3.584, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1104, nobs = 600, Jo/n = 1.84, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 83.31, nobs = 300, Jo/n = 0.2777, err = 12 +CostFunction: Nonlinear J = 3369 + Norm reduction all members ( 1) = 0.7216, 0.7109, 0.7893, 0.8465 + Quadratic cost function all members: J ( 1) = 3027, 3825, 2014, 2814 + Norm reduction all members ( 2) = 1.021, 0.9841, 1.035, 1.024 + Quadratic cost function all members: J ( 2) = 2506, 3217, 1761, 2438 + Norm reduction all members ( 3) = 0.7978, 0.7637, 0.8287, 0.8148 + Quadratic cost function all members: J ( 3) = 2136, 2681, 1581, 2157 + Norm reduction all members ( 4) = 0.426, 0.5472, 0.4854, 0.5317 + Quadratic cost function all members: J ( 4) = 1923, 2450, 1427, 1979 + Norm reduction all members ( 5) = 0.4426, 0.3654, 0.394, 0.4996 + Quadratic cost function all members: J ( 5) = 1784, 2130, 1326, 1798 + Norm reduction all members ( 6) = 0.3488, 0.2975, 0.3185, 0.3828 + Quadratic cost function all members: J ( 6) = 1652, 1980, 1247, 1648 + Norm reduction all members ( 7) = 0.2445, 0.2136, 0.2516, 0.3026 + Quadratic cost function all members: J ( 7) = 1550, 1898, 1189, 1576 + Norm reduction all members ( 8) = 0.1635, 0.1692, 0.1936, 0.2002 + Quadratic cost function all members: J ( 8) = 1447, 1789, 1136, 1491 + Norm reduction all members ( 9) = 0.1256, 0.1218, 0.1586, 0.1376 + Quadratic cost function all members: J ( 9) = 1403, 1740, 1113, 1437 +DRPBlockLanczosMinimizer: reduction in residual norm = 0.1256 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6332e+08, Max= 1.0396e+08, RMS= 1.7698e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 162.6 +CostJo : Nonlinear Jo(Stream) = 563.7, nobs = 600, Jo/n = 0.9395, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 490.7, nobs = 600, Jo/n = 0.8179, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 74.86, nobs = 300, Jo/n = 0.2495, err = 12 +CostFunction: Nonlinear J = 1292 diff --git a/qg/test/testoutput/eda_4dvar.test b/qg/test/testoutput/eda_4dvar.test index e0bb91d02..4716c7794 100644 --- a/qg/test/testoutput/eda_4dvar.test +++ b/qg/test/testoutput/eda_4dvar.test @@ -1,36 +1,32 @@ -Test : CostJb : Nonlinear Jb = 0 -Test : CostJo : Nonlinear Jo(Stream) = 19013.2, nobs = 800, Jo/n = 23.7666, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 2088.99, nobs = 800, Jo/n = 2.61123, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 286.549, nobs = 400, Jo/n = 0.716372, err = 12 -Test : CostJcDFI: Nonlinear Jc = 17.009 -Test : CostFunction: Nonlinear J = 21405.8 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.0121582 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -2.9668, Max= 1.2573, RMS= 1.2524 -Test : Scaling= 1e+08, Min= -2.5200, Max= -0.0000, RMS= 1.2988 -Test : Scaling= 0.001, Min= -0.4243, Max= 0.3645, RMS= 0.2811 -Test : CostJb : Nonlinear Jb = 139.7 -Test : CostJo : Nonlinear Jo(Stream) = 997.4, nobs = 800, Jo/n = 1.247, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 251, nobs = 800, Jo/n = 0.3138, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 31.84, nobs = 400, Jo/n = 0.0796, err = 12 -Test : CostJcDFI: Nonlinear Jc = 40.17 -Test : CostFunction: Nonlinear J = 1460 -Test : DRPLanczosMinimizer: reduction in residual norm = 0.02358 -Test : CostFunction::addIncrement: Analysis: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Variable = streamfunction -Test : Boundary conditions are activated -Test : Scaling= 1e+08, Min= -2.9797, Max= 1.2582, RMS= 1.2586 -Test : Scaling= 1e+08, Min= -2.5200, Max= -0.0000, RMS= 1.2988 -Test : Scaling= 0.001, Min= -0.4243, Max= 0.3645, RMS= 0.2811 -Test : CostJb : Nonlinear Jb = 145.1 -Test : CostJo : Nonlinear Jo(Stream) = 79.85, nobs = 800, Jo/n = 0.09982, err = 4e+06 -Test : CostJo : Nonlinear Jo(Wind) = 42.7, nobs = 800, Jo/n = 0.05337, err = 6 -Test : CostJo : Nonlinear Jo(WSpeed) = 9.777, nobs = 400, Jo/n = 0.02444, err = 12 -Test : CostJcDFI: Nonlinear Jc = 34.01 -Test : CostFunction: Nonlinear J = 311.4 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Stream) = 11062.1, nobs = 800, Jo/n = 13.8277, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 1356.83, nobs = 800, Jo/n = 1.69604, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 179.84, nobs = 400, Jo/n = 0.4496, err = 12 +CostJcDFI: Nonlinear Jc = 57.9613 +CostFunction: Nonlinear J = 12656.8 +DRPLanczosMinimizer: reduction in residual norm = 0.0336644 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.2116e+08, Max= 1.0307e+08, RMS= 1.8911e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 28.45 +CostJo : Nonlinear Jo(Stream) = 558.1, nobs = 800, Jo/n = 0.6977, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 217.8, nobs = 800, Jo/n = 0.2722, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 39.41, nobs = 400, Jo/n = 0.09852, err = 12 +CostJcDFI: Nonlinear Jc = 60.36 +CostFunction: Nonlinear J = 904.1 +DRPLanczosMinimizer: reduction in residual norm = 0.1265 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.1935e+08, Max= 1.0477e+08, RMS= 1.9089e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +CostJb : Nonlinear Jb = 34.18 +CostJo : Nonlinear Jo(Stream) = 153.5, nobs = 800, Jo/n = 0.1919, err = 4e+06 +CostJo : Nonlinear Jo(Wind) = 104.6, nobs = 800, Jo/n = 0.1308, err = 6 +CostJo : Nonlinear Jo(WSpeed) = 17.99, nobs = 400, Jo/n = 0.04499, err = 12 +CostJcDFI: Nonlinear Jc = 65.93 +CostFunction: Nonlinear J = 376.2 diff --git a/qg/test/testoutput/ens_hofx.test b/qg/test/testoutput/ens_hofx.test index 7fdc1e504..1c19d7ad6 100644 --- a/qg/test/testoutput/ens_hofx.test +++ b/qg/test/testoutput/ens_hofx.test @@ -1,17 +1,17 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Final state: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4224e+08, Max= 1.2418e+08, RMS= 1.7584e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : H(x): -Test : Stream nobs= 300 Min= -5.3856e+08, Max= 8.6950e+07, Average= -1.2072e+08 -Test : Wind nobs= 160 Min= -1.2440e+02, Max= 9.4257e+01, Average= 1.1768e+01 -Test : WSpeed nobs= 300 Min= 1.2710e+00, Max= 1.2510e+02, Average= 4.6830e+01 -Test : End H(x) +Initial state: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Final state: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4224e+08, Max= 1.2418e+08, RMS= 1.7584e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +H(x): +Stream nobs= 300 Min= -5.3856e+08, Max= 8.6950e+07, Average= -1.2072e+08 +Wind nobs= 160 Min= -1.2440e+02, Max= 9.4257e+01, Average= 1.1768e+01 +WSpeed nobs= 300 Min= 1.2710e+00, Max= 1.2510e+02, Average= 4.6830e+01 +End H(x) diff --git a/qg/test/testoutput/ens_recenter.test b/qg/test/testoutput/ens_recenter.test index b61758f76..abe117527 100644 --- a/qg/test/testoutput/ens_recenter.test +++ b/qg/test/testoutput/ens_recenter.test @@ -1,66 +1,67 @@ -Test : Original member 0 : -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Original member 1 : -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6931e+08, Max= 9.9327e+07, RMS= 1.7976e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Original member 2 : -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.9074e+08, Max= 1.0042e+08, RMS= 1.8589e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Original member 3 : -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.7276e+08, Max= 9.4808e+07, RMS= 1.7978e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Original member 4 : -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.7215e+08, Max= 9.9052e+07, RMS= 1.8304e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Ensemble mean: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.7486e+08, Max= 9.6575e+07, RMS= 1.8216e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Recentered member 0 : -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.7348e+08, Max= 1.5063e+08, RMS= 1.8806e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Recentered member 1 : -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6614e+08, Max= 9.2017e+07, RMS= 1.8151e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Recentered member 2 : -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8521e+08, Max= 9.5093e+07, RMS= 1.8805e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Recentered member 3 : -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6890e+08, Max= 8.9656e+07, RMS= 1.8163e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Recentered member 4 : -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6887e+08, Max= 9.3721e+07, RMS= 1.8456e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Original member 0 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Original member 1 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6931e+08, Max= 9.9327e+07, RMS= 1.7976e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Original member 2 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9074e+08, Max= 1.0042e+08, RMS= 1.8589e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Original member 3 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7276e+08, Max= 9.4808e+07, RMS= 1.7978e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Original member 4 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7215e+08, Max= 9.9052e+07, RMS= 1.8304e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Ensemble mean: + + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7486e+08, Max= 9.6575e+07, RMS= 1.8216e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 0 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7348e+08, Max= 1.5063e+08, RMS= 1.8806e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 1 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6614e+08, Max= 9.2017e+07, RMS= 1.8151e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 2 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8521e+08, Max= 9.5093e+07, RMS= 1.8805e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 3 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6890e+08, Max= 8.9656e+07, RMS= 1.8163e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 4 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6887e+08, Max= 9.3721e+07, RMS= 1.8456e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/ens_variance.test b/qg/test/testoutput/ens_variance.test index aadfc0b72..75c1dc635 100644 --- a/qg/test/testoutput/ens_variance.test +++ b/qg/test/testoutput/ens_variance.test @@ -1,4 +1,5 @@ -Test : Variance: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 4.7139e+11, Max= 1.5009e+15, RMS= 2.8630e+14 +Variance: + + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 4.7139e+11, Max= 1.5009e+15, RMS= 2.8630e+14 diff --git a/qg/test/testoutput/ens_variance_inflation_field.test b/qg/test/testoutput/ens_variance_inflation_field.test index 822269926..10950a250 100644 --- a/qg/test/testoutput/ens_variance_inflation_field.test +++ b/qg/test/testoutput/ens_variance_inflation_field.test @@ -1,4 +1,5 @@ -Test : Variance: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 1.8856e+12, Max= 6.0037e+15, RMS= 1.1452e+15 +Variance: + + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 1.8856e+12, Max= 6.0037e+15, RMS= 1.1452e+15 diff --git a/qg/test/testoutput/ens_variance_inflation_value.test b/qg/test/testoutput/ens_variance_inflation_value.test index 822269926..10950a250 100644 --- a/qg/test/testoutput/ens_variance_inflation_value.test +++ b/qg/test/testoutput/ens_variance_inflation_value.test @@ -1,4 +1,5 @@ -Test : Variance: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 1.8856e+12, Max= 6.0037e+15, RMS= 1.1452e+15 +Variance: + + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 1.8856e+12, Max= 6.0037e+15, RMS= 1.1452e+15 diff --git a/qg/test/testoutput/forecast.test b/qg/test/testoutput/forecast.test index 17bd52b83..31f83614f 100644 --- a/qg/test/testoutput/forecast.test +++ b/qg/test/testoutput/forecast.test @@ -1,12 +1,12 @@ -Test : Initial state: -Test : Valid time: 2009-12-31T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8796e+08, Max= 1.0889e+08, RMS= 2.0493e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Final state: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.1708e+08, Max= 1.0232e+08, RMS= 1.6357e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Initial state: + Valid time: 2009-12-31T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8796e+08, Max= 1.0889e+08, RMS= 2.0493e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Final state: + Valid time: 2010-01-02T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.1708e+08, Max= 1.0232e+08, RMS= 1.6357e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/gen_ens_pert_B.test b/qg/test/testoutput/gen_ens_pert_B.test index 4dd5a10c7..4d76ac722 100644 --- a/qg/test/testoutput/gen_ens_pert_B.test +++ b/qg/test/testoutput/gen_ens_pert_B.test @@ -1,66 +1,66 @@ -Test : Initial state: -Test : Valid time: 2009-12-31T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.8796e+08, Max= 1.0889e+08, RMS= 2.0493e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Member 0 final state: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4224e+08, Max= 1.2418e+08, RMS= 1.7584e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Member 1 final state: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.3426e+08, Max= 9.8104e+07, RMS= 1.6992e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Member 2 final state: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4601e+08, Max= 1.0169e+08, RMS= 1.7665e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Member 3 final state: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.3446e+08, Max= 1.0277e+08, RMS= 1.6981e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Member 4 final state: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.3100e+08, Max= 9.5883e+07, RMS= 1.6904e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Member 5 final state: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.3282e+08, Max= 1.0319e+08, RMS= 1.7295e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Member 6 final state: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.3246e+08, Max= 9.6905e+07, RMS= 1.7096e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Member 7 final state: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.2419e+08, Max= 1.2494e+08, RMS= 1.6860e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Member 8 final state: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.3878e+08, Max= 9.3644e+07, RMS= 1.7330e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Member 9 final state: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.3011e+08, Max= 1.0896e+08, RMS= 1.6876e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Initial state: + Valid time: 2009-12-31T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.8796e+08, Max= 1.0889e+08, RMS= 2.0493e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Member 0 final state: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4224e+08, Max= 1.2418e+08, RMS= 1.7584e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Member 1 final state: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.3426e+08, Max= 9.8104e+07, RMS= 1.6992e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Member 2 final state: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4601e+08, Max= 1.0169e+08, RMS= 1.7665e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Member 3 final state: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.3446e+08, Max= 1.0277e+08, RMS= 1.6981e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Member 4 final state: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.3100e+08, Max= 9.5883e+07, RMS= 1.6904e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Member 5 final state: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.3282e+08, Max= 1.0319e+08, RMS= 1.7295e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Member 6 final state: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.3246e+08, Max= 9.6905e+07, RMS= 1.7096e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Member 7 final state: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.2419e+08, Max= 1.2494e+08, RMS= 1.6860e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Member 8 final state: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.3878e+08, Max= 9.3644e+07, RMS= 1.7330e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Member 9 final state: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.3011e+08, Max= 1.0896e+08, RMS= 1.6876e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/hofx.test b/qg/test/testoutput/hofx.test index 803f972a2..77e5d1311 100644 --- a/qg/test/testoutput/hofx.test +++ b/qg/test/testoutput/hofx.test @@ -1,17 +1,17 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Final state: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.5859e+08, Max= 1.0654e+08, RMS= 1.7877e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : H(x): -Test : Stream nobs= 300 Min= -5.6944e+08, Max= 9.0622e+07, Average= -1.3204e+08 -Test : Wind nobs= 160 Min= -8.3586e+01, Max= 1.0673e+02, Average= 1.0558e+01 -Test : WSpeed nobs= 300 Min= 8.1358e+00, Max= 1.4522e+02, Average= 4.9940e+01 -Test : End H(x) +Initial state: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Final state: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.5859e+08, Max= 1.0654e+08, RMS= 1.7877e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +H(x): +Stream nobs= 300 Min= -5.6944e+08, Max= 9.0622e+07, Average= -1.3204e+08 +Wind nobs= 160 Min= -8.3586e+01, Max= 1.0673e+02, Average= 1.0558e+01 +WSpeed nobs= 300 Min= 8.1358e+00, Max= 1.4522e+02, Average= 4.9940e+01 +End H(x) diff --git a/qg/test/testoutput/hofx3d.test b/qg/test/testoutput/hofx3d.test index 2df8e55f1..584665c1c 100644 --- a/qg/test/testoutput/hofx3d.test +++ b/qg/test/testoutput/hofx3d.test @@ -1,11 +1,11 @@ -Test : State: -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.0526e+08, Max= 1.1159e+08, RMS= 1.8719e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : H(x): -Test : Stream nobs= 300 Min= -5.8159e+08, Max= 8.9514e+07, Average= -1.3681e+08 -Test : Wind nobs= 160 Min= -8.3304e+01, Max= 1.0663e+02, Average= 1.0113e+01 -Test : WSpeed nobs= 300 Min= 1.6435e+01, Max= 1.4412e+02, Average= 5.1343e+01 -Test : End H(x) +State: + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.0526e+08, Max= 1.1159e+08, RMS= 1.8719e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +H(x): +Stream nobs= 300 Min= -5.8159e+08, Max= 8.9514e+07, Average= -1.3681e+08 +Wind nobs= 160 Min= -8.3304e+01, Max= 1.0663e+02, Average= 1.0113e+01 +WSpeed nobs= 300 Min= 1.6435e+01, Max= 1.4412e+02, Average= 5.1343e+01 +End H(x) diff --git a/qg/test/testoutput/letkf.test b/qg/test/testoutput/letkf.test index b4434c856..e2df4c7bf 100644 --- a/qg/test/testoutput/letkf.test +++ b/qg/test/testoutput/letkf.test @@ -1,170 +1,205 @@ -Test : Initial state for member 1: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.5492e+08, Max= 1.1027e+08, RMS= 1.7951e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4224e+08, Max= 1.2418e+08, RMS= 1.7584e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Initial state for member 2: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6931e+08, Max= 9.9327e+07, RMS= 1.7976e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.5020e+08, Max= 9.8531e+07, RMS= 1.7427e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.3426e+08, Max= 9.8104e+07, RMS= 1.6992e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Initial state for member 3: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.9074e+08, Max= 1.0042e+08, RMS= 1.8589e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6561e+08, Max= 1.0048e+08, RMS= 1.8093e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4601e+08, Max= 1.0169e+08, RMS= 1.7665e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Initial state for member 4: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.7276e+08, Max= 9.4808e+07, RMS= 1.7978e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.5080e+08, Max= 9.8565e+07, RMS= 1.7375e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.3446e+08, Max= 1.0277e+08, RMS= 1.6981e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Initial state for member 5: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.7215e+08, Max= 9.9052e+07, RMS= 1.8304e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4705e+08, Max= 9.6271e+07, RMS= 1.7467e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.3100e+08, Max= 9.5883e+07, RMS= 1.6904e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : H(x) for member 1: -Test : Stream nobs= 300 Min= -5.4020e+08, Max= 8.6950e+07, Average= -1.2080e+08 -Test : Wind nobs= 160 Min= -1.2372e+02, Max= 9.5123e+01, Average= 1.1752e+01 -Test : WSpeed nobs= 300 Min= 1.5996e+00, Max= 1.2635e+02, Average= 4.6914e+01 -Test : H(x) for member 2: -Test : Stream nobs= 300 Min= -5.3042e+08, Max= 8.9085e+07, Average= -1.2262e+08 -Test : Wind nobs= 160 Min= -1.0544e+02, Max= 9.4639e+01, Average= 1.1232e+01 -Test : WSpeed nobs= 300 Min= 3.1631e+00, Max= 1.4462e+02, Average= 4.6283e+01 -Test : H(x) for member 3: -Test : Stream nobs= 300 Min= -5.4203e+08, Max= 8.4745e+07, Average= -1.2861e+08 -Test : Wind nobs= 160 Min= -7.8772e+01, Max= 9.9178e+01, Average= 1.2338e+01 -Test : WSpeed nobs= 300 Min= 4.5818e+00, Max= 1.2667e+02, Average= 4.5105e+01 -Test : H(x) for member 4: -Test : Stream nobs= 300 Min= -5.3782e+08, Max= 8.7050e+07, Average= -1.2265e+08 -Test : Wind nobs= 160 Min= -9.7651e+01, Max= 1.0620e+02, Average= 1.1342e+01 -Test : WSpeed nobs= 300 Min= 3.4839e-01, Max= 1.3021e+02, Average= 4.6108e+01 -Test : H(x) for member 5: -Test : Stream nobs= 300 Min= -5.2606e+08, Max= 7.6970e+07, Average= -1.2415e+08 -Test : Wind nobs= 160 Min= -6.4074e+01, Max= 9.9153e+01, Average= 1.3381e+01 -Test : WSpeed nobs= 300 Min= 5.6407e+00, Max= 1.2156e+02, Average= 4.4377e+01 -Test : H(x) ensemble background mean: -Test : Stream nobs= 300 Min= -5.3531e+08, Max= 8.4960e+07, Average= -1.2376e+08 -Test : Wind nobs= 160 Min= -9.3325e+01, Max= 9.8546e+01, Average= 1.2009e+01 -Test : WSpeed nobs= 300 Min= 6.1089e+00, Max= 1.2988e+02, Average= 4.5757e+01 -Test : background y - H(x): -Test : Stream nobs= 300 Min= -4.2183e+07, Max= 3.7845e+07, Average= -8.2739e+06 -Test : Wind nobs= 160 Min= -3.7898e+01, Max= 1.9094e+01, Average= -1.4507e+00 -Test : WSpeed nobs= 300 Min= -2.7821e+01, Max= 3.2298e+01, Average= 4.1823e+00 -Test : Background mean : -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.7486e+08, Max= 9.6575e+07, RMS= 1.8216e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.5279e+08, Max= 9.3048e+07, RMS= 1.7609e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.3566e+08, Max= 9.5381e+07, RMS= 1.7157e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Analysis mean : -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.9026e+08, Max= 9.5856e+07, RMS= 1.8792e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.6092e+08, Max= 9.1362e+07, RMS= 1.8251e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4655e+08, Max= 9.6366e+07, RMS= 1.7841e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : H(x) for member 1: -Test : Stream nobs= 300 Min= -5.4486e+08, Max= 9.1832e+07, Average= -1.2800e+08 -Test : Wind nobs= 160 Min= -9.6559e+01, Max= 1.0881e+02, Average= 1.1563e+01 -Test : WSpeed nobs= 300 Min= 3.3554e+00, Max= 1.2154e+02, Average= 4.6685e+01 -Test : H(x) for member 2: -Test : Stream nobs= 300 Min= -5.3983e+08, Max= 9.3072e+07, Average= -1.2890e+08 -Test : Wind nobs= 160 Min= -8.3423e+01, Max= 1.0781e+02, Average= 1.1252e+01 -Test : WSpeed nobs= 300 Min= 5.8725e+00, Max= 1.3087e+02, Average= 4.6272e+01 -Test : H(x) for member 3: -Test : Stream nobs= 300 Min= -5.4582e+08, Max= 9.0372e+07, Average= -1.3205e+08 -Test : Wind nobs= 160 Min= -9.1028e+01, Max= 1.1127e+02, Average= 1.1876e+01 -Test : WSpeed nobs= 300 Min= 5.2300e+00, Max= 1.2200e+02, Average= 4.6677e+01 -Test : H(x) for member 4: -Test : Stream nobs= 300 Min= -5.4362e+08, Max= 9.1775e+07, Average= -1.2892e+08 -Test : Wind nobs= 160 Min= -8.4658e+01, Max= 1.1873e+02, Average= 1.1356e+01 -Test : WSpeed nobs= 300 Min= 1.9489e+00, Max= 1.2352e+02, Average= 4.6564e+01 -Test : H(x) for member 5: -Test : Stream nobs= 300 Min= -5.3753e+08, Max= 8.5962e+07, Average= -1.2972e+08 -Test : Wind nobs= 160 Min= -8.3380e+01, Max= 1.1201e+02, Average= 1.2450e+01 -Test : WSpeed nobs= 300 Min= 7.9598e+00, Max= 1.1894e+02, Average= 4.5836e+01 -Test : H(x) ensemble analysis mean: -Test : Stream nobs= 300 Min= -5.4233e+08, Max= 9.0602e+07, Average= -1.2952e+08 -Test : Wind nobs= 160 Min= -8.5660e+01, Max= 1.1173e+02, Average= 1.1699e+01 -Test : WSpeed nobs= 300 Min= 8.7572e+00, Max= 1.2337e+02, Average= 4.6407e+01 -Test : analysis y - H(x): -Test : Stream nobs= 300 Min= -3.3262e+07, Max= 2.5898e+07, Average= -2.5176e+06 -Test : Wind nobs= 160 Min= -3.1217e+01, Max= 2.2379e+01, Average= -1.1411e+00 -Test : WSpeed nobs= 300 Min= -2.7367e+01, Max= 4.4216e+01, Average= 3.5329e+00 -Test : ombg RMS: 1.105e+07 -Test : oman RMS: 6.689e+06 +Initial state for member 1: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.5492e+08, Max= 1.1027e+08, RMS= 1.7951e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4224e+08, Max= 1.2418e+08, RMS= 1.7584e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + +Initial state for member 2: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6931e+08, Max= 9.9327e+07, RMS= 1.7976e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.5020e+08, Max= 9.8531e+07, RMS= 1.7427e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.3426e+08, Max= 9.8104e+07, RMS= 1.6992e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + +Initial state for member 3: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9074e+08, Max= 1.0042e+08, RMS= 1.8589e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6561e+08, Max= 1.0048e+08, RMS= 1.8093e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4601e+08, Max= 1.0169e+08, RMS= 1.7665e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + +Initial state for member 4: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7276e+08, Max= 9.4808e+07, RMS= 1.7978e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.5080e+08, Max= 9.8565e+07, RMS= 1.7375e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.3446e+08, Max= 1.0277e+08, RMS= 1.6981e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + +Initial state for member 5: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7215e+08, Max= 9.9052e+07, RMS= 1.8304e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4705e+08, Max= 9.6271e+07, RMS= 1.7467e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.3100e+08, Max= 9.5883e+07, RMS= 1.6904e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + +H(x) for member 1: +Stream nobs= 300 Min= -5.4020e+08, Max= 8.6950e+07, Average= -1.2080e+08 +Wind nobs= 160 Min= -1.2372e+02, Max= 9.5123e+01, Average= 1.1752e+01 +WSpeed nobs= 300 Min= 1.5996e+00, Max= 1.2635e+02, Average= 4.6914e+01 + +H(x) for member 2: +Stream nobs= 300 Min= -5.3042e+08, Max= 8.9085e+07, Average= -1.2262e+08 +Wind nobs= 160 Min= -1.0544e+02, Max= 9.4639e+01, Average= 1.1232e+01 +WSpeed nobs= 300 Min= 3.1631e+00, Max= 1.4462e+02, Average= 4.6283e+01 + +H(x) for member 3: +Stream nobs= 300 Min= -5.4203e+08, Max= 8.4745e+07, Average= -1.2861e+08 +Wind nobs= 160 Min= -7.8772e+01, Max= 9.9178e+01, Average= 1.2338e+01 +WSpeed nobs= 300 Min= 4.5818e+00, Max= 1.2667e+02, Average= 4.5105e+01 + +H(x) for member 4: +Stream nobs= 300 Min= -5.3782e+08, Max= 8.7050e+07, Average= -1.2265e+08 +Wind nobs= 160 Min= -9.7651e+01, Max= 1.0620e+02, Average= 1.1342e+01 +WSpeed nobs= 300 Min= 3.4839e-01, Max= 1.3021e+02, Average= 4.6108e+01 + +H(x) for member 5: +Stream nobs= 300 Min= -5.2606e+08, Max= 7.6970e+07, Average= -1.2415e+08 +Wind nobs= 160 Min= -6.4074e+01, Max= 9.9153e+01, Average= 1.3381e+01 +WSpeed nobs= 300 Min= 5.6407e+00, Max= 1.2156e+02, Average= 4.4377e+01 + +H(x) ensemble background mean: +Stream nobs= 300 Min= -5.3531e+08, Max= 8.4960e+07, Average= -1.2376e+08 +Wind nobs= 160 Min= -9.3325e+01, Max= 9.8546e+01, Average= 1.2009e+01 +WSpeed nobs= 300 Min= 6.1089e+00, Max= 1.2988e+02, Average= 4.5757e+01 + +background y - H(x): +Stream nobs= 300 Min= -4.2183e+07, Max= 3.7845e+07, Average= -8.2739e+06 +Wind nobs= 160 Min= -3.7898e+01, Max= 1.9094e+01, Average= -1.4507e+00 +WSpeed nobs= 300 Min= -2.7821e+01, Max= 3.2298e+01, Average= 4.1823e+00 + +Background mean : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7486e+08, Max= 9.6575e+07, RMS= 1.8216e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.5279e+08, Max= 9.3048e+07, RMS= 1.7609e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.3566e+08, Max= 9.5381e+07, RMS= 1.7157e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + +Analysis mean : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9026e+08, Max= 9.5856e+07, RMS= 1.8792e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6092e+08, Max= 9.1362e+07, RMS= 1.8251e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4655e+08, Max= 9.6366e+07, RMS= 1.7841e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 + +H(x) for member 1: +Stream nobs= 300 Min= -5.4486e+08, Max= 9.1832e+07, Average= -1.2800e+08 +Wind nobs= 160 Min= -9.6559e+01, Max= 1.0881e+02, Average= 1.1563e+01 +WSpeed nobs= 300 Min= 3.3554e+00, Max= 1.2154e+02, Average= 4.6685e+01 + +H(x) for member 2: +Stream nobs= 300 Min= -5.3983e+08, Max= 9.3072e+07, Average= -1.2890e+08 +Wind nobs= 160 Min= -8.3423e+01, Max= 1.0781e+02, Average= 1.1252e+01 +WSpeed nobs= 300 Min= 5.8725e+00, Max= 1.3087e+02, Average= 4.6272e+01 + +H(x) for member 3: +Stream nobs= 300 Min= -5.4582e+08, Max= 9.0372e+07, Average= -1.3205e+08 +Wind nobs= 160 Min= -9.1028e+01, Max= 1.1127e+02, Average= 1.1876e+01 +WSpeed nobs= 300 Min= 5.2300e+00, Max= 1.2200e+02, Average= 4.6677e+01 + +H(x) for member 4: +Stream nobs= 300 Min= -5.4362e+08, Max= 9.1775e+07, Average= -1.2892e+08 +Wind nobs= 160 Min= -8.4658e+01, Max= 1.1873e+02, Average= 1.1356e+01 +WSpeed nobs= 300 Min= 1.9489e+00, Max= 1.2352e+02, Average= 4.6564e+01 + +H(x) for member 5: +Stream nobs= 300 Min= -5.3753e+08, Max= 8.5962e+07, Average= -1.2972e+08 +Wind nobs= 160 Min= -8.3380e+01, Max= 1.1201e+02, Average= 1.2450e+01 +WSpeed nobs= 300 Min= 7.9598e+00, Max= 1.1894e+02, Average= 4.5836e+01 + +H(x) ensemble analysis mean: +Stream nobs= 300 Min= -5.4233e+08, Max= 9.0602e+07, Average= -1.2952e+08 +Wind nobs= 160 Min= -8.5660e+01, Max= 1.1173e+02, Average= 1.1699e+01 +WSpeed nobs= 300 Min= 8.7572e+00, Max= 1.2337e+02, Average= 4.6407e+01 + +analysis y - H(x): +Stream nobs= 300 Min= -3.3262e+07, Max= 2.5898e+07, Average= -2.5176e+06 +Wind nobs= 160 Min= -3.1217e+01, Max= 2.2379e+01, Average= -1.1411e+00 +WSpeed nobs= 300 Min= -2.7367e+01, Max= 4.4216e+01, Average= 3.5329e+00 + +ombg RMS: 1.105e+07 +oman RMS: 6.689e+06 diff --git a/qg/test/testoutput/make_obs_3d.test b/qg/test/testoutput/make_obs_3d.test index f5bfd1c47..6f7f2ffc9 100644 --- a/qg/test/testoutput/make_obs_3d.test +++ b/qg/test/testoutput/make_obs_3d.test @@ -1,17 +1,17 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Final state: -Test : Valid time: 2010-01-01T15:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4390e+08, Max= 1.0635e+08, RMS= 1.7643e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : H(x): -Test : Stream nobs= 600 Min= -5.7803e+08, Max= 1.2841e+08, Average= -1.1552e+08 -Test : Wind nobs= 600 Min= -7.1086e+01, Max= 1.1626e+02, Average= 1.3189e+01 -Test : WSpeed nobs= 300 Min= 2.4533e+00, Max= 1.6880e+02, Average= 4.9588e+01 -Test : End H(x) +Initial state: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Final state: + Valid time: 2010-01-01T15:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4390e+08, Max= 1.0635e+08, RMS= 1.7643e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +H(x): +Stream nobs= 600 Min= -5.7803e+08, Max= 1.2841e+08, Average= -1.1552e+08 +Wind nobs= 600 Min= -7.1086e+01, Max= 1.1626e+02, Average= 1.3189e+01 +WSpeed nobs= 300 Min= 2.4533e+00, Max= 1.6880e+02, Average= 4.9588e+01 +End H(x) diff --git a/qg/test/testoutput/make_obs_4d_12h.test b/qg/test/testoutput/make_obs_4d_12h.test index 803f972a2..77e5d1311 100644 --- a/qg/test/testoutput/make_obs_4d_12h.test +++ b/qg/test/testoutput/make_obs_4d_12h.test @@ -1,17 +1,17 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Final state: -Test : Valid time: 2010-01-01T12:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.5859e+08, Max= 1.0654e+08, RMS= 1.7877e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : H(x): -Test : Stream nobs= 300 Min= -5.6944e+08, Max= 9.0622e+07, Average= -1.3204e+08 -Test : Wind nobs= 160 Min= -8.3586e+01, Max= 1.0673e+02, Average= 1.0558e+01 -Test : WSpeed nobs= 300 Min= 8.1358e+00, Max= 1.4522e+02, Average= 4.9940e+01 -Test : End H(x) +Initial state: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Final state: + Valid time: 2010-01-01T12:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.5859e+08, Max= 1.0654e+08, RMS= 1.7877e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +H(x): +Stream nobs= 300 Min= -5.6944e+08, Max= 9.0622e+07, Average= -1.3204e+08 +Wind nobs= 160 Min= -8.3586e+01, Max= 1.0673e+02, Average= 1.0558e+01 +WSpeed nobs= 300 Min= 8.1358e+00, Max= 1.4522e+02, Average= 4.9940e+01 +End H(x) diff --git a/qg/test/testoutput/make_obs_4d_24h.test b/qg/test/testoutput/make_obs_4d_24h.test index be6aada2a..c320682bd 100644 --- a/qg/test/testoutput/make_obs_4d_24h.test +++ b/qg/test/testoutput/make_obs_4d_24h.test @@ -1,17 +1,17 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Final state: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4100e+08, Max= 1.1396e+08, RMS= 1.7043e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : H(x): -Test : Stream nobs= 800 Min= -6.1451e+08, Max= 1.4118e+08, Average= -1.1514e+08 -Test : Wind nobs= 800 Min= -7.0969e+01, Max= 1.1687e+02, Average= 1.3047e+01 -Test : WSpeed nobs= 400 Min= 1.6011e+00, Max= 1.9114e+02, Average= 4.9728e+01 -Test : End H(x) +Initial state: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Final state: + Valid time: 2010-01-02T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4100e+08, Max= 1.1396e+08, RMS= 1.7043e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +H(x): +Stream nobs= 800 Min= -6.1451e+08, Max= 1.4118e+08, Average= -1.1514e+08 +Wind nobs= 800 Min= -7.0969e+01, Max= 1.1687e+02, Average= 1.3047e+01 +WSpeed nobs= 400 Min= 1.6011e+00, Max= 1.9114e+02, Average= 4.9728e+01 +End H(x) diff --git a/qg/test/testoutput/make_obs_4d_biased.test b/qg/test/testoutput/make_obs_4d_biased.test index 3308845df..2c659fb9a 100644 --- a/qg/test/testoutput/make_obs_4d_biased.test +++ b/qg/test/testoutput/make_obs_4d_biased.test @@ -1,17 +1,17 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Final state: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4100e+08, Max= 1.1396e+08, RMS= 1.7043e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : H(x): -Test : Stream nobs= 800 Min= -6.2451e+08, Max= 1.3118e+08, Average= -1.2514e+08 -Test : Wind nobs= 800 Min= -6.9902e+01, Max= 1.2687e+02, Average= 1.8047e+01 -Test : WSpeed nobs= 400 Min= 1.6011e+00, Max= 1.9114e+02, Average= 4.9728e+01 -Test : End H(x) +Initial state: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -5.1807e+08, Max= 1.0739e+08, RMS= 1.9152e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Final state: + Valid time: 2010-01-02T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4100e+08, Max= 1.1396e+08, RMS= 1.7043e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +H(x): +Stream nobs= 800 Min= -6.2451e+08, Max= 1.3118e+08, Average= -1.2514e+08 +Wind nobs= 800 Min= -6.9902e+01, Max= 1.2687e+02, Average= 1.8047e+01 +WSpeed nobs= 400 Min= 1.6011e+00, Max= 1.9114e+02, Average= 4.9728e+01 +End H(x) diff --git a/qg/test/testoutput/rtpp.test b/qg/test/testoutput/rtpp.test index 70b372e67..b6eb94e49 100644 --- a/qg/test/testoutput/rtpp.test +++ b/qg/test/testoutput/rtpp.test @@ -1,24 +1,24 @@ -Test : Background member 1: -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.5492e+08, Max= 1.1027e+08, RMS= 1.7951e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Analysis member 1: -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.5020e+08, Max= 9.8531e+07, RMS= 1.7427e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Updated Analysis member 1: -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.5256e+08, Max= 9.3779e+07, RMS= 1.7629e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Analysis mean: -Test : Valid time: 2010-01-01T06:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.5256e+08, Max= 9.3779e+07, RMS= 1.7629e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Background member 1: + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.5492e+08, Max= 1.1027e+08, RMS= 1.7951e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Analysis member 1: + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.5020e+08, Max= 9.8531e+07, RMS= 1.7427e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Updated Analysis member 1: + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.5256e+08, Max= 9.3779e+07, RMS= 1.7629e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Analysis mean: + Valid time: 2010-01-01T06:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.5256e+08, Max= 9.3779e+07, RMS= 1.7629e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/static_b_init.test b/qg/test/testoutput/static_b_init.test index 69182978b..bd11c1490 100644 --- a/qg/test/testoutput/static_b_init.test +++ b/qg/test/testoutput/static_b_init.test @@ -1,3 +1,4 @@ -Test : Valid time: 2009-12-31T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.5233e+07, Max= 3.6083e+07, RMS= 1.5609e+07 + + Valid time: 2009-12-31T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.5233e+07, Max= 3.6083e+07, RMS= 1.5609e+07 diff --git a/qg/test/testoutput/truth.test b/qg/test/testoutput/truth.test index 1c8ffe43c..65ab5fc92 100644 --- a/qg/test/testoutput/truth.test +++ b/qg/test/testoutput/truth.test @@ -1,12 +1,12 @@ -Test : Initial state: -Test : Valid time: 2009-12-15T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -3.8125e+08, Max= -4.7657e+06, RMS= 1.6644e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Test : Final state: -Test : Valid time: 2010-01-02T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= -4.4486e+08, Max= 1.2473e+08, RMS= 1.7599e+08 -Test : Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 -Test : Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Initial state: + Valid time: 2009-12-15T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -3.8125e+08, Max= -4.7657e+06, RMS= 1.6644e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Final state: + Valid time: 2010-01-02T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.4486e+08, Max= 1.2473e+08, RMS= 1.7599e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/qg/test/testoutput/uniform_field_hybrid.test b/qg/test/testoutput/uniform_field_hybrid.test index 47818d51e..334f2d645 100644 --- a/qg/test/testoutput/uniform_field_hybrid.test +++ b/qg/test/testoutput/uniform_field_hybrid.test @@ -1,12 +1,12 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 5.0000e-01, Max= 5.0000e-01, RMS= 5.0000e-01 -Test : Streamfunction LBC : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Potential vorticity LBC: Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 -Test : Final state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 5.0000e-01, Max= 5.0000e-01, RMS= 5.0000e-01 -Test : Streamfunction LBC : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Potential vorticity LBC: Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 +Initial state: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 5.0000e-01, Max= 5.0000e-01, RMS= 5.0000e-01 + Streamfunction LBC : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Potential vorticity LBC: Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 +Final state: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 5.0000e-01, Max= 5.0000e-01, RMS= 5.0000e-01 + Streamfunction LBC : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Potential vorticity LBC: Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 diff --git a/qg/test/testoutput/uniform_field_inflation.test b/qg/test/testoutput/uniform_field_inflation.test index 9b0959826..dcbca21ca 100644 --- a/qg/test/testoutput/uniform_field_inflation.test +++ b/qg/test/testoutput/uniform_field_inflation.test @@ -1,12 +1,12 @@ -Test : Initial state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 2.0000e+00, Max= 2.0000e+00, RMS= 2.0000e+00 -Test : Streamfunction LBC : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Potential vorticity LBC: Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 -Test : Final state: -Test : Valid time: 2010-01-01T00:00:00Z -Test : Resolution = 40, 20, 2 -Test : Streamfunction : Min= 2.0000e+00, Max= 2.0000e+00, RMS= 2.0000e+00 -Test : Streamfunction LBC : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 -Test : Potential vorticity LBC: Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 +Initial state: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 2.0000e+00, Max= 2.0000e+00, RMS= 2.0000e+00 + Streamfunction LBC : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Potential vorticity LBC: Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 +Final state: + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= 2.0000e+00, Max= 2.0000e+00, RMS= 2.0000e+00 + Streamfunction LBC : Min= 0.0000e+00, Max= 0.0000e+00, RMS= 0.0000e+00 + Potential vorticity LBC: Min= -7.1485e-05, Max= 7.8634e-05, RMS= 7.5144e-05 From 6cda4e863533d650db9ce4f1f0bbf0f24235c646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Fri, 30 Apr 2021 15:41:32 +0100 Subject: [PATCH 117/142] Allow linear obs operator tests to indicate the operator's constructor should throw an exception. (#1190) --- src/test/interface/LinearObsOperator.h | 32 +++++++++++++++++++++----- src/test/interface/ObsOperator.h | 2 +- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/test/interface/LinearObsOperator.h b/src/test/interface/LinearObsOperator.h index 42737726b..f1a03e46b 100644 --- a/src/test/interface/LinearObsOperator.h +++ b/src/test/interface/LinearObsOperator.h @@ -25,12 +25,15 @@ #include "oops/interface/ObsOperator.h" #include "oops/runs/Test.h" #include "oops/util/dot_product.h" +#include "oops/util/Expect.h" #include "oops/util/Logger.h" #include "test/interface/ObsTestsFixture.h" #include "test/TestEnvironment.h" namespace test { +const char *expectConstructorToThrow = "expect constructor to throw exception with message"; + // ----------------------------------------------------------------------------- /// \brief tests constructor and print method template void testConstructor() { @@ -43,12 +46,20 @@ template void testConstructor() { std::string confname = "obs operator"; if (conf.has("linear obs operator")) confname = "linear obs operator"; eckit::LocalConfiguration linobsopconf(conf, confname); - std::unique_ptr linobsop( - new LinearObsOperator_(Test_::obspace()[jj], linobsopconf)); - EXPECT(linobsop.get()); - oops::Log::test() << "Testing LinearObsOperator: " << *linobsop << std::endl; - linobsop.reset(); - EXPECT(!linobsop.get()); + + if (!Test_::config(jj).has(expectConstructorToThrow)) { + std::unique_ptr linobsop( + new LinearObsOperator_(Test_::obspace()[jj], linobsopconf)); + EXPECT(linobsop.get()); + oops::Log::test() << "Testing LinearObsOperator: " << *linobsop << std::endl; + linobsop.reset(); + EXPECT(!linobsop.get()); + } else { + // The constructor is expected to throw an exception containing the specified string. + const std::string expectedMessage = Test_::config(jj).getString(expectConstructorToThrow); + EXPECT_THROWS_MSG(LinearObsOperator_(Test_::obspace()[jj], linobsopconf), + expectedMessage.c_str()); + } } } @@ -70,6 +81,9 @@ template void testLinearity() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { const eckit::LocalConfiguration & conf = Test_::config(jj); + if (conf.has(expectConstructorToThrow)) + continue; + // initialize observation operator (set variables requested from the model, // variables simulated by the observation operator, other init) eckit::LocalConfiguration obsopconf(conf, "obs operator"); @@ -147,6 +161,9 @@ template void testAdjoint() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { const eckit::LocalConfiguration & conf = Test_::config(jj); + if (conf.has(expectConstructorToThrow)) + continue; + // initialize observation operator (set variables requested from the model, // variables simulated by the observation operator, other init) eckit::LocalConfiguration obsopconf(conf, "obs operator"); @@ -228,6 +245,9 @@ template void testTangentLinear() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { const eckit::LocalConfiguration & conf = Test_::config(jj); + if (conf.has(expectConstructorToThrow)) + continue; + // initialize observation operator (set variables requested from the model, // variables simulated by the observation operator, other init) eckit::LocalConfiguration obsopconf(conf, "obs operator"); diff --git a/src/test/interface/ObsOperator.h b/src/test/interface/ObsOperator.h index 411d20ea7..2d6d02ef4 100644 --- a/src/test/interface/ObsOperator.h +++ b/src/test/interface/ObsOperator.h @@ -70,7 +70,7 @@ template void testSimulateObs() { for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { const eckit::LocalConfiguration & conf = Test_::config(jj); - if (Test_::config(jj).has(expectConstructorToThrow)) + if (conf.has(expectConstructorToThrow)) continue; // initialize observation operator (set variables requested from the model, From 24596a2da2c24cd708e38d123cb583a3dea1c463 Mon Sep 17 00:00:00 2001 From: Claude Gibert Date: Wed, 5 May 2021 19:46:35 +0100 Subject: [PATCH 118/142] Feature/hofx4d ewok (#1197) * have the background substituted at the last minute * needed quotes * changed templates * renamed the file for 4d Co-authored-by: Anna Shlyaeva --- ewok/hofx3d.yaml | 3 ++- ewok/{hofx4dhack.yaml => hofx4d.yaml} | 9 ++------- 2 files changed, 4 insertions(+), 8 deletions(-) rename ewok/{hofx4dhack.yaml => hofx4d.yaml} (50%) diff --git a/ewok/hofx3d.yaml b/ewok/hofx3d.yaml index a160804b0..824a56a45 100644 --- a/ewok/hofx3d.yaml +++ b/ewok/hofx3d.yaml @@ -1,8 +1,9 @@ geometry: $(GEOMETRY) state: - $(BACKGROUND) + '{{BACKGROUND}}' observations: $(OBSERVATIONS) window begin: '{{window_begin}}' window length: $(window_length) + diff --git a/ewok/hofx4dhack.yaml b/ewok/hofx4d.yaml similarity index 50% rename from ewok/hofx4dhack.yaml rename to ewok/hofx4d.yaml index 09e926cd3..d688f3f92 100644 --- a/ewok/hofx4dhack.yaml +++ b/ewok/hofx4d.yaml @@ -1,14 +1,9 @@ geometry: $(GEOMETRY) states: - - $(BACKGROUND) - - $(BACKGROUND) - - $(BACKGROUND) - - $(BACKGROUND) - - $(BACKGROUND) - - $(BACKGROUND) - - $(BACKGROUND) + '{{BACKGROUND}}' observations: $(OBSERVATIONS) window begin: '{{window_begin}}' window length: $(window_length) + From df25cf742bf70b81062671951dda81f6a1e2ccbc Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Thu, 6 May 2021 12:31:22 -0600 Subject: [PATCH 119/142] refine ObsVector::packEigen test (#1201) * refine packEigen test of ObsVector * add comments before some checks --- l95/test/testinput/interfaces.yaml | 2 +- qg/test/testinput/interfaces.yaml | 6 +++--- src/test/interface/ObsVector.h | 34 +++++++++++++++++++----------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/l95/test/testinput/interfaces.yaml b/l95/test/testinput/interfaces.yaml index 73696e20c..d5812664f 100644 --- a/l95/test/testinput/interfaces.yaml +++ b/l95/test/testinput/interfaces.yaml @@ -123,7 +123,7 @@ observations: lats: [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] localization reduces values: true rms ref: 8.3207407741318846 - reference nobs: 160 + reference global nobs: 160 tolerance: 1.0e-10 obs iterator test: tolerance: 0.0 diff --git a/qg/test/testinput/interfaces.yaml b/qg/test/testinput/interfaces.yaml index 684ea2074..664a5594b 100644 --- a/qg/test/testinput/interfaces.yaml +++ b/qg/test/testinput/interfaces.yaml @@ -74,7 +74,7 @@ observations: norm: 10.0 relative tolerance: 0.0 rms ref: 183502589.5028424 - reference nobs: 800 + reference global nobs: 800 tolerance: 1.0e-8 obs localization: localization method: Heaviside @@ -117,7 +117,7 @@ observations: norm: 10.0 relative tolerance: 0.0 rms ref: 39.644266100943696 - reference nobs: 800 + reference global nobs: 800 tolerance: 1.0e-8 obs localization: localization method: Heaviside @@ -153,7 +153,7 @@ observations: tolerance AD: 1.0e-10 tolerance TL: 1.0e-6 rms ref: 58.474969231121605 - reference nobs: 400 + reference global nobs: 400 tolerance: 1.0e-8 obs localization: localization method: Heaviside diff --git a/src/test/interface/ObsVector.h b/src/test/interface/ObsVector.h index 9ab9c93f1..50a552d67 100644 --- a/src/test/interface/ObsVector.h +++ b/src/test/interface/ObsVector.h @@ -162,7 +162,8 @@ template void testReadWrite() { } } // ----------------------------------------------------------------------------- -/// \brief Tests ObsVector::mask method. +/// \brief Tests ObsVector::mask, ObsVector::packEigen and +/// ObsVector::packEigenSize methods. /// \details Tests that: /// - mask of all zeros (nothing to mask) applied to ObsVector doesn't change /// its size and content; @@ -170,13 +171,15 @@ template void testReadWrite() { /// from the file applied to ObsVector changes its size. /// - linear algebra operations with ObsVector that were masked out produce /// ObsVectors that have the same number of obs masked out. +/// - size returned by packEigenSize is consistent with size of Eigen Vector +/// returned by packEigen, and is the same as reference value for each MPI +/// task. template void testMask() { typedef ObsTestsFixture Test_; typedef oops::ObsDataVector ObsDataVector_; typedef oops::ObsSpace ObsSpace_; typedef oops::ObsVector ObsVector_; - const double tolerance = 1.0e-8; for (std::size_t jj = 0; jj < Test_::obspace().size(); ++jj) { const ObsSpace_ & obspace = Test_::obspace()[jj]; @@ -184,7 +187,7 @@ template void testMask() { reference.random(); oops::Log::test() << "ObsVector before masking: " << reference << std::endl; - const size_t nobs_all = Test_::config(jj).getInt("reference nobs"); + const size_t nobs_all = Test_::config(jj).getInt("reference global nobs"); EXPECT_EQUAL(reference.nobs(), nobs_all); EXPECT(nobs_all > 0); @@ -200,18 +203,23 @@ template void testMask() { /// apply non-empty mask, check number of observations std::string maskvarname; - size_t nobs_after_mask; + /// by default (else statement below), apply mask that masks out everything + size_t nobs_after_mask = 0; + std::vector nobs_after_mask_local(Test_::comm().size(), 0); /// if mask variable is available, apply mask from file and read reference number of masked obs if (Test_::config(jj).has("mask variable")) { maskvarname = Test_::config(jj).getString("mask variable"); - nobs_after_mask = Test_::config(jj).getInt("reference masked nobs"); + nobs_after_mask = Test_::config(jj).getUnsigned("reference global masked nobs"); + nobs_after_mask_local = Test_::config(jj).getUnsignedVector("reference local masked nobs"); + // check that specified mask masks out something EXPECT_NOT_EQUAL(nobs_after_mask, nobs_all); + // check that "reference local masked nobs" are defined for all MPI tasks + EXPECT_EQUAL(Test_::comm().size(), nobs_after_mask_local.size()); /// if mask variable is unavailable, apply mask with all ones } else { // Hack for mask with ones: use ObsVector set to ones, write to ObsSpace, // then read as ObsDataVector. maskvarname = "set_mask"; - nobs_after_mask = 0; ObsVector_ tmp(obspace); tmp.ones(); tmp.save(maskvarname); @@ -281,13 +289,15 @@ template void testMask() { EXPECT_EQUAL(with_mask.nobs(), nobs_after_mask); /// test packEigen + test.random(); Eigen::VectorXd with_mask_vec = test.packEigen(mask); - EXPECT(with_mask_vec.size() == test.packEigenSize(mask)); - if (with_mask_vec.size() > 0) { - double rms1 = with_mask.rms(); - double rms2 = sqrt(with_mask_vec.squaredNorm() / with_mask_vec.size()); - EXPECT(std::abs(rms1-rms2) < tolerance); - } + // check that the size of returned Eigen Vector is consistent with size + // returned by packEigenSize() + EXPECT_EQUAL(with_mask_vec.size(), test.packEigenSize(mask)); + oops::Log::debug() << "Local number of masked observations is: " << + with_mask_vec.size() << std::endl; + // check that the size is consistent with reference for this MPI task + EXPECT_EQUAL(with_mask_vec.size(), nobs_after_mask_local[Test_::comm().rank()]); } } // ----------------------------------------------------------------------------- From d7cab89d3d23bab237e6b9bd8cfcd1735a6b000e Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Fri, 7 May 2021 12:58:38 -0600 Subject: [PATCH 120/142] link cdash in CI (#1054) * cdash for oops codebuild intel * bugfix * upload to cdash from codebuild intel * bugfix * debugging * debugging... * use git-credential-helper in codebuild intel * debug * remove ctest update * modify DartConfiguration.tcl * exclude saddlepoint test * determine cdash url * bugfix * bugfix * debug * debug * bugfix * upload artifact * bugfix * bugfix * bugfix * complete artificats * bugfix * test * test * test * test * changing gnu * changing gnu * debug * bugfix * add clang * bugfix * debug * debug * bugfix * bugfix * bugfix * bugfix * bugfix * bugfix * bugfix * bugfix * bugfix * use make * clean up * Update CI/cmake/CTestConfig.cmake Co-authored-by: Ryan Honeyager * Update CI/CMakeLists.txt Co-authored-by: Ryan Honeyager * Update CI/CMakeLists.txt Co-authored-by: Ryan Honeyager * Update CI/CMakeLists.txt Co-authored-by: Ryan Honeyager * bugfix * bugfix * clean up * bugfix for intel * artificial bugs in test references * continue on failure * upload to cdash in finally stage * trigger tests * revert intentional errors in tests * set pipefail codebuild clang * use PIPESTATUS * try building with ctest again and check build time * using ctest to build increase the build time. move back to using make * final clean up Co-authored-by: Ryan Honeyager Co-authored-by: Anna Shlyaeva --- CI/CMakeLists.txt | 24 +++--- CI/buildspec_clang.yml | 36 +++++++-- CI/buildspec_gnu.yml | 115 +++++++++++++++------------- CI/buildspec_intel.yml | 77 ++++++++++++++----- CI/cdash-url.sh | 8 ++ CI/cmake/CTestConfig.cmake | 9 +++ CI/cmake/CTestCustom.ctest.in | 9 +++ CI/cmake/cdash-integration.cmake | 126 +++++++++++++++++++++++++++++++ 8 files changed, 315 insertions(+), 89 deletions(-) create mode 100755 CI/cdash-url.sh create mode 100644 CI/cmake/CTestConfig.cmake create mode 100644 CI/cmake/CTestCustom.ctest.in create mode 100644 CI/cmake/cdash-integration.cmake diff --git a/CI/CMakeLists.txt b/CI/CMakeLists.txt index 63e5e718d..d04b64bdb 100644 --- a/CI/CMakeLists.txt +++ b/CI/CMakeLists.txt @@ -1,4 +1,4 @@ - + # (C) Copyright 2017 UCAR # # This software is licensed under the terms of the Apache Licence Version 2.0 @@ -8,25 +8,23 @@ # OOPS bundle # -project( oops-bundle C CXX Fortran ) +cmake_minimum_required( VERSION 3.12 FATAL_ERROR ) -cmake_minimum_required( VERSION 3.3.2 FATAL_ERROR ) +project( oops-bundle VERSION 1.0.0 LANGUAGES C CXX Fortran ) -set( CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake;${CMAKE_CURRENT_SOURCE_DIR}/ecbuild/cmake;${CMAKE_MODULE_PATH}") +find_package(ecbuild REQUIRED) +list( APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) include( ecbuild_bundle ) set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" ) ecbuild_bundle_initialize() -ecbuild_requires_macro_version( 2.7 ) - -ecbuild_add_option( FEATURE OMP - DEFAULT ON - DESCRIPTION "Use OpenMP" ) - -ecbuild_bundle( PROJECT fckit GIT "https://github.com/JCSDA-internal/fckit.git" ) -ecbuild_bundle( PROJECT atlas GIT "https://github.com/JCSDA-internal/atlas.git" ) -ecbuild_bundle( PROJECT oops GIT "https://github.com/JCSDA-internal/oops.git" ) +ecbuild_bundle( PROJECT fckit GIT "https://github.com/jcsda-internal/fckit.git" ) +ecbuild_bundle( PROJECT atlas GIT "https://github.com/jcsda-internal/atlas.git" ) +ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" ) ecbuild_bundle_finalize() + +include(cmake/cdash-integration.cmake) +include(CTest) diff --git a/CI/buildspec_clang.yml b/CI/buildspec_clang.yml index b43f2a7d7..48692b3e3 100644 --- a/CI/buildspec_clang.yml +++ b/CI/buildspec_clang.yml @@ -1,6 +1,7 @@ version: 0.2 env: + shell: bash parameter-store: GIT_USER: "/CodeBuild/Git_USER" GIT_PASS: "/CodeBuild/Git_PASS" @@ -21,6 +22,10 @@ phases: - echo $CODEBUILD_WEBHOOK_TRIGGER - echo $CODEBUILD_WEBHOOK_BASE_REF + - gh_source=$(pwd) + - echo ${gh_source} + - echo $CODEBUILD_SRC_DIR + # Codebuild only runs on PUSH events if HEAD_REF # is refs/heads/develop (merge to develop). In this # case CODEBUILD_GIT_BRANCH="develop" @@ -35,9 +40,9 @@ phases: - echo "CODEBUILD_SOURCE_VERSION=${CODEBUILD_SOURCE_VERSION}" - echo "check for same branch names, except for develop" - - if ! [ "$CODEBUILD_GIT_BRANCH" = "develop" ]; then + - if ! [ "$CODEBUILD_GIT_BRANCH" = "develop" ]; then pip install boto3; - pip install pygithub; + pip install pygithub; python CI/update_webhook_branchname.py $CODEBUILD_GIT_BRANCH $CODEBUILD_RESOLVED_SOURCE_VERSION; fi @@ -54,10 +59,29 @@ phases: - cd jedi-build-package - pip install --user -e . - - ~/.local/bin/jedi-build --gh-token=$GIT_PASS -j 4 --env-id=docker-clang --branch-map atlas:release-stable fckit:release-stable -br ${CODEBUILD_GIT_BRANCH} -p oops -vvv -3 -u --submit-dashboard --site CodeBuild --abort-on-build-errors --abort-on-test-errors + - ~/.local/bin/jedi-build --gh-token=$GIT_PASS -j 4 --env-id=docker-clang --branch-map atlas:release-stable fckit:release-stable -br ${CODEBUILD_GIT_BRANCH} -p oops -vvv -3 -u --submit-dashboard --site CodeBuild --abort-on-build-errors --abort-on-test-errors --cleanup=False 2>&1 | tee jedi_build_output.txt ; test ${PIPESTATUS[0]} -eq 0 + + #find CDASH URL + - Done_path_line=$(grep 'Done.xml' jedi_build_output.txt | tail -1) + - echo $Done_path_line + - Done_path=$(echo ${Done_path_line} | sed 's/Uploaded://') + - echo $Done_path + - cat $Done_path + # retrieve buildID from Done.xml + - buildID=$(cat $Done_path | grep -o -P '(?<=buildId>).*(?= /jcsda/artifacts/cdash-url.txt + - cat /jcsda/artifacts/cdash-url.txt + - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt + - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.txt + - ls /jcsda/artifacts/ - artifacts: files: - - 'artifacts/*' - name: artifact-oops-clang + - '/jcsda/artifacts/*' + name: oops-clang-url diff --git a/CI/buildspec_gnu.yml b/CI/buildspec_gnu.yml index b49917cb6..a2ca2e5bf 100644 --- a/CI/buildspec_gnu.yml +++ b/CI/buildspec_gnu.yml @@ -37,50 +37,26 @@ phases: - echo "CODEBUILD_GIT_BRANCH=${CODEBUILD_GIT_BRANCH}" - echo "CODEBUILD_SOURCE_VERSION=${CODEBUILD_SOURCE_VERSION}" - - echo MPI setup for Docker - - mkdir -p /var/run/sshd - - ssh-keygen -A - - sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config - - sed -i 's/#RSAAuthentication yes/RSAAuthentication yes/g' /etc/ssh/sshd_config - - sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config - - - groupadd jcsda -g 9999 - - useradd jcsdauser - - mkdir -p /jcsda /build_container - - chown -R jcsdauser:jcsda /build_container /usr/local - - chmod 6755 /jcsda /build_container /usr/local - - - mkdir /jcsda/.ssh ; echo "StrictHostKeyChecking no" > /jcsda/.ssh/config - - mkdir -p /jcsda/.openmpi - - mkdir -p /jcsda/oops-bundle - - mkdir -p /home/jcsdauser/.openmpi - - - cp CI/default-mca-params.conf /home/jcsdauser/.openmpi/mca-params.conf - - cat /home/jcsdauser/.openmpi/mca-params.conf - - chown -R jcsdauser:jcsda /jcsda/ - - - su - jcsdauser -c "ssh-keygen -f /jcsda/.ssh/id_rsa -t rsa -N '' - && chmod 600 /jcsda/.ssh/config - && chmod 700 /jcsda/.ssh - && cp /jcsda/.ssh/id_rsa.pub /jcsda/.ssh/authorized_keys - && echo MPI setup for Docker done" - pre_build: commands: - echo Executing pre_build phase + - mkdir -p /jcsda/oops-bundle - git lfs install - - cp ~/.gitconfig /home/jcsdauser/ - - cp CI/CMakeLists.txt /jcsda/oops-bundle - + - cp ~/.gitconfig /home/jedi/ - cd CI - + - if [ "$CODEBUILD_GIT_BRANCH" = "develop" ]; then export CODEBUILD_GIT_BRANCH_FORK="release-stable"; else export CODEBUILD_GIT_BRANCH_FORK=${CODEBUILD_GIT_BRANCH}; echo "CODEBUILD_GIT_BRANCH_FORK=${CODEBUILD_GIT_BRANCH_FORK}"; fi + # Upload branch name and commit sha as CodeBuild artifact to S3 + - mkdir /jcsda/artifacts + - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt + - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.txt + # oops - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/oops $CODEBUILD_GIT_BRANCH oops /jcsda/oops-bundle develop # fckit @@ -88,24 +64,41 @@ phases: # atlas - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/atlas $CODEBUILD_GIT_BRANCH_FORK atlas /jcsda/oops-bundle release-stable - - cd /jcsda/oops-bundle - - ls - + + # move CMakeLists.txt from oops/CI to bundle directory + - cp CMakeLists.txt /jcsda/oops-bundle/CMakeLists.txt + + # cdash upload setup + - mkdir /jcsda/oops-bundle/cmake + - cp cmake/CTestCustom.ctest.in /jcsda/oops-bundle/cmake/ + - cp cmake/cdash-integration.cmake /jcsda/oops-bundle/cmake/ + - cp cmake/CTestConfig.cmake /jcsda/oops-bundle/ + build: + on-failure: CONTINUE commands: - echo Executing build phase - - su - jcsdauser -c "cd /build_container + - export BUILD_STATUS="0" + - echo $BUILD_STATUS + - echo $CODEBUILD_BUILD_SUCCEEDING + + # configure and build + - su - jedi -c "cd /home/jedi && ls && export FC=mpifort && export CC=mpicc && export CXX=mpicxx - && cmake -DCMAKE_MODULE_PATH=/usr/local/share/ecbuild/cmake/ -DCMAKE_BUILD_TYPE=Debug -DENABLE_GPROF=ON -DCMAKE_CXX_COMPILER=$(which g++) -DCMAKE_C_COMPILER=$(which gcc) -DCMAKE_Fortran_COMPILER=$(which gfortran) /jcsda/oops-bundle/ - && cd oops - && make -j4" + && ecbuild -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE /jcsda/oops-bundle/" - - export BUILD_STATUS="0" - - echo $BUILD_STATUS - - echo $CODEBUILD_BUILD_SUCCEEDING + - su - jedi -c "cd /home/jedi/oops + && export FC=mpifort + && export CC=mpicc + && export CXX=mpicxx + && cp ../DartConfiguration.tcl . + && sed -i 's/oops-bundle/oops-bundle\/oops/' DartConfiguration.tcl + && sed -i 's/jedi/jedi\/oops/' DartConfiguration.tcl + && cat DartConfiguration.tcl + && make -j4" - if [ "$CODEBUILD_BUILD_SUCCEEDING" = "1" ]; then export BUILD_STATUS="1"; @@ -113,26 +106,44 @@ phases: fi - echo $BUILD_STATUS - - su - jcsdauser -c "cd /build_container - && ls + # run ctest and upload to cdash + - su - jedi -c "cd /home/jedi/oops + && export FC=mpifort + && export CC=mpicc + && export CXX=mpicxx + && ctest -C RelWithDebInfo -D ExperimentalTest -E test_qg_4dvar_saddlepoint" + + finally: + - su - jedi -c "cd /home/jedi/oops && export FC=mpifort && export CC=mpicc && export CXX=mpicxx - && cd oops - && ctest" - + && ctest -C RelWithDebInfo -D ExperimentalSubmit -M Continuous -- --track Continuous --group Continuous" + post_build: commands: - echo Executing post_build phase - echo $CODEBUILD_BUILD_SUCCEEDING - - if [ "$BUILD_STATUS" = "1" ] && [ "$CODEBUILD_BUILD_SUCCEEDING" = "0" ]; - then echo "Build passed, rerun failed tests"; - su - jcsdauser -c "cd /build_container/oops - && ctest -VV --rerun-failed"; + - echo $BUILD_STATUS + + # upload find cdash url and upload it as CodeBuild artifact to S3 + - if [ "$BUILD_STATUS" = "1" ]; + then echo "Build & tests passed, find cdash url"; + bash /jcsda/oops-bundle/oops/CI/cdash-url.sh /home/jedi/oops/Testing; + url=$(bash /jcsda/oops-bundle/oops/CI/cdash-url.sh /home/jedi/oops/Testing); + echo $url; + echo ${url} > /jcsda/artifacts/cdash-url.txt; + cat /jcsda/artifacts/cdash-url.txt; else echo "Build failed"; fi + - echo 'Connect to CodeCov' - - cd /build_container/oops + - cd /home/jedi/oops - pwd - ls - bash /jcsda/oops-bundle/oops/CI/codecov_script_$org_name.sh + +artifacts: + files: + - '/jcsda/artifacts/*' + name: oops-gnu-url diff --git a/CI/buildspec_intel.yml b/CI/buildspec_intel.yml index d4a6d1523..327c65eb1 100644 --- a/CI/buildspec_intel.yml +++ b/CI/buildspec_intel.yml @@ -2,6 +2,7 @@ version: 0.2 env: shell: bash + git-credential-helper: yes parameter-store: GIT_USER: "/CodeBuild/Git_USER" GIT_PASS: "/CodeBuild/Git_PASS" @@ -46,6 +47,11 @@ phases: echo "CODEBUILD_GIT_BRANCH_FORK=${CODEBUILD_GIT_BRANCH_FORK}"; fi + # Upload branch name and commit sha as CodeBuild artifact to S3 + - mkdir -p /jcsda/artifacts + - echo ${CODEBUILD_GIT_BRANCH} > /jcsda/artifacts/branch_name.txt + - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > /jcsda/artifacts/commit_sha.txt + - cd CI - . /etc/profile.d/intel.sh @@ -57,29 +63,49 @@ phases: - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/atlas $CODEBUILD_GIT_BRANCH_FORK atlas /jcsda/oops-bundle release-stable # move CMakeLists.txt from oops/CI to bundle directory - - cp CMakeLists.txt /jcsda/oops-bundle + - cp CMakeLists.txt /jcsda/oops-bundle/CMakeLists.txt + + # cdash upload setup + - mkdir /jcsda/oops-bundle/cmake + - cp cmake/CTestCustom.ctest.in /jcsda/oops-bundle/cmake/ + - cp cmake/cdash-integration.cmake /jcsda/oops-bundle/cmake/ + - cp cmake/CTestConfig.cmake /jcsda/oops-bundle/ + - chmod 777 -R /jcsda/oops-bundle - cd /jcsda/oops-bundle - ls + - ls -ln /jcsda/oops-bundle/oops/CI build: + on-failure: CONTINUE commands: - echo Executing build phase - ## cannot source /etc/bash.bashrc so copy what's there for jscdauser + - export BUILD_STATUS="0" + - echo $BUILD_STATUS + - echo $CODEBUILD_BUILD_SUCCEEDING + + # configure and build - su - jedi -c "export CC=mpiicc && export FC=mpiifort && export CXX=mpiicpc && export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH && export PATH=/usr/local/lib:$PATH && cd /home/jedi - && ecbuild /jcsda/oops-bundle/ - && cd /home/jedi/oops - && make -j2" + && ecbuild -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE /jcsda/oops-bundle/" - - export BUILD_STATUS="0" - - echo $BUILD_STATUS - - echo $CODEBUILD_BUILD_SUCCEEDING + - su - jedi -c "export CC=mpiicc + && export FC=mpiifort + && export CXX=mpiicpc + && export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + && export PATH=/usr/local/lib:$PATH + && cd /home/jedi/oops + && ls -ln /home/jedi/oops + && cp ../DartConfiguration.tcl . + && sed -i 's/oops-bundle/oops-bundle\/oops/' DartConfiguration.tcl + && sed -i 's/jedi/jedi\/oops/' DartConfiguration.tcl + && cat DartConfiguration.tcl + && make -j4" - if [ "$CODEBUILD_BUILD_SUCCEEDING" = "1" ]; then export BUILD_STATUS="1"; @@ -87,27 +113,42 @@ phases: fi - echo $BUILD_STATUS + # run ctest and upload to cdash - su - jedi -c "export CC=mpiicc && export FC=mpiifort && export CXX=mpiicpc && export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH && export PATH=/usr/local/lib:$PATH && cd /home/jedi/oops - && ctest -E test_qg_4dvar_saddlepoint" - - post_build: - commands: - - echo Executing post_build phase - - echo $CODEBUILD_BUILD_SUCCEEDING + && ctest -C RelWithDebInfo -D ExperimentalTest -E test_qg_4dvar_saddlepoint" - - if [ "$BUILD_STATUS" = "1" ] && [ "$CODEBUILD_BUILD_SUCCEEDING" = "0" ]; - then echo "Build passed, rerun failed tests"; - su - jedi -c "export CC=mpiicc + finally: + - su - jedi -c "export CC=mpiicc && export FC=mpiifort && export CXX=mpiicpc && export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH && export PATH=/usr/local/lib:$PATH && cd /home/jedi/oops - && ctest -VV --rerun-failed"; + && ctest -C RelWithDebInfo -D ExperimentalSubmit -M Continuous -- --track Continuous --group Continuous" + + post_build: + commands: + - echo Executing post_build phase + - echo $CODEBUILD_BUILD_SUCCEEDING + + # upload find cdash url and upload it as CodeBuild artifact to S3 + + - if [ "$BUILD_STATUS" = "1" ]; + then echo "Build & tests passed, find cdash url"; + bash /jcsda/oops-bundle/oops/CI/cdash-url.sh /home/jedi/oops/Testing; + url=$(bash /jcsda/oops-bundle/oops/CI/cdash-url.sh /home/jedi/oops/Testing); + echo $url; + echo ${url} > /jcsda/artifacts/cdash-url.txt; + cat /jcsda/artifacts/cdash-url.txt; else echo "Build failed"; fi + +artifacts: + files: + - '/jcsda/artifacts/*' + name: oops-intel-url diff --git a/CI/cdash-url.sh b/CI/cdash-url.sh new file mode 100755 index 000000000..dbf9ed38c --- /dev/null +++ b/CI/cdash-url.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +dir=$1 +tag=$(head -1 $dir/TAG) +Done=$(cat $dir/$tag/Done.xml) +buildID=$(echo $Done | grep -o -P '(?<=buildId>).*(?= Date: Fri, 7 May 2021 13:17:18 -0600 Subject: [PATCH 121/142] adds additional testing for obs localization (#1194) * added optional rmse test to obs localization test * added local obs testing for ms to l95 * coding norm * changed reference tests * Update src/test/interface/ObsLocalization.h Co-authored-by: Anna Shlyaeva * added prints Co-authored-by: Anna Shlyaeva --- l95/test/testinput/interfaces.yaml | 2 +- qg/test/testinput/interfaces.yaml | 6 +++--- src/test/interface/ObsLocalization.h | 27 +++++++++++++++++---------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/l95/test/testinput/interfaces.yaml b/l95/test/testinput/interfaces.yaml index d5812664f..7a9b6872c 100644 --- a/l95/test/testinput/interfaces.yaml +++ b/l95/test/testinput/interfaces.yaml @@ -121,7 +121,7 @@ observations: reference gridpoints: lons: [ 0, 0.025, 0.05, 0.075, 0.1, 0.875, 0.9, 0.925, 0.95, 0.975 ] lats: [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - localization reduces values: true + reference rms: [0.52443,0.489544,0.52443,0.489544,0.52443,0.489544,0.52443,0.489544,0.52443,0.489544] rms ref: 8.3207407741318846 reference global nobs: 160 tolerance: 1.0e-10 diff --git a/qg/test/testinput/interfaces.yaml b/qg/test/testinput/interfaces.yaml index 664a5594b..424c8cfac 100644 --- a/qg/test/testinput/interfaces.yaml +++ b/qg/test/testinput/interfaces.yaml @@ -83,7 +83,7 @@ observations: reference gridpoints: lons: [-175.5] lats: [5.623] - localization reduces values: false + reference rms: [0] obs iterator test: tolerance: 1.0e-6 reference nlocs: 800 @@ -126,7 +126,7 @@ observations: reference gridpoints: lons: [-175.5] lats: [5.623] - localization reduces values: false + reference rms: [1] obs iterator test: tolerance: 1.0e-6 reference nlocs: 400 @@ -162,7 +162,7 @@ observations: reference gridpoints: lons: [-175.5] lats: [5.623] - localization reduces values: false + reference rms: [1] obs iterator test: tolerance: 1.0e-6 reference nlocs: 400 diff --git a/src/test/interface/ObsLocalization.h b/src/test/interface/ObsLocalization.h index 1221c7603..0d0156532 100644 --- a/src/test/interface/ObsLocalization.h +++ b/src/test/interface/ObsLocalization.h @@ -8,6 +8,7 @@ #ifndef TEST_INTERFACE_OBSLOCALIZATION_H_ #define TEST_INTERFACE_OBSLOCALIZATION_H_ +#include #include #include #include @@ -82,7 +83,10 @@ template void testObsLocalization() { for (GeometryIterator_ ii = geometry.begin(); ii != geometry.end(); ++ii) { // debug print to help decide which points to specify for reference // set OOPS_DEBUG environment variable to -1 to see prints from all MPI tasks - oops::Log::debug() << "Iterating over " << ii << ": " << *ii << std::endl; + if (locconf.getBool("print iterator", false)) { + oops::Log::debug() << "Iterating over " << std::setprecision(9) << ii << ": " + << *ii << std::endl; + } // check if we need to test at this location (if there are any points in the // reference point list within 1e-5 of this locationn) const auto & it = std::find_if(reference_points.begin(), reference_points.end(), @@ -98,7 +102,10 @@ template void testObsLocalization() { oops::Log::test() << "Localization values: " << locvector << std::endl; locvector.mask(outside); oops::Log::test() << "Local vector nobs and reference: " << locvector.nobs() << ", " - << nobs_local[index] << std::endl; + << nobs_local_ref[index] << std::endl; + oops::Log::debug() << "Local vector stats lat,lon,nobs,rms: " << *ii << ", " + << locvector.nobs() << ", " << locvector.rms() << std::endl; + // save number of local obs to be tested later nobs_local[index] = locvector.nobs(); @@ -120,14 +127,14 @@ template void testObsLocalization() { EXPECT_EQUAL(total_tested, lons.size()); // Test that computed number of local obs is the same as reference EXPECT_EQUAL(nobs_local_ref, nobs_local); - // check whether localization is expected to reduce obsvector.rms() - if (locconf.getBool("localization reduces values")) { - for (size_t jpoint = 0; jpoint < nobs_local.size(); ++jpoint) { - EXPECT(locvector_rms[jpoint] < 1.0); - } - } else { - for (size_t jpoint = 0; jpoint < nobs_local.size(); ++jpoint) { - EXPECT((nobs_local[jpoint] == 0) || (locvector_rms[jpoint] == 1.0)); + // check value of the rms is close to reference + const std::vector ref_locvector_rms = locconf.getDoubleVector("reference rms"); + ASSERT(lons.size() == ref_locvector_rms.size()); + oops::Log::debug() << "reference RMS" << ref_locvector_rms << std::endl; + oops::Log::debug() << "computed RMS" << locvector_rms << std::endl; + for (size_t jpoint = 0; jpoint < nobs_local.size(); ++jpoint) { + if (nobs_local[jpoint] > 0) { + EXPECT(std::abs(locvector_rms[jpoint]-ref_locvector_rms[jpoint]) < 1.e-5); } } } From 7b85ea98bfdcf0ef9fab2e7b89ecf6b314f048cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Fri, 7 May 2021 19:51:44 -0600 Subject: [PATCH 122/142] Simple memory usage estimate (#1196) * Simple memory usage estimate * Code reviews --- src/CMakeLists.txt | 6 +- src/oops/base/EnsembleCovariance.h | 2 + src/oops/base/ObsErrorBase.h | 1 + src/oops/interface/ErrorCovariance.h | 2 + src/oops/interface/Increment.h | 4 + src/oops/interface/ObsErrorCovariance.h | 3 +- src/oops/interface/ObsFilter.h | 2 +- src/oops/interface/ObsSpace.h | 8 +- src/oops/interface/ObsVector.h | 3 + src/oops/interface/State.h | 5 + src/oops/runs/Run.cc | 5 +- src/oops/util/MemoryCounter.cc | 35 ++++++ src/oops/util/MemoryCounter.h | 31 +++++ src/oops/util/MemoryHelper.cc | 153 ++++++++++++++++++++++++ src/oops/util/MemoryHelper.h | 40 +++++++ 15 files changed, 293 insertions(+), 7 deletions(-) create mode 100644 src/oops/util/MemoryCounter.cc create mode 100644 src/oops/util/MemoryCounter.h create mode 100644 src/oops/util/MemoryHelper.cc create mode 100644 src/oops/util/MemoryHelper.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5917bca9b..2328a2583 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -290,13 +290,17 @@ oops/util/LibOOPS.h oops/util/LocalEnvironment.cc oops/util/LocalEnvironment.h oops/util/Logger.h +oops/util/MemoryCounter.cc +oops/util/MemoryCounter.h +oops/util/MemoryHelper.cc +oops/util/MemoryHelper.h oops/util/missing_values_f.cc oops/util/missing_values_f.h oops/util/missing_values_mod.F90 -oops/util/netcdf_utils_mod.f90 oops/util/missingValues.cc oops/util/missingValues.h oops/util/NamedEnumerator.h +oops/util/netcdf_utils_mod.f90 oops/util/ObjectCounter.h oops/util/ObjectCountHelper.cc oops/util/ObjectCountHelper.h diff --git a/src/oops/base/EnsembleCovariance.h b/src/oops/base/EnsembleCovariance.h index 4f55a6de0..321406b3c 100644 --- a/src/oops/base/EnsembleCovariance.h +++ b/src/oops/base/EnsembleCovariance.h @@ -26,6 +26,7 @@ #include "oops/interface/Increment.h" #include "oops/interface/State.h" #include "oops/util/Logger.h" +#include "oops/util/MemoryCounter.h" namespace oops { @@ -67,6 +68,7 @@ EnsembleCovariance::EnsembleCovariance(const Geometry_ & resol, const Var : ModelSpaceCovarianceBase(xb, fg, resol, conf), ens_(), loc_() { Log::trace() << "EnsembleCovariance::EnsembleCovariance start" << std::endl; + util::MemoryCounter mem("oops::EnsembleCovariance"); ens_.reset(new Ensemble_(conf, xb, fg, resol, vars)); if (conf.has("localization")) { const eckit::LocalConfiguration confloc(conf, "localization"); diff --git a/src/oops/base/ObsErrorBase.h b/src/oops/base/ObsErrorBase.h index d63417619..7963fbd3d 100644 --- a/src/oops/base/ObsErrorBase.h +++ b/src/oops/base/ObsErrorBase.h @@ -19,6 +19,7 @@ #include "eckit/config/Configuration.h" #include "oops/interface/ObsSpace.h" +#include "oops/interface/ObsVector.h" #include "oops/util/abor1_cpp.h" #include "oops/util/Logger.h" #include "oops/util/Printable.h" diff --git a/src/oops/interface/ErrorCovariance.h b/src/oops/interface/ErrorCovariance.h index 3fff64358..a828cab09 100644 --- a/src/oops/interface/ErrorCovariance.h +++ b/src/oops/interface/ErrorCovariance.h @@ -22,6 +22,7 @@ #include "oops/interface/Increment.h" #include "oops/interface/State.h" #include "oops/util/Logger.h" +#include "oops/util/MemoryCounter.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" #include "oops/util/Timer.h" @@ -87,6 +88,7 @@ ErrorCovariance::ErrorCovariance(const Geometry_ & resol, const Variables { Log::trace() << "ErrorCovariance::ErrorCovariance starting" << std::endl; util::Timer timer(classname(), "ErrorCovariance"); + util::MemoryCounter mem(classname()); covariance_.reset(new Covariance_(resol.geometry(), vars, parametersOrConfiguration::value>( parameters), diff --git a/src/oops/interface/Increment.h b/src/oops/interface/Increment.h index d52443ffc..b65212b85 100644 --- a/src/oops/interface/Increment.h +++ b/src/oops/interface/Increment.h @@ -28,6 +28,7 @@ #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/gatherPrint.h" +#include "oops/util/MemoryCounter.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Serializable.h" #include "oops/util/Timer.h" @@ -142,6 +143,7 @@ Increment::Increment(const Geometry_ & resol, const Variables & vars, { Log::trace() << "Increment::Increment starting" << std::endl; util::Timer timer(classname(), "Increment"); + util::MemoryCounter mem(classname()); increment_.reset(new Increment_(resol.geometry(), vars, time)); Log::trace() << "Increment::Increment done" << std::endl; } @@ -154,6 +156,7 @@ Increment::Increment(const Geometry_ & resol, const Increment & other) { Log::trace() << "Increment::Increment starting" << std::endl; util::Timer timer(classname(), "Increment"); + util::MemoryCounter mem(classname()); increment_.reset(new Increment_(resol.geometry(), *other.increment_)); Log::trace() << "Increment::Increment done" << std::endl; } @@ -166,6 +169,7 @@ Increment::Increment(const Increment & other, const bool copy) { Log::trace() << "Increment::Increment copy starting" << std::endl; util::Timer timer(classname(), "Increment"); + util::MemoryCounter mem(classname()); increment_.reset(new Increment_(*other.increment_, copy)); Log::trace() << "Increment::Increment copy done" << std::endl; } diff --git a/src/oops/interface/ObsErrorCovariance.h b/src/oops/interface/ObsErrorCovariance.h index 619f8b1fe..ccaf4686e 100644 --- a/src/oops/interface/ObsErrorCovariance.h +++ b/src/oops/interface/ObsErrorCovariance.h @@ -14,10 +14,10 @@ #include #include - #include "oops/base/ObsErrorBase.h" #include "oops/interface/ObsSpace.h" #include "oops/interface/ObsVector.h" +#include "oops/util/MemoryCounter.h" namespace eckit { class Configuration; @@ -66,6 +66,7 @@ ObsErrorCovariance::ObsErrorCovariance(const eckit::Configuration & const Variables & obsvar) : covar_() { Log::trace() << "ObsErrorCovariance::ObsErrorCovariance starting" << std::endl; util::Timer timer(classname(), "ObsErrorCovariance"); + util::MemoryCounter mem(classname()); covar_.reset(new OBSERR(conf, obsdb, obsvar)); Log::trace() << "ObsErrorCovariance::ObsErrorCovariance done" << std::endl; } diff --git a/src/oops/interface/ObsFilter.h b/src/oops/interface/ObsFilter.h index 4c7f7e023..114c310c5 100644 --- a/src/oops/interface/ObsFilter.h +++ b/src/oops/interface/ObsFilter.h @@ -75,7 +75,7 @@ class ObsFilter : public ObsFilterBase { private: void print(std::ostream &) const override; - ObsSpace_ obsdb_; + const ObsSpace_ & obsdb_; const std::unique_ptr parameters_; std::unique_ptr ofilt_; }; diff --git a/src/oops/interface/ObsSpace.h b/src/oops/interface/ObsSpace.h index 08910429f..064aa53e2 100644 --- a/src/oops/interface/ObsSpace.h +++ b/src/oops/interface/ObsSpace.h @@ -21,6 +21,7 @@ #include "oops/interface/GeometryIterator.h" #include "oops/mpi/mpi.h" #include "oops/util/Logger.h" +#include "oops/util/MemoryCounter.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" #include "oops/util/Timer.h" @@ -34,7 +35,6 @@ namespace util { } namespace oops { - template class ObsVector; // ----------------------------------------------------------------------------- @@ -42,7 +42,6 @@ template class ObsSpace : public util::Printable, private util::ObjectCounter > { typedef typename OBS::ObsSpace ObsSpace_; - typedef ObsVector ObsVector_; typedef GeometryIterator ObsIterator_; public: @@ -53,6 +52,8 @@ class ObsSpace : public util::Printable, const eckit::mpi::Comm & time = oops::mpi::myself()); ~ObsSpace(); + ObsSpace(const ObsSpace &) = delete; + /// Interfacing ObsSpace_ & obsspace() const {return *obsdb_;} // const problem? YT @@ -75,7 +76,7 @@ class ObsSpace : public util::Printable, private: void print(std::ostream &) const; - std::shared_ptr obsdb_; + std::unique_ptr obsdb_; const eckit::mpi::Comm & time_; }; @@ -89,6 +90,7 @@ ObsSpace::ObsSpace(const eckit::Configuration & conf, const eckit::mpi::Comm & time) : obsdb_(), time_(time) { Log::trace() << "ObsSpace::ObsSpace starting" << std::endl; util::Timer timer(classname(), "ObsSpace"); + util::MemoryCounter mem(classname()); obsdb_.reset(new ObsSpace_(conf, comm, bgn, end, time)); Log::trace() << "ObsSpace::ObsSpace done" << std::endl; } diff --git a/src/oops/interface/ObsVector.h b/src/oops/interface/ObsVector.h index 8f0e40734..30130e1f0 100644 --- a/src/oops/interface/ObsVector.h +++ b/src/oops/interface/ObsVector.h @@ -21,6 +21,7 @@ #include "oops/interface/ObsSpace.h" #include "oops/util/gatherPrint.h" #include "oops/util/Logger.h" +#include "oops/util/MemoryCounter.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" #include "oops/util/Timer.h" @@ -101,6 +102,7 @@ ObsVector::ObsVector(const ObsSpace & os, const std::string name) : data_(), commTime_(os.timeComm()) { Log::trace() << "ObsVector::ObsVector starting " << name << std::endl; util::Timer timer(classname(), "ObsVector"); + util::MemoryCounter mem(classname()); data_.reset(new ObsVector_(os.obsspace(), name)); @@ -111,6 +113,7 @@ template ObsVector::ObsVector(const ObsVector & other): data_(), commTime_(other.commTime_) { Log::trace() << "ObsVector::ObsVector starting" << std::endl; util::Timer timer(classname(), "ObsVector"); + util::MemoryCounter mem(classname()); data_.reset(new ObsVector_(*other.data_)); diff --git a/src/oops/interface/State.h b/src/oops/interface/State.h index b9655071a..3c2cac8e8 100644 --- a/src/oops/interface/State.h +++ b/src/oops/interface/State.h @@ -22,6 +22,7 @@ #include "oops/interface/Geometry.h" #include "oops/util/DateTime.h" #include "oops/util/gatherPrint.h" +#include "oops/util/MemoryCounter.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" #include "oops/util/Serializable.h" @@ -89,6 +90,7 @@ State::State(const Geometry_ & resol, const Variables & vars, { Log::trace() << "State::State starting" << std::endl; util::Timer timer(classname(), "State"); + util::MemoryCounter mem(classname()); state_.reset(new State_(resol.geometry(), vars, time)); Log::trace() << "State::State done" << std::endl; } @@ -101,6 +103,7 @@ State::State(const Geometry_ & resol, const eckit::Configuration & conf) { Log::trace() << "State::State read starting" << std::endl; util::Timer timer(classname(), "State"); + util::MemoryCounter mem(classname()); eckit::LocalConfiguration myconf; if (conf.has("states")) { @@ -126,6 +129,7 @@ State::State(const Geometry_ & resol, const State & other) { Log::trace() << "State::State interpolated starting" << std::endl; util::Timer timer(classname(), "State"); + util::MemoryCounter mem(classname()); state_.reset(new State_(resol.geometry(), *other.state_)); Log::trace() << "State::State interpolated done" << std::endl; } @@ -137,6 +141,7 @@ State::State(const State & other) : state_(), commTime_(other.commTime_) { Log::trace() << "State::State starting copy" << std::endl; util::Timer timer(classname(), "State"); + util::MemoryCounter mem(classname()); state_.reset(new State_(*other.state_)); Log::trace() << "State::State copy done" << std::endl; } diff --git a/src/oops/runs/Run.cc b/src/oops/runs/Run.cc index 38d0b2f6e..1d82798fa 100644 --- a/src/oops/runs/Run.cc +++ b/src/oops/runs/Run.cc @@ -19,6 +19,7 @@ #include "oops/runs/Application.h" #include "oops/util/LibOOPS.h" #include "oops/util/Logger.h" +#include "oops/util/MemoryHelper.h" #include "oops/util/ObjectCountHelper.h" #include "oops/util/TimerHelper.h" @@ -100,8 +101,9 @@ Run::Run(int argc, char** argv) : eckit::Main(argc, argv, "OOPS_HOME"), config_( Log::info() << "Full configuration is:" << *config_ << std::endl; // Start measuring performance - util::ObjectCountHelper::start(); + util::MemoryHelper::start(); util::TimerHelper::start(); + util::ObjectCountHelper::start(); } // ----------------------------------------------------------------------------- @@ -138,6 +140,7 @@ int Run::execute(const Application & app) { // Performance diagnostics util::ObjectCountHelper::stop(); util::TimerHelper::stop(); + util::MemoryHelper::stop(); Log::info() << "Run: Finishing " << app << " with status = " << status << std::endl; return status; diff --git a/src/oops/util/MemoryCounter.cc b/src/oops/util/MemoryCounter.cc new file mode 100644 index 000000000..edfa61c8e --- /dev/null +++ b/src/oops/util/MemoryCounter.cc @@ -0,0 +1,35 @@ +/* + * (C) Copyright 2021-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/util/MemoryCounter.h" + +#include + +#include "eckit/system/ResourceUsage.h" +#include "oops/util/MemoryHelper.h" + +namespace util { + +// ----------------------------------------------------------------------------- + +MemoryCounter::MemoryCounter(const std::string & classname) + : name_(classname), rss_(eckit::system::ResourceUsage().maxResidentSetSize()) +{} + +// ----------------------------------------------------------------------------- + +MemoryCounter::~MemoryCounter() { + size_t current = eckit::system::ResourceUsage().maxResidentSetSize(); +// convert bytes to MiB + double mem = (static_cast(current)-static_cast(rss_))/1048576.0; + MemoryHelper::add(name_, mem); +} + +// ----------------------------------------------------------------------------- + +} // namespace util + diff --git a/src/oops/util/MemoryCounter.h b/src/oops/util/MemoryCounter.h new file mode 100644 index 000000000..92b71be29 --- /dev/null +++ b/src/oops/util/MemoryCounter.h @@ -0,0 +1,31 @@ +/* + * (C) Copyright 2021-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include + +namespace util { + +// ----------------------------------------------------------------------------- + +class MemoryCounter { + public: + explicit MemoryCounter(const std::string &); + ~MemoryCounter(); + + MemoryCounter(const MemoryCounter&) = delete; + + private: + std::string name_; + size_t rss_; +}; + +// ----------------------------------------------------------------------------- + +} // namespace util + diff --git a/src/oops/util/MemoryHelper.cc b/src/oops/util/MemoryHelper.cc new file mode 100644 index 000000000..6c64789d8 --- /dev/null +++ b/src/oops/util/MemoryHelper.cc @@ -0,0 +1,153 @@ +/* + * (C) Copyright 2021-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/util/MemoryHelper.h" + +#include +#include +#include + +#include "eckit/io/ResizableBuffer.h" +#include "eckit/mpi/Comm.h" +#include "eckit/serialisation/ResizableMemoryStream.h" +#include "eckit/system/ResourceUsage.h" + +#include "oops/mpi/mpi.h" +#include "oops/util/Logger.h" +#include "oops/util/MemoryCounter.h" + +namespace util { + +// ----------------------------------------------------------------------------- + +MemoryHelper & MemoryHelper::getHelper() { + static MemoryHelper theHelper; + return theHelper; +} + +// ----------------------------------------------------------------------------- + +void MemoryHelper::start() { + getHelper().on_ = true; +} + +// ----------------------------------------------------------------------------- + +void MemoryHelper::stop() { + oops::Log::stats() << getHelper() << std::endl; + getHelper().stats_.clear(); + getHelper().on_ = false; +} + +// ----------------------------------------------------------------------------- + +void MemoryHelper::add(const std::string & name, const double rss) { + if (getHelper().on_) { + getHelper().stats_[name][0] += 1.0; + getHelper().stats_[name][1] += rss; + if (rss > getHelper().stats_[name][2]) getHelper().stats_[name][2] = rss; + } +} + +// ----------------------------------------------------------------------------- + +MemoryHelper::MemoryHelper(): on_(false), stats_() {} + +// ----------------------------------------------------------------------------- + +MemoryHelper::~MemoryHelper() {} + +// ----------------------------------------------------------------------------- + +void MemoryHelper::print(std::ostream & os) const { + size_t ntasks = oops::mpi::world().size(); + int tag = 1235; + size_t rssbyte = eckit::system::ResourceUsage().maxResidentSetSize(); + double rss = static_cast(rssbyte)/(1024*1024); + + if (oops::mpi::world().rank() > 0) { +// Tasks send their numbers to task 0 + eckit::ResizableBuffer bufr(8000); + eckit::ResizableMemoryStream sstr(bufr); + sstr << stats_.size(); + for (auto jt = stats_.begin(); jt != stats_.end(); ++jt) { + sstr << jt->first << jt->second[0] << jt->second[1] << jt->second[2]; + } + sstr << rss; + oops::mpi::world().send(static_cast(bufr.data()), sstr.position(), 0, tag); + } else { +// Task 0 receives stats from other tasks + std::map> stats = stats_; + double rssmin = rss; + double rssmax = rss; + double rsstot = rss; + for (size_t from = 1; from < ntasks; ++from) { + eckit::mpi::Status st = oops::mpi::world().probe(from, tag); + size_t size = oops::mpi::world().getCount(st); + eckit::ResizableBuffer bufr(size); + bufr.zero(); + + oops::mpi::world().receive(static_cast(bufr.data()), bufr.size(), from, tag); + eckit::ResizableMemoryStream sstr(bufr); + + std::string name; + double count; + double sum; + double max; + size_t nstats; + sstr >> nstats; + for (size_t jj = 0; jj < nstats; ++jj) { + sstr >> name; + sstr >> count; + sstr >> sum; + sstr >> max; + stats[name][0] += count; + stats[name][1] += sum; + if (max > stats[name][2]) stats[name][2] = max; + } + sstr >> rss; + if (rss < rssmin) rssmin = rss; + if (rss > rssmax) rssmax = rss; + rsstot += rss; + } + +// Print global statistics + int table_width = 64; + std::ostringstream title_s; + title_s << " Memory Statistics (" << std::setw(5) << ntasks << " MPI tasks) "; + std::string title = title_s.str(); + float title_half_width = (table_width-title.size())/2.; + os << std::endl << std::string(table_width, '-') << std::endl + << std::string(std::floor(title_half_width), '-') + << title << std::string(std::ceil(title_half_width), '-') << std::endl + << std::string(table_width, '-') << std::endl + << std::setw(30) << std::left << " " + << std::setw(11) << std::right << "Count/task" + << std::setw(11) << std::right << "Avg (MiB)" + << std::setw(12) << std::right << "Max (MiB)" + << std::endl; + for (auto jt = stats.begin(); jt != stats.end(); ++jt) { + os << std::setw(30) << std::left << jt->first << ": " + << std::setw(12) << std::right << std::fixed << std::setprecision(0) + << std::setw(8) << jt->second[0]/ntasks + << std::setw(12) << std::right << std::fixed << std::setprecision(2) + << std::setw(12) << jt->second[1]/jt->second[0] + << std::setw(12) << jt->second[2] + << std::endl; + } + os << std::string(table_width, '-') << std::endl; + + os << "Total memory used per MPI task: min = " << std::setprecision(0) + << rssmin << " MiB, max = " << rssmax << " MiB." << std::endl; + os << "Total memory used: " << std::setprecision(0) << rsstot << " MiB." << std::endl; + os << std::string(table_width, '-') << std::endl; + } +} + +// ----------------------------------------------------------------------------- + +} // namespace util diff --git a/src/oops/util/MemoryHelper.h b/src/oops/util/MemoryHelper.h new file mode 100644 index 000000000..aa27021c1 --- /dev/null +++ b/src/oops/util/MemoryHelper.h @@ -0,0 +1,40 @@ +/* + * (C) Copyright 2021-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#pragma once + +#include +#include +#include + +#include "oops/util/Printable.h" + +namespace util { + +// ----------------------------------------------------------------------------- + +class MemoryHelper : public util::Printable { + public: + static void start(); + static void stop(); + static void add(const std::string &, const double); + MemoryHelper(const MemoryHelper&) = delete; + ~MemoryHelper(); + + private: + static MemoryHelper & getHelper(); + MemoryHelper(); + void print(std::ostream &) const; + + bool on_; + std::map> stats_; +}; + +// ----------------------------------------------------------------------------- + +} // namespace util + From faa2213abb4b08f274570da331fdc8f911763066 Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Mon, 10 May 2021 19:02:05 -0600 Subject: [PATCH 123/142] bugfix to upload test coverage report (#1206) * add -DENABLE_GPROF=ON to ecbuild * change cdash url to point to test results --- CI/buildspec_gnu.yml | 2 +- CI/cdash-url.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CI/buildspec_gnu.yml b/CI/buildspec_gnu.yml index a2ca2e5bf..7222ac41b 100644 --- a/CI/buildspec_gnu.yml +++ b/CI/buildspec_gnu.yml @@ -88,7 +88,7 @@ phases: && export FC=mpifort && export CC=mpicc && export CXX=mpicxx - && ecbuild -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE /jcsda/oops-bundle/" + && ecbuild -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCDASH_OVERRIDE_GIT_BRANCH=$CODEBUILD_GIT_BRANCH -DCTEST_UPDATE_VERSION_ONLY=FALSE -DENABLE_GPROF=ON /jcsda/oops-bundle/" - su - jedi -c "cd /home/jedi/oops && export FC=mpifort diff --git a/CI/cdash-url.sh b/CI/cdash-url.sh index dbf9ed38c..ba6b9a621 100755 --- a/CI/cdash-url.sh +++ b/CI/cdash-url.sh @@ -4,5 +4,5 @@ dir=$1 tag=$(head -1 $dir/TAG) Done=$(cat $dir/$tag/Done.xml) buildID=$(echo $Done | grep -o -P '(?<=buildId>).*(?= Date: Wed, 12 May 2021 11:42:56 -0600 Subject: [PATCH 124/142] Add hybrid gain application (#1207) * first commit * add ref files. fix compile bug * codingnorms * change year --- qg/mains/CMakeLists.txt | 5 ++ qg/mains/qgHybridGain.cc | 15 +++++ qg/test/CMakeLists.txt | 8 +++ qg/test/testinput/hybridgain.yaml | 37 +++++++++++ qg/test/testoutput/hybridgain.test | 50 +++++++++++++++ src/oops/runs/HybridGain.h | 100 +++++++++++++++++++++++++++++ 6 files changed, 215 insertions(+) create mode 100644 qg/mains/qgHybridGain.cc create mode 100644 qg/test/testinput/hybridgain.yaml create mode 100644 qg/test/testoutput/hybridgain.test create mode 100644 src/oops/runs/HybridGain.h diff --git a/qg/mains/CMakeLists.txt b/qg/mains/CMakeLists.txt index 3d5eb45be..67f925228 100644 --- a/qg/mains/CMakeLists.txt +++ b/qg/mains/CMakeLists.txt @@ -88,6 +88,11 @@ ecbuild_add_executable( TARGET qg_ens_recenter.x LIBS qg ) +ecbuild_add_executable( TARGET qg_hybridgain.x + SOURCES qgHybridGain.cc + LIBS qg + ) + ecbuild_add_executable( TARGET qg_rtpp.x SOURCES qgRTPP.cc LIBS qg diff --git a/qg/mains/qgHybridGain.cc b/qg/mains/qgHybridGain.cc new file mode 100644 index 000000000..6e54693a7 --- /dev/null +++ b/qg/mains/qgHybridGain.cc @@ -0,0 +1,15 @@ +/* + * (C) Copyright 2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ +#include "model/QgTraits.h" +#include "oops/runs/HybridGain.h" +#include "oops/runs/Run.h" + +int main(int argc, char ** argv) { + oops::Run run(argc, argv); + oops::HybridGain var; + return run.execute(var); +} diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index e1485055e..fc67e78fd 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -65,6 +65,7 @@ list( APPEND qg_testinput testinput/getvalues.yaml testinput/hofx.yaml testinput/hofx3d.yaml + testinput/hybridgain.yaml testinput/interfaces.yaml testinput/letkf.yaml testinput/lineargetvalues.yaml @@ -124,6 +125,7 @@ list( APPEND qg_testoutput testoutput/gen_ens_pert_B.test testoutput/hofx.test testoutput/hofx3d.test + testoutput/hybridgain.test testoutput/letkf.test testoutput/make_obs_3d.test testoutput/make_obs_4d_12h.test @@ -443,6 +445,12 @@ ecbuild_add_test( TARGET test_qg_ens_recenter COMMAND qg_ens_recenter.x TEST_DEPENDS test_qg_gen_ens_pert_B ) +ecbuild_add_test( TARGET test_qg_hybridgain + OMP 2 + ARGS testinput/hybridgain.yaml + COMMAND qg_hybridgain.x + TEST_DEPENDS test_qg_gen_ens_pert_B ) + ecbuild_add_test( TARGET test_qg_uniform_field_inflation OMP 2 ARGS testinput/uniform_field_inflation.yaml diff --git a/qg/test/testinput/hybridgain.yaml b/qg/test/testinput/hybridgain.yaml new file mode 100644 index 000000000..26d21d076 --- /dev/null +++ b/qg/test/testinput/hybridgain.yaml @@ -0,0 +1,37 @@ +hybrid weights: + control: 0.2 + ensemble: 0.8 + +control: + date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1D.nc + +ensemble mean: + date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1D.nc + +ensemble: +- date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1D.nc +- date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1D.nc +- date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1D.nc +- date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1D.nc +- date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1D.nc + +geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + +recentered output: + datadir: Data + exp: hybridgain + type: ens + date: 2010-01-01T00:00:00Z + +test: + reference filename: testoutput/hybridgain.test diff --git a/qg/test/testoutput/hybridgain.test b/qg/test/testoutput/hybridgain.test new file mode 100644 index 000000000..adb8323b5 --- /dev/null +++ b/qg/test/testoutput/hybridgain.test @@ -0,0 +1,50 @@ +Control prior: + + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Ensemble mean: + + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6931e+08, Max= 9.9327e+07, RMS= 1.7976e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +new center : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6931e+08, Max= 9.6127e+07, RMS= 1.8043e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 0 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7189e+08, Max= 1.0866e+08, RMS= 1.8566e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 1 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6931e+08, Max= 9.6127e+07, RMS= 1.8043e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 2 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9075e+08, Max= 9.9203e+07, RMS= 1.8684e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 3 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7276e+08, Max= 9.3587e+07, RMS= 1.8062e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 4 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7215e+08, Max= 9.7831e+07, RMS= 1.8379e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= 0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/src/oops/runs/HybridGain.h b/src/oops/runs/HybridGain.h new file mode 100644 index 000000000..7fa6d16f3 --- /dev/null +++ b/src/oops/runs/HybridGain.h @@ -0,0 +1,100 @@ +/* + * (C) Copyright 2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#ifndef OOPS_RUNS_HYBRIDGAIN_H_ +#define OOPS_RUNS_HYBRIDGAIN_H_ + +#include +#include +#include + + +#include "eckit/config/LocalConfiguration.h" +#include "oops/base/Variables.h" +#include "oops/interface/Geometry.h" +#include "oops/interface/Increment.h" +#include "oops/interface/State.h" +#include "oops/mpi/mpi.h" +#include "oops/runs/Application.h" +#include "oops/util/DateTime.h" + +namespace oops { + +template class HybridGain : public Application { + typedef Geometry Geometry_; + typedef Increment Increment_; + typedef State State_; + + public: + // ----------------------------------------------------------------------------- + explicit HybridGain(const eckit::mpi::Comm & comm = oops::mpi::world()) : Application(comm) {} + // ----------------------------------------------------------------------------- + virtual ~HybridGain() {} + // ----------------------------------------------------------------------------- + int execute(const eckit::Configuration & fullConfig) const { + // Setup Geometry + const eckit::LocalConfiguration resolConfig(fullConfig, "geometry"); + const Geometry_ resol(resolConfig, this->getComm()); + + // Read averaging weights + double alphaControl = fullConfig.getDouble("hybrid weights.control"); + double alphaEnsemble = fullConfig.getDouble("hybrid weights.ensemble"); + + // Get control state + const eckit::LocalConfiguration bkgConfig(fullConfig, "control"); + State_ x_control(resol, bkgConfig); + Log::test() << "Control prior: " << std::endl << x_control << std::endl; + const Variables vars = x_control.variables(); + + // Get ens mean + const eckit::LocalConfiguration emeanConfig(fullConfig, "ensemble mean"); + State_ x_emean(resol, emeanConfig); + Log::test() << "Ensemble mean: " << std::endl << x_emean << std::endl; + + // Get ensemble configuration + std::vector ensConfig; + fullConfig.get("ensemble", ensConfig); + unsigned nens = ensConfig.size(); + + // Compute new center and save + State_ x_new_center(resol, vars, x_control.validTime()); + x_new_center.zero(); + x_new_center.accumul(alphaControl, x_control); + x_new_center.accumul(alphaEnsemble, x_emean); + eckit::LocalConfiguration centerOut(fullConfig, "recentered output"); + centerOut.set("member", static_cast(0) ); + x_new_center.write(centerOut); + Log::test() << "new center : " << x_new_center << std::endl; + + // Recenter ensemble around new centr and save + for (unsigned jj = 0; jj < nens; ++jj) { + State_ x(resol, ensConfig[jj]); + Increment_ pert(resol, vars, x.validTime()); + pert.diff(x, x_emean); + x = x_new_center; + x += pert; + + // Save recentered member + eckit::LocalConfiguration recenterout(fullConfig, "recentered output"); + recenterout.set("member", static_cast(jj+1) ); + x.write(recenterout); + Log::test() << "Recentered member " << jj << " : " << x << std::endl; + } + + return 0; + } + // ----------------------------------------------------------------------------- + private: + std::string appname() const { + return "oops::HybridGain<" + MODEL::name() + ">"; + } + // ----------------------------------------------------------------------------- +}; + +} // namespace oops + +#endif // OOPS_RUNS_HYBRIDGAIN_H_ From 4f8ac35a178a500a732bc856b0fabc417692b55b Mon Sep 17 00:00:00 2001 From: Mark J Olah Date: Wed, 12 May 2021 13:45:58 -0600 Subject: [PATCH 125/142] Bugfix/ecbuild import cleanup (#1208) * Cleanup oops-imports.cmake.in to use conditional find_dependency() calls. * Fix conditional Co-authored-by: Anna Shlyaeva --- oops-import.cmake.in | 76 ++++++++++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/oops-import.cmake.in b/oops-import.cmake.in index 8e016ff65..6e436acc9 100644 --- a/oops-import.cmake.in +++ b/oops-import.cmake.in @@ -6,29 +6,59 @@ include(CMakeFindDependencyMacro) set(oops_qg_FOUND @ENABLE_QG_MODEL@) #COMPONENT qg set(oops_lorenz95_FOUND @ENABLE_LORENZ95_MODEL@) #COMPONENT lorenz95 -if( @MKL_FOUND@ ) - find_dependency( MKL REQUIRED ) - set( LAPACK_LIBRARIES ${MKL_LIBRARIES} ) -else() - find_dependency( LAPACK REQUIRED ) -endif() - -find_dependency( Eigen3 REQUIRED NO_MODULE HINTS - $ENV{Eigen3_ROOT} $ENV{EIGEN3_ROOT} $ENV{Eigen_ROOT} $ENV{EIGEN_ROOT} - $ENV{Eigen3_PATH} $ENV{EIGEN3_PATH} $ENV{Eigen_PATH} $ENV{EIGEN_PATH} ) - -if(@OpenMP_FOUND@) - find_package( OpenMP REQUIRED COMPONENTS CXX Fortran ) -endif() -find_package( MPI REQUIRED COMPONENTS CXX Fortran ) -find_package( NetCDF REQUIRED COMPONENTS Fortran ) -find_dependency( Boost REQUIRED ) -find_package( eckit REQUIRED COMPONENTS MPI ) -find_dependency( fckit REQUIRED ) -if(@OpenMP_FOUND@) - find_package( atlas REQUIRED COMPONENTS OMP OMP_Fortran ) -else() - find_dependency( atlas REQUIRED ) +if(@jedicmake_FOUND@ AND NOT jedicmake_FOUND) + find_dependency(jedicmake REQUIRED) +endif() + +if((@LAPACK_FOUND@ AND NOT LAPACK_FOUND) OR (@MKL_FOUND@ AND NOT MKL_FOUND)) + if( @MKL_FOUND@ ) + find_dependency( MKL REQUIRED ) + set( LAPACK_LIBRARIES ${MKL_LIBRARIES} ) + else() + find_dependency( LAPACK REQUIRED ) + endif() +endif() + +if(NOT Eigen3_FOUND) + find_dependency( Eigen3 REQUIRED NO_MODULE HINTS @EIGEN3_ROOT_DIR@ ) +endif() + +if(NOT (OpenMP_CXX_FOUND AND OpenMP_Fortran_FOUND)) + if(@OpenMP_FOUND@) + find_dependency( OpenMP REQUIRED COMPONENTS CXX Fortran ) + endif() +endif() + +if(NOT (MPI_CXX_FOUND AND MPI_Fortran_FOUND)) + find_dependency( MPI REQUIRED COMPONENTS CXX Fortran ) +endif() + +if(NOT NetCDF_Fortran_FOUND) + find_dependency( NetCDF REQUIRED COMPONENTS Fortran ) +endif() + +if(NOT Boost_FOUND) + find_dependency( Boost REQUIRED ) +endif() + +if(NOT eckit_FOUND) + find_dependency( eckit REQUIRED COMPONENTS MPI ) +endif() + +if(NOT fckit_FOUND) + find_dependency( fckit REQUIRED ) +endif() + +if(NOT atlas_FOUND) + if(@OpenMP_FOUND@) + find_dependency( atlas REQUIRED COMPONENTS OMP OMP_Fortran ) + else() + find_dependency( atlas REQUIRED ) + endif() +endif() + +if(@ENABLE_GPTL@ AND NOT GPTL_FOUND) + find_dependency( GPTL REQUIRED ) endif() #Export Fortran compiler version for checking module compatibility From 82cb85dd32d131c55e44fc1c373beca55dabf0ef Mon Sep 17 00:00:00 2001 From: Fabio L R Diniz <45880035+fabiolrdiniz@users.noreply.github.com> Date: Thu, 13 May 2021 22:40:21 -0300 Subject: [PATCH 126/142] Adding an option to calculate the adjoint of the solver through a recomputation mode (#1195) * adding fsoi for primal minimizers * making fsoi to use 4dvar executable for now * fix for the name of the tests according to ecbuild_add_test * updating compare tests for fsoi * fix for reference file * avoiding float relative comparison * cleaning up dualvector and primalminimizer * adding description and removing comment * removing iter from RinvH and fixing indentation in PrimalMinimizer * fix for coding norms * moving ObSens calculation to happen inside PrimalMinimizer --- l95/test/CMakeLists.txt | 19 ++++++ l95/test/testinput/3dvar_4fsoi_pcg.yaml | 51 ++++++++++++++ l95/test/testinput/fsoi_3dvar_pcg.yaml | 49 ++++++++++++++ l95/test/testoutput/3dvar_4fsoi_pcg.test | 10 +++ l95/test/testoutput/fsoi_3dvar_pcg.test | 11 +++ src/oops/assimilation/DualVector.h | 16 ++++- src/oops/assimilation/PrimalMinimizer.h | 47 +++++++++++++ src/oops/assimilation/RinvHMatrix.h | 86 ++++++++++++++++++++++++ 8 files changed, 286 insertions(+), 3 deletions(-) create mode 100644 l95/test/testinput/3dvar_4fsoi_pcg.yaml create mode 100644 l95/test/testinput/fsoi_3dvar_pcg.yaml create mode 100644 l95/test/testoutput/3dvar_4fsoi_pcg.test create mode 100644 l95/test/testoutput/fsoi_3dvar_pcg.test create mode 100644 src/oops/assimilation/RinvHMatrix.h diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index ebc94a0aa..78e7449b1 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -1,6 +1,7 @@ list( APPEND l95_test_input testinput/3dvar.yaml + testinput/3dvar_4fsoi_pcg.yaml testinput/3dvar_noobs.yaml testinput/3dvar_qc.yaml testinput/3dvar_qc_obserr.yaml @@ -57,6 +58,7 @@ list( APPEND l95_test_input testinput/forecast.yaml testinput/forecast_pseudomodel.yaml testinput/forecast_identitymodel.yaml + testinput/fsoi_3dvar_pcg.yaml testinput/genenspert.yaml testinput/getkf.yaml testinput/getkf_offline_hofx.yaml @@ -105,6 +107,7 @@ list( APPEND l95_test_data list( APPEND l95_testoutput testoutput/3dvar.test + testoutput/3dvar_4fsoi_pcg.test testoutput/3dvar_noobs.test testoutput/3dvar_qc.test testoutput/3dvar_qc_obserr.test @@ -143,6 +146,7 @@ list( APPEND l95_testoutput testoutput/forecast.test testoutput/forecast_pseudomodel.test testoutput/forecast_identitymodel.test + testoutput/fsoi_3dvar_pcg.test testoutput/genenspert.test testoutput/getkf.test testoutput/getkf_offline_hofx.test @@ -655,6 +659,21 @@ ecbuild_add_test( TARGET test_l95_eda_3dvar_block COMMAND l95_eda.x ARGS testinput/eda.3dvar.block.yaml TEST_DEPENDS test_l95_genenspert test_l95_makeobs3d ) + +##################################################################### +# FSOI tests +##################################################################### + +ecbuild_add_test( TARGET test_l95_3dvar_4fsoi_pcg + COMMAND l95_4dvar.x + ARGS testinput/3dvar_4fsoi_pcg.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) + +ecbuild_add_test( TARGET test_l95_fsoi_3dvar_pcg + COMMAND l95_4dvar.x + ARGS testinput/fsoi_3dvar_pcg.yaml + TEST_DEPENDS test_l95_3dvar_4fsoi_pcg ) + ##################################################################### # state-related tests ##################################################################### diff --git a/l95/test/testinput/3dvar_4fsoi_pcg.yaml b/l95/test/testinput/3dvar_4fsoi_pcg.yaml new file mode 100644 index 000000000..3fd95c637 --- /dev/null +++ b/l95/test/testinput/3dvar_4fsoi_pcg.yaml @@ -0,0 +1,51 @@ +cost function: + cost type: 3D-Var + window begin: 2010-01-01T21:00:00Z + window length: PT6H + geometry: + resol: 40 + analysis variables: [x] + background: + date: 2010-01-02T00:00:00Z + filename: Data/test.fc.2010-01-01T00:00:00Z.P1D + background error: + covariance model: L95Error + date: 2010-01-02T00:00:00Z + length_scale: 1.0 + standard_deviation: 0.6 + observations: + - obs operator: {} + obs space: + obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt + obsdataout: Data/l95.3dvar_4fsoi_pcg.2010-01-02T00:00:00Z.obt + obs error: + covariance model: diagonal +variational: + minimizer: + algorithm: PCG + iterations: + - ninner: 20 + gradient norm reduction: 1e-10 + geometry: + resol: 40 + diagnostics: + departures: ombg + test: on + online diagnostics: + write increment: true + increment: + datadir: Data + date: 2010-01-02T00:00:00Z + exp: 3dvar_4fsoi_pcg.iter1 + type: in +final: + diagnostics: + departures: oman +output: + datadir: Data + exp: l95.3dvar_4fsoi_pcg + frequency: PT6H + type: an + +test: + reference filename: testoutput/3dvar_4fsoi_pcg.test diff --git a/l95/test/testinput/fsoi_3dvar_pcg.yaml b/l95/test/testinput/fsoi_3dvar_pcg.yaml new file mode 100644 index 000000000..44f0d8124 --- /dev/null +++ b/l95/test/testinput/fsoi_3dvar_pcg.yaml @@ -0,0 +1,49 @@ +cost function: + cost type: 3D-Var + window begin: 2010-01-01T21:00:00Z + window length: PT6H + geometry: + resol: 40 + analysis variables: [x] + background: + date: 2010-01-02T00:00:00Z + filename: Data/test.fc.2010-01-01T00:00:00Z.P1D + background error: + covariance model: L95Error + date: 2010-01-02T00:00:00Z + length_scale: 1.0 + standard_deviation: 0.6 + observations: + - obs operator: {} + obs space: + obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt + obsdataout: Data/l95.fsoi_3dvar_pcg.2010-01-02T00:00:00Z.obt + obs error: + covariance model: diagonal +variational: + minimizer: + algorithm: PCG + iterations: + - ninner: 20 + gradient norm reduction: 1e-10 + geometry: + resol: 40 + diagnostics: + departures: ombg + test: on + fsoi: + increment test: true + input forecast sensitivity: + date: 2010-01-02T00:00:00Z + filename: Data/3dvar_4fsoi_pcg.iter1.in.2010-01-02T00:00:00Z.PT0S +final: + diagnostics: + departures: oman +output: + datadir: Data + exp: l95.fsoi_3dvar_pcg + frequency: PT6H + type: an + +test: + reference filename: testoutput/fsoi_3dvar_pcg.test diff --git a/l95/test/testoutput/3dvar_4fsoi_pcg.test b/l95/test/testoutput/3dvar_4fsoi_pcg.test new file mode 100644 index 000000000..a86382407 --- /dev/null +++ b/l95/test/testoutput/3dvar_4fsoi_pcg.test @@ -0,0 +1,10 @@ +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114.535, nobs = 120, Jo/n = 0.954461, err = 0.4 +CostFunction: Nonlinear J = 114.535 +PCGMinimizer: reduction in residual norm = 3.48414e-10 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.75305, Max=9.06719, Average=8.00913 +CostJb : Nonlinear Jb = 12.2983 +CostJo : Nonlinear Jo(Lorenz 95) = 3.17231, nobs = 120, Jo/n = 0.0264359, err = 0.4 +CostFunction: Nonlinear J = 15.4706 diff --git a/l95/test/testoutput/fsoi_3dvar_pcg.test b/l95/test/testoutput/fsoi_3dvar_pcg.test new file mode 100644 index 000000000..1a7e53b7c --- /dev/null +++ b/l95/test/testoutput/fsoi_3dvar_pcg.test @@ -0,0 +1,11 @@ +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114.535, nobs = 120, Jo/n = 0.954461, err = 0.4 +CostFunction: Nonlinear J = 114.535 +PCGMinimizer: reduction in residual norm = 7.88245e-11 +Into tolerance FSOI increment test. +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.65953, Max=9.39191, Average=7.97085 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114.535, nobs = 120, Jo/n = 0.954461, err = 0.4 +CostFunction: Nonlinear J = 114.535 diff --git a/src/oops/assimilation/DualVector.h b/src/oops/assimilation/DualVector.h index ee105132d..5febe488f 100644 --- a/src/oops/assimilation/DualVector.h +++ b/src/oops/assimilation/DualVector.h @@ -1,9 +1,9 @@ /* * (C) Copyright 2009-2016 ECMWF. - * + * * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ @@ -12,6 +12,7 @@ #define OOPS_ASSIMILATION_DUALVECTOR_H_ #include +#include #include #include @@ -59,6 +60,8 @@ template class DualVector { void axpy(const double, const DualVector &); double dot_product_with(const DualVector &) const; + void saveDep(const std::string &) const; + // Clear everything void clear(); @@ -246,6 +249,13 @@ bool DualVector::compatible(const DualVector & other) const { return lcheck; } // ----------------------------------------------------------------------------- +template +void DualVector::saveDep(const std::string & name) const { + for (unsigned jj = 0; jj < dxjo_.size(); ++jj) { + dxjo_[jj]->save(name); + } +} +// ----------------------------------------------------------------------------- } // namespace oops #endif // OOPS_ASSIMILATION_DUALVECTOR_H_ diff --git a/src/oops/assimilation/PrimalMinimizer.h b/src/oops/assimilation/PrimalMinimizer.h index fc3167d26..c70a32a32 100644 --- a/src/oops/assimilation/PrimalMinimizer.h +++ b/src/oops/assimilation/PrimalMinimizer.h @@ -11,6 +11,7 @@ #ifndef OOPS_ASSIMILATION_PRIMALMINIMIZER_H_ #define OOPS_ASSIMILATION_PRIMALMINIMIZER_H_ +#include #include #include "eckit/config/Configuration.h" @@ -18,8 +19,12 @@ #include "oops/assimilation/ControlIncrement.h" #include "oops/assimilation/CostFunction.h" #include "oops/assimilation/Minimizer.h" +#include "oops/assimilation/RinvHMatrix.h" +#include "oops/util/FloatCompare.h" #include "oops/util/Logger.h" +#include "oops/assimilation/DualVector.h" + namespace oops { /// Primal Minimizer @@ -36,6 +41,7 @@ template class PrimalMinimizer : public Minimizer< typedef BMatrix Bmat_; typedef HessianMatrix Hessian_; typedef Minimizer Minimizer_; + typedef RinvHMatrix RinvH_; public: explicit PrimalMinimizer(const CostFct_ & J): Minimizer_(J), J_(J) {} @@ -90,6 +96,47 @@ PrimalMinimizer::doMinimize(const eckit::Configuration & config) { Log::test() << classname() << ": reduction in residual norm = " << reduc << std::endl; Log::info() << classname() << " output" << *dx << std::endl; + if (config.has("fsoi")) { + Log::info() << classname() << " Entering Observation Sensitivity Calculation" << std::endl; + + // Multiply result of solver by RinvH to get observation sensitivity (ys) + DualVector ys; + const RinvH_ RinvH(J_); + RinvH.multiply(*dx, ys); + + // Write out observation sensitivity + const std::string osensname = "ObsSensitivity"; + ys.saveDep(osensname); + + bool runFSOIincTest = config.getBool("fsoi.increment test", false); + if (runFSOIincTest) { + // Get departures + DualVector dp; + for (unsigned jj = 0; jj < J_.nterms(); ++jj) { + std::unique_ptr ww(J_.jterm(jj).newGradientFG()); + dp.append(J_.jterm(jj).multiplyCovar(*ww)); + } + + // , where dx = K dp + double adj_tst_fwd = dot_product(rhs, rhs); + // , where K = Hessian Ht Rinv; dp=departures + double adj_tst_bwd = dot_product(ys, dp); + + Log::info() << "Online FSOI increment test: " << std::endl + << util::PrintAdjTest(adj_tst_fwd, adj_tst_bwd, "K") << std::endl; + + bool passed = oops::is_close_absolute(adj_tst_fwd, adj_tst_bwd, 1.0e-5); + if (passed) { + Log::test() << "Into tolerance FSOI increment test." << std::endl; + } else { + Log::test() << "Out of tolerance FSOI increment test." << std::endl; + } + } + + // Make sure not to update state in FSOI mode + dx->zero(); + } + return dx; } diff --git a/src/oops/assimilation/RinvHMatrix.h b/src/oops/assimilation/RinvHMatrix.h new file mode 100644 index 000000000..8c15918e2 --- /dev/null +++ b/src/oops/assimilation/RinvHMatrix.h @@ -0,0 +1,86 @@ +/* + * (C) Copyright 2009-2016 ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#ifndef OOPS_ASSIMILATION_RINVHMATRIX_H_ +#define OOPS_ASSIMILATION_RINVHMATRIX_H_ + +#include + +#include +#include + +#include "oops/assimilation/ControlIncrement.h" +#include "oops/assimilation/CostFunction.h" +#include "oops/assimilation/DualVector.h" +#include "oops/base/Departures.h" +#include "oops/base/PostProcessorTLAD.h" +#include "oops/interface/Increment.h" +#include "oops/util/formats.h" +#include "oops/util/Logger.h" +#include "oops/util/PrintAdjTest.h" + +namespace oops { + +/// The \f$ R^{-1} H \f$ matrix. +/*! + * The solvers represent matrices as objects that implement a "multiply" + * method. This class defines objects that apply a generalized + * \f$ R^{-1} H \f$ matrix that also includes the equivalent + * operators for the other terms of the cost function. + */ + +template class RinvHMatrix : private boost::noncopyable { + typedef ControlIncrement CtrlInc_; + typedef CostFunction CostFct_; + typedef DualVector Dual_; + + public: + explicit RinvHMatrix(const CostFct_ & j); + + void multiply(const CtrlInc_ & dx, Dual_ & zz) const; + + private: + CostFct_ const & j_; +}; + +// ----------------------------------------------------------------------------- + +template +RinvHMatrix::RinvHMatrix(const CostFct_ & j) + : j_(j) +{} + +// ----------------------------------------------------------------------------- + +template +void RinvHMatrix::multiply(const CtrlInc_ & dx, Dual_ & zz) const { +// Setup TL terms of cost function + PostProcessorTLAD costtl; + for (unsigned jj = 0; jj < j_.nterms(); ++jj) { + j_.jterm(jj).setPostProcTL(dx, costtl); + } + +// Run TLM + CtrlInc_ mdx(dx); + j_.runTLM(mdx, costtl); + +// Get TLM outputs, multiply by covariance inverses + for (unsigned jj = 0; jj < j_.nterms(); ++jj) { + std::unique_ptr wtmp(j_.jterm(jj).newDualVector()); + j_.jterm(jj).computeCostTL(dx, *wtmp); + zz.append(j_.jterm(jj).multiplyCoInv(*wtmp)); + } +} + +// ----------------------------------------------------------------------------- + +} // namespace oops + +#endif // OOPS_ASSIMILATION_RINVHMATRIX_H_ From ef128c4de927b82bf3202cb5788aa03114aca8d9 Mon Sep 17 00:00:00 2001 From: Travis Sluka Date: Mon, 17 May 2021 15:13:01 -0600 Subject: [PATCH 127/142] add missing RinvHMatrix to cmakelists (#1212) --- src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2328a2583..f73136ba9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -69,6 +69,7 @@ oops/assimilation/PLanczos.h oops/assimilation/PLanczosMinimizer.h oops/assimilation/PrimalMinimizer.h oops/assimilation/QNewtonLMP.h +oops/assimilation/RinvHMatrix.h oops/assimilation/RinvMatrix.h oops/assimilation/rotmat.h oops/assimilation/RPCGMinimizer.h From 8f517dd3ffb36bf5a2f680af4a810aaeba8b1355 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Tue, 18 May 2021 10:28:28 -0600 Subject: [PATCH 128/142] refactor ObsLocalization to be consistent with documentation (#1211) * refactor ObsLocalization * add missing parameter names --- l95/src/lorenz95/ObsLocGC99.cc | 4 +- l95/src/lorenz95/ObsLocGC99.h | 7 ++-- qg/model/ObsLocQG.cc | 4 +- qg/model/ObsLocQG.h | 9 ++--- src/oops/base/ObsLocalizationBase.h | 26 ++++++++++--- src/oops/interface/ObsLocalization.h | 56 ++++++++++++++++------------ 6 files changed, 62 insertions(+), 44 deletions(-) diff --git a/l95/src/lorenz95/ObsLocGC99.cc b/l95/src/lorenz95/ObsLocGC99.cc index f7ecf8eb1..2a3cf6673 100644 --- a/l95/src/lorenz95/ObsLocGC99.cc +++ b/l95/src/lorenz95/ObsLocGC99.cc @@ -20,13 +20,11 @@ #include "lorenz95/ObsVec1D.h" #include "oops/generic/gc99.h" -#include "oops/interface/ObsLocalization.h" // ----------------------------------------------------------------------------- namespace lorenz95 { -static oops::ObsLocalizationMaker> makerGC_("Gaspari-Cohn"); +static oops::ObsLocalizationMaker makerGC_("Gaspari-Cohn"); // ----------------------------------------------------------------------------- diff --git a/l95/src/lorenz95/ObsLocGC99.h b/l95/src/lorenz95/ObsLocGC99.h index f6b634cfb..c7695cb6b 100644 --- a/l95/src/lorenz95/ObsLocGC99.h +++ b/l95/src/lorenz95/ObsLocGC99.h @@ -11,8 +11,9 @@ #include #include "eckit/config/Configuration.h" -#include "oops/util/Printable.h" +#include "oops/base/ObsLocalizationBase.h" +#include "lorenz95/L95Traits.h" #include "lorenz95/ObsData1D.h" // Forward declarations @@ -22,7 +23,7 @@ namespace lorenz95 { class ObsVec1D; /// Observation space localization for Lorenz 95 model (Gaspari-Cohn) -class ObsLocGC99: public util::Printable { +class ObsLocGC99: public oops::ObsLocalizationBase { public: ObsLocGC99(const eckit::Configuration &, const ObsTable &); @@ -30,7 +31,7 @@ class ObsLocGC99: public util::Printable { /// localization flags (1: outside of localization; 0: inside localization area) /// in \p outside void computeLocalization(const Iterator &, ObsData1D & outside, - ObsVec1D & obsvector) const; + ObsVec1D & obsvector) const override; private: void print(std::ostream &) const override; diff --git a/qg/model/ObsLocQG.cc b/qg/model/ObsLocQG.cc index 4acecae96..b566dfacd 100644 --- a/qg/model/ObsLocQG.cc +++ b/qg/model/ObsLocQG.cc @@ -14,7 +14,6 @@ #include "eckit/geometry/Point2.h" #include "eckit/geometry/Sphere.h" -#include "oops/interface/ObsLocalization.h" #include "oops/util/Logger.h" #include "model/GeometryQGIterator.h" @@ -26,8 +25,7 @@ using atlas::array::make_view; namespace qg { -static oops::ObsLocalizationMaker> makerObsLoc_("Heaviside"); +static oops::ObsLocalizationMaker makerObsLoc_("Heaviside"); // ----------------------------------------------------------------------------- diff --git a/qg/model/ObsLocQG.h b/qg/model/ObsLocQG.h index f49b238f5..aad101be2 100644 --- a/qg/model/ObsLocQG.h +++ b/qg/model/ObsLocQG.h @@ -10,11 +10,10 @@ #include -#include "atlas/field.h" - -#include "oops/util/Printable.h" +#include "oops/base/ObsLocalizationBase.h" #include "oops/qg/ObsDataQG.h" +#include "oops/qg/QgTraits.h" namespace eckit { class Configuration; @@ -27,11 +26,11 @@ namespace qg { /// \brief Observation-space localization for QG model (Heaviside function /// with prescribed lengthscale). -class ObsLocQG : public util::Printable { +class ObsLocQG : public oops::ObsLocalizationBase { public: ObsLocQG(const eckit::Configuration &, const ObsSpaceQG &); - void computeLocalization(const GeometryQGIterator &, ObsDataQG &, ObsVecQG &) const; + void computeLocalization(const GeometryQGIterator &, ObsDataQG &, ObsVecQG &) const override; private: void print(std::ostream &) const override; diff --git a/src/oops/base/ObsLocalizationBase.h b/src/oops/base/ObsLocalizationBase.h index cfb6e5be1..01f5711a7 100644 --- a/src/oops/base/ObsLocalizationBase.h +++ b/src/oops/base/ObsLocalizationBase.h @@ -21,17 +21,31 @@ namespace oops { -/// Base class for observation-space localization +/// Base class for observation-space localization. +/// Defines the interfaces for observation space localization. +/// Use this class as a base class for OBS- and MODEL-specific implementations. template class ObsLocalizationBase : public util::Printable, - private boost::noncopyable { - typedef GeometryIterator GeometryIterator_; - typedef ObsDataVector ObsDataVector_; - typedef ObsVector ObsVector_; + private boost::noncopyable { + typedef typename MODEL::GeometryIterator GeometryIterator_; + typedef typename OBS::ObsVector ObsVector_; + typedef typename OBS::template ObsDataVector ObsDataVector_; public: ObsLocalizationBase() = default; virtual ~ObsLocalizationBase() = default; + /// compute obs-space localization: fill \p obsvector with observation-space + /// localization values between observations and \p point in model-space, and + /// fill \p outside with flags on whether obs is local or not (1: outside of + /// localization, 0: inside of localization, local) + /// Method used in oops. Calls `computeLocalization` abstract method, and + /// passes OBS- and MODEL-specific classes to the OBS- and MODEL-specific + /// implementations of ObsLocalization. + void computeLocalization(const GeometryIterator & point, + ObsDataVector & flags, ObsVector & obsvector) const { + computeLocalization(point.geometryiter(), flags.obsdatavector(), obsvector.obsvector()); + } + /// compute obs-space localization: fill \p obsvector with observation-space /// localization values between observations and \p point in model-space, and /// fill \p outside with flags on whether obs is local or not (1: outside of @@ -67,7 +81,7 @@ class ObsLocalizationMaker : public ObsLocalizationFactory { typedef ObsSpace ObsSpace_; virtual ObsLocalizationBase * make(const eckit::Configuration & conf, const ObsSpace_ & obspace) - { return new T(conf, obspace); } + { return new T(conf, obspace.obsspace()); } public: explicit ObsLocalizationMaker(const std::string & name) : ObsLocalizationFactory(name) {} diff --git a/src/oops/interface/ObsLocalization.h b/src/oops/interface/ObsLocalization.h index bad655656..39f169568 100644 --- a/src/oops/interface/ObsLocalization.h +++ b/src/oops/interface/ObsLocalization.h @@ -22,9 +22,13 @@ namespace oops { // ----------------------------------------------------------------------------- - -template -class ObsLocalization : public ObsLocalizationBase { +/// \brief Encapsulates the observation-space localization +/// Note: to see methods that need to be implemented in the ObsLocalization implementation, +/// see ObsLocalizationBase class +template +class ObsLocalization : public util::Printable, + private boost::noncopyable { + typedef ObsLocalizationBase ObsLocBase_; typedef GeometryIterator GeometryIterator_; typedef ObsSpace ObsSpace_; typedef ObsDataVector ObsDataVector_; @@ -36,53 +40,57 @@ class ObsLocalization : public ObsLocalizationBase { ObsLocalization(const eckit::Configuration &, const ObsSpace_ &); ~ObsLocalization(); - void computeLocalization(const GeometryIterator_ &, - ObsDataVector_ &, ObsVector_ &) const override; + /// compute obs-space localization: fill \p obsvector with observation-space + /// localization values between observations and \p point in model-space, and + /// fill \p outside with flags on whether obs is local or not (1: outside of + /// localization, 0: inside of localization, local) + void computeLocalization(const GeometryIterator_ & point, + ObsDataVector_ & outside, ObsVector_ & obsvector) const override; private: void print(std::ostream &) const override; - std::unique_ptr obsloc_; + std::unique_ptr obsloc_; }; // ----------------------------------------------------------------------------- -template -ObsLocalization::ObsLocalization(const eckit::Configuration & conf, - const ObsSpace_ & obspace) +template +ObsLocalization::ObsLocalization(const eckit::Configuration & conf, + const ObsSpace_ & obspace) : obsloc_() { - Log::trace() << "ObsLocalization::ObsLocalization starting" << std::endl; + Log::trace() << "ObsLocalization::ObsLocalization starting" << std::endl; util::Timer timer(classname(), "ObsLocalization"); - obsloc_.reset(new LOC(conf, obspace.obsspace())); - Log::trace() << "ObsLocalization::ObsLocalization done" << std::endl; + obsloc_.reset(ObsLocalizationFactory::create(conf, obspace)); + Log::trace() << "ObsLocalization::ObsLocalization done" << std::endl; } // ----------------------------------------------------------------------------- -template -ObsLocalization::~ObsLocalization() { - Log::trace() << "ObsLocalization::~ObsLocalization starting" << std::endl; +template +ObsLocalization::~ObsLocalization() { + Log::trace() << "ObsLocalization::~ObsLocalization starting" << std::endl; util::Timer timer(classname(), "~ObsLocalization"); obsloc_.reset(); - Log::trace() << "ObsLocalization::~ObsLocalization done" << std::endl; + Log::trace() << "ObsLocalization::~ObsLocalization done" << std::endl; } // ----------------------------------------------------------------------------- -template -void ObsLocalization::computeLocalization(const GeometryIterator_ & p, - ObsDataVector_ & local, ObsVector_ & obsvector) const { - Log::trace() << "ObsLocalization:: computeLocalization starting" << std::endl; +template +void ObsLocalization::computeLocalization(const GeometryIterator_ & p, + ObsDataVector_ & local, ObsVector_ & obsvector) const { + Log::trace() << "ObsLocalization:: computeLocalization starting" << std::endl; util::Timer timer(classname(), "computeLocalization"); - obsloc_->computeLocalization(p.geometryiter(), local.obsdatavector(), obsvector.obsvector()); - Log::trace() << "ObsLocalization:: computeLocalization done" << std::endl; + obsloc_->computeLocalization(p, local, obsvector); + Log::trace() << "ObsLocalization:: computeLocalization done" << std::endl; } // ----------------------------------------------------------------------------- -template -void ObsLocalization::print(std::ostream & os) const { +template +void ObsLocalization::print(std::ostream & os) const { os << *obsloc_; } From 0d213953fab06e9a7229926c6cd738c510181b9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Tue, 25 May 2021 14:36:27 +0100 Subject: [PATCH 129/142] Added an mpi::exclusiveScan() function. (#1221) --- src/oops/mpi/mpi.cc | 11 +++++++++++ src/oops/mpi/mpi.h | 6 ++++++ src/test/mpi/mpi.h | 14 ++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/src/oops/mpi/mpi.cc b/src/oops/mpi/mpi.cc index 448cc0f07..dde3d9fd0 100644 --- a/src/oops/mpi/mpi.cc +++ b/src/oops/mpi/mpi.cc @@ -10,6 +10,7 @@ #include "oops/mpi/mpi.h" +#include // for accumulate() #include #include @@ -171,5 +172,15 @@ void allGatherv(const eckit::mpi::Comm & comm, std::vector &x) { x = decodeStrings(charBuffer.buffer, lengthBuffer.buffer); } +// ------------------------------------------------------------------------------------------------ + +void exclusiveScan(const eckit::mpi::Comm &comm, size_t &x) { + // Could be done with MPI_Exscan, but there's no wrapper for it in eckit::mpi. + + std::vector xs(comm.size()); + comm.allGather(x, xs.begin(), xs.end()); + x = std::accumulate(xs.begin(), xs.begin() + comm.rank(), 0); +} + } // namespace mpi } // namespace oops diff --git a/src/oops/mpi/mpi.h b/src/oops/mpi/mpi.h index 2899e0e03..fe59e5614 100644 --- a/src/oops/mpi/mpi.h +++ b/src/oops/mpi/mpi.h @@ -183,5 +183,11 @@ void allGatherv(const eckit::mpi::Comm & comm, std::vector &x); // ------------------------------------------------------------------------------------------------ +/// \brief Perform the exclusive scan operation. +/// +/// On output, `x` is set to the sum of the values of `x` passed to this function +/// on all ranks lower than the calling rank (and to 0 on rank 0). +void exclusiveScan(const eckit::mpi::Comm &comm, size_t &x); + } // namespace mpi } // namespace oops diff --git a/src/test/mpi/mpi.h b/src/test/mpi/mpi.h index db2203491..f7785266c 100644 --- a/src/test/mpi/mpi.h +++ b/src/test/mpi/mpi.h @@ -234,6 +234,20 @@ CASE("mpi/mpi/allGatherEigen") { EXPECT_EQUAL(expectedEigen, globalEigen); } // ----------------------------------------------------------------------------------------------- +CASE("mpi/mpi/exclusiveScan") { + const eckit::mpi::Comm &comm = oops::mpi::world(); + const size_t rank = comm.rank(); + const size_t size = comm.size(); + + size_t expectedResult = 0; + for (size_t lowerRank = 0; lowerRank < rank; ++lowerRank) + expectedResult += lowerRank; + + size_t result = rank; + oops::mpi::exclusiveScan(comm, result); + EXPECT_EQUAL(result, expectedResult); +} +// ----------------------------------------------------------------------------------------------- class Mpi : public oops::Test { private: From 06620f183899543096b40473f2359dfa920ab6a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Tue, 25 May 2021 08:22:07 -0600 Subject: [PATCH 130/142] Remove HofX3Dslow (#1189) Co-authored-by: Anna Shlyaeva --- src/CMakeLists.txt | 1 - src/oops/runs/HofX3Dslow.h | 120 ------------------------------------- 2 files changed, 121 deletions(-) delete mode 100644 src/oops/runs/HofX3Dslow.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f73136ba9..6e895fadb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -236,7 +236,6 @@ oops/runs/ExternalDFI.h oops/runs/Forecast.h oops/runs/GenEnsPertB.h oops/runs/HofX3D.h -oops/runs/HofX3Dslow.h oops/runs/HofX4D.h oops/runs/HofX4Dhack.h oops/runs/LocalEnsembleDA.h diff --git a/src/oops/runs/HofX3Dslow.h b/src/oops/runs/HofX3Dslow.h deleted file mode 100644 index b89e08af3..000000000 --- a/src/oops/runs/HofX3Dslow.h +++ /dev/null @@ -1,120 +0,0 @@ -/* - * (C) Copyright 2018-2020 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#ifndef OOPS_RUNS_HOFX3DSLOW_H_ -#define OOPS_RUNS_HOFX3DSLOW_H_ - -#include -#include -#include -#include - -#include "eckit/config/LocalConfiguration.h" -#include "oops/assimilation/CalcHofX.h" -#include "oops/base/instantiateObsFilterFactory.h" -#include "oops/base/Observations.h" -#include "oops/base/ObsSpaces.h" -#include "oops/base/Variables.h" -#include "oops/interface/Geometry.h" -#include "oops/interface/GeoVaLs.h" -#include "oops/interface/GetValues.h" -#include "oops/interface/Locations.h" -#include "oops/interface/State.h" -#include "oops/mpi/mpi.h" -#include "oops/runs/Application.h" -#include "oops/util/ConfigFunctions.h" -#include "oops/util/DateTime.h" -#include "oops/util/Duration.h" -#include "oops/util/Logger.h" - -namespace oops { - -template class HofX3Dslow : public Application { - typedef CalcHofX CalcHofX_; - typedef Geometry Geometry_; - typedef GeoVaLs GeoVaLs_; - typedef GetValues GetValues_; - typedef Locations Locations_; - typedef ObsAuxControls ObsAux_; - typedef Observations Observations_; - typedef ObsSpaces ObsSpaces_; - typedef State State_; - - typedef std::vector> GeoVaLsVec_; - typedef std::vector> LocationsVec_; - typedef std::vector VariablesVec_; - - - public: -// ----------------------------------------------------------------------------- - explicit HofX3Dslow(const eckit::mpi::Comm & comm = oops::mpi::world()) : Application(comm) { - instantiateObsFilterFactory(); - } -// ----------------------------------------------------------------------------- - virtual ~HofX3Dslow() {} -// ----------------------------------------------------------------------------- - int execute(const eckit::Configuration & fullConfig) const { - // Setup observation window - const util::Duration winlen(fullConfig.getString("window length")); - const util::DateTime winbgn(fullConfig.getString("window begin")); - const util::DateTime winend(winbgn + winlen); - Log::info() << "Observation window from " << winbgn << " to " << winend << std::endl; - - // Setup geometry - const eckit::LocalConfiguration geometryConfig(fullConfig, "geometry"); - const Geometry_ geometry(geometryConfig, this->getComm()); - - // Setup states for H(x) - const eckit::LocalConfiguration stateConfig(fullConfig, "state"); - Log::info() << "State configuration is:" << stateConfig << std::endl; - State_ xx(geometry, stateConfig); - Log::test() << "State: " << xx << std::endl; - - // Setup observations - const eckit::LocalConfiguration obsConfig(fullConfig, "observations"); - ObsSpaces_ obspaces(obsConfig, this->getComm(), winbgn, winend); - ObsAux_ obsaux(obspaces, obsConfig); - CalcHofX_ hofx(obspaces, obsConfig); - hofx.initialize(obsaux); - - // fill in GeoVaLs - GeoVaLsVec_ geovals; - const LocationsVec_ & locations = hofx.locations(); - const VariablesVec_ & vars = hofx.requiredVars(); - - std::vector getValuesConfig = - util::vectoriseAndFilter(obsConfig, "get values"); - - // loop over all observation types - for (size_t jj = 0; jj < obspaces.size(); ++jj) { - GetValues_ getvals(geometry, *locations[jj], getValuesConfig[jj]); - geovals.emplace_back(new GeoVaLs_(*locations[jj], vars[jj])); - getvals.fillGeoVaLs(xx, winbgn, winend, *geovals[jj]); - } - - // Compute H(x) on filled in geovals and run the filters - Observations_ yobs = hofx.compute(geovals); - hofx.saveQcFlags("EffectiveQC"); - hofx.saveObsErrors("EffectiveError"); - - // Save H(x) - Log::test() << "H(x): " << std::endl << yobs << "End H(x)" << std::endl; - yobs.save("hofx"); - - return 0; - } -// ----------------------------------------------------------------------------- - private: - std::string appname() const { - return "oops::HofX3Dslow<" + MODEL::name() + ", " + OBS::name() + ">"; - } -// ----------------------------------------------------------------------------- -}; - -} // namespace oops - -#endif // OOPS_RUNS_HOFX3DSLOW_H_ From 0962d17f8b725a1ec20584c5cca9ba2697ec275c Mon Sep 17 00:00:00 2001 From: Fabio L R Diniz <45880035+fabiolrdiniz@users.noreply.github.com> Date: Wed, 26 May 2021 04:00:53 -0300 Subject: [PATCH 131/142] Extending the observation sensitivity calculation for DRMinimizer (#1213) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * adding fsoi for primal minimizers * making fsoi to use 4dvar executable for now * fix for the name of the tests according to ecbuild_add_test * updating compare tests for fsoi * fix for reference file * avoiding float relative comparison * cleaning up dualvector and primalminimizer * adding description and removing comment * removing iter from RinvH and fixing indentation in PrimalMinimizer * fix for coding norms * moving ObSens calculation to happen inside PrimalMinimizer * extending fsoi for dr-minimizers * rephrasing warning and making tolerance optional through yaml Co-authored-by: Yannick Trémolet <30638944+ytremolet@users.noreply.github.com> --- l95/test/CMakeLists.txt | 14 ++++++ l95/test/testinput/3dvar_4fsoi_dripcg.yaml | 51 +++++++++++++++++++++ l95/test/testinput/fsoi_3dvar_dripcg.yaml | 50 ++++++++++++++++++++ l95/test/testinput/fsoi_3dvar_pcg.yaml | 1 + l95/test/testoutput/3dvar_4fsoi_dripcg.test | 10 ++++ l95/test/testoutput/fsoi_3dvar_dripcg.test | 11 +++++ l95/test/testoutput/fsoi_3dvar_pcg.test | 2 +- src/oops/assimilation/DRMinimizer.h | 48 +++++++++++++++++++ src/oops/assimilation/PrimalMinimizer.h | 7 +-- 9 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 l95/test/testinput/3dvar_4fsoi_dripcg.yaml create mode 100644 l95/test/testinput/fsoi_3dvar_dripcg.yaml create mode 100644 l95/test/testoutput/3dvar_4fsoi_dripcg.test create mode 100644 l95/test/testoutput/fsoi_3dvar_dripcg.test diff --git a/l95/test/CMakeLists.txt b/l95/test/CMakeLists.txt index 78e7449b1..e883dfe72 100644 --- a/l95/test/CMakeLists.txt +++ b/l95/test/CMakeLists.txt @@ -1,6 +1,7 @@ list( APPEND l95_test_input testinput/3dvar.yaml + testinput/3dvar_4fsoi_dripcg.yaml testinput/3dvar_4fsoi_pcg.yaml testinput/3dvar_noobs.yaml testinput/3dvar_qc.yaml @@ -58,6 +59,7 @@ list( APPEND l95_test_input testinput/forecast.yaml testinput/forecast_pseudomodel.yaml testinput/forecast_identitymodel.yaml + testinput/fsoi_3dvar_dripcg.yaml testinput/fsoi_3dvar_pcg.yaml testinput/genenspert.yaml testinput/getkf.yaml @@ -107,6 +109,7 @@ list( APPEND l95_test_data list( APPEND l95_testoutput testoutput/3dvar.test + testoutput/3dvar_4fsoi_dripcg.test testoutput/3dvar_4fsoi_pcg.test testoutput/3dvar_noobs.test testoutput/3dvar_qc.test @@ -146,6 +149,7 @@ list( APPEND l95_testoutput testoutput/forecast.test testoutput/forecast_pseudomodel.test testoutput/forecast_identitymodel.test + testoutput/fsoi_3dvar_dripcg.test testoutput/fsoi_3dvar_pcg.test testoutput/genenspert.test testoutput/getkf.test @@ -674,6 +678,16 @@ ecbuild_add_test( TARGET test_l95_fsoi_3dvar_pcg ARGS testinput/fsoi_3dvar_pcg.yaml TEST_DEPENDS test_l95_3dvar_4fsoi_pcg ) +ecbuild_add_test( TARGET test_l95_3dvar_4fsoi_dripcg + COMMAND l95_4dvar.x + ARGS testinput/3dvar_4fsoi_dripcg.yaml + TEST_DEPENDS test_l95_forecast test_l95_makeobs3d ) + +ecbuild_add_test( TARGET test_l95_fsoi_3dvar_dripcg + COMMAND l95_4dvar.x + ARGS testinput/fsoi_3dvar_dripcg.yaml + TEST_DEPENDS test_l95_3dvar_4fsoi_dripcg ) + ##################################################################### # state-related tests ##################################################################### diff --git a/l95/test/testinput/3dvar_4fsoi_dripcg.yaml b/l95/test/testinput/3dvar_4fsoi_dripcg.yaml new file mode 100644 index 000000000..77eb9b43b --- /dev/null +++ b/l95/test/testinput/3dvar_4fsoi_dripcg.yaml @@ -0,0 +1,51 @@ +cost function: + cost type: 3D-Var + window begin: 2010-01-01T21:00:00Z + window length: PT6H + geometry: + resol: 40 + analysis variables: [x] + background: + date: 2010-01-02T00:00:00Z + filename: Data/test.fc.2010-01-01T00:00:00Z.P1D + background error: + covariance model: L95Error + date: 2010-01-02T00:00:00Z + length_scale: 1.0 + standard_deviation: 0.6 + observations: + - obs operator: {} + obs space: + obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt + obsdataout: Data/l95.3dvar_4fsoi_dripcg.2010-01-02T00:00:00Z.obt + obs error: + covariance model: diagonal +variational: + minimizer: + algorithm: DRIPCG + iterations: + - ninner: 20 + gradient norm reduction: 1e-10 + geometry: + resol: 40 + diagnostics: + departures: ombg + test: on + online diagnostics: + write increment: true + increment: + datadir: Data + date: 2010-01-02T00:00:00Z + exp: 3dvar_4fsoi_dripcg.iter1 + type: in +final: + diagnostics: + departures: oman +output: + datadir: Data + exp: l95.3dvar_4fsoi_dripcg + frequency: PT6H + type: an + +test: + reference filename: testoutput/3dvar_4fsoi_dripcg.test diff --git a/l95/test/testinput/fsoi_3dvar_dripcg.yaml b/l95/test/testinput/fsoi_3dvar_dripcg.yaml new file mode 100644 index 000000000..c462ad342 --- /dev/null +++ b/l95/test/testinput/fsoi_3dvar_dripcg.yaml @@ -0,0 +1,50 @@ +cost function: + cost type: 3D-Var + window begin: 2010-01-01T21:00:00Z + window length: PT6H + geometry: + resol: 40 + analysis variables: [x] + background: + date: 2010-01-02T00:00:00Z + filename: Data/test.fc.2010-01-01T00:00:00Z.P1D + background error: + covariance model: L95Error + date: 2010-01-02T00:00:00Z + length_scale: 1.0 + standard_deviation: 0.6 + observations: + - obs operator: {} + obs space: + obsdatain: Data/l95.truth3d.2010-01-02T00:00:00Z.obt + obsdataout: Data/l95.fsoi_3dvar_dripcg.2010-01-02T00:00:00Z.obt + obs error: + covariance model: diagonal +variational: + minimizer: + algorithm: DRIPCG + iterations: + - ninner: 20 + gradient norm reduction: 1e-10 + geometry: + resol: 40 + diagnostics: + departures: ombg + test: on + fsoi: + increment test: true + increment test tolerance: 1.0e-6 + input forecast sensitivity: + date: 2010-01-02T00:00:00Z + filename: Data/3dvar_4fsoi_dripcg.iter1.in.2010-01-02T00:00:00Z.PT0S +final: + diagnostics: + departures: oman +output: + datadir: Data + exp: l95.fsoi_3dvar_dripcg + frequency: PT6H + type: an + +test: + reference filename: testoutput/fsoi_3dvar_dripcg.test diff --git a/l95/test/testinput/fsoi_3dvar_pcg.yaml b/l95/test/testinput/fsoi_3dvar_pcg.yaml index 44f0d8124..5d699ac18 100644 --- a/l95/test/testinput/fsoi_3dvar_pcg.yaml +++ b/l95/test/testinput/fsoi_3dvar_pcg.yaml @@ -33,6 +33,7 @@ variational: test: on fsoi: increment test: true + increment test tolerance: 1.0e-6 input forecast sensitivity: date: 2010-01-02T00:00:00Z filename: Data/3dvar_4fsoi_pcg.iter1.in.2010-01-02T00:00:00Z.PT0S diff --git a/l95/test/testoutput/3dvar_4fsoi_dripcg.test b/l95/test/testoutput/3dvar_4fsoi_dripcg.test new file mode 100644 index 000000000..6865123c6 --- /dev/null +++ b/l95/test/testoutput/3dvar_4fsoi_dripcg.test @@ -0,0 +1,10 @@ +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114.535, nobs = 120, Jo/n = 0.954461, err = 0.4 +CostFunction: Nonlinear J = 114.535 +DRIPCGMinimizer: reduction in residual norm = 1.94488e-09 +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.75305, Max=9.06719, Average=8.00913 +CostJb : Nonlinear Jb = 12.2983 +CostJo : Nonlinear Jo(Lorenz 95) = 3.17231, nobs = 120, Jo/n = 0.0264359, err = 0.4 +CostFunction: Nonlinear J = 15.4706 diff --git a/l95/test/testoutput/fsoi_3dvar_dripcg.test b/l95/test/testoutput/fsoi_3dvar_dripcg.test new file mode 100644 index 000000000..b045f12e7 --- /dev/null +++ b/l95/test/testoutput/fsoi_3dvar_dripcg.test @@ -0,0 +1,11 @@ +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114.535, nobs = 120, Jo/n = 0.954461, err = 0.4 +CostFunction: Nonlinear J = 114.535 +DRIPCGMinimizer: reduction in residual norm = 4.49681e-10 +FSOI increment test within tolerance. +CostFunction::addIncrement: Analysis: + Valid time: 2010-01-02T00:00:00Z + Min=6.65953, Max=9.39191, Average=7.97085 +CostJb : Nonlinear Jb = 0 +CostJo : Nonlinear Jo(Lorenz 95) = 114.535, nobs = 120, Jo/n = 0.954461, err = 0.4 +CostFunction: Nonlinear J = 114.535 diff --git a/l95/test/testoutput/fsoi_3dvar_pcg.test b/l95/test/testoutput/fsoi_3dvar_pcg.test index 1a7e53b7c..c75da604b 100644 --- a/l95/test/testoutput/fsoi_3dvar_pcg.test +++ b/l95/test/testoutput/fsoi_3dvar_pcg.test @@ -2,7 +2,7 @@ CostJb : Nonlinear Jb = 0 CostJo : Nonlinear Jo(Lorenz 95) = 114.535, nobs = 120, Jo/n = 0.954461, err = 0.4 CostFunction: Nonlinear J = 114.535 PCGMinimizer: reduction in residual norm = 7.88245e-11 -Into tolerance FSOI increment test. +FSOI increment test within tolerance. CostFunction::addIncrement: Analysis: Valid time: 2010-01-02T00:00:00Z Min=6.65953, Max=9.39191, Average=7.97085 diff --git a/src/oops/assimilation/DRMinimizer.h b/src/oops/assimilation/DRMinimizer.h index f42a6b4c9..1921d7ab4 100644 --- a/src/oops/assimilation/DRMinimizer.h +++ b/src/oops/assimilation/DRMinimizer.h @@ -22,8 +22,10 @@ #include "oops/assimilation/CostFunction.h" #include "oops/assimilation/HtRinvHMatrix.h" #include "oops/assimilation/Minimizer.h" +#include "oops/assimilation/RinvHMatrix.h" #include "oops/util/dot_product.h" +#include "oops/util/FloatCompare.h" #include "oops/util/formats.h" #include "oops/util/Logger.h" @@ -47,6 +49,7 @@ template class DRMinimizer : public Minimizer CtrlInc_; typedef HtRinvHMatrix HtRinvH_; typedef Minimizer Minimizer_; + typedef RinvHMatrix RinvH_; public: explicit DRMinimizer(const CostFct_ & J): Minimizer_(J), J_(J), gradJb_(), costJ0Jb_(0) {} @@ -93,9 +96,12 @@ DRMinimizer::doMinimize(const eckit::Configuration & config) { // Compute RHS (sum B^{-1} dx_{i}) + H^T R^{-1} d // dx_i = x_i - x_{i-1}; dx_1 = x_1 - x_b CtrlInc_ rhs(J_.jb()); + CtrlInc_ dx0(rhs); if (config.has("fsoi")) { const eckit::LocalConfiguration FcSensitivityConfig(config, "fsoi.input forecast sensitivity"); rhs.read(FcSensitivityConfig); + dx0 = rhs; + dx0 *= -1.0; Log::info() << classname() << " rhs has forecast sensitivity" << std::endl; } else { J_.computeGradientFG(rhs); @@ -131,6 +137,48 @@ DRMinimizer::doMinimize(const eckit::Configuration & config) { costJ0Jb_ += dot_product(*dx, dxhtmp); } + if (config.has("fsoi")) { + Log::info() << classname() << " Entering Observation Sensitivity Calculation" << std::endl; + + // Multiply result of solver by RinvH to get observation sensitivity (ys) + DualVector ys; + const RinvH_ RinvH(J_); + RinvH.multiply(*dx, ys); + + // Write out observation sensitivity + const std::string osensname = "ObsSensitivity"; + ys.saveDep(osensname); + + bool runFSOIincTest = config.getBool("fsoi.increment test", false); + if (runFSOIincTest) { + // Get departures + DualVector dp; + for (unsigned jj = 0; jj < J_.nterms(); ++jj) { + std::unique_ptr ww(J_.jterm(jj).newGradientFG()); + dp.append(J_.jterm(jj).multiplyCovar(*ww)); + } + + // , where dx = K dp + double adj_tst_fwd = dot_product(dx0, dx0); + // , where K = Hessian Ht Rinv; dp=departures + double adj_tst_bwd = dot_product(ys, dp); + + Log::info() << "Online FSOI increment test: " << std::endl + << util::PrintAdjTest(adj_tst_fwd, adj_tst_bwd, "K") << std::endl; + + double fsoi_inctest_tolerance = config.getDouble("fsoi.increment test tolerance", 1.0e-5); + bool passed = oops::is_close_absolute(adj_tst_fwd, adj_tst_bwd, fsoi_inctest_tolerance); + if (passed) { + Log::test() << "FSOI increment test within tolerance." << std::endl; + } else { + Log::test() << "FSOI increment test fails tolerance bound." << std::endl; + } + } + + // Make sure not to update state in FSOI mode + dx->zero(); + } + return dx; } diff --git a/src/oops/assimilation/PrimalMinimizer.h b/src/oops/assimilation/PrimalMinimizer.h index c70a32a32..1d7ceec17 100644 --- a/src/oops/assimilation/PrimalMinimizer.h +++ b/src/oops/assimilation/PrimalMinimizer.h @@ -125,11 +125,12 @@ PrimalMinimizer::doMinimize(const eckit::Configuration & config) { Log::info() << "Online FSOI increment test: " << std::endl << util::PrintAdjTest(adj_tst_fwd, adj_tst_bwd, "K") << std::endl; - bool passed = oops::is_close_absolute(adj_tst_fwd, adj_tst_bwd, 1.0e-5); + double fsoi_inctest_tolerance = config.getDouble("fsoi.increment test tolerance", 1.0e-5); + bool passed = oops::is_close_absolute(adj_tst_fwd, adj_tst_bwd, fsoi_inctest_tolerance); if (passed) { - Log::test() << "Into tolerance FSOI increment test." << std::endl; + Log::test() << "FSOI increment test within tolerance." << std::endl; } else { - Log::test() << "Out of tolerance FSOI increment test." << std::endl; + Log::test() << "FSOI increment test fails tolerance bound." << std::endl; } } From 31af36f588cd0b2ca3b73df2934fea845266e851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Wed, 26 May 2021 07:19:13 -0600 Subject: [PATCH 132/142] Better prints of runtime and memory usage (#1215) * Improve prints of runtime stats * Add timing prints ni DRPLanczos Co-authored-by: Anna Shlyaeva --- qg/model/ObsVecQG.cc | 10 +- qg/model/ObsVecQG.h | 1 + qg/model/QgFortran.h | 1 + qg/model/qg_obsvec_interface.F90 | 22 ++- qg/model/qg_obsvec_mod.F90 | 15 +- src/CMakeLists.txt | 6 +- src/oops/assimilation/ControlIncrement.h | 4 + src/oops/assimilation/DRIPCGMinimizer.h | 4 + src/oops/assimilation/DRPLanczosMinimizer.h | 4 + .../assimilation/IncrementalAssimilation.h | 6 + src/oops/base/EnsembleCovariance.h | 14 +- src/oops/base/ObsAuxIncrements.h | 12 +- src/oops/interface/ErrorCovariance.h | 7 +- src/oops/interface/Increment.h | 7 +- src/oops/interface/ModelAuxIncrement.h | 3 + src/oops/interface/ObsAuxIncrement.h | 6 +- src/oops/interface/ObsErrorCovariance.h | 7 +- src/oops/interface/ObsSpace.h | 7 +- src/oops/interface/ObsVector.h | 11 +- src/oops/interface/State.h | 9 +- src/oops/runs/Run.cc | 8 +- src/oops/runs/Variational.h | 3 + src/oops/util/MemoryCounter.cc | 35 ---- src/oops/util/MemoryHelper.cc | 153 ------------------ src/oops/util/MemoryHelper.h | 40 ----- src/oops/util/ObjectCountHelper.cc | 58 ++++--- src/oops/util/ObjectCountHelper.h | 14 +- src/oops/util/ObjectCounter.h | 17 +- src/oops/util/Timer.cc | 16 ++ src/oops/util/Timer.h | 6 + src/oops/util/printRunStats.cc | 68 ++++++++ .../util/{MemoryCounter.h => printRunStats.h} | 14 +- 32 files changed, 279 insertions(+), 309 deletions(-) delete mode 100644 src/oops/util/MemoryCounter.cc delete mode 100644 src/oops/util/MemoryHelper.cc delete mode 100644 src/oops/util/MemoryHelper.h create mode 100644 src/oops/util/printRunStats.cc rename src/oops/util/{MemoryCounter.h => printRunStats.h} (68%) diff --git a/qg/model/ObsVecQG.cc b/qg/model/ObsVecQG.cc index 97d4b7374..a2d27e42e 100644 --- a/qg/model/ObsVecQG.cc +++ b/qg/model/ObsVecQG.cc @@ -22,8 +22,7 @@ namespace qg { // ----------------------------------------------------------------------------- -ObsVecQG::ObsVecQG(const ObsSpaceQG & obsdb, - const std::string & name) +ObsVecQG::ObsVecQG(const ObsSpaceQG & obsdb, const std::string & name) : obsdb_(obsdb), keyOvec_(0) { qg_obsvec_setup_f90(keyOvec_, obsdb.obsvariables().size(), obsdb.nobs()); @@ -170,4 +169,11 @@ unsigned int ObsVecQG::nobs() const { return nobs; } // ----------------------------------------------------------------------------- +size_t ObsVecQG::size() const { + int iobs; + qg_obsvec_size_f90(keyOvec_, iobs); + size_t nobs(iobs); + return nobs; +} +// ----------------------------------------------------------------------------- } // namespace qg diff --git a/qg/model/ObsVecQG.h b/qg/model/ObsVecQG.h index a21281152..3445cd67c 100644 --- a/qg/model/ObsVecQG.h +++ b/qg/model/ObsVecQG.h @@ -47,6 +47,7 @@ class ObsVecQG : public util::Printable, Eigen::VectorXd packEigen(const ObsDataQG &) const; size_t packEigenSize(const ObsDataQG &) const; + size_t size() const; /// set all values to zero void zero(); diff --git a/qg/model/QgFortran.h b/qg/model/QgFortran.h index 375a021dd..883448b5d 100644 --- a/qg/model/QgFortran.h +++ b/qg/model/QgFortran.h @@ -232,6 +232,7 @@ extern "C" { void qg_obsvec_dotprod_f90(const F90ovec &, const F90ovec &, double &); void qg_obsvec_stats_f90(const F90ovec &, double &, double &, double &); void qg_obsvec_nobs_f90(const F90ovec &, int &); + void qg_obsvec_size_f90(const F90ovec &, int &); /// fill \p data (size \p nobs) with all non-masked out (non-missing) values void qg_obsvec_get_withmask_f90(const F90ovec &, const F90ovec & mask_key, double * data, const int & nobs); diff --git a/qg/model/qg_obsvec_interface.F90 b/qg/model/qg_obsvec_interface.F90 index 9ec95fc23..53b19bd64 100644 --- a/qg/model/qg_obsvec_interface.F90 +++ b/qg/model/qg_obsvec_interface.F90 @@ -395,7 +395,7 @@ subroutine qg_obsvec_stats_c(c_key_self,zmin,zmax,zavg) bind(c,name='qg_obsvec_s end subroutine qg_obsvec_stats_c ! ------------------------------------------------------------------------------ -!> Get observation vector size +!> Get number of observations (not missing) subroutine qg_obsvec_nobs_c(c_key_self,kobs) bind(c,name='qg_obsvec_nobs_f90') implicit none @@ -415,6 +415,26 @@ subroutine qg_obsvec_nobs_c(c_key_self,kobs) bind(c,name='qg_obsvec_nobs_f90') end subroutine qg_obsvec_nobs_c ! ------------------------------------------------------------------------------ +!> Get observation vector size +subroutine qg_obsvec_size_c(c_key_self,kobs) bind(c,name='qg_obsvec_size_f90') + +implicit none + +! Passed variables +integer(c_int),intent(in) :: c_key_self !< Observation vector +integer(c_int),intent(inout) :: kobs !< Observation vector size + +! Local vector +type(qg_obsvec),pointer :: self + +! Interface +call qg_obsvec_registry%get(c_key_self,self) + +! Call Fortran +call qg_obsvec_size(self,kobs) + +end subroutine qg_obsvec_size_c +! ------------------------------------------------------------------------------ !> Get observation vector size (only non-masked observations) subroutine qg_obsvec_nobs_withmask_c(c_key_self,c_key_mask,kobs) bind(c,name='qg_obsvec_nobs_withmask_f90') diff --git a/qg/model/qg_obsvec_mod.F90 b/qg/model/qg_obsvec_mod.F90 index 40f400f46..f062e97f0 100644 --- a/qg/model/qg_obsvec_mod.F90 +++ b/qg/model/qg_obsvec_mod.F90 @@ -21,8 +21,9 @@ module qg_obsvec_mod public :: qg_obsvec_registry public :: qg_obsvec_setup,qg_obsvec_clone,qg_obsvec_delete,qg_obsvec_copy,qg_obsvec_zero, & & qg_obsvec_zero_ith, qg_obsvec_ones, qg_obsvec_mask, qg_obsvec_mul_scal,qg_obsvec_add, & - & qg_obsvec_sub,qg_obsvec_mul,qg_obsvec_div,qg_obsvec_axpy,qg_obsvec_invert,qg_obsvec_random,qg_obsvec_dotprod, & - & qg_obsvec_stats,qg_obsvec_nobs,qg_obsvec_nobs_withmask,qg_obsvec_get_withmask + & qg_obsvec_sub,qg_obsvec_mul,qg_obsvec_div,qg_obsvec_axpy,qg_obsvec_invert, & + & qg_obsvec_random,qg_obsvec_dotprod,qg_obsvec_stats, & + & qg_obsvec_size,qg_obsvec_nobs,qg_obsvec_nobs_withmask,qg_obsvec_get_withmask ! ------------------------------------------------------------------------------ interface subroutine qg_obsvec_random_i(odb,nn,zz) bind(c,name='qg_obsvec_random_f') @@ -400,6 +401,16 @@ subroutine qg_obsvec_stats(self,zmin,zmax,zavg) endif end subroutine qg_obsvec_stats + +! ------------------------------------------------------------------------------ +!> Get observation vector size +subroutine qg_obsvec_size(self,kobs) +implicit none +type(qg_obsvec),intent(in) :: self !< Observation vector +integer,intent(inout) :: kobs !< Observation vector size +kobs = size(self%values) + 2 +end subroutine qg_obsvec_size + ! ------------------------------------------------------------------------------ !> Get observation vector size subroutine qg_obsvec_nobs(self,kobs) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6e895fadb..d331dfa83 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -290,10 +290,6 @@ oops/util/LibOOPS.h oops/util/LocalEnvironment.cc oops/util/LocalEnvironment.h oops/util/Logger.h -oops/util/MemoryCounter.cc -oops/util/MemoryCounter.h -oops/util/MemoryHelper.cc -oops/util/MemoryHelper.h oops/util/missing_values_f.cc oops/util/missing_values_f.h oops/util/missing_values_mod.F90 @@ -306,6 +302,8 @@ oops/util/ObjectCountHelper.cc oops/util/ObjectCountHelper.h oops/util/Printable.h oops/util/PrintAdjTest.h +oops/util/printRunStats.cc +oops/util/printRunStats.h oops/util/PropertiesOfNVectors.h oops/util/random_f.cc oops/util/random_f.h diff --git a/src/oops/assimilation/ControlIncrement.h b/src/oops/assimilation/ControlIncrement.h index 5d1f52961..d2a170608 100644 --- a/src/oops/assimilation/ControlIncrement.h +++ b/src/oops/assimilation/ControlIncrement.h @@ -121,6 +121,7 @@ ControlIncrement::ControlIncrement(const JbTotal_ & jb) obsbias_(jb.jbObsBias().obspaces(), jb.jbObsBias().config()), windowBegin_(jb.windowBegin()), windowEnd_(jb.windowEnd()) { + this->setObjectSize(this->serialSize()*sizeof(double)); Log::trace() << "ControlIncrement:ControlIncrement created." << std::endl; } // ----------------------------------------------------------------------------- @@ -129,6 +130,7 @@ ControlIncrement::ControlIncrement(const ControlIncrement & other, c : increment_(other.increment_, copy), modbias_(other.modbias_, copy), obsbias_(other.obsbias_, copy), windowBegin_(other.windowBegin_), windowEnd_(other.windowEnd_) { + this->setObjectSize(this->serialSize()*sizeof(double)); Log::trace() << "ControlIncrement:ControlIncrement copied." << std::endl; } // ----------------------------------------------------------------------------- @@ -138,6 +140,7 @@ ControlIncrement::ControlIncrement(const ControlIncrement & other, : increment_(other.increment_, tlConf), modbias_(other.modbias_, tlConf), obsbias_(other.obsbias_, tlConf), windowBegin_(other.windowBegin_), windowEnd_(other.windowEnd_) { + this->setObjectSize(this->serialSize()*sizeof(double)); Log::trace() << "ControlIncrement:ControlIncrement copied." << std::endl; } // ----------------------------------------------------------------------------- @@ -147,6 +150,7 @@ ControlIncrement::ControlIncrement(const Geometry_ & geom, : increment_(geom, other.increment_), modbias_(other.modbias_, true), obsbias_(other.obsbias_, true), windowBegin_(other.windowBegin_), windowEnd_(other.windowEnd_) { + this->setObjectSize(this->serialSize()*sizeof(double)); Log::trace() << "ControlIncrement:ControlIncrement copied." << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/assimilation/DRIPCGMinimizer.h b/src/oops/assimilation/DRIPCGMinimizer.h index c495be918..8b7fd4c52 100644 --- a/src/oops/assimilation/DRIPCGMinimizer.h +++ b/src/oops/assimilation/DRIPCGMinimizer.h @@ -25,6 +25,7 @@ #include "oops/util/dot_product.h" #include "oops/util/formats.h" #include "oops/util/Logger.h" +#include "oops/util/printRunStats.h" namespace oops { @@ -103,6 +104,7 @@ double DRIPCGMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, CtrlInc_ const Bmat_ & B, const HtRinvH_ & HtRinvH, const double costJ0Jb, const double costJ0JoJc, const int maxiter, const double tolerance) { + util::printRunStats("DRIPCG start"); CtrlInc_ ap(xh); CtrlInc_ pp(xh); CtrlInc_ ph(xh); @@ -139,6 +141,7 @@ double DRIPCGMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, CtrlInc_ Log::info() << std::endl; for (int jiter = 0; jiter < maxiter; ++jiter) { Log::info() << " DRIPCG Starting Iteration " << jiter+1 << std::endl; + util::printRunStats("DRIPCG iteration " + std::to_string(jiter+1)); if (jiter == 0) { pp = ss; @@ -212,6 +215,7 @@ double DRIPCGMinimizer::solve(CtrlInc_ & xx, CtrlInc_ & xh, CtrlInc_ // Generate the (second-level) Limited Memory Preconditioner lmp_.update(B); + util::printRunStats("DRIPCG end"); return normReduction; } diff --git a/src/oops/assimilation/DRPLanczosMinimizer.h b/src/oops/assimilation/DRPLanczosMinimizer.h index 2da522f16..0ec0cca65 100644 --- a/src/oops/assimilation/DRPLanczosMinimizer.h +++ b/src/oops/assimilation/DRPLanczosMinimizer.h @@ -27,6 +27,7 @@ #include "oops/util/dot_product.h" #include "oops/util/formats.h" #include "oops/util/Logger.h" +#include "oops/util/printRunStats.h" namespace oops { @@ -113,6 +114,7 @@ double DRPLanczosMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, Ctr const Bmat_ & B, const HtRinvH_ & HtRinvH, const double costJ0Jb, const double costJ0JoJc, const int maxiter, const double tolerance) { + util::printRunStats("DRPLanczos start"); // dx increment // dxh B^{-1} dx // rr (sum B^{-1} dx_i^{b} +) G^T H^{-1} d @@ -156,6 +158,7 @@ double DRPLanczosMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, Ctr Log::info() << std::endl; for (int jiter = 0; jiter < maxiter; ++jiter) { Log::info() << "DRPLanczos Starting Iteration " << jiter+1 << std::endl; + util::printRunStats("DRPLanczos iteration " + std::to_string(jiter+1)); // v_{i+1} = ( pr_{i} + H^T R^{-1} H z_{i} ) - beta * v_{i-1} HtRinvH.multiply(zz, vv); @@ -248,6 +251,7 @@ double DRPLanczosMinimizer::solve(CtrlInc_ & dx, CtrlInc_ & dxh, Ctr } ++outerLoop_; + util::printRunStats("DRPLanczos end"); return normReduction; } diff --git a/src/oops/assimilation/IncrementalAssimilation.h b/src/oops/assimilation/IncrementalAssimilation.h index 740069a32..614923c91 100644 --- a/src/oops/assimilation/IncrementalAssimilation.h +++ b/src/oops/assimilation/IncrementalAssimilation.h @@ -23,6 +23,7 @@ #include "oops/base/StateInfo.h" #include "oops/interface/State.h" #include "oops/util/Logger.h" +#include "oops/util/printRunStats.h" namespace oops { @@ -33,6 +34,8 @@ int IncrementalAssimilation(ControlVariable & xx, CostFunction Minimizer_; typedef State State_; + util::printRunStats("IncrementalAssimilation start"); + // Setup outer loop std::vector iterconfs; config.get("iterations", iterconfs); @@ -50,6 +53,7 @@ int IncrementalAssimilation(ControlVariable & xx, CostFunction post; @@ -60,6 +64,7 @@ int IncrementalAssimilation(ControlVariable & xx, CostFunction dx(minim->minimize(iterconfs[jouter])); @@ -70,6 +75,7 @@ int IncrementalAssimilation(ControlVariable & xx, CostFunction +#include #include #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" +#include "eckit/system/ResourceUsage.h" + #include "oops/assimilation/GMRESR.h" #include "oops/base/IdentityMatrix.h" #include "oops/base/IncrementEnsemble.h" @@ -26,7 +29,7 @@ #include "oops/interface/Increment.h" #include "oops/interface/State.h" #include "oops/util/Logger.h" -#include "oops/util/MemoryCounter.h" +#include "oops/util/ObjectCounter.h" namespace oops { @@ -34,7 +37,8 @@ namespace oops { // ----------------------------------------------------------------------------- template -class EnsembleCovariance : public ModelSpaceCovarianceBase { +class EnsembleCovariance : public ModelSpaceCovarianceBase, + private util::ObjectCounter> { typedef Geometry Geometry_; typedef Increment Increment_; typedef LocalizationBase Localization_; @@ -43,6 +47,8 @@ class EnsembleCovariance : public ModelSpaceCovarianceBase { typedef std::shared_ptr> EnsemblePtr_; public: + static const std::string classname() {return "oops::EnsembleCovariance";} + EnsembleCovariance(const Geometry_ &, const Variables &, const eckit::Configuration &, const State_ &, const State_ &); ~EnsembleCovariance(); @@ -68,12 +74,14 @@ EnsembleCovariance::EnsembleCovariance(const Geometry_ & resol, const Var : ModelSpaceCovarianceBase(xb, fg, resol, conf), ens_(), loc_() { Log::trace() << "EnsembleCovariance::EnsembleCovariance start" << std::endl; - util::MemoryCounter mem("oops::EnsembleCovariance"); + size_t init = eckit::system::ResourceUsage().maxResidentSetSize(); ens_.reset(new Ensemble_(conf, xb, fg, resol, vars)); if (conf.has("localization")) { const eckit::LocalConfiguration confloc(conf, "localization"); loc_ = LocalizationFactory::create(resol, xb.validTime(), confloc); } + size_t current = eckit::system::ResourceUsage().maxResidentSetSize(); + this->setObjectSize(current - init); Log::trace() << "EnsembleCovariance::EnsembleCovariance done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/base/ObsAuxIncrements.h b/src/oops/base/ObsAuxIncrements.h index 126f1ce67..6b7f944d1 100644 --- a/src/oops/base/ObsAuxIncrements.h +++ b/src/oops/base/ObsAuxIncrements.h @@ -17,6 +17,7 @@ #include "oops/base/ObsSpaces.h" #include "oops/interface/ObsAuxIncrement.h" #include "oops/util/Logger.h" +#include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" #include "oops/util/Serializable.h" @@ -26,7 +27,8 @@ namespace oops { template class ObsAuxIncrements : public util::Printable, - public util::Serializable { + public util::Serializable, + private util::ObjectCounter > { typedef ObsAuxIncrement ObsAuxIncrement_; typedef ObsAuxControls ObsAuxControls_; typedef ObsSpaces ObsSpaces_; @@ -88,6 +90,8 @@ template ObsAuxIncrements::ObsAuxIncrements(const ObsSpaces_ & odb, const eckit::Configuration & conf) : auxs_(0) { + Log::trace() << "ObsAuxIncrements::ObsAuxIncrements starting" << std::endl; + size_t bytes = 0; std::vector obsconf = conf.getSubConfigurations(); for (std::size_t jobs = 0; jobs < obsconf.size(); ++jobs) { eckit::LocalConfiguration obsauxconf = obsconf[jobs].getSubConfiguration("obs bias"); @@ -95,7 +99,10 @@ ObsAuxIncrements::ObsAuxIncrements(const ObsSpaces_ & odb, const eckit::Con obsauxparams.validateAndDeserialize(obsauxconf); auxs_.push_back( std::unique_ptr(new ObsAuxIncrement_(odb[jobs], obsauxparams))); + bytes += auxs_[jobs]->serialSize(); } + this->setObjectSize(bytes*sizeof(double)); + Log::trace() << "ObsAuxIncrements::ObsAuxIncrements done" << std::endl; } // ----------------------------------------------------------------------------- template @@ -103,10 +110,13 @@ ObsAuxIncrements::ObsAuxIncrements(const ObsAuxIncrements & other, const bo : auxs_(other.size()) { Log::trace() << "ObsAuxIncrements::ObsAuxIncrements copy starting" << std::endl; + size_t bytes = 0; ASSERT(size() == other.size()); for (std::size_t jobs = 0; jobs < other.size(); ++jobs) { auxs_[jobs].reset(new ObsAuxIncrement_(other[jobs], copy)); + bytes += auxs_[jobs]->serialSize(); } + this->setObjectSize(bytes*sizeof(double)); Log::trace() << "ObsAuxIncrements::ObsAuxIncrements copy done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/interface/ErrorCovariance.h b/src/oops/interface/ErrorCovariance.h index a828cab09..9704694ee 100644 --- a/src/oops/interface/ErrorCovariance.h +++ b/src/oops/interface/ErrorCovariance.h @@ -16,13 +16,14 @@ #include +#include "eckit/system/ResourceUsage.h" + #include "oops/base/ModelSpaceCovarianceBase.h" #include "oops/base/Variables.h" #include "oops/interface/Geometry.h" #include "oops/interface/Increment.h" #include "oops/interface/State.h" #include "oops/util/Logger.h" -#include "oops/util/MemoryCounter.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" #include "oops/util/Timer.h" @@ -88,11 +89,13 @@ ErrorCovariance::ErrorCovariance(const Geometry_ & resol, const Variables { Log::trace() << "ErrorCovariance::ErrorCovariance starting" << std::endl; util::Timer timer(classname(), "ErrorCovariance"); - util::MemoryCounter mem(classname()); + size_t init = eckit::system::ResourceUsage().maxResidentSetSize(); covariance_.reset(new Covariance_(resol.geometry(), vars, parametersOrConfiguration::value>( parameters), xb.state(), fg.state())); + size_t current = eckit::system::ResourceUsage().maxResidentSetSize(); + this->setObjectSize(current - init); Log::trace() << "ErrorCovariance::ErrorCovariance done" << std::endl; } diff --git a/src/oops/interface/Increment.h b/src/oops/interface/Increment.h index b65212b85..c00138cac 100644 --- a/src/oops/interface/Increment.h +++ b/src/oops/interface/Increment.h @@ -28,7 +28,6 @@ #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/gatherPrint.h" -#include "oops/util/MemoryCounter.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Serializable.h" #include "oops/util/Timer.h" @@ -143,8 +142,8 @@ Increment::Increment(const Geometry_ & resol, const Variables & vars, { Log::trace() << "Increment::Increment starting" << std::endl; util::Timer timer(classname(), "Increment"); - util::MemoryCounter mem(classname()); increment_.reset(new Increment_(resol.geometry(), vars, time)); + this->setObjectSize(increment_->serialSize()*sizeof(double)); Log::trace() << "Increment::Increment done" << std::endl; } @@ -156,8 +155,8 @@ Increment::Increment(const Geometry_ & resol, const Increment & other) { Log::trace() << "Increment::Increment starting" << std::endl; util::Timer timer(classname(), "Increment"); - util::MemoryCounter mem(classname()); increment_.reset(new Increment_(resol.geometry(), *other.increment_)); + this->setObjectSize(increment_->serialSize()*sizeof(double)); Log::trace() << "Increment::Increment done" << std::endl; } @@ -169,8 +168,8 @@ Increment::Increment(const Increment & other, const bool copy) { Log::trace() << "Increment::Increment copy starting" << std::endl; util::Timer timer(classname(), "Increment"); - util::MemoryCounter mem(classname()); increment_.reset(new Increment_(*other.increment_, copy)); + this->setObjectSize(increment_->serialSize()*sizeof(double)); Log::trace() << "Increment::Increment copy done" << std::endl; } diff --git a/src/oops/interface/ModelAuxIncrement.h b/src/oops/interface/ModelAuxIncrement.h index 7b0b04c08..6593d0b3f 100644 --- a/src/oops/interface/ModelAuxIncrement.h +++ b/src/oops/interface/ModelAuxIncrement.h @@ -100,6 +100,7 @@ ModelAuxIncrement::ModelAuxIncrement(const Geometry_ & resol, Log::trace() << "ModelAuxIncrement::ModelAuxIncrement starting" << std::endl; util::Timer timer(classname(), "ModelAuxIncrement"); aux_.reset(new ModelAuxIncrement_(resol.geometry(), conf)); + this->setObjectSize(aux_->serialSize()*sizeof(double)); Log::trace() << "ModelAuxIncrement::ModelAuxIncrement done" << std::endl; } // ----------------------------------------------------------------------------- @@ -110,6 +111,7 @@ ModelAuxIncrement::ModelAuxIncrement(const ModelAuxIncrement & other, Log::trace() << "ModelAuxIncrement::ModelAuxIncrement copy starting" << std::endl; util::Timer timer(classname(), "ModelAuxIncrement"); aux_.reset(new ModelAuxIncrement_(*other.aux_, copy)); + this->setObjectSize(aux_->serialSize()*sizeof(double)); Log::trace() << "ModelAuxIncrement::ModelAuxIncrement copy done" << std::endl; } // ----------------------------------------------------------------------------- @@ -120,6 +122,7 @@ ModelAuxIncrement::ModelAuxIncrement(const ModelAuxIncrement & other, Log::trace() << "ModelAuxIncrement::ModelAuxIncrement interpolated starting" << std::endl; util::Timer timer(classname(), "ModelAuxIncrement"); aux_.reset(new ModelAuxIncrement_(*other.aux_, conf)); + this->setObjectSize(aux_->serialSize()*sizeof(double)); Log::trace() << "ModelAuxIncrement::ModelAuxIncrement interpolated done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/interface/ObsAuxIncrement.h b/src/oops/interface/ObsAuxIncrement.h index 01a364d6a..3a7ba9207 100644 --- a/src/oops/interface/ObsAuxIncrement.h +++ b/src/oops/interface/ObsAuxIncrement.h @@ -95,21 +95,23 @@ ObsAuxControl & operator+=(ObsAuxControl & xx, const ObsAuxIncrement ObsAuxIncrement::ObsAuxIncrement(const ObsSpace & os, - const Parameters_ & params) : aux_() + const Parameters_ & params) : aux_() { Log::trace() << "ObsAuxIncrement::ObsAuxIncrement starting" << std::endl; util::Timer timer(classname(), "ObsAuxIncrement"); aux_.reset(new ObsAuxIncrement_(os.obsspace(), params)); + this->setObjectSize(aux_->serialSize()*sizeof(double)); Log::trace() << "ObsAuxIncrement::ObsAuxIncrement done" << std::endl; } // ----------------------------------------------------------------------------- template ObsAuxIncrement::ObsAuxIncrement(const ObsAuxIncrement & other, - const bool copy) : aux_() + const bool copy) : aux_() { Log::trace() << "ObsAuxIncrement::ObsAuxIncrement copy starting" << std::endl; util::Timer timer(classname(), "ObsAuxIncrement"); aux_.reset(new ObsAuxIncrement_(*other.aux_, copy)); + this->setObjectSize(aux_->serialSize()*sizeof(double)); Log::trace() << "ObsAuxIncrement::ObsAuxIncrement copy done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/interface/ObsErrorCovariance.h b/src/oops/interface/ObsErrorCovariance.h index ccaf4686e..a47386dfb 100644 --- a/src/oops/interface/ObsErrorCovariance.h +++ b/src/oops/interface/ObsErrorCovariance.h @@ -14,10 +14,11 @@ #include #include +#include "eckit/system/ResourceUsage.h" + #include "oops/base/ObsErrorBase.h" #include "oops/interface/ObsSpace.h" #include "oops/interface/ObsVector.h" -#include "oops/util/MemoryCounter.h" namespace eckit { class Configuration; @@ -66,8 +67,10 @@ ObsErrorCovariance::ObsErrorCovariance(const eckit::Configuration & const Variables & obsvar) : covar_() { Log::trace() << "ObsErrorCovariance::ObsErrorCovariance starting" << std::endl; util::Timer timer(classname(), "ObsErrorCovariance"); - util::MemoryCounter mem(classname()); + size_t init = eckit::system::ResourceUsage().maxResidentSetSize(); covar_.reset(new OBSERR(conf, obsdb, obsvar)); + size_t current = eckit::system::ResourceUsage().maxResidentSetSize(); + this->setObjectSize(current - init); Log::trace() << "ObsErrorCovariance::ObsErrorCovariance done" << std::endl; } diff --git a/src/oops/interface/ObsSpace.h b/src/oops/interface/ObsSpace.h index 064aa53e2..6d3ac817c 100644 --- a/src/oops/interface/ObsSpace.h +++ b/src/oops/interface/ObsSpace.h @@ -17,11 +17,12 @@ #include #include "eckit/geometry/Point2.h" +#include "eckit/system/ResourceUsage.h" + #include "oops/base/Variables.h" #include "oops/interface/GeometryIterator.h" #include "oops/mpi/mpi.h" #include "oops/util/Logger.h" -#include "oops/util/MemoryCounter.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" #include "oops/util/Timer.h" @@ -90,8 +91,10 @@ ObsSpace::ObsSpace(const eckit::Configuration & conf, const eckit::mpi::Comm & time) : obsdb_(), time_(time) { Log::trace() << "ObsSpace::ObsSpace starting" << std::endl; util::Timer timer(classname(), "ObsSpace"); - util::MemoryCounter mem(classname()); + size_t init = eckit::system::ResourceUsage().maxResidentSetSize(); obsdb_.reset(new ObsSpace_(conf, comm, bgn, end, time)); + size_t current = eckit::system::ResourceUsage().maxResidentSetSize(); + this->setObjectSize(current - init); Log::trace() << "ObsSpace::ObsSpace done" << std::endl; } diff --git a/src/oops/interface/ObsVector.h b/src/oops/interface/ObsVector.h index 30130e1f0..1c24a081f 100644 --- a/src/oops/interface/ObsVector.h +++ b/src/oops/interface/ObsVector.h @@ -21,7 +21,6 @@ #include "oops/interface/ObsSpace.h" #include "oops/util/gatherPrint.h" #include "oops/util/Logger.h" -#include "oops/util/MemoryCounter.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" #include "oops/util/Timer.h" @@ -102,10 +101,8 @@ ObsVector::ObsVector(const ObsSpace & os, const std::string name) : data_(), commTime_(os.timeComm()) { Log::trace() << "ObsVector::ObsVector starting " << name << std::endl; util::Timer timer(classname(), "ObsVector"); - util::MemoryCounter mem(classname()); - data_.reset(new ObsVector_(os.obsspace(), name)); - + this->setObjectSize(data_->size() * sizeof(double)); Log::trace() << "ObsVector::ObsVector done" << std::endl; } // ----------------------------------------------------------------------------- @@ -113,10 +110,8 @@ template ObsVector::ObsVector(const ObsVector & other): data_(), commTime_(other.commTime_) { Log::trace() << "ObsVector::ObsVector starting" << std::endl; util::Timer timer(classname(), "ObsVector"); - util::MemoryCounter mem(classname()); - data_.reset(new ObsVector_(*other.data_)); - + this->setObjectSize(data_->size() * sizeof(double)); Log::trace() << "ObsVector::ObsVector done" << std::endl; } // ----------------------------------------------------------------------------- @@ -124,9 +119,7 @@ template ObsVector::~ObsVector() { Log::trace() << "ObsVector::~ObsVector starting" << std::endl; util::Timer timer(classname(), "~ObsVector"); - data_.reset(); - Log::trace() << "ObsVector::~ObsVector done" << std::endl; } // ----------------------------------------------------------------------------- diff --git a/src/oops/interface/State.h b/src/oops/interface/State.h index 3c2cac8e8..37deb13c7 100644 --- a/src/oops/interface/State.h +++ b/src/oops/interface/State.h @@ -22,7 +22,6 @@ #include "oops/interface/Geometry.h" #include "oops/util/DateTime.h" #include "oops/util/gatherPrint.h" -#include "oops/util/MemoryCounter.h" #include "oops/util/ObjectCounter.h" #include "oops/util/Printable.h" #include "oops/util/Serializable.h" @@ -90,8 +89,8 @@ State::State(const Geometry_ & resol, const Variables & vars, { Log::trace() << "State::State starting" << std::endl; util::Timer timer(classname(), "State"); - util::MemoryCounter mem(classname()); state_.reset(new State_(resol.geometry(), vars, time)); + this->setObjectSize(state_->serialSize()*sizeof(double)); Log::trace() << "State::State done" << std::endl; } @@ -103,7 +102,6 @@ State::State(const Geometry_ & resol, const eckit::Configuration & conf) { Log::trace() << "State::State read starting" << std::endl; util::Timer timer(classname(), "State"); - util::MemoryCounter mem(classname()); eckit::LocalConfiguration myconf; if (conf.has("states")) { @@ -118,6 +116,7 @@ State::State(const Geometry_ & resol, const eckit::Configuration & conf) } state_.reset(new State_(resol.geometry(), myconf)); + this->setObjectSize(state_->serialSize()*sizeof(double)); Log::trace() << "State::State read done" << std::endl; } @@ -129,8 +128,8 @@ State::State(const Geometry_ & resol, const State & other) { Log::trace() << "State::State interpolated starting" << std::endl; util::Timer timer(classname(), "State"); - util::MemoryCounter mem(classname()); state_.reset(new State_(resol.geometry(), *other.state_)); + this->setObjectSize(state_->serialSize()*sizeof(double)); Log::trace() << "State::State interpolated done" << std::endl; } @@ -141,8 +140,8 @@ State::State(const State & other) : state_(), commTime_(other.commTime_) { Log::trace() << "State::State starting copy" << std::endl; util::Timer timer(classname(), "State"); - util::MemoryCounter mem(classname()); state_.reset(new State_(*other.state_)); + this->setObjectSize(state_->serialSize()*sizeof(double)); Log::trace() << "State::State copy done" << std::endl; } diff --git a/src/oops/runs/Run.cc b/src/oops/runs/Run.cc index 1d82798fa..0cbecd3d7 100644 --- a/src/oops/runs/Run.cc +++ b/src/oops/runs/Run.cc @@ -19,8 +19,8 @@ #include "oops/runs/Application.h" #include "oops/util/LibOOPS.h" #include "oops/util/Logger.h" -#include "oops/util/MemoryHelper.h" #include "oops/util/ObjectCountHelper.h" +#include "oops/util/printRunStats.h" #include "oops/util/TimerHelper.h" #ifdef ENABLE_GPTL @@ -101,7 +101,6 @@ Run::Run(int argc, char** argv) : eckit::Main(argc, argv, "OOPS_HOME"), config_( Log::info() << "Full configuration is:" << *config_ << std::endl; // Start measuring performance - util::MemoryHelper::start(); util::TimerHelper::start(); util::ObjectCountHelper::start(); } @@ -115,6 +114,7 @@ Run::~Run() { // ----------------------------------------------------------------------------- int Run::execute(const Application & app) { + util::printRunStats("Run start", true); int status = 1; Log::info() << "Run: Starting " << app << std::endl; try { @@ -135,12 +135,12 @@ int Run::execute(const Application & app) { status = 1; Log::error() << "Unknown exception: " << app << " terminating..." << std::endl; } - Log::info() << "Run: Finishing " << app << std::endl; + Log::info() << std::endl << "Run: Finishing " << app << std::endl; // Performance diagnostics util::ObjectCountHelper::stop(); util::TimerHelper::stop(); - util::MemoryHelper::stop(); + util::printRunStats("Run end", true); Log::info() << "Run: Finishing " << app << " with status = " << status << std::endl; return status; diff --git a/src/oops/runs/Variational.h b/src/oops/runs/Variational.h index f49b0510f..c57968443 100644 --- a/src/oops/runs/Variational.h +++ b/src/oops/runs/Variational.h @@ -38,6 +38,7 @@ #include "oops/util/DateTime.h" #include "oops/util/Duration.h" #include "oops/util/Logger.h" +#include "oops/util/printRunStats.h" namespace oops { @@ -64,6 +65,7 @@ template class Variational : public Application { /// The background is constructed inside the cost function because its valid /// time within the assimilation window can be different (3D-Var vs. 4D-Var), /// it can be 3D or 4D (strong vs weak constraint), etc... + util::printRunStats("Variational start"); // Setup cost function eckit::LocalConfiguration cfConf(fullConfig, "cost function"); @@ -99,6 +101,7 @@ template class Variational : public Application { // Save ObsAux xx.obsVar().write(cfConf); + util::printRunStats("Variational end"); return 0; } // ----------------------------------------------------------------------------- diff --git a/src/oops/util/MemoryCounter.cc b/src/oops/util/MemoryCounter.cc deleted file mode 100644 index edfa61c8e..000000000 --- a/src/oops/util/MemoryCounter.cc +++ /dev/null @@ -1,35 +0,0 @@ -/* - * (C) Copyright 2021-2021 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "oops/util/MemoryCounter.h" - -#include - -#include "eckit/system/ResourceUsage.h" -#include "oops/util/MemoryHelper.h" - -namespace util { - -// ----------------------------------------------------------------------------- - -MemoryCounter::MemoryCounter(const std::string & classname) - : name_(classname), rss_(eckit::system::ResourceUsage().maxResidentSetSize()) -{} - -// ----------------------------------------------------------------------------- - -MemoryCounter::~MemoryCounter() { - size_t current = eckit::system::ResourceUsage().maxResidentSetSize(); -// convert bytes to MiB - double mem = (static_cast(current)-static_cast(rss_))/1048576.0; - MemoryHelper::add(name_, mem); -} - -// ----------------------------------------------------------------------------- - -} // namespace util - diff --git a/src/oops/util/MemoryHelper.cc b/src/oops/util/MemoryHelper.cc deleted file mode 100644 index 6c64789d8..000000000 --- a/src/oops/util/MemoryHelper.cc +++ /dev/null @@ -1,153 +0,0 @@ -/* - * (C) Copyright 2021-2021 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#include "oops/util/MemoryHelper.h" - -#include -#include -#include - -#include "eckit/io/ResizableBuffer.h" -#include "eckit/mpi/Comm.h" -#include "eckit/serialisation/ResizableMemoryStream.h" -#include "eckit/system/ResourceUsage.h" - -#include "oops/mpi/mpi.h" -#include "oops/util/Logger.h" -#include "oops/util/MemoryCounter.h" - -namespace util { - -// ----------------------------------------------------------------------------- - -MemoryHelper & MemoryHelper::getHelper() { - static MemoryHelper theHelper; - return theHelper; -} - -// ----------------------------------------------------------------------------- - -void MemoryHelper::start() { - getHelper().on_ = true; -} - -// ----------------------------------------------------------------------------- - -void MemoryHelper::stop() { - oops::Log::stats() << getHelper() << std::endl; - getHelper().stats_.clear(); - getHelper().on_ = false; -} - -// ----------------------------------------------------------------------------- - -void MemoryHelper::add(const std::string & name, const double rss) { - if (getHelper().on_) { - getHelper().stats_[name][0] += 1.0; - getHelper().stats_[name][1] += rss; - if (rss > getHelper().stats_[name][2]) getHelper().stats_[name][2] = rss; - } -} - -// ----------------------------------------------------------------------------- - -MemoryHelper::MemoryHelper(): on_(false), stats_() {} - -// ----------------------------------------------------------------------------- - -MemoryHelper::~MemoryHelper() {} - -// ----------------------------------------------------------------------------- - -void MemoryHelper::print(std::ostream & os) const { - size_t ntasks = oops::mpi::world().size(); - int tag = 1235; - size_t rssbyte = eckit::system::ResourceUsage().maxResidentSetSize(); - double rss = static_cast(rssbyte)/(1024*1024); - - if (oops::mpi::world().rank() > 0) { -// Tasks send their numbers to task 0 - eckit::ResizableBuffer bufr(8000); - eckit::ResizableMemoryStream sstr(bufr); - sstr << stats_.size(); - for (auto jt = stats_.begin(); jt != stats_.end(); ++jt) { - sstr << jt->first << jt->second[0] << jt->second[1] << jt->second[2]; - } - sstr << rss; - oops::mpi::world().send(static_cast(bufr.data()), sstr.position(), 0, tag); - } else { -// Task 0 receives stats from other tasks - std::map> stats = stats_; - double rssmin = rss; - double rssmax = rss; - double rsstot = rss; - for (size_t from = 1; from < ntasks; ++from) { - eckit::mpi::Status st = oops::mpi::world().probe(from, tag); - size_t size = oops::mpi::world().getCount(st); - eckit::ResizableBuffer bufr(size); - bufr.zero(); - - oops::mpi::world().receive(static_cast(bufr.data()), bufr.size(), from, tag); - eckit::ResizableMemoryStream sstr(bufr); - - std::string name; - double count; - double sum; - double max; - size_t nstats; - sstr >> nstats; - for (size_t jj = 0; jj < nstats; ++jj) { - sstr >> name; - sstr >> count; - sstr >> sum; - sstr >> max; - stats[name][0] += count; - stats[name][1] += sum; - if (max > stats[name][2]) stats[name][2] = max; - } - sstr >> rss; - if (rss < rssmin) rssmin = rss; - if (rss > rssmax) rssmax = rss; - rsstot += rss; - } - -// Print global statistics - int table_width = 64; - std::ostringstream title_s; - title_s << " Memory Statistics (" << std::setw(5) << ntasks << " MPI tasks) "; - std::string title = title_s.str(); - float title_half_width = (table_width-title.size())/2.; - os << std::endl << std::string(table_width, '-') << std::endl - << std::string(std::floor(title_half_width), '-') - << title << std::string(std::ceil(title_half_width), '-') << std::endl - << std::string(table_width, '-') << std::endl - << std::setw(30) << std::left << " " - << std::setw(11) << std::right << "Count/task" - << std::setw(11) << std::right << "Avg (MiB)" - << std::setw(12) << std::right << "Max (MiB)" - << std::endl; - for (auto jt = stats.begin(); jt != stats.end(); ++jt) { - os << std::setw(30) << std::left << jt->first << ": " - << std::setw(12) << std::right << std::fixed << std::setprecision(0) - << std::setw(8) << jt->second[0]/ntasks - << std::setw(12) << std::right << std::fixed << std::setprecision(2) - << std::setw(12) << jt->second[1]/jt->second[0] - << std::setw(12) << jt->second[2] - << std::endl; - } - os << std::string(table_width, '-') << std::endl; - - os << "Total memory used per MPI task: min = " << std::setprecision(0) - << rssmin << " MiB, max = " << rssmax << " MiB." << std::endl; - os << "Total memory used: " << std::setprecision(0) << rsstot << " MiB." << std::endl; - os << std::string(table_width, '-') << std::endl; - } -} - -// ----------------------------------------------------------------------------- - -} // namespace util diff --git a/src/oops/util/MemoryHelper.h b/src/oops/util/MemoryHelper.h deleted file mode 100644 index aa27021c1..000000000 --- a/src/oops/util/MemoryHelper.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * (C) Copyright 2021-2021 UCAR - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - */ - -#pragma once - -#include -#include -#include - -#include "oops/util/Printable.h" - -namespace util { - -// ----------------------------------------------------------------------------- - -class MemoryHelper : public util::Printable { - public: - static void start(); - static void stop(); - static void add(const std::string &, const double); - MemoryHelper(const MemoryHelper&) = delete; - ~MemoryHelper(); - - private: - static MemoryHelper & getHelper(); - MemoryHelper(); - void print(std::ostream &) const; - - bool on_; - std::map> stats_; -}; - -// ----------------------------------------------------------------------------- - -} // namespace util - diff --git a/src/oops/util/ObjectCountHelper.cc b/src/oops/util/ObjectCountHelper.cc index 66be60125..7de7412c0 100644 --- a/src/oops/util/ObjectCountHelper.cc +++ b/src/oops/util/ObjectCountHelper.cc @@ -21,36 +21,39 @@ namespace util { // ----------------------------------------------------------------------------- -std::map > * ObjectCountHelper::counters_ = 0; +std::map > ObjectCountHelper::counters_; // ----------------------------------------------------------------------------- void ObjectCountHelper::start() { - ASSERT(!counters_); - counters_ = new std::map < std::string, std::shared_ptr >(); - ASSERT(counters_); oops::Log::stats() << "ObjectCountHelper started." << std::endl; } // ----------------------------------------------------------------------------- void ObjectCountHelper::stop() { - ASSERT(counters_); typedef std::map >::iterator it; oops::Log::stats() << " " << std::endl; oops::Log::stats() << "----------------------------------------------------------------------" - << std::endl; + << "------------" << std::endl; oops::Log::stats() << "--------------------------- Object counts ----------------------------" - << std::endl; + << "------------" << std::endl; oops::Log::stats() << "----------------------------------------------------------------------" + << "------------" << std::endl; + oops::Log::stats() << std::setw(34) << std::left << " " + << std::setw(8) << std::right << "Total" + << std::setw(9) << std::right << "Simult." + << std::setw(7) << std::right << "Remain" + << std::setw(12) << std::right << "Avg (Mb)" + << std::setw(12) << std::right << "HWM (Mb)" << std::endl; - for (it jc = counters_->begin(); jc != counters_->end(); ++jc) { + for (it jc = counters_.begin(); jc != counters_.end(); ++jc) { oops::Log::stats() << std::setw(32) << std::left << jc->first << ": " << *(jc->second) << std::endl; } oops::Log::stats() << "----------------------------- Object counts --------------------------" - << std::endl; - counters_->clear(); + << "------------" << std::endl; + counters_.clear(); } // ----------------------------------------------------------------------------- @@ -58,10 +61,10 @@ void ObjectCountHelper::stop() { std::shared_ptr ObjectCountHelper::create(const std::string & cname) { std::shared_ptr pcount; typedef std::map >::iterator it; - it jj = counters_->find(cname); - if (jj == counters_->end()) { + it jj = counters_.find(cname); + if (jj == counters_.end()) { pcount.reset(new ObjectCountHelper(cname)); - (*counters_)[cname] = pcount; + counters_[cname] = pcount; } else { pcount = jj->second; } @@ -71,7 +74,7 @@ std::shared_ptr ObjectCountHelper::create(const std::string & // ----------------------------------------------------------------------------- ObjectCountHelper::ObjectCountHelper(const std::string & cname) - : current_(0), created_(0), max_(0) {} + : current_(0), created_(0), max_(0), bytes_(0), maxbytes_(0), totbytes_(0) {} // ----------------------------------------------------------------------------- @@ -87,16 +90,35 @@ void ObjectCountHelper::oneMore() { // ----------------------------------------------------------------------------- -void ObjectCountHelper::oneLess() { +void ObjectCountHelper::oneLess(const size_t & bytes) { --current_; + bytes_ -= bytes; +} + +// ----------------------------------------------------------------------------- + +void ObjectCountHelper::setSize(const size_t & bytes) { + bytes_ += bytes; + maxbytes_ = std::max(maxbytes_, bytes_); + totbytes_ += bytes; } // ----------------------------------------------------------------------------- void ObjectCountHelper::print(std::ostream & out) const { - out << "simultaneous = " << std::setw(5) << std::right << max_ - << ", total = " << std::setw(6) << std::right << created_; - if (current_ > 0) out << ", not destructed = " << std::setw(5) << std::right << current_; + out << std::setw(8) << std::right << created_ + << std::setw(8) << std::right << max_; + if (current_ > 0) { + out << std::setw(8) << std::right << current_; + } else { + out << std::setw(8) << " "; + } + if (maxbytes_ > 0) { + double size = static_cast(totbytes_) / static_cast(created_) / 1.e+6; + out << std::setw(12) << std::right << std::fixed << std::setprecision(2) << size; + double hwm = static_cast(maxbytes_) / 1.e+6; + out << std::setw(12) << std::right << std::fixed << std::setprecision(2) << hwm; + } } // ----------------------------------------------------------------------------- diff --git a/src/oops/util/ObjectCountHelper.h b/src/oops/util/ObjectCountHelper.h index 1892bab6a..f8fd39db5 100644 --- a/src/oops/util/ObjectCountHelper.h +++ b/src/oops/util/ObjectCountHelper.h @@ -32,17 +32,21 @@ class ObjectCountHelper : public util::Printable, ~ObjectCountHelper(); void oneMore(); - void oneLess(); + void oneLess(const size_t &); + void setSize(const size_t &); private: - static std::map< std::string, std::shared_ptr > * counters_; + static std::map< std::string, std::shared_ptr > counters_; explicit ObjectCountHelper(const std::string &); void print(std::ostream &) const; - unsigned int current_; - unsigned int created_; - unsigned int max_; + size_t current_; + size_t created_; + size_t max_; + size_t bytes_; + size_t maxbytes_; + size_t totbytes_; }; // ----------------------------------------------------------------------------- diff --git a/src/oops/util/ObjectCounter.h b/src/oops/util/ObjectCounter.h index 6a2fb500c..e4ee366bd 100644 --- a/src/oops/util/ObjectCounter.h +++ b/src/oops/util/ObjectCounter.h @@ -13,6 +13,8 @@ #include +#include "eckit/exception/Exceptions.h" + #include "oops/util/ObjectCountHelper.h" namespace util { @@ -22,17 +24,26 @@ namespace util { template class ObjectCounter { public: - ObjectCounter(): count_(ObjectCountHelper::create(T::classname())) + ObjectCounter(): count_(ObjectCountHelper::create(T::classname())), bytes_(0) {count_->oneMore();} - ObjectCounter(const ObjectCounter & other) : count_(other.count_) + ObjectCounter(const ObjectCounter & other) : count_(other.count_), bytes_(0) {count_->oneMore();} ~ObjectCounter() - {count_->oneLess();} + {count_->oneLess(bytes_);} + + protected: +// Optionally set object size (in bytes) + void setObjectSize(const size_t & bytes) { + ASSERT(bytes_ == 0); // can only be set once + bytes_ = bytes; + count_->setSize(bytes_); + } private: std::shared_ptr count_; + size_t bytes_; }; // ----------------------------------------------------------------------------- diff --git a/src/oops/util/Timer.cc b/src/oops/util/Timer.cc index 745814ae1..aa178a632 100644 --- a/src/oops/util/Timer.cc +++ b/src/oops/util/Timer.cc @@ -23,6 +23,8 @@ namespace { // Local global constants constexpr int METHOD_PRINT_WIDTH = 60; } +// ----------------------------------------------------------------------------- + /** InitTime * * A class to be static initialized that measures time deltas since program initialization @@ -49,15 +51,21 @@ Timer::Timer(const std::string & class_name, const std::string & method_name) : name_(class_name + "::" + method_name), start_(ClockT::now()) { } +// ----------------------------------------------------------------------------- + Timer::~Timer() { std::chrono::duration dt = ClockT::now() - start_; // elapsed millisecs TimerHelper::add(name_, dt.count()); } +// ----------------------------------------------------------------------------- + LoggingTimer::LoggingTimer(const std::string & class_name, const std::string & method_name) : LoggingTimer(class_name, method_name, oops::Log::timer()) { } +// ----------------------------------------------------------------------------- + LoggingTimer::LoggingTimer(const std::string & class_name, const std::string & method_name, std::ostream& log) : Timer(class_name, method_name), log_(log) @@ -67,6 +75,8 @@ LoggingTimer::LoggingTimer(const std::string & class_name, << " Start: " << std::fixed << std::setprecision(2) << st << std::endl; } +// ----------------------------------------------------------------------------- + LoggingTimer::~LoggingTimer() { double st = init_time.elapsed_sec(start_); @@ -78,4 +88,10 @@ LoggingTimer::~LoggingTimer() // ----------------------------------------------------------------------------- +double timeStamp() { + return init_time.elapsed_sec(Timer::ClockT::now()); +} + +// ----------------------------------------------------------------------------- + } // namespace util diff --git a/src/oops/util/Timer.h b/src/oops/util/Timer.h index 05073c4c5..6ece000c4 100644 --- a/src/oops/util/Timer.h +++ b/src/oops/util/Timer.h @@ -34,6 +34,8 @@ class Timer { TimeT start_; }; +// ----------------------------------------------------------------------------- + class LoggingTimer : public Timer { public: LoggingTimer(const std::string &class_name, const std::string &method_name); @@ -46,6 +48,10 @@ class LoggingTimer : public Timer { // ----------------------------------------------------------------------------- +double timeStamp(); + +// ----------------------------------------------------------------------------- + } // namespace util #endif // OOPS_UTIL_TIMER_H_ diff --git a/src/oops/util/printRunStats.cc b/src/oops/util/printRunStats.cc new file mode 100644 index 000000000..e2317b174 --- /dev/null +++ b/src/oops/util/printRunStats.cc @@ -0,0 +1,68 @@ +/* + * (C) Copyright 2021-2021 UCAR + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + */ + +#include "oops/util/printRunStats.h" + +#include +#include +#include + +#include "eckit/system/ResourceUsage.h" + +#include "oops/mpi/mpi.h" +#include "oops/util/Logger.h" +#include "oops/util/Timer.h" + +namespace util { + +// ----------------------------------------------------------------------------- + +void printRunStats(const std::string & name, const bool alltasks) { + size_t rssbyte = eckit::system::ResourceUsage().maxResidentSetSize(); + double rss = static_cast(rssbyte); + + double factor = 1.0e+6; + std::string unit = " Mb"; + + if (alltasks) { + size_t ntasks = oops::mpi::world().size(); + std::vector zss(ntasks); + oops::mpi::world().gather(rss, zss, 0); + + if (oops::mpi::world().rank() == 0) { + double rssmin = rss; + double rssmax = rss; + double rsstot = rss; + for (size_t jj = 1; jj < ntasks; ++jj) { + if (zss[jj] < rssmin) rssmin = zss[jj]; + if (zss[jj] > rssmax) rssmax = zss[jj]; + rsstot += zss[jj]; + } + + if (rsstot >= 1.0e+9) {factor = 1.0e+9; unit = " Gb";} + oops::Log::stats() << std::left << std::setw(40) << name << " - Runtime: " + << std::fixed << std::right << std::setprecision(2) + << std::setw(9) << timeStamp() << " sec, Memory: total: " + << std::setw(8) << rsstot / factor << unit; + if (rssmin >= 1.0e+9) {factor = 1.0e+9; unit = " Gb";} else {factor = 1.0e+6; unit = " Mb";} + oops::Log::stats() << ", per task: min = " + << std::right << std::setprecision(2) << std::fixed + << std::setw(8) << rssmin / factor << unit << ", max = " + << std::setw(8) << rssmax / factor << unit << std::endl; + } + } else { + if (rss >= 1.0e+9) {factor = 1.0e+9; unit = " Gb";} + oops::Log::stats() << std::left << std::setw(40) << name << " - Runtime: " + << std::fixed << std::right << std::setprecision(2) + << std::setw(8) << timeStamp() << " sec, Local Memory: " + << std::setw(8) << rss / factor << unit << std::endl; + } +} + +// ----------------------------------------------------------------------------- + +} // namespace util diff --git a/src/oops/util/MemoryCounter.h b/src/oops/util/printRunStats.h similarity index 68% rename from src/oops/util/MemoryCounter.h rename to src/oops/util/printRunStats.h index 92b71be29..c62de9d61 100644 --- a/src/oops/util/MemoryCounter.h +++ b/src/oops/util/printRunStats.h @@ -13,19 +13,9 @@ namespace util { // ----------------------------------------------------------------------------- -class MemoryCounter { - public: - explicit MemoryCounter(const std::string &); - ~MemoryCounter(); - - MemoryCounter(const MemoryCounter&) = delete; - - private: - std::string name_; - size_t rss_; -}; +// Print runtime and memory statistics (only for task 0 by default) +void printRunStats(const std::string &, const bool alltasks = false); // ----------------------------------------------------------------------------- } // namespace util - From a9b7291eb2228e8c6c11188a4b535372a18c7962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Wed, 26 May 2021 10:05:32 -0600 Subject: [PATCH 133/142] Fix geovals bias variables in trajectory (#1225) --- src/oops/base/ObserverTLAD.h | 24 ++++++++++++++++-------- src/oops/base/ObserversTLAD.h | 8 +++----- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/oops/base/ObserverTLAD.h b/src/oops/base/ObserverTLAD.h index 39f7768e0..dc0b8c22c 100644 --- a/src/oops/base/ObserverTLAD.h +++ b/src/oops/base/ObserverTLAD.h @@ -48,8 +48,8 @@ class ObserverTLAD { ObserverTLAD(const ObsSpace_ &, const eckit::Configuration &); ~ObserverTLAD() {} - std::shared_ptr initializeTraj(const Geometry_ &); - void finalizeTraj(const ObsAuxCtrl_ &); + std::shared_ptr initializeTraj(const Geometry_ &, const ObsAuxCtrl_ &); + void finalizeTraj(); std::shared_ptr initializeTL(); void finalizeTL(const ObsAuxIncr_ &, ObsVector_ &); @@ -65,6 +65,7 @@ class ObserverTLAD { std::unique_ptr locations_; // locations util::DateTime winbgn_; // Begining of assimilation window util::DateTime winend_; // End of assimilation window + const ObsAuxCtrl_ * ybias_; bool init_; }; @@ -75,17 +76,19 @@ ObserverTLAD::ObserverTLAD(const ObsSpace_ & obsdb, const eckit::Con hoptlad_(obspace_, conf.has("linear obs operator") ? eckit::LocalConfiguration(conf, "linear obs operator") : eckit::LocalConfiguration(conf, "obs operator")), - getvals_(), locations_(), winbgn_(obsdb.windowStart()), winend_(obsdb.windowEnd()), init_(false) + getvals_(), locations_(), winbgn_(obsdb.windowStart()), winend_(obsdb.windowEnd()), + ybias_(nullptr), init_(false) { Log::trace() << "ObserverTLAD::ObserverTLAD" << std::endl; } // ----------------------------------------------------------------------------- template std::shared_ptr> -ObserverTLAD::initializeTraj(const Geometry_ & geom) { +ObserverTLAD::initializeTraj(const Geometry_ & geom, const ObsAuxCtrl_ & ybias) { Log::trace() << "ObserverTLAD::initializeTraj start" << std::endl; + ybias_ = &ybias; -// hop is only needed to get locations +// hop is only needed to get locations and requiredVars ObsOperator_ hop(obspace_, eckit::LocalConfiguration(obsconfig_, "obs operator")); locations_.reset(new Locations_(hop.locations())); @@ -95,8 +98,13 @@ ObserverTLAD::initializeTraj(const Geometry_ & geom) { eckit::LocalConfiguration(obsconfig_, "get values") : eckit::LocalConfiguration(obsconfig_, ""))); +// Set up variables that will be requested from the model + Variables geovars; + geovars += hop.requiredVars(); + geovars += ybias_->requiredVars(); + getvals_.reset(new GetValTLAD_(gvconf, geom, winbgn_, winend_, - *locations_, hop.requiredVars(), hoptlad_.requiredVars())); + *locations_, geovars, hoptlad_.requiredVars())); init_ = true; Log::trace() << "ObserverTLAD::initializeTraj done" << std::endl; @@ -104,7 +112,7 @@ ObserverTLAD::initializeTraj(const Geometry_ & geom) { } // ----------------------------------------------------------------------------- template -void ObserverTLAD::finalizeTraj(const ObsAuxCtrl_ & ybias) { +void ObserverTLAD::finalizeTraj() { Log::trace() << "ObserverTLAD::finalizeTraj start" << std::endl; ASSERT(init_); @@ -112,7 +120,7 @@ void ObserverTLAD::finalizeTraj(const ObsAuxCtrl_ & ybias) { std::unique_ptr geovals = getvals_->finalize(); /// Set linearization trajectory for H(x) - hoptlad_.setTrajectory(*geovals, ybias); + hoptlad_.setTrajectory(*geovals, *ybias_); init_ = false; Log::trace() << "ObserverTLAD::finalizeTraj done" << std::endl; diff --git a/src/oops/base/ObserversTLAD.h b/src/oops/base/ObserversTLAD.h index 6beb7d8b4..bb2731b9c 100644 --- a/src/oops/base/ObserversTLAD.h +++ b/src/oops/base/ObserversTLAD.h @@ -60,7 +60,6 @@ class ObserversTLAD { private: std::vector> observers_; - const ObsAuxCtrls_ * ybias_; util::DateTime winbgn_; util::DateTime winend_; }; @@ -69,7 +68,7 @@ class ObserversTLAD { template ObserversTLAD::ObserversTLAD(const ObsSpaces_ & obspaces, const eckit::Configuration & obsConfig) - : observers_(), ybias_(nullptr), winbgn_(obspaces.windowStart()), winend_(obspaces.windowEnd()) + : observers_(), winbgn_(obspaces.windowStart()), winend_(obspaces.windowEnd()) { Log::trace() << "ObserversTLAD::ObserversTLAD start" << std::endl; std::vector obsconfs = obsConfig.getSubConfigurations(); @@ -86,10 +85,9 @@ template void ObserversTLAD::initializeTraj(const Geometry_ & geom, const ObsAuxCtrls_ & ybias, PostProcTLAD_ & pp) { Log::trace() << "ObserversTLAD::initializeTraj start" << std::endl; - ybias_ = &ybias; std::shared_ptr getvals(new GetValueTLADs_(winbgn_, winend_)); for (size_t jj = 0; jj < observers_.size(); ++jj) { - if (observers_[jj]) getvals->append(observers_[jj]->initializeTraj(geom)); + if (observers_[jj]) getvals->append(observers_[jj]->initializeTraj(geom, ybias[jj])); } pp.enrollProcessor(getvals); Log::trace() << "ObserversTLAD::initializeTraj done" << std::endl; @@ -99,7 +97,7 @@ template void ObserversTLAD::finalizeTraj() { Log::trace() << "ObserversTLAD::finalizeTraj start" << std::endl; for (size_t jj = 0; jj < observers_.size(); ++jj) { - if (observers_[jj]) observers_[jj]->finalizeTraj((*ybias_)[jj]); + if (observers_[jj]) observers_[jj]->finalizeTraj(); } Log::trace() << "ObserversTLAD::finalizeTraj done" << std::endl; } From 8fc4374832b328a8485ab39e3bcd59456ec75723 Mon Sep 17 00:00:00 2001 From: Maryam Abdi Date: Tue, 1 Jun 2021 09:47:35 -0600 Subject: [PATCH 134/142] update ci containers (#1232) * remove ecmwf * use ecbuild35 branch of jedi-build --- CI/CMakeLists.txt | 2 -- CI/buildspec_clang.yml | 2 +- CI/buildspec_gnu.yml | 5 ----- CI/buildspec_intel.yml | 4 ---- 4 files changed, 1 insertion(+), 12 deletions(-) diff --git a/CI/CMakeLists.txt b/CI/CMakeLists.txt index d04b64bdb..ed68d35b0 100644 --- a/CI/CMakeLists.txt +++ b/CI/CMakeLists.txt @@ -20,8 +20,6 @@ set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" ) ecbuild_bundle_initialize() -ecbuild_bundle( PROJECT fckit GIT "https://github.com/jcsda-internal/fckit.git" ) -ecbuild_bundle( PROJECT atlas GIT "https://github.com/jcsda-internal/atlas.git" ) ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" ) ecbuild_bundle_finalize() diff --git a/CI/buildspec_clang.yml b/CI/buildspec_clang.yml index 48692b3e3..b1ff8b769 100644 --- a/CI/buildspec_clang.yml +++ b/CI/buildspec_clang.yml @@ -54,7 +54,7 @@ phases: - echo ${CODEBUILD_RESOLVED_SOURCE_VERSION} > artifacts/commit_sha.txt # get jedi-build-package - - git clone https://$GIT_USER:$GIT_PASS@github.com/jcsda-internal/jedi-build-package + - git clone -b feature/ecbuild35 https://$GIT_USER:$GIT_PASS@github.com/jcsda-internal/jedi-build-package - cd jedi-build-package - pip install --user -e . diff --git a/CI/buildspec_gnu.yml b/CI/buildspec_gnu.yml index 7222ac41b..40e92347e 100644 --- a/CI/buildspec_gnu.yml +++ b/CI/buildspec_gnu.yml @@ -59,11 +59,6 @@ phases: # oops - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/oops $CODEBUILD_GIT_BRANCH oops /jcsda/oops-bundle develop - # fckit - - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/fckit $CODEBUILD_GIT_BRANCH_FORK fckit /jcsda/oops-bundle release-stable - # atlas - - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/atlas $CODEBUILD_GIT_BRANCH_FORK atlas /jcsda/oops-bundle release-stable - # move CMakeLists.txt from oops/CI to bundle directory - cp CMakeLists.txt /jcsda/oops-bundle/CMakeLists.txt diff --git a/CI/buildspec_intel.yml b/CI/buildspec_intel.yml index 327c65eb1..143025eaa 100644 --- a/CI/buildspec_intel.yml +++ b/CI/buildspec_intel.yml @@ -57,10 +57,6 @@ phases: # oops - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/oops $CODEBUILD_GIT_BRANCH oops /jcsda/oops-bundle develop - # fckit - - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/fckit $CODEBUILD_GIT_BRANCH_FORK fckit /jcsda/oops-bundle release-stable - # atlas - - ./clone.sh $GIT_USER $GIT_PASS jcsda-internal/atlas $CODEBUILD_GIT_BRANCH_FORK atlas /jcsda/oops-bundle release-stable # move CMakeLists.txt from oops/CI to bundle directory - cp CMakeLists.txt /jcsda/oops-bundle/CMakeLists.txt From cb043ae271409e1240c417f5d4c165d610894128 Mon Sep 17 00:00:00 2001 From: frolovsa <55715838+frolovsa@users.noreply.github.com> Date: Wed, 2 Jun 2021 08:47:18 -0600 Subject: [PATCH 135/142] Feature/hybrid gain (#1227) * add oops/runs/HybridGain.h to Cmake file * added new version of the hybrid gain * coding norms * add assert * Update HybridGain.h Co-authored-by: Anna Shlyaeva --- qg/test/CMakeLists.txt | 16 +++- ...bridgain.yaml => hybridgain_analysis.yaml} | 6 +- qg/test/testinput/hybridgain_increment.yaml | 44 +++++++++++ ...bridgain.test => hybridgain_analysis.test} | 4 +- qg/test/testoutput/hybridgain_increment.test | 57 ++++++++++++++ src/CMakeLists.txt | 1 + src/oops/runs/HybridGain.h | 77 ++++++++++++++----- 7 files changed, 176 insertions(+), 29 deletions(-) rename qg/test/testinput/{hybridgain.yaml => hybridgain_analysis.yaml} (88%) create mode 100644 qg/test/testinput/hybridgain_increment.yaml rename qg/test/testoutput/{hybridgain.test => hybridgain_analysis.test} (98%) create mode 100644 qg/test/testoutput/hybridgain_increment.test diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index fc67e78fd..dffeebce5 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -65,7 +65,8 @@ list( APPEND qg_testinput testinput/getvalues.yaml testinput/hofx.yaml testinput/hofx3d.yaml - testinput/hybridgain.yaml + testinput/hybridgain_analysis.yaml + testinput/hybridgain_increment.yaml testinput/interfaces.yaml testinput/letkf.yaml testinput/lineargetvalues.yaml @@ -125,7 +126,8 @@ list( APPEND qg_testoutput testoutput/gen_ens_pert_B.test testoutput/hofx.test testoutput/hofx3d.test - testoutput/hybridgain.test + testoutput/hybridgain_analysis.test + testoutput/hybridgain_increment.test testoutput/letkf.test testoutput/make_obs_3d.test testoutput/make_obs_4d_12h.test @@ -445,9 +447,15 @@ ecbuild_add_test( TARGET test_qg_ens_recenter COMMAND qg_ens_recenter.x TEST_DEPENDS test_qg_gen_ens_pert_B ) -ecbuild_add_test( TARGET test_qg_hybridgain +ecbuild_add_test( TARGET test_qg_hybridgain_analysis OMP 2 - ARGS testinput/hybridgain.yaml + ARGS testinput/hybridgain_analysis.yaml + COMMAND qg_hybridgain.x + TEST_DEPENDS test_qg_gen_ens_pert_B ) + +ecbuild_add_test( TARGET test_qg_hybridgain_increment + OMP 2 + ARGS testinput/hybridgain_increment.yaml COMMAND qg_hybridgain.x TEST_DEPENDS test_qg_gen_ens_pert_B ) diff --git a/qg/test/testinput/hybridgain.yaml b/qg/test/testinput/hybridgain_analysis.yaml similarity index 88% rename from qg/test/testinput/hybridgain.yaml rename to qg/test/testinput/hybridgain_analysis.yaml index 26d21d076..bf3c710a7 100644 --- a/qg/test/testinput/hybridgain.yaml +++ b/qg/test/testinput/hybridgain_analysis.yaml @@ -2,11 +2,13 @@ hybrid weights: control: 0.2 ensemble: 0.8 +hybrid type: average analysis + control: date: 2010-01-01T00:00:00Z filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1D.nc -ensemble mean: +ensemble mean posterior: date: 2010-01-01T00:00:00Z filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1D.nc @@ -34,4 +36,4 @@ recentered output: date: 2010-01-01T00:00:00Z test: - reference filename: testoutput/hybridgain.test + reference filename: testoutput/hybridgain_analysis.test diff --git a/qg/test/testinput/hybridgain_increment.yaml b/qg/test/testinput/hybridgain_increment.yaml new file mode 100644 index 000000000..e1f1421e0 --- /dev/null +++ b/qg/test/testinput/hybridgain_increment.yaml @@ -0,0 +1,44 @@ +hybrid weights: + control: 0.2 + ensemble: 0.8 + +hybrid type: average increment + +control: + date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1D.nc + +ensemble mean prior: + date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1D.nc + +ensemble mean posterior: + date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1D.nc + +ensemble: +- date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.1.2009-12-31T00:00:00Z.P1D.nc +- date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.2.2009-12-31T00:00:00Z.P1D.nc +- date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.3.2009-12-31T00:00:00Z.P1D.nc +- date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.4.2009-12-31T00:00:00Z.P1D.nc +- date: 2010-01-01T00:00:00Z + filename: Data/forecast.ens.5.2009-12-31T00:00:00Z.P1D.nc + +geometry: + nx: 40 + ny: 20 + depths: [4500.0, 5500.0] + +recentered output: + datadir: Data + exp: hybridgain + type: ens + date: 2010-01-01T00:00:00Z + +test: + reference filename: testoutput/hybridgain_increment.test + test output filename: testoutput/hybridgain_increment.test.out diff --git a/qg/test/testoutput/hybridgain.test b/qg/test/testoutput/hybridgain_analysis.test similarity index 98% rename from qg/test/testoutput/hybridgain.test rename to qg/test/testoutput/hybridgain_analysis.test index adb8323b5..ae7708cf5 100644 --- a/qg/test/testoutput/hybridgain.test +++ b/qg/test/testoutput/hybridgain_analysis.test @@ -1,11 +1,11 @@ -Control prior: +Control: Valid time: 2010-01-01T00:00:00Z Resolution = 40, 20, 2 Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 -Ensemble mean: +Ensemble mean posterior: Valid time: 2010-01-01T00:00:00Z Resolution = 40, 20, 2 diff --git a/qg/test/testoutput/hybridgain_increment.test b/qg/test/testoutput/hybridgain_increment.test new file mode 100644 index 000000000..1a9ed2b36 --- /dev/null +++ b/qg/test/testoutput/hybridgain_increment.test @@ -0,0 +1,57 @@ +Control: + + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Ensemble mean posterior: + + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Ensemble mean prior: + + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +new center : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 0 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7030e+08, Max= 9.3328e+07, RMS= 1.8437e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 1 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.6931e+08, Max= 9.9327e+07, RMS= 1.7976e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 2 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.9074e+08, Max= 1.0042e+08, RMS= 1.8589e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 3 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7276e+08, Max= 9.4808e+07, RMS= 1.7978e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 +Recentered member 4 : + Valid time: 2010-01-01T00:00:00Z + Resolution = 40, 20, 2 + Streamfunction : Min= -4.7215e+08, Max= 9.9052e+07, RMS= 1.8304e+08 + Streamfunction LBC : Min= -4.0032e+08, Max= -0.0000e+00, RMS= 2.0632e+08 + Potential vorticity LBC: Min= -6.7294e-04, Max= 5.7903e-04, RMS= 4.4640e-04 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d331dfa83..2882aba5f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -238,6 +238,7 @@ oops/runs/GenEnsPertB.h oops/runs/HofX3D.h oops/runs/HofX4D.h oops/runs/HofX4Dhack.h +oops/runs/HybridGain.h oops/runs/LocalEnsembleDA.h oops/runs/RTPP.h oops/runs/Run.cc diff --git a/src/oops/runs/HybridGain.h b/src/oops/runs/HybridGain.h index 7fa6d16f3..87ee0379b 100644 --- a/src/oops/runs/HybridGain.h +++ b/src/oops/runs/HybridGain.h @@ -20,6 +20,7 @@ #include "oops/interface/State.h" #include "oops/mpi/mpi.h" #include "oops/runs/Application.h" +#include "oops/util/abor1_cpp.h" #include "oops/util/DateTime.h" namespace oops { @@ -44,38 +45,72 @@ template class HybridGain : public Application { double alphaControl = fullConfig.getDouble("hybrid weights.control"); double alphaEnsemble = fullConfig.getDouble("hybrid weights.ensemble"); + // Read hybrid type + std::string hybridType = fullConfig.getString("hybrid type", "average analysis"); + // Get control state const eckit::LocalConfiguration bkgConfig(fullConfig, "control"); - State_ x_control(resol, bkgConfig); - Log::test() << "Control prior: " << std::endl << x_control << std::endl; - const Variables vars = x_control.variables(); - - // Get ens mean - const eckit::LocalConfiguration emeanConfig(fullConfig, "ensemble mean"); - State_ x_emean(resol, emeanConfig); - Log::test() << "Ensemble mean: " << std::endl << x_emean << std::endl; + State_ xaControl(resol, bkgConfig); + Log::test() << "Control: " << std::endl << xaControl << std::endl; + const Variables vars = xaControl.variables(); + + // Get posterior ens mean + const eckit::LocalConfiguration emeanConfig(fullConfig, "ensemble mean posterior"); + State_ xaEmeanPost(resol, emeanConfig); + Log::test() << "Ensemble mean posterior: " << std::endl << xaEmeanPost << std::endl; + + // Compute new center + State_ xNewCenter(resol, vars, xaControl.validTime()); + if (hybridType == "average analysis") { + // using average of analysis (following Bonavita) + // xa_hybrid = a1*xa1 + a2*xa2 + // a1+a2 have to equal to 1 + + ASSERT(alphaControl + alphaEnsemble == 1.0); + xNewCenter.zero(); + xNewCenter.accumul(alphaControl, xaControl); + xNewCenter.accumul(alphaEnsemble, xaEmeanPost); + } else if (hybridType == "average increment") { + // using average of analysis incerments (following Whitaker) + // xa_hybrid = xf_prior + a1*xinc1 + a2*xinc2 + // Note: a1+a2 no longer need to add to one + + // Get prior ens mean + const eckit::LocalConfiguration emeanConfigPrior(fullConfig, "ensemble mean prior"); + State_ xfEmeanPrior(resol, emeanConfig); + Log::test() << "Ensemble mean prior: " << std::endl << xfEmeanPrior << std::endl; + // compute ensemble mean increment + Increment_ pertEns(resol, vars, xaControl.validTime()); + pertEns.diff(xaEmeanPost, xfEmeanPrior); + pertEns *= alphaEnsemble; + // compute control increment + Increment_ pertControl(resol, vars, xaControl.validTime()); + pertControl.diff(xaControl, xfEmeanPrior); + pertControl *= alphaControl; + // compute hybrid posterior + xNewCenter = xfEmeanPrior; + xNewCenter += pertEns; + xNewCenter += pertControl; + } else { + ABORT("Unknown hybrid gain type: " + hybridType); + } + // Output new center + eckit::LocalConfiguration centerOut(fullConfig, "recentered output"); + centerOut.set("member", static_cast(0) ); + xNewCenter.write(centerOut); + Log::test() << "new center : " << xNewCenter << std::endl; // Get ensemble configuration std::vector ensConfig; fullConfig.get("ensemble", ensConfig); unsigned nens = ensConfig.size(); - // Compute new center and save - State_ x_new_center(resol, vars, x_control.validTime()); - x_new_center.zero(); - x_new_center.accumul(alphaControl, x_control); - x_new_center.accumul(alphaEnsemble, x_emean); - eckit::LocalConfiguration centerOut(fullConfig, "recentered output"); - centerOut.set("member", static_cast(0) ); - x_new_center.write(centerOut); - Log::test() << "new center : " << x_new_center << std::endl; - - // Recenter ensemble around new centr and save + // Recenter ensemble around new center and save for (unsigned jj = 0; jj < nens; ++jj) { State_ x(resol, ensConfig[jj]); Increment_ pert(resol, vars, x.validTime()); - pert.diff(x, x_emean); - x = x_new_center; + pert.diff(x, xaEmeanPost); + x = xNewCenter; x += pert; // Save recentered member From e2a6305b183a17f1cfc6f0b80194b90c5f87ec29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20Tr=C3=A9molet?= <30638944+ytremolet@users.noreply.github.com> Date: Wed, 2 Jun 2021 17:07:01 -0600 Subject: [PATCH 136/142] Use ObsSpace close() (#1224) * Fix geovals bias variables in trajectory * Call ObsSpace::close * Change close to save * Trigger CI testing * Trigger CI testing * Trigger CI testing Co-authored-by: Stephen Herbener Co-authored-by: Maryam Abdi Co-authored-by: Anna Shlyaeva --- l95/src/lorenz95/ObsTable.cc | 9 ++++++--- l95/src/lorenz95/ObsTable.h | 2 ++ qg/model/ObsSpaceQG.cc | 6 +++++- qg/model/ObsSpaceQG.h | 5 ++++- src/oops/assimilation/CostJo.h | 10 +++++++++- src/oops/base/ObsSpaces.h | 12 ++++++++++++ src/oops/interface/ObsSpace.h | 15 ++++++++++++++- src/oops/runs/HofX3D.h | 1 + src/oops/runs/HofX4D.h | 1 + src/oops/runs/HofX4Dhack.h | 1 + src/oops/runs/LocalEnsembleDA.h | 6 +++++- 11 files changed, 60 insertions(+), 8 deletions(-) diff --git a/l95/src/lorenz95/ObsTable.cc b/l95/src/lorenz95/ObsTable.cc index 24be3b216..0852fe98f 100644 --- a/l95/src/lorenz95/ObsTable.cc +++ b/l95/src/lorenz95/ObsTable.cc @@ -67,14 +67,17 @@ ObsTable::ObsTable(const eckit::Configuration & config, const eckit::mpi::Comm & // ----------------------------------------------------------------------------- ObsTable::~ObsTable() { - if (!nameOut_.empty()) { - otWrite(nameOut_); - } oops::Log::trace() << "ObsTable::ObsTable destructed" << std::endl; } // ----------------------------------------------------------------------------- +void ObsTable::save() const { + if (!nameOut_.empty()) otWrite(nameOut_); +} + +// ----------------------------------------------------------------------------- + bool ObsTable::has(const std::string & col) const { return (data_.find(col) != data_.end()); } diff --git a/l95/src/lorenz95/ObsTable.h b/l95/src/lorenz95/ObsTable.h index f4db40911..ea20f8292 100644 --- a/l95/src/lorenz95/ObsTable.h +++ b/l95/src/lorenz95/ObsTable.h @@ -46,6 +46,8 @@ class ObsTable : public oops::ObsSpaceBase, const util::DateTime &, const util::DateTime &, const eckit::mpi::Comm &); ~ObsTable(); + void save() const; + void putdb(const std::string &, const std::vector &) const; void putdb(const std::string &, const std::vector &) const; void putdb(const std::string &, const std::vector &) const; diff --git a/qg/model/ObsSpaceQG.cc b/qg/model/ObsSpaceQG.cc index a21616c3d..1ff0ce0a5 100644 --- a/qg/model/ObsSpaceQG.cc +++ b/qg/model/ObsSpaceQG.cc @@ -109,7 +109,11 @@ ObsSpaceQG::ObsSpaceQG(const eckit::Configuration & config, const eckit::mpi::Co // ----------------------------------------------------------------------------- -ObsSpaceQG::~ObsSpaceQG() { +ObsSpaceQG::~ObsSpaceQG() {} + +// ----------------------------------------------------------------------------- + +void ObsSpaceQG::save() const { ASSERT(theObsFileCount_ > 0); theObsFileCount_--; if (theObsFileCount_ == 0) { diff --git a/qg/model/ObsSpaceQG.h b/qg/model/ObsSpaceQG.h index 90f8343b1..0447c530b 100644 --- a/qg/model/ObsSpaceQG.h +++ b/qg/model/ObsSpaceQG.h @@ -49,6 +49,9 @@ class ObsSpaceQG : public oops::ObsSpaceBase { const util::DateTime &, const util::DateTime &, const eckit::mpi::Comm &); ~ObsSpaceQG(); + /// save and close file + void save() const; + /// read data or metadata void getdb(const std::string &, int &) const; /// save data or metadata @@ -77,7 +80,7 @@ class ObsSpaceQG : public oops::ObsSpaceBase { private: void print(std::ostream &) const; - F90odb key_; // pointer to Fortran structure + mutable F90odb key_; // pointer to Fortran structure const std::string obsname_; // corresponds with obstype const util::DateTime winbgn_; // window for the observations const util::DateTime winend_; diff --git a/src/oops/assimilation/CostJo.h b/src/oops/assimilation/CostJo.h index bc332c7fe..218265b44 100644 --- a/src/oops/assimilation/CostJo.h +++ b/src/oops/assimilation/CostJo.h @@ -73,7 +73,7 @@ template class CostJo : public CostTermBase::CostJo(const eckit::Configuration & joConf, const eckit::mpi // ----------------------------------------------------------------------------- +template +CostJo::~CostJo() { + obspaces_.save(); + Log::trace() << "CostJo::~CostJo" << std::endl; +} + +// ----------------------------------------------------------------------------- + template void CostJo::setPostProc(const CtrlVar_ & xx, const eckit::Configuration & conf, PostProc_ & pp) { diff --git a/src/oops/base/ObsSpaces.h b/src/oops/base/ObsSpaces.h index 006707b27..557998b76 100644 --- a/src/oops/base/ObsSpaces.h +++ b/src/oops/base/ObsSpaces.h @@ -46,6 +46,9 @@ class ObsSpaces : public util::Printable, const eckit::mpi::Comm & time = oops::mpi::myself()); ~ObsSpaces(); +/// Save files + void save() const; + /// Access std::size_t size() const {return spaces_.size();} ObsSpace_ & operator[](const std::size_t ii) {return *spaces_.at(ii);} @@ -88,6 +91,15 @@ ObsSpaces::~ObsSpaces() {} // ----------------------------------------------------------------------------- +template +void ObsSpaces::save() const { + for (std::size_t jj = 0; jj < spaces_.size(); ++jj) { + spaces_[jj]->save(); + } +} + +// ----------------------------------------------------------------------------- + template void ObsSpaces::print(std::ostream & os) const { for (std::size_t jj = 0; jj < spaces_.size(); ++jj) { diff --git a/src/oops/interface/ObsSpace.h b/src/oops/interface/ObsSpace.h index 6d3ac817c..5d4cb7edd 100644 --- a/src/oops/interface/ObsSpace.h +++ b/src/oops/interface/ObsSpace.h @@ -74,6 +74,9 @@ class ObsSpace : public util::Printable, const eckit::mpi::Comm & timeComm() const {return time_;} +// Save to file + void save() const; + private: void print(std::ostream &) const; @@ -110,6 +113,16 @@ ObsSpace::~ObsSpace() { // ----------------------------------------------------------------------------- +template +void ObsSpace::save() const { + Log::trace() << "ObsSpace::save starting" << std::endl; + util::Timer timer(classname(), "save"); + obsdb_->save(); + Log::trace() << "ObsSpace::save done" << std::endl; +} + +// ----------------------------------------------------------------------------- + template void ObsSpace::print(std::ostream & os) const { Log::trace() << "ObsSpace::print starting" << std::endl; @@ -119,7 +132,7 @@ void ObsSpace::print(std::ostream & os) const { } // ----------------------------------------------------------------------------- -// + template const Variables & ObsSpace::obsvariables() const { Log::trace() << "ObsSpace::obsvariables starting" << std::endl; diff --git a/src/oops/runs/HofX3D.h b/src/oops/runs/HofX3D.h index 6b6ac6031..905ff4112 100644 --- a/src/oops/runs/HofX3D.h +++ b/src/oops/runs/HofX3D.h @@ -122,6 +122,7 @@ template class HofX3D : public Application { // Save H(x) Log::test() << "H(x): " << std::endl << yobs << "End H(x)" << std::endl; yobs.save("hofx"); + obspaces.save(); return 0; } diff --git a/src/oops/runs/HofX4D.h b/src/oops/runs/HofX4D.h index 88db8412f..b5d789b30 100644 --- a/src/oops/runs/HofX4D.h +++ b/src/oops/runs/HofX4D.h @@ -127,6 +127,7 @@ template class HofX4D : public Application { // Save H(x) as observations (if "make obs" == true) const bool makeobs = fullConfig.getBool("make obs", false); if (makeobs) yobs.save("ObsValue"); + obspaces.save(); return 0; } diff --git a/src/oops/runs/HofX4Dhack.h b/src/oops/runs/HofX4Dhack.h index 080d7e92a..335c7ac72 100644 --- a/src/oops/runs/HofX4Dhack.h +++ b/src/oops/runs/HofX4Dhack.h @@ -176,6 +176,7 @@ template class HofX4Dhack : public Application { Log::test() << "H(x): " << std::endl << yobs << "End H(x)" << std::endl; Log::info() << "H(x): " << std::endl << yobs << "End H(x)" << std::endl; yobs.save("hofx"); + obspaces.save(); return 0; } diff --git a/src/oops/runs/LocalEnsembleDA.h b/src/oops/runs/LocalEnsembleDA.h index da2690772..baf6c50a3 100644 --- a/src/oops/runs/LocalEnsembleDA.h +++ b/src/oops/runs/LocalEnsembleDA.h @@ -120,7 +120,10 @@ template class LocalEnsembleDA : public Applicati // quit early if running in observer-only mode bool observerOnly = driverConfig.getBool("run as observer only", false); - if (observerOnly) {return 0;} + if (observerOnly) { + obsdb.save(); + return 0; + } // calculate background mean State4D_ bkg_mean = ens_xx.mean(); @@ -232,6 +235,7 @@ template class LocalEnsembleDA : public Applicati Log::test() << "ombg RMS: " << ombg.rms() << std::endl << "oman RMS: " << oman.rms() << std::endl; } + obsdb.save(); return 0; } From 50261aa55311ba8c77bee949d573a5c760ae3f74 Mon Sep 17 00:00:00 2001 From: adammartins Date: Thu, 3 Jun 2021 16:33:49 +0100 Subject: [PATCH 137/142] Bugfix/fix comparefloat (#1231) * all maxUlpsDiff and increase tolerance * add tests using maxulpsdiff and old failing test * rephrase description * fixes for various comments Co-authored-by: Wojciech Smigaj --- l95/test/testinput/interfaces.yaml | 2 +- src/oops/util/FloatCompare.h | 68 +++++++++++++---- src/test/interface/State.h | 4 +- src/test/util/FloatCompare.h | 113 +++++++++++++++++++---------- 4 files changed, 130 insertions(+), 57 deletions(-) diff --git a/l95/test/testinput/interfaces.yaml b/l95/test/testinput/interfaces.yaml index 7a9b6872c..36dc15dee 100644 --- a/l95/test/testinput/interfaces.yaml +++ b/l95/test/testinput/interfaces.yaml @@ -105,7 +105,7 @@ observations: obs operator: {} linear obs operator test: coef TL: '0.1' - tolerance AD: '1.0e-16' + tolerance AD: '1.0e-15' tolerance TL: '1.0e-14' obs bias: bias: 0.3 diff --git a/src/oops/util/FloatCompare.h b/src/oops/util/FloatCompare.h index 7627cd83c..c4827f8d4 100644 --- a/src/oops/util/FloatCompare.h +++ b/src/oops/util/FloatCompare.h @@ -43,13 +43,18 @@ enum class TestVerbosity { /// \param verbosity /// Determines whether the test result will be logged both on success and failure, only on /// failure, or not at all. +/// \param max_ulps_diff +/// Maximum spacing between \p a and \p b in ULPs (where 1 ULP, or the unit of least precision, +/// is the spacing between two consecutive floating-point numbers) for which the function +/// returns true even if the relative difference between \p a and \p b is larger than +/// maximum_relative_difference. By default, 0. /// /// \returns False if |a - b| > max(|a|, |b|) * max_relative_difference or if either \p a or \p b /// is NaN or infinite; true otherwise. (wsmigaj: Not sure why false is returned for two infinities /// of the same sign.) template bool is_close_relative( - T a, T b, T max_relative_difference, + T a, T b, T max_relative_difference, int max_ulps_diff, const LogPrefixGenerator & log_prefix_generator, TestVerbosity verbosity = TestVerbosity::LOG_FAILURE_ONLY) { // if nan or inf values, always return false @@ -62,7 +67,7 @@ bool is_close_relative( T MaxAbs = (AbsA < AbsB ? AbsB : AbsA); // greater of AbsA, AbsB times max_relative_difference T EpsAB = MaxAbs * max_relative_difference; - bool passed = eckit::types::is_approximately_equal(a, b, EpsAB); + bool passed = eckit::types::is_approximately_equal(a, b, EpsAB, max_ulps_diff); std::size_t num_digits = std::numeric_limits::max_digits10; if (passed) { if (verbosity == TestVerbosity::LOG_SUCCESS_AND_FAILURE) { @@ -93,22 +98,32 @@ bool is_close_relative( /// \param verbosity /// Determines whether the test result will be logged both on success and failure, only on /// failure, or not at all. +/// \param max_ulps_diff +/// Maximum spacing between \p a and \p b in ULPs (where 1 ULP, or the unit of least precision, +/// is the spacing between two consecutive floating-point numbers) for which the function +/// returns true even if the relative difference between \p a and \p b is larger than +/// maximum_relative_difference. By default, 0. /// /// \returns False if |a - b| > max(|a|, |b|) * max_relative_difference or if either \p a or \p b /// is NaN or infinite; true otherwise. (wsmigaj: Not sure why false is returned for two infinities /// of the same sign.) template bool is_close_relative( - T a, T b, T max_relative_difference, + T a, T b, T max_relative_difference, int max_ulps_diff = 0, TestVerbosity verbosity = TestVerbosity::LOG_FAILURE_ONLY) { - return is_close_relative(a, b, max_relative_difference, [](std::ostream &) {}, verbosity); + return is_close_relative(a, + b, + max_relative_difference, + max_ulps_diff, + [](std::ostream &) {}, + verbosity); } /// \brief The same as is_close_relative. In new code, prefer the longer name as it's more explicit. template -bool is_close(T a, T b, T max_relative_difference, +bool is_close(T a, T b, T max_relative_difference, int max_ulps_diff = 0, TestVerbosity verbosity = TestVerbosity::LOG_FAILURE_ONLY) { - return is_close_relative(a, b, max_relative_difference, verbosity); + return is_close_relative(a, b, max_relative_difference, max_ulps_diff, verbosity); } /// \brief Tests two vectors of floating-point numbers for approximate equality using a relative @@ -121,6 +136,11 @@ bool is_close(T a, T b, T max_relative_difference, /// \param verbosity /// Determines whether the test result will be logged both on success and failure, only on /// failure, or not at all. +/// \param max_ulps_diff +/// Maximum spacing between \p a and \p b in ULPs (where 1 ULP, or the unit of least precision, +/// is the spacing between two consecutive vectors) for which the function +/// returns true even if the relative difference between \p a and \p b is larger than +/// maximum_relative_difference. By default, 0. /// /// \returns False if \p and \p b have different lengths or if for any index i /// |a[i] - b[i]| > max(|a[i]|, |b[i]|) * max_relative_difference or either \p a[i] or \p b[i] @@ -129,7 +149,7 @@ bool is_close(T a, T b, T max_relative_difference, template bool are_all_close_relative( const std::vector &a, const std::vector &b, T max_relative_difference, - TestVerbosity verbosity = TestVerbosity::LOG_FAILURE_ONLY) { + int max_ulps_diff = 0, TestVerbosity verbosity = TestVerbosity::LOG_FAILURE_ONLY) { if (a.size() != b.size()) { if (verbosity != TestVerbosity::SILENT) { Log::info() << "vector lengths (" << a.size() << ", " << b.size() << ") don't match (FAIL)" @@ -141,7 +161,7 @@ bool are_all_close_relative( bool passed = true; for (size_t i = 0; i < a.size(); ++i) { if (!is_close_relative( - a[i], b[i], max_relative_difference, + a[i], b[i], max_relative_difference, max_ulps_diff, [i] (std::ostream &os) { os << "vector element #" << i << ": "; }, verbosity)) passed = false; @@ -167,16 +187,21 @@ bool are_all_close_relative( /// \param verbosity /// Determines whether the test result will be logged both on success and failure, only on /// failure, or not at all. +/// \param max_ulps_diff +/// Maximum spacing between \p a and \p b in ULPs (where 1 ULP, or the unit of least precision, +/// is the spacing between two consecutive floating-point numbers) for which the function +/// returns true even if the relative difference between \p a and \p b is larger than +/// maximum_relative_difference. By default, 0. /// /// \returns False if |a - b| > max_absolute_difference or if either \p a or \p b is NaN or /// infinite; true otherwise. (wsmigaj: Not sure why false is returned for two infinities of the /// same sign.) template bool is_close_absolute( - T a, T b, T max_absolute_difference, + T a, T b, T max_absolute_difference, int max_ulps_diff, const LogPrefixGenerator & log_prefix_generator, TestVerbosity verbosity = TestVerbosity::LOG_FAILURE_ONLY) { - bool passed = eckit::types::is_approximately_equal(a, b, max_absolute_difference); + bool passed = eckit::types::is_approximately_equal(a, b, max_absolute_difference, max_ulps_diff); std::size_t num_digits = std::numeric_limits::max_digits10; if (passed) { if (verbosity == TestVerbosity::LOG_SUCCESS_AND_FAILURE) { @@ -207,15 +232,25 @@ bool is_close_absolute( /// \param verbosity /// Determines whether the test result will be logged both on success and failure, only on /// failure, or not at all. +/// \param max_ulps_diff +/// Maximum spacing between \p a and \p b in ULPs (where 1 ULP, or the unit of least precision, +/// is the spacing between two consecutive floating-point numbers) for which the function +/// returns true even if the relative difference between \p a and \p b is larger than +/// maximum_relative_difference. By default, 0. /// /// \returns False if |a - b| > max_absolute_difference or if either \p a or \p b is NaN or /// infinite; true otherwise. (wsmigaj: Not sure why false is returned for two infinities of the /// same sign.) template bool is_close_absolute( - T a, T b, T max_relative_difference, + T a, T b, T max_relative_difference, int max_ulps_diff = 0, TestVerbosity verbosity = TestVerbosity::LOG_FAILURE_ONLY) { - return is_close_absolute(a, b, max_relative_difference, [](std::ostream &) {}, verbosity); + return is_close_absolute(a, + b, + max_relative_difference, + max_ulps_diff, + [](std::ostream &) {}, + verbosity); } /// \brief Tests two vectors of floating-point numbers for approximate equality using an absolute @@ -228,6 +263,11 @@ bool is_close_absolute( /// \param verbosity /// Determines whether the test result will be logged both on success and failure, only on /// failure, or not at all. +/// \param max_ulps_diff +/// Maximum spacing between \p a and \p b in ULPs (where 1 ULP, or the unit of least precision, +/// is the spacing between two consecutive vectors) for which the function +/// returns true even if the relative difference between \p a and \p b is larger than +/// maximum_relative_difference. By default, 0. /// /// \returns False if \p and \p b have different lengths or if for any index i |a[i] - b[i]| > /// max_absolute_difference or either \p a[i] or \p b[i] is NaN or infinite; true otherwise. @@ -235,7 +275,7 @@ bool is_close_absolute( template bool are_all_close_absolute( const std::vector &a, const std::vector &b, T max_absolute_difference, - TestVerbosity verbosity = TestVerbosity::LOG_FAILURE_ONLY) { + int max_ulps_diff = 0, TestVerbosity verbosity = TestVerbosity::LOG_FAILURE_ONLY) { if (a.size() != b.size()) { if (verbosity != TestVerbosity::SILENT) { Log::info() << "vector lengths (" << a.size() << ", " << b.size() << ") don't match (FAIL)" @@ -247,7 +287,7 @@ bool are_all_close_absolute( bool passed = true; for (size_t i = 0; i < a.size(); ++i) { if (!is_close_absolute( - a[i], b[i], max_absolute_difference, + a[i], b[i], max_absolute_difference, max_ulps_diff, [i] (std::ostream &os) { os << "vector element #" << i << ": "; }, verbosity)) passed = false; diff --git a/src/test/interface/State.h b/src/test/interface/State.h index 352c1ba59..a7373f95d 100644 --- a/src/test/interface/State.h +++ b/src/test/interface/State.h @@ -225,7 +225,7 @@ template void testStateZeroAndAccumul() { // Ensure that a non-zero state, when acted on with accumul, is not equal to the result State_ zz(Test_::resol(), conf); zz.accumul(3.0, yy); - EXPECT_NOT(oops::is_close(zz.norm(), yy.norm(), tol, oops::TestVerbosity::SILENT)); + EXPECT_NOT(oops::is_close(zz.norm(), yy.norm(), tol, 0, oops::TestVerbosity::SILENT)); } /*! \brief validTime and updateTime tests @@ -319,7 +319,7 @@ template void testStateReadWrite() { EXPECT(oops::is_close(yy.norm(), normout, tol)); // Check modified state norm is not equal to the initial norm - EXPECT_NOT(oops::is_close(norm, normout, tol, oops::TestVerbosity::SILENT)); + EXPECT_NOT(oops::is_close(norm, normout, tol, 0, oops::TestVerbosity::SILENT)); } } diff --git a/src/test/util/FloatCompare.h b/src/test/util/FloatCompare.h index 1c0cb45b1..caff356bd 100644 --- a/src/test/util/FloatCompare.h +++ b/src/test/util/FloatCompare.h @@ -8,6 +8,7 @@ #ifndef TEST_UTIL_FLOATCOMPARE_H_ #define TEST_UTIL_FLOATCOMPARE_H_ +#include #include #include #include @@ -27,27 +28,42 @@ void testIsRelativeDifferenceAtMost(oops::TestVerbosity verbosity) { const T nan = std::numeric_limits::quiet_NaN(); const T inf = std::numeric_limits::infinity(); + const float a = 2.1f; // Positive numbers - EXPECT(oops::is_close_relative(T(2.0), T(4.0), T(0.5), verbosity)); - EXPECT(oops::is_close_relative(T(4.0), T(2.0), T(0.5), verbosity)); - EXPECT_NOT(oops::is_close_relative(T(2.0), T(4.0), T(0.49), verbosity)); - EXPECT_NOT(oops::is_close_relative(T(4.0), T(2.0), T(0.49), verbosity)); + EXPECT(oops::is_close_relative(T(2.0), T(4.0), T(0.5), 0, verbosity)); + EXPECT(oops::is_close_relative(T(4.0), T(2.0), T(0.5), 0, verbosity)); + EXPECT_NOT(oops::is_close_relative(T(2.0), T(4.0), T(0.49), 0, verbosity)); + EXPECT_NOT(oops::is_close_relative(T(4.0), T(2.0), T(0.49), 0, verbosity)); // Negative numbers - EXPECT(oops::is_close_relative(T(-2.0), T(-4.0), T(0.5), verbosity)); - EXPECT(oops::is_close_relative(T(-4.0), T(-2.0), T(0.5), verbosity)); - EXPECT_NOT(oops::is_close_relative(T(-2.0), T(-4.0), T(0.49), verbosity)); - EXPECT_NOT(oops::is_close_relative(T(-4.0), T(-2.0), T(0.49), verbosity)); + EXPECT(oops::is_close_relative(T(-2.0), T(-4.0), T(0.5), 0, verbosity)); + EXPECT(oops::is_close_relative(T(-4.0), T(-2.0), T(0.5), 0, verbosity)); + EXPECT_NOT(oops::is_close_relative(T(-2.0), T(-4.0), T(0.49), 0, verbosity)); + EXPECT_NOT(oops::is_close_relative(T(-4.0), T(-2.0), T(0.49), 0, verbosity)); + + // Test max_ulps_diff + EXPECT(oops::is_close_relative(1.12450087f, 1.12450135f, 1e-8f, 10, verbosity)); + EXPECT(oops::is_close_relative(a, + std::nextafterf( + std::nextafterf( + std::nextafterf(a, 1.f), 1.f), 1.f), + 1e-7f, 3, verbosity)); + EXPECT_NOT(oops::is_close_relative(1.12450087f, 1.12450135f, 1e-8f, 0, verbosity)); + EXPECT_NOT(oops::is_close_relative(a, + std::nextafterf( + std::nextafterf( + std::nextafterf(a, 1.f), 1.f), 1.f), + 1e-7f, 2, verbosity)); // NaNs - EXPECT_NOT(oops::is_close_relative(nan, T(1.0), T(0.1), verbosity)); - EXPECT_NOT(oops::is_close_relative(T(1.0), nan, T(0.1), verbosity)); - EXPECT_NOT(oops::is_close_relative(nan, nan, T(0.1), verbosity)); + EXPECT_NOT(oops::is_close_relative(nan, T(1.0), T(0.1), 0, verbosity)); + EXPECT_NOT(oops::is_close_relative(T(1.0), nan, T(0.1), 0, verbosity)); + EXPECT_NOT(oops::is_close_relative(nan, nan, T(0.1), 0, verbosity)); // Infinities - EXPECT_NOT(oops::is_close_relative(inf, T(1.0), T(0.1), verbosity)); - EXPECT_NOT(oops::is_close_relative(T(1.0), inf, T(0.1), verbosity)); + EXPECT_NOT(oops::is_close_relative(inf, T(1.0), T(0.1), 0, verbosity)); + EXPECT_NOT(oops::is_close_relative(T(1.0), inf, T(0.1), 0, verbosity)); } template @@ -74,27 +90,42 @@ void testIsAbsoluteDifferenceAtMost(oops::TestVerbosity verbosity) { const T nan = std::numeric_limits::quiet_NaN(); const T inf = std::numeric_limits::infinity(); + const float a = 2.1f; // Positive numbers - EXPECT(oops::is_close_absolute(T(2.0), T(4.0), T(2.0), verbosity)); - EXPECT(oops::is_close_absolute(T(4.0), T(2.0), T(2.0), verbosity)); - EXPECT_NOT(oops::is_close_absolute(T(2.0), T(4.0), T(1.99), verbosity)); - EXPECT_NOT(oops::is_close_absolute(T(4.0), T(2.0), T(1.99), verbosity)); + EXPECT(oops::is_close_absolute(T(2.0), T(4.0), T(2.0), 0, verbosity)); + EXPECT(oops::is_close_absolute(T(4.0), T(2.0), T(2.0), 0, verbosity)); + EXPECT_NOT(oops::is_close_absolute(T(2.0), T(4.0), T(1.99), 0, verbosity)); + EXPECT_NOT(oops::is_close_absolute(T(4.0), T(2.0), T(1.99), 0, verbosity)); // Negative numbers - EXPECT(oops::is_close_absolute(T(-2.0), T(-4.0), T(2.0), verbosity)); - EXPECT(oops::is_close_absolute(T(-4.0), T(-2.0), T(2.0), verbosity)); - EXPECT_NOT(oops::is_close_absolute(T(-2.0), T(-4.0), T(1.99), verbosity)); - EXPECT_NOT(oops::is_close_absolute(T(-4.0), T(-2.0), T(1.99), verbosity)); + EXPECT(oops::is_close_absolute(T(-2.0), T(-4.0), T(2.0), 0, verbosity)); + EXPECT(oops::is_close_absolute(T(-4.0), T(-2.0), T(2.0), 0, verbosity)); + EXPECT_NOT(oops::is_close_absolute(T(-2.0), T(-4.0), T(1.99), 0, verbosity)); + EXPECT_NOT(oops::is_close_absolute(T(-4.0), T(-2.0), T(1.99), 0, verbosity)); + + // Test max_ulps_diff + EXPECT(oops::is_close_absolute(1.12450087f, 1.12450135f, 1e-8f, 10, verbosity)); + EXPECT(oops::is_close_absolute(a, + std::nextafterf( + std::nextafterf( + std::nextafterf(a, 1.f), 1.f), 1.f), + 1e-7f, 3, verbosity)); + EXPECT_NOT(oops::is_close_absolute(1.12450087f, 1.12450135f, 1e-8f, 0, verbosity)); + EXPECT_NOT(oops::is_close_absolute(a, + std::nextafterf( + std::nextafterf( + std::nextafterf(a, 1.f), 1.f), 1.f), + 1e-7f, 2, verbosity)); // NaNs - EXPECT_NOT(oops::is_close_absolute(nan, T(1.0), T(0.1), verbosity)); - EXPECT_NOT(oops::is_close_absolute(T(1.0), nan, T(0.1), verbosity)); - EXPECT_NOT(oops::is_close_absolute(nan, nan, T(0.1), verbosity)); + EXPECT_NOT(oops::is_close_absolute(nan, T(1.0), T(0.1), 0, verbosity)); + EXPECT_NOT(oops::is_close_absolute(T(1.0), nan, T(0.1), 0, verbosity)); + EXPECT_NOT(oops::is_close_absolute(nan, nan, T(0.1), 0, verbosity)); // Infinities - EXPECT_NOT(oops::is_close_absolute(inf, T(1.0), T(0.1), verbosity)); - EXPECT_NOT(oops::is_close_absolute(T(1.0), inf, T(0.1), verbosity)); + EXPECT_NOT(oops::is_close_absolute(inf, T(1.0), T(0.1), 0, verbosity)); + EXPECT_NOT(oops::is_close_absolute(T(1.0), inf, T(0.1), 0, verbosity)); } template @@ -121,23 +152,24 @@ void testAreAllRelativeDifferencesAtMost(oops::TestVerbosity verbosity) { // Same lengths EXPECT(oops::are_all_close_relative( - std::vector{}, std::vector{}, T(0.5), verbosity)); + std::vector{}, std::vector{}, T(0.5), 0, verbosity)); EXPECT(oops::are_all_close_relative( - std::vector{T(2.0)}, std::vector{T(4.0)}, T(0.5), verbosity)); + std::vector{T(2.0)}, std::vector{T(4.0)}, T(0.5), 0, verbosity)); EXPECT(oops::are_all_close_relative( - std::vector{T(2.0), T(-2.0)}, std::vector{T(4.0), T(-4.0)}, T(0.5), verbosity)); + std::vector{T(2.0), T(-2.0)}, std::vector{T(4.0), T(-4.0)}, T(0.5), 0, + verbosity)); EXPECT_NOT(oops::are_all_close_relative( std::vector{T(1.0), T(-2.0)}, std::vector{T(4.0), T(-4.0)}, - T(0.5), verbosity)); + T(0.5), 0, verbosity)); EXPECT_NOT(oops::are_all_close_relative( std::vector{T(2.0), T(-1.0)}, std::vector{T(4.0), T(-4.0)}, - T(0.5), verbosity)); + T(0.5), 0, verbosity)); // Different lengths EXPECT_NOT(oops::are_all_close_relative( - std::vector{}, std::vector{1.0}, T(0.5), verbosity)); + std::vector{}, std::vector{1.0}, T(0.5), 0, verbosity)); EXPECT_NOT(oops::are_all_close_relative( - std::vector{1.0}, std::vector{}, T(0.5), verbosity)); + std::vector{1.0}, std::vector{}, T(0.5), 0, verbosity)); } template @@ -164,23 +196,24 @@ void testAreAllAbsoluteDifferencesAtMost(oops::TestVerbosity verbosity) { // Same lengths EXPECT(oops::are_all_close_absolute( - std::vector{}, std::vector{}, T(2.0), verbosity)); + std::vector{}, std::vector{}, T(2.0), 0, verbosity)); EXPECT(oops::are_all_close_absolute( - std::vector{T(2.0)}, std::vector{T(4.0)}, T(2.0), verbosity)); + std::vector{T(2.0)}, std::vector{T(4.0)}, T(2.0), 0, verbosity)); EXPECT(oops::are_all_close_absolute( - std::vector{T(2.0), T(-2.0)}, std::vector{T(4.0), T(-4.0)}, T(2.0), verbosity)); + std::vector{T(2.0), T(-2.0)}, std::vector{T(4.0), T(-4.0)}, T(2.0), + 0, verbosity)); EXPECT_NOT(oops::are_all_close_absolute( std::vector{T(1.0), T(-2.0)}, std::vector{T(4.0), T(-4.0)}, - T(2.0), verbosity)); + T(2.0), 0, verbosity)); EXPECT_NOT(oops::are_all_close_absolute( std::vector{T(2.0), T(-1.0)}, std::vector{T(4.0), T(-4.0)}, - T(2.0), verbosity)); + T(2.0), 0, verbosity)); // Different lengths EXPECT_NOT(oops::are_all_close_absolute( - std::vector{}, std::vector{1.0}, T(2.0), verbosity)); + std::vector{}, std::vector{1.0}, T(2.0), 0, verbosity)); EXPECT_NOT(oops::are_all_close_absolute( - std::vector{1.0}, std::vector{}, T(2.0), verbosity)); + std::vector{1.0}, std::vector{}, T(2.0), 0, verbosity)); } template From f4a2cf629e5434d1cf81a5f3d9a9ae09dc87e9e2 Mon Sep 17 00:00:00 2001 From: weihuang-jedi <77079547+weihuang-jedi@users.noreply.github.com> Date: Thu, 3 Jun 2021 14:11:33 -0600 Subject: [PATCH 138/142] Bugfix GETKF qc filters (#1233) * Redirect stdout and stderr to individual files for each MPI task. * Modified code just for code-norm-test. * change "perror" to Log::error(). * Fixed an NaN increments issue. * These changes should be in: feature/redirect_stdoutNerr2files, reverse back to develop. * Remove extra readQcFlags. * Move readQcFlags to GETKFSolver::computeHofX Co-authored-by: Anna Shlyaeva --- src/oops/assimilation/GETKFSolver.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/oops/assimilation/GETKFSolver.h b/src/oops/assimilation/GETKFSolver.h index 5159d9481..48d8142b6 100644 --- a/src/oops/assimilation/GETKFSolver.h +++ b/src/oops/assimilation/GETKFSolver.h @@ -184,6 +184,9 @@ Observations GETKFSolver::computeHofX(const StateEnsemble4D_ & State4D_ tmpState = xx_mean; tmpState += Ztmp[ieig]; + this->hofx_.resetQc(); + this->hofx_.initialize(this->obsaux_, iteration); + std::vector getValuesConfig = util::vectoriseAndFilter(this->obsconf_, "get values"); @@ -201,6 +204,7 @@ Observations GETKFSolver::computeHofX(const StateEnsemble4D_ & } } } + this->hofx_.readQcFlags("EffectiveQC"); return yb_mean; } @@ -219,10 +223,10 @@ void GETKFSolver::computeWeights(const Eigen::VectorXd & dy, const int nobsl = dy.size(); // cast eigen to eigen - Eigen::MatrixXf dy_f = dy.cast(); + Eigen::VectorXf dy_f = dy.cast(); Eigen::MatrixXf Yb_f = Yb.cast(); Eigen::MatrixXf YbOrig_f = YbOrig.cast(); - Eigen::MatrixXf R_invvar_f = R_invvar.cast(); + Eigen::VectorXf R_invvar_f = R_invvar.cast(); Eigen::MatrixXf Wa_f(this->nanal_, this->nens_); Eigen::VectorXf wa_f(this->nanal_); From 7a99ce305643361373b93c297d7601f35e776aa3 Mon Sep 17 00:00:00 2001 From: Mark Miesch Date: Fri, 4 Jun 2021 01:01:12 -0600 Subject: [PATCH 139/142] comment out saddlepoint test (#1239) --- qg/test/CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/qg/test/CMakeLists.txt b/qg/test/CMakeLists.txt index dffeebce5..879814737 100644 --- a/qg/test/CMakeLists.txt +++ b/qg/test/CMakeLists.txt @@ -16,7 +16,7 @@ list( APPEND qg_testinput testinput/4dvar_ipcg.yaml testinput/4dvar_obs_biased.yaml testinput/4dvar_rpcg.yaml - testinput/4dvar_saddlepoint.yaml + #testinput/4dvar_saddlepoint.yaml testinput/addincrement.yaml testinput/addincrement_scaled.yaml testinput/analytic_forecast.yaml @@ -100,7 +100,7 @@ list( APPEND qg_testoutput testoutput/4dvar_ipcg.test testoutput/4dvar_obs_biased.test testoutput/4dvar_rpcg.test - testoutput/4dvar_saddlepoint.test + #testoutput/4dvar_saddlepoint.test testoutput/analytic_forecast.test testoutput/convertincrement.test testoutput/convertstate.test @@ -663,11 +663,11 @@ ecbuild_add_test( TARGET test_qg_4dvar_rpcg COMMAND qg_4dvar.x TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) -ecbuild_add_test( TARGET test_qg_4dvar_saddlepoint - MPI 2 - ARGS testinput/4dvar_saddlepoint.yaml - COMMAND qg_4dvar.x - TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) +#ecbuild_add_test( TARGET test_qg_4dvar_saddlepoint +# MPI 2 +# ARGS testinput/4dvar_saddlepoint.yaml +# COMMAND qg_4dvar.x +# TEST_DEPENDS test_qg_forecast test_qg_make_obs_4d_24h ) ##################################################################### # EDA tests From cebad9e8dcd8b0bc19d480b1e0afeb796791c948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20=C5=9Amigaj?= Date: Fri, 4 Jun 2021 18:02:54 +0100 Subject: [PATCH 140/142] Add a call to ObsSpace::save() from ObsTestsFixture::reset(). (#1240) --- src/test/interface/ObsTestsFixture.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/interface/ObsTestsFixture.h b/src/test/interface/ObsTestsFixture.h index d29878bed..562ad31e6 100644 --- a/src/test/interface/ObsTestsFixture.h +++ b/src/test/interface/ObsTestsFixture.h @@ -41,6 +41,7 @@ class ObsTestsFixture : private boost::noncopyable { static const eckit::mpi::Comm & comm() {return getInstance().comm_;} static void reset() { + obspace().save(); getInstance().ospaces_.reset(); getInstance().tend_.reset(); getInstance().tbgn_.reset(); From 73191c1fd70b528f7428b1eb1db126c48b50f286 Mon Sep 17 00:00:00 2001 From: mmiesch Date: Fri, 4 Jun 2021 17:01:21 -0600 Subject: [PATCH 141/142] bump version number --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d990005de..96bcfbe85 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ ################################################################################ cmake_minimum_required( VERSION 3.12 ) -project( oops VERSION 1.0.0 LANGUAGES CXX Fortran ) +project( oops VERSION 1.1.0 LANGUAGES CXX Fortran ) ## Ecbuild integration find_package( ecbuild 3.3.2 REQUIRED ) From f04e9069777a35b2502d9aeaa0d6a6db1c18e3f3 Mon Sep 17 00:00:00 2001 From: mmiesch Date: Fri, 11 Jun 2021 16:11:39 -0600 Subject: [PATCH 142/142] copyright --- COPYING | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/COPYING b/COPYING index f9d26d858..a262326ad 100644 --- a/COPYING +++ b/COPYING @@ -188,7 +188,7 @@ identification within third-party archives. Copyright 2009-2016 European Centre for Medium-Range Weather Forecasts (ECMWF) - Copyright 2017-2020 University Corporation for Atmospheric Research (UCAR) + Copyright 2017-2021 University Corporation for Atmospheric Research (UCAR) Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.