From d689b0373b939a721f824ebc922c8a7fed8412c8 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 25 Apr 2023 12:59:23 +0100 Subject: [PATCH 01/16] So much MA work. --- bgcval2/bgcval.py | 10 +- bgcval2/bgcval2_make_report.py | 2 +- bgcval2/functions/circulation.py | 35 +++++-- bgcval2/p2p/matchDataAndModel.py | 2 + bgcval2/p2p/testsuite_p2p.py | 4 + input_yml/mission_atlantic.yml | 130 ++++++++++++++++++++---- key_files/ma_aeu.yml | 2 +- key_files/ma_ammonium.yml | 2 +- key_files/ma_amoc_26n.yml | 2 +- key_files/ma_dic.yml | 2 +- key_files/ma_drake.yml | 2 +- key_files/ma_gpp.yml | 2 +- key_files/ma_iron.yml | 2 +- key_files/ma_mld.yml | 5 +- key_files/ma_mld_sigma.yml | 2 +- key_files/ma_nitrate.yml | 2 +- key_files/ma_northerntotaliceextent.yml | 2 +- key_files/ma_npp.yml | 2 +- key_files/ma_o2.yml | 9 +- key_files/ma_p1c.yml | 2 +- key_files/ma_ph.yml | 2 +- key_files/ma_phosphate.yml | 2 +- key_files/ma_silicate.yml | 4 +- key_files/ma_southerntotaliceextent.yml | 2 +- key_files/ma_sss.yml | 8 +- key_files/ma_sst.yml | 2 +- key_files/ma_ta.yml | 2 +- key_files/ma_totalairseafluxco2.yml | 2 +- key_files/ma_totalchlorophyll.yml | 2 +- key_files/ma_totaliceextent.yml | 2 +- key_files/ma_totalintgpp.yml | 2 +- key_files/ma_totalintnpp.yml | 2 +- key_files/ma_totalphytoc.yml | 2 +- key_files/ma_totalzooc.yml | 2 +- key_files/ma_zos.yml | 2 +- key_lists/missionatlantic_kmf.yml | 44 ++++++++ key_lists/missionatlantic_p2p.yml | 42 ++++++++ 37 files changed, 279 insertions(+), 66 deletions(-) create mode 100644 key_lists/missionatlantic_kmf.yml create mode 100644 key_lists/missionatlantic_p2p.yml diff --git a/bgcval2/bgcval.py b/bgcval2/bgcval.py index 02ff2959..894b4bad 100755 --- a/bgcval2/bgcval.py +++ b/bgcval2/bgcval.py @@ -38,13 +38,13 @@ import sys from multiprocessing import Pool -from .download_from_mass import findLastFinishedYear -from .analysis_timeseries import analysis_timeseries, singleTimeSeries, singleTimeSeriesProfile +from bgcval2.download_from_mass import findLastFinishedYear +from bgcval2.analysis_timeseries import analysis_timeseries, singleTimeSeries, singleTimeSeriesProfile # these modules are no more and break the tests # from .analysis_timeseries import level1KeysDict, physKeysDict, bgcKeysDict, keymetricsfirstDict -from .analysis_p2p import analysis_p2p, p2pDict_level2, p2pDict_physics, single_p2p -from .bgcval2_make_report import html5Maker -from .bv2tools import folder +from bgcval2.analysis_p2p import analysis_p2p, p2pDict_level2, p2pDict_physics, single_p2p +from bgcval2.bgcval2_make_report import html5Maker +from bgcval2.bv2tools import folder def timeseriesParrallelL1(index): diff --git a/bgcval2/bgcval2_make_report.py b/bgcval2/bgcval2_make_report.py index aa10aafc..bfaf3e32 100755 --- a/bgcval2/bgcval2_make_report.py +++ b/bgcval2/bgcval2_make_report.py @@ -1004,7 +1004,7 @@ def newImageLocation(fn): for s in slices: if s in [ 'Surface', - '1000m', + '*', ]: vfiles.extend( glob(imagedir + '/' + jobID + '/P2Pplots/*/*' + key + diff --git a/bgcval2/functions/circulation.py b/bgcval2/functions/circulation.py index fc212807..65e03115 100644 --- a/bgcval2/functions/circulation.py +++ b/bgcval2/functions/circulation.py @@ -240,7 +240,6 @@ def TwentySixNorth(nc,keys,**kwargs): # Atlantic Mask not loaded assert 0 - zv = np.ma.array(nc.variables[keys[0]][..., latslice26Nnm, :]) # m/s atlmoc = np.array(np.zeros_like(zv[0, :, :, 0])) print('TwentySixNorth:', e3v_AMOC26N.shape, atlmoc.shape, zv.shape) @@ -286,11 +285,35 @@ def twentysixnorth025(nc,keys,**kwargs): depths = np.ma.masked_where(thkcello.mask + np.abs(depths)<500., depths) # masked above 500m depth. e1v = e1v_AMOC26N[:,None, :, :] - flux = vo * depths * e1v_AMOC26N[:,None, :, :]/1.E06 - - moc=np.ma.zeros_like(flux) - np.ma.cumsum(flux[:,::-1], axis=1, out=moc ) # sum floor to surface - return moc.max() +# flux = vo * thkcello * e1v_AMOC26N[:,None, :, :]/1.E06 +# moc=np.ma.zeros_like(flux) +# np.ma.cumsum(flux[:,::-1], axis=1, out=moc ) # sum floor to surface +# return moc.max() + atlmoc = np.array(np.zeros_like(vo.squeeze())) + #print(atlmoc.shape, thkcello.shape, depths.shape, vo.shape, e1v_AMOC26N.shape, e3v_AMOC26N.shape) + # (75, 264) (1, 75, 1, 264) (1, 75, 1, 264) (1, 75, 1, 264) (1, 1, 264) (75, 264) + + #assert 0 + + for (t, z, la, lo), vox in np.ndenumerate(vo): + if not vox: + continue + if not depths[t, z, la, lo] or np.ma.is_masked(depths[t, z, la, lo]): + continue +# if not alttmask_AMOC26N[la, lo]: +# continue +# if not tmask_AMOC26N[z, la, lo]: +# continue +# if np.ma.is_masked(zv[0, z, la, lo]): +# continue + atlmoc[z, la] = atlmoc[z, la] - e1v[t, 0, la, lo] * thkcello[t, z, la, lo] * vo[t, z, la, lo] / 1.E06 + + for z in range(thkcello.shape[1] -2, 1, -1): # add from the bottom up + atlmoc[z, :] = atlmoc[z+1, :] + atlmoc[z, :] + print('AMOC:', atlmoc.max()) + #assert 0 + return atlmoc.max() + def AMOC26N(nc, keys, **kwargs): diff --git a/bgcval2/p2p/matchDataAndModel.py b/bgcval2/p2p/matchDataAndModel.py index 81c86451..3b11aee3 100755 --- a/bgcval2/p2p/matchDataAndModel.py +++ b/bgcval2/p2p/matchDataAndModel.py @@ -270,6 +270,7 @@ def _convertDataTo1D_(self, ): '500m', '1000m', '2000m', + '4000m', ]: print( 'matchDataAndModel:\tconvertDataTo1D:\tSlicing along depth direction.' @@ -281,6 +282,7 @@ def _convertDataTo1D_(self, ): if self.depthLevel == '500m': z = 500. if self.depthLevel == '1000m': z = 1000. if self.depthLevel == '2000m': z = 2000. + if self.depthLevel == '4000m': z = 4000. if nc.variables[self.datacoords['z']].ndim == 1: k = bvt.getORCAdepth( diff --git a/bgcval2/p2p/testsuite_p2p.py b/bgcval2/p2p/testsuite_p2p.py index 0b59037a..2d12a840 100644 --- a/bgcval2/p2p/testsuite_p2p.py +++ b/bgcval2/p2p/testsuite_p2p.py @@ -27,6 +27,7 @@ """ #Standard Python modules: +import os from sys import argv, exit from os.path import exists from calendar import month_name @@ -162,6 +163,9 @@ def testsuite_p2p( # Match observations and model. # Does not produce and plots. #annual = True + if not os.path.exists(av[name].get('dataFiles', '')): + print('Data file missing:', name, av[name].get('dataFiles', '')) + assert 0 b = matchDataAndModel(av[name]['dataFiles'], av[name]['modelFiles'], dataType=name, diff --git a/input_yml/mission_atlantic.yml b/input_yml/mission_atlantic.yml index 2dad7d70..e79d0a64 100644 --- a/input_yml/mission_atlantic.yml +++ b/input_yml/mission_atlantic.yml @@ -26,34 +26,130 @@ jobs: linestyle: '-' shifttime: 0. suite: missionatlantic - test17a: - description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.1' - colour: red +# test17a: +# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.1' +# colour: red +# thickness: 1.7 +# linestyle: '-' +# shifttime: 0. +# suite: missionatlantic + +# test17aa: +# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 1' +# colour: blue +# thickness: 1.7 +# linestyle: '-' +# shifttime: 0. +# suite: missionatlantic +# test17b: +# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 2' +# colour: blue +# thickness: 1.7 +# linestyle: '-' +# shifttime: 1. +# suite: missionatlantic +# test17c: +# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 3' +# colour: blue +# thickness: 1.7 +# linestyle: '-' +# shifttime: 2. +# suite: missionatlantic + +# Broken with weird DIC/pH +# test18a: +# description: 'NEMO-ERSEM eORCA025 - new river input - spinup year 1' +# colour: purple +# thickness: 1.7 +# linestyle: '-' +# shifttime: 0. +# suite: missionatlantic +# test18b: +# description: 'NEMO-ERSEM eORCA025 - new river input - spinup year 2' +# colour: purple +# thickness: 1.7 +# linestyle: '-' +# shifttime: 1. +# suite: missionatlantic +# test18c: +# description: 'NEMO-ERSEM eORCA025 - new river input - spinup year 3' +# colour: purple +# thickness: 1.7 +# linestyle: '-' +# shifttime: 2. +# suite: missionatlantic + +# test19_S1: +# description: 'NEMO-ERSEM eORCA025 - same as 18 but better - spinup year 1' +# colour: orange +# thickness: 1.7 +# linestyle: '-' +# shifttime: 0. +# suite: missionatlantic +# test19_S2: +# description: 'NEMO-ERSEM eORCA025 - same as 18 but better - spinup year 2' +# colour: orange +# thickness: 1.7 +# linestyle: '-' +# shifttime: 1. +# suite: missionatlantic +# test19_S3: +# description: 'NEMO-ERSEM eORCA025 - same as 18 but better - spinup year 3' +# colour: orange +# thickness: 1.7 +# linestyle: '-' +# shifttime: 2. +# suite: missionatlantic + + test19b_S1: + description: 'NEMO-ERSEM eORCA025 - same as 19 but better - spinup year 1' + colour: dodgerblue thickness: 1.7 linestyle: '-' shifttime: 0. suite: missionatlantic - test17aa: - description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 1' - colour: blue +# test19b_S2: +# description: 'NEMO-ERSEM eORCA025 - same as 19 but better - spinup year 2' +# colour: dodgerblue +# thickness: 1.7 +# linestyle: '-' +# shifttime: 1. +# suite: missionatlantic +# test19b_S3: +# description: 'NEMO-ERSEM eORCA025 - same as 19 but better - spinup year 3' +# colour: dodgerblue +# thickness: 1.7 +# linestyle: '-' +# shifttime: 2. +# suite: missionatlantic + + test25: + description: 'NEMO-ERSEM eORCA025 - stable(?!)' + colour: purple thickness: 1.7 linestyle: '-' - shifttime: 0. + shifttime: 02. suite: missionatlantic - test17b: - description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 2' - colour: blue + +# test25S1: +# description: 'NEMO-ERSEM eORCA025 - stablE(?!) spinup s1' +# colour: purple +# thickness: 1.7 +# linestyle: '-' +# shifttime: -3. +# suite: missionatlantic + test25S2: + description: 'NEMO-ERSEM eORCA025 - stable(?!) spinup 2' + colour: purple thickness: 1.7 linestyle: '-' - shifttime: 1. + shifttime: -2. suite: missionatlantic - test17c: - description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 3' - colour: blue + test25S3: + description: 'NEMO-ERSEM eORCA025 - stable(?!) spinup3' + colour: purple thickness: 1.7 linestyle: '-' - shifttime: 2. + shifttime: -1. suite: missionatlantic - - diff --git a/key_files/ma_aeu.yml b/key_files/ma_aeu.yml index c24219f3..5d66f9b5 100644 --- a/key_files/ma_aeu.yml +++ b/key_files/ma_aeu.yml @@ -3,7 +3,7 @@ name: AEU #AtlanticEquatorialUndercurent units: Sv dimensions: 1 model: NEMO -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_U*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_U*.nc modelgrid : eORCA025 model_vars: uo model_convert: diff --git a/key_files/ma_ammonium.yml b/key_files/ma_ammonium.yml index 8106c8d1..d9eb2c0c 100644 --- a/key_files/ma_ammonium.yml +++ b/key_files/ma_ammonium.yml @@ -4,7 +4,7 @@ units : mmol N/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N4_n model_convert : NoChange diff --git a/key_files/ma_amoc_26n.yml b/key_files/ma_amoc_26n.yml index 9388b75a..7d47ec86 100644 --- a/key_files/ma_amoc_26n.yml +++ b/key_files/ma_amoc_26n.yml @@ -4,7 +4,7 @@ name: AMOC_26N units: Sv dimensions: 1 model: NEMO -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_V*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_V*.nc modelgrid : eORCA025 model_vars: vo model_convert: diff --git a/key_files/ma_dic.yml b/key_files/ma_dic.yml index 434ed5b8..eb7e3d79 100644 --- a/key_files/ma_dic.yml +++ b/key_files/ma_dic.yml @@ -4,7 +4,7 @@ units : mmol C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_c model_convert : NoChange diff --git a/key_files/ma_drake.yml b/key_files/ma_drake.yml index 899b8af3..b8d9695c 100644 --- a/key_files/ma_drake.yml +++ b/key_files/ma_drake.yml @@ -3,7 +3,7 @@ name: DrakePassageTransport units: Sv dimensions: 1 model: NEMO -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_U*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_U*.nc modelgrid : eORCA025 model_vars: uo model_convert: diff --git a/key_files/ma_gpp.yml b/key_files/ma_gpp.yml index abd2bbd9..7edc2643 100644 --- a/key_files/ma_gpp.yml +++ b/key_files/ma_gpp.yml @@ -4,7 +4,7 @@ units : "" dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_GPP_result model_convert : NoChange diff --git a/key_files/ma_iron.yml b/key_files/ma_iron.yml index 015e398c..efb25930 100644 --- a/key_files/ma_iron.yml +++ b/key_files/ma_iron.yml @@ -4,7 +4,7 @@ units : umol Fe/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N7_f model_convert : NoChange diff --git a/key_files/ma_mld.yml b/key_files/ma_mld.yml index 47288a3d..28f9c681 100644 --- a/key_files/ma_mld.yml +++ b/key_files/ma_mld.yml @@ -4,9 +4,10 @@ units : m dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc -model_vars : somxzint1 +model_convert : choose_best_var +model_vars : somxzint1 mldr10_1 layers : layerless regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean ITCZ metrics : mean, max #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_mld_sigma.yml b/key_files/ma_mld_sigma.yml index 8d6987ac..5eb9b743 100644 --- a/key_files/ma_mld_sigma.yml +++ b/key_files/ma_mld_sigma.yml @@ -4,7 +4,7 @@ units : m dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : mldr10_1 layers : layerless diff --git a/key_files/ma_nitrate.yml b/key_files/ma_nitrate.yml index 49272554..28592115 100644 --- a/key_files/ma_nitrate.yml +++ b/key_files/ma_nitrate.yml @@ -4,7 +4,7 @@ units : mmol N/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N3_n model_convert : NoChange diff --git a/key_files/ma_northerntotaliceextent.yml b/key_files/ma_northerntotaliceextent.yml index c24311c6..6e37db25 100644 --- a/key_files/ma_northerntotaliceextent.yml +++ b/key_files/ma_northerntotaliceextent.yml @@ -3,7 +3,7 @@ name : NorthernTotalIceExtent units : 1E6 km^2 dimensions : 1 model : CICE -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc model_vars : soicecov model_convert: path: bgcval2/functions/ice.py diff --git a/key_files/ma_npp.yml b/key_files/ma_npp.yml index 245859a4..d7676aeb 100644 --- a/key_files/ma_npp.yml +++ b/key_files/ma_npp.yml @@ -4,7 +4,7 @@ units : "" dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_NPP_result model_convert : NoChange diff --git a/key_files/ma_o2.yml b/key_files/ma_o2.yml index c511258c..fb6ef8d7 100644 --- a/key_files/ma_o2.yml +++ b/key_files/ma_o2.yml @@ -4,14 +4,15 @@ units : mmol O_2/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O2_o model_convert : NoChange -dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_O00_01.nc -data_vars : O_an + #dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_O00_01.nc +dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_o00_01.nc +data_vars : o_an data_convert : NoChange -data_source : WOA13 +data_source : WOA18 layers : 500m #Surface #50m 100m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_p1c.yml b/key_files/ma_p1c.yml index f1b9f6f0..26cae13a 100644 --- a/key_files/ma_p1c.yml +++ b/key_files/ma_p1c.yml @@ -4,7 +4,7 @@ units : mg C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : P1_c model_convert : NoChange diff --git a/key_files/ma_ph.yml b/key_files/ma_ph.yml index 85c13500..e83de34a 100644 --- a/key_files/ma_ph.yml +++ b/key_files/ma_ph.yml @@ -4,7 +4,7 @@ units : None dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_pH model_convert : NoChange diff --git a/key_files/ma_phosphate.yml b/key_files/ma_phosphate.yml index 1439526b..dedfbd3b 100644 --- a/key_files/ma_phosphate.yml +++ b/key_files/ma_phosphate.yml @@ -4,7 +4,7 @@ units : mmol P/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N1_p model_convert : NoChange diff --git a/key_files/ma_silicate.yml b/key_files/ma_silicate.yml index 62e2ffc6..647f76c8 100644 --- a/key_files/ma_silicate.yml +++ b/key_files/ma_silicate.yml @@ -4,7 +4,7 @@ units : mmol Si/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N5_s model_convert : NoChange @@ -12,6 +12,6 @@ dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_i00_01.nc data_vars : i_an data_convert : NoChange data_source : WOA13 -layers : Surface 500m +layers : Surface 100m 200m 500m 1000m 2000m 4000m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_southerntotaliceextent.yml b/key_files/ma_southerntotaliceextent.yml index 51d0fcae..a2046df7 100644 --- a/key_files/ma_southerntotaliceextent.yml +++ b/key_files/ma_southerntotaliceextent.yml @@ -3,7 +3,7 @@ name : SouthernTotalIceExtent units : 1E6 km^2 dimensions : 1 model : CICE -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc model_vars : soicecov model_convert: path: bgcval2/functions/ice.py diff --git a/key_files/ma_sss.yml b/key_files/ma_sss.yml index 02445cf6..e6668d7f 100644 --- a/key_files/ma_sss.yml +++ b/key_files/ma_sss.yml @@ -4,13 +4,13 @@ units : PSU dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : so_abs so -dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_t00_01v2_AMM.nc -data_vars : t_an +dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_s00_01v2_AMM.nc +data_vars : s_an data_convert : NoChange data_source : WOA13 -layers : Surface #50m 100m +layers : Surface 50m 100m 200m 500m 1000m 2000m 4000m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean ITCZ metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_sst.yml b/key_files/ma_sst.yml index 119a88dd..ccb51937 100644 --- a/key_files/ma_sst.yml +++ b/key_files/ma_sst.yml @@ -4,7 +4,7 @@ units : degrees C dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : thetao thetao_con votemper model_convert : choose_best_var diff --git a/key_files/ma_ta.yml b/key_files/ma_ta.yml index b962379c..1174e228 100644 --- a/key_files/ma_ta.yml +++ b/key_files/ma_ta.yml @@ -4,7 +4,7 @@ units : umol/kg dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_TA model_convert : NoChange diff --git a/key_files/ma_totalairseafluxco2.yml b/key_files/ma_totalairseafluxco2.yml index a6fee744..abc8945b 100644 --- a/key_files/ma_totalairseafluxco2.yml +++ b/key_files/ma_totalairseafluxco2.yml @@ -4,7 +4,7 @@ units : Pg C/yr model : NEMO-ERSEM modelgrid : eORCA025 dimensions : 1 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc model_vars : O3_fair model_convert : path: bgcval2/functions/AirSeaFluxCO2.py diff --git a/key_files/ma_totalchlorophyll.yml b/key_files/ma_totalchlorophyll.yml index c98711df..505e0e48 100644 --- a/key_files/ma_totalchlorophyll.yml +++ b/key_files/ma_totalchlorophyll.yml @@ -4,7 +4,7 @@ units : "" dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_Chl_result model_convert : NoChange diff --git a/key_files/ma_totaliceextent.yml b/key_files/ma_totaliceextent.yml index 1f71a436..b08a342a 100644 --- a/key_files/ma_totaliceextent.yml +++ b/key_files/ma_totaliceextent.yml @@ -4,7 +4,7 @@ name : TotalIceExtent units : 1E6 km^2 dimensions : 1 model : NEMO-ERSEM -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc model_vars : soicecov model_convert: path: bgcval2/functions/ice.py diff --git a/key_files/ma_totalintgpp.yml b/key_files/ma_totalintgpp.yml index 986759dc..94e4d8bd 100644 --- a/key_files/ma_totalintgpp.yml +++ b/key_files/ma_totalintgpp.yml @@ -2,7 +2,7 @@ name : TotalIntGPP units : Gt/yr dimensions : 1 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc model_vars : Ptot_NPP_result model_convert : path: bgcval2/functions/TotalIntPP.py diff --git a/key_files/ma_totalintnpp.yml b/key_files/ma_totalintnpp.yml index 6baad60b..c17101bb 100644 --- a/key_files/ma_totalintnpp.yml +++ b/key_files/ma_totalintnpp.yml @@ -2,7 +2,7 @@ name : TotalIntNPP units : Gt/yr dimensions : 1 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc model_vars : Ptot_NPP_result model_convert : path: bgcval2/functions/TotalIntPP.py diff --git a/key_files/ma_totalphytoc.yml b/key_files/ma_totalphytoc.yml index f4a97a28..42d44339 100644 --- a/key_files/ma_totalphytoc.yml +++ b/key_files/ma_totalphytoc.yml @@ -4,7 +4,7 @@ units : mg C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_c_result model_convert : NoChange diff --git a/key_files/ma_totalzooc.yml b/key_files/ma_totalzooc.yml index 4553066f..722e5d22 100644 --- a/key_files/ma_totalzooc.yml +++ b/key_files/ma_totalzooc.yml @@ -4,7 +4,7 @@ units : mg C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ztot_c_result model_convert : NoChange diff --git a/key_files/ma_zos.yml b/key_files/ma_zos.yml index 8cbe23cb..e20363f4 100644 --- a/key_files/ma_zos.yml +++ b/key_files/ma_zos.yml @@ -4,7 +4,7 @@ units : m dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO_*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : zos layers : layerless diff --git a/key_lists/missionatlantic_kmf.yml b/key_lists/missionatlantic_kmf.yml new file mode 100644 index 00000000..279b6148 --- /dev/null +++ b/key_lists/missionatlantic_kmf.yml @@ -0,0 +1,44 @@ +--- +# Key metrics first - KMF +# A short list of key metrics to run first to get a very quick idea +# of how a model is working. +keys: + # Physics: + # + # MA_SST: True + #MA_SSS: True + MA_AEU: True + MA_Drake: True + MA_AMOC_26N: True + #MA_ZOS: True + #MA_MLD: True + #MA_MLD_Sigma: True + + # Ice + #MA_SouthernTotalIceExtent: True + #MA_NorthernTotalIceExtent: True + MA_TotalIceExtent: True + + # BGC + # MA_Totalairseafluxco2: True # Not in test17 + # MA_TotalIntGPP: True + MA_TotalIntNPP: True + #MA_Nitrate: True + # MA_Ammonium: True + # MA_Phosphate: True + # MA_Iron: True + # MA_Silicate: True + # MA_TotalChlorophyll: True + # MA_TotalPhytoC: True + # MA_P1c: True + # MA_TotalZooC: True + # MA_NPP: True + # MA_GPP: True + # MA_pH: True + # MA_O2: True + # MA_DIC: True + # MA_TA: True + + +# what do we need here? +# Air Sea Flux of CO2 - need new data. diff --git a/key_lists/missionatlantic_p2p.yml b/key_lists/missionatlantic_p2p.yml new file mode 100644 index 00000000..b534d770 --- /dev/null +++ b/key_lists/missionatlantic_p2p.yml @@ -0,0 +1,42 @@ +--- +# Mission Atlantic point to point fields. +keys: + # Physics: + # + MA_SST: True + MA_SSS: True + #MA_AEU: True + #MA_Drake: True + #MA_AMOC_26N: True + #MA_ZOS: True + # MA_MLD: True + #MA_MLD_Sigma: True + + # Ice + #MA_SouthernTotalIceExtent: True + #MA_NorthernTotalIceExtent: True + #MA_TotalIceExtent: True + + # BGC + # MA_Totalairseafluxco2: True # Not in test17 + # MA_TotalIntGPP: True + #MA_TotalIntNPP: True + MA_Nitrate: True + # MA_Ammonium: True + MA_Phosphate: True + #MA_Iron: True + MA_Silicate: True + # MA_TotalChlorophyll: True + # MA_TotalPhytoC: True + # MA_P1c: True + # MA_TotalZooC: True + # MA_NPP: True + # MA_GPP: True + # MA_pH: True + MA_O2: True + # MA_DIC: True + # MA_TA: True + + +# what do we need here? +# Air Sea Flux of CO2 - need new data. From 43e52db38cae4d0f1be09c7320aee31970d4e901 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 25 Apr 2023 15:08:36 +0100 Subject: [PATCH 02/16] Update bgcval2_make_report.py --- bgcval2/bgcval2_make_report.py | 57 ++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/bgcval2/bgcval2_make_report.py b/bgcval2/bgcval2_make_report.py index bfaf3e32..dbb84d71 100755 --- a/bgcval2/bgcval2_make_report.py +++ b/bgcval2/bgcval2_make_report.py @@ -165,6 +165,7 @@ def newImageLocation(fn): Level1Profiles = True level2Horizontal = True level2Physics = False + level2_auto = True summarySections = False level3OMZ = False Level3Salinity = False @@ -954,27 +955,51 @@ def newImageLocation(fn): FileLists=FileLists, FileOrder=FileOrder) - if level2Physics: - l2Fields = [ - 'Temperature', - 'Salinity', - 'MLD', - 'ZonalCurrent', - 'MeridionalCurrent', - 'VerticalCurrent', - ] + + + + + if level2Physics or level2_auto: + if level2Physics: + l2Fields = [ + 'Temperature', + 'Salinity', + 'MLD', + 'ZonalCurrent', + 'MeridionalCurrent', + 'VerticalCurrent', + ] + slices = [ + 'Surface', + '1000m', + 'Transect', + ] + SectionTitle = 'Level 2 - Physics' + region = 'Global' + + if level2_auto: + l2Fields = glob(imagedir + '/' + jobID + '/P2Pplots') + l2Fields = [os.path.basename(fn) for fn in sorted(l2Fields)] + levels = ['Surface', '50m', '100m', '200m', '500m', '750m', '1000m', '1500m', '2000m', '4000m','Transect'] + outdict = {} + outlevels = {} + for i, fn in enumerate(l2Fields): + for lv in levels: + if fn.find(lv)>-1: + outlevels[lv]= True + fn = fn.replace(lv, '') + outdict[fn]= True + + l2Fields = [key for key, v in outdict.items()] + slices = [key for key, v in outlevels.items()] + SectionTitle = 'Level 2' + region = '*' + hrefs = [] Titles = {} SidebarTitles = {} Descriptions = {} FileLists = {} - SectionTitle = 'Level 2 - Physics' - region = 'Global' - slices = [ - 'Surface', - '1000m', - 'Transect', - ] FileOrder = {} for key in sorted(l2Fields): From 31a7bb0b32a4186017842b442d3ca49953ba730b Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Wed, 26 Apr 2023 10:30:03 +0100 Subject: [PATCH 03/16] Updated to woa 18. --- bgcval2/bgcval2_make_report.py | 12 +++++++++--- bgcval2/p2p/matchDataAndModel.py | 4 ++++ key_files/ma_mld.yml | 4 ++++ key_files/ma_nitrate.yml | 3 ++- key_files/ma_o2.yml | 2 +- key_files/ma_phosphate.yml | 3 ++- key_files/ma_silicate.yml | 3 ++- key_files/ma_sss.yml | 3 ++- key_files/ma_sst.yml | 3 ++- key_lists/missionatlantic_p2p.yml | 2 +- 10 files changed, 29 insertions(+), 10 deletions(-) diff --git a/bgcval2/bgcval2_make_report.py b/bgcval2/bgcval2_make_report.py index dbb84d71..bf97dff1 100755 --- a/bgcval2/bgcval2_make_report.py +++ b/bgcval2/bgcval2_make_report.py @@ -960,7 +960,7 @@ def newImageLocation(fn): if level2Physics or level2_auto: - if level2Physics: + if level2Physics: l2Fields = [ 'Temperature', 'Salinity', @@ -978,9 +978,9 @@ def newImageLocation(fn): region = 'Global' if level2_auto: - l2Fields = glob(imagedir + '/' + jobID + '/P2Pplots') + l2Fields = glob(imagedir + '/' + jobID + '/P2Pplots/*/*') l2Fields = [os.path.basename(fn) for fn in sorted(l2Fields)] - levels = ['Surface', '50m', '100m', '200m', '500m', '750m', '1000m', '1500m', '2000m', '4000m','Transect'] + levels = ['Surface', '4000m', '2000m', '1000m', '750m','500m','200m', '100m', '50m', 'Transect'] outdict = {} outlevels = {} for i, fn in enumerate(l2Fields): @@ -1001,6 +1001,7 @@ def newImageLocation(fn): Descriptions = {} FileLists = {} FileOrder = {} + #print(l2Fields, slices) for key in sorted(l2Fields): #if key not in ['Alkalinity','Nitrate']: continue @@ -1039,6 +1040,11 @@ def newImageLocation(fn): glob(imagedir + '/' + jobID + '/P2Pplots/*/*' + key + '*/*/*' + s + '*' + region + '*' + key + '*' + year + '*robinquad-cartopy.png')) + vfiles.extend( + glob(imagedir + '/' + jobID + '/P2Pplots/*/*' + key + + '*/*/*' + s + '*' + region + '*' + key + '*' + + year + '*.png')) + if s in [ 'Transect', ]: diff --git a/bgcval2/p2p/matchDataAndModel.py b/bgcval2/p2p/matchDataAndModel.py index 3b11aee3..471932b2 100755 --- a/bgcval2/p2p/matchDataAndModel.py +++ b/bgcval2/p2p/matchDataAndModel.py @@ -268,6 +268,7 @@ def _convertDataTo1D_(self, ): '100m', '200m', '500m', + '750m', '1000m', '2000m', '4000m', @@ -280,6 +281,7 @@ def _convertDataTo1D_(self, ): if self.depthLevel == '100m': z = 100. if self.depthLevel == '200m': z = 200. if self.depthLevel == '500m': z = 500. + if self.depthLevel == '750m': z = 750. if self.depthLevel == '1000m': z = 1000. if self.depthLevel == '2000m': z = 2000. if self.depthLevel == '4000m': z = 4000. @@ -1124,6 +1126,8 @@ def var_to_datetime(ncvar): if units in ['months since 0000-01-01 00:00:00', ]: units = 'months since 2000-01-01 00:00:00' return num2date(ncvar[:], 'months since 2000-01-01 00:00:00', calendar='360_day') + elif units.find('months since')>-1: + return num2date(ncvar[:], units, calendar='360_day') return num2date(ncvar[:], ncvar.units, calendar=calendar) diff --git a/key_files/ma_mld.yml b/key_files/ma_mld.yml index 28f9c681..c63b8222 100644 --- a/key_files/ma_mld.yml +++ b/key_files/ma_mld.yml @@ -8,6 +8,10 @@ modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_convert : choose_best_var model_vars : somxzint1 mldr10_1 +dataFile : $BASEDIR_OBS/WOA/annual/woa18_A5B7_M0200_01.nc +data_vars : M_an +data_convert : NoChange +data_source : WOA13 layers : layerless regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean ITCZ metrics : mean, max #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_nitrate.yml b/key_files/ma_nitrate.yml index 28592115..5b56cf82 100644 --- a/key_files/ma_nitrate.yml +++ b/key_files/ma_nitrate.yml @@ -8,7 +8,8 @@ modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N3_n model_convert : NoChange -dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_n00_01.nc + #dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_n00_01.nc +dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_n00_01.nc data_vars : n_an data_convert : NoChange data_source : WOA13 diff --git a/key_files/ma_o2.yml b/key_files/ma_o2.yml index fb6ef8d7..52735742 100644 --- a/key_files/ma_o2.yml +++ b/key_files/ma_o2.yml @@ -13,6 +13,6 @@ dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_o00_01.nc data_vars : o_an data_convert : NoChange data_source : WOA18 -layers : 500m #Surface #50m 100m +layers : 500m 750m 1000m #Surface #50m 100m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_phosphate.yml b/key_files/ma_phosphate.yml index dedfbd3b..f24ecf14 100644 --- a/key_files/ma_phosphate.yml +++ b/key_files/ma_phosphate.yml @@ -8,7 +8,8 @@ modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N1_p model_convert : NoChange -dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_p00_01.nc + #dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_p00_01.nc +dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_p00_01.nc data_vars : p_an data_convert : NoChange data_source : WOA13 diff --git a/key_files/ma_silicate.yml b/key_files/ma_silicate.yml index 647f76c8..b9db9b62 100644 --- a/key_files/ma_silicate.yml +++ b/key_files/ma_silicate.yml @@ -8,7 +8,8 @@ modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N5_s model_convert : NoChange -dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_i00_01.nc + #ataFile : $BASEDIR_OBS/WOA/annual/woa13_all_i00_01.nc +dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_i00_01.nc data_vars : i_an data_convert : NoChange data_source : WOA13 diff --git a/key_files/ma_sss.yml b/key_files/ma_sss.yml index e6668d7f..8b1f1120 100644 --- a/key_files/ma_sss.yml +++ b/key_files/ma_sss.yml @@ -7,7 +7,8 @@ modelgrid : eORCA025 modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : so_abs so -dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_s00_01v2_AMM.nc + # dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_s00_01v2_AMM.nc +dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa18_decav_s00_01.nc data_vars : s_an data_convert : NoChange data_source : WOA13 diff --git a/key_files/ma_sst.yml b/key_files/ma_sst.yml index ccb51937..20558e9e 100644 --- a/key_files/ma_sst.yml +++ b/key_files/ma_sst.yml @@ -8,7 +8,8 @@ modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : thetao thetao_con votemper model_convert : choose_best_var -dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_t00_01v2_AMM.nc + #dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_t00_01v2_AMM.nc +dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa18_decav_t00_01.nc data_vars : t_an data_convert : NoChange data_source : WOA13 diff --git a/key_lists/missionatlantic_p2p.yml b/key_lists/missionatlantic_p2p.yml index b534d770..8b60dcfd 100644 --- a/key_lists/missionatlantic_p2p.yml +++ b/key_lists/missionatlantic_p2p.yml @@ -9,7 +9,7 @@ keys: #MA_Drake: True #MA_AMOC_26N: True #MA_ZOS: True - # MA_MLD: True + #MA_MLD: True #MA_MLD_Sigma: True # Ice From ea9d2fc098e3234577c83ea840d9fcfc682334f1 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Fri, 13 Oct 2023 12:06:44 +0100 Subject: [PATCH 04/16] add ma files. --- bgcval2/p2p/matchDataAndModel.py | 11 ++++++++++- bgcval2/timeseries/timeseriesTools.py | 16 +++++----------- key_files/ma_mld.yml | 2 +- key_files/ma_nitrate.yml | 2 +- key_files/ma_phosphate.yml | 2 +- key_files/ma_silicate.yml | 2 +- key_files/ma_sss.yml | 2 +- key_files/ma_sst.yml | 2 +- key_lists/ma_debug.yml | 6 ++++-- 9 files changed, 25 insertions(+), 20 deletions(-) diff --git a/bgcval2/p2p/matchDataAndModel.py b/bgcval2/p2p/matchDataAndModel.py index 471932b2..2a9593fb 100755 --- a/bgcval2/p2p/matchDataAndModel.py +++ b/bgcval2/p2p/matchDataAndModel.py @@ -44,6 +44,7 @@ # local imports import bgcval2.bgcvaltools.bv2tools as bvt from bgcval2.bgcvaltools.pftnames import CMIP5models + ##### # These are availalble in the module: # https://gitlab.ecosystem-modelling.pml.ac.uk/ledm/netcdf_manip @@ -925,6 +926,7 @@ def loadMesh(self, modelfile=None): if self.loncc.ndim == 1 and self.loncc.shape != self.latcc.shape: self.loncc, self.latcc = np.meshgrid(self.loncc, self.latcc) + # self.depthcc = choose_best_ncvar(ncER, depthNames)[:] # if 'deptht' in list(ncER.variables.keys()): # self.depthcc = ncER.variables['deptht'][:].squeeze() @@ -934,7 +936,14 @@ def loadMesh(self, modelfile=None): # self.depthcc = ncER.variables['lev'][:].squeeze() # else: # print(self.modelcoords['z'], ncER.variables.keys()) - self.depthcc = ncER.variables[self.modelcoords['z']][:] + if self.modelcoords['z'] in ncER.variables.keys(): + self.depthcc = ncER.variables[self.modelcoords['z']][:] + else: + print('load mesh:', self.modelcoords['z'], 'is missing', ncER.variables.keys()) + if not set(ncER.variables.keys()).intersection(set(depthNames)): + print('load mesh: no depth field available in ', modelfile) + self.depthcc = np.array([0]) +# assert 0 # self.depthcc = choose_best_ncvar(ncER, depthNames)[:] self.datescc = var_to_datetime(ncER.variables[self.modelcoords['t']]) diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py index 68dbed8d..934e8385 100644 --- a/bgcval2/timeseries/timeseriesTools.py +++ b/bgcval2/timeseries/timeseriesTools.py @@ -164,22 +164,16 @@ def getHorizontalSlice(nc, coords, details, layer, data=''): '200m', '300m', '500m', + '750m', '1000m', '2000m', '3000m', '4000m', ]: - if layer == 'Surface': z = 0. - if layer == '50m': z = 50. - if layer == '100m': z = 100. - if layer == '200m': z = 200. - if layer == '300m': z = 300. - if layer == '500m': z = 500. - if layer == '1000m': z = 1000. - if layer == '2000m': z = 2000. - if layer == '3000m': z = 3000. - if layer == '4000m': z = 4000. - print(z) + if layer == 'Surface': + z = 0. + else: + z = float(layer.replace('m', '')) k = bvt.getORCAdepth(z, nc.variables[coords['z']][:], debug=False) if isinstance(data, str): data = std_extractData(nc, details) diff --git a/key_files/ma_mld.yml b/key_files/ma_mld.yml index c63b8222..0dbacda0 100644 --- a/key_files/ma_mld.yml +++ b/key_files/ma_mld.yml @@ -11,7 +11,7 @@ model_vars : somxzint1 mldr10_1 dataFile : $BASEDIR_OBS/WOA/annual/woa18_A5B7_M0200_01.nc data_vars : M_an data_convert : NoChange -data_source : WOA13 +data_source : WOA18 layers : layerless regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean ITCZ metrics : mean, max #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_nitrate.yml b/key_files/ma_nitrate.yml index 5b56cf82..638fc596 100644 --- a/key_files/ma_nitrate.yml +++ b/key_files/ma_nitrate.yml @@ -12,7 +12,7 @@ model_convert : NoChange dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_n00_01.nc data_vars : n_an data_convert : NoChange -data_source : WOA13 +data_source : WOA18 layers : Surface 500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_phosphate.yml b/key_files/ma_phosphate.yml index f24ecf14..b7d578eb 100644 --- a/key_files/ma_phosphate.yml +++ b/key_files/ma_phosphate.yml @@ -12,7 +12,7 @@ model_convert : NoChange dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_p00_01.nc data_vars : p_an data_convert : NoChange -data_source : WOA13 +data_source : WOA18 layers : Surface 500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_silicate.yml b/key_files/ma_silicate.yml index b9db9b62..abf7f862 100644 --- a/key_files/ma_silicate.yml +++ b/key_files/ma_silicate.yml @@ -12,7 +12,7 @@ model_convert : NoChange dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_i00_01.nc data_vars : i_an data_convert : NoChange -data_source : WOA13 +data_source : WOA18 layers : Surface 100m 200m 500m 1000m 2000m 4000m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_sss.yml b/key_files/ma_sss.yml index 8b1f1120..67b336bf 100644 --- a/key_files/ma_sss.yml +++ b/key_files/ma_sss.yml @@ -11,7 +11,7 @@ model_vars : so_abs so dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa18_decav_s00_01.nc data_vars : s_an data_convert : NoChange -data_source : WOA13 +data_source : WOA18 layers : Surface 50m 100m 200m 500m 1000m 2000m 4000m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean ITCZ metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_sst.yml b/key_files/ma_sst.yml index 20558e9e..666527aa 100644 --- a/key_files/ma_sst.yml +++ b/key_files/ma_sst.yml @@ -12,7 +12,7 @@ model_convert : choose_best_var dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa18_decav_t00_01.nc data_vars : t_an data_convert : NoChange -data_source : WOA13 +data_source : WOA18 layers : Surface #50m 100m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean ITCZ metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_lists/ma_debug.yml b/key_lists/ma_debug.yml index e48ec8c2..727ed8cb 100644 --- a/key_lists/ma_debug.yml +++ b/key_lists/ma_debug.yml @@ -4,5 +4,7 @@ # of how a model is working. keys: # BGC - MA_TotalIntGPP: True - MA_TotalIntNPP: True + MA_SST: True + + # MA_TotalIntGPP: True + # MA_TotalIntNPP: True From d63c66db973412a458b6a9aa5260795eb20f9b14 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Fri, 13 Oct 2023 12:13:20 +0100 Subject: [PATCH 05/16] Removed hard link to SENEMO in files. --- .idea/bgcval2.iml | 10 ++++++++++ .idea/inspectionProfiles/profiles_settings.xml | 6 ++++++ .idea/misc.xml | 4 ++++ .idea/vcs.xml | 6 ++++++ key_files/ma_aeu.yml | 2 +- key_files/ma_ammonium.yml | 2 +- key_files/ma_amoc_26n.yml | 2 +- key_files/ma_dic.yml | 2 +- key_files/ma_drake.yml | 2 +- key_files/ma_gpp.yml | 2 +- key_files/ma_iron.yml | 2 +- key_files/ma_mld.yml | 2 +- key_files/ma_mld_sigma.yml | 2 +- key_files/ma_nitrate.yml | 2 +- key_files/ma_northerntotaliceextent.yml | 2 +- key_files/ma_npp.yml | 2 +- key_files/ma_o2.yml | 2 +- key_files/ma_p1c.yml | 2 +- key_files/ma_ph.yml | 2 +- key_files/ma_phosphate.yml | 2 +- key_files/ma_silicate.yml | 2 +- key_files/ma_southerntotaliceextent.yml | 2 +- key_files/ma_sss.yml | 2 +- key_files/ma_sst.yml | 2 +- key_files/ma_ta.yml | 2 +- key_files/ma_totalairseafluxco2.yml | 2 +- key_files/ma_totalchlorophyll.yml | 2 +- key_files/ma_totaliceextent.yml | 2 +- key_files/ma_totalintgpp.yml | 2 +- key_files/ma_totalintnpp.yml | 2 +- key_files/ma_totalphytoc.yml | 2 +- key_files/ma_totalzooc.yml | 2 +- key_files/ma_zos.yml | 2 +- 33 files changed, 55 insertions(+), 29 deletions(-) create mode 100644 .idea/bgcval2.iml create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/vcs.xml diff --git a/.idea/bgcval2.iml b/.idea/bgcval2.iml new file mode 100644 index 00000000..aad402c4 --- /dev/null +++ b/.idea/bgcval2.iml @@ -0,0 +1,10 @@ + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 00000000..105ce2da --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..c47c9106 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..35eb1ddf --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/key_files/ma_aeu.yml b/key_files/ma_aeu.yml index 5d66f9b5..b272f033 100644 --- a/key_files/ma_aeu.yml +++ b/key_files/ma_aeu.yml @@ -3,7 +3,7 @@ name: AEU #AtlanticEquatorialUndercurent units: Sv dimensions: 1 model: NEMO -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_U*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_U*.nc modelgrid : eORCA025 model_vars: uo model_convert: diff --git a/key_files/ma_ammonium.yml b/key_files/ma_ammonium.yml index d9eb2c0c..04bae9a4 100644 --- a/key_files/ma_ammonium.yml +++ b/key_files/ma_ammonium.yml @@ -4,7 +4,7 @@ units : mmol N/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N4_n model_convert : NoChange diff --git a/key_files/ma_amoc_26n.yml b/key_files/ma_amoc_26n.yml index 7d47ec86..40a011cc 100644 --- a/key_files/ma_amoc_26n.yml +++ b/key_files/ma_amoc_26n.yml @@ -4,7 +4,7 @@ name: AMOC_26N units: Sv dimensions: 1 model: NEMO -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_V*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/**_1m_*grid_V*.nc modelgrid : eORCA025 model_vars: vo model_convert: diff --git a/key_files/ma_dic.yml b/key_files/ma_dic.yml index eb7e3d79..ddb8e1dc 100644 --- a/key_files/ma_dic.yml +++ b/key_files/ma_dic.yml @@ -4,7 +4,7 @@ units : mmol C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/**_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_c model_convert : NoChange diff --git a/key_files/ma_drake.yml b/key_files/ma_drake.yml index b8d9695c..6299c09d 100644 --- a/key_files/ma_drake.yml +++ b/key_files/ma_drake.yml @@ -3,7 +3,7 @@ name: DrakePassageTransport units: Sv dimensions: 1 model: NEMO -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_U*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/**_1m_*grid_U*.nc modelgrid : eORCA025 model_vars: uo model_convert: diff --git a/key_files/ma_gpp.yml b/key_files/ma_gpp.yml index 7edc2643..60daa286 100644 --- a/key_files/ma_gpp.yml +++ b/key_files/ma_gpp.yml @@ -4,7 +4,7 @@ units : "" dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_GPP_result model_convert : NoChange diff --git a/key_files/ma_iron.yml b/key_files/ma_iron.yml index efb25930..e9ab964b 100644 --- a/key_files/ma_iron.yml +++ b/key_files/ma_iron.yml @@ -4,7 +4,7 @@ units : umol Fe/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N7_f model_convert : NoChange diff --git a/key_files/ma_mld.yml b/key_files/ma_mld.yml index 0dbacda0..23732889 100644 --- a/key_files/ma_mld.yml +++ b/key_files/ma_mld.yml @@ -4,7 +4,7 @@ units : m dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_convert : choose_best_var model_vars : somxzint1 mldr10_1 diff --git a/key_files/ma_mld_sigma.yml b/key_files/ma_mld_sigma.yml index 5eb9b743..eba56b17 100644 --- a/key_files/ma_mld_sigma.yml +++ b/key_files/ma_mld_sigma.yml @@ -4,7 +4,7 @@ units : m dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : mldr10_1 layers : layerless diff --git a/key_files/ma_nitrate.yml b/key_files/ma_nitrate.yml index 638fc596..90924791 100644 --- a/key_files/ma_nitrate.yml +++ b/key_files/ma_nitrate.yml @@ -4,7 +4,7 @@ units : mmol N/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N3_n model_convert : NoChange diff --git a/key_files/ma_northerntotaliceextent.yml b/key_files/ma_northerntotaliceextent.yml index 6e37db25..a1745651 100644 --- a/key_files/ma_northerntotaliceextent.yml +++ b/key_files/ma_northerntotaliceextent.yml @@ -3,7 +3,7 @@ name : NorthernTotalIceExtent units : 1E6 km^2 dimensions : 1 model : CICE -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc model_vars : soicecov model_convert: path: bgcval2/functions/ice.py diff --git a/key_files/ma_npp.yml b/key_files/ma_npp.yml index d7676aeb..eafa4541 100644 --- a/key_files/ma_npp.yml +++ b/key_files/ma_npp.yml @@ -4,7 +4,7 @@ units : "" dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_NPP_result model_convert : NoChange diff --git a/key_files/ma_o2.yml b/key_files/ma_o2.yml index 52735742..4830b9c8 100644 --- a/key_files/ma_o2.yml +++ b/key_files/ma_o2.yml @@ -4,7 +4,7 @@ units : mmol O_2/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O2_o model_convert : NoChange diff --git a/key_files/ma_p1c.yml b/key_files/ma_p1c.yml index 26cae13a..e4d1e04e 100644 --- a/key_files/ma_p1c.yml +++ b/key_files/ma_p1c.yml @@ -4,7 +4,7 @@ units : mg C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : P1_c model_convert : NoChange diff --git a/key_files/ma_ph.yml b/key_files/ma_ph.yml index e83de34a..a929ec32 100644 --- a/key_files/ma_ph.yml +++ b/key_files/ma_ph.yml @@ -4,7 +4,7 @@ units : None dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_pH model_convert : NoChange diff --git a/key_files/ma_phosphate.yml b/key_files/ma_phosphate.yml index b7d578eb..0158515b 100644 --- a/key_files/ma_phosphate.yml +++ b/key_files/ma_phosphate.yml @@ -4,7 +4,7 @@ units : mmol P/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N1_p model_convert : NoChange diff --git a/key_files/ma_silicate.yml b/key_files/ma_silicate.yml index abf7f862..c651fc53 100644 --- a/key_files/ma_silicate.yml +++ b/key_files/ma_silicate.yml @@ -4,7 +4,7 @@ units : mmol Si/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N5_s model_convert : NoChange diff --git a/key_files/ma_southerntotaliceextent.yml b/key_files/ma_southerntotaliceextent.yml index a2046df7..f965a1c7 100644 --- a/key_files/ma_southerntotaliceextent.yml +++ b/key_files/ma_southerntotaliceextent.yml @@ -3,7 +3,7 @@ name : SouthernTotalIceExtent units : 1E6 km^2 dimensions : 1 model : CICE -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc model_vars : soicecov model_convert: path: bgcval2/functions/ice.py diff --git a/key_files/ma_sss.yml b/key_files/ma_sss.yml index 67b336bf..8887d416 100644 --- a/key_files/ma_sss.yml +++ b/key_files/ma_sss.yml @@ -4,7 +4,7 @@ units : PSU dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : so_abs so # dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_s00_01v2_AMM.nc diff --git a/key_files/ma_sst.yml b/key_files/ma_sst.yml index 666527aa..639870b7 100644 --- a/key_files/ma_sst.yml +++ b/key_files/ma_sst.yml @@ -4,7 +4,7 @@ units : degrees C dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : thetao thetao_con votemper model_convert : choose_best_var diff --git a/key_files/ma_ta.yml b/key_files/ma_ta.yml index 1174e228..9f8c7811 100644 --- a/key_files/ma_ta.yml +++ b/key_files/ma_ta.yml @@ -4,7 +4,7 @@ units : umol/kg dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_TA model_convert : NoChange diff --git a/key_files/ma_totalairseafluxco2.yml b/key_files/ma_totalairseafluxco2.yml index abc8945b..cdb59bc5 100644 --- a/key_files/ma_totalairseafluxco2.yml +++ b/key_files/ma_totalairseafluxco2.yml @@ -4,7 +4,7 @@ units : Pg C/yr model : NEMO-ERSEM modelgrid : eORCA025 dimensions : 1 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc model_vars : O3_fair model_convert : path: bgcval2/functions/AirSeaFluxCO2.py diff --git a/key_files/ma_totalchlorophyll.yml b/key_files/ma_totalchlorophyll.yml index 505e0e48..8079db9b 100644 --- a/key_files/ma_totalchlorophyll.yml +++ b/key_files/ma_totalchlorophyll.yml @@ -4,7 +4,7 @@ units : "" dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_Chl_result model_convert : NoChange diff --git a/key_files/ma_totaliceextent.yml b/key_files/ma_totaliceextent.yml index b08a342a..c2562d05 100644 --- a/key_files/ma_totaliceextent.yml +++ b/key_files/ma_totaliceextent.yml @@ -4,7 +4,7 @@ name : TotalIceExtent units : 1E6 km^2 dimensions : 1 model : NEMO-ERSEM -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc model_vars : soicecov model_convert: path: bgcval2/functions/ice.py diff --git a/key_files/ma_totalintgpp.yml b/key_files/ma_totalintgpp.yml index 94e4d8bd..108f2ae9 100644 --- a/key_files/ma_totalintgpp.yml +++ b/key_files/ma_totalintgpp.yml @@ -2,7 +2,7 @@ name : TotalIntGPP units : Gt/yr dimensions : 1 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc model_vars : Ptot_NPP_result model_convert : path: bgcval2/functions/TotalIntPP.py diff --git a/key_files/ma_totalintnpp.yml b/key_files/ma_totalintnpp.yml index c17101bb..d931cbed 100644 --- a/key_files/ma_totalintnpp.yml +++ b/key_files/ma_totalintnpp.yml @@ -2,7 +2,7 @@ name : TotalIntNPP units : Gt/yr dimensions : 1 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc model_vars : Ptot_NPP_result model_convert : path: bgcval2/functions/TotalIntPP.py diff --git a/key_files/ma_totalphytoc.yml b/key_files/ma_totalphytoc.yml index 42d44339..4ab925d1 100644 --- a/key_files/ma_totalphytoc.yml +++ b/key_files/ma_totalphytoc.yml @@ -4,7 +4,7 @@ units : mg C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_c_result model_convert : NoChange diff --git a/key_files/ma_totalzooc.yml b/key_files/ma_totalzooc.yml index 722e5d22..32f61327 100644 --- a/key_files/ma_totalzooc.yml +++ b/key_files/ma_totalzooc.yml @@ -4,7 +4,7 @@ units : mg C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ztot_c_result model_convert : NoChange diff --git a/key_files/ma_zos.yml b/key_files/ma_zos.yml index e20363f4..a8fc799e 100644 --- a/key_files/ma_zos.yml +++ b/key_files/ma_zos.yml @@ -4,7 +4,7 @@ units : m dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/SENEMO*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : zos layers : layerless From 7b25785af97f188a609d4deb20c41b2df151dc54 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 19 Oct 2023 09:43:18 +0100 Subject: [PATCH 06/16] Added new mission atlantic code. --- bgcval2/default-bgcval2-config.yml | 2 +- input_yml/mission_atlantic_CNRM.yml | 46 +++++++++++ input_yml/mission_atlantic_GS1p0.yml | 112 +++++++++++++++++++++++++++ 3 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 input_yml/mission_atlantic_CNRM.yml create mode 100644 input_yml/mission_atlantic_GS1p0.yml diff --git a/bgcval2/default-bgcval2-config.yml b/bgcval2/default-bgcval2-config.yml index 2d2cfa58..4571f81d 100644 --- a/bgcval2/default-bgcval2-config.yml +++ b/bgcval2/default-bgcval2-config.yml @@ -70,7 +70,7 @@ standard-paths: p2p_ppDir: "/local1/data/scratch/ledm/bgcval2_postProcessed" imagedir: "images" #ModelFolder_pref: "gle/dare_baseline" #AMM7 - ModelFolder_pref: "ledm/MissionAtlantic/SENEMO" + ModelFolder_pref: "ledm/MissionAtlantic/" orcaGridfn: "/data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc" amm7Gridfn: "/users/modellers/ledm/workspace/coast/AMM7_grid/mesh_mask.nc" ObsFolder: "/data/sthenno1/scratch/ledm/Observations/" diff --git a/input_yml/mission_atlantic_CNRM.yml b/input_yml/mission_atlantic_CNRM.yml new file mode 100644 index 00000000..f2d1546d --- /dev/null +++ b/input_yml/mission_atlantic_CNRM.yml @@ -0,0 +1,46 @@ +--- +# GC5 N96 ORCA1 spinup analysis +name: MissionAtlantic_CNRM + +# Run the single job analysis +do_analysis_timeseries: True + +# Download from mass: +do_mass_download: False + +# Strict file check (fails if True and files are missing) +strictFileCheck: False + +# master analysis suite +master_suites: missionatlantic + +# Auto download jobs (can also be set for individual jobs). +auto_download: False + +# Job ID's suites as named by Rose/Cylc +jobs: + CNRM_hist: + description: 'CNRM historical run.' + colour: green + thickness: 1.8 + linestyle: '-' + shifttime: 0. + suite: missionatlantic + + CNRM_ssp370: + description: 'CNRM SSP3-7.0' + colour: purple + thickness: 1.8 + linestyle: '-' + shifttime: 0. + suite: missionatlantic + + CNRM_ssp126: + description: 'CNRM SSP1-2.6' + colour: dodgerblue + thickness: 1.8 + linestyle: '-' + shifttime: 0. + suite: missionatlantic + + diff --git a/input_yml/mission_atlantic_GS1p0.yml b/input_yml/mission_atlantic_GS1p0.yml new file mode 100644 index 00000000..4e3f150b --- /dev/null +++ b/input_yml/mission_atlantic_GS1p0.yml @@ -0,0 +1,112 @@ +--- +# GC5 N96 ORCA1 spinup analysis +name: MissionAtlantic_GS1p0 + +# Run the single job analysis +do_analysis_timeseries: True + +# Download from mass: +do_mass_download: False + +# Strict file check (fails if True and files are missing) +strictFileCheck: False + +# master analysis suite +master_suites: missionatlantic + +# Auto download jobs (can also be set for individual jobs). +auto_download: False + +# Job ID's suites as named by Rose/Cylc +jobs: + GS1p0_hind: + description: 'NEMO-ERSEM eORCA025 - GS1p0_hind' + colour: green + thickness: 1.8 + linestyle: '-' + shifttime: 0. + suite: missionatlantic + +# test17a: +# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.1' +# colour: red +# thickness: 1.7 +# linestyle: '-' +# shifttime: 0. +# suite: missionatlantic + +# test17aa: +# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 1' +# colour: blue +# thickness: 1.7 +# linestyle: '-' +# shifttime: 0. +# suite: missionatlantic +# test17b: +# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 2' +# colour: blue +# thickness: 1.7 +# linestyle: '-' +# shifttime: 1. +# suite: missionatlantic +# test17c: +# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 3' +# colour: blue +# thickness: 1.7 +# linestyle: '-' +# shifttime: 2. +# suite: missionatlantic + +# Broken with weird DIC/pH +# test18a: +# description: 'NEMO-ERSEM eORCA025 - new river input - spinup year 1' +# colour: purple +# thickness: 1.7 +# linestyle: '-' +# shifttime: 0. +# suite: missionatlantic +# test18b: +# description: 'NEMO-ERSEM eORCA025 - new river input - spinup year 2' +# colour: purple +# thickness: 1.7 +# linestyle: '-' +# shifttime: 1. +# suite: missionatlantic +# test18c: +# description: 'NEMO-ERSEM eORCA025 - new river input - spinup year 3' +# colour: purple +# thickness: 1.7 +# linestyle: '-' +# shifttime: 2. +# suite: missionatlantic + +# test19_S1: +# description: 'NEMO-ERSEM eORCA025 - same as 18 but better - spinup year 1' +# colour: orange +# thickness: 1.7 +# linestyle: '-' +# shifttime: 0. +# suite: missionatlantic +# test19_S2: +# description: 'NEMO-ERSEM eORCA025 - same as 18 but better - spinup year 2' +# colour: orange +# thickness: 1.7 +# linestyle: '-' +# shifttime: 1. +# suite: missionatlantic +# test19_S3: +# description: 'NEMO-ERSEM eORCA025 - same as 18 but better - spinup year 3' +# colour: orange +# thickness: 1.7 +# linestyle: '-' +# shifttime: 2. +# suite: missionatlantic + + test25: + description: 'NEMO-ERSEM eORCA025 - stable(?!)' + colour: purple + thickness: 1.7 + linestyle: '-' + shifttime: 02. + suite: missionatlantic + From 7f645d5693e6c8d369a7867ad92f668bb9068997 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 19 Oct 2023 09:44:11 +0100 Subject: [PATCH 07/16] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6284a3c6..de6e55f9 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ _sidebar.rst.inc local_test/BGC_data/valeriu mass_scripts CompareReports2 +.idea/workspace.xml From dd88b1ad3bcb32242857cda391ab133820e2b617 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 19 Oct 2023 09:48:15 +0100 Subject: [PATCH 08/16] Added new paths to remove linking of MA runs. --- key_files/ma_aeu.yml | 2 +- key_files/ma_ammonium.yml | 2 +- key_files/ma_amoc_26n.yml | 2 +- key_files/ma_dic.yml | 2 +- key_files/ma_drake.yml | 2 +- key_files/ma_gpp.yml | 2 +- key_files/ma_iron.yml | 2 +- key_files/ma_mld.yml | 2 +- key_files/ma_mld_sigma.yml | 2 +- key_files/ma_nitrate.yml | 2 +- key_files/ma_northerntotaliceextent.yml | 2 +- key_files/ma_npp.yml | 2 +- key_files/ma_o2.yml | 2 +- key_files/ma_p1c.yml | 2 +- key_files/ma_ph.yml | 2 +- key_files/ma_phosphate.yml | 2 +- key_files/ma_silicate.yml | 2 +- key_files/ma_southerntotaliceextent.yml | 2 +- key_files/ma_sss.yml | 2 +- key_files/ma_sst.yml | 2 +- key_files/ma_ta.yml | 2 +- key_files/ma_totalairseafluxco2.yml | 2 +- key_files/ma_totalchlorophyll.yml | 2 +- key_files/ma_totaliceextent.yml | 2 +- key_files/ma_totalintgpp.yml | 2 +- key_files/ma_totalintnpp.yml | 2 +- key_files/ma_totalphytoc.yml | 2 +- key_files/ma_totalzooc.yml | 2 +- key_files/ma_zos.yml | 2 +- 29 files changed, 29 insertions(+), 29 deletions(-) diff --git a/key_files/ma_aeu.yml b/key_files/ma_aeu.yml index b272f033..3f277cc7 100644 --- a/key_files/ma_aeu.yml +++ b/key_files/ma_aeu.yml @@ -3,7 +3,7 @@ name: AEU #AtlanticEquatorialUndercurent units: Sv dimensions: 1 model: NEMO -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_U*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_U*.nc modelgrid : eORCA025 model_vars: uo model_convert: diff --git a/key_files/ma_ammonium.yml b/key_files/ma_ammonium.yml index 04bae9a4..a86a9843 100644 --- a/key_files/ma_ammonium.yml +++ b/key_files/ma_ammonium.yml @@ -4,7 +4,7 @@ units : mmol N/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N4_n model_convert : NoChange diff --git a/key_files/ma_amoc_26n.yml b/key_files/ma_amoc_26n.yml index 40a011cc..cb0a0aca 100644 --- a/key_files/ma_amoc_26n.yml +++ b/key_files/ma_amoc_26n.yml @@ -4,7 +4,7 @@ name: AMOC_26N units: Sv dimensions: 1 model: NEMO -modelFiles : $BASEDIR_MODEL/$JOBID/**_1m_*grid_V*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_V*.nc modelgrid : eORCA025 model_vars: vo model_convert: diff --git a/key_files/ma_dic.yml b/key_files/ma_dic.yml index ddb8e1dc..e8140b8f 100644 --- a/key_files/ma_dic.yml +++ b/key_files/ma_dic.yml @@ -4,7 +4,7 @@ units : mmol C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/**_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_c model_convert : NoChange diff --git a/key_files/ma_drake.yml b/key_files/ma_drake.yml index 6299c09d..d4b79187 100644 --- a/key_files/ma_drake.yml +++ b/key_files/ma_drake.yml @@ -3,7 +3,7 @@ name: DrakePassageTransport units: Sv dimensions: 1 model: NEMO -modelFiles : $BASEDIR_MODEL/$JOBID/**_1m_*grid_U*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_U*.nc modelgrid : eORCA025 model_vars: uo model_convert: diff --git a/key_files/ma_gpp.yml b/key_files/ma_gpp.yml index 60daa286..56058c30 100644 --- a/key_files/ma_gpp.yml +++ b/key_files/ma_gpp.yml @@ -4,7 +4,7 @@ units : "" dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_GPP_result model_convert : NoChange diff --git a/key_files/ma_iron.yml b/key_files/ma_iron.yml index e9ab964b..b8e73fb6 100644 --- a/key_files/ma_iron.yml +++ b/key_files/ma_iron.yml @@ -4,7 +4,7 @@ units : umol Fe/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N7_f model_convert : NoChange diff --git a/key_files/ma_mld.yml b/key_files/ma_mld.yml index 23732889..bdaadbc9 100644 --- a/key_files/ma_mld.yml +++ b/key_files/ma_mld.yml @@ -4,7 +4,7 @@ units : m dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_convert : choose_best_var model_vars : somxzint1 mldr10_1 diff --git a/key_files/ma_mld_sigma.yml b/key_files/ma_mld_sigma.yml index eba56b17..e22254d3 100644 --- a/key_files/ma_mld_sigma.yml +++ b/key_files/ma_mld_sigma.yml @@ -4,7 +4,7 @@ units : m dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : mldr10_1 layers : layerless diff --git a/key_files/ma_nitrate.yml b/key_files/ma_nitrate.yml index 90924791..869900a6 100644 --- a/key_files/ma_nitrate.yml +++ b/key_files/ma_nitrate.yml @@ -4,7 +4,7 @@ units : mmol N/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N3_n model_convert : NoChange diff --git a/key_files/ma_northerntotaliceextent.yml b/key_files/ma_northerntotaliceextent.yml index a1745651..6ecb3db6 100644 --- a/key_files/ma_northerntotaliceextent.yml +++ b/key_files/ma_northerntotaliceextent.yml @@ -3,7 +3,7 @@ name : NorthernTotalIceExtent units : 1E6 km^2 dimensions : 1 model : CICE -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc model_vars : soicecov model_convert: path: bgcval2/functions/ice.py diff --git a/key_files/ma_npp.yml b/key_files/ma_npp.yml index eafa4541..5f94bcb0 100644 --- a/key_files/ma_npp.yml +++ b/key_files/ma_npp.yml @@ -4,7 +4,7 @@ units : "" dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_NPP_result model_convert : NoChange diff --git a/key_files/ma_o2.yml b/key_files/ma_o2.yml index 4830b9c8..832d2801 100644 --- a/key_files/ma_o2.yml +++ b/key_files/ma_o2.yml @@ -4,7 +4,7 @@ units : mmol O_2/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O2_o model_convert : NoChange diff --git a/key_files/ma_p1c.yml b/key_files/ma_p1c.yml index e4d1e04e..826f89de 100644 --- a/key_files/ma_p1c.yml +++ b/key_files/ma_p1c.yml @@ -4,7 +4,7 @@ units : mg C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : P1_c model_convert : NoChange diff --git a/key_files/ma_ph.yml b/key_files/ma_ph.yml index a929ec32..9b55f665 100644 --- a/key_files/ma_ph.yml +++ b/key_files/ma_ph.yml @@ -4,7 +4,7 @@ units : None dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_pH model_convert : NoChange diff --git a/key_files/ma_phosphate.yml b/key_files/ma_phosphate.yml index 0158515b..3cd26121 100644 --- a/key_files/ma_phosphate.yml +++ b/key_files/ma_phosphate.yml @@ -4,7 +4,7 @@ units : mmol P/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N1_p model_convert : NoChange diff --git a/key_files/ma_silicate.yml b/key_files/ma_silicate.yml index c651fc53..4e364ff7 100644 --- a/key_files/ma_silicate.yml +++ b/key_files/ma_silicate.yml @@ -4,7 +4,7 @@ units : mmol Si/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N5_s model_convert : NoChange diff --git a/key_files/ma_southerntotaliceextent.yml b/key_files/ma_southerntotaliceextent.yml index f965a1c7..eee63f1a 100644 --- a/key_files/ma_southerntotaliceextent.yml +++ b/key_files/ma_southerntotaliceextent.yml @@ -3,7 +3,7 @@ name : SouthernTotalIceExtent units : 1E6 km^2 dimensions : 1 model : CICE -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc model_vars : soicecov model_convert: path: bgcval2/functions/ice.py diff --git a/key_files/ma_sss.yml b/key_files/ma_sss.yml index 8887d416..e049a85a 100644 --- a/key_files/ma_sss.yml +++ b/key_files/ma_sss.yml @@ -4,7 +4,7 @@ units : PSU dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : so_abs so # dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_s00_01v2_AMM.nc diff --git a/key_files/ma_sst.yml b/key_files/ma_sst.yml index 639870b7..278bf75f 100644 --- a/key_files/ma_sst.yml +++ b/key_files/ma_sst.yml @@ -4,7 +4,7 @@ units : degrees C dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : thetao thetao_con votemper model_convert : choose_best_var diff --git a/key_files/ma_ta.yml b/key_files/ma_ta.yml index 9f8c7811..09f0945a 100644 --- a/key_files/ma_ta.yml +++ b/key_files/ma_ta.yml @@ -4,7 +4,7 @@ units : umol/kg dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*ptrc_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_TA model_convert : NoChange diff --git a/key_files/ma_totalairseafluxco2.yml b/key_files/ma_totalairseafluxco2.yml index cdb59bc5..249a7d46 100644 --- a/key_files/ma_totalairseafluxco2.yml +++ b/key_files/ma_totalairseafluxco2.yml @@ -4,7 +4,7 @@ units : Pg C/yr model : NEMO-ERSEM modelgrid : eORCA025 dimensions : 1 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc model_vars : O3_fair model_convert : path: bgcval2/functions/AirSeaFluxCO2.py diff --git a/key_files/ma_totalchlorophyll.yml b/key_files/ma_totalchlorophyll.yml index 8079db9b..b2e00e80 100644 --- a/key_files/ma_totalchlorophyll.yml +++ b/key_files/ma_totalchlorophyll.yml @@ -4,7 +4,7 @@ units : "" dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_Chl_result model_convert : NoChange diff --git a/key_files/ma_totaliceextent.yml b/key_files/ma_totaliceextent.yml index c2562d05..a8fbbeff 100644 --- a/key_files/ma_totaliceextent.yml +++ b/key_files/ma_totaliceextent.yml @@ -4,7 +4,7 @@ name : TotalIceExtent units : 1E6 km^2 dimensions : 1 model : NEMO-ERSEM -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc model_vars : soicecov model_convert: path: bgcval2/functions/ice.py diff --git a/key_files/ma_totalintgpp.yml b/key_files/ma_totalintgpp.yml index 108f2ae9..e96d18ad 100644 --- a/key_files/ma_totalintgpp.yml +++ b/key_files/ma_totalintgpp.yml @@ -2,7 +2,7 @@ name : TotalIntGPP units : Gt/yr dimensions : 1 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc model_vars : Ptot_NPP_result model_convert : path: bgcval2/functions/TotalIntPP.py diff --git a/key_files/ma_totalintnpp.yml b/key_files/ma_totalintnpp.yml index d931cbed..0cd3ce7b 100644 --- a/key_files/ma_totalintnpp.yml +++ b/key_files/ma_totalintnpp.yml @@ -2,7 +2,7 @@ name : TotalIntNPP units : Gt/yr dimensions : 1 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc model_vars : Ptot_NPP_result model_convert : path: bgcval2/functions/TotalIntPP.py diff --git a/key_files/ma_totalphytoc.yml b/key_files/ma_totalphytoc.yml index 4ab925d1..8d29f1b7 100644 --- a/key_files/ma_totalphytoc.yml +++ b/key_files/ma_totalphytoc.yml @@ -4,7 +4,7 @@ units : mg C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ptot_c_result model_convert : NoChange diff --git a/key_files/ma_totalzooc.yml b/key_files/ma_totalzooc.yml index 32f61327..1391278b 100644 --- a/key_files/ma_totalzooc.yml +++ b/key_files/ma_totalzooc.yml @@ -4,7 +4,7 @@ units : mg C/m^3 dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*diag_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*diag_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : Ztot_c_result model_convert : NoChange diff --git a/key_files/ma_zos.yml b/key_files/ma_zos.yml index a8fc799e..3e6d3440 100644 --- a/key_files/ma_zos.yml +++ b/key_files/ma_zos.yml @@ -4,7 +4,7 @@ units : m dimensions : 1 model : NEMO025-ERSEM modelgrid : eORCA025 -modelFiles : $BASEDIR_MODEL/$JOBID/*_1m_*grid_T*.nc +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : zos layers : layerless From a1d04070cddd132401c337e28b597be820e9e8af Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 30 Oct 2023 11:19:58 +0000 Subject: [PATCH 09/16] fixed error when file is missing --- bgcval2/functions/TotalIntPP.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bgcval2/functions/TotalIntPP.py b/bgcval2/functions/TotalIntPP.py index 6226fc56..57437b6f 100644 --- a/bgcval2/functions/TotalIntPP.py +++ b/bgcval2/functions/TotalIntPP.py @@ -31,6 +31,8 @@ from bgcval2.bgcvaltools.dataset import dataset from bgcval2.functions.get_kwarg_file import get_kwarg_file from bgcval2.functions.standard_functions import find_best_var +import os + global loadedArea global model_area @@ -84,6 +86,10 @@ def MA_TotalIntPP(nc, keys, **kwargs): # This will only work for NEMO models: thkfn = nc.filename.replace('diag_T', 'grid_T').replace('ptrc_T', 'grid_T') print('MA_TotalIntPP: opening:', thkfn) + if not os.path.exists(thkfn): + print('MA_TotalIntPP: ERROR: does not exist:', thkfn) + return np.ma.masked + thknc = dataset(thkfn, 'r') thick = thknc.variables['thkcello'][:] thknc.close() From 290ee4db414982416e47b230d647fb54428e43cb Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 9 Nov 2023 14:57:35 +0000 Subject: [PATCH 10/16] added metric to path --- bgcval2/analysis_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/analysis_compare.py b/bgcval2/analysis_compare.py index e1c3be68..edfc376f 100755 --- a/bgcval2/analysis_compare.py +++ b/bgcval2/analysis_compare.py @@ -412,7 +412,7 @@ def timeseries_compare(jobs, data=-999, # in situ data distribution title=title, filename=bvt.folder(imageFolder) + - '_'.join([name, region, layer, ts, ls + '.png']), + '_'.join([name, region, layer, ts, metric, ls + '.png']), units=units, plotStyle=ts, lineStyle=ls, From 8ec9ecb1015d212b5662d42b0a127de594f0eacb Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 12 Dec 2023 11:42:18 +0000 Subject: [PATCH 11/16] Added MA fixes and new revert shelves tool. --- bgcval2/bgcval2_make_report.py | 6 +- input_yml/mission_atlantic_CNRM.yml | 6 +- key_files/ma_aeu.yml | 2 + key_files/ma_ammonium.yml | 2 +- key_files/ma_amoc_26n.yml | 2 + key_files/ma_drake.yml | 2 + key_files/ma_iron.yml | 2 +- key_files/ma_nitrate.yml | 2 +- key_files/ma_ph.yml | 4 +- key_files/ma_phosphate.yml | 2 +- key_files/ma_silicate.yml | 2 +- key_files/ma_sss.yml | 5 +- key_files/ma_ta.yml | 4 +- key_lists/missionatlantic_kmf.yml | 3 +- revert_shelves.py | 154 ++++++++++++++++++++++++++++ 15 files changed, 181 insertions(+), 17 deletions(-) create mode 100644 revert_shelves.py diff --git a/bgcval2/bgcval2_make_report.py b/bgcval2/bgcval2_make_report.py index a4266042..8c12805c 100755 --- a/bgcval2/bgcval2_make_report.py +++ b/bgcval2/bgcval2_make_report.py @@ -1504,8 +1504,8 @@ def newImageLocation(fn): 'Salinty_Global_Surface', 'FreshwaterFlux_Global', 'TotalHeatFlux', - 'MA_SST', - 'MA_SSS', + 'MA_SST_Global_Surface', + 'MA_SSS_Global_Surface', 'MA_Drake', 'MA_AMOC_26N', 'MA_AEU', @@ -1546,6 +1546,8 @@ def newImageLocation(fn): if found: continue sectionTitle = 'Physics Key Metrics' if fn.find(key) > -1: +# if key in ['MA_SST', 'MA_SSS', 'MA_Nitrate',] and fn.find('Global_Surface') < 0: +# continue try: categories[sectionTitle].append(fn) except: diff --git a/input_yml/mission_atlantic_CNRM.yml b/input_yml/mission_atlantic_CNRM.yml index f2d1546d..163f8802 100644 --- a/input_yml/mission_atlantic_CNRM.yml +++ b/input_yml/mission_atlantic_CNRM.yml @@ -25,7 +25,7 @@ jobs: thickness: 1.8 linestyle: '-' shifttime: 0. - suite: missionatlantic + suite: missionatlantic missionatlantic_kmf CNRM_ssp370: description: 'CNRM SSP3-7.0' @@ -33,7 +33,7 @@ jobs: thickness: 1.8 linestyle: '-' shifttime: 0. - suite: missionatlantic + suite: missionatlantic missionatlantic_kmf CNRM_ssp126: description: 'CNRM SSP1-2.6' @@ -41,6 +41,6 @@ jobs: thickness: 1.8 linestyle: '-' shifttime: 0. - suite: missionatlantic + suite: missionatlantic missionatlantic_kmf diff --git a/key_files/ma_aeu.yml b/key_files/ma_aeu.yml index 3f277cc7..1744824d 100644 --- a/key_files/ma_aeu.yml +++ b/key_files/ma_aeu.yml @@ -14,3 +14,5 @@ model_convert: max_depth: 500. layers: layerless regions: regionless +smoothings : DataOnly both5 + diff --git a/key_files/ma_ammonium.yml b/key_files/ma_ammonium.yml index a86a9843..b7824c04 100644 --- a/key_files/ma_ammonium.yml +++ b/key_files/ma_ammonium.yml @@ -8,6 +8,6 @@ modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N4_n model_convert : NoChange -layers : Surface 500m +layers : Surface #00m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_amoc_26n.yml b/key_files/ma_amoc_26n.yml index cb0a0aca..bfbe5760 100644 --- a/key_files/ma_amoc_26n.yml +++ b/key_files/ma_amoc_26n.yml @@ -15,3 +15,5 @@ model_convert: grid: eORCA025 layers: layerless regions: regionless +smoothings : DataOnly both5 + diff --git a/key_files/ma_drake.yml b/key_files/ma_drake.yml index d4b79187..8b1d33cd 100644 --- a/key_files/ma_drake.yml +++ b/key_files/ma_drake.yml @@ -13,3 +13,5 @@ model_convert: grid: eORCA025 layers: layerless regions: regionless +smoothings : DataOnly both5 + diff --git a/key_files/ma_iron.yml b/key_files/ma_iron.yml index b8e73fb6..05830953 100644 --- a/key_files/ma_iron.yml +++ b/key_files/ma_iron.yml @@ -8,6 +8,6 @@ modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : N7_f model_convert : NoChange -layers : Surface 500m +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_nitrate.yml b/key_files/ma_nitrate.yml index 869900a6..e0b6e7b5 100644 --- a/key_files/ma_nitrate.yml +++ b/key_files/ma_nitrate.yml @@ -13,6 +13,6 @@ dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_n00_01.nc data_vars : n_an data_convert : NoChange data_source : WOA18 -layers : Surface 500m +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_ph.yml b/key_files/ma_ph.yml index 9b55f665..e329942c 100644 --- a/key_files/ma_ph.yml +++ b/key_files/ma_ph.yml @@ -8,6 +8,6 @@ modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_pH model_convert : NoChange -layers : Surface 500m +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean -metrics : mean, min #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max +metrics : mean #, min #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_phosphate.yml b/key_files/ma_phosphate.yml index 3cd26121..14f4b90d 100644 --- a/key_files/ma_phosphate.yml +++ b/key_files/ma_phosphate.yml @@ -13,6 +13,6 @@ dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_p00_01.nc data_vars : p_an data_convert : NoChange data_source : WOA18 -layers : Surface 500m +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_silicate.yml b/key_files/ma_silicate.yml index 4e364ff7..53fd8787 100644 --- a/key_files/ma_silicate.yml +++ b/key_files/ma_silicate.yml @@ -13,6 +13,6 @@ dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_i00_01.nc data_vars : i_an data_convert : NoChange data_source : WOA18 -layers : Surface 100m 200m 500m 1000m 2000m 4000m +layers : Surface #100m 200m 500m 1000m 2000m 4000m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_sss.yml b/key_files/ma_sss.yml index e049a85a..2fdc3299 100644 --- a/key_files/ma_sss.yml +++ b/key_files/ma_sss.yml @@ -12,6 +12,7 @@ dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa18_deca data_vars : s_an data_convert : NoChange data_source : WOA18 -layers : Surface 50m 100m 200m 500m 1000m 2000m 4000m -regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean ITCZ +layers : Surface #50m 100m 200m 500m 1000m 2000m + #layers : Surface 50m 100m 200m 500m 1000m 2000m 4000m +regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean #TCZ metrics : mean #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_files/ma_ta.yml b/key_files/ma_ta.yml index 09f0945a..3973702d 100644 --- a/key_files/ma_ta.yml +++ b/key_files/ma_ta.yml @@ -8,6 +8,6 @@ modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc model_vars : O3_TA model_convert : NoChange -layers : Surface 500m +layers : Surface #500m regions : Global NorthAtlanticOcean SouthAtlanticOcean EquatorialAtlanticOcean -metrics : mean, max #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max +metrics : mean #, max #, median, 10pc, 20pc, 30pc, 40pc, 50pc, 60pc, 70pc, 80pc, 90pc, min, max diff --git a/key_lists/missionatlantic_kmf.yml b/key_lists/missionatlantic_kmf.yml index 279b6148..662fa563 100644 --- a/key_lists/missionatlantic_kmf.yml +++ b/key_lists/missionatlantic_kmf.yml @@ -5,7 +5,7 @@ keys: # Physics: # - # MA_SST: True + MA_SST_KMF: True #MA_SSS: True MA_AEU: True MA_Drake: True @@ -23,6 +23,7 @@ keys: # MA_Totalairseafluxco2: True # Not in test17 # MA_TotalIntGPP: True MA_TotalIntNPP: True + MA_Nitrate_KMF: True #MA_Nitrate: True # MA_Ammonium: True # MA_Phosphate: True diff --git a/revert_shelves.py b/revert_shelves.py new file mode 100644 index 00000000..fb7f0963 --- /dev/null +++ b/revert_shelves.py @@ -0,0 +1,154 @@ + +""" +In this script, we remove years of data from a shelve +""" +from shelve import open as shOpen +import glob +import os + +from bgcval2.bgcvaltools import bv2tools as bvt +from bgcval2._runtime_config import get_run_configuration +from bgcval2.Paths.paths import paths_setter + + + + +def get_paths( + config_user=None + ): + # get runtime configuration + if config_user: + paths_dict, config_user = get_run_configuration(config_user) + else: + paths_dict, config_user = get_run_configuration("defaults") + + # filter paths dict into an object that's usable below + paths = paths_setter(paths_dict) + return paths.shelvedir + + +def load_all_datatypes(shelvedir, jobID): + """ + Returns a + """ + wildcards = shelvedir + '_'.join([ + jobID, + '*', + ]) + '.shelve.dat' + files = glob.glob(wildcards) + datatypes = [] + for fn in files: + basename = os.path.basename(fn) + basename = basename.replace(jobID+'_', '') + basename = basename.replace('.shelve.dat', '') + datatypes.append(basename) + return datatypes + + + +def remove_data(jobID, + year, + month=None, + config_user=None, + dataTypes= ['MA_AMOC_26N', ], + dry_run = True, + ): + """ + Remove all data from the year and month from shelve files. + """ + + path_shelvedir = get_paths(config_user=config_user) + + shelvedir = bvt.folder([path_shelvedir, "timeseries", jobID]) + + if dataTypes == ['all', ]: + dataTypes = load_all_datatypes(shelvedir, jobID) + + print(dataTypes) + return + for dataType in dataTypes: + shelvefn = shelvedir + '_'.join([ + jobID, + dataType, + ]) + '.shelve' + + print(shelvefn) + + if glob.glob(shelvefn+'*'): + sh = shOpen(shelvefn) + print('Shelve loads okay:', shelvefn +'*') + sh = shOpen(shelvefn) + readFiles = sh['readFiles'] + modeldataD = sh['modeldata'] + sh.close() + + if month: + time_key = year+month + else: + time_key = year + + changes = 0 + remove_files = [] + for readFile in readFiles: + if readFile.find(time_key) > -1: + remove_files.append(readFile) + changes +=1 + + if not changes: + print('Nothing to remove') + return + + # Remove files from list + for remove_file in remove_files: + print('Removing', remove_file) + if dry_run: + pass + else: + readFiles.remove(remove_file) + + # remove processed data from file. + for (r, l, m), values in modeldataD.items(): + key_removes = [] + for time in values.keys(): + if int(time) == int(year): + # year matches: + if not month: + # No month (delete all entries from this year + key_removes.append(time) + continue + # search for month: + mn = int((time - int(year)) *12) + if mn == int(month): + # found same month + key_removes.append(time) + + for key_remove in key_removes: + print('Removing', key_remove) + if dry_run: + pass + else: + del values[key_remove] + + # Save file. + if not dry_run: + print('Saving:', shelvefn) + sh = shOpen(shelvefn) + sh['readFiles'] = readFiles + sh['modeldata'] = modeldataD + sh.close() + else: + print('Not saving (dry_run):', shelvefn) + + +def main(): + jobID = 'GFDL_hist' + years = ['2007', ] + months = ['07', ] + dataTypes = ['all', ] + dry_run = True + for year in years: + for month in months: + remove_data(jobID, year, month=month,dataTypes=dataTypes, dry_run=dry_run) #, month) + + +main() From b3af6e4aed1d14a6fef6ce4a86c597fbbd2b81a3 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 12 Dec 2023 12:13:42 +0000 Subject: [PATCH 12/16] added new files and command revert --- .../revert_shelves.py | 100 +++++++++++++++--- input_yml/mission_atlantic_ALL.yml | 59 +++++++++++ key_files/ma_nitrate_kmf.yml | 18 ++++ key_files/ma_sst_kmf.yml | 20 ++++ setup.py | 1 + 5 files changed, 183 insertions(+), 15 deletions(-) rename revert_shelves.py => bgcval2/revert_shelves.py (57%) create mode 100644 input_yml/mission_atlantic_ALL.yml create mode 100644 key_files/ma_nitrate_kmf.yml create mode 100644 key_files/ma_sst_kmf.yml diff --git a/revert_shelves.py b/bgcval2/revert_shelves.py similarity index 57% rename from revert_shelves.py rename to bgcval2/revert_shelves.py index fb7f0963..564efa83 100644 --- a/revert_shelves.py +++ b/bgcval2/revert_shelves.py @@ -5,6 +5,8 @@ from shelve import open as shOpen import glob import os +import argparse + from bgcval2.bgcvaltools import bv2tools as bvt from bgcval2._runtime_config import get_run_configuration @@ -38,6 +40,8 @@ def load_all_datatypes(shelvedir, jobID): files = glob.glob(wildcards) datatypes = [] for fn in files: + if fn.find('insitu') > -1: + continue basename = os.path.basename(fn) basename = basename.replace(jobID+'_', '') basename = basename.replace('.shelve.dat', '') @@ -64,16 +68,12 @@ def remove_data(jobID, if dataTypes == ['all', ]: dataTypes = load_all_datatypes(shelvedir, jobID) - print(dataTypes) - return for dataType in dataTypes: shelvefn = shelvedir + '_'.join([ jobID, dataType, ]) + '.shelve' - print(shelvefn) - if glob.glob(shelvefn+'*'): sh = shOpen(shelvefn) print('Shelve loads okay:', shelvefn +'*') @@ -82,7 +82,13 @@ def remove_data(jobID, modeldataD = sh['modeldata'] sh.close() - if month: + # generate a time key from year/month. + if month: + if isinstance(month, str) and len(month) ==1: + month = int(month) + if isinstance(month, int): + month = bvt.mnStr(month) + time_key = year+month else: time_key = year @@ -96,7 +102,7 @@ def remove_data(jobID, if not changes: print('Nothing to remove') - return + continue # Remove files from list for remove_file in remove_files: @@ -140,15 +146,79 @@ def remove_data(jobID, print('Not saving (dry_run):', shelvefn) +def get_args(): + """Parse command line arguments.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('-c', + '--config-file', + default=os.path.join(os.getcwd(), + 'config-user.yml'), + help='User configuration file', + required=False) + + parser.add_argument('-j', + '--jobids', + default=None, + nargs='+', + type=str, + help='One or more jobIDs (required)', + required=True) + + parser.add_argument('-y', + '--years', + default=None, + nargs='+', + type=str, + help='One or more years (required)', + required=True) + parser.add_argument('-m', + '--months', + default=[None, ], + nargs='+', + type=str, + help='One or more months - default is all months', + required=False) + + parser.add_argument('-k', + '--keys', + default=['all',], + nargs='+', + type=str, + help='One or more datasets - default is everything. ', + required=False) + + parser.add_argument('-d', + '--dry-run', + action='store_true', + help='Dry run: Do not edit any files.', + required=False + ) + + + args = parser.parse_args() + + return args + + + def main(): - jobID = 'GFDL_hist' - years = ['2007', ] - months = ['07', ] - dataTypes = ['all', ] - dry_run = True - for year in years: - for month in months: - remove_data(jobID, year, month=month,dataTypes=dataTypes, dry_run=dry_run) #, month) + args = get_args() + jobIDs = args.jobids + years = args.years + months = args.months + dataTypes = args.keys + dry_run = args.dry_run + + for jobID in jobIDs: + for year in years: + for month in months: + print('start: remove_data', jobID, year,month, dataTypes, dry_run) + remove_data(jobID, year, month=month,dataTypes=dataTypes, dry_run=dry_run) + + +if __name__ == "__main__": + main() -main() diff --git a/input_yml/mission_atlantic_ALL.yml b/input_yml/mission_atlantic_ALL.yml new file mode 100644 index 00000000..b65eadeb --- /dev/null +++ b/input_yml/mission_atlantic_ALL.yml @@ -0,0 +1,59 @@ +--- +# GC5 N96 ORCA1 spinup analysis +name: MissionAtlantic_All + +# Run the single job analysis +do_analysis_timeseries: True + +# Download from mass: +do_mass_download: False + +# Strict file check (fails if True and files are missing) +strictFileCheck: False + +# master analysis suite +master_suites: missionatlantic + +# Auto download jobs (can also be set for individual jobs). +auto_download: False + +# Job ID's suites as named by Rose/Cylc +jobs: + CNRM_hist: + description: 'CNRM historical run.' + label: 'CNRM-hist' + colour: green + thickness: 1.8 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + GFDL_hist: + description: 'GFDL historical run.' + label: 'GFDL-hist' + colour: black + thickness: 1.2 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + + CNRM_ssp370: + description: 'CNRM SSP3-7.0' + label: 'CNRM-ssp370' + colour: purple + thickness: 1.2 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + CNRM_ssp126: + description: 'CNRM SSP1-2.6' + label: 'CNRM-ssp126' + colour: dodgerblue + thickness: 1.2 + linestyle: '-' + shifttime: 0. + suite: missionatlantic missionatlantic_kmf + + diff --git a/key_files/ma_nitrate_kmf.yml b/key_files/ma_nitrate_kmf.yml new file mode 100644 index 00000000..3d74191d --- /dev/null +++ b/key_files/ma_nitrate_kmf.yml @@ -0,0 +1,18 @@ +--- +name : MA_Nitrate +units : mmol N/m^3 +dimensions : 1 +model : NEMO025-ERSEM +modelgrid : eORCA025 +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*ptrc_T*.nc +gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc +model_vars : N3_n +model_convert : NoChange + #dataFile : $BASEDIR_OBS/WOA/annual/woa13_all_n00_01.nc +dataFile : $BASEDIR_OBS/WOA/annual/woa18_all_n00_01.nc +data_vars : n_an +data_convert : NoChange +data_source : WOA18 +layers : Surface +regions : Global +metrics : mean diff --git a/key_files/ma_sst_kmf.yml b/key_files/ma_sst_kmf.yml new file mode 100644 index 00000000..131339fa --- /dev/null +++ b/key_files/ma_sst_kmf.yml @@ -0,0 +1,20 @@ +--- +name : MA_SST +units : degrees C +dimensions : 1 +model : NEMO025-ERSEM +modelgrid : eORCA025 +modelFiles : $BASEDIR_MODEL/$JOBID/*/*/*_1m_*grid_T*.nc +gridFile : /data/proteus2/scratch/ledm/MissionAtlantic/SENEMO/eORCA025_mesh_mask_mes_v2.nc +model_vars : thetao thetao_con votemper +model_convert : choose_best_var + #dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa13_decav_t00_01v2_AMM.nc +dataFile : /data/sthenno1/scratch/ledm/Observations/WOA/annual/woa18_decav_t00_01.nc +data_vars : t_an +data_convert : NoChange +data_source : WOA18 +layers : Surface #50m 100m +regions : Global +metrics : mean +smoothings : DataOnly both5 + diff --git a/setup.py b/setup.py index 015512b2..ac37167e 100755 --- a/setup.py +++ b/setup.py @@ -203,6 +203,7 @@ def read_authors(filename): 'analysis_timeseries = bgcval2.analysis_timeseries:main', 'download_from_mass = bgcval2.download_from_mass:main', 'bgcval2_make_report = bgcval2.bgcval2_make_report:main', + 'revert_shelves = bgcval2.revert_shelves:main', ], }, cmdclass={ From 918c6c807ce3214ff02543bfef1d64333ed1ef42 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 12 Dec 2023 12:23:28 +0000 Subject: [PATCH 13/16] added documentation --- README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3c9e8aee..2187781b 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Executable name | What it does | Command `bgcval` | runs time series and point to point. | bgcval jobID `bgcval2_make_report` | makes the single model HTML report. | bgcval2_make_report jobID `analysis_compare` | runs comparison of multiple single jobs | analysis_compare - +`revert_shelves` | Removes specific years or months from processed shelves | revert_shelves -j jobID -y years -m months ### Checking out development branches @@ -562,6 +562,31 @@ The third place that these plots are kept is on the public facing jasmin directo This is where the report is hosted. + + +Fixing mistakes +--------------- + +The `revert_shelves` tool is built to remove erroneous processed data from bgcval2 +shelves (where processed data is stored. +This is useful for processed updated simulations, new runs, post hoc fixes +and for debugging. + +The revert Shelves tool is set up to remove for a given jobID, year, and even month. + +A typical useage would be: +``` +revert_shelves -j jobID1 jobID2 -y 2009 2010 -m 02 11 -k AMOC +``` + +This would remove all AMOC data from February and November from the +years 2009 and 2010. +The default behaviour for the `-k` argument is to run this over all keys. + +Test this command using the dry run argument `-d`. + + + Point to point analysis ----------------------- From e9d9debaeb7cd1d2e36164ae053db059c7077ab6 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 12 Dec 2023 15:57:19 +0000 Subject: [PATCH 14/16] removed .idea files. --- .idea/bgcval2.iml | 10 ---------- .idea/inspectionProfiles/profiles_settings.xml | 6 ------ .idea/misc.xml | 4 ---- .idea/vcs.xml | 6 ------ 4 files changed, 26 deletions(-) delete mode 100644 .idea/bgcval2.iml delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/vcs.xml diff --git a/.idea/bgcval2.iml b/.idea/bgcval2.iml deleted file mode 100644 index aad402c4..00000000 --- a/.idea/bgcval2.iml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2da..00000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index c47c9106..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1ddf..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From c78c1ea20fc42a1e1f660e8d700f00e1d16ace8d Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 12 Dec 2023 16:00:54 +0000 Subject: [PATCH 15/16] removed old runs --- input_yml/mission_atlantic.yml | 155 --------------------------- input_yml/mission_atlantic_debug.yml | 26 ----- 2 files changed, 181 deletions(-) delete mode 100644 input_yml/mission_atlantic.yml delete mode 100644 input_yml/mission_atlantic_debug.yml diff --git a/input_yml/mission_atlantic.yml b/input_yml/mission_atlantic.yml deleted file mode 100644 index e79d0a64..00000000 --- a/input_yml/mission_atlantic.yml +++ /dev/null @@ -1,155 +0,0 @@ ---- -# GC5 N96 ORCA1 spinup analysis -name: MissionAtlantic_PML - -# Run the single job analysis -do_analysis_timeseries: True - -# Download from mass: -do_mass_download: False - -# Strict file check (fails if True and files are missing) -strictFileCheck: False - -# master analysis suite -master_suites: missionatlantic - -# Auto download jobs (can also be set for individual jobs). -auto_download: False - -# Job ID's suites as named by Rose/Cylc -jobs: - test17: - description: 'NEMO-ERSEM eORCA025 - revised rivers ' - colour: green - thickness: 1.8 - linestyle: '-' - shifttime: 0. - suite: missionatlantic -# test17a: -# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.1' -# colour: red -# thickness: 1.7 -# linestyle: '-' -# shifttime: 0. -# suite: missionatlantic - -# test17aa: -# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 1' -# colour: blue -# thickness: 1.7 -# linestyle: '-' -# shifttime: 0. -# suite: missionatlantic -# test17b: -# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 2' -# colour: blue -# thickness: 1.7 -# linestyle: '-' -# shifttime: 1. -# suite: missionatlantic -# test17c: -# description: 'NEMO-ERSEM eORCA025 - with iron deposition x0.59 spinup year 3' -# colour: blue -# thickness: 1.7 -# linestyle: '-' -# shifttime: 2. -# suite: missionatlantic - -# Broken with weird DIC/pH -# test18a: -# description: 'NEMO-ERSEM eORCA025 - new river input - spinup year 1' -# colour: purple -# thickness: 1.7 -# linestyle: '-' -# shifttime: 0. -# suite: missionatlantic -# test18b: -# description: 'NEMO-ERSEM eORCA025 - new river input - spinup year 2' -# colour: purple -# thickness: 1.7 -# linestyle: '-' -# shifttime: 1. -# suite: missionatlantic -# test18c: -# description: 'NEMO-ERSEM eORCA025 - new river input - spinup year 3' -# colour: purple -# thickness: 1.7 -# linestyle: '-' -# shifttime: 2. -# suite: missionatlantic - -# test19_S1: -# description: 'NEMO-ERSEM eORCA025 - same as 18 but better - spinup year 1' -# colour: orange -# thickness: 1.7 -# linestyle: '-' -# shifttime: 0. -# suite: missionatlantic -# test19_S2: -# description: 'NEMO-ERSEM eORCA025 - same as 18 but better - spinup year 2' -# colour: orange -# thickness: 1.7 -# linestyle: '-' -# shifttime: 1. -# suite: missionatlantic -# test19_S3: -# description: 'NEMO-ERSEM eORCA025 - same as 18 but better - spinup year 3' -# colour: orange -# thickness: 1.7 -# linestyle: '-' -# shifttime: 2. -# suite: missionatlantic - - test19b_S1: - description: 'NEMO-ERSEM eORCA025 - same as 19 but better - spinup year 1' - colour: dodgerblue - thickness: 1.7 - linestyle: '-' - shifttime: 0. - suite: missionatlantic -# test19b_S2: -# description: 'NEMO-ERSEM eORCA025 - same as 19 but better - spinup year 2' -# colour: dodgerblue -# thickness: 1.7 -# linestyle: '-' -# shifttime: 1. -# suite: missionatlantic -# test19b_S3: -# description: 'NEMO-ERSEM eORCA025 - same as 19 but better - spinup year 3' -# colour: dodgerblue -# thickness: 1.7 -# linestyle: '-' -# shifttime: 2. -# suite: missionatlantic - - test25: - description: 'NEMO-ERSEM eORCA025 - stable(?!)' - colour: purple - thickness: 1.7 - linestyle: '-' - shifttime: 02. - suite: missionatlantic - -# test25S1: -# description: 'NEMO-ERSEM eORCA025 - stablE(?!) spinup s1' -# colour: purple -# thickness: 1.7 -# linestyle: '-' -# shifttime: -3. -# suite: missionatlantic - test25S2: - description: 'NEMO-ERSEM eORCA025 - stable(?!) spinup 2' - colour: purple - thickness: 1.7 - linestyle: '-' - shifttime: -2. - suite: missionatlantic - test25S3: - description: 'NEMO-ERSEM eORCA025 - stable(?!) spinup3' - colour: purple - thickness: 1.7 - linestyle: '-' - shifttime: -1. - suite: missionatlantic - diff --git a/input_yml/mission_atlantic_debug.yml b/input_yml/mission_atlantic_debug.yml deleted file mode 100644 index a96d6af1..00000000 --- a/input_yml/mission_atlantic_debug.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -# GC5 N96 ORCA1 spinup analysis -name: MissionAtlantic_PML_debug - -# Run the single job analysis -do_analysis_timeseries: True - -# Download from mass: -do_mass_download: False - -# master analysis suite -master_suites: ma_debug - -# Auto download jobs (can also be set for individual jobs). -auto_download: False - -# Job ID's suites as named by Rose/Cylc -jobs: - test16: - description: 'NEMO-ERSEM eORCA025' - colour: purple - thickness: 1.5 - linestyle: '-' - shifttime: 0. - suite: ma_debug - From d8a888412eea75938ca56a9147af936728810a7a Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Wed, 13 Dec 2023 10:36:53 +0000 Subject: [PATCH 16/16] Apply suggestions from code review Co-authored-by: Valeriu Predoi --- bgcval2/bgcval2_make_report.py | 9 +++------ bgcval2/functions/circulation.py | 7 ------- bgcval2/p2p/matchDataAndModel.py | 3 +-- bgcval2/revert_shelves.py | 8 +++----- 4 files changed, 7 insertions(+), 20 deletions(-) diff --git a/bgcval2/bgcval2_make_report.py b/bgcval2/bgcval2_make_report.py index 8c12805c..4eae28b2 100755 --- a/bgcval2/bgcval2_make_report.py +++ b/bgcval2/bgcval2_make_report.py @@ -984,10 +984,10 @@ def newImageLocation(fn): outlevels = {} for i, fn in enumerate(l2Fields): for lv in levels: - if fn.find(lv)>-1: - outlevels[lv]= True + if fn.find(lv) > -1: + outlevels[lv] = True fn = fn.replace(lv, '') - outdict[fn]= True + outdict[fn] = True l2Fields = [key for key, v in outdict.items()] slices = [key for key, v in outlevels.items()] @@ -1000,7 +1000,6 @@ def newImageLocation(fn): Descriptions = {} FileLists = {} FileOrder = {} - #print(l2Fields, slices) for key in sorted(l2Fields): #if key not in ['Alkalinity','Nitrate']: continue @@ -1546,8 +1545,6 @@ def newImageLocation(fn): if found: continue sectionTitle = 'Physics Key Metrics' if fn.find(key) > -1: -# if key in ['MA_SST', 'MA_SSS', 'MA_Nitrate',] and fn.find('Global_Surface') < 0: -# continue try: categories[sectionTitle].append(fn) except: diff --git a/bgcval2/functions/circulation.py b/bgcval2/functions/circulation.py index 775e16f9..7af0d624 100644 --- a/bgcval2/functions/circulation.py +++ b/bgcval2/functions/circulation.py @@ -304,18 +304,11 @@ def twentysixnorth025(nc,keys,**kwargs): continue if not depths[t, z, la, lo] or np.ma.is_masked(depths[t, z, la, lo]): continue -# if not alttmask_AMOC26N[la, lo]: -# continue -# if not tmask_AMOC26N[z, la, lo]: -# continue -# if np.ma.is_masked(zv[0, z, la, lo]): -# continue atlmoc[z, la] = atlmoc[z, la] - e1v[t, 0, la, lo] * thkcello[t, z, la, lo] * vo[t, z, la, lo] / 1.E06 for z in range(thkcello.shape[1] -2, 1, -1): # add from the bottom up atlmoc[z, :] = atlmoc[z+1, :] + atlmoc[z, :] print('AMOC:', atlmoc.max()) - #assert 0 return atlmoc.max() diff --git a/bgcval2/p2p/matchDataAndModel.py b/bgcval2/p2p/matchDataAndModel.py index 2a9593fb..f470c46f 100755 --- a/bgcval2/p2p/matchDataAndModel.py +++ b/bgcval2/p2p/matchDataAndModel.py @@ -943,7 +943,6 @@ def loadMesh(self, modelfile=None): if not set(ncER.variables.keys()).intersection(set(depthNames)): print('load mesh: no depth field available in ', modelfile) self.depthcc = np.array([0]) -# assert 0 # self.depthcc = choose_best_ncvar(ncER, depthNames)[:] self.datescc = var_to_datetime(ncER.variables[self.modelcoords['t']]) @@ -1135,7 +1134,7 @@ def var_to_datetime(ncvar): if units in ['months since 0000-01-01 00:00:00', ]: units = 'months since 2000-01-01 00:00:00' return num2date(ncvar[:], 'months since 2000-01-01 00:00:00', calendar='360_day') - elif units.find('months since')>-1: + elif units.find('months since') > -1: return num2date(ncvar[:], units, calendar='360_day') return num2date(ncvar[:], ncvar.units, calendar=calendar) diff --git a/bgcval2/revert_shelves.py b/bgcval2/revert_shelves.py index 564efa83..b9a2d152 100644 --- a/bgcval2/revert_shelves.py +++ b/bgcval2/revert_shelves.py @@ -13,8 +13,6 @@ from bgcval2.Paths.paths import paths_setter - - def get_paths( config_user=None ): @@ -84,12 +82,12 @@ def remove_data(jobID, # generate a time key from year/month. if month: - if isinstance(month, str) and len(month) ==1: + if isinstance(month, str) and len(month) == 1: month = int(month) if isinstance(month, int): month = bvt.mnStr(month) - time_key = year+month + time_key = year + month else: time_key = year @@ -186,7 +184,7 @@ def get_args(): default=['all',], nargs='+', type=str, - help='One or more datasets - default is everything. ', + help='One or more datasets - default is everything (all available keys). ', required=False) parser.add_argument('-d',