From a2ec969d40069d58bf08237f6c22e838d510607e Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 25 Mar 2024 11:20:36 +0000 Subject: [PATCH 1/7] Added SubpolarNorthAtlantic region --- bgcval2/bgcvaltools/makeMask.py | 6 ++++++ bgcval2/bgcvaltools/pftnames.py | 2 ++ key_files/temperature.yml | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/bgcval2/bgcvaltools/makeMask.py b/bgcval2/bgcvaltools/makeMask.py index 1a4f7a57..60e4c0fa 100644 --- a/bgcval2/bgcvaltools/makeMask.py +++ b/bgcval2/bgcvaltools/makeMask.py @@ -245,6 +245,12 @@ def makeMask(name, newSlice, xt, xz, xy, xx, xd, debug=False): xy, 60., 80.).mask return mx + if newSlice in ['SubpolarNorthAtlantic', 'SPNA',]: + # Based on SPNA region here: https://www.nature.com/articles/s43247-021-00120-y#citeas + mx = np.ma.masked_outside(xx, -35., -10.).mask + np.ma.masked_outside( + xy, 40., 65.).mask + return mx + if newSlice == 'AtlanticSOcean': mx = np.ma.masked_outside(xx, -40., 20.).mask + np.ma.masked_outside( xy, -50., -75.).mask diff --git a/bgcval2/bgcvaltools/pftnames.py b/bgcval2/bgcvaltools/pftnames.py index c98d93e1..1e8a3d73 100644 --- a/bgcval2/bgcvaltools/pftnames.py +++ b/bgcval2/bgcvaltools/pftnames.py @@ -527,6 +527,8 @@ def makeLongNameDict(): lnd['Remainder'] = "Oligotrophic Gyres" lnd['ArcticOcean'] = "Arctic Ocean" lnd['NorthernSubpolarAtlantic'] = "Northern Subpolar Atlantic" + lnd['SPNA'] = "Subpolar North Atlantic" + lnd['SubpolarNorthAtlantic'] = lnd['SPNA'] lnd['NorthernSubpolarPacific'] = "Northern Subpolar Pacific" lnd['SouthernOcean'] = "Southern Ocean" diff --git a/key_files/temperature.yml b/key_files/temperature.yml index 7a9ee346..ddc83627 100644 --- a/key_files/temperature.yml +++ b/key_files/temperature.yml @@ -21,7 +21,7 @@ data_convert : NoChange data_tdict : ZeroToZero layers : Surface -regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean +regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean SPNA From 5e342b255335cc23c02b024c7e10a0d36b2b1f5e Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 25 Mar 2024 11:26:23 +0000 Subject: [PATCH 2/7] Turned off inefficient calculation for print only --- bgcval2/timeseries/timeseriesTools.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py index 38ff1ca2..280f6fc9 100644 --- a/bgcval2/timeseries/timeseriesTools.py +++ b/bgcval2/timeseries/timeseriesTools.py @@ -336,17 +336,17 @@ def run(self): print("DataLoader:\tLoaded", self.name, 'in', end=' ') print('{:<24} layer: {:<8}'.format(region, layer), end=' ') print('\tdata length:', - len(self.load[(region, layer)]), - end=' ') - print('\tmean:', - self.load[(region, layer)].mean(), - 'of', - len(self.load[(region, layer)]), - end=' ') - print('\trange:', [ - self.load[(region, layer)].min(), - self.load[(region, layer)].max() - ]) + len(self.load[(region, layer)]),) + #end=' ') + #print('\tmean:', + # self.load[(region, layer)].mean(), + # 'of', + # len(self.load[(region, layer)]), + # end=' ') + #print('\trange:', [ + # self.load[(region, layer)].min(), + # self.load[(region, layer)].max() + #]) def _makeTimeDict_(self, ): """ Make a dictionairy linking the time index with the float time. From 9d88b1ff8b00b3be32fde6dc7bd5715cd5a34249 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 25 Mar 2024 11:50:37 +0000 Subject: [PATCH 3/7] an attempt to speed up analysis of new regions --- bgcval2/timeseries/timeseriesAnalysis.py | 1 + bgcval2/timeseries/timeseriesTools.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/bgcval2/timeseries/timeseriesAnalysis.py b/bgcval2/timeseries/timeseriesAnalysis.py index c064cab8..33ff2bb4 100644 --- a/bgcval2/timeseries/timeseriesAnalysis.py +++ b/bgcval2/timeseries/timeseriesAnalysis.py @@ -272,6 +272,7 @@ def loadModel(self): self.modeldetails, regions=self.regions, layers=self.layers, + modeldataD=self.modeldataD, ) for l in self.layers: diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py index 280f6fc9..6dfbd439 100644 --- a/bgcval2/timeseries/timeseriesTools.py +++ b/bgcval2/timeseries/timeseriesTools.py @@ -267,6 +267,9 @@ def __init__(self, layers=[ 'Surface', ], + modeldataD = {}, + metrics = ['mean', ], + meantime = 0, data=''): self.fn = fn if type(nc) == type('filename'): @@ -276,6 +279,9 @@ def __init__(self, self.details = details self.regions = regions self.layers = layers + self.metrics = metrics + self.modeldataD = modeldataD + self.meantime = meantime self.name = self.details['name'] if data == '': data = std_extractData(nc, self.details) self.Fulldata = data @@ -294,7 +300,7 @@ def run(self): depths = {} lays = self.layers[:] lays.reverse() - + for l in lays: try: layer = int(l) @@ -317,6 +323,15 @@ def run(self): continue for region in self.regions: + loadthisregion = 0 + for m in metrics: + if self.modeldataD.get((r, l, m), False) and self.meantime in modeldataD[(r, l, m)]: + # Data arra already exists and This time point's data already exists + continue + # This region does not exist in the processed data, so add it. + loadthisregion +=1 + if loadthisregion == 0: + continue arr, arr_t, arr_z, arr_lat, arr_lon = self.createDataArray( region, layer) if len(arr) == 0: From 1efd64ac483d17b84dd5e37f9f9038532027d020 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 25 Mar 2024 11:54:05 +0000 Subject: [PATCH 4/7] Added bug fixes. --- bgcval2/timeseries/timeseriesAnalysis.py | 3 ++- bgcval2/timeseries/timeseriesTools.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bgcval2/timeseries/timeseriesAnalysis.py b/bgcval2/timeseries/timeseriesAnalysis.py index 33ff2bb4..9d098f4d 100644 --- a/bgcval2/timeseries/timeseriesAnalysis.py +++ b/bgcval2/timeseries/timeseriesAnalysis.py @@ -272,7 +272,8 @@ def loadModel(self): self.modeldetails, regions=self.regions, layers=self.layers, - modeldataD=self.modeldataD, + metrics= self.metrics, + modeldataD=modeldataD, ) for l in self.layers: diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py index 6dfbd439..15ec9569 100644 --- a/bgcval2/timeseries/timeseriesTools.py +++ b/bgcval2/timeseries/timeseriesTools.py @@ -324,8 +324,8 @@ def run(self): for region in self.regions: loadthisregion = 0 - for m in metrics: - if self.modeldataD.get((r, l, m), False) and self.meantime in modeldataD[(r, l, m)]: + for m in self.metrics: + if self.modeldataD.get((region, l, m), False) and self.meantime in self.modeldataD[(region, l, m)]: # Data arra already exists and This time point's data already exists continue # This region does not exist in the processed data, so add it. From a0e7a73658cb5a8a96a487dc117447f6b3abf370 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 25 Mar 2024 11:57:00 +0000 Subject: [PATCH 5/7] added prints --- bgcval2/timeseries/timeseriesAnalysis.py | 1 + bgcval2/timeseries/timeseriesTools.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/bgcval2/timeseries/timeseriesAnalysis.py b/bgcval2/timeseries/timeseriesAnalysis.py index 9d098f4d..2f20ab77 100644 --- a/bgcval2/timeseries/timeseriesAnalysis.py +++ b/bgcval2/timeseries/timeseriesAnalysis.py @@ -274,6 +274,7 @@ def loadModel(self): layers=self.layers, metrics= self.metrics, modeldataD=modeldataD, + meantime=meantime, ) for l in self.layers: diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py index 15ec9569..3dd61806 100644 --- a/bgcval2/timeseries/timeseriesTools.py +++ b/bgcval2/timeseries/timeseriesTools.py @@ -327,7 +327,10 @@ def run(self): for m in self.metrics: if self.modeldataD.get((region, l, m), False) and self.meantime in self.modeldataD[(region, l, m)]: # Data arra already exists and This time point's data already exists + print('Dataloader: No need to load this setting', (region, l, m)) continue + print('Dataloader: We need to re-load this setting', (region, l, m), self.meantime) + # This region does not exist in the processed data, so add it. loadthisregion +=1 if loadthisregion == 0: From 423cdede2f911afc4a8876cee1a121f324c007c5 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 25 Mar 2024 12:13:20 +0000 Subject: [PATCH 6/7] Added SPNA to lots of analyses --- key_files/alkalinity.yml | 2 +- key_files/atmospco2.yml | 2 +- key_files/chlorophyll.yml | 2 +- key_files/dic.yml | 2 +- key_files/dust.yml | 2 +- key_files/iron.yml | 2 +- key_files/mld.yml | 4 +--- key_files/mpa_mld.yml | 2 +- key_files/nitrate.yml | 2 +- key_files/oxygen.yml | 2 +- key_files/salinity.yml | 2 +- key_files/silicate.yml | 2 +- key_files/temperature.yml | 2 +- 13 files changed, 13 insertions(+), 15 deletions(-) diff --git a/key_files/alkalinity.yml b/key_files/alkalinity.yml index 252c7a03..d8f5e46b 100644 --- a/key_files/alkalinity.yml +++ b/key_files/alkalinity.yml @@ -19,5 +19,5 @@ model_convert : NoChange # function: convertmeqm3TOumolkg layers : Surface #50m #;100m 200m 500m 1000m 2000m -regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean +regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean SPNA diff --git a/key_files/atmospco2.yml b/key_files/atmospco2.yml index 66e71bfb..a341d12e 100644 --- a/key_files/atmospco2.yml +++ b/key_files/atmospco2.yml @@ -8,7 +8,7 @@ modelFiles : $BASEDIR_MODEL/$JOBID/medusa*$JOBIDo_1y_*_diad-T.nc model_vars : ATM_PCO2 model_convert : NoChange layers : layerless -regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean +regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean SPNA diff --git a/key_files/chlorophyll.yml b/key_files/chlorophyll.yml index 439e0858..6ed0aa17 100644 --- a/key_files/chlorophyll.yml +++ b/key_files/chlorophyll.yml @@ -14,7 +14,7 @@ model_vars : CHD CHN model_convert : sum layers : Surface -regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean +regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean SPNA diff --git a/key_files/dic.yml b/key_files/dic.yml index 8a6c5ec8..e6862e2d 100644 --- a/key_files/dic.yml +++ b/key_files/dic.yml @@ -15,5 +15,5 @@ model_convert : NoChange layers : Surface #50m #;100m 200m 500m 1000m 2000m -regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean +regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean SPNA diff --git a/key_files/dust.yml b/key_files/dust.yml index 17f72342..909eca9d 100644 --- a/key_files/dust.yml +++ b/key_files/dust.yml @@ -9,4 +9,4 @@ model_vars : AEOLIAN model_convert: function: NoChange layers : layerless -regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean +regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean SPNA diff --git a/key_files/iron.yml b/key_files/iron.yml index a240f3c0..e1948d07 100644 --- a/key_files/iron.yml +++ b/key_files/iron.yml @@ -25,7 +25,7 @@ model_convert : #data_convert_factor : 1000 layers : Surface #Transect ;CanRusTransect PTransect SOTransect Equator ArcTransect AntTransect ArcTransect AntTransect -regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean +regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean SPNA diff --git a/key_files/mld.yml b/key_files/mld.yml index 82398e3f..4c8c1eef 100644 --- a/key_files/mld.yml +++ b/key_files/mld.yml @@ -30,6 +30,4 @@ data_convert: maskname : mask areafile: $BASEDIR_OBS/IFREMER-MLD/mld_DT02_c1m_reg2.0-annual.nc #layers : Surface -#regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean -regions : Global OnShelf OffShelf ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthAtlanticOcean SouthAtlanticOcean NorthPacificOcean SouthPacificOcean #;Remainder NorthernSubpolarAtlantic NorthernSubpolarPacific - +regions : Global OnShelf OffShelf ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthAtlanticOcean SouthAtlanticOcean NorthPacificOcean SouthPacificOcean SPNA diff --git a/key_files/mpa_mld.yml b/key_files/mpa_mld.yml index 3d37268e..d6f341cb 100644 --- a/key_files/mpa_mld.yml +++ b/key_files/mpa_mld.yml @@ -30,6 +30,6 @@ data_convert: maskname : mask areafile: $BASEDIR_OBS/IFREMER-MLD/mld_DT02_c1m_reg2.0-annual.nc #layers : Surface -#regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean +#regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean SPNA regions : Global Ascension TristandaCunha Pitcairn Cornwall diff --git a/key_files/nitrate.yml b/key_files/nitrate.yml index 427e45f5..c7ba88da 100644 --- a/key_files/nitrate.yml +++ b/key_files/nitrate.yml @@ -21,7 +21,7 @@ data_convert : NoChange data_tdict : ZeroToZero layers : Surface 50m #;100m 200m 500m 1000m 2000m -regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean +regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean SPNA diff --git a/key_files/oxygen.yml b/key_files/oxygen.yml index 74017a0a..bf3b92ba 100644 --- a/key_files/oxygen.yml +++ b/key_files/oxygen.yml @@ -21,5 +21,5 @@ data_convert: factor : 44.661 layers : Surface 100m 500m -regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean +regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean SPNA diff --git a/key_files/salinity.yml b/key_files/salinity.yml index 2fb7b53f..6ca7f429 100644 --- a/key_files/salinity.yml +++ b/key_files/salinity.yml @@ -20,4 +20,4 @@ data_convert : NoChange data_tdict : ZeroToZero layers : Surface -regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean +regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean SPNA diff --git a/key_files/silicate.yml b/key_files/silicate.yml index 67e50b93..7f1c0dc0 100644 --- a/key_files/silicate.yml +++ b/key_files/silicate.yml @@ -19,7 +19,7 @@ data_convert : NoChange data_tdict : ZeroToZero layers : Surface -regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean +regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean SPNA diff --git a/key_files/temperature.yml b/key_files/temperature.yml index ddc83627..7d108b61 100644 --- a/key_files/temperature.yml +++ b/key_files/temperature.yml @@ -21,7 +21,7 @@ data_convert : NoChange data_tdict : ZeroToZero layers : Surface -regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean SPNA +regions : Global ignoreInlandSeas SouthernOcean ArcticOcean Equator10 NorthPacificOcean SouthPacificOcean NorthAtlanticOcean SouthAtlanticOcean SPNA SPNA From d19974dd6acf5bfc99020358ffb227c292afddfb Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Tue, 26 Mar 2024 11:55:24 +0000 Subject: [PATCH 7/7] Apply suggestions from code review --- bgcval2/timeseries/timeseriesTools.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/bgcval2/timeseries/timeseriesTools.py b/bgcval2/timeseries/timeseriesTools.py index 3dd61806..319511b2 100644 --- a/bgcval2/timeseries/timeseriesTools.py +++ b/bgcval2/timeseries/timeseriesTools.py @@ -355,16 +355,6 @@ def run(self): print('{:<24} layer: {:<8}'.format(region, layer), end=' ') print('\tdata length:', len(self.load[(region, layer)]),) - #end=' ') - #print('\tmean:', - # self.load[(region, layer)].mean(), - # 'of', - # len(self.load[(region, layer)]), - # end=' ') - #print('\trange:', [ - # self.load[(region, layer)].min(), - # self.load[(region, layer)].max() - #]) def _makeTimeDict_(self, ): """ Make a dictionairy linking the time index with the float time.