Merge pull request #126 from valeriupredoi/dev_fixing_shelve_opens

Fix shelve opens to "with"style opening

Author: ledm

bgcval2/bgcvaltools/linkTwoJobs.py

@@ -35,7 +35,7 @@
 import os
 import shutil
 from glob import glob
-from shelve import open as shopen
+from shelve import open as shOpen
 from datetime import datetime
 
 from bgcval2.bgcvaltools.bv2tools import folder
@@ -169,7 +169,7 @@ def linkTwoJobs(jobID1, jobID2, copyShelves=False):
 
             if os.path.exists(fn2) and fn2.find('insitu') == -1:
                 try:
-                    s = shopen(fn2)
+                    s = shOpen(fn2)
                     if 'readFiles' not in list(s.keys()):
                         s.close()
                         print("Shelve has no readFiles:", fn2)

bgcval2/bgcvaltools/makeMask.py

@@ -31,6 +31,7 @@
 from calendar import month_name
 from shelve import open as shOpen
 import os
+from glob import glob
 from bgcval2.Paths.paths import paths
 from netCDF4 import Dataset
 import bgcval2.bgcvaltools.bv2tools as bvt
@@ -483,11 +484,11 @@ def makeMask(name, newSlice, xt, xz, xy, xx, xd, debug=False):
 
         shelveFn = bvt.folder(os.path.join(paths.shelvedir, "MatchingMasks/"))+ newSlice+"_diag_maskMask.shelve"
 
-        try:
-            s = shOpen(shelveFn)
-            lldict = s['lldict']
-            s.close()
-        except:
+        if len(glob(shelveFn+'*')):
+            # Shelve already exists:
+            with shOpen(shelveFn) as sh:
+                lldict = sh['lldict']
+        else:
             lldict = {}
 
         print("Bathy mask: before mask:", newSlice, nmask.sum(), 'of',
@@ -517,9 +518,8 @@ def makeMask(name, newSlice, xt, xz, xy, xx, xd, debug=False):
                     nmask[i] = 1
         if i > 0:
             try:
-                s = shOpen(shelveFn)
-                s['lldict'] = lldict
-                s.close()
+                with shOpen(shelveFn) as sh:
+                    sh['lldict'] = lldict
             except:
                 print(
                     "makeMask:\tWARNING:\tUnable to save lldict at this time")

bgcval2/bgcvaltools/removeMaskFromshelve.py

@@ -31,27 +31,26 @@
 
 """
 
-from shelve import open as shopen
+from shelve import open as shOpen
 from glob import glob
 
 
 def removeFromShelves(fn, removeRegions):
     print('removing:', removeRegions, 'from', fn)
-    sh = shopen(fn)
-
-    modeldata = sh['modeldata']
-
-    for key in list(modeldata.keys()):
-        try:
-            (r, l, m) = key
-        except:
-            continue
-        if r in removeRegions:
-            print('modeldata[', (r, l, m), '] will be deleted')
-            del modeldata[(r, l, m)]
-
-    sh['modeldata'] = modeldata
-    sh.close()
+    with shOpen(fn) as sh:
+        modeldata = sh['modeldata']
+        for key in list(modeldata.keys()):
+            try:
+                (r, l, m) = key
+            except ValueError:
+                continue
+            if r in removeRegions:
+                print('modeldata[', (r, l, m), '] will be deleted')
+                if (r, l, m) in modeldata:
+                    del modeldata[(r, l, m)]
+
+        sh['modeldata'] = modeldata
+    # sh.close()
 
 
 removeRegions = [

bgcval2/custom_plots/PIControl_sst.py

@@ -3,7 +3,7 @@
 import os
 import numpy as np
 from matplotlib import pyplot
-from shelve import open as shopen
+from shelve import open as shOpen
 from mpl_toolkits.axes_grid.inset_locator import (inset_axes, InsetPosition,
                                                   mark_inset,
                                                   zoomed_inset_axes)
@@ -174,7 +174,7 @@ def get_data(j, field='AMOC'):
 
     if not os.path.exists(fn):
         print('Does not exist:', fn)
-    shelve = shopen(fn)
+    shelve = shOpen(fn)
     data = shelve['modeldata'][index]
     shelve.close()
     return data

bgcval2/custom_plots/ensemble_plots.py

@@ -10,7 +10,7 @@
 import os
 import numpy as np
 from matplotlib import pyplot
-from shelve import open as shopen
+from shelve import open as shOpen
 
 
 def movingaverage_DT(data, times, window_len=10., window_units='years'):
@@ -106,7 +106,7 @@ def getAMOCdata(j, field='AMOC'):
 
     if not os.path.exists(fn):
         print('Does not exist:', fn)
-    shelve = shopen(fn)
+    shelve = shOpen(fn)
     data = shelve['modeldata'][index]
     shelve.close()
     #	times = sorted(data.keys())

bgcval2/custom_plots/u-aw310_timeseries.py

@@ -5,7 +5,7 @@
 
 import numpy as np
 from matplotlib import pyplot
-from shelve import open as shopen
+from shelve import open as shOpen
 
 
 def movingaverage_DT(data, times, window_len=10., window_units='years'):
@@ -162,7 +162,7 @@ def getAMOCdata(j, field='AMOC'):
         fn = prefix + j + '_MLD.shelve'
         index = ('Wilkes', 'layerless', 'mean')
 
-    shelve = shopen(fn)
+    shelve = shOpen(fn)
     data = shelve['modeldata'][index]
     shelve.close()
     return data

bgcval2/timeseries/analysis_level0.py

@@ -29,7 +29,7 @@
 
 """
 
-from shelve import open as shopen
+from shelve import open as shOpen
 from sys import argv, exit
 import os
 import numpy as np
@@ -80,7 +80,7 @@ def analysis_level0(
         print("This shelve fn doesn't exists", shelvefn)
         return name, False, False
     try:
-        s = shopen(shelvefn)
+        s = shOpen(shelvefn)
         modeldata = s['modeldata']
         readFiles = s['readFiles']
         s.close()
@@ -150,13 +150,13 @@ def analysis_level0_insitu(
         print("This shelve fn doesn't exists", shelvefn)
         return False
     try:
-        s = shopen(shelvefn)
+        s = shOpen(shelvefn)
         dataD = s['dataD']
         s.close()
     except:
         print("This shelve fn doesn't work properly", shelvefn)
         try:
-            s = shopen(shelvefn)
+            s = shOpen(shelvefn)
             print("s.keys:", list(s.keys()))
             s.close()
         except:

bgcval2/timeseries/profileAnalysis.py

@@ -33,6 +33,7 @@
 from netCDF4 import num2date
 import os
 import shutil
+from glob import glob
 
 #Specific local code:
 from bgcval2.bgcvaltools import bv2tools as bvt
@@ -181,27 +182,19 @@ def loadModel(self):
         # load and calculate the model info
         self.setmlayers()
 
-        try:
-            if self.clean:
-                print("profileAnalysis:\tloadModel:\tUser requested clean run. Wiping old data.")
-                assert 0
-            sh = shOpen(self.shelvefn)
-            readFiles = sh['readFiles']
-            modeldataD = sh['modeldata']
-            sh.close()
-
-            print("OprofileAnalysis:\tloadModel:\tpened shelve:", self.shelvefn, '\tread', len(
-                readFiles))
-        except:
+        if self.clean or not len(glob(self.shelvefn+'*')):
+            #print("profileAnalysis:\tloadModel:\tUser requested clean run. Wiping old data.")
             readFiles = []
             modeldataD = {}
-            #		self.setmlayers()
             for r in self.regions:
                 for l in self.mlayers:
                     for m in self.metrics:
                         modeldataD[(r, l, m)] = {}
-
-            print("profileAnalysis:\tloadModel:\tCould not open shelve:", self.shelvefn, '\tread', len(
+        else:
+            with shOpen(self.shelvefn) as sh:
+                readFiles = sh['readFiles']
+                modeldataD = sh['modeldata']
+            print("ProfileAnalysis:\tloadModel:\tOpened shelve:", self.shelvefn, '\tread', len(
                 readFiles))
 
         ###############
@@ -300,18 +293,16 @@ def loadModel(self):
             if openedFiles:
                 print("Saving shelve:", self.dataType, self.shelvefn, '\tread', len(
                     readFiles))
-                sh = shOpen(self.shelvefn)
-                sh['readFiles'] = readFiles
-                sh['modeldata'] = modeldataD
-                sh.close()
+                with shOpen(self.shelvefn) as sh:
+                    sh['readFiles'] = readFiles
+                    sh['modeldata'] = modeldataD
                 openedFiles = 0
         if openedFiles:
             print("Saving shelve:", self.dataType, self.shelvefn, '\tread', len(
                 readFiles))
-            sh = shOpen(self.shelvefn)
-            sh['readFiles'] = readFiles
-            sh['modeldata'] = modeldataD
-            sh.close()
+            with shOpen(self.shelvefn) as sh:
+                sh['readFiles'] = readFiles
+                sh['modeldata'] = modeldataD
 
         self.modeldataD = modeldataD
         if self.debug:
@@ -371,18 +362,14 @@ def loadData(self):
 
         ###############
         # load and calculate the real data info
-        try:
-            if self.clean:
-                print("profileAnalysis:\t loadData\tUser requested clean run. Wiping old data.")
-                assert 0
-            sh = shOpen(self.shelvefn_insitu)
-            dataD = sh['dataD']
-            sh.close()
+        if self.clean or not len(glob(self.shelvefn_insitu+'*')):
+            print("profileAnalysis:\t loadData\tClean run.")
+            dataD = {}
+        else:
+            with shOpen(self.shelvefn_insitu) as sh:
+                dataD = sh['dataD']
             print("profileAnalysis:\t loadData\tOpened shelve:", self.shelvefn_insitu)
             self.dataD = dataD
-        except:
-            dataD = {}
-            print("profileAnalysis:\t loadData\tCould not open shelve:", self.shelvefn_insitu)
 
         ###############
         # Test to find out if we need to load the netcdf, or if we can just return the dict as a self.object.
@@ -488,38 +475,22 @@ def loadData(self):
                         r, l), 'mean:\t', meandatad)
 
             print("profileAnalysis:\t loadData.\tSaving shelve: (layer", l, ")", self.shelvefn_insitu)
-            sh = shOpen(self.shelvefn_insitu)
-            sh['dataD'] = dataD
-            sh.close()
-
+            with shOpen(self.shelvefn_insitu) as sh:
+                sh['dataD'] = dataD
         ###############
         # Savng shelve
         print("profileAnalysis:\t loadData.\tSaving shelve:", self.shelvefn_insitu)
-        try:
-            sh = shOpen(self.shelvefn_insitu)
-            sh['dataD'] = dataD
-            sh.close()
-
-            print("profileAnalysis:\t loadData.\tSaved shelve:", self.shelvefn_insitu)
-
-        except:
-            print("profileAnalysis:\t WARNING.\tSaving shelve failed, trying again.:", self.shelvefn_insitu)
-            print("Data is", list(dataD.keys()))
+        with shOpen(self.shelvefn_insitu) as sh:
+            sh['dataD'] = dataD        
 
-            for key in sorted(dataD.keys()):
 
-                print(key, ':\t', dataD[key])
-                sh = shOpen(bvt.folder('./tmpshelves') + 'tmshelve.shelve')
-                sh['dataD'] = dataD[key]
-                sh.close()
-                print("saved fine:\t./tmpshelves/tmshelve.shelve")
 
-            shutil.move(self.shelvefn_insitu, self.shelvefn_insitu + '.broken')
+        #     for key in sorted(dataD.keys()):
 
-            #		try:
-            sh = shOpen(self.shelvefn_insitu)
-            sh['dataD'] = dataD
-            sh.close()
+        #         print(key, ':\t', dataD[key])
+        #         with shOpen(bvt.folder('./tmpshelves') + 'tmshelve.shelve') as sh:
+        #             sh['dataD'] = dataD[key]
+        #         print("saved fine:\t./tmpshelves/tmshelve.shelve")
 
 
 #		except: