From 0d641759c67e58f3ff35db553dcfa099a27af1b0 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 23 Mar 2023 14:45:45 +0000 Subject: [PATCH 01/11] Added a tool to load area so it is the same everywhere --- bgcval2/functions/AirSeaFluxCO2.py | 3 ++- bgcval2/functions/BenCa.py | 3 ++- bgcval2/functions/InvtAlk.py | 3 ++- bgcval2/functions/TotalIntPP.py | 6 ++---- bgcval2/functions/dust.py | 3 ++- bgcval2/functions/globalVolMean.py | 9 ++++++--- bgcval2/functions/ice.py | 5 +++-- bgcval2/functions/tools.py | 17 +++++++++++++++++ 8 files changed, 36 insertions(+), 13 deletions(-) create mode 100644 bgcval2/functions/tools.py diff --git a/bgcval2/functions/AirSeaFluxCO2.py b/bgcval2/functions/AirSeaFluxCO2.py index 93116547..fe4acbef 100644 --- a/bgcval2/functions/AirSeaFluxCO2.py +++ b/bgcval2/functions/AirSeaFluxCO2.py @@ -32,6 +32,7 @@ import numpy as np from bgcval2.bgcvaltools.dataset import dataset from bgcval2.functions.get_kwarg_file import get_kwarg_file +from tools import load_area global loaded_area_and_mask global tmask @@ -46,7 +47,7 @@ def load_area_and_mask(areafile): if isinstance(areafile, list) and len(areafile)==1: areafile = areafile[0] nc = dataset(areafile, 'r') - area = nc.variables['e2t'][:] * nc.variables['e1t'][:] + area = load_area(nc) nc.close() loaded_area_and_mask = True return area diff --git a/bgcval2/functions/BenCa.py b/bgcval2/functions/BenCa.py index e62cba89..65c66007 100644 --- a/bgcval2/functions/BenCa.py +++ b/bgcval2/functions/BenCa.py @@ -32,6 +32,7 @@ import numpy as np from bgcval2.bgcvaltools.dataset import dataset from bgcval2.functions.get_kwarg_file import get_kwarg_file +from tools import load_area global loaded_area_and_mask global tmask @@ -48,7 +49,7 @@ def load_area_and_mask(areafile): else: raise ValueError(f"{areafile} must be a length=1 list containing a single file descriptor.") nc = dataset(areafile, 'r') - area = nc.variables['e2t'][:] * nc.variables['e1t'][:] + area = load_area(nc) nc.close() loaded_area_and_mask = True return area diff --git a/bgcval2/functions/InvtAlk.py b/bgcval2/functions/InvtAlk.py index 88b7d30e..356a44e3 100644 --- a/bgcval2/functions/InvtAlk.py +++ b/bgcval2/functions/InvtAlk.py @@ -32,6 +32,7 @@ import numpy as np from bgcval2.bgcvaltools.dataset import dataset from bgcval2.functions.get_kwarg_file import get_kwarg_file +from tools import load_area global loaded_area_and_mask global tmask @@ -48,7 +49,7 @@ def load_area_and_mask(areafile): else: raise ValueError(f"{areafile} must be a length=1 list containing a single file descriptor.") nc = dataset(areafile, 'r') - area = nc.variables['e2t'][:] * nc.variables['e1t'][:] + area = load_area(nc) nc.close() loaded_area_and_mask = True return area diff --git a/bgcval2/functions/TotalIntPP.py b/bgcval2/functions/TotalIntPP.py index 6226fc56..5b0e0bdf 100644 --- a/bgcval2/functions/TotalIntPP.py +++ b/bgcval2/functions/TotalIntPP.py @@ -31,6 +31,7 @@ from bgcval2.bgcvaltools.dataset import dataset from bgcval2.functions.get_kwarg_file import get_kwarg_file from bgcval2.functions.standard_functions import find_best_var +from tools import load_area global loadedArea global model_area @@ -41,10 +42,7 @@ def loadDataMask(gridfn): nc = dataset(gridfn,'r') - try: - model_area = nc.variables['area'][:] - except: - model_area = nc.variables['e1t'][:]*nc.variables['e2t'][:] + model_area = load_area(nc) nc.close() loadedArea = True return model_area diff --git a/bgcval2/functions/dust.py b/bgcval2/functions/dust.py index 702f71f9..02aa64db 100644 --- a/bgcval2/functions/dust.py +++ b/bgcval2/functions/dust.py @@ -42,7 +42,8 @@ def loadDataMask(gridfn): global loadedArea global masked_area nc = dataset(gridfn,'r') - masked_area = nc.variables['e2t'][:] * nc.variables['e1t'][:]*nc.variables['tmask'][0] + area = load_area(nc) + masked_area = area *nc.variables['tmask'][0] nc.close() loadedArea = True diff --git a/bgcval2/functions/globalVolMean.py b/bgcval2/functions/globalVolMean.py index 5d7f9ebb..86812f36 100644 --- a/bgcval2/functions/globalVolMean.py +++ b/bgcval2/functions/globalVolMean.py @@ -31,15 +31,18 @@ import numpy as np from bgcval2.functions.standard_functions import choose_best_var - +from tools import load_area def calc_vol(nc): """ Calculate volume from the (grid-T) netcdf file. """ - area = nc.variables['area'][:] + area = load_area(area) thkcello = nc.variables['thkcello'][:] - area = area[None, None,:, :] + if area.ndim == 4: + area = area[None, None,:, :] + if area.ndim == 3: + area = area[None, :, :] return thkcello * area diff --git a/bgcval2/functions/ice.py b/bgcval2/functions/ice.py index e719d22b..c74b0735 100644 --- a/bgcval2/functions/ice.py +++ b/bgcval2/functions/ice.py @@ -32,6 +32,7 @@ import numpy as np from bgcval2.bgcvaltools.dataset import dataset from bgcval2.functions.get_kwarg_file import get_kwarg_file +from tools import load_area global loaded_area_and_mask global tmask @@ -50,7 +51,7 @@ def load_area_and_mask(gridfn, maskname,): gridfn = gridfn[0] nc = dataset(gridfn, 'r') tmask = nc.variables[maskname][0] - area = nc.variables['e2t'][:] * nc.variables['e1t'][:] + area = load_area(nc) lat = nc.variables['nav_lat'][:] nc.close() loaded_area_and_mask = True @@ -74,7 +75,7 @@ def calculate_ice_extent(nc, keys, **kwargs): if 'area' not in nc.variables and not loaded_area_and_mask: area, tmask, lat = load_area_and_mask(areafile, maskname) else: - area = nc.variables['area'][:] + area = load_area(nc) lat = nc.variables['nav_lat'][:] tmask = nc.variables[keys[0]][:].squeeze().mask diff --git a/bgcval2/functions/tools.py b/bgcval2/functions/tools.py new file mode 100644 index 00000000..b7173e82 --- /dev/null +++ b/bgcval2/functions/tools.py @@ -0,0 +1,17 @@ + + +import numpy as np +from bgcval2.functions.standard_functions import choose_best_var + + +def load_area(nc): + """ + Generic tool for loading area: + """ + area_keys = ['area', 'area_grid_T', 'area_grid_W', 'area_grid_V', 'area_grid_U'] + if set(area_keys).intersection(set(nc.variables.keys()): + return choose_best_var(nc, area_keys) + else: + area = nc.variables['e1t'][:]*nc.variables['e2t'][:] + + From 05c2d6017236c5fc0806dc4d2c1440e918873f0f Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 23 Mar 2023 15:57:29 +0000 Subject: [PATCH 02/11] Working version --- bgcval2/download_from_mass.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bgcval2/download_from_mass.py b/bgcval2/download_from_mass.py index 461e24c8..cdd8488b 100755 --- a/bgcval2/download_from_mass.py +++ b/bgcval2/download_from_mass.py @@ -409,7 +409,9 @@ def download_from_mass( outputFold = folder([paths.ModelFolder_pref, jobID,] ) # make this folder group writeable. st = os.stat(outputFold) - os.chmod(outputFold, st.st_mode | stat.S_IWGRP) + + try: os.chmod(outputFold, st.st_mode | stat.S_IWGRP) + except: pass deleteBadLinksAndZeroSize(outputFold, jobID) @@ -417,7 +419,8 @@ def download_from_mass( deleteBadLinksAndZeroSize(outputFold, jobID) # Set up a file to save command to a new file. - download_script_path = ''.join([folder('mass_scripts/'), jobID,'.sh']) + username = os.getlogin() + download_script_path = ''.join([folder('mass_scripts/'), jobID,'_',username, '.sh']) header_lines = ['# Run this script on mass-cli1.jasmin.ac.uk\n',] header_lines.append('# from login1.jasmin.ac.uk, ssh to the mass machine:\n# ssh -X mass-cli\n') header_lines.append(''.join(['# run script with:\n# source ', os.path.abspath(download_script_path),'\n'])) @@ -483,7 +486,8 @@ def download_from_mass( if auto_download: shared_file_path = os.path.join(paths.shared_mass_scripts, os.path.basename(download_script_path)) print('writing file in shared path', shared_file_path) - shutil.copy(download_script_path, shared_file_path) + try: shutil.copy(download_script_path, shared_file_path) + except: pass fixFilePaths(outputFold, jobID, debug=False,) deleteBadLinksAndZeroSize(outputFold, jobID, debug=False,) From fce11db726a37c73de5b0c29877e41e3e21baff5 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 23 Mar 2023 16:06:26 +0000 Subject: [PATCH 03/11] bugfix here. --- bgcval2/functions/AirSeaFluxCO2.py | 2 +- bgcval2/functions/BenCa.py | 2 +- bgcval2/functions/InvtAlk.py | 2 +- bgcval2/functions/TotalIntPP.py | 2 +- bgcval2/functions/globalVolMean.py | 2 +- bgcval2/functions/ice.py | 2 +- bgcval2/functions/tools.py | 40 +++++++++++++++++++++++++++--- 7 files changed, 42 insertions(+), 10 deletions(-) diff --git a/bgcval2/functions/AirSeaFluxCO2.py b/bgcval2/functions/AirSeaFluxCO2.py index fe4acbef..b320435d 100644 --- a/bgcval2/functions/AirSeaFluxCO2.py +++ b/bgcval2/functions/AirSeaFluxCO2.py @@ -32,7 +32,7 @@ import numpy as np from bgcval2.bgcvaltools.dataset import dataset from bgcval2.functions.get_kwarg_file import get_kwarg_file -from tools import load_area +from bgcval2.functions.tools import load_area global loaded_area_and_mask global tmask diff --git a/bgcval2/functions/BenCa.py b/bgcval2/functions/BenCa.py index 65c66007..40ea0d18 100644 --- a/bgcval2/functions/BenCa.py +++ b/bgcval2/functions/BenCa.py @@ -32,7 +32,7 @@ import numpy as np from bgcval2.bgcvaltools.dataset import dataset from bgcval2.functions.get_kwarg_file import get_kwarg_file -from tools import load_area +from bgcval2.functions.tools import load_area global loaded_area_and_mask global tmask diff --git a/bgcval2/functions/InvtAlk.py b/bgcval2/functions/InvtAlk.py index 356a44e3..dc8d05f3 100644 --- a/bgcval2/functions/InvtAlk.py +++ b/bgcval2/functions/InvtAlk.py @@ -32,7 +32,7 @@ import numpy as np from bgcval2.bgcvaltools.dataset import dataset from bgcval2.functions.get_kwarg_file import get_kwarg_file -from tools import load_area +from bgcval2.functions.tools import load_area global loaded_area_and_mask global tmask diff --git a/bgcval2/functions/TotalIntPP.py b/bgcval2/functions/TotalIntPP.py index 5b0e0bdf..294ee877 100644 --- a/bgcval2/functions/TotalIntPP.py +++ b/bgcval2/functions/TotalIntPP.py @@ -31,7 +31,7 @@ from bgcval2.bgcvaltools.dataset import dataset from bgcval2.functions.get_kwarg_file import get_kwarg_file from bgcval2.functions.standard_functions import find_best_var -from tools import load_area +from bgcval2.functions.tools import load_area global loadedArea global model_area diff --git a/bgcval2/functions/globalVolMean.py b/bgcval2/functions/globalVolMean.py index 86812f36..5130e5f5 100644 --- a/bgcval2/functions/globalVolMean.py +++ b/bgcval2/functions/globalVolMean.py @@ -31,7 +31,7 @@ import numpy as np from bgcval2.functions.standard_functions import choose_best_var -from tools import load_area +from bgcval2.functions.tools import load_area def calc_vol(nc): """ diff --git a/bgcval2/functions/ice.py b/bgcval2/functions/ice.py index c74b0735..0b830584 100644 --- a/bgcval2/functions/ice.py +++ b/bgcval2/functions/ice.py @@ -32,7 +32,7 @@ import numpy as np from bgcval2.bgcvaltools.dataset import dataset from bgcval2.functions.get_kwarg_file import get_kwarg_file -from tools import load_area +from bgcval2.functions.tools import load_area global loaded_area_and_mask global tmask diff --git a/bgcval2/functions/tools.py b/bgcval2/functions/tools.py index b7173e82..c92d69b2 100644 --- a/bgcval2/functions/tools.py +++ b/bgcval2/functions/tools.py @@ -1,3 +1,33 @@ +# Copyright 2023, Plymouth Marine Laboratory +# +# This file is part of the bgc-val library. +# +# bgc-val is free software: you can redistribute it and/or modify it +# under the terms of the Revised Berkeley Software Distribution (BSD) 3-clause license. + +# bgc-val is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the revised BSD license for more details. +# You should have received a copy of the revised BSD license along with bgc-val. +# If not, see . +# +# Address: +# Plymouth Marine Laboratory +# Prospect Place, The Hoe +# Plymouth, PL1 3DH, UK +# +# Email: +# ledm@pml.ac.uk +# +""" +.. module:: tools + :platform: Unix + :synopsis: This uses some tools for the functions library. + +.. moduleauthor:: Lee de Mora + +""" + import numpy as np @@ -9,9 +39,11 @@ def load_area(nc): Generic tool for loading area: """ area_keys = ['area', 'area_grid_T', 'area_grid_W', 'area_grid_V', 'area_grid_U'] - if set(area_keys).intersection(set(nc.variables.keys()): - return choose_best_var(nc, area_keys) - else: + if set(area_keys).intersection(set(nc.variables.keys())): + area = choose_best_var(nc, area_keys) + elif set(['e1t', 'e2t']).intersection(set(nc.variables.keys())): area = nc.variables['e1t'][:]*nc.variables['e2t'][:] - + else: + raise ValueError('Unable to load or calculate area from this file.') + return area From 544e45f6fcac17fff8ca3126c1dfdf36cef0b378 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Thu, 23 Mar 2023 16:48:37 +0000 Subject: [PATCH 04/11] fix how code gets user name for portability --- bgcval2/download_from_mass.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bgcval2/download_from_mass.py b/bgcval2/download_from_mass.py index cdd8488b..e79804ab 100755 --- a/bgcval2/download_from_mass.py +++ b/bgcval2/download_from_mass.py @@ -419,7 +419,11 @@ def download_from_mass( deleteBadLinksAndZeroSize(outputFold, jobID) # Set up a file to save command to a new file. - username = os.getlogin() + try: + username = os.getlogin() + except OSError: + import getpass + username = getpass.getuser() download_script_path = ''.join([folder('mass_scripts/'), jobID,'_',username, '.sh']) header_lines = ['# Run this script on mass-cli1.jasmin.ac.uk\n',] header_lines.append('# from login1.jasmin.ac.uk, ssh to the mass machine:\n# ssh -X mass-cli\n') From 5d42c9474e7da7823cafe88a6f0246bd854894a2 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 23 Mar 2023 17:13:22 +0000 Subject: [PATCH 05/11] bugfix --- bgcval2/functions/globalVolMean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/functions/globalVolMean.py b/bgcval2/functions/globalVolMean.py index 5130e5f5..e9b2bb43 100644 --- a/bgcval2/functions/globalVolMean.py +++ b/bgcval2/functions/globalVolMean.py @@ -37,7 +37,7 @@ def calc_vol(nc): """ Calculate volume from the (grid-T) netcdf file. """ - area = load_area(area) + area = load_area(nc) thkcello = nc.variables['thkcello'][:] if area.ndim == 4: area = area[None, None,:, :] From 4e73a34af1771fdf0b4f47cbfdf1330675036722 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 23 Mar 2023 17:20:33 +0000 Subject: [PATCH 06/11] Update bgcval2/download_from_mass.py Co-authored-by: Valeriu Predoi --- bgcval2/download_from_mass.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bgcval2/download_from_mass.py b/bgcval2/download_from_mass.py index e79804ab..b5d41a94 100755 --- a/bgcval2/download_from_mass.py +++ b/bgcval2/download_from_mass.py @@ -411,7 +411,8 @@ def download_from_mass( st = os.stat(outputFold) try: os.chmod(outputFold, st.st_mode | stat.S_IWGRP) - except: pass + except OSError: + pass deleteBadLinksAndZeroSize(outputFold, jobID) From 7a80b9a02f7e929b9b64ffa1e6135b791d4b5d70 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 23 Mar 2023 17:20:51 +0000 Subject: [PATCH 07/11] Update bgcval2/functions/tools.py Co-authored-by: Valeriu Predoi --- bgcval2/functions/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/functions/tools.py b/bgcval2/functions/tools.py index c92d69b2..142081b7 100644 --- a/bgcval2/functions/tools.py +++ b/bgcval2/functions/tools.py @@ -44,6 +44,6 @@ def load_area(nc): elif set(['e1t', 'e2t']).intersection(set(nc.variables.keys())): area = nc.variables['e1t'][:]*nc.variables['e2t'][:] else: - raise ValueError('Unable to load or calculate area from this file.') + raise ValueError(f"Provided {nc} file doesn't contain any of permitted area variables {area_keys}") return area From b98cb269f4afac96a8359c7ce2a13ed8d6458586 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 23 Mar 2023 17:21:01 +0000 Subject: [PATCH 08/11] Update bgcval2/download_from_mass.py --- bgcval2/download_from_mass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgcval2/download_from_mass.py b/bgcval2/download_from_mass.py index b5d41a94..e91881e3 100755 --- a/bgcval2/download_from_mass.py +++ b/bgcval2/download_from_mass.py @@ -425,7 +425,7 @@ def download_from_mass( except OSError: import getpass username = getpass.getuser() - download_script_path = ''.join([folder('mass_scripts/'), jobID,'_',username, '.sh']) + download_script_path = ''.join([folder('mass_scripts/'), jobID, '_', username, '.sh']) header_lines = ['# Run this script on mass-cli1.jasmin.ac.uk\n',] header_lines.append('# from login1.jasmin.ac.uk, ssh to the mass machine:\n# ssh -X mass-cli\n') header_lines.append(''.join(['# run script with:\n# source ', os.path.abspath(download_script_path),'\n'])) From eaf113465de555a9abb0bf3431b26d500871ff83 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Thu, 23 Mar 2023 17:21:12 +0000 Subject: [PATCH 09/11] Update bgcval2/download_from_mass.py Co-authored-by: Valeriu Predoi --- bgcval2/download_from_mass.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bgcval2/download_from_mass.py b/bgcval2/download_from_mass.py index e91881e3..32819f4e 100755 --- a/bgcval2/download_from_mass.py +++ b/bgcval2/download_from_mass.py @@ -410,7 +410,8 @@ def download_from_mass( # make this folder group writeable. st = os.stat(outputFold) - try: os.chmod(outputFold, st.st_mode | stat.S_IWGRP) + try: + os.chmod(outputFold, st.st_mode | stat.S_IWGRP) except OSError: pass From faefa20e691e2c77626e48b298bf7a6a0c7c46c2 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 3 Apr 2023 16:15:37 +0100 Subject: [PATCH 10/11] Update bgcval2/functions/tools.py --- bgcval2/functions/tools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bgcval2/functions/tools.py b/bgcval2/functions/tools.py index 142081b7..74937784 100644 --- a/bgcval2/functions/tools.py +++ b/bgcval2/functions/tools.py @@ -29,7 +29,6 @@ """ - import numpy as np from bgcval2.functions.standard_functions import choose_best_var From 8f9805af7e162a7e82194bf15b63bae5123487f3 Mon Sep 17 00:00:00 2001 From: Lee de Mora Date: Mon, 3 Apr 2023 16:15:53 +0100 Subject: [PATCH 11/11] Added error handling and output. --- bgcval2/download_from_mass.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bgcval2/download_from_mass.py b/bgcval2/download_from_mass.py index 32819f4e..17b9f8fe 100755 --- a/bgcval2/download_from_mass.py +++ b/bgcval2/download_from_mass.py @@ -413,7 +413,7 @@ def download_from_mass( try: os.chmod(outputFold, st.st_mode | stat.S_IWGRP) except OSError: - pass + print('Unable to change permissions:', outputFold) deleteBadLinksAndZeroSize(outputFold, jobID) @@ -492,8 +492,10 @@ def download_from_mass( if auto_download: shared_file_path = os.path.join(paths.shared_mass_scripts, os.path.basename(download_script_path)) print('writing file in shared path', shared_file_path) - try: shutil.copy(download_script_path, shared_file_path) - except: pass + try: + shutil.copy(download_script_path, shared_file_path) + except EnvironmentError: + print('Error copying file', download_script_path, 'to', shared_file_path) fixFilePaths(outputFold, jobID, debug=False,) deleteBadLinksAndZeroSize(outputFold, jobID, debug=False,)