diff --git a/Coupled_Drivers/cice_driver.py b/Coupled_Drivers/cice_driver.py index bc0f1a1..304b858 100644 --- a/Coupled_Drivers/cice_driver.py +++ b/Coupled_Drivers/cice_driver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2023-2025 Met Office. All rights reserved. + (C) Crown copyright 2023-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -11,6 +11,10 @@ Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom *****************************COPYRIGHT****************************** + +# Some of the content of this file has been produced with the assistance of +# Claude Sonnet 4.5. + NAME cice_driver.py @@ -30,6 +34,7 @@ import time2days import inc_days import common +import shellout import error import dr_env_lib.cice_def import dr_env_lib.env_lib @@ -184,24 +189,21 @@ def _setup_executable(common_env): #any variables containing things that can be globbed will start with gl_ gl_step_int_match = '^dt=' - _, step_int_val = common.exec_subproc(['grep', gl_step_int_match, - cice_nl]) + _, step_int_val = shellout._exec_subprocess('grep %s %s' % (gl_step_int_match, cice_nl)) cice_step_int = int(re.findall(r'^dt=(\d*)\.?', step_int_val)[0]) cice_steps = (tot_runlen_sec - last_dump_seconds) // cice_step_int - _, cice_histfreq_val = common.exec_subproc(['grep', 'histfreq', cice_nl]) + _, cice_histfreq_val = shellout._exec_subprocess('grep histfreq %s' % cice_nl) cice_histfreq_val = re.findall(r'histfreq\s*=\s*(.*)', cice_histfreq_val)[0] cice_histfreq = __expand_array(cice_histfreq_val)[1] - _, cice_histfreq_n_val = common.exec_subproc([ \ - 'grep', 'histfreq_n', cice_nl]) + _, cice_histfreq_n_val = shellout._exec_subprocess('grep histfreq_n %s' % cice_nl) cice_histfreq_n_val = re.findall(r'histfreq_n\s*=\s*(.*)', cice_histfreq_n_val)[0] cice_histfreq_n = __expand_array(cice_histfreq_n_val) cice_histfreq_n = int(cice_histfreq_n.split(',')[0]) - _, cice_age_rest_val = common.exec_subproc([ \ - 'grep', '^restart_age', cice_nl]) + _, cice_age_rest_val = shellout._exec_subprocess('grep ^restart_age %s' % cice_nl) cice_age_rest = re.findall(r'restart_age\s*=\s*(.*)', cice_age_rest_val)[0] @@ -216,7 +218,7 @@ def _setup_executable(common_env): cice_envar['SHARED_FNAME']) sys.exit(error.MISSING_DRIVER_FILE_ERROR) if not common_env['MODELBASIS']: - _, modelbasis_val = common.exec_subproc('grep', 'model_basis_time', + _, modelbasis_val = shellout._exec_subprocess('grep model_basis_time %s' % cice_envar['SHARED_FNAME']) modelbasis_val = re.findall(r'model_basis_time\s*=\s*(.*)', modelbasis_val) @@ -225,7 +227,7 @@ def _setup_executable(common_env): if not common_env['TASKSTART']: common_env.add('TASKSTART', common_env['MODELBASIS']) if not common_env['TASKLENGTH']: - _, tasklength_val = common.exec_subproc('grep', 'run_target_end', + _, tasklength_val = shellout._exec_subprocess('grep run_target_end %s' % cice_envar['SHARED_FNAME']) tasklength_val = re.findall(r'run_target_end\s*=\s*(.*)', tasklength_val) @@ -241,20 +243,18 @@ def _setup_executable(common_env): // cice_step_int else: # This is probably a coupled NWP suite - cmd = ['rose', 'date', str(run_start[0])+'0101T0000Z', - cice_envar['TASK_START_TIME']] - _, time_since_year_start = common.exec_subproc(cmd) + cmd = 'rose date %s0101T0000Z %s' % (str(run_start[0]), cice_envar['TASK_START_TIME']) + _, time_since_year_start = shellout._exec_subprocess(cmd) #The next command works because rose date assumes # 19700101T0000Z is second 0 - cmd = ['rose', 'date', '--print-format=%s', '19700101T00Z', - '--offset='+time_since_year_start] + cmd = 'rose date --print-format=%%s 19700101T00Z --offset=%s' % time_since_year_start # Account for restarting from a failure in next line - # common.exec_subproc returns a tuple containing (return_code, output) - seconds_since_year_start = int(common.exec_subproc(cmd)[1]) \ + # shellout._exec_subprocess returns a tuple containing (return_code, output) + seconds_since_year_start = int(shellout._exec_subprocess(cmd)[1]) \ + last_dump_seconds cice_istep0 = seconds_since_year_start/cice_step_int - _, cice_rst_val = common.exec_subproc(['grep', 'restart_dir', cice_nl]) + _, cice_rst_val = shellout._exec_subprocess('grep restart_dir %s' % cice_nl) cice_rst = re.findall(r'restart_dir\s*=\s*\'(.*)\',', cice_rst_val)[0] if cice_rst[-1] == '/': cice_rst = cice_rst[:-1] @@ -266,9 +266,9 @@ def _setup_executable(common_env): cice_restart = os.path.join(cice_rst, cice_envar['CICE_RESTART']) - _, cice_hist_val = common.exec_subproc(['grep', 'history_dir', cice_nl]) + _, cice_hist_val = shellout._exec_subprocess('grep history_dir %s' % cice_nl) cice_hist = re.findall(r'history_dir\s*=\s*\'(.*)\',', cice_hist_val)[0] - _, cice_incond_val = common.exec_subproc(['grep', 'incond_dir', cice_nl]) + _, cice_incond_val = shellout._exec_subprocess('grep incond_dir %s' % cice_nl) cice_incond = re.findall(r'incond_dir\s*=\s*\'(.*)\',', cice_incond_val)[0] for direc in (cice_rst, cice_hist, cice_incond): @@ -312,8 +312,7 @@ def _setup_executable(common_env): if cice_age_rest == 'true': cice_runtype = 'continue' ice_ic = 'set in pointer file' - _, _ = common.exec_subproc([cice_envar['CICE_START'], - '>', cice_restart]) + _, _ = shellout._exec_subprocess('%s > %s' % (cice_envar['CICE_START'], cice_restart)) sys.stdout.write('[INFO] %s > %s' % (cice_envar['CICE_START'], cice_restart)) diff --git a/Coupled_Drivers/common.py b/Coupled_Drivers/common.py index d3470cf..fdcb0be 100644 --- a/Coupled_Drivers/common.py +++ b/Coupled_Drivers/common.py @@ -228,73 +228,6 @@ def setup_runtime(common_env): return runlen_sec -def exec_subproc_timeout(cmd, timeout_sec=10): - ''' - Execute a given shell command with a timeout. Takes a list containing - the commands to be run, and an integer timeout_sec for how long to - wait for the command to run. Returns the return code from the process - and the standard out from the command or 'None' if the command times out. - ''' - process = subprocess.Popen(cmd, shell=False, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - timer = threading.Timer(timeout_sec, process.kill) - try: - timer.start() - stdout, err = process.communicate() - if err: - sys.stderr.write('[SUBPROCESS ERROR] %s\n' % error) - rcode = process.returncode - finally: - timer.cancel() - if sys.version_info[0] >= 3: - output = stdout.decode() - else: - output = stdout - return rcode, output - - -def exec_subproc(cmd, verbose=True): - ''' - Execute given shell command. Takes a list containing the commands to be - run, and a logical verbose which if set to true will write the output of - the command to stdout. - ''' - process = subprocess.Popen(cmd, shell=False, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output, err = process.communicate() - if verbose and output: - sys.stdout.write('[SUBPROCESS OUTPUT] %s\n' % output) - if err: - sys.stderr.write('[SUBPROCESS ERROR] %s\n' % error) - if sys.version_info[0] >= 3: - output = output.decode() - return process.returncode, output - - -def __exec_subproc_true_shell(cmd, verbose=True): - ''' - Execute given shell command, with shell=True. Only use this function if - exec_subproc does not work correctly. Takes a list containing the commands - to be run, and a logical verbose which if set to true will write the - output of the command to stdout. - ''' - process = subprocess.Popen(cmd, shell=True, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output, err = process.communicate() - if verbose and output: - sys.stdout.write('[SUBPROCESS OUTPUT] %s\n' % output) - if err: - sys.stderr.write('[SUBPROCESS ERROR] %s\n' % error) - if sys.version_info[0] >= 3: - output = output.decode() - return process.returncode, output - def _calculate_ppn_values(nproc, nodes): ''' diff --git a/Coupled_Drivers/cpmip_utils.py b/Coupled_Drivers/cpmip_utils.py index f6bfcf2..ca0ab67 100644 --- a/Coupled_Drivers/cpmip_utils.py +++ b/Coupled_Drivers/cpmip_utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2023-2025 Met Office. All rights reserved. + (C) Crown copyright 2023-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -11,6 +11,10 @@ Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom *****************************COPYRIGHT****************************** + +# Some of the content of this file has been produced with the assistance of +# Claude Sonnet 4.5. + NAME cpmip_utils.py @@ -23,6 +27,7 @@ import sys import error import common +import shellout def get_component_resolution(nlist_file, resolution_variables): ''' @@ -32,7 +37,7 @@ def get_component_resolution(nlist_file, resolution_variables): ''' resolution = 1 for res_var in resolution_variables: - _, out = common.exec_subproc(['grep', res_var, nlist_file], + _, out = shellout._exec_subprocess('grep %s %s' % (res_var, nlist_file), verbose=True) try: i_res = int(re.search(r'(\d+)', out).group(0)) @@ -56,7 +61,7 @@ def get_glob_usage(glob_path, timeout=60): filelist = glob.glob(glob_path) if filelist: du_command = ['du', '-c'] + filelist - rcode, output = common.exec_subproc_timeout(du_command, timeout) + rcode, output = shellout._exec_subprocess(du_command, timeout) if rcode == 0: size_k = float(output.split()[-2]) else: @@ -131,7 +136,7 @@ def get_workdir_netcdf_output(timeout=60): i_f.split('.')[-1] == 'nc' and not os.path.islink(i_f)] size_k = -1.0 du_command = ['du', '-c'] + output_files - rcode, output = common.exec_subproc_timeout(du_command, timeout) + rcode, output = shellout._exec_subprocess(du_command, timeout) if rcode == 0: size_k = float(output.split()[-2]) return size_k diff --git a/Coupled_Drivers/cpmip_xios.py b/Coupled_Drivers/cpmip_xios.py index d9dc65a..0c84e9f 100644 --- a/Coupled_Drivers/cpmip_xios.py +++ b/Coupled_Drivers/cpmip_xios.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -11,6 +11,10 @@ Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom *****************************COPYRIGHT****************************** + +# Some of the content of this file has been produced with the assistance of +# Claude Sonnet 4.5. + NAME cpmip_xios.py @@ -21,6 +25,7 @@ import shutil import sys import common +import shellout def data_metrics_setup_nemo(): ''' @@ -58,8 +63,8 @@ def measure_xios_client_times(timeout=120): 'xios_client' in i_f and 'out' in i_f] total_files = len(files) for i_f in files: - rcode, out = common.exec_subproc_timeout( - ['grep', 'total time', i_f], timeout) + rcode, out = shellout._exec_subprocess( + 'grep "total time" %s' % i_f, timeout) if rcode == 0: meas_time = float(out.split()[-2]) total_measured += 1 diff --git a/Coupled_Drivers/mct_driver.py b/Coupled_Drivers/mct_driver.py index 0b02996..d6d4731 100644 --- a/Coupled_Drivers/mct_driver.py +++ b/Coupled_Drivers/mct_driver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy of the code, the use, duplication or disclosure of it is strictly @@ -10,6 +10,10 @@ Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom *****************************COPYRIGHT****************************** + +# Some of the content of this file has been produced with the assistance of +# Claude Sonnet 4.5. + NAME mct_driver.py @@ -23,6 +27,7 @@ import sys import glob import common +import shellout import error import update_namcouple import dr_env_lib.mct_def @@ -238,7 +243,7 @@ def _setup_executable(common_env, envarinsts, run_info): # Create transient field namelist (note if we're creating a # namcouple on the fly, this will have to wait until after # the namcouple have been created). - _, _ = common.exec_subproc('./OASIS_fields') + _, _ = shellout._exec_subprocess('./OASIS_fields') for component in mct_envar['COUPLING_COMPONENTS'].split(): if not component in common_env['models']: diff --git a/Coupled_Drivers/nemo_driver.py b/Coupled_Drivers/nemo_driver.py index e816772..274f39a 100644 --- a/Coupled_Drivers/nemo_driver.py +++ b/Coupled_Drivers/nemo_driver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2023-2025 Met Office. All rights reserved. + (C) Crown copyright 2023-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -11,6 +11,10 @@ Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom *****************************COPYRIGHT****************************** + +# Some of the content of this file has been produced with the assistance of +# Claude Sonnet 4.5. + NAME nemo_driver.py @@ -27,6 +31,7 @@ import shutil import inc_days import common +import shellout import error try: @@ -78,8 +83,8 @@ def _get_nemorst(nemo_nl_file): ''' Retrieve the nemo restart directory from the nemo namelist file ''' - ocerst_rcode, ocerst_val = common.exec_subproc([ \ - 'grep', 'cn_ocerst_outdir', nemo_nl_file]) + ocerst_rcode, ocerst_val = shellout._exec_subprocess( + 'grep cn_ocerst_outdir %s' % nemo_nl_file) if ocerst_rcode == 0: nemo_rst = re.findall(r'[\"\'](.*?)[\"\']', ocerst_val)[0] if nemo_rst[-1] == '/': @@ -92,8 +97,8 @@ def _get_ln_icebergs(nemo_nl_file): Interrogate the nemo namelist to see if we are running with icebergs, Returns boolean, True if icebergs are used, False if not ''' - icb_rcode, icb_val = common.exec_subproc([ \ - 'grep', 'ln_icebergs', nemo_nl_file]) + icb_rcode, icb_val = shellout._exec_subprocess( + 'grep ln_icebergs %s' % nemo_nl_file) if icb_rcode != 0: sys.stderr.write('Unable to read ln_icebergs in &namberg namelist' ' in the NEMO namelist file %s\n' @@ -303,8 +308,8 @@ def _setup_executable(common_env): nemo_rst = _get_nemorst(nemo_envar['NEMO_NL']) if nemo_rst: restart_direcs.append(nemo_rst) - icerst_rcode, icerst_val = common.exec_subproc([ \ - 'grep', 'cn_icerst_dir', nemo_envar['NEMO_NL']]) + icerst_rcode, icerst_val = shellout._exec_subprocess( + 'grep cn_icerst_dir %s' % nemo_envar['NEMO_NL']) if icerst_rcode == 0: ice_rst = re.findall(r'[\"\'](.*?)[\"\']', icerst_val)[0] if ice_rst[-1] == '/': @@ -440,14 +445,14 @@ def _setup_executable(common_env): sys.exit(error.MISSING_MODEL_FILE_ERROR) # First timestep of the previous cycle - _, first_step_val = common.exec_subproc(['grep', gl_first_step_match, - history_nemo_nl]) + _, first_step_val = shellout._exec_subprocess('grep %s %s' % (gl_first_step_match, + history_nemo_nl)) nemo_first_step = int(re.findall(r'.+=(.+),', first_step_val)[0]) # Last timestep of the previous cycle - _, last_step_val = common.exec_subproc(['grep', gl_last_step_match, - history_nemo_nl]) + _, last_step_val = shellout._exec_subprocess('grep %s %s' % (gl_last_step_match, + history_nemo_nl)) nemo_last_step = re.findall(r'.+=(.+),', last_step_val)[0] # The string in the nemo time step field might have any one of @@ -460,15 +465,15 @@ def _setup_executable(common_env): nemo_last_step = 0 # Determine (as an integer) the number of seconds per model timestep - _, nemo_step_int_val = common.exec_subproc(['grep', gl_step_int_match, - nemo_envar['NEMO_NL']]) + _, nemo_step_int_val = shellout._exec_subprocess('grep %s %s' % (gl_step_int_match, + nemo_envar['NEMO_NL'])) nemo_step_int = int(re.findall(r'.+=(\d*)', nemo_step_int_val)[0]) # If the value for nemo_rst_date_value is true then the model uses # absolute date convention, otherwise the dump times are relative to the # start of the model run and have an integer representation - _, nemo_rst_date_value = common.exec_subproc([ \ - 'grep', gl_nemo_restart_date_match, history_nemo_nl]) + _, nemo_rst_date_value = shellout._exec_subprocess( + 'grep %s %s' % (gl_nemo_restart_date_match, history_nemo_nl)) if 'true' in nemo_rst_date_value: nemo_rst_date_bool = True else: @@ -480,8 +485,8 @@ def _setup_executable(common_env): nemo_dump_time = "00000000" # Get the model basis time for this run (YYYYMMDD) - _, model_basis_val = common.exec_subproc( - ['grep', gl_model_basis_time, history_nemo_nl]) + _, model_basis_val = shellout._exec_subprocess( + 'grep %s %s' % (gl_model_basis_time, history_nemo_nl)) nemo_model_basis = re.findall(r'.+=(.+),', model_basis_val)[0] if os.path.isfile(latest_nemo_dump): @@ -768,8 +773,7 @@ def _setup_executable(common_env): update_nl_cmd = './update_nemo_nl %s' % update_nl_cmd # REFACTOR TO USE THE SAFE EXEC SUBPROC - update_nl_rcode, _ = common.__exec_subproc_true_shell([ \ - update_nl_cmd]) + update_nl_rcode, _ = shellout._exec_subprocess(update_nl_cmd) if update_nl_rcode != 0: sys.stderr.write('[FAIL] Error updating nemo namelist\n') sys.exit(error.SUBPROC_ERROR) @@ -989,8 +993,8 @@ def _finalize_executable(common_env): write_ocean_out_to_stdout() - _, error_count = common.__exec_subproc_true_shell([ \ - 'grep "E R R O R" ocean.output | wc -l']) + _, error_count = shellout._exec_subprocess( + 'grep "E R R O R" ocean.output | wc -l') if int(error_count) >= 1: sys.stderr.write('[FAIL] An error has been found with the NEMO run.' ' Please investigate the ocean.output file for more' diff --git a/Coupled_Drivers/rivers_driver.py b/Coupled_Drivers/rivers_driver.py index 8dfab29..2d526d2 100644 --- a/Coupled_Drivers/rivers_driver.py +++ b/Coupled_Drivers/rivers_driver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2025 Met Office. All rights reserved. + (C) Crown copyright 2025-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -11,6 +11,10 @@ Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom *****************************COPYRIGHT****************************** + +# Some of the content of this file has been produced with the assistance of +# Claude Sonnet 4.5. + NAME rivers_driver.py @@ -23,6 +27,7 @@ import re import pathlib import common +import shellout import error import dr_env_lib.rivers_def import dr_env_lib.env_lib @@ -58,12 +63,12 @@ def _setup_dates(common_envar): task_length[2], task_length[3], task_length[4]) - start_cmd = ['isodatetime', '%s' % start_date, '-f', '%s' % format_date] - end_cmd = ['isodatetime', '%s' % start_date, '-f', '%s' % format_date, - '-s', '%s' % length_date, '--calendar', '%s' % calendar] + start_cmd = 'isodatetime %s -f "%s"' % (start_date, format_date) + end_cmd = 'isodatetime %s -f "%s" -s %s --calendar %s' % (start_date, format_date, + length_date, calendar) - _, run_start = common.exec_subproc(start_cmd) - _, run_end = common.exec_subproc(end_cmd) + _, run_start = shellout._exec_subprocess(start_cmd) + _, run_end = shellout._exec_subprocess(end_cmd) return run_start.strip(), run_end.strip() @@ -97,7 +102,7 @@ def _update_river_nl(river_envar, run_start, run_end): mod_timenl.replace() # Create the output directory, do not rely on f90nml - rcode, val = common.exec_subproc(['grep', 'output_dir', output_nl]) + rcode, val = shellout._exec_subprocess('grep output_dir %s' % output_nl) if rcode == 0: try: output_dir = re.findall(r'[\"\'](.*?)[\"\']', val)[0].rstrip('/') diff --git a/Coupled_Drivers/si3_controller.py b/Coupled_Drivers/si3_controller.py index b0a93e2..bd4087f 100644 --- a/Coupled_Drivers/si3_controller.py +++ b/Coupled_Drivers/si3_controller.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy of the code, the use, duplication or disclosure of it is strictly @@ -10,6 +10,10 @@ Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom *****************************COPYRIGHT****************************** + +# Some of the content of this file has been produced with the assistance of +# Claude Sonnet 4.5. + NAME si3_controller.py @@ -22,6 +26,7 @@ import sys import glob import common +import shellout import error import dr_env_lib.ocn_cont_def import dr_env_lib.env_lib @@ -44,8 +49,8 @@ def _get_si3rst(si3_nl_file): ''' Retrieve the SI3 restart directory from the nemo namelist file ''' - si3rst_rcode, si3rst_val = common.exec_subproc([ \ - 'grep', 'cn_icerst_outdir', si3_nl_file]) + si3rst_rcode, si3rst_val = shellout._exec_subprocess( + 'grep cn_icerst_outdir %s' % si3_nl_file) if si3rst_rcode == 0: si3_rst = re.findall('[\"\'](.*?)[\"\']', si3rst_val)[0] if si3_rst[-1] == '/': diff --git a/Coupled_Drivers/top_controller.py b/Coupled_Drivers/top_controller.py index f1d2478..0b6a4ca 100644 --- a/Coupled_Drivers/top_controller.py +++ b/Coupled_Drivers/top_controller.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -11,6 +11,10 @@ Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom *****************************COPYRIGHT****************************** + +# Some of the content of this file has been produced with the assistance of +# Claude Sonnet 4.5. + NAME top_controller.py @@ -69,6 +73,7 @@ import glob import shutil import common +import shellout import error import dr_env_lib.ocn_cont_def import dr_env_lib.env_lib @@ -100,8 +105,8 @@ def _get_toprst_dir(top_nl_file): something different. ''' - toprst_rcode, toprst_val = common.exec_subproc([ \ - 'grep', 'cn_trcrst_outdir', top_nl_file]) + toprst_rcode, toprst_val = shellout._exec_subprocess( + 'grep cn_trcrst_outdir %s' % top_nl_file) if toprst_rcode == 0: top_rst_dir = re.findall('[\"\'](.*?)[\"\']', toprst_val)[0] diff --git a/Coupled_Drivers/unittests/test_cpmip_utils.py b/Coupled_Drivers/unittests/test_cpmip_utils.py index 964b97c..d82ab71 100644 --- a/Coupled_Drivers/unittests/test_cpmip_utils.py +++ b/Coupled_Drivers/unittests/test_cpmip_utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2023-2025 Met Office. All rights reserved. + (C) Crown copyright 2023-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -11,6 +11,10 @@ Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom *****************************COPYRIGHT****************************** + +# Some of the content of this file has been produced with the assistance of +# Claude Sonnet 4.5. + ''' import unittest @@ -29,7 +33,7 @@ class TestGetComponentResolution(unittest.TestCase): ''' Test the construction of component resolution from namelist ''' - @mock.patch('cpmip_utils.common.exec_subproc') + @mock.patch('cpmip_utils.shellout._exec_subprocess') def test_get_component_resolution(self, mock_subproc): ''' Test construction of total resolution @@ -44,7 +48,7 @@ def test_get_component_resolution(self, mock_subproc): 6000) subproc_calls = [] for res_var in res_vars: - subproc_calls.append(mock.call(['grep', res_var, 'NEMO_NL'], + subproc_calls.append(mock.call('grep %s NEMO_NL' % res_var, verbose=True)) mock_subproc.assert_has_calls(subproc_calls) @@ -67,7 +71,7 @@ def test_get_glob_usage_nofile(self, mock_glob): self.assertEqual(patch_output.getvalue(), expected_output) @mock.patch('cpmip_utils.glob.glob', return_value=['file1', 'file2']) - @mock.patch('cpmip_utils.common.exec_subproc_timeout', + @mock.patch('cpmip_utils.shellout._exec_subprocess', return_value=(0, '\n128 file1\n128 file2\n256 total\n')) def test_get_glob_usage(self, mock_subproc, mock_glob): ''' @@ -81,7 +85,7 @@ class TestNCDFOutput(unittest.TestCase): Test measurment of NCDF file sizes ''' @mock.patch('cpmip_utils.os.listdir', return_value=[]) - @mock.patch('cpmip_utils.common.exec_subproc_timeout', + @mock.patch('cpmip_utils.shellout._exec_subprocess', return_value=(1, None)) def test_no_files_output(self, mock_subproc, mock_ncdffiles): ''' @@ -91,7 +95,7 @@ def test_no_files_output(self, mock_subproc, mock_ncdffiles): @mock.patch('cpmip_utils.os.listdir', return_value=['file1.nc', 'file2.nc']) - @mock.patch('cpmip_utils.common.exec_subproc_timeout', + @mock.patch('cpmip_utils.shellout._exec_subprocess', return_value=(0, '\n128 file1.nc\n128 file2.nc\n256 total\n')) def test_files_output(self, mock_subproc, mock_ncdffiles): ''' diff --git a/Coupled_Drivers/unittests/test_cpmip_xios.py b/Coupled_Drivers/unittests/test_cpmip_xios.py index 2d1c087..ce361db 100644 --- a/Coupled_Drivers/unittests/test_cpmip_xios.py +++ b/Coupled_Drivers/unittests/test_cpmip_xios.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -11,6 +11,10 @@ Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom *****************************COPYRIGHT****************************** + +# Some of the content of this file has been produced with the assistance of +# Claude Sonnet 4.5. + ''' import unittest @@ -93,7 +97,7 @@ def test_no_files(self, mock_listdir): ['xios_client0.out', 'xios_client1.out', 'xios_client2.out']) - @mock.patch('cpmip_xios.common.exec_subproc_timeout') + @mock.patch('cpmip_xios.shellout._exec_subprocess') def test_three_files(self, mock_exec_subproc, mock_listdir): ''' Test that three files with no timeout give mean and max @@ -116,7 +120,7 @@ def test_three_files(self, mock_exec_subproc, mock_listdir): ['xios_client0.out', 'xios_client1.out', 'xios_client2.out']) - @mock.patch('cpmip_xios.common.exec_subproc_timeout') + @mock.patch('cpmip_xios.shellout._exec_subprocess') def test_one_timeout(self, mock_exec_subproc, mock_listdir): ''' Test what happens if there is a timeout diff --git a/Coupled_Drivers/unittests/test_rivers_driver.py b/Coupled_Drivers/unittests/test_rivers_driver.py index 5d9a206..1070ddb 100644 --- a/Coupled_Drivers/unittests/test_rivers_driver.py +++ b/Coupled_Drivers/unittests/test_rivers_driver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2025 Met Office. All rights reserved. + (C) Crown copyright 2025-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -11,6 +11,10 @@ Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom *****************************COPYRIGHT****************************** + +# Some of the content of this file has been produced with the assistance of +# Claude Sonnet 4.5. + ''' import sys import unittest @@ -30,26 +34,24 @@ class TestPrivateMethods(unittest.TestCase): Test the private methods of the JULES river standalone driver ''' - @mock.patch('rivers_driver.common.exec_subproc', return_value=[0, 'output']) + @mock.patch('rivers_driver.shellout._exec_subprocess', return_value=[0, 'output']) def test_setup_dates(self, mock_exec): ''' Test the _setup_dates method ''' start, end = rivers_driver._setup_dates(COMMON_ENV) - self.assertIn(mock.call(['isodatetime', '19790901T0000Z', - '-f', '%Y-%m-%d %H:%M:%S']), + self.assertIn(mock.call('isodatetime 19790901T0000Z -f "%Y-%m-%d %H:%M:%S"'), mock_exec.mock_calls) - self.assertIn(mock.call(['isodatetime', '19790901T0000Z', '-f', - '%Y-%m-%d %H:%M:%S', '-s', 'P1Y4M10DT0H0M', - '--calendar', 'gregorian']), + self.assertIn(mock.call('isodatetime 19790901T0000Z -f "%Y-%m-%d %H:%M:%S" -s P1Y4M10DT0H0M --calendar gregorian'), mock_exec.mock_calls) self.assertEqual(len(mock_exec.mock_calls), 2) @mock.patch('rivers_driver.common') + @mock.patch('rivers_driver.shellout') @mock.patch('rivers_driver.os.path.isfile') @mock.patch('rivers_driver.pathlib') - def test_update_river_nl(self, mock_lib, mock_path, mock_common): + def test_update_river_nl(self, mock_lib, mock_path, mock_shellout, mock_common): ''' Test the _update_river_nl method ''' - mock_common.exec_subproc.return_value = (0, 'dir="this/path/"') + mock_shellout._exec_subprocess.returnvalue = (0, 'dir="this/path/"') rivers_driver._update_river_nl(RIVER_ENV, '19790901T0000Z', '19810121T0000Z') @@ -64,14 +66,14 @@ def test_update_river_nl(self, mock_lib, mock_path, mock_common): self.assertIn(mock.call().var_val('output_start', '19790901T0000Z'), nml_calls) self.assertIn(mock.call().replace(), nml_calls) - + self.assertIn(mock.call().var_val('main_run_start', '19790901T0000Z'), nml_calls) self.assertIn(mock.call().var_val('main_run_end', '19810121T0000Z'), nml_calls) - mock_common.exec_subproc.assert_called_once_with( - ['grep', 'output_dir', 'output.nml'] + mock_shellout._exec_subprocess.assert_called_once_with( + 'grep output_dir output.nml' ) mock_lib.Path.assert_called_once_with('this/path') mock_lib.Path().mkdir.assert_called_once_with(parents=True, diff --git a/Coupled_Drivers/write_namcouple.py b/Coupled_Drivers/write_namcouple.py index 5d1fb85..23b1334 100644 --- a/Coupled_Drivers/write_namcouple.py +++ b/Coupled_Drivers/write_namcouple.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2021-2025 Met Office. All rights reserved. + (C) Crown copyright 2021-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy of the code, the use, duplication or disclosure of it is strictly @@ -10,6 +10,10 @@ Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom *****************************COPYRIGHT****************************** + +# Some of the content of this file has been produced with the assistance of +# Claude Sonnet 4.5. + NAME write_namcouple.py @@ -19,6 +23,7 @@ import sys import itertools import common +import shellout import default_couplings import error import write_cf_name_table @@ -332,4 +337,4 @@ def write_namcouple(common_env, run_info, coupling_list): # Now that namcouple has been created, we can create the transient # field namelist - _, _ = common.exec_subproc('./OASIS_fields') + _, _ = shellout._exec_subprocess('./OASIS_fields') diff --git a/Coupled_Drivers/xios_driver.py b/Coupled_Drivers/xios_driver.py index 3834673..fd514c4 100644 --- a/Coupled_Drivers/xios_driver.py +++ b/Coupled_Drivers/xios_driver.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 ''' *****************************COPYRIGHT****************************** - (C) Crown copyright 2023-2025 Met Office. All rights reserved. + (C) Crown copyright 2023-2026 Met Office. All rights reserved. Use, duplication or disclosure of this code is subject to the restrictions as set forth in the licence. If no licence has been raised with this copy @@ -11,6 +11,10 @@ Met Office, FitzRoy Road, Exeter, Devon, EX1 3PB, United Kingdom *****************************COPYRIGHT****************************** + +# Some of the content of this file has been produced with the assistance of +# Claude Sonnet 4.5. + NAME xios_driver.py @@ -24,6 +28,7 @@ import os import shutil import common +import shellout import dr_env_lib.xios_def import dr_env_lib.env_lib @@ -43,7 +48,7 @@ def _update_iodef( ''' # Work-around in lieu of viable multi component iodef.xml handling - _, _ = common.exec_subproc(['cp', 'mydef.xml', iodef_fname]) + _, _ = shellout._exec_subprocess('cp mydef.xml %s' % iodef_fname) # Note we do not use python's xml module for this job, as the comment # line prevalent in the first line of the GO5 iodef.xml files renders diff --git a/Postprocessing/common/utils.py b/Postprocessing/common/utils.py index 9a79af4..159d664 100644 --- a/Postprocessing/common/utils.py +++ b/Postprocessing/common/utils.py @@ -23,7 +23,7 @@ import os import errno import shutil -import subprocess +import shellout import timer @@ -153,61 +153,13 @@ def finalcycle(): return fcycle -@timer.run_timer -def exec_subproc(cmd, verbose=True, cwd=os.getcwd()): - ''' - Execute given shell command. - 'cmd' input should be in the form of either a: - string - "cd DIR; command arg1 arg2" - list of words - ["command", "arg1", "arg2"] - Optional arguments: - verbose = False: only reproduce the command std.out upon - failure of the command - True: reproduce std.out regardless of outcome - cwd = Directory in which to execute the command - ''' - import shlex - - cmd_array = [cmd] - if not isinstance(cmd, list): - cmd_array = cmd.split(';') - for i, cmd in enumerate(cmd_array): - # Use shlex.split to cope with arguments that contain whitespace - cmd_array[i] = shlex.split(cmd) - - # Initialise rcode, in the event there is no command - rcode = 99 - output = 'No command provided' - - for cmd in cmd_array: - try: - output = subprocess.check_output(cmd, - stderr=subprocess.STDOUT, - universal_newlines=True, cwd=cwd) - rcode = 0 - if verbose: - log_msg('[SUBPROCESS]: ' + str(output)) - except subprocess.CalledProcessError as exc: - output = exc.output - rcode = exc.returncode - except OSError as exc: - output = exc.strerror - rcode = exc.errno - if rcode != 0: - msg = '[SUBPROCESS]: Command: {}\n[SUBPROCESS]: Error = {}:\n\t{}' - log_msg(msg.format(' '.join(cmd), rcode, output), level='WARN') - break - - return rcode, output - - def get_utility_avail(utility): '''Return True/False if shell command is available''' try: status = shutil.which(utility) except AttributeError: # subprocess.getstatusoutput does not exist at Python2.7 - status, _ = utils.exec_subproc(utility + ' --help', verbose=False) + status, _ = shellout.exec_subprocess(utility + ' --help') return bool(status) @@ -496,7 +448,7 @@ def _mod_all_calendars_date(indate, delta, cal): cmd = '{} {} --calendar {} --offset {} --print-format ' \ '%Y,%m,%d,%H,%M'.format(datecmd, dateinput, cal, offset) - rcode, output = exec_subproc(cmd, verbose=False) + rcode, output = shellout._exec_subprocess(cmd) else: log_msg('add_period_to_date: Invalid date for conversion to ' 'ISO 8601 date representation: ' + str(outdate), diff --git a/Utilities/lib/shellout.py b/Utilities/lib/shellout.py new file mode 100644 index 0000000..c0effa8 --- /dev/null +++ b/Utilities/lib/shellout.py @@ -0,0 +1,47 @@ +import timer +import subprocess +import os +import sys +import shlex + + +@timer.run_timer +def _exec_subprocess(cmd, verbose=False, current_working_directory=os.getcwd()): + """ + Execute a given shell command + + :param cmd: The command to be executed given as a string + :param verbose: A boolean value to determine if the stdout + stream is displayed during the runtime. + :param current_working_directory: The directory in which the + command should be executed. + """ + + cmd = shlex.split(cmd) + + try: + + output = subprocess.run( + cmd, + stdin=subprocess.PIPE, + capture_output=True, + cwd=current_working_directory, + timeout=10, + ) + rcode = output.returncode + + if verbose and output: + sys.stdout.write(f"[DEBUG]{output.stdout}\n") + if output.stderr and output.returncode != 0: + sys.stderr.write(f"[ERROR] {output.stderr}\n") + if sys.version_info[0] >= 3: + output.stdout = output.stdout.decode() + + except subprocess.CalledProcessError as exc: + output = exc.output + rcode = exc.returncode + except OSError as exc: + output = exc.strerror + rcode = exc.errno + + return rcode, output