From cbb4e0eef4139a66fd5da5a3950ae999a34b55bb Mon Sep 17 00:00:00 2001 From: jedbakerMO Date: Tue, 14 Oct 2025 13:23:44 +0100 Subject: [PATCH 1/7] Adding feature and tidying relevant code --- post-processing/post-process.py | 51 ++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/post-processing/post-process.py b/post-processing/post-process.py index 2978b895..a7214fd1 100755 --- a/post-processing/post-process.py +++ b/post-processing/post-process.py @@ -29,10 +29,11 @@ def parse_cli_arguments(input_arguments: list[str] = None, """ parser = argparse.ArgumentParser(description="This script is for merging the outputs from a test that uses Vernier callipers into one file. For full documentation please see the post-processing section of the user guide.") - parser.add_argument("-p", "--path", type=Path, default=(os.getcwd()), help="Path to Vernier output files") - parser.add_argument("-o", "--output_name", type=str, default=str("vernier-merged-output"), help="Name of file to write to") - parser.add_argument("-i", "--input_name", type=str, default=str("vernier-output-"), help="Vernier files to read from") - parser.add_argument("-b", "--basic_output", action="store_true", default=False, help="Outputs only mean values across MPI ranks") + parser.add_argument("-p", "--path", type=Path, default=(os.getcwd()), help="Path to Vernier output files") + parser.add_argument("-o", "--output_name", type=str, default=str("vernier-merged-output"), help="Name of file to write to") + parser.add_argument("-i", "--input_name", type=str, default=str("vernier-output-"), help="Vernier files to read from") + parser.add_argument("-b", "--basic_output", action="store_true", default=False, help="Outputs only mean values across MPI ranks") + parser.add_argument("-r", "--recursive_process", action="store_true", default=False, help="Recursively processes all found vernier outputs in the given directory") return parser.parse_args(args=input_arguments) @@ -116,7 +117,6 @@ def merge_and_analyse(file_path: Path, The merged dataframe, containing the routine names and the mean 'Self' and 'Total' values across all outputs. """ - print(f"Path to open: {file_path}") print(f"Detected {mpiranks} files.") for rank in range(0,mpiranks): @@ -163,33 +163,44 @@ def merge_and_analyse(file_path: Path, def main(): """ Read in command line arguments, assigning them to variables. Determine how many outputs to merge """ - args = parse_cli_arguments() - file_path = args.path - merged_file_name = args.output_name - input_name = args.input_name + args = parse_cli_arguments() + file_path = args.path + merged_file_name = args.output_name + input_name = args.input_name basic_output_bool = args.basic_output - mpiranks = read_mpi_ranks(file_path, input_name) + recursive_process = args.recursive_process - if mpiranks == 0: - - print("Error, no vernier-outputs detected") - print("Searched in: ", file_path) + if recursive_process: + print("\nVernier outputs found in: \n") + valid_directories = [] + for root, dirs, files in os.walk("."): + if (input_name+"0") in files: + valid_directories.append(os.path.relpath(root)) + print( os.path.relpath(root)) else: + valid_directories = [file_path] + + mpiranks = read_mpi_ranks(output_path, input_name) + + if mpiranks == 0: - print("\nReading and Merging...") + print("Error, no vernier-outputs detected") + print("Searched in: ", output_path) + for output_path in valid_directories: - merged_frame = merge_and_analyse(file_path, int(mpiranks), input_name, basic_output_bool) + mpiranks = read_mpi_ranks(output_path, input_name) + print("\nMerging files in: ", output_path) + + merged_frame = merge_and_analyse(output_path, int(mpiranks), input_name, basic_output_bool) thread_string = "@0" merged_frame["Routine"] = merged_frame["Routine"].str.replace(thread_string, '') - print("\nWriting...") - with open(f"{merged_file_name}", 'w') as f: + with open(f"{output_path}/{merged_file_name}", 'w') as f: f.write(merged_frame.to_string(index=False, col_space=10)) - - print(f"Merged outputs written to {merged_file_name}\n") + print(f"Merged outputs written to {output_path}/{merged_file_name}") if __name__ == '__main__': From 6cd74311269d7459eb524a94555115d5c87141ab Mon Sep 17 00:00:00 2001 From: jedbakerMO Date: Tue, 14 Oct 2025 14:09:38 +0100 Subject: [PATCH 2/7] Updating documentation --- documentation/Sphinx/user_guide/output/post-processing.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/documentation/Sphinx/user_guide/output/post-processing.rst b/documentation/Sphinx/user_guide/output/post-processing.rst index 907e49d5..89ea3f5c 100644 --- a/documentation/Sphinx/user_guide/output/post-processing.rst +++ b/documentation/Sphinx/user_guide/output/post-processing.rst @@ -53,6 +53,9 @@ script, these are given in the table below. * - ``-b/ --basic_output`` - False - If used will only output mean values across MPI ranks + * - ``-r/ --recursive_process`` + - False + - If used will process all vernier outputs found within subdirectories of given path Below is an example of the current script output. From d55d35ba697963c02b93a46445b7517f58b42f51 Mon Sep 17 00:00:00 2001 From: Jed Baker Date: Tue, 14 Oct 2025 14:32:18 +0100 Subject: [PATCH 3/7] Update post-processing.rst --- documentation/Sphinx/user_guide/output/post-processing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/Sphinx/user_guide/output/post-processing.rst b/documentation/Sphinx/user_guide/output/post-processing.rst index 2d4f1ae7..25d21e4f 100644 --- a/documentation/Sphinx/user_guide/output/post-processing.rst +++ b/documentation/Sphinx/user_guide/output/post-processing.rst @@ -52,8 +52,8 @@ script, these are given in the table below. - Vernier files to read from * - ``-m/ --max_only`` - False - - If used will only output max values across MPI ranks - * - ``-f/ --full_info`` + - If used will only output maximum values across MPI ranks + * - ``-f/ --full_info`` - False - Enables full Vernier output to be merged and output * - ``-r/ --recursive_process`` From 46f9297ab2304c573adef1ad7c3e135c240845a1 Mon Sep 17 00:00:00 2001 From: Jed Baker Date: Tue, 14 Oct 2025 14:35:16 +0100 Subject: [PATCH 4/7] Update post-processing.rst --- documentation/Sphinx/user_guide/output/post-processing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/Sphinx/user_guide/output/post-processing.rst b/documentation/Sphinx/user_guide/output/post-processing.rst index 25d21e4f..62b4e120 100644 --- a/documentation/Sphinx/user_guide/output/post-processing.rst +++ b/documentation/Sphinx/user_guide/output/post-processing.rst @@ -52,7 +52,7 @@ script, these are given in the table below. - Vernier files to read from * - ``-m/ --max_only`` - False - - If used will only output maximum values across MPI ranks + - If used will only calculate maximum values across MPI ranks * - ``-f/ --full_info`` - False - Enables full Vernier output to be merged and output From ffd6ce843bc2578c6e3b9b4d46c6de1aa62cbbda Mon Sep 17 00:00:00 2001 From: jedbakerMO Date: Tue, 14 Oct 2025 14:38:54 +0100 Subject: [PATCH 5/7] Fixing merge issues and tidying up code --- post-processing/post-process.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/post-processing/post-process.py b/post-processing/post-process.py index 1fc13895..2374a6a8 100755 --- a/post-processing/post-process.py +++ b/post-processing/post-process.py @@ -35,7 +35,7 @@ def parse_cli_arguments(input_arguments: list[str] = None, parser.add_argument("-b", "--basic_output", action="store_true", default=False, help="Outputs only mean values across MPI ranks") parser.add_argument("-m", "--max_only", action="store_true", default=False, help="Only calculates the maximum value across all ranks") parser.add_argument("-f", "--full_info", action="store_true", default=False, help="Enables merging and displaying of all information Vernier records") - parser.add_argument("-r", "--recursive_process", action="store_true", default=False, help="Recursively processes all found vernier outputs in the given directory" + parser.add_argument("-r", "--recursive_process", action="store_true", default=False, help="Recursively processes all found vernier outputs in the given directory") return parser.parse_args(args=input_arguments) def read_mpi_ranks(directory_path: Path, @@ -196,8 +196,6 @@ def main(): full_info_bool = args.full_info recursive_process = args.recursive_process - mpiranks = read_mpi_ranks(file_path, input_name) - if recursive_process: print("\nVernier outputs found in: \n") valid_directories = [] @@ -207,11 +205,10 @@ def main(): print( os.path.relpath(root)) else: - valid_directories = [file_path] - mpiranks = read_mpi_ranks(output_path, input_name) + valid_directories = [file_path] - if mpiranks == 0: + if read_mpi_ranks(output_path, input_name) == 0: print("Error, no vernier-outputs detected") print("Searched in: ", output_path) From 3f966ea428610a9955d58f53475f5378b58ad07d Mon Sep 17 00:00:00 2001 From: jedbakerMO Date: Tue, 14 Oct 2025 14:42:00 +0100 Subject: [PATCH 6/7] Added error checking for recursive processing --- post-processing/post-process.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/post-processing/post-process.py b/post-processing/post-process.py index 2374a6a8..6e2279f7 100755 --- a/post-processing/post-process.py +++ b/post-processing/post-process.py @@ -197,13 +197,20 @@ def main(): recursive_process = args.recursive_process if recursive_process: + print("\nVernier outputs found in: \n") + valid_directories = [] + for root, dirs, files in os.walk("."): if (input_name+"0") in files: valid_directories.append(os.path.relpath(root)) print( os.path.relpath(root)) + if len(valid_directories) == 0: + + print("Error, no vernier-outputs detected") + else: valid_directories = [file_path] From 2005c7ffc628fa1593fa7f03fbf96efb86794e45 Mon Sep 17 00:00:00 2001 From: jedbakerMO Date: Tue, 14 Oct 2025 16:32:27 +0100 Subject: [PATCH 7/7] Fixing merge issue --- post-processing/post-process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/post-processing/post-process.py b/post-processing/post-process.py index 6e2279f7..d099abf6 100755 --- a/post-processing/post-process.py +++ b/post-processing/post-process.py @@ -226,7 +226,7 @@ def main(): print("\nMerging files in: ", output_path) - merged_frame = merge_and_analyse(file_path, int(mpiranks), input_name, max_only_bool, full_info_bool) + merged_frame = merge_and_analyse(output_path, int(mpiranks), input_name, max_only_bool, full_info_bool) thread_string = "@0" merged_frame["Routine"] = merged_frame["Routine"].str.replace(thread_string, '')