From d871cc1306e7eb2ef8c35cc7347af91ea5896363 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 8 Nov 2019 16:19:57 +0000 Subject: [PATCH 01/82] Add tool for updating v4 factory uses to v5 style --- src/bout-v5-factory-upgrader.py | 251 ++++++++++++++++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100755 src/bout-v5-factory-upgrader.py diff --git a/src/bout-v5-factory-upgrader.py b/src/bout-v5-factory-upgrader.py new file mode 100755 index 0000000..1a985b7 --- /dev/null +++ b/src/bout-v5-factory-upgrader.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 + +import argparse +import copy +import difflib +import re + + +# Dictionary of factory methods that may need updating +factories = { + "Laplacian": { + "factory_name": "Laplacian", + "type_name": "Laplacian", + "create_method": "create", + }, + "SolverFactory": { + "factory_name": "SolverFactory", + "type_name": "Solver", + "create_method": "createSolver", + }, + "Solver": { + "factory_name": "Solver", + "type_name": "Solver", + "create_method": "create", + }, +} + + +def find_factory_calls(factory, source): + """Find all the places where the factory creation method is called, + and return a list of the variable names + + Parameters + ---------- + factory + Dictionary containing 'factory_name' and 'create_method' + source + Text to search + + """ + return re.findall( + r""" + \s*([\w_]+) # variable + \s*=\s* + {factory_name}:: + .*{create_method}.* + """.format( + **factory + ), + source, + re.VERBOSE, + ) + + +def find_type_pointers(factory, source): + return re.findall( + r""" + \b{type_name}\s*\*\s* # Type name and pointer + ([\w_]+)\s*; # Variable name + """.format( + **factory + ), + source, + re.VERBOSE, + ) + + +def fix_declarations(factory, variables, source): + """Fix the declaration of varables in source. Returns modified source + + Replaces `Type*` with either `std::unique_ptr` for + declarations, or with `auto` for initialisations. + + Parameters + ---------- + factory + Dictionary of factory information + variables + List of variable names + source + Text to update + + """ + + for variable in variables: + # Declarations + source = re.sub( + r""" + (.*?)(class\s*)? # optional "class" keyword + \b({type_name})\s*\*\s* # Type-pointer + ({variable_name})\s*; # Variable + """.format( + type_name=factory["type_name"], variable_name=variable + ), + r"\1std::unique_ptr<\3> \4{nullptr};", + source, + flags=re.VERBOSE, + ) + + # Declarations with initialisation + source = re.sub( + r""" + (.*?)(class\s*)? # optional "class" keyword + ({type_name})\s*\*\s* # Type-pointer + ({variable_name})\s* # Variable + =\s* # Assignment from factory + ({factory_name}::.*{create_method}.*); + """.format( + variable_name=variable, **factory + ), + r"\1auto \4 = \5;", + source, + flags=re.VERBOSE, + ) + + return source + + +def fix_deletions(variables, source): + """Remove `delete` statements of variables. Returns modified source + + Parameters + ---------- + variables + List of variable names + source + Text to update + + """ + + for variable in variables: + source = re.sub( + r"(.*;?)\s*(delete\s*{variable})\s*;".format(variable=variable), + r"\1", + source, + ) + + return source + + +def yes_or_no(question): + """Convert user input from yes/no variations to True/False + + """ + while True: + reply = input(question + " [y/N] ").lower().strip() + if not reply or reply[0] == "n": + return False + if reply[0] == "y": + return True + + +def apply_fixes(factories, source, all_declarations=False): + """Apply the various fixes for each factory to source. Returns + modified source + + Parameters + ---------- + factories + Dictionary of factory properties + source + Text to update + """ + + modified = source + + for factory in factories.values(): + variables = find_factory_calls(factory, modified) + if all_declarations: + variables = variables + find_type_pointers(factory, modified) + modified = fix_declarations(factory, variables, modified) + modified = fix_deletions(variables, modified) + + return modified + + +def create_patch(filename, original, modified): + """Create a unified diff between original and modified + """ + + patch = "\n".join( + difflib.unified_diff( + original.splitlines(), + modified.splitlines(), + fromfile=filename, + tofile=filename, + lineterm="", + ) + ) + + return patch + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Fix types of factory-created objects") + + parser.add_argument("files", action="store", nargs="+", help="Input files") + parser.add_argument( + "--force", "-f", action="store_true", help="Make changes without asking" + ) + parser.add_argument( + "--quiet", "-q", action="store_true", help="Don't print patches" + ) + parser.add_argument( + "--patch-only", "-p", action="store_true", help="Print the patches and exit" + ) + parser.add_argument( + "--all-declarations", + "-a", + action="store_true", + help="Fix all declarations of factory types, not just variables created from factories", + ) + + args = parser.parse_args() + + if args.force and args.patch_only: + raise ValueError("Incompatible options: --force and --patch") + + for filename in args.files: + with open(filename, "r") as f: + contents = f.read() + original = copy.deepcopy(contents) + + modified = apply_fixes( + factories, contents, all_declarations=args.all_declarations + ) + patch = create_patch(filename, original, modified) + + if args.patch_only: + print(patch) + continue + + if not patch: + if not args.quiet: + print("No changes to make to {}".format(filename)) + continue + + if not args.quiet: + print("\n******************************************") + print("Changes to {}\n".format(filename)) + print(patch) + print("\n******************************************") + + if args.force: + make_change = True + else: + make_change = yes_or_no("Make changes to {}?".format(filename)) + + if make_change: + with open(filename, "w") as f: + f.write(modified) From ef2eae89c90c43478e70ff6d886340b8b01be4f1 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 14 Nov 2019 12:02:57 +0000 Subject: [PATCH 02/82] Add LaplaceXZ to factory-upgrader helper script --- src/bout-v5-factory-upgrader.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bout-v5-factory-upgrader.py b/src/bout-v5-factory-upgrader.py index 1a985b7..6377392 100755 --- a/src/bout-v5-factory-upgrader.py +++ b/src/bout-v5-factory-upgrader.py @@ -13,6 +13,11 @@ "type_name": "Laplacian", "create_method": "create", }, + "LaplaceXZ": { + "factory_name": "LaplaceXZ", + "type_name": "LaplaceXZ", + "create_method": "create", + }, "SolverFactory": { "factory_name": "SolverFactory", "type_name": "Solver", From 1a3ef9bc8a54b60a75cbcde5b83b8e85b2c76808 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 14 Nov 2019 13:41:04 +0000 Subject: [PATCH 03/82] Add InvertPar to factory-upgrader helper script --- src/bout-v5-factory-upgrader.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bout-v5-factory-upgrader.py b/src/bout-v5-factory-upgrader.py index 6377392..bf64d2b 100755 --- a/src/bout-v5-factory-upgrader.py +++ b/src/bout-v5-factory-upgrader.py @@ -8,6 +8,11 @@ # Dictionary of factory methods that may need updating factories = { + "InvertPar": { + "factory_name": "InvertPar", + "type_name": "InvertPar", + "create_method": "Create", + }, "Laplacian": { "factory_name": "Laplacian", "type_name": "Laplacian", From 43ecd303fb0edb900743ad0e2183d6477677573f Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 14 Nov 2019 14:30:26 +0000 Subject: [PATCH 04/82] Catch static variables with zero initialisation in factory upgrader --- src/bout-v5-factory-upgrader.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/bout-v5-factory-upgrader.py b/src/bout-v5-factory-upgrader.py index bf64d2b..23aba18 100755 --- a/src/bout-v5-factory-upgrader.py +++ b/src/bout-v5-factory-upgrader.py @@ -107,7 +107,7 @@ def fix_declarations(factory, variables, source): flags=re.VERBOSE, ) - # Declarations with initialisation + # Declarations with initialisation from factory source = re.sub( r""" (.*?)(class\s*)? # optional "class" keyword @@ -123,6 +123,22 @@ def fix_declarations(factory, variables, source): flags=re.VERBOSE, ) + # Declarations with zero initialisation + source = re.sub( + r""" + (.*?)(?:class\s*)? # optional "class" keyword + ({type_name})\s*\*\s* # Type-pointer + ({variable_name})\s* # Variable + =\s* # Assignment + (0|nullptr|NULL); + """.format( + variable_name=variable, **factory + ), + r"\1std::unique_ptr<\2> \3{nullptr};", + source, + flags=re.VERBOSE, + ) + return source From b50ca681ab6ff110be489ffb2a19bcbc17d47f95 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 14 Nov 2019 17:39:24 +0000 Subject: [PATCH 05/82] Add Mesh to factory updater --- src/bout-v5-factory-upgrader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bout-v5-factory-upgrader.py b/src/bout-v5-factory-upgrader.py index 23aba18..ef2b206 100755 --- a/src/bout-v5-factory-upgrader.py +++ b/src/bout-v5-factory-upgrader.py @@ -13,6 +13,7 @@ "type_name": "InvertPar", "create_method": "Create", }, + "Mesh": {"factory_name": "Mesh", "type_name": "Mesh", "create_method": "Create"}, "Laplacian": { "factory_name": "Laplacian", "type_name": "Laplacian", From 5d3e07b49a9a65f34f41a91c893eeddddb23515e Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 15 Nov 2019 11:17:45 +0000 Subject: [PATCH 06/82] Add InterpolationFactory to factory upgrader --- src/bout-v5-factory-upgrader.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bout-v5-factory-upgrader.py b/src/bout-v5-factory-upgrader.py index ef2b206..be33cc4 100755 --- a/src/bout-v5-factory-upgrader.py +++ b/src/bout-v5-factory-upgrader.py @@ -8,6 +8,11 @@ # Dictionary of factory methods that may need updating factories = { + "Interpolation": { + "factory_name": "InterpolationFactory", + "type_name": "Interpolation", + "create_method": "create", + }, "InvertPar": { "factory_name": "InvertPar", "type_name": "InvertPar", From 0db1788c0c7a9ea2258c4a74f93371ca7e519028 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 19 Nov 2019 13:53:16 +0000 Subject: [PATCH 07/82] Add format-upgrader to replace printf formats with fmt formats --- src/bout-v5-format-upgrader.py | 158 +++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100755 src/bout-v5-format-upgrader.py diff --git a/src/bout-v5-format-upgrader.py b/src/bout-v5-format-upgrader.py new file mode 100755 index 0000000..4435119 --- /dev/null +++ b/src/bout-v5-format-upgrader.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 + +import argparse +import copy +import difflib +import re + + +format_replacements = { + "%c": "{:c}", + "%d": "{:d}", + "%e": "{:e}", + "%f": "{:f}", + "%g": "{:g}", + "%i": "{:d}", + "%ld": "{:d}", + "%le": "{:e}", + "%lu": "{:d}", + "%p": "{:p}", + "%s": "{:s}", + "%zu": "{:d}", +} + + +def fix_format_replacement(format_replacement, source): + """Replace printf format with fmt format + + """ + return re.sub(format_replacement[0], format_replacement[1], source) + + +def fix_trivial_format(source): + """Reduce trivial formatting of strings to just the string + + """ + + def trivial_replace(match): + if match.group(2): + return "{}{}{}".format(match.group(1), match.group(2), match.group(4)) + if match.group(3): + return "{}{}{}".format(match.group(1), match.group(3), match.group(4)) + raise ValueError("Found an unexpected match: {}".format(match)) + + return re.sub( + r""" + (.*)? + "{:s}",\s* # Entire format is just a string + (?:([\w_]+)\.c_str\(\) # And replacement is std::string::c_str + |(".*?")) + (.*)? + """, + trivial_replace, + source, + flags=re.VERBOSE, + ) + + +def apply_fixes(format_replacements, source): + """Apply the various fixes for each factory to source. Returns + modified source + + Parameters + ---------- + factories + Dictionary of factory properties + source + Text to update + """ + + modified = source + + for format_replacement in format_replacements.items(): + modified = fix_format_replacement(format_replacement, modified) + + modified = fix_trivial_format(modified) + + return modified + + +def yes_or_no(question): + """Convert user input from yes/no variations to True/False + + """ + while True: + reply = input(question + " [y/N] ").lower().strip() + if not reply or reply[0] == "n": + return False + if reply[0] == "y": + return True + + +def create_patch(filename, original, modified): + """Create a unified diff between original and modified + """ + + patch = "\n".join( + difflib.unified_diff( + original.splitlines(), + modified.splitlines(), + fromfile=filename, + tofile=filename, + lineterm="", + ) + ) + + return patch + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Fix format specifiers") + + parser.add_argument("files", action="store", nargs="+", help="Input files") + parser.add_argument( + "--force", "-f", action="store_true", help="Make changes without asking" + ) + parser.add_argument( + "--quiet", "-q", action="store_true", help="Don't print patches" + ) + parser.add_argument( + "--patch-only", "-p", action="store_true", help="Print the patches and exit" + ) + + args = parser.parse_args() + + if args.force and args.patch_only: + raise ValueError("Incompatible options: --force and --patch") + + for filename in args.files: + with open(filename, "r") as f: + contents = f.read() + original = copy.deepcopy(contents) + + modified = apply_fixes(format_replacements, contents) + patch = create_patch(filename, original, modified) + + if args.patch_only: + print(patch) + continue + + if not patch: + if not args.quiet: + print("No changes to make to {}".format(filename)) + continue + + if not args.quiet: + print("\n******************************************") + print("Changes to {}\n".format(filename)) + print(patch) + print("\n******************************************") + + if args.force: + make_change = True + else: + make_change = yes_or_no("Make changes to {}?".format(filename)) + + if make_change: + with open(filename, "w") as f: + f.write(modified) From b652493c0bf475864422d9dfb6a2245ab009b2ce Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 19 Nov 2019 17:23:52 +0000 Subject: [PATCH 08/82] Make format upgrader fix uses of {:s} with std::string::c_str --- src/bout-v5-format-upgrader.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/bout-v5-format-upgrader.py b/src/bout-v5-format-upgrader.py index 4435119..bb57b6d 100755 --- a/src/bout-v5-format-upgrader.py +++ b/src/bout-v5-format-upgrader.py @@ -45,7 +45,7 @@ def trivial_replace(match): r""" (.*)? "{:s}",\s* # Entire format is just a string - (?:([\w_]+)\.c_str\(\) # And replacement is std::string::c_str + (?:([\w_]+)\.c_str\(\) # And replacement is std::string::c_str |(".*?")) (.*)? """, @@ -55,6 +55,36 @@ def trivial_replace(match): ) +def fix_string_c_str(source): + """Fix formats that use {:s} where the replacement is using std::string::c_str + + """ + return re.sub( + r""" + (".*{:s}[^;]*?",) # A format string containing {:s} + \s*([^);]+?)\.c_str\(\) # Replacement of std::string::c_str + """, + r"\1 \2", + source, + flags=re.DOTALL | re.VERBOSE, + ) + + +def fix_trace(source): + """Fix TRACE macros where fix_string_c_str has failed for some reason + + """ + return re.sub( + r""" + (TRACE\(".*{:s}.*",) + \s*([\w_]+)\.c_str\(\)\); # Replacement of std::string::c_str + """, + r"\1 \2);", + source, + flags=re.VERBOSE, + ) + + def apply_fixes(format_replacements, source): """Apply the various fixes for each factory to source. Returns modified source @@ -73,6 +103,8 @@ def apply_fixes(format_replacements, source): modified = fix_format_replacement(format_replacement, modified) modified = fix_trivial_format(modified) + modified = fix_string_c_str(modified) + modified = fix_trace(modified) return modified From 4dd3a7d4dc8d0630051818c863b74e350fda859b Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 20 Nov 2019 12:11:47 +0000 Subject: [PATCH 09/82] Add formats with width specifier to v5 format fixer --- src/bout-v5-format-upgrader.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/bout-v5-format-upgrader.py b/src/bout-v5-format-upgrader.py index bb57b6d..5cf3876 100755 --- a/src/bout-v5-format-upgrader.py +++ b/src/bout-v5-format-upgrader.py @@ -7,18 +7,18 @@ format_replacements = { - "%c": "{:c}", - "%d": "{:d}", - "%e": "{:e}", - "%f": "{:f}", - "%g": "{:g}", - "%i": "{:d}", - "%ld": "{:d}", - "%le": "{:e}", - "%lu": "{:d}", - "%p": "{:p}", - "%s": "{:s}", - "%zu": "{:d}", + "c": "c", + "d": "d", + "e": "e", + "f": "f", + "g": "g", + "i": "d", + "ld": "d", + "le": "e", + "lu": "d", + "p": "p", + "s": "s", + "zu": "d", } @@ -26,7 +26,11 @@ def fix_format_replacement(format_replacement, source): """Replace printf format with fmt format """ - return re.sub(format_replacement[0], format_replacement[1], source) + return re.sub( + r"%([0-9]*\.?[0-9]*){}".format(format_replacement[0]), + r"{{:\1{}}}".format(format_replacement[1]), + source, + ) def fix_trivial_format(source): From d1f83bcd1e84e47ff522c8f3ae5e68ae95e46c9f Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 20 Nov 2019 15:04:36 +0000 Subject: [PATCH 10/82] Fix all BoutException calls in library --- src/bout-v5-format-upgrader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bout-v5-format-upgrader.py b/src/bout-v5-format-upgrader.py index 5cf3876..758ea54 100755 --- a/src/bout-v5-format-upgrader.py +++ b/src/bout-v5-format-upgrader.py @@ -70,7 +70,7 @@ def fix_string_c_str(source): """, r"\1 \2", source, - flags=re.DOTALL | re.VERBOSE, + flags=re.VERBOSE, ) From 59727bb8ed0a223b36f64843bd71febb258b13c5 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 20 Nov 2019 15:05:12 +0000 Subject: [PATCH 11/82] Fix uses of toString().c_str() with formatting --- src/bout-v5-format-upgrader.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/bout-v5-format-upgrader.py b/src/bout-v5-format-upgrader.py index 758ea54..215fc8b 100755 --- a/src/bout-v5-format-upgrader.py +++ b/src/bout-v5-format-upgrader.py @@ -66,7 +66,7 @@ def fix_string_c_str(source): return re.sub( r""" (".*{:s}[^;]*?",) # A format string containing {:s} - \s*([^);]+?)\.c_str\(\) # Replacement of std::string::c_str + \s*([^;]+?)\.c_str\(\) # Replacement of std::string::c_str """, r"\1 \2", source, @@ -89,6 +89,22 @@ def fix_trace(source): ) +def fix_toString_c_str(source): + """Fix formats that call toString where the replacement is using std::string::c_str + + + """ + return re.sub( + r""" + (".*{:s}[^;]*?",.*?) # A format string containing {:s} + (toString\(.*?\))\.c_str\(\) # Replacement of std::string::c_str + """, + r"\1\2", + source, + flags=re.VERBOSE, + ) + + def apply_fixes(format_replacements, source): """Apply the various fixes for each factory to source. Returns modified source From 9da6d8697196e953a354650ee2b11e958dc55468 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 20 Feb 2020 14:15:46 +0000 Subject: [PATCH 12/82] Add updater for Interpolation -> XZInterpolation types --- src/bout-v5-xzinterpolation-upgrader.py | 275 ++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100755 src/bout-v5-xzinterpolation-upgrader.py diff --git a/src/bout-v5-xzinterpolation-upgrader.py b/src/bout-v5-xzinterpolation-upgrader.py new file mode 100755 index 0000000..5aa565e --- /dev/null +++ b/src/bout-v5-xzinterpolation-upgrader.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 + + +import argparse +import copy +import difflib +import re + +try: + import clang.cindex + + has_clang = True +except ImportError: + has_clang = False + + +headers = {"interpolation": {"old": "interpolation.hxx", "new": "interpolation_xz.hxx"}} + +interpolations = { + "Hermite": {"old": "HermiteSpline", "new": "XZHermiteSpline"}, + "Interpolation": {"old": "Interpolation", "new": "XZInterpolation"}, + "MonotonicHermite": { + "old": "MonotonicHermiteSpline", + "new": "XZMonotonicHermiteSpline", + }, + "Bilinear": {"old": "Bilinear", "new": "XZBilinear"}, + "Lagrange4pt": {"old": "Lagrange4pt", "new": "XZLagrange4pt"}, +} + +factories = { + "InterpolationFactory": { + "old": "InterpolationFactory", + "new": "XZInterpolationFactory", + } +} + + +def fix_header_includes(old_header, new_header, source): + """Replace old_header with new_header in source + + Parameters + ---------- + old_header: str + Name of header to be replaced + new_header: str + Name of replacement header + source: str + Text to search + + """ + return re.sub( + r""" + (\s*\#\s*include\s*) # Preprocessor include + (<|") + ({header}) # Header name + (>|") + """.format( + header=old_header + ), + r"\1\2{header}\4".format(header=new_header), + source, + flags=re.VERBOSE, + ) + + +def fix_interpolations(old_interpolation, new_interpolation, source): + + return re.sub( + r""" + \b{}\b + """.format( + old_interpolation + ), + r"{}".format(new_interpolation), + source, + flags=re.VERBOSE, + ) + + +def clang_parse(filename, source): + index = clang.cindex.Index.create() + return index.parse(filename, unsaved_files=[(filename, source)]) + + +def clang_find_interpolations(node, typename, nodes=None): + if nodes is None: + nodes = [] + if node.kind == clang.cindex.CursorKind.TYPE_REF: + if node.type.spelling == typename: + nodes.append(node) + for child in node.get_children(): + clang_find_interpolations(child, typename, nodes) + return nodes + + +def clang_fix_single_interpolation( + old_interpolation, new_interpolation, source, location +): + modified = source + line = modified[location.line - 1] + new_line = ( + line[: location.column - 1] + + new_interpolation + + line[location.column + len(old_interpolation) - 1 :] + ) + modified[location.line - 1] = new_line + return modified + + +def clang_fix_interpolation(old_interpolation, new_interpolation, node, source): + nodes = clang_find_interpolations(node, old_interpolation) + modified = source + for node in nodes: + modified = clang_fix_single_interpolation( + old_interpolation, new_interpolation, modified, node.location + ) + return modified + + +def fix_factories(old_factory, new_factory, source): + + return re.sub( + r""" + \b{}\b + """.format( + old_factory + ), + r"{}".format(new_factory), + source, + flags=re.VERBOSE, + ) + + +def create_patch(filename, original, modified): + """Create a unified diff between original and modified + """ + + patch = "\n".join( + difflib.unified_diff( + original.splitlines(), + modified.splitlines(), + fromfile=filename, + tofile=filename, + lineterm="", + ) + ) + + return patch + + +def yes_or_no(question): + """Convert user input from yes/no variations to True/False + + """ + while True: + reply = input(question + " [y/N] ").lower().strip() + if not reply or reply[0] == "n": + return False + if reply[0] == "y": + return True + + +def apply_fixes(headers, interpolations, factories, source): + """Apply all Interpolation fixes to source + + Parameters + ---------- + headers + Dictionary of old/new headers + interpolations + Dictionary of old/new Interpolation types + source + Text to update + + """ + + modified = copy.deepcopy(source) + + for header in headers.values(): + modified = fix_header_includes(header["old"], header["new"], modified) + for interpolation in interpolations.values(): + modified = fix_interpolations( + interpolation["old"], interpolation["new"], modified + ) + for factory in factories.values(): + modified = fix_factories(factory["old"], factory["new"], modified) + + return modified + + +def clang_apply_fixes(headers, interpolations, factories, filename, source): + + # translation unit + tu = clang_parse(filename, source) + + modified = source + + for header in headers.values(): + modified = fix_header_includes(header["old"], header["new"], modified) + + modified = modified.split("\n") + for interpolation in interpolations.values(): + modified = clang_fix_interpolation( + interpolation["old"], interpolation["new"], tu.cursor, modified + ) + modified = "\n".join(modified) + for factory in factories.values(): + modified = fix_factories(factory["old"], factory["new"], modified) + + return modified + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Fix types of Interpolation objects") + + parser.add_argument("files", action="store", nargs="+", help="Input files") + parser.add_argument( + "--force", "-f", action="store_true", help="Make changes without asking" + ) + parser.add_argument( + "--quiet", "-q", action="store_true", help="Don't print patches" + ) + parser.add_argument( + "--patch-only", "-p", action="store_true", help="Print the patches and exit" + ) + parser.add_argument( + "--clang", action="store_true", help="Use libclang if available" + ) + + args = parser.parse_args() + + if args.force and args.patch_only: + raise ValueError("Incompatible options: --force and --patch") + + if args.clang and not has_clang: + raise RuntimeError( + "libclang is not available. Please install libclang Python bindings" + ) + + for filename in args.files: + with open(filename, "r") as f: + contents = f.read() + original = copy.deepcopy(contents) + + if args.clang and has_clang: + modified = clang_apply_fixes( + headers, interpolations, factories, filename, contents + ) + else: + modified = apply_fixes(headers, interpolations, factories, contents) + patch = create_patch(filename, original, modified) + + if args.patch_only: + print(patch) + continue + + if not patch: + if not args.quiet: + print("No changes to make to {}".format(filename)) + continue + + if not args.quiet: + print("\n******************************************") + print("Changes to {}\n".format(filename)) + print(patch) + print("\n******************************************") + + if args.force: + make_change = True + else: + make_change = yes_or_no("Make changes to {}?".format(filename)) + + if make_change: + with open(filename, "w") as f: + f.write(modified) From d9a25487294d4d906715fb8fe730f942c3a8b44a Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 24 Feb 2020 15:46:42 +0000 Subject: [PATCH 13/82] Add input file upgrader for v4 -> v5 Currently just "canonicalises" input files and adds "bout_version" option --- src/bout-v5-input-file-upgrader.py | 196 +++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100755 src/bout-v5-input-file-upgrader.py diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py new file mode 100755 index 0000000..70bd112 --- /dev/null +++ b/src/bout-v5-input-file-upgrader.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 + +import argparse +import copy +import difflib +import textwrap +import warnings + +from boutdata.data import BoutOptionsFile +from boututils.boutwarnings import AlwaysWarning + +# This should be a list of dicts, each containing exactly two keys: +# "old" and "new". The values of these keys should be the old/new +# names of input file values or sections +REPLACEMENTS = [] + + +def fix_replacements(replacements, options_file): + """Change the names of options in options_file according to the list + of dicts replacements + + """ + for replacement in replacements: + try: + options_file.rename(replacement["old"], replacement["new"]) + except KeyError: + pass + except TypeError as e: + raise RuntimeError( + "Could not apply transformation: '{old}' -> '{new}' to file '{0}', due to error:" + "\n\t{1}".format(options_file.filename, e.args[0], **replacement) + ) from e + + +def apply_fixes(replacements, options_file): + """Apply all fixes in this module + """ + + modified = copy.deepcopy(options_file) + + fix_replacements(replacements, options_file) + + return modified + + +def yes_or_no(question): + """Convert user input from yes/no variations to True/False + + """ + while True: + reply = input(question + " [y/N] ").lower().strip() + if not reply or reply[0] == "n": + return False + if reply[0] == "y": + return True + + +def create_patch(filename, original, modified): + """Create a unified diff between original and modified + """ + + patch = "\n".join( + difflib.unified_diff( + original.splitlines(), + modified.splitlines(), + fromfile=filename, + tofile=filename, + lineterm="", + ) + ) + + return patch + + +def possibly_apply_patch(patch, options_file, quiet=False, force=False): + """Possibly apply patch to options_file. If force is True, applies the + patch without asking, overwriting any existing file. Otherwise, + ask for confirmation from stdin + + """ + if not quiet: + print("\n******************************************") + print("Changes to {}\n".format(options_file.filename)) + print(patch) + print("\n******************************************") + + if force: + make_change = True + else: + make_change = yes_or_no("Make changes to {}?".format(options_file.filename)) + if make_change: + options_file.write(overwrite=True) + return make_change + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=textwrap.dedent( + """\ + Fix input files for BOUT++ v5+ + + Please note that this will only fix input options in sections with + standard or default names. You may also need to fix options in custom + sections. + + Warning! Even with no fixes, there may still be changes as this script + will "canonicalise" the input files: + + * nested sections are moved to be under their parent section, while + preserving relative order + + * empty sections are removed + + * floating point numbers may have their format changed, although the + value will not change + + * consecutive blank lines will be reduced to a single blank line + + * whitespace around equals signs will be changed to exactly one space + + * trailing whitespace will be removed + + Files that change in this way will have the "canonicalisation" patch + presented first. If you choose not to apply this patch, the "upgrade + fixer" patch will still include it.""" + ), + ) + + parser.add_argument("files", action="store", nargs="+", help="Input files") + + force_patch_group = parser.add_mutually_exclusive_group() + force_patch_group.add_argument( + "--force", "-f", action="store_true", help="Make changes without asking" + ) + force_patch_group.add_argument( + "--patch-only", "-p", action="store_true", help="Print the patches and exit" + ) + + parser.add_argument( + "--quiet", "-q", action="store_true", help="Don't print patches" + ) + parser.add_argument( + "--accept-canonical", + "-c", + action="store_true", + help="Automatically accept the canonical patch", + ) + + args = parser.parse_args() + + warnings.simplefilter("ignore", AlwaysWarning) + + for filename in args.files: + with open(filename, "r") as f: + original_source = f.read() + + try: + original = BoutOptionsFile(filename) + except ValueError: + pass + + canonicalised_patch = create_patch(filename, original_source, str(original)) + if canonicalised_patch and not args.patch_only: + print( + "WARNING: original input file '{}' not in canonical form!".format( + filename + ) + ) + applied_patch = possibly_apply_patch( + canonicalised_patch, + original, + args.quiet, + args.force or args.accept_canonical, + ) + # Re-read input file + if applied_patch: + original_source = str(original) + + try: + modified = apply_fixes(REPLACEMENTS, original) + except RuntimeError as e: + print(e) + continue + patch = create_patch(filename, original_source, str(modified)) + + if args.patch_only: + print(patch) + continue + + if not patch: + if not args.quiet: + print("No changes to make to {}".format(filename)) + continue + + possibly_apply_patch(patch, modified, args.quiet, args.force) From b8836a6acbe48c9d8d875b556e74b0efee340d28 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 24 Feb 2020 17:09:47 +0000 Subject: [PATCH 14/82] Add input replacements for paralleltransform changes --- src/bout-v5-input-file-upgrader.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 70bd112..ac749d5 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -12,7 +12,11 @@ # This should be a list of dicts, each containing exactly two keys: # "old" and "new". The values of these keys should be the old/new # names of input file values or sections -REPLACEMENTS = [] +REPLACEMENTS = [ + {"old": "mesh:paralleltransform", "new": "mesh:paralleltransform:type"}, + {"old": "fci", "new": "mesh:paralleltransform"}, + {"old": "interpolation", "new": "mesh:paralleltransform:xzinterpolation"}, +] def fix_replacements(replacements, options_file): From 8dc28ddea86a720229b4da75e9eb3c1079ed02cc Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 26 Feb 2020 14:24:35 +0000 Subject: [PATCH 15/82] Fix typo in input file upgrader --- src/bout-v5-input-file-upgrader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index ac749d5..98a6878 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -42,7 +42,7 @@ def apply_fixes(replacements, options_file): modified = copy.deepcopy(options_file) - fix_replacements(replacements, options_file) + fix_replacements(replacements, modified) return modified From e4efa55eaa3beefb4a0f6d4fa0dbf6b6619ee8ea Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 26 Feb 2020 17:09:10 +0000 Subject: [PATCH 16/82] Add 4to5 macro updater script --- src/bout-v5-macro-upgrader.py | 216 ++++++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100755 src/bout-v5-macro-upgrader.py diff --git a/src/bout-v5-macro-upgrader.py b/src/bout-v5-macro-upgrader.py new file mode 100755 index 0000000..ab2c444 --- /dev/null +++ b/src/bout-v5-macro-upgrader.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 + +import argparse +import copy +import difflib +import re +import textwrap + +MACRO_REPLACEMENTS = [ + {"old": "REVISION", "new": "bout::version::revision", "header": "bout/version.hxx"}, + { + "old": "BOUT_VERSION_DOUBLE", + "new": "bout::version::as_double", + "header": "bout/version.hxx", + }, + { + "old": "BOUT_VERSION_STRING", + "new": "bout::version::full", + "header": "bout/version.hxx", + }, + # Next one is not technically a macro, but near enough + {"old": "BOUT_VERSION", "new": "bout::version::full", "header": "bout/version.hxx"}, +] + + +def fix_include_version_header(old, header, source): + """Make sure version.hxx header is included + """ + + # If header is already included, we can skip this fix + if ( + re.search( + r'^#\s*include.*(<|"){}(>|")'.format(header), source, flags=re.MULTILINE + ) + is not None + ): + return source + + # If the old macro isn't in the file, we can skip this fix + if re.search(r"\b{}\b".format(old), source) is None: + return source + + # Now we want to find a suitable place to stick the new include + # Good candidates are includes of BOUT++ headers + includes = [] + source_lines = source.splitlines() + for linenumber, line in enumerate(source_lines): + if re.match(r"^#\s*include.*bout/", line): + includes.append(linenumber) + if re.match(r"^#\s*include.*physicsmodel", line): + includes.append(linenumber) + + if includes: + last_include = includes[-1] + 1 + else: + # No suitable includes, so just stick at the top of the file + last_include = 0 + source_lines.insert(last_include, '#include "{}"'.format(header)) + + return "\n".join(source_lines) + + +def fix_ifdefs(old, source): + """Remove any #ifdef/#endif pairs that use the old macro + """ + source_lines = source.splitlines() + + in_ifdef = None + lines_to_pop = [] + for linenumber, line in enumerate(source_lines): + if_def = re.match(r"#\s*ifdef\s*(.*)", line) + endif = re.match(r"#\s*endif", line) + if not (if_def or endif): + continue + # Now we need to keep track of whether we're inside an + # interesting #ifdef, as they might be nested, and we want to + # find the matching #endif + if endif: + if in_ifdef is not None: + in_ifdef -= 1 + if in_ifdef == 0: + in_ifdef = None + lines_to_pop.append(linenumber) + continue + if if_def.group(1) == old: + in_ifdef = 1 + lines_to_pop.append(linenumber) + elif in_ifdef is not None: + in_ifdef += 1 + + # Go over the source lines in reverse so that we don't need to + # recompute indices + for line in reversed(lines_to_pop): + del source_lines[line] + + return "\n".join(source_lines) + + +def fix_replacement(old, new, source): + """Straight replacements + """ + return re.sub(r'([^"])\b{}\b([^"])'.format(old), r"\1{}\2".format(new), source) + + +def apply_fixes(replacements, source): + """Apply all fixes in this module + """ + modified = copy.deepcopy(source) + + for replacement in replacements: + modified = fix_include_version_header( + replacement["old"], replacement["header"], modified + ) + modified = fix_ifdefs(replacement["old"], modified) + modified = fix_replacement(replacement["old"], replacement["new"], modified) + + return modified + + +def yes_or_no(question): + """Convert user input from yes/no variations to True/False + + """ + while True: + reply = input(question + " [y/N] ").lower().strip() + if not reply or reply[0] == "n": + return False + if reply[0] == "y": + return True + + +def create_patch(filename, original, modified): + """Create a unified diff between original and modified + """ + + patch = "\n".join( + difflib.unified_diff( + original.splitlines(), + modified.splitlines(), + fromfile=filename, + tofile=filename, + lineterm="", + ) + ) + + return patch + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=textwrap.dedent( + """\ + Fix macro defines for BOUT++ v4 -> v5 + + Please note that this is only slightly better than dumb text replacement. It + will fix the following: + + * replacement of macros with variables + * inclusion of correct headers for new variables + * removal of #ifdef/#endif pairs that do simple checks for the old macro + + It will try not to replace quoted macro names. + + Please check the diff output carefully! + """ + ), + ) + + parser.add_argument("files", action="store", nargs="+", help="Input files") + parser.add_argument( + "--force", "-f", action="store_true", help="Make changes without asking" + ) + parser.add_argument( + "--quiet", "-q", action="store_true", help="Don't print patches" + ) + parser.add_argument( + "--patch-only", "-p", action="store_true", help="Print the patches and exit" + ) + + args = parser.parse_args() + + if args.force and args.patch_only: + raise ValueError("Incompatible options: --force and --patch") + + for filename in args.files: + with open(filename, "r") as f: + contents = f.read() + original = copy.deepcopy(contents) + + modified = apply_fixes(MACRO_REPLACEMENTS, contents) + patch = create_patch(filename, original, modified) + + if args.patch_only: + print(patch) + continue + + if not patch: + if not args.quiet: + print("No changes to make to {}".format(filename)) + continue + + if not args.quiet: + print("\n******************************************") + print("Changes to {}\n".format(filename)) + print(patch) + print("\n******************************************") + + if args.force: + make_change = True + else: + make_change = yes_or_no("Make changes to {}?".format(filename)) + + if make_change: + with open(filename, "w") as f: + f.write(modified) From af193d585db395d50357988207fc093c1e39609c Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 26 Feb 2020 17:37:40 +0000 Subject: [PATCH 17/82] Move bout::version::revision into standalone file Makefile-build system now also configures this file to include the commit from configure-time, so that bout::version::revision will always have a meaningful value even if -DBOUT_REVISION is not passed on command line --- src/bout-v5-macro-upgrader.py | 39 ++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/bout-v5-macro-upgrader.py b/src/bout-v5-macro-upgrader.py index ab2c444..a9770c3 100755 --- a/src/bout-v5-macro-upgrader.py +++ b/src/bout-v5-macro-upgrader.py @@ -6,35 +6,46 @@ import re import textwrap +# List of macros, their replacements and what header to find them +# in. Each element should be a dict with "old", "new" and "headers" +# keys, with "old" and "new" values being strings, and "headers" being a +# list of strings MACRO_REPLACEMENTS = [ - {"old": "REVISION", "new": "bout::version::revision", "header": "bout/version.hxx"}, + { + "old": "REVISION", + "new": "bout::version::revision", + "headers": ["bout/revision.hxx"], + }, { "old": "BOUT_VERSION_DOUBLE", "new": "bout::version::as_double", - "header": "bout/version.hxx", + "headers": ["bout/version.hxx", "bout.hxx"], }, { "old": "BOUT_VERSION_STRING", "new": "bout::version::full", - "header": "bout/version.hxx", + "headers": ["bout/version.hxx", "bout.hxx"], }, # Next one is not technically a macro, but near enough - {"old": "BOUT_VERSION", "new": "bout::version::full", "header": "bout/version.hxx"}, + { + "old": "BOUT_VERSION", + "new": "bout::version::full", + "headers": ["bout/version.hxx", "bout.hxx"], + }, ] -def fix_include_version_header(old, header, source): +def fix_include_version_header(old, headers, source): """Make sure version.hxx header is included """ # If header is already included, we can skip this fix - if ( - re.search( - r'^#\s*include.*(<|"){}(>|")'.format(header), source, flags=re.MULTILINE - ) - is not None - ): - return source + for header in headers: + if ( + re.search(r'^#\s*include.*(<|"){}(>|")'.format(header), source, flags=re.M) + is not None + ): + return source # If the old macro isn't in the file, we can skip this fix if re.search(r"\b{}\b".format(old), source) is None: @@ -55,7 +66,7 @@ def fix_include_version_header(old, header, source): else: # No suitable includes, so just stick at the top of the file last_include = 0 - source_lines.insert(last_include, '#include "{}"'.format(header)) + source_lines.insert(last_include, '#include "{}"'.format(headers[0])) return "\n".join(source_lines) @@ -109,7 +120,7 @@ def apply_fixes(replacements, source): for replacement in replacements: modified = fix_include_version_header( - replacement["old"], replacement["header"], modified + replacement["old"], replacement["headers"], modified ) modified = fix_ifdefs(replacement["old"], modified) modified = fix_replacement(replacement["old"], replacement["new"], modified) From 720ac0d1ed4bc1eb3dd6b5955c4048c810a5cc37 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 28 Feb 2020 13:41:17 +0000 Subject: [PATCH 18/82] Write comments using '#' in BoutOptions Canonical form for v5 --- src/bout-v5-input-file-upgrader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 98a6878..1d30c83 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -125,6 +125,8 @@ def possibly_apply_patch(patch, options_file, quiet=False, force=False): * trailing whitespace will be removed + * comments will always use '#' + Files that change in this way will have the "canonicalisation" patch presented first. If you choose not to apply this patch, the "upgrade fixer" patch will still include it.""" From cec0075098c3db5ae20de832c995c23743f4be55 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 2 Mar 2020 15:18:44 +0000 Subject: [PATCH 19/82] Fix uncompiled parts of #ifdef blocks in macro upgrader --- src/bout-v5-macro-upgrader.py | 58 ++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/src/bout-v5-macro-upgrader.py b/src/bout-v5-macro-upgrader.py index a9770c3..174becd 100755 --- a/src/bout-v5-macro-upgrader.py +++ b/src/bout-v5-macro-upgrader.py @@ -72,39 +72,66 @@ def fix_include_version_header(old, headers, source): def fix_ifdefs(old, source): - """Remove any #ifdef/#endif pairs that use the old macro + """Remove any code inside #ifdef/#ifndef blocks that would now not be compiled + """ source_lines = source.splitlines() + # Something to keep track of nested sections in_ifdef = None - lines_to_pop = [] + # List of (#ifdef or #ifndef, dict of start/else/end lines) + macro_blocks = [] for linenumber, line in enumerate(source_lines): - if_def = re.match(r"#\s*ifdef\s*(.*)", line) + if_def = re.match(r"#\s*(ifn?def)\s*(.*)", line) + else_block = re.match(r"#\s*else", line) endif = re.match(r"#\s*endif", line) - if not (if_def or endif): + if not (if_def or else_block or endif): continue # Now we need to keep track of whether we're inside an - # interesting #ifdef, as they might be nested, and we want to - # find the matching #endif + # interesting #ifdef/ifndef, as they might be nested, and we + # want to find the matching #endif and #else if endif: if in_ifdef is not None: in_ifdef -= 1 if in_ifdef == 0: in_ifdef = None - lines_to_pop.append(linenumber) + macro_blocks[-1]["end"] = linenumber + continue + if else_block: + if in_ifdef == 1: + macro_blocks[-1]["else"] = linenumber continue - if if_def.group(1) == old: + if if_def.group(2) == old: in_ifdef = 1 - lines_to_pop.append(linenumber) + macro_blocks.append({"start": linenumber, "if_def_type": if_def.group(1)}) elif in_ifdef is not None: in_ifdef += 1 - # Go over the source lines in reverse so that we don't need to - # recompute indices - for line in reversed(lines_to_pop): - del source_lines[line] + if macro_blocks == []: + return source - return "\n".join(source_lines) + # Get all of the lines to be removed + lines_to_remove = set() + for block in macro_blocks: + lines_to_remove |= set(block.values()) + if block["if_def_type"] == "ifdef": + if "else" in block: + # Delete the #else block for #ifdef + lines_to_remove |= set(range(block["else"], block["end"])) + else: + # Keep the #else block for #ifndef if there is one, otherwise remove the + # whole block + lines_to_remove |= set( + range(block["start"], block.get("else", block["end"])) + ) + + # Apparently this is actually the best way of removing a bunch of (possibly) + # non-contiguous indices + modified_lines = [ + line for num, line in enumerate(source_lines) if num not in lines_to_remove + ] + + return "\n".join(modified_lines) def fix_replacement(old, new, source): @@ -169,7 +196,8 @@ def create_patch(filename, original, modified): * replacement of macros with variables * inclusion of correct headers for new variables - * removal of #ifdef/#endif pairs that do simple checks for the old macro + * removal of #if(n)def/#endif blocks that do simple checks for the old + macro, keeping the appriopriate part It will try not to replace quoted macro names. From 435e885232a8c1a8282158906b35dc532bb0c5eb Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 4 May 2020 17:24:54 +0100 Subject: [PATCH 20/82] Add renamed feature macros to bout-v5-macro-upgrader --- src/bout-v5-macro-upgrader.py | 152 +++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 4 deletions(-) diff --git a/src/bout-v5-macro-upgrader.py b/src/bout-v5-macro-upgrader.py index 174becd..0a383b4 100755 --- a/src/bout-v5-macro-upgrader.py +++ b/src/bout-v5-macro-upgrader.py @@ -15,22 +15,149 @@ "old": "REVISION", "new": "bout::version::revision", "headers": ["bout/revision.hxx"], + "macro": False, + "always_defined": True, }, { "old": "BOUT_VERSION_DOUBLE", "new": "bout::version::as_double", "headers": ["bout/version.hxx", "bout.hxx"], + "macro": False, + "always_defined": True, }, { "old": "BOUT_VERSION_STRING", "new": "bout::version::full", "headers": ["bout/version.hxx", "bout.hxx"], + "macro": False, + "always_defined": True, }, # Next one is not technically a macro, but near enough { "old": "BOUT_VERSION", "new": "bout::version::full", "headers": ["bout/version.hxx", "bout.hxx"], + "macro": False, + "always_defined": True, + }, + { + "old": "BACKTRACE", + "new": "BOUT_USE_BACKTRACE", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "HAS_ARKODE", + "new": "BOUT_HAS_ARKODE", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "HAS_CVODE", + "new": "BOUT_HAS_CVODE", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "HAS_HDF5", + "new": "BOUT_HAS_HDF5", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "HAS_IDA", + "new": "BOUT_HAS_IDA", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "HAS_LAPACK", + "new": "BOUT_HAS_LAPACK", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "LAPACK", + "new": "BOUT_HAS_LAPACK", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "HAS_NETCDF", + "new": "BOUT_HAS_NETCDF", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "HAS_PETSC", + "new": "BOUT_HAS_PETSC", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "HAS_PRETTY_FUNCTION", + "new": "BOUT_HAS_PRETTY_FUNCTION", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "HAS_PVODE", + "new": "BOUT_HAS_PVODE", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "TRACK", + "new": "BOUT_USE_TRACK", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "NCDF4", + "new": "BOUT_HAS_NETCDF", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "HDF5", + "new": "BOUT_HAS_HDF5", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "DEBUG_ENABLED", + "new": "BOUT_USE_OUTPUT_DEBUG", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "BOUT_FPE", + "new": "BOUT_USE_SIGFPE", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, + { + "old": "LOGCOLOR", + "new": "BOUT_USE_COLOR", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, }, ] @@ -39,6 +166,9 @@ def fix_include_version_header(old, headers, source): """Make sure version.hxx header is included """ + if not isinstance(headers, list): + headers = [headers] + # If header is already included, we can skip this fix for header in headers: if ( @@ -134,6 +264,13 @@ def fix_ifdefs(old, source): return "\n".join(modified_lines) +def fix_always_defined_macros(old, new, source): + """Fix '#ifdef's that should become plain '#if' + """ + new_source = re.sub(r"#ifdef\s+{}".format(old), r"#if {}".format(new), source) + return re.sub(r"#ifndef\s+{}".format(old), r"#if !{}".format(new), new_source) + + def fix_replacement(old, new, source): """Straight replacements """ @@ -149,7 +286,12 @@ def apply_fixes(replacements, source): modified = fix_include_version_header( replacement["old"], replacement["headers"], modified ) - modified = fix_ifdefs(replacement["old"], modified) + if replacement["macro"] and replacement["always_defined"]: + modified = fix_always_defined_macros( + replacement["old"], replacement["new"], modified + ) + elif replacement["always_defined"]: + modified = fix_ifdefs(replacement["old"], modified) modified = fix_replacement(replacement["old"], replacement["new"], modified) return modified @@ -194,12 +336,14 @@ def create_patch(filename, original, modified): Please note that this is only slightly better than dumb text replacement. It will fix the following: - * replacement of macros with variables + * replacement of macros with variables or new names * inclusion of correct headers for new variables * removal of #if(n)def/#endif blocks that do simple checks for the old - macro, keeping the appriopriate part + macro, keeping the appriopriate part, if replaced by a variable + * change '#if(n)def' for '#if (!)' if the replacment is always defined - It will try not to replace quoted macro names. + It will try not to replace quoted macro names, but may + still replace them in strings or comments. Please check the diff output carefully! """ From b0f6e5078372374d32254e296ce5eafffbfc7116 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 5 May 2020 09:50:26 +0100 Subject: [PATCH 21/82] Fix a couple of uses of renamed macros --- src/bout-v5-macro-upgrader.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/bout-v5-macro-upgrader.py b/src/bout-v5-macro-upgrader.py index 0a383b4..7f79d49 100755 --- a/src/bout-v5-macro-upgrader.py +++ b/src/bout-v5-macro-upgrader.py @@ -159,6 +159,13 @@ "macro": True, "always_defined": True, }, + { + "old": "OPENMP_SCHEDULE", + "new": "BOUT_OPENMP_SCHEDULE", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, ] From 2ee94f839122a7a99bd129f1c60f07a6ae359c0e Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 23 Jul 2020 14:40:46 +0100 Subject: [PATCH 22/82] Consider underscores as word boundaries when updating macros --- src/bout-v5-macro-upgrader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bout-v5-macro-upgrader.py b/src/bout-v5-macro-upgrader.py index 7f79d49..5cd2f83 100755 --- a/src/bout-v5-macro-upgrader.py +++ b/src/bout-v5-macro-upgrader.py @@ -274,14 +274,14 @@ def fix_ifdefs(old, source): def fix_always_defined_macros(old, new, source): """Fix '#ifdef's that should become plain '#if' """ - new_source = re.sub(r"#ifdef\s+{}".format(old), r"#if {}".format(new), source) - return re.sub(r"#ifndef\s+{}".format(old), r"#if !{}".format(new), new_source) + new_source = re.sub(r"#ifdef\s+{}\b".format(old), r"#if {}".format(new), source) + return re.sub(r"#ifndef\s+{}\b".format(old), r"#if !{}".format(new), new_source) def fix_replacement(old, new, source): """Straight replacements """ - return re.sub(r'([^"])\b{}\b([^"])'.format(old), r"\1{}\2".format(new), source) + return re.sub(r'([^"_])\b{}\b([^"_])'.format(old), r"\1{}\2".format(new), source) def apply_fixes(replacements, source): From e5764ed654a072c1558d00e44b4b643392fea8ea Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 23 Jul 2020 14:45:48 +0100 Subject: [PATCH 23/82] Fix bug with default file format when using legacy netcdf --- src/bout-v5-macro-upgrader.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/bout-v5-macro-upgrader.py b/src/bout-v5-macro-upgrader.py index 5cd2f83..16d3b3e 100755 --- a/src/bout-v5-macro-upgrader.py +++ b/src/bout-v5-macro-upgrader.py @@ -131,6 +131,13 @@ "macro": True, "always_defined": True, }, + { + "old": "NCDF", + "new": "BOUT_HAS_LEGACY_NETCDF", + "headers": "bout/build_config.hxx", + "macro": True, + "always_defined": True, + }, { "old": "HDF5", "new": "BOUT_HAS_HDF5", From dd0b50548527ec166c6b386397f7ab72facefd24 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 4 Sep 2020 15:35:11 +0100 Subject: [PATCH 24/82] Add fixer for change of factory create method name --- src/bout-v5-factory-upgrader.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/bout-v5-factory-upgrader.py b/src/bout-v5-factory-upgrader.py index be33cc4..91a3804 100755 --- a/src/bout-v5-factory-upgrader.py +++ b/src/bout-v5-factory-upgrader.py @@ -16,7 +16,8 @@ "InvertPar": { "factory_name": "InvertPar", "type_name": "InvertPar", - "create_method": "Create", + "create_method": "create", + "old_create_method": "Create", }, "Mesh": {"factory_name": "Mesh", "type_name": "Mesh", "create_method": "Create"}, "Laplacian": { @@ -170,6 +171,21 @@ def fix_deletions(variables, source): return source +def fix_create_method(factory, source): + """Fix change of name of factory `create` method + + """ + + if "old_create_method" not in factory: + return source + return re.sub( + r"({factory_name})\s*::\s*{old_create_method}\b".format(**factory), + r"\1::{create_method}".format(**factory), + source, + flags=re.VERBOSE, + ) + + def yes_or_no(question): """Convert user input from yes/no variations to True/False @@ -197,6 +213,7 @@ def apply_fixes(factories, source, all_declarations=False): modified = source for factory in factories.values(): + modified = fix_create_method(factory, modified) variables = find_factory_calls(factory, modified) if all_declarations: variables = variables + find_type_pointers(factory, modified) From a58540d830c10884fa6519bcc151470be5fe829f Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 7 Sep 2020 11:26:29 +0100 Subject: [PATCH 25/82] Add warning in upgrader if factory arguments have changed --- src/bout-v5-factory-upgrader.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/bout-v5-factory-upgrader.py b/src/bout-v5-factory-upgrader.py index 91a3804..c245f40 100755 --- a/src/bout-v5-factory-upgrader.py +++ b/src/bout-v5-factory-upgrader.py @@ -18,6 +18,7 @@ "type_name": "InvertPar", "create_method": "create", "old_create_method": "Create", + "arguments_changed": True, }, "Mesh": {"factory_name": "Mesh", "type_name": "Mesh", "create_method": "Create"}, "Laplacian": { @@ -178,11 +179,22 @@ def fix_create_method(factory, source): if "old_create_method" not in factory: return source + old_create_pattern = re.compile( + r"({factory_name})\s*::\s*{old_create_method}\b".format(**factory) + ) + if not old_create_pattern.findall(source): + return source + + if factory.get("arguments_changed", False): + print( + "**WARNING** Arguments of {factory_name}::{create_method} have changed, and your current arguments may not work." + " Please consult the documentation for the new arguments.".format(**factory) + ) + return re.sub( r"({factory_name})\s*::\s*{old_create_method}\b".format(**factory), r"\1::{create_method}".format(**factory), source, - flags=re.VERBOSE, ) From 5f322fdfab3722031af2c0137ff1376d585e8b41 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Mon, 2 Nov 2020 22:09:49 +0000 Subject: [PATCH 26/82] Option to update values in bout-v5-input-file-upgrader.py --- src/bout-v5-input-file-upgrader.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 1d30c83..c74dc39 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -9,9 +9,10 @@ from boutdata.data import BoutOptionsFile from boututils.boutwarnings import AlwaysWarning -# This should be a list of dicts, each containing exactly two keys: -# "old" and "new". The values of these keys should be the old/new -# names of input file values or sections +# This should be a list of dicts, each containing "old", "new" and optionally "values". +# The values of "old"/"new" keys should be the old/new names of input file values or +# sections. The value of "values" is a dict containing replacements for values of the +# option. REPLACEMENTS = [ {"old": "mesh:paralleltransform", "new": "mesh:paralleltransform:type"}, {"old": "fci", "new": "mesh:paralleltransform"}, @@ -34,6 +35,12 @@ def fix_replacements(replacements, options_file): "Could not apply transformation: '{old}' -> '{new}' to file '{0}', due to error:" "\n\t{1}".format(options_file.filename, e.args[0], **replacement) ) from e + else: + if "values" in replacement: + old_value = options_file[replacement["new"]] + options_file[replacement["new"]] = replacement["values"].get( + old_value, old_value + ) def apply_fixes(replacements, options_file): From d24b39134f1c7cfdf4b05611477887664cb443ce Mon Sep 17 00:00:00 2001 From: John Omotani Date: Mon, 2 Nov 2020 22:11:06 +0000 Subject: [PATCH 27/82] Deprecate fft_measure option Only keep fft_measurement_flag. --- src/bout-v5-input-file-upgrader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index c74dc39..aadf708 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -17,6 +17,8 @@ {"old": "mesh:paralleltransform", "new": "mesh:paralleltransform:type"}, {"old": "fci", "new": "mesh:paralleltransform"}, {"old": "interpolation", "new": "mesh:paralleltransform:xzinterpolation"}, + {"old": "fft:fft_measure", "new": "fft:fft_measurement_flag", + "values": {"false": "estimate", "true": "measure"}} ] From 4bf348485466cf838479be60f9310d9258e903f8 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 3 Nov 2020 13:21:43 +0000 Subject: [PATCH 28/82] Handle bool-like expressions in the same way as BOUT++ --- src/bout-v5-input-file-upgrader.py | 56 ++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index aadf708..8414e04 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -12,16 +12,34 @@ # This should be a list of dicts, each containing "old", "new" and optionally "values". # The values of "old"/"new" keys should be the old/new names of input file values or # sections. The value of "values" is a dict containing replacements for values of the -# option. +# option. "type" optionally specifies the type of the old value of the option; for +# example this is needed for special handling of boolean values. REPLACEMENTS = [ {"old": "mesh:paralleltransform", "new": "mesh:paralleltransform:type"}, {"old": "fci", "new": "mesh:paralleltransform"}, {"old": "interpolation", "new": "mesh:paralleltransform:xzinterpolation"}, {"old": "fft:fft_measure", "new": "fft:fft_measurement_flag", - "values": {"false": "estimate", "true": "measure"}} + "type": bool, "values": {False: "estimate", True: "measure"}} ] +def parse_bool(bool_expression): + try: + bool_expression_lower = bool_expression.lower() + except AttributeError: + # bool_expression was not a str: no need to lower + bool_expression_lower = bool_expression + + if bool_expression_lower in ["true", "y", "t", 1, True]: + return True + elif bool_expression_lower in ["false", "n", "f", 0, False]: + return False + else: + raise RuntimeError( + f"Expected boolean option. Could not parse {bool_expression}" + ) + + def fix_replacements(replacements, options_file): """Change the names of options in options_file according to the list of dicts replacements @@ -38,11 +56,35 @@ def fix_replacements(replacements, options_file): "\n\t{1}".format(options_file.filename, e.args[0], **replacement) ) from e else: - if "values" in replacement: - old_value = options_file[replacement["new"]] - options_file[replacement["new"]] = replacement["values"].get( - old_value, old_value - ) + if "type" in replacement: + # Special handling for certain types, replicating what BOUT++ does + if replacement["type"] is bool: + # The original value must be something that BOUT++ recognises as a + # bool. + # replacement["values"] must contain both True and False keys. + old_value = parse_bool(options_file[replacement["new"]]) + options_file[replacement["new"]] = replacement["values"][old_value] + else: + raise ValueError( + f"Error in REPLACEMENTS: type {replacement['type']} is not handled" + ) + else: + # Option values are just a string + if "values" in replacement: + old_value = options_file[replacement["new"]] + try: + old_value = old_value.lower() + except AttributeError: + # old_value was not a str, so no need to convert to lower case + pass + + try: + options_file[replacement["new"]] = replacement["values"][ + old_value + ] + except KeyError: + # No replacement given for this value: keep the old one + pass def apply_fixes(replacements, options_file): From 363e1c3efdab269377b3da0bcd74202dd06a1f18 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 8 Jun 2020 17:03:52 +0100 Subject: [PATCH 29/82] Add tool to upgrade legacy models --- src/bout-v5-physics-model-upgrader.py | 210 ++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100755 src/bout-v5-physics-model-upgrader.py diff --git a/src/bout-v5-physics-model-upgrader.py b/src/bout-v5-physics-model-upgrader.py new file mode 100755 index 0000000..727c3b1 --- /dev/null +++ b/src/bout-v5-physics-model-upgrader.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 + +import argparse +import copy +import difflib +import pathlib +import re +import textwrap + + +PHYSICS_MODEL_INCLUDE = '#include "bout/physicsmodel.hxx"' + +PHYSICS_MODEL_SKELETON = """ +class {name} : public PhysicsModel {{ +protected: + int init(bool{restarting}) override; + int rhs(BoutReal{time}) override; +}}; +""" + +BOUTMAIN = "\n\nBOUTMAIN({})\n" + +# Regular expression for a PhysicsModel +PHYSICS_MODEL_RE = re.compile(r":\s*public\s*PhysicsModel") + +# Regular expressions for a legacy physics model +LEGACY_MODEL_RE_TEMPLATE = r"""int\s+physics_{}\s*\({} + (\s+ # Require spaces only if the argument is named + (?PUNUSED\()? # Possible UNUSED macro + [a-zA-Z_0-9]* # Argument name + (?(unused)\)) # If UNUSED macro was present, we need an extra closing bracket + )? + \)""" + +LEGACY_MODEL_INIT_RE = re.compile( + LEGACY_MODEL_RE_TEMPLATE.format("init", "bool"), re.VERBOSE | re.MULTILINE +) +LEGACY_MODEL_RUN_RE = re.compile( + LEGACY_MODEL_RE_TEMPLATE.format("run", "BoutReal"), re.VERBOSE | re.MULTILINE +) + +LEGACY_MODEL_INCLUDE_RE = re.compile( + r'^#\s*include.*(<|")boutmain.hxx(>|")', re.MULTILINE +) + + +def is_legacy_model(source): + """Return true if the source is a legacy physics model + + """ + return LEGACY_MODEL_INCLUDE_RE.search(source) is not None + + +def find_last_include(source_lines): + """Return the line number after the last #include (or 0 if no includes) + """ + for number, line in enumerate(reversed(source_lines)): + if line.startswith("#include"): + return len(source_lines) - number + return 0 + + +def convert_legacy_model(source, name): + """Convert a legacy physics model to a PhysicsModel + """ + + if not is_legacy_model(source): + return source + + # Replace legacy header + replaced_header = LEGACY_MODEL_INCLUDE_RE.sub( + r"#include \1bout/physicsmodel.hxx\2", source + ) + + source_lines = replaced_header.splitlines() + last_include = find_last_include(source_lines) + + init_function = LEGACY_MODEL_INIT_RE.search(source) + if init_function is not None: + restarting = init_function.group(1) + else: + restarting = "" + + run_function = LEGACY_MODEL_RUN_RE.search(source) + if run_function is not None: + time = run_function.group(1) + else: + time = "" + + source_lines.insert( + last_include, + PHYSICS_MODEL_SKELETON.format(name=name, restarting=restarting, time=time), + ) + + added_class = "\n".join(source_lines) + + fixed_init = LEGACY_MODEL_INIT_RE.sub( + r"int {}::init(bool\1)".format(name), added_class + ) + fixed_run = LEGACY_MODEL_RUN_RE.sub( + r"int {}::rhs(BoutReal\1)".format(name), fixed_init + ) + + added_main = fixed_run + BOUTMAIN.format(name) + return added_main + + +def yes_or_no(question): + """Convert user input from yes/no variations to True/False + + """ + while True: + reply = input(question + " [y/N] ").lower().strip() + if not reply or reply[0] == "n": + return False + if reply[0] == "y": + return True + + +def create_patch(filename, original, modified): + """Create a unified diff between original and modified + """ + + patch = "\n".join( + difflib.unified_diff( + original.splitlines(), + modified.splitlines(), + fromfile=filename, + tofile=filename, + lineterm="", + ) + ) + + return patch + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=textwrap.dedent( + """\ + Upgrade legacy physics models to use the PhysicsModel class + + This will do the bare minimum required to compile, and + won't make global objects (like Field3Ds) members of the + new class, or free functions other than + `physics_init`/`physics_run` methods of the new class. + + By default, this will use the file name stripped of file + extensions as the name of the new class. Use '--name=' to give a different name. + """ + ), + ) + + parser.add_argument("files", action="store", nargs="+", help="Files to fix") + parser.add_argument( + "--force", "-f", action="store_true", help="Make changes without asking" + ) + parser.add_argument( + "--quiet", "-q", action="store_true", help="Don't print patches" + ) + parser.add_argument( + "--patch-only", "-p", action="store_true", help="Print the patches and exist" + ) + parser.add_argument( + "--name", + "-n", + action="store", + nargs="?", + type=str, + help="Name for new PhysicsModel class, default is from filename", + ) + + args = parser.parse_args() + + if args.force and args.patch_only: + raise ValueError("Incompatible options: --force and --patch") + + for filename in args.files: + with open(filename, "r") as f: + contents = f.read() + + if not is_legacy_model(contents): + if not args.quiet: + print("No changes to make to {}".format(filename)) + continue + + original = copy.deepcopy(contents) + + new_name = args.name or pathlib.Path(filename).stem.capitalize() + + modified = convert_legacy_model(original, new_name) + patch = create_patch(filename, original, modified) + + if args.patch_only: + print(patch) + continue + + if not args.quiet: + print("\n******************************************") + print("Changes to {}\n".format(filename)) + print(patch) + print("\n******************************************") + + make_change = args.force or yes_or_no("Make changes to {}?".format(filename)) + + if make_change: + with open(filename, "w") as f: + f.write(modified) From 8c8851b6238946a335d482690cc2d10a4382377d Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 9 Jun 2020 11:16:56 +0100 Subject: [PATCH 30/82] Catch (some) invalid names in physics model upgrader --- src/bout-v5-physics-model-upgrader.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/bout-v5-physics-model-upgrader.py b/src/bout-v5-physics-model-upgrader.py index 727c3b1..19986af 100755 --- a/src/bout-v5-physics-model-upgrader.py +++ b/src/bout-v5-physics-model-upgrader.py @@ -188,7 +188,14 @@ def create_patch(filename, original, modified): original = copy.deepcopy(contents) - new_name = args.name or pathlib.Path(filename).stem.capitalize() + new_name = args.name or pathlib.Path(filename).stem.capitalize().replace( + " ", "_" + ) + + if re.match(r"^[0-9]+.*", new_name): + raise ValueError( + f"Invalid name: '{new_name}'. Use --name to specify a valid C++ identifier" + ) modified = convert_legacy_model(original, new_name) patch = create_patch(filename, original, modified) From 12c7d9608b73c6881a60749f7956424f90652289 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 9 Jun 2020 11:17:20 +0100 Subject: [PATCH 31/82] Fix deprecated bout_solve/bout_constrain calls --- src/bout-v5-physics-model-upgrader.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/bout-v5-physics-model-upgrader.py b/src/bout-v5-physics-model-upgrader.py index 19986af..6a33993 100755 --- a/src/bout-v5-physics-model-upgrader.py +++ b/src/bout-v5-physics-model-upgrader.py @@ -43,6 +43,9 @@ class {name} : public PhysicsModel {{ r'^#\s*include.*(<|")boutmain.hxx(>|")', re.MULTILINE ) +BOUT_CONSTRAIN_RE = re.compile(r"bout_constrain\(([^,)]+,\s*[^,)]+,\s*[^,)]+)\)") +BOUT_SOLVE_RE = re.compile(r"bout_solve\(([^,)]+,\s*[^,)]+)\)") + def is_legacy_model(source): """Return true if the source is a legacy physics model @@ -101,7 +104,10 @@ def convert_legacy_model(source, name): r"int {}::rhs(BoutReal\1)".format(name), fixed_init ) - added_main = fixed_run + BOUTMAIN.format(name) + fixed_constraint = BOUT_CONSTRAIN_RE.sub(r"solver->constraint(\1)", fixed_run) + fixed_solve = BOUT_CONSTRAIN_RE.sub(r"solver->add(\1)", fixed_constraint) + + added_main = fixed_solve + BOUTMAIN.format(name) return added_main From 137ba7c2b1adb976c293a94fd85cb4a20c4b8731 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 11 Jun 2020 12:03:36 +0100 Subject: [PATCH 32/82] Allows physics model upgrader to fix split operator models --- src/bout-v5-physics-model-upgrader.py | 140 ++++++++++++++++++-------- 1 file changed, 98 insertions(+), 42 deletions(-) diff --git a/src/bout-v5-physics-model-upgrader.py b/src/bout-v5-physics-model-upgrader.py index 6a33993..4a809aa 100755 --- a/src/bout-v5-physics-model-upgrader.py +++ b/src/bout-v5-physics-model-upgrader.py @@ -6,6 +6,7 @@ import pathlib import re import textwrap +import warnings PHYSICS_MODEL_INCLUDE = '#include "bout/physicsmodel.hxx"' @@ -13,18 +14,19 @@ PHYSICS_MODEL_SKELETON = """ class {name} : public PhysicsModel {{ protected: - int init(bool{restarting}) override; - int rhs(BoutReal{time}) override; + {methods} }}; """ +PHYSICS_MODEL_RHS_SKELETON = "int {function}({arg_type}{time}) override;" + BOUTMAIN = "\n\nBOUTMAIN({})\n" # Regular expression for a PhysicsModel PHYSICS_MODEL_RE = re.compile(r":\s*public\s*PhysicsModel") # Regular expressions for a legacy physics model -LEGACY_MODEL_RE_TEMPLATE = r"""int\s+physics_{}\s*\({} +LEGACY_MODEL_RE_TEMPLATE = r"""int\s+{}\s*\({} (\s+ # Require spaces only if the argument is named (?PUNUSED\()? # Possible UNUSED macro [a-zA-Z_0-9]* # Argument name @@ -32,19 +34,25 @@ class {name} : public PhysicsModel {{ )? \)""" -LEGACY_MODEL_INIT_RE = re.compile( - LEGACY_MODEL_RE_TEMPLATE.format("init", "bool"), re.VERBOSE | re.MULTILINE -) -LEGACY_MODEL_RUN_RE = re.compile( - LEGACY_MODEL_RE_TEMPLATE.format("run", "BoutReal"), re.VERBOSE | re.MULTILINE -) - LEGACY_MODEL_INCLUDE_RE = re.compile( r'^#\s*include.*(<|")boutmain.hxx(>|")', re.MULTILINE ) -BOUT_CONSTRAIN_RE = re.compile(r"bout_constrain\(([^,)]+,\s*[^,)]+,\s*[^,)]+)\)") -BOUT_SOLVE_RE = re.compile(r"bout_solve\(([^,)]+,\s*[^,)]+)\)") +SPLIT_OPERATOR_RE = re.compile( + r"solver\s*->\s*setSplitOperator\(([a-zA-Z0-9_]+),\s*([a-zA-Z0-9_]+)\s*\)" +) + + +def has_split_operator(source): + """Return the names of the split operator functions if set, otherwise False + + """ + + match = SPLIT_OPERATOR_RE.search(source) + if not match: + return False + + return match.group(1), match.group(2) def is_legacy_model(source): @@ -63,6 +71,50 @@ def find_last_include(source_lines): return 0 +def fix_model_operator(source, model_name, operator_name, operator_type, new_name): + """Fix any definitions of the operator, and return the new declaration + + May modify source + """ + + operator_re = re.compile( + LEGACY_MODEL_RE_TEMPLATE.format(operator_name, operator_type), + re.VERBOSE | re.MULTILINE, + ) + + matches = list(operator_re.finditer(source)) + if matches == []: + warnings.warn( + f"Could not find {operator_name}; is it defined in another file? If so, you will need to fix it manually" + ) + return source, False + + if len(matches) > 1: + source = re.sub( + LEGACY_MODEL_RE_TEMPLATE.format(operator_name, operator_type) + r"\s*;", + "", + source, + flags=re.VERBOSE | re.MULTILINE, + ) + + arg_name = operator_re.search(source).group(1) + else: + arg_name = matches[0].group(1) + + # Fix definition and any existing declarations + modified = operator_re.sub( + fr"int {model_name}::{new_name}({operator_type}\1)", source + ) + + # Create the declaration + return ( + modified, + PHYSICS_MODEL_RHS_SKELETON.format( + function=new_name, arg_type=operator_type, time=arg_name + ), + ) + + def convert_legacy_model(source, name): """Convert a legacy physics model to a PhysicsModel """ @@ -71,44 +123,48 @@ def convert_legacy_model(source, name): return source # Replace legacy header - replaced_header = LEGACY_MODEL_INCLUDE_RE.sub( - r"#include \1bout/physicsmodel.hxx\2", source - ) + source = LEGACY_MODEL_INCLUDE_RE.sub(r"#include \1bout/physicsmodel.hxx\2", source) - source_lines = replaced_header.splitlines() - last_include = find_last_include(source_lines) + method_decls = [] - init_function = LEGACY_MODEL_INIT_RE.search(source) - if init_function is not None: - restarting = init_function.group(1) - else: - restarting = "" + source, decl = fix_model_operator(source, name, "physics_init", "bool", "init") + if decl: + method_decls.append(decl) - run_function = LEGACY_MODEL_RUN_RE.search(source) - if run_function is not None: - time = run_function.group(1) - else: - time = "" + split_operators = has_split_operator(source) + if split_operators: + source = SPLIT_OPERATOR_RE.sub(r"setSplitOperator(true)", source) - source_lines.insert( - last_include, - PHYSICS_MODEL_SKELETON.format(name=name, restarting=restarting, time=time), - ) + convective, diffusive = split_operators + # Fix the free functions + source, decl = fix_model_operator( + source, name, convective, "BoutReal", "convective" + ) + if decl: + method_decls.append(decl) + source, decl = fix_model_operator( + source, name, diffusive, "BoutReal", "diffusive" + ) + if decl: + method_decls.append(decl) + else: + # Fix the rhs free function + source, decl = fix_model_operator( + source, name, "physics_run", "BoutReal", "rhs" + ) + if decl: + method_decls.append(decl) - added_class = "\n".join(source_lines) + source_lines = source.splitlines() + last_include = find_last_include(source_lines) - fixed_init = LEGACY_MODEL_INIT_RE.sub( - r"int {}::init(bool\1)".format(name), added_class - ) - fixed_run = LEGACY_MODEL_RUN_RE.sub( - r"int {}::rhs(BoutReal\1)".format(name), fixed_init - ) + methods = "\n ".join(method_decls) + physics_model = PHYSICS_MODEL_SKELETON.format(methods=methods, name=name) - fixed_constraint = BOUT_CONSTRAIN_RE.sub(r"solver->constraint(\1)", fixed_run) - fixed_solve = BOUT_CONSTRAIN_RE.sub(r"solver->add(\1)", fixed_constraint) + source_lines.insert(last_include, physics_model) + source_lines.append(BOUTMAIN.format(name)) - added_main = fixed_solve + BOUTMAIN.format(name) - return added_main + return "\n".join(source_lines) def yes_or_no(question): From d8ecd76f8b30fa4b436351df3f6a72122ec108a7 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 11 Jun 2020 17:03:22 +0100 Subject: [PATCH 33/82] Attempt a better fix of bout_constrain --- src/bout-v5-physics-model-upgrader.py | 79 +++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 5 deletions(-) diff --git a/src/bout-v5-physics-model-upgrader.py b/src/bout-v5-physics-model-upgrader.py index 4a809aa..8a30900 100755 --- a/src/bout-v5-physics-model-upgrader.py +++ b/src/bout-v5-physics-model-upgrader.py @@ -38,6 +38,8 @@ class {name} : public PhysicsModel {{ r'^#\s*include.*(<|")boutmain.hxx(>|")', re.MULTILINE ) +BOUT_SOLVE_RE = re.compile(r"bout_solve\(([^,)]+,\s*[^,)]+)\)", re.MULTILINE) + SPLIT_OPERATOR_RE = re.compile( r"solver\s*->\s*setSplitOperator\(([a-zA-Z0-9_]+),\s*([a-zA-Z0-9_]+)\s*\)" ) @@ -115,13 +117,70 @@ def fix_model_operator(source, model_name, operator_name, operator_type, new_nam ) -def convert_legacy_model(source, name): +def fix_bout_constrain(source, error_on_warning): + """Fix uses of bout_constrain. This is complicated because it might be + in a conditional, and Solver::constraint returns void + + """ + + if "bout_constrain" not in source: + return source + + modified = re.sub( + r"""if\s*\(\s*(?:!|not)\s* # in a conditional, checking for false + bout_constrain\(([^;]+,[^;]+,[^;]+)\) # actual function call + \s*\) # end of conditional + (?P\s*{\s*)? # possible open brace + (?:\s*\n)? # possible newline + \s*throw\s+BoutException\(.*\);(?:\s*\n)? # throwing an exception + (?(brace)\s*})? # consume matching closing brace + """, + r"solver->constraint(\1);\n", + source, + flags=re.VERBOSE | re.MULTILINE, + ) + + remaining_matches = list(re.finditer("bout_constrain", modified)) + if remaining_matches == []: + # We fixed everything! + return modified + + # Construct a useful error message + source_lines = source.splitlines() + lines_context = [] + for match in remaining_matches: + bad_line = source[: match.end()].count("\n") + line_range = range(max(0, bad_line - 1), min(len(source_lines), bad_line + 2)) + lines_context.append( + "\n ".join(["{}:{}".format(i, source_lines[i]) for i in line_range]) + ) + + message = textwrap.dedent( + """\ + Some uses of `bout_constrain` remain, but we could not automatically + convert them to use `Solver::constraint`. Please fix them before + continuing: + """ + ) + message += " " + "\n ".join(lines_context) + + if error_on_warning: + raise RuntimeError(message) + print(message) + return modified + + +def convert_legacy_model(source, name, error_on_warning): """Convert a legacy physics model to a PhysicsModel """ if not is_legacy_model(source): return source + source = fix_bout_constrain(source, error_on_warning) + + source = BOUT_SOLVE_RE.sub(r"solver->add(\1)", source) + # Replace legacy header source = LEGACY_MODEL_INCLUDE_RE.sub(r"#include \1bout/physicsmodel.hxx\2", source) @@ -254,12 +313,22 @@ def create_patch(filename, original, modified): " ", "_" ) - if re.match(r"^[0-9]+.*", new_name): - raise ValueError( - f"Invalid name: '{new_name}'. Use --name to specify a valid C++ identifier" + try: + if re.match(r"^[0-9]+.*", new_name) and not args.force: + raise ValueError( + f"Invalid name: '{new_name}'. Use --name to specify a valid C++ identifier" + ) + + modified = convert_legacy_model( + original, new_name, not (args.force or args.patch_only) ) + except (RuntimeError, ValueError) as e: + error_message = textwrap.indent(f"{e}", " ") + print( + f"There was a problem applying automatic fixes to {filename}:\n\n{error_message}" + ) + continue - modified = convert_legacy_model(original, new_name) patch = create_patch(filename, original, modified) if args.patch_only: From 9c2b49df775333ebbdaaef946f2a558e891daa92 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 12 Jan 2021 23:41:24 +0100 Subject: [PATCH 34/82] Support macros being removed in bout-v5-macro-upgrader.py --- src/bout-v5-macro-upgrader.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/bout-v5-macro-upgrader.py b/src/bout-v5-macro-upgrader.py index 16d3b3e..d39303b 100755 --- a/src/bout-v5-macro-upgrader.py +++ b/src/bout-v5-macro-upgrader.py @@ -9,7 +9,8 @@ # List of macros, their replacements and what header to find them # in. Each element should be a dict with "old", "new" and "headers" # keys, with "old" and "new" values being strings, and "headers" being a -# list of strings +# list of strings. "new" can also be None if the macro has been removed, which +# will cause an error to be printed if the macro is found. MACRO_REPLACEMENTS = [ { "old": "REVISION", @@ -297,6 +298,13 @@ def apply_fixes(replacements, source): modified = copy.deepcopy(source) for replacement in replacements: + if replacement["new"] is None: + print( + "'%s' has been removed, please delete from your code" + % replacement["old"] + ) + continue + modified = fix_include_version_header( replacement["old"], replacement["headers"], modified ) From 97b99f55ad3dbc164d3fac8ec11fadcd3f45b26c Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 12 Jan 2021 23:44:22 +0100 Subject: [PATCH 35/82] Print error for HDF5 macros when running bout-v5-macro-upgrader.py --- src/bout-v5-macro-upgrader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bout-v5-macro-upgrader.py b/src/bout-v5-macro-upgrader.py index d39303b..b6db6da 100755 --- a/src/bout-v5-macro-upgrader.py +++ b/src/bout-v5-macro-upgrader.py @@ -64,8 +64,8 @@ }, { "old": "HAS_HDF5", - "new": "BOUT_HAS_HDF5", - "headers": "bout/build_config.hxx", + "new": None, + "headers": [], "macro": True, "always_defined": True, }, @@ -141,8 +141,8 @@ }, { "old": "HDF5", - "new": "BOUT_HAS_HDF5", - "headers": "bout/build_config.hxx", + "new": None, + "headers": [], "macro": True, "always_defined": True, }, From f50e14c87fcc3364116665eeafc43dca1b14d736 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 5 Feb 2021 11:03:02 +0000 Subject: [PATCH 36/82] Use lowercase options for setting derivative methods --- src/bout-v5-input-file-upgrader.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 8414e04..87ea80c 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -3,6 +3,7 @@ import argparse import copy import difflib +import itertools import textwrap import warnings @@ -22,6 +23,16 @@ "type": bool, "values": {False: "estimate", True: "measure"}} ] +for section, derivative in itertools.product( + ["ddx", "ddy", "ddz", "diff"], ["First", "Second", "Fourth", "Flux", "Upwind"] +): + REPLACEMENTS.append( + { + "old": f"mesh:{section}:{derivative}", + "new": f"mesh:{section}:{derivative.lower()}", + } + ) + def parse_bool(bool_expression): try: From d1f0daa04228dc1290582e04d30bf38e0e6a4c39 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 5 Feb 2021 11:04:27 +0000 Subject: [PATCH 37/82] User lowercase for nout, timestep inputs --- src/bout-v5-input-file-upgrader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 87ea80c..6af5f2e 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -21,6 +21,8 @@ {"old": "interpolation", "new": "mesh:paralleltransform:xzinterpolation"}, {"old": "fft:fft_measure", "new": "fft:fft_measurement_flag", "type": bool, "values": {False: "estimate", True: "measure"}} + {"old": "TIMESTEP", "new": "timestep"}, + {"old": "NOUT", "new": "nout"}, ] for section, derivative in itertools.product( From 19e977e156282c6b29f2ca4f667110c2aa91d4ec Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 5 Feb 2021 11:05:29 +0000 Subject: [PATCH 38/82] Use lowercase for mesh:staggergrids Also move setting mesh members into init-list, and reorder members for correct init order --- src/bout-v5-input-file-upgrader.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 6af5f2e..396cd35 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -19,10 +19,23 @@ {"old": "mesh:paralleltransform", "new": "mesh:paralleltransform:type"}, {"old": "fci", "new": "mesh:paralleltransform"}, {"old": "interpolation", "new": "mesh:paralleltransform:xzinterpolation"}, - {"old": "fft:fft_measure", "new": "fft:fft_measurement_flag", - "type": bool, "values": {False: "estimate", True: "measure"}} + { + "old": "fft:fft_measure", + "new": "fft:fft_measurement_flag", + "type": bool, + "values": {False: "estimate", True: "measure"}, + }, {"old": "TIMESTEP", "new": "timestep"}, {"old": "NOUT", "new": "nout"}, + # The following haven't been changed, but are frequently spelt with the wrong case + {"old": "mxg", "new": "MXG"}, + {"old": "myg", "new": "MYG"}, + {"old": "nxpe", "new": "NXPE"}, + {"old": "nype", "new": "NYPE"}, + {"old": "mesh:StaggerGrids", "new": "mesh:staggergrids"}, + {"old": "zmin", "new": "ZMIN"}, + {"old": "zmax", "new": "ZMAX"}, + {"old": "zperiod", "new": "ZPERIOD"}, ] for section, derivative in itertools.product( From 2af51510e1c6a5be06d96da1503368e44b4f4f8b Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 5 Feb 2021 11:43:42 +0000 Subject: [PATCH 39/82] Consistently use lowercase for "All" section WARNING: currently, boutdata.data.BoutOptionsFile just removes the section instead of renaming. As this is now a submodule, needs to be fixed in the boutdata repo --- src/bout-v5-input-file-upgrader.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 396cd35..c073bd7 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -27,6 +27,10 @@ }, {"old": "TIMESTEP", "new": "timestep"}, {"old": "NOUT", "new": "nout"}, + { + "old": "All:bndry_all", + "new": "all:bndry_all", + }, # This was inconsistent in the library # The following haven't been changed, but are frequently spelt with the wrong case {"old": "mxg", "new": "MXG"}, {"old": "myg", "new": "MYG"}, From a80166647ae753d65b879a87ca766cbc80c1a2e7 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 8 Feb 2021 13:30:36 +0000 Subject: [PATCH 40/82] Replace entire "All" section in input upgrader --- src/bout-v5-input-file-upgrader.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index c073bd7..701ca32 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -27,10 +27,8 @@ }, {"old": "TIMESTEP", "new": "timestep"}, {"old": "NOUT", "new": "nout"}, - { - "old": "All:bndry_all", - "new": "all:bndry_all", - }, # This was inconsistent in the library + # This was inconsistent in the library + {"old": "All", "new": "all"}, # The following haven't been changed, but are frequently spelt with the wrong case {"old": "mxg", "new": "MXG"}, {"old": "myg", "new": "MYG"}, From b46035a6e35a7617c35ae57cc8cf20383eb1a22c Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 8 Feb 2021 17:29:24 +0000 Subject: [PATCH 41/82] Add argument to input upgrader to just apply canonicalisation That is, allow just fixing whitespace etc. --- src/bout-v5-input-file-upgrader.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 701ca32..3d93ffc 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -116,8 +116,7 @@ def fix_replacements(replacements, options_file): def apply_fixes(replacements, options_file): - """Apply all fixes in this module - """ + """Apply all fixes in this module""" modified = copy.deepcopy(options_file) @@ -127,9 +126,7 @@ def apply_fixes(replacements, options_file): def yes_or_no(question): - """Convert user input from yes/no variations to True/False - - """ + """Convert user input from yes/no variations to True/False""" while True: reply = input(question + " [y/N] ").lower().strip() if not reply or reply[0] == "n": @@ -139,8 +136,7 @@ def yes_or_no(question): def create_patch(filename, original, modified): - """Create a unified diff between original and modified - """ + """Create a unified diff between original and modified""" patch = "\n".join( difflib.unified_diff( @@ -231,6 +227,12 @@ def possibly_apply_patch(patch, options_file, quiet=False, force=False): action="store_true", help="Automatically accept the canonical patch", ) + parser.add_argument( + "--canonical-only", + "-k", + action="store_true", + help="Only check/fix canonicalisation", + ) args = parser.parse_args() @@ -262,6 +264,9 @@ def possibly_apply_patch(patch, options_file, quiet=False, force=False): if applied_patch: original_source = str(original) + if args.canonical_only: + continue + try: modified = apply_fixes(REPLACEMENTS, original) except RuntimeError as e: From e91a145cf70da7e1197a402e78cdcb8625595195 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 9 Feb 2021 10:34:23 +0000 Subject: [PATCH 42/82] Monkey-patch BoutOptions to make it case-sensitive --- src/bout-v5-input-file-upgrader.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 3d93ffc..6d0e930 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -7,10 +7,23 @@ import textwrap import warnings -from boutdata.data import BoutOptionsFile +from boutdata.data import BoutOptionsFile, BoutOptions from boututils.boutwarnings import AlwaysWarning -# This should be a list of dicts, each containing "old", "new" and optionally "values". + +def case_sensitive_init(self, name="root", parent=None): + self._sections = dict() + self._keys = dict() + self._name = name + self._parent = parent + self.comments = dict() + self.inline_comments = dict() + self._comment_whitespace = dict() + + +# Monky-patch BoutOptions to make sure it's case sensitive +BoutOptions.__init__ = case_sensitive_init + # The values of "old"/"new" keys should be the old/new names of input file values or # sections. The value of "values" is a dict containing replacements for values of the # option. "type" optionally specifies the type of the old value of the option; for From 3e94de1d8a1081a3b22c113f8bc8363fa6728dab Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 9 Feb 2021 10:37:26 +0000 Subject: [PATCH 43/82] Tweak replacement key names in input upgrader --- src/bout-v5-input-file-upgrader.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 6d0e930..8a87630 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -24,9 +24,11 @@ def case_sensitive_init(self, name="root", parent=None): # Monky-patch BoutOptions to make sure it's case sensitive BoutOptions.__init__ = case_sensitive_init + +# This should be a list of dicts, each containing "old", "new" and optionally "new_values". # The values of "old"/"new" keys should be the old/new names of input file values or -# sections. The value of "values" is a dict containing replacements for values of the -# option. "type" optionally specifies the type of the old value of the option; for +# sections. The value of "new_values" is a dict containing replacements for values of the +# option. "old_type" optionally specifies the type of the old value of the option; for # example this is needed for special handling of boolean values. REPLACEMENTS = [ {"old": "mesh:paralleltransform", "new": "mesh:paralleltransform:type"}, @@ -35,8 +37,8 @@ def case_sensitive_init(self, name="root", parent=None): { "old": "fft:fft_measure", "new": "fft:fft_measurement_flag", - "type": bool, - "values": {False: "estimate", True: "measure"}, + "old_type": bool, + "new_values": {False: "estimate", True: "measure"}, }, {"old": "TIMESTEP", "new": "timestep"}, {"old": "NOUT", "new": "nout"}, @@ -97,21 +99,23 @@ def fix_replacements(replacements, options_file): "\n\t{1}".format(options_file.filename, e.args[0], **replacement) ) from e else: - if "type" in replacement: + if "old_type" in replacement: # Special handling for certain types, replicating what BOUT++ does - if replacement["type"] is bool: + if replacement["old_type"] is bool: # The original value must be something that BOUT++ recognises as a # bool. - # replacement["values"] must contain both True and False keys. + # replacement["new_values"] must contain both True and False keys. old_value = parse_bool(options_file[replacement["new"]]) - options_file[replacement["new"]] = replacement["values"][old_value] + options_file[replacement["new"]] = replacement["new_values"][ + old_value + ] else: raise ValueError( f"Error in REPLACEMENTS: type {replacement['type']} is not handled" ) else: # Option values are just a string - if "values" in replacement: + if "new_values" in replacement: old_value = options_file[replacement["new"]] try: old_value = old_value.lower() @@ -120,7 +124,7 @@ def fix_replacements(replacements, options_file): pass try: - options_file[replacement["new"]] = replacement["values"][ + options_file[replacement["new"]] = replacement["new_values"][ old_value ] except KeyError: From 0eb786aabb88ed4a94448359a9b858d85654cbec Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 9 Feb 2021 10:38:06 +0000 Subject: [PATCH 44/82] Add helper function to work out if an option needs upgrading --- src/bout-v5-input-file-upgrader.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 8a87630..9836a2d 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -83,6 +83,21 @@ def parse_bool(bool_expression): ) +def already_fixed(replacement, options_file): + """Check if the options_file already has already had this particular fix applied""" + # The old key is there and the new one isn't, then it's definitely not fixed + if replacement["old"] in options_file and replacement["new"] not in options_file: + return False + # If the new isn't there, there's nothing to fix + if replacement["new"] not in options_file: + return True + # If we don't need to fix values, we're done + if "new_values" not in replacement: + return True + # Check if the current value is acceptable + return options_file[replacement["new"]] in replacement["new_values"].values() + + def fix_replacements(replacements, options_file): """Change the names of options in options_file according to the list of dicts replacements @@ -90,6 +105,8 @@ def fix_replacements(replacements, options_file): """ for replacement in replacements: try: + if already_fixed(replacement, options_file): + continue options_file.rename(replacement["old"], replacement["new"]) except KeyError: pass From 25bab631537af9effb85e2a4754f0b8107c66fce Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 9 Feb 2021 16:00:35 +0000 Subject: [PATCH 45/82] Lowercase 'twistshift' option --- src/bout-v5-input-file-upgrader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 9836a2d..cec3fc2 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -50,6 +50,7 @@ def case_sensitive_init(self, name="root", parent=None): {"old": "nxpe", "new": "NXPE"}, {"old": "nype", "new": "NYPE"}, {"old": "mesh:StaggerGrids", "new": "mesh:staggergrids"}, + {"old": "TwistShift", "new": "twistshift"}, {"old": "zmin", "new": "ZMIN"}, {"old": "zmax", "new": "ZMAX"}, {"old": "zperiod", "new": "ZPERIOD"}, From c6c43a21bbc43a7c9d1efff57bb9d0130b58c960 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 9 Feb 2021 16:13:46 +0000 Subject: [PATCH 46/82] Fix some more common spellings of input options --- src/bout-v5-input-file-upgrader.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index cec3fc2..0d4e013 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -49,11 +49,15 @@ def case_sensitive_init(self, name="root", parent=None): {"old": "myg", "new": "MYG"}, {"old": "nxpe", "new": "NXPE"}, {"old": "nype", "new": "NYPE"}, + {"old": "mesh:NX", "new": "mesh:nx"}, + {"old": "mesh:NY", "new": "mesh:ny"}, + {"old": "mesh:shiftangle", "new": "mesh:ShiftAngle"}, + {"old": "mesh:shiftAngle", "new": "mesh:ShiftAngle"}, {"old": "mesh:StaggerGrids", "new": "mesh:staggergrids"}, {"old": "TwistShift", "new": "twistshift"}, {"old": "zmin", "new": "ZMIN"}, {"old": "zmax", "new": "ZMAX"}, - {"old": "zperiod", "new": "ZPERIOD"}, + {"old": "ZPERIOD", "new": "zperiod"}, ] for section, derivative in itertools.product( From 66175297543cba3027ea523c5c7640116675e337 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 9 Feb 2021 16:41:15 +0000 Subject: [PATCH 47/82] Fix some example input files with wrong derivatives sections --- src/bout-v5-input-file-upgrader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 0d4e013..2a55e2b 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -42,6 +42,9 @@ def case_sensitive_init(self, name="root", parent=None): }, {"old": "TIMESTEP", "new": "timestep"}, {"old": "NOUT", "new": "nout"}, + {"old": "ddx", "new": "mesh:ddx"}, + {"old": "ddy", "new": "mesh:ddy"}, + {"old": "ddz", "new": "mesh:ddz"}, # This was inconsistent in the library {"old": "All", "new": "all"}, # The following haven't been changed, but are frequently spelt with the wrong case From b800570e1b52d34468421aa08a961109b8953c4c Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 10 Feb 2021 16:29:19 +0000 Subject: [PATCH 48/82] Fix examples using old `laplace_nonuniform` option --- src/bout-v5-input-file-upgrader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 2a55e2b..0938fb6 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -45,6 +45,7 @@ def case_sensitive_init(self, name="root", parent=None): {"old": "ddx", "new": "mesh:ddx"}, {"old": "ddy", "new": "mesh:ddy"}, {"old": "ddz", "new": "mesh:ddz"}, + {"old": "laplace:laplace_nonuniform", "new": "laplace:nonuniform"}, # This was inconsistent in the library {"old": "All", "new": "all"}, # The following haven't been changed, but are frequently spelt with the wrong case From 58d31d07f745f276dfc0b47a59fbfb734eb8ad41 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 10 Feb 2021 16:32:05 +0000 Subject: [PATCH 49/82] Fix example with wrong capitalisation of output:shiftOutput --- src/bout-v5-input-file-upgrader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 0938fb6..177c20a 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -58,6 +58,8 @@ def case_sensitive_init(self, name="root", parent=None): {"old": "mesh:shiftangle", "new": "mesh:ShiftAngle"}, {"old": "mesh:shiftAngle", "new": "mesh:ShiftAngle"}, {"old": "mesh:StaggerGrids", "new": "mesh:staggergrids"}, + {"old": "output:shiftoutput", "new": "output:shiftOutput"}, + {"old": "output:ShiftOutput", "new": "output:shiftOutput"}, {"old": "TwistShift", "new": "twistshift"}, {"old": "zmin", "new": "ZMIN"}, {"old": "zmax", "new": "ZMAX"}, From be00a676bfb78b0360eaba67f4f6686b620b9273 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 11 Feb 2021 11:53:56 +0000 Subject: [PATCH 50/82] Fix bad capitalisation of zShift option in input files --- src/bout-v5-input-file-upgrader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 177c20a..19be1ed 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -57,6 +57,7 @@ def case_sensitive_init(self, name="root", parent=None): {"old": "mesh:NY", "new": "mesh:ny"}, {"old": "mesh:shiftangle", "new": "mesh:ShiftAngle"}, {"old": "mesh:shiftAngle", "new": "mesh:ShiftAngle"}, + {"old": "mesh:zshift", "new": "mesh:zShift"}, {"old": "mesh:StaggerGrids", "new": "mesh:staggergrids"}, {"old": "output:shiftoutput", "new": "output:shiftOutput"}, {"old": "output:ShiftOutput", "new": "output:shiftOutput"}, From c641e8a20e18aab2e01479a5e614123873fedf81 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 11 Feb 2021 15:52:02 +0000 Subject: [PATCH 51/82] Use consistent case for Solver atol/rtol options Some Solvers used uppercase, some lowercase --- src/bout-v5-input-file-upgrader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 19be1ed..3ce45b9 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -46,6 +46,8 @@ def case_sensitive_init(self, name="root", parent=None): {"old": "ddy", "new": "mesh:ddy"}, {"old": "ddz", "new": "mesh:ddz"}, {"old": "laplace:laplace_nonuniform", "new": "laplace:nonuniform"}, + {"old": "solver:ATOL", "new": "solver:atol"}, + {"old": "solver:RTOL", "new": "solver:rtol"}, # This was inconsistent in the library {"old": "All", "new": "all"}, # The following haven't been changed, but are frequently spelt with the wrong case From 520de5e255587fa34f40133dbddcaa28c6b77ce6 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 11 Feb 2021 16:54:19 +0000 Subject: [PATCH 52/82] Fix for some tests with dump_format in wrong section Actually this is no longer relevant and we can probably remove it entirely --- src/bout-v5-input-file-upgrader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 3ce45b9..88ad924 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -46,6 +46,7 @@ def case_sensitive_init(self, name="root", parent=None): {"old": "ddy", "new": "mesh:ddy"}, {"old": "ddz", "new": "mesh:ddz"}, {"old": "laplace:laplace_nonuniform", "new": "laplace:nonuniform"}, + {"old": "mesh:dump_format", "new": "dump_format"}, {"old": "solver:ATOL", "new": "solver:atol"}, {"old": "solver:RTOL", "new": "solver:rtol"}, # This was inconsistent in the library From a6b76f6b27c18dedea3cd2a02ae1db17c26e6c9b Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 2 Mar 2021 14:32:54 +0000 Subject: [PATCH 53/82] Convert output camelCase options to lowercase --- src/bout-v5-input-file-upgrader.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 88ad924..20e0c9f 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -62,8 +62,12 @@ def case_sensitive_init(self, name="root", parent=None): {"old": "mesh:shiftAngle", "new": "mesh:ShiftAngle"}, {"old": "mesh:zshift", "new": "mesh:zShift"}, {"old": "mesh:StaggerGrids", "new": "mesh:staggergrids"}, - {"old": "output:shiftoutput", "new": "output:shiftOutput"}, - {"old": "output:ShiftOutput", "new": "output:shiftOutput"}, + {"old": "output:shiftOutput", "new": "output:shiftoutput"}, + {"old": "output:ShiftOutput", "new": "output:shiftoutput"}, + {"old": "output:shiftInput", "new": "output:shiftinput"}, + {"old": "output:ShiftInput", "new": "output:shiftinput"}, + {"old": "output:flushFrequency", "new": "output:flushfrequency"}, + {"old": "output:FlushFrequency", "new": "output:flushfrequency"}, {"old": "TwistShift", "new": "twistshift"}, {"old": "zmin", "new": "ZMIN"}, {"old": "zmax", "new": "ZMAX"}, From 65c3c8f4dfadc8737a0dafb1893d9f3f472f19cf Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 9 Apr 2021 10:41:40 +0100 Subject: [PATCH 54/82] Rename 'restart' section to 'restart_files' --- src/bout-v5-input-file-upgrader.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index 20e0c9f..f922a62 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -72,6 +72,18 @@ def case_sensitive_init(self, name="root", parent=None): {"old": "zmin", "new": "ZMIN"}, {"old": "zmax", "new": "ZMAX"}, {"old": "ZPERIOD", "new": "zperiod"}, + # 'restart' can be either a section or a value, so move all the + # section:values instead + {"old": "restart:parallel", "new": "restart_files:parallel"}, + {"old": "restart:flush", "new": "restart_files:flush"}, + {"old": "restart:guards", "new": "restart_files:guards"}, + {"old": "restart:floats", "new": "restart_files:floats"}, + {"old": "restart:openclose", "new": "restart_files:openclose"}, + {"old": "restart:enabled", "new": "restart_files:enabled"}, + {"old": "restart:init_missing", "new": "restart_files:init_missing"}, + {"old": "restart:shiftOutput", "new": "restart_files:shiftOutput"}, + {"old": "restart:shiftInput", "new": "restart_files:shiftInput"}, + {"old": "restart:flushFrequency", "new": "restart_files:flushFrequency"}, ] for section, derivative in itertools.product( From dbfd34863ee60c492532aaaf7692144b24d51e89 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 14 Apr 2021 10:58:13 +0100 Subject: [PATCH 55/82] Update physics model upgrader for additional Solver::add argument --- src/bout-v5-physics-model-upgrader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bout-v5-physics-model-upgrader.py b/src/bout-v5-physics-model-upgrader.py index 8a30900..bbe8023 100755 --- a/src/bout-v5-physics-model-upgrader.py +++ b/src/bout-v5-physics-model-upgrader.py @@ -38,7 +38,7 @@ class {name} : public PhysicsModel {{ r'^#\s*include.*(<|")boutmain.hxx(>|")', re.MULTILINE ) -BOUT_SOLVE_RE = re.compile(r"bout_solve\(([^,)]+,\s*[^,)]+)\)", re.MULTILINE) +BOUT_SOLVE_RE = re.compile(r"bout_solve\(([^,)]+,\s*[^,)]+(,\s*[^,)]+)?)\)", re.MULTILINE) SPLIT_OPERATOR_RE = re.compile( r"solver\s*->\s*setSplitOperator\(([a-zA-Z0-9_]+),\s*([a-zA-Z0-9_]+)\s*\)" From 61adb7e5371e0ebff877003b7bb99336b8cc648d Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 15 Apr 2021 09:32:50 +0100 Subject: [PATCH 56/82] Add ability to upgrade physics model preconditioners/Jacobians --- src/bout-v5-physics-model-upgrader.py | 169 ++++++++++++++++++++------ 1 file changed, 133 insertions(+), 36 deletions(-) diff --git a/src/bout-v5-physics-model-upgrader.py b/src/bout-v5-physics-model-upgrader.py index bbe8023..dbc3a7e 100755 --- a/src/bout-v5-physics-model-upgrader.py +++ b/src/bout-v5-physics-model-upgrader.py @@ -18,30 +18,59 @@ class {name} : public PhysicsModel {{ }}; """ -PHYSICS_MODEL_RHS_SKELETON = "int {function}({arg_type}{time}) override;" +PHYSICS_MODEL_RHS_SKELETON = "int {function}({arguments}){override};" BOUTMAIN = "\n\nBOUTMAIN({})\n" # Regular expression for a PhysicsModel -PHYSICS_MODEL_RE = re.compile(r":\s*public\s*PhysicsModel") +PHYSICS_MODEL_RE = re.compile( + r"""class\s+(?P[a-zA-Z0-9_]+)\s*: # Class name + \s*(?:public)?\s*PhysicsModel[\n\s]*{ # Inherits from PhysicsModel + """, + re.VERBOSE | re.MULTILINE, +) -# Regular expressions for a legacy physics model -LEGACY_MODEL_RE_TEMPLATE = r"""int\s+{}\s*\({} - (\s+ # Require spaces only if the argument is named - (?PUNUSED\()? # Possible UNUSED macro +FUNCTION_SIGNATURE_ARGUMENT_RE = r"""({arg_type} + \s+ # Require spaces only if the argument is named + (?PUNUSED\()? # Possible UNUSED macro [a-zA-Z_0-9]* # Argument name - (?(unused)\)) # If UNUSED macro was present, we need an extra closing bracket + (?(unused{arg_num})\)) # If UNUSED macro was present, we need an extra closing bracket )? - \)""" +""" + + +def create_function_signature_re(function_name, argument_types): + """Create a regular expression for a legacy physics model function""" + + if not isinstance(argument_types, list): + argument_types = [argument_types] + + arguments = r",\s*".join( + [ + FUNCTION_SIGNATURE_ARGUMENT_RE.format(arg_type=argument, arg_num=num) + for num, argument in enumerate(argument_types) + ] + ) + + return fr"int\s+{function_name}\s*\({arguments}\)" + LEGACY_MODEL_INCLUDE_RE = re.compile( r'^#\s*include.*(<|")boutmain.hxx(>|")', re.MULTILINE ) -BOUT_SOLVE_RE = re.compile(r"bout_solve\(([^,)]+,\s*[^,)]+(,\s*[^,)]+)?)\)", re.MULTILINE) +BOUT_SOLVE_RE = re.compile( + r"bout_solve\(([^,)]+,\s*[^,)]+(,\s*[^,)]+)?)\)", re.MULTILINE +) + +RHS_RE = re.compile(r"solver\s*->\s*setRHS\(\s*([a-zA-Z0-9_]+)\s*\)") + +PRECON_RE = re.compile(r"solver\s*->\s*setPrecon\(\s*([a-zA-Z0-9_]+)\s*\)") + +JACOBIAN_RE = re.compile(r"solver\s*->\s*setJacobian\(\s*([a-zA-Z0-9_]+)\s*\)") SPLIT_OPERATOR_RE = re.compile( - r"solver\s*->\s*setSplitOperator\(([a-zA-Z0-9_]+),\s*([a-zA-Z0-9_]+)\s*\)" + r"solver\s*->\s*setSplitOperator\(\s*([a-zA-Z0-9_]+),\s*([a-zA-Z0-9_]+)\s*\)" ) @@ -73,14 +102,19 @@ def find_last_include(source_lines): return 0 -def fix_model_operator(source, model_name, operator_name, operator_type, new_name): +def fix_model_operator( + source, model_name, operator_name, operator_type, new_name, override +): """Fix any definitions of the operator, and return the new declaration May modify source """ + if not isinstance(operator_type, list): + operator_type = [operator_type] + operator_re = re.compile( - LEGACY_MODEL_RE_TEMPLATE.format(operator_name, operator_type), + create_function_signature_re(operator_name, operator_type), re.VERBOSE | re.MULTILINE, ) @@ -93,26 +127,28 @@ def fix_model_operator(source, model_name, operator_name, operator_type, new_nam if len(matches) > 1: source = re.sub( - LEGACY_MODEL_RE_TEMPLATE.format(operator_name, operator_type) + r"\s*;", + create_function_signature_re(operator_name, operator_type) + r"\s*;", "", source, flags=re.VERBOSE | re.MULTILINE, ) - arg_name = operator_re.search(source).group(1) + arg_names = operator_re.search(source).groups()[::2] else: - arg_name = matches[0].group(1) + arg_names = matches[0].groups()[::2] # Fix definition and any existing declarations - modified = operator_re.sub( - fr"int {model_name}::{new_name}({operator_type}\1)", source - ) + arguments = ", ".join(arg_names) + + modified = operator_re.sub(fr"int {model_name}::{new_name}({arguments})", source) # Create the declaration return ( modified, PHYSICS_MODEL_RHS_SKELETON.format( - function=new_name, arg_type=operator_type, time=arg_name + function=new_name, + arguments=arguments, + override=" override" if override else "", ), ) @@ -170,6 +206,59 @@ def fix_bout_constrain(source, error_on_warning): return modified +def convert_old_solver_api(source, name): + """Fix or remove old Solver API calls""" + + source = BOUT_SOLVE_RE.sub(r"solver->add(\1)", source) + + # Remove calls to Solver::setRHS + source = RHS_RE.sub("", source) + + method_decls = [] + + precons = PRECON_RE.findall(source) + for precon in precons: + source, decl = fix_model_operator( + source, + name, + precon, + ["BoutReal", "BoutReal", "BoutReal"], + precon, + override=False, + ) + if decl: + method_decls.append(decl) + source = PRECON_RE.sub(fr"setPrecon(&{name}::\1)", source) + + jacobians = JACOBIAN_RE.findall(source) + for jacobian in jacobians: + source, decl = fix_model_operator( + source, name, jacobian, "BoutReal", jacobian, override=False + ) + if decl: + method_decls.append(decl) + source = JACOBIAN_RE.sub(fr"setJacobian(&{name}::\1)", source) + + if not method_decls: + return source + + match = PHYSICS_MODEL_RE.search(source) + if match is None: + warnings.warn( + f"Could not find the '{name}' class to add" + "preconditioner and/or Jacobian declarations; is it defined" + "in another file? If so, you will need to fix it manually" + ) + return source, False + + last_line_of_class = source[: match.end() + 1].count("\n") + methods = "\n ".join(method_decls) + source_lines = source.splitlines() + source_lines.insert(last_line_of_class, f" {methods}") + + return "\n".join(source_lines) + + def convert_legacy_model(source, name, error_on_warning): """Convert a legacy physics model to a PhysicsModel """ @@ -179,14 +268,14 @@ def convert_legacy_model(source, name, error_on_warning): source = fix_bout_constrain(source, error_on_warning) - source = BOUT_SOLVE_RE.sub(r"solver->add(\1)", source) - # Replace legacy header source = LEGACY_MODEL_INCLUDE_RE.sub(r"#include \1bout/physicsmodel.hxx\2", source) method_decls = [] - source, decl = fix_model_operator(source, name, "physics_init", "bool", "init") + source, decl = fix_model_operator( + source, name, "physics_init", "bool", "init", override=True + ) if decl: method_decls.append(decl) @@ -197,19 +286,19 @@ def convert_legacy_model(source, name, error_on_warning): convective, diffusive = split_operators # Fix the free functions source, decl = fix_model_operator( - source, name, convective, "BoutReal", "convective" + source, name, convective, "BoutReal", "convective", override=True ) if decl: method_decls.append(decl) source, decl = fix_model_operator( - source, name, diffusive, "BoutReal", "diffusive" + source, name, diffusive, "BoutReal", "diffusive", override=True ) if decl: method_decls.append(decl) else: # Fix the rhs free function source, decl = fix_model_operator( - source, name, "physics_run", "BoutReal", "rhs" + source, name, "physics_run", "BoutReal", "rhs", override=True ) if decl: method_decls.append(decl) @@ -264,8 +353,10 @@ def create_patch(filename, original, modified): This will do the bare minimum required to compile, and won't make global objects (like Field3Ds) members of the - new class, or free functions other than - `physics_init`/`physics_run` methods of the new class. + new class, or free functions (other than + `physics_init`/`physics_run`, preconditioners, and + Jacobians) methods of the new class. Comments may also be + left behind. By default, this will use the file name stripped of file extensions as the name of the new class. Use '--name= Date: Fri, 16 Apr 2021 11:12:45 +0100 Subject: [PATCH 57/82] Add some comments to physics model upgrader [skip ci] --- src/bout-v5-physics-model-upgrader.py | 63 ++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/src/bout-v5-physics-model-upgrader.py b/src/bout-v5-physics-model-upgrader.py index dbc3a7e..4070343 100755 --- a/src/bout-v5-physics-model-upgrader.py +++ b/src/bout-v5-physics-model-upgrader.py @@ -108,16 +108,34 @@ def fix_model_operator( """Fix any definitions of the operator, and return the new declaration May modify source + + Parameters + ---------- + source: str + Source code to fix + model_name: str + Name of the PhysicsModel class to create or add methods to + operator_name: str + Name of the free function to fix + operator_type: str, [str] + Function argument types + new_name: str + Name of the PhysicsModel method + override: bool + Is `new_name` overriding a virtual? """ + # Make sure we have a list of types if not isinstance(operator_type, list): operator_type = [operator_type] + # Get a regex for the function signature operator_re = re.compile( create_function_signature_re(operator_name, operator_type), re.VERBOSE | re.MULTILINE, ) + # Find any declarations of the free function matches = list(operator_re.finditer(source)) if matches == []: warnings.warn( @@ -125,6 +143,8 @@ def fix_model_operator( ) return source, False + # If we found more than one, remove the first one as it's probably + # a declaration and not a definition if len(matches) > 1: source = re.sub( create_function_signature_re(operator_name, operator_type) + r"\s*;", @@ -133,6 +153,8 @@ def fix_model_operator( flags=re.VERBOSE | re.MULTILINE, ) + # Get the names of the function arguments. Every other group + # from the regex, as the other groups match the `UNUSED` macro arg_names = operator_re.search(source).groups()[::2] else: arg_names = matches[0].groups()[::2] @@ -140,6 +162,7 @@ def fix_model_operator( # Fix definition and any existing declarations arguments = ", ".join(arg_names) + # Modify the definition: it's out-of-line so we need the qualified name modified = operator_re.sub(fr"int {model_name}::{new_name}({arguments})", source) # Create the declaration @@ -162,6 +185,9 @@ def fix_bout_constrain(source, error_on_warning): if "bout_constrain" not in source: return source + # The bout_constrain free function returns False if the Solver + # doesn't have constraints, but the Solver::constraint method does + # the checking itself, so we don't need to repeat it modified = re.sub( r"""if\s*\(\s*(?:!|not)\s* # in a conditional, checking for false bout_constrain\(([^;]+,[^;]+,[^;]+)\) # actual function call @@ -176,6 +202,7 @@ def fix_bout_constrain(source, error_on_warning): flags=re.VERBOSE | re.MULTILINE, ) + # The above might not fix everything, so best check if there are any uses left remaining_matches = list(re.finditer("bout_constrain", modified)) if remaining_matches == []: # We fixed everything! @@ -207,15 +234,28 @@ def fix_bout_constrain(source, error_on_warning): def convert_old_solver_api(source, name): - """Fix or remove old Solver API calls""" + """Fix or remove old Solver API calls + + Parameters + ---------- + source: str + The source code to modify + name: str + The PhysicsModel class name + """ + # Fixing `bout_solve` is a straight replacement, easy source = BOUT_SOLVE_RE.sub(r"solver->add(\1)", source) - # Remove calls to Solver::setRHS + # Completely remove calls to Solver::setRHS source = RHS_RE.sub("", source) + # List of old free functions that now need declarations inside the + # class definition method_decls = [] + # Fix uses of solver->setPrecon + # Get the name of any free functions passed as arguments to setPrecon precons = PRECON_RE.findall(source) for precon in precons: source, decl = fix_model_operator( @@ -228,8 +268,10 @@ def convert_old_solver_api(source, name): ) if decl: method_decls.append(decl) + # Almost a straight replacement, but it's now a member-function pointer source = PRECON_RE.sub(fr"setPrecon(&{name}::\1)", source) + # Fix uses of solver->setJacobian, basically the same as for setPrecon jacobians = JACOBIAN_RE.findall(source) for jacobian in jacobians: source, decl = fix_model_operator( @@ -239,11 +281,14 @@ def convert_old_solver_api(source, name): method_decls.append(decl) source = JACOBIAN_RE.sub(fr"setJacobian(&{name}::\1)", source) + # If we didn't find any free functions that need to be made into + # methods, we're done if not method_decls: return source - match = PHYSICS_MODEL_RE.search(source) - if match is None: + # We need to find the class defintion + class_def = PHYSICS_MODEL_RE.search(source) + if class_def is None: warnings.warn( f"Could not find the '{name}' class to add" "preconditioner and/or Jacobian declarations; is it defined" @@ -251,10 +296,16 @@ def convert_old_solver_api(source, name): ) return source, False - last_line_of_class = source[: match.end() + 1].count("\n") + # The easiest place to stick the method declaration is on the line + # immediately following the open brace of the class def, and the + # easiest way to insert it is to split the source into a list, + # insert in the list, then join the list back into a string. + # The regex from above finds the offset in the source which we + # need to turn into a line number + first_line_of_class = source[: class_def.end() + 1].count("\n") methods = "\n ".join(method_decls) source_lines = source.splitlines() - source_lines.insert(last_line_of_class, f" {methods}") + source_lines.insert(first_line_of_class, f" {methods}") return "\n".join(source_lines) From 8c9fe1b56f26db171f6d18a60d32d3ae74650e75 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 9 Jun 2021 17:21:39 +0100 Subject: [PATCH 58/82] Remove some now obsolete input options for DataFile --- src/bout-v5-input-file-upgrader.py | 35 ++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index f922a62..e2940ff 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -96,6 +96,25 @@ def case_sensitive_init(self, name="root", parent=None): } ) +DELETED = ["dump_format"] + +for section, value in itertools.product( + ["output", "restart"], + [ + "floats", + # Following are not yet implemented in OptionsNetCDF. Not yet + # clear if they need to be, or can be safely removed + # "shiftoutput", + # "shiftinput", + # "flushfrequency", + # "parallel", + # "guards", + # "openclose", + # "init_missing", + ], +): + DELETED.append(f"{section}:{value}") + def parse_bool(bool_expression): try: @@ -180,13 +199,25 @@ def fix_replacements(replacements, options_file): pass -def apply_fixes(replacements, options_file): +def remove_deleted(deleted, options_file): + """Remove each key that appears in 'deleted' from 'options_file'""" + + for key in deleted: + # Better would be options_file.pop(key, None), but there's a + # bug in current implementation + if key in options_file: + del options_file[key] + + +def apply_fixes(replacements, deleted, options_file): """Apply all fixes in this module""" modified = copy.deepcopy(options_file) fix_replacements(replacements, modified) + remove_deleted(deleted, modified) + return modified @@ -333,7 +364,7 @@ def possibly_apply_patch(patch, options_file, quiet=False, force=False): continue try: - modified = apply_fixes(REPLACEMENTS, original) + modified = apply_fixes(REPLACEMENTS, DELETED, original) except RuntimeError as e: print(e) continue From 6f1585c1947b603e7e4dfd34999525a68df1bf48 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 26 Jul 2022 16:37:45 +0100 Subject: [PATCH 59/82] Add tool to move headers under `include/bout` and fix `#include`s Replaces similar bash tool --- src/bout-v5-header-upgrader.py | 217 +++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100755 src/bout-v5-header-upgrader.py diff --git a/src/bout-v5-header-upgrader.py b/src/bout-v5-header-upgrader.py new file mode 100755 index 0000000..8e67bb0 --- /dev/null +++ b/src/bout-v5-header-upgrader.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 + +import argparse +import copy +import difflib +import re +import textwrap +from pathlib import Path +from typing import List +from subprocess import run + + +header_shim_sentinel = "// BOUT++ header shim" + +header_warning = f"""\ +#pragma once +{header_shim_sentinel} +#warning Header "{{0}}" has moved to "bout/{{0}}". Run `bin/bout-v5-header-upgrader.py` to fix +#include "bout/{{0}}" +""" + + +def header_needs_moving(header: Path) -> bool: + """Check if `header` has not yet been moved""" + with open(header, "r") as f: + return header_shim_sentinel not in f.read() + + +def deprecated_header_list(include_path: Path = Path("./include")): + """List of deprecated header paths (that is, those in bare + ``include`` directory) + + """ + return include_path.glob("*.hxx") + + +def write_header_shim(header: Path): + """Write 'shim' for header, that ``include``s new location""" + with open(header, "w") as f: + f.write(header_warning.format(header.name)) + + +def fix_library_header_locations( + include_path: Path = Path("./include"), quiet: bool = False +): + unmoved_headers = list( + filter(header_needs_moving, deprecated_header_list(include_path)) + ) + include_bout_path = include_path / "bout" + + if unmoved_headers == []: + print("No headers to move!") + return + + out = run("git diff-index --cached HEAD --quiet", shell=True) + if out.returncode: + raise RuntimeError( + "git index not clean! Please commit or discard any changes before continuing" + ) + + # First we move any existing headers and commit this change, so + # that history is preserved + for header in unmoved_headers: + new_path = include_bout_path / header.name + if not quiet: + print(f"Moving '{header}' to '{new_path}'") + run(f"git mv {header} {new_path}", shell=True, check=True) + + run(r"git commit -m 'Move headers to `include/bout`'", shell=True, check=True) + + # Now we can write the compatibility shim + for header in unmoved_headers: + write_header_shim(header) + + run(f"git add {' '.join(map(str, unmoved_headers))}", shell=True, check=True) + run( + r"git commit -m 'Add compatibility shims for old header locations'", + shell=True, + check=True, + ) + + +def make_header_regex(deprecated_headers: List[str]) -> re.Pattern: + """Create a regular expression to match deprecated header locations""" + deprecated_header_joined = "|".join((header.name for header in deprecated_headers)) + return re.compile(rf'(#include\s+<|")(?:\.\./)({deprecated_header_joined})(>|")') + + +def apply_fixes(header_regex, source): + """Apply all fixes in this module""" + modified = copy.deepcopy(source) + + return header_regex.sub(r"\1bout/\2\3", modified) + + +def yes_or_no(question): + """Convert user input from yes/no variations to True/False""" + while True: + reply = input(question + " [y/N] ").lower().strip() + if not reply or reply[0] == "n": + return False + if reply[0] == "y": + return True + + +def create_patch(filename, original, modified): + """Create a unified diff between original and modified""" + + patch = "\n".join( + difflib.unified_diff( + original.splitlines(), + modified.splitlines(), + fromfile=filename, + tofile=filename, + lineterm="", + ) + ) + + return patch + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=textwrap.dedent( + """\ + Fix deprecated header locations for BOUT++ v4 -> v5 + + All BOUT++ headers are now under ``include/bout`` and + should be included as ``#include ``. This + tool will fix such includes. + + For developers: the option ``--move-deprecated-headers`` + will move the headers from ``include`` to + ``include/bout``, and add a compatibility shim in the old + location. This option is mutually exclusive with + ``--files``, and should be used after running this tool + over the library files. + + WARNING: If any files do need moving, this will create a + git commit in order to preserve history across file moves. + If you have staged changes, this tool will not work, so to + avoid committing undesired or unrelated changes. + + """ + ), + ) + + parser.add_argument( + "--force", "-f", action="store_true", help="Make changes without asking" + ) + parser.add_argument( + "--quiet", "-q", action="store_true", help="Don't print patches" + ) + parser.add_argument( + "--patch-only", "-p", action="store_true", help="Print the patches and exit" + ) + parser.add_argument( + "--include-path", + "-i", + help="Path to `include` directory", + default="./include", + type=Path, + ) + + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--files", nargs="*", action="store", help="Input files") + group.add_argument( + "--move-deprecated-headers", + action="store_true", + help="Move the deprecated headers", + ) + + args = parser.parse_args() + + if args.force and args.patch_only: + raise ValueError("Incompatible options: --force and --patch") + + deprecated_headers = deprecated_header_list(args.include_path) + + if args.move_deprecated_headers: + fix_library_header_locations(args.include_path, args.quiet) + exit(0) + + header_regex = make_header_regex(deprecated_headers) + + for filename in args.files: + with open(filename, "r") as f: + contents = f.read() + original = copy.deepcopy(contents) + + modified = apply_fixes(header_regex, contents) + patch = create_patch(filename, original, modified) + + if args.patch_only: + print(patch) + continue + + if not patch: + if not args.quiet: + print(f"No changes to make to {filename}") + continue + + if not args.quiet: + print("\n******************************************") + print(f"Changes to {filename}\n") + print(patch) + print("\n******************************************") + + if args.force: + make_change = True + else: + make_change = yes_or_no(f"Make changes to {filename}?") + + if make_change: + with open(filename, "w") as f: + f.write(modified) From f83cadbdfe30ce6846c64843da1f91b0f101943b Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Thu, 2 Feb 2023 08:14:21 -0800 Subject: [PATCH 60/82] Update bin/bout-v5-header-upgrader.py --- src/bout-v5-header-upgrader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bout-v5-header-upgrader.py b/src/bout-v5-header-upgrader.py index 8e67bb0..49a8fbc 100755 --- a/src/bout-v5-header-upgrader.py +++ b/src/bout-v5-header-upgrader.py @@ -83,7 +83,7 @@ def fix_library_header_locations( def make_header_regex(deprecated_headers: List[str]) -> re.Pattern: """Create a regular expression to match deprecated header locations""" deprecated_header_joined = "|".join((header.name for header in deprecated_headers)) - return re.compile(rf'(#include\s+<|")(?:\.\./)({deprecated_header_joined})(>|")') + return re.compile(rf'(#include\s+<|")(?:\.\./)?({deprecated_header_joined})(>|")') def apply_fixes(header_regex, source): From 050131a89be2fdb27abf8aa53c113bbe7d2f9e2f Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 3 Jul 2024 15:17:33 +0100 Subject: [PATCH 61/82] Pull out duplicated code from upgrader scripts --- src/bout-v5-factory-upgrader.py | 75 ++-------------- src/bout-v5-format-upgrader.py | 75 ++-------------- src/bout-v5-header-upgrader.py | 68 ++------------- src/bout-v5-input-file-upgrader.py | 51 ++--------- src/bout-v5-macro-upgrader.py | 72 ++-------------- src/bout-v5-physics-model-upgrader.py | 72 ++-------------- src/bout-v5-xzinterpolation-upgrader.py | 73 ++-------------- src/boutupgrader/__init__.py | 110 ++++++++++++++++++++++++ 8 files changed, 154 insertions(+), 442 deletions(-) create mode 100644 src/boutupgrader/__init__.py diff --git a/src/bout-v5-factory-upgrader.py b/src/bout-v5-factory-upgrader.py index c245f40..10151c7 100755 --- a/src/bout-v5-factory-upgrader.py +++ b/src/bout-v5-factory-upgrader.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 +import boutupgrader + import argparse import copy -import difflib import re @@ -198,18 +199,6 @@ def fix_create_method(factory, source): ) -def yes_or_no(question): - """Convert user input from yes/no variations to True/False - - """ - while True: - reply = input(question + " [y/N] ").lower().strip() - if not reply or reply[0] == "n": - return False - if reply[0] == "y": - return True - - def apply_fixes(factories, source, all_declarations=False): """Apply the various fixes for each factory to source. Returns modified source @@ -235,36 +224,10 @@ def apply_fixes(factories, source, all_declarations=False): return modified -def create_patch(filename, original, modified): - """Create a unified diff between original and modified - """ - - patch = "\n".join( - difflib.unified_diff( - original.splitlines(), - modified.splitlines(), - fromfile=filename, - tofile=filename, - lineterm="", - ) - ) - - return patch - - if __name__ == "__main__": parser = argparse.ArgumentParser(description="Fix types of factory-created objects") + parser = boutupgrader.default_args(parser) - parser.add_argument("files", action="store", nargs="+", help="Input files") - parser.add_argument( - "--force", "-f", action="store_true", help="Make changes without asking" - ) - parser.add_argument( - "--quiet", "-q", action="store_true", help="Don't print patches" - ) - parser.add_argument( - "--patch-only", "-p", action="store_true", help="Print the patches and exit" - ) parser.add_argument( "--all-declarations", "-a", @@ -274,9 +237,6 @@ def create_patch(filename, original, modified): args = parser.parse_args() - if args.force and args.patch_only: - raise ValueError("Incompatible options: --force and --patch") - for filename in args.files: with open(filename, "r") as f: contents = f.read() @@ -285,28 +245,7 @@ def create_patch(filename, original, modified): modified = apply_fixes( factories, contents, all_declarations=args.all_declarations ) - patch = create_patch(filename, original, modified) - - if args.patch_only: - print(patch) - continue - - if not patch: - if not args.quiet: - print("No changes to make to {}".format(filename)) - continue - - if not args.quiet: - print("\n******************************************") - print("Changes to {}\n".format(filename)) - print(patch) - print("\n******************************************") - - if args.force: - make_change = True - else: - make_change = yes_or_no("Make changes to {}?".format(filename)) - - if make_change: - with open(filename, "w") as f: - f.write(modified) + + boutupgrader.apply_or_display_patch( + filename, original, modified, args.patch_only, args.quiet, args.force + ) diff --git a/src/bout-v5-format-upgrader.py b/src/bout-v5-format-upgrader.py index 215fc8b..f1e5983 100755 --- a/src/bout-v5-format-upgrader.py +++ b/src/bout-v5-format-upgrader.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 +import boutupgrader + import argparse import copy -import difflib import re @@ -129,49 +130,10 @@ def apply_fixes(format_replacements, source): return modified -def yes_or_no(question): - """Convert user input from yes/no variations to True/False - - """ - while True: - reply = input(question + " [y/N] ").lower().strip() - if not reply or reply[0] == "n": - return False - if reply[0] == "y": - return True - - -def create_patch(filename, original, modified): - """Create a unified diff between original and modified - """ - - patch = "\n".join( - difflib.unified_diff( - original.splitlines(), - modified.splitlines(), - fromfile=filename, - tofile=filename, - lineterm="", - ) - ) - - return patch - - if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Fix format specifiers") - - parser.add_argument("files", action="store", nargs="+", help="Input files") - parser.add_argument( - "--force", "-f", action="store_true", help="Make changes without asking" - ) - parser.add_argument( - "--quiet", "-q", action="store_true", help="Don't print patches" + parser = boutupgrader.default_args( + argparse.ArgumentParser(description="Fix format specifiers") ) - parser.add_argument( - "--patch-only", "-p", action="store_true", help="Print the patches and exit" - ) - args = parser.parse_args() if args.force and args.patch_only: @@ -183,28 +145,7 @@ def create_patch(filename, original, modified): original = copy.deepcopy(contents) modified = apply_fixes(format_replacements, contents) - patch = create_patch(filename, original, modified) - - if args.patch_only: - print(patch) - continue - - if not patch: - if not args.quiet: - print("No changes to make to {}".format(filename)) - continue - - if not args.quiet: - print("\n******************************************") - print("Changes to {}\n".format(filename)) - print(patch) - print("\n******************************************") - - if args.force: - make_change = True - else: - make_change = yes_or_no("Make changes to {}?".format(filename)) - - if make_change: - with open(filename, "w") as f: - f.write(modified) + + boutupgrader.apply_or_display_patch( + filename, original, modified, args.patch_only, args.quiet, args.force + ) diff --git a/src/bout-v5-header-upgrader.py b/src/bout-v5-header-upgrader.py index 49a8fbc..785e263 100755 --- a/src/bout-v5-header-upgrader.py +++ b/src/bout-v5-header-upgrader.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 +import boutupgrader + import argparse import copy -import difflib import re import textwrap from pathlib import Path @@ -93,32 +94,6 @@ def apply_fixes(header_regex, source): return header_regex.sub(r"\1bout/\2\3", modified) -def yes_or_no(question): - """Convert user input from yes/no variations to True/False""" - while True: - reply = input(question + " [y/N] ").lower().strip() - if not reply or reply[0] == "n": - return False - if reply[0] == "y": - return True - - -def create_patch(filename, original, modified): - """Create a unified diff between original and modified""" - - patch = "\n".join( - difflib.unified_diff( - original.splitlines(), - modified.splitlines(), - fromfile=filename, - tofile=filename, - lineterm="", - ) - ) - - return patch - - if __name__ == "__main__": parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, @@ -145,16 +120,8 @@ def create_patch(filename, original, modified): """ ), ) + parser = boutupgrader.default_args(parser, pos_files=False) - parser.add_argument( - "--force", "-f", action="store_true", help="Make changes without asking" - ) - parser.add_argument( - "--quiet", "-q", action="store_true", help="Don't print patches" - ) - parser.add_argument( - "--patch-only", "-p", action="store_true", help="Print the patches and exit" - ) parser.add_argument( "--include-path", "-i", @@ -190,28 +157,7 @@ def create_patch(filename, original, modified): original = copy.deepcopy(contents) modified = apply_fixes(header_regex, contents) - patch = create_patch(filename, original, modified) - - if args.patch_only: - print(patch) - continue - - if not patch: - if not args.quiet: - print(f"No changes to make to {filename}") - continue - - if not args.quiet: - print("\n******************************************") - print(f"Changes to {filename}\n") - print(patch) - print("\n******************************************") - - if args.force: - make_change = True - else: - make_change = yes_or_no(f"Make changes to {filename}?") - - if make_change: - with open(filename, "w") as f: - f.write(modified) + + boutupgrader.apply_or_display_patch( + filename, original, modified, args.patch_only, args.quiet, args.force + ) diff --git a/src/bout-v5-input-file-upgrader.py b/src/bout-v5-input-file-upgrader.py index e2940ff..93a95de 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/bout-v5-input-file-upgrader.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 +from boutupgrader import create_patch, yes_or_no, default_args + import argparse import copy -import difflib import itertools import textwrap import warnings @@ -221,32 +222,6 @@ def apply_fixes(replacements, deleted, options_file): return modified -def yes_or_no(question): - """Convert user input from yes/no variations to True/False""" - while True: - reply = input(question + " [y/N] ").lower().strip() - if not reply or reply[0] == "n": - return False - if reply[0] == "y": - return True - - -def create_patch(filename, original, modified): - """Create a unified diff between original and modified""" - - patch = "\n".join( - difflib.unified_diff( - original.splitlines(), - modified.splitlines(), - fromfile=filename, - tofile=filename, - lineterm="", - ) - ) - - return patch - - def possibly_apply_patch(patch, options_file, quiet=False, force=False): """Possibly apply patch to options_file. If force is True, applies the patch without asking, overwriting any existing file. Otherwise, @@ -303,20 +278,8 @@ def possibly_apply_patch(patch, options_file, quiet=False, force=False): fixer" patch will still include it.""" ), ) + parser = default_args(parser) - parser.add_argument("files", action="store", nargs="+", help="Input files") - - force_patch_group = parser.add_mutually_exclusive_group() - force_patch_group.add_argument( - "--force", "-f", action="store_true", help="Make changes without asking" - ) - force_patch_group.add_argument( - "--patch-only", "-p", action="store_true", help="Print the patches and exit" - ) - - parser.add_argument( - "--quiet", "-q", action="store_true", help="Don't print patches" - ) parser.add_argument( "--accept-canonical", "-c", @@ -345,11 +308,7 @@ def possibly_apply_patch(patch, options_file, quiet=False, force=False): canonicalised_patch = create_patch(filename, original_source, str(original)) if canonicalised_patch and not args.patch_only: - print( - "WARNING: original input file '{}' not in canonical form!".format( - filename - ) - ) + print(f"WARNING: original input file '{filename}' not in canonical form!") applied_patch = possibly_apply_patch( canonicalised_patch, original, @@ -376,7 +335,7 @@ def possibly_apply_patch(patch, options_file, quiet=False, force=False): if not patch: if not args.quiet: - print("No changes to make to {}".format(filename)) + print(f"No changes to make to {filename}") continue possibly_apply_patch(patch, modified, args.quiet, args.force) diff --git a/src/bout-v5-macro-upgrader.py b/src/bout-v5-macro-upgrader.py index b6db6da..6066ff9 100755 --- a/src/bout-v5-macro-upgrader.py +++ b/src/bout-v5-macro-upgrader.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 +import boutupgrader + import argparse import copy -import difflib import re import textwrap @@ -319,35 +320,6 @@ def apply_fixes(replacements, source): return modified -def yes_or_no(question): - """Convert user input from yes/no variations to True/False - - """ - while True: - reply = input(question + " [y/N] ").lower().strip() - if not reply or reply[0] == "n": - return False - if reply[0] == "y": - return True - - -def create_patch(filename, original, modified): - """Create a unified diff between original and modified - """ - - patch = "\n".join( - difflib.unified_diff( - original.splitlines(), - modified.splitlines(), - fromfile=filename, - tofile=filename, - lineterm="", - ) - ) - - return patch - - if __name__ == "__main__": parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, @@ -371,17 +343,7 @@ def create_patch(filename, original, modified): """ ), ) - - parser.add_argument("files", action="store", nargs="+", help="Input files") - parser.add_argument( - "--force", "-f", action="store_true", help="Make changes without asking" - ) - parser.add_argument( - "--quiet", "-q", action="store_true", help="Don't print patches" - ) - parser.add_argument( - "--patch-only", "-p", action="store_true", help="Print the patches and exit" - ) + parser = boutupgrader.default_args(parser) args = parser.parse_args() @@ -394,28 +356,6 @@ def create_patch(filename, original, modified): original = copy.deepcopy(contents) modified = apply_fixes(MACRO_REPLACEMENTS, contents) - patch = create_patch(filename, original, modified) - - if args.patch_only: - print(patch) - continue - - if not patch: - if not args.quiet: - print("No changes to make to {}".format(filename)) - continue - - if not args.quiet: - print("\n******************************************") - print("Changes to {}\n".format(filename)) - print(patch) - print("\n******************************************") - - if args.force: - make_change = True - else: - make_change = yes_or_no("Make changes to {}?".format(filename)) - - if make_change: - with open(filename, "w") as f: - f.write(modified) + boutupgrader.apply_or_display_patch( + filename, original, modified, args.patch_only, args.quiet, args.force + ) diff --git a/src/bout-v5-physics-model-upgrader.py b/src/bout-v5-physics-model-upgrader.py index 4070343..e607a9a 100755 --- a/src/bout-v5-physics-model-upgrader.py +++ b/src/bout-v5-physics-model-upgrader.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 +import boutupgrader + import argparse import copy -import difflib import pathlib import re import textwrap @@ -366,35 +367,6 @@ def convert_legacy_model(source, name, error_on_warning): return "\n".join(source_lines) -def yes_or_no(question): - """Convert user input from yes/no variations to True/False - - """ - while True: - reply = input(question + " [y/N] ").lower().strip() - if not reply or reply[0] == "n": - return False - if reply[0] == "y": - return True - - -def create_patch(filename, original, modified): - """Create a unified diff between original and modified - """ - - patch = "\n".join( - difflib.unified_diff( - original.splitlines(), - modified.splitlines(), - fromfile=filename, - tofile=filename, - lineterm="", - ) - ) - - return patch - - if __name__ == "__main__": parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, @@ -415,17 +387,7 @@ def create_patch(filename, original, modified): """ ), ) - - parser.add_argument("files", action="store", nargs="+", help="Files to fix") - parser.add_argument( - "--force", "-f", action="store_true", help="Make changes without asking" - ) - parser.add_argument( - "--quiet", "-q", action="store_true", help="Don't print patches" - ) - parser.add_argument( - "--patch-only", "-p", action="store_true", help="Print the patches and exit" - ) + parser = boutupgrader.default_args(parser) parser.add_argument( "--name", "-n", @@ -437,9 +399,6 @@ def create_patch(filename, original, modified): args = parser.parse_args() - if args.force and args.patch_only: - raise ValueError("Incompatible options: --force and --patch") - for filename in args.files: with open(filename, "r") as f: contents = f.read() @@ -472,25 +431,6 @@ def create_patch(filename, original, modified): ) continue - patch = create_patch(filename, original, modified) - - if args.patch_only: - print(patch) - continue - - if not patch: - if not args.quiet: - print("No changes to make to {}".format(filename)) - continue - - if not args.quiet: - print("\n******************************************") - print("Changes to {}\n".format(filename)) - print(patch) - print("\n******************************************") - - make_change = args.force or yes_or_no("Make changes to {}?".format(filename)) - - if make_change: - with open(filename, "w") as f: - f.write(modified) + boutupgrader.apply_or_display_patch( + filename, original, modified, args.patch_only, args.quiet, args.force + ) diff --git a/src/bout-v5-xzinterpolation-upgrader.py b/src/bout-v5-xzinterpolation-upgrader.py index 5aa565e..6d4bf7f 100755 --- a/src/bout-v5-xzinterpolation-upgrader.py +++ b/src/bout-v5-xzinterpolation-upgrader.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 +import boutupgrader import argparse import copy -import difflib import re try: @@ -131,35 +131,6 @@ def fix_factories(old_factory, new_factory, source): ) -def create_patch(filename, original, modified): - """Create a unified diff between original and modified - """ - - patch = "\n".join( - difflib.unified_diff( - original.splitlines(), - modified.splitlines(), - fromfile=filename, - tofile=filename, - lineterm="", - ) - ) - - return patch - - -def yes_or_no(question): - """Convert user input from yes/no variations to True/False - - """ - while True: - reply = input(question + " [y/N] ").lower().strip() - if not reply or reply[0] == "n": - return False - if reply[0] == "y": - return True - - def apply_fixes(headers, interpolations, factories, source): """Apply all Interpolation fixes to source @@ -212,26 +183,13 @@ def clang_apply_fixes(headers, interpolations, factories, filename, source): if __name__ == "__main__": parser = argparse.ArgumentParser(description="Fix types of Interpolation objects") - - parser.add_argument("files", action="store", nargs="+", help="Input files") - parser.add_argument( - "--force", "-f", action="store_true", help="Make changes without asking" - ) - parser.add_argument( - "--quiet", "-q", action="store_true", help="Don't print patches" - ) - parser.add_argument( - "--patch-only", "-p", action="store_true", help="Print the patches and exit" - ) + parser = boutupgrader.default_args(parser) parser.add_argument( "--clang", action="store_true", help="Use libclang if available" ) args = parser.parse_args() - if args.force and args.patch_only: - raise ValueError("Incompatible options: --force and --patch") - if args.clang and not has_clang: raise RuntimeError( "libclang is not available. Please install libclang Python bindings" @@ -248,28 +206,7 @@ def clang_apply_fixes(headers, interpolations, factories, filename, source): ) else: modified = apply_fixes(headers, interpolations, factories, contents) - patch = create_patch(filename, original, modified) - if args.patch_only: - print(patch) - continue - - if not patch: - if not args.quiet: - print("No changes to make to {}".format(filename)) - continue - - if not args.quiet: - print("\n******************************************") - print("Changes to {}\n".format(filename)) - print(patch) - print("\n******************************************") - - if args.force: - make_change = True - else: - make_change = yes_or_no("Make changes to {}?".format(filename)) - - if make_change: - with open(filename, "w") as f: - f.write(modified) + boutupgrader.apply_or_display_patch( + filename, original, modified, args.patch_only, args.quiet, args.force + ) diff --git a/src/boutupgrader/__init__.py b/src/boutupgrader/__init__.py new file mode 100644 index 0000000..a33ef68 --- /dev/null +++ b/src/boutupgrader/__init__.py @@ -0,0 +1,110 @@ +import argparse +import difflib + + +def default_args( + parser: argparse.ArgumentParser, pos_files: bool = True +) -> argparse.ArgumentParser: + """Add some default arguments to the parser + + Parameters + ---------- + parser : + pos_files : + If ``True``, add positional argument for input files + + """ + + if pos_files: + parser.add_argument("files", action="store", nargs="+", help="Input files") + + parser.add_argument( + "--quiet", "-q", action="store_true", help="Don't print patches" + ) + + group = parser.add_mutually_exclusive_group() + group.add_argument( + "--force", "-f", action="store_true", help="Make changes without asking" + ) + group.add_argument( + "--patch-only", "-p", action="store_true", help="Print the patches and exit" + ) + + return parser + + +def yes_or_no(question: str) -> bool: + """Convert user input from yes/no variations to True/False""" + while True: + reply = input(f"{question} [y/N] ").lower().strip() + if not reply or reply[0] == "n": + return False + if reply[0] == "y": + return True + + +def create_patch(filename: str, original: str, modified: str) -> str: + """Create a unified diff between original and modified""" + + patch = "\n".join( + difflib.unified_diff( + original.splitlines(), + modified.splitlines(), + fromfile=filename, + tofile=filename, + lineterm="", + ) + ) + + return patch + + +def apply_or_display_patch( + filename: str, + original: str, + modified: str, + patch_only: bool, + quiet: bool, + force: bool, +): + """Given the original and modified versions of a file, display and/or apply it + + Parameters + ---------- + filename : str + Name of file + original : str + Original text of file + modified : str + Modified text of file + patch_only : bool + If ``True``, only print the patch + quiet : bool + If ``True``, don't print to screen, unless ``patch_only`` is + ``True`` + force : bool + If ``True``, always apply modifications to file + + """ + + patch = create_patch(filename, original, modified) + + if patch_only: + print(patch) + return + + if not patch: + if not quiet: + print(f"No changes to make to {filename}") + return + + if not quiet: + print("\n******************************************") + print(f"Changes to {filename}\n{patch}") + print("\n******************************************") + + make_change = force or yes_or_no(f"Make changes to {filename}?") + + if make_change: + with open(filename, "w") as f: + f.write(modified) From b3547d7b8c26926d9759d5f0912ae4fc2189511f Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 17 Jul 2024 15:06:20 +0100 Subject: [PATCH 62/82] Move upgrader scripts from BOUT-dev repo to boututils --- .../upgrader/bout_v5_factory_upgrader.py} | 6 +++--- .../upgrader/bout_v5_format_upgrader.py} | 6 +++--- .../upgrader/bout_v5_header_upgrader.py} | 6 +++--- .../upgrader/bout_v5_input_file_upgrader.py} | 4 ++-- .../upgrader/bout_v5_macro_upgrader.py} | 6 +++--- .../upgrader/bout_v5_physics_model_upgrader.py} | 6 +++--- .../upgrader/bout_v5_xzinterpolation_upgrader.py} | 6 +++--- .../__init__.py => boututils/upgrader/common.py} | 0 8 files changed, 20 insertions(+), 20 deletions(-) rename src/{bout-v5-factory-upgrader.py => boututils/upgrader/bout_v5_factory_upgrader.py} (98%) rename src/{bout-v5-format-upgrader.py => boututils/upgrader/bout_v5_format_upgrader.py} (96%) rename src/{bout-v5-header-upgrader.py => boututils/upgrader/bout_v5_header_upgrader.py} (97%) rename src/{bout-v5-input-file-upgrader.py => boututils/upgrader/bout_v5_input_file_upgrader.py} (99%) rename src/{bout-v5-macro-upgrader.py => boututils/upgrader/bout_v5_macro_upgrader.py} (98%) rename src/{bout-v5-physics-model-upgrader.py => boututils/upgrader/bout_v5_physics_model_upgrader.py} (99%) rename src/{bout-v5-xzinterpolation-upgrader.py => boututils/upgrader/bout_v5_xzinterpolation_upgrader.py} (97%) rename src/{boutupgrader/__init__.py => boututils/upgrader/common.py} (100%) diff --git a/src/bout-v5-factory-upgrader.py b/src/boututils/upgrader/bout_v5_factory_upgrader.py similarity index 98% rename from src/bout-v5-factory-upgrader.py rename to src/boututils/upgrader/bout_v5_factory_upgrader.py index 10151c7..be26fc8 100755 --- a/src/bout-v5-factory-upgrader.py +++ b/src/boututils/upgrader/bout_v5_factory_upgrader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import boutupgrader +from .common import default_args, apply_or_display_patch import argparse import copy @@ -226,7 +226,7 @@ def apply_fixes(factories, source, all_declarations=False): if __name__ == "__main__": parser = argparse.ArgumentParser(description="Fix types of factory-created objects") - parser = boutupgrader.default_args(parser) + parser = default_args(parser) parser.add_argument( "--all-declarations", @@ -246,6 +246,6 @@ def apply_fixes(factories, source, all_declarations=False): factories, contents, all_declarations=args.all_declarations ) - boutupgrader.apply_or_display_patch( + apply_or_display_patch( filename, original, modified, args.patch_only, args.quiet, args.force ) diff --git a/src/bout-v5-format-upgrader.py b/src/boututils/upgrader/bout_v5_format_upgrader.py similarity index 96% rename from src/bout-v5-format-upgrader.py rename to src/boututils/upgrader/bout_v5_format_upgrader.py index f1e5983..62553be 100755 --- a/src/bout-v5-format-upgrader.py +++ b/src/boututils/upgrader/bout_v5_format_upgrader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import boutupgrader +from .common import default_args, apply_or_display_patch import argparse import copy @@ -131,7 +131,7 @@ def apply_fixes(format_replacements, source): if __name__ == "__main__": - parser = boutupgrader.default_args( + parser = default_args( argparse.ArgumentParser(description="Fix format specifiers") ) args = parser.parse_args() @@ -146,6 +146,6 @@ def apply_fixes(format_replacements, source): modified = apply_fixes(format_replacements, contents) - boutupgrader.apply_or_display_patch( + apply_or_display_patch( filename, original, modified, args.patch_only, args.quiet, args.force ) diff --git a/src/bout-v5-header-upgrader.py b/src/boututils/upgrader/bout_v5_header_upgrader.py similarity index 97% rename from src/bout-v5-header-upgrader.py rename to src/boututils/upgrader/bout_v5_header_upgrader.py index 785e263..e9dc4fa 100755 --- a/src/bout-v5-header-upgrader.py +++ b/src/boututils/upgrader/bout_v5_header_upgrader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import boutupgrader +from .common import default_args, apply_or_display_patch import argparse import copy @@ -120,7 +120,7 @@ def apply_fixes(header_regex, source): """ ), ) - parser = boutupgrader.default_args(parser, pos_files=False) + parser = default_args(parser, pos_files=False) parser.add_argument( "--include-path", @@ -158,6 +158,6 @@ def apply_fixes(header_regex, source): modified = apply_fixes(header_regex, contents) - boutupgrader.apply_or_display_patch( + apply_or_display_patch( filename, original, modified, args.patch_only, args.quiet, args.force ) diff --git a/src/bout-v5-input-file-upgrader.py b/src/boututils/upgrader/bout_v5_input_file_upgrader.py similarity index 99% rename from src/bout-v5-input-file-upgrader.py rename to src/boututils/upgrader/bout_v5_input_file_upgrader.py index 93a95de..aa3d04b 100755 --- a/src/bout-v5-input-file-upgrader.py +++ b/src/boututils/upgrader/bout_v5_input_file_upgrader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from boutupgrader import create_patch, yes_or_no, default_args +from .common import create_patch, yes_or_no, default_args import argparse import copy @@ -9,7 +9,7 @@ import warnings from boutdata.data import BoutOptionsFile, BoutOptions -from boututils.boutwarnings import AlwaysWarning +from ..boutwarnings import AlwaysWarning def case_sensitive_init(self, name="root", parent=None): diff --git a/src/bout-v5-macro-upgrader.py b/src/boututils/upgrader/bout_v5_macro_upgrader.py similarity index 98% rename from src/bout-v5-macro-upgrader.py rename to src/boututils/upgrader/bout_v5_macro_upgrader.py index 6066ff9..5f5e505 100755 --- a/src/bout-v5-macro-upgrader.py +++ b/src/boututils/upgrader/bout_v5_macro_upgrader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import boutupgrader +from .common import default_args, apply_or_display_patch import argparse import copy @@ -343,7 +343,7 @@ def apply_fixes(replacements, source): """ ), ) - parser = boutupgrader.default_args(parser) + parser = default_args(parser) args = parser.parse_args() @@ -356,6 +356,6 @@ def apply_fixes(replacements, source): original = copy.deepcopy(contents) modified = apply_fixes(MACRO_REPLACEMENTS, contents) - boutupgrader.apply_or_display_patch( + apply_or_display_patch( filename, original, modified, args.patch_only, args.quiet, args.force ) diff --git a/src/bout-v5-physics-model-upgrader.py b/src/boututils/upgrader/bout_v5_physics_model_upgrader.py similarity index 99% rename from src/bout-v5-physics-model-upgrader.py rename to src/boututils/upgrader/bout_v5_physics_model_upgrader.py index e607a9a..bcb1399 100755 --- a/src/bout-v5-physics-model-upgrader.py +++ b/src/boututils/upgrader/bout_v5_physics_model_upgrader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import boutupgrader +from .common import default_args, apply_or_display_patch import argparse import copy @@ -387,7 +387,7 @@ def convert_legacy_model(source, name, error_on_warning): """ ), ) - parser = boutupgrader.default_args(parser) + parser = default_args(parser) parser.add_argument( "--name", "-n", @@ -431,6 +431,6 @@ def convert_legacy_model(source, name, error_on_warning): ) continue - boutupgrader.apply_or_display_patch( + apply_or_display_patch( filename, original, modified, args.patch_only, args.quiet, args.force ) diff --git a/src/bout-v5-xzinterpolation-upgrader.py b/src/boututils/upgrader/bout_v5_xzinterpolation_upgrader.py similarity index 97% rename from src/bout-v5-xzinterpolation-upgrader.py rename to src/boututils/upgrader/bout_v5_xzinterpolation_upgrader.py index 6d4bf7f..a226a0d 100755 --- a/src/bout-v5-xzinterpolation-upgrader.py +++ b/src/boututils/upgrader/bout_v5_xzinterpolation_upgrader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import boutupgrader +from .common import default_args, apply_or_display_patch import argparse import copy @@ -183,7 +183,7 @@ def clang_apply_fixes(headers, interpolations, factories, filename, source): if __name__ == "__main__": parser = argparse.ArgumentParser(description="Fix types of Interpolation objects") - parser = boutupgrader.default_args(parser) + parser = default_args(parser) parser.add_argument( "--clang", action="store_true", help="Use libclang if available" ) @@ -207,6 +207,6 @@ def clang_apply_fixes(headers, interpolations, factories, filename, source): else: modified = apply_fixes(headers, interpolations, factories, contents) - boutupgrader.apply_or_display_patch( + apply_or_display_patch( filename, original, modified, args.patch_only, args.quiet, args.force ) diff --git a/src/boutupgrader/__init__.py b/src/boututils/upgrader/common.py similarity index 100% rename from src/boutupgrader/__init__.py rename to src/boututils/upgrader/common.py From 7fe4f041ccf1570afb4237c1a149134277e9e744 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 23 Jul 2024 15:47:09 +0100 Subject: [PATCH 63/82] Add upgrader scripts as subcommands of top-level `bout-upgrader` --- pyproject.toml | 1 + src/boututils/upgrader/__init__.py | 44 +++++++++++++++++++ .../upgrader/bout_v5_factory_upgrader.py | 18 +++++--- .../upgrader/bout_v5_format_upgrader.py | 18 ++++---- .../upgrader/bout_v5_header_upgrader.py | 28 ++++++------ .../upgrader/bout_v5_input_file_upgrader.py | 14 +++--- .../upgrader/bout_v5_macro_upgrader.py | 16 +++---- .../bout_v5_physics_model_upgrader.py | 13 +++--- .../bout_v5_xzinterpolation_upgrader.py | 16 ++++--- 9 files changed, 115 insertions(+), 53 deletions(-) create mode 100644 src/boututils/upgrader/__init__.py diff --git a/pyproject.toml b/pyproject.toml index 2f83530..4daba6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ docs = [ [project.scripts] bout-squashoutput = "boutdata.scripts.bout_squashoutput:main" +bout-upgrader = "boututils.upgrader:main" [tool.setuptools.dynamic] version = { attr = "setuptools_scm.get_version" } diff --git a/src/boututils/upgrader/__init__.py b/src/boututils/upgrader/__init__.py new file mode 100644 index 0000000..cf9fb4e --- /dev/null +++ b/src/boututils/upgrader/__init__.py @@ -0,0 +1,44 @@ +import argparse + +from .bout_v5_factory_upgrader import add_parser as add_factory_parser +from .bout_v5_format_upgrader import add_parser as add_format_parser +from .bout_v5_header_upgrader import add_parser as add_header_parser +from .bout_v5_input_file_upgrader import add_parser as add_input_parser +from .bout_v5_macro_upgrader import add_parser as add_macro_parser +from .bout_v5_physics_model_upgrader import add_parser as add_model_parser +from .bout_v5_xzinterpolation_upgrader import add_parser as add_xzinterp_parser + + +def main(): + # Parent parser that has arguments common to all subcommands + common_args = argparse.ArgumentParser(add_help=False) + common_args.add_argument( + "--quiet", "-q", action="store_true", help="Don't print patches" + ) + force_or_patch_group = common_args.add_mutually_exclusive_group() + force_or_patch_group.add_argument( + "--force", "-f", action="store_true", help="Make changes without asking" + ) + force_or_patch_group.add_argument( + "--patch-only", "-p", action="store_true", help="Print the patches and exit" + ) + + # Parent parser for commands that always take a list of files + files_args = argparse.ArgumentParser(add_help=False) + files_args.add_argument("files", action="store", nargs="+", help="Input files") + + parser = argparse.ArgumentParser( + description="Upgrade BOUT++ source and input files to newer versions" + ) + subcommand = parser.add_subparsers(title="subcommands", required=True) + + add_factory_parser(subcommand, common_args, files_args) + add_format_parser(subcommand, common_args, files_args) + add_header_parser(subcommand, common_args) + add_input_parser(subcommand, common_args, files_args) + add_macro_parser(subcommand, common_args, files_args) + add_model_parser(subcommand, common_args, files_args) + add_xzinterp_parser(subcommand, common_args, files_args) + + args = parser.parse_args() + args.func(args) diff --git a/src/boututils/upgrader/bout_v5_factory_upgrader.py b/src/boututils/upgrader/bout_v5_factory_upgrader.py index be26fc8..12100ca 100755 --- a/src/boututils/upgrader/bout_v5_factory_upgrader.py +++ b/src/boututils/upgrader/bout_v5_factory_upgrader.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 -from .common import default_args, apply_or_display_patch +from .common import apply_or_display_patch -import argparse import copy import re @@ -224,19 +223,24 @@ def apply_fixes(factories, source, all_declarations=False): return modified -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Fix types of factory-created objects") - parser = default_args(parser) - +def add_parser(subcommand, default_args, files_args): + factory_help = "Fix types of factory-created objects" + parser = subcommand.add_parser( + "factory", + help=factory_help, + description=factory_help, + parents=[default_args, files_args], + ) parser.add_argument( "--all-declarations", "-a", action="store_true", help="Fix all declarations of factory types, not just variables created from factories", ) + parser.set_defaults(func=run) - args = parser.parse_args() +def run(args): for filename in args.files: with open(filename, "r") as f: contents = f.read() diff --git a/src/boututils/upgrader/bout_v5_format_upgrader.py b/src/boututils/upgrader/bout_v5_format_upgrader.py index 62553be..bc2c4c5 100755 --- a/src/boututils/upgrader/bout_v5_format_upgrader.py +++ b/src/boututils/upgrader/bout_v5_format_upgrader.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 -from .common import default_args, apply_or_display_patch +from .common import apply_or_display_patch -import argparse import copy import re @@ -130,15 +129,18 @@ def apply_fixes(format_replacements, source): return modified -if __name__ == "__main__": - parser = default_args( - argparse.ArgumentParser(description="Fix format specifiers") +def add_parser(subcommand, default_args, files_args): + format_help = "Fix format specifiers" + parser = subcommand.add_parser( + "format", + description=format_help, + help=format_help, + parents=[default_args, files_args], ) - args = parser.parse_args() + parser.set_defaults(func=run) - if args.force and args.patch_only: - raise ValueError("Incompatible options: --force and --patch") +def run(args): for filename in args.files: with open(filename, "r") as f: contents = f.read() diff --git a/src/boututils/upgrader/bout_v5_header_upgrader.py b/src/boututils/upgrader/bout_v5_header_upgrader.py index e9dc4fa..a7e7198 100755 --- a/src/boututils/upgrader/bout_v5_header_upgrader.py +++ b/src/boututils/upgrader/bout_v5_header_upgrader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from .common import default_args, apply_or_display_patch +from .common import apply_or_display_patch import argparse import copy @@ -8,7 +8,7 @@ import textwrap from pathlib import Path from typing import List -from subprocess import run +import subprocess header_shim_sentinel = "// BOUT++ header shim" @@ -16,7 +16,7 @@ header_warning = f"""\ #pragma once {header_shim_sentinel} -#warning Header "{{0}}" has moved to "bout/{{0}}". Run `bin/bout-v5-header-upgrader.py` to fix +#warning Header "{{0}}" has moved to "bout/{{0}}". Run `bout-upgrader header` to fix #include "bout/{{0}}" """ @@ -53,7 +53,7 @@ def fix_library_header_locations( print("No headers to move!") return - out = run("git diff-index --cached HEAD --quiet", shell=True) + out = subprocess.run("git diff-index --cached HEAD --quiet", shell=True) if out.returncode: raise RuntimeError( "git index not clean! Please commit or discard any changes before continuing" @@ -94,8 +94,10 @@ def apply_fixes(header_regex, source): return header_regex.sub(r"\1bout/\2\3", modified) -if __name__ == "__main__": - parser = argparse.ArgumentParser( +def add_parser(subcommand, default_args): + parser = subcommand.add_parser( + "header", + help="Fix deprecated header locations", formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent( """\ @@ -119,8 +121,8 @@ def apply_fixes(header_regex, source): """ ), + parents=[default_args], ) - parser = default_args(parser, pos_files=False) parser.add_argument( "--include-path", @@ -130,19 +132,17 @@ def apply_fixes(header_regex, source): type=Path, ) - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument("--files", nargs="*", action="store", help="Input files") - group.add_argument( + header_group = parser.add_mutually_exclusive_group(required=True) + header_group.add_argument("--files", nargs="*", action="store", help="Input files") + header_group.add_argument( "--move-deprecated-headers", action="store_true", help="Move the deprecated headers", ) + parser.set_defaults(func=run) - args = parser.parse_args() - - if args.force and args.patch_only: - raise ValueError("Incompatible options: --force and --patch") +def run(args): deprecated_headers = deprecated_header_list(args.include_path) if args.move_deprecated_headers: diff --git a/src/boututils/upgrader/bout_v5_input_file_upgrader.py b/src/boututils/upgrader/bout_v5_input_file_upgrader.py index aa3d04b..d9255e3 100755 --- a/src/boututils/upgrader/bout_v5_input_file_upgrader.py +++ b/src/boututils/upgrader/bout_v5_input_file_upgrader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from .common import create_patch, yes_or_no, default_args +from .common import create_patch, yes_or_no import argparse import copy @@ -243,9 +243,11 @@ def possibly_apply_patch(patch, options_file, quiet=False, force=False): return make_change -if __name__ == "__main__": - parser = argparse.ArgumentParser( +def add_parser(subcommand, default_args, files_args): + parser = subcommand.add_parser( + "input", formatter_class=argparse.RawDescriptionHelpFormatter, + help="Fix input files", description=textwrap.dedent( """\ Fix input files for BOUT++ v5+ @@ -277,8 +279,8 @@ def possibly_apply_patch(patch, options_file, quiet=False, force=False): presented first. If you choose not to apply this patch, the "upgrade fixer" patch will still include it.""" ), + parents=[default_args, files_args], ) - parser = default_args(parser) parser.add_argument( "--accept-canonical", @@ -292,8 +294,10 @@ def possibly_apply_patch(patch, options_file, quiet=False, force=False): action="store_true", help="Only check/fix canonicalisation", ) + parser.set_defaults(func=run) - args = parser.parse_args() + +def run(args): warnings.simplefilter("ignore", AlwaysWarning) diff --git a/src/boututils/upgrader/bout_v5_macro_upgrader.py b/src/boututils/upgrader/bout_v5_macro_upgrader.py index 5f5e505..334d07a 100755 --- a/src/boututils/upgrader/bout_v5_macro_upgrader.py +++ b/src/boututils/upgrader/bout_v5_macro_upgrader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from .common import default_args, apply_or_display_patch +from .common import apply_or_display_patch import argparse import copy @@ -320,8 +320,10 @@ def apply_fixes(replacements, source): return modified -if __name__ == "__main__": - parser = argparse.ArgumentParser( +def add_parser(subcommand, default_args, files_args): + parser = subcommand.add_parser( + "macro", + help="Fix macro defines", formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent( """\ @@ -342,14 +344,12 @@ def apply_fixes(replacements, source): Please check the diff output carefully! """ ), + parents=[default_args, files_args], ) - parser = default_args(parser) + parser.set_defaults(func=run) - args = parser.parse_args() - - if args.force and args.patch_only: - raise ValueError("Incompatible options: --force and --patch") +def run(args): for filename in args.files: with open(filename, "r") as f: contents = f.read() diff --git a/src/boututils/upgrader/bout_v5_physics_model_upgrader.py b/src/boututils/upgrader/bout_v5_physics_model_upgrader.py index bcb1399..a93e7c9 100755 --- a/src/boututils/upgrader/bout_v5_physics_model_upgrader.py +++ b/src/boututils/upgrader/bout_v5_physics_model_upgrader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from .common import default_args, apply_or_display_patch +from .common import apply_or_display_patch import argparse import copy @@ -367,8 +367,10 @@ def convert_legacy_model(source, name, error_on_warning): return "\n".join(source_lines) -if __name__ == "__main__": - parser = argparse.ArgumentParser( +def add_parser(subcommand, default_args, files_args): + parser = subcommand.add_parser( + "model", + help="Upgrade legacy physics models", formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent( """\ @@ -386,8 +388,8 @@ def convert_legacy_model(source, name, error_on_warning): name>' to give a different name. """ ), + parents=[default_args, files_args], ) - parser = default_args(parser) parser.add_argument( "--name", "-n", @@ -396,9 +398,10 @@ def convert_legacy_model(source, name, error_on_warning): type=str, help="Name for new PhysicsModel class, default is from filename", ) + parser.set_defaults(func=run) - args = parser.parse_args() +def run(args): for filename in args.files: with open(filename, "r") as f: contents = f.read() diff --git a/src/boututils/upgrader/bout_v5_xzinterpolation_upgrader.py b/src/boututils/upgrader/bout_v5_xzinterpolation_upgrader.py index a226a0d..1e342a3 100755 --- a/src/boututils/upgrader/bout_v5_xzinterpolation_upgrader.py +++ b/src/boututils/upgrader/bout_v5_xzinterpolation_upgrader.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 -from .common import default_args, apply_or_display_patch +from .common import apply_or_display_patch -import argparse import copy import re @@ -181,15 +180,20 @@ def clang_apply_fixes(headers, interpolations, factories, filename, source): return modified -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Fix types of Interpolation objects") - parser = default_args(parser) +def add_parser(subcommand, default_args, files_args): + parser = subcommand.add_parser( + "xzinterp", + help="Fix types of Interpolation objects", + description="Fix types of Interpolation objects", + parents=[default_args, files_args], + ) parser.add_argument( "--clang", action="store_true", help="Use libclang if available" ) + parser.set_defaults(func=run) - args = parser.parse_args() +def run(args): if args.clang and not has_clang: raise RuntimeError( "libclang is not available. Please install libclang Python bindings" From cf58eca53263d4b2e4a96137106177a8baab5101 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 23 Jul 2024 15:47:29 +0100 Subject: [PATCH 64/82] Move import of `boutdata` into function to avoid circular dependency --- src/boututils/upgrader/bout_v5_input_file_upgrader.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/boututils/upgrader/bout_v5_input_file_upgrader.py b/src/boututils/upgrader/bout_v5_input_file_upgrader.py index d9255e3..5576c47 100755 --- a/src/boututils/upgrader/bout_v5_input_file_upgrader.py +++ b/src/boututils/upgrader/bout_v5_input_file_upgrader.py @@ -8,7 +8,6 @@ import textwrap import warnings -from boutdata.data import BoutOptionsFile, BoutOptions from ..boutwarnings import AlwaysWarning @@ -22,10 +21,6 @@ def case_sensitive_init(self, name="root", parent=None): self._comment_whitespace = dict() -# Monky-patch BoutOptions to make sure it's case sensitive -BoutOptions.__init__ = case_sensitive_init - - # This should be a list of dicts, each containing "old", "new" and optionally "new_values". # The values of "old"/"new" keys should be the old/new names of input file values or # sections. The value of "new_values" is a dict containing replacements for values of the @@ -298,6 +293,9 @@ def add_parser(subcommand, default_args, files_args): def run(args): + from boutdata.data import BoutOptionsFile, BoutOptions + # Monkey-patch BoutOptions to make sure it's case sensitive + BoutOptions.__init__ = case_sensitive_init warnings.simplefilter("ignore", AlwaysWarning) From 050437e1d08b7c9a4a49fbd1745ebeb9322af3f3 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Tue, 23 Jul 2024 15:48:34 +0100 Subject: [PATCH 65/82] Apply black --- .../upgrader/bout_v5_factory_upgrader.py | 4 +--- .../upgrader/bout_v5_format_upgrader.py | 21 +++++------------- .../upgrader/bout_v5_input_file_upgrader.py | 1 + .../upgrader/bout_v5_macro_upgrader.py | 16 +++++--------- .../bout_v5_physics_model_upgrader.py | 22 +++++++------------ 5 files changed, 20 insertions(+), 44 deletions(-) diff --git a/src/boututils/upgrader/bout_v5_factory_upgrader.py b/src/boututils/upgrader/bout_v5_factory_upgrader.py index 12100ca..1c01a2b 100755 --- a/src/boututils/upgrader/bout_v5_factory_upgrader.py +++ b/src/boututils/upgrader/bout_v5_factory_upgrader.py @@ -173,9 +173,7 @@ def fix_deletions(variables, source): def fix_create_method(factory, source): - """Fix change of name of factory `create` method - - """ + """Fix change of name of factory `create` method""" if "old_create_method" not in factory: return source diff --git a/src/boututils/upgrader/bout_v5_format_upgrader.py b/src/boututils/upgrader/bout_v5_format_upgrader.py index bc2c4c5..fc96f22 100755 --- a/src/boututils/upgrader/bout_v5_format_upgrader.py +++ b/src/boututils/upgrader/bout_v5_format_upgrader.py @@ -23,9 +23,7 @@ def fix_format_replacement(format_replacement, source): - """Replace printf format with fmt format - - """ + """Replace printf format with fmt format""" return re.sub( r"%([0-9]*\.?[0-9]*){}".format(format_replacement[0]), r"{{:\1{}}}".format(format_replacement[1]), @@ -34,9 +32,7 @@ def fix_format_replacement(format_replacement, source): def fix_trivial_format(source): - """Reduce trivial formatting of strings to just the string - - """ + """Reduce trivial formatting of strings to just the string""" def trivial_replace(match): if match.group(2): @@ -60,9 +56,7 @@ def trivial_replace(match): def fix_string_c_str(source): - """Fix formats that use {:s} where the replacement is using std::string::c_str - - """ + """Fix formats that use {:s} where the replacement is using std::string::c_str""" return re.sub( r""" (".*{:s}[^;]*?",) # A format string containing {:s} @@ -75,9 +69,7 @@ def fix_string_c_str(source): def fix_trace(source): - """Fix TRACE macros where fix_string_c_str has failed for some reason - - """ + """Fix TRACE macros where fix_string_c_str has failed for some reason""" return re.sub( r""" (TRACE\(".*{:s}.*",) @@ -90,10 +82,7 @@ def fix_trace(source): def fix_toString_c_str(source): - """Fix formats that call toString where the replacement is using std::string::c_str - - - """ + """Fix formats that call toString where the replacement is using std::string::c_str""" return re.sub( r""" (".*{:s}[^;]*?",.*?) # A format string containing {:s} diff --git a/src/boututils/upgrader/bout_v5_input_file_upgrader.py b/src/boututils/upgrader/bout_v5_input_file_upgrader.py index 5576c47..eb9fb6f 100755 --- a/src/boututils/upgrader/bout_v5_input_file_upgrader.py +++ b/src/boututils/upgrader/bout_v5_input_file_upgrader.py @@ -294,6 +294,7 @@ def add_parser(subcommand, default_args, files_args): def run(args): from boutdata.data import BoutOptionsFile, BoutOptions + # Monkey-patch BoutOptions to make sure it's case sensitive BoutOptions.__init__ = case_sensitive_init diff --git a/src/boututils/upgrader/bout_v5_macro_upgrader.py b/src/boututils/upgrader/bout_v5_macro_upgrader.py index 334d07a..2fe2343 100755 --- a/src/boututils/upgrader/bout_v5_macro_upgrader.py +++ b/src/boututils/upgrader/bout_v5_macro_upgrader.py @@ -179,8 +179,7 @@ def fix_include_version_header(old, headers, source): - """Make sure version.hxx header is included - """ + """Make sure version.hxx header is included""" if not isinstance(headers, list): headers = [headers] @@ -218,9 +217,7 @@ def fix_include_version_header(old, headers, source): def fix_ifdefs(old, source): - """Remove any code inside #ifdef/#ifndef blocks that would now not be compiled - - """ + """Remove any code inside #ifdef/#ifndef blocks that would now not be compiled""" source_lines = source.splitlines() # Something to keep track of nested sections @@ -281,21 +278,18 @@ def fix_ifdefs(old, source): def fix_always_defined_macros(old, new, source): - """Fix '#ifdef's that should become plain '#if' - """ + """Fix '#ifdef's that should become plain '#if'""" new_source = re.sub(r"#ifdef\s+{}\b".format(old), r"#if {}".format(new), source) return re.sub(r"#ifndef\s+{}\b".format(old), r"#if !{}".format(new), new_source) def fix_replacement(old, new, source): - """Straight replacements - """ + """Straight replacements""" return re.sub(r'([^"_])\b{}\b([^"_])'.format(old), r"\1{}\2".format(new), source) def apply_fixes(replacements, source): - """Apply all fixes in this module - """ + """Apply all fixes in this module""" modified = copy.deepcopy(source) for replacement in replacements: diff --git a/src/boututils/upgrader/bout_v5_physics_model_upgrader.py b/src/boututils/upgrader/bout_v5_physics_model_upgrader.py index a93e7c9..9ca72e8 100755 --- a/src/boututils/upgrader/bout_v5_physics_model_upgrader.py +++ b/src/boututils/upgrader/bout_v5_physics_model_upgrader.py @@ -53,7 +53,7 @@ def create_function_signature_re(function_name, argument_types): ] ) - return fr"int\s+{function_name}\s*\({arguments}\)" + return rf"int\s+{function_name}\s*\({arguments}\)" LEGACY_MODEL_INCLUDE_RE = re.compile( @@ -76,9 +76,7 @@ def create_function_signature_re(function_name, argument_types): def has_split_operator(source): - """Return the names of the split operator functions if set, otherwise False - - """ + """Return the names of the split operator functions if set, otherwise False""" match = SPLIT_OPERATOR_RE.search(source) if not match: @@ -88,15 +86,12 @@ def has_split_operator(source): def is_legacy_model(source): - """Return true if the source is a legacy physics model - - """ + """Return true if the source is a legacy physics model""" return LEGACY_MODEL_INCLUDE_RE.search(source) is not None def find_last_include(source_lines): - """Return the line number after the last #include (or 0 if no includes) - """ + """Return the line number after the last #include (or 0 if no includes)""" for number, line in enumerate(reversed(source_lines)): if line.startswith("#include"): return len(source_lines) - number @@ -164,7 +159,7 @@ def fix_model_operator( arguments = ", ".join(arg_names) # Modify the definition: it's out-of-line so we need the qualified name - modified = operator_re.sub(fr"int {model_name}::{new_name}({arguments})", source) + modified = operator_re.sub(rf"int {model_name}::{new_name}({arguments})", source) # Create the declaration return ( @@ -270,7 +265,7 @@ def convert_old_solver_api(source, name): if decl: method_decls.append(decl) # Almost a straight replacement, but it's now a member-function pointer - source = PRECON_RE.sub(fr"setPrecon(&{name}::\1)", source) + source = PRECON_RE.sub(rf"setPrecon(&{name}::\1)", source) # Fix uses of solver->setJacobian, basically the same as for setPrecon jacobians = JACOBIAN_RE.findall(source) @@ -280,7 +275,7 @@ def convert_old_solver_api(source, name): ) if decl: method_decls.append(decl) - source = JACOBIAN_RE.sub(fr"setJacobian(&{name}::\1)", source) + source = JACOBIAN_RE.sub(rf"setJacobian(&{name}::\1)", source) # If we didn't find any free functions that need to be made into # methods, we're done @@ -312,8 +307,7 @@ def convert_old_solver_api(source, name): def convert_legacy_model(source, name, error_on_warning): - """Convert a legacy physics model to a PhysicsModel - """ + """Convert a legacy physics model to a PhysicsModel""" if not is_legacy_model(source): return source From 1ac849924be7ee21f19bc87c3e201531c0dc8ff2 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 12 Sep 2024 17:01:05 +0100 Subject: [PATCH 66/82] Move upgrader script to own package --- pyproject.toml | 2 +- src/{boututils/upgrader => boutupgrader}/__init__.py | 0 .../upgrader => boutupgrader}/bout_v5_factory_upgrader.py | 0 .../upgrader => boutupgrader}/bout_v5_format_upgrader.py | 0 .../upgrader => boutupgrader}/bout_v5_header_upgrader.py | 0 .../upgrader => boutupgrader}/bout_v5_input_file_upgrader.py | 2 +- .../upgrader => boutupgrader}/bout_v5_macro_upgrader.py | 0 .../upgrader => boutupgrader}/bout_v5_physics_model_upgrader.py | 0 .../bout_v5_xzinterpolation_upgrader.py | 0 src/{boututils/upgrader => boutupgrader}/common.py | 0 10 files changed, 2 insertions(+), 2 deletions(-) rename src/{boututils/upgrader => boutupgrader}/__init__.py (100%) rename src/{boututils/upgrader => boutupgrader}/bout_v5_factory_upgrader.py (100%) rename src/{boututils/upgrader => boutupgrader}/bout_v5_format_upgrader.py (100%) rename src/{boututils/upgrader => boutupgrader}/bout_v5_header_upgrader.py (100%) rename src/{boututils/upgrader => boutupgrader}/bout_v5_input_file_upgrader.py (99%) rename src/{boututils/upgrader => boutupgrader}/bout_v5_macro_upgrader.py (100%) rename src/{boututils/upgrader => boutupgrader}/bout_v5_physics_model_upgrader.py (100%) rename src/{boututils/upgrader => boutupgrader}/bout_v5_xzinterpolation_upgrader.py (100%) rename src/{boututils/upgrader => boutupgrader}/common.py (100%) diff --git a/pyproject.toml b/pyproject.toml index 4daba6b..8c025df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ docs = [ [project.scripts] bout-squashoutput = "boutdata.scripts.bout_squashoutput:main" -bout-upgrader = "boututils.upgrader:main" +bout-upgrader = "boutupgrader:main" [tool.setuptools.dynamic] version = { attr = "setuptools_scm.get_version" } diff --git a/src/boututils/upgrader/__init__.py b/src/boutupgrader/__init__.py similarity index 100% rename from src/boututils/upgrader/__init__.py rename to src/boutupgrader/__init__.py diff --git a/src/boututils/upgrader/bout_v5_factory_upgrader.py b/src/boutupgrader/bout_v5_factory_upgrader.py similarity index 100% rename from src/boututils/upgrader/bout_v5_factory_upgrader.py rename to src/boutupgrader/bout_v5_factory_upgrader.py diff --git a/src/boututils/upgrader/bout_v5_format_upgrader.py b/src/boutupgrader/bout_v5_format_upgrader.py similarity index 100% rename from src/boututils/upgrader/bout_v5_format_upgrader.py rename to src/boutupgrader/bout_v5_format_upgrader.py diff --git a/src/boututils/upgrader/bout_v5_header_upgrader.py b/src/boutupgrader/bout_v5_header_upgrader.py similarity index 100% rename from src/boututils/upgrader/bout_v5_header_upgrader.py rename to src/boutupgrader/bout_v5_header_upgrader.py diff --git a/src/boututils/upgrader/bout_v5_input_file_upgrader.py b/src/boutupgrader/bout_v5_input_file_upgrader.py similarity index 99% rename from src/boututils/upgrader/bout_v5_input_file_upgrader.py rename to src/boutupgrader/bout_v5_input_file_upgrader.py index eb9fb6f..6eec352 100755 --- a/src/boututils/upgrader/bout_v5_input_file_upgrader.py +++ b/src/boutupgrader/bout_v5_input_file_upgrader.py @@ -8,7 +8,7 @@ import textwrap import warnings -from ..boutwarnings import AlwaysWarning +from boututils.boutwarnings import AlwaysWarning def case_sensitive_init(self, name="root", parent=None): diff --git a/src/boututils/upgrader/bout_v5_macro_upgrader.py b/src/boutupgrader/bout_v5_macro_upgrader.py similarity index 100% rename from src/boututils/upgrader/bout_v5_macro_upgrader.py rename to src/boutupgrader/bout_v5_macro_upgrader.py diff --git a/src/boututils/upgrader/bout_v5_physics_model_upgrader.py b/src/boutupgrader/bout_v5_physics_model_upgrader.py similarity index 100% rename from src/boututils/upgrader/bout_v5_physics_model_upgrader.py rename to src/boutupgrader/bout_v5_physics_model_upgrader.py diff --git a/src/boututils/upgrader/bout_v5_xzinterpolation_upgrader.py b/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py similarity index 100% rename from src/boututils/upgrader/bout_v5_xzinterpolation_upgrader.py rename to src/boutupgrader/bout_v5_xzinterpolation_upgrader.py diff --git a/src/boututils/upgrader/common.py b/src/boutupgrader/common.py similarity index 100% rename from src/boututils/upgrader/common.py rename to src/boutupgrader/common.py From 960b69b5974f7f84365922a2160d6176d5a49de9 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 12 Sep 2024 17:02:20 +0100 Subject: [PATCH 67/82] Remove execute permissions from individual upgrader modules --- src/boutupgrader/bout_v5_factory_upgrader.py | 2 -- src/boutupgrader/bout_v5_format_upgrader.py | 2 -- src/boutupgrader/bout_v5_header_upgrader.py | 2 -- src/boutupgrader/bout_v5_input_file_upgrader.py | 2 -- src/boutupgrader/bout_v5_macro_upgrader.py | 2 -- src/boutupgrader/bout_v5_physics_model_upgrader.py | 2 -- src/boutupgrader/bout_v5_xzinterpolation_upgrader.py | 2 -- 7 files changed, 14 deletions(-) mode change 100755 => 100644 src/boutupgrader/bout_v5_factory_upgrader.py mode change 100755 => 100644 src/boutupgrader/bout_v5_format_upgrader.py mode change 100755 => 100644 src/boutupgrader/bout_v5_header_upgrader.py mode change 100755 => 100644 src/boutupgrader/bout_v5_input_file_upgrader.py mode change 100755 => 100644 src/boutupgrader/bout_v5_macro_upgrader.py mode change 100755 => 100644 src/boutupgrader/bout_v5_physics_model_upgrader.py mode change 100755 => 100644 src/boutupgrader/bout_v5_xzinterpolation_upgrader.py diff --git a/src/boutupgrader/bout_v5_factory_upgrader.py b/src/boutupgrader/bout_v5_factory_upgrader.py old mode 100755 new mode 100644 index 1c01a2b..bbc4aa8 --- a/src/boutupgrader/bout_v5_factory_upgrader.py +++ b/src/boutupgrader/bout_v5_factory_upgrader.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - from .common import apply_or_display_patch import copy diff --git a/src/boutupgrader/bout_v5_format_upgrader.py b/src/boutupgrader/bout_v5_format_upgrader.py old mode 100755 new mode 100644 index fc96f22..8aff6b0 --- a/src/boutupgrader/bout_v5_format_upgrader.py +++ b/src/boutupgrader/bout_v5_format_upgrader.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - from .common import apply_or_display_patch import copy diff --git a/src/boutupgrader/bout_v5_header_upgrader.py b/src/boutupgrader/bout_v5_header_upgrader.py old mode 100755 new mode 100644 index a7e7198..77ec93f --- a/src/boutupgrader/bout_v5_header_upgrader.py +++ b/src/boutupgrader/bout_v5_header_upgrader.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - from .common import apply_or_display_patch import argparse diff --git a/src/boutupgrader/bout_v5_input_file_upgrader.py b/src/boutupgrader/bout_v5_input_file_upgrader.py old mode 100755 new mode 100644 index 6eec352..47bbae9 --- a/src/boutupgrader/bout_v5_input_file_upgrader.py +++ b/src/boutupgrader/bout_v5_input_file_upgrader.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - from .common import create_patch, yes_or_no import argparse diff --git a/src/boutupgrader/bout_v5_macro_upgrader.py b/src/boutupgrader/bout_v5_macro_upgrader.py old mode 100755 new mode 100644 index 2fe2343..33e02e0 --- a/src/boutupgrader/bout_v5_macro_upgrader.py +++ b/src/boutupgrader/bout_v5_macro_upgrader.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - from .common import apply_or_display_patch import argparse diff --git a/src/boutupgrader/bout_v5_physics_model_upgrader.py b/src/boutupgrader/bout_v5_physics_model_upgrader.py old mode 100755 new mode 100644 index 9ca72e8..84e6c74 --- a/src/boutupgrader/bout_v5_physics_model_upgrader.py +++ b/src/boutupgrader/bout_v5_physics_model_upgrader.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - from .common import apply_or_display_patch import argparse diff --git a/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py b/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py old mode 100755 new mode 100644 index 1e342a3..c0d60b4 --- a/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py +++ b/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - from .common import apply_or_display_patch import copy From 2f8df50ddb9159a506ed40b15d8feca22c3335b6 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 12 Sep 2024 17:02:56 +0100 Subject: [PATCH 68/82] Remove unneeded function --- src/boutupgrader/common.py | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/src/boutupgrader/common.py b/src/boutupgrader/common.py index a33ef68..60a00d1 100644 --- a/src/boutupgrader/common.py +++ b/src/boutupgrader/common.py @@ -1,38 +1,6 @@ -import argparse import difflib -def default_args( - parser: argparse.ArgumentParser, pos_files: bool = True -) -> argparse.ArgumentParser: - """Add some default arguments to the parser - - Parameters - ---------- - parser : - pos_files : - If ``True``, add positional argument for input files - - """ - - if pos_files: - parser.add_argument("files", action="store", nargs="+", help="Input files") - - parser.add_argument( - "--quiet", "-q", action="store_true", help="Don't print patches" - ) - - group = parser.add_mutually_exclusive_group() - group.add_argument( - "--force", "-f", action="store_true", help="Make changes without asking" - ) - group.add_argument( - "--patch-only", "-p", action="store_true", help="Print the patches and exit" - ) - - return parser - - def yes_or_no(question: str) -> bool: """Convert user input from yes/no variations to True/False""" while True: From 5ac848b34eb19e306c977f3ebe7634f6fa4f0964 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 12 Sep 2024 17:09:36 +0100 Subject: [PATCH 69/82] Sort headers --- src/boutupgrader/bout_v5_factory_upgrader.py | 3 +-- src/boutupgrader/bout_v5_format_upgrader.py | 3 +-- src/boutupgrader/bout_v5_header_upgrader.py | 5 ++--- src/boutupgrader/bout_v5_input_file_upgrader.py | 6 +++--- src/boutupgrader/bout_v5_macro_upgrader.py | 4 ++-- src/boutupgrader/bout_v5_physics_model_upgrader.py | 3 +-- src/boutupgrader/bout_v5_xzinterpolation_upgrader.py | 4 ++-- 7 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/boutupgrader/bout_v5_factory_upgrader.py b/src/boutupgrader/bout_v5_factory_upgrader.py index bbc4aa8..6f0976f 100644 --- a/src/boutupgrader/bout_v5_factory_upgrader.py +++ b/src/boutupgrader/bout_v5_factory_upgrader.py @@ -1,8 +1,7 @@ -from .common import apply_or_display_patch - import copy import re +from .common import apply_or_display_patch # Dictionary of factory methods that may need updating factories = { diff --git a/src/boutupgrader/bout_v5_format_upgrader.py b/src/boutupgrader/bout_v5_format_upgrader.py index 8aff6b0..036ce14 100644 --- a/src/boutupgrader/bout_v5_format_upgrader.py +++ b/src/boutupgrader/bout_v5_format_upgrader.py @@ -1,8 +1,7 @@ -from .common import apply_or_display_patch - import copy import re +from .common import apply_or_display_patch format_replacements = { "c": "c", diff --git a/src/boutupgrader/bout_v5_header_upgrader.py b/src/boutupgrader/bout_v5_header_upgrader.py index 77ec93f..7c1d2f8 100644 --- a/src/boutupgrader/bout_v5_header_upgrader.py +++ b/src/boutupgrader/bout_v5_header_upgrader.py @@ -1,13 +1,12 @@ -from .common import apply_or_display_patch - import argparse import copy import re +import subprocess import textwrap from pathlib import Path from typing import List -import subprocess +from .common import apply_or_display_patch header_shim_sentinel = "// BOUT++ header shim" diff --git a/src/boutupgrader/bout_v5_input_file_upgrader.py b/src/boutupgrader/bout_v5_input_file_upgrader.py index 47bbae9..3d9b090 100644 --- a/src/boutupgrader/bout_v5_input_file_upgrader.py +++ b/src/boutupgrader/bout_v5_input_file_upgrader.py @@ -1,5 +1,3 @@ -from .common import create_patch, yes_or_no - import argparse import copy import itertools @@ -8,6 +6,8 @@ from boututils.boutwarnings import AlwaysWarning +from .common import create_patch, yes_or_no + def case_sensitive_init(self, name="root", parent=None): self._sections = dict() @@ -291,7 +291,7 @@ def add_parser(subcommand, default_args, files_args): def run(args): - from boutdata.data import BoutOptionsFile, BoutOptions + from boutdata.data import BoutOptions, BoutOptionsFile # Monkey-patch BoutOptions to make sure it's case sensitive BoutOptions.__init__ = case_sensitive_init diff --git a/src/boutupgrader/bout_v5_macro_upgrader.py b/src/boutupgrader/bout_v5_macro_upgrader.py index 33e02e0..db621e6 100644 --- a/src/boutupgrader/bout_v5_macro_upgrader.py +++ b/src/boutupgrader/bout_v5_macro_upgrader.py @@ -1,10 +1,10 @@ -from .common import apply_or_display_patch - import argparse import copy import re import textwrap +from .common import apply_or_display_patch + # List of macros, their replacements and what header to find them # in. Each element should be a dict with "old", "new" and "headers" # keys, with "old" and "new" values being strings, and "headers" being a diff --git a/src/boutupgrader/bout_v5_physics_model_upgrader.py b/src/boutupgrader/bout_v5_physics_model_upgrader.py index 84e6c74..9f0d7c4 100644 --- a/src/boutupgrader/bout_v5_physics_model_upgrader.py +++ b/src/boutupgrader/bout_v5_physics_model_upgrader.py @@ -1,5 +1,3 @@ -from .common import apply_or_display_patch - import argparse import copy import pathlib @@ -7,6 +5,7 @@ import textwrap import warnings +from .common import apply_or_display_patch PHYSICS_MODEL_INCLUDE = '#include "bout/physicsmodel.hxx"' diff --git a/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py b/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py index c0d60b4..0930d6f 100644 --- a/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py +++ b/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py @@ -1,8 +1,8 @@ -from .common import apply_or_display_patch - import copy import re +from .common import apply_or_display_patch + try: import clang.cindex From 1e38d87b3525d99bac73d05c75225229f764063c Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 12 Sep 2024 17:14:11 +0100 Subject: [PATCH 70/82] Replace `.format` with f-strings --- src/boutupgrader/bout_v5_factory_upgrader.py | 2 +- src/boutupgrader/bout_v5_format_upgrader.py | 10 +++---- .../bout_v5_input_file_upgrader.py | 4 +-- src/boutupgrader/bout_v5_macro_upgrader.py | 15 +++++----- .../bout_v5_physics_model_upgrader.py | 2 +- .../bout_v5_xzinterpolation_upgrader.py | 30 ++++++++----------- 6 files changed, 28 insertions(+), 35 deletions(-) diff --git a/src/boutupgrader/bout_v5_factory_upgrader.py b/src/boutupgrader/bout_v5_factory_upgrader.py index 6f0976f..0dba61e 100644 --- a/src/boutupgrader/bout_v5_factory_upgrader.py +++ b/src/boutupgrader/bout_v5_factory_upgrader.py @@ -161,7 +161,7 @@ def fix_deletions(variables, source): for variable in variables: source = re.sub( - r"(.*;?)\s*(delete\s*{variable})\s*;".format(variable=variable), + rf"(.*;?)\s*(delete\s*{variable})\s*;", r"\1", source, ) diff --git a/src/boutupgrader/bout_v5_format_upgrader.py b/src/boutupgrader/bout_v5_format_upgrader.py index 036ce14..ef0b01f 100644 --- a/src/boutupgrader/bout_v5_format_upgrader.py +++ b/src/boutupgrader/bout_v5_format_upgrader.py @@ -22,8 +22,8 @@ def fix_format_replacement(format_replacement, source): """Replace printf format with fmt format""" return re.sub( - r"%([0-9]*\.?[0-9]*){}".format(format_replacement[0]), - r"{{:\1{}}}".format(format_replacement[1]), + rf"%([0-9]*\.?[0-9]*){format_replacement[0]}", + rf"{{:\1{format_replacement[1]}}}", source, ) @@ -33,10 +33,10 @@ def fix_trivial_format(source): def trivial_replace(match): if match.group(2): - return "{}{}{}".format(match.group(1), match.group(2), match.group(4)) + return f"{match.group(1)}{match.group(2)}{match.group(4)}" if match.group(3): - return "{}{}{}".format(match.group(1), match.group(3), match.group(4)) - raise ValueError("Found an unexpected match: {}".format(match)) + return f"{match.group(1)}{match.group(3)}{match.group(4)}" + raise ValueError(f"Found an unexpected match: {match}") return re.sub( r""" diff --git a/src/boutupgrader/bout_v5_input_file_upgrader.py b/src/boutupgrader/bout_v5_input_file_upgrader.py index 3d9b090..d62d631 100644 --- a/src/boutupgrader/bout_v5_input_file_upgrader.py +++ b/src/boutupgrader/bout_v5_input_file_upgrader.py @@ -223,14 +223,14 @@ def possibly_apply_patch(patch, options_file, quiet=False, force=False): """ if not quiet: print("\n******************************************") - print("Changes to {}\n".format(options_file.filename)) + print(f"Changes to {options_file.filename}\n") print(patch) print("\n******************************************") if force: make_change = True else: - make_change = yes_or_no("Make changes to {}?".format(options_file.filename)) + make_change = yes_or_no(f"Make changes to {options_file.filename}?") if make_change: options_file.write(overwrite=True) return make_change diff --git a/src/boutupgrader/bout_v5_macro_upgrader.py b/src/boutupgrader/bout_v5_macro_upgrader.py index db621e6..922b48d 100644 --- a/src/boutupgrader/bout_v5_macro_upgrader.py +++ b/src/boutupgrader/bout_v5_macro_upgrader.py @@ -185,13 +185,13 @@ def fix_include_version_header(old, headers, source): # If header is already included, we can skip this fix for header in headers: if ( - re.search(r'^#\s*include.*(<|"){}(>|")'.format(header), source, flags=re.M) + re.search(rf'^#\s*include.*(<|"){header}(>|")', source, flags=re.M) is not None ): return source # If the old macro isn't in the file, we can skip this fix - if re.search(r"\b{}\b".format(old), source) is None: + if re.search(rf"\b{old}\b", source) is None: return source # Now we want to find a suitable place to stick the new include @@ -209,7 +209,7 @@ def fix_include_version_header(old, headers, source): else: # No suitable includes, so just stick at the top of the file last_include = 0 - source_lines.insert(last_include, '#include "{}"'.format(headers[0])) + source_lines.insert(last_include, f'#include "{headers[0]}"') return "\n".join(source_lines) @@ -277,13 +277,13 @@ def fix_ifdefs(old, source): def fix_always_defined_macros(old, new, source): """Fix '#ifdef's that should become plain '#if'""" - new_source = re.sub(r"#ifdef\s+{}\b".format(old), r"#if {}".format(new), source) - return re.sub(r"#ifndef\s+{}\b".format(old), r"#if !{}".format(new), new_source) + new_source = re.sub(rf"#ifdef\s+{old}\b", rf"#if {new}", source) + return re.sub(rf"#ifndef\s+{old}\b", rf"#if !{new}", new_source) def fix_replacement(old, new, source): """Straight replacements""" - return re.sub(r'([^"_])\b{}\b([^"_])'.format(old), r"\1{}\2".format(new), source) + return re.sub(rf'([^"_])\b{old}\b([^"_])', rf"\1{new}\2", source) def apply_fixes(replacements, source): @@ -293,8 +293,7 @@ def apply_fixes(replacements, source): for replacement in replacements: if replacement["new"] is None: print( - "'%s' has been removed, please delete from your code" - % replacement["old"] + f"{replacement['old']} has been removed, please delete from your code" ) continue diff --git a/src/boutupgrader/bout_v5_physics_model_upgrader.py b/src/boutupgrader/bout_v5_physics_model_upgrader.py index 9f0d7c4..fc815ce 100644 --- a/src/boutupgrader/bout_v5_physics_model_upgrader.py +++ b/src/boutupgrader/bout_v5_physics_model_upgrader.py @@ -208,7 +208,7 @@ def fix_bout_constrain(source, error_on_warning): bad_line = source[: match.end()].count("\n") line_range = range(max(0, bad_line - 1), min(len(source_lines), bad_line + 2)) lines_context.append( - "\n ".join(["{}:{}".format(i, source_lines[i]) for i in line_range]) + "\n ".join([f"{i}:{source_lines[i]}" for i in line_range]) ) message = textwrap.dedent( diff --git a/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py b/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py index 0930d6f..620780c 100644 --- a/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py +++ b/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py @@ -46,15 +46,13 @@ def fix_header_includes(old_header, new_header, source): """ return re.sub( - r""" + rf""" (\s*\#\s*include\s*) # Preprocessor include (<|") - ({header}) # Header name + ({old_header}) # Header name (>|") - """.format( - header=old_header - ), - r"\1\2{header}\4".format(header=new_header), + """, + rf"\1\2{new_header}\4", source, flags=re.VERBOSE, ) @@ -63,12 +61,10 @@ def fix_header_includes(old_header, new_header, source): def fix_interpolations(old_interpolation, new_interpolation, source): return re.sub( - r""" - \b{}\b - """.format( - old_interpolation - ), - r"{}".format(new_interpolation), + rf""" + \b{old_interpolation}\b + """, + rf"{new_interpolation}", source, flags=re.VERBOSE, ) @@ -117,12 +113,10 @@ def clang_fix_interpolation(old_interpolation, new_interpolation, node, source): def fix_factories(old_factory, new_factory, source): return re.sub( - r""" - \b{}\b - """.format( - old_factory - ), - r"{}".format(new_factory), + rf""" + \b{old_factory}\b + """, + new_factory, source, flags=re.VERBOSE, ) From c835b75f1675750ae886877200bab8005bad3958 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 12 Sep 2024 17:14:36 +0100 Subject: [PATCH 71/82] Remove default `open` flags --- src/boutupgrader/bout_v5_factory_upgrader.py | 2 +- src/boutupgrader/bout_v5_format_upgrader.py | 2 +- src/boutupgrader/bout_v5_header_upgrader.py | 4 ++-- src/boutupgrader/bout_v5_input_file_upgrader.py | 2 +- src/boutupgrader/bout_v5_macro_upgrader.py | 2 +- src/boutupgrader/bout_v5_physics_model_upgrader.py | 2 +- src/boutupgrader/bout_v5_xzinterpolation_upgrader.py | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/boutupgrader/bout_v5_factory_upgrader.py b/src/boutupgrader/bout_v5_factory_upgrader.py index 0dba61e..1558fae 100644 --- a/src/boutupgrader/bout_v5_factory_upgrader.py +++ b/src/boutupgrader/bout_v5_factory_upgrader.py @@ -237,7 +237,7 @@ def add_parser(subcommand, default_args, files_args): def run(args): for filename in args.files: - with open(filename, "r") as f: + with open(filename) as f: contents = f.read() original = copy.deepcopy(contents) diff --git a/src/boutupgrader/bout_v5_format_upgrader.py b/src/boutupgrader/bout_v5_format_upgrader.py index ef0b01f..5c0b9b0 100644 --- a/src/boutupgrader/bout_v5_format_upgrader.py +++ b/src/boutupgrader/bout_v5_format_upgrader.py @@ -128,7 +128,7 @@ def add_parser(subcommand, default_args, files_args): def run(args): for filename in args.files: - with open(filename, "r") as f: + with open(filename) as f: contents = f.read() original = copy.deepcopy(contents) diff --git a/src/boutupgrader/bout_v5_header_upgrader.py b/src/boutupgrader/bout_v5_header_upgrader.py index 7c1d2f8..223a398 100644 --- a/src/boutupgrader/bout_v5_header_upgrader.py +++ b/src/boutupgrader/bout_v5_header_upgrader.py @@ -20,7 +20,7 @@ def header_needs_moving(header: Path) -> bool: """Check if `header` has not yet been moved""" - with open(header, "r") as f: + with open(header) as f: return header_shim_sentinel not in f.read() @@ -149,7 +149,7 @@ def run(args): header_regex = make_header_regex(deprecated_headers) for filename in args.files: - with open(filename, "r") as f: + with open(filename) as f: contents = f.read() original = copy.deepcopy(contents) diff --git a/src/boutupgrader/bout_v5_input_file_upgrader.py b/src/boutupgrader/bout_v5_input_file_upgrader.py index d62d631..fb4aeb7 100644 --- a/src/boutupgrader/bout_v5_input_file_upgrader.py +++ b/src/boutupgrader/bout_v5_input_file_upgrader.py @@ -299,7 +299,7 @@ def run(args): warnings.simplefilter("ignore", AlwaysWarning) for filename in args.files: - with open(filename, "r") as f: + with open(filename) as f: original_source = f.read() try: diff --git a/src/boutupgrader/bout_v5_macro_upgrader.py b/src/boutupgrader/bout_v5_macro_upgrader.py index 922b48d..e253c05 100644 --- a/src/boutupgrader/bout_v5_macro_upgrader.py +++ b/src/boutupgrader/bout_v5_macro_upgrader.py @@ -342,7 +342,7 @@ def add_parser(subcommand, default_args, files_args): def run(args): for filename in args.files: - with open(filename, "r") as f: + with open(filename) as f: contents = f.read() original = copy.deepcopy(contents) diff --git a/src/boutupgrader/bout_v5_physics_model_upgrader.py b/src/boutupgrader/bout_v5_physics_model_upgrader.py index fc815ce..5763f4a 100644 --- a/src/boutupgrader/bout_v5_physics_model_upgrader.py +++ b/src/boutupgrader/bout_v5_physics_model_upgrader.py @@ -394,7 +394,7 @@ def add_parser(subcommand, default_args, files_args): def run(args): for filename in args.files: - with open(filename, "r") as f: + with open(filename) as f: contents = f.read() original = copy.deepcopy(contents) diff --git a/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py b/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py index 620780c..8de673c 100644 --- a/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py +++ b/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py @@ -192,7 +192,7 @@ def run(args): ) for filename in args.files: - with open(filename, "r") as f: + with open(filename) as f: contents = f.read() original = copy.deepcopy(contents) From b728c869afb1dda8dad41e034df3838f7a9f3282 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 12 Sep 2024 17:14:45 +0100 Subject: [PATCH 72/82] Use generator in place of tuple --- src/boutupgrader/bout_v5_header_upgrader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/boutupgrader/bout_v5_header_upgrader.py b/src/boutupgrader/bout_v5_header_upgrader.py index 223a398..1c7b26f 100644 --- a/src/boutupgrader/bout_v5_header_upgrader.py +++ b/src/boutupgrader/bout_v5_header_upgrader.py @@ -80,7 +80,7 @@ def fix_library_header_locations( def make_header_regex(deprecated_headers: List[str]) -> re.Pattern: """Create a regular expression to match deprecated header locations""" - deprecated_header_joined = "|".join((header.name for header in deprecated_headers)) + deprecated_header_joined = "|".join(header.name for header in deprecated_headers) return re.compile(rf'(#include\s+<|")(?:\.\./)?({deprecated_header_joined})(>|")') From 24a1386334f965fab56bbeb71b4040a835489757 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 12 Sep 2024 17:20:57 +0100 Subject: [PATCH 73/82] Group v5 upgrade subcommands --- src/boutupgrader/__init__.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/boutupgrader/__init__.py b/src/boutupgrader/__init__.py index cf9fb4e..94860d8 100644 --- a/src/boutupgrader/__init__.py +++ b/src/boutupgrader/__init__.py @@ -31,14 +31,17 @@ def main(): description="Upgrade BOUT++ source and input files to newer versions" ) subcommand = parser.add_subparsers(title="subcommands", required=True) + v5_subcommand = subcommand.add_parser( + "v5", help="BOUT++ v5 upgrades" + ).add_subparsers(title="v5 subcommands", required=True) - add_factory_parser(subcommand, common_args, files_args) - add_format_parser(subcommand, common_args, files_args) - add_header_parser(subcommand, common_args) - add_input_parser(subcommand, common_args, files_args) - add_macro_parser(subcommand, common_args, files_args) - add_model_parser(subcommand, common_args, files_args) - add_xzinterp_parser(subcommand, common_args, files_args) + add_factory_parser(v5_subcommand, common_args, files_args) + add_format_parser(v5_subcommand, common_args, files_args) + add_header_parser(v5_subcommand, common_args) + add_input_parser(v5_subcommand, common_args, files_args) + add_macro_parser(v5_subcommand, common_args, files_args) + add_model_parser(v5_subcommand, common_args, files_args) + add_xzinterp_parser(v5_subcommand, common_args, files_args) args = parser.parse_args() args.func(args) From f948fc23246fa9019a980b33a8463dbc4c0c2263 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 12 Sep 2024 17:30:16 +0100 Subject: [PATCH 74/82] Add version argument to bout-upgrader --- src/boutupgrader/__init__.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/boutupgrader/__init__.py b/src/boutupgrader/__init__.py index 94860d8..b592f51 100644 --- a/src/boutupgrader/__init__.py +++ b/src/boutupgrader/__init__.py @@ -1,4 +1,5 @@ import argparse +from importlib.metadata import PackageNotFoundError, version from .bout_v5_factory_upgrader import add_parser as add_factory_parser from .bout_v5_format_upgrader import add_parser as add_format_parser @@ -8,6 +9,18 @@ from .bout_v5_physics_model_upgrader import add_parser as add_model_parser from .bout_v5_xzinterpolation_upgrader import add_parser as add_xzinterp_parser +try: + # This gives the version if the boututils package was installed + __version__ = version("boutdata") +except PackageNotFoundError: + # This branch handles the case when boututils is used from the git repo + try: + from setuptools_scm import get_version + + __version__ = get_version(root="..", relative_to=__file__) + except (ModuleNotFoundError, LookupError): + __version__ = "dev" + def main(): # Parent parser that has arguments common to all subcommands @@ -30,6 +43,9 @@ def main(): parser = argparse.ArgumentParser( description="Upgrade BOUT++ source and input files to newer versions" ) + parser.add_argument( + "--version", action="version", version=f"%(prog)s {__version__}" + ) subcommand = parser.add_subparsers(title="subcommands", required=True) v5_subcommand = subcommand.add_parser( "v5", help="BOUT++ v5 upgrades" From 14c7dc3de478f589f8a94c0778a6ce7db596cc98 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 28 Sep 2016 17:43:45 +0100 Subject: [PATCH 75/82] Add tool to help with updating files from v3 to v4 --- bin/bout_3to4.py | 129 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100755 bin/bout_3to4.py diff --git a/bin/bout_3to4.py b/bin/bout_3to4.py new file mode 100755 index 0000000..73c5830 --- /dev/null +++ b/bin/bout_3to4.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 + +import argparse +import re +import fileinput + +nonmembers = { + 'DC': ['DC', 1], + 'slice': ['sliceXZ', 2], +} + +coordinates = [ + "outputVars", + "dx", "dy", "dz", + "non_uniform", + "d1_dx", "d1_dy", + "J", "Bxy", + "g11", "g22", "g33", "g12", "g13", "g23", + "g_11", "g_22", "g_33", "g_12", "g_13", "g_23", + "G1_11", "G1_22", "G1_33", "G1_12", "G1_13", + "G2_11", "G2_22", "G2_33", "G2_12", "G2_23", + "G3_11", "G3_22", "G3_33", "G3_13", "G3_23", + "G1", "G2", "G3", + "ShiftTorsion", "IntShiftTorsion", + "geometry", "calcCovariant", "calcContravariant", "jacobian" +] + + +def fix_nonmembers(line_text, filename, line_num, replace=False): + """Replace member functions with nonmembers + """ + + old_line_text = line_text + + for old, (new, num_args) in nonmembers.items(): + pattern = re.compile("(\w*)\.{}\(".format(old)) + matches = re.findall(pattern, line_text) + for match in matches: + replacement = "{func}({object}".format(func=new, object=match) + if num_args > 1: + replacement += ", " + line_text = re.sub(pattern, replacement, line_text) + if not replace: + name_num = "{name}:{num}:".format(name=filename, num=line_num) + print("{name_num}{line}".format(name_num=name_num, line=old_line_text), end='') + print(" "*len(name_num) + line_text) + if replace: + return line_text + + +def fix_subscripts(line_text, filename, line_num, replace=False): + """Replace triple square brackets with round brackets + + Should also check that the variable is a Field3D - but doesn't + """ + + old_line_text = line_text + pattern = re.compile("\[([^[]*)\]\[([^[]*)\]\[([^[]*)\]") + matches = re.findall(pattern, line_text) + for match in matches: + line_text = re.sub(pattern, "(\1, \2, \3)", line_text) + if not replace: + name_num = "{name}:{num}:".format(name=filename, num=line_num) + print("{name_num}{line}".format(name_num=name_num, line=old_line_text), end='') + print(" "*len(name_num) + line_text) + if replace: + return line_text + + +def fix_coordinates(line_text, filename, line_num, replace=False): + """Fix variables that have moved from mesh to coordinates + """ + + old_line_text = line_text + + for var in coordinates: + pattern = re.compile("mesh->{}".format(var)) + matches = re.findall(pattern, line_text) + for match in matches: + line_text = re.sub(pattern, "coords->{}".format(var), line_text) + if not replace: + name_num = "{name}:{num}:".format(name=filename, num=line_num) + print("{name_num}{line}".format(name_num=name_num, line=old_line_text), end='') + print(" "*len(name_num) + line_text) + if replace: + return line_text + + +if __name__ == '__main__': + + epilog = """ + Currently bout_3to4 can detect the following transformations are needed: + - Triple square brackets instead of round brackets for subscripts + - Field member functions that are now non-members + - Variables/functions that have moved from Mesh to Coordinates + + Note that in the latter case of transformations, you will still need to manually add + Coordinates *coords = mesh->coordinates(); + to the correct scopes + """ + + parser = argparse.ArgumentParser(description="A little helper for upgrading from BOUT++ version 3 to version 4", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=epilog) + parser.add_argument("-r", "--replace", action="store_true", + help="Actually make the fix") + parser.add_argument("files", nargs='+', + help="Files to process") + + args = parser.parse_args() + + # Loops over all lines across all files + for line in fileinput.input(files=args.files, inplace=args.replace): + filename = fileinput.filename() + line_num = fileinput.filelineno() + + # Apply the transformations and then update the line if we're doing a replacement + new_line = fix_nonmembers(line, filename, line_num, args.replace) + line = new_line if args.replace else line + + new_line = fix_subscripts(line, filename, line_num, args.replace) + line = new_line if args.replace else line + + new_line = fix_coordinates(line, filename, line_num, args.replace) + line = new_line if args.replace else line + + # If we're doing a replacement, then we need to print all lines, without a newline + if args.replace: + print(line, end='') From 5f3d19e717a8511a8378575de4dedbc9ebc3e08e Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 29 Sep 2016 14:08:44 +0100 Subject: [PATCH 76/82] Use mesh->coordinates() instead of coords in 3to4 Slightly uglier, but resulting file doesn't need additional lines manually adding --- bin/bout_3to4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/bout_3to4.py b/bin/bout_3to4.py index 73c5830..cf9d35e 100755 --- a/bin/bout_3to4.py +++ b/bin/bout_3to4.py @@ -77,7 +77,7 @@ def fix_coordinates(line_text, filename, line_num, replace=False): pattern = re.compile("mesh->{}".format(var)) matches = re.findall(pattern, line_text) for match in matches: - line_text = re.sub(pattern, "coords->{}".format(var), line_text) + line_text = re.sub(pattern, "mesh->coordinates()->{}".format(var), line_text) if not replace: name_num = "{name}:{num}:".format(name=filename, num=line_num) print("{name_num}{line}".format(name_num=name_num, line=old_line_text), end='') From 5bbe434a41426860984b64cf60333858833c8190 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 21 Nov 2016 10:35:07 +0000 Subject: [PATCH 77/82] Enable bout_3to4 to check 2D subscripts --- bin/bout_3to4.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/bin/bout_3to4.py b/bin/bout_3to4.py index cf9d35e..fa1781e 100755 --- a/bin/bout_3to4.py +++ b/bin/bout_3to4.py @@ -51,14 +51,20 @@ def fix_nonmembers(line_text, filename, line_num, replace=False): def fix_subscripts(line_text, filename, line_num, replace=False): """Replace triple square brackets with round brackets - Should also check that the variable is a Field3D - but doesn't + Should also check that the variable is a Field3D/Field2D - but doesn't """ old_line_text = line_text - pattern = re.compile("\[([^[]*)\]\[([^[]*)\]\[([^[]*)\]") + # Catch both 2D and 3D arrays + pattern = re.compile(r"\[([^[]*)\]\[([^[]*)\](?:\[([^[]*)\])?") matches = re.findall(pattern, line_text) for match in matches: - line_text = re.sub(pattern, "(\1, \2, \3)", line_text) + # If the last group is non-empty, then it was a 3D array + if len(match[2]): + replacement = r"(\1, \2, \3)" + else: + replacement = r"(\1, \2)" + line_text = re.sub(pattern, replacement, line_text) if not replace: name_num = "{name}:{num}:".format(name=filename, num=line_num) print("{name_num}{line}".format(name_num=name_num, line=old_line_text), end='') From 3f35847aa22a5f4233effcbb6132065c51680a29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20L=C3=B8iten?= Date: Tue, 18 Apr 2017 13:42:10 +0200 Subject: [PATCH 78/82] Added ng@ -> LocalN@ replacement As a sidenote, the following replacements would be handy, but was not implemented in this commit a^b -> pow(a,b) Field3D.max(bool_or_NULL) -> max(Field3D, bool_or_NULL --- bin/bout_3to4.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/bin/bout_3to4.py b/bin/bout_3to4.py index fa1781e..7e81eb2 100755 --- a/bin/bout_3to4.py +++ b/bin/bout_3to4.py @@ -25,6 +25,11 @@ "geometry", "calcCovariant", "calcContravariant", "jacobian" ] +local_mesh = [ + ("ngx", "LocalNx"), + ("ngy", "LocalNy"), + ("ngz", "LocalNz"), +] def fix_nonmembers(line_text, filename, line_num, replace=False): """Replace member functions with nonmembers @@ -92,6 +97,24 @@ def fix_coordinates(line_text, filename, line_num, replace=False): return line_text +def fix_local_mesh_size(line_text, filename, line_num, replace=False): + """Replaces ng@ with LocalNg@, where @ is in {x,y,z} + """ + + old_line_text = line_text + + for lm in local_mesh: + pattern = re.compile(lm[0]) + matches = re.findall(pattern, line_text) + for match in matches: + line_text = re.sub(pattern, lm[1], line_text) + if not replace: + name_num = "{name}:{num}:".format(name=filename, num=line_num) + print("{name_num}{line}".format(name_num=name_num, line=old_line_text), end='') + print(" "*len(name_num) + line_text) + if replace: + return line_text + if __name__ == '__main__': epilog = """ @@ -130,6 +153,9 @@ def fix_coordinates(line_text, filename, line_num, replace=False): new_line = fix_coordinates(line, filename, line_num, args.replace) line = new_line if args.replace else line + new_line = fix_local_mesh_size(line, filename, line_num, args.replace) + line = new_line if args.replace else line + # If we're doing a replacement, then we need to print all lines, without a newline if args.replace: print(line, end='') From 4969437d39912b0cccd137e03026bdfb359c0cac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20L=C3=B8iten?= Date: Tue, 18 Apr 2017 22:24:46 +0200 Subject: [PATCH 79/82] Changed ngz to warning, added more warnings --- bin/bout_3to4.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/bin/bout_3to4.py b/bin/bout_3to4.py index 7e81eb2..9199abb 100755 --- a/bin/bout_3to4.py +++ b/bin/bout_3to4.py @@ -3,6 +3,7 @@ import argparse import re import fileinput +import sys nonmembers = { 'DC': ['DC', 1], @@ -28,7 +29,16 @@ local_mesh = [ ("ngx", "LocalNx"), ("ngy", "LocalNy"), - ("ngz", "LocalNz"), +] + +warnings = [ + (r"\^", "Use pow(a,b) instead of a^b"), + (r"\.max\(", "Use max(a) instead of a.max()"), + ("ngz", ("ngz is changed to LocalNz in v4." + " The extra point in z has been removed." + " Change ngz -> LocalNz, and ensure that" + " the number of points are correct") + ) ] def fix_nonmembers(line_text, filename, line_num, replace=False): @@ -115,6 +125,23 @@ def fix_local_mesh_size(line_text, filename, line_num, replace=False): if replace: return line_text + +def throw_warnings(line_text, filename, line_num): + """Throws a warning for ^, .max() and ngz + """ + + for warn in warnings: + pattern = re.compile(warn[0]) + matches = re.findall(pattern, line_text) + for match in matches: + name_num = "{name}:{num}:".format(name=filename, num=line_num) + # stdout is redirected to the file if --replace is given, + # therefore use stderr + sys.stderr.write("{name_num}{line}".format(name_num=name_num, line=line_text)) + # Coloring with \033[91m, end coloring with \033[0m\n + sys.stderr.write(" "*len(name_num) + "\033[91m!!!WARNING: {}\033[0m\n\n".format(warn[1])) + + if __name__ == '__main__': epilog = """ @@ -156,6 +183,8 @@ def fix_local_mesh_size(line_text, filename, line_num, replace=False): new_line = fix_local_mesh_size(line, filename, line_num, args.replace) line = new_line if args.replace else line + new_line = throw_warnings(line, filename, line_num) + # If we're doing a replacement, then we need to print all lines, without a newline if args.replace: print(line, end='') From b66b192a1951429fa5e96fb8df498978cc0031bb Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 12 Sep 2024 17:52:19 +0100 Subject: [PATCH 80/82] Move bout_3to4 into boutupgrader structure --- src/boutupgrader/__init__.py | 7 +++++++ {bin => src/boutupgrader}/bout_3to4.py | 20 ++++++++++---------- 2 files changed, 17 insertions(+), 10 deletions(-) rename {bin => src/boutupgrader}/bout_3to4.py (92%) diff --git a/src/boutupgrader/__init__.py b/src/boutupgrader/__init__.py index b592f51..288b599 100644 --- a/src/boutupgrader/__init__.py +++ b/src/boutupgrader/__init__.py @@ -8,6 +8,7 @@ from .bout_v5_macro_upgrader import add_parser as add_macro_parser from .bout_v5_physics_model_upgrader import add_parser as add_model_parser from .bout_v5_xzinterpolation_upgrader import add_parser as add_xzinterp_parser +from .bout_3to4 import add_parser as add_3to4_parser try: # This gives the version if the boututils package was installed @@ -47,6 +48,12 @@ def main(): "--version", action="version", version=f"%(prog)s {__version__}" ) subcommand = parser.add_subparsers(title="subcommands", required=True) + + v4_subcommand = subcommand.add_parser( + "v4", help="BOUT++ v4 upgrades" + ).add_subparsers(title="v4 subcommands", required=True) + add_3to4_parser(v4_subcommand, common_args, files_args) + v5_subcommand = subcommand.add_parser( "v5", help="BOUT++ v5 upgrades" ).add_subparsers(title="v5 subcommands", required=True) diff --git a/bin/bout_3to4.py b/src/boutupgrader/bout_3to4.py similarity index 92% rename from bin/bout_3to4.py rename to src/boutupgrader/bout_3to4.py index 9199abb..41fa492 100755 --- a/bin/bout_3to4.py +++ b/src/boutupgrader/bout_3to4.py @@ -142,8 +142,7 @@ def throw_warnings(line_text, filename, line_num): sys.stderr.write(" "*len(name_num) + "\033[91m!!!WARNING: {}\033[0m\n\n".format(warn[1])) -if __name__ == '__main__': - +def add_parser(subcommand, default_args, files_args): epilog = """ Currently bout_3to4 can detect the following transformations are needed: - Triple square brackets instead of round brackets for subscripts @@ -155,16 +154,17 @@ def throw_warnings(line_text, filename, line_num): to the correct scopes """ - parser = argparse.ArgumentParser(description="A little helper for upgrading from BOUT++ version 3 to version 4", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=epilog) - parser.add_argument("-r", "--replace", action="store_true", - help="Actually make the fix") - parser.add_argument("files", nargs='+', - help="Files to process") + parser = subcommand.add_parser( + "3to4", + help="A little helper for upgrading from BOUT++ version 3 to version 4", + description="A little helper for upgrading from BOUT++ version 3 to version 4", + parents=[default_args, files_args], + epilog=epilog + ) + parser.set_defaults(func=run) - args = parser.parse_args() +def run(args): # Loops over all lines across all files for line in fileinput.input(files=args.files, inplace=args.replace): filename = fileinput.filename() From 8c0669dcd2c08e14b88bf6a029d3e09e0a336531 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 12 Sep 2024 18:01:03 +0100 Subject: [PATCH 81/82] Apply black and ruff fixes --- src/boutupgrader/__init__.py | 2 +- src/boutupgrader/bout_3to4.py | 132 +++++++++++++++++++++++----------- 2 files changed, 92 insertions(+), 42 deletions(-) diff --git a/src/boutupgrader/__init__.py b/src/boutupgrader/__init__.py index 288b599..5741adf 100644 --- a/src/boutupgrader/__init__.py +++ b/src/boutupgrader/__init__.py @@ -1,6 +1,7 @@ import argparse from importlib.metadata import PackageNotFoundError, version +from .bout_3to4 import add_parser as add_3to4_parser from .bout_v5_factory_upgrader import add_parser as add_factory_parser from .bout_v5_format_upgrader import add_parser as add_format_parser from .bout_v5_header_upgrader import add_parser as add_header_parser @@ -8,7 +9,6 @@ from .bout_v5_macro_upgrader import add_parser as add_macro_parser from .bout_v5_physics_model_upgrader import add_parser as add_model_parser from .bout_v5_xzinterpolation_upgrader import add_parser as add_xzinterp_parser -from .bout_3to4 import add_parser as add_3to4_parser try: # This gives the version if the boututils package was installed diff --git a/src/boutupgrader/bout_3to4.py b/src/boutupgrader/bout_3to4.py index 41fa492..1fbdc52 100755 --- a/src/boutupgrader/bout_3to4.py +++ b/src/boutupgrader/bout_3to4.py @@ -1,29 +1,60 @@ #!/usr/bin/env python3 -import argparse -import re import fileinput +import re import sys nonmembers = { - 'DC': ['DC', 1], - 'slice': ['sliceXZ', 2], + "DC": ["DC", 1], + "slice": ["sliceXZ", 2], } coordinates = [ "outputVars", - "dx", "dy", "dz", + "dx", + "dy", + "dz", "non_uniform", - "d1_dx", "d1_dy", - "J", "Bxy", - "g11", "g22", "g33", "g12", "g13", "g23", - "g_11", "g_22", "g_33", "g_12", "g_13", "g_23", - "G1_11", "G1_22", "G1_33", "G1_12", "G1_13", - "G2_11", "G2_22", "G2_33", "G2_12", "G2_23", - "G3_11", "G3_22", "G3_33", "G3_13", "G3_23", - "G1", "G2", "G3", - "ShiftTorsion", "IntShiftTorsion", - "geometry", "calcCovariant", "calcContravariant", "jacobian" + "d1_dx", + "d1_dy", + "J", + "Bxy", + "g11", + "g22", + "g33", + "g12", + "g13", + "g23", + "g_11", + "g_22", + "g_33", + "g_12", + "g_13", + "g_23", + "G1_11", + "G1_22", + "G1_33", + "G1_12", + "G1_13", + "G2_11", + "G2_22", + "G2_33", + "G2_12", + "G2_23", + "G3_11", + "G3_22", + "G3_33", + "G3_13", + "G3_23", + "G1", + "G2", + "G3", + "ShiftTorsion", + "IntShiftTorsion", + "geometry", + "calcCovariant", + "calcContravariant", + "jacobian", ] local_mesh = [ @@ -34,16 +65,20 @@ warnings = [ (r"\^", "Use pow(a,b) instead of a^b"), (r"\.max\(", "Use max(a) instead of a.max()"), - ("ngz", ("ngz is changed to LocalNz in v4." - " The extra point in z has been removed." - " Change ngz -> LocalNz, and ensure that" - " the number of points are correct") - ) + ( + "ngz", + ( + "ngz is changed to LocalNz in v4." + " The extra point in z has been removed." + " Change ngz -> LocalNz, and ensure that" + " the number of points are correct" + ), + ), ] + def fix_nonmembers(line_text, filename, line_num, replace=False): - """Replace member functions with nonmembers - """ + """Replace member functions with nonmembers""" old_line_text = line_text @@ -57,8 +92,11 @@ def fix_nonmembers(line_text, filename, line_num, replace=False): line_text = re.sub(pattern, replacement, line_text) if not replace: name_num = "{name}:{num}:".format(name=filename, num=line_num) - print("{name_num}{line}".format(name_num=name_num, line=old_line_text), end='') - print(" "*len(name_num) + line_text) + print( + "{name_num}{line}".format(name_num=name_num, line=old_line_text), + end="", + ) + print(" " * len(name_num) + line_text) if replace: return line_text @@ -82,15 +120,16 @@ def fix_subscripts(line_text, filename, line_num, replace=False): line_text = re.sub(pattern, replacement, line_text) if not replace: name_num = "{name}:{num}:".format(name=filename, num=line_num) - print("{name_num}{line}".format(name_num=name_num, line=old_line_text), end='') - print(" "*len(name_num) + line_text) + print( + "{name_num}{line}".format(name_num=name_num, line=old_line_text), end="" + ) + print(" " * len(name_num) + line_text) if replace: return line_text def fix_coordinates(line_text, filename, line_num, replace=False): - """Fix variables that have moved from mesh to coordinates - """ + """Fix variables that have moved from mesh to coordinates""" old_line_text = line_text @@ -98,18 +137,22 @@ def fix_coordinates(line_text, filename, line_num, replace=False): pattern = re.compile("mesh->{}".format(var)) matches = re.findall(pattern, line_text) for match in matches: - line_text = re.sub(pattern, "mesh->coordinates()->{}".format(var), line_text) + line_text = re.sub( + pattern, "mesh->coordinates()->{}".format(var), line_text + ) if not replace: name_num = "{name}:{num}:".format(name=filename, num=line_num) - print("{name_num}{line}".format(name_num=name_num, line=old_line_text), end='') - print(" "*len(name_num) + line_text) + print( + "{name_num}{line}".format(name_num=name_num, line=old_line_text), + end="", + ) + print(" " * len(name_num) + line_text) if replace: return line_text def fix_local_mesh_size(line_text, filename, line_num, replace=False): - """Replaces ng@ with LocalNg@, where @ is in {x,y,z} - """ + """Replaces ng@ with LocalNg@, where @ is in {x,y,z}""" old_line_text = line_text @@ -120,15 +163,17 @@ def fix_local_mesh_size(line_text, filename, line_num, replace=False): line_text = re.sub(pattern, lm[1], line_text) if not replace: name_num = "{name}:{num}:".format(name=filename, num=line_num) - print("{name_num}{line}".format(name_num=name_num, line=old_line_text), end='') - print(" "*len(name_num) + line_text) + print( + "{name_num}{line}".format(name_num=name_num, line=old_line_text), + end="", + ) + print(" " * len(name_num) + line_text) if replace: return line_text def throw_warnings(line_text, filename, line_num): - """Throws a warning for ^, .max() and ngz - """ + """Throws a warning for ^, .max() and ngz""" for warn in warnings: pattern = re.compile(warn[0]) @@ -137,9 +182,14 @@ def throw_warnings(line_text, filename, line_num): name_num = "{name}:{num}:".format(name=filename, num=line_num) # stdout is redirected to the file if --replace is given, # therefore use stderr - sys.stderr.write("{name_num}{line}".format(name_num=name_num, line=line_text)) + sys.stderr.write( + "{name_num}{line}".format(name_num=name_num, line=line_text) + ) # Coloring with \033[91m, end coloring with \033[0m\n - sys.stderr.write(" "*len(name_num) + "\033[91m!!!WARNING: {}\033[0m\n\n".format(warn[1])) + sys.stderr.write( + " " * len(name_num) + + "\033[91m!!!WARNING: {}\033[0m\n\n".format(warn[1]) + ) def add_parser(subcommand, default_args, files_args): @@ -159,7 +209,7 @@ def add_parser(subcommand, default_args, files_args): help="A little helper for upgrading from BOUT++ version 3 to version 4", description="A little helper for upgrading from BOUT++ version 3 to version 4", parents=[default_args, files_args], - epilog=epilog + epilog=epilog, ) parser.set_defaults(func=run) @@ -187,4 +237,4 @@ def run(args): # If we're doing a replacement, then we need to print all lines, without a newline if args.replace: - print(line, end='') + print(line, end="") From cbe7da9fef0033679569ffcc8760cd35c8575836 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 26 Sep 2024 15:59:13 +0100 Subject: [PATCH 82/82] Fix some more ruff/flake8 warnings --- src/boutupgrader/bout_3to4.py | 40 +++++++------------ src/boutupgrader/bout_v5_macro_upgrader.py | 2 +- .../bout_v5_xzinterpolation_upgrader.py | 3 -- 3 files changed, 16 insertions(+), 29 deletions(-) diff --git a/src/boutupgrader/bout_3to4.py b/src/boutupgrader/bout_3to4.py index 1fbdc52..f208634 100755 --- a/src/boutupgrader/bout_3to4.py +++ b/src/boutupgrader/bout_3to4.py @@ -83,19 +83,16 @@ def fix_nonmembers(line_text, filename, line_num, replace=False): old_line_text = line_text for old, (new, num_args) in nonmembers.items(): - pattern = re.compile("(\w*)\.{}\(".format(old)) + pattern = re.compile(rf"(\w*)\.{old}\(") matches = re.findall(pattern, line_text) for match in matches: - replacement = "{func}({object}".format(func=new, object=match) + replacement = f"{new}({match}" if num_args > 1: replacement += ", " line_text = re.sub(pattern, replacement, line_text) if not replace: - name_num = "{name}:{num}:".format(name=filename, num=line_num) - print( - "{name_num}{line}".format(name_num=name_num, line=old_line_text), - end="", - ) + name_num = f"{filename}:{line_num}:" + print(f"{name_num}{old_line_text}", end="") print(" " * len(name_num) + line_text) if replace: return line_text @@ -119,10 +116,8 @@ def fix_subscripts(line_text, filename, line_num, replace=False): replacement = r"(\1, \2)" line_text = re.sub(pattern, replacement, line_text) if not replace: - name_num = "{name}:{num}:".format(name=filename, num=line_num) - print( - "{name_num}{line}".format(name_num=name_num, line=old_line_text), end="" - ) + name_num = f"{filename}:{line_num}:" + print(f"{name_num}{old_line_text}", end="") print(" " * len(name_num) + line_text) if replace: return line_text @@ -134,16 +129,14 @@ def fix_coordinates(line_text, filename, line_num, replace=False): old_line_text = line_text for var in coordinates: - pattern = re.compile("mesh->{}".format(var)) + pattern = re.compile(f"mesh->{var}") matches = re.findall(pattern, line_text) for match in matches: - line_text = re.sub( - pattern, "mesh->coordinates()->{}".format(var), line_text - ) + line_text = re.sub(pattern, f"mesh->coordinates()->{var}", line_text) if not replace: - name_num = "{name}:{num}:".format(name=filename, num=line_num) + name_num = f"{filename}:{line_num}:" print( - "{name_num}{line}".format(name_num=name_num, line=old_line_text), + f"{name_num}{old_line_text}", end="", ) print(" " * len(name_num) + line_text) @@ -162,9 +155,9 @@ def fix_local_mesh_size(line_text, filename, line_num, replace=False): for match in matches: line_text = re.sub(pattern, lm[1], line_text) if not replace: - name_num = "{name}:{num}:".format(name=filename, num=line_num) + name_num = f"{filename}:{line_num}:" print( - "{name_num}{line}".format(name_num=name_num, line=old_line_text), + f"{name_num}{old_line_text}", end="", ) print(" " * len(name_num) + line_text) @@ -179,16 +172,13 @@ def throw_warnings(line_text, filename, line_num): pattern = re.compile(warn[0]) matches = re.findall(pattern, line_text) for match in matches: - name_num = "{name}:{num}:".format(name=filename, num=line_num) + name_num = f"{filename}:{line_num}:" # stdout is redirected to the file if --replace is given, # therefore use stderr - sys.stderr.write( - "{name_num}{line}".format(name_num=name_num, line=line_text) - ) + sys.stderr.write(f"{name_num}{line_text}") # Coloring with \033[91m, end coloring with \033[0m\n sys.stderr.write( - " " * len(name_num) - + "\033[91m!!!WARNING: {}\033[0m\n\n".format(warn[1]) + " " * len(name_num) + f"\033[91m!!!WARNING: {warn[1]}\033[0m\n\n" ) diff --git a/src/boutupgrader/bout_v5_macro_upgrader.py b/src/boutupgrader/bout_v5_macro_upgrader.py index e253c05..c271a6e 100644 --- a/src/boutupgrader/bout_v5_macro_upgrader.py +++ b/src/boutupgrader/bout_v5_macro_upgrader.py @@ -185,7 +185,7 @@ def fix_include_version_header(old, headers, source): # If header is already included, we can skip this fix for header in headers: if ( - re.search(rf'^#\s*include.*(<|"){header}(>|")', source, flags=re.M) + re.search(rf'^#\s*include.*(<|"){header}(>|")', source, flags=re.MULTILINE) is not None ): return source diff --git a/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py b/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py index 8de673c..7cad47c 100644 --- a/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py +++ b/src/boutupgrader/bout_v5_xzinterpolation_upgrader.py @@ -59,7 +59,6 @@ def fix_header_includes(old_header, new_header, source): def fix_interpolations(old_interpolation, new_interpolation, source): - return re.sub( rf""" \b{old_interpolation}\b @@ -111,7 +110,6 @@ def clang_fix_interpolation(old_interpolation, new_interpolation, node, source): def fix_factories(old_factory, new_factory, source): - return re.sub( rf""" \b{old_factory}\b @@ -151,7 +149,6 @@ def apply_fixes(headers, interpolations, factories, source): def clang_apply_fixes(headers, interpolations, factories, filename, source): - # translation unit tu = clang_parse(filename, source)