From 136030d66acd53c30a39cbdd1a003579cafb8b16 Mon Sep 17 00:00:00 2001 From: Carl Recine Date: Fri, 15 Mar 2024 16:24:16 -0700 Subject: [PATCH 1/3] adjusted print level of driver based on verbosity --- aviary/interface/methods_for_level2.py | 28 +++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/aviary/interface/methods_for_level2.py b/aviary/interface/methods_for_level2.py index 6126d7a39..dad9e67f7 100644 --- a/aviary/interface/methods_for_level2.py +++ b/aviary/interface/methods_for_level2.py @@ -1591,23 +1591,45 @@ def add_driver(self, optimizer=None, use_coloring=None, max_iter=50, verbosity=V driver.declare_coloring() if driver.options["optimizer"] == "SNOPT": + if verbosity == Verbosity.QUIET: + isumm = 0 + iprint = 0 + elif verbosity == Verbosity.BRIEF: + isumm = 6 + iprint = 0 + else: + isumm = 6 + iprint = 9 driver.opt_settings["Major iterations limit"] = max_iter driver.opt_settings["Major optimality tolerance"] = 1e-4 driver.opt_settings["Major feasibility tolerance"] = 1e-7 - driver.opt_settings["iSumm"] = 6 + driver.opt_settings["iSumm"] = isumm + driver.opt_settings["iPrint"] = iprint elif driver.options["optimizer"] == "IPOPT": + if verbosity == Verbosity.QUIET: + print_level = 0 + elif verbosity == Verbosity.BRIEF: + print_level = 2 + elif verbosity == Verbosity.VERBOSE: + print_level = 5 + else: + print_level = 8 driver.opt_settings['tol'] = 1.0E-6 driver.opt_settings['mu_init'] = 1e-5 driver.opt_settings['max_iter'] = max_iter - driver.opt_settings['print_level'] = 5 + driver.opt_settings['print_level'] = print_level # for faster convergence driver.opt_settings['nlp_scaling_method'] = 'gradient-based' driver.opt_settings['alpha_for_y'] = 'safer-min-dual-infeas' driver.opt_settings['mu_strategy'] = 'monotone' elif driver.options["optimizer"] == "SLSQP": + if verbosity == Verbosity.QUIET: + disp = False + else: + disp = True driver.options["tol"] = 1e-9 driver.options["maxiter"] = max_iter - driver.options["disp"] = True + driver.options["disp"] = disp if verbosity != Verbosity.QUIET: if isinstance(verbosity, list): From 7a5b73822a768d5786a3d87bcaf303698d662276 Mon Sep 17 00:00:00 2001 From: Carl Recine Date: Fri, 15 Mar 2024 16:32:26 -0700 Subject: [PATCH 2/3] Switched tests from BRIEF to QUIET --- aviary/interface/methods_for_level1.py | 6 +++--- aviary/interface/test/test_basic_report.py | 4 ++-- .../interface/test/test_height_energy_mission.py | 11 ++++++----- aviary/interface/test/test_interface_bugs.py | 3 ++- aviary/interface/test/test_linkage_logic.py | 3 ++- .../propulsion/test/test_custom_engine_model.py | 3 ++- .../benchmark_tests/test_0_iters.py | 3 ++- .../benchmark_tests/test_bench_FwFm.py | 16 ++++++++++------ .../benchmark_tests/test_bench_FwGm.py | 5 +++-- .../benchmark_tests/test_bench_GwFm.py | 5 +++-- .../benchmark_tests/test_bench_GwGm.py | 10 +++++----- .../test_subsystems_within_a_mission.py | 3 ++- 12 files changed, 42 insertions(+), 30 deletions(-) diff --git a/aviary/interface/methods_for_level1.py b/aviary/interface/methods_for_level1.py index 5205611c9..3ee2ff20e 100644 --- a/aviary/interface/methods_for_level1.py +++ b/aviary/interface/methods_for_level1.py @@ -5,7 +5,7 @@ from pathlib import Path import openmdao.api as om -from aviary.variable_info.enums import AnalysisScheme +from aviary.variable_info.enums import AnalysisScheme, Verbosity from aviary.interface.methods_for_level2 import AviaryProblem @@ -13,7 +13,7 @@ def run_aviary(aircraft_filename, phase_info, optimizer=None, analysis_scheme=AnalysisScheme.COLLOCATION, objective_type=None, record_filename='dymos_solution.db', restart_filename=None, max_iter=50, run_driver=True, make_plots=True, phase_info_parameterization=None, - optimization_history_filename=None): + optimization_history_filename=None, verbosity=Verbosity.BRIEF): """ Run the Aviary optimization problem for a specified aircraft configuration and mission. @@ -79,7 +79,7 @@ def run_aviary(aircraft_filename, phase_info, optimizer=None, # Link phases and variables prob.link_phases() - prob.add_driver(optimizer, max_iter=max_iter) + prob.add_driver(optimizer, max_iter=max_iter, verbosity=verbosity) prob.add_design_variables() diff --git a/aviary/interface/test/test_basic_report.py b/aviary/interface/test/test_basic_report.py index 01b166d9f..da5856f00 100644 --- a/aviary/interface/test/test_basic_report.py +++ b/aviary/interface/test/test_basic_report.py @@ -7,7 +7,7 @@ from aviary.interface.default_phase_info.height_energy import phase_info from aviary.interface.methods_for_level1 import run_aviary -from openmdao.utils.testing_utils import use_tempdirs +from aviary.variable_info.enums import Verbosity @unittest.skip("Skipping test due to sensitivity in setup. Need tolerances on actual values") @@ -16,7 +16,7 @@ class BasicReportTestCase(unittest.TestCase): def setUp(self): local_phase_info = deepcopy(phase_info) self.prob = run_aviary('models/test_aircraft/aircraft_for_bench_FwFm.csv', - local_phase_info, + local_phase_info, verbosity=Verbosity.QUIET, mission_method="FLOPS", mass_method="FLOPS", optimizer='IPOPT') def test_text_report(self): diff --git a/aviary/interface/test/test_height_energy_mission.py b/aviary/interface/test/test_height_energy_mission.py index 81f5685ae..11f9bd7c3 100644 --- a/aviary/interface/test/test_height_energy_mission.py +++ b/aviary/interface/test/test_height_energy_mission.py @@ -9,6 +9,7 @@ from aviary.subsystems.test.test_dummy_subsystem import ArrayGuessSubsystemBuilder from aviary.mission.flops_based.phases.energy_phase import EnergyPhase from aviary.variable_info.variables import Dynamic +from aviary.variable_info.enums import Verbosity @use_tempdirs @@ -118,7 +119,7 @@ def run_mission(self, phase_info, optimizer): return run_aviary( self.aircraft_definition_file, phase_info, make_plots=self.make_plots, max_iter=self.max_iter, optimizer=optimizer, - optimization_history_filename="driver_test.db") + optimization_history_filename="driver_test.db", verbosity=Verbosity.QUIET) def test_mission_basic_and_dashboard(self): # We need to remove the TESTFLO_RUNNING environment variable for this test to run. @@ -220,16 +221,16 @@ def test_custom_phase_builder(self): local_phase_info = self.phase_info.copy() local_phase_info['climb']['phase_builder'] = EnergyPhase - run_aviary(self.aircraft_definition_file, - local_phase_info, max_iter=1, optimizer='SLSQP') + run_aviary(self.aircraft_definition_file, local_phase_info, + verbosity=Verbosity.QUIET, max_iter=1, optimizer='SLSQP') def test_custom_phase_builder_error(self): local_phase_info = self.phase_info.copy() local_phase_info['climb']['phase_builder'] = "fake phase object" with self.assertRaises(TypeError): - run_aviary(self.aircraft_definition_file, - local_phase_info, max_iter=1, optimizer='SLSQP') + run_aviary(self.aircraft_definition_file, local_phase_info, + verbosity=Verbosity.QUIET, max_iter=1, optimizer='SLSQP') if __name__ == '__main__': diff --git a/aviary/interface/test/test_interface_bugs.py b/aviary/interface/test/test_interface_bugs.py index f298b66ed..4259ecc21 100644 --- a/aviary/interface/test/test_interface_bugs.py +++ b/aviary/interface/test/test_interface_bugs.py @@ -8,6 +8,7 @@ from aviary.subsystems.subsystem_builder_base import SubsystemBuilderBase from aviary.interface.default_phase_info.height_energy import phase_info as ph_in from aviary.variable_info.variables import Aircraft +from aviary.variable_info.enums import Verbosity from openmdao.utils.testing_utils import use_tempdirs @@ -87,7 +88,7 @@ def test_post_mission_promotion(self): # Link phases and variables prob.link_phases() - prob.add_driver("SLSQP") + prob.add_driver("SLSQP", verbosity=Verbosity.QUIET) prob.add_design_variables() diff --git a/aviary/interface/test/test_linkage_logic.py b/aviary/interface/test/test_linkage_logic.py index 502b17165..ef623bcaa 100644 --- a/aviary/interface/test/test_linkage_logic.py +++ b/aviary/interface/test/test_linkage_logic.py @@ -5,6 +5,7 @@ from openmdao.core.problem import _clear_problem_names from aviary.interface.methods_for_level2 import AviaryProblem +from aviary.variable_info.enums import Verbosity @use_tempdirs @@ -132,7 +133,7 @@ def test_linkages(self): # Link phases and variables prob.link_phases() - prob.add_driver('SLSQP') + prob.add_driver('SLSQP', verbosity=Verbosity.QUIET) prob.add_design_variables() diff --git a/aviary/subsystems/propulsion/test/test_custom_engine_model.py b/aviary/subsystems/propulsion/test/test_custom_engine_model.py index 2222168a5..d3b91d87a 100644 --- a/aviary/subsystems/propulsion/test/test_custom_engine_model.py +++ b/aviary/subsystems/propulsion/test/test_custom_engine_model.py @@ -7,6 +7,7 @@ from aviary.subsystems.propulsion.engine_model import EngineModel from aviary.utils.aviary_values import AviaryValues from aviary.variable_info.variables import Dynamic +from aviary.variable_info.enums import Verbosity from openmdao.utils.assert_utils import assert_near_equal from openmdao.utils.testing_utils import use_tempdirs @@ -179,7 +180,7 @@ def test_custom_engine(self): # Link phases and variables prob.link_phases() - prob.add_driver("SLSQP") + prob.add_driver("SLSQP", verbosity=Verbosity.QUIET) prob.add_design_variables() diff --git a/aviary/validation_cases/benchmark_tests/test_0_iters.py b/aviary/validation_cases/benchmark_tests/test_0_iters.py index 94ff3e779..b565b8cb9 100644 --- a/aviary/validation_cases/benchmark_tests/test_0_iters.py +++ b/aviary/validation_cases/benchmark_tests/test_0_iters.py @@ -4,6 +4,7 @@ from openmdao.utils.testing_utils import require_pyoptsparse, use_tempdirs from aviary.interface.methods_for_level2 import AviaryProblem +from aviary.variable_info.enums import Verbosity from aviary.interface.default_phase_info.two_dof import phase_info as two_dof_phase_info from aviary.interface.default_phase_info.height_energy import phase_info as height_energy_phase_info from aviary.models.N3CC.N3CC_data import inputs @@ -22,7 +23,7 @@ def build_and_run_problem(self, input_filename, phase_info, objective_type=None) prob.add_phases() prob.add_post_mission_systems() prob.link_phases() - prob.add_driver("SLSQP", max_iter=0) + prob.add_driver("SLSQP", max_iter=0, verbosity=Verbosity.QUIET) prob.add_design_variables() prob.add_objective(objective_type if objective_type else None) prob.setup() diff --git a/aviary/validation_cases/benchmark_tests/test_bench_FwFm.py b/aviary/validation_cases/benchmark_tests/test_bench_FwFm.py index 3a52bfcf5..4518a8f93 100644 --- a/aviary/validation_cases/benchmark_tests/test_bench_FwFm.py +++ b/aviary/validation_cases/benchmark_tests/test_bench_FwFm.py @@ -8,6 +8,7 @@ from openmdao.core.problem import _clear_problem_names from aviary.interface.methods_for_level1 import run_aviary +from aviary.variable_info.enums import Verbosity from aviary.validation_cases.benchmark_utils import \ compare_against_expected_values @@ -364,15 +365,17 @@ class TestBenchFwFmSerial(ProblemPhaseTestCase): @require_pyoptsparse(optimizer="IPOPT") def test_bench_FwFm_IPOPT(self): - prob = run_aviary( - 'models/test_aircraft/aircraft_for_bench_FwFm.csv', self.phase_info, max_iter=50, optimizer='IPOPT') + prob = run_aviary('models/test_aircraft/aircraft_for_bench_FwFm.csv', + self.phase_info, verbosity=Verbosity.QUIET, + max_iter=50, optimizer='IPOPT') compare_against_expected_values(prob, self.expected_dict) @require_pyoptsparse(optimizer="SNOPT") def test_bench_FwFm_SNOPT(self): - prob = run_aviary( - 'models/test_aircraft/aircraft_for_bench_FwFm.csv', self.phase_info, max_iter=50, optimizer='SNOPT') + prob = run_aviary('models/test_aircraft/aircraft_for_bench_FwFm.csv', + self.phase_info, verbosity=Verbosity.QUIET, + max_iter=50, optimizer='SNOPT') compare_against_expected_values(prob, self.expected_dict) @@ -385,8 +388,9 @@ class TestBenchFwFmParallel(ProblemPhaseTestCase): @require_pyoptsparse(optimizer="SNOPT") def test_bench_FwFm_SNOPT_MPI(self): - prob = run_aviary( - 'models/test_aircraft/aircraft_for_bench_FwFm.csv', self.phase_info, max_iter=50, optimizer='SNOPT') + prob = run_aviary('models/test_aircraft/aircraft_for_bench_FwFm.csv', + self.phase_info, verbosity=Verbosity.QUIET, + max_iter=50, optimizer='SNOPT') compare_against_expected_values(prob, self.expected_dict) diff --git a/aviary/validation_cases/benchmark_tests/test_bench_FwGm.py b/aviary/validation_cases/benchmark_tests/test_bench_FwGm.py index 4d896ee22..4a5e28ce7 100644 --- a/aviary/validation_cases/benchmark_tests/test_bench_FwGm.py +++ b/aviary/validation_cases/benchmark_tests/test_bench_FwGm.py @@ -8,6 +8,7 @@ from aviary.interface.default_phase_info.two_dof import phase_info from aviary.interface.methods_for_level1 import run_aviary from aviary.variable_info.variables import Aircraft, Mission +from aviary.variable_info.enums import Verbosity @use_tempdirs @@ -20,7 +21,7 @@ def setUp(self): def bench_test_swap_3_FwGm_IPOPT(self): local_phase_info = deepcopy(phase_info) prob = run_aviary('models/test_aircraft/aircraft_for_bench_FwGm.csv', - local_phase_info, optimizer='IPOPT') + local_phase_info, verbosity=Verbosity.QUIET, optimizer='IPOPT') rtol = 1e-2 @@ -44,7 +45,7 @@ def bench_test_swap_3_FwGm_IPOPT(self): def bench_test_swap_3_FwGm_SNOPT(self): local_phase_info = deepcopy(phase_info) prob = run_aviary('models/test_aircraft/aircraft_for_bench_FwGm.csv', - local_phase_info, optimizer='SNOPT') + local_phase_info, verbosity=Verbosity.QUIET, optimizer='SNOPT') rtol = 1e-2 diff --git a/aviary/validation_cases/benchmark_tests/test_bench_GwFm.py b/aviary/validation_cases/benchmark_tests/test_bench_GwFm.py index e33c73d16..08d1d50f4 100644 --- a/aviary/validation_cases/benchmark_tests/test_bench_GwFm.py +++ b/aviary/validation_cases/benchmark_tests/test_bench_GwFm.py @@ -13,6 +13,7 @@ from openmdao.core.problem import _clear_problem_names from aviary.interface.methods_for_level1 import run_aviary +from aviary.variable_info.enums import Verbosity from aviary.validation_cases.benchmark_utils import \ compare_against_expected_values @@ -361,14 +362,14 @@ def setUp(self): @require_pyoptsparse(optimizer="IPOPT") def bench_test_swap_1_GwFm(self): prob = run_aviary('models/test_aircraft/aircraft_for_bench_GwFm.csv', self.phase_info, - max_iter=50, optimizer='IPOPT') + max_iter=50, optimizer='IPOPT', verbosity=Verbosity.QUIET) compare_against_expected_values(prob, self.expected_dict) @require_pyoptsparse(optimizer="SNOPT") def bench_test_swap_1_GwFm_SNOPT(self): prob = run_aviary('models/test_aircraft/aircraft_for_bench_GwFm.csv', self.phase_info, - max_iter=50, optimizer='SNOPT') + max_iter=50, optimizer='SNOPT', verbosity=Verbosity.QUIET) compare_against_expected_values(prob, self.expected_dict) diff --git a/aviary/validation_cases/benchmark_tests/test_bench_GwGm.py b/aviary/validation_cases/benchmark_tests/test_bench_GwGm.py index 686e9edb4..e57fd9663 100644 --- a/aviary/validation_cases/benchmark_tests/test_bench_GwGm.py +++ b/aviary/validation_cases/benchmark_tests/test_bench_GwGm.py @@ -8,7 +8,7 @@ from aviary.interface.default_phase_info.two_dof import phase_info from aviary.interface.methods_for_level1 import run_aviary from aviary.variable_info.variables import Aircraft, Mission -from aviary.variable_info.enums import AnalysisScheme +from aviary.variable_info.enums import AnalysisScheme, Verbosity @use_tempdirs @@ -21,7 +21,7 @@ def setUp(self): def test_bench_GwGm(self): local_phase_info = deepcopy(phase_info) prob = run_aviary('models/test_aircraft/aircraft_for_bench_GwGm.csv', - local_phase_info, optimizer='IPOPT') + local_phase_info, optimizer='IPOPT', verbosity=Verbosity.QUIET) rtol = 0.01 @@ -48,7 +48,7 @@ def test_bench_GwGm(self): def test_bench_GwGm_SNOPT(self): local_phase_info = deepcopy(phase_info) prob = run_aviary('models/test_aircraft/aircraft_for_bench_GwGm.csv', - local_phase_info, optimizer='SNOPT') + local_phase_info, optimizer='SNOPT', verbosity=Verbosity.QUIET) rtol = 0.01 @@ -75,7 +75,7 @@ def test_bench_GwGm_SNOPT(self): def test_bench_GwGm_SNOPT_lbm_s(self): local_phase_info = deepcopy(phase_info) prob = run_aviary('models/test_aircraft/aircraft_for_bench_GwGm_lbm_s.csv', - local_phase_info, optimizer='SNOPT') + local_phase_info, optimizer='SNOPT', verbosity=Verbosity.QUIET) rtol = 0.01 @@ -103,7 +103,7 @@ def test_bench_GwGm_shooting(self): local_phase_info = deepcopy(phase_info) prob = run_aviary('models/test_aircraft/aircraft_for_bench_GwGm.csv', local_phase_info, optimizer='IPOPT', run_driver=False, - analysis_scheme=AnalysisScheme.SHOOTING) + analysis_scheme=AnalysisScheme.SHOOTING, verbosity=Verbosity.QUIET) rtol = 0.01 diff --git a/aviary/validation_cases/benchmark_tests/test_subsystems_within_a_mission.py b/aviary/validation_cases/benchmark_tests/test_subsystems_within_a_mission.py index fa600e998..6966166bc 100644 --- a/aviary/validation_cases/benchmark_tests/test_subsystems_within_a_mission.py +++ b/aviary/validation_cases/benchmark_tests/test_subsystems_within_a_mission.py @@ -5,6 +5,7 @@ from openmdao.utils.testing_utils import use_tempdirs from aviary.interface.methods_for_level2 import AviaryProblem +from aviary.variable_info.enums import Verbosity from aviary.subsystems.test.test_dummy_subsystem import ( PostOnlyBuilder, ArrayGuessSubsystemBuilder, AdditionalArrayGuessSubsystemBuilder, @@ -70,7 +71,7 @@ def test_subsystems_in_a_mission(self): # Link phases and variables prob.link_phases() - prob.add_driver("SLSQP", max_iter=0) + prob.add_driver("SLSQP", max_iter=0, verbosity=Verbosity.QUIET) prob.add_design_variables() From 24d03fe8d62f783b07da8eb9b1273b3e3cb80db3 Mon Sep 17 00:00:00 2001 From: Carl Recine Date: Mon, 18 Mar 2024 14:47:59 -0700 Subject: [PATCH 3/3] Adding documentation --- aviary/docs/developer_guide/coding_standards.md | 3 +++ aviary/docs/user_guide/troubleshooting.md | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/aviary/docs/developer_guide/coding_standards.md b/aviary/docs/developer_guide/coding_standards.md index 024638747..619e12c7d 100644 --- a/aviary/docs/developer_guide/coding_standards.md +++ b/aviary/docs/developer_guide/coding_standards.md @@ -16,6 +16,9 @@ To set up pre-commit in your development python environment, there are a few one The Aviary repository contains a configuration file that defines what is run when commits are made and with what options enabled. Currently this is limited to autopep8 with a max line length restriction. +### Controlling Display Levels +To make debugging issues easier, it is strongly recommended to make use of the `VERBOSITY` setting. This allows control over how much information is displayed to a user; too much information makes finding relevant information difficult and not enough information can make tracking difficult. `Brief` should be the default in most cases; however, `Quiet` should be the default for tests. + ## Naming Conventions ### Variables When it comes to variable naming, always be verbose! The Aviary team considers long but clear and descriptive names superior to shortened or vague names. Typing out a long name is only difficult once, as most IDEs will help you auto-complete long variable names, but the readability they add lasts a lifetime! diff --git a/aviary/docs/user_guide/troubleshooting.md b/aviary/docs/user_guide/troubleshooting.md index 1a8770efb..17a54be33 100644 --- a/aviary/docs/user_guide/troubleshooting.md +++ b/aviary/docs/user_guide/troubleshooting.md @@ -36,6 +36,12 @@ Once you get good results with the simple mission and understand the results, yo ### Interpreting optimized results +```{note} +A `VERBOSITY` control has been added to minimize the amount of unnecessary information that will be displayed. +Currently Quiet, Brief [default], Verbose, and Debug are supported. Quiet will suppress practically everything other than warnings and errors. Verbose will include information such as the progress of the optimization, instead of just a final summary. And Debug will contain detailed information about many of the steps as they happen. +Some of Aviary's CLI functions, such as `fortran_to_aviary`, allow the verbosity to be set directly with a command line argument. `run_mission` uses the variable `settings:verbosity` to control the print levels. +``` + Once you've built up an understanding of your model and have successfully performed optimization, you can start to interpret the results. This is where a mix of aircraft engineering knowledge and optimization knowledge is extremely helpful.