From d8ca8aab957b1d51714d4ad503ed2426e1285e3d Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 30 Aug 2024 17:23:58 -0600 Subject: [PATCH 01/40] Panzer: adding transient sensitivity analysis capability Signed-off-by: Roger Pawlowski --- .../example/main_driver/CMakeLists.txt | 18 +- .../energy-transient-tempus-sa-blocked.xml | 465 ++++++++++++++++++ .../example/main_driver/main_driver_sa.cpp | 419 ++++++++++++++++ 3 files changed, 899 insertions(+), 3 deletions(-) create mode 100644 packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml create mode 100644 packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp diff --git a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt index 0ed82da7ca7d..98546631cd7e 100644 --- a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt +++ b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt @@ -5,7 +5,6 @@ TRIBITS_INCLUDE_DIRECTORIES(${PARENT_PACKAGE_SOURCE_DIR}/disc-fe/test/closure_mo TRIBITS_INCLUDE_DIRECTORIES(${PACKAGE_SOURCE_DIR}/test/bcstrategy) SET(main_driver_SOURCES - main_driver.cpp user_app_NOXObserverFactory.hpp user_app_NOXObserver_WriteToExodus.hpp user_app_NOXObserver_NeumannBCAnalyticSystemTest.hpp @@ -13,7 +12,12 @@ SET(main_driver_SOURCES TRIBITS_ADD_EXECUTABLE( main_driver - SOURCES ${main_driver_SOURCES} + SOURCES main_driver.cpp ${main_driver_SOURCES} + ) + +TRIBITS_ADD_EXECUTABLE( + main_driver_sa + SOURCES main_driver_sa.cpp ${main_driver_SOURCES} ) TRIBITS_COPY_FILES_TO_BINARY_DIR(main_driver_files @@ -25,6 +29,7 @@ TRIBITS_COPY_FILES_TO_BINARY_DIR(main_driver_files energy-ss-loca-eigenvalue.xml energy-ss-blocked.xml energy-transient-tempus-blocked.xml + energy-transient-tempus-sa-blocked.xml energy-neumann.xml energy-ss-blocked-tp.xml periodic_wedge.xml @@ -57,7 +62,7 @@ TRIBITS_ADD_ADVANCED_TEST( PASS_REGULAR_EXPRESSION "panzer::MainDriver run completed." ) - TRIBITS_ADD_ADVANCED_TEST( +TRIBITS_ADD_ADVANCED_TEST( main_driver_energy-ss-point-calc TEST_0 EXEC main_driver ARGS --i=energy-ss-point-calc.xml --exodus-io-num-procs=1 --point-calc @@ -109,6 +114,13 @@ IF(${PACKAGE_NAME}_ENABLE_Teko) ARGS --i=energy-transient-tempus-blocked.xml PASS_REGULAR_EXPRESSION "panzer::MainDriver run completed." ) + + TRIBITS_ADD_ADVANCED_TEST( + main_driver_energy-transient-tempus-sa-blocked + TEST_0 EXEC main_driver_sa + ARGS --i=energy-transient-tempus-sa-blocked.xml --point-calc + PASS_REGULAR_EXPRESSION "panzer::MainDriver run completed." + ) ENDIF() ENDIF() diff --git a/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml b/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml new file mode 100644 index 000000000000..27210907529d --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml @@ -0,0 +1,465 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp b/packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp new file mode 100644 index 000000000000..9862c9ee6808 --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp @@ -0,0 +1,419 @@ +// @HEADER +// ***************************************************************************** +// Panzer: A partial differential equation assembly +// engine for strongly coupled complex multiphysics systems +// +// Copyright 2011 NTESS and the Panzer contributors. +// SPDX-License-Identifier: BSD-3-Clause +// ***************************************************************************** +// @HEADER + +#include "Teuchos_ConfigDefs.hpp" +#include "Teuchos_RCP.hpp" +#include "Teuchos_TimeMonitor.hpp" +#include "Teuchos_StackedTimer.hpp" +#include "Teuchos_DefaultComm.hpp" +#include "Teuchos_CommHelpers.hpp" +#include "Teuchos_GlobalMPISession.hpp" +#include "Teuchos_CommandLineProcessor.hpp" +#include "Teuchos_XMLParameterListHelpers.hpp" +#include "Teuchos_FancyOStream.hpp" +#include "Teuchos_oblackholestream.hpp" +#include "Teuchos_Assert.hpp" +#include "Teuchos_as.hpp" + +#include "Panzer_NodeType.hpp" +#include "PanzerAdaptersSTK_config.hpp" +#include "Panzer_STK_ModelEvaluatorFactory.hpp" +#include "Panzer_ClosureModel_Factory_TemplateManager.hpp" +#include "Panzer_PauseToAttach.hpp" +#include "Panzer_String_Utilities.hpp" +#include "Panzer_ThyraObjContainer.hpp" +#include "Thyra_VectorSpaceBase.hpp" + +#include "NOX_Utils.H" +#include "NOX_Observer_Print.hpp" + +#include "Tempus_IntegratorBasic.hpp" +#include "Tempus_StepperFactory.hpp" + +#include "user_app_ClosureModel_Factory_TemplateBuilder.hpp" +#include "user_app_EquationSetFactory.hpp" +#include "user_app_BCStrategy_Factory.hpp" +#include "user_app_NOXObserverFactory.hpp" +#ifdef PANZER_HAVE_TEMPUS +#include "user_app_TempusObserverFactory.hpp" +#endif +#include "user_app_ResponseEvaluatorFactory_HOFlux.hpp" + +#include "Panzer_ResponseEvaluatorFactory_Probe.hpp" + +#include + +#include +#include + +// ************************************************* +// ************************************************* + +// Main Driver for a transient sensitivity analysis simulation + +// ************************************************* +// ************************************************* + +int main(int argc, char *argv[]) +{ + int status = 0; + + Teuchos::oblackholestream blackhole; + Teuchos::GlobalMPISession mpiSession(&argc, &argv, &blackhole); + Kokkos::initialize(argc,argv); + + Teuchos::RCP out = Teuchos::rcp(new Teuchos::FancyOStream(Teuchos::rcp(&std::cout,false))); + Teuchos::RCP pout = Teuchos::rcp(new Teuchos::FancyOStream(Teuchos::rcp(&std::cout,false))); + if (mpiSession.getNProc() > 1) { + out->setShowProcRank(true); + out->setOutputToRootOnly(0); + } + + try { + const auto stackedTimer = Teuchos::rcp(new Teuchos::StackedTimer("Panzer Main Driver Sensitivity Analysis")); + Teuchos::TimeMonitor::setStackedTimer(stackedTimer); + + Teuchos::RCP > comm = Teuchos::DefaultComm::getComm(); + + // Parse the command line arguments + std::string input_file_name = "user_app.xml"; + int exodus_io_num_procs = 0; + bool pauseToAttachOn = false; + bool pointCalculation = false; + bool printTimers = true; + bool printInputPL = false; + bool overrideNoxOutput = true; + { + Teuchos::CommandLineProcessor clp; + + clp.setOption("i", &input_file_name, "User_App input xml filename"); + clp.setOption("exodus-io-num-procs", &exodus_io_num_procs, "Number of processes that can access the file system at the same time to read their portion of a sliced exodus file in parallel. Defaults to 0 - implies all processes for the run can access the file system at the same time."); + clp.setOption("pause-to-attach","disable-pause-to-attach", &pauseToAttachOn, "Call pause to attach, default is off."); + clp.setOption("point-calc","disable-point-calc", &pointCalculation, "Enable the probe evaluator unit test."); + clp.setOption("time","no-time", &printTimers, "Print the timing information."); + clp.setOption("pl","no-pl", &printTimers, "Print the input ParameterList at the start of the run."); + clp.setOption("override-nox-output","default-nox-output", &overrideNoxOutput, "Override default nox printing with new print observer."); + + Teuchos::CommandLineProcessor::EParseCommandLineReturn parse_return = + clp.parse(argc,argv,&std::cerr); + + TEUCHOS_TEST_FOR_EXCEPTION(parse_return != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL, + std::runtime_error, "Failed to parse command line!"); + } + + if(pauseToAttachOn) + panzer::pauseToAttach(); + + TEUCHOS_ASSERT(exodus_io_num_procs <= comm->getSize()); + if (exodus_io_num_procs != 0) + Ioss::SerializeIO::setGroupFactor(exodus_io_num_procs); + + // Parse the input file and broadcast to other processes + Teuchos::RCP input_params = Teuchos::rcp(new Teuchos::ParameterList("User_App Parameters")); + Teuchos::updateParametersFromXmlFileAndBroadcast(input_file_name, input_params.ptr(), *comm); + + if (printInputPL) + *out << *input_params << std::endl; + + Teuchos::ParameterList solver_factories = input_params->sublist("Solver Factories"); + input_params->remove("Solver Factories"); + + // Add in the application specific equation set factory + Teuchos::RCP eqset_factory = Teuchos::rcp(new user_app::MyFactory); + + // Add in the application specific closure model factory + user_app::MyModelFactory_TemplateBuilder cm_builder; + panzer::ClosureModelFactory_TemplateManager cm_factory; + cm_factory.buildObjects(cm_builder); + + // Add in the application specific bc factory + user_app::BCFactory bc_factory; + + // Create the global data + Teuchos::RCP global_data = panzer::createGlobalData(); + + // A GlobalStatistics closure model requires the comm to be set in the user data. + input_params->sublist("User Data").set("Comm", comm); + + Teuchos::RCP > stkIOResponseLibrary + = Teuchos::rcp(new panzer::ResponseLibrary()); + + Teuchos::RCP > physics; + Teuchos::RCP > solver; + Teuchos::RCP > rLibrary; + std::vector > physicsBlocks; + Teuchos::RCP > linObjFactory; + std::map responseIndexToName; + { + Teuchos::ParameterList responses = input_params->sublist("Responses"); + input_params->remove("Responses"); + + panzer_stk::ModelEvaluatorFactory me_factory; + me_factory.setParameterList(input_params); + me_factory.buildObjects(comm,global_data,eqset_factory,bc_factory,cm_factory); + + // add a volume response functional for each field + for(Teuchos::ParameterList::ConstIterator itr=responses.begin();itr!=responses.end();++itr) { + const std::string name = responses.name(itr); + TEUCHOS_ASSERT(responses.entry(itr).isList()); + Teuchos::ParameterList & lst = Teuchos::getValue(responses.entry(itr)); + + // parameterize the builder + panzer::FunctionalResponse_Builder builder; + builder.comm = MPI_COMM_WORLD; // good enough + builder.cubatureDegree = 2; + builder.requiresCellIntegral = lst.isType("Requires Cell Integral") ? lst.get("Requires Cell Integral"): false; + std::cout << "ROGER requires ce=" << builder.requiresCellIntegral << std::endl; + builder.quadPointField = lst.get("Field Name"); + + // add the response + std::vector eblocks; + panzer::StringTokenizer(eblocks,lst.get("Element Blocks"),",",true); + + std::vector wkst_descs; + for(std::size_t i=0;i nof; + { + nof = Teuchos::rcp(new user_app::NOXObserverFactory(stkIOResponseLibrary)); + + Teuchos::RCP observers_to_build = + Teuchos::parameterList(solver_factories.sublist("NOX Observers")); + + nof->setParameterList(observers_to_build); + } + +#ifdef PANZER_HAVE_TEMPUS + Teuchos::RCP tof; + { + tof = Teuchos::rcp(new user_app::TempusObserverFactory(stkIOResponseLibrary,rLibrary->getWorksetContainer())); + } + solver = me_factory.buildResponseOnlyModelEvaluator(physics,global_data,Teuchos::null,nof.ptr(),tof.ptr()); +#else + solver = me_factory.buildResponseOnlyModelEvaluator(physics,global_data,nof.ptr()); +#endif + } + } + + // setup outputs to mesh on the stkIOResponseLibrary + //////////////////////////////////////////////////////////////// + + stkIOResponseLibrary->initialize(*rLibrary); + + // 1. Register correct aggregator and reserve response - this was done in the appropriate observer object + + // 2. Build volume field managers + { + Teuchos::ParameterList user_data(input_params->sublist("User Data")); + user_data.set("Workset Size",input_params->sublist("Assembly").get("Workset Size")); + + stkIOResponseLibrary->buildResponseEvaluators(physicsBlocks, + cm_factory, + input_params->sublist("Closure Models"), + user_data); + } + + // setup outputs for the point calculation + //////////////////////////////////////////////////////////////// + + Teuchos::RCP > pointResponseLibrary + = Teuchos::rcp(new panzer::ResponseLibrary); + + if(pointCalculation) { + pointResponseLibrary->initialize(*rLibrary); + + { + panzer::ProbeResponse_Builder builder; + builder.comm = MPI_COMM_WORLD; + builder.point = Teuchos::Array{0.5,0.5}; // Bottom + builder.cubatureDegree = 2; + builder.fieldName = "TEMPERATURE"; + builder.applyDirichletToDerivative = false; + + std::vector descriptors; + descriptors.push_back(panzer::WorksetDescriptor("eblock-0_0")); + + pointResponseLibrary->addResponse("Value In Middle",descriptors,builder); + } + + { + Teuchos::ParameterList user_data(input_params->sublist("User Data")); + user_data.set("Workset Size",input_params->sublist("Assembly").get("Workset Size")); + + pointResponseLibrary->buildResponseEvaluators(physicsBlocks, + *eqset_factory, + cm_factory, + input_params->sublist("Closure Models"), + user_data); + } + + { + Teuchos::RCP > resp + = Teuchos::rcp_dynamic_cast >(pointResponseLibrary->getResponse("Value In Middle"),true); + + const auto vec = Thyra::createMember(*resp->getVectorSpace(),"Value In Middle Response Thyra Vector"); + resp->setVector(vec); + } + } + + // Tempus stepper + using namespace Teuchos; + using namespace Tempus; + + RCP tempus_pl = parameterList(std::string("Tempus")); + *tempus_pl = input_params->sublist("Solution Control").sublist("Tempus"); + + if (overrideNoxOutput) { + auto nox_utils = Teuchos::rcp(new NOX::Utils(NOX::Utils::Error,comm->getRank(),0,3)); + auto print_nonlinear = Teuchos::rcp(new NOX::ObserverPrint(nox_utils)); + tempus_pl->sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true).sublist("Solver Options",true).set>("Observer",print_nonlinear); + } + + auto integrator = createIntegratorBasic(tempus_pl, physics, false); + + std::cout << "ROGER solver=" << integrator->getStepper()->getSolver() << std::endl; + + integrator->initialize(); + + std::cout << "ROGER solver=" << integrator->getStepper()->getSolver() << std::endl; + + RCP> x0 = physics->getNominalValues().get_x()->clone_v(); + integrator->initializeSolutionHistory(0.0, x0); + + bool integratorStatus = integrator->advanceTime(); + TEUCHOS_ASSERT(integratorStatus); + + auto x = integrator->getSolutionHistory()->getCurrentState()->getX(); + auto x_dot = integrator->getSolutionHistory()->getCurrentState()->getXDot(); + + { + // get responses if there are any + ////////////////////////////////////////////// + if(physics->Ng()>0) { + + Thyra::ModelEvaluatorBase::InArgs respInArgs = physics->createInArgs(); + Thyra::ModelEvaluatorBase::OutArgs respOutArgs = physics->createOutArgs(); + + TEUCHOS_ASSERT(physics->Ng()==respOutArgs.Ng()); + + respInArgs.set_x(x); + respInArgs.set_x_dot(x_dot); + + // set up response out args + for(int i=0;i > response = Thyra::createMember(*physics->get_g_space(i)); + respOutArgs.set_g(i,response); + } + + // Now, solve the problem and return the responses + physics->evalModel(respInArgs, respOutArgs); + + // loop over out args for printing + for(int i=0;i > response = respOutArgs.get_g(i); + + TEUCHOS_ASSERT(response!=Teuchos::null); // should not be null! + + *out << "Response Value \"" << responseIndexToName[i] << "\": " << Thyra::get_ele(*response,0) << std::endl; + } + } + + if(pointCalculation) { + stackedTimer->start("Point Value Response Calculation"); + // initialize the assembly container + panzer::AssemblyEngineInArgs ae_inargs; + ae_inargs.container_ = linObjFactory->buildLinearObjContainer(); + ae_inargs.ghostedContainer_ = linObjFactory->buildGhostedLinearObjContainer(); + ae_inargs.alpha = 0.0; + ae_inargs.beta = 1.0; + ae_inargs.evaluate_transient_terms = false; + + // initialize the ghosted container + linObjFactory->initializeGhostedContainer(panzer::LinearObjContainer::X,*ae_inargs.ghostedContainer_); + + const Teuchos::RCP> thGlobalContainer + = Teuchos::rcp_dynamic_cast>(ae_inargs.container_,true); + thGlobalContainer->set_x_th(x); + + // evaluate current on contacts + pointResponseLibrary->addResponsesToInArgs(ae_inargs); + pointResponseLibrary->evaluate(ae_inargs); + + // output current values + *out << "\nPoint Values: \n"; + { + std::string currentRespName = "Value In Middle"; + + Teuchos::RCP > resp = + Teuchos::rcp_dynamic_cast>(pointResponseLibrary->getResponse(currentRespName),true); + + // Linear problem with analytic solution + const double gold_value = 0.5; + const double tol = 1.0e-8; + *out << " " << currentRespName << " = " << resp->value << ", error = " << fabs(resp->value - gold_value) << ", tol = " << tol << std::endl; + // TEUCHOS_ASSERT(fabs(resp->value - gold_value) < tol); + } + + stackedTimer->stop("Point Value Response Calculation"); + } + } + + stackedTimer->stopBaseTimer(); + if (printTimers) { + Teuchos::StackedTimer::OutputOptions options; + options.output_fraction = true; + options.output_minmax = false; + options.output_histogram = false; + options.num_histogram = 5; + std::string timing_file_name = input_file_name+"_timing.log"; + std::fstream timing_file{timing_file_name,std::ios::out | std::ios::trunc}; + stackedTimer->report(timing_file, Teuchos::DefaultComm::getComm(), options); + } + + } + catch (std::exception& e) { + *out << "*********** Caught Exception: Begin Error Report ***********" << std::endl; + *out << e.what() << std::endl; + *out << "************ Caught Exception: End Error Report ************" << std::endl; + status = -1; + } + catch (std::string& msg) { + *out << "*********** Caught Exception: Begin Error Report ***********" << std::endl; + *out << msg << std::endl; + *out << "************ Caught Exception: End Error Report ************" << std::endl; + status = -1; + } + catch (...) { + *out << "*********** Caught Exception: Begin Error Report ***********" << std::endl; + *out << "Caught UNKOWN exception" << std::endl; + *out << "************ Caught Exception: End Error Report ************" << std::endl; + status = -1; + } + + if (status == 0) + *out << "panzer::MainDriver run completed." << std::endl; + + return status; +} From 1c433855a80b0d0df3c4999fae70d0106d3d8ff6 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Tue, 3 Sep 2024 08:13:42 -0600 Subject: [PATCH 02/40] Panzer: xml to yaml tools Signed-off-by: Roger Pawlowski --- .../example/main_driver/CMakeLists.txt | 5 +++ .../main_driver/Panzer_xml_to_yaml.cpp | 34 +++++++++++++++++++ .../example/main_driver/main_driver.cpp | 2 +- 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 packages/panzer/adapters-stk/example/main_driver/Panzer_xml_to_yaml.cpp diff --git a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt index 98546631cd7e..c0cb3417c20e 100644 --- a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt +++ b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt @@ -20,6 +20,11 @@ TRIBITS_ADD_EXECUTABLE( SOURCES main_driver_sa.cpp ${main_driver_SOURCES} ) +TRIBITS_ADD_EXECUTABLE( + xml_to_yaml + SOURCES Panzer_xml_to_yaml.cpp + ) + TRIBITS_COPY_FILES_TO_BINARY_DIR(main_driver_files SOURCE_FILES energy-ss.xml diff --git a/packages/panzer/adapters-stk/example/main_driver/Panzer_xml_to_yaml.cpp b/packages/panzer/adapters-stk/example/main_driver/Panzer_xml_to_yaml.cpp new file mode 100644 index 000000000000..4c331f044a72 --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/Panzer_xml_to_yaml.cpp @@ -0,0 +1,34 @@ +// @HEADER +// ***************************************************************************** +// Panzer: A partial differential equation assembly +// engine for strongly coupled complex multiphysics systems +// +// Copyright 2011 NTESS and the Panzer contributors. +// SPDX-License-Identifier: BSD-3-Clause +// ***************************************************************************** +// @HEADER + +#include "Teuchos_ParameterList.hpp" +#include "Teuchos_XMLParser.hpp" +#include "Teuchos_YamlParser_decl.hpp" +#include "Teuchos_XMLParameterListCoreHelpers.hpp" +#include "Teuchos_YamlParameterListCoreHelpers.hpp" +#include "Teuchos_Assert.hpp" +#include + +using namespace Teuchos; + +int main(int argc, char *argv[]) { + + TEUCHOS_ASSERT(argc == 2); + + std::string file_prefix(argv[1]); + + std::string input_xml_file_name(file_prefix+".xml"); + auto pList = getParametersFromXmlFile(input_xml_file_name); + + std::string output_yaml_file_name(file_prefix+".yaml"); + writeParameterListToYamlFile(*pList,output_yaml_file_name); + + return 0; +} diff --git a/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp b/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp index 3dbcbcb1daa7..6877fef5e0db 100644 --- a/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp +++ b/packages/panzer/adapters-stk/example/main_driver/main_driver.cpp @@ -94,7 +94,7 @@ int main(int argc, char *argv[]) clp.setOption("flux-calc","disable-flux-calc", &fluxCalculation, "Enable the flux calculation."); clp.setOption("point-calc","disable-point-calc", &pointCalculation, "Enable the probe evaluator unit test."); clp.setOption("time","no-time", &printTimers, "Print the timing information."); - clp.setOption("pl","no-pl", &printTimers, "Print the input ParameterList at the start of the run."); + clp.setOption("pl","no-pl", &printInputPL, "Print the input ParameterList at the start of the run."); Teuchos::CommandLineProcessor::EParseCommandLineReturn parse_return = clp.parse(argc,argv,&std::cerr); From 39f373d48e15fbdb0d631a363520ea0cdf41508c Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Tue, 3 Sep 2024 09:11:17 -0600 Subject: [PATCH 03/40] NOX: cleanup shadow warnings Signed-off-by: Roger Pawlowski --- .../nox/test/lapack/StatusTests/StatusTests.C | 203 +++++++++--------- 1 file changed, 102 insertions(+), 101 deletions(-) diff --git a/packages/nox/test/lapack/StatusTests/StatusTests.C b/packages/nox/test/lapack/StatusTests/StatusTests.C index 631a649e9571..98ff82c210d4 100644 --- a/packages/nox/test/lapack/StatusTests/StatusTests.C +++ b/packages/nox/test/lapack/StatusTests/StatusTests.C @@ -206,10 +206,8 @@ private: }; -//! Main subroutine of Broyden.C int main(int argc, char *argv[]) { - // Basic declarations to clean up the code using Teuchos::RCP; using Teuchos::rcp; using Teuchos::ParameterList; @@ -231,31 +229,82 @@ int main(int argc, char *argv[]) rcp(new Teuchos::ParameterList); Teuchos::ParameterList& solverParameters = *solverParametersPtr; - // Set the nonlinear solver method - //solverParameters.set("Nonlinear Solver", "Tensor-Krylov Based"); - //solverParameters.set("Nonlinear Solver", "Tensor Based"); - solverParameters.set("Nonlinear Solver", "Line Search Based"); + Teuchos::ParameterList status_test_list; - // Sublist for printing parameters - Teuchos::ParameterList& printParams = solverParameters.sublist("Printing"); - //printParams.set("MyPID", 0); - printParams.set("Output Precision", 3); - printParams.set("Output Processor", 0); - printParams.set("Output Information", - NOX::Utils::OuterIteration + - NOX::Utils::OuterIterationStatusTest + - NOX::Utils::InnerIteration + - NOX::Utils::Details + - NOX::Utils::Warning); - NOX::Utils utils(printParams); + { + // Set the nonlinear solver method + //solverParameters.set("Nonlinear Solver", "Tensor-Krylov Based"); + //solverParameters.set("Nonlinear Solver", "Tensor Based"); + solverParameters.set("Nonlinear Solver", "Line Search Based"); + + // Sublist for printing parameters + Teuchos::ParameterList& printParams = solverParameters.sublist("Printing"); + //printParams.set("MyPID", 0); + printParams.set("Output Precision", 3); + printParams.set("Output Processor", 0); + printParams.set("Output Information", + NOX::Utils::OuterIteration + + NOX::Utils::OuterIterationStatusTest + + NOX::Utils::InnerIteration + + NOX::Utils::Details + + NOX::Utils::Warning); + std::cout << "\n\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" << std::endl; + std::cout << "Testing Convergence tests (NormF, NormUpdate, NormWRMS) ..." + << std::endl; + std::cout << "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" << std::endl; - // Convergence tests and factory + // ************************************** + // Create the convergence tests + // ************************************** + status_test_list.set("Test Type", "Combo"); + status_test_list.set("Combo Type", "OR"); + status_test_list.set("Number of Tests", 5); + Teuchos::ParameterList& conv = status_test_list.sublist("Test 0"); + Teuchos::ParameterList& fv = status_test_list.sublist("Test 1"); + Teuchos::ParameterList& divergence = status_test_list.sublist("Test 2"); + Teuchos::ParameterList& stagnation = status_test_list.sublist("Test 3"); + Teuchos::ParameterList& maxiters = status_test_list.sublist("Test 4"); + + conv.set("Test Type", "Combo"); + conv.set("Combo Type", "AND"); + conv.set("Number of Tests", 3); + Teuchos::ParameterList& normF = conv.sublist("Test 0"); + Teuchos::ParameterList& normWRMS = conv.sublist("Test 1"); + Teuchos::ParameterList& normUpdate = conv.sublist("Test 2"); + normF.set("Test Type", "NormF"); + normF.set("Tolerance", 1.0e-12); + normF.set("Norm Type", "Two Norm"); + normF.set("Scale Type", "Unscaled"); + normWRMS.set("Test Type", "NormWRMS"); + normWRMS.set("Absolute Tolerance", 1.0e-8); + normWRMS.set("Relative Tolerance", 1.0e-5); + normWRMS.set("Tolerance", 1.0); + normWRMS.set("BDF Multiplier", 1.0); + normWRMS.set("Alpha", 1.0); + normWRMS.set("Beta", 0.5); + normWRMS.set("Disable Implicit Weighting", true); + normUpdate.set("Test Type", "NormUpdate"); + normUpdate.set("Norm Type", "One Norm"); + normUpdate.set("Scale Type", "Scaled"); + + fv.set("Test Type", "FiniteValue"); + fv.set("Vector Type", "F Vector"); + fv.set("Norm Type", "Two Norm"); + + divergence.set("Test Type", "Divergence"); + divergence.set("Tolerance", 1.0e+20); + divergence.set("Consecutive Iterations", 3); + + stagnation.set("Test Type", "Stagnation"); + stagnation.set("Tolerance", 1.0); + stagnation.set("Consecutive Iterations", 5); - std::cout << "\n\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" << std::endl; - std::cout << "Testing Convergence tests (NormF, NormUpdate, NormWRMS) ..." - << std::endl; - std::cout << "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" << std::endl; + maxiters.set("Test Type", "MaxIters"); + maxiters.set("Maximum Iterations", 20); + } + + NOX::Utils utils(solverParameters.sublist("Printing")); // Set up the problem interface Broyden broyden(100,0.99); @@ -265,88 +314,40 @@ int main(int argc, char *argv[]) // specified problem. RCP grp = rcp(new NOX::LAPACK::Group(broyden)); - // ************************************** - // Create the convergence tests - // ************************************** - Teuchos::ParameterList stl; - stl.set("Test Type", "Combo"); - stl.set("Combo Type", "OR"); - stl.set("Number of Tests", 5); - Teuchos::ParameterList& conv = stl.sublist("Test 0"); - Teuchos::ParameterList& fv = stl.sublist("Test 1"); - Teuchos::ParameterList& divergence = stl.sublist("Test 2"); - Teuchos::ParameterList& stagnation = stl.sublist("Test 3"); - Teuchos::ParameterList& maxiters = stl.sublist("Test 4"); - - conv.set("Test Type", "Combo"); - conv.set("Combo Type", "AND"); - conv.set("Number of Tests", 3); - Teuchos::ParameterList& normF = conv.sublist("Test 0"); - Teuchos::ParameterList& normWRMS = conv.sublist("Test 1"); - Teuchos::ParameterList& normUpdate = conv.sublist("Test 2"); - normF.set("Test Type", "NormF"); - normF.set("Tolerance", 1.0e-12); - normF.set("Norm Type", "Two Norm"); - normF.set("Scale Type", "Unscaled"); - normWRMS.set("Test Type", "NormWRMS"); - normWRMS.set("Absolute Tolerance", 1.0e-8); - normWRMS.set("Relative Tolerance", 1.0e-5); - normWRMS.set("Tolerance", 1.0); - normWRMS.set("BDF Multiplier", 1.0); - normWRMS.set("Alpha", 1.0); - normWRMS.set("Beta", 0.5); - normWRMS.set("Disable Implicit Weighting", true); - normUpdate.set("Test Type", "NormUpdate"); - normUpdate.set("Norm Type", "One Norm"); - normUpdate.set("Scale Type", "Scaled"); - - fv.set("Test Type", "FiniteValue"); - fv.set("Vector Type", "F Vector"); - fv.set("Norm Type", "Two Norm"); - - divergence.set("Test Type", "Divergence"); - divergence.set("Tolerance", 1.0e+20); - divergence.set("Consecutive Iterations", 3); - - stagnation.set("Test Type", "Stagnation"); - stagnation.set("Tolerance", 1.0); - stagnation.set("Consecutive Iterations", 5); - - maxiters.set("Test Type", "MaxIters"); - maxiters.set("Maximum Iterations", 20); - Teuchos::RCP statusTestsCombo; Teuchos::RCP st_params; #ifdef HAVE_TEUCHOS_EXTENDED std::cout << "Writing parameter list to \"input.xml\"" << std::endl; - Teuchos::writeParameterListToXmlFile(stl, "input.xml"); + Teuchos::writeParameterListToXmlFile(status_test_list, "input.xml"); std::cout << "Reading parameter list from \"input.xml\"" << std::endl; statusTestsCombo = NOX::StatusTest::buildStatusTests("input.xml", utils); #else - statusTestsCombo = NOX::StatusTest::buildStatusTests(stl, utils); + statusTestsCombo = NOX::StatusTest::buildStatusTests(status_test_list, utils); #endif // ************************************** // Finished: Create the convergence tests // ************************************** - // Create the solver - Teuchos::RCP solver = - NOX::Solver::buildSolver(grp, statusTestsCombo, solverParametersPtr); + { + // Create the solver + Teuchos::RCP solver = + NOX::Solver::buildSolver(grp, statusTestsCombo, solverParametersPtr); - // Solve the nonlinear system - NOX::StatusTest::StatusType status = solver->solve(); + // Solve the nonlinear system + NOX::StatusTest::StatusType status = solver->solve(); - // Print final status - if (status == NOX::StatusTest::Converged && - solver->getNumIterations() == 12) { - final_status_value += 0; - std::cout << "\nConvergence tests passed!" << std::endl; - } - else { - final_status_value += 1; - std::cout << "\nConvergence tests Failed!" << std::endl; + // Print final status + if (status == NOX::StatusTest::Converged && + solver->getNumIterations() == 12) { + final_status_value += 0; + std::cout << "\nConvergence tests passed!" << std::endl; + } + else { + final_status_value += 1; + std::cout << "\nConvergence tests Failed!" << std::endl; + } } // Re-run test with complete checks of status tests @@ -371,7 +372,7 @@ int main(int argc, char *argv[]) Teuchos::RCP solver = NOX::Solver::buildSolver(group, statusTestsCombo, tmpParams); - status = solver->solve(); + NOX::StatusTest::StatusType status = solver->solve(); std::cout << *tmpParams << std::endl; @@ -417,7 +418,7 @@ int main(int argc, char *argv[]) Teuchos::RCP solver = NOX::Solver::buildSolver(group, combo, solverParametersPtr); - status = solver->solve(); + NOX::StatusTest::StatusType status = solver->solve(); if (status == NOX::StatusTest::Converged && solver->getNumIterations() == 11) { @@ -438,16 +439,16 @@ int main(int argc, char *argv[]) std::cout << "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" << std::endl; std::map< std::string, Teuchos::RCP > my_map; - normF.set("Tag", "Norm F Status Test"); + status_test_list.sublist("Test 0").sublist("Test 0").set("Tag", "Norm F Status Test"); Teuchos::RCP all_tests = - NOX::StatusTest::buildStatusTests(stl, utils, &my_map); + NOX::StatusTest::buildStatusTests(status_test_list, utils, &my_map); Teuchos::RCP my_test = my_map["Norm F Status Test"]; Teuchos::RCP my_normF_test= - Teuchos::rcp_dynamic_cast(my_test); + Teuchos::rcp_dynamic_cast(my_test,true); my_normF_test->print(std::cout); @@ -516,7 +517,7 @@ int main(int argc, char *argv[]) RCP sp = rcp(new Teuchos::ParameterList); sp->set("Nonlinear Solver", "Line Search Based"); - sp->sublist("Printing") = printParams; + sp->sublist("Printing") = solverParameters.sublist("Printing"); sp->sublist("Solver Options").set("Status Test Check Type", "Complete"); Teuchos::RCP solver = @@ -593,7 +594,7 @@ int main(int argc, char *argv[]) RCP sp = rcp(new Teuchos::ParameterList); sp->set("Nonlinear Solver", "Line Search Based"); - sp->sublist("Printing") = printParams; + sp->sublist("Printing") = solverParameters.sublist("Printing"); sp->sublist("Solver Options").set("Status Test Check Type", "Complete"); Teuchos::RCP solver = @@ -644,7 +645,7 @@ int main(int argc, char *argv[]) NOX::Solver::buildSolver(group, NOX::StatusTest::buildStatusTests(p, utils), solverParametersPtr); - status = solver->solve(); + NOX::StatusTest::StatusType status = solver->solve(); // A failure reported by max iters is a passing test if (status == NOX::StatusTest::Failed) { @@ -674,7 +675,7 @@ int main(int argc, char *argv[]) Teuchos::RCP solver = NOX::Solver::buildSolver(group, combo, solverParametersPtr); - status = solver->solve(); + NOX::StatusTest::StatusType status = solver->solve(); // A failure reported by finite value is a passing test if (status == NOX::StatusTest::Failed && @@ -705,7 +706,7 @@ int main(int argc, char *argv[]) Teuchos::RCP solver = NOX::Solver::buildSolver(group, combo, solverParametersPtr); - status = solver->solve(); + NOX::StatusTest::StatusType status = solver->solve(); // A failure reported by divergence is a passing test if (status == NOX::StatusTest::Failed && @@ -737,7 +738,7 @@ int main(int argc, char *argv[]) Teuchos::RCP solver = NOX::Solver::buildSolver(group, combo, solverParametersPtr); - status = solver->solve(); + NOX::StatusTest::StatusType status = solver->solve(); // A failure reported by stagnation is a passing test if (status == NOX::StatusTest::Failed && @@ -783,7 +784,7 @@ int main(int argc, char *argv[]) NOX::Solver::buildSolver(group, st, solverParametersPtr); // first time step - status = solver->solve(); + NOX::StatusTest::StatusType status = solver->solve(); TEUCHOS_ASSERT(solver->getNumIterations() == 3); // second time step From 8f5fb08dcfaf465bc8e02018526e80430e03d29e Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Tue, 3 Sep 2024 09:11:45 -0600 Subject: [PATCH 04/40] NOX: fix documentation Signed-off-by: Roger Pawlowski --- packages/nox/src/NOX_Direction_Factory.C | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nox/src/NOX_Direction_Factory.C b/packages/nox/src/NOX_Direction_Factory.C index 3200e1a68d18..451e8689e228 100644 --- a/packages/nox/src/NOX_Direction_Factory.C +++ b/packages/nox/src/NOX_Direction_Factory.C @@ -80,7 +80,7 @@ buildDirection(const Teuchos::RCP& gd, } } else { - std::string msg = "Error - NOX::Direction::Facotry::buildDirection() - Invalid choice for \"Method\" in \"Direction\" sublist!"; + std::string msg = "Error - NOX::Direction::Factory::buildDirection() - Invalid choice for \"Method\" in \"Direction\" sublist!"; TEUCHOS_TEST_FOR_EXCEPTION(true, std::logic_error, msg); } From dffc40e64ef0e68d9e134fb32078bb2f32199067 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Tue, 3 Sep 2024 09:18:35 -0600 Subject: [PATCH 05/40] NOX: added parameter validation to the newton direction reset object Signed-off-by: Roger Pawlowski --- packages/nox/src/NOX_Direction_Newton.C | 43 ++++++++++++++++++------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/packages/nox/src/NOX_Direction_Newton.C b/packages/nox/src/NOX_Direction_Newton.C index d9d7f05ef51e..8a339320765a 100644 --- a/packages/nox/src/NOX_Direction_Newton.C +++ b/packages/nox/src/NOX_Direction_Newton.C @@ -15,7 +15,7 @@ #include "NOX_Solver_LineSearchBased.H" #include "NOX_Utils.H" #include "NOX_GlobalData.H" - +#include "Teuchos_StandardParameterEntryValidators.hpp" NOX::Direction::Newton:: Newton(const Teuchos::RCP& gd, @@ -39,23 +39,42 @@ reset(const Teuchos::RCP& gd, Teuchos::ParameterList& p = params.sublist("Newton"); - doRescue = p.get("Rescue Bad Newton Solve", true); - if (!p.sublist("Linear Solver").isParameter("Tolerance")) + // Validate and set defaults + { + Teuchos::ParameterList validParams; + validParams.sublist("Linear Solver").disableRecursiveValidation(); + validParams.sublist("Stratimikos Linear Solver").disableRecursiveValidation(); + validParams.set("Forcing Term Method", "Constant", + "Choice of forcing term used by the linear solver", + Teuchos::rcp(new Teuchos::StringValidator(Teuchos::tuple("Constant", "Type 1", "Type 2")))); + validParams.set("Forcing Term Minimum Tolerance", 1.0e-4); + validParams.set("Forcing Term Maximum Tolerance", 0.9); + validParams.set("Forcing Term Initial Tolerance", 0.01); + validParams.set("Forcing Term Alpha", 1.5); + validParams.set("Forcing Term Gamma", 0.9); + validParams.set("Rescue Bad Newton Solve", true); + + p.validateParametersAndSetDefaults(validParams); + } + + doRescue = p.get("Rescue Bad Newton Solve"); + + if (!p.sublist("Linear Solver").isType("Tolerance")) p.sublist("Linear Solver").get("Tolerance", 1.0e-10); + method = p.get("Forcing Term Method"); - if ( p.get("Forcing Term Method", "Constant") == "Constant" ) { + if ( method == "Constant" ) { useAdjustableForcingTerm = false; - eta_k = p.sublist("Linear Solver").get("Tolerance", 1.0e-4); + eta_k = p.sublist("Linear Solver").get("Tolerance"); } else { useAdjustableForcingTerm = true; - method = p.get("Forcing Term Method", "Type 1"); - eta_min = p.get("Forcing Term Minimum Tolerance", 1.0e-4); - eta_max = p.get("Forcing Term Maximum Tolerance", 0.9); - eta_initial = p.get("Forcing Term Initial Tolerance", 0.01); - alpha = p.get("Forcing Term Alpha", 1.5); - gamma = p.get("Forcing Term Gamma", 0.9); + eta_min = p.get("Forcing Term Minimum Tolerance"); + eta_max = p.get("Forcing Term Maximum Tolerance"); + eta_initial = p.get("Forcing Term Initial Tolerance"); + alpha = p.get("Forcing Term Alpha"); + gamma = p.get("Forcing Term Gamma"); eta_k = eta_min; } @@ -76,7 +95,7 @@ bool NOX::Direction::Newton::compute(NOX::Abstract::Vector& dir, // Reset the linear solver tolerance. if (useAdjustableForcingTerm) { resetForcingTerm(soln, solver.getPreviousSolutionGroup(), - solver.getNumIterations(), solver); + solver.getNumIterations(), solver); } else { if (utils->isPrintType(Utils::Details)) { From 8ab6ed1352b198801e9feb0b9d5f407db8b473e8 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Tue, 3 Sep 2024 09:21:09 -0600 Subject: [PATCH 06/40] Panzer: cleanup Signed-off-by: Roger Pawlowski --- .../adapters-stk/example/main_driver/Panzer_xml_to_yaml.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/panzer/adapters-stk/example/main_driver/Panzer_xml_to_yaml.cpp b/packages/panzer/adapters-stk/example/main_driver/Panzer_xml_to_yaml.cpp index 4c331f044a72..735745e0f642 100644 --- a/packages/panzer/adapters-stk/example/main_driver/Panzer_xml_to_yaml.cpp +++ b/packages/panzer/adapters-stk/example/main_driver/Panzer_xml_to_yaml.cpp @@ -21,12 +21,12 @@ using namespace Teuchos; int main(int argc, char *argv[]) { TEUCHOS_ASSERT(argc == 2); - + std::string file_prefix(argv[1]); std::string input_xml_file_name(file_prefix+".xml"); - auto pList = getParametersFromXmlFile(input_xml_file_name); - + auto pList = getParametersFromXmlFile(input_xml_file_name); + std::string output_yaml_file_name(file_prefix+".yaml"); writeParameterListToYamlFile(*pList,output_yaml_file_name); From 0a1993ea710ed8cc4147d569755a50b2dc439b75 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Tue, 3 Sep 2024 12:20:28 -0600 Subject: [PATCH 07/40] NOX: add offset to print observer Signed-off-by: Roger Pawlowski --- packages/nox/src/NOX_Observer_Print.cpp | 18 ++++++++++++++++-- packages/nox/src/NOX_Observer_Print.hpp | 4 +++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/packages/nox/src/NOX_Observer_Print.cpp b/packages/nox/src/NOX_Observer_Print.cpp index e22cbc19dbcb..3fe8b223b516 100644 --- a/packages/nox/src/NOX_Observer_Print.cpp +++ b/packages/nox/src/NOX_Observer_Print.cpp @@ -14,8 +14,10 @@ #include "NOX_Utils.H" #include "NOX_Solver_LineSearchBased.H" -NOX::ObserverPrint::ObserverPrint(const Teuchos::RCP& os) : - os_(os) +NOX::ObserverPrint::ObserverPrint(const Teuchos::RCP& os, + const size_t offset) : + os_(os), + offset_(offset) {} void NOX::ObserverPrint::runPreIterate(const NOX::Solver::Generic& solver) @@ -24,6 +26,12 @@ void NOX::ObserverPrint::runPreIterate(const NOX::Solver::Generic& solver) auto& os = os_->out(); auto original_flags = os.flags(); + if (offset_ > 0) { + for (size_t i=0; i < offset_; ++i) { + os << " "; + } + } + os.setf(std::ios::left); os.width(5); os << "N"; @@ -69,6 +77,12 @@ void NOX::ObserverPrint::printStep(const NOX::Solver::Generic& solver) auto original_flags = os.flags(); const int precision = 6; + if (offset_ > 0) { + for (size_t i=0; i < offset_; ++i) { + os << " "; + } + } + os.width(5); os.setf(std::ios::left); os << stats.numNonlinearIterations; diff --git a/packages/nox/src/NOX_Observer_Print.hpp b/packages/nox/src/NOX_Observer_Print.hpp index 838790ba8949..dae2efbce936 100644 --- a/packages/nox/src/NOX_Observer_Print.hpp +++ b/packages/nox/src/NOX_Observer_Print.hpp @@ -27,13 +27,15 @@ namespace NOX { */ class ObserverPrint : public NOX::Observer { public: - ObserverPrint(const Teuchos::RCP& printUtils); + ObserverPrint(const Teuchos::RCP& printUtils, + const size_t offset=0); // Derived methods void runPreIterate(const NOX::Solver::Generic& solver); void runPostIterate(const NOX::Solver::Generic& solver); private: void printStep(const NOX::Solver::Generic& solver); Teuchos::RCP os_; + const size_t offset_; }; } From 6e5fb107d63560b80aea15a461f23eecb9a8b37e Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Tue, 3 Sep 2024 13:47:26 -0600 Subject: [PATCH 08/40] Panzer: allow override of stratimikos list in ME factory Signed-off-by: Roger Pawlowski --- .../src/Panzer_STK_ModelEvaluatorFactory.hpp | 10 ++++++++++ .../Panzer_STK_ModelEvaluatorFactory_impl.hpp | 20 +++++++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp index c88c5c668217..f839535b9300 100644 --- a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp +++ b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp @@ -85,6 +85,15 @@ namespace panzer_stk { Teuchos::RCP getValidParameters() const; //@} + /** \brief Set the Stratimikos Linear Solver sublist used to + create the LinearOpWithSolve factory. + + This function is optional to call. The default list is pull + from the main parameter list in "Solution + Control->NOX->Direction->Newton->Stratimikos Linear Solver". + */ + void setStratimikosList(const Teuchos::RCP& paramList); + /** \brief Builds the model evaluators for a panzer assembly \param[in] comm (Required) Teuchos communicator. Must be non-null. \param[in] global_data (Required) A fully constructed (all members allocated) global data object used to control parameter library and output support. Must be non-null. @@ -313,6 +322,7 @@ namespace panzer_stk { Teuchos::RCP m_wkstContainer; bool useDynamicCoordinates_; + Teuchos::RCP m_stratimikos_params; }; template diff --git a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory_impl.hpp b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory_impl.hpp index 0f6e8de8c8f7..e102766078f1 100644 --- a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory_impl.hpp +++ b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory_impl.hpp @@ -103,6 +103,12 @@ namespace panzer_stk { this->setMyParamList(paramList); } + template + void ModelEvaluatorFactory::setStratimikosList(const Teuchos::RCP& paramList) + { + m_stratimikos_params = paramList; + } + template Teuchos::RCP ModelEvaluatorFactory::getValidParameters() const { @@ -1563,10 +1569,16 @@ namespace panzer_stk { const Teuchos::ParameterList & p = *this->getParameterList(); const Teuchos::ParameterList & solncntl_params = p.sublist("Solution Control"); - // Build stratimikos solver (note that this is a hard coded path to linear solver options in nox list!) - Teuchos::RCP strat_params - = Teuchos::rcp(new Teuchos::ParameterList(solncntl_params.sublist("NOX").sublist("Direction"). - sublist("Newton").sublist("Stratimikos Linear Solver").sublist("Stratimikos"))); + // Build stratimikos solver (note that this defaults to a hard + // coded path to linear solver options in nox list!) + Teuchos::RCP strat_params; + if (nonnull(m_stratimikos_params)) { + strat_params = m_stratimikos_params; + } + else { + strat_params = Teuchos::rcp(new Teuchos::ParameterList(solncntl_params.sublist("NOX").sublist("Direction"). + sublist("Newton").sublist("Stratimikos Linear Solver").sublist("Stratimikos"))); + } bool writeCoordinates = false; if(p.sublist("Options").isType("Write Coordinates")) From 23932558e43c41a0a262b8585f96ec058b34e4b9 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Tue, 3 Sep 2024 13:48:13 -0600 Subject: [PATCH 09/40] Panzer: updates to main driver sa for solver construction Signed-off-by: Roger Pawlowski --- .../energy-transient-tempus-sa-blocked.xml | 204 +++++------------- .../example/main_driver/main_driver_sa.cpp | 46 ++-- 2 files changed, 79 insertions(+), 171 deletions(-) diff --git a/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml b/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml index 27210907529d..3b545c237277 100644 --- a/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml +++ b/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml @@ -1,7 +1,7 @@ - + @@ -46,41 +46,41 @@ - + - + - + - + - + - - + + - - + + - - + + - - + + @@ -92,6 +92,11 @@ + + + + + @@ -112,6 +117,9 @@ + + + @@ -137,10 +145,10 @@ - - - - + + + + @@ -148,10 +156,10 @@ - - - - + + + + @@ -159,10 +167,10 @@ - - - - + + + + @@ -170,16 +178,16 @@ - - - - + + + + - + @@ -187,112 +195,14 @@ - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -344,7 +254,7 @@ - + @@ -355,30 +265,17 @@ - - - - - - - - - - - - - + - + - + - - + @@ -392,15 +289,15 @@ - + - + - - + + @@ -462,4 +359,3 @@ - diff --git a/packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp b/packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp index 9862c9ee6808..b0911d5ac028 100644 --- a/packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp +++ b/packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp @@ -17,6 +17,7 @@ #include "Teuchos_GlobalMPISession.hpp" #include "Teuchos_CommandLineProcessor.hpp" #include "Teuchos_XMLParameterListHelpers.hpp" +#include "Teuchos_YamlParameterListHelpers.hpp" #include "Teuchos_FancyOStream.hpp" #include "Teuchos_oblackholestream.hpp" #include "Teuchos_Assert.hpp" @@ -26,7 +27,6 @@ #include "PanzerAdaptersSTK_config.hpp" #include "Panzer_STK_ModelEvaluatorFactory.hpp" #include "Panzer_ClosureModel_Factory_TemplateManager.hpp" -#include "Panzer_PauseToAttach.hpp" #include "Panzer_String_Utilities.hpp" #include "Panzer_ThyraObjContainer.hpp" #include "Thyra_VectorSpaceBase.hpp" @@ -85,8 +85,7 @@ int main(int argc, char *argv[]) // Parse the command line arguments std::string input_file_name = "user_app.xml"; int exodus_io_num_procs = 0; - bool pauseToAttachOn = false; - bool pointCalculation = false; + bool pointCalculation = true; bool printTimers = true; bool printInputPL = false; bool overrideNoxOutput = true; @@ -95,10 +94,9 @@ int main(int argc, char *argv[]) clp.setOption("i", &input_file_name, "User_App input xml filename"); clp.setOption("exodus-io-num-procs", &exodus_io_num_procs, "Number of processes that can access the file system at the same time to read their portion of a sliced exodus file in parallel. Defaults to 0 - implies all processes for the run can access the file system at the same time."); - clp.setOption("pause-to-attach","disable-pause-to-attach", &pauseToAttachOn, "Call pause to attach, default is off."); clp.setOption("point-calc","disable-point-calc", &pointCalculation, "Enable the probe evaluator unit test."); clp.setOption("time","no-time", &printTimers, "Print the timing information."); - clp.setOption("pl","no-pl", &printTimers, "Print the input ParameterList at the start of the run."); + clp.setOption("pl","no-pl", &printInputPL, "Print the input ParameterList at the start of the run."); clp.setOption("override-nox-output","default-nox-output", &overrideNoxOutput, "Override default nox printing with new print observer."); Teuchos::CommandLineProcessor::EParseCommandLineReturn parse_return = @@ -108,17 +106,22 @@ int main(int argc, char *argv[]) std::runtime_error, "Failed to parse command line!"); } - if(pauseToAttachOn) - panzer::pauseToAttach(); - TEUCHOS_ASSERT(exodus_io_num_procs <= comm->getSize()); if (exodus_io_num_procs != 0) Ioss::SerializeIO::setGroupFactor(exodus_io_num_procs); // Parse the input file and broadcast to other processes Teuchos::RCP input_params = Teuchos::rcp(new Teuchos::ParameterList("User_App Parameters")); - Teuchos::updateParametersFromXmlFileAndBroadcast(input_file_name, input_params.ptr(), *comm); - + { + const std::string check_yaml = ".yaml"; + const auto search_yaml = input_file_name.find(check_yaml); + if (search_yaml != std::string::npos) { + Teuchos::updateParametersFromYamlFileAndBroadcast(input_file_name, input_params.ptr(), *comm); + } else { + Teuchos::updateParametersFromXmlFileAndBroadcast(input_file_name, input_params.ptr(), *comm); + } + } + if (printInputPL) *out << *input_params << std::endl; @@ -157,6 +160,9 @@ int main(int argc, char *argv[]) panzer_stk::ModelEvaluatorFactory me_factory; me_factory.setParameterList(input_params); + auto strat_list = Teuchos::parameterList(); + *strat_list = input_params->sublist("Solution Control",true).sublist("Tempus",true).sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true).sublist("Direction",true).sublist("Newton",true).sublist("Stratimikos Linear Solver",true).sublist("Stratimikos",true); + me_factory.setStratimikosList(strat_list); me_factory.buildObjects(comm,global_data,eqset_factory,bc_factory,cm_factory); // add a volume response functional for each field @@ -170,7 +176,6 @@ int main(int argc, char *argv[]) builder.comm = MPI_COMM_WORLD; // good enough builder.cubatureDegree = 2; builder.requiresCellIntegral = lst.isType("Requires Cell Integral") ? lst.get("Requires Cell Integral"): false; - std::cout << "ROGER requires ce=" << builder.requiresCellIntegral << std::endl; builder.quadPointField = lst.get("Field Name"); // add the response @@ -287,17 +292,24 @@ int main(int argc, char *argv[]) if (overrideNoxOutput) { auto nox_utils = Teuchos::rcp(new NOX::Utils(NOX::Utils::Error,comm->getRank(),0,3)); - auto print_nonlinear = Teuchos::rcp(new NOX::ObserverPrint(nox_utils)); + auto print_nonlinear = Teuchos::rcp(new NOX::ObserverPrint(nox_utils,10)); tempus_pl->sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true).sublist("Solver Options",true).set>("Observer",print_nonlinear); } - - auto integrator = createIntegratorBasic(tempus_pl, physics, false); - std::cout << "ROGER solver=" << integrator->getStepper()->getSolver() << std::endl; + // Tempus is overriding NOX parameters with its own defaults. Need + // to fix this. It also does not validate because of arbitrary + // solver name. Should validate but use + // disableRecursiveValidation() to avoid errors with arbitrary + // names. - integrator->initialize(); + const bool doInitialization = false; + auto integrator = createIntegratorBasic(tempus_pl, physics, doInitialization); - std::cout << "ROGER solver=" << integrator->getStepper()->getSolver() << std::endl; + RCP noxList = parameterList("Correct NOX Params"); + *noxList = tempus_pl->sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true); + noxList->print(std::cout); + integrator->getStepper()->getSolver()->setParameterList(noxList); + integrator->initialize(); RCP> x0 = physics->getNominalValues().get_x()->clone_v(); integrator->initializeSolutionHistory(0.0, x0); From 637f2dbc111f44aca3d2a40940b5452b69317eb5 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Wed, 4 Sep 2024 15:42:06 -0600 Subject: [PATCH 10/40] Panzer: start of parameter sensitivity test Signed-off-by: Roger Pawlowski --- .../energy-transient-tempus-sa-blocked.xml | 16 +++++++ .../example/main_driver/main_driver_sa.cpp | 44 ++++++++++++------- .../src/Panzer_STK_ModelEvaluatorFactory.hpp | 2 +- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml b/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml index 3b545c237277..7e3fbab9a6f9 100644 --- a/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml +++ b/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml @@ -45,6 +45,20 @@ + + + + + + + + + + + + + + @@ -255,6 +269,7 @@ + @@ -315,6 +330,7 @@ + diff --git a/packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp b/packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp index b0911d5ac028..729a7ce3d71d 100644 --- a/packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp +++ b/packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp @@ -121,7 +121,7 @@ int main(int argc, char *argv[]) Teuchos::updateParametersFromXmlFileAndBroadcast(input_file_name, input_params.ptr(), *comm); } } - + if (printInputPL) *out << *input_params << std::endl; @@ -203,7 +203,7 @@ int main(int argc, char *argv[]) Teuchos::RCP nof; { nof = Teuchos::rcp(new user_app::NOXObserverFactory(stkIOResponseLibrary)); - + Teuchos::RCP observers_to_build = Teuchos::parameterList(solver_factories.sublist("NOX Observers")); @@ -282,11 +282,11 @@ int main(int argc, char *argv[]) resp->setVector(vec); } } - + // Tempus stepper using namespace Teuchos; using namespace Tempus; - + RCP tempus_pl = parameterList(std::string("Tempus")); *tempus_pl = input_params->sublist("Solution Control").sublist("Tempus"); @@ -310,17 +310,17 @@ int main(int argc, char *argv[]) noxList->print(std::cout); integrator->getStepper()->getSolver()->setParameterList(noxList); integrator->initialize(); - + RCP> x0 = physics->getNominalValues().get_x()->clone_v(); integrator->initializeSolutionHistory(0.0, x0); - + bool integratorStatus = integrator->advanceTime(); TEUCHOS_ASSERT(integratorStatus); - + auto x = integrator->getSolutionHistory()->getCurrentState()->getX(); auto x_dot = integrator->getSolutionHistory()->getCurrentState()->getXDot(); - - { + + { // get responses if there are any ////////////////////////////////////////////// if(physics->Ng()>0) { @@ -333,22 +333,34 @@ int main(int argc, char *argv[]) respInArgs.set_x(x); respInArgs.set_x_dot(x_dot); - // set up response out args for(int i=0;i > response = Thyra::createMember(*physics->get_g_space(i)); respOutArgs.set_g(i,response); } - // Now, solve the problem and return the responses physics->evalModel(respInArgs, respOutArgs); - // loop over out args for printing - for(int i=0;i > response = respOutArgs.get_g(i); + { + auto parameter_library = global_data->pl; + TEUCHOS_ASSERT(nonnull(parameter_library)); + for (auto i=parameter_library->begin();i != parameter_library->end(); ++i) { + *out << "Sacado::ParameterLibrary: " << i->first << std::endl; + } + } - TEUCHOS_ASSERT(response!=Teuchos::null); // should not be null! + { + auto nominalValues = physics->getNominalValues(); + for (int i=0; i < respInArgs.Np();++i) { + auto p = nominalValues.get_p(i); + auto p_names = physics->get_p_names(i); + *out << "ModelEvaluator::NominalValues Parameter Value: \"" << (*p_names)[0] << "\" = " << Thyra::get_ele(*p,0) << std::endl; + } + } - *out << "Response Value \"" << responseIndexToName[i] << "\": " << Thyra::get_ele(*response,0) << std::endl; + for (int i=0;i > response = respOutArgs.get_g(i); + TEUCHOS_ASSERT(response!=Teuchos::null); + *out << "Response Value: \"" << responseIndexToName[i] << "\" = " << Thyra::get_ele(*response,0) << std::endl; } } diff --git a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp index f839535b9300..adf6221a5bd1 100644 --- a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp +++ b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp @@ -344,7 +344,7 @@ addResponse(const std::string & responseName,const std::vectoraddResponse(responseName,wkstDesc,builder); } else if(panzer_me!=Teuchos::null && thyra_ep_me==Teuchos::null) { - return panzer_me->addResponse(responseName,wkstDesc,builder); + return panzer_me->addFlexibleResponse(responseName,wkstDesc,builder); } #else Teuchos::RCP panzer_me = Teuchos::rcp_dynamic_cast(m_physics_me); From abaa994a43e53d4ccbd003669368c8b6d0feb390 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 13 Sep 2024 15:12:52 -0600 Subject: [PATCH 11/40] Panzer: start of a transient optimization example using Tempus and ROL Signed-off-by: Roger Pawlowski --- .../adapters-stk/cmake/Dependencies.cmake | 2 +- .../example/main_driver/CMakeLists.txt | 27 +- .../Panzer_ROLTempusReducedObjective.hpp | 483 ++++++++++++++++++ .../energy-transient-tempus-opt-blocked.xml | 397 ++++++++++++++ .../example/main_driver/main_driver_opt.cpp | 404 +++++++++++++++ .../example/main_driver/rol_params.xml | 308 +++++++++++ .../src/Panzer_STK_ModelEvaluatorFactory.hpp | 2 +- 7 files changed, 1616 insertions(+), 7 deletions(-) create mode 100644 packages/panzer/adapters-stk/example/main_driver/Panzer_ROLTempusReducedObjective.hpp create mode 100644 packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-opt-blocked.xml create mode 100644 packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp create mode 100644 packages/panzer/adapters-stk/example/main_driver/rol_params.xml diff --git a/packages/panzer/adapters-stk/cmake/Dependencies.cmake b/packages/panzer/adapters-stk/cmake/Dependencies.cmake index 17c1750fa8ee..720eafead127 100644 --- a/packages/panzer/adapters-stk/cmake/Dependencies.cmake +++ b/packages/panzer/adapters-stk/cmake/Dependencies.cmake @@ -7,7 +7,7 @@ SET(LIB_REQUIRED_DEP_PACKAGES TeuchosCore STKUtil STKTopology STKMesh STKIO Zoltan Stratimikos Piro NOX PanzerCore PanzerDiscFE) SET(LIB_OPTIONAL_DEP_PACKAGES STKSearch SEACASIoss SEACASExodus UMR Percept Teko MueLu Ifpack2 Tempus) SET(TEST_REQUIRED_DEP_PACKAGES SEACASIoss SEACASExodus Teko MueLu Ifpack2) -SET(TEST_OPTIONAL_DEP_PACKAGES Pamgen STKSearch) +SET(TEST_OPTIONAL_DEP_PACKAGES Pamgen STKSearch ROL) SET(LIB_REQUIRED_DEP_TPLS) SET(LIB_OPTIONAL_DEP_TPLS) SET(TEST_REQUIRED_DEP_TPLS) diff --git a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt index c0cb3417c20e..a2781918056b 100644 --- a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt +++ b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt @@ -15,11 +15,6 @@ TRIBITS_ADD_EXECUTABLE( SOURCES main_driver.cpp ${main_driver_SOURCES} ) -TRIBITS_ADD_EXECUTABLE( - main_driver_sa - SOURCES main_driver_sa.cpp ${main_driver_SOURCES} - ) - TRIBITS_ADD_EXECUTABLE( xml_to_yaml SOURCES Panzer_xml_to_yaml.cpp @@ -35,6 +30,8 @@ TRIBITS_COPY_FILES_TO_BINARY_DIR(main_driver_files energy-ss-blocked.xml energy-transient-tempus-blocked.xml energy-transient-tempus-sa-blocked.xml + energy-transient-tempus-opt-blocked.xml + rol_params.xml energy-neumann.xml energy-ss-blocked-tp.xml periodic_wedge.xml @@ -113,6 +110,7 @@ TRIBITS_ADD_ADVANCED_TEST( IF(${PACKAGE_NAME}_ENABLE_Teko) IF(${PACKAGE_NAME}_ENABLE_Tempus) + # Simple Tempus transient test TRIBITS_ADD_ADVANCED_TEST( main_driver_energy-transient-tempus-blocked TEST_0 EXEC main_driver @@ -120,12 +118,31 @@ IF(${PACKAGE_NAME}_ENABLE_Teko) PASS_REGULAR_EXPRESSION "panzer::MainDriver run completed." ) + # Transient Sensitivity Analysis + TRIBITS_ADD_EXECUTABLE( + main_driver_sa + SOURCES main_driver_sa.cpp ${main_driver_SOURCES} + ) TRIBITS_ADD_ADVANCED_TEST( main_driver_energy-transient-tempus-sa-blocked TEST_0 EXEC main_driver_sa ARGS --i=energy-transient-tempus-sa-blocked.xml --point-calc PASS_REGULAR_EXPRESSION "panzer::MainDriver run completed." ) + + # Optimization with ROL + IF(${PACKAGE_NAME}_ENABLE_ROL) + TRIBITS_ADD_EXECUTABLE( + main_driver_opt + SOURCES main_driver_opt.cpp ${main_driver_SOURCES} + ) + TRIBITS_ADD_ADVANCED_TEST( + main_driver_energy-transient-tempus-opt-blocked + TEST_0 EXEC main_driver_opt + PASS_REGULAR_EXPRESSION "panzer::MainDriver run completed." + ) + ENDIF() + ENDIF() ENDIF() diff --git a/packages/panzer/adapters-stk/example/main_driver/Panzer_ROLTempusReducedObjective.hpp b/packages/panzer/adapters-stk/example/main_driver/Panzer_ROLTempusReducedObjective.hpp new file mode 100644 index 000000000000..f927e8d6b9fd --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/Panzer_ROLTempusReducedObjective.hpp @@ -0,0 +1,483 @@ +// @HEADER +// ***************************************************************************** +// Rapid Optimization Library (ROL) Package +// +// Copyright 2014 NTESS and the ROL contributors. +// SPDX-License-Identifier: BSD-3-Clause +// ***************************************************************************** +// @HEADER + +#ifndef ROL_TEMPUS_REDUCED_OBJECTIVE_HPP +#define ROL_TEMPUS_REDUCED_OBJECTIVE_HPP + +#include + +#include "Teuchos_RCP.hpp" +#include "Teuchos_ParameterList.hpp" + +#include "Tempus_IntegratorBasic.hpp" +#include "Tempus_IntegratorForwardSensitivity.hpp" +#include "Tempus_IntegratorAdjointSensitivity.hpp" +#include "Tempus_IntegratorPseudoTransientForwardSensitivity.hpp" +#include "Tempus_IntegratorPseudoTransientAdjointSensitivity.hpp" + +#include "Thyra_ModelEvaluator.hpp" +#include "Thyra_DefaultNominalBoundsOverrideModelEvaluator.hpp" +#include "Thyra_VectorStdOps.hpp" + +#include "ROL_Objective.hpp" +#include "ROL_Vector.hpp" +#include "ROL_ThyraVector.hpp" + +namespace ROL { + +template +class TempusReducedObjective : public virtual ROL::Objective { +public: + + TempusReducedObjective( + const Teuchos::RCP >& model, + const Teuchos::RCP& tempus_params, + const Teuchos::RCP& objective_params); + + virtual ~TempusReducedObjective() {} + + //! Compute value of objective + Real value( const ROL::Vector &x, Real &tol ); + + //! Compute gradient of objective + void gradient( ROL::Vector &g, const ROL::Vector &x, Real &tol ); + + //! Set response target for computing objective + void set_target(const Teuchos::RCP >& target); + void set_target(const Teuchos::RCP >& target); + + //! Helper function to create optimization vector + Teuchos::RCP > create_design_vector() const; + + //! Helper function to create a response vector + Teuchos::RCP > create_response_vector() const; + + //! Helper function to run tempus, computing responses and derivatives + void run_tempus(ROL::Vector& r, const ROL::Vector& p); + void run_tempus(const Thyra::ModelEvaluatorBase::InArgs& inArgs, + const Thyra::ModelEvaluatorBase::OutArgs& outArgs); + +private: + + Teuchos::RCP > model_; + Teuchos::RCP tempus_params_; + Teuchos::RCP > target_; + std::string objective_type_; + std::string sensitivity_method_; + int param_index_; + int response_index_; + bool use_fd_gradient_; + +}; // class TempusReducedObjective + +template +TempusReducedObjective:: +TempusReducedObjective( + const Teuchos::RCP >& model, + const Teuchos::RCP& tempus_params, + const Teuchos::RCP& objective_params) : + model_(model), + tempus_params_(tempus_params), + objective_type_("Sum of Squares"), + sensitivity_method_("Forward"), + param_index_(0), + response_index_(0), + use_fd_gradient_(false) +{ + objective_type_ = + objective_params->get("Objective Type", objective_type_); + sensitivity_method_ = + objective_params->get("Sensitivity Method", sensitivity_method_); + param_index_ = + objective_params->get("Sensitivity Parameter Index", param_index_); + response_index_ = + objective_params->get("Response Function Index", response_index_); + use_fd_gradient_ = + objective_params->get("Use FD Gradient", use_fd_gradient_); +} + +template +Real +TempusReducedObjective:: +value( const ROL::Vector &p, Real &tol ) +{ + using Teuchos::RCP; + typedef Thyra::ModelEvaluatorBase MEB; + + // Run tempus and compute response for specified parameter values + MEB::InArgs inArgs = model_->getNominalValues(); + MEB::OutArgs outArgs = model_->createOutArgs(); + const ROL::ThyraVector& thyra_p = + Teuchos::dyn_cast >(p); + inArgs.set_p(param_index_, thyra_p.getVector()); + RCP > g = + Thyra::createMember(model_->get_g_space(response_index_)); + outArgs.set_g(response_index_, g); + run_tempus(inArgs, outArgs); + + // Compute objective + if (target_ != Teuchos::null) + Thyra::Vp_StV(g.ptr(), -1.0, *target_); // g <- g - target + Real val = 0.0; + if (objective_type_ == "Sum of Squares") { + Thyra::ele_wise_scale(*g, g.ptr()); // g <- g.*g + val = Thyra::sum(*g); // val = \sum_i g_i + } + else if (objective_type_ == "Sum") + val = Thyra::sum(*g); // val = \sum_i g_i + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Invalid objective type " << objective_type_ << "!" << std::endl + << "Valid choices are: \"Sum\" and \"Sum of Squares\"."); + + return val; +} + +template +void +TempusReducedObjective:: +gradient(ROL::Vector &grad, const ROL::Vector &p, Real &tol) +{ + if (use_fd_gradient_) { + ROL::Objective::gradient(grad, p, tol); + return; + } + + using Teuchos::RCP; + typedef Thyra::ModelEvaluatorBase MEB; + + // Run tempus and compute response gradient for specified parameter values + const int num_p = model_->get_p_space(param_index_)->dim(); + const int num_g = model_->get_g_space(response_index_)->dim(); + MEB::InArgs inArgs = model_->getNominalValues(); + MEB::OutArgs outArgs = model_->createOutArgs(); + const ROL::ThyraVector& thyra_p = + Teuchos::dyn_cast >(p); + inArgs.set_p(param_index_, thyra_p.getVector()); + RCP > g = + Thyra::createMember(model_->get_g_space(response_index_)); + RCP > dgdp; + MEB::EDerivativeMultiVectorOrientation dgdp_orientation = + MEB::DERIV_MV_JACOBIAN_FORM; + if (sensitivity_method_ == "Adjoint"|| + sensitivity_method_ == "Pseudotransient Adjoint") { + // Adjoint, PseudoTransientAdjoint integrators require gradient form + // (but does not necessarily require the model to support gradient form) + dgdp = + Thyra::createMembers(model_->get_p_space(param_index_), num_g); + dgdp_orientation = MEB::DERIV_MV_GRADIENT_FORM; + } + else { + // Form determined by what model supports + MEB::DerivativeSupport support = + outArgs.supports(MEB::OUT_ARG_DgDp, response_index_, param_index_); + if (support.supports(MEB::DERIV_MV_JACOBIAN_FORM)) { + dgdp = + Thyra::createMembers(model_->get_g_space(response_index_), num_p); + dgdp_orientation = MEB::DERIV_MV_JACOBIAN_FORM; + } + else if (support.supports(MEB::DERIV_MV_GRADIENT_FORM)) { + dgdp = + Thyra::createMembers(model_->get_p_space(param_index_), num_g); + dgdp_orientation = MEB::DERIV_MV_GRADIENT_FORM; + } + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Model must support Jacobian or gradient forms for dg/dp!"); + } + outArgs.set_DgDp(response_index_, param_index_, + MEB::DerivativeMultiVector(dgdp, dgdp_orientation)); + outArgs.set_g(response_index_, g); + run_tempus(inArgs, outArgs); + + // Compute objective gradient + RCP > final_grad = + Teuchos::dyn_cast >(grad).getVector()->col(0); + if (target_ != Teuchos::null) + Thyra::Vp_StV(g.ptr(), -1.0, *target_); // g <- g - target + if (dgdp_orientation == MEB::DERIV_MV_JACOBIAN_FORM) { + for (int j=0; j > dgdp_j = dgdp->col(j); + Real grad_val = 0.0; + if (objective_type_ == "Sum of Squares") { + Thyra::ele_wise_prod_update(2.0, *g, dgdp_j.ptr()); + grad_val = Thyra::sum(*dgdp_j); + } + else if (objective_type_ == "Sum") + grad_val = Thyra::sum(*dgdp_j); + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Invalid objective type " << objective_type_ << "!" << std::endl + << "Valid choices are: \"Sum\" and \"Sum of Squares\"."); + Thyra::set_ele(j, grad_val, final_grad.ptr()); + } + } + else { // gradient form + Thyra::assign(final_grad.ptr(), 0.0); + for (int i=0; icol(i))); + } + else if (objective_type_ == "Sum") { + Thyra::Vp_StV(final_grad.ptr(), 1.0, *(dgdp->col(i))); + } + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Invalid objective type " << objective_type_ << "!" << std::endl + << "Valid choices are: \"Sum\" and \"Sum of Squares\"."); + } + } +} + +template +void +TempusReducedObjective:: +set_target(const Teuchos::RCP >& target) { + target_ = target; +} + +template +void +TempusReducedObjective:: +set_target(const Teuchos::RCP >& target) { + using Teuchos::rcp_dynamic_cast; + target_ = + rcp_dynamic_cast >(target, true)->getVector(); +} + +template +Teuchos::RCP > +TempusReducedObjective:: +create_design_vector() const { + typedef Thyra::ModelEvaluatorBase MEB; + + Teuchos::RCP > p = + Thyra::createMember(model_->get_p_space(param_index_)); + MEB::InArgs nominalValues = model_->getNominalValues(); + if (nominalValues.get_p(param_index_) != Teuchos::null) + Thyra::assign(p.ptr(), *(nominalValues.get_p(param_index_))); + else + Thyra::assign(p.ptr(), Teuchos::ScalarTraits::zero()); + return Teuchos::rcp(new ROL::ThyraVector(p)); +} + +template +Teuchos::RCP > +TempusReducedObjective:: +create_response_vector() const { + Teuchos::RCP > g = + Thyra::createMember(model_->get_g_space(response_index_)); + Thyra::assign(g.ptr(), Teuchos::ScalarTraits::zero()); + return Teuchos::rcp(new ROL::ThyraVector(g)); +} + +template +void +TempusReducedObjective:: +run_tempus(ROL::Vector& r, const ROL::Vector& p) +{ + typedef Thyra::ModelEvaluatorBase MEB; + + MEB::InArgs inArgs = model_->getNominalValues(); + MEB::OutArgs outArgs = model_->createOutArgs(); + const ROL::ThyraVector& thyra_p = + Teuchos::dyn_cast >(p); + ROL::ThyraVector& thyra_r = + Teuchos::dyn_cast >(r); + inArgs.set_p(param_index_, thyra_p.getVector()); + outArgs.set_g(response_index_, thyra_r.getVector()); + run_tempus(inArgs, outArgs); +} + +template +void +TempusReducedObjective:: +run_tempus(const Thyra::ModelEvaluatorBase::InArgs& inArgs, + const Thyra::ModelEvaluatorBase::OutArgs& outArgs) +{ + using Teuchos::rcp; + using Teuchos::RCP; + using Teuchos::rcpFromRef; + typedef Thyra::ModelEvaluatorBase MEB; + typedef Thyra::DefaultNominalBoundsOverrideModelEvaluator DNBOME; + + // Override nominal values in model to supplied inArgs + RCP wrapped_model = rcp(new DNBOME(model_, rcpFromRef(inArgs))); + + Real t; + RCP > x, x_dot; + RCP > dxdp, dxdotdp; + RCP > g = outArgs.get_g(response_index_); + RCP > dgdp = + outArgs.get_DgDp(response_index_, param_index_).getMultiVector(); + MEB::EDerivativeMultiVectorOrientation dgdp_orientation = + outArgs.get_DgDp(response_index_, param_index_).getMultiVectorOrientation(); + + // Create and run integrator + if (dgdp != Teuchos::null && sensitivity_method_ == "Forward") { + RCP > integrator = + Tempus::createIntegratorForwardSensitivity(tempus_params_, wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + dxdp = integrator->getDxDp(); + dxdotdp = integrator->getDXDotDp(); + } + else if (dgdp != Teuchos::null && sensitivity_method_ == "Adjoint") { + RCP > integrator = + Tempus::createIntegratorAdjointSensitivity(tempus_params_, wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + Thyra::assign(dgdp.ptr(), *(integrator->getDgDp())); + } + else if (dgdp != Teuchos::null && + sensitivity_method_ == "Pseudotransient Forward") { + RCP > integrator = + Tempus::createIntegratorPseudoTransientForwardSensitivity(tempus_params_, + wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + dxdp = integrator->getDxDp(); + dxdotdp = integrator->getDXDotDp(); + } + else if (dgdp != Teuchos::null && + sensitivity_method_ == "Pseudotransient Adjoint") { + RCP > integrator = + Tempus::integratorPseudoTransientAdjointSensitivity(tempus_params_, + wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + Thyra::assign(dgdp.ptr(), *(integrator->getDgDp())); + } + else if (dgdp != Teuchos::null) { + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Invalid sensitivity method " << sensitivity_method_ << "!" << std::endl + << "Valid choices are: Forward, Adjoint, Pseudotransient Forward, " + << " and Pseudotransient Adjoint."); + } + else { + RCP > integrator = + Tempus::createIntegratorBasic(tempus_params_, wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + } + + // Evaluate response at final state + const int num_g = model_->get_g_space(response_index_)->dim(); + MEB::InArgs modelInArgs = inArgs; + MEB::OutArgs modelOutArgs = outArgs; + modelInArgs.set_x(x); + if (modelInArgs.supports(MEB::IN_ARG_x_dot)) modelInArgs.set_x_dot(x_dot); + if (modelInArgs.supports(MEB::IN_ARG_t)) modelInArgs.set_t(t); + RCP > dgdx, dgdxdot; + MEB::EDerivativeMultiVectorOrientation dgdx_orientation = + MEB::DERIV_MV_JACOBIAN_FORM; + MEB::EDerivativeMultiVectorOrientation dgdxdot_orientation = + MEB::DERIV_MV_JACOBIAN_FORM; + if (dgdp != Teuchos::null && + (sensitivity_method_ == "Forward" || + sensitivity_method_ == "Pseudotransient Forward")) { + MEB::DerivativeSupport dgdx_support = + outArgs.supports(MEB::OUT_ARG_DgDx, response_index_); + if (!dgdx_support.none()) { + if (dgdx_support.supports(MEB::DERIV_MV_GRADIENT_FORM)) { + dgdx = Thyra::createMembers(model_->get_x_space(), num_g); + dgdx_orientation = MEB::DERIV_MV_GRADIENT_FORM; + } + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Model must support gradient forms for dg/dx!"); + modelOutArgs.set_DgDx( + response_index_, + MEB::DerivativeMultiVector(dgdx, dgdx_orientation)); + } + + MEB::DerivativeSupport dgdxdot_support = + modelOutArgs.supports(MEB::OUT_ARG_DgDx_dot, response_index_); + if (!dgdxdot_support.none()) { + if (dgdxdot_support.supports(MEB::DERIV_MV_GRADIENT_FORM)) { + dgdxdot = Thyra::createMembers(model_->get_x_space(), num_g); + dgdxdot_orientation = MEB::DERIV_MV_GRADIENT_FORM; + } + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Model must support Jacobian or gradient forms for dg/dx!"); + modelOutArgs.set_DgDx_dot( + response_index_, + MEB::DerivativeMultiVector(dgdxdot, dgdxdot_orientation)); + } + } + else if (dgdp != Teuchos::null && + (sensitivity_method_ == "Adjoint" || + sensitivity_method_ == "Pseudotransient Adjoint")) { + // Clear dg/dp as an out arg since it was already computed by the adjoint + // integrator + modelOutArgs.set_DgDp(response_index_, param_index_, + MEB::Derivative()); + } + + model_->evalModel(modelInArgs, modelOutArgs); + + // dg/dp = dg/dp + dg/dx*dx/dp + dg/dx_dot*dx_dot/dp + // We assume dg/dx, dg/dxdot are in gradient form while dxdp, dxdotdp are in + // Jacobian form + if (dgdp != Teuchos::null && dgdx != Teuchos::null) { + if (dgdp_orientation == MEB::DERIV_MV_JACOBIAN_FORM) + dgdx->apply(Thyra::TRANS, *dxdp, dgdp.ptr(), Real(1.0), Real(1.0)); + else + dxdp->apply(Thyra::TRANS, *dgdx, dgdp.ptr(), Real(1.0), Real(1.0)); + } + if (dgdp != Teuchos::null && dgdxdot != Teuchos::null) { + if (dgdp_orientation == MEB::DERIV_MV_JACOBIAN_FORM) + dgdxdot->apply(Thyra::TRANS, *dxdotdp, dgdp.ptr(), Real(1.0), Real(1.0)); + else + dxdotdp->apply(Thyra::TRANS, *dgdxdot, dgdp.ptr(), Real(1.0), Real(1.0)); + } +} + +} // namespace ROL + +#endif // ROL_TEMPUS_REDUCED_OBJECTIVE_HPP diff --git a/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-opt-blocked.xml b/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-opt-blocked.xml new file mode 100644 index 000000000000..fc06b2ef718d --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-opt-blocked.xml @@ -0,0 +1,397 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp b/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp new file mode 100644 index 000000000000..b519eecd1893 --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp @@ -0,0 +1,404 @@ +// @HEADER +// ***************************************************************************** +// Panzer: A partial differential equation assembly +// engine for strongly coupled complex multiphysics systems +// +// Copyright 2011 NTESS and the Panzer contributors. +// SPDX-License-Identifier: BSD-3-Clause +// ***************************************************************************** +// @HEADER + +#include "Teuchos_ConfigDefs.hpp" +#include "Teuchos_RCP.hpp" +#include "Teuchos_TimeMonitor.hpp" +#include "Teuchos_StackedTimer.hpp" +#include "Teuchos_DefaultComm.hpp" +#include "Teuchos_CommHelpers.hpp" +#include "Teuchos_GlobalMPISession.hpp" +#include "Teuchos_CommandLineProcessor.hpp" +#include "Teuchos_XMLParameterListHelpers.hpp" +#include "Teuchos_YamlParameterListHelpers.hpp" +#include "Teuchos_FancyOStream.hpp" +#include "Teuchos_oblackholestream.hpp" +#include "Teuchos_Assert.hpp" +#include "Teuchos_as.hpp" + +#include "Panzer_NodeType.hpp" +#include "PanzerAdaptersSTK_config.hpp" +#include "Panzer_STK_ModelEvaluatorFactory.hpp" +#include "Panzer_ClosureModel_Factory_TemplateManager.hpp" +#include "Panzer_String_Utilities.hpp" +#include "Panzer_ThyraObjContainer.hpp" +#include "Thyra_VectorSpaceBase.hpp" + +#include "NOX_Utils.H" +#include "NOX_Observer_Print.hpp" + +#include "Tempus_IntegratorBasic.hpp" +#include "Tempus_StepperFactory.hpp" + +#include "user_app_ClosureModel_Factory_TemplateBuilder.hpp" +#include "user_app_EquationSetFactory.hpp" +#include "user_app_BCStrategy_Factory.hpp" +#include "user_app_NOXObserverFactory.hpp" +#ifdef PANZER_HAVE_TEMPUS +#include "user_app_TempusObserverFactory.hpp" +#endif +#include "user_app_ResponseEvaluatorFactory_HOFlux.hpp" + +#include "Panzer_ROLTempusReducedObjective.hpp" + +#include "Panzer_ResponseEvaluatorFactory_Probe.hpp" + +#include + +#include +#include + +// ************************************************* +// applicaiton functions defined in this file +// ************************************************* +namespace user_app { + void addResponsesToModelEvaluatorFactory(const Teuchos::ParameterList& response_sublist, + panzer_stk::ModelEvaluatorFactory& me_factory); + void computeAndPrintResponses(const Teuchos::RCP>& me, + const auto& x, + const auto& x_dot, + const auto& t, + const auto& global_data, + std::ostream& out); +} + +// ************************************************* +// ************************************************* +// Main Driver for a transient optimization problem using ROL and +// Tempus +// ************************************************* +// ************************************************* + +int main(int argc, char *argv[]) +{ + int status = 0; + + std::ostream blackhole{nullptr}; + Teuchos::GlobalMPISession mpiSession(&argc, &argv, &blackhole); + Kokkos::initialize(argc,argv); + + Teuchos::RCP out = Teuchos::rcp(new Teuchos::FancyOStream(Teuchos::rcp(&std::cout,false))); + Teuchos::RCP pout = Teuchos::rcp(new Teuchos::FancyOStream(Teuchos::rcp(&std::cout,false))); + if (mpiSession.getNProc() > 1) { + out->setShowProcRank(true); + out->setOutputToRootOnly(0); + } + + try { + const auto stackedTimer = Teuchos::rcp(new Teuchos::StackedTimer("Panzer Main Driver: Transient Optimization")); + Teuchos::TimeMonitor::setStackedTimer(stackedTimer); + + Teuchos::RCP > comm = Teuchos::DefaultComm::getComm(); + + // Parse the command line arguments + std::string input_file_name = "energy-transient-tempus-opt-blocked.xml"; + int exodus_io_num_procs = 0; + bool printTimers = true; + bool printInputPL = false; + bool overrideNoxOutput = true; + { + Teuchos::CommandLineProcessor clp; + + clp.setOption("i", &input_file_name, "Input file name, supports xml or yaml"); + clp.setOption("exodus-io-num-procs", &exodus_io_num_procs, "Number of processes that can access the file system at the same time to read their portion of a sliced exodus file in parallel. Defaults to 0 - implies all processes for the run can access the file system at the same time."); + clp.setOption("time","no-time", &printTimers, "Print the timing information."); + clp.setOption("pl","no-pl", &printInputPL, "Print the input ParameterList at the start of the run."); + clp.setOption("override-nox-output","default-nox-output", &overrideNoxOutput, "Override default nox printing with new print observer."); + + auto parse_return = clp.parse(argc,argv,&std::cerr); + + TEUCHOS_TEST_FOR_EXCEPTION(parse_return != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL, + std::runtime_error, "Failed to parse command line!"); + } + + TEUCHOS_ASSERT(exodus_io_num_procs <= comm->getSize()); + if (exodus_io_num_procs != 0) + Ioss::SerializeIO::setGroupFactor(exodus_io_num_procs); + + // Parse the input file and broadcast to other processes + Teuchos::RCP input_params = Teuchos::rcp(new Teuchos::ParameterList("User_App Parameters")); + { + const std::string check_yaml = ".yaml"; + const auto search_yaml = input_file_name.find(check_yaml); + if (search_yaml != std::string::npos) { + Teuchos::updateParametersFromYamlFileAndBroadcast(input_file_name, input_params.ptr(), *comm); + } else { + Teuchos::updateParametersFromXmlFileAndBroadcast(input_file_name, input_params.ptr(), *comm); + } + } + + if (printInputPL) + *out << *input_params << std::endl; + + Teuchos::ParameterList solver_factories = input_params->sublist("Solver Factories"); + input_params->remove("Solver Factories"); + + // Add in the application specific equation set factory + Teuchos::RCP eqset_factory = Teuchos::rcp(new user_app::MyFactory); + + // Add in the application specific closure model factory + user_app::MyModelFactory_TemplateBuilder cm_builder; + panzer::ClosureModelFactory_TemplateManager cm_factory; + cm_factory.buildObjects(cm_builder); + + // Add in the application specific bc factory + user_app::BCFactory bc_factory; + + // Create the global data + Teuchos::RCP global_data = panzer::createGlobalData(); + input_params->sublist("User Data").set("Comm", comm); // Required for GlobalStatistics + + Teuchos::RCP > stkIOResponseLibrary + = Teuchos::rcp(new panzer::ResponseLibrary()); + + Teuchos::RCP > physics; + Teuchos::RCP > solver; + Teuchos::RCP > rLibrary; + std::vector > physicsBlocks; + Teuchos::RCP > linObjFactory; + Teuchos::RCP globalIndexer; + Teuchos::RCP mesh; + { + Teuchos::ParameterList responses = input_params->sublist("Responses"); + input_params->remove("Responses"); + + panzer_stk::ModelEvaluatorFactory me_factory; + me_factory.setParameterList(input_params); + auto strat_list = Teuchos::parameterList(); + *strat_list = input_params->sublist("Solution Control",true).sublist("Tempus",true).sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true).sublist("Direction",true).sublist("Newton",true).sublist("Stratimikos Linear Solver",true).sublist("Stratimikos",true); + me_factory.setStratimikosList(strat_list); + me_factory.buildObjects(comm,global_data,eqset_factory,bc_factory,cm_factory); + + user_app::addResponsesToModelEvaluatorFactory(responses,me_factory); + me_factory.buildResponses(cm_factory); + + physicsBlocks = me_factory.getPhysicsBlocks(); + physics = me_factory.getPhysicsModelEvaluator(); + rLibrary = me_factory.getResponseLibrary(); + linObjFactory = me_factory.getLinearObjFactory(); + globalIndexer = me_factory.getGlobalIndexer(); + mesh = me_factory.getMesh(); + } + + // setup outputs to mesh on the stkIOResponseLibrary + { + stkIOResponseLibrary->initialize(*rLibrary); + + Teuchos::ParameterList user_data(input_params->sublist("User Data")); + user_data.set("Workset Size",input_params->sublist("Assembly").get("Workset Size")); + + stkIOResponseLibrary->buildResponseEvaluators(physicsBlocks, + cm_factory, + input_params->sublist("Closure Models"), + user_data); + } + + // Tempus stepper + using namespace Teuchos; + using namespace Tempus; + + RCP tempus_pl = parameterList(std::string("Tempus")); + *tempus_pl = input_params->sublist("Solution Control").sublist("Tempus"); + + if (overrideNoxOutput) { + auto nox_utils = Teuchos::rcp(new NOX::Utils(NOX::Utils::Error,comm->getRank(),0,3)); + auto print_nonlinear = Teuchos::rcp(new NOX::ObserverPrint(nox_utils,10)); + tempus_pl->sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true).sublist("Solver Options",true).set>("Observer",print_nonlinear); + } + + // Tempus is overriding NOX parameters with its own defaults. Need + // to fix this. It also does not validate because of arbitrary + // solver name. Should validate but use + // disableRecursiveValidation() to avoid errors with arbitrary + // names. + + const bool doInitialization = false; + auto integrator = createIntegratorBasic(tempus_pl, physics, doInitialization); + + RCP noxList = parameterList("Correct NOX Params"); + *noxList = tempus_pl->sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true); + // noxList->print(std::cout); + integrator->getStepper()->getSolver()->setParameterList(noxList); + integrator->initialize(); + + RCP tof = Teuchos::rcp(new user_app::TempusObserverFactory(stkIOResponseLibrary,rLibrary->getWorksetContainer())); + auto tempus_observers = tof->buildTempusObserver(mesh,globalIndexer,linObjFactory); + integrator->setObserver(tempus_observers); + + RCP> x0 = physics->getNominalValues().get_x()->clone_v(); + integrator->initializeSolutionHistory(0.0, x0); + + bool integratorStatus = integrator->advanceTime(); + TEUCHOS_ASSERT(integratorStatus); + + auto x = integrator->getSolutionHistory()->getCurrentState()->getX(); + auto x_dot = integrator->getSolutionHistory()->getCurrentState()->getXDot(); + auto t = integrator->getSolutionHistory()->getCurrentState()->getTime(); + + user_app::computeAndPrintResponses(physics,x,x_dot,t,global_data,*out); + + { + Teuchos::RCP rol_params = Teuchos::rcp(new Teuchos::ParameterList("ROL Parameters")); + { + const std::string rol_input_file = "rol_params.xml"; + Teuchos::updateParametersFromXmlFileAndBroadcast(rol_input_file, rol_params.ptr(), *comm); + + RCP pl = sublist(rol_params, "Tempus", true); + RCP objective_params = sublist(rol_params, "Objective", true); + ROL::Ptr > objective = + ROL::makePtr >(physics, pl, objective_params); + + // Create target -- do forward integration with perturbed parameter values + // RCP > zs = objective->create_design_vector(); + // zs->setScalar(1.5); + // RCP > r = objective->create_response_vector(); + // objective->run_tempus(*r, *zs); + // objective->set_target(r); + + } + } + + stackedTimer->stopBaseTimer(); + if (printTimers) { + Teuchos::StackedTimer::OutputOptions options; + options.output_fraction = true; + options.output_minmax = false; + options.output_histogram = false; + options.num_histogram = 5; + std::string timing_file_name = input_file_name+"_timing.log"; + std::fstream timing_file{timing_file_name,std::ios::out | std::ios::trunc}; + stackedTimer->report(timing_file, Teuchos::DefaultComm::getComm(), options); + } + + } + catch (std::exception& e) { + *out << "*********** Caught Exception: Begin Error Report ***********" << std::endl; + *out << e.what() << std::endl; + *out << "************ Caught Exception: End Error Report ************" << std::endl; + status = -1; + } + + if (status == 0) + *out << "panzer::MainDriver run completed." << std::endl; + + return status; +} + +void user_app::addResponsesToModelEvaluatorFactory(const Teuchos::ParameterList& responses, + panzer_stk::ModelEvaluatorFactory& me_factory) +{ + for(Teuchos::ParameterList::ConstIterator itr=responses.begin();itr!=responses.end();++itr) { + const std::string name = responses.name(itr); + TEUCHOS_ASSERT(responses.entry(itr).isList()); + Teuchos::ParameterList & lst = Teuchos::getValue(responses.entry(itr)); + + if (lst.get("Type") == "Functional") { + + panzer::FunctionalResponse_Builder builder; + builder.comm = MPI_COMM_WORLD; + builder.cubatureDegree = 2; + builder.requiresCellIntegral = true; + builder.quadPointField = lst.get("Field Name"); + + std::vector eblocks; + panzer::StringTokenizer(eblocks,lst.get("Element Blocks"),",",true); + + std::vector wkst_descs; + for(std::size_t i=0;i("Type") == "Point Value") { + panzer::ProbeResponse_Builder builder; + builder.comm = MPI_COMM_WORLD; + builder.point = Teuchos::Array{0.5,0.5}; + builder.cubatureDegree = 2; + builder.fieldName = lst.get("Field Name"); + builder.applyDirichletToDerivative = false; + + std::vector eblocks; + panzer::StringTokenizer(eblocks,lst.get("Element Blocks"),",",true); + + std::vector descriptors; + for(std::size_t i=0;i("Type") << "\" is not supported!"); + } + } +} + +void user_app::computeAndPrintResponses(const Teuchos::RCP>& physics, + const auto& x, + const auto& x_dot, + const auto& t, + const auto& global_data, + std::ostream& os) +{ + if(physics->Ng()>0) { + + os << "Ng = " << physics->Ng() << std::endl; + os << "Np = " << physics->Np() << std::endl; + Thyra::ModelEvaluatorBase::InArgs respInArgs = physics->createInArgs(); + Thyra::ModelEvaluatorBase::OutArgs respOutArgs = physics->createOutArgs(); + + TEUCHOS_ASSERT(physics->Ng()==respOutArgs.Ng()); + + respInArgs.set_x(x); + respInArgs.set_x_dot(x_dot); + + for(int g=0;g > response = Thyra::createMember(*physics->get_g_space(g)); + respOutArgs.set_g(g,response); + + for (int p = 0; p < physics->Np(); ++p) { + bool derivative_supported = respOutArgs.supports(Thyra::ModelEvaluatorBase::OUT_ARG_DgDp,g,p).supports(Thyra::ModelEvaluatorBase::DERIV_MV_JACOBIAN_FORM); + os << "DgDp(" << g << "," << p << ") supports = " << derivative_supported << std::endl; + if (derivative_supported) { + auto dgdp = physics->create_DgDp_op(g,p); + respOutArgs.set_DgDp(g,p,Thyra::ModelEvaluatorBase::Derivative(dgdp)); + } + } + } + + physics->evalModel(respInArgs, respOutArgs); + + { + auto parameter_library = global_data->pl; + TEUCHOS_ASSERT(nonnull(parameter_library)); + for (auto i=parameter_library->begin();i != parameter_library->end(); ++i) { + os << "Sacado::ParameterLibrary: " << i->first << std::endl; + } + } + + { + auto nominalValues = physics->getNominalValues(); + for (int i=0; i < respInArgs.Np();++i) { + auto p = nominalValues.get_p(i); + auto p_names = physics->get_p_names(i); + os << "ModelEvaluator::NominalValues Parameter Value: \"" << (*p_names)[0] << "\" = " << Thyra::get_ele(*p,0) << std::endl; + } + } + + for (int i=0;i > response = respOutArgs.get_g(i); + TEUCHOS_ASSERT(response!=Teuchos::null); + os << "Response Value: \"" << physics->get_g_names(i)[0] << "\" = " << Thyra::get_ele(*response,0) << std::endl; + for (int j=0; j < respOutArgs.Np(); ++j) { + // os << " dg(" << i << ")/dp(" << j << ") supports(GRAD_FORM) = " << respOutArgs.supports(Thyra::ModelEvaluatorBase::OUT_ARG_DgDp,i,j).supports(Thyra::ModelEvaluatorBase::DERIV_MV_GRADIENT_FORM) << std::endl; + // os << " dg(" << i << ")/dp(" << j << ") supports(JAC_FORM) = " << respOutArgs.supports(Thyra::ModelEvaluatorBase::OUT_ARG_DgDp,i,j).supports(Thyra::ModelEvaluatorBase::DERIV_MV_JACOBIAN_FORM) << std::endl; + } + } + } +} diff --git a/packages/panzer/adapters-stk/example/main_driver/rol_params.xml b/packages/panzer/adapters-stk/example/main_driver/rol_params.xml new file mode 100644 index 000000000000..8777e117ca47 --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/rol_params.xml @@ -0,0 +1,308 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp index adf6221a5bd1..f839535b9300 100644 --- a/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp +++ b/packages/panzer/adapters-stk/src/Panzer_STK_ModelEvaluatorFactory.hpp @@ -344,7 +344,7 @@ addResponse(const std::string & responseName,const std::vectoraddResponse(responseName,wkstDesc,builder); } else if(panzer_me!=Teuchos::null && thyra_ep_me==Teuchos::null) { - return panzer_me->addFlexibleResponse(responseName,wkstDesc,builder); + return panzer_me->addResponse(responseName,wkstDesc,builder); } #else Teuchos::RCP panzer_me = Teuchos::rcp_dynamic_cast(m_physics_me); From 0df0487d6070af42e3e1b87a1eea3969b1608e14 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 13 Sep 2024 16:16:11 -0600 Subject: [PATCH 12/40] Panzer: add exodus io to transient opt example Signed-off-by: Roger Pawlowski --- .../example/main_driver/main_driver_opt.cpp | 18 +++++++++++--- .../user_app_TempusObserver_WriteToExodus.hpp | 24 +++++++++++-------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp b/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp index b519eecd1893..739eaaa4512c 100644 --- a/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp +++ b/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp @@ -35,6 +35,8 @@ #include "NOX_Observer_Print.hpp" #include "Tempus_IntegratorBasic.hpp" +#include "Tempus_IntegratorObserverComposite.hpp" +#include "Tempus_IntegratorObserverBasic.hpp" #include "Tempus_StepperFactory.hpp" #include "user_app_ClosureModel_Factory_TemplateBuilder.hpp" @@ -228,9 +230,16 @@ int main(int argc, char *argv[]) integrator->getStepper()->getSolver()->setParameterList(noxList); integrator->initialize(); - RCP tof = Teuchos::rcp(new user_app::TempusObserverFactory(stkIOResponseLibrary,rLibrary->getWorksetContainer())); - auto tempus_observers = tof->buildTempusObserver(mesh,globalIndexer,linObjFactory); - integrator->setObserver(tempus_observers); + // Setting observers on tempus breaks the screen output of the + // time steps! It replaces IntegratorObserverBasic which handles + // IO. Is this at least documented? + { + RCP> tempus_observers = rcp(new Tempus::IntegratorObserverComposite); + RCP tof = Teuchos::rcp(new user_app::TempusObserverFactory(stkIOResponseLibrary,rLibrary->getWorksetContainer())); + tempus_observers->addObserver(Teuchos::rcp(new Tempus::IntegratorObserverBasic)); + tempus_observers->addObserver(tof->buildTempusObserver(mesh,globalIndexer,linObjFactory)); + integrator->setObserver(tempus_observers); + } RCP> x0 = physics->getNominalValues().get_x()->clone_v(); integrator->initializeSolutionHistory(0.0, x0); @@ -291,6 +300,9 @@ int main(int argc, char *argv[]) return status; } +// ************************************************* +// ************************************************* + void user_app::addResponsesToModelEvaluatorFactory(const Teuchos::ParameterList& responses, panzer_stk::ModelEvaluatorFactory& me_factory) { diff --git a/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp b/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp index 3873d08f191e..ffd9a9fdd5a3 100644 --- a/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp +++ b/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp @@ -34,7 +34,8 @@ namespace user_app { m_mesh(mesh), m_dof_manager(dof_manager), m_lof(lof), - m_response_library(response_library) + m_response_library(response_library), + m_print_debug(false) { // get all element blocks and add them to the list std::vector eBlocks; @@ -45,18 +46,20 @@ namespace user_app { m_response_library->addResponse("Main Field Output",eBlocks,builder); } - void observeStartIntegrator(const Tempus::Integrator& integrator) override {this->writeToExodus(integrator);}; - void observeStartTimeStep(const Tempus::Integrator& ) override {}; - void observeNextTimeStep(const Tempus::Integrator& ) override {}; - void observeBeforeTakeStep(const Tempus::Integrator& ) override {}; - void observeAfterTakeStep(const Tempus::Integrator& integrator) override {this->writeToExodus(integrator);} - void observeAfterCheckTimeStep(const Tempus::Integrator& ) override {}; - void observeEndTimeStep(const Tempus::Integrator& ) override {}; - void observeEndIntegrator(const Tempus::Integrator& ) override {}; + void observeStartIntegrator(const Tempus::Integrator& integrator) override {this->writeToExodus(integrator);} + void observeStartTimeStep(const Tempus::Integrator& ) override {} + void observeNextTimeStep(const Tempus::Integrator& ) override {} + void observeBeforeTakeStep(const Tempus::Integrator& ) override {} + void observeAfterTakeStep(const Tempus::Integrator& ) override {} + void observeAfterCheckTimeStep(const Tempus::Integrator& ) override {} + void observeEndTimeStep(const Tempus::Integrator& integrator) override {this->writeToExodus(integrator);} + void observeEndIntegrator(const Tempus::Integrator& ) override {} void writeToExodus(const Tempus::Integrator& integrator) { - + if (m_print_debug) { + std::cout << "Writing to expdus with time step=" << integrator.getSolutionHistory()->getCurrentTime() << std::endl; + } Teuchos::RCP > solution = integrator.getSolutionHistory()->getStateTimeIndexN()->getX(); // initialize the assembly container @@ -88,6 +91,7 @@ namespace user_app { Teuchos::RCP m_dof_manager; Teuchos::RCP > m_lof; Teuchos::RCP > m_response_library; + bool m_print_debug; }; } From 08f12154a5b626433fc8c5e37901d47c010490cb Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Mon, 16 Sep 2024 12:15:10 -0600 Subject: [PATCH 13/40] Panzer: fixed some IO issues in opt test Signed-off-by: Roger Pawlowski --- .../example/main_driver/main_driver_opt.cpp | 7 +++++++ .../user_app_TempusObserver_WriteToExodus.hpp | 11 ++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp b/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp index 739eaaa4512c..fd1cc98857ab 100644 --- a/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp +++ b/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp @@ -196,6 +196,13 @@ int main(int argc, char *argv[]) Teuchos::ParameterList user_data(input_params->sublist("User Data")); user_data.set("Workset Size",input_params->sublist("Assembly").get("Workset Size")); + std::vector eBlocks; + mesh->getElementBlockNames(eBlocks); + + panzer_stk::RespFactorySolnWriter_Builder builder; + builder.mesh = mesh; + stkIOResponseLibrary->addResponse("Main Field Output",eBlocks,builder); + stkIOResponseLibrary->buildResponseEvaluators(physicsBlocks, cm_factory, input_params->sublist("Closure Models"), diff --git a/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp b/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp index ffd9a9fdd5a3..04cec8bceb14 100644 --- a/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp +++ b/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp @@ -46,17 +46,21 @@ namespace user_app { m_response_library->addResponse("Main Field Output",eBlocks,builder); } - void observeStartIntegrator(const Tempus::Integrator& integrator) override {this->writeToExodus(integrator);} + void observeStartIntegrator(const Tempus::Integrator& integrator) override + {this->writeToExodus(integrator,"Tempus::Observer::writeToExodus(INITIAL_CONDITION)");} void observeStartTimeStep(const Tempus::Integrator& ) override {} void observeNextTimeStep(const Tempus::Integrator& ) override {} void observeBeforeTakeStep(const Tempus::Integrator& ) override {} void observeAfterTakeStep(const Tempus::Integrator& ) override {} void observeAfterCheckTimeStep(const Tempus::Integrator& ) override {} - void observeEndTimeStep(const Tempus::Integrator& integrator) override {this->writeToExodus(integrator);} + void observeEndTimeStep(const Tempus::Integrator& integrator) override + {this->writeToExodus(integrator,"Tempus::Observer::writeToExodus(END_TIME_STEP)");} void observeEndIntegrator(const Tempus::Integrator& ) override {} - void writeToExodus(const Tempus::Integrator& integrator) + void writeToExodus(const Tempus::Integrator& integrator, const std::string& timer_name) { + auto timer = Teuchos::TimeMonitor::getStackedTimer(); + timer->start(timer_name); if (m_print_debug) { std::cout << "Writing to expdus with time step=" << integrator.getSolutionHistory()->getCurrentTime() << std::endl; } @@ -83,6 +87,7 @@ namespace user_app { m_response_library->addResponsesToInArgs(ae_inargs); m_response_library->evaluate(ae_inargs); m_mesh->writeToExodus(integrator.getSolutionHistory()->getCurrentTime()); + timer->stop(timer_name); } protected: From b5171505c0f7f9036d5be225666ff854943d99fb Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Mon, 16 Sep 2024 13:03:33 -0600 Subject: [PATCH 14/40] Panzer: remove sa problem for now Signed-off-by: Roger Pawlowski --- .../example/main_driver/CMakeLists.txt | 13 - .../energy-transient-tempus-sa-blocked.xml | 377 --------------- .../example/main_driver/main_driver_sa.cpp | 443 ------------------ 3 files changed, 833 deletions(-) delete mode 100644 packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml delete mode 100644 packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp diff --git a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt index a2781918056b..0b0e890247f6 100644 --- a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt +++ b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt @@ -29,7 +29,6 @@ TRIBITS_COPY_FILES_TO_BINARY_DIR(main_driver_files energy-ss-loca-eigenvalue.xml energy-ss-blocked.xml energy-transient-tempus-blocked.xml - energy-transient-tempus-sa-blocked.xml energy-transient-tempus-opt-blocked.xml rol_params.xml energy-neumann.xml @@ -118,18 +117,6 @@ IF(${PACKAGE_NAME}_ENABLE_Teko) PASS_REGULAR_EXPRESSION "panzer::MainDriver run completed." ) - # Transient Sensitivity Analysis - TRIBITS_ADD_EXECUTABLE( - main_driver_sa - SOURCES main_driver_sa.cpp ${main_driver_SOURCES} - ) - TRIBITS_ADD_ADVANCED_TEST( - main_driver_energy-transient-tempus-sa-blocked - TEST_0 EXEC main_driver_sa - ARGS --i=energy-transient-tempus-sa-blocked.xml --point-calc - PASS_REGULAR_EXPRESSION "panzer::MainDriver run completed." - ) - # Optimization with ROL IF(${PACKAGE_NAME}_ENABLE_ROL) TRIBITS_ADD_EXECUTABLE( diff --git a/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml b/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml deleted file mode 100644 index 7e3fbab9a6f9..000000000000 --- a/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-sa-blocked.xml +++ /dev/null @@ -1,377 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp b/packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp deleted file mode 100644 index 729a7ce3d71d..000000000000 --- a/packages/panzer/adapters-stk/example/main_driver/main_driver_sa.cpp +++ /dev/null @@ -1,443 +0,0 @@ -// @HEADER -// ***************************************************************************** -// Panzer: A partial differential equation assembly -// engine for strongly coupled complex multiphysics systems -// -// Copyright 2011 NTESS and the Panzer contributors. -// SPDX-License-Identifier: BSD-3-Clause -// ***************************************************************************** -// @HEADER - -#include "Teuchos_ConfigDefs.hpp" -#include "Teuchos_RCP.hpp" -#include "Teuchos_TimeMonitor.hpp" -#include "Teuchos_StackedTimer.hpp" -#include "Teuchos_DefaultComm.hpp" -#include "Teuchos_CommHelpers.hpp" -#include "Teuchos_GlobalMPISession.hpp" -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_XMLParameterListHelpers.hpp" -#include "Teuchos_YamlParameterListHelpers.hpp" -#include "Teuchos_FancyOStream.hpp" -#include "Teuchos_oblackholestream.hpp" -#include "Teuchos_Assert.hpp" -#include "Teuchos_as.hpp" - -#include "Panzer_NodeType.hpp" -#include "PanzerAdaptersSTK_config.hpp" -#include "Panzer_STK_ModelEvaluatorFactory.hpp" -#include "Panzer_ClosureModel_Factory_TemplateManager.hpp" -#include "Panzer_String_Utilities.hpp" -#include "Panzer_ThyraObjContainer.hpp" -#include "Thyra_VectorSpaceBase.hpp" - -#include "NOX_Utils.H" -#include "NOX_Observer_Print.hpp" - -#include "Tempus_IntegratorBasic.hpp" -#include "Tempus_StepperFactory.hpp" - -#include "user_app_ClosureModel_Factory_TemplateBuilder.hpp" -#include "user_app_EquationSetFactory.hpp" -#include "user_app_BCStrategy_Factory.hpp" -#include "user_app_NOXObserverFactory.hpp" -#ifdef PANZER_HAVE_TEMPUS -#include "user_app_TempusObserverFactory.hpp" -#endif -#include "user_app_ResponseEvaluatorFactory_HOFlux.hpp" - -#include "Panzer_ResponseEvaluatorFactory_Probe.hpp" - -#include - -#include -#include - -// ************************************************* -// ************************************************* - -// Main Driver for a transient sensitivity analysis simulation - -// ************************************************* -// ************************************************* - -int main(int argc, char *argv[]) -{ - int status = 0; - - Teuchos::oblackholestream blackhole; - Teuchos::GlobalMPISession mpiSession(&argc, &argv, &blackhole); - Kokkos::initialize(argc,argv); - - Teuchos::RCP out = Teuchos::rcp(new Teuchos::FancyOStream(Teuchos::rcp(&std::cout,false))); - Teuchos::RCP pout = Teuchos::rcp(new Teuchos::FancyOStream(Teuchos::rcp(&std::cout,false))); - if (mpiSession.getNProc() > 1) { - out->setShowProcRank(true); - out->setOutputToRootOnly(0); - } - - try { - const auto stackedTimer = Teuchos::rcp(new Teuchos::StackedTimer("Panzer Main Driver Sensitivity Analysis")); - Teuchos::TimeMonitor::setStackedTimer(stackedTimer); - - Teuchos::RCP > comm = Teuchos::DefaultComm::getComm(); - - // Parse the command line arguments - std::string input_file_name = "user_app.xml"; - int exodus_io_num_procs = 0; - bool pointCalculation = true; - bool printTimers = true; - bool printInputPL = false; - bool overrideNoxOutput = true; - { - Teuchos::CommandLineProcessor clp; - - clp.setOption("i", &input_file_name, "User_App input xml filename"); - clp.setOption("exodus-io-num-procs", &exodus_io_num_procs, "Number of processes that can access the file system at the same time to read their portion of a sliced exodus file in parallel. Defaults to 0 - implies all processes for the run can access the file system at the same time."); - clp.setOption("point-calc","disable-point-calc", &pointCalculation, "Enable the probe evaluator unit test."); - clp.setOption("time","no-time", &printTimers, "Print the timing information."); - clp.setOption("pl","no-pl", &printInputPL, "Print the input ParameterList at the start of the run."); - clp.setOption("override-nox-output","default-nox-output", &overrideNoxOutput, "Override default nox printing with new print observer."); - - Teuchos::CommandLineProcessor::EParseCommandLineReturn parse_return = - clp.parse(argc,argv,&std::cerr); - - TEUCHOS_TEST_FOR_EXCEPTION(parse_return != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL, - std::runtime_error, "Failed to parse command line!"); - } - - TEUCHOS_ASSERT(exodus_io_num_procs <= comm->getSize()); - if (exodus_io_num_procs != 0) - Ioss::SerializeIO::setGroupFactor(exodus_io_num_procs); - - // Parse the input file and broadcast to other processes - Teuchos::RCP input_params = Teuchos::rcp(new Teuchos::ParameterList("User_App Parameters")); - { - const std::string check_yaml = ".yaml"; - const auto search_yaml = input_file_name.find(check_yaml); - if (search_yaml != std::string::npos) { - Teuchos::updateParametersFromYamlFileAndBroadcast(input_file_name, input_params.ptr(), *comm); - } else { - Teuchos::updateParametersFromXmlFileAndBroadcast(input_file_name, input_params.ptr(), *comm); - } - } - - if (printInputPL) - *out << *input_params << std::endl; - - Teuchos::ParameterList solver_factories = input_params->sublist("Solver Factories"); - input_params->remove("Solver Factories"); - - // Add in the application specific equation set factory - Teuchos::RCP eqset_factory = Teuchos::rcp(new user_app::MyFactory); - - // Add in the application specific closure model factory - user_app::MyModelFactory_TemplateBuilder cm_builder; - panzer::ClosureModelFactory_TemplateManager cm_factory; - cm_factory.buildObjects(cm_builder); - - // Add in the application specific bc factory - user_app::BCFactory bc_factory; - - // Create the global data - Teuchos::RCP global_data = panzer::createGlobalData(); - - // A GlobalStatistics closure model requires the comm to be set in the user data. - input_params->sublist("User Data").set("Comm", comm); - - Teuchos::RCP > stkIOResponseLibrary - = Teuchos::rcp(new panzer::ResponseLibrary()); - - Teuchos::RCP > physics; - Teuchos::RCP > solver; - Teuchos::RCP > rLibrary; - std::vector > physicsBlocks; - Teuchos::RCP > linObjFactory; - std::map responseIndexToName; - { - Teuchos::ParameterList responses = input_params->sublist("Responses"); - input_params->remove("Responses"); - - panzer_stk::ModelEvaluatorFactory me_factory; - me_factory.setParameterList(input_params); - auto strat_list = Teuchos::parameterList(); - *strat_list = input_params->sublist("Solution Control",true).sublist("Tempus",true).sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true).sublist("Direction",true).sublist("Newton",true).sublist("Stratimikos Linear Solver",true).sublist("Stratimikos",true); - me_factory.setStratimikosList(strat_list); - me_factory.buildObjects(comm,global_data,eqset_factory,bc_factory,cm_factory); - - // add a volume response functional for each field - for(Teuchos::ParameterList::ConstIterator itr=responses.begin();itr!=responses.end();++itr) { - const std::string name = responses.name(itr); - TEUCHOS_ASSERT(responses.entry(itr).isList()); - Teuchos::ParameterList & lst = Teuchos::getValue(responses.entry(itr)); - - // parameterize the builder - panzer::FunctionalResponse_Builder builder; - builder.comm = MPI_COMM_WORLD; // good enough - builder.cubatureDegree = 2; - builder.requiresCellIntegral = lst.isType("Requires Cell Integral") ? lst.get("Requires Cell Integral"): false; - builder.quadPointField = lst.get("Field Name"); - - // add the response - std::vector eblocks; - panzer::StringTokenizer(eblocks,lst.get("Element Blocks"),",",true); - - std::vector wkst_descs; - for(std::size_t i=0;i nof; - { - nof = Teuchos::rcp(new user_app::NOXObserverFactory(stkIOResponseLibrary)); - - Teuchos::RCP observers_to_build = - Teuchos::parameterList(solver_factories.sublist("NOX Observers")); - - nof->setParameterList(observers_to_build); - } - -#ifdef PANZER_HAVE_TEMPUS - Teuchos::RCP tof; - { - tof = Teuchos::rcp(new user_app::TempusObserverFactory(stkIOResponseLibrary,rLibrary->getWorksetContainer())); - } - solver = me_factory.buildResponseOnlyModelEvaluator(physics,global_data,Teuchos::null,nof.ptr(),tof.ptr()); -#else - solver = me_factory.buildResponseOnlyModelEvaluator(physics,global_data,nof.ptr()); -#endif - } - } - - // setup outputs to mesh on the stkIOResponseLibrary - //////////////////////////////////////////////////////////////// - - stkIOResponseLibrary->initialize(*rLibrary); - - // 1. Register correct aggregator and reserve response - this was done in the appropriate observer object - - // 2. Build volume field managers - { - Teuchos::ParameterList user_data(input_params->sublist("User Data")); - user_data.set("Workset Size",input_params->sublist("Assembly").get("Workset Size")); - - stkIOResponseLibrary->buildResponseEvaluators(physicsBlocks, - cm_factory, - input_params->sublist("Closure Models"), - user_data); - } - - // setup outputs for the point calculation - //////////////////////////////////////////////////////////////// - - Teuchos::RCP > pointResponseLibrary - = Teuchos::rcp(new panzer::ResponseLibrary); - - if(pointCalculation) { - pointResponseLibrary->initialize(*rLibrary); - - { - panzer::ProbeResponse_Builder builder; - builder.comm = MPI_COMM_WORLD; - builder.point = Teuchos::Array{0.5,0.5}; // Bottom - builder.cubatureDegree = 2; - builder.fieldName = "TEMPERATURE"; - builder.applyDirichletToDerivative = false; - - std::vector descriptors; - descriptors.push_back(panzer::WorksetDescriptor("eblock-0_0")); - - pointResponseLibrary->addResponse("Value In Middle",descriptors,builder); - } - - { - Teuchos::ParameterList user_data(input_params->sublist("User Data")); - user_data.set("Workset Size",input_params->sublist("Assembly").get("Workset Size")); - - pointResponseLibrary->buildResponseEvaluators(physicsBlocks, - *eqset_factory, - cm_factory, - input_params->sublist("Closure Models"), - user_data); - } - - { - Teuchos::RCP > resp - = Teuchos::rcp_dynamic_cast >(pointResponseLibrary->getResponse("Value In Middle"),true); - - const auto vec = Thyra::createMember(*resp->getVectorSpace(),"Value In Middle Response Thyra Vector"); - resp->setVector(vec); - } - } - - // Tempus stepper - using namespace Teuchos; - using namespace Tempus; - - RCP tempus_pl = parameterList(std::string("Tempus")); - *tempus_pl = input_params->sublist("Solution Control").sublist("Tempus"); - - if (overrideNoxOutput) { - auto nox_utils = Teuchos::rcp(new NOX::Utils(NOX::Utils::Error,comm->getRank(),0,3)); - auto print_nonlinear = Teuchos::rcp(new NOX::ObserverPrint(nox_utils,10)); - tempus_pl->sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true).sublist("Solver Options",true).set>("Observer",print_nonlinear); - } - - // Tempus is overriding NOX parameters with its own defaults. Need - // to fix this. It also does not validate because of arbitrary - // solver name. Should validate but use - // disableRecursiveValidation() to avoid errors with arbitrary - // names. - - const bool doInitialization = false; - auto integrator = createIntegratorBasic(tempus_pl, physics, doInitialization); - - RCP noxList = parameterList("Correct NOX Params"); - *noxList = tempus_pl->sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true); - noxList->print(std::cout); - integrator->getStepper()->getSolver()->setParameterList(noxList); - integrator->initialize(); - - RCP> x0 = physics->getNominalValues().get_x()->clone_v(); - integrator->initializeSolutionHistory(0.0, x0); - - bool integratorStatus = integrator->advanceTime(); - TEUCHOS_ASSERT(integratorStatus); - - auto x = integrator->getSolutionHistory()->getCurrentState()->getX(); - auto x_dot = integrator->getSolutionHistory()->getCurrentState()->getXDot(); - - { - // get responses if there are any - ////////////////////////////////////////////// - if(physics->Ng()>0) { - - Thyra::ModelEvaluatorBase::InArgs respInArgs = physics->createInArgs(); - Thyra::ModelEvaluatorBase::OutArgs respOutArgs = physics->createOutArgs(); - - TEUCHOS_ASSERT(physics->Ng()==respOutArgs.Ng()); - - respInArgs.set_x(x); - respInArgs.set_x_dot(x_dot); - - for(int i=0;i > response = Thyra::createMember(*physics->get_g_space(i)); - respOutArgs.set_g(i,response); - } - - physics->evalModel(respInArgs, respOutArgs); - - { - auto parameter_library = global_data->pl; - TEUCHOS_ASSERT(nonnull(parameter_library)); - for (auto i=parameter_library->begin();i != parameter_library->end(); ++i) { - *out << "Sacado::ParameterLibrary: " << i->first << std::endl; - } - } - - { - auto nominalValues = physics->getNominalValues(); - for (int i=0; i < respInArgs.Np();++i) { - auto p = nominalValues.get_p(i); - auto p_names = physics->get_p_names(i); - *out << "ModelEvaluator::NominalValues Parameter Value: \"" << (*p_names)[0] << "\" = " << Thyra::get_ele(*p,0) << std::endl; - } - } - - for (int i=0;i > response = respOutArgs.get_g(i); - TEUCHOS_ASSERT(response!=Teuchos::null); - *out << "Response Value: \"" << responseIndexToName[i] << "\" = " << Thyra::get_ele(*response,0) << std::endl; - } - } - - if(pointCalculation) { - stackedTimer->start("Point Value Response Calculation"); - // initialize the assembly container - panzer::AssemblyEngineInArgs ae_inargs; - ae_inargs.container_ = linObjFactory->buildLinearObjContainer(); - ae_inargs.ghostedContainer_ = linObjFactory->buildGhostedLinearObjContainer(); - ae_inargs.alpha = 0.0; - ae_inargs.beta = 1.0; - ae_inargs.evaluate_transient_terms = false; - - // initialize the ghosted container - linObjFactory->initializeGhostedContainer(panzer::LinearObjContainer::X,*ae_inargs.ghostedContainer_); - - const Teuchos::RCP> thGlobalContainer - = Teuchos::rcp_dynamic_cast>(ae_inargs.container_,true); - thGlobalContainer->set_x_th(x); - - // evaluate current on contacts - pointResponseLibrary->addResponsesToInArgs(ae_inargs); - pointResponseLibrary->evaluate(ae_inargs); - - // output current values - *out << "\nPoint Values: \n"; - { - std::string currentRespName = "Value In Middle"; - - Teuchos::RCP > resp = - Teuchos::rcp_dynamic_cast>(pointResponseLibrary->getResponse(currentRespName),true); - - // Linear problem with analytic solution - const double gold_value = 0.5; - const double tol = 1.0e-8; - *out << " " << currentRespName << " = " << resp->value << ", error = " << fabs(resp->value - gold_value) << ", tol = " << tol << std::endl; - // TEUCHOS_ASSERT(fabs(resp->value - gold_value) < tol); - } - - stackedTimer->stop("Point Value Response Calculation"); - } - } - - stackedTimer->stopBaseTimer(); - if (printTimers) { - Teuchos::StackedTimer::OutputOptions options; - options.output_fraction = true; - options.output_minmax = false; - options.output_histogram = false; - options.num_histogram = 5; - std::string timing_file_name = input_file_name+"_timing.log"; - std::fstream timing_file{timing_file_name,std::ios::out | std::ios::trunc}; - stackedTimer->report(timing_file, Teuchos::DefaultComm::getComm(), options); - } - - } - catch (std::exception& e) { - *out << "*********** Caught Exception: Begin Error Report ***********" << std::endl; - *out << e.what() << std::endl; - *out << "************ Caught Exception: End Error Report ************" << std::endl; - status = -1; - } - catch (std::string& msg) { - *out << "*********** Caught Exception: Begin Error Report ***********" << std::endl; - *out << msg << std::endl; - *out << "************ Caught Exception: End Error Report ************" << std::endl; - status = -1; - } - catch (...) { - *out << "*********** Caught Exception: Begin Error Report ***********" << std::endl; - *out << "Caught UNKOWN exception" << std::endl; - *out << "************ Caught Exception: End Error Report ************" << std::endl; - status = -1; - } - - if (status == 0) - *out << "panzer::MainDriver run completed." << std::endl; - - return status; -} From 0c30d2dee7a494dd8c2906423a8520985d30c767 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Wed, 18 Sep 2024 12:03:09 -0600 Subject: [PATCH 15/40] splitting out solver pieces for rol Signed-off-by: Roger Pawlowski --- .../example/main_driver/CMakeLists.txt | 2 +- .../example/main_driver/main_driver_opt.cpp | 267 +++++----- .../user_app_ROLTempusReducedObjective.hpp | 477 ++++++++++++++++++ 3 files changed, 628 insertions(+), 118 deletions(-) create mode 100644 packages/panzer/adapters-stk/example/main_driver/user_app_ROLTempusReducedObjective.hpp diff --git a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt index 0b0e890247f6..5dff70456774 100644 --- a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt +++ b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt @@ -121,7 +121,7 @@ IF(${PACKAGE_NAME}_ENABLE_Teko) IF(${PACKAGE_NAME}_ENABLE_ROL) TRIBITS_ADD_EXECUTABLE( main_driver_opt - SOURCES main_driver_opt.cpp ${main_driver_SOURCES} + SOURCES main_driver_opt.cpp ${main_driver_SOURCES} user_app_ROLTempusReducedObjective.hpp ) TRIBITS_ADD_ADVANCED_TEST( main_driver_energy-transient-tempus-opt-blocked diff --git a/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp b/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp index fd1cc98857ab..af09ef0223cf 100644 --- a/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp +++ b/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp @@ -52,10 +52,15 @@ #include "Panzer_ResponseEvaluatorFactory_Probe.hpp" +#include "ROL_OptimizationProblem.hpp" +#include "ROL_OptimizationSolver.hpp" +#include "ROL_RandomVector.hpp" + #include #include #include +#include // ************************************************* // applicaiton functions defined in this file @@ -63,12 +68,20 @@ namespace user_app { void addResponsesToModelEvaluatorFactory(const Teuchos::ParameterList& response_sublist, panzer_stk::ModelEvaluatorFactory& me_factory); + void computeAndPrintResponses(const Teuchos::RCP>& me, const auto& x, const auto& x_dot, const auto& t, const auto& global_data, std::ostream& out); + + std::tuple>, + Teuchos::RCP, + Teuchos::RCP>> + buildTempusIntegrator(const Teuchos::RCP& input_params, + const Teuchos::RCP>& comm, + const bool overrideNoxOutput); } // ************************************************* @@ -80,6 +93,9 @@ namespace user_app { int main(int argc, char *argv[]) { + using namespace Teuchos; + using namespace Tempus; + int status = 0; std::ostream blackhole{nullptr}; @@ -139,118 +155,10 @@ int main(int argc, char *argv[]) if (printInputPL) *out << *input_params << std::endl; - Teuchos::ParameterList solver_factories = input_params->sublist("Solver Factories"); - input_params->remove("Solver Factories"); - - // Add in the application specific equation set factory - Teuchos::RCP eqset_factory = Teuchos::rcp(new user_app::MyFactory); - - // Add in the application specific closure model factory - user_app::MyModelFactory_TemplateBuilder cm_builder; - panzer::ClosureModelFactory_TemplateManager cm_factory; - cm_factory.buildObjects(cm_builder); - - // Add in the application specific bc factory - user_app::BCFactory bc_factory; - - // Create the global data - Teuchos::RCP global_data = panzer::createGlobalData(); - input_params->sublist("User Data").set("Comm", comm); // Required for GlobalStatistics - - Teuchos::RCP > stkIOResponseLibrary - = Teuchos::rcp(new panzer::ResponseLibrary()); - - Teuchos::RCP > physics; - Teuchos::RCP > solver; - Teuchos::RCP > rLibrary; - std::vector > physicsBlocks; - Teuchos::RCP > linObjFactory; - Teuchos::RCP globalIndexer; - Teuchos::RCP mesh; - { - Teuchos::ParameterList responses = input_params->sublist("Responses"); - input_params->remove("Responses"); - - panzer_stk::ModelEvaluatorFactory me_factory; - me_factory.setParameterList(input_params); - auto strat_list = Teuchos::parameterList(); - *strat_list = input_params->sublist("Solution Control",true).sublist("Tempus",true).sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true).sublist("Direction",true).sublist("Newton",true).sublist("Stratimikos Linear Solver",true).sublist("Stratimikos",true); - me_factory.setStratimikosList(strat_list); - me_factory.buildObjects(comm,global_data,eqset_factory,bc_factory,cm_factory); - - user_app::addResponsesToModelEvaluatorFactory(responses,me_factory); - me_factory.buildResponses(cm_factory); - - physicsBlocks = me_factory.getPhysicsBlocks(); - physics = me_factory.getPhysicsModelEvaluator(); - rLibrary = me_factory.getResponseLibrary(); - linObjFactory = me_factory.getLinearObjFactory(); - globalIndexer = me_factory.getGlobalIndexer(); - mesh = me_factory.getMesh(); - } - - // setup outputs to mesh on the stkIOResponseLibrary - { - stkIOResponseLibrary->initialize(*rLibrary); - - Teuchos::ParameterList user_data(input_params->sublist("User Data")); - user_data.set("Workset Size",input_params->sublist("Assembly").get("Workset Size")); - - std::vector eBlocks; - mesh->getElementBlockNames(eBlocks); - - panzer_stk::RespFactorySolnWriter_Builder builder; - builder.mesh = mesh; - stkIOResponseLibrary->addResponse("Main Field Output",eBlocks,builder); - - stkIOResponseLibrary->buildResponseEvaluators(physicsBlocks, - cm_factory, - input_params->sublist("Closure Models"), - user_data); - } - - // Tempus stepper - using namespace Teuchos; - using namespace Tempus; - - RCP tempus_pl = parameterList(std::string("Tempus")); - *tempus_pl = input_params->sublist("Solution Control").sublist("Tempus"); - - if (overrideNoxOutput) { - auto nox_utils = Teuchos::rcp(new NOX::Utils(NOX::Utils::Error,comm->getRank(),0,3)); - auto print_nonlinear = Teuchos::rcp(new NOX::ObserverPrint(nox_utils,10)); - tempus_pl->sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true).sublist("Solver Options",true).set>("Observer",print_nonlinear); - } - - // Tempus is overriding NOX parameters with its own defaults. Need - // to fix this. It also does not validate because of arbitrary - // solver name. Should validate but use - // disableRecursiveValidation() to avoid errors with arbitrary - // names. - - const bool doInitialization = false; - auto integrator = createIntegratorBasic(tempus_pl, physics, doInitialization); - - RCP noxList = parameterList("Correct NOX Params"); - *noxList = tempus_pl->sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true); - // noxList->print(std::cout); - integrator->getStepper()->getSolver()->setParameterList(noxList); - integrator->initialize(); - - // Setting observers on tempus breaks the screen output of the - // time steps! It replaces IntegratorObserverBasic which handles - // IO. Is this at least documented? - { - RCP> tempus_observers = rcp(new Tempus::IntegratorObserverComposite); - RCP tof = Teuchos::rcp(new user_app::TempusObserverFactory(stkIOResponseLibrary,rLibrary->getWorksetContainer())); - tempus_observers->addObserver(Teuchos::rcp(new Tempus::IntegratorObserverBasic)); - tempus_observers->addObserver(tof->buildTempusObserver(mesh,globalIndexer,linObjFactory)); - integrator->setObserver(tempus_observers); - } - - RCP> x0 = physics->getNominalValues().get_x()->clone_v(); - integrator->initializeSolutionHistory(0.0, x0); + RCP params_copy = parameterList(*input_params); + auto [integrator,global_data,physics] = user_app::buildTempusIntegrator(params_copy,comm,overrideNoxOutput); + bool integratorStatus = integrator->advanceTime(); TEUCHOS_ASSERT(integratorStatus); @@ -259,7 +167,7 @@ int main(int argc, char *argv[]) auto t = integrator->getSolutionHistory()->getCurrentState()->getTime(); user_app::computeAndPrintResponses(physics,x,x_dot,t,global_data,*out); - + { Teuchos::RCP rol_params = Teuchos::rcp(new Teuchos::ParameterList("ROL Parameters")); { @@ -272,13 +180,17 @@ int main(int argc, char *argv[]) ROL::makePtr >(physics, pl, objective_params); // Create target -- do forward integration with perturbed parameter values - // RCP > zs = objective->create_design_vector(); - // zs->setScalar(1.5); - // RCP > r = objective->create_response_vector(); - // objective->run_tempus(*r, *zs); - // objective->set_target(r); + RCP > zs = objective->create_design_vector(); + zs->setScalar(1.5); + RCP > r = objective->create_response_vector(); + objective->run_tempus(*r, *zs); + objective->set_target(r); } + + ROL::Ptr outStream = ROL::makePtrFromRef(std::cout); + + } stackedTimer->stopBaseTimer(); @@ -421,3 +333,124 @@ void user_app::computeAndPrintResponses(const Teuchos::RCP>, + Teuchos::RCP, + Teuchos::RCP>> +user_app::buildTempusIntegrator(const Teuchos::RCP& input_params, + const Teuchos::RCP>& comm, + const bool overrideNoxOutput) +{ + using namespace Teuchos; + using namespace Tempus; + + Teuchos::ParameterList solver_factories = input_params->sublist("Solver Factories"); + input_params->remove("Solver Factories"); + + // Add in the application specific equation set factory + Teuchos::RCP eqset_factory = Teuchos::rcp(new user_app::MyFactory); + + // Add in the application specific closure model factory + user_app::MyModelFactory_TemplateBuilder cm_builder; + panzer::ClosureModelFactory_TemplateManager cm_factory; + cm_factory.buildObjects(cm_builder); + + // Add in the application specific bc factory + user_app::BCFactory bc_factory; + + // Create the global data + Teuchos::RCP global_data = panzer::createGlobalData(); + input_params->sublist("User Data").set("Comm", comm); // Required for GlobalStatistics + + Teuchos::RCP > stkIOResponseLibrary + = Teuchos::rcp(new panzer::ResponseLibrary()); + + Teuchos::RCP> physics; + Teuchos::RCP> solver; + Teuchos::RCP> rLibrary; + std::vector> physicsBlocks; + Teuchos::RCP> linObjFactory; + Teuchos::RCP globalIndexer; + Teuchos::RCP mesh; + { + Teuchos::ParameterList responses = input_params->sublist("Responses"); + input_params->remove("Responses"); + + panzer_stk::ModelEvaluatorFactory me_factory; + me_factory.setParameterList(input_params); + auto strat_list = Teuchos::parameterList(); + *strat_list = input_params->sublist("Solution Control",true).sublist("Tempus",true).sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true).sublist("Direction",true).sublist("Newton",true).sublist("Stratimikos Linear Solver",true).sublist("Stratimikos",true); + me_factory.setStratimikosList(strat_list); + me_factory.buildObjects(comm,global_data,eqset_factory,bc_factory,cm_factory); + + user_app::addResponsesToModelEvaluatorFactory(responses,me_factory); + me_factory.buildResponses(cm_factory); + + physicsBlocks = me_factory.getPhysicsBlocks(); + physics = me_factory.getPhysicsModelEvaluator(); + rLibrary = me_factory.getResponseLibrary(); + linObjFactory = me_factory.getLinearObjFactory(); + globalIndexer = me_factory.getGlobalIndexer(); + mesh = me_factory.getMesh(); + } + + // setup outputs to mesh on the stkIOResponseLibrary + { + stkIOResponseLibrary->initialize(*rLibrary); + + Teuchos::ParameterList user_data(input_params->sublist("User Data")); + user_data.set("Workset Size",input_params->sublist("Assembly").get("Workset Size")); + + std::vector eBlocks; + mesh->getElementBlockNames(eBlocks); + + panzer_stk::RespFactorySolnWriter_Builder builder; + builder.mesh = mesh; + stkIOResponseLibrary->addResponse("Main Field Output",eBlocks,builder); + + stkIOResponseLibrary->buildResponseEvaluators(physicsBlocks, + cm_factory, + input_params->sublist("Closure Models"), + user_data); + } + + RCP tempus_pl = parameterList(std::string("Tempus")); + *tempus_pl = input_params->sublist("Solution Control").sublist("Tempus"); + + if (overrideNoxOutput) { + auto nox_utils = Teuchos::rcp(new NOX::Utils(NOX::Utils::Error,comm->getRank(),0,3)); + auto print_nonlinear = Teuchos::rcp(new NOX::ObserverPrint(nox_utils,10)); + tempus_pl->sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true).sublist("Solver Options",true).set>("Observer",print_nonlinear); + } + + // Tempus is overriding NOX parameters with its own defaults. Need + // to fix this. It also does not validate because of arbitrary + // solver name. Should validate but use + // disableRecursiveValidation() to avoid errors with arbitrary + // names. + + const bool doInitialization = false; + auto integrator = createIntegratorBasic(tempus_pl, physics, doInitialization); + + RCP noxList = parameterList("Correct NOX Params"); + *noxList = tempus_pl->sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true); + // noxList->print(std::cout); + integrator->getStepper()->getSolver()->setParameterList(noxList); + integrator->initialize(); + + // Setting observers on tempus breaks the screen output of the + // time steps! It replaces IntegratorObserverBasic which handles + // IO. Is this at least documented? + { + RCP> tempus_observers = rcp(new Tempus::IntegratorObserverComposite); + RCP tof = Teuchos::rcp(new user_app::TempusObserverFactory(stkIOResponseLibrary,rLibrary->getWorksetContainer())); + tempus_observers->addObserver(Teuchos::rcp(new Tempus::IntegratorObserverBasic)); + tempus_observers->addObserver(tof->buildTempusObserver(mesh,globalIndexer,linObjFactory)); + integrator->setObserver(tempus_observers); + } + + RCP> x0 = physics->getNominalValues().get_x()->clone_v(); + integrator->initializeSolutionHistory(0.0, x0); + + return {integrator,global_data,physics}; +} diff --git a/packages/panzer/adapters-stk/example/main_driver/user_app_ROLTempusReducedObjective.hpp b/packages/panzer/adapters-stk/example/main_driver/user_app_ROLTempusReducedObjective.hpp new file mode 100644 index 000000000000..8ca53b74f86e --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/user_app_ROLTempusReducedObjective.hpp @@ -0,0 +1,477 @@ +// @HEADER +// @HEADER + +#ifndef ROL_TEMPUS_REDUCED_OBJECTIVE_HPP +#define ROL_TEMPUS_REDUCED_OBJECTIVE_HPP + +#include + +#include "Teuchos_RCP.hpp" +#include "Teuchos_ParameterList.hpp" + +#include "Tempus_IntegratorBasic.hpp" +#include "Tempus_IntegratorForwardSensitivity.hpp" +#include "Tempus_IntegratorAdjointSensitivity.hpp" +#include "Tempus_IntegratorPseudoTransientForwardSensitivity.hpp" +#include "Tempus_IntegratorPseudoTransientAdjointSensitivity.hpp" + +#include "Thyra_ModelEvaluator.hpp" +#include "Thyra_DefaultNominalBoundsOverrideModelEvaluator.hpp" +#include "Thyra_VectorStdOps.hpp" + +#include "ROL_Objective.hpp" +#include "ROL_Vector.hpp" +#include "ROL_ThyraVector.hpp" + +namespace ROL { + +template +class TempusReducedObjective : public virtual ROL::Objective { +public: + + TempusReducedObjective( + const Teuchos::RCP >& model, + const Teuchos::RCP& tempus_params, + const Teuchos::RCP& objective_params); + + virtual ~TempusReducedObjective() {} + + //! Compute value of objective + Real value( const ROL::Vector &x, Real &tol ); + + //! Compute gradient of objective + void gradient( ROL::Vector &g, const ROL::Vector &x, Real &tol ); + + //! Set response target for computing objective + void set_target(const Teuchos::RCP >& target); + void set_target(const Teuchos::RCP >& target); + + //! Helper function to create optimization vector + Teuchos::RCP > create_design_vector() const; + + //! Helper function to create a response vector + Teuchos::RCP > create_response_vector() const; + + //! Helper function to run tempus, computing responses and derivatives + void run_tempus(ROL::Vector& r, const ROL::Vector& p); + void run_tempus(const Thyra::ModelEvaluatorBase::InArgs& inArgs, + const Thyra::ModelEvaluatorBase::OutArgs& outArgs); + +private: + + Teuchos::RCP > model_; + Teuchos::RCP tempus_params_; + Teuchos::RCP > target_; + std::string objective_type_; + std::string sensitivity_method_; + int param_index_; + int response_index_; + bool use_fd_gradient_; + +}; // class TempusReducedObjective + +template +TempusReducedObjective:: +TempusReducedObjective( + const Teuchos::RCP >& model, + const Teuchos::RCP& tempus_params, + const Teuchos::RCP& objective_params) : + model_(model), + tempus_params_(tempus_params), + objective_type_("Sum of Squares"), + sensitivity_method_("Forward"), + param_index_(0), + response_index_(0), + use_fd_gradient_(false) +{ + objective_type_ = + objective_params->get("Objective Type", objective_type_); + sensitivity_method_ = + objective_params->get("Sensitivity Method", sensitivity_method_); + param_index_ = + objective_params->get("Sensitivity Parameter Index", param_index_); + response_index_ = + objective_params->get("Response Function Index", response_index_); + use_fd_gradient_ = + objective_params->get("Use FD Gradient", use_fd_gradient_); +} + +template +Real +TempusReducedObjective:: +value( const ROL::Vector &p, Real &tol ) +{ + using Teuchos::RCP; + typedef Thyra::ModelEvaluatorBase MEB; + + // Run tempus and compute response for specified parameter values + MEB::InArgs inArgs = model_->getNominalValues(); + MEB::OutArgs outArgs = model_->createOutArgs(); + const ROL::ThyraVector& thyra_p = + Teuchos::dyn_cast >(p); + inArgs.set_p(param_index_, thyra_p.getVector()); + RCP > g = + Thyra::createMember(model_->get_g_space(response_index_)); + outArgs.set_g(response_index_, g); + run_tempus(inArgs, outArgs); + + // Compute objective + if (target_ != Teuchos::null) + Thyra::Vp_StV(g.ptr(), -1.0, *target_); // g <- g - target + Real val = 0.0; + if (objective_type_ == "Sum of Squares") { + Thyra::ele_wise_scale(*g, g.ptr()); // g <- g.*g + val = Thyra::sum(*g); // val = \sum_i g_i + } + else if (objective_type_ == "Sum") + val = Thyra::sum(*g); // val = \sum_i g_i + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Invalid objective type " << objective_type_ << "!" << std::endl + << "Valid choices are: \"Sum\" and \"Sum of Squares\"."); + + return val; +} + +template +void +TempusReducedObjective:: +gradient(ROL::Vector &grad, const ROL::Vector &p, Real &tol) +{ + if (use_fd_gradient_) { + ROL::Objective::gradient(grad, p, tol); + return; + } + + using Teuchos::RCP; + typedef Thyra::ModelEvaluatorBase MEB; + + // Run tempus and compute response gradient for specified parameter values + const int num_p = model_->get_p_space(param_index_)->dim(); + const int num_g = model_->get_g_space(response_index_)->dim(); + MEB::InArgs inArgs = model_->getNominalValues(); + MEB::OutArgs outArgs = model_->createOutArgs(); + const ROL::ThyraVector& thyra_p = + Teuchos::dyn_cast >(p); + inArgs.set_p(param_index_, thyra_p.getVector()); + RCP > g = + Thyra::createMember(model_->get_g_space(response_index_)); + RCP > dgdp; + MEB::EDerivativeMultiVectorOrientation dgdp_orientation = + MEB::DERIV_MV_JACOBIAN_FORM; + if (sensitivity_method_ == "Adjoint"|| + sensitivity_method_ == "Pseudotransient Adjoint") { + // Adjoint, PseudoTransientAdjoint integrators require gradient form + // (but does not necessarily require the model to support gradient form) + dgdp = + Thyra::createMembers(model_->get_p_space(param_index_), num_g); + dgdp_orientation = MEB::DERIV_MV_GRADIENT_FORM; + } + else { + // Form determined by what model supports + MEB::DerivativeSupport support = + outArgs.supports(MEB::OUT_ARG_DgDp, response_index_, param_index_); + if (support.supports(MEB::DERIV_MV_JACOBIAN_FORM)) { + dgdp = + Thyra::createMembers(model_->get_g_space(response_index_), num_p); + dgdp_orientation = MEB::DERIV_MV_JACOBIAN_FORM; + } + else if (support.supports(MEB::DERIV_MV_GRADIENT_FORM)) { + dgdp = + Thyra::createMembers(model_->get_p_space(param_index_), num_g); + dgdp_orientation = MEB::DERIV_MV_GRADIENT_FORM; + } + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Model must support Jacobian or gradient forms for dg/dp!"); + } + outArgs.set_DgDp(response_index_, param_index_, + MEB::DerivativeMultiVector(dgdp, dgdp_orientation)); + outArgs.set_g(response_index_, g); + run_tempus(inArgs, outArgs); + + // Compute objective gradient + RCP > final_grad = + Teuchos::dyn_cast >(grad).getVector()->col(0); + if (target_ != Teuchos::null) + Thyra::Vp_StV(g.ptr(), -1.0, *target_); // g <- g - target + if (dgdp_orientation == MEB::DERIV_MV_JACOBIAN_FORM) { + for (int j=0; j > dgdp_j = dgdp->col(j); + Real grad_val = 0.0; + if (objective_type_ == "Sum of Squares") { + Thyra::ele_wise_prod_update(2.0, *g, dgdp_j.ptr()); + grad_val = Thyra::sum(*dgdp_j); + } + else if (objective_type_ == "Sum") + grad_val = Thyra::sum(*dgdp_j); + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Invalid objective type " << objective_type_ << "!" << std::endl + << "Valid choices are: \"Sum\" and \"Sum of Squares\"."); + Thyra::set_ele(j, grad_val, final_grad.ptr()); + } + } + else { // gradient form + Thyra::assign(final_grad.ptr(), 0.0); + for (int i=0; icol(i))); + } + else if (objective_type_ == "Sum") { + Thyra::Vp_StV(final_grad.ptr(), 1.0, *(dgdp->col(i))); + } + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Invalid objective type " << objective_type_ << "!" << std::endl + << "Valid choices are: \"Sum\" and \"Sum of Squares\"."); + } + } +} + +template +void +TempusReducedObjective:: +set_target(const Teuchos::RCP >& target) { + target_ = target; +} + +template +void +TempusReducedObjective:: +set_target(const Teuchos::RCP >& target) { + using Teuchos::rcp_dynamic_cast; + target_ = + rcp_dynamic_cast >(target, true)->getVector(); +} + +template +Teuchos::RCP > +TempusReducedObjective:: +create_design_vector() const { + typedef Thyra::ModelEvaluatorBase MEB; + + Teuchos::RCP > p = + Thyra::createMember(model_->get_p_space(param_index_)); + MEB::InArgs nominalValues = model_->getNominalValues(); + if (nominalValues.get_p(param_index_) != Teuchos::null) + Thyra::assign(p.ptr(), *(nominalValues.get_p(param_index_))); + else + Thyra::assign(p.ptr(), Teuchos::ScalarTraits::zero()); + return Teuchos::rcp(new ROL::ThyraVector(p)); +} + +template +Teuchos::RCP > +TempusReducedObjective:: +create_response_vector() const { + Teuchos::RCP > g = + Thyra::createMember(model_->get_g_space(response_index_)); + Thyra::assign(g.ptr(), Teuchos::ScalarTraits::zero()); + return Teuchos::rcp(new ROL::ThyraVector(g)); +} + +template +void +TempusReducedObjective:: +run_tempus(ROL::Vector& r, const ROL::Vector& p) +{ + typedef Thyra::ModelEvaluatorBase MEB; + + MEB::InArgs inArgs = model_->getNominalValues(); + MEB::OutArgs outArgs = model_->createOutArgs(); + const ROL::ThyraVector& thyra_p = + Teuchos::dyn_cast >(p); + ROL::ThyraVector& thyra_r = + Teuchos::dyn_cast >(r); + inArgs.set_p(param_index_, thyra_p.getVector()); + outArgs.set_g(response_index_, thyra_r.getVector()); + run_tempus(inArgs, outArgs); +} + +template +void +TempusReducedObjective:: +run_tempus(const Thyra::ModelEvaluatorBase::InArgs& inArgs, + const Thyra::ModelEvaluatorBase::OutArgs& outArgs) +{ + using Teuchos::rcp; + using Teuchos::RCP; + using Teuchos::rcpFromRef; + typedef Thyra::ModelEvaluatorBase MEB; + typedef Thyra::DefaultNominalBoundsOverrideModelEvaluator DNBOME; + + // Override nominal values in model to supplied inArgs + RCP wrapped_model = rcp(new DNBOME(model_, rcpFromRef(inArgs))); + + Real t; + RCP > x, x_dot; + RCP > dxdp, dxdotdp; + RCP > g = outArgs.get_g(response_index_); + RCP > dgdp = + outArgs.get_DgDp(response_index_, param_index_).getMultiVector(); + MEB::EDerivativeMultiVectorOrientation dgdp_orientation = + outArgs.get_DgDp(response_index_, param_index_).getMultiVectorOrientation(); + + // Create and run integrator + if (dgdp != Teuchos::null && sensitivity_method_ == "Forward") { + RCP > integrator = + Tempus::createIntegratorForwardSensitivity(tempus_params_, wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + dxdp = integrator->getDxDp(); + dxdotdp = integrator->getDXDotDp(); + } + else if (dgdp != Teuchos::null && sensitivity_method_ == "Adjoint") { + RCP > integrator = + Tempus::createIntegratorAdjointSensitivity(tempus_params_, wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + Thyra::assign(dgdp.ptr(), *(integrator->getDgDp())); + } + else if (dgdp != Teuchos::null && + sensitivity_method_ == "Pseudotransient Forward") { + RCP > integrator = + Tempus::createIntegratorPseudoTransientForwardSensitivity(tempus_params_, + wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + dxdp = integrator->getDxDp(); + dxdotdp = integrator->getDXDotDp(); + } + else if (dgdp != Teuchos::null && + sensitivity_method_ == "Pseudotransient Adjoint") { + RCP > integrator = + Tempus::integratorPseudoTransientAdjointSensitivity(tempus_params_, + wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + Thyra::assign(dgdp.ptr(), *(integrator->getDgDp())); + } + else if (dgdp != Teuchos::null) { + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Invalid sensitivity method " << sensitivity_method_ << "!" << std::endl + << "Valid choices are: Forward, Adjoint, Pseudotransient Forward, " + << " and Pseudotransient Adjoint."); + } + else { + RCP > integrator = + Tempus::createIntegratorBasic(tempus_params_, wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + } + + // Evaluate response at final state + const int num_g = model_->get_g_space(response_index_)->dim(); + MEB::InArgs modelInArgs = inArgs; + MEB::OutArgs modelOutArgs = outArgs; + modelInArgs.set_x(x); + if (modelInArgs.supports(MEB::IN_ARG_x_dot)) modelInArgs.set_x_dot(x_dot); + if (modelInArgs.supports(MEB::IN_ARG_t)) modelInArgs.set_t(t); + RCP > dgdx, dgdxdot; + MEB::EDerivativeMultiVectorOrientation dgdx_orientation = + MEB::DERIV_MV_JACOBIAN_FORM; + MEB::EDerivativeMultiVectorOrientation dgdxdot_orientation = + MEB::DERIV_MV_JACOBIAN_FORM; + if (dgdp != Teuchos::null && + (sensitivity_method_ == "Forward" || + sensitivity_method_ == "Pseudotransient Forward")) { + MEB::DerivativeSupport dgdx_support = + outArgs.supports(MEB::OUT_ARG_DgDx, response_index_); + if (!dgdx_support.none()) { + if (dgdx_support.supports(MEB::DERIV_MV_GRADIENT_FORM)) { + dgdx = Thyra::createMembers(model_->get_x_space(), num_g); + dgdx_orientation = MEB::DERIV_MV_GRADIENT_FORM; + } + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Model must support gradient forms for dg/dx!"); + modelOutArgs.set_DgDx( + response_index_, + MEB::DerivativeMultiVector(dgdx, dgdx_orientation)); + } + + MEB::DerivativeSupport dgdxdot_support = + modelOutArgs.supports(MEB::OUT_ARG_DgDx_dot, response_index_); + if (!dgdxdot_support.none()) { + if (dgdxdot_support.supports(MEB::DERIV_MV_GRADIENT_FORM)) { + dgdxdot = Thyra::createMembers(model_->get_x_space(), num_g); + dgdxdot_orientation = MEB::DERIV_MV_GRADIENT_FORM; + } + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Model must support Jacobian or gradient forms for dg/dx!"); + modelOutArgs.set_DgDx_dot( + response_index_, + MEB::DerivativeMultiVector(dgdxdot, dgdxdot_orientation)); + } + } + else if (dgdp != Teuchos::null && + (sensitivity_method_ == "Adjoint" || + sensitivity_method_ == "Pseudotransient Adjoint")) { + // Clear dg/dp as an out arg since it was already computed by the adjoint + // integrator + modelOutArgs.set_DgDp(response_index_, param_index_, + MEB::Derivative()); + } + + model_->evalModel(modelInArgs, modelOutArgs); + + // dg/dp = dg/dp + dg/dx*dx/dp + dg/dx_dot*dx_dot/dp + // We assume dg/dx, dg/dxdot are in gradient form while dxdp, dxdotdp are in + // Jacobian form + if (dgdp != Teuchos::null && dgdx != Teuchos::null) { + if (dgdp_orientation == MEB::DERIV_MV_JACOBIAN_FORM) + dgdx->apply(Thyra::TRANS, *dxdp, dgdp.ptr(), Real(1.0), Real(1.0)); + else + dxdp->apply(Thyra::TRANS, *dgdx, dgdp.ptr(), Real(1.0), Real(1.0)); + } + if (dgdp != Teuchos::null && dgdxdot != Teuchos::null) { + if (dgdp_orientation == MEB::DERIV_MV_JACOBIAN_FORM) + dgdxdot->apply(Thyra::TRANS, *dxdotdp, dgdp.ptr(), Real(1.0), Real(1.0)); + else + dxdotdp->apply(Thyra::TRANS, *dgdxdot, dgdp.ptr(), Real(1.0), Real(1.0)); + } +} + +} // namespace ROL + +#endif // ROL_TEMPUS_REDUCED_OBJECTIVE_HPP From c1f6a42c42933dbbf41687fafd9faaf0a550ad7c Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Fri, 20 Sep 2024 15:34:28 -0600 Subject: [PATCH 16/40] Panzer: added ROL transient optimization problem Signed-off-by: Roger Pawlowski --- .../example/main_driver/CMakeLists.txt | 6 +- .../energy-transient-tempus-opt-blocked.xml | 204 +++++++- .../example/main_driver/main_driver_opt.cpp | 331 ++----------- .../example/main_driver/rol_params.xml | 109 +---- .../user_app_TempusObserver_WriteToExodus.hpp | 2 + .../user_app_TransientObjective.cpp | 5 + .../user_app_TransientObjective.hpp | 91 ++++ .../user_app_TransientObjective_impl.hpp | 455 ++++++++++++++++++ .../main_driver/user_app_Utilities.cpp | 309 ++++++++++++ .../main_driver/user_app_Utilities.hpp | 145 ++++++ .../src/Panzer_ModelEvaluator_impl.hpp | 10 +- .../src/evaluators/Panzer_Parameter_impl.hpp | 3 +- .../user_app_EquationSetFactory.hpp | 5 + 13 files changed, 1265 insertions(+), 410 deletions(-) create mode 100644 packages/panzer/adapters-stk/example/main_driver/user_app_TransientObjective.cpp create mode 100644 packages/panzer/adapters-stk/example/main_driver/user_app_TransientObjective.hpp create mode 100644 packages/panzer/adapters-stk/example/main_driver/user_app_TransientObjective_impl.hpp create mode 100644 packages/panzer/adapters-stk/example/main_driver/user_app_Utilities.cpp create mode 100644 packages/panzer/adapters-stk/example/main_driver/user_app_Utilities.hpp diff --git a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt index 5dff70456774..9de8071715b0 100644 --- a/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt +++ b/packages/panzer/adapters-stk/example/main_driver/CMakeLists.txt @@ -117,11 +117,11 @@ IF(${PACKAGE_NAME}_ENABLE_Teko) PASS_REGULAR_EXPRESSION "panzer::MainDriver run completed." ) - # Optimization with ROL - IF(${PACKAGE_NAME}_ENABLE_ROL) + # Optimization with ROL (currently doesn't support GPU devices) + IF(${PACKAGE_NAME}_ENABLE_ROL AND NOT Kokkos_ENABLE_CUDA AND NOT Kokkos_ENABLE_HIP) TRIBITS_ADD_EXECUTABLE( main_driver_opt - SOURCES main_driver_opt.cpp ${main_driver_SOURCES} user_app_ROLTempusReducedObjective.hpp + SOURCES main_driver_opt.cpp ${main_driver_SOURCES} user_app_ROLTempusReducedObjective.hpp user_app_TransientObjective.hpp user_app_TransientObjective_impl.hpp user_app_TransientObjective.cpp user_app_Utilities.hpp user_app_Utilities.cpp ) TRIBITS_ADD_ADVANCED_TEST( main_driver_energy-transient-tempus-opt-blocked diff --git a/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-opt-blocked.xml b/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-opt-blocked.xml index fc06b2ef718d..5e377070a278 100644 --- a/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-opt-blocked.xml +++ b/packages/panzer/adapters-stk/example/main_driver/energy-transient-tempus-opt-blocked.xml @@ -288,7 +288,7 @@ - + @@ -394,4 +394,206 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp b/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp index af09ef0223cf..c0fae5d462e6 100644 --- a/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp +++ b/packages/panzer/adapters-stk/example/main_driver/main_driver_opt.cpp @@ -19,7 +19,6 @@ #include "Teuchos_XMLParameterListHelpers.hpp" #include "Teuchos_YamlParameterListHelpers.hpp" #include "Teuchos_FancyOStream.hpp" -#include "Teuchos_oblackholestream.hpp" #include "Teuchos_Assert.hpp" #include "Teuchos_as.hpp" @@ -49,6 +48,7 @@ #include "user_app_ResponseEvaluatorFactory_HOFlux.hpp" #include "Panzer_ROLTempusReducedObjective.hpp" +#include "user_app_TransientObjective.hpp" #include "Panzer_ResponseEvaluatorFactory_Probe.hpp" @@ -58,38 +58,16 @@ #include +#include "user_app_Utilities.hpp" + #include #include #include -// ************************************************* -// applicaiton functions defined in this file -// ************************************************* -namespace user_app { - void addResponsesToModelEvaluatorFactory(const Teuchos::ParameterList& response_sublist, - panzer_stk::ModelEvaluatorFactory& me_factory); - - void computeAndPrintResponses(const Teuchos::RCP>& me, - const auto& x, - const auto& x_dot, - const auto& t, - const auto& global_data, - std::ostream& out); - - std::tuple>, - Teuchos::RCP, - Teuchos::RCP>> - buildTempusIntegrator(const Teuchos::RCP& input_params, - const Teuchos::RCP>& comm, - const bool overrideNoxOutput); -} - -// ************************************************* // ************************************************* // Main Driver for a transient optimization problem using ROL and // Tempus // ************************************************* -// ************************************************* int main(int argc, char *argv[]) { @@ -98,8 +76,8 @@ int main(int argc, char *argv[]) int status = 0; - std::ostream blackhole{nullptr}; - Teuchos::GlobalMPISession mpiSession(&argc, &argv, &blackhole); + std::ostream os_dev_null{nullptr}; + Teuchos::GlobalMPISession mpiSession(&argc, &argv, &os_dev_null); Kokkos::initialize(argc,argv); Teuchos::RCP out = Teuchos::rcp(new Teuchos::FancyOStream(Teuchos::rcp(&std::cout,false))); @@ -155,44 +133,43 @@ int main(int argc, char *argv[]) if (printInputPL) *out << *input_params << std::endl; + // Pull off objective and rol lists so the panzer validation does not fail + RCP objective_params = parameterList(input_params->sublist("Objective",true)); + RCP rol_params = parameterList(input_params->sublist("ROL", true)); + input_params->remove("Objective"); + input_params->remove("ROL"); - RCP params_copy = parameterList(*input_params); - auto [integrator,global_data,physics] = user_app::buildTempusIntegrator(params_copy,comm,overrideNoxOutput); - - bool integratorStatus = integrator->advanceTime(); - TEUCHOS_ASSERT(integratorStatus); - - auto x = integrator->getSolutionHistory()->getCurrentState()->getX(); - auto x_dot = integrator->getSolutionHistory()->getCurrentState()->getXDot(); - auto t = integrator->getSolutionHistory()->getCurrentState()->getTime(); - - user_app::computeAndPrintResponses(physics,x,x_dot,t,global_data,*out); - { - Teuchos::RCP rol_params = Teuchos::rcp(new Teuchos::ParameterList("ROL Parameters")); - { - const std::string rol_input_file = "rol_params.xml"; - Teuchos::updateParametersFromXmlFileAndBroadcast(rol_input_file, rol_params.ptr(), *comm); - - RCP pl = sublist(rol_params, "Tempus", true); - RCP objective_params = sublist(rol_params, "Objective", true); - ROL::Ptr > objective = - ROL::makePtr >(physics, pl, objective_params); - - // Create target -- do forward integration with perturbed parameter values - RCP > zs = objective->create_design_vector(); - zs->setScalar(1.5); - RCP > r = objective->create_response_vector(); - objective->run_tempus(*r, *zs); - objective->set_target(r); - - } - + RCP tempus_params = parameterList(input_params->sublist("Solution Control",true).sublist("Tempus",true)); + auto objective = ROL::makePtr>(input_params,comm,objective_params,out); + + // Create target -- do forward integration with perturbed parameter values + RCP> p = objective->create_design_vector(); + p->setScalar(2.0); + RCP> r = objective->create_response_vector(); + objective->run_tempus(*r, *p); + objective->set_target(r); + + p->setScalar(1.5); + ROL::OptimizationProblem problem(objective, p); + ROL::OptimizationSolver solver(problem, *rol_params); ROL::Ptr outStream = ROL::makePtrFromRef(std::cout); + if (comm->getRank() == 0) + solver.solve(*outStream); + else + solver.solve(os_dev_null); - + { + const ROL::ThyraVector& thyra_p = dyn_cast >(*p); + ROL::ThyraVector& thyra_r = dyn_cast >(*r); + *out << "Final Values: p = " << Thyra::get_ele(*(thyra_p.getVector()),0) + << ", g = " << Thyra::get_ele(*(thyra_r.getVector()),0) + << std::endl; + } } + // user_app::computeAndPrintResponses(physics,x,x_dot,t,global_data,*out); + stackedTimer->stopBaseTimer(); if (printTimers) { Teuchos::StackedTimer::OutputOptions options; @@ -218,239 +195,3 @@ int main(int argc, char *argv[]) return status; } - -// ************************************************* -// ************************************************* - -void user_app::addResponsesToModelEvaluatorFactory(const Teuchos::ParameterList& responses, - panzer_stk::ModelEvaluatorFactory& me_factory) -{ - for(Teuchos::ParameterList::ConstIterator itr=responses.begin();itr!=responses.end();++itr) { - const std::string name = responses.name(itr); - TEUCHOS_ASSERT(responses.entry(itr).isList()); - Teuchos::ParameterList & lst = Teuchos::getValue(responses.entry(itr)); - - if (lst.get("Type") == "Functional") { - - panzer::FunctionalResponse_Builder builder; - builder.comm = MPI_COMM_WORLD; - builder.cubatureDegree = 2; - builder.requiresCellIntegral = true; - builder.quadPointField = lst.get("Field Name"); - - std::vector eblocks; - panzer::StringTokenizer(eblocks,lst.get("Element Blocks"),",",true); - - std::vector wkst_descs; - for(std::size_t i=0;i("Type") == "Point Value") { - panzer::ProbeResponse_Builder builder; - builder.comm = MPI_COMM_WORLD; - builder.point = Teuchos::Array{0.5,0.5}; - builder.cubatureDegree = 2; - builder.fieldName = lst.get("Field Name"); - builder.applyDirichletToDerivative = false; - - std::vector eblocks; - panzer::StringTokenizer(eblocks,lst.get("Element Blocks"),",",true); - - std::vector descriptors; - for(std::size_t i=0;i("Type") << "\" is not supported!"); - } - } -} - -void user_app::computeAndPrintResponses(const Teuchos::RCP>& physics, - const auto& x, - const auto& x_dot, - const auto& t, - const auto& global_data, - std::ostream& os) -{ - if(physics->Ng()>0) { - - os << "Ng = " << physics->Ng() << std::endl; - os << "Np = " << physics->Np() << std::endl; - Thyra::ModelEvaluatorBase::InArgs respInArgs = physics->createInArgs(); - Thyra::ModelEvaluatorBase::OutArgs respOutArgs = physics->createOutArgs(); - - TEUCHOS_ASSERT(physics->Ng()==respOutArgs.Ng()); - - respInArgs.set_x(x); - respInArgs.set_x_dot(x_dot); - - for(int g=0;g > response = Thyra::createMember(*physics->get_g_space(g)); - respOutArgs.set_g(g,response); - - for (int p = 0; p < physics->Np(); ++p) { - bool derivative_supported = respOutArgs.supports(Thyra::ModelEvaluatorBase::OUT_ARG_DgDp,g,p).supports(Thyra::ModelEvaluatorBase::DERIV_MV_JACOBIAN_FORM); - os << "DgDp(" << g << "," << p << ") supports = " << derivative_supported << std::endl; - if (derivative_supported) { - auto dgdp = physics->create_DgDp_op(g,p); - respOutArgs.set_DgDp(g,p,Thyra::ModelEvaluatorBase::Derivative(dgdp)); - } - } - } - - physics->evalModel(respInArgs, respOutArgs); - - { - auto parameter_library = global_data->pl; - TEUCHOS_ASSERT(nonnull(parameter_library)); - for (auto i=parameter_library->begin();i != parameter_library->end(); ++i) { - os << "Sacado::ParameterLibrary: " << i->first << std::endl; - } - } - - { - auto nominalValues = physics->getNominalValues(); - for (int i=0; i < respInArgs.Np();++i) { - auto p = nominalValues.get_p(i); - auto p_names = physics->get_p_names(i); - os << "ModelEvaluator::NominalValues Parameter Value: \"" << (*p_names)[0] << "\" = " << Thyra::get_ele(*p,0) << std::endl; - } - } - - for (int i=0;i > response = respOutArgs.get_g(i); - TEUCHOS_ASSERT(response!=Teuchos::null); - os << "Response Value: \"" << physics->get_g_names(i)[0] << "\" = " << Thyra::get_ele(*response,0) << std::endl; - for (int j=0; j < respOutArgs.Np(); ++j) { - // os << " dg(" << i << ")/dp(" << j << ") supports(GRAD_FORM) = " << respOutArgs.supports(Thyra::ModelEvaluatorBase::OUT_ARG_DgDp,i,j).supports(Thyra::ModelEvaluatorBase::DERIV_MV_GRADIENT_FORM) << std::endl; - // os << " dg(" << i << ")/dp(" << j << ") supports(JAC_FORM) = " << respOutArgs.supports(Thyra::ModelEvaluatorBase::OUT_ARG_DgDp,i,j).supports(Thyra::ModelEvaluatorBase::DERIV_MV_JACOBIAN_FORM) << std::endl; - } - } - } -} - -std::tuple>, - Teuchos::RCP, - Teuchos::RCP>> -user_app::buildTempusIntegrator(const Teuchos::RCP& input_params, - const Teuchos::RCP>& comm, - const bool overrideNoxOutput) -{ - using namespace Teuchos; - using namespace Tempus; - - Teuchos::ParameterList solver_factories = input_params->sublist("Solver Factories"); - input_params->remove("Solver Factories"); - - // Add in the application specific equation set factory - Teuchos::RCP eqset_factory = Teuchos::rcp(new user_app::MyFactory); - - // Add in the application specific closure model factory - user_app::MyModelFactory_TemplateBuilder cm_builder; - panzer::ClosureModelFactory_TemplateManager cm_factory; - cm_factory.buildObjects(cm_builder); - - // Add in the application specific bc factory - user_app::BCFactory bc_factory; - - // Create the global data - Teuchos::RCP global_data = panzer::createGlobalData(); - input_params->sublist("User Data").set("Comm", comm); // Required for GlobalStatistics - - Teuchos::RCP > stkIOResponseLibrary - = Teuchos::rcp(new panzer::ResponseLibrary()); - - Teuchos::RCP> physics; - Teuchos::RCP> solver; - Teuchos::RCP> rLibrary; - std::vector> physicsBlocks; - Teuchos::RCP> linObjFactory; - Teuchos::RCP globalIndexer; - Teuchos::RCP mesh; - { - Teuchos::ParameterList responses = input_params->sublist("Responses"); - input_params->remove("Responses"); - - panzer_stk::ModelEvaluatorFactory me_factory; - me_factory.setParameterList(input_params); - auto strat_list = Teuchos::parameterList(); - *strat_list = input_params->sublist("Solution Control",true).sublist("Tempus",true).sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true).sublist("Direction",true).sublist("Newton",true).sublist("Stratimikos Linear Solver",true).sublist("Stratimikos",true); - me_factory.setStratimikosList(strat_list); - me_factory.buildObjects(comm,global_data,eqset_factory,bc_factory,cm_factory); - - user_app::addResponsesToModelEvaluatorFactory(responses,me_factory); - me_factory.buildResponses(cm_factory); - - physicsBlocks = me_factory.getPhysicsBlocks(); - physics = me_factory.getPhysicsModelEvaluator(); - rLibrary = me_factory.getResponseLibrary(); - linObjFactory = me_factory.getLinearObjFactory(); - globalIndexer = me_factory.getGlobalIndexer(); - mesh = me_factory.getMesh(); - } - - // setup outputs to mesh on the stkIOResponseLibrary - { - stkIOResponseLibrary->initialize(*rLibrary); - - Teuchos::ParameterList user_data(input_params->sublist("User Data")); - user_data.set("Workset Size",input_params->sublist("Assembly").get("Workset Size")); - - std::vector eBlocks; - mesh->getElementBlockNames(eBlocks); - - panzer_stk::RespFactorySolnWriter_Builder builder; - builder.mesh = mesh; - stkIOResponseLibrary->addResponse("Main Field Output",eBlocks,builder); - - stkIOResponseLibrary->buildResponseEvaluators(physicsBlocks, - cm_factory, - input_params->sublist("Closure Models"), - user_data); - } - - RCP tempus_pl = parameterList(std::string("Tempus")); - *tempus_pl = input_params->sublist("Solution Control").sublist("Tempus"); - - if (overrideNoxOutput) { - auto nox_utils = Teuchos::rcp(new NOX::Utils(NOX::Utils::Error,comm->getRank(),0,3)); - auto print_nonlinear = Teuchos::rcp(new NOX::ObserverPrint(nox_utils,10)); - tempus_pl->sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true).sublist("Solver Options",true).set>("Observer",print_nonlinear); - } - - // Tempus is overriding NOX parameters with its own defaults. Need - // to fix this. It also does not validate because of arbitrary - // solver name. Should validate but use - // disableRecursiveValidation() to avoid errors with arbitrary - // names. - - const bool doInitialization = false; - auto integrator = createIntegratorBasic(tempus_pl, physics, doInitialization); - - RCP noxList = parameterList("Correct NOX Params"); - *noxList = tempus_pl->sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true); - // noxList->print(std::cout); - integrator->getStepper()->getSolver()->setParameterList(noxList); - integrator->initialize(); - - // Setting observers on tempus breaks the screen output of the - // time steps! It replaces IntegratorObserverBasic which handles - // IO. Is this at least documented? - { - RCP> tempus_observers = rcp(new Tempus::IntegratorObserverComposite); - RCP tof = Teuchos::rcp(new user_app::TempusObserverFactory(stkIOResponseLibrary,rLibrary->getWorksetContainer())); - tempus_observers->addObserver(Teuchos::rcp(new Tempus::IntegratorObserverBasic)); - tempus_observers->addObserver(tof->buildTempusObserver(mesh,globalIndexer,linObjFactory)); - integrator->setObserver(tempus_observers); - } - - RCP> x0 = physics->getNominalValues().get_x()->clone_v(); - integrator->initializeSolutionHistory(0.0, x0); - - return {integrator,global_data,physics}; -} diff --git a/packages/panzer/adapters-stk/example/main_driver/rol_params.xml b/packages/panzer/adapters-stk/example/main_driver/rol_params.xml index 8777e117ca47..27ecf899e374 100644 --- a/packages/panzer/adapters-stk/example/main_driver/rol_params.xml +++ b/packages/panzer/adapters-stk/example/main_driver/rol_params.xml @@ -1,114 +1,9 @@ - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp b/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp index 04cec8bceb14..dc965c8dbf0e 100644 --- a/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp +++ b/packages/panzer/adapters-stk/example/main_driver/user_app_TempusObserver_WriteToExodus.hpp @@ -15,6 +15,8 @@ #include "Tempus_IntegratorObserver.hpp" #include "Teuchos_RCP.hpp" #include "Teuchos_Assert.hpp" +#include "Teuchos_TimeMonitor.hpp" +#include "Teuchos_StackedTimer.hpp" #include "Panzer_STK_Interface.hpp" #include "Panzer_GlobalIndexer.hpp" diff --git a/packages/panzer/adapters-stk/example/main_driver/user_app_TransientObjective.cpp b/packages/panzer/adapters-stk/example/main_driver/user_app_TransientObjective.cpp new file mode 100644 index 000000000000..56e1bbf8d563 --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/user_app_TransientObjective.cpp @@ -0,0 +1,5 @@ + +#include "user_app_TransientObjective.hpp" +#include "user_app_TransientObjective_impl.hpp" + +template class ROL::TransientReducedObjective; diff --git a/packages/panzer/adapters-stk/example/main_driver/user_app_TransientObjective.hpp b/packages/panzer/adapters-stk/example/main_driver/user_app_TransientObjective.hpp new file mode 100644 index 000000000000..65b041d76222 --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/user_app_TransientObjective.hpp @@ -0,0 +1,91 @@ +// @HEADER +// @HEADER + +#ifndef USER_APP_TRANSIENT_REDUCED_OBJECTIVE_HPP +#define USER_APP_TRANSIENT_REDUCED_OBJECTIVE_HPP + +#include + +#include "Teuchos_RCP.hpp" +#include "Teuchos_ParameterList.hpp" + +#include "Tempus_IntegratorBasic.hpp" +#include "Tempus_IntegratorForwardSensitivity.hpp" +#include "Tempus_IntegratorAdjointSensitivity.hpp" +#include "Tempus_IntegratorPseudoTransientForwardSensitivity.hpp" +#include "Tempus_IntegratorPseudoTransientAdjointSensitivity.hpp" + +#include "Thyra_ModelEvaluator.hpp" +#include "Thyra_DefaultNominalBoundsOverrideModelEvaluator.hpp" +#include "Thyra_VectorStdOps.hpp" + +#include "ROL_Objective.hpp" +#include "ROL_Vector.hpp" +#include "ROL_ThyraVector.hpp" + +#include "user_app_Utilities.hpp" + +namespace ROL { + +template +class TransientReducedObjective : public virtual ROL::Objective { +public: + + TransientReducedObjective( + const Teuchos::RCP& input_params, + const Teuchos::RCP>& comm, + const Teuchos::RCP& objective_params, + const Teuchos::RCP& os); + + virtual ~TransientReducedObjective() {} + + //! Compute value of objective + Real value( const ROL::Vector &x, Real &tol ); + + //! Compute gradient of objective + void gradient( ROL::Vector &g, const ROL::Vector &x, Real &tol ); + + //! Set response target for computing objective + void set_target(const Teuchos::RCP >& target); + void set_target(const Teuchos::RCP >& target); + + //! Helper function to create optimization vector + Teuchos::RCP > create_design_vector() const; + + //! Helper function to create a response vector + Teuchos::RCP > create_response_vector() const; + + //! Helper function to run tempus, computing responses and derivatives + void run_tempus(ROL::Vector& r, const ROL::Vector& p); + void run_tempus(const Thyra::ModelEvaluatorBase::InArgs& inArgs, + const Thyra::ModelEvaluatorBase::OutArgs& outArgs); + +private: + + // Teuchos::RCP > model_; + Teuchos::RCP tempus_params_; + Teuchos::RCP > target_; + std::string objective_type_; + std::string sensitivity_method_; + int param_index_; + int response_index_; + bool use_fd_gradient_; + + // Objects neeeded to rebuild the time integrator since it seems to + // not be able to reset itself + Teuchos::RCP input_params_; + Teuchos::RCP> comm_; + Teuchos::RCP> model_; + Teuchos::RCP global_data_; + Teuchos::RCP mesh_; + Teuchos::RCP> response_library_; + Teuchos::RCP> stk_io_response_library_; + Teuchos::RCP> lin_obj_factory_; + Teuchos::RCP global_indexer_; + Teuchos::RCP os_; + bool print_debug_; +}; + +} + +#endif diff --git a/packages/panzer/adapters-stk/example/main_driver/user_app_TransientObjective_impl.hpp b/packages/panzer/adapters-stk/example/main_driver/user_app_TransientObjective_impl.hpp new file mode 100644 index 000000000000..41e6053f922a --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/user_app_TransientObjective_impl.hpp @@ -0,0 +1,455 @@ +#include "user_app_Utilities.hpp" +#include "Teuchos_StandardParameterEntryValidators.hpp" + +namespace ROL { + +template +TransientReducedObjective:: +TransientReducedObjective(const Teuchos::RCP& input_params, + const Teuchos::RCP>& comm, + const Teuchos::RCP& objective_params, + const Teuchos::RCP& os) : + objective_type_("Sum of Squares"), + sensitivity_method_("Forward"), + param_index_(0), + response_index_(0), + use_fd_gradient_(false), + input_params_(input_params), + comm_(comm), + os_(os), + print_debug_(true) +{ + TEUCHOS_ASSERT(nonnull(input_params)); + TEUCHOS_ASSERT(nonnull(comm)); + TEUCHOS_ASSERT(nonnull(objective_params)); + + Teuchos::ParameterList valid_params; + valid_params.set("Objective Type", + "Sum of Squares", + "How to compute the objective function", + Teuchos::rcp(new Teuchos::StringValidator({"Sum of Squares","Sum"}))); + valid_params.set("Sensitivity Method", + "Forward", + "Algorithm to compute parameter sensitivities", + Teuchos::rcp(new Teuchos::StringValidator({"Forward", + "Adjoint", + "Pseudotransient Forward", + "Pseudotransient Adjoint"}))); + valid_params.set("Parameter Name","NOT SET BY USER"); + valid_params.set("Response Name","NOT SET BY USER"); + valid_params.set("Use FD Gradient",true); + + objective_params->validateParametersAndSetDefaults(valid_params); + + objective_type_ = objective_params->get("Objective Type"); + sensitivity_method_ = objective_params->get("Sensitivity Method"); + use_fd_gradient_ = objective_params->get("Use FD Gradient"); + + std::tie(model_,global_data_,mesh_,response_library_,stk_io_response_library_,lin_obj_factory_,global_indexer_) = + user_app::buildModelEvaluator(input_params_,comm_); + + param_index_ = std::get<0>(user_app::findParameterIndex(objective_params->get("Parameter Name"),*model_)); + response_index_ = std::get<0>(user_app::findResponseIndex(objective_params->get("Response Name"),*model_)); +} + +template +Real +TransientReducedObjective:: +value( const ROL::Vector &p, Real &tol ) +{ + using Teuchos::RCP; + typedef Thyra::ModelEvaluatorBase MEB; + + // Run tempus and compute response for specified parameter values + MEB::InArgs inArgs = model_->getNominalValues(); + MEB::OutArgs outArgs = model_->createOutArgs(); + const ROL::ThyraVector& thyra_p = + Teuchos::dyn_cast >(p); + inArgs.set_p(param_index_, thyra_p.getVector()); + RCP > g = + Thyra::createMember(model_->get_g_space(response_index_)); + outArgs.set_g(response_index_, g); + run_tempus(inArgs, outArgs); + + // Compute objective + if (target_ != Teuchos::null) + Thyra::Vp_StV(g.ptr(), -1.0, *target_); // g <- g - target + Real val = 0.0; + if (objective_type_ == "Sum of Squares") { + Thyra::ele_wise_scale(*g, g.ptr()); // g <- g.*g + val = Thyra::sum(*g); // val = \sum_i g_i + } + else if (objective_type_ == "Sum") + val = Thyra::sum(*g); // val = \sum_i g_i + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Invalid objective type " << objective_type_ << "!" << std::endl + << "Valid choices are: \"Sum\" and \"Sum of Squares\"."); + + if (print_debug_) { + // const ROL::ThyraVector& thyra_p = Teuchos::dyn_cast >(*p); + // ROL::ThyraVector& thyra_g = Teuchos::dyn_cast >(*g); + *os_ << "TransientReducedObjective::value(): p = " << Thyra::get_ele(*(thyra_p.getVector()),0) + << ", g = " << Thyra::get_ele(*g,0) + << ", objective_val=" << val + << std::endl; + } + + return val; +} + +template +void +TransientReducedObjective:: +gradient(ROL::Vector &grad, const ROL::Vector &p, Real &tol) +{ + if (use_fd_gradient_) { + ROL::Objective::gradient(grad, p, tol); + return; + } + + using Teuchos::RCP; + typedef Thyra::ModelEvaluatorBase MEB; + + // Run tempus and compute response gradient for specified parameter values + const int num_p = model_->get_p_space(param_index_)->dim(); + const int num_g = model_->get_g_space(response_index_)->dim(); + MEB::InArgs inArgs = model_->getNominalValues(); + MEB::OutArgs outArgs = model_->createOutArgs(); + const ROL::ThyraVector& thyra_p = + Teuchos::dyn_cast >(p); + inArgs.set_p(param_index_, thyra_p.getVector()); + RCP > g = + Thyra::createMember(model_->get_g_space(response_index_)); + RCP > dgdp; + MEB::EDerivativeMultiVectorOrientation dgdp_orientation = + MEB::DERIV_MV_JACOBIAN_FORM; + if (sensitivity_method_ == "Adjoint"|| + sensitivity_method_ == "Pseudotransient Adjoint") { + // Adjoint, PseudoTransientAdjoint integrators require gradient form + // (but does not necessarily require the model to support gradient form) + dgdp = + Thyra::createMembers(model_->get_p_space(param_index_), num_g); + dgdp_orientation = MEB::DERIV_MV_GRADIENT_FORM; + } + else { + // Form determined by what model supports + MEB::DerivativeSupport support = + outArgs.supports(MEB::OUT_ARG_DgDp, response_index_, param_index_); + if (support.supports(MEB::DERIV_MV_JACOBIAN_FORM)) { + dgdp = + Thyra::createMembers(model_->get_g_space(response_index_), num_p); + dgdp_orientation = MEB::DERIV_MV_JACOBIAN_FORM; + } + else if (support.supports(MEB::DERIV_MV_GRADIENT_FORM)) { + dgdp = + Thyra::createMembers(model_->get_p_space(param_index_), num_g); + dgdp_orientation = MEB::DERIV_MV_GRADIENT_FORM; + } + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Model must support Jacobian or gradient forms for dg/dp!"); + } + outArgs.set_DgDp(response_index_, param_index_, + MEB::DerivativeMultiVector(dgdp, dgdp_orientation)); + outArgs.set_g(response_index_, g); + run_tempus(inArgs, outArgs); + + // Compute objective gradient + RCP > final_grad = + Teuchos::dyn_cast >(grad).getVector()->col(0); + if (target_ != Teuchos::null) + Thyra::Vp_StV(g.ptr(), -1.0, *target_); // g <- g - target + if (dgdp_orientation == MEB::DERIV_MV_JACOBIAN_FORM) { + for (int j=0; j > dgdp_j = dgdp->col(j); + Real grad_val = 0.0; + if (objective_type_ == "Sum of Squares") { + Thyra::ele_wise_prod_update(2.0, *g, dgdp_j.ptr()); + grad_val = Thyra::sum(*dgdp_j); + } + else if (objective_type_ == "Sum") + grad_val = Thyra::sum(*dgdp_j); + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Invalid objective type " << objective_type_ << "!" << std::endl + << "Valid choices are: \"Sum\" and \"Sum of Squares\"."); + Thyra::set_ele(j, grad_val, final_grad.ptr()); + } + } + else { // gradient form + Thyra::assign(final_grad.ptr(), 0.0); + for (int i=0; icol(i))); + } + else if (objective_type_ == "Sum") { + Thyra::Vp_StV(final_grad.ptr(), 1.0, *(dgdp->col(i))); + } + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Invalid objective type " << objective_type_ << "!" << std::endl + << "Valid choices are: \"Sum\" and \"Sum of Squares\"."); + } + } +} + +template +void +TransientReducedObjective:: +set_target(const Teuchos::RCP >& target) { + target_ = target; +} + +template +void +TransientReducedObjective:: +set_target(const Teuchos::RCP >& target) { + using Teuchos::rcp_dynamic_cast; + target_ = + rcp_dynamic_cast >(target, true)->getVector(); +} + +template +Teuchos::RCP > +TransientReducedObjective:: +create_design_vector() const { + typedef Thyra::ModelEvaluatorBase MEB; + + Teuchos::RCP > p = + Thyra::createMember(model_->get_p_space(param_index_)); + MEB::InArgs nominalValues = model_->getNominalValues(); + if (nominalValues.get_p(param_index_) != Teuchos::null) + Thyra::assign(p.ptr(), *(nominalValues.get_p(param_index_))); + else + Thyra::assign(p.ptr(), Teuchos::ScalarTraits::zero()); + return Teuchos::rcp(new ROL::ThyraVector(p)); +} + +template +Teuchos::RCP > +TransientReducedObjective:: +create_response_vector() const { + Teuchos::RCP > g = + Thyra::createMember(model_->get_g_space(response_index_)); + Thyra::assign(g.ptr(), Teuchos::ScalarTraits::zero()); + return Teuchos::rcp(new ROL::ThyraVector(g)); +} + +template +void +TransientReducedObjective:: +run_tempus(ROL::Vector& r, const ROL::Vector& p) +{ + typedef Thyra::ModelEvaluatorBase MEB; + + MEB::InArgs inArgs = model_->getNominalValues(); + MEB::OutArgs outArgs = model_->createOutArgs(); + const ROL::ThyraVector& thyra_p = + Teuchos::dyn_cast >(p); + ROL::ThyraVector& thyra_r = + Teuchos::dyn_cast >(r); + inArgs.set_p(param_index_, thyra_p.getVector()); + outArgs.set_g(response_index_, thyra_r.getVector()); + run_tempus(inArgs, outArgs); +} + +template +void +TransientReducedObjective:: +run_tempus(const Thyra::ModelEvaluatorBase::InArgs& inArgs, + const Thyra::ModelEvaluatorBase::OutArgs& outArgs) +{ + using Teuchos::rcp; + using Teuchos::RCP; + using Teuchos::rcpFromRef; + typedef Thyra::ModelEvaluatorBase MEB; + typedef Thyra::DefaultNominalBoundsOverrideModelEvaluator DNBOME; + + // Override nominal values in model to supplied inArgs + RCP wrapped_model = rcp(new DNBOME(model_, rcpFromRef(inArgs))); + + Real t; + RCP > x, x_dot; + RCP > dxdp, dxdotdp; + RCP > g = outArgs.get_g(response_index_); + RCP > dgdp; + MEB::EDerivativeMultiVectorOrientation dgdp_orientation = Thyra::ModelEvaluatorBase::DERIV_MV_JACOBIAN_FORM; + bool dgdp_supported = outArgs.supports(Thyra::ModelEvaluatorBase::OUT_ARG_DgDp,response_index_,param_index_).supports(Thyra::ModelEvaluatorBase::DERIV_MV_JACOBIAN_FORM) || outArgs.supports(Thyra::ModelEvaluatorBase::OUT_ARG_DgDp,response_index_,param_index_).supports(Thyra::ModelEvaluatorBase::DERIV_MV_GRADIENT_FORM); + if (dgdp_supported) { + dgdp = outArgs.get_DgDp(response_index_, param_index_).getMultiVector(); + dgdp_orientation = outArgs.get_DgDp(response_index_, param_index_).getMultiVectorOrientation(); + } + + // Create and run integrator + if (dgdp != Teuchos::null && sensitivity_method_ == "Forward") { + RCP > integrator = + Tempus::createIntegratorForwardSensitivity(tempus_params_, wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + dxdp = integrator->getDxDp(); + dxdotdp = integrator->getDXDotDp(); + } + else if (dgdp != Teuchos::null && sensitivity_method_ == "Adjoint") { + RCP > integrator = + Tempus::createIntegratorAdjointSensitivity(tempus_params_, wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + Thyra::assign(dgdp.ptr(), *(integrator->getDgDp())); + } + else if (dgdp != Teuchos::null && + sensitivity_method_ == "Pseudotransient Forward") { + RCP > integrator = + Tempus::createIntegratorPseudoTransientForwardSensitivity(tempus_params_, + wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + dxdp = integrator->getDxDp(); + dxdotdp = integrator->getDXDotDp(); + } + else if (dgdp != Teuchos::null && + sensitivity_method_ == "Pseudotransient Adjoint") { + RCP > integrator = + Tempus::integratorPseudoTransientAdjointSensitivity(tempus_params_, + wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + Thyra::assign(dgdp.ptr(), *(integrator->getDgDp())); + } + else if (dgdp != Teuchos::null) { + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Invalid sensitivity method " << sensitivity_method_ << "!" << std::endl + << "Valid choices are: Forward, Adjoint, Pseudotransient Forward, " + << " and Pseudotransient Adjoint."); + } + else { + /* + RCP > integrator = + Tempus::createIntegratorBasic(tempus_params_, wrapped_model); + const bool integratorStatus = integrator->advanceTime(); + TEUCHOS_TEST_FOR_EXCEPTION( + !integratorStatus, std::logic_error, "Integrator failed!"); + */ + + auto input_params_copy = Teuchos::parameterList(*input_params_); + const bool override_nox_output = true; + auto integrator = user_app::buildTimeIntegrator(input_params_copy,comm_,wrapped_model,mesh_,response_library_, + stk_io_response_library_, + lin_obj_factory_,global_indexer_, + override_nox_output); + + bool integratorStatus = integrator->advanceTime(); + TEUCHOS_ASSERT(integratorStatus); + + // Get final state + t = integrator->getTime(); + x = integrator->getX(); + x_dot = integrator->getXDot(); + } + + // Evaluate response at final state + const int num_g = model_->get_g_space(response_index_)->dim(); + MEB::InArgs modelInArgs = inArgs; + MEB::OutArgs modelOutArgs = outArgs; + modelInArgs.set_x(x); + if (modelInArgs.supports(MEB::IN_ARG_x_dot)) modelInArgs.set_x_dot(x_dot); + if (modelInArgs.supports(MEB::IN_ARG_t)) modelInArgs.set_t(t); + RCP > dgdx, dgdxdot; + MEB::EDerivativeMultiVectorOrientation dgdx_orientation = + MEB::DERIV_MV_JACOBIAN_FORM; + MEB::EDerivativeMultiVectorOrientation dgdxdot_orientation = + MEB::DERIV_MV_JACOBIAN_FORM; + if (dgdp != Teuchos::null && + (sensitivity_method_ == "Forward" || + sensitivity_method_ == "Pseudotransient Forward")) { + MEB::DerivativeSupport dgdx_support = + outArgs.supports(MEB::OUT_ARG_DgDx, response_index_); + if (!dgdx_support.none()) { + if (dgdx_support.supports(MEB::DERIV_MV_GRADIENT_FORM)) { + dgdx = Thyra::createMembers(model_->get_x_space(), num_g); + dgdx_orientation = MEB::DERIV_MV_GRADIENT_FORM; + } + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Model must support gradient forms for dg/dx!"); + modelOutArgs.set_DgDx( + response_index_, + MEB::DerivativeMultiVector(dgdx, dgdx_orientation)); + } + + MEB::DerivativeSupport dgdxdot_support = + modelOutArgs.supports(MEB::OUT_ARG_DgDx_dot, response_index_); + if (!dgdxdot_support.none()) { + if (dgdxdot_support.supports(MEB::DERIV_MV_GRADIENT_FORM)) { + dgdxdot = Thyra::createMembers(model_->get_x_space(), num_g); + dgdxdot_orientation = MEB::DERIV_MV_GRADIENT_FORM; + } + else + TEUCHOS_TEST_FOR_EXCEPTION( + true, std::logic_error, + "Model must support Jacobian or gradient forms for dg/dx!"); + modelOutArgs.set_DgDx_dot( + response_index_, + MEB::DerivativeMultiVector(dgdxdot, dgdxdot_orientation)); + } + } + else if (dgdp != Teuchos::null && + (sensitivity_method_ == "Adjoint" || + sensitivity_method_ == "Pseudotransient Adjoint")) { + // Clear dg/dp as an out arg since it was already computed by the adjoint + // integrator + modelOutArgs.set_DgDp(response_index_, param_index_, + MEB::Derivative()); + } + + model_->evalModel(modelInArgs, modelOutArgs); + + // dg/dp = dg/dp + dg/dx*dx/dp + dg/dx_dot*dx_dot/dp + // We assume dg/dx, dg/dxdot are in gradient form while dxdp, dxdotdp are in + // Jacobian form + if (dgdp != Teuchos::null && dgdx != Teuchos::null) { + if (dgdp_orientation == MEB::DERIV_MV_JACOBIAN_FORM) + dgdx->apply(Thyra::TRANS, *dxdp, dgdp.ptr(), Real(1.0), Real(1.0)); + else + dxdp->apply(Thyra::TRANS, *dgdx, dgdp.ptr(), Real(1.0), Real(1.0)); + } + if (dgdp != Teuchos::null && dgdxdot != Teuchos::null) { + if (dgdp_orientation == MEB::DERIV_MV_JACOBIAN_FORM) + dgdxdot->apply(Thyra::TRANS, *dxdotdp, dgdp.ptr(), Real(1.0), Real(1.0)); + else + dxdotdp->apply(Thyra::TRANS, *dgdxdot, dgdp.ptr(), Real(1.0), Real(1.0)); + } +} + +} // namespace ROL diff --git a/packages/panzer/adapters-stk/example/main_driver/user_app_Utilities.cpp b/packages/panzer/adapters-stk/example/main_driver/user_app_Utilities.cpp new file mode 100644 index 000000000000..2d4dadc65a78 --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/user_app_Utilities.cpp @@ -0,0 +1,309 @@ +// @HEADER +// ***************************************************************************** +// Panzer: A partial differential equation assembly +// engine for strongly coupled complex multiphysics systems +// +// Copyright 2011 NTESS and the Panzer contributors. +// SPDX-License-Identifier: BSD-3-Clause +// ***************************************************************************** +// @HEADER + +#include "Teuchos_ConfigDefs.hpp" +#include "Teuchos_RCP.hpp" +#include "Teuchos_TimeMonitor.hpp" +#include "Teuchos_StackedTimer.hpp" +#include "Teuchos_DefaultComm.hpp" +#include "Teuchos_CommHelpers.hpp" +#include "Teuchos_GlobalMPISession.hpp" +#include "Teuchos_CommandLineProcessor.hpp" +#include "Teuchos_XMLParameterListHelpers.hpp" +#include "Teuchos_YamlParameterListHelpers.hpp" +#include "Teuchos_FancyOStream.hpp" +#include "Teuchos_oblackholestream.hpp" +#include "Teuchos_Assert.hpp" +#include "Teuchos_as.hpp" + +#include "Panzer_NodeType.hpp" +#include "PanzerAdaptersSTK_config.hpp" +#include "Panzer_STK_ModelEvaluatorFactory.hpp" +#include "Panzer_ClosureModel_Factory_TemplateManager.hpp" +#include "Panzer_String_Utilities.hpp" +#include "Panzer_ThyraObjContainer.hpp" +#include "Thyra_VectorSpaceBase.hpp" + +#include "NOX_Utils.H" +#include "NOX_Observer_Print.hpp" + +#include "Tempus_IntegratorBasic.hpp" +#include "Tempus_IntegratorObserverComposite.hpp" +#include "Tempus_IntegratorObserverBasic.hpp" +#include "Tempus_StepperFactory.hpp" + +#include "user_app_ClosureModel_Factory_TemplateBuilder.hpp" +#include "user_app_EquationSetFactory.hpp" +#include "user_app_BCStrategy_Factory.hpp" +#include "user_app_NOXObserverFactory.hpp" +#ifdef PANZER_HAVE_TEMPUS +#include "user_app_TempusObserverFactory.hpp" +#endif +#include "user_app_ResponseEvaluatorFactory_HOFlux.hpp" + +#include "Panzer_ROLTempusReducedObjective.hpp" +#include "user_app_TransientObjective.hpp" + +#include "Panzer_ResponseEvaluatorFactory_Probe.hpp" + +#include "ROL_OptimizationProblem.hpp" +#include "ROL_OptimizationSolver.hpp" +#include "ROL_RandomVector.hpp" + +#include + +#include +#include +#include + +#include "user_app_Utilities.hpp" + +void user_app::addResponsesToModelEvaluatorFactory(const Teuchos::ParameterList& responses, + panzer_stk::ModelEvaluatorFactory& me_factory) +{ + for(Teuchos::ParameterList::ConstIterator itr=responses.begin();itr!=responses.end();++itr) { + const std::string name = responses.name(itr); + TEUCHOS_ASSERT(responses.entry(itr).isList()); + Teuchos::ParameterList & lst = Teuchos::getValue(responses.entry(itr)); + + if (lst.get("Type") == "Functional") { + + panzer::FunctionalResponse_Builder builder; + builder.comm = MPI_COMM_WORLD; + builder.cubatureDegree = 2; + builder.requiresCellIntegral = true; + builder.quadPointField = lst.get("Field Name"); + + std::vector eblocks; + panzer::StringTokenizer(eblocks,lst.get("Element Blocks"),",",true); + + std::vector wkst_descs; + for(std::size_t i=0;i("Type") == "Point Value") { + panzer::ProbeResponse_Builder builder; + builder.comm = MPI_COMM_WORLD; + builder.point = Teuchos::Array{0.5,0.5}; + builder.cubatureDegree = 2; + builder.fieldName = lst.get("Field Name"); + builder.applyDirichletToDerivative = false; + + std::vector eblocks; + panzer::StringTokenizer(eblocks,lst.get("Element Blocks"),",",true); + + std::vector descriptors; + for(std::size_t i=0;i("Type") << "\" is not supported!"); + } + } +} + +std::tuple< Teuchos::RCP>, + Teuchos::RCP, + Teuchos::RCP, + Teuchos::RCP>, + Teuchos::RCP>, + Teuchos::RCP>, + Teuchos::RCP + > +user_app::buildModelEvaluator(const Teuchos::RCP& input_params, + const Teuchos::RCP>& comm) +{ + using namespace Teuchos; + using namespace Tempus; + + Teuchos::ParameterList solver_factories = input_params->sublist("Solver Factories"); + input_params->remove("Solver Factories"); + + // Add in the application specific equation set factory + Teuchos::RCP eqset_factory = Teuchos::rcp(new user_app::MyFactory); + + // Add in the application specific closure model factory + user_app::MyModelFactory_TemplateBuilder cm_builder; + panzer::ClosureModelFactory_TemplateManager cm_factory; + cm_factory.buildObjects(cm_builder); + + // Add in the application specific bc factory + user_app::BCFactory bc_factory; + + // Create the global data + Teuchos::RCP global_data = panzer::createGlobalData(); + input_params->sublist("User Data").set("Comm", comm); // Required for GlobalStatistics + + Teuchos::RCP > stkIOResponseLibrary + = Teuchos::rcp(new panzer::ResponseLibrary()); + + Teuchos::RCP> physics; + Teuchos::RCP> solver; + Teuchos::RCP> rLibrary; + std::vector> physicsBlocks; + Teuchos::RCP> linObjFactory; + Teuchos::RCP globalIndexer; + Teuchos::RCP mesh; + { + Teuchos::ParameterList responses = input_params->sublist("Responses"); + input_params->remove("Responses"); + + panzer_stk::ModelEvaluatorFactory me_factory; + me_factory.setParameterList(input_params); + auto strat_list = Teuchos::parameterList(); + *strat_list = input_params->sublist("Solution Control",true).sublist("Tempus",true).sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true).sublist("Direction",true).sublist("Newton",true).sublist("Stratimikos Linear Solver",true).sublist("Stratimikos",true); + me_factory.setStratimikosList(strat_list); + me_factory.buildObjects(comm,global_data,eqset_factory,bc_factory,cm_factory); + + user_app::addResponsesToModelEvaluatorFactory(responses,me_factory); + me_factory.buildResponses(cm_factory); + + physicsBlocks = me_factory.getPhysicsBlocks(); + physics = me_factory.getPhysicsModelEvaluator(); + rLibrary = me_factory.getResponseLibrary(); + linObjFactory = me_factory.getLinearObjFactory(); + globalIndexer = me_factory.getGlobalIndexer(); + mesh = me_factory.getMesh(); + } + + // setup outputs to mesh on the stkIOResponseLibrary + { + stkIOResponseLibrary->initialize(*rLibrary); + + Teuchos::ParameterList user_data(input_params->sublist("User Data")); + user_data.set("Workset Size",input_params->sublist("Assembly").get("Workset Size")); + + std::vector eBlocks; + mesh->getElementBlockNames(eBlocks); + + panzer_stk::RespFactorySolnWriter_Builder builder; + builder.mesh = mesh; + stkIOResponseLibrary->addResponse("Main Field Output",eBlocks,builder); + + stkIOResponseLibrary->buildResponseEvaluators(physicsBlocks, + cm_factory, + input_params->sublist("Closure Models"), + user_data); + } + + return {physics,global_data,mesh,rLibrary,stkIOResponseLibrary,linObjFactory,globalIndexer}; +} + +Teuchos::RCP> +user_app::buildTimeIntegrator(const Teuchos::RCP& input_params, + const Teuchos::RCP>& comm, + Teuchos::RCP> physics, + Teuchos::RCP mesh, + Teuchos::RCP> rLibrary, + Teuchos::RCP> stkIOResponseLibrary, + Teuchos::RCP> linObjFactory, + Teuchos::RCP globalIndexer, + const bool overrideNoxOutput) +{ + using namespace Teuchos; + using namespace Tempus; + + RCP tempus_pl = parameterList(input_params->sublist("Solution Control").sublist("Tempus")); + + if (overrideNoxOutput) { + auto nox_utils = Teuchos::rcp(new NOX::Utils(NOX::Utils::Error,comm->getRank(),0,3)); + auto print_nonlinear = Teuchos::rcp(new NOX::ObserverPrint(nox_utils,10)); + tempus_pl->sublist("My Example Stepper",true) + .sublist("My Example Solver",true) + .sublist("NOX",true) + .sublist("Solver Options",true) + .set>("Observer",print_nonlinear); + } + + // Tempus is overriding NOX parameters with its own defaults. Need + // to fix this. It also does not validate because of arbitrary + // solver name. Should validate but use + // disableRecursiveValidation() to avoid errors with arbitrary + // names. + + const bool doInitialization = false; + auto integrator = createIntegratorBasic(tempus_pl, physics, doInitialization); + + RCP noxList = parameterList("Correct NOX Params"); + *noxList = tempus_pl->sublist("My Example Stepper",true).sublist("My Example Solver",true).sublist("NOX",true); + // noxList->print(std::cout); + integrator->getStepper()->getSolver()->setParameterList(noxList); + integrator->initialize(); + + // Setting observers on tempus breaks the screen output of the + // time steps! It replaces IntegratorObserverBasic which handles + // IO. Is this at least documented? + { + RCP> tempus_observers = rcp(new Tempus::IntegratorObserverComposite); + RCP tof = + Teuchos::rcp(new user_app::TempusObserverFactory(stkIOResponseLibrary,rLibrary->getWorksetContainer())); + tempus_observers->addObserver(Teuchos::rcp(new Tempus::IntegratorObserverBasic)); + tempus_observers->addObserver(tof->buildTempusObserver(mesh,globalIndexer,linObjFactory)); + integrator->setObserver(tempus_observers); + } + + RCP> x0 = physics->getNominalValues().get_x()->clone_v(); + integrator->initializeSolutionHistory(0.0, x0); + + return integrator; +} + +std::tuple user_app::findParameterIndex(const std::string& p_name,const Thyra::ModelEvaluator& me) +{ + bool found = false; + int index = -1; + int sub_index = -1; + for (int p = 0; p < me.Np(); ++p) { + auto p_names = me.get_p_names(p); + for (Teuchos::Ordinal p_sub=0; p_sub < p_names->size(); ++p_sub) { + if ((*p_names)[p_sub] == p_name) { + found = true; + index = p; + sub_index = p_sub; + break; + } + } + if (found) break; + } + + TEUCHOS_TEST_FOR_EXCEPTION(!found,std::runtime_error, + "ERROR: the parameter \"" << p_name << "\" is not a valid parameter name in the model evaluator!"); + + return {index,sub_index}; +} + +std::tuple user_app::findResponseIndex(const std::string& g_name,const Thyra::ModelEvaluator& me) +{ + bool found = false; + int index = -1; + int sub_index = -1; + for (int g = 0; g < me.Ng(); ++g) { + auto g_names = me.get_g_names(g); + for (Teuchos::Ordinal g_sub=0; g_sub < g_names.size(); ++g_sub) { + std::cout << "g(" << g << "," << g_sub << ")=" << g_names[g_sub] << std::endl; + if (g_names[g_sub] == g_name) { + found = true; + index = g; + sub_index = g_sub; + break; + } + } + if (found) break; + } + + TEUCHOS_TEST_FOR_EXCEPTION(!found,std::runtime_error, + "ERROR: the response \"" << g_name << "\" is not a valid response name in the model evaluator!"); + + return {index,sub_index}; +} diff --git a/packages/panzer/adapters-stk/example/main_driver/user_app_Utilities.hpp b/packages/panzer/adapters-stk/example/main_driver/user_app_Utilities.hpp new file mode 100644 index 000000000000..14aeca4a2494 --- /dev/null +++ b/packages/panzer/adapters-stk/example/main_driver/user_app_Utilities.hpp @@ -0,0 +1,145 @@ +// @HEADER +// ***************************************************************************** +// Panzer: A partial differential equation assembly +// engine for strongly coupled complex multiphysics systems +// +// Copyright 2011 NTESS and the Panzer contributors. +// SPDX-License-Identifier: BSD-3-Clause +// ***************************************************************************** +// @HEADER + +#ifndef PANZER_ADAPTERS_STK_MAIN_DRIVER_USER_APP_UTILITIES_HPP +#define PANZER_ADAPTERS_STK_MAIN_DRIVER_USER_APP_UTILITIES_HPP + +#include "Teuchos_ConfigDefs.hpp" +#include "Teuchos_RCP.hpp" +#include "Teuchos_DefaultComm.hpp" +#include "Teuchos_CommHelpers.hpp" + +#include "Panzer_NodeType.hpp" +#include "PanzerAdaptersSTK_config.hpp" +#include "Panzer_STK_ModelEvaluatorFactory.hpp" +#include "Panzer_ClosureModel_Factory_TemplateManager.hpp" +#include "Panzer_String_Utilities.hpp" +#include "Panzer_ThyraObjContainer.hpp" +#include "Thyra_VectorSpaceBase.hpp" + +#include "NOX_Utils.H" +#include "NOX_Observer_Print.hpp" + +#include "Tempus_IntegratorBasic.hpp" + +#include "user_app_ClosureModel_Factory_TemplateBuilder.hpp" +#include "user_app_EquationSetFactory.hpp" +#include "user_app_BCStrategy_Factory.hpp" +#include "user_app_NOXObserverFactory.hpp" +#include "user_app_TempusObserverFactory.hpp" +#include "user_app_ResponseEvaluatorFactory_HOFlux.hpp" + +#include +#include + +namespace user_app { + + void addResponsesToModelEvaluatorFactory(const Teuchos::ParameterList& response_sublist, + panzer_stk::ModelEvaluatorFactory& me_factory); + + void computeAndPrintResponses(const Teuchos::RCP>& me, + const auto& x, + const auto& x_dot, + const auto& t, + const auto& global_data, + std::ostream& out); + + std::tuple< Teuchos::RCP>, + Teuchos::RCP, + Teuchos::RCP, + Teuchos::RCP>, // normal response lib + Teuchos::RCP>, // stk io response lib + Teuchos::RCP>, + Teuchos::RCP + > + buildModelEvaluator(const Teuchos::RCP& input_params, + const Teuchos::RCP>& comm); + + Teuchos::RCP> + buildTimeIntegrator(const Teuchos::RCP& input_params, + const Teuchos::RCP>& comm, + Teuchos::RCP> me, + Teuchos::RCP mesh, + Teuchos::RCP> rLibrary, + Teuchos::RCP> stkIOResponseLibrary, + Teuchos::RCP> linObjFactory, + Teuchos::RCP globalIndexer, + const bool overrideNoxOutput); + + std::tuple findParameterIndex(const std::string& p_name,const Thyra::ModelEvaluator& me); + + std::tuple findResponseIndex(const std::string& g_name,const Thyra::ModelEvaluator& me); +} + +void user_app::computeAndPrintResponses(const Teuchos::RCP>& physics, + const auto& x, + const auto& x_dot, + const auto& t, + const auto& global_data, + std::ostream& os) +{ + if(physics->Ng()>0) { + + os << "Ng = " << physics->Ng() << std::endl; + os << "Np = " << physics->Np() << std::endl; + Thyra::ModelEvaluatorBase::InArgs respInArgs = physics->createInArgs(); + Thyra::ModelEvaluatorBase::OutArgs respOutArgs = physics->createOutArgs(); + + TEUCHOS_ASSERT(physics->Ng()==respOutArgs.Ng()); + + respInArgs.set_x(x); + respInArgs.set_x_dot(x_dot); + + for(int g=0;g > response = Thyra::createMember(*physics->get_g_space(g)); + respOutArgs.set_g(g,response); + + for (int p = 0; p < physics->Np(); ++p) { + bool derivative_supported = respOutArgs.supports(Thyra::ModelEvaluatorBase::OUT_ARG_DgDp,g,p).supports(Thyra::ModelEvaluatorBase::DERIV_MV_JACOBIAN_FORM); + os << "DgDp(" << g << "," << p << ") supports = " << derivative_supported << std::endl; + if (derivative_supported) { + auto dgdp = physics->create_DgDp_op(g,p); + respOutArgs.set_DgDp(g,p,Thyra::ModelEvaluatorBase::Derivative(dgdp)); + } + } + } + + physics->evalModel(respInArgs, respOutArgs); + + { + auto parameter_library = global_data->pl; + TEUCHOS_ASSERT(nonnull(parameter_library)); + for (auto i=parameter_library->begin();i != parameter_library->end(); ++i) { + os << "Sacado::ParameterLibrary: " << i->first << std::endl; + } + } + + { + auto nominalValues = physics->getNominalValues(); + for (int i=0; i < respInArgs.Np();++i) { + auto p = nominalValues.get_p(i); + auto p_names = physics->get_p_names(i); + os << "ModelEvaluator::NominalValues Parameter Value: \"" << (*p_names)[0] << "\" = " << Thyra::get_ele(*p,0) << std::endl; + } + } + + for (int i=0;i > response = respOutArgs.get_g(i); + TEUCHOS_ASSERT(response!=Teuchos::null); + os << "Response Value: \"" << physics->get_g_names(i)[0] << "\" = " << Thyra::get_ele(*response,0) << std::endl; + for (int j=0; j < respOutArgs.Np(); ++j) { + // os << " dg(" << i << ")/dp(" << j << ") supports(GRAD_FORM) = " << respOutArgs.supports(Thyra::ModelEvaluatorBase::OUT_ARG_DgDp,i,j).supports(Thyra::ModelEvaluatorBase::DERIV_MV_GRADIENT_FORM) << std::endl; + // os << " dg(" << i << ")/dp(" << j << ") supports(JAC_FORM) = " << respOutArgs.supports(Thyra::ModelEvaluatorBase::OUT_ARG_DgDp,i,j).supports(Thyra::ModelEvaluatorBase::DERIV_MV_JACOBIAN_FORM) << std::endl; + } + } + } +} + +#endif diff --git a/packages/panzer/disc-fe/src/Panzer_ModelEvaluator_impl.hpp b/packages/panzer/disc-fe/src/Panzer_ModelEvaluator_impl.hpp index 1f3101881359..92d60c06fcc1 100644 --- a/packages/panzer/disc-fe/src/Panzer_ModelEvaluator_impl.hpp +++ b/packages/panzer/disc-fe/src/Panzer_ModelEvaluator_impl.hpp @@ -1647,6 +1647,7 @@ void panzer::ModelEvaluator:: evalModelImpl_basic_g(const Thyra::ModelEvaluatorBase::InArgs &inArgs, const Thyra::ModelEvaluatorBase::OutArgs &outArgs) const { + PANZER_FUNC_TIME_MONITOR("panzer::ModelEvaluator::evalModelImpl_basic_g()"); // optional sanity check // TEUCHOS_ASSERT(required_basic_g(outArgs)); @@ -1684,6 +1685,7 @@ panzer::ModelEvaluator:: evalModelImpl_basic_dgdx(const Thyra::ModelEvaluatorBase::InArgs &inArgs, const Thyra::ModelEvaluatorBase::OutArgs &outArgs) const { + PANZER_FUNC_TIME_MONITOR("panzer::ModelEvaluator::evalModelImpl_basic_dgdx()"); typedef Thyra::ModelEvaluatorBase MEB; // optional sanity check @@ -1730,6 +1732,7 @@ panzer::ModelEvaluator:: evalModelImpl_basic_dgdp_scalar(const Thyra::ModelEvaluatorBase::InArgs &inArgs, const Thyra::ModelEvaluatorBase::OutArgs &outArgs) const { + PANZER_FUNC_TIME_MONITOR("panzer::ModelEvaluator::evalModelImpl_basic_dgdp_scalar()"); using Teuchos::RCP; using Teuchos::rcp; using Teuchos::rcp_dynamic_cast; @@ -1818,6 +1821,7 @@ panzer::ModelEvaluator:: evalModelImpl_basic_dgdp_distro(const Thyra::ModelEvaluatorBase::InArgs &inArgs, const Thyra::ModelEvaluatorBase::OutArgs &outArgs) const { + PANZER_FUNC_TIME_MONITOR("panzer::ModelEvaluator::evalModelImpl_basic_dgdp_distro()"); typedef Thyra::ModelEvaluatorBase MEB; // optional sanity check @@ -1877,6 +1881,7 @@ panzer::ModelEvaluator:: evalModelImpl_basic_dfdp_scalar(const Thyra::ModelEvaluatorBase::InArgs &inArgs, const Thyra::ModelEvaluatorBase::OutArgs &outArgs) const { + PANZER_FUNC_TIME_MONITOR("panzer::ModelEvaluator::evalModelImpl_basic_dfdp_scalar()"); using Teuchos::RCP; using Teuchos::rcp_dynamic_cast; @@ -1982,7 +1987,7 @@ evalModelImpl_basic_dfdp_scalar(const Thyra::ModelEvaluatorBase::InArgs /////////////////////////////////////////////////////////////////////////////////////// if(totalParameterCount>0) { - PANZER_FUNC_TIME_MONITOR("panzer::ModelEvaluator::evalModel(df/dp)"); + PANZER_FUNC_TIME_MONITOR_DIFF("panzer::ModelEvaluator::evalModel(df/dp)",dfdp_eval); ae_tm_.getAsObject()->evaluate(ae_inargs); } } @@ -1993,7 +1998,7 @@ panzer::ModelEvaluator:: evalModelImpl_basic_dfdp_scalar_fd(const Thyra::ModelEvaluatorBase::InArgs &inArgs, const Thyra::ModelEvaluatorBase::OutArgs &outArgs) const { - PANZER_FUNC_TIME_MONITOR("panzer::ModelEvaluator::evalModel(df/dp)"); + PANZER_FUNC_TIME_MONITOR("panzer::ModelEvaluator::evalModelImpl_basic_dfdp_scalar_fd()"); using Teuchos::RCP; using Teuchos::rcp_dynamic_cast; @@ -2097,6 +2102,7 @@ panzer::ModelEvaluator:: evalModelImpl_basic_dfdp_distro(const Thyra::ModelEvaluatorBase::InArgs &inArgs, const Thyra::ModelEvaluatorBase::OutArgs &outArgs) const { + PANZER_FUNC_TIME_MONITOR("panzer::ModelEvaluator::evalModelImpl_basic_dfdp_distro()"); using Teuchos::RCP; using Teuchos::rcp_dynamic_cast; using Teuchos::null; diff --git a/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp b/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp index ce108d8ab80a..6c77f855f345 100644 --- a/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp +++ b/packages/panzer/disc-fe/src/evaluators/Panzer_Parameter_impl.hpp @@ -49,13 +49,12 @@ evaluateFields(typename TRAITS::EvalData workset) auto param_val = param->getValue(); auto target_field_v = target_field.get_static_view(); auto target_field_h = Kokkos::create_mirror_view(target_field_v); - + for (int cell=0; cell < workset.num_cells; ++cell) { for (std::size_t pt=0; pt Date: Mon, 6 Jan 2025 11:37:11 -0700 Subject: [PATCH 17/40] Add argument `skip-run-tests` to PR scripts Add argument `skip-run-tests` to PR script to allow finer control for building and/or running tests for resource control purposes. The argument will propogate through to SimpleTesting CMake scripts as a cache variable which can be utilized to further control the execution of the testing ctest-stage. Signed-off-by: Anderson --- .../framework/pr_tools/PullRequestLinuxDriverTest.py | 7 +++++++ .../trilinosprhelpers/TrilinosPRConfigurationBase.py | 9 +++++++++ .../trilinosprhelpers/TrilinosPRConfigurationStandard.py | 1 + .../unittests/test_TrilinosPRConfigurationStandard.py | 3 ++- 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/framework/pr_tools/PullRequestLinuxDriverTest.py b/packages/framework/pr_tools/PullRequestLinuxDriverTest.py index 54e0e8bd8731..ed22723b03a2 100755 --- a/packages/framework/pr_tools/PullRequestLinuxDriverTest.py +++ b/packages/framework/pr_tools/PullRequestLinuxDriverTest.py @@ -258,6 +258,12 @@ def parse_args(): default=False, help="Enable dry-run mode. Script will run but not execute the build steps. Default = %(default)s") + optional.add_argument("--skip-run-tests", + dest="skip_run_tests", + action="store_true", + default=False, + help="Skip running tests in SimpleTesting test stage. Default = %(default)s") + optional.add_argument("--extra-configure-args", dest="extra_configure_args", action="store", @@ -299,6 +305,7 @@ def parse_args(): print("| - [O] req-mem-per-core : {req_mem_per_core}".format(**vars(arguments))) print("| - [O] test-mode : {test_mode}".format(**vars(arguments))) print("| - [O] workspace-dir : {workspace_dir}".format(**vars(arguments))) + print("| - [O] skip-run-tests : {skip_run_tests}".format(**vars(arguments))) print("| - [O] extra_configure_args : {extra_configure_args}".format(**vars(arguments))) print("| - [O] dashboard_build_name : {dashboard_build_name}".format(**vars(arguments))) print("| - [O] use_explicit_cachefile : {use_explicit_cachefile}".format(**vars(arguments))) diff --git a/packages/framework/pr_tools/trilinosprhelpers/TrilinosPRConfigurationBase.py b/packages/framework/pr_tools/trilinosprhelpers/TrilinosPRConfigurationBase.py index 9587b1f5fa97..653edcdd9784 100644 --- a/packages/framework/pr_tools/trilinosprhelpers/TrilinosPRConfigurationBase.py +++ b/packages/framework/pr_tools/trilinosprhelpers/TrilinosPRConfigurationBase.py @@ -352,6 +352,15 @@ def arg_ccache_enable(self): """Is ccache enabled?""" return self.args.ccache_enable + @property + def arg_skip_run_tests(self): + """ + Control whether tests should run for this build. Used for cases + where resources are limited such that you choose to only compile tests + but skip running them. + """ + return self.args.skip_run_tests + # -------------------- # P R O P E R T I E S # -------------------- diff --git a/packages/framework/pr_tools/trilinosprhelpers/TrilinosPRConfigurationStandard.py b/packages/framework/pr_tools/trilinosprhelpers/TrilinosPRConfigurationStandard.py index d585aa7a0fb3..e5458dcc37de 100644 --- a/packages/framework/pr_tools/trilinosprhelpers/TrilinosPRConfigurationStandard.py +++ b/packages/framework/pr_tools/trilinosprhelpers/TrilinosPRConfigurationStandard.py @@ -87,6 +87,7 @@ def execute_test(self): f"-Dsubprojects_file:FILEPATH={self.arg_filename_subprojects}", f"-DCTEST_DROP_SITE:STRING={self.arg_ctest_drop_site}", "-DUSE_EXPLICIT_TRILINOS_CACHEFILE:BOOL=" + ("ON" if self.arg_use_explicit_cachefile else "OFF"), + "-DSKIP_RUN_TESTS:BOOL=" + ("ON" if self.arg_skip_run_tests else "OFF"), ] if self.arg_extra_configure_args: diff --git a/packages/framework/pr_tools/trilinosprhelpers/unittests/test_TrilinosPRConfigurationStandard.py b/packages/framework/pr_tools/trilinosprhelpers/unittests/test_TrilinosPRConfigurationStandard.py index 47586711a32d..6f6cbb952571 100755 --- a/packages/framework/pr_tools/trilinosprhelpers/unittests/test_TrilinosPRConfigurationStandard.py +++ b/packages/framework/pr_tools/trilinosprhelpers/unittests/test_TrilinosPRConfigurationStandard.py @@ -175,7 +175,8 @@ def dummy_args(self): ccache_enable=False, dry_run = False, use_explicit_cachefile = False, - extra_configure_args = "" + extra_configure_args = "", + skip_run_tests = False ) return output From a5ecfafb93df1a665b8d7d198c7d5c348b288982 Mon Sep 17 00:00:00 2001 From: Anderson Date: Mon, 6 Jan 2025 16:09:16 -0700 Subject: [PATCH 18/40] Update RHEL8 CUDA UVM config to enable tests Enable tests for RHEL8 CUDA UVM config to allow for finer control at the PR scripting level. In order to manage resources for GPU nodes, PR scripts will have ability to dictate if tests should build AND run. Story ref: TRILFRAME-675 Signed-off-by: Anderson --- packages/framework/ini-files/config-specs.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/framework/ini-files/config-specs.ini b/packages/framework/ini-files/config-specs.ini index e872329eac1e..4fb31fa85fcd 100644 --- a/packages/framework/ini-files/config-specs.ini +++ b/packages/framework/ini-files/config-specs.ini @@ -1517,7 +1517,9 @@ use PACKAGE-ENABLES|NO-EPETRA use COMMON_SPACK_TPLS use SEMS_CUDA -opt-set-cmake-var Trilinos_ENABLE_TESTS BOOL FORCE : OFF +# If run through PR testing, PR scripts will control whether tests +# should build and run, or ONLY build (TRILFRAME-675) +opt-set-cmake-var Trilinos_ENABLE_TESTS BOOL FORCE : ON [rhel8_sems-cuda-11.4.2-gnu-10.1.0-openmpi-4.1.6_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_uvm_deprecated-on_all] use rhel8_sems-cuda-11.4.2-gnu-10.1.0-openmpi-4.1.6_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_uvm_deprecated-on_no-package-enables From 96a38269d6ba59ff9502147a759e6bfb29c8629e Mon Sep 17 00:00:00 2001 From: Anderson Date: Mon, 6 Jan 2025 16:12:37 -0700 Subject: [PATCH 19/40] Fix RHEL8 CUDA UVM config inheritance and names Fix RHEL8 CUDA UVM config inheritance and naming for "all" and "all-no-epetra" versions. Signed-off-by: Anderson --- packages/framework/ini-files/config-specs.ini | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/framework/ini-files/config-specs.ini b/packages/framework/ini-files/config-specs.ini index 4fb31fa85fcd..e41812e05117 100644 --- a/packages/framework/ini-files/config-specs.ini +++ b/packages/framework/ini-files/config-specs.ini @@ -1523,12 +1523,11 @@ opt-set-cmake-var Trilinos_ENABLE_TESTS BOOL FORCE : ON [rhel8_sems-cuda-11.4.2-gnu-10.1.0-openmpi-4.1.6_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_uvm_deprecated-on_all] use rhel8_sems-cuda-11.4.2-gnu-10.1.0-openmpi-4.1.6_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_uvm_deprecated-on_no-package-enables -use PACKAGE-ENABLES|ALL-NO-EPETRA +use PACKAGE-ENABLES|ALL [rhel8_sems-cuda-11.4.2-gnu-10.1.0-openmpi-4.1.6_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_uvm_deprecated-on_all-no-epetra] -use rhel8_sems-cuda-11.4.2-gnu-10.1.0-openmpi-4.1.6_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_uvm_deprecated-on_all - -opt-set-cmake-var Trilinos_ENABLE_TESTS BOOL FORCE : ON +use rhel8_sems-cuda-11.4.2-gnu-10.1.0-openmpi-4.1.6_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_uvm_deprecated-on_no-package-enables +use PACKAGE-ENABLES|ALL-NO-EPETRA [rhel8_sems-cuda-11.4.2-gnu-10.1.0-openmpi-4.1.6_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_no-uvm_deprecated-on_no-package-enables] # uses sems-v2 modules From ae2a931288ff7a9eaf81ec5e41fef639391dbf29 Mon Sep 17 00:00:00 2001 From: Anderson Date: Tue, 7 Jan 2025 17:38:19 -0700 Subject: [PATCH 20/40] Add ctest-test-stage logic to skip running tests If -DSKIP_RUN_TESTS=ON is passed to the SimpleTesting cmake scripts, added logic to skip the ctest test stage and return success. Signed-off-by: Anderson --- cmake/SimpleTesting/cmake/ctest-common.cmake | 3 ++ .../cmake/ctest-stage-test.cmake | 30 ++++++++++++------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/cmake/SimpleTesting/cmake/ctest-common.cmake b/cmake/SimpleTesting/cmake/ctest-common.cmake index 812e9ebfb3b9..d19eaabb1fcf 100644 --- a/cmake/SimpleTesting/cmake/ctest-common.cmake +++ b/cmake/SimpleTesting/cmake/ctest-common.cmake @@ -76,6 +76,9 @@ if( NOT DEFINED skip_upload_config_files ) set( skip_upload_config_files OFF ) endif() +if( NOT DEFINED skip_run_tests ) + set (skip_run_tests OFF) +endif() # ----------------------------------------------------------- # -- Miscellaneous Settings diff --git a/cmake/SimpleTesting/cmake/ctest-stage-test.cmake b/cmake/SimpleTesting/cmake/ctest-stage-test.cmake index 76ec3a0656f3..e4d66bed850c 100644 --- a/cmake/SimpleTesting/cmake/ctest-stage-test.cmake +++ b/cmake/SimpleTesting/cmake/ctest-stage-test.cmake @@ -13,20 +13,28 @@ banner("START test step") set(STAGE_TEST_ERROR OFF) -if(CTEST_BUILD_NAME MATCHES .*_asan_.*) - set(CTEST_MEMORYCHECK_TYPE "AddressSanitizer") - set(ENV{LSAN_OPTIONS} "suppressions=${CTEST_SOURCE_DIRECTORY}/packages/framework/asan_assets/lsan.supp") - set(ENV{LD_PRELOAD} ${CTEST_SOURCE_DIRECTORY}/packages/framework/asan_assets/dummy_dlclose.so) - ctest_memcheck(PARALLEL_LEVEL ${TEST_PARALLEL_LEVEL} +message(STATUS "=== skip_run_tests is ${skip_run_tests}") +if(NOT skip_run_tests) + if(CTEST_BUILD_NAME MATCHES .*_asan_.*) + set(CTEST_MEMORYCHECK_TYPE "AddressSanitizer") + set(ENV{LSAN_OPTIONS} "suppressions=${CTEST_SOURCE_DIRECTORY}/packages/framework/asan_assets/lsan.supp") + set(ENV{LD_PRELOAD} ${CTEST_SOURCE_DIRECTORY}/packages/framework/asan_assets/dummy_dlclose.so) + ctest_memcheck(PARALLEL_LEVEL ${TEST_PARALLEL_LEVEL} + CAPTURE_CMAKE_ERROR captured_cmake_error + RETURN_VALUE test_error) + unset(ENV{LD_PRELOAD}) + submit_by_parts( "MemCheck" ) + else() + ctest_test(PARALLEL_LEVEL ${TEST_PARALLEL_LEVEL} CAPTURE_CMAKE_ERROR captured_cmake_error RETURN_VALUE test_error) - unset(ENV{LD_PRELOAD}) - submit_by_parts( "MemCheck" ) + submit_by_parts( "Test" ) + endif() else() - ctest_test(PARALLEL_LEVEL ${TEST_PARALLEL_LEVEL} - CAPTURE_CMAKE_ERROR captured_cmake_error - RETURN_VALUE test_error) - submit_by_parts( "Test" ) + message(">>> SKIPPED RUNNING TESTS (skip_run_tests=${skip_run_tests})") + set(test_error 0) + submit_by_parts("Test") + endif() # Print out final stage banner From bf488061900fa9f827659c188cdf020b6d4cc222 Mon Sep 17 00:00:00 2001 From: Anderson Date: Wed, 8 Jan 2025 17:30:46 -0700 Subject: [PATCH 21/40] Fix case sensitive instances of names in CMake scripts Signed-off-by: Anderson --- cmake/SimpleTesting/cmake/ctest-common.cmake | 4 ++-- cmake/SimpleTesting/cmake/ctest-stage-test.cmake | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/SimpleTesting/cmake/ctest-common.cmake b/cmake/SimpleTesting/cmake/ctest-common.cmake index d19eaabb1fcf..6663c00e4dbd 100644 --- a/cmake/SimpleTesting/cmake/ctest-common.cmake +++ b/cmake/SimpleTesting/cmake/ctest-common.cmake @@ -76,8 +76,8 @@ if( NOT DEFINED skip_upload_config_files ) set( skip_upload_config_files OFF ) endif() -if( NOT DEFINED skip_run_tests ) - set (skip_run_tests OFF) +if( NOT DEFINED SKIP_RUN_TESTS ) + set (SKIP_RUN_TESTS OFF) endif() # ----------------------------------------------------------- diff --git a/cmake/SimpleTesting/cmake/ctest-stage-test.cmake b/cmake/SimpleTesting/cmake/ctest-stage-test.cmake index e4d66bed850c..e98cb78980db 100644 --- a/cmake/SimpleTesting/cmake/ctest-stage-test.cmake +++ b/cmake/SimpleTesting/cmake/ctest-stage-test.cmake @@ -13,8 +13,8 @@ banner("START test step") set(STAGE_TEST_ERROR OFF) -message(STATUS "=== skip_run_tests is ${skip_run_tests}") -if(NOT skip_run_tests) +message(STATUS "=== SKIP_RUN_TESTS is ${SKIP_RUN_TESTS}") +if(NOT SKIP_RUN_TESTS) if(CTEST_BUILD_NAME MATCHES .*_asan_.*) set(CTEST_MEMORYCHECK_TYPE "AddressSanitizer") set(ENV{LSAN_OPTIONS} "suppressions=${CTEST_SOURCE_DIRECTORY}/packages/framework/asan_assets/lsan.supp") @@ -31,7 +31,7 @@ if(NOT skip_run_tests) submit_by_parts( "Test" ) endif() else() - message(">>> SKIPPED RUNNING TESTS (skip_run_tests=${skip_run_tests})") + message(">>> SKIPPED RUNNING TESTS (skip_run_tests=${SKIP_RUN_TESTS})") set(test_error 0) submit_by_parts("Test") From 5e490f11b7c2b46e41f4375560d5d9a0d34dcece Mon Sep 17 00:00:00 2001 From: Anderson Date: Fri, 10 Jan 2025 15:17:21 -0700 Subject: [PATCH 22/40] Clean up print statements Signed-off-by: Anderson --- cmake/SimpleTesting/cmake/ctest-stage-test.cmake | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmake/SimpleTesting/cmake/ctest-stage-test.cmake b/cmake/SimpleTesting/cmake/ctest-stage-test.cmake index e98cb78980db..2a22ed2722a2 100644 --- a/cmake/SimpleTesting/cmake/ctest-stage-test.cmake +++ b/cmake/SimpleTesting/cmake/ctest-stage-test.cmake @@ -13,7 +13,6 @@ banner("START test step") set(STAGE_TEST_ERROR OFF) -message(STATUS "=== SKIP_RUN_TESTS is ${SKIP_RUN_TESTS}") if(NOT SKIP_RUN_TESTS) if(CTEST_BUILD_NAME MATCHES .*_asan_.*) set(CTEST_MEMORYCHECK_TYPE "AddressSanitizer") @@ -31,10 +30,9 @@ if(NOT SKIP_RUN_TESTS) submit_by_parts( "Test" ) endif() else() - message(">>> SKIPPED RUNNING TESTS (skip_run_tests=${SKIP_RUN_TESTS})") + message(">>> SKIPPED RUNNING TESTS (SKIP_RUN_TESTS=${SKIP_RUN_TESTS})") set(test_error 0) submit_by_parts("Test") - endif() # Print out final stage banner From 5fc81c1bf41bd8635e714ba59d7742214d551c05 Mon Sep 17 00:00:00 2001 From: Anderson Date: Fri, 10 Jan 2025 15:18:23 -0700 Subject: [PATCH 23/40] Add `SKIP_RUN_TESTS` option to SimpleTesting docs Signed-off-by: Anderson --- cmake/SimpleTesting/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/SimpleTesting/README.md b/cmake/SimpleTesting/README.md index 948ce7ab403f..b63c95bb96db 100644 --- a/cmake/SimpleTesting/README.md +++ b/cmake/SimpleTesting/README.md @@ -92,6 +92,8 @@ the [`ctest-driver.cmake`](ctest-driver.cmake) file in its place. | `build_dir` | STRING | NO | `${build_root}/${CTEST_BUILD_NAME}` | Path to the build directory. | | `PARALLEL_LEVEL` | STRING | NO | `` | | | `TEST_PARALLEL_LEVEL` | STRING | NO | `${PARALLEL_LEVEL}` | | +| `SKIP_RUN_TESTS` | BOOL | NO | OFF | Skip running any tests (any tests enabled will still compile) | + 1. It might worthwhile to remove `build_root` since it's only used to create `build_dir` IF `build_dir` is not passed in via a `-D` option. From 404f306814a80174e24c3ab769f81433134e2308 Mon Sep 17 00:00:00 2001 From: Anderson Date: Fri, 10 Jan 2025 15:19:03 -0700 Subject: [PATCH 24/40] Removed out of date SimpleTesting documentation Removed references to old ctest driver frameworks as its been long enough for any of what was written to be relevent. Additionally, a lot of the referenced files/frameworks no longer exists. Signed-off-by: Anderson --- cmake/SimpleTesting/README.md | 59 ++--------------------------------- 1 file changed, 3 insertions(+), 56 deletions(-) diff --git a/cmake/SimpleTesting/README.md b/cmake/SimpleTesting/README.md index b63c95bb96db..0426aaebfaa4 100644 --- a/cmake/SimpleTesting/README.md +++ b/cmake/SimpleTesting/README.md @@ -1,20 +1,3 @@ -CMake File Structure -==================== -This CMake structure attempts to capture the structure of the original -`TFW_testing_single_configure_prototype` that is used by the current / old Trilinos -Pull-Request (PR) framework. - -**Note**: _Trilinos uses the **`pull_request_changes`** branch from this repository_. - -Specifically, this new configuration will replace the existing `simple_testing.cmake` -script. - -This diagram describes the current structure of the CMake files and how they -interact with each other. -
- CMake Structure Diagram -
- CMake Files ----------- @@ -32,42 +15,6 @@ CMake Files The _guarded_ files use the CMake command [`include_guard()`][1] which should prevent that file from being include more than once in an include chain. -Options and Variables (`simple_testing.cmake`) ----------------------------------------------- -The `simple_testing.cmake` file has a number of optional parameters that can be sent into the -CMake system via `-D:BOOL=` parameters: - -| Option | Required? | Default | PR Override | Purpose | -|----------------------------|:---------:|---------------------------------------------|-------------|----------------------------------| -| `build_name` | YES | N/A | YES | Sets `CTEST_BUILD_NAME` | -| `ctest_submit_retry_count` | NO | 5 | | | -| `ctest_submit_retry_delay` | NO | 3 | | | -| `dashboard_model` | NO | `Experimental` | YES but NO | Set to the same value as default | -| `dashboard_track` | NO | `Experimental` | YES but NO | Set to the same value as default | -| `skip_by_parts_submit` | NO | `ON` | YES | | -| `skip_clean_build_dir` | NO | `ON` | | | -| `skip_single_submit` | NO | `ON` | | | -| `skip_update_step` | NO | `OFF` | YES | | -| `skip_upload_config_files` | NO | `OFF` | | | -| `PARALLEL_LEVEL` | NO | _num cores_ | YES | | -| `TEST_PARALLEL_LEVEL` | NO | `${PARALLEL_LEVEL}` | YES | | -| `build_root` | NO | `${CTEST_SOURCE_DIRECTORY}/nightly_testing` | | | -| `build_dir` | NO | `${build_root}/${CTEST_BUILD_NAME}` | YES | The CMake build dir | -| `configure_script` | YES | N/A | YES | See note below table | -| `package_enables` | YES | N/A | YES | `packageEnables.cmake` | -| `subprojects_file` | YES | N/A | YES | `package_subproject_list.cmake` | - -1. `configure_script` points to the `cmake/std/PullRequestLinuxTestingSettings.cmake` file. - - Example: `${WORKSPACE}/Trilinos/cmake/std/PullRequestLinuxGCC8.3.0TestingSettings.cmake` - -See `TrilinosPRConfigurationStandard.py`[2] for information on what options are set to something -other than the default during normal Trilinos PR operations. - -Expected Operation ------------------- -The expected operation of this set of files to replace the old `simple_testing.cmake` is to load -the [`ctest-driver.cmake`](ctest-driver.cmake) file in its place. - `ctest-driver.cmake` Options ---------------------------- @@ -103,11 +50,11 @@ the [`ctest-driver.cmake`](ctest-driver.cmake) file in its place. Example CTest call from a Trilinos PR ------------------------------------- -This is an example, for reference, of how the `ctest` command is invoked in the current/old Trilinos +This is an example, for reference, of how the `ctest` command is invoked in the current Trilinos PR test driver. ```bash ctest \ - -S simple_testing.cmake \ + -S ctest-driver.cmake \ -Dsource_dir=${WORKSPACE}/Trilinos \ -Dbuild_name=PR-9495-test-Trilinos_pullrequest_gcc_8.3.0-5164 \ -Dskip_by_parts_submit=OFF \ @@ -138,4 +85,4 @@ when only documentation or perhaps the testing framework itself is modified and not need to spend O(5 hours) for the test suite to run. [1]: https://cmake.org/cmake/help/latest/command/include_guard.html -[2]: https://github.com/trilinos/Trilinos/blob/master/cmake/std/trilinosprhelpers/TrilinosPRConfigurationStandard.py +[2]: https://github.com/trilinos/Trilinos/blob/master/packages/framework/pr_tools/trilinosprhelpers/TrilinosPRConfigurationStandard.py From 25dce806d1c5969b6a7ff3cd058da0090cd0e020 Mon Sep 17 00:00:00 2001 From: Anderson Date: Fri, 10 Jan 2025 15:39:37 -0700 Subject: [PATCH 25/40] Add reference to TrilinosPRConfigurationStandard.py to SimpleTesting doc Signed-off-by: Anderson --- cmake/SimpleTesting/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/SimpleTesting/README.md b/cmake/SimpleTesting/README.md index 0426aaebfaa4..8554dcf20330 100644 --- a/cmake/SimpleTesting/README.md +++ b/cmake/SimpleTesting/README.md @@ -47,7 +47,6 @@ from being include more than once in an include chain. 2. Related to (1), we might also change `build_dir` to be `BUILD_DIR` and pass that in. - Example CTest call from a Trilinos PR ------------------------------------- This is an example, for reference, of how the `ctest` command is invoked in the current Trilinos @@ -69,6 +68,9 @@ ctest \ -Dsubprojects_file=../package_subproject_list.cmake ``` +See `TrilinosPRConfigurationStandard.py`[2] for information on what options are set to something +other than the default during normal Trilinos PR operations. + Additional Notes and Pitfalls ============================= From 1973284905abe4bfba9a7a61955541499442f3a6 Mon Sep 17 00:00:00 2001 From: Anderson Chauphan Date: Fri, 10 Jan 2025 16:19:17 -0700 Subject: [PATCH 26/40] Update test's default optional arguments Signed-off-by: Anderson Chauphan --- .../pr_tools/unittests/test_PullRequestLinuxDriverTest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/framework/pr_tools/unittests/test_PullRequestLinuxDriverTest.py b/packages/framework/pr_tools/unittests/test_PullRequestLinuxDriverTest.py index 46cf59176cd8..dc9a4cb1d265 100755 --- a/packages/framework/pr_tools/unittests/test_PullRequestLinuxDriverTest.py +++ b/packages/framework/pr_tools/unittests/test_PullRequestLinuxDriverTest.py @@ -81,6 +81,7 @@ def setUp(self): filename_packageenables='../packageEnables.cmake', filename_subprojects='../package_subproject_list.cmake', skip_create_packageenables=False, + skip_run_tests=False, test_mode='standard', req_mem_per_core=3.0, max_cores_allowed=12, From bc6496ec88899ce3823e8062a01bc5ed41a5ef69 Mon Sep 17 00:00:00 2001 From: Maarten Arnst Date: Mon, 13 Jan 2025 16:19:42 +0100 Subject: [PATCH 27/40] Intrepid2: Fix the implicit by-copy capture of this is deprecated (#13698) Signed-off-by: Maarten Arnst --- .../Basis/Intrepid2_HCURL_HEX_In_FEMDef.hpp | 8 ++++---- .../Basis/Intrepid2_HCURL_QUAD_In_FEMDef.hpp | 8 ++++---- .../Basis/Intrepid2_HCURL_TET_In_FEMDef.hpp | 8 ++++---- .../Basis/Intrepid2_HCURL_TRI_In_FEMDef.hpp | 8 ++++---- .../Basis/Intrepid2_HDIV_HEX_In_FEMDef.hpp | 8 ++++---- .../Basis/Intrepid2_HDIV_QUAD_In_FEMDef.hpp | 8 ++++---- .../Basis/Intrepid2_HDIV_TET_In_FEMDef.hpp | 8 ++++---- .../Basis/Intrepid2_HDIV_TRI_In_FEMDef.hpp | 8 ++++---- .../Basis/Intrepid2_HGRAD_HEX_Cn_FEMDef.hpp | 8 ++++---- .../Basis/Intrepid2_HGRAD_LINE_Cn_FEMDef.hpp | 8 ++++---- .../Basis/Intrepid2_HGRAD_QUAD_Cn_FEMDef.hpp | 12 ++++++------ .../Basis/Intrepid2_HGRAD_TET_Cn_FEMDef.hpp | 8 ++++---- .../Basis/Intrepid2_HGRAD_TRI_Cn_FEMDef.hpp | 12 ++++++------ .../Basis/Intrepid2_HVOL_HEX_Cn_FEMDef.hpp | 4 ++-- .../Basis/Intrepid2_HVOL_LINE_Cn_FEMDef.hpp | 4 ++-- .../Basis/Intrepid2_HVOL_QUAD_Cn_FEMDef.hpp | 4 ++-- .../Basis/Intrepid2_HVOL_TET_Cn_FEMDef.hpp | 4 ++-- .../Basis/Intrepid2_HVOL_TRI_Cn_FEMDef.hpp | 4 ++-- 18 files changed, 66 insertions(+), 66 deletions(-) diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_HEX_In_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_HEX_In_FEMDef.hpp index 182c05d721b0..806809284cd0 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_HEX_In_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_HEX_In_FEMDef.hpp @@ -630,19 +630,19 @@ namespace Intrepid2 { switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinvLine_ = this->vinvLine_, &vinvBubble_ = this->vinvBubble_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HCURL_HEX_In_FEM::Serial::getValues( output, input, work, this->vinvLine_, this->vinvBubble_ ); + Impl::Basis_HCURL_HEX_In_FEM::Serial::getValues( output, input, work, vinvLine_, vinvBubble_ ); }); break; case OPERATOR_CURL: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinvLine_ = this->vinvLine_, &vinvBubble_ = this->vinvBubble_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HCURL_HEX_In_FEM::Serial::getValues( output, input, work, this->vinvLine_, this->vinvBubble_ ); + Impl::Basis_HCURL_HEX_In_FEM::Serial::getValues( output, input, work, vinvLine_, vinvBubble_ ); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_QUAD_In_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_QUAD_In_FEMDef.hpp index b00248a51fc8..37d857aa203e 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_QUAD_In_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_QUAD_In_FEMDef.hpp @@ -421,19 +421,19 @@ namespace Intrepid2 { switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinvLine_ = this->vinvLine_, &vinvBubble_ = this-> vinvBubble_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HCURL_QUAD_In_FEM::Serial::getValues( output, input, work, this->vinvLine_, this->vinvBubble_ ); + Impl::Basis_HCURL_QUAD_In_FEM::Serial::getValues( output, input, work, vinvLine_, vinvBubble_ ); }); break; case OPERATOR_CURL: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinvLine_ = this->vinvLine_, &vinvBubble_ = this->vinvBubble_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HCURL_QUAD_In_FEM::Serial::getValues( output, input, work, this->vinvLine_, this->vinvBubble_ ); + Impl::Basis_HCURL_QUAD_In_FEM::Serial::getValues( output, input, work, vinvLine_, vinvBubble_ ); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_TET_In_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_TET_In_FEMDef.hpp index 56149a4a1820..c47b2a2a156b 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_TET_In_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_TET_In_FEMDef.hpp @@ -600,19 +600,19 @@ Basis_HCURL_TET_In_FEM::getValues( switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &coeffs_ = this->coeffs_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HCURL_TET_In_FEM::Serial::getValues( output, input, work, this->coeffs_ ); + Impl::Basis_HCURL_TET_In_FEM::Serial::getValues( output, input, work, coeffs_ ); }); break; case OPERATOR_CURL: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &coeffs_ = this->coeffs_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HCURL_TET_In_FEM::Serial::getValues( output, input, work, this->coeffs_ ); + Impl::Basis_HCURL_TET_In_FEM::Serial::getValues( output, input, work, coeffs_ ); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_TRI_In_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_TRI_In_FEMDef.hpp index 6cb65ab386de..83ccf78a9278 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_TRI_In_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HCURL_TRI_In_FEMDef.hpp @@ -489,19 +489,19 @@ namespace Intrepid2 { switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &coeffs_ = this->coeffs_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HCURL_TRI_In_FEM::Serial::getValues( output, input, work, this->coeffs_ ); + Impl::Basis_HCURL_TRI_In_FEM::Serial::getValues( output, input, work, coeffs_ ); }); break; case OPERATOR_CURL: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &coeffs_ = this->coeffs_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HCURL_TRI_In_FEM::Serial::getValues( output, input, work, this->coeffs_ ); + Impl::Basis_HCURL_TRI_In_FEM::Serial::getValues( output, input, work, coeffs_ ); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_HEX_In_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_HEX_In_FEMDef.hpp index 0d5d25113bdb..9d97011885ca 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_HEX_In_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_HEX_In_FEMDef.hpp @@ -544,19 +544,19 @@ namespace Intrepid2 { switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinvLine_ = this->vinvLine_, &vinvBubble_ = this->vinvBubble_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HDIV_HEX_In_FEM::Serial::getValues( output, input, work, this->vinvLine_, this->vinvBubble_ ); + Impl::Basis_HDIV_HEX_In_FEM::Serial::getValues( output, input, work, vinvLine_, vinvBubble_ ); }); break; case OPERATOR_DIV: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinvLine_ = this->vinvLine_, &vinvBubble_ = this->vinvBubble_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HDIV_HEX_In_FEM::Serial::getValues( output, input, work, this->vinvLine_, this->vinvBubble_ ); + Impl::Basis_HDIV_HEX_In_FEM::Serial::getValues( output, input, work, vinvLine_, vinvBubble_ ); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_QUAD_In_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_QUAD_In_FEMDef.hpp index ee5bdc9458c4..e053e76c990d 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_QUAD_In_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_QUAD_In_FEMDef.hpp @@ -415,19 +415,19 @@ namespace Intrepid2 { switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinvLine_ = this->vinvLine_, &vinvBubble_ = this->vinvBubble_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HDIV_QUAD_In_FEM::Serial::getValues( output, input, work, this->vinvLine_, this->vinvBubble_ ); + Impl::Basis_HDIV_QUAD_In_FEM::Serial::getValues( output, input, work, vinvLine_, vinvBubble_ ); }); break; case OPERATOR_DIV: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinvLine_ = this->vinvLine_, &vinvBubble_ = this->vinvBubble_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HDIV_QUAD_In_FEM::Serial::getValues( output, input, work, this->vinvLine_, this->vinvBubble_ ); + Impl::Basis_HDIV_QUAD_In_FEM::Serial::getValues( output, input, work, vinvLine_, vinvBubble_ ); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_TET_In_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_TET_In_FEMDef.hpp index 96e0e7cf2267..c82cc837211b 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_TET_In_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_TET_In_FEMDef.hpp @@ -488,19 +488,19 @@ Basis_HDIV_TET_In_FEM::getValues( switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &coeffs_ = this->coeffs_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HDIV_TET_In_FEM::Serial::getValues( output, input, work, this->coeffs_ ); + Impl::Basis_HDIV_TET_In_FEM::Serial::getValues( output, input, work, coeffs_ ); }); break; case OPERATOR_DIV: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &coeffs_ = this->coeffs_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HDIV_TET_In_FEM::Serial::getValues( output, input, work, this->coeffs_ ); + Impl::Basis_HDIV_TET_In_FEM::Serial::getValues( output, input, work, coeffs_ ); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_TRI_In_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_TRI_In_FEMDef.hpp index 89c86f5274e9..507e2d290120 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_TRI_In_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HDIV_TRI_In_FEMDef.hpp @@ -484,19 +484,19 @@ Basis_HDIV_TRI_In_FEM( const ordinal_type order, switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &coeffs_ = this->coeffs_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HDIV_TRI_In_FEM::Serial::getValues( output, input, work, this->coeffs_ ); + Impl::Basis_HDIV_TRI_In_FEM::Serial::getValues( output, input, work, coeffs_ ); }); break; case OPERATOR_DIV: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &coeffs_ = this->coeffs_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HDIV_TRI_In_FEM::Serial::getValues( output, input, work, this->coeffs_ ); + Impl::Basis_HDIV_TRI_In_FEM::Serial::getValues( output, input, work, coeffs_ ); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_HEX_Cn_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_HEX_Cn_FEMDef.hpp index 36139dfb95f4..8908598c15b8 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_HEX_Cn_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_HEX_Cn_FEMDef.hpp @@ -419,19 +419,19 @@ namespace Intrepid2 { switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HGRAD_HEX_Cn_FEM::Serial::getValues( output, input, work, this->vinv_ ); + Impl::Basis_HGRAD_HEX_Cn_FEM::Serial::getValues( output, input, work, vinv_ ); }); break; case OPERATOR_GRAD: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HGRAD_HEX_Cn_FEM::Serial::getValues( output, input, work, this->vinv_ ); + Impl::Basis_HGRAD_HEX_Cn_FEM::Serial::getValues( output, input, work, vinv_ ); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_LINE_Cn_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_LINE_Cn_FEMDef.hpp index 6140c13821cb..24a209424ae4 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_LINE_Cn_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_LINE_Cn_FEMDef.hpp @@ -362,19 +362,19 @@ namespace Intrepid2 { switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HGRAD_LINE_Cn_FEM::Serial::getValues( output, input, work, this->vinv_ ); + Impl::Basis_HGRAD_LINE_Cn_FEM::Serial::getValues( output, input, work, vinv_ ); }); break; case OPERATOR_GRAD: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HGRAD_LINE_Cn_FEM::Serial::getValues( output, input, work, this->vinv_ ); + Impl::Basis_HGRAD_LINE_Cn_FEM::Serial::getValues( output, input, work, vinv_ ); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_QUAD_Cn_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_QUAD_Cn_FEMDef.hpp index 9e232352285e..f11aca244836 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_QUAD_Cn_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_QUAD_Cn_FEMDef.hpp @@ -393,27 +393,27 @@ namespace Intrepid2 { switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HGRAD_QUAD_Cn_FEM::Serial::getValues( output, input, work, this->vinv_ ); + Impl::Basis_HGRAD_QUAD_Cn_FEM::Serial::getValues( output, input, work, vinv_ ); }); break; case OPERATOR_GRAD: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HGRAD_QUAD_Cn_FEM::Serial::getValues( output, input, work, this->vinv_ ); + Impl::Basis_HGRAD_QUAD_Cn_FEM::Serial::getValues( output, input, work, vinv_ ); }); break; case OPERATOR_CURL: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HGRAD_QUAD_Cn_FEM::Serial::getValues( output, input, work, this->vinv_ ); + Impl::Basis_HGRAD_QUAD_Cn_FEM::Serial::getValues( output, input, work, vinv_ ); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_TET_Cn_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_TET_Cn_FEMDef.hpp index 44f09ce456eb..7e2072d74656 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_TET_Cn_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_TET_Cn_FEMDef.hpp @@ -458,19 +458,19 @@ Basis_HGRAD_TET_Cn_FEM( const ordinal_type order, using range_type = Kokkos::pair; switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_, basisDegree_ = this->basisDegree_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HGRAD_TET_Cn_FEM::Serial::getValues( output, input, work, this->vinv_, this->basisDegree_); + Impl::Basis_HGRAD_TET_Cn_FEM::Serial::getValues( output, input, work, vinv_, basisDegree_); }); break; case OPERATOR_GRAD: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_, basisDegree_ = this->basisDegree_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HGRAD_TET_Cn_FEM::Serial::getValues( output, input, work, this->vinv_, this->basisDegree_); + Impl::Basis_HGRAD_TET_Cn_FEM::Serial::getValues( output, input, work, vinv_, basisDegree_); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_TRI_Cn_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_TRI_Cn_FEMDef.hpp index c7b7a40cfa7b..e6b11aa87140 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_TRI_Cn_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HGRAD_TRI_Cn_FEMDef.hpp @@ -408,27 +408,27 @@ Basis_HGRAD_TRI_Cn_FEM( const ordinal_type order, using range_type = Kokkos::pair; switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_, basisDegree_ = this->basisDegree_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HGRAD_TRI_Cn_FEM::Serial::getValues( output, input, work, this->vinv_, this->basisDegree_); + Impl::Basis_HGRAD_TRI_Cn_FEM::Serial::getValues( output, input, work, vinv_, basisDegree_); }); break; case OPERATOR_GRAD: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_, basisDegree_ = this->basisDegree_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HGRAD_TRI_Cn_FEM::Serial::getValues( output, input, work, this->vinv_, this->basisDegree_); + Impl::Basis_HGRAD_TRI_Cn_FEM::Serial::getValues( output, input, work, vinv_, basisDegree_); }); break; case OPERATOR_CURL: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_, basisDegree_ = this->basisDegree_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type(pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt,pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HGRAD_TRI_Cn_FEM::Serial::getValues( output, input, work, this->vinv_, this->basisDegree_); + Impl::Basis_HGRAD_TRI_Cn_FEM::Serial::getValues( output, input, work, vinv_, basisDegree_); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_HEX_Cn_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_HEX_Cn_FEMDef.hpp index 617eeb9cad84..3d3049c44e03 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_HEX_Cn_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_HEX_Cn_FEMDef.hpp @@ -351,11 +351,11 @@ namespace Intrepid2 { using range_type = Kokkos::pair; switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_, basisDegree_ = this->basisDegree_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HVOL_HEX_Cn_FEM::Serial::getValues( output, input, work, this->vinv_, this->basisDegree_); + Impl::Basis_HVOL_HEX_Cn_FEM::Serial::getValues( output, input, work, vinv_, basisDegree_); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_LINE_Cn_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_LINE_Cn_FEMDef.hpp index dc8f25d3cd7e..38ce394d20eb 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_LINE_Cn_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_LINE_Cn_FEMDef.hpp @@ -325,11 +325,11 @@ namespace Intrepid2 { switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HVOL_LINE_Cn_FEM::Serial::getValues( output, input, work, this->vinv_ ); + Impl::Basis_HVOL_LINE_Cn_FEM::Serial::getValues( output, input, work, vinv_ ); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_QUAD_Cn_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_QUAD_Cn_FEMDef.hpp index f492b6a65f7c..ae18d0ee7369 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_QUAD_Cn_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_QUAD_Cn_FEMDef.hpp @@ -317,11 +317,11 @@ namespace Intrepid2 { using range_type = Kokkos::pair; switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_, basisDegree_ = this->basisDegree_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HVOL_QUAD_Cn_FEM::Serial::getValues( output, input, work, this->vinv_, this->basisDegree_); + Impl::Basis_HVOL_QUAD_Cn_FEM::Serial::getValues( output, input, work, vinv_, basisDegree_); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_TET_Cn_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_TET_Cn_FEMDef.hpp index 7927a1e124f6..3fb8c2454755 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_TET_Cn_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_TET_Cn_FEMDef.hpp @@ -345,11 +345,11 @@ namespace Intrepid2 { using range_type = Kokkos::pair; switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HVOL_TET_Cn_FEM::Serial::getValues( output, input, work, this->vinv_); + Impl::Basis_HVOL_TET_Cn_FEM::Serial::getValues( output, input, work, vinv_); }); break; default: { diff --git a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_TRI_Cn_FEMDef.hpp b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_TRI_Cn_FEMDef.hpp index aa6f54065ff6..b703fd1d84b9 100644 --- a/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_TRI_Cn_FEMDef.hpp +++ b/packages/intrepid2/src/Discretization/Basis/Intrepid2_HVOL_TRI_Cn_FEMDef.hpp @@ -337,11 +337,11 @@ Basis_HVOL_TRI_Cn_FEM( const ordinal_type order, using range_type = Kokkos::pair; switch(operatorType) { case OPERATOR_VALUE: - Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=] (ordinal_type& pt) { + Kokkos::parallel_for (Kokkos::TeamThreadRange (team_member, numPoints), [=, &vinv_ = this->vinv_] (ordinal_type& pt) { auto output = Kokkos::subview( outputValues, Kokkos::ALL(), range_type (pt,pt+1), Kokkos::ALL() ); const auto input = Kokkos::subview( inputPoints, range_type(pt, pt+1), Kokkos::ALL() ); WorkViewType work(workView.data() + sizePerPoint*team_member.team_rank(), sizePerPoint); - Impl::Basis_HVOL_TRI_Cn_FEM::Serial::getValues( output, input, work, this->vinv_); + Impl::Basis_HVOL_TRI_Cn_FEM::Serial::getValues( output, input, work, vinv_); }); break; default: { From 833d7bd33f026276cf360318f979b0e771323fb8 Mon Sep 17 00:00:00 2001 From: Anderson Chauphan Date: Mon, 13 Jan 2025 10:39:39 -0700 Subject: [PATCH 28/40] Add use of --skip-run-tests if build-name is uvm Add injection to the start of the PR script driver to include the argument `--skip-run-tests` if the `GENCONFIG_BUILD_NAME` contains `_uvm_` and `no-package-enables`. This is to ensure that this option is only being applied to PR testing configurations and will not affect Nightly test runs which go through the same PR scripts. Signed-off-by: Anderson Chauphan --- packages/framework/pr_tools/PullRequestLinuxDriver.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/framework/pr_tools/PullRequestLinuxDriver.sh b/packages/framework/pr_tools/PullRequestLinuxDriver.sh index 4198a87db296..3f041240fc8c 100755 --- a/packages/framework/pr_tools/PullRequestLinuxDriver.sh +++ b/packages/framework/pr_tools/PullRequestLinuxDriver.sh @@ -279,6 +279,11 @@ then test_cmd_options+=( "--skip-create-packageenables ") fi +if [[ ${GENCONFIG_BUILD_NAME} == *"_uvm_"* && ${GENCONFIG_BUILD_NAME} == *"no-package-enables"* ]] +then + test_cmd_options+=( "--skip-run-tests" ) +fi + test_cmd="${PYTHON_EXE:?} ${REPO_ROOT:?}/packages/framework/pr_tools/PullRequestLinuxDriverTest.py ${test_cmd_options[@]}" # Call the script to launch the tests From 5f1171617dc7dcea4711ad8b4d4e9c5fa31315c5 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Mon, 13 Jan 2025 12:10:54 -0700 Subject: [PATCH 29/40] NOX: fix plist validation for inexact utils Signed-off-by: Roger Pawlowski --- packages/nox/src/NOX_Direction_Newton.C | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/nox/src/NOX_Direction_Newton.C b/packages/nox/src/NOX_Direction_Newton.C index 8a339320765a..fde2c022fad4 100644 --- a/packages/nox/src/NOX_Direction_Newton.C +++ b/packages/nox/src/NOX_Direction_Newton.C @@ -54,6 +54,11 @@ reset(const Teuchos::RCP& gd, validParams.set("Forcing Term Gamma", 0.9); validParams.set("Rescue Bad Newton Solve", true); + // Not used in this direction object, but needed in the Inexact + // Newton Utils which uses the same parameter list. Need to + // consolidate the two objects eventually. + validParams.set("Set Tolerance in Parameter List",true); + p.validateParametersAndSetDefaults(validParams); } From b71d29e601012c427d812450f80954606aa2ee16 Mon Sep 17 00:00:00 2001 From: "Samuel E. Browne" Date: Mon, 13 Jan 2025 14:24:18 -0700 Subject: [PATCH 30/40] Add check for remotes being as expected Since we moved GenConfig and some of its associated repositories, get_dependencies.sh should detect and tell the user that they will need to allow it to re-clone them if their remotes are out-of-date. Otherwise pulls can/will fail. Signed-off-by: Samuel E. Browne --- packages/framework/get_dependencies.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/framework/get_dependencies.sh b/packages/framework/get_dependencies.sh index 956857ecaf50..d70e82c5eb99 100755 --- a/packages/framework/get_dependencies.sh +++ b/packages/framework/get_dependencies.sh @@ -36,11 +36,23 @@ function tril_genconfig_clone_or_update_repo() { echo if [[ -d ${sub_dir} ]] ; then - echo "STATUS: ${sub_dir}: Fetching remote repo" cd ${sub_dir} - tril_genconfig_assert_pwd_is_git_repo - cmd="git fetch" - retry_command "${cmd}" + remote=$(git remote get-url origin) + if [[ ${git_url} == *"${remote}"* ]] + then + echo "STATUS: ${sub_dir}: Fetching remote repo" + tril_genconfig_assert_pwd_is_git_repo + cmd="git fetch" + retry_command "${cmd}" + else + echo "ERROR: Current remote origin does not match expected!" >&2 + echo "Please remove/move '$(pwd)' and re-run this script" >&2 + echo "" >&2 + echo "Current: ${remote}" >&2 + echo "Expected: ${git_url}" >&2 + echo "" >&2 + exit 1 + fi else echo "STATUS: ${sub_dir}: Cloning from '${git_url}'" cmd="git clone ${git_url} ${sub_dir}" From 2a29ccdb851604245c92af6dfe2a003aba447b96 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 22:35:39 +0000 Subject: [PATCH 31/40] Bump actions/upload-artifact from 4.5.0 to 4.6.0 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.5.0 to 4.6.0. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/6f51ac03b9356f520e9adb1b1b7802705f340c2b...65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/clang_format.yml | 2 +- .github/workflows/scorecards.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clang_format.yml b/.github/workflows/clang_format.yml index db061f4a4272..8edc9a0495f1 100644 --- a/.github/workflows/clang_format.yml +++ b/.github/workflows/clang_format.yml @@ -22,7 +22,7 @@ jobs: - run: git diff HEAD > format_patch.txt - run: if [ "$(cat format_patch.txt)" == "" ] ; then rm format_patch.txt ; else cat format_patch.txt; fi - - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 id: upload-artf if: ${{ hashFiles('format_patch.txt') != '' }} with: diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 711c3a947c5b..0ba48d4899ee 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -58,7 +58,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 with: name: SARIF file path: results.sarif From 778e2fb95f7a3608be26e7ea3a9cef27625c8321 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 22:35:47 +0000 Subject: [PATCH 32/40] Bump github/codeql-action from 3.28.0 to 3.28.1 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.28.0 to 3.28.1. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/48ab28a6f5dbc2a99bf1e0131198dd8f1df78169...b6a472f63d85b9c78a3ac5e89422239fc15e9b3c) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql.yml | 4 ++-- .github/workflows/scorecards.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 6e067a99b3bb..0d3fa53c932a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -45,7 +45,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Initialize CodeQL - uses: github/codeql-action/init@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0 + uses: github/codeql-action/init@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -108,6 +108,6 @@ jobs: ninja -j 16 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0 + uses: github/codeql-action/analyze@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 711c3a947c5b..83f4594c983d 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -66,6 +66,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0 + uses: github/codeql-action/upload-sarif@b6a472f63d85b9c78a3ac5e89422239fc15e9b3c # v3.28.1 with: sarif_file: results.sarif From ae4a2c3d146e8afee3549e00588ce8d096bcc1b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 22:35:49 +0000 Subject: [PATCH 33/40] Bump step-security/harden-runner from 2.10.2 to 2.10.3 Bumps [step-security/harden-runner](https://github.com/step-security/harden-runner) from 2.10.2 to 2.10.3. - [Release notes](https://github.com/step-security/harden-runner/releases) - [Commits](https://github.com/step-security/harden-runner/compare/0080882f6c36860b6ba35c610c98ce87d4e2f26f...c95a14d0e5bab51a9f56296a4eb0e416910cd350) --- updated-dependencies: - dependency-name: step-security/harden-runner dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/dependency-review.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 21a469b132cb..922916c51016 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 + uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3 with: egress-policy: audit From 7da1b38e5e29ef968ea7632074301f8cf93f75e3 Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Tue, 14 Jan 2025 07:22:40 -0700 Subject: [PATCH 34/40] Normalize URLs to allow for git@ protocol in check This allows for developers to use a git@ protocol for these repos instead of the HTTPs protocol so they can do development on these GenConfig repos not still be able to run the get_dependencies.sh script. This just makes development and testing easier. Otherwise, you have to switch back and forth between different sets of repos, which is error-prone and clunky. I also reformatted the error message some to make more clear (IMHO). I tested this manually in the following scenarios: 1) Starting with old GenConfig repo with wrong URL. That gave the error message: ERROR: Current remote origin does not match expected! Current Remote: git@gitlab-ex.sandia.gov:trilinos-devops-consolidation/code/GenConfig.git Expected Remote: https://github.com/sandialabs/GenConfig.git Please remove/move '/scratch/rabartl/Trilinos.base/Trilinos/packages/framework/GenConfig' and re-run this script 2) Starting with no GenConfig repo. That gave seemed to clone the repos correctly. 3) With existing GenConfig repo pointing to the correct URL. This seemed to update the repos correctly. 4) With existing GenConfig repo pointing to the correct URL but now with git@ URL instead of https:// URL and wrong SHA1 for GenConfig repo. This seemed to update the repos correctly. NOTE: The new utility script `normalize_git_repo_url.py` is maintained with unit tests in the little internal repo: * https://gitlab-ex.sandia.gov/rabartl/normalize_git_repo_url That makes this reusable for other such purposes in other projects as well. Signed-off-by: Roscoe A. Bartlett --- packages/framework/get_dependencies.sh | 21 +++++++----- packages/framework/normalize_git_repo_url.py | 36 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 9 deletions(-) create mode 100755 packages/framework/normalize_git_repo_url.py diff --git a/packages/framework/get_dependencies.sh b/packages/framework/get_dependencies.sh index d70e82c5eb99..323e3bee45e9 100755 --- a/packages/framework/get_dependencies.sh +++ b/packages/framework/get_dependencies.sh @@ -37,22 +37,25 @@ function tril_genconfig_clone_or_update_repo() { if [[ -d ${sub_dir} ]] ; then cd ${sub_dir} + # Validate correct remote and abort if not correct remote=$(git remote get-url origin) - if [[ ${git_url} == *"${remote}"* ]] - then - echo "STATUS: ${sub_dir}: Fetching remote repo" - tril_genconfig_assert_pwd_is_git_repo - cmd="git fetch" - retry_command "${cmd}" - else + normalized_remote=$(python3 ${script_dir}/normalize_git_repo_url.py ${remote}) + normalized_git_url=$(python3 ${script_dir}/normalize_git_repo_url.py ${git_url}) + if [[ "${normalized_remote}" != "${normalized_git_url}" ]] ; then echo "ERROR: Current remote origin does not match expected!" >&2 + echo " Current Remote: ${remote}" >&2 + echo " Expected Remote: ${git_url}" >&2 + echo "" >&2 echo "Please remove/move '$(pwd)' and re-run this script" >&2 echo "" >&2 - echo "Current: ${remote}" >&2 - echo "Expected: ${git_url}" >&2 echo "" >&2 exit 1 fi + # Update remote repo (which points to correct remote) + echo "STATUS: ${sub_dir}: Fetching remote repo" + tril_genconfig_assert_pwd_is_git_repo + cmd="git fetch" + retry_command "${cmd}" else echo "STATUS: ${sub_dir}: Cloning from '${git_url}'" cmd="git clone ${git_url} ${sub_dir}" diff --git a/packages/framework/normalize_git_repo_url.py b/packages/framework/normalize_git_repo_url.py new file mode 100755 index 000000000000..fcafca2b1cba --- /dev/null +++ b/packages/framework/normalize_git_repo_url.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import sys + +def removeStartingSubstr(inputStr, substrToRemove): + if inputStr.startswith(substrToRemove): + substrToRemoveLen = len(substrToRemove) + return inputStr[substrToRemoveLen:] + return inputStr + +def removeTrailingSubstr(inputStr, substrToRemove): + if inputStr.endswith(substrToRemove): + substrToRemoveLen = len(substrToRemove) + return inputStr[:-substrToRemoveLen] + return inputStr + +def normalizeGitRepoUrl(inputUrl): + url = inputUrl + url = removeStartingSubstr(url, "https://") + url = removeStartingSubstr(url, "git@") + url = removeTrailingSubstr(url, ".git") + url = url.replace(":", "/") + url = url.lower() + return url + +# Main + +def main(): + if len(sys.argv) != 2: + print("Usage: normalize_git_repo_url.py ") + sys.exit(1) + inputUrl = sys.argv[1] + print(normalizeGitRepoUrl(inputUrl)) + +if __name__ == "__main__": + main() From 8a84a21f6adf32b11aaf93224df0a3d56c9e69b3 Mon Sep 17 00:00:00 2001 From: Roger Pawlowski Date: Tue, 14 Jan 2025 11:37:22 -0700 Subject: [PATCH 35/40] Panzer: comment out code that requires c++20 Signed-off-by: Roger Pawlowski --- .../example/main_driver/user_app_Utilities.hpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/panzer/adapters-stk/example/main_driver/user_app_Utilities.hpp b/packages/panzer/adapters-stk/example/main_driver/user_app_Utilities.hpp index 14aeca4a2494..2d659ea15a72 100644 --- a/packages/panzer/adapters-stk/example/main_driver/user_app_Utilities.hpp +++ b/packages/panzer/adapters-stk/example/main_driver/user_app_Utilities.hpp @@ -44,12 +44,12 @@ namespace user_app { void addResponsesToModelEvaluatorFactory(const Teuchos::ParameterList& response_sublist, panzer_stk::ModelEvaluatorFactory& me_factory); - void computeAndPrintResponses(const Teuchos::RCP>& me, - const auto& x, - const auto& x_dot, - const auto& t, - const auto& global_data, - std::ostream& out); + // void computeAndPrintResponses(const Teuchos::RCP>& me, + // const auto& x, + // const auto& x_dot, + // const auto& t, + // const auto& global_data, + // std::ostream& out); std::tuple< Teuchos::RCP>, Teuchos::RCP, @@ -78,6 +78,7 @@ namespace user_app { std::tuple findResponseIndex(const std::string& g_name,const Thyra::ModelEvaluator& me); } +/* void user_app::computeAndPrintResponses(const Teuchos::RCP>& physics, const auto& x, const auto& x_dot, @@ -141,5 +142,6 @@ void user_app::computeAndPrintResponses(const Teuchos::RCP Date: Wed, 15 Jan 2025 16:15:54 +0000 Subject: [PATCH 36/40] [StepSecurity] ci: Pin dependencies Signed-off-by: StepSecurity Bot --- .github/workflows/AT2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/AT2.yml b/.github/workflows/AT2.yml index 8855f236ee1f..909cd82a08bd 100644 --- a/.github/workflows/AT2.yml +++ b/.github/workflows/AT2.yml @@ -31,7 +31,7 @@ jobs: should_skip: ${{ steps.skip_check.outputs.should_skip }} steps: - id: skip_check - uses: fkirc/skip-duplicate-actions@v5 + uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1 with: skip_after_successful_duplicate: 'true' From 32eda68459d496b18edd3abe0ce777ae420e5450 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Wed, 15 Jan 2025 11:03:16 -0700 Subject: [PATCH 37/40] README.md: More badges! Signed-off-by: Chris Siefert --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 75df4f80f3b5..663c536fb2de 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,9 @@ [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/trilinos/Trilinos/badge)](https://securityscorecards.dev/viewer/?uri=github.com/trilinos/Trilinos) [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/9452/badge)](https://www.bestpractices.dev/projects/9452) +![CodeFactor Grade](https://img.shields.io/codefactor/grade/github/trilinos/Trilinos) +![GitHub contributors](https://img.shields.io/github/contributors/trilinos/Trilinos) +![GitHub commit activity](https://img.shields.io/github/commit-activity/w/trilinos/Trilinos) The Trilinos Project is an effort to develop algorithms and enabling From 8fd2e8e893e27f8fb3110c4825cab617876c20cd Mon Sep 17 00:00:00 2001 From: Anderson Date: Wed, 15 Jan 2025 11:56:48 -0700 Subject: [PATCH 38/40] Update cuda-uvm config to disable in broken packages Update cuda-uvm config to disable package tests containing broken tests when building with UVM enabled. These package tests are force enabled in the configuration used for Nightly testing. Signed-off-by: Anderson --- packages/framework/ini-files/config-specs.ini | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/framework/ini-files/config-specs.ini b/packages/framework/ini-files/config-specs.ini index e41812e05117..b74053fe85ca 100644 --- a/packages/framework/ini-files/config-specs.ini +++ b/packages/framework/ini-files/config-specs.ini @@ -1521,6 +1521,13 @@ use SEMS_CUDA # should build and run, or ONLY build (TRILFRAME-675) opt-set-cmake-var Trilinos_ENABLE_TESTS BOOL FORCE : ON +# Packages containing tests broken when UVM enabled +opt-set-cmake-var Amesos2_ENABLE_TESTS BOOL FORCE : OFF +opt-set-cmake-var NOX_ENABLE_TESTS BOOL FORCE : OFF +opt-set-cmake-var Panzer_ENABLE_TESTS BOOL FORCE : OFF +opt-set-cmake-var Stokhos_ENABLE_TESTS BOOL FORCE : OFF +opt-set-cmake-var TrilinosCouplings_ENABLE_TESTS BOOL FORCE : OFF + [rhel8_sems-cuda-11.4.2-gnu-10.1.0-openmpi-4.1.6_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_uvm_deprecated-on_all] use rhel8_sems-cuda-11.4.2-gnu-10.1.0-openmpi-4.1.6_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_uvm_deprecated-on_no-package-enables use PACKAGE-ENABLES|ALL @@ -1529,6 +1536,14 @@ use PACKAGE-ENABLES|ALL use rhel8_sems-cuda-11.4.2-gnu-10.1.0-openmpi-4.1.6_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_uvm_deprecated-on_no-package-enables use PACKAGE-ENABLES|ALL-NO-EPETRA +# Packages containing tests broken when UVM enabled +# Enabled in this configuration to be caught in Nightly +opt-set-cmake-var Amesos2_ENABLE_TESTS BOOL FORCE : ON +opt-set-cmake-var NOX_ENABLE_TESTS BOOL FORCE : ON +opt-set-cmake-var Panzer_ENABLE_TESTS BOOL FORCE : ON +opt-set-cmake-var Stokhos_ENABLE_TESTS BOOL FORCE : ON +opt-set-cmake-var TrilinosCouplings_ENABLE_TESTS BOOL FORCE : ON + [rhel8_sems-cuda-11.4.2-gnu-10.1.0-openmpi-4.1.6_release_static_Volta70_no-asan_complex_no-fpic_mpi_pt_no-rdc_no-uvm_deprecated-on_no-package-enables] # uses sems-v2 modules use RHEL8 From 542a84fad98d45f5811c2d1729a726923cb61148 Mon Sep 17 00:00:00 2001 From: Chris Siefert Date: Wed, 15 Jan 2025 15:37:21 -0700 Subject: [PATCH 39/40] README.md: Fixing badge Got the codefactor badge fixed Signed-off-by: Chris Siefert --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 663c536fb2de..3c6c50d8034b 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/trilinos/Trilinos/badge)](https://securityscorecards.dev/viewer/?uri=github.com/trilinos/Trilinos) [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/9452/badge)](https://www.bestpractices.dev/projects/9452) -![CodeFactor Grade](https://img.shields.io/codefactor/grade/github/trilinos/Trilinos) +[![CodeFactor](https://www.codefactor.io/repository/github/trilinos/trilinos/badge)](https://www.codefactor.io/repository/github/trilinos/trilinos) ![GitHub contributors](https://img.shields.io/github/contributors/trilinos/Trilinos) ![GitHub commit activity](https://img.shields.io/github/commit-activity/w/trilinos/Trilinos) From 36d7541558e50f40141c787c9e5a6c993a437e8e Mon Sep 17 00:00:00 2001 From: "Roscoe A. Bartlett" Date: Thu, 16 Jan 2025 08:24:52 -0500 Subject: [PATCH 40/40] Increase default maxRcpRawObjAccessRatio from 13.5 to 20.0 (#8648, #13728) That should be high enough to avoid every random failure of this check ever observed in Trilinos PR testing. It is debatable if a test such as this should be run in all builds or in just dedicated performance builds. (The default timing ratios are very loose.) We just want to make sure these tests are not broken in every build so that this test will be able to run in performance builds. Signed-off-by: Roscoe A. Bartlett --- .../core/test/MemoryManagement/RCP_Performance_UnitTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/teuchos/core/test/MemoryManagement/RCP_Performance_UnitTests.cpp b/packages/teuchos/core/test/MemoryManagement/RCP_Performance_UnitTests.cpp index 3cd208618580..245c6d71c8d8 100644 --- a/packages/teuchos/core/test/MemoryManagement/RCP_Performance_UnitTests.cpp +++ b/packages/teuchos/core/test/MemoryManagement/RCP_Performance_UnitTests.cpp @@ -30,7 +30,7 @@ double maxRcpRawAdjustRefCountRatio = 100.0; #ifdef HAVE_TEUCHOSCORE_CXX11 double maxRcpSpAdjustRefCountRatio = 5.0; #endif -double maxRcpRawObjAccessRatio = 13.5; +double maxRcpRawObjAccessRatio = 20.0; // See trilinos/Trilinos#13728 const int intPrec = 8; const int dblPrec = 6;