From f2066ea16e4a92d390f41cd32ee4debc38a1dbd1 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 18 May 2025 09:14:22 -0600 Subject: [PATCH 01/33] working on a model observer --- pyomo/contrib/observer/__init__.py | 0 pyomo/contrib/observer/model_observer.py | 741 +++++++++++++++++++++++ 2 files changed, 741 insertions(+) create mode 100644 pyomo/contrib/observer/__init__.py create mode 100644 pyomo/contrib/observer/model_observer.py diff --git a/pyomo/contrib/observer/__init__.py b/pyomo/contrib/observer/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py new file mode 100644 index 00000000000..b644676f3d2 --- /dev/null +++ b/pyomo/contrib/observer/model_observer.py @@ -0,0 +1,741 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# __________________________________________________________________________ + +import abc +import datetime +from typing import List, Sequence + +from pyomo.common.config import ConfigDict, ConfigValue +from pyomo.core.base.constraint import ConstraintData, Constraint +from pyomo.core.base.sos import SOSConstraintData, SOSConstraint +from pyomo.core.base.var import VarData +from pyomo.core.base.param import ParamData, Param +from pyomo.core.base.objective import ObjectiveData +from pyomo.core.staleflag import StaleFlagManager +from pyomo.common.collections import ComponentMap +from pyomo.common.timing import HierarchicalTimer +from pyomo.contrib.solver.common.results import Results +from pyomo.contrib.solver.common.util import collect_vars_and_named_exprs, get_objective +from pyomo.common.numeric_types import native_numeric_types + + +class AutoUpdateConfig(ConfigDict): + """ + Control which parts of the model are automatically checked and/or updated upon re-solve + """ + + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + if doc is None: + doc = 'Configuration options to detect changes in model between solves' + super().__init__( + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + + self.check_for_new_or_removed_constraints: bool = self.declare( + 'check_for_new_or_removed_constraints', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, new/old constraints will not be automatically detected on subsequent + solves. Use False only when manually updating the solver with opt.add_constraints() + and opt.remove_constraints() or when you are certain constraints are not being + added to/removed from the model.""", + ), + ) + self.check_for_new_or_removed_vars: bool = self.declare( + 'check_for_new_or_removed_vars', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, new/old variables will not be automatically detected on subsequent + solves. Use False only when manually updating the solver with opt.add_variables() and + opt.remove_variables() or when you are certain variables are not being added to / + removed from the model.""", + ), + ) + self.check_for_new_or_removed_params: bool = self.declare( + 'check_for_new_or_removed_params', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, new/old parameters will not be automatically detected on subsequent + solves. Use False only when manually updating the solver with opt.add_parameters() and + opt.remove_parameters() or when you are certain parameters are not being added to / + removed from the model.""", + ), + ) + self.check_for_new_objective: bool = self.declare( + 'check_for_new_objective', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, new/old objectives will not be automatically detected on subsequent + solves. Use False only when manually updating the solver with opt.set_objective() or + when you are certain objectives are not being added to / removed from the model.""", + ), + ) + self.update_constraints: bool = self.declare( + 'update_constraints', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to existing constraints will not be automatically detected on + subsequent solves. This includes changes to the lower, body, and upper attributes of + constraints. Use False only when manually updating the solver with + opt.remove_constraints() and opt.add_constraints() or when you are certain constraints + are not being modified.""", + ), + ) + self.update_vars: bool = self.declare( + 'update_vars', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to existing variables will not be automatically detected on + subsequent solves. This includes changes to the lb, ub, domain, and fixed + attributes of variables. Use False only when manually updating the solver with + opt.update_variables() or when you are certain variables are not being modified.""", + ), + ) + self.update_parameters: bool = self.declare( + 'update_parameters', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to parameter values will not be automatically detected on + subsequent solves. Use False only when manually updating the solver with + opt.update_parameters() or when you are certain parameters are not being modified.""", + ), + ) + self.update_named_expressions: bool = self.declare( + 'update_named_expressions', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to Expressions will not be automatically detected on + subsequent solves. Use False only when manually updating the solver with + opt.remove_constraints() and opt.add_constraints() or when you are certain + Expressions are not being modified.""", + ), + ) + self.update_objective: bool = self.declare( + 'update_objective', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to objectives will not be automatically detected on + subsequent solves. This includes the expr and sense attributes of objectives. Use + False only when manually updating the solver with opt.set_objective() or when you are + certain objectives are not being modified.""", + ), + ) + + +class Observer(abc.ABC): + def __init__(self): + pass + + @abc.abstractmethod + def add_variables(self, variables: List[VarData]): + pass + + @abc.abstractmethod + def add_parameters(self, params: List[ParamData]): + pass + + @abc.abstractmethod + def add_constraints(self, cons: List[ConstraintData]): + pass + + @abc.abstractmethod + def add_sos_constraints(self, cons: List[SOSConstraintData]): + pass + + @abc.abstractmethod + def set_objective(self, obj: ObjectiveData): + pass + + @abc.abstractmethod + def remove_constraints(self, cons: List[ConstraintData]): + pass + + @abc.abstractmethod + def remove_sos_constraints(self, cons: List[SOSConstraintData]): + pass + + @abc.abstractmethod + def remove_variables(self, variables: List[VarData]): + pass + + @abc.abstractmethod + def remove_parameters(self, params: List[ParamData]): + pass + + +class ModelChangeDetector: + def __init__( + self, observers: Sequence[Observer], + treat_fixed_vars_as_params=True, + **kwds, + ): + """ + Parameters + ---------- + observers: Sequence[Observer] + The objects to notify when changes are made to the model + treat_fixed_vars_as_params: bool + This is an advanced option that should only be used in special circumstances. + With the default setting of True, fixed variables will be treated like parameters. + This means that z == x*y will be linear if x or y is fixed and the constraint + can be written to an LP file. If the value of the fixed variable gets changed, we have + to completely reprocess all constraints using that variable. If + treat_fixed_vars_as_params is False, then constraints will be processed as if fixed + variables are not fixed, and the solver will be told the variable is fixed. This means + z == x*y could not be written to an LP file even if x and/or y is fixed. However, + updating the values of fixed variables is much faster this way. + """ + self._observers: List[Observer] = list(observers) + self._model = None + self._active_constraints = {} # maps constraint to (lower, body, upper) + self._vars = {} # maps var id to (var, lb, ub, fixed, domain, value) + self._params = {} # maps param id to param + self._objective = None + self._objective_expr = None + self._objective_sense = None + self._named_expressions = ( + {} + ) # maps constraint to list of tuples (named_expr, named_expr.expr) + self._external_functions = ComponentMap() + self._obj_named_expressions = [] + self._referenced_variables = ( + {} + ) # var_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + self._referenced_params = ( + {} + ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + self._vars_referenced_by_con = {} + self._vars_referenced_by_obj = [] + self._params_referenced_by_con = {} + self._params_referenced_by_obj = [] + self._expr_types = None + self._treat_fixed_vars_as_params = treat_fixed_vars_as_params + self.config: AutoUpdateConfig = AutoUpdateConfig()(value=kwds, preserve_implicit=True) + + def set_instance(self, model): + saved_config = self.config + self.__init__(observers=self._observers, treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) + self.config = saved_config + self._model = model + self.add_block(model) + if self._objective is None: + self.set_objective(None) + + def _add_variables(self, variables: List[VarData]): + for v in variables: + if id(v) in self._referenced_variables: + raise ValueError(f'Variable {v.name} has already been added') + self._referenced_variables[id(v)] = [{}, {}, None] + self._vars[id(v)] = ( + v, + v._lb, + v._ub, + v.fixed, + v.domain.get_interval(), + v.value, + ) + for obs in self._observers: + obs.add_variables(variables) + + def _add_parameters(self, params: List[ParamData]): + for p in params: + pid = id(p) + if pid in self._referenced_params: + raise ValueError(f'Parameter {p.name} has already been added') + self._referenced_params[pid] = [{}, {}, None] + self._params[id(p)] = (p, p.value) + for obs in self._observers: + obs.add_parameters(params) + + def _check_for_new_vars(self, variables: List[VarData]): + new_vars = {} + for v in variables: + v_id = id(v) + if v_id not in self._referenced_variables: + new_vars[v_id] = v + self._add_variables(list(new_vars.values())) + + def _check_to_remove_vars(self, variables: List[VarData]): + vars_to_remove = {} + for v in variables: + v_id = id(v) + ref_cons, ref_sos, ref_obj = self._referenced_variables[v_id] + if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: + vars_to_remove[v_id] = v + self._remove_variables(list(vars_to_remove.values())) + + def _check_for_new_params(self, params: List[ParamData]): + new_params = {} + for p in params: + pid = id(p) + if pid not in self._referenced_params: + new_params[pid] = p + self._add_parameters(list(new_params.values())) + + def _check_to_remove_params(self, params: List[ParamData]): + params_to_remove = {} + for p in params: + p_id = id(p) + ref_cons, ref_sos, ref_obj = self._referenced_params[p_id] + if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: + params_to_remove[p_id] = p + self._remove_parameters(list(params_to_remove.values())) + + def _add_constraints(self, cons: List[ConstraintData]): + all_fixed_vars = {} + for con in cons: + if con in self._named_expressions: + raise ValueError(f'Constraint {con.name} has already been added') + self._active_constraints[con] = con.expr + tmp = collect_vars_and_named_exprs(con.expr) + named_exprs, variables, fixed_vars, parameters, external_functions = tmp + self._check_for_new_vars(variables) + self._check_for_new_params(parameters) + self._named_expressions[con] = [(e, e.expr) for e in named_exprs] + if len(external_functions) > 0: + self._external_functions[con] = external_functions + self._vars_referenced_by_con[con] = variables + self._params_referenced_by_con[con] = parameters + for v in variables: + self._referenced_variables[id(v)][0][con] = None + for p in parameters: + self._referenced_params[id(p)][0][con] = None + if not self._treat_fixed_vars_as_params: + for v in fixed_vars: + v.unfix() + all_fixed_vars[id(v)] = v + for obs in self._observers: + obs.add_constraints(cons) + for v in all_fixed_vars.values(): + v.fix() + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + all_fixed_vars = {} + for con in cons: + if con in self._vars_referenced_by_con: + raise ValueError(f'Constraint {con.name} has already been added') + self._active_constraints[con] = tuple() + variables = [] + params = [] + for v, p in con.get_items(): + variables.append(v) + if type(p) in native_numeric_types: + continue + if p.is_parameter_type(): + params.append(p) + self._check_for_new_vars(variables) + self._check_for_new_params(params) + self._named_expressions[con] = [] + self._vars_referenced_by_con[con] = variables + self._params_referenced_by_con[con] = params + for v in variables: + self._referenced_variables[id(v)][1][con] = None + for p in params: + self._referenced_params[id(p)][1][con] = None + if not self._treat_fixed_vars_as_params: + for v in variables: + if v.is_fixed(): + v.unfix() + all_fixed_vars[id(v)] = v + for obs in self._observers: + obs.add_sos_constraints(cons) + for v in all_fixed_vars.values(): + v.fix() + + def _set_objective(self, obj: ObjectiveData): + if self._objective is not None: + for v in self._vars_referenced_by_obj: + self._referenced_variables[id(v)][2] = None + self._check_to_remove_vars(self._vars_referenced_by_obj) + self._external_functions.pop(self._objective, None) + if obj is not None: + self._objective = obj + self._objective_expr = obj.expr + self._objective_sense = obj.sense + tmp = collect_vars_and_named_exprs(obj.expr) + named_exprs, variables, fixed_vars, external_functions = tmp + self._check_for_new_vars(variables) + self._obj_named_expressions = [(i, i.expr) for i in named_exprs] + if len(external_functions) > 0: + self._external_functions[obj] = external_functions + self._vars_referenced_by_obj = variables + for v in variables: + self._referenced_variables[id(v)][2] = obj + if not self._treat_fixed_vars_as_params: + for v in fixed_vars: + v.unfix() + for obs in self._observers: + obs.set_objective(obj) + for v in fixed_vars: + v.fix() + else: + self._vars_referenced_by_obj = [] + self._objective = None + self._objective_expr = None + self._objective_sense = None + self._obj_named_expressions = [] + for obs in self._observers: + obs.set_objective(obj) + + def add_block(self, block): + param_dict = {} + for p in block.component_objects(Param, descend_into=True): + if p.mutable: + for _p in p.values(): + param_dict[id(_p)] = _p + self._add_parameters(list(param_dict.values())) + self._add_constraints( + list( + block.component_data_objects(Constraint, descend_into=True, active=True) + ) + ) + self._add_sos_constraints( + list( + block.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + ) + ) + obj = get_objective(block) + if obj is not None: + self._set_objective(obj) + + def _remove_constraints(self, cons: List[ConstraintData]): + for obs in self._observers: + obs.remove_constraints(cons) + for con in cons: + if con not in self._named_expressions: + raise ValueError( + f'Cannot remove constraint {con.name} - it was not added' + ) + for v in self._vars_referenced_by_con[con]: + self._referenced_variables[id(v)][0].pop(con) + self._check_to_remove_vars(self._vars_referenced_by_con[con]) + del self._active_constraints[con] + del self._named_expressions[con] + self._external_functions.pop(con, None) + del self._vars_referenced_by_con[con] + + def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + for obs in self._observers: + obs.remove_sos_constraints(cons) + for con in cons: + if con not in self._vars_referenced_by_con: + raise ValueError( + f'Cannot remove constraint {con.name} - it was not added' + ) + for v in self._vars_referenced_by_con[con]: + self._referenced_variables[id(v)][1].pop(con) + self._check_to_remove_vars(self._vars_referenced_by_con[con]) + del self._active_constraints[con] + del self._named_expressions[con] + del self._vars_referenced_by_con[con] + + def _remove_variables(self, variables: List[VarData]): + for obs in self._observers: + obs.remove_variables(variables) + for v in variables: + v_id = id(v) + if v_id not in self._referenced_variables: + raise ValueError( + f'Cannot remove variable {v.name} - it has not been added' + ) + cons_using, sos_using, obj_using = self._referenced_variables[v_id] + if cons_using or sos_using or (obj_using is not None): + raise ValueError( + f'Cannot remove variable {v.name} - it is still being used by constraints or the objective' + ) + del self._referenced_variables[v_id] + del self._vars[v_id] + + def _remove_parameters(self, params: List[ParamData]): + for obs in self._observers: + obs.remove_parameters(params) + for p in params: + del self._params[id(p)] + + def _remove_block(self, block): + self._remove_constraints( + list( + block.component_data_objects( + ctype=Constraint, descend_into=True, active=True + ) + ) + ) + self._remove_sos_constraints( + list( + block.component_data_objects( + ctype=SOSConstraint, descend_into=True, active=True + ) + ) + ) + self._remove_parameters( + list( + dict( + (id(p), p) + for p in block.component_data_objects( + ctype=Param, descend_into=True + ) + ).values() + ) + ) + + @abc.abstractmethod + def _update_variables(self, variables: List[VarData]): + pass + + def update_variables(self, variables: List[VarData]): + for v in variables: + self._vars[id(v)] = ( + v, + v._lb, + v._ub, + v.fixed, + v.domain.get_interval(), + v.value, + ) + self._update_variables(variables) + + @abc.abstractmethod + def update_parameters(self): + pass + + def update(self, timer: HierarchicalTimer = None): + if timer is None: + timer = HierarchicalTimer() + config = self._active_config.auto_updates + new_vars = [] + old_vars = [] + new_params = [] + old_params = [] + new_cons = [] + old_cons = [] + old_sos = [] + new_sos = [] + current_cons_dict = {} + current_sos_dict = {} + timer.start('vars') + if config.update_vars: + start_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} + timer.stop('vars') + timer.start('params') + if config.check_for_new_or_removed_params: + current_params_dict = {} + for p in self._model.component_objects(Param, descend_into=True): + if p.mutable: + for _p in p.values(): + current_params_dict[id(_p)] = _p + for p_id, p in current_params_dict.items(): + if p_id not in self._params: + new_params.append(p) + for p_id, p in self._params.items(): + if p_id not in current_params_dict: + old_params.append(p) + timer.stop('params') + timer.start('cons') + if config.check_for_new_or_removed_constraints or config.update_constraints: + current_cons_dict = { + c: None + for c in self._model.component_data_objects( + Constraint, descend_into=True, active=True + ) + } + current_sos_dict = { + c: None + for c in self._model.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + } + for c in current_cons_dict.keys(): + if c not in self._vars_referenced_by_con: + new_cons.append(c) + for c in current_sos_dict.keys(): + if c not in self._vars_referenced_by_con: + new_sos.append(c) + for c in self._vars_referenced_by_con: + if c not in current_cons_dict and c not in current_sos_dict: + if (c.ctype is Constraint) or ( + c.ctype is None and isinstance(c, ConstraintData) + ): + old_cons.append(c) + else: + assert (c.ctype is SOSConstraint) or ( + c.ctype is None and isinstance(c, SOSConstraintData) + ) + old_sos.append(c) + self.remove_constraints(old_cons) + self.remove_sos_constraints(old_sos) + timer.stop('cons') + timer.start('params') + self.remove_parameters(old_params) + + # sticking this between removal and addition + # is important so that we don't do unnecessary work + if config.update_parameters: + self.update_parameters() + + self.add_parameters(new_params) + timer.stop('params') + timer.start('vars') + self.add_variables(new_vars) + timer.stop('vars') + timer.start('cons') + self.add_constraints(new_cons) + self.add_sos_constraints(new_sos) + new_cons_set = set(new_cons) + new_sos_set = set(new_sos) + cons_to_remove_and_add = {} + need_to_set_objective = False + if config.update_constraints: + for c in current_cons_dict.keys(): + if c not in new_cons_set and c.expr is not self._active_constraints[c]: + cons_to_remove_and_add[c] = None + sos_to_update = [] + for c in current_sos_dict.keys(): + if c not in new_sos_set: + sos_to_update.append(c) + self.remove_sos_constraints(sos_to_update) + self.add_sos_constraints(sos_to_update) + timer.stop('cons') + timer.start('vars') + if config.update_vars: + end_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} + vars_to_check = [v for v_id, v in end_vars.items() if v_id in start_vars] + if config.update_vars: + vars_to_update = [] + for v in vars_to_check: + _v, lb, ub, fixed, domain_interval, value = self._vars[id(v)] + if (fixed != v.fixed) or (fixed and (value != v.value)): + vars_to_update.append(v) + if self._treat_fixed_vars_as_params: + for c in self._referenced_variables[id(v)][0]: + cons_to_remove_and_add[c] = None + if self._referenced_variables[id(v)][2] is not None: + need_to_set_objective = True + elif lb is not v._lb: + vars_to_update.append(v) + elif ub is not v._ub: + vars_to_update.append(v) + elif domain_interval != v.domain.get_interval(): + vars_to_update.append(v) + self.update_variables(vars_to_update) + timer.stop('vars') + timer.start('cons') + cons_to_remove_and_add = list(cons_to_remove_and_add.keys()) + self.remove_constraints(cons_to_remove_and_add) + self.add_constraints(cons_to_remove_and_add) + timer.stop('cons') + timer.start('named expressions') + if config.update_named_expressions: + cons_to_update = [] + for c, expr_list in self._named_expressions.items(): + if c in new_cons_set: + continue + for named_expr, old_expr in expr_list: + if named_expr.expr is not old_expr: + cons_to_update.append(c) + break + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) + for named_expr, old_expr in self._obj_named_expressions: + if named_expr.expr is not old_expr: + need_to_set_objective = True + break + timer.stop('named expressions') + timer.start('objective') + if self._active_config.auto_updates.check_for_new_objective: + pyomo_obj = get_objective(self._model) + if pyomo_obj is not self._objective: + need_to_set_objective = True + else: + pyomo_obj = self._objective + if self._active_config.auto_updates.update_objective: + if pyomo_obj is not None and pyomo_obj.expr is not self._objective_expr: + need_to_set_objective = True + elif pyomo_obj is not None and pyomo_obj.sense is not self._objective_sense: + # we can definitely do something faster here than resetting the whole objective + need_to_set_objective = True + if need_to_set_objective: + self.set_objective(pyomo_obj) + timer.stop('objective') + + # this has to be done after the objective and constraints in case the + # old objective/constraints use old variables + timer.start('vars') + self.remove_variables(old_vars) + timer.stop('vars') + + +class PersistentSolverMixin: + """ + The `solve` method in Gurobi and Highs is exactly the same, so this Mixin + minimizes the duplicate code + """ + + def solve(self, model, **kwds) -> Results: + start_timestamp = datetime.datetime.now(datetime.timezone.utc) + self._active_config = config = self.config(value=kwds, preserve_implicit=True) + StaleFlagManager.mark_all_as_stale() + + if self._last_results_object is not None: + self._last_results_object.solution_loader.invalidate() + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + if model is not self._model: + timer.start('set_instance') + self.set_instance(model) + timer.stop('set_instance') + else: + timer.start('update') + self.update(timer=timer) + timer.stop('update') + + res = self._solve() + self._last_results_object = res + + end_timestamp = datetime.datetime.now(datetime.timezone.utc) + res.timing_info.start_timestamp = start_timestamp + res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() + res.timing_info.timer = timer + self._active_config = self.config + + return res From 97aeb31fd2c6fb2fc30349a19644d719af92b4dc Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 30 Jul 2025 06:37:17 -0600 Subject: [PATCH 02/33] working on a model observer --- pyomo/contrib/observer/model_observer.py | 392 ++++++++++-------- pyomo/contrib/observer/tests/__init__.py | 0 .../observer/tests/test_change_detector.py | 91 ++++ 3 files changed, 306 insertions(+), 177 deletions(-) create mode 100644 pyomo/contrib/observer/tests/__init__.py create mode 100644 pyomo/contrib/observer/tests/test_change_detector.py diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index b644676f3d2..b6768f2ac6b 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -11,7 +11,7 @@ import abc import datetime -from typing import List, Sequence +from typing import List, Sequence, Optional from pyomo.common.config import ConfigDict, ConfigValue from pyomo.core.base.constraint import ConstraintData, Constraint @@ -27,6 +27,27 @@ from pyomo.common.numeric_types import native_numeric_types +""" +The ModelChangeDetector is meant to be used to automatically identify changes +in a Pyomo model or block. Here is a list of changes that will be detected. +Note that inactive components (e.g., constraints) are treated as "removed". + - new constraints that have been added to the model + - constraints that have been removed from the model + - new variables that have been detected in new or modified constraints/objectives + - old variables that are no longer used in any constraints/objectives + - new parameters that have been detected in new or modified constraints/objectives + - old parameters that are no longer used in any constraints/objectives + - new objectives that have been added to the model + - objectives that have been removed from the model + - modified constraint expressions (relies on expressions being immutable) + - modified objective expressions (relies on expressions being immutable) + - modified objective sense + - changes to variable bounds, domains, and "fixed" flags + - changes to named expressions (relies on expressions being immutable) + - changes to parameter values and fixed variable values +""" + + class AutoUpdateConfig(ConfigDict): """ Control which parts of the model are automatically checked and/or updated upon re-solve @@ -62,30 +83,6 @@ def __init__( added to/removed from the model.""", ), ) - self.check_for_new_or_removed_vars: bool = self.declare( - 'check_for_new_or_removed_vars', - ConfigValue( - domain=bool, - default=True, - description=""" - If False, new/old variables will not be automatically detected on subsequent - solves. Use False only when manually updating the solver with opt.add_variables() and - opt.remove_variables() or when you are certain variables are not being added to / - removed from the model.""", - ), - ) - self.check_for_new_or_removed_params: bool = self.declare( - 'check_for_new_or_removed_params', - ConfigValue( - domain=bool, - default=True, - description=""" - If False, new/old parameters will not be automatically detected on subsequent - solves. Use False only when manually updating the solver with opt.add_parameters() and - opt.remove_parameters() or when you are certain parameters are not being added to / - removed from the model.""", - ), - ) self.check_for_new_objective: bool = self.declare( 'check_for_new_objective', ConfigValue( @@ -118,19 +115,23 @@ def __init__( description=""" If False, changes to existing variables will not be automatically detected on subsequent solves. This includes changes to the lb, ub, domain, and fixed - attributes of variables. Use False only when manually updating the solver with - opt.update_variables() or when you are certain variables are not being modified.""", + attributes of variables. Use False only when manually updating the observer with + opt.update_variables() or when you are certain variables are not being modified. + Note that changes to values of fixed variables is handled by + update_parameters_and_fixed_vars.""", ), ) - self.update_parameters: bool = self.declare( + self.update_parameters_and_fixed_vars: bool = self.declare( 'update_parameters', ConfigValue( domain=bool, default=True, description=""" - If False, changes to parameter values will not be automatically detected on - subsequent solves. Use False only when manually updating the solver with - opt.update_parameters() or when you are certain parameters are not being modified.""", + If False, changes to parameter values and fixed variable values will + not be automatically detected on subsequent solves. Use False only + when manually updating the observer with + opt.update_parameters_and_fixed_variables() or when you are certain + parameters are not being modified.""", ), ) self.update_named_expressions: bool = self.declare( @@ -199,6 +200,18 @@ def remove_variables(self, variables: List[VarData]): def remove_parameters(self, params: List[ParamData]): pass + @abc.abstractmethod + def update_variables(self, variables: List[VarData]): + pass + + @abc.abstractmethod + def update_parameters_and_fixed_variables( + self, + params: List[ParamData], + variables: List[VarData], + ): + pass + class ModelChangeDetector: def __init__( @@ -224,7 +237,8 @@ def __init__( """ self._observers: List[Observer] = list(observers) self._model = None - self._active_constraints = {} # maps constraint to (lower, body, upper) + self._active_constraints = {} # maps constraint to expression + self._active_sos = {} self._vars = {} # maps var id to (var, lb, ub, fixed, domain, value) self._params = {} # maps param id to param self._objective = None @@ -254,9 +268,7 @@ def set_instance(self, model): self.__init__(observers=self._observers, treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) self.config = saved_config self._model = model - self.add_block(model) - if self._objective is None: - self.set_objective(None) + self._add_block(model) def _add_variables(self, variables: List[VarData]): for v in variables: @@ -321,7 +333,7 @@ def _check_to_remove_params(self, params: List[ParamData]): def _add_constraints(self, cons: List[ConstraintData]): all_fixed_vars = {} for con in cons: - if con in self._named_expressions: + if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') self._active_constraints[con] = con.expr tmp = collect_vars_and_named_exprs(con.expr) @@ -351,10 +363,11 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): for con in cons: if con in self._vars_referenced_by_con: raise ValueError(f'Constraint {con.name} has already been added') - self._active_constraints[con] = tuple() + sos_items = list(con.get_items()) + self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) variables = [] params = [] - for v, p in con.get_items(): + for v, p in sos_items: variables.append(v) if type(p) in native_numeric_types: continue @@ -384,20 +397,25 @@ def _set_objective(self, obj: ObjectiveData): for v in self._vars_referenced_by_obj: self._referenced_variables[id(v)][2] = None self._check_to_remove_vars(self._vars_referenced_by_obj) + self._check_to_remove_params(self._params_referenced_by_obj) self._external_functions.pop(self._objective, None) if obj is not None: self._objective = obj self._objective_expr = obj.expr self._objective_sense = obj.sense tmp = collect_vars_and_named_exprs(obj.expr) - named_exprs, variables, fixed_vars, external_functions = tmp + named_exprs, variables, fixed_vars, parameters, external_functions = tmp self._check_for_new_vars(variables) + self._check_for_new_params(parameters) self._obj_named_expressions = [(i, i.expr) for i in named_exprs] if len(external_functions) > 0: self._external_functions[obj] = external_functions self._vars_referenced_by_obj = variables + self._params_referenced_by_obj = parameters for v in variables: self._referenced_variables[id(v)][2] = obj + for p in parameters: + self._referenced_params[id(p)][2] = obj if not self._treat_fixed_vars_as_params: for v in fixed_vars: v.unfix() @@ -407,6 +425,7 @@ def _set_objective(self, obj: ObjectiveData): v.fix() else: self._vars_referenced_by_obj = [] + self._params_referenced_by_obj = [] self._objective = None self._objective_expr = None self._objective_sense = None @@ -414,13 +433,7 @@ def _set_objective(self, obj: ObjectiveData): for obs in self._observers: obs.set_objective(obj) - def add_block(self, block): - param_dict = {} - for p in block.component_objects(Param, descend_into=True): - if p.mutable: - for _p in p.values(): - param_dict[id(_p)] = _p - self._add_parameters(list(param_dict.values())) + def _add_block(self, block): self._add_constraints( list( block.component_data_objects(Constraint, descend_into=True, active=True) @@ -447,11 +460,15 @@ def _remove_constraints(self, cons: List[ConstraintData]): ) for v in self._vars_referenced_by_con[con]: self._referenced_variables[id(v)][0].pop(con) + for p in self._params_referenced_by_con[con]: + self._referenced_params[id(p)][0].pop(con) self._check_to_remove_vars(self._vars_referenced_by_con[con]) + self._check_to_remove_params(self._params_referenced_by_con[con]) del self._active_constraints[con] del self._named_expressions[con] self._external_functions.pop(con, None) del self._vars_referenced_by_con[con] + del self._params_referenced_by_con[con] def _remove_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: @@ -463,10 +480,14 @@ def _remove_sos_constraints(self, cons: List[SOSConstraintData]): ) for v in self._vars_referenced_by_con[con]: self._referenced_variables[id(v)][1].pop(con) + for p in self._params_referenced_by_con[con]: + self._referenced_params[id(p)][1].pop(con) self._check_to_remove_vars(self._vars_referenced_by_con[con]) - del self._active_constraints[con] + self._check_to_remove_params(self._params_referenced_by_con[con]) + del self._active_sos[con] del self._named_expressions[con] del self._vars_referenced_by_con[con] + del self._params_referenced_by_con[con] def _remove_variables(self, variables: List[VarData]): for obs in self._observers: @@ -489,39 +510,20 @@ def _remove_parameters(self, params: List[ParamData]): for obs in self._observers: obs.remove_parameters(params) for p in params: - del self._params[id(p)] - - def _remove_block(self, block): - self._remove_constraints( - list( - block.component_data_objects( - ctype=Constraint, descend_into=True, active=True + p_id = id(p) + if p_id not in self._referenced_params: + raise ValueError( + f'Cannot remove parameter {p.name} - it has not been added' ) - ) - ) - self._remove_sos_constraints( - list( - block.component_data_objects( - ctype=SOSConstraint, descend_into=True, active=True + cons_using, sos_using, obj_using = self._referenced_params[p_id] + if cons_using or sos_using or (obj_using is not None): + raise ValueError( + f'Cannot remove parameter {p.name} - it is still being used by constraints or the objective' ) - ) - ) - self._remove_parameters( - list( - dict( - (id(p), p) - for p in block.component_data_objects( - ctype=Param, descend_into=True - ) - ).values() - ) - ) + del self._referenced_params[p_id] + del self._params[p_id] - @abc.abstractmethod def _update_variables(self, variables: List[VarData]): - pass - - def update_variables(self, variables: List[VarData]): for v in variables: self._vars[id(v)] = ( v, @@ -531,109 +533,138 @@ def update_variables(self, variables: List[VarData]): v.domain.get_interval(), v.value, ) - self._update_variables(variables) + for obs in self._observers: + obs.update_variables(variables) - @abc.abstractmethod - def update_parameters(self): - pass + def _update_parameters_and_fixed_variables(self, params, variables): + for p in params: + self._params[id(p)] = (p, p.value) + for v in variables: + self._vars[id(v)][5] = v.value + for obs in self._observers: + obs.update_parameters_and_fixed_variables(params, variables) - def update(self, timer: HierarchicalTimer = None): - if timer is None: - timer = HierarchicalTimer() - config = self._active_config.auto_updates - new_vars = [] - old_vars = [] - new_params = [] - old_params = [] + def _check_for_new_or_removed_sos(self): + new_sos = [] + old_sos = [] + current_sos_dict = { + c: None + for c in self._model.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + } + for c in current_sos_dict.keys(): + if c not in self._active_sos: + new_sos.append(c) + for c in self._active_sos: + if c not in current_sos_dict: + old_sos.append(c) + return new_sos, old_sos + + def _check_for_new_or_removed_constraints(self): new_cons = [] old_cons = [] - old_sos = [] - new_sos = [] - current_cons_dict = {} - current_sos_dict = {} - timer.start('vars') - if config.update_vars: - start_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} - timer.stop('vars') - timer.start('params') - if config.check_for_new_or_removed_params: - current_params_dict = {} - for p in self._model.component_objects(Param, descend_into=True): - if p.mutable: - for _p in p.values(): - current_params_dict[id(_p)] = _p - for p_id, p in current_params_dict.items(): - if p_id not in self._params: - new_params.append(p) - for p_id, p in self._params.items(): - if p_id not in current_params_dict: - old_params.append(p) - timer.stop('params') - timer.start('cons') - if config.check_for_new_or_removed_constraints or config.update_constraints: - current_cons_dict = { - c: None - for c in self._model.component_data_objects( - Constraint, descend_into=True, active=True - ) - } - current_sos_dict = { - c: None - for c in self._model.component_data_objects( - SOSConstraint, descend_into=True, active=True - ) - } - for c in current_cons_dict.keys(): - if c not in self._vars_referenced_by_con: - new_cons.append(c) - for c in current_sos_dict.keys(): - if c not in self._vars_referenced_by_con: - new_sos.append(c) - for c in self._vars_referenced_by_con: - if c not in current_cons_dict and c not in current_sos_dict: - if (c.ctype is Constraint) or ( - c.ctype is None and isinstance(c, ConstraintData) - ): - old_cons.append(c) - else: - assert (c.ctype is SOSConstraint) or ( - c.ctype is None and isinstance(c, SOSConstraintData) - ) - old_sos.append(c) - self.remove_constraints(old_cons) - self.remove_sos_constraints(old_sos) - timer.stop('cons') - timer.start('params') - self.remove_parameters(old_params) + current_cons_dict = { + c: None + for c in self._model.component_data_objects( + Constraint, descend_into=True, active=True + ) + } + for c in current_cons_dict.keys(): + if c not in self._active_constraints: + new_cons.append(c) + for c in self._active_constraints: + if c not in current_cons_dict: + old_cons.append(c) + return new_cons, old_cons + + def _check_for_modified_sos(self): + sos_to_update = [] + for c, (old_vlist, old_plist) in self._active_sos.items(): + sos_items = list(c.get_items()) + new_vlist = [i[0] for i in sos_items] + new_plist = [i[1] for i in sos_items] + if len(old_vlist) != len(new_vlist): + sos_to_update.append(c) + elif len(old_plist) != len(new_plist): + sos_to_update.append(c) + else: + needs_update = False + for v1, v2 in zip(old_vlist, new_vlist): + if v1 is not v2: + needs_update = True + break + for p1, p2 in zip(old_plist, new_plist): + if p1 is not p2: + needs_update = True + if needs_update: + break + if needs_update: + sos_to_update.append(c) + return sos_to_update + + def _check_for_modified_constraints(self): + cons_to_update = [] + for c, expr in self._active_constraints.items(): + if c.expr is not expr: + cons_to_update.append(c) + return cons_to_update + + def _check_for_var_changes(self): + vars_to_update = [] + cons_to_update = {} + update_obj = False + for vid, (v, _lb, _ub, _fixed, _domain_interval, _value) in self._vars.items(): + if v.fixed != _fixed: + vars_to_update.append(v) + if self._treat_fixed_vars_as_params: + for c in self._referenced_variables[vid][0]: + cons_to_update[c] = None + + elif v._lb is not _lb: + vars_to_update.append(v) + elif v._ub is not _ub: + vars_to_update.append(v) + + + def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): + if timer is None: + timer = HierarchicalTimer() + config: AutoUpdateConfig = self.config(value=kwds, preserve_implicit=True) + + added_cons = set() + added_sos = set() + + if config.check_for_new_or_removed_constraints: + timer.start('sos') + new_sos, old_sos = self._check_for_new_or_removed_sos() + self._add_sos_constraints(new_sos) + self._remove_sos_constraints(old_sos) + added_sos.update(new_sos) + timer.stop('cons') + timer.start('cons') + new_cons, old_cons = self._check_for_new_or_removed_constraints() + self._add_constraints(new_cons) + self._remove_constraints(old_cons) + added_cons.update(new_cons) + timer.stop('cons') - # sticking this between removal and addition - # is important so that we don't do unnecessary work - if config.update_parameters: - self.update_parameters() + if config.update_constraints: + timer.start('cons') + cons_to_update = self._check_for_modified_constraints() + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + timer.stop('cons') + timer.start('sos') + sos_to_update = self._check_for_modified_sos() + self._remove_sos_constraints(sos_to_update) + self._add_sos_constraints(sos_to_update) + added_sos.update(sos_to_update) + timer.stop('sos') - self.add_parameters(new_params) - timer.stop('params') - timer.start('vars') - self.add_variables(new_vars) - timer.stop('vars') - timer.start('cons') - self.add_constraints(new_cons) - self.add_sos_constraints(new_sos) - new_cons_set = set(new_cons) - new_sos_set = set(new_sos) - cons_to_remove_and_add = {} need_to_set_objective = False - if config.update_constraints: - for c in current_cons_dict.keys(): - if c not in new_cons_set and c.expr is not self._active_constraints[c]: - cons_to_remove_and_add[c] = None - sos_to_update = [] - for c in current_sos_dict.keys(): - if c not in new_sos_set: - sos_to_update.append(c) - self.remove_sos_constraints(sos_to_update) - self.add_sos_constraints(sos_to_update) - timer.stop('cons') + timer.start('vars') if config.update_vars: end_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} @@ -696,11 +727,18 @@ def update(self, timer: HierarchicalTimer = None): self.set_objective(pyomo_obj) timer.stop('objective') - # this has to be done after the objective and constraints in case the - # old objective/constraints use old variables - timer.start('vars') - self.remove_variables(old_vars) - timer.stop('vars') + if config.update_parameters: + timer.start('params') + modified_params = [] + for pid, (p, old_val) in self._params.items(): + if p.value != old_val: + modified_params.append(p) + modified_vars = [] + for vid, (v, _lb, _ub, _fixed, _domain_interval, _val) in self._vars.items(): + if _fixed and _val != v.value: + modified_vars.append(v) + self._update_parameters_and_fixed_variables(modified_params, modified_vars) + timer.stop('params') class PersistentSolverMixin: diff --git a/pyomo/contrib/observer/tests/__init__.py b/pyomo/contrib/observer/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py new file mode 100644 index 00000000000..46faacae9bb --- /dev/null +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -0,0 +1,91 @@ +from pyomo.core.base.constraint import ConstraintData +from pyomo.core.base.objective import ObjectiveData +from pyomo.core.base.param import ParamData +from pyomo.core.base.sos import SOSConstraintData +from pyomo.core.base.var import VarData +import pyomo.environ as pe +from pyomo.common import unittest +from typing import List +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.common.collections import ComponentMap +import logging + + +logger = logging.getLogger(__name__) + + +class ObserverChecker(Observer): + def __init__(self): + super().__init__() + self.counts = ComponentMap() + """ + counts is be a mapping from component (e.g., variable) to another + mapping from string ('add', 'remove', 'update', or 'value') to an int that + indicates the number of times the corresponding method has been called + """ + + def check(self, expected): + unittest.assertStructuredAlmostEqual( + first=expected, + second=self.counts, + places=7, + ) + + def _process(self, comps, key): + for c in comps: + if c not in self.counts: + self.counts[c] = {'add': 0, 'remove': 0, 'update': 0, 'value': 0} + self.counts[c][key] += 1 + + def add_variables(self, variables: List[VarData]): + self._process(variables, 'add') + + def add_parameters(self, params: List[ParamData]): + self._process(params, 'add') + + def add_constraints(self, cons: List[ConstraintData]): + self._process(cons, 'add') + + def add_sos_constraints(self, cons: List[SOSConstraintData]): + self._process(cons, 'add') + + def set_objective(self, obj: ObjectiveData): + self._process([obj], 'add') + + def remove_constraints(self, cons: List[ConstraintData]): + self._process(cons, 'remove') + + def remove_sos_constraints(self, cons: List[SOSConstraintData]): + self._process(cons, 'remove') + + def remove_variables(self, variables: List[VarData]): + self._process(variables, 'remove') + + def remove_parameters(self, params: List[ParamData]): + self._process(params, 'remove') + + def update_variables(self, variables: List[VarData]): + self._process(variables, 'update') + + def update_parameters_and_fixed_variables(self, params: List[ParamData], variables: List[VarData]): + self._process(params, 'value') + self._process(variables, 'value') + + +class TestChangeDetector(unittest.TestCase): + def test_basics(self): + m = pe.ConcreteModel() + m.x = pe.Var() + m.y = pe.Var() + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) + + detector.set_instance(m) + + expected = ComponentMap() + + obs.check(expected) + + def test_vars_and_params_elsewhere(self): + pass \ No newline at end of file From a69d5e69ea9729a4c1ae0396719fd173065d55a9 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 6 Aug 2025 06:20:19 -0600 Subject: [PATCH 03/33] working on model change detector --- pyomo/contrib/observer/component_collector.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 pyomo/contrib/observer/component_collector.py diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py new file mode 100644 index 00000000000..149c4038bc6 --- /dev/null +++ b/pyomo/contrib/observer/component_collector.py @@ -0,0 +1,77 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +from pyomo.core.expr.visitor import StreamBasedExpressionVisitor +from pyomo.core.expr.numeric_expr import ExternalFunctionExpression, NPV_ExternalFunctionExpression +from pyomo.core.base.var import VarData, ScalarVar +from pyomo.core.base.param import ParamData, ScalarParam +from pyomo.core.base.expression import ExpressionData, ScalarExpression + + +def handle_var(node, collector): + collector.variables[id(node)] = node + return None + + +def handle_param(node, collector): + collector.params[id(node)] = node + return None + + +def handle_named_expression(node, collector): + collector.named_expressions[id(node)] = node + return None + + +def handle_external_function(node, collector): + collector.external_functions[id(node)] = node + return None + + +collector_handlers = { + VarData: handle_var, + ScalarVar: handle_var, + ParamData: handle_param, + ScalarParam: handle_param, + ExpressionData: handle_named_expression, + ScalarExpression: handle_named_expression, + ExternalFunctionExpression: handle_external_function, + NPV_ExternalFunctionExpression: handle_external_function, +} + + +class _ComponentFromExprCollector(StreamBasedExpressionVisitor): + def __init__(self): + self.named_expressions = {} + self.variables = {} + self.params = {} + self.external_functions = {} + + def exitNode(self, node, data): + nt = type(node) + if nt in collector_handlers: + return collector_handlers[nt](node, self) + else: + return None + + +_visitor = _ComponentFromExprCollector() + + +def collect_components_from_expr(expr): + _visitor.__init__() + _visitor.walk_expression(expr) + return ( + list(_visitor.named_expressions.values()), + list(_visitor.variables.values()), + list(_visitor.params.values()), + list(_visitor.external_functions.values()), + ) From 9763e9a2f594e74a26bd5003682b963812d0acb3 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 7 Aug 2025 08:26:14 -0600 Subject: [PATCH 04/33] observer --- pyomo/contrib/observer/component_collector.py | 3 +- pyomo/contrib/observer/model_observer.py | 313 ++++++++---------- .../observer/tests/test_change_detector.py | 118 ++++++- 3 files changed, 250 insertions(+), 184 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index 149c4038bc6..5cbbdaf31bd 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -49,11 +49,12 @@ def handle_external_function(node, collector): class _ComponentFromExprCollector(StreamBasedExpressionVisitor): - def __init__(self): + def __init__(self, **kwds): self.named_expressions = {} self.variables = {} self.params = {} self.external_functions = {} + super().__init__(**kwds) def exitNode(self, node, data): nt = type(node) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index b6768f2ac6b..422eb1da574 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -23,7 +23,8 @@ from pyomo.common.collections import ComponentMap from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.results import Results -from pyomo.contrib.solver.common.util import collect_vars_and_named_exprs, get_objective +from pyomo.contrib.solver.common.util import get_objective +from .component_collector import collect_components_from_expr from pyomo.common.numeric_types import native_numeric_types @@ -121,7 +122,7 @@ def __init__( update_parameters_and_fixed_vars.""", ), ) - self.update_parameters_and_fixed_vars: bool = self.declare( + self.update_parameters: bool = self.declare( 'update_parameters', ConfigValue( domain=bool, @@ -181,7 +182,7 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): pass @abc.abstractmethod - def set_objective(self, obj: ObjectiveData): + def set_objective(self, obj: Optional[ObjectiveData]): pass @abc.abstractmethod @@ -205,18 +206,13 @@ def update_variables(self, variables: List[VarData]): pass @abc.abstractmethod - def update_parameters_and_fixed_variables( - self, - params: List[ParamData], - variables: List[VarData], - ): + def update_parameters(self, params: List[ParamData]): pass class ModelChangeDetector: def __init__( self, observers: Sequence[Observer], - treat_fixed_vars_as_params=True, **kwds, ): """ @@ -224,16 +220,6 @@ def __init__( ---------- observers: Sequence[Observer] The objects to notify when changes are made to the model - treat_fixed_vars_as_params: bool - This is an advanced option that should only be used in special circumstances. - With the default setting of True, fixed variables will be treated like parameters. - This means that z == x*y will be linear if x or y is fixed and the constraint - can be written to an LP file. If the value of the fixed variable gets changed, we have - to completely reprocess all constraints using that variable. If - treat_fixed_vars_as_params is False, then constraints will be processed as if fixed - variables are not fixed, and the solver will be told the variable is fixed. This means - z == x*y could not be written to an LP file even if x and/or y is fixed. However, - updating the values of fixed variables is much faster this way. """ self._observers: List[Observer] = list(observers) self._model = None @@ -260,12 +246,11 @@ def __init__( self._params_referenced_by_con = {} self._params_referenced_by_obj = [] self._expr_types = None - self._treat_fixed_vars_as_params = treat_fixed_vars_as_params self.config: AutoUpdateConfig = AutoUpdateConfig()(value=kwds, preserve_implicit=True) def set_instance(self, model): saved_config = self.config - self.__init__(observers=self._observers, treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) + self.__init__(observers=self._observers) self.config = saved_config self._model = model self._add_block(model) @@ -331,37 +316,38 @@ def _check_to_remove_params(self, params: List[ParamData]): self._remove_parameters(list(params_to_remove.values())) def _add_constraints(self, cons: List[ConstraintData]): - all_fixed_vars = {} + vars_to_check = [] + params_to_check = [] for con in cons: if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') self._active_constraints[con] = con.expr - tmp = collect_vars_and_named_exprs(con.expr) - named_exprs, variables, fixed_vars, parameters, external_functions = tmp - self._check_for_new_vars(variables) - self._check_for_new_params(parameters) + tmp = collect_components_from_expr(con.expr) + named_exprs, variables, parameters, external_functions = tmp + vars_to_check.extend(variables) + params_to_check.extend(parameters) self._named_expressions[con] = [(e, e.expr) for e in named_exprs] if len(external_functions) > 0: self._external_functions[con] = external_functions self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = parameters + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) + for con in cons: + variables = self._vars_referenced_by_con[con] + parameters = self._params_referenced_by_con[con] for v in variables: self._referenced_variables[id(v)][0][con] = None for p in parameters: self._referenced_params[id(p)][0][con] = None - if not self._treat_fixed_vars_as_params: - for v in fixed_vars: - v.unfix() - all_fixed_vars[id(v)] = v for obs in self._observers: obs.add_constraints(cons) - for v in all_fixed_vars.values(): - v.fix() def _add_sos_constraints(self, cons: List[SOSConstraintData]): - all_fixed_vars = {} + vars_to_check = [] + params_to_check = [] for con in cons: - if con in self._vars_referenced_by_con: + if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') sos_items = list(con.get_items()) self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) @@ -373,8 +359,8 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): continue if p.is_parameter_type(): params.append(p) - self._check_for_new_vars(variables) - self._check_for_new_params(params) + vars_to_check.extend(variables) + params_to_check.extend(params) self._named_expressions[con] = [] self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params @@ -382,29 +368,28 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): self._referenced_variables[id(v)][1][con] = None for p in params: self._referenced_params[id(p)][1][con] = None - if not self._treat_fixed_vars_as_params: - for v in variables: - if v.is_fixed(): - v.unfix() - all_fixed_vars[id(v)] = v + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) for obs in self._observers: obs.add_sos_constraints(cons) - for v in all_fixed_vars.values(): - v.fix() - def _set_objective(self, obj: ObjectiveData): + def _set_objective(self, obj: Optional[ObjectiveData]): + vars_to_remove_check = [] + params_to_remove_check = [] if self._objective is not None: for v in self._vars_referenced_by_obj: self._referenced_variables[id(v)][2] = None - self._check_to_remove_vars(self._vars_referenced_by_obj) - self._check_to_remove_params(self._params_referenced_by_obj) + for p in self._params_referenced_by_obj: + self._referenced_params[id(p)][2] = None + vars_to_remove_check.extend(self._vars_referenced_by_obj) + params_to_remove_check.extend(self._params_referenced_by_obj) self._external_functions.pop(self._objective, None) if obj is not None: self._objective = obj self._objective_expr = obj.expr self._objective_sense = obj.sense - tmp = collect_vars_and_named_exprs(obj.expr) - named_exprs, variables, fixed_vars, parameters, external_functions = tmp + tmp = collect_components_from_expr(obj.expr) + named_exprs, variables, parameters, external_functions = tmp self._check_for_new_vars(variables) self._check_for_new_params(parameters) self._obj_named_expressions = [(i, i.expr) for i in named_exprs] @@ -416,13 +401,6 @@ def _set_objective(self, obj: ObjectiveData): self._referenced_variables[id(v)][2] = obj for p in parameters: self._referenced_params[id(p)][2] = obj - if not self._treat_fixed_vars_as_params: - for v in fixed_vars: - v.unfix() - for obs in self._observers: - obs.set_objective(obj) - for v in fixed_vars: - v.fix() else: self._vars_referenced_by_obj = [] self._params_referenced_by_obj = [] @@ -430,8 +408,10 @@ def _set_objective(self, obj: ObjectiveData): self._objective_expr = None self._objective_sense = None self._obj_named_expressions = [] - for obs in self._observers: - obs.set_objective(obj) + for obs in self._observers: + obs.set_objective(obj) + self._check_to_remove_vars(vars_to_remove_check) + self._check_to_remove_params(params_to_remove_check) def _add_block(self, block): self._add_constraints( @@ -447,14 +427,15 @@ def _add_block(self, block): ) ) obj = get_objective(block) - if obj is not None: - self._set_objective(obj) + self._set_objective(obj) def _remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.remove_constraints(cons) + vars_to_check = [] + params_to_check = [] for con in cons: - if con not in self._named_expressions: + if con not in self._active_constraints: raise ValueError( f'Cannot remove constraint {con.name} - it was not added' ) @@ -462,19 +443,23 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._referenced_variables[id(v)][0].pop(con) for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][0].pop(con) - self._check_to_remove_vars(self._vars_referenced_by_con[con]) - self._check_to_remove_params(self._params_referenced_by_con[con]) + vars_to_check.extend(self._vars_referenced_by_con[con]) + params_to_check.extend(self._params_referenced_by_con[con]) del self._active_constraints[con] del self._named_expressions[con] self._external_functions.pop(con, None) del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] + self._check_to_remove_vars(vars_to_check) + self._check_to_remove_params(params_to_check) def _remove_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.remove_sos_constraints(cons) + vars_to_check = [] + params_to_check = [] for con in cons: - if con not in self._vars_referenced_by_con: + if con not in self._active_sos: raise ValueError( f'Cannot remove constraint {con.name} - it was not added' ) @@ -482,12 +467,14 @@ def _remove_sos_constraints(self, cons: List[SOSConstraintData]): self._referenced_variables[id(v)][1].pop(con) for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][1].pop(con) - self._check_to_remove_vars(self._vars_referenced_by_con[con]) - self._check_to_remove_params(self._params_referenced_by_con[con]) + vars_to_check.extend(self._vars_referenced_by_con[con]) + params_to_check.extend(self._params_referenced_by_con[con]) del self._active_sos[con] del self._named_expressions[con] del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] + self._check_to_remove_vars(vars_to_check) + self._check_to_remove_params(params_to_check) def _remove_variables(self, variables: List[VarData]): for obs in self._observers: @@ -536,13 +523,11 @@ def _update_variables(self, variables: List[VarData]): for obs in self._observers: obs.update_variables(variables) - def _update_parameters_and_fixed_variables(self, params, variables): + def _update_parameters(self, params): for p in params: self._params[id(p)] = (p, p.value) - for v in variables: - self._vars[id(v)][5] = v.value for obs in self._observers: - obs.update_parameters_and_fixed_variables(params, variables) + obs.update_parameters(params) def _check_for_new_or_removed_sos(self): new_sos = [] @@ -617,15 +602,60 @@ def _check_for_var_changes(self): for vid, (v, _lb, _ub, _fixed, _domain_interval, _value) in self._vars.items(): if v.fixed != _fixed: vars_to_update.append(v) - if self._treat_fixed_vars_as_params: - for c in self._referenced_variables[vid][0]: - cons_to_update[c] = None - + for c in self._referenced_variables[vid][0]: + cons_to_update[c] = None + if self._referenced_variables[vid][2] is not None: + update_obj = True elif v._lb is not _lb: vars_to_update.append(v) elif v._ub is not _ub: vars_to_update.append(v) - + elif _domain_interval != v.domain.get_interval(): + vars_to_update.append(v) + elif v.value != _value: + vars_to_update.append(v) + cons_to_update = list(cons_to_update.keys()) + return vars_to_update, cons_to_update, update_obj + + def _check_for_param_changes(self): + params_to_update = [] + for pid, (p, val) in self._params.items(): + if p.value != val: + params_to_update.append(p) + return params_to_update + + def _check_for_named_expression_changes(self): + cons_to_update = [] + for con, ne_list in self._named_expressions.items(): + for named_expr, old_expr in ne_list: + if named_expr.expr is not old_expr: + cons_to_update.append(con) + break + update_obj = False + ne_list = self._obj_named_expressions + for named_expr, old_expr in ne_list: + if named_expr.expr is not old_expr: + update_obj = True + break + return cons_to_update, update_obj + + def _check_for_new_objective(self): + update_obj = False + new_obj = get_objective(self._model) + if new_obj is not self._objective: + update_obj = True + return new_obj, update_obj + + def _check_for_objective_changes(self): + update_obj = False + if self._objective is None: + return update_obj + if self._objective.expr is not self._objective_expr: + update_obj = True + elif self._objective.sense != self._objective_sense: + # we can definitely do something faster here than resetting the whole objective + update_obj = True + return update_obj def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if timer is None: @@ -641,7 +671,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): self._add_sos_constraints(new_sos) self._remove_sos_constraints(old_sos) added_sos.update(new_sos) - timer.stop('cons') + timer.stop('sos') timer.start('cons') new_cons, old_cons = self._check_for_new_or_removed_constraints() self._add_constraints(new_cons) @@ -665,115 +695,46 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): need_to_set_objective = False - timer.start('vars') if config.update_vars: - end_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} - vars_to_check = [v for v_id, v in end_vars.items() if v_id in start_vars] - if config.update_vars: - vars_to_update = [] - for v in vars_to_check: - _v, lb, ub, fixed, domain_interval, value = self._vars[id(v)] - if (fixed != v.fixed) or (fixed and (value != v.value)): - vars_to_update.append(v) - if self._treat_fixed_vars_as_params: - for c in self._referenced_variables[id(v)][0]: - cons_to_remove_and_add[c] = None - if self._referenced_variables[id(v)][2] is not None: - need_to_set_objective = True - elif lb is not v._lb: - vars_to_update.append(v) - elif ub is not v._ub: - vars_to_update.append(v) - elif domain_interval != v.domain.get_interval(): - vars_to_update.append(v) - self.update_variables(vars_to_update) - timer.stop('vars') - timer.start('cons') - cons_to_remove_and_add = list(cons_to_remove_and_add.keys()) - self.remove_constraints(cons_to_remove_and_add) - self.add_constraints(cons_to_remove_and_add) - timer.stop('cons') - timer.start('named expressions') + timer.start('vars') + vars_to_update, cons_to_update, update_obj = self._check_for_var_changes() + self._update_variables(vars_to_update) + cons_to_update = [i for i in cons_to_update if i not in added_cons] + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + if update_obj: + need_to_set_objective = True + timer.stop('vars') + if config.update_named_expressions: - cons_to_update = [] - for c, expr_list in self._named_expressions.items(): - if c in new_cons_set: - continue - for named_expr, old_expr in expr_list: - if named_expr.expr is not old_expr: - cons_to_update.append(c) - break - self.remove_constraints(cons_to_update) - self.add_constraints(cons_to_update) - for named_expr, old_expr in self._obj_named_expressions: - if named_expr.expr is not old_expr: - need_to_set_objective = True - break - timer.stop('named expressions') - timer.start('objective') - if self._active_config.auto_updates.check_for_new_objective: - pyomo_obj = get_objective(self._model) - if pyomo_obj is not self._objective: + timer.start('named expressions') + cons_to_update, update_obj = self._check_for_named_expression_changes() + cons_to_update = [i for i in cons_to_update if i not in added_cons] + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + if update_obj: need_to_set_objective = True - else: - pyomo_obj = self._objective - if self._active_config.auto_updates.update_objective: - if pyomo_obj is not None and pyomo_obj.expr is not self._objective_expr: + timer.stop('named expressions') + + timer.start('objective') + new_obj = self._objective + if config.check_for_new_objective: + new_obj, update_obj = self._check_for_new_objective() + if update_obj: need_to_set_objective = True - elif pyomo_obj is not None and pyomo_obj.sense is not self._objective_sense: - # we can definitely do something faster here than resetting the whole objective + if config.update_objective: + update_obj = self._check_for_objective_changes() + if update_obj: need_to_set_objective = True + if need_to_set_objective: - self.set_objective(pyomo_obj) + self._set_objective(new_obj) timer.stop('objective') if config.update_parameters: timer.start('params') - modified_params = [] - for pid, (p, old_val) in self._params.items(): - if p.value != old_val: - modified_params.append(p) - modified_vars = [] - for vid, (v, _lb, _ub, _fixed, _domain_interval, _val) in self._vars.items(): - if _fixed and _val != v.value: - modified_vars.append(v) - self._update_parameters_and_fixed_variables(modified_params, modified_vars) + params_to_update = self._check_for_param_changes() + self._update_parameters(params_to_update) timer.stop('params') - - -class PersistentSolverMixin: - """ - The `solve` method in Gurobi and Highs is exactly the same, so this Mixin - minimizes the duplicate code - """ - - def solve(self, model, **kwds) -> Results: - start_timestamp = datetime.datetime.now(datetime.timezone.utc) - self._active_config = config = self.config(value=kwds, preserve_implicit=True) - StaleFlagManager.mark_all_as_stale() - - if self._last_results_object is not None: - self._last_results_object.solution_loader.invalidate() - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer - - if model is not self._model: - timer.start('set_instance') - self.set_instance(model) - timer.stop('set_instance') - else: - timer.start('update') - self.update(timer=timer) - timer.stop('update') - - res = self._solve() - self._last_results_object = res - - end_timestamp = datetime.datetime.now(datetime.timezone.utc) - res.timing_info.start_timestamp = start_timestamp - res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() - res.timing_info.timer = timer - self._active_config = self.config - - return res diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 46faacae9bb..4ed0fff8e45 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -14,13 +14,18 @@ logger = logging.getLogger(__name__) +def make_count_dict(): + d = {'add': 0, 'remove': 0, 'update': 0, 'set': 0} + return d + + class ObserverChecker(Observer): def __init__(self): super().__init__() self.counts = ComponentMap() """ counts is be a mapping from component (e.g., variable) to another - mapping from string ('add', 'remove', 'update', or 'value') to an int that + mapping from string ('add', 'remove', 'update', or 'set') to an int that indicates the number of times the corresponding method has been called """ @@ -34,57 +39,156 @@ def check(self, expected): def _process(self, comps, key): for c in comps: if c not in self.counts: - self.counts[c] = {'add': 0, 'remove': 0, 'update': 0, 'value': 0} + self.counts[c] = make_count_dict() self.counts[c][key] += 1 def add_variables(self, variables: List[VarData]): + for v in variables: + assert v.is_variable_type() self._process(variables, 'add') def add_parameters(self, params: List[ParamData]): + for p in params: + assert p.is_parameter_type() self._process(params, 'add') def add_constraints(self, cons: List[ConstraintData]): + for c in cons: + assert isinstance(c, ConstraintData) self._process(cons, 'add') def add_sos_constraints(self, cons: List[SOSConstraintData]): + for c in cons: + assert isinstance(c, SOSConstraintData) self._process(cons, 'add') def set_objective(self, obj: ObjectiveData): - self._process([obj], 'add') + assert obj is None or isinstance(obj, ObjectiveData) + self._process([obj], 'set') def remove_constraints(self, cons: List[ConstraintData]): + for c in cons: + assert isinstance(c, ConstraintData) self._process(cons, 'remove') def remove_sos_constraints(self, cons: List[SOSConstraintData]): + for c in cons: + assert isinstance(c, SOSConstraintData) self._process(cons, 'remove') def remove_variables(self, variables: List[VarData]): + for v in variables: + assert v.is_variable_type() self._process(variables, 'remove') def remove_parameters(self, params: List[ParamData]): + for p in params: + assert p.is_parameter_type() self._process(params, 'remove') def update_variables(self, variables: List[VarData]): + for v in variables: + assert v.is_variable_type() self._process(variables, 'update') - def update_parameters_and_fixed_variables(self, params: List[ParamData], variables: List[VarData]): - self._process(params, 'value') - self._process(variables, 'value') + def update_parameters(self, params: List[ParamData]): + for p in params: + assert p.is_parameter_type() + self._process(params, 'update') class TestChangeDetector(unittest.TestCase): - def test_basics(self): + def test_objective(self): m = pe.ConcreteModel() m.x = pe.Var() m.y = pe.Var() + m.p = pe.Param(mutable=True, initialize=1) obs = ObserverChecker() detector = ModelChangeDetector([obs]) + expected = ComponentMap() + expected[None] = make_count_dict() + expected[None]['set'] += 1 + detector.set_instance(m) + obs.check(expected) + + m.obj = pe.Objective(expr=m.x**2 + m.p*m.y**2) + detector.update() + expected[m.obj] = make_count_dict() + expected[m.obj]['set'] += 1 + expected[m.x] = make_count_dict() + expected[m.x]['add'] += 1 + expected[m.y] = make_count_dict() + expected[m.y]['add'] += 1 + expected[m.p] = make_count_dict() + expected[m.p]['add'] += 1 + obs.check(expected) + + m.y.setlb(0) + detector.update() + expected[m.y]['update'] += 1 + obs.check(expected) + + m.x.fix(2) + detector.update() + expected[m.x]['update'] += 1 + expected[m.obj]['set'] += 1 + obs.check(expected) + + m.x.unfix() + detector.update() + expected[m.x]['update'] += 1 + expected[m.obj]['set'] += 1 + obs.check(expected) + + m.p.value = 2 + detector.update() + expected[m.p]['update'] += 1 + obs.check(expected) + + m.obj.expr = m.x**2 + m.y**2 + detector.update() + expected[m.p]['remove'] += 1 + expected[m.obj]['set'] += 1 + obs.check(expected) + + del m.obj + m.obj = pe.Objective(expr=m.p*m.x) + detector.update() + expected[m.p]['add'] += 1 + expected[m.y]['remove'] += 1 + # remember, m.obj is a different object now + expected[m.obj] = make_count_dict() + expected[m.obj]['set'] += 1 + + def test_constraints(self): + m = pe.ConcreteModel() + m.x = pe.Var() + m.y = pe.Var() + m.p = pe.Param(mutable=True, initialize=1) + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) expected = ComponentMap() + expected[None] = make_count_dict() + expected[None]['set'] += 1 + + detector.set_instance(m) + obs.check(expected) + m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p)**2) + detector.update() + expected[m.x] = make_count_dict() + expected[m.y] = make_count_dict() + expected[m.p] = make_count_dict() + expected[m.x]['add'] += 1 + expected[m.y]['add'] += 1 + expected[m.p]['add'] += 1 + expected[m.c1] = make_count_dict() + expected[m.c1]['add'] += 1 obs.check(expected) def test_vars_and_params_elsewhere(self): From ff635b88c03e6a8ba22148cf32dfe7061c066954 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 8 Aug 2025 07:31:55 -0600 Subject: [PATCH 05/33] working on a model observer --- .../observer/tests/test_change_detector.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 4ed0fff8e45..efda8a181d9 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -42,6 +42,12 @@ def _process(self, comps, key): self.counts[c] = make_count_dict() self.counts[c][key] += 1 + def pprint(self): + for k, d in self.counts.items(): + print(f'{k}:') + for a, v in d.items(): + print(f' {a}: {v}') + def add_variables(self, variables: List[VarData]): for v in variables: assert v.is_variable_type() @@ -179,6 +185,7 @@ def test_constraints(self): detector.set_instance(m) obs.check(expected) + m.obj = pe.Objective(expr=m.y) m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p)**2) detector.update() expected[m.x] = make_count_dict() @@ -189,6 +196,27 @@ def test_constraints(self): expected[m.p]['add'] += 1 expected[m.c1] = make_count_dict() expected[m.c1]['add'] += 1 + expected[m.obj] = make_count_dict() + expected[m.obj]['set'] += 1 + obs.check(expected) + + # now fix a variable and make sure the + # constraint gets removed and added + m.x.fix(1) + obs.pprint() + detector.update() + obs.pprint() + expected[m.c1]['remove'] += 1 + expected[m.c1]['add'] += 1 + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then + # added again when the constraint is added + expected[m.x]['update'] += 1 + expected[m.x]['remove'] += 1 + expected[m.x]['add'] += 1 + expected[m.p]['remove'] += 1 + expected[m.p]['add'] += 1 obs.check(expected) def test_vars_and_params_elsewhere(self): From b57ab07cde98cc387f8ee0c4f1724f209b05084a Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 9 Aug 2025 07:06:26 -0600 Subject: [PATCH 06/33] updating solution loader --- .../contrib/solver/common/solution_loader.py | 106 +++++++++++++++--- pyomo/contrib/solver/solvers/highs.py | 4 +- 2 files changed, 92 insertions(+), 18 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index 911d8bee50d..065c00185f6 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -9,7 +9,7 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -from typing import Sequence, Dict, Optional, Mapping, NoReturn +from typing import Sequence, Dict, Optional, Mapping, List, Any from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.var import VarData @@ -23,24 +23,75 @@ class SolutionLoaderBase: Intent of this class and its children is to load the solution back into the model. """ - def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> NoReturn: + def get_solution_ids(self) -> List[Any]: """ - Load the solution of the primal variables into the value attribute of the variables. + If there are multiple solutions available, this will return a + list of the solution ids which can then be used with other + methods like `load_soltuion`. If only one solution is + available, this will return [None]. If no solutions + are available, this will return None + + Returns + ------- + solutions_ids: List[Any] + The identifiers for multiple solutions + """ + return NotImplemented + + def get_number_of_solutions(self) -> int: + """ + Returns + ------- + num_solutions: int + Indicates the number of solutions found + """ + return NotImplemented + + def load_solution(self, solution_id=None): + """ + Load the solution (everything that can be) back into the model + + Parameters + ---------- + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be loaded. If None, the default solution will be used. + """ + # this should load everything it can + self.load_vars(solution_id=solution_id) + self.load_import_suffixes(solution_id=solution_id) + + def load_vars( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, + ) -> None: + """ + Load the solution of the primal variables into the value attribute + of the variables. Parameters ---------- vars_to_load: list - The minimum set of variables whose solution should be loaded. If vars_to_load - is None, then the solution to all primal variables will be loaded. Even if - vars_to_load is specified, the values of other variables may also be - loaded depending on the interface. + The minimum set of variables whose solution should be loaded. If + vars_to_load is None, then the solution to all primal variables + will be loaded. Even if vars_to_load is specified, the values of + other variables may also be loaded depending on the interface. + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be loaded. If None, the default solution will be used. """ - for var, val in self.get_primals(vars_to_load=vars_to_load).items(): + for var, val in self.get_vars( + vars_to_load=vars_to_load, + solution_id=solution_id + ).items(): var.set_value(val, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) - def get_primals( - self, vars_to_load: Optional[Sequence[VarData]] = None + def get_vars( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: """ Returns a ComponentMap mapping variable to var value. @@ -50,6 +101,9 @@ def get_primals( vars_to_load: list A list of the variables whose solution value should be retrieved. If vars_to_load is None, then the values for all variables will be retrieved. + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be retrieved. If None, the default solution will be used. Returns ------- @@ -57,11 +111,13 @@ def get_primals( Maps variables to solution values """ raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'get_primals'." + f"Derived class {self.__class__.__name__} failed to implement required method 'get_vars'." ) def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, + cons_to_load: Optional[Sequence[ConstraintData]] = None, + solution_id=None, ) -> Dict[ConstraintData, float]: """ Returns a dictionary mapping constraint to dual value. @@ -71,16 +127,21 @@ def get_duals( cons_to_load: list A list of the constraints whose duals should be retrieved. If cons_to_load is None, then the duals for all constraints will be retrieved. + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be retrieved. If None, the default solution will be used. Returns ------- duals: dict Maps constraints to dual values """ - raise NotImplementedError(f'{type(self)} does not support the get_duals method') + return NotImplemented def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: """ Returns a ComponentMap mapping variable to reduced cost. @@ -90,15 +151,26 @@ def get_reduced_costs( vars_to_load: list A list of the variables whose reduced cost should be retrieved. If vars_to_load is None, then the reduced costs for all variables will be loaded. + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be retrieved. If None, the default solution will be used. Returns ------- reduced_costs: ComponentMap Maps variables to reduced costs """ - raise NotImplementedError( - f'{type(self)} does not support the get_reduced_costs method' - ) + return NotImplemented + + def load_import_suffixes(self, solution_id=None): + """ + Parameters + ---------- + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be loaded. If None, the default solution will be used. + """ + return NotImplemented class PersistentSolutionLoader(SolutionLoaderBase): diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index 2fdac4942c8..6eb4afa828a 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -306,7 +306,9 @@ def _solve(self): self._solver_model.run() timer.stop('optimize') - return self._postsolve() + res = self._postsolve() + res.solver_log = ostreams[0].getvalue() + return res def _process_domain_and_bounds(self, var_id): _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[var_id] From 070811d91c992691f16bbf7b97d9de2310fc7398 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 9 Aug 2025 07:20:11 -0600 Subject: [PATCH 07/33] refactoring gurobi interfaces --- .../solver/solvers/gurobi_persistent.py | 156 ++++++++++++------ 1 file changed, 108 insertions(+), 48 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi_persistent.py index ea3693c1c70..899b7915e80 100644 --- a/pyomo/contrib/solver/solvers/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi_persistent.py @@ -30,7 +30,7 @@ from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types from pyomo.repn import generate_standard_repn from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability +from pyomo.contrib.solver.common.base import PersistentSolverBase, SolverBase, Availability from pyomo.contrib.solver.common.results import ( Results, TerminationCondition, @@ -233,44 +233,15 @@ def __init__(self): self.var2 = None -class GurobiPersistent( - GurobiSolverMixin, - PersistentSolverMixin, - PersistentSolverUtils, - PersistentSolverBase, -): - """ - Interface to Gurobi persistent - """ - +class GurobiBase(SolverBase): CONFIG = GurobiConfig() _gurobipy_available = gurobipy_available def __init__(self, **kwds): - treat_fixed_vars_as_params = kwds.pop('treat_fixed_vars_as_params', True) - PersistentSolverBase.__init__(self, **kwds) - PersistentSolverUtils.__init__( - self, treat_fixed_vars_as_params=treat_fixed_vars_as_params - ) + super().__init__(**kwds) self._register_env_client() self._solver_model = None - self._symbol_map = SymbolMap() - self._labeler = None - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._solver_con_to_pyomo_con_map = {} - self._pyomo_sos_to_solver_sos_map = {} - self._range_constraints = OrderedSet() - self._mutable_helpers = {} - self._mutable_bounds = {} - self._mutable_quadratic_helpers = {} - self._mutable_objective = None - self._needs_updated = True self._callback = None - self._callback_func = None - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._last_results_object: Optional[Results] = None def release_license(self): self._reinit() @@ -280,12 +251,10 @@ def __del__(self): if not python_is_shutting_down(): self._release_env_client() - @property - def symbol_map(self): - return self._symbol_map + def _mipstart(self): + raise NotImplementedError('should be implemented by derived classes') - def _solve(self): - config = self._active_config + def _solve(self, config): timer = config.timer ostreams = [io.StringIO()] + config.tee @@ -304,13 +273,7 @@ def _solve(self): self._solver_model.setParam('MIPGapAbs', config.abs_gap) if config.use_mipstart: - for ( - pyomo_var_id, - gurobi_var, - ) in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[pyomo_var_id][0] - if pyomo_var.is_integer() and pyomo_var.value is not None: - self.set_var_attr(pyomo_var, 'Start', pyomo_var.value) + self._mipstart() for key, option in options.items(): self._solver_model.setParam(key, option) @@ -319,18 +282,99 @@ def _solve(self): self._solver_model.optimize(self._callback) timer.stop('optimize') - self._needs_updated = False res = self._postsolve(timer) res.solver_config = config res.solver_name = 'Gurobi' res.solver_version = self.version() res.solver_log = ostreams[0].getvalue() return res + + +class GurobiDirect(GurobiBase): + def __init__(self, **kwds): + super().__init__(**kwds) + + +class GurobiQuadraticBase(GurobiBase): + def __init__(self, **kwds): + super().__init__(**kwds) + self._vars = {} # from id(v) to v + self._symbol_map = SymbolMap() + self._labeler = None + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + + @property + def symbol_map(self): + return self._symbol_map + + def _mipstart(self): + for ( + pyomo_var_id, + gurobi_var, + ) in self._pyomo_var_to_solver_var_map.items(): + pyomo_var = self._vars[pyomo_var_id] + if pyomo_var.is_integer() and pyomo_var.value is not None: + gurobi_var.setAttr('Start', pyomo_var.value) + + def _proces_domain_and_bounds(self, var): + lb, ub, step = var.domain.get_interval() + if lb is None: + lb = -gurobipy.GRB.INFINITY + if ub is None: + ub = gurobipy.GRB.INFINITY + if step == 0: + vtype = gurobipy.GRB.CONTINUOUS + elif step == 1: + if lb == 0 and ub == 1: + vtype = gurobipy.GRB.BINARY + else: + vtype = gurobipy.GRB.INTEGER + else: + raise ValueError( + f'Unrecognized domain step: {step} (should be either 0 or 1)' + ) + if var.fixed: + lb = var.value + ub = lb + else: + lb = max(lb, value(var._lb)) + ub = min(ub, value(var._ub)) + return lb, ub, vtype + + +class GurobiDirectQuadratic(GurobiQuadraticBase): + def __init__(self, **kwds): + super().__init__(**kwds) + + +class GurobiPersistentQuadratic(GurobiQuadraticBase): + def __init__(self, **kwds): + super().__init__(**kwds) + self._solver_con_to_pyomo_con_map = {} + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._callback_func = None + self._last_results_object: Optional[Results] = None + + def _solve(self, config): + super()._solve(config) + self._needs_updated = False def _process_domain_and_bounds( self, var, var_id, mutable_lbs, mutable_ubs, ndx, gurobipy_var ): - _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[id(var)] + _lb = var._lb + _ub = var._ub + _fixed = var.fixed + _domain_interval = var.domain.get_interval() + _value = var.value lb, ub, step = _domain_interval if lb is None: lb = -gurobipy.GRB.INFINITY @@ -372,6 +416,24 @@ def _process_domain_and_bounds( return lb, ub, vtype + +class _GurobiPersistent( + GurobiSolverMixin, + PersistentSolverMixin, + PersistentSolverUtils, + PersistentSolverBase, +): + """ + Interface to Gurobi persistent + """ + + + def __init__(self, **kwds): + PersistentSolverBase.__init__(self, **kwds) + PersistentSolverUtils.__init__( + self, treat_fixed_vars_as_params=treat_fixed_vars_as_params + ) + def _add_variables(self, variables: List[VarData]): var_names = [] vtypes = [] @@ -522,7 +584,6 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobipy_con = self._solver_model.addRange( gurobi_expr, lhs_val, rhs_val, name=conname ) - self._range_constraints.add(con) if not is_constant(lhs_expr) or not is_constant(rhs_expr): mutable_range_constant = _MutableRangeConstant() mutable_range_constant.lhs_expr = lhs_expr @@ -654,7 +715,6 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._symbol_map.removeSymbol(con) del self._pyomo_con_to_solver_con_map[con] del self._solver_con_to_pyomo_con_map[id(solver_con)] - self._range_constraints.discard(con) self._mutable_helpers.pop(con, None) self._mutable_quadratic_helpers.pop(con, None) self._needs_updated = True From d70dbb52a28aeb3500bdd430919f43f94c7a862c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 10 Aug 2025 15:36:39 -0600 Subject: [PATCH 08/33] revert_gurobi_persistent --- .../solver/solvers/gurobi_persistent.py | 156 ++++++------------ 1 file changed, 48 insertions(+), 108 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi_persistent.py index 899b7915e80..ea3693c1c70 100644 --- a/pyomo/contrib/solver/solvers/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi_persistent.py @@ -30,7 +30,7 @@ from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types from pyomo.repn import generate_standard_repn from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, SolverBase, Availability +from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability from pyomo.contrib.solver.common.results import ( Results, TerminationCondition, @@ -233,15 +233,44 @@ def __init__(self): self.var2 = None -class GurobiBase(SolverBase): +class GurobiPersistent( + GurobiSolverMixin, + PersistentSolverMixin, + PersistentSolverUtils, + PersistentSolverBase, +): + """ + Interface to Gurobi persistent + """ + CONFIG = GurobiConfig() _gurobipy_available = gurobipy_available def __init__(self, **kwds): - super().__init__(**kwds) + treat_fixed_vars_as_params = kwds.pop('treat_fixed_vars_as_params', True) + PersistentSolverBase.__init__(self, **kwds) + PersistentSolverUtils.__init__( + self, treat_fixed_vars_as_params=treat_fixed_vars_as_params + ) self._register_env_client() self._solver_model = None + self._symbol_map = SymbolMap() + self._labeler = None + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._solver_con_to_pyomo_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + self._range_constraints = OrderedSet() + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True self._callback = None + self._callback_func = None + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._last_results_object: Optional[Results] = None def release_license(self): self._reinit() @@ -251,10 +280,12 @@ def __del__(self): if not python_is_shutting_down(): self._release_env_client() - def _mipstart(self): - raise NotImplementedError('should be implemented by derived classes') + @property + def symbol_map(self): + return self._symbol_map - def _solve(self, config): + def _solve(self): + config = self._active_config timer = config.timer ostreams = [io.StringIO()] + config.tee @@ -273,7 +304,13 @@ def _solve(self, config): self._solver_model.setParam('MIPGapAbs', config.abs_gap) if config.use_mipstart: - self._mipstart() + for ( + pyomo_var_id, + gurobi_var, + ) in self._pyomo_var_to_solver_var_map.items(): + pyomo_var = self._vars[pyomo_var_id][0] + if pyomo_var.is_integer() and pyomo_var.value is not None: + self.set_var_attr(pyomo_var, 'Start', pyomo_var.value) for key, option in options.items(): self._solver_model.setParam(key, option) @@ -282,99 +319,18 @@ def _solve(self, config): self._solver_model.optimize(self._callback) timer.stop('optimize') + self._needs_updated = False res = self._postsolve(timer) res.solver_config = config res.solver_name = 'Gurobi' res.solver_version = self.version() res.solver_log = ostreams[0].getvalue() return res - - -class GurobiDirect(GurobiBase): - def __init__(self, **kwds): - super().__init__(**kwds) - - -class GurobiQuadraticBase(GurobiBase): - def __init__(self, **kwds): - super().__init__(**kwds) - self._vars = {} # from id(v) to v - self._symbol_map = SymbolMap() - self._labeler = None - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._pyomo_sos_to_solver_sos_map = {} - - @property - def symbol_map(self): - return self._symbol_map - - def _mipstart(self): - for ( - pyomo_var_id, - gurobi_var, - ) in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[pyomo_var_id] - if pyomo_var.is_integer() and pyomo_var.value is not None: - gurobi_var.setAttr('Start', pyomo_var.value) - - def _proces_domain_and_bounds(self, var): - lb, ub, step = var.domain.get_interval() - if lb is None: - lb = -gurobipy.GRB.INFINITY - if ub is None: - ub = gurobipy.GRB.INFINITY - if step == 0: - vtype = gurobipy.GRB.CONTINUOUS - elif step == 1: - if lb == 0 and ub == 1: - vtype = gurobipy.GRB.BINARY - else: - vtype = gurobipy.GRB.INTEGER - else: - raise ValueError( - f'Unrecognized domain step: {step} (should be either 0 or 1)' - ) - if var.fixed: - lb = var.value - ub = lb - else: - lb = max(lb, value(var._lb)) - ub = min(ub, value(var._ub)) - return lb, ub, vtype - - -class GurobiDirectQuadratic(GurobiQuadraticBase): - def __init__(self, **kwds): - super().__init__(**kwds) - - -class GurobiPersistentQuadratic(GurobiQuadraticBase): - def __init__(self, **kwds): - super().__init__(**kwds) - self._solver_con_to_pyomo_con_map = {} - self._mutable_helpers = {} - self._mutable_bounds = {} - self._mutable_quadratic_helpers = {} - self._mutable_objective = None - self._needs_updated = True - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._callback_func = None - self._last_results_object: Optional[Results] = None - - def _solve(self, config): - super()._solve(config) - self._needs_updated = False def _process_domain_and_bounds( self, var, var_id, mutable_lbs, mutable_ubs, ndx, gurobipy_var ): - _lb = var._lb - _ub = var._ub - _fixed = var.fixed - _domain_interval = var.domain.get_interval() - _value = var.value + _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[id(var)] lb, ub, step = _domain_interval if lb is None: lb = -gurobipy.GRB.INFINITY @@ -416,24 +372,6 @@ def _process_domain_and_bounds( return lb, ub, vtype - -class _GurobiPersistent( - GurobiSolverMixin, - PersistentSolverMixin, - PersistentSolverUtils, - PersistentSolverBase, -): - """ - Interface to Gurobi persistent - """ - - - def __init__(self, **kwds): - PersistentSolverBase.__init__(self, **kwds) - PersistentSolverUtils.__init__( - self, treat_fixed_vars_as_params=treat_fixed_vars_as_params - ) - def _add_variables(self, variables: List[VarData]): var_names = [] vtypes = [] @@ -584,6 +522,7 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobipy_con = self._solver_model.addRange( gurobi_expr, lhs_val, rhs_val, name=conname ) + self._range_constraints.add(con) if not is_constant(lhs_expr) or not is_constant(rhs_expr): mutable_range_constant = _MutableRangeConstant() mutable_range_constant.lhs_expr = lhs_expr @@ -715,6 +654,7 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._symbol_map.removeSymbol(con) del self._pyomo_con_to_solver_con_map[con] del self._solver_con_to_pyomo_con_map[id(solver_con)] + self._range_constraints.discard(con) self._mutable_helpers.pop(con, None) self._mutable_quadratic_helpers.pop(con, None) self._needs_updated = True From 5b1d3f9cfb551598599a8f2ecf99313517bf06f7 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 10 Aug 2025 20:43:45 -0600 Subject: [PATCH 09/33] refactoring gurobi interfaces --- .../contrib/solver/solvers/gurobi/__init__.py | 0 .../solver/solvers/gurobi/gurobi_direct.py | 201 +++++++++++ .../solvers/gurobi/gurobi_direct_base.py | 328 ++++++++++++++++++ 3 files changed, 529 insertions(+) create mode 100644 pyomo/contrib/solver/solvers/gurobi/__init__.py create mode 100644 pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py create mode 100644 pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py diff --git a/pyomo/contrib/solver/solvers/gurobi/__init__.py b/pyomo/contrib/solver/solvers/gurobi/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py new file mode 100644 index 00000000000..5c36372ef72 --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -0,0 +1,201 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import datetime +import io +import math +import operator +import os + +from pyomo.common.collections import ComponentMap, ComponentSet +from pyomo.common.config import ConfigValue +from pyomo.common.dependencies import attempt_import +from pyomo.common.enums import ObjectiveSense +from pyomo.common.errors import MouseTrap, ApplicationError +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.timing import HierarchicalTimer +from pyomo.core.staleflag import StaleFlagManager + +from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.config import BranchAndBoundConfig +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, + NoDualsError, + NoReducedCostsError, + NoSolutionError, + IncompatibleModelError, +) +from pyomo.contrib.solver.common.results import ( + Results, + SolutionStatus, + TerminationCondition, +) +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +import logging +from .gurobi_direct_base import GurobiDirectBase, gurobipy + + +class GurobiDirectSolutionLoader(SolutionLoaderBase): + def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars): + self._grb_model = grb_model + self._grb_cons = grb_cons + self._grb_vars = grb_vars + self._pyo_cons = pyo_cons + self._pyo_vars = pyo_vars + GurobiDirectBase._register_env_client() + + def __del__(self): + if python_is_shutting_down(): + return + # Free the associated model + if self._grb_model is not None: + self._grb_cons = None + self._grb_vars = None + self._pyo_cons = None + self._pyo_vars = None + # explicitly release the model + self._grb_model.dispose() + self._grb_model = None + # Release the gurobi license if this is the last reference to + # the environment (either through a results object or solver + # interface) + GurobiDirectBase._release_env_client() + + def load_vars(self, vars_to_load=None, solution_number=0): + assert solution_number == 0 + if self._grb_model.SolCount == 0: + raise NoSolutionError() + + iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) + if vars_to_load: + vars_to_load = ComponentSet(vars_to_load) + iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) + for p_var, g_var in iterator: + p_var.set_value(g_var, skip_validation=True) + StaleFlagManager.mark_all_as_stale(delayed=True) + + def get_primals(self, vars_to_load=None, solution_number=0): + assert solution_number == 0 + if self._grb_model.SolCount == 0: + raise NoSolutionError() + + iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) + if vars_to_load: + vars_to_load = ComponentSet(vars_to_load) + iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) + return ComponentMap(iterator) + + def get_duals(self, cons_to_load=None): + if self._grb_model.Status != gurobipy.GRB.OPTIMAL: + raise NoDualsError() + + def dedup(_iter): + last = None + for con_info_dual in _iter: + if not con_info_dual[1] and con_info_dual[0][0] is last: + continue + last = con_info_dual[0][0] + yield con_info_dual + + iterator = dedup(zip(self._pyo_cons, self._grb_cons.getAttr('Pi').tolist())) + if cons_to_load: + cons_to_load = set(cons_to_load) + iterator = filter( + lambda con_info_dual: con_info_dual[0][0] in cons_to_load, iterator + ) + return {con_info[0]: dual for con_info, dual in iterator} + + def get_reduced_costs(self, vars_to_load=None): + if self._grb_model.Status != gurobipy.GRB.OPTIMAL: + raise NoReducedCostsError() + + iterator = zip(self._pyo_vars, self._grb_vars.getAttr('Rc').tolist()) + if vars_to_load: + vars_to_load = ComponentSet(vars_to_load) + iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) + return ComponentMap(iterator) + + +class GurobiDirect(GurobiDirectBase): + def __init__(self, **kwds): + super().__init__(**kwds) + self._gurobi_vars = None + self._pyomo_vars = None + + def _pyomo_gurobi_var_iter(self): + return zip(self._pyomo_vars, self._gurobi_vars.tolist()) + + def _create_solver_model(self, pyomo_model, config): + timer = config.timer + + timer.start('compile_model') + repn = LinearStandardFormCompiler().write( + pyomo_model, mixed_form=True, set_sense=None + ) + timer.stop('compile_model') + + if len(repn.objectives) > 1: + raise IncompatibleModelError( + f"The {self.__class__.__name__} solver only supports models " + f"with zero or one objectives (received {len(repn.objectives)})." + ) + + timer.start('prepare_matrices') + inf = float('inf') + ninf = -inf + bounds = list(map(operator.attrgetter('bounds'), repn.columns)) + lb = [ninf if _b is None else _b for _b in map(operator.itemgetter(0), bounds)] + ub = [inf if _b is None else _b for _b in map(operator.itemgetter(1), bounds)] + CON = gurobipy.GRB.CONTINUOUS + BIN = gurobipy.GRB.BINARY + INT = gurobipy.GRB.INTEGER + vtype = [ + ( + CON + if v.is_continuous() + else BIN if v.is_binary() else INT if v.is_integer() else '?' + ) + for v in repn.columns + ] + sense_type = list('=<>') # Note: ordering matches 0, 1, -1 + sense = [sense_type[r[1]] for r in repn.rows] + timer.stop('prepare_matrices') + + gurobi_model = gurobipy.Model(env=self.env()) + + timer.start('transfer_model') + x = gurobi_model.addMVar( + len(repn.columns), + lb=lb, + ub=ub, + obj=repn.c.todense()[0] if repn.c.shape[0] else 0, + vtype=vtype, + ) + A = gurobi_model.addMConstr(repn.A, x, sense, repn.rhs) + if repn.c.shape[0]: + gurobi_model.setAttr('ObjCon', repn.c_offset[0]) + gurobi_model.setAttr('ModelSense', int(repn.objectives[0].sense)) + # Note: calling gurobi_model.update() here is not + # necessary (it will happen as part of optimize()): + # gurobi_model.update() + timer.stop('transfer_model') + + self._pyomo_vars = repn.columns + self._gurobi_vars = x + + solution_loader = GurobiDirectSolutionLoader( + gurobi_model, A, x, repn.rows, repn.columns + ) + has_obj = len(repn.objectives) > 0 + + return gurobi_model, solution_loader, has_obj diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py new file mode 100644 index 00000000000..01c91b8b2ed --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -0,0 +1,328 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import datetime +import io +import math +import operator +import os + +from pyomo.common.collections import ComponentMap, ComponentSet +from pyomo.common.config import ConfigValue +from pyomo.common.dependencies import attempt_import +from pyomo.common.enums import ObjectiveSense +from pyomo.common.errors import MouseTrap, ApplicationError +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.timing import HierarchicalTimer +from pyomo.core.staleflag import StaleFlagManager + +from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.config import BranchAndBoundConfig +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, + NoDualsError, + NoReducedCostsError, + NoSolutionError, + IncompatibleModelError, +) +from pyomo.contrib.solver.common.results import ( + Results, + SolutionStatus, + TerminationCondition, +) +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +import logging + + +logger = logging.getLogger(__name__) + + +gurobipy, gurobipy_available = attempt_import('gurobipy') + + +class GurobiConfig(BranchAndBoundConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + BranchAndBoundConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.use_mipstart: bool = self.declare( + 'use_mipstart', + ConfigValue( + default=False, + domain=bool, + description="If True, the current values of the integer variables " + "will be passed to Gurobi.", + ), + ) + + +class GurobiDirectBase(SolverBase): + + _num_gurobipy_env_clients = 0 + _gurobipy_env = None + _available = None + _gurobipy_available = gurobipy_available + _tc_map = None + + CONFIG = GurobiConfig() + + def __init__(self, **kwds): + super().__init__(**kwds) + self._register_env_client() + + def __del__(self): + if not python_is_shutting_down(): + self._release_env_client() + + def available(self): + if self._available is None: + # this triggers the deferred import, and for the persistent + # interface, may update the _available flag + # + # Note that we set the _available flag on the *most derived + # class* and not on the instance, or on the base class. That + # allows different derived interfaces to have different + # availability (e.g., persistent has a minimum version + # requirement that the direct interface doesn't - is that true?) + if not self._gurobipy_available: + if self._available is None: + self.__class__._available = Availability.NotFound + else: + self.__class__._available = self._check_license() + return self._available + + @staticmethod + def release_license(): + if GurobiDirectBase._gurobipy_env is None: + return + if GurobiDirectBase._num_gurobipy_env_clients: + logger.warning( + "Call to GurobiDirectBase.release_license() with %s remaining " + "environment clients." % (GurobiDirectBase._num_gurobipy_env_clients,) + ) + GurobiDirectBase._gurobipy_env.close() + GurobiDirectBase._gurobipy_env = None + + @staticmethod + def env(): + if GurobiDirectBase._gurobipy_env is None: + with capture_output(capture_fd=True): + GurobiDirectBase._gurobipy_env = gurobipy.Env() + return GurobiDirectBase._gurobipy_env + + @staticmethod + def _register_env_client(): + GurobiDirectBase._num_gurobipy_env_clients += 1 + + @staticmethod + def _release_env_client(): + GurobiDirectBase._num_gurobipy_env_clients -= 1 + if GurobiDirectBase._num_gurobipy_env_clients <= 0: + # Note that _num_gurobipy_env_clients should never be <0, + # but if it is, release_license will issue a warning (that + # we want to know about) + GurobiDirectBase.release_license() + + def _check_license(self): + try: + model = gurobipy.Model(env=self.env()) + except gurobipy.GurobiError: + return Availability.BadLicense + + model.setParam('OutputFlag', 0) + try: + model.addVars(range(2001)) + model.optimize() + return Availability.FullLicense + except gurobipy.GurobiError: + return Availability.LimitedLicense + finally: + model.dispose() + + def version(self): + version = ( + gurobipy.GRB.VERSION_MAJOR, + gurobipy.GRB.VERSION_MINOR, + gurobipy.GRB.VERSION_TECHNICAL, + ) + return version + + def _create_solver_model(self, pyomo_model, config): + # should return gurobi_model, solution_loader, has_objective + raise NotImplementedError('should be implemented by derived classes') + + def _pyomo_gurobi_var_iter(self): + # generator of tuples (pyomo_var, gurobi_var) + raise NotImplementedError('should be implemented by derived classes') + + def _mipstart(self): + for pyomo_var, gurobi_var in self._pyomo_gurobi_var_iter(): + if pyomo_var.is_integer() and pyomo_var.value is not None: + gurobi_var.setAttr('Start', pyomo_var.value) + + def solve(self, model, **kwds) -> Results: + start_timestamp = datetime.datetime.now(datetime.timezone.utc) + config: GurobiConfig = self.config( + value=kwds, + preserve_implicit=True, + ) + if not self.available(): + c = self.__class__ + raise ApplicationError( + f'Solver {c.__module__}.{c.__qualname__} is not available ' + f'({self.available()}).' + ) + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + StaleFlagManager.mark_all_as_stale() + ostreams = [io.StringIO()] + config.tee + + orig_cwd = os.getcwd() + try: + if config.working_dir: + os.chdir(config.working_dir) + with capture_output(TeeStream(*ostreams), capture_fd=False): + gurobi_model, solution_loader, has_obj = self._create_solver_model(model, config) + options = config.solver_options + + gurobi_model.setParam('LogToConsole', 1) + + if config.threads is not None: + gurobi_model.setParam('Threads', config.threads) + if config.time_limit is not None: + gurobi_model.setParam('TimeLimit', config.time_limit) + if config.rel_gap is not None: + gurobi_model.setParam('MIPGap', config.rel_gap) + if config.abs_gap is not None: + gurobi_model.setParam('MIPGapAbs', config.abs_gap) + + if config.use_mipstart: + self._mipstart() + + for key, option in options.items(): + gurobi_model.setParam(key, option) + + timer.start('optimize') + gurobi_model.optimize() + timer.stop('optimize') + finally: + os.chdir(orig_cwd) + + res = self._postsolve( + grb_model=gurobi_model, + config=config, + ) + + res.solution_loader = solution_loader + res.solver_log = ostreams[0].getvalue() + end_timestamp = datetime.datetime.now(datetime.timezone.utc) + res.timing_info.start_timestamp = start_timestamp + res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() + res.timing_info.timer = timer + return res + + def _get_tc_map(self): + if GurobiDirectBase._tc_map is None: + grb = gurobipy.GRB + tc = TerminationCondition + GurobiDirectBase._tc_map = { + grb.LOADED: tc.unknown, # problem is loaded, but no solution + grb.OPTIMAL: tc.convergenceCriteriaSatisfied, + grb.INFEASIBLE: tc.provenInfeasible, + grb.INF_OR_UNBD: tc.infeasibleOrUnbounded, + grb.UNBOUNDED: tc.unbounded, + grb.CUTOFF: tc.objectiveLimit, + grb.ITERATION_LIMIT: tc.iterationLimit, + grb.NODE_LIMIT: tc.iterationLimit, + grb.TIME_LIMIT: tc.maxTimeLimit, + grb.SOLUTION_LIMIT: tc.unknown, + grb.INTERRUPTED: tc.interrupted, + grb.NUMERIC: tc.unknown, + grb.SUBOPTIMAL: tc.unknown, + grb.USER_OBJ_LIMIT: tc.objectiveLimit, + } + return GurobiDirectBase._tc_map + + def _postsolve(self, grb_model, config, has_obj): + status = grb_model.Status + + results = Results() + results.timing_info.gurobi_time = grb_model.Runtime + + if grb_model.SolCount > 0: + if status == gurobipy.GRB.OPTIMAL: + results.solution_status = SolutionStatus.optimal + else: + results.solution_status = SolutionStatus.feasible + else: + results.solution_status = SolutionStatus.noSolution + + results.termination_condition = self._get_tc_map().get( + status, TerminationCondition.unknown + ) + + if ( + results.termination_condition + != TerminationCondition.convergenceCriteriaSatisfied + and config.raise_exception_on_nonoptimal_result + ): + raise NoOptimalSolutionError() + + if has_obj: + try: + if math.isfinite(grb_model.ObjVal): + results.incumbent_objective = grb_model.ObjVal + else: + results.incumbent_objective = None + except (gurobipy.GurobiError, AttributeError): + results.incumbent_objective = None + try: + results.objective_bound = grb_model.ObjBound + except (gurobipy.GurobiError, AttributeError): + if grb_model.ModelSense == ObjectiveSense.minimize: + results.objective_bound = -math.inf + else: + results.objective_bound = math.inf + else: + results.incumbent_objective = None + results.objective_bound = None + + results.iteration_count = grb_model.getAttr('IterCount') + + config.timer.start('load solution') + if config.load_solutions: + if grb_model.SolCount > 0: + results.solution_loader.load_vars() + else: + raise NoFeasibleSolutionError() + config.timer.stop('load solution') + + results.solver_config = config + results.solver_name = self.name + results.solver_version = self.version() + + return results From 4818130badff45f8873326b32710bfba2e87e7fd Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 11 Aug 2025 06:23:20 -0600 Subject: [PATCH 10/33] refactoring gurobi interfaces --- .../solver/solvers/gurobi/gurobi_direct.py | 7 +- .../solvers/gurobi/gurobi_direct_base.py | 72 ++-- .../solvers/gurobi/gurobi_persistent.py | 335 ++++++++++++++++++ 3 files changed, 381 insertions(+), 33 deletions(-) create mode 100644 pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 5c36372ef72..f4a33e2cc54 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -24,6 +24,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer from pyomo.core.staleflag import StaleFlagManager +from pyomo.repn.plugins.standard_form import LinearStandardFormCompiler from pyomo.contrib.solver.common.base import SolverBase, Availability from pyomo.contrib.solver.common.config import BranchAndBoundConfig @@ -127,6 +128,8 @@ def get_reduced_costs(self, vars_to_load=None): class GurobiDirect(GurobiDirectBase): + _minimum_version = (9, 0, 0) + def __init__(self, **kwds): super().__init__(**kwds) self._gurobi_vars = None @@ -135,8 +138,8 @@ def __init__(self, **kwds): def _pyomo_gurobi_var_iter(self): return zip(self._pyomo_vars, self._gurobi_vars.tolist()) - def _create_solver_model(self, pyomo_model, config): - timer = config.timer + def _create_solver_model(self, pyomo_model): + timer = self.config.timer timer.start('compile_model') repn = LinearStandardFormCompiler().write( diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 01c91b8b2ed..b314a39b49a 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -85,12 +85,14 @@ class GurobiDirectBase(SolverBase): _available = None _gurobipy_available = gurobipy_available _tc_map = None + _minimum_version = (0, 0, 0) CONFIG = GurobiConfig() def __init__(self, **kwds): super().__init__(**kwds) self._register_env_client() + self._callback = None def __del__(self): if not python_is_shutting_down(): @@ -111,6 +113,8 @@ def available(self): self.__class__._available = Availability.NotFound else: self.__class__._available = self._check_license() + if self.version() < self._minimum_version: + self.__class__._available = Availability.BadVersion return self._available @staticmethod @@ -169,7 +173,7 @@ def version(self): ) return version - def _create_solver_model(self, pyomo_model, config): + def _create_solver_model(self, pyomo_model): # should return gurobi_model, solution_loader, has_objective raise NotImplementedError('should be implemented by derived classes') @@ -184,29 +188,30 @@ def _mipstart(self): def solve(self, model, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) - config: GurobiConfig = self.config( - value=kwds, - preserve_implicit=True, - ) - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer - - StaleFlagManager.mark_all_as_stale() - ostreams = [io.StringIO()] + config.tee - + orig_config = self.config orig_cwd = os.getcwd() try: + self.config = config = self.config( + value=kwds, + preserve_implicit=True, + ) + if not self.available(): + c = self.__class__ + raise ApplicationError( + f'Solver {c.__module__}.{c.__qualname__} is not available ' + f'({self.available()}).' + ) + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + StaleFlagManager.mark_all_as_stale() + ostreams = [io.StringIO()] + config.tee + if config.working_dir: os.chdir(config.working_dir) with capture_output(TeeStream(*ostreams), capture_fd=False): - gurobi_model, solution_loader, has_obj = self._create_solver_model(model, config) + gurobi_model, solution_loader, has_obj = self._create_solver_model(model) options = config.solver_options gurobi_model.setParam('LogToConsole', 1) @@ -227,15 +232,16 @@ def solve(self, model, **kwds) -> Results: gurobi_model.setParam(key, option) timer.start('optimize') - gurobi_model.optimize() + gurobi_model.optimize(self._callback) timer.stop('optimize') + + res = self._postsolve( + grb_model=gurobi_model, + has_obj=has_obj, + ) finally: os.chdir(orig_cwd) - - res = self._postsolve( - grb_model=gurobi_model, - config=config, - ) + self.config = orig_config res.solution_loader = solution_loader res.solver_log = ostreams[0].getvalue() @@ -267,7 +273,7 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map - def _postsolve(self, grb_model, config, has_obj): + def _postsolve(self, grb_model, has_obj): status = grb_model.Status results = Results() @@ -288,7 +294,7 @@ def _postsolve(self, grb_model, config, has_obj): if ( results.termination_condition != TerminationCondition.convergenceCriteriaSatisfied - and config.raise_exception_on_nonoptimal_result + and self.config.raise_exception_on_nonoptimal_result ): raise NoOptimalSolutionError() @@ -313,15 +319,19 @@ def _postsolve(self, grb_model, config, has_obj): results.iteration_count = grb_model.getAttr('IterCount') - config.timer.start('load solution') - if config.load_solutions: + self.config.timer.start('load solution') + if self.config.load_solutions: if grb_model.SolCount > 0: results.solution_loader.load_vars() else: raise NoFeasibleSolutionError() - config.timer.stop('load solution') + self.config.timer.stop('load solution') - results.solver_config = config + # self.config gets copied a the beginning of + # solve and restored at the end, so modifying + # results.solver_config will not actually + # modify self.config + results.solver_config = self.config results.solver_name = self.name results.solver_version = self.version() diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py new file mode 100644 index 00000000000..fa269c1d3c5 --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -0,0 +1,335 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import io +import logging +import math +from typing import List, Optional +from collections.abc import Iterable + +from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet +from pyomo.common.dependencies import attempt_import +from pyomo.common.errors import ApplicationError +from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.timing import HierarchicalTimer +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.core.kernel.objective import minimize, maximize +from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler +from pyomo.core.base.var import VarData +from pyomo.core.base.constraint import ConstraintData, Constraint +from pyomo.core.base.sos import SOSConstraintData, SOSConstraint +from pyomo.core.base.param import ParamData +from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types +from pyomo.repn import generate_standard_repn +from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression +from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability +from pyomo.contrib.solver.common.results import ( + Results, + TerminationCondition, + SolutionStatus, +) +from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig +from pyomo.contrib.solver.solvers.gurobi_direct import ( + GurobiConfigMixin, + GurobiSolverMixin, +) +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, + NoDualsError, + NoReducedCostsError, + NoSolutionError, + IncompatibleModelError, +) +from pyomo.contrib.solver.common.persistent import ( + PersistentSolverUtils, + PersistentSolverMixin, +) +from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader +from pyomo.core.staleflag import StaleFlagManager +from .gurobi_direct_base import GurobiConfig, GurobiDirectBase, gurobipy +from pyomo.contrib.solver.common.util import get_objective +from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor + + +logger = logging.getLogger(__name__) + + +class GurobiSolutionLoader(PersistentSolutionLoader): + def load_vars(self, vars_to_load=None, solution_number=0): + self._assert_solution_still_valid() + self._solver._load_vars( + vars_to_load=vars_to_load, solution_number=solution_number + ) + + def get_primals(self, vars_to_load=None, solution_number=0): + self._assert_solution_still_valid() + return self._solver._get_primals( + vars_to_load=vars_to_load, solution_number=solution_number + ) + + +class _MutableLowerBound: + def __init__(self, var_id, expr, var_map): + self.var_id = var_id + self.expr = expr + self.var_map = var_map + + def update(self): + self.var_map[self.var_id].setAttr('lb', value(self.expr)) + + +class _MutableUpperBound: + def __init__(self, var_id, expr, var_map): + self.var_id = var_id + self.expr = expr + self.var_map = var_map + + def update(self): + self.var_map[self.var_id].setAttr('ub', value(self.expr)) + + +class _MutableLinearCoefficient: + def __init__(self): + self.expr = None + self.var = None + self.con = None + self.gurobi_model = None + + def update(self): + self.gurobi_model.chgCoeff(self.con, self.var, value(self.expr)) + + +class _MutableRangeConstant: + def __init__(self): + self.lhs_expr = None + self.rhs_expr = None + self.con = None + self.slack_name = None + self.gurobi_model = None + + def update(self): + rhs_val = value(self.rhs_expr) + lhs_val = value(self.lhs_expr) + self.con.rhs = rhs_val + slack = self.gurobi_model.getVarByName(self.slack_name) + slack.ub = rhs_val - lhs_val + + +class _MutableConstant: + def __init__(self): + self.expr = None + self.con = None + + def update(self): + self.con.rhs = value(self.expr) + + +class _MutableQuadraticConstraint: + def __init__( + self, gurobi_model, gurobi_con, constant, linear_coefs, quadratic_coefs + ): + self.con = gurobi_con + self.gurobi_model = gurobi_model + self.constant = constant + self.last_constant_value = value(self.constant.expr) + self.linear_coefs = linear_coefs + self.last_linear_coef_values = [value(i.expr) for i in self.linear_coefs] + self.quadratic_coefs = quadratic_coefs + self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + + def get_updated_expression(self): + gurobi_expr = self.gurobi_model.getQCRow(self.con) + for ndx, coef in enumerate(self.linear_coefs): + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_linear_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.var + self.last_linear_coef_values[ndx] = current_coef_value + for ndx, coef in enumerate(self.quadratic_coefs): + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_quadratic_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 + self.last_quadratic_coef_values[ndx] = current_coef_value + return gurobi_expr + + def get_updated_rhs(self): + return value(self.constant.expr) + + +class _MutableObjective: + def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): + self.gurobi_model = gurobi_model + self.constant = constant + self.linear_coefs = linear_coefs + self.quadratic_coefs = quadratic_coefs + self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + + def get_updated_expression(self): + for ndx, coef in enumerate(self.linear_coefs): + coef.var.obj = value(coef.expr) + self.gurobi_model.ObjCon = value(self.constant.expr) + + gurobi_expr = None + for ndx, coef in enumerate(self.quadratic_coefs): + if value(coef.expr) != self.last_quadratic_coef_values[ndx]: + if gurobi_expr is None: + self.gurobi_model.update() + gurobi_expr = self.gurobi_model.getObjective() + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_quadratic_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 + self.last_quadratic_coef_values[ndx] = current_coef_value + return gurobi_expr + + +class _MutableQuadraticCoefficient: + def __init__(self): + self.expr = None + self.var1 = None + self.var2 = None + + +class GurobiDirectQuadratic(GurobiDirectBase): + _minimum_version = (7, 0, 0) + + def __init__(self, **kwds): + super().__init__(**kwds) + self._solver_model = None + self._vars = {} # from id(v) to v + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + + def _create_solver_model(self, pyomo_model): + self._clear() + self._solver_model = gurobipy.Model(env=self.env()) + cons = list(pyomo_model.component_data_objects(Constraint, descend_into=True, active=True)) + self._add_constraints(cons) + sos = list(pyomo_model.component_data_objects(SOSConstraint, descend_into=True, active=True)) + self._add_sos_constraints(sos) + obj = get_objective(pyomo_model) + self._set_objective(obj) + + def _clear(self): + self._solver_model = None + self._vars = {} + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + + def _pyomo_gurobi_var_iter(self): + for vid, v in self._vars.items(): + yield v, self._pyomo_var_to_solver_var_map[vid] + + def _process_domain_and_bounds(self, var): + lb, ub, step = var.domain.get_interval() + if lb is None: + lb = -gurobipy.GRB.INFINITY + if ub is None: + ub = gurobipy.GRB.INFINITY + if step == 0: + vtype = gurobipy.GRB.CONTINUOUS + elif step == 1: + if lb == 0 and ub == 1: + vtype = gurobipy.GRB.BINARY + else: + vtype = gurobipy.GRB.INTEGER + else: + raise ValueError( + f'Unrecognized domain: {var.domain}' + ) + if var.fixed: + lb = var.value + ub = lb + else: + lb = max(lb, value(var._lb)) + ub = min(ub, value(var._ub)) + return lb, ub, vtype + + def _add_variables(self, variables: List[VarData]): + vtypes = [] + lbs = [] + ubs = [] + for ndx, var in enumerate(variables): + self._vars[id(var)] = var + lb, ub, vtype = self._process_domain_and_bounds(var) + vtypes.append(vtype) + lbs.append(lb) + ubs.append(ub) + + gurobi_vars = self._solver_model.addVars( + len(variables), lb=lbs, ub=ubs, vtype=vtypes + ) + + for pyomo_var, gurobi_var in zip(variables, gurobi_vars): + self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var + + def _get_expr_from_pyomo_expr(self, expr): + repn = generate_standard_repn(expr, quadratic=True, compute_values=True) + + if repn.nonlinear_expr is not None: + raise IncompatibleModelError( + f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' + ) + + if len(repn.linear_vars) > 0: + missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] + self._add_variables(missing_vars) + new_expr = gurobipy.LinExpr( + repn.linear_coefs, + [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars], + ) + else: + new_expr = 0.0 + + for coef, v in zip(repn.quadratic_coefs, repn.quadratic_vars): + x, y = v + gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] + gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] + new_expr += coef * gurobi_x * gurobi_y + + return new_expr, repn.constant + + def _add_constraints(self, cons: List[ConstraintData]): + gurobi_expr_list = [] + for con in cons: + lb, body, ub = con.to_bounded_expression(evaluate_bounds=True) + gurobi_expr, repn_constant = self._get_expr_from_pyomo_expr(body) + if lb is None and ub is None: + raise ValueError( + "Constraint does not have a lower " + f"or an upper bound: {con} \n" + ) + elif lb is None: + gurobi_expr_list.append(gurobi_expr <= ub - repn_constant) + elif ub is None: + gurobi_expr_list.append(lb - repn_constant <= gurobi_expr) + elif lb == ub: + gurobi_expr_list.append(gurobi_expr == lb - repn_constant) + else: + gurobi_expr_list.append(gurobi_expr == [lb-repn_constant, ub-repn_constant]) + + gurobi_cons = self._solver_model.addConstrs(gurobi_expr_list) + self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) + self._pyomo_con_to_solver_con_map[con] = gurobipy_con + self._solver_con_to_pyomo_con_map[id(gurobipy_con)] = con + self._constraints_added_since_update.update(cons) + self._needs_updated = True + + +class GurobiPersistentQuadratic(GurobiDirectQuadratic): + _minimum_version = (7, 0, 0) From db0fda4310d498ff7d225df06a035a377356b7bb Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Tue, 12 Aug 2025 07:31:39 -0600 Subject: [PATCH 11/33] Apply black --- pyomo/contrib/observer/component_collector.py | 5 +++- pyomo/contrib/observer/model_observer.py | 24 ++++++++-------- .../observer/tests/test_change_detector.py | 28 ++++++++++--------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index 5cbbdaf31bd..d52ec46086c 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -10,7 +10,10 @@ # ___________________________________________________________________________ from pyomo.core.expr.visitor import StreamBasedExpressionVisitor -from pyomo.core.expr.numeric_expr import ExternalFunctionExpression, NPV_ExternalFunctionExpression +from pyomo.core.expr.numeric_expr import ( + ExternalFunctionExpression, + NPV_ExternalFunctionExpression, +) from pyomo.core.base.var import VarData, ScalarVar from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.expression import ExpressionData, ScalarExpression diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 422eb1da574..39b832cc266 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -211,10 +211,7 @@ def update_parameters(self, params: List[ParamData]): class ModelChangeDetector: - def __init__( - self, observers: Sequence[Observer], - **kwds, - ): + def __init__(self, observers: Sequence[Observer], **kwds): """ Parameters ---------- @@ -240,13 +237,15 @@ def __init__( ) # var_id: [dict[constraints, None], dict[sos constraints, None], None or objective] self._referenced_params = ( {} - ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] self._vars_referenced_by_con = {} self._vars_referenced_by_obj = [] self._params_referenced_by_con = {} self._params_referenced_by_obj = [] self._expr_types = None - self.config: AutoUpdateConfig = AutoUpdateConfig()(value=kwds, preserve_implicit=True) + self.config: AutoUpdateConfig = AutoUpdateConfig()( + value=kwds, preserve_implicit=True + ) def set_instance(self, model): saved_config = self.config @@ -350,7 +349,10 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') sos_items = list(con.get_items()) - self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) + self._active_sos[con] = ( + [i[0] for i in sos_items], + [i[1] for i in sos_items], + ) variables = [] params = [] for v, p in sos_items: @@ -616,14 +618,14 @@ def _check_for_var_changes(self): vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) return vars_to_update, cons_to_update, update_obj - + def _check_for_param_changes(self): params_to_update = [] for pid, (p, val) in self._params.items(): if p.value != val: params_to_update.append(p) return params_to_update - + def _check_for_named_expression_changes(self): cons_to_update = [] for con, ne_list in self._named_expressions.items(): @@ -644,7 +646,7 @@ def _check_for_new_objective(self): new_obj = get_objective(self._model) if new_obj is not self._objective: update_obj = True - return new_obj, update_obj + return new_obj, update_obj def _check_for_objective_changes(self): update_obj = False @@ -717,7 +719,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if update_obj: need_to_set_objective = True timer.stop('named expressions') - + timer.start('objective') new_obj = self._objective if config.check_for_new_objective: diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index efda8a181d9..29e0de01eb9 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -6,7 +6,11 @@ import pyomo.environ as pe from pyomo.common import unittest from typing import List -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, +) from pyomo.common.collections import ComponentMap import logging @@ -31,11 +35,9 @@ def __init__(self): def check(self, expected): unittest.assertStructuredAlmostEqual( - first=expected, - second=self.counts, - places=7, + first=expected, second=self.counts, places=7 ) - + def _process(self, comps, key): for c in comps: if c not in self.counts: @@ -120,7 +122,7 @@ def test_objective(self): detector.set_instance(m) obs.check(expected) - m.obj = pe.Objective(expr=m.x**2 + m.p*m.y**2) + m.obj = pe.Objective(expr=m.x**2 + m.p * m.y**2) detector.update() expected[m.obj] = make_count_dict() expected[m.obj]['set'] += 1 @@ -131,7 +133,7 @@ def test_objective(self): expected[m.p] = make_count_dict() expected[m.p]['add'] += 1 obs.check(expected) - + m.y.setlb(0) detector.update() expected[m.y]['update'] += 1 @@ -161,7 +163,7 @@ def test_objective(self): obs.check(expected) del m.obj - m.obj = pe.Objective(expr=m.p*m.x) + m.obj = pe.Objective(expr=m.p * m.x) detector.update() expected[m.p]['add'] += 1 expected[m.y]['remove'] += 1 @@ -186,7 +188,7 @@ def test_constraints(self): obs.check(expected) m.obj = pe.Objective(expr=m.y) - m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p)**2) + m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p) ** 2) detector.update() expected[m.x] = make_count_dict() expected[m.y] = make_count_dict() @@ -208,9 +210,9 @@ def test_constraints(self): obs.pprint() expected[m.c1]['remove'] += 1 expected[m.c1]['add'] += 1 - # because x and p are only used in the - # one constraint, they get removed when - # the constraint is removed and then + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then # added again when the constraint is added expected[m.x]['update'] += 1 expected[m.x]['remove'] += 1 @@ -220,4 +222,4 @@ def test_constraints(self): obs.check(expected) def test_vars_and_params_elsewhere(self): - pass \ No newline at end of file + pass From 42c8cc8423ea87b2e839c8b820b6e3bb643934b2 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 09:54:25 -0600 Subject: [PATCH 12/33] adding copyright statements --- pyomo/contrib/observer/__init__.py | 10 ++++++++++ pyomo/contrib/observer/tests/__init__.py | 10 ++++++++++ pyomo/contrib/observer/tests/test_change_detector.py | 11 +++++++++++ 3 files changed, 31 insertions(+) diff --git a/pyomo/contrib/observer/__init__.py b/pyomo/contrib/observer/__init__.py index e69de29bb2d..6eb9ea8b81d 100644 --- a/pyomo/contrib/observer/__init__.py +++ b/pyomo/contrib/observer/__init__.py @@ -0,0 +1,10 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ diff --git a/pyomo/contrib/observer/tests/__init__.py b/pyomo/contrib/observer/tests/__init__.py index e69de29bb2d..6eb9ea8b81d 100644 --- a/pyomo/contrib/observer/tests/__init__.py +++ b/pyomo/contrib/observer/tests/__init__.py @@ -0,0 +1,10 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 29e0de01eb9..dd7951342da 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -1,3 +1,14 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.objective import ObjectiveData from pyomo.core.base.param import ParamData From 7998fda55861cde77931d14d58b27444d9f3a501 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 10:04:55 -0600 Subject: [PATCH 13/33] refactoring gurobi interfaces --- pyomo/contrib/solver/plugins.py | 8 +- .../solvers/gurobi/gurobi_direct_base.py | 132 ++++++++++- .../solvers/gurobi/gurobi_persistent.py | 220 ++++++++++++++++-- .../solver/tests/solvers/test_solvers.py | 26 ++- 4 files changed, 350 insertions(+), 36 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 86c05f2bd70..7630c614aa2 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -13,7 +13,8 @@ from .common.factory import SolverFactory from .solvers.ipopt import Ipopt from .solvers.gurobi_persistent import GurobiPersistent -from .solvers.gurobi_direct import GurobiDirect +from .solvers.gurobi.gurobi_direct import GurobiDirect +from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic from .solvers.highs import Highs @@ -31,6 +32,11 @@ def load(): legacy_name='gurobi_direct_v2', doc='Direct (scipy-based) interface to Gurobi', )(GurobiDirect) + SolverFactory.register( + name='gurobi_direct_quadratic', + legacy_name='gurobi_direct_quadratic_v2', + doc='Direct interface to Gurobi', + )(GurobiDirect) SolverFactory.register( name='highs', legacy_name='highs', doc='Persistent interface to HiGHS' )(Highs) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index b314a39b49a..d26dbf54c83 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -78,6 +78,122 @@ def __init__( ) +def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_number): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + if ( + solver_model.getAttr('NumIntVars') == 0 + and solver_model.getAttr('NumBinVars') == 0 + ): + raise ValueError( + 'Cannot obtain suboptimal solutions for a continuous model' + ) + original_solution_number = solver_model.getParamInfo('SolutionNumber')[2] + solver_model.setParam('SolutionNumber', solution_number) + gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + vals = solver_model.getAttr("Xn", gurobi_vars_to_load) + res = ComponentMap() + for var, val in zip(vars_to_load, vals): + res[var] = val + solver_model.setParam('SolutionNumber', original_solution_number) + return res + + +def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + for v, val in _get_primals( + solver_model=solver_model, + var_map=var_map, + vars_to_load=vars_to_load, + solution_number=solution_number, + ).items(): + v.set_value(val, skip_validation=True) + StaleFlagManager.mark_all_as_stale(delayed=True) + + +def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + if solver_model.SolCount == 0: + raise NoSolutionError() + + if solution_number != 0: + return _load_suboptimal_mip_solution( + solver_model=solver_model, + var_map=var_map, + vars_to_load=vars_to_load, + solution_number=solution_number, + ) + + gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + vals = solver_model.getAttr("X", gurobi_vars_to_load) + + res = ComponentMap() + for var, val in zip(vars_to_load, vals): + res[var] = val + return res + + +def _get_reduced_costs(solver_model, var_map, vars_to_load): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + """ + if solver_model.Status != gurobipy.GRB.OPTIMAL: + raise NoReducedCostsError() + + res = ComponentMap() + gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + vals = solver_model.getAttr("Rc", gurobi_vars_to_load) + + for var, val in zip(vars_to_load, vals): + res[var] = val + + return res + + +def _get_duals(solver_model, con_map, linear_cons_to_load, quadratic_cons_to_load): + """ + solver_model: gurobipy.Model + con_map: Dict[ConstraintData, gurobipy.Constr] + Maps the pyomo constraint to the gurobipy constraint + linear_cons_to_load: List[ConstraintData] + quadratic_cons_to_load: List[ConstraintData] + """ + if solver_model.Status != gurobipy.GRB.OPTIMAL: + raise NoDualsError() + + linear_gurobi_cons = [con_map[c] for c in linear_cons_to_load] + quadratic_gurobi_cons = [con_map[c] for c in quadratic_cons_to_load] + linear_vals = solver_model.getAttr("Pi", linear_gurobi_cons) + quadratic_vals = solver_model.getAttr("QCPi", quadratic_gurobi_cons) + + duals = {} + for c, val in zip(linear_cons_to_load, linear_vals): + duals[c] = val + for c, val in zip(quadratic_cons_to_load, quadratic_vals): + duals[c] = val + return duals + + class GurobiDirectBase(SolverBase): _num_gurobipy_env_clients = 0 @@ -191,10 +307,15 @@ def solve(self, model, **kwds) -> Results: orig_config = self.config orig_cwd = os.getcwd() try: - self.config = config = self.config( + config = self.config( value=kwds, preserve_implicit=True, ) + + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = config + object.__setattr__(self, 'config', config) + if not self.available(): c = self.__class__ raise ApplicationError( @@ -237,13 +358,17 @@ def solve(self, model, **kwds) -> Results: res = self._postsolve( grb_model=gurobi_model, + solution_loader=solution_loader, has_obj=has_obj, ) finally: os.chdir(orig_cwd) + + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = orig_config + object.__setattr__(self, 'config', orig_config) self.config = orig_config - res.solution_loader = solution_loader res.solver_log = ostreams[0].getvalue() end_timestamp = datetime.datetime.now(datetime.timezone.utc) res.timing_info.start_timestamp = start_timestamp @@ -273,10 +398,11 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map - def _postsolve(self, grb_model, has_obj): + def _postsolve(self, grb_model, solution_loader, has_obj): status = grb_model.Status results = Results() + results.solution_loader = solution_loader results.timing_info.gurobi_time = grb_model.Runtime if grb_model.SolCount > 0: diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index fa269c1d3c5..3ae6e86526c 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -12,7 +12,7 @@ import io import logging import math -from typing import List, Optional +from typing import Dict, List, NoReturn, Optional, Sequence, Mapping from collections.abc import Iterable from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet @@ -53,9 +53,18 @@ PersistentSolverUtils, PersistentSolverMixin, ) -from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader +from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader, SolutionLoaderBase from pyomo.core.staleflag import StaleFlagManager -from .gurobi_direct_base import GurobiConfig, GurobiDirectBase, gurobipy +from .gurobi_direct_base import ( + GurobiConfig, + GurobiDirectBase, + gurobipy, + _load_suboptimal_mip_solution, + _load_vars, + _get_primals, + _get_duals, + _get_reduced_costs, +) from pyomo.contrib.solver.common.util import get_objective from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor @@ -63,7 +72,87 @@ logger = logging.getLogger(__name__) -class GurobiSolutionLoader(PersistentSolutionLoader): +class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): + def __init__( + self, + solver_model, + var_id_map, + var_map, + con_map, + linear_cons, + quadratic_cons, + ) -> None: + super().__init__() + self._solver_model = solver_model + self._vars = var_id_map + self._var_map = var_map + self._con_map = con_map + self._linear_cons = linear_cons + self._quadratic_cons = quadratic_cons + + def load_vars( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=0, + ) -> None: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + _load_vars( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + solution_number=solution_id, + ) + + def get_primals( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=0, + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + return _get_primals( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + solution_number=solution_id, + ) + + def get_reduced_costs( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + return _get_reduced_costs( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + ) + + def get_duals( + self, + cons_to_load: Optional[Sequence[ConstraintData]] = None, + ) -> Dict[ConstraintData, float]: + if cons_to_load is None: + cons_to_load = list(self._con_map.keys()) + linear_cons_to_load = [] + quadratic_cons_to_load = [] + for c in cons_to_load: + if c in self._linear_cons: + linear_cons_to_load.append(c) + else: + assert c in self._quadratic_cons + quadratic_cons_to_load.append(c) + return _get_duals( + solver_model=self._solver_model, + con_map=self._con_map, + linear_cons_to_load=linear_cons_to_load, + quadratic_cons_to_load=quadratic_cons_to_load, + ) + + +class GurobiPersistentSolutionLoader(PersistentSolutionLoader): def load_vars(self, vars_to_load=None, solution_number=0): self._assert_solution_still_valid() self._solver._load_vars( @@ -212,23 +301,50 @@ def __init__(self, **kwds): self._vars = {} # from id(v) to v self._pyomo_var_to_solver_var_map = {} self._pyomo_con_to_solver_con_map = {} + self._linear_cons = set() + self._quadratic_cons = set() self._pyomo_sos_to_solver_sos_map = {} def _create_solver_model(self, pyomo_model): + timer = self.config.timer + timer.start('create gurobipy model') self._clear() self._solver_model = gurobipy.Model(env=self.env()) + timer.start('collect constraints') cons = list(pyomo_model.component_data_objects(Constraint, descend_into=True, active=True)) + timer.stop('collect constraints') + timer.start('translate constraints') self._add_constraints(cons) + timer.stop('translate constraints') + timer.start('sos') sos = list(pyomo_model.component_data_objects(SOSConstraint, descend_into=True, active=True)) self._add_sos_constraints(sos) + timer.stop('sos') + timer.start('get objective') obj = get_objective(pyomo_model) + timer.stop('get objective') + timer.start('translate objective') self._set_objective(obj) + timer.stop('translate objective') + has_obj = obj is not None + solution_loader = GurobiDirectQuadraticSolutionLoader( + solver_model=self._solver_model, + var_id_map=self._vars, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + linear_cons=self._linear_cons, + quadratic_cons=self._quadratic_cons, + ) + timer.stop('create gurobipy model') + return self._solver_model, solution_loader, has_obj def _clear(self): self._solver_model = None self._vars = {} self._pyomo_var_to_solver_var_map = {} self._pyomo_con_to_solver_con_map = {} + self._linear_cons = set() + self._quadratic_cons = set() self._pyomo_sos_to_solver_sos_map = {} def _pyomo_gurobi_var_iter(self): @@ -256,8 +372,10 @@ def _process_domain_and_bounds(self, var): lb = var.value ub = lb else: - lb = max(lb, value(var._lb)) - ub = min(ub, value(var._ub)) + if var._lb is not None: + lb = max(lb, value(var._lb)) + if var._ub is not None: + ub = min(ub, value(var._ub)) return lb, ub, vtype def _add_variables(self, variables: List[VarData]): @@ -273,14 +391,12 @@ def _add_variables(self, variables: List[VarData]): gurobi_vars = self._solver_model.addVars( len(variables), lb=lbs, ub=ubs, vtype=vtypes - ) + ).values() for pyomo_var, gurobi_var in zip(variables, gurobi_vars): self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var - def _get_expr_from_pyomo_expr(self, expr): - repn = generate_standard_repn(expr, quadratic=True, compute_values=True) - + def _get_expr_from_pyomo_repn(self, repn): if repn.nonlinear_expr is not None: raise IncompatibleModelError( f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' @@ -289,18 +405,26 @@ def _get_expr_from_pyomo_expr(self, expr): if len(repn.linear_vars) > 0: missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] self._add_variables(missing_vars) + vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr( repn.linear_coefs, - [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars], + vlist, ) else: new_expr = 0.0 - for coef, v in zip(repn.quadratic_coefs, repn.quadratic_vars): - x, y = v - gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] - gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - new_expr += coef * gurobi_x * gurobi_y + if len(repn.quadratic_vars) > 0: + missing_vars = {} + for x, y in repn.quadratic_vars: + for v in [x, y]: + vid = id(v) + if vid not in self._vars: + missing_vars[vid] = v + self._add_variables(list(missing_vars.values())) + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] + gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] + new_expr += coef * gurobi_x * gurobi_y return new_expr, repn.constant @@ -308,26 +432,72 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobi_expr_list = [] for con in cons: lb, body, ub = con.to_bounded_expression(evaluate_bounds=True) - gurobi_expr, repn_constant = self._get_expr_from_pyomo_expr(body) + repn = generate_standard_repn(body, quadratic=True, compute_values=True) + if len(repn.quadratic_vars) > 0: + self._quadratic_cons.add(con) + else: + self._linear_cons.add(con) + gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) if lb is None and ub is None: raise ValueError( "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: - gurobi_expr_list.append(gurobi_expr <= ub - repn_constant) + gurobi_expr_list.append(gurobi_expr <= float(ub - repn_constant)) elif ub is None: - gurobi_expr_list.append(lb - repn_constant <= gurobi_expr) + gurobi_expr_list.append(float(lb - repn_constant) <= gurobi_expr) elif lb == ub: - gurobi_expr_list.append(gurobi_expr == lb - repn_constant) + gurobi_expr_list.append(gurobi_expr == float(lb - repn_constant)) else: - gurobi_expr_list.append(gurobi_expr == [lb-repn_constant, ub-repn_constant]) + gurobi_expr_list.append(gurobi_expr == [float(lb-repn_constant), float(ub-repn_constant)]) - gurobi_cons = self._solver_model.addConstrs(gurobi_expr_list) + gurobi_cons = self._solver_model.addConstrs((gurobi_expr_list[i] for i in range(len(gurobi_expr_list)))).values() self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) - self._pyomo_con_to_solver_con_map[con] = gurobipy_con - self._solver_con_to_pyomo_con_map[id(gurobipy_con)] = con - self._constraints_added_since_update.update(cons) + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + for con in cons: + level = con.level + if level == 1: + sos_type = gurobipy.GRB.SOS_TYPE1 + elif level == 2: + sos_type = gurobipy.GRB.SOS_TYPE2 + else: + raise ValueError( + f"Solver does not support SOS level {level} constraints" + ) + + gurobi_vars = [] + weights = [] + + missing_vars = {id(v): v for v, w in con.get_items() if id(v) not in self._vars} + self._add_variables(list(missing_vars.values())) + + for v, w in con.get_items(): + v_id = id(v) + gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) + weights.append(w) + + gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) + self._pyomo_sos_to_solver_sos_map[con] = gurobipy_con + + def _set_objective(self, obj): + if obj is None: + sense = gurobipy.GRB.MINIMIZE + gurobi_expr = 0 + repn_constant = 0 + else: + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') + + repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=True) + gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) + + self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) self._needs_updated = True diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 5ab36554061..748e0127151 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -30,8 +30,9 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.solvers.ipopt import Ipopt -from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent -from pyomo.contrib.solver.solvers.gurobi_direct import GurobiDirect +# from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -47,20 +48,31 @@ raise unittest.SkipTest('Parameterized is not available.') all_solvers = [ - ('gurobi_persistent', GurobiPersistent), + # ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('highs', Highs), ] mip_solvers = [ - ('gurobi_persistent', GurobiPersistent), + # ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), ] -nlp_solvers = [('ipopt', Ipopt)] -qcp_solvers = [('gurobi_persistent', GurobiPersistent), ('ipopt', Ipopt)] +nlp_solvers = [ + ('ipopt', Ipopt), +] +qcp_solvers = [ + # ('gurobi_persistent', GurobiPersistent), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), + ('ipopt', Ipopt), +] qp_solvers = qcp_solvers + [("highs", Highs)] -miqcqp_solvers = [('gurobi_persistent', GurobiPersistent)] +miqcqp_solvers = [ + # ('gurobi_persistent', GurobiPersistent), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), +] nl_solvers = [('ipopt', Ipopt)] nl_solvers_set = {i[0] for i in nl_solvers} From 909be8815c6c585a5c2c700b9444599868c26598 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:02:49 -0600 Subject: [PATCH 14/33] refactoring gurobi interfaces --- pyomo/contrib/observer/model_observer.py | 69 +- pyomo/contrib/solver/plugins.py | 3 +- .../solvers/gurobi/gurobi_persistent.py | 934 +++++++++++++++++- .../solver/tests/solvers/test_solvers.py | 11 +- 4 files changed, 925 insertions(+), 92 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 422eb1da574..8f7238c2ee9 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -162,9 +162,6 @@ def __init__( class Observer(abc.ABC): - def __init__(self): - pass - @abc.abstractmethod def add_variables(self, variables: List[VarData]): pass @@ -255,7 +252,7 @@ def set_instance(self, model): self._model = model self._add_block(model) - def _add_variables(self, variables: List[VarData]): + def add_variables(self, variables: List[VarData]): for v in variables: if id(v) in self._referenced_variables: raise ValueError(f'Variable {v.name} has already been added') @@ -271,7 +268,7 @@ def _add_variables(self, variables: List[VarData]): for obs in self._observers: obs.add_variables(variables) - def _add_parameters(self, params: List[ParamData]): + def add_parameters(self, params: List[ParamData]): for p in params: pid = id(p) if pid in self._referenced_params: @@ -287,7 +284,7 @@ def _check_for_new_vars(self, variables: List[VarData]): v_id = id(v) if v_id not in self._referenced_variables: new_vars[v_id] = v - self._add_variables(list(new_vars.values())) + self.add_variables(list(new_vars.values())) def _check_to_remove_vars(self, variables: List[VarData]): vars_to_remove = {} @@ -296,7 +293,7 @@ def _check_to_remove_vars(self, variables: List[VarData]): ref_cons, ref_sos, ref_obj = self._referenced_variables[v_id] if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: vars_to_remove[v_id] = v - self._remove_variables(list(vars_to_remove.values())) + self.remove_variables(list(vars_to_remove.values())) def _check_for_new_params(self, params: List[ParamData]): new_params = {} @@ -304,7 +301,7 @@ def _check_for_new_params(self, params: List[ParamData]): pid = id(p) if pid not in self._referenced_params: new_params[pid] = p - self._add_parameters(list(new_params.values())) + self.add_parameters(list(new_params.values())) def _check_to_remove_params(self, params: List[ParamData]): params_to_remove = {} @@ -313,9 +310,9 @@ def _check_to_remove_params(self, params: List[ParamData]): ref_cons, ref_sos, ref_obj = self._referenced_params[p_id] if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: params_to_remove[p_id] = p - self._remove_parameters(list(params_to_remove.values())) + self.remove_parameters(list(params_to_remove.values())) - def _add_constraints(self, cons: List[ConstraintData]): + def add_constraints(self, cons: List[ConstraintData]): vars_to_check = [] params_to_check = [] for con in cons: @@ -343,7 +340,7 @@ def _add_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.add_constraints(cons) - def _add_sos_constraints(self, cons: List[SOSConstraintData]): + def add_sos_constraints(self, cons: List[SOSConstraintData]): vars_to_check = [] params_to_check = [] for con in cons: @@ -373,7 +370,7 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.add_sos_constraints(cons) - def _set_objective(self, obj: Optional[ObjectiveData]): + def set_objective(self, obj: Optional[ObjectiveData]): vars_to_remove_check = [] params_to_remove_check = [] if self._objective is not None: @@ -414,12 +411,12 @@ def _set_objective(self, obj: Optional[ObjectiveData]): self._check_to_remove_params(params_to_remove_check) def _add_block(self, block): - self._add_constraints( + self.add_constraints( list( block.component_data_objects(Constraint, descend_into=True, active=True) ) ) - self._add_sos_constraints( + self.add_sos_constraints( list( block.component_data_objects( SOSConstraint, descend_into=True, active=True @@ -427,9 +424,9 @@ def _add_block(self, block): ) ) obj = get_objective(block) - self._set_objective(obj) + self.set_objective(obj) - def _remove_constraints(self, cons: List[ConstraintData]): + def remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.remove_constraints(cons) vars_to_check = [] @@ -453,7 +450,7 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._check_to_remove_vars(vars_to_check) self._check_to_remove_params(params_to_check) - def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + def remove_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.remove_sos_constraints(cons) vars_to_check = [] @@ -476,7 +473,7 @@ def _remove_sos_constraints(self, cons: List[SOSConstraintData]): self._check_to_remove_vars(vars_to_check) self._check_to_remove_params(params_to_check) - def _remove_variables(self, variables: List[VarData]): + def remove_variables(self, variables: List[VarData]): for obs in self._observers: obs.remove_variables(variables) for v in variables: @@ -493,7 +490,7 @@ def _remove_variables(self, variables: List[VarData]): del self._referenced_variables[v_id] del self._vars[v_id] - def _remove_parameters(self, params: List[ParamData]): + def remove_parameters(self, params: List[ParamData]): for obs in self._observers: obs.remove_parameters(params) for p in params: @@ -510,7 +507,7 @@ def _remove_parameters(self, params: List[ParamData]): del self._referenced_params[p_id] del self._params[p_id] - def _update_variables(self, variables: List[VarData]): + def update_variables(self, variables: List[VarData]): for v in variables: self._vars[id(v)] = ( v, @@ -523,7 +520,7 @@ def _update_variables(self, variables: List[VarData]): for obs in self._observers: obs.update_variables(variables) - def _update_parameters(self, params): + def update_parameters(self, params): for p in params: self._params[id(p)] = (p, p.value) for obs in self._observers: @@ -668,28 +665,28 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.check_for_new_or_removed_constraints: timer.start('sos') new_sos, old_sos = self._check_for_new_or_removed_sos() - self._add_sos_constraints(new_sos) - self._remove_sos_constraints(old_sos) + self.add_sos_constraints(new_sos) + self.remove_sos_constraints(old_sos) added_sos.update(new_sos) timer.stop('sos') timer.start('cons') new_cons, old_cons = self._check_for_new_or_removed_constraints() - self._add_constraints(new_cons) - self._remove_constraints(old_cons) + self.add_constraints(new_cons) + self.remove_constraints(old_cons) added_cons.update(new_cons) timer.stop('cons') if config.update_constraints: timer.start('cons') cons_to_update = self._check_for_modified_constraints() - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) added_cons.update(cons_to_update) timer.stop('cons') timer.start('sos') sos_to_update = self._check_for_modified_sos() - self._remove_sos_constraints(sos_to_update) - self._add_sos_constraints(sos_to_update) + self.remove_sos_constraints(sos_to_update) + self.add_sos_constraints(sos_to_update) added_sos.update(sos_to_update) timer.stop('sos') @@ -698,10 +695,10 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.update_vars: timer.start('vars') vars_to_update, cons_to_update, update_obj = self._check_for_var_changes() - self._update_variables(vars_to_update) + self.update_variables(vars_to_update) cons_to_update = [i for i in cons_to_update if i not in added_cons] - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) added_cons.update(cons_to_update) if update_obj: need_to_set_objective = True @@ -711,8 +708,8 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): timer.start('named expressions') cons_to_update, update_obj = self._check_for_named_expression_changes() cons_to_update = [i for i in cons_to_update if i not in added_cons] - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) added_cons.update(cons_to_update) if update_obj: need_to_set_objective = True @@ -730,11 +727,11 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): need_to_set_objective = True if need_to_set_objective: - self._set_objective(new_obj) + self.set_objective(new_obj) timer.stop('objective') if config.update_parameters: timer.start('params') params_to_update = self._check_for_param_changes() - self._update_parameters(params_to_update) + self.update_parameters(params_to_update) timer.stop('params') diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 7630c614aa2..f29c4f61c4e 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -12,9 +12,8 @@ from .common.factory import SolverFactory from .solvers.ipopt import Ipopt -from .solvers.gurobi_persistent import GurobiPersistent from .solvers.gurobi.gurobi_direct import GurobiDirect -from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic +from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from .solvers.highs import Highs diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 3ae6e86526c..844502ca476 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -9,6 +9,7 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +from __future__ import annotations import io import logging import math @@ -21,6 +22,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer from pyomo.common.shutdown import python_is_shutting_down +from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler from pyomo.core.base.var import VarData @@ -67,6 +69,7 @@ ) from pyomo.contrib.solver.common.util import get_objective from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector logger = logging.getLogger(__name__) @@ -152,18 +155,33 @@ def get_duals( ) -class GurobiPersistentSolutionLoader(PersistentSolutionLoader): - def load_vars(self, vars_to_load=None, solution_number=0): +class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): + def __init__(self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) -> None: + super().__init__(solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) + self._valid = True + + def invalidate(self): + self._valid = False + + def _assert_solution_still_valid(self): + if not self._valid: + raise RuntimeError('The results in the solver are no longer valid.') + + def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> None: self._assert_solution_still_valid() - self._solver._load_vars( - vars_to_load=vars_to_load, solution_number=solution_number - ) + return super().load_vars(vars_to_load, solution_id) + + def get_primals(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_primals(vars_to_load, solution_id) - def get_primals(self, vars_to_load=None, solution_number=0): + def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() - return self._solver._get_primals( - vars_to_load=vars_to_load, solution_number=solution_number - ) + return super().get_duals(cons_to_load) + + def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_reduced_costs(vars_to_load) class _MutableLowerBound: @@ -187,46 +205,61 @@ def update(self): class _MutableLinearCoefficient: - def __init__(self): - self.expr = None - self.var = None - self.con = None - self.gurobi_model = None + def __init__(self, expr, pyomo_con, con_map, pyomo_var_id, var_map, gurobi_model): + self.expr = expr + self.pyomo_con = pyomo_con + self.pyomo_var_id = pyomo_var_id + self.con_map = con_map + self.var_map = var_map + self.gurobi_model = gurobi_model + + @property + def gurobi_var(self): + return self.var_map[self.pyomo_var_id] + + @property + def gurobi_con(self): + return self.con_map[self.pyomo_con] def update(self): - self.gurobi_model.chgCoeff(self.con, self.var, value(self.expr)) + self.gurobi_model.chgCoeff(self.gurobi_con, self.gurobi_var, value(self.expr)) class _MutableRangeConstant: - def __init__(self): - self.lhs_expr = None - self.rhs_expr = None - self.con = None - self.slack_name = None - self.gurobi_model = None + def __init__(self, lhs_expr, rhs_expr, pyomo_con, con_map, slack_name, gurobi_model): + self.lhs_expr = lhs_expr + self.rhs_expr = rhs_expr + self.pyomo_con = pyomo_con + self.con_map = con_map + self.slack_name = slack_name + self.gurobi_model = gurobi_model def update(self): rhs_val = value(self.rhs_expr) lhs_val = value(self.lhs_expr) - self.con.rhs = rhs_val + con = self.con_map[self.pyomo_con] + con.rhs = rhs_val slack = self.gurobi_model.getVarByName(self.slack_name) slack.ub = rhs_val - lhs_val class _MutableConstant: - def __init__(self): - self.expr = None - self.con = None + def __init__(self, expr, pyomo_con, con_map): + self.expr = expr + self.pyomo_con = pyomo_con + self.con_map = con_map def update(self): - self.con.rhs = value(self.expr) + con = self.con_map[self.pyomo_con] + con.rhs = value(self.expr) class _MutableQuadraticConstraint: def __init__( - self, gurobi_model, gurobi_con, constant, linear_coefs, quadratic_coefs + self, gurobi_model, pyomo_con, con_map, constant, linear_coefs, quadratic_coefs ): - self.con = gurobi_con + self.pyomo_con = pyomo_con + self.con_map = con_map self.gurobi_model = gurobi_model self.constant = constant self.last_constant_value = value(self.constant.expr) @@ -235,8 +268,12 @@ def __init__( self.quadratic_coefs = quadratic_coefs self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + @property + def gurobi_con(self): + return self.con_map[self.pyomo_con] + def get_updated_expression(self): - gurobi_expr = self.gurobi_model.getQCRow(self.con) + gurobi_expr = self.gurobi_model.getQCRow(self.gurobi_con) for ndx, coef in enumerate(self.linear_coefs): current_coef_value = value(coef.expr) incremental_coef_value = ( @@ -260,14 +297,14 @@ def get_updated_rhs(self): class _MutableObjective: def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): self.gurobi_model = gurobi_model - self.constant = constant - self.linear_coefs = linear_coefs - self.quadratic_coefs = quadratic_coefs - self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + self.constant: _MutableConstant = constant + self.linear_coefs: List[_MutableLinearCoefficient] = linear_coefs + self.quadratic_coefs: List[_MutableQuadraticCoefficient] = quadratic_coefs + self.last_quadratic_coef_values: List[float] = [value(i.expr) for i in self.quadratic_coefs] def get_updated_expression(self): for ndx, coef in enumerate(self.linear_coefs): - coef.var.obj = value(coef.expr) + coef.gurobi_var.obj = value(coef.expr) self.gurobi_model.ObjCon = value(self.constant.expr) gurobi_expr = None @@ -286,10 +323,19 @@ def get_updated_expression(self): class _MutableQuadraticCoefficient: - def __init__(self): + def __init__(self, expr, v1id, v2id, var_map): self.expr = None - self.var1 = None - self.var2 = None + self.var_map = var_map + self.v1id = v1id + self.v2id = v2id + + @property + def var1(self): + return self.var_map[self.v1id] + + @property + def var2(self): + return self.var_map[self.v2id] class GurobiDirectQuadratic(GurobiDirectBase): @@ -405,9 +451,10 @@ def _get_expr_from_pyomo_repn(self, repn): if len(repn.linear_vars) > 0: missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] self._add_variables(missing_vars) + coef_list = [value(i) for i in repn.linear_coefs] vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr( - repn.linear_coefs, + coef_list, vlist, ) else: @@ -424,9 +471,9 @@ def _get_expr_from_pyomo_repn(self, repn): for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - new_expr += coef * gurobi_x * gurobi_y + new_expr += value(coef) * gurobi_x * gurobi_y - return new_expr, repn.constant + return new_expr def _add_constraints(self, cons: List[ConstraintData]): gurobi_expr_list = [] @@ -437,20 +484,20 @@ def _add_constraints(self, cons: List[ConstraintData]): self._quadratic_cons.add(con) else: self._linear_cons.add(con) - gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) if lb is None and ub is None: raise ValueError( "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: - gurobi_expr_list.append(gurobi_expr <= float(ub - repn_constant)) + gurobi_expr_list.append(gurobi_expr <= float(ub - repn.constant)) elif ub is None: - gurobi_expr_list.append(float(lb - repn_constant) <= gurobi_expr) + gurobi_expr_list.append(float(lb - repn.constant) <= gurobi_expr) elif lb == ub: - gurobi_expr_list.append(gurobi_expr == float(lb - repn_constant)) + gurobi_expr_list.append(gurobi_expr == float(lb - repn.constant)) else: - gurobi_expr_list.append(gurobi_expr == [float(lb-repn_constant), float(ub-repn_constant)]) + gurobi_expr_list.append(gurobi_expr == [float(lb-repn.constant), float(ub-repn.constant)]) gurobi_cons = self._solver_model.addConstrs((gurobi_expr_list[i] for i in range(len(gurobi_expr_list)))).values() self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) @@ -495,11 +542,802 @@ def _set_objective(self, obj): raise ValueError(f'Objective sense is not recognized: {obj.sense}') repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=True) - gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + repn_constant = repn.constant self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) - self._needs_updated = True -class GurobiPersistentQuadratic(GurobiDirectQuadratic): +class _GurobiObserver(Observer): + def __init__(self, opt: GurobiPersistentQuadratic) -> None: + self.opt = opt + + def add_variables(self, variables: List[VarData]): + self.opt._add_variables(variables) + + def add_parameters(self, params: List[ParamData]): + pass + + def add_constraints(self, cons: List[ConstraintData]): + self.opt._add_constraints(cons) + + def add_sos_constraints(self, cons: List[SOSConstraintData]): + self.opt._add_sos_constraints(cons) + + def set_objective(self, obj: ObjectiveData | None): + self.opt._set_objective(obj) + + def remove_constraints(self, cons: List[ConstraintData]): + self.opt._remove_constraints(cons) + + def remove_sos_constraints(self, cons: List[SOSConstraintData]): + self.opt._remove_sos_constraints(cons) + + def remove_variables(self, variables: List[VarData]): + self.opt._remove_variables(variables) + + def remove_parameters(self, params: List[ParamData]): + pass + + def update_variables(self, variables: List[VarData]): + self.opt._update_variables(variables) + + def update_parameters(self, params: List[ParamData]): + self.opt._update_parameters(params) + + +class GurobiPersistent(GurobiDirectQuadratic): _minimum_version = (7, 0, 0) + + def __init__(self, **kwds): + super().__init__(**kwds) + self._pyomo_model = None + self._objective = None + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True + self._callback_func = None + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._last_results_object: Optional[Results] = None + self._observer = _GurobiObserver(self) + self._change_detector = ModelChangeDetector(observers=[self._observer]) + self._constraint_ndx = 0 + + @property + def auto_updates(self): + return self._change_detector.config + + def _clear(self): + super()._clear() + self._pyomo_model = None + self._objective = None + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._last_results_object = None + self._constraint_ndx = 0 + + def _create_solver_model(self, pyomo_model): + if pyomo_model is self._pyomo_model: + self.update() + else: + self.set_instance(pyomo_model) + + solution_loader = GurobiPersistentSolutionLoader( + solver_model=self._solver_model, + var_id_map=self._vars, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + linear_cons=self._linear_cons, + quadratic_cons=self._quadratic_cons, + ) + has_obj = self._objective is not None + return self._solver_model, solution_loader, has_obj + + def release_license(self): + self._clear() + self.__class__.release_license() + + def solve(self, model, **kwds) -> Results: + res = super().solve(model, **kwds) + self._needs_updated = False + return res + + def _process_domain_and_bounds(self, var): + res = super()._process_domain_and_bounds(var) + if not is_constant(var._lb): + mutable_lb = _MutableLowerBound(id(var), var.lower, self._pyomo_var_to_solver_var_map) + self._mutable_bounds[id(var), 'lb'] = (var, mutable_lb) + if not is_constant(var._ub): + mutable_ub = _MutableUpperBound(id(var), var.upper, self._pyomo_var_to_solver_var_map) + self._mutable_bounds[id(var), 'ub'] = (var, mutable_ub) + return res + + def _add_variables(self, variables: List[VarData]): + self._invalidate_last_results() + super()._add_variables(variables) + self._vars_added_since_update.update(variables) + self._needs_updated = True + + def set_instance(self, pyomo_model): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + self._clear() + self._pyomo_model = pyomo_model + self._solver_model = gurobipy.Model(env=self.env()) + timer.start('set_instance') + self._change_detector.set_instance(pyomo_model) + timer.stop('set_instance') + + def update(self): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + if self._pyomo_model is None: + raise RuntimeError('must call set_instance or solve before update') + timer.start('update') + if self._needs_updated: + self._update_gurobi_model() + self._change_detector.update(timer=timer) + timer.stop('update') + + def _add_constraints(self, cons: List[ConstraintData]): + self._invalidate_last_results() + gurobi_expr_list = [] + for ndx, con in enumerate(cons): + lb, body, ub = con.to_bounded_expression(evaluate_bounds=False) + repn = generate_standard_repn(body, quadratic=True, compute_values=False) + if len(repn.quadratic_vars) > 0: + self._quadratic_cons.add(con) + else: + self._linear_cons.add(con) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + mutable_constant = None + if lb is None and ub is None: + raise ValueError( + "Constraint does not have a lower " + f"or an upper bound: {con} \n" + ) + elif lb is None: + rhs_expr = ub - repn.constant + gurobi_expr_list.append(gurobi_expr <= float(value(rhs_expr))) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + elif ub is None: + rhs_expr = lb - repn.constant + gurobi_expr_list.append(float(value(rhs_expr)) <= gurobi_expr) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + elif con.equality: + rhs_expr = lb - repn.constant + gurobi_expr_list.append(gurobi_expr == float(value(rhs_expr))) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + else: + assert len(repn.quadratic_vars) == 0, "Quadratic range constraints are not supported" + lhs_expr = lb - repn.constant + rhs_expr = ub - repn.constant + gurobi_expr_list.append(gurobi_expr == [float(value(lhs_expr)), float(value(rhs_expr))]) + if not is_constant(lhs_expr) or not is_constant(rhs_expr): + conname = f'c{self._constraint_ndx}[{ndx}]' + mutable_constant = _MutableRangeConstant(lhs_expr, rhs_expr, con, self._pyomo_con_to_solver_con_map, 'Rg' + conname, self._solver_model) + + mlc_list = [] + for c, v in zip(repn.linear_coefs, repn.linear_vars): + if not is_constant(c): + mlc = _MutableLinearCoefficient(c, con, self._pyomo_con_to_solver_con_map, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc_list.append(mlc) + + if len(repn.quadratic_vars) == 0: + if len(mlc_list) > 0: + self._mutable_helpers[con] = mlc_list + if mutable_constant is not None: + if con not in self._mutable_helpers: + self._mutable_helpers[con] = [] + self._mutable_helpers[con].append(mutable_constant) + else: + if mutable_constant is None: + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mqc_list = [] + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + if not is_constant(coef): + mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc_list.append(mqc) + mqc = _MutableQuadraticConstraint( + self._solver_model, + con, + self._pyomo_con_to_solver_con_map, + mutable_constant, + mlc_list, + mqc_list, + ) + self._mutable_quadratic_helpers[con] = mqc + + gurobi_cons = list(self._solver_model.addConstrs( + (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))), + name=f'c{self._constraint_ndx}' + ).values()) + self._constraint_ndx += 1 + self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) + self._constraints_added_since_update.update(cons) + self._needs_updated = True + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + self._invalidate_last_results() + super()._add_sos_constraints(cons) + self._constraints_added_since_update.update(cons) + self._needs_updated = True + + def _set_objective(self, obj): + self._invalidate_last_results() + if obj is None: + sense = gurobipy.GRB.MINIMIZE + gurobi_expr = 0 + repn_constant = 0 + self._mutable_objective = None + else: + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') + + repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=False) + repn_constant = value(repn.constant) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + mutable_constant = _MutableConstant(repn.constant, None, None) + + mlc_list = [] + for c, v in zip(repn.linear_coefs, repn.linear_vars): + if not is_constant(c): + mlc = _MutableLinearCoefficient(c, None, None, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc_list.append(mlc) + + mqc_list = [] + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + if not is_constant(coef): + mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc_list.append(mqc) + + self._mutable_objective = _MutableObjective(self._solver_model, mutable_constant, mlc_list, mqc_list) + + # hack + # see PR #2454 + if self._objective is not None: + self._solver_model.setObjective(0) + self._solver_model.update() + + self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) + self._objective = obj + self._needs_updated = True + + def _update_gurobi_model(self): + self._solver_model.update() + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._needs_updated = False + + def _remove_constraints(self, cons: List[ConstraintData]): + self._invalidate_last_results() + for con in cons: + if con in self._constraints_added_since_update: + self._update_gurobi_model() + solver_con = self._pyomo_con_to_solver_con_map[con] + self._solver_model.remove(solver_con) + del self._pyomo_con_to_solver_con_map[con] + self._mutable_helpers.pop(con, None) + self._mutable_quadratic_helpers.pop(con, None) + self._needs_updated = True + + def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + self._invalidate_last_results() + for con in cons: + if con in self._constraints_added_since_update: + self._update_gurobi_model() + solver_sos_con = self._pyomo_sos_to_solver_sos_map[con] + self._solver_model.remove(solver_sos_con) + del self._pyomo_sos_to_solver_sos_map[con] + self._needs_updated = True + + def _remove_variables(self, variables: List[VarData]): + self._invalidate_last_results() + for var in variables: + v_id = id(var) + if var in self._vars_added_since_update: + self._update_gurobi_model() + solver_var = self._pyomo_var_to_solver_var_map[v_id] + self._solver_model.remove(solver_var) + del self._pyomo_var_to_solver_var_map[v_id] + self._mutable_bounds.pop(v_id, None) + self._needs_updated = True + + def _update_variables(self, variables: List[VarData]): + self._invalidate_last_results() + for var in variables: + var_id = id(var) + if var_id not in self._pyomo_var_to_solver_var_map: + raise ValueError( + f'The Var provided to update_var needs to be added first: {var}' + ) + self._mutable_bounds.pop((var_id, 'lb'), None) + self._mutable_bounds.pop((var_id, 'ub'), None) + gurobipy_var = self._pyomo_var_to_solver_var_map[var_id] + lb, ub, vtype = self._process_domain_and_bounds(var) + gurobipy_var.setAttr('lb', lb) + gurobipy_var.setAttr('ub', ub) + gurobipy_var.setAttr('vtype', vtype) + self._needs_updated = True + + def _update_parameters(self, params: List[ParamData]): + self._invalidate_last_results() + for con, helpers in self._mutable_helpers.items(): + for helper in helpers: + helper.update() + for k, (v, helper) in self._mutable_bounds.items(): + helper.update() + + for con, helper in self._mutable_quadratic_helpers.items(): + if con in self._constraints_added_since_update: + self._update_gurobi_model() + gurobi_con = helper.gurobi_con + new_gurobi_expr = helper.get_updated_expression() + new_rhs = helper.get_updated_rhs() + new_sense = gurobi_con.qcsense + self._solver_model.remove(gurobi_con) + new_con = self._solver_model.addQConstr( + new_gurobi_expr, new_sense, new_rhs, + ) + self._pyomo_con_to_solver_con_map[con] = new_con + helper.pyomo_con = con + self._constraints_added_since_update.add(con) + + if self._mutable_objective is not None: + new_gurobi_expr = self._mutable_objective.get_updated_expression() + if new_gurobi_expr is not None: + if self._objective.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + else: + sense = gurobipy.GRB.MAXIMIZE + # TODO: need a test for when part of the object is linear + # and part of the objective is quadratic, but both + # parts have mutable coefficients + self._solver_model.setObjective(new_gurobi_expr, sense=sense) + + def _invalidate_last_results(self): + if self._last_results_object is not None: + self._last_results_object.solution_loader.invalidate() + + def get_model_attr(self, attr): + """ + Get the value of an attribute on the Gurobi model. + + Parameters + ---------- + attr: str + The attribute to get. See Gurobi documentation for descriptions of the attributes. + """ + if self._needs_updated: + self._update_gurobi_model() + return self._solver_model.getAttr(attr) + + def write(self, filename): + """ + Write the model to a file (e.g., and lp file). + + Parameters + ---------- + filename: str + Name of the file to which the model should be written. + """ + self._solver_model.write(filename) + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._needs_updated = False + + def set_linear_constraint_attr(self, con, attr, val): + """ + Set the value of an attribute on a gurobi linear constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be modified. + attr: str + The attribute to be modified. Options are: + CBasis + DStart + Lazy + val: any + See gurobi documentation for acceptable values. + """ + if attr in {'Sense', 'RHS', 'ConstrName'}: + raise ValueError( + f'Linear constraint attr {attr} cannot be set with' + ' the set_linear_constraint_attr method. Please use' + ' the remove_constraint and add_constraint methods.' + ) + self._pyomo_con_to_solver_con_map[con].setAttr(attr, val) + self._needs_updated = True + + def set_var_attr(self, var, attr, val): + """ + Set the value of an attribute on a gurobi variable. + + Parameters + ---------- + var: pyomo.core.base.var.VarData + The pyomo var for which the corresponding gurobi var attribute + should be modified. + attr: str + The attribute to be modified. Options are: + Start + VarHintVal + VarHintPri + BranchPriority + VBasis + PStart + val: any + See gurobi documentation for acceptable values. + """ + if attr in {'LB', 'UB', 'VType', 'VarName'}: + raise ValueError( + f'Var attr {attr} cannot be set with' + ' the set_var_attr method. Please use' + ' the update_var method.' + ) + if attr == 'Obj': + raise ValueError( + 'Var attr Obj cannot be set with' + ' the set_var_attr method. Please use' + ' the set_objective method.' + ) + self._pyomo_var_to_solver_var_map[id(var)].setAttr(attr, val) + self._needs_updated = True + + def get_var_attr(self, var, attr): + """ + Get the value of an attribute on a gurobi var. + + Parameters + ---------- + var: pyomo.core.base.var.VarData + The pyomo var for which the corresponding gurobi var attribute + should be retrieved. + attr: str + The attribute to get. See gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_var_to_solver_var_map[id(var)].getAttr(attr) + + def get_linear_constraint_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi linear constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_con_to_solver_con_map[con].getAttr(attr) + + def get_sos_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi sos constraint. + + Parameters + ---------- + con: pyomo.core.base.sos.SOSConstraintData + The pyomo SOS constraint for which the corresponding gurobi SOS constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_sos_to_solver_sos_map[con].getAttr(attr) + + def get_quadratic_constraint_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi quadratic constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_con_to_solver_con_map[con].getAttr(attr) + + def set_gurobi_param(self, param, val): + """ + Set a gurobi parameter. + + Parameters + ---------- + param: str + The gurobi parameter to set. Options include any gurobi parameter. + Please see the Gurobi documentation for options. + val: any + The value to set the parameter to. See Gurobi documentation for possible values. + """ + self._solver_model.setParam(param, val) + + def get_gurobi_param_info(self, param): + """ + Get information about a gurobi parameter. + + Parameters + ---------- + param: str + The gurobi parameter to get info for. See Gurobi documentation for possible options. + + Returns + ------- + six-tuple containing the parameter name, type, value, minimum value, maximum value, and default value. + """ + return self._solver_model.getParamInfo(param) + + def _intermediate_callback(self): + def f(gurobi_model, where): + self._callback_func(self._pyomo_model, self, where) + + return f + + def set_callback(self, func=None): + """ + Specify a callback for gurobi to use. + + Parameters + ---------- + func: function + The function to call. The function should have three arguments. The first will be the pyomo model being + solved. The second will be the GurobiPersistent instance. The third will be an enum member of + gurobipy.GRB.Callback. This will indicate where in the branch and bound algorithm gurobi is at. For + example, suppose we want to solve + + .. math:: + + min 2*x + y + + s.t. + + y >= (x-2)**2 + + 0 <= x <= 4 + + y >= 0 + + y integer + + as an MILP using extended cutting planes in callbacks. + + >>> from gurobipy import GRB # doctest:+SKIP + >>> import pyomo.environ as pyo + >>> from pyomo.core.expr.taylor_series import taylor_series_expansion + >>> from pyomo.contrib import appsi + >>> + >>> m = pyo.ConcreteModel() + >>> m.x = pyo.Var(bounds=(0, 4)) + >>> m.y = pyo.Var(within=pyo.Integers, bounds=(0, None)) + >>> m.obj = pyo.Objective(expr=2*m.x + m.y) + >>> m.cons = pyo.ConstraintList() # for the cutting planes + >>> + >>> def _add_cut(xval): + ... # a function to generate the cut + ... m.x.value = xval + ... return m.cons.add(m.y >= taylor_series_expansion((m.x - 2)**2)) + ... + >>> _c = _add_cut(0) # start with 2 cuts at the bounds of x + >>> _c = _add_cut(4) # this is an arbitrary choice + >>> + >>> opt = appsi.solvers.Gurobi() + >>> opt.config.stream_solver = True + >>> opt.set_instance(m) # doctest:+SKIP + >>> opt.gurobi_options['PreCrush'] = 1 + >>> opt.gurobi_options['LazyConstraints'] = 1 + >>> + >>> def my_callback(cb_m, cb_opt, cb_where): + ... if cb_where == GRB.Callback.MIPSOL: + ... cb_opt.cbGetSolution(variables=[m.x, m.y]) + ... if m.y.value < (m.x.value - 2)**2 - 1e-6: + ... cb_opt.cbLazy(_add_cut(m.x.value)) + ... + >>> opt.set_callback(my_callback) + >>> res = opt.solve(m) # doctest:+SKIP + + """ + if func is not None: + self._callback_func = func + self._callback = self._intermediate_callback() + else: + self._callback = None + self._callback_func = None + + def cbCut(self, con): + """ + Add a cut within a callback. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The cut to add + """ + if not con.active: + raise ValueError('cbCut expected an active constraint.') + + if is_fixed(con.body): + raise ValueError('cbCut expected a non-trivial constraint') + + repn = generate_standard_repn(con.body, quadratic=True, compute_values=True) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + if con.has_lb(): + if con.has_ub(): + raise ValueError('Range constraints are not supported in cbCut.') + if not is_fixed(con.lower): + raise ValueError(f'Lower bound of constraint {con} is not constant.') + if con.has_ub(): + if not is_fixed(con.upper): + raise ValueError(f'Upper bound of constraint {con} is not constant.') + + if con.equality: + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_lb() and (value(con.lower) > -float('inf')): + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.GREATER_EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_ub() and (value(con.upper) < float('inf')): + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.LESS_EQUAL, + rhs=value(con.upper - repn.constant), + ) + else: + raise ValueError( + f'Constraint does not have a lower or an upper bound {con} \n' + ) + + def cbGet(self, what): + return self._solver_model.cbGet(what) + + def cbGetNodeRel(self, variables): + """ + Parameters + ---------- + variables: Var or iterable of Var + """ + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + var_values = self._solver_model.cbGetNodeRel(gurobi_vars) + for i, v in enumerate(variables): + v.set_value(var_values[i], skip_validation=True) + + def cbGetSolution(self, variables): + """ + Parameters + ---------- + variables: iterable of vars + """ + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + var_values = self._solver_model.cbGetSolution(gurobi_vars) + for i, v in enumerate(variables): + v.set_value(var_values[i], skip_validation=True) + + def cbLazy(self, con): + """ + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The lazy constraint to add + """ + if not con.active: + raise ValueError('cbLazy expected an active constraint.') + + if is_fixed(con.body): + raise ValueError('cbLazy expected a non-trivial constraint') + + repn = generate_standard_repn(con.body, quadratic=True, compute_values=True) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + if con.has_lb(): + if con.has_ub(): + raise ValueError('Range constraints are not supported in cbLazy.') + if not is_fixed(con.lower): + raise ValueError(f'Lower bound of constraint {con} is not constant.') + if con.has_ub(): + if not is_fixed(con.upper): + raise ValueError(f'Upper bound of constraint {con} is not constant.') + + if con.equality: + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_lb() and (value(con.lower) > -float('inf')): + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.GREATER_EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_ub() and (value(con.upper) < float('inf')): + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.LESS_EQUAL, + rhs=value(con.upper - repn.constant), + ) + else: + raise ValueError( + f'Constraint does not have a lower or an upper bound {con} \n' + ) + + def cbSetSolution(self, variables, solution): + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + self._solver_model.cbSetSolution(gurobi_vars, solution) + + def cbUseSolution(self): + return self._solver_model.cbUseSolution() + + def reset(self): + self._solver_model.reset() + + def add_variables(self, variables): + self._change_detector.add_variables(variables) + + def add_constraints(self, cons): + self._change_detector.add_constraints(cons) + + def add_sos_constraints(self, cons): + self._change_detector.add_sos_constraints(cons) + + def set_objective(self, obj): + self._change_detector.set_objective(obj) + + def remove_constrains(self, cons): + self._change_detector.remove_constraints(cons) + + def remove_sos_constraints(self, cons): + self._change_detector.remove_sos_constraints(cons) + + def remove_variables(self, variables): + self._change_detector.remove_variables(variables) + + def update_variables(self, variables): + self._change_detector.update_variables(variables) + + def update_parameters(self, params): + self._change_detector.update_parameters(params) \ No newline at end of file diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 748e0127151..a0d87835e13 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -30,9 +30,8 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.solvers.ipopt import Ipopt -# from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect -from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -48,14 +47,14 @@ raise unittest.SkipTest('Parameterized is not available.') all_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('highs', Highs), ] mip_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), @@ -64,13 +63,13 @@ ('ipopt', Ipopt), ] qcp_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ] qp_solvers = qcp_solvers + [("highs", Highs)] miqcqp_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ] nl_solvers = [('ipopt', Ipopt)] From 862c387a8e6478d9b9c3f176b0059046d02f1198 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:18:04 -0600 Subject: [PATCH 15/33] bugs --- pyomo/contrib/solver/plugins.py | 4 ++-- pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index f29c4f61c4e..fed739232ad 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -35,7 +35,7 @@ def load(): name='gurobi_direct_quadratic', legacy_name='gurobi_direct_quadratic_v2', doc='Direct interface to Gurobi', - )(GurobiDirect) + )(GurobiDirectQuadratic) SolverFactory.register( - name='highs', legacy_name='highs', doc='Persistent interface to HiGHS' + name='highs', legacy_name='highs_v2', doc='Persistent interface to HiGHS' )(Highs) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 844502ca476..b8a8f46d1f6 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -279,7 +279,7 @@ def get_updated_expression(self): incremental_coef_value = ( current_coef_value - self.last_linear_coef_values[ndx] ) - gurobi_expr += incremental_coef_value * coef.var + gurobi_expr += incremental_coef_value * coef.gurobi_var self.last_linear_coef_values[ndx] = current_coef_value for ndx, coef in enumerate(self.quadratic_coefs): current_coef_value = value(coef.expr) @@ -324,7 +324,7 @@ def get_updated_expression(self): class _MutableQuadraticCoefficient: def __init__(self, expr, v1id, v2id, var_map): - self.expr = None + self.expr = expr self.var_map = var_map self.v1id = v1id self.v2id = v2id @@ -860,6 +860,7 @@ def _remove_variables(self, variables: List[VarData]): solver_var = self._pyomo_var_to_solver_var_map[v_id] self._solver_model.remove(solver_var) del self._pyomo_var_to_solver_var_map[v_id] + del self._vars[v_id] self._mutable_bounds.pop(v_id, None) self._needs_updated = True From 8f7a61ed3bf8eafc8eee12544755f61097a2b7be Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:35:33 -0600 Subject: [PATCH 16/33] refactoring gurobi interfaces --- pyomo/contrib/observer/model_observer.py | 7 +- .../solvers/gurobi/gurobi_persistent.py | 6 +- pyomo/contrib/solver/solvers/gurobi_direct.py | 470 ------ .../solver/solvers/gurobi_persistent.py | 1409 ----------------- .../tests/solvers/test_gurobi_persistent.py | 49 +- 5 files changed, 25 insertions(+), 1916 deletions(-) delete mode 100644 pyomo/contrib/solver/solvers/gurobi_direct.py delete mode 100644 pyomo/contrib/solver/solvers/gurobi_persistent.py diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 8f7238c2ee9..bd905e1c61d 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -361,12 +361,15 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): self._named_expressions[con] = [] self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) + for con in cons: + variables = self._vars_referenced_by_con[con] + params = self._params_referenced_by_con[con] for v in variables: self._referenced_variables[id(v)][1][con] = None for p in params: self._referenced_params[id(p)][1][con] = None - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) for obs in self._observers: obs.add_sos_constraints(cons) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index b8a8f46d1f6..e91381f41a3 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -39,10 +39,6 @@ SolutionStatus, ) from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig -from pyomo.contrib.solver.solvers.gurobi_direct import ( - GurobiConfigMixin, - GurobiSolverMixin, -) from pyomo.contrib.solver.common.util import ( NoFeasibleSolutionError, NoOptimalSolutionError, @@ -1328,7 +1324,7 @@ def add_sos_constraints(self, cons): def set_objective(self, obj): self._change_detector.set_objective(obj) - def remove_constrains(self, cons): + def remove_constraints(self, cons): self._change_detector.remove_constraints(cons) def remove_sos_constraints(self, cons): diff --git a/pyomo/contrib/solver/solvers/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi_direct.py deleted file mode 100644 index 45ea9dcc873..00000000000 --- a/pyomo/contrib/solver/solvers/gurobi_direct.py +++ /dev/null @@ -1,470 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import datetime -import io -import math -import operator -import os - -from pyomo.common.collections import ComponentMap, ComponentSet -from pyomo.common.config import ConfigValue -from pyomo.common.dependencies import attempt_import -from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import MouseTrap, ApplicationError -from pyomo.common.shutdown import python_is_shutting_down -from pyomo.common.tee import capture_output, TeeStream -from pyomo.common.timing import HierarchicalTimer -from pyomo.core.staleflag import StaleFlagManager -from pyomo.repn.plugins.standard_form import LinearStandardFormCompiler - -from pyomo.contrib.solver.common.base import SolverBase, Availability -from pyomo.contrib.solver.common.config import BranchAndBoundConfig -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoDualsError, - NoReducedCostsError, - NoSolutionError, - IncompatibleModelError, -) -from pyomo.contrib.solver.common.results import ( - Results, - SolutionStatus, - TerminationCondition, -) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase - - -gurobipy, gurobipy_available = attempt_import('gurobipy') - - -class GurobiConfigMixin: - """ - Mixin class for Gurobi-specific configurations - """ - - def __init__(self): - self.use_mipstart: bool = self.declare( - 'use_mipstart', - ConfigValue( - default=False, - domain=bool, - description="If True, the current values of the integer variables " - "will be passed to Gurobi.", - ), - ) - - -class GurobiConfig(BranchAndBoundConfig, GurobiConfigMixin): - def __init__( - self, - description=None, - doc=None, - implicit=False, - implicit_domain=None, - visibility=0, - ): - BranchAndBoundConfig.__init__( - self, - description=description, - doc=doc, - implicit=implicit, - implicit_domain=implicit_domain, - visibility=visibility, - ) - GurobiConfigMixin.__init__(self) - - -class GurobiDirectSolutionLoader(SolutionLoaderBase): - def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars, pyo_obj): - self._grb_model = grb_model - self._grb_cons = grb_cons - self._grb_vars = grb_vars - self._pyo_cons = pyo_cons - self._pyo_vars = pyo_vars - self._pyo_obj = pyo_obj - GurobiDirect._register_env_client() - - def __del__(self): - if python_is_shutting_down(): - return - # Free the associated model - if self._grb_model is not None: - self._grb_cons = None - self._grb_vars = None - self._pyo_cons = None - self._pyo_vars = None - self._pyo_obj = None - # explicitly release the model - self._grb_model.dispose() - self._grb_model = None - # Release the gurobi license if this is the last reference to - # the environment (either through a results object or solver - # interface) - GurobiDirect._release_env_client() - - def load_vars(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 - if self._grb_model.SolCount == 0: - raise NoSolutionError() - - iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) - for p_var, g_var in iterator: - p_var.set_value(g_var, skip_validation=True) - StaleFlagManager.mark_all_as_stale(delayed=True) - - def get_primals(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 - if self._grb_model.SolCount == 0: - raise NoSolutionError() - - iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) - return ComponentMap(iterator) - - def get_duals(self, cons_to_load=None): - if self._grb_model.Status != gurobipy.GRB.OPTIMAL: - raise NoDualsError() - - def dedup(_iter): - last = None - for con_info_dual in _iter: - if not con_info_dual[1] and con_info_dual[0][0] is last: - continue - last = con_info_dual[0][0] - yield con_info_dual - - iterator = dedup(zip(self._pyo_cons, self._grb_cons.getAttr('Pi').tolist())) - if cons_to_load: - cons_to_load = set(cons_to_load) - iterator = filter( - lambda con_info_dual: con_info_dual[0][0] in cons_to_load, iterator - ) - return {con_info[0]: dual for con_info, dual in iterator} - - def get_reduced_costs(self, vars_to_load=None): - if self._grb_model.Status != gurobipy.GRB.OPTIMAL: - raise NoReducedCostsError() - - iterator = zip(self._pyo_vars, self._grb_vars.getAttr('Rc').tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) - return ComponentMap(iterator) - - -class GurobiSolverMixin: - """ - gurobi_direct and gurobi_persistent check availability and set versions - in the same way. This moves the logic to a central location to reduce - duplicate code. - """ - - _num_gurobipy_env_clients = 0 - _gurobipy_env = None - _available = None - _gurobipy_available = gurobipy_available - - def available(self): - if self._available is None: - # this triggers the deferred import, and for the persistent - # interface, may update the _available flag - # - # Note that we set the _available flag on the *most derived - # class* and not on the instance, or on the base class. That - # allows different derived interfaces to have different - # availability (e.g., persistent has a minimum version - # requirement that the direct interface doesn't) - if not self._gurobipy_available: - if self._available is None: - self.__class__._available = Availability.NotFound - else: - self.__class__._available = self._check_license() - return self._available - - @staticmethod - def release_license(): - if GurobiSolverMixin._gurobipy_env is None: - return - if GurobiSolverMixin._num_gurobipy_env_clients: - logger.warning( - "Call to GurobiSolverMixin.release_license() with %s remaining " - "environment clients." % (GurobiSolverMixin._num_gurobipy_env_clients,) - ) - GurobiSolverMixin._gurobipy_env.close() - GurobiSolverMixin._gurobipy_env = None - - @staticmethod - def env(): - if GurobiSolverMixin._gurobipy_env is None: - with capture_output(capture_fd=True): - GurobiSolverMixin._gurobipy_env = gurobipy.Env() - return GurobiSolverMixin._gurobipy_env - - @staticmethod - def _register_env_client(): - GurobiSolverMixin._num_gurobipy_env_clients += 1 - - @staticmethod - def _release_env_client(): - GurobiSolverMixin._num_gurobipy_env_clients -= 1 - if GurobiSolverMixin._num_gurobipy_env_clients <= 0: - # Note that _num_gurobipy_env_clients should never be <0, - # but if it is, release_license will issue a warning (that - # we want to know about) - GurobiSolverMixin.release_license() - - def _check_license(self): - try: - model = gurobipy.Model(env=self.env()) - except gurobipy.GurobiError: - return Availability.BadLicense - - model.setParam('OutputFlag', 0) - try: - model.addVars(range(2001)) - model.optimize() - return Availability.FullLicense - except gurobipy.GurobiError: - return Availability.LimitedLicense - finally: - model.dispose() - - def version(self): - version = ( - gurobipy.GRB.VERSION_MAJOR, - gurobipy.GRB.VERSION_MINOR, - gurobipy.GRB.VERSION_TECHNICAL, - ) - return version - - -class GurobiDirect(GurobiSolverMixin, SolverBase): - """ - Interface to Gurobi using gurobipy - """ - - CONFIG = GurobiConfig() - - _tc_map = None - - def __init__(self, **kwds): - super().__init__(**kwds) - self._register_env_client() - - def __del__(self): - if not python_is_shutting_down(): - self._release_env_client() - - def solve(self, model, **kwds) -> Results: - start_timestamp = datetime.datetime.now(datetime.timezone.utc) - config = self.config(value=kwds, preserve_implicit=True) - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer - - StaleFlagManager.mark_all_as_stale() - - timer.start('compile_model') - repn = LinearStandardFormCompiler().write( - model, mixed_form=True, set_sense=None - ) - timer.stop('compile_model') - - if len(repn.objectives) > 1: - raise IncompatibleModelError( - f"The {self.__class__.__name__} solver only supports models " - f"with zero or one objectives (received {len(repn.objectives)})." - ) - - timer.start('prepare_matrices') - inf = float('inf') - ninf = -inf - bounds = list(map(operator.attrgetter('bounds'), repn.columns)) - lb = [ninf if _b is None else _b for _b in map(operator.itemgetter(0), bounds)] - ub = [inf if _b is None else _b for _b in map(operator.itemgetter(1), bounds)] - CON = gurobipy.GRB.CONTINUOUS - BIN = gurobipy.GRB.BINARY - INT = gurobipy.GRB.INTEGER - vtype = [ - ( - CON - if v.is_continuous() - else BIN if v.is_binary() else INT if v.is_integer() else '?' - ) - for v in repn.columns - ] - sense_type = list('=<>') # Note: ordering matches 0, 1, -1 - sense = [sense_type[r[1]] for r in repn.rows] - timer.stop('prepare_matrices') - - ostreams = [io.StringIO()] + config.tee - res = Results() - - orig_cwd = os.getcwd() - try: - if config.working_dir: - os.chdir(config.working_dir) - with capture_output(TeeStream(*ostreams), capture_fd=False): - gurobi_model = gurobipy.Model(env=self.env()) - - timer.start('transfer_model') - x = gurobi_model.addMVar( - len(repn.columns), - lb=lb, - ub=ub, - obj=repn.c.todense()[0] if repn.c.shape[0] else 0, - vtype=vtype, - ) - A = gurobi_model.addMConstr(repn.A, x, sense, repn.rhs) - if repn.c.shape[0]: - gurobi_model.setAttr('ObjCon', repn.c_offset[0]) - gurobi_model.setAttr('ModelSense', int(repn.objectives[0].sense)) - # Note: calling gurobi_model.update() here is not - # necessary (it will happen as part of optimize()): - # gurobi_model.update() - timer.stop('transfer_model') - - options = config.solver_options - - gurobi_model.setParam('LogToConsole', 1) - - if config.threads is not None: - gurobi_model.setParam('Threads', config.threads) - if config.time_limit is not None: - gurobi_model.setParam('TimeLimit', config.time_limit) - if config.rel_gap is not None: - gurobi_model.setParam('MIPGap', config.rel_gap) - if config.abs_gap is not None: - gurobi_model.setParam('MIPGapAbs', config.abs_gap) - - if config.use_mipstart: - raise MouseTrap("MIPSTART not yet supported") - - for key, option in options.items(): - gurobi_model.setParam(key, option) - - timer.start('optimize') - gurobi_model.optimize() - timer.stop('optimize') - finally: - os.chdir(orig_cwd) - - res = self._postsolve( - timer, - config, - GurobiDirectSolutionLoader( - gurobi_model, A, x, repn.rows, repn.columns, repn.objectives - ), - ) - - res.solver_config = config - res.solver_name = 'Gurobi' - res.solver_version = self.version() - res.solver_log = ostreams[0].getvalue() - - end_timestamp = datetime.datetime.now(datetime.timezone.utc) - res.timing_info.start_timestamp = start_timestamp - res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() - res.timing_info.timer = timer - return res - - def _postsolve(self, timer: HierarchicalTimer, config, loader): - grb_model = loader._grb_model - status = grb_model.Status - - results = Results() - results.solution_loader = loader - results.timing_info.gurobi_time = grb_model.Runtime - - if grb_model.SolCount > 0: - if status == gurobipy.GRB.OPTIMAL: - results.solution_status = SolutionStatus.optimal - else: - results.solution_status = SolutionStatus.feasible - else: - results.solution_status = SolutionStatus.noSolution - - results.termination_condition = self._get_tc_map().get( - status, TerminationCondition.unknown - ) - - if ( - results.termination_condition - != TerminationCondition.convergenceCriteriaSatisfied - and config.raise_exception_on_nonoptimal_result - ): - raise NoOptimalSolutionError() - - if loader._pyo_obj: - try: - if math.isfinite(grb_model.ObjVal): - results.incumbent_objective = grb_model.ObjVal - else: - results.incumbent_objective = None - except (gurobipy.GurobiError, AttributeError): - results.incumbent_objective = None - try: - results.objective_bound = grb_model.ObjBound - except (gurobipy.GurobiError, AttributeError): - if grb_model.ModelSense == ObjectiveSense.minimize: - results.objective_bound = -math.inf - else: - results.objective_bound = math.inf - else: - results.incumbent_objective = None - results.objective_bound = None - - results.iteration_count = grb_model.getAttr('IterCount') - - timer.start('load solution') - if config.load_solutions: - if grb_model.SolCount > 0: - results.solution_loader.load_vars() - else: - raise NoFeasibleSolutionError() - timer.stop('load solution') - - return results - - def _get_tc_map(self): - if GurobiDirect._tc_map is None: - grb = gurobipy.GRB - tc = TerminationCondition - GurobiDirect._tc_map = { - grb.LOADED: tc.unknown, # problem is loaded, but no solution - grb.OPTIMAL: tc.convergenceCriteriaSatisfied, - grb.INFEASIBLE: tc.provenInfeasible, - grb.INF_OR_UNBD: tc.infeasibleOrUnbounded, - grb.UNBOUNDED: tc.unbounded, - grb.CUTOFF: tc.objectiveLimit, - grb.ITERATION_LIMIT: tc.iterationLimit, - grb.NODE_LIMIT: tc.iterationLimit, - grb.TIME_LIMIT: tc.maxTimeLimit, - grb.SOLUTION_LIMIT: tc.unknown, - grb.INTERRUPTED: tc.interrupted, - grb.NUMERIC: tc.unknown, - grb.SUBOPTIMAL: tc.unknown, - grb.USER_OBJ_LIMIT: tc.objectiveLimit, - } - return GurobiDirect._tc_map diff --git a/pyomo/contrib/solver/solvers/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi_persistent.py deleted file mode 100644 index ea3693c1c70..00000000000 --- a/pyomo/contrib/solver/solvers/gurobi_persistent.py +++ /dev/null @@ -1,1409 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import io -import logging -import math -from typing import List, Optional -from collections.abc import Iterable - -from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet -from pyomo.common.dependencies import attempt_import -from pyomo.common.errors import ApplicationError -from pyomo.common.tee import capture_output, TeeStream -from pyomo.common.timing import HierarchicalTimer -from pyomo.common.shutdown import python_is_shutting_down -from pyomo.core.kernel.objective import minimize, maximize -from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler -from pyomo.core.base.var import VarData -from pyomo.core.base.constraint import ConstraintData -from pyomo.core.base.sos import SOSConstraintData -from pyomo.core.base.param import ParamData -from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types -from pyomo.repn import generate_standard_repn -from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability -from pyomo.contrib.solver.common.results import ( - Results, - TerminationCondition, - SolutionStatus, -) -from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig -from pyomo.contrib.solver.solvers.gurobi_direct import ( - GurobiConfigMixin, - GurobiSolverMixin, -) -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoDualsError, - NoReducedCostsError, - NoSolutionError, - IncompatibleModelError, -) -from pyomo.contrib.solver.common.persistent import ( - PersistentSolverUtils, - PersistentSolverMixin, -) -from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader -from pyomo.core.staleflag import StaleFlagManager - - -logger = logging.getLogger(__name__) - - -def _import_gurobipy(): - try: - import gurobipy - except ImportError: - GurobiPersistent._available = Availability.NotFound - raise - if gurobipy.GRB.VERSION_MAJOR < 7: - GurobiPersistent._available = Availability.BadVersion - raise ImportError('The Persistent Gurobi interface requires gurobipy>=7.0.0') - return gurobipy - - -gurobipy, gurobipy_available = attempt_import('gurobipy', importer=_import_gurobipy) - - -class GurobiConfig(PersistentBranchAndBoundConfig, GurobiConfigMixin): - def __init__( - self, - description=None, - doc=None, - implicit=False, - implicit_domain=None, - visibility=0, - ): - PersistentBranchAndBoundConfig.__init__( - self, - description=description, - doc=doc, - implicit=implicit, - implicit_domain=implicit_domain, - visibility=visibility, - ) - GurobiConfigMixin.__init__(self) - - -class GurobiSolutionLoader(PersistentSolutionLoader): - def load_vars(self, vars_to_load=None, solution_number=0): - self._assert_solution_still_valid() - self._solver._load_vars( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - def get_primals(self, vars_to_load=None, solution_number=0): - self._assert_solution_still_valid() - return self._solver._get_primals( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - -class _MutableLowerBound: - def __init__(self, expr): - self.var = None - self.expr = expr - - def update(self): - self.var.setAttr('lb', value(self.expr)) - - -class _MutableUpperBound: - def __init__(self, expr): - self.var = None - self.expr = expr - - def update(self): - self.var.setAttr('ub', value(self.expr)) - - -class _MutableLinearCoefficient: - def __init__(self): - self.expr = None - self.var = None - self.con = None - self.gurobi_model = None - - def update(self): - self.gurobi_model.chgCoeff(self.con, self.var, value(self.expr)) - - -class _MutableRangeConstant: - def __init__(self): - self.lhs_expr = None - self.rhs_expr = None - self.con = None - self.slack_name = None - self.gurobi_model = None - - def update(self): - rhs_val = value(self.rhs_expr) - lhs_val = value(self.lhs_expr) - self.con.rhs = rhs_val - slack = self.gurobi_model.getVarByName(self.slack_name) - slack.ub = rhs_val - lhs_val - - -class _MutableConstant: - def __init__(self): - self.expr = None - self.con = None - - def update(self): - self.con.rhs = value(self.expr) - - -class _MutableQuadraticConstraint: - def __init__( - self, gurobi_model, gurobi_con, constant, linear_coefs, quadratic_coefs - ): - self.con = gurobi_con - self.gurobi_model = gurobi_model - self.constant = constant - self.last_constant_value = value(self.constant.expr) - self.linear_coefs = linear_coefs - self.last_linear_coef_values = [value(i.expr) for i in self.linear_coefs] - self.quadratic_coefs = quadratic_coefs - self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] - - def get_updated_expression(self): - gurobi_expr = self.gurobi_model.getQCRow(self.con) - for ndx, coef in enumerate(self.linear_coefs): - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_linear_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var - self.last_linear_coef_values[ndx] = current_coef_value - for ndx, coef in enumerate(self.quadratic_coefs): - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_quadratic_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 - self.last_quadratic_coef_values[ndx] = current_coef_value - return gurobi_expr - - def get_updated_rhs(self): - return value(self.constant.expr) - - -class _MutableObjective: - def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): - self.gurobi_model = gurobi_model - self.constant = constant - self.linear_coefs = linear_coefs - self.quadratic_coefs = quadratic_coefs - self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] - - def get_updated_expression(self): - for ndx, coef in enumerate(self.linear_coefs): - coef.var.obj = value(coef.expr) - self.gurobi_model.ObjCon = value(self.constant.expr) - - gurobi_expr = None - for ndx, coef in enumerate(self.quadratic_coefs): - if value(coef.expr) != self.last_quadratic_coef_values[ndx]: - if gurobi_expr is None: - self.gurobi_model.update() - gurobi_expr = self.gurobi_model.getObjective() - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_quadratic_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 - self.last_quadratic_coef_values[ndx] = current_coef_value - return gurobi_expr - - -class _MutableQuadraticCoefficient: - def __init__(self): - self.expr = None - self.var1 = None - self.var2 = None - - -class GurobiPersistent( - GurobiSolverMixin, - PersistentSolverMixin, - PersistentSolverUtils, - PersistentSolverBase, -): - """ - Interface to Gurobi persistent - """ - - CONFIG = GurobiConfig() - _gurobipy_available = gurobipy_available - - def __init__(self, **kwds): - treat_fixed_vars_as_params = kwds.pop('treat_fixed_vars_as_params', True) - PersistentSolverBase.__init__(self, **kwds) - PersistentSolverUtils.__init__( - self, treat_fixed_vars_as_params=treat_fixed_vars_as_params - ) - self._register_env_client() - self._solver_model = None - self._symbol_map = SymbolMap() - self._labeler = None - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._solver_con_to_pyomo_con_map = {} - self._pyomo_sos_to_solver_sos_map = {} - self._range_constraints = OrderedSet() - self._mutable_helpers = {} - self._mutable_bounds = {} - self._mutable_quadratic_helpers = {} - self._mutable_objective = None - self._needs_updated = True - self._callback = None - self._callback_func = None - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._last_results_object: Optional[Results] = None - - def release_license(self): - self._reinit() - self.__class__.release_license() - - def __del__(self): - if not python_is_shutting_down(): - self._release_env_client() - - @property - def symbol_map(self): - return self._symbol_map - - def _solve(self): - config = self._active_config - timer = config.timer - ostreams = [io.StringIO()] + config.tee - - with capture_output(TeeStream(*ostreams), capture_fd=False): - options = config.solver_options - - self._solver_model.setParam('LogToConsole', 1) - - if config.threads is not None: - self._solver_model.setParam('Threads', config.threads) - if config.time_limit is not None: - self._solver_model.setParam('TimeLimit', config.time_limit) - if config.rel_gap is not None: - self._solver_model.setParam('MIPGap', config.rel_gap) - if config.abs_gap is not None: - self._solver_model.setParam('MIPGapAbs', config.abs_gap) - - if config.use_mipstart: - for ( - pyomo_var_id, - gurobi_var, - ) in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[pyomo_var_id][0] - if pyomo_var.is_integer() and pyomo_var.value is not None: - self.set_var_attr(pyomo_var, 'Start', pyomo_var.value) - - for key, option in options.items(): - self._solver_model.setParam(key, option) - - timer.start('optimize') - self._solver_model.optimize(self._callback) - timer.stop('optimize') - - self._needs_updated = False - res = self._postsolve(timer) - res.solver_config = config - res.solver_name = 'Gurobi' - res.solver_version = self.version() - res.solver_log = ostreams[0].getvalue() - return res - - def _process_domain_and_bounds( - self, var, var_id, mutable_lbs, mutable_ubs, ndx, gurobipy_var - ): - _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[id(var)] - lb, ub, step = _domain_interval - if lb is None: - lb = -gurobipy.GRB.INFINITY - if ub is None: - ub = gurobipy.GRB.INFINITY - if step == 0: - vtype = gurobipy.GRB.CONTINUOUS - elif step == 1: - if lb == 0 and ub == 1: - vtype = gurobipy.GRB.BINARY - else: - vtype = gurobipy.GRB.INTEGER - else: - raise ValueError( - f'Unrecognized domain step: {step} (should be either 0 or 1)' - ) - if _fixed: - lb = _value - ub = _value - else: - if _lb is not None: - if not is_constant(_lb): - mutable_bound = _MutableLowerBound(NPV_MaxExpression((_lb, lb))) - if gurobipy_var is None: - mutable_lbs[ndx] = mutable_bound - else: - mutable_bound.var = gurobipy_var - self._mutable_bounds[var_id, 'lb'] = (var, mutable_bound) - lb = max(value(_lb), lb) - if _ub is not None: - if not is_constant(_ub): - mutable_bound = _MutableUpperBound(NPV_MinExpression((_ub, ub))) - if gurobipy_var is None: - mutable_ubs[ndx] = mutable_bound - else: - mutable_bound.var = gurobipy_var - self._mutable_bounds[var_id, 'ub'] = (var, mutable_bound) - ub = min(value(_ub), ub) - - return lb, ub, vtype - - def _add_variables(self, variables: List[VarData]): - var_names = [] - vtypes = [] - lbs = [] - ubs = [] - mutable_lbs = {} - mutable_ubs = {} - for ndx, var in enumerate(variables): - varname = self._symbol_map.getSymbol(var, self._labeler) - lb, ub, vtype = self._process_domain_and_bounds( - var, id(var), mutable_lbs, mutable_ubs, ndx, None - ) - var_names.append(varname) - vtypes.append(vtype) - lbs.append(lb) - ubs.append(ub) - - gurobi_vars = self._solver_model.addVars( - len(variables), lb=lbs, ub=ubs, vtype=vtypes, name=var_names - ) - - for ndx, pyomo_var in enumerate(variables): - gurobi_var = gurobi_vars[ndx] - self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var - for ndx, mutable_bound in mutable_lbs.items(): - mutable_bound.var = gurobi_vars[ndx] - for ndx, mutable_bound in mutable_ubs.items(): - mutable_bound.var = gurobi_vars[ndx] - self._vars_added_since_update.update(variables) - self._needs_updated = True - - def _add_parameters(self, params: List[ParamData]): - pass - - def _reinit(self): - saved_config = self.config - saved_tmp_config = self._active_config - self.__init__(treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) - # Note that __init__ registers a new env client, so we need to - # release it here: - self._release_env_client() - self.config = saved_config - self._active_config = saved_tmp_config - - def set_instance(self, model): - if self._last_results_object is not None: - self._last_results_object.solution_loader.invalidate() - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - self._reinit() - self._model = model - - if self.config.symbolic_solver_labels: - self._labeler = TextLabeler() - else: - self._labeler = NumericLabeler('x') - - self._solver_model = gurobipy.Model(name=model.name or '', env=self.env()) - - self.add_block(model) - if self._objective is None: - self.set_objective(None) - - def _get_expr_from_pyomo_expr(self, expr): - mutable_linear_coefficients = [] - mutable_quadratic_coefficients = [] - repn = generate_standard_repn(expr, quadratic=True, compute_values=False) - - degree = repn.polynomial_degree() - if (degree is None) or (degree > 2): - raise IncompatibleModelError( - f'GurobiAuto does not support expressions of degree {degree}.' - ) - - if len(repn.linear_vars) > 0: - linear_coef_vals = [] - for ndx, coef in enumerate(repn.linear_coefs): - if not is_constant(coef): - mutable_linear_coefficient = _MutableLinearCoefficient() - mutable_linear_coefficient.expr = coef - mutable_linear_coefficient.var = self._pyomo_var_to_solver_var_map[ - id(repn.linear_vars[ndx]) - ] - mutable_linear_coefficients.append(mutable_linear_coefficient) - linear_coef_vals.append(value(coef)) - new_expr = gurobipy.LinExpr( - linear_coef_vals, - [self._pyomo_var_to_solver_var_map[id(i)] for i in repn.linear_vars], - ) - else: - new_expr = 0.0 - - for ndx, v in enumerate(repn.quadratic_vars): - x, y = v - gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] - gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - coef = repn.quadratic_coefs[ndx] - if not is_constant(coef): - mutable_quadratic_coefficient = _MutableQuadraticCoefficient() - mutable_quadratic_coefficient.expr = coef - mutable_quadratic_coefficient.var1 = gurobi_x - mutable_quadratic_coefficient.var2 = gurobi_y - mutable_quadratic_coefficients.append(mutable_quadratic_coefficient) - coef_val = value(coef) - new_expr += coef_val * gurobi_x * gurobi_y - - return ( - new_expr, - repn.constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - - def _add_constraints(self, cons: List[ConstraintData]): - for con in cons: - conname = self._symbol_map.getSymbol(con, self._labeler) - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if ( - gurobi_expr.__class__ in {gurobipy.LinExpr, gurobipy.Var} - or gurobi_expr.__class__ in native_numeric_types - ): - if con.equality: - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - elif con.has_lb() and con.has_ub(): - lhs_expr = con.lower - repn_constant - rhs_expr = con.upper - repn_constant - lhs_val = value(lhs_expr) - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addRange( - gurobi_expr, lhs_val, rhs_val, name=conname - ) - self._range_constraints.add(con) - if not is_constant(lhs_expr) or not is_constant(rhs_expr): - mutable_range_constant = _MutableRangeConstant() - mutable_range_constant.lhs_expr = lhs_expr - mutable_range_constant.rhs_expr = rhs_expr - mutable_range_constant.con = gurobipy_con - mutable_range_constant.slack_name = 'Rg' + conname - mutable_range_constant.gurobi_model = self._solver_model - self._mutable_helpers[con] = [mutable_range_constant] - elif con.has_lb(): - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.GREATER_EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - elif con.has_ub(): - rhs_expr = con.upper - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.LESS_EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - else: - raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" - ) - for tmp in mutable_linear_coefficients: - tmp.con = gurobipy_con - tmp.gurobi_model = self._solver_model - if len(mutable_linear_coefficients) > 0: - if con not in self._mutable_helpers: - self._mutable_helpers[con] = mutable_linear_coefficients - else: - self._mutable_helpers[con].extend(mutable_linear_coefficients) - elif gurobi_expr.__class__ is gurobipy.QuadExpr: - if con.equality: - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.EQUAL, rhs_val, name=conname - ) - elif con.has_lb() and con.has_ub(): - raise NotImplementedError( - 'Quadratic range constraints are not supported' - ) - elif con.has_lb(): - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.GREATER_EQUAL, rhs_val, name=conname - ) - elif con.has_ub(): - rhs_expr = con.upper - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.LESS_EQUAL, rhs_val, name=conname - ) - else: - raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" - ) - if ( - len(mutable_linear_coefficients) > 0 - or len(mutable_quadratic_coefficients) > 0 - or not is_constant(repn_constant) - ): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_quadratic_constraint = _MutableQuadraticConstraint( - self._solver_model, - gurobipy_con, - mutable_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - self._mutable_quadratic_helpers[con] = mutable_quadratic_constraint - else: - raise ValueError( - f'Unrecognized Gurobi expression type: {str(gurobi_expr.__class__)}' - ) - - self._pyomo_con_to_solver_con_map[con] = gurobipy_con - self._solver_con_to_pyomo_con_map[id(gurobipy_con)] = con - self._constraints_added_since_update.update(cons) - self._needs_updated = True - - def _add_sos_constraints(self, cons: List[SOSConstraintData]): - for con in cons: - conname = self._symbol_map.getSymbol(con, self._labeler) - level = con.level - if level == 1: - sos_type = gurobipy.GRB.SOS_TYPE1 - elif level == 2: - sos_type = gurobipy.GRB.SOS_TYPE2 - else: - raise ValueError( - f"Solver does not support SOS level {level} constraints" - ) - - gurobi_vars = [] - weights = [] - - for v, w in con.get_items(): - v_id = id(v) - gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) - weights.append(w) - - gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) - self._pyomo_sos_to_solver_sos_map[con] = gurobipy_con - self._constraints_added_since_update.update(cons) - self._needs_updated = True - - def _remove_constraints(self, cons: List[ConstraintData]): - for con in cons: - if con in self._constraints_added_since_update: - self._update_gurobi_model() - solver_con = self._pyomo_con_to_solver_con_map[con] - self._solver_model.remove(solver_con) - self._symbol_map.removeSymbol(con) - del self._pyomo_con_to_solver_con_map[con] - del self._solver_con_to_pyomo_con_map[id(solver_con)] - self._range_constraints.discard(con) - self._mutable_helpers.pop(con, None) - self._mutable_quadratic_helpers.pop(con, None) - self._needs_updated = True - - def _remove_sos_constraints(self, cons: List[SOSConstraintData]): - for con in cons: - if con in self._constraints_added_since_update: - self._update_gurobi_model() - solver_sos_con = self._pyomo_sos_to_solver_sos_map[con] - self._solver_model.remove(solver_sos_con) - self._symbol_map.removeSymbol(con) - del self._pyomo_sos_to_solver_sos_map[con] - self._needs_updated = True - - def _remove_variables(self, variables: List[VarData]): - for var in variables: - v_id = id(var) - if var in self._vars_added_since_update: - self._update_gurobi_model() - solver_var = self._pyomo_var_to_solver_var_map[v_id] - self._solver_model.remove(solver_var) - self._symbol_map.removeSymbol(var) - del self._pyomo_var_to_solver_var_map[v_id] - self._mutable_bounds.pop(v_id, None) - self._needs_updated = True - - def _remove_parameters(self, params: List[ParamData]): - pass - - def _update_variables(self, variables: List[VarData]): - for var in variables: - var_id = id(var) - if var_id not in self._pyomo_var_to_solver_var_map: - raise ValueError( - f'The Var provided to update_var needs to be added first: {var}' - ) - self._mutable_bounds.pop((var_id, 'lb'), None) - self._mutable_bounds.pop((var_id, 'ub'), None) - gurobipy_var = self._pyomo_var_to_solver_var_map[var_id] - lb, ub, vtype = self._process_domain_and_bounds( - var, var_id, None, None, None, gurobipy_var - ) - gurobipy_var.setAttr('lb', lb) - gurobipy_var.setAttr('ub', ub) - gurobipy_var.setAttr('vtype', vtype) - self._needs_updated = True - - def update_parameters(self): - for con, helpers in self._mutable_helpers.items(): - for helper in helpers: - helper.update() - for k, (v, helper) in self._mutable_bounds.items(): - helper.update() - - for con, helper in self._mutable_quadratic_helpers.items(): - if con in self._constraints_added_since_update: - self._update_gurobi_model() - gurobi_con = helper.con - new_gurobi_expr = helper.get_updated_expression() - new_rhs = helper.get_updated_rhs() - new_sense = gurobi_con.qcsense - pyomo_con = self._solver_con_to_pyomo_con_map[id(gurobi_con)] - name = self._symbol_map.getSymbol(pyomo_con, self._labeler) - self._solver_model.remove(gurobi_con) - new_con = self._solver_model.addQConstr( - new_gurobi_expr, new_sense, new_rhs, name=name - ) - self._pyomo_con_to_solver_con_map[id(pyomo_con)] = new_con - del self._solver_con_to_pyomo_con_map[id(gurobi_con)] - self._solver_con_to_pyomo_con_map[id(new_con)] = pyomo_con - helper.con = new_con - self._constraints_added_since_update.add(con) - - helper = self._mutable_objective - pyomo_obj = self._objective - new_gurobi_expr = helper.get_updated_expression() - if new_gurobi_expr is not None: - if pyomo_obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - else: - sense = gurobipy.GRB.MAXIMIZE - self._solver_model.setObjective(new_gurobi_expr, sense=sense) - - def _set_objective(self, obj): - if obj is None: - sense = gurobipy.GRB.MINIMIZE - gurobi_expr = 0 - repn_constant = 0 - mutable_linear_coefficients = [] - mutable_quadratic_coefficients = [] - else: - if obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - elif obj.sense == maximize: - sense = gurobipy.GRB.MAXIMIZE - else: - raise ValueError(f'Objective sense is not recognized: {obj.sense}') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(obj.expr) - - mutable_constant = _MutableConstant() - mutable_constant.expr = repn_constant - mutable_objective = _MutableObjective( - self._solver_model, - mutable_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - self._mutable_objective = mutable_objective - - # These two lines are needed as a workaround - # see PR #2454 - self._solver_model.setObjective(0) - self._solver_model.update() - - self._solver_model.setObjective(gurobi_expr + value(repn_constant), sense=sense) - self._needs_updated = True - - def _postsolve(self, timer: HierarchicalTimer): - config = self._active_config - - gprob = self._solver_model - grb = gurobipy.GRB - status = gprob.Status - - results = Results() - results.solution_loader = GurobiSolutionLoader(self) - results.timing_info.gurobi_time = gprob.Runtime - - if gprob.SolCount > 0: - if status == grb.OPTIMAL: - results.solution_status = SolutionStatus.optimal - else: - results.solution_status = SolutionStatus.feasible - else: - results.solution_status = SolutionStatus.noSolution - - if status == grb.LOADED: # problem is loaded, but no solution - results.termination_condition = TerminationCondition.unknown - elif status == grb.OPTIMAL: # optimal - results.termination_condition = ( - TerminationCondition.convergenceCriteriaSatisfied - ) - elif status == grb.INFEASIBLE: - results.termination_condition = TerminationCondition.provenInfeasible - elif status == grb.INF_OR_UNBD: - results.termination_condition = TerminationCondition.infeasibleOrUnbounded - elif status == grb.UNBOUNDED: - results.termination_condition = TerminationCondition.unbounded - elif status == grb.CUTOFF: - results.termination_condition = TerminationCondition.objectiveLimit - elif status == grb.ITERATION_LIMIT: - results.termination_condition = TerminationCondition.iterationLimit - elif status == grb.NODE_LIMIT: - results.termination_condition = TerminationCondition.iterationLimit - elif status == grb.TIME_LIMIT: - results.termination_condition = TerminationCondition.maxTimeLimit - elif status == grb.SOLUTION_LIMIT: - results.termination_condition = TerminationCondition.unknown - elif status == grb.INTERRUPTED: - results.termination_condition = TerminationCondition.interrupted - elif status == grb.NUMERIC: - results.termination_condition = TerminationCondition.unknown - elif status == grb.SUBOPTIMAL: - results.termination_condition = TerminationCondition.unknown - elif status == grb.USER_OBJ_LIMIT: - results.termination_condition = TerminationCondition.objectiveLimit - else: - results.termination_condition = TerminationCondition.unknown - - if ( - results.termination_condition - != TerminationCondition.convergenceCriteriaSatisfied - and config.raise_exception_on_nonoptimal_result - ): - raise NoOptimalSolutionError() - - results.incumbent_objective = None - results.objective_bound = None - if self._objective is not None: - try: - results.incumbent_objective = gprob.ObjVal - except (gurobipy.GurobiError, AttributeError): - results.incumbent_objective = None - try: - results.objective_bound = gprob.ObjBound - except (gurobipy.GurobiError, AttributeError): - if self._objective.sense == minimize: - results.objective_bound = -math.inf - else: - results.objective_bound = math.inf - - if results.incumbent_objective is not None and not math.isfinite( - results.incumbent_objective - ): - results.incumbent_objective = None - - results.iteration_count = gprob.getAttr('IterCount') - - timer.start('load solution') - if config.load_solutions: - if gprob.SolCount > 0: - self._load_vars() - else: - raise NoFeasibleSolutionError() - timer.stop('load solution') - - return results - - def _load_suboptimal_mip_solution(self, vars_to_load, solution_number): - if ( - self.get_model_attr('NumIntVars') == 0 - and self.get_model_attr('NumBinVars') == 0 - ): - raise ValueError( - 'Cannot obtain suboptimal solutions for a continuous model' - ) - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - original_solution_number = self.get_gurobi_param_info('SolutionNumber')[2] - self.set_gurobi_param('SolutionNumber', solution_number) - gurobi_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] - vals = self._solver_model.getAttr("Xn", gurobi_vars_to_load) - res = ComponentMap() - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - self.set_gurobi_param('SolutionNumber', original_solution_number) - return res - - def _load_vars(self, vars_to_load=None, solution_number=0): - for v, val in self._get_primals( - vars_to_load=vars_to_load, solution_number=solution_number - ).items(): - v.set_value(val, skip_validation=True) - StaleFlagManager.mark_all_as_stale(delayed=True) - - def _get_primals(self, vars_to_load=None, solution_number=0): - if self._needs_updated: - self._update_gurobi_model() # this is needed to ensure that solutions cannot be loaded after the model has been changed - - if self._solver_model.SolCount == 0: - raise NoSolutionError() - - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - if vars_to_load is None: - vars_to_load = self._pyomo_var_to_solver_var_map.keys() - else: - vars_to_load = [id(v) for v in vars_to_load] - - if solution_number != 0: - return self._load_suboptimal_mip_solution( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - gurobi_vars_to_load = [var_map[pyomo_var_id] for pyomo_var_id in vars_to_load] - vals = self._solver_model.getAttr("X", gurobi_vars_to_load) - - res = ComponentMap() - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - return res - - def _get_reduced_costs(self, vars_to_load=None): - if self._needs_updated: - self._update_gurobi_model() - - if self._solver_model.Status != gurobipy.GRB.OPTIMAL: - raise NoReducedCostsError() - - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - res = ComponentMap() - if vars_to_load is None: - vars_to_load = self._pyomo_var_to_solver_var_map.keys() - else: - vars_to_load = [id(v) for v in vars_to_load] - - gurobi_vars_to_load = [var_map[pyomo_var_id] for pyomo_var_id in vars_to_load] - vals = self._solver_model.getAttr("Rc", gurobi_vars_to_load) - - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - - return res - - def _get_duals(self, cons_to_load=None): - if self._needs_updated: - self._update_gurobi_model() - - if self._solver_model.Status != gurobipy.GRB.OPTIMAL: - raise NoDualsError() - - con_map = self._pyomo_con_to_solver_con_map - reverse_con_map = self._solver_con_to_pyomo_con_map - dual = {} - - if cons_to_load is None: - linear_cons_to_load = self._solver_model.getConstrs() - quadratic_cons_to_load = self._solver_model.getQConstrs() - else: - gurobi_cons_to_load = OrderedSet( - [con_map[pyomo_con] for pyomo_con in cons_to_load] - ) - linear_cons_to_load = list( - gurobi_cons_to_load.intersection( - OrderedSet(self._solver_model.getConstrs()) - ) - ) - quadratic_cons_to_load = list( - gurobi_cons_to_load.intersection( - OrderedSet(self._solver_model.getQConstrs()) - ) - ) - linear_vals = self._solver_model.getAttr("Pi", linear_cons_to_load) - quadratic_vals = self._solver_model.getAttr("QCPi", quadratic_cons_to_load) - - for gurobi_con, val in zip(linear_cons_to_load, linear_vals): - pyomo_con = reverse_con_map[id(gurobi_con)] - dual[pyomo_con] = val - for gurobi_con, val in zip(quadratic_cons_to_load, quadratic_vals): - pyomo_con = reverse_con_map[id(gurobi_con)] - dual[pyomo_con] = val - - return dual - - def update(self, timer: HierarchicalTimer = None): - if self._needs_updated: - self._update_gurobi_model() - super().update(timer=timer) - self._update_gurobi_model() - - def _update_gurobi_model(self): - self._solver_model.update() - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._needs_updated = False - - def get_model_attr(self, attr): - """ - Get the value of an attribute on the Gurobi model. - - Parameters - ---------- - attr: str - The attribute to get. See Gurobi documentation for descriptions of the attributes. - """ - if self._needs_updated: - self._update_gurobi_model() - return self._solver_model.getAttr(attr) - - def write(self, filename): - """ - Write the model to a file (e.g., and lp file). - - Parameters - ---------- - filename: str - Name of the file to which the model should be written. - """ - self._solver_model.write(filename) - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._needs_updated = False - - def set_linear_constraint_attr(self, con, attr, val): - """ - Set the value of an attribute on a gurobi linear constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be modified. - attr: str - The attribute to be modified. Options are: - CBasis - DStart - Lazy - val: any - See gurobi documentation for acceptable values. - """ - if attr in {'Sense', 'RHS', 'ConstrName'}: - raise ValueError( - f'Linear constraint attr {attr} cannot be set with' - ' the set_linear_constraint_attr method. Please use' - ' the remove_constraint and add_constraint methods.' - ) - self._pyomo_con_to_solver_con_map[con].setAttr(attr, val) - self._needs_updated = True - - def set_var_attr(self, var, attr, val): - """ - Set the value of an attribute on a gurobi variable. - - Parameters - ---------- - var: pyomo.core.base.var.VarData - The pyomo var for which the corresponding gurobi var attribute - should be modified. - attr: str - The attribute to be modified. Options are: - Start - VarHintVal - VarHintPri - BranchPriority - VBasis - PStart - val: any - See gurobi documentation for acceptable values. - """ - if attr in {'LB', 'UB', 'VType', 'VarName'}: - raise ValueError( - f'Var attr {attr} cannot be set with' - ' the set_var_attr method. Please use' - ' the update_var method.' - ) - if attr == 'Obj': - raise ValueError( - 'Var attr Obj cannot be set with' - ' the set_var_attr method. Please use' - ' the set_objective method.' - ) - self._pyomo_var_to_solver_var_map[id(var)].setAttr(attr, val) - self._needs_updated = True - - def get_var_attr(self, var, attr): - """ - Get the value of an attribute on a gurobi var. - - Parameters - ---------- - var: pyomo.core.base.var.VarData - The pyomo var for which the corresponding gurobi var attribute - should be retrieved. - attr: str - The attribute to get. See gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_var_to_solver_var_map[id(var)].getAttr(attr) - - def get_linear_constraint_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi linear constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_con_to_solver_con_map[con].getAttr(attr) - - def get_sos_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi sos constraint. - - Parameters - ---------- - con: pyomo.core.base.sos.SOSConstraintData - The pyomo SOS constraint for which the corresponding gurobi SOS constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_sos_to_solver_sos_map[con].getAttr(attr) - - def get_quadratic_constraint_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi quadratic constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_con_to_solver_con_map[con].getAttr(attr) - - def set_gurobi_param(self, param, val): - """ - Set a gurobi parameter. - - Parameters - ---------- - param: str - The gurobi parameter to set. Options include any gurobi parameter. - Please see the Gurobi documentation for options. - val: any - The value to set the parameter to. See Gurobi documentation for possible values. - """ - self._solver_model.setParam(param, val) - - def get_gurobi_param_info(self, param): - """ - Get information about a gurobi parameter. - - Parameters - ---------- - param: str - The gurobi parameter to get info for. See Gurobi documentation for possible options. - - Returns - ------- - six-tuple containing the parameter name, type, value, minimum value, maximum value, and default value. - """ - return self._solver_model.getParamInfo(param) - - def _intermediate_callback(self): - def f(gurobi_model, where): - self._callback_func(self._model, self, where) - - return f - - def set_callback(self, func=None): - """ - Specify a callback for gurobi to use. - - Parameters - ---------- - func: function - The function to call. The function should have three arguments. The first will be the pyomo model being - solved. The second will be the GurobiPersistent instance. The third will be an enum member of - gurobipy.GRB.Callback. This will indicate where in the branch and bound algorithm gurobi is at. For - example, suppose we want to solve - - .. math:: - - min 2*x + y - - s.t. - - y >= (x-2)**2 - - 0 <= x <= 4 - - y >= 0 - - y integer - - as an MILP using extended cutting planes in callbacks. - - >>> from gurobipy import GRB # doctest:+SKIP - >>> import pyomo.environ as pyo - >>> from pyomo.core.expr.taylor_series import taylor_series_expansion - >>> from pyomo.contrib import appsi - >>> - >>> m = pyo.ConcreteModel() - >>> m.x = pyo.Var(bounds=(0, 4)) - >>> m.y = pyo.Var(within=pyo.Integers, bounds=(0, None)) - >>> m.obj = pyo.Objective(expr=2*m.x + m.y) - >>> m.cons = pyo.ConstraintList() # for the cutting planes - >>> - >>> def _add_cut(xval): - ... # a function to generate the cut - ... m.x.value = xval - ... return m.cons.add(m.y >= taylor_series_expansion((m.x - 2)**2)) - ... - >>> _c = _add_cut(0) # start with 2 cuts at the bounds of x - >>> _c = _add_cut(4) # this is an arbitrary choice - >>> - >>> opt = appsi.solvers.Gurobi() - >>> opt.config.stream_solver = True - >>> opt.set_instance(m) # doctest:+SKIP - >>> opt.gurobi_options['PreCrush'] = 1 - >>> opt.gurobi_options['LazyConstraints'] = 1 - >>> - >>> def my_callback(cb_m, cb_opt, cb_where): - ... if cb_where == GRB.Callback.MIPSOL: - ... cb_opt.cbGetSolution(variables=[m.x, m.y]) - ... if m.y.value < (m.x.value - 2)**2 - 1e-6: - ... cb_opt.cbLazy(_add_cut(m.x.value)) - ... - >>> opt.set_callback(my_callback) - >>> res = opt.solve(m) # doctest:+SKIP - - """ - if func is not None: - self._callback_func = func - self._callback = self._intermediate_callback() - else: - self._callback = None - self._callback_func = None - - def cbCut(self, con): - """ - Add a cut within a callback. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The cut to add - """ - if not con.active: - raise ValueError('cbCut expected an active constraint.') - - if is_fixed(con.body): - raise ValueError('cbCut expected a non-trivial constraint') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if con.has_lb(): - if con.has_ub(): - raise ValueError('Range constraints are not supported in cbCut.') - if not is_fixed(con.lower): - raise ValueError(f'Lower bound of constraint {con} is not constant.') - if con.has_ub(): - if not is_fixed(con.upper): - raise ValueError(f'Upper bound of constraint {con} is not constant.') - - if con.equality: - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_lb() and (value(con.lower) > -float('inf')): - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.GREATER_EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_ub() and (value(con.upper) < float('inf')): - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.LESS_EQUAL, - rhs=value(con.upper - repn_constant), - ) - else: - raise ValueError( - f'Constraint does not have a lower or an upper bound {con} \n' - ) - - def cbGet(self, what): - return self._solver_model.cbGet(what) - - def cbGetNodeRel(self, variables): - """ - Parameters - ---------- - variables: Var or iterable of Var - """ - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - var_values = self._solver_model.cbGetNodeRel(gurobi_vars) - for i, v in enumerate(variables): - v.set_value(var_values[i], skip_validation=True) - - def cbGetSolution(self, variables): - """ - Parameters - ---------- - variables: iterable of vars - """ - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - var_values = self._solver_model.cbGetSolution(gurobi_vars) - for i, v in enumerate(variables): - v.set_value(var_values[i], skip_validation=True) - - def cbLazy(self, con): - """ - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The lazy constraint to add - """ - if not con.active: - raise ValueError('cbLazy expected an active constraint.') - - if is_fixed(con.body): - raise ValueError('cbLazy expected a non-trivial constraint') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if con.has_lb(): - if con.has_ub(): - raise ValueError('Range constraints are not supported in cbLazy.') - if not is_fixed(con.lower): - raise ValueError(f'Lower bound of constraint {con} is not constant.') - if con.has_ub(): - if not is_fixed(con.upper): - raise ValueError(f'Upper bound of constraint {con} is not constant.') - - if con.equality: - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_lb() and (value(con.lower) > -float('inf')): - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.GREATER_EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_ub() and (value(con.upper) < float('inf')): - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.LESS_EQUAL, - rhs=value(con.upper - repn_constant), - ) - else: - raise ValueError( - f'Constraint does not have a lower or an upper bound {con} \n' - ) - - def cbSetSolution(self, variables, solution): - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - self._solver_model.cbSetSolution(gurobi_vars, solution) - - def cbUseSolution(self): - return self._solver_model.cbUseSolution() - - def reset(self): - self._solver_model.reset() diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py index 8703ae9edff..96cd1498956 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py @@ -11,7 +11,7 @@ import pyomo.common.unittest as unittest import pyomo.environ as pyo -from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiPersistent from pyomo.contrib.solver.common.results import SolutionStatus from pyomo.core.expr.taylor_series import taylor_series_expansion @@ -471,11 +471,11 @@ def test_solution_number(self): res = opt.solve(m) num_solutions = opt.get_model_attr('SolCount') self.assertEqual(num_solutions, 3) - res.solution_loader.load_vars(solution_number=0) + res.solution_loader.load_vars(solution_id=0) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.431184939357673) - res.solution_loader.load_vars(solution_number=1) + res.solution_loader.load_vars(solution_id=1) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.584793218502477) - res.solution_loader.load_vars(solution_number=2) + res.solution_loader.load_vars(solution_id=2) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.592304628123309) def test_zero_time_limit(self): @@ -496,16 +496,14 @@ def test_zero_time_limit(self): self.assertIsNone(res.incumbent_objective) -class TestManualModel(unittest.TestCase): +class TestManualMode(unittest.TestCase): def setUp(self): opt = GurobiPersistent() - opt.config.auto_updates.check_for_new_or_removed_params = False - opt.config.auto_updates.check_for_new_or_removed_vars = False - opt.config.auto_updates.check_for_new_or_removed_constraints = False - opt.config.auto_updates.update_parameters = False - opt.config.auto_updates.update_vars = False - opt.config.auto_updates.update_constraints = False - opt.config.auto_updates.update_named_expressions = False + opt.auto_updates.check_for_new_or_removed_constraints = False + opt.auto_updates.update_parameters = False + opt.auto_updates.update_vars = False + opt.auto_updates.update_constraints = False + opt.auto_updates.update_named_expressions = False self.opt = opt def test_basics(self): @@ -603,16 +601,13 @@ def test_update1(self): opt = self.opt opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 1) opt.remove_constraints([m.c1]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 0) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 0) opt.add_constraints([m.c1]) - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 1) def test_update2(self): m = pyo.ConcreteModel() @@ -625,16 +620,13 @@ def test_update2(self): opt = self.opt opt.config.symbolic_solver_labels = True opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumConstrs'), 1) opt.remove_constraints([m.c2]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 0) + self.assertEqual(opt.get_model_attr('NumConstrs'), 0) opt.add_constraints([m.c2]) - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumConstrs'), 1) def test_update3(self): m = pyo.ConcreteModel() @@ -684,16 +676,13 @@ def test_update5(self): opt = self.opt opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 1) + self.assertEqual(opt.get_model_attr('NumSOS'), 1) opt.remove_sos_constraints([m.c1]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 0) + self.assertEqual(opt.get_model_attr('NumSOS'), 0) opt.add_sos_constraints([m.c1]) - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 1) + self.assertEqual(opt.get_model_attr('NumSOS'), 1) def test_update6(self): m = pyo.ConcreteModel() From 92fa4f5c72a4d26c5568a40e7ce7726ddd12e991 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:49:29 -0600 Subject: [PATCH 17/33] remove unused imports --- .../solver/solvers/gurobi/gurobi_direct.py | 20 ---------- .../solvers/gurobi/gurobi_direct_base.py | 7 +--- .../solvers/gurobi/gurobi_persistent.py | 38 +++---------------- 3 files changed, 7 insertions(+), 58 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index f4a33e2cc54..16c633c7d7c 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -9,40 +9,20 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -import datetime -import io -import math import operator -import os from pyomo.common.collections import ComponentMap, ComponentSet -from pyomo.common.config import ConfigValue -from pyomo.common.dependencies import attempt_import -from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import MouseTrap, ApplicationError from pyomo.common.shutdown import python_is_shutting_down -from pyomo.common.tee import capture_output, TeeStream -from pyomo.common.timing import HierarchicalTimer from pyomo.core.staleflag import StaleFlagManager from pyomo.repn.plugins.standard_form import LinearStandardFormCompiler -from pyomo.contrib.solver.common.base import SolverBase, Availability -from pyomo.contrib.solver.common.config import BranchAndBoundConfig from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, NoDualsError, NoReducedCostsError, NoSolutionError, IncompatibleModelError, ) -from pyomo.contrib.solver.common.results import ( - Results, - SolutionStatus, - TerminationCondition, -) from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase -import logging from .gurobi_direct_base import GurobiDirectBase, gurobipy diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index d26dbf54c83..41bdb244743 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -12,14 +12,13 @@ import datetime import io import math -import operator import os -from pyomo.common.collections import ComponentMap, ComponentSet +from pyomo.common.collections import ComponentMap from pyomo.common.config import ConfigValue from pyomo.common.dependencies import attempt_import from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import MouseTrap, ApplicationError +from pyomo.common.errors import ApplicationError from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer @@ -33,14 +32,12 @@ NoDualsError, NoReducedCostsError, NoSolutionError, - IncompatibleModelError, ) from pyomo.contrib.solver.common.results import ( Results, SolutionStatus, TerminationCondition, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase import logging diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index e91381f41a3..7b0463d2cf1 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -10,61 +10,33 @@ # ___________________________________________________________________________ from __future__ import annotations -import io import logging -import math -from typing import Dict, List, NoReturn, Optional, Sequence, Mapping +from typing import Dict, List, Optional, Sequence, Mapping from collections.abc import Iterable -from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet -from pyomo.common.dependencies import attempt_import -from pyomo.common.errors import ApplicationError -from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.collections import ComponentSet, OrderedSet from pyomo.common.timing import HierarchicalTimer -from pyomo.common.shutdown import python_is_shutting_down from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize -from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler from pyomo.core.base.var import VarData from pyomo.core.base.constraint import ConstraintData, Constraint from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.param import ParamData from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types from pyomo.repn import generate_standard_repn -from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability -from pyomo.contrib.solver.common.results import ( - Results, - TerminationCondition, - SolutionStatus, -) -from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoDualsError, - NoReducedCostsError, - NoSolutionError, - IncompatibleModelError, -) -from pyomo.contrib.solver.common.persistent import ( - PersistentSolverUtils, - PersistentSolverMixin, -) -from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader, SolutionLoaderBase +from pyomo.contrib.solver.common.results import Results +from pyomo.contrib.solver.common.util import IncompatibleModelError +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( - GurobiConfig, GurobiDirectBase, gurobipy, - _load_suboptimal_mip_solution, _load_vars, _get_primals, _get_duals, _get_reduced_costs, ) from pyomo.contrib.solver.common.util import get_objective -from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector From 8a9fc46b802dcc181a2b00a131ab089dfc8c59a4 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:50:15 -0600 Subject: [PATCH 18/33] run black --- .../solvers/gurobi/gurobi_direct_base.py | 29 +-- .../solvers/gurobi/gurobi_persistent.py | 213 +++++++++++------- .../solver/tests/solvers/test_solvers.py | 11 +- 3 files changed, 153 insertions(+), 100 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 41bdb244743..df6bb8b5327 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -87,9 +87,7 @@ def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_ solver_model.getAttr('NumIntVars') == 0 and solver_model.getAttr('NumBinVars') == 0 ): - raise ValueError( - 'Cannot obtain suboptimal solutions for a continuous model' - ) + raise ValueError('Cannot obtain suboptimal solutions for a continuous model') original_solution_number = solver_model.getParamInfo('SolutionNumber')[2] solver_model.setParam('SolutionNumber', solution_number) gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] @@ -112,7 +110,7 @@ def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): for v, val in _get_primals( solver_model=solver_model, var_map=var_map, - vars_to_load=vars_to_load, + vars_to_load=vars_to_load, solution_number=solution_number, ).items(): v.set_value(val, skip_validation=True) @@ -177,7 +175,7 @@ def _get_duals(solver_model, con_map, linear_cons_to_load, quadratic_cons_to_loa """ if solver_model.Status != gurobipy.GRB.OPTIMAL: raise NoDualsError() - + linear_gurobi_cons = [con_map[c] for c in linear_cons_to_load] quadratic_gurobi_cons = [con_map[c] for c in quadratic_cons_to_load] linear_vals = solver_model.getAttr("Pi", linear_gurobi_cons) @@ -293,7 +291,7 @@ def _create_solver_model(self, pyomo_model): def _pyomo_gurobi_var_iter(self): # generator of tuples (pyomo_var, gurobi_var) raise NotImplementedError('should be implemented by derived classes') - + def _mipstart(self): for pyomo_var, gurobi_var in self._pyomo_gurobi_var_iter(): if pyomo_var.is_integer() and pyomo_var.value is not None: @@ -304,11 +302,8 @@ def solve(self, model, **kwds) -> Results: orig_config = self.config orig_cwd = os.getcwd() try: - config = self.config( - value=kwds, - preserve_implicit=True, - ) - + config = self.config(value=kwds, preserve_implicit=True) + # hack to work around legacy solver wrapper __setattr__ # otherwise, this would just be self.config = config object.__setattr__(self, 'config', config) @@ -329,7 +324,9 @@ def solve(self, model, **kwds) -> Results: if config.working_dir: os.chdir(config.working_dir) with capture_output(TeeStream(*ostreams), capture_fd=False): - gurobi_model, solution_loader, has_obj = self._create_solver_model(model) + gurobi_model, solution_loader, has_obj = self._create_solver_model( + model + ) options = config.solver_options gurobi_model.setParam('LogToConsole', 1) @@ -354,9 +351,7 @@ def solve(self, model, **kwds) -> Results: timer.stop('optimize') res = self._postsolve( - grb_model=gurobi_model, - solution_loader=solution_loader, - has_obj=has_obj, + grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj ) finally: os.chdir(orig_cwd) @@ -450,9 +445,9 @@ def _postsolve(self, grb_model, solution_loader, has_obj): raise NoFeasibleSolutionError() self.config.timer.stop('load solution') - # self.config gets copied a the beginning of + # self.config gets copied a the beginning of # solve and restored at the end, so modifying - # results.solver_config will not actually + # results.solver_config will not actually # modify self.config results.solver_config = self.config results.solver_name = self.name diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 7b0463d2cf1..6628f001421 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -29,8 +29,8 @@ from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( - GurobiDirectBase, - gurobipy, + GurobiDirectBase, + gurobipy, _load_vars, _get_primals, _get_duals, @@ -45,13 +45,7 @@ class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): def __init__( - self, - solver_model, - var_id_map, - var_map, - con_map, - linear_cons, - quadratic_cons, + self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons ) -> None: super().__init__() self._solver_model = solver_model @@ -62,9 +56,7 @@ def __init__( self._quadratic_cons = quadratic_cons def load_vars( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=0, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> None: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -76,9 +68,7 @@ def load_vars( ) def get_primals( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=0, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -88,10 +78,9 @@ def get_primals( vars_to_load=vars_to_load, solution_number=solution_id, ) - + def get_reduced_costs( - self, - vars_to_load: Optional[Sequence[VarData]] = None, + self, vars_to_load: Optional[Sequence[VarData]] = None ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -100,10 +89,9 @@ def get_reduced_costs( var_map=self._var_map, vars_to_load=vars_to_load, ) - + def get_duals( - self, - cons_to_load: Optional[Sequence[ConstraintData]] = None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None ) -> Dict[ConstraintData, float]: if cons_to_load is None: cons_to_load = list(self._con_map.keys()) @@ -124,8 +112,12 @@ def get_duals( class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): - def __init__(self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) -> None: - super().__init__(solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) + def __init__( + self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + ) -> None: + super().__init__( + solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + ) self._valid = True def invalidate(self): @@ -135,19 +127,27 @@ def _assert_solution_still_valid(self): if not self._valid: raise RuntimeError('The results in the solver are no longer valid.') - def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> None: + def load_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + ) -> None: self._assert_solution_still_valid() return super().load_vars(vars_to_load, solution_id) - - def get_primals(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> Mapping[VarData, float]: + + def get_primals( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_primals(vars_to_load, solution_id) - def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None) -> Dict[ConstraintData, float]: + def get_duals( + self, cons_to_load: Sequence[ConstraintData] | None = None + ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return super().get_duals(cons_to_load) - - def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None) -> Mapping[VarData, float]: + + def get_reduced_costs( + self, vars_to_load: Sequence[VarData] | None = None + ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_reduced_costs(vars_to_load) @@ -194,7 +194,9 @@ def update(self): class _MutableRangeConstant: - def __init__(self, lhs_expr, rhs_expr, pyomo_con, con_map, slack_name, gurobi_model): + def __init__( + self, lhs_expr, rhs_expr, pyomo_con, con_map, slack_name, gurobi_model + ): self.lhs_expr = lhs_expr self.rhs_expr = rhs_expr self.pyomo_con = pyomo_con @@ -268,7 +270,9 @@ def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): self.constant: _MutableConstant = constant self.linear_coefs: List[_MutableLinearCoefficient] = linear_coefs self.quadratic_coefs: List[_MutableQuadraticCoefficient] = quadratic_coefs - self.last_quadratic_coef_values: List[float] = [value(i.expr) for i in self.quadratic_coefs] + self.last_quadratic_coef_values: List[float] = [ + value(i.expr) for i in self.quadratic_coefs + ] def get_updated_expression(self): for ndx, coef in enumerate(self.linear_coefs): @@ -300,7 +304,7 @@ def __init__(self, expr, v1id, v2id, var_map): @property def var1(self): return self.var_map[self.v1id] - + @property def var2(self): return self.var_map[self.v2id] @@ -325,13 +329,21 @@ def _create_solver_model(self, pyomo_model): self._clear() self._solver_model = gurobipy.Model(env=self.env()) timer.start('collect constraints') - cons = list(pyomo_model.component_data_objects(Constraint, descend_into=True, active=True)) + cons = list( + pyomo_model.component_data_objects( + Constraint, descend_into=True, active=True + ) + ) timer.stop('collect constraints') timer.start('translate constraints') self._add_constraints(cons) timer.stop('translate constraints') timer.start('sos') - sos = list(pyomo_model.component_data_objects(SOSConstraint, descend_into=True, active=True)) + sos = list( + pyomo_model.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + ) self._add_sos_constraints(sos) timer.stop('sos') timer.start('get objective') @@ -351,7 +363,7 @@ def _create_solver_model(self, pyomo_model): ) timer.stop('create gurobipy model') return self._solver_model, solution_loader, has_obj - + def _clear(self): self._solver_model = None self._vars = {} @@ -379,9 +391,7 @@ def _process_domain_and_bounds(self, var): else: vtype = gurobipy.GRB.INTEGER else: - raise ValueError( - f'Unrecognized domain: {var.domain}' - ) + raise ValueError(f'Unrecognized domain: {var.domain}') if var.fixed: lb = var.value ub = lb @@ -415,16 +425,13 @@ def _get_expr_from_pyomo_repn(self, repn): raise IncompatibleModelError( f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' ) - + if len(repn.linear_vars) > 0: missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] self._add_variables(missing_vars) coef_list = [value(i) for i in repn.linear_coefs] vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] - new_expr = gurobipy.LinExpr( - coef_list, - vlist, - ) + new_expr = gurobipy.LinExpr(coef_list, vlist) else: new_expr = 0.0 @@ -455,8 +462,7 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobi_expr = self._get_expr_from_pyomo_repn(repn) if lb is None and ub is None: raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" + "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: gurobi_expr_list.append(gurobi_expr <= float(ub - repn.constant)) @@ -465,9 +471,14 @@ def _add_constraints(self, cons: List[ConstraintData]): elif lb == ub: gurobi_expr_list.append(gurobi_expr == float(lb - repn.constant)) else: - gurobi_expr_list.append(gurobi_expr == [float(lb-repn.constant), float(ub-repn.constant)]) + gurobi_expr_list.append( + gurobi_expr + == [float(lb - repn.constant), float(ub - repn.constant)] + ) - gurobi_cons = self._solver_model.addConstrs((gurobi_expr_list[i] for i in range(len(gurobi_expr_list)))).values() + gurobi_cons = self._solver_model.addConstrs( + (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))) + ).values() self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) def _add_sos_constraints(self, cons: List[SOSConstraintData]): @@ -485,7 +496,9 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): gurobi_vars = [] weights = [] - missing_vars = {id(v): v for v, w in con.get_items() if id(v) not in self._vars} + missing_vars = { + id(v): v for v, w in con.get_items() if id(v) not in self._vars + } self._add_variables(list(missing_vars.values())) for v, w in con.get_items(): @@ -608,7 +621,7 @@ def _create_solver_model(self, pyomo_model): ) has_obj = self._objective is not None return self._solver_model, solution_loader, has_obj - + def release_license(self): self._clear() self.__class__.release_license() @@ -617,17 +630,21 @@ def solve(self, model, **kwds) -> Results: res = super().solve(model, **kwds) self._needs_updated = False return res - + def _process_domain_and_bounds(self, var): res = super()._process_domain_and_bounds(var) if not is_constant(var._lb): - mutable_lb = _MutableLowerBound(id(var), var.lower, self._pyomo_var_to_solver_var_map) + mutable_lb = _MutableLowerBound( + id(var), var.lower, self._pyomo_var_to_solver_var_map + ) self._mutable_bounds[id(var), 'lb'] = (var, mutable_lb) if not is_constant(var._ub): - mutable_ub = _MutableUpperBound(id(var), var.upper, self._pyomo_var_to_solver_var_map) + mutable_ub = _MutableUpperBound( + id(var), var.upper, self._pyomo_var_to_solver_var_map + ) self._mutable_bounds[id(var), 'ub'] = (var, mutable_ub) return res - + def _add_variables(self, variables: List[VarData]): self._invalidate_last_results() super()._add_variables(variables) @@ -673,37 +690,60 @@ def _add_constraints(self, cons: List[ConstraintData]): mutable_constant = None if lb is None and ub is None: raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" + "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: rhs_expr = ub - repn.constant gurobi_expr_list.append(gurobi_expr <= float(value(rhs_expr))) if not is_constant(rhs_expr): - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) elif ub is None: rhs_expr = lb - repn.constant gurobi_expr_list.append(float(value(rhs_expr)) <= gurobi_expr) if not is_constant(rhs_expr): - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) elif con.equality: rhs_expr = lb - repn.constant gurobi_expr_list.append(gurobi_expr == float(value(rhs_expr))) if not is_constant(rhs_expr): - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) else: - assert len(repn.quadratic_vars) == 0, "Quadratic range constraints are not supported" + assert ( + len(repn.quadratic_vars) == 0 + ), "Quadratic range constraints are not supported" lhs_expr = lb - repn.constant rhs_expr = ub - repn.constant - gurobi_expr_list.append(gurobi_expr == [float(value(lhs_expr)), float(value(rhs_expr))]) + gurobi_expr_list.append( + gurobi_expr == [float(value(lhs_expr)), float(value(rhs_expr))] + ) if not is_constant(lhs_expr) or not is_constant(rhs_expr): conname = f'c{self._constraint_ndx}[{ndx}]' - mutable_constant = _MutableRangeConstant(lhs_expr, rhs_expr, con, self._pyomo_con_to_solver_con_map, 'Rg' + conname, self._solver_model) + mutable_constant = _MutableRangeConstant( + lhs_expr, + rhs_expr, + con, + self._pyomo_con_to_solver_con_map, + 'Rg' + conname, + self._solver_model, + ) mlc_list = [] for c, v in zip(repn.linear_coefs, repn.linear_vars): if not is_constant(c): - mlc = _MutableLinearCoefficient(c, con, self._pyomo_con_to_solver_con_map, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc = _MutableLinearCoefficient( + c, + con, + self._pyomo_con_to_solver_con_map, + id(v), + self._pyomo_var_to_solver_var_map, + self._solver_model, + ) mlc_list.append(mlc) if len(repn.quadratic_vars) == 0: @@ -715,15 +755,19 @@ def _add_constraints(self, cons: List[ConstraintData]): self._mutable_helpers[con].append(mutable_constant) else: if mutable_constant is None: - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) mqc_list = [] for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): if not is_constant(coef): - mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc = _MutableQuadraticCoefficient( + coef, id(x), id(y), self._pyomo_var_to_solver_var_map + ) mqc_list.append(mqc) mqc = _MutableQuadraticConstraint( self._solver_model, - con, + con, self._pyomo_con_to_solver_con_map, mutable_constant, mlc_list, @@ -731,10 +775,12 @@ def _add_constraints(self, cons: List[ConstraintData]): ) self._mutable_quadratic_helpers[con] = mqc - gurobi_cons = list(self._solver_model.addConstrs( - (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))), - name=f'c{self._constraint_ndx}' - ).values()) + gurobi_cons = list( + self._solver_model.addConstrs( + (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))), + name=f'c{self._constraint_ndx}', + ).values() + ) self._constraint_ndx += 1 self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) self._constraints_added_since_update.update(cons) @@ -761,7 +807,9 @@ def _set_objective(self, obj): else: raise ValueError(f'Objective sense is not recognized: {obj.sense}') - repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=False) + repn = generate_standard_repn( + obj.expr, quadratic=True, compute_values=False + ) repn_constant = value(repn.constant) gurobi_expr = self._get_expr_from_pyomo_repn(repn) @@ -770,16 +818,27 @@ def _set_objective(self, obj): mlc_list = [] for c, v in zip(repn.linear_coefs, repn.linear_vars): if not is_constant(c): - mlc = _MutableLinearCoefficient(c, None, None, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc = _MutableLinearCoefficient( + c, + None, + None, + id(v), + self._pyomo_var_to_solver_var_map, + self._solver_model, + ) mlc_list.append(mlc) mqc_list = [] for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): if not is_constant(coef): - mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc = _MutableQuadraticCoefficient( + coef, id(x), id(y), self._pyomo_var_to_solver_var_map + ) mqc_list.append(mqc) - self._mutable_objective = _MutableObjective(self._solver_model, mutable_constant, mlc_list, mqc_list) + self._mutable_objective = _MutableObjective( + self._solver_model, mutable_constant, mlc_list, mqc_list + ) # hack # see PR #2454 @@ -865,9 +924,7 @@ def _update_parameters(self, params: List[ParamData]): new_rhs = helper.get_updated_rhs() new_sense = gurobi_con.qcsense self._solver_model.remove(gurobi_con) - new_con = self._solver_model.addQConstr( - new_gurobi_expr, new_sense, new_rhs, - ) + new_con = self._solver_model.addQConstr(new_gurobi_expr, new_sense, new_rhs) self._pyomo_con_to_solver_con_map[con] = new_con helper.pyomo_con = con self._constraints_added_since_update.add(con) @@ -880,7 +937,7 @@ def _update_parameters(self, params: List[ParamData]): else: sense = gurobipy.GRB.MAXIMIZE # TODO: need a test for when part of the object is linear - # and part of the objective is quadratic, but both + # and part of the objective is quadratic, but both # parts have mutable coefficients self._solver_model.setObjective(new_gurobi_expr, sense=sense) @@ -1309,4 +1366,4 @@ def update_variables(self, variables): self._change_detector.update_variables(variables) def update_parameters(self, params): - self._change_detector.update_parameters(params) \ No newline at end of file + self._change_detector.update_parameters(params) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index a0d87835e13..96e2e7b2c38 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -31,7 +31,10 @@ from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.solvers.ipopt import Ipopt from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect -from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import ( + GurobiDirectQuadratic, + GurobiPersistent, +) from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -59,11 +62,9 @@ ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), ] -nlp_solvers = [ - ('ipopt', Ipopt), -] +nlp_solvers = [('ipopt', Ipopt)] qcp_solvers = [ - ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ] From 7249b1941c6b3aad5cd3f7a3bd34f7ab28ef3566 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:57:56 -0600 Subject: [PATCH 19/33] update solution loader --- .../solvers/gurobi/gurobi_persistent.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 6628f001421..05acfef2b4f 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -15,6 +15,7 @@ from collections.abc import Iterable from pyomo.common.collections import ComponentSet, OrderedSet +from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize @@ -54,6 +55,25 @@ def __init__( self._con_map = con_map self._linear_cons = linear_cons self._quadratic_cons = quadratic_cons + GurobiDirectBase._register_env_client() + + def __del__(self): + if python_is_shutting_down(): + return + # Free the associated model + if self._solver_model is not None: + self._vars = None + self._var_map = None + self._con_map = None + self._linear_cons = None + self._quadratic_cons = None + # explicitly release the model + self._solver_model.dispose() + self._solver_model = None + # Release the gurobi license if this is the last reference to + # the environment (either through a results object or solver + # interface) + GurobiDirectBase._release_env_client() def load_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 From ac42345de6f87fe5010af92bbdf2cf7d772d95d4 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 17:41:27 -0600 Subject: [PATCH 20/33] updating solution loader --- pyomo/contrib/solver/common/base.py | 7 +-- .../contrib/solver/common/solution_loader.py | 47 +++++++++++++++++-- .../solver/solvers/gurobi/gurobi_direct.py | 32 +++++++++---- pyomo/contrib/solver/solvers/highs.py | 4 +- pyomo/contrib/solver/solvers/ipopt.py | 36 ++++++-------- pyomo/contrib/solver/solvers/sol_reader.py | 30 +++++++++--- 6 files changed, 107 insertions(+), 49 deletions(-) diff --git a/pyomo/contrib/solver/common/base.py b/pyomo/contrib/solver/common/base.py index 280b80629a3..f935f3d4988 100644 --- a/pyomo/contrib/solver/common/base.py +++ b/pyomo/contrib/solver/common/base.py @@ -567,12 +567,7 @@ def _solution_handler( legacy_results._smap_id = id(symbol_map) delete_legacy_soln = True if load_solutions: - if hasattr(model, 'dual') and model.dual.import_enabled(): - for con, val in results.solution_loader.get_duals().items(): - model.dual[con] = val - if hasattr(model, 'rc') and model.rc.import_enabled(): - for var, val in results.solution_loader.get_reduced_costs().items(): - model.rc[var] = val + results.solution_loader.load_import_suffixes() elif results.incumbent_objective is not None: delete_legacy_soln = False for var, val in results.solution_loader.get_primals().items(): diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index 065c00185f6..e399d6bea55 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -9,11 +9,32 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +from __future__ import annotations + from typing import Sequence, Dict, Optional, Mapping, List, Any from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.var import VarData from pyomo.core.staleflag import StaleFlagManager +from pyomo.core.base.suffix import Suffix + + +def load_import_suffixes(pyomo_model, solution_loader: SolutionLoaderBase, solution_id=None): + dual_suffix = None + rc_suffix = None + for suffix in pyomo_model.component_objects(Suffix, descend_into=True, active=True): + if not suffix.import_enabled(): + continue + if suffix.local_name == 'dual': + dual_suffix = suffix + elif suffix.local_name == 'rc': + rc_suffix = suffix + if dual_suffix is not None: + for k, v in solution_loader.get_duals(solution_id=solution_id).items(): + dual_suffix[k] = v + if rc_suffix is not None: + for k, v in solution_loader.get_reduced_costs(solution_id=solution_id).items(): + rc_suffix[k] = v class SolutionLoaderBase: @@ -178,29 +199,45 @@ class PersistentSolutionLoader(SolutionLoaderBase): Loader for persistent solvers """ - def __init__(self, solver): + def __init__(self, solver, pyomo_model): self._solver = solver self._valid = True + self._pyomo_model = pyomo_model def _assert_solution_still_valid(self): if not self._valid: raise RuntimeError('The results in the solver are no longer valid.') - def get_primals(self, vars_to_load=None): + def get_solution_ids(self) -> List[Any]: + self._assert_solution_still_valid() + return super().get_solution_ids() + + def get_number_of_solutions(self) -> int: + self._assert_solution_still_valid() + return super().get_number_of_solutions() + + def get_vars(self, vars_to_load=None, solution_id=None): self._assert_solution_still_valid() - return self._solver._get_primals(vars_to_load=vars_to_load) + return self._solver._get_primals(vars_to_load=vars_to_load, solution_id=solution_id) def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, + cons_to_load: Optional[Sequence[ConstraintData]] = None, + solution_id=None, ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return self._solver._get_duals(cons_to_load=cons_to_load) def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return self._solver._get_reduced_costs(vars_to_load=vars_to_load) + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) + def invalidate(self): self._valid = False diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 16c633c7d7c..cca23315b1a 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -10,6 +10,7 @@ # ___________________________________________________________________________ import operator +from typing import List from pyomo.common.collections import ComponentMap, ComponentSet from pyomo.common.shutdown import python_is_shutting_down @@ -22,17 +23,18 @@ NoSolutionError, IncompatibleModelError, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes from .gurobi_direct_base import GurobiDirectBase, gurobipy class GurobiDirectSolutionLoader(SolutionLoaderBase): - def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars): + def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars, pyomo_model): self._grb_model = grb_model self._grb_cons = grb_cons self._grb_vars = grb_vars self._pyo_cons = pyo_cons self._pyo_vars = pyo_vars + self._pyomo_model = pyomo_model GurobiDirectBase._register_env_client() def __del__(self): @@ -44,6 +46,7 @@ def __del__(self): self._grb_vars = None self._pyo_cons = None self._pyo_vars = None + self._pyomo_model = None # explicitly release the model self._grb_model.dispose() self._grb_model = None @@ -52,8 +55,16 @@ def __del__(self): # interface) GurobiDirectBase._release_env_client() - def load_vars(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 + def get_number_of_solutions(self) -> int: + if self._grb_model.SolCount == 0: + return 0 + return 1 + + def get_solution_ids(self) -> List[Any]: + return [0] + + def load_vars(self, vars_to_load=None, solution_id=0): + assert solution_id == 0 if self._grb_model.SolCount == 0: raise NoSolutionError() @@ -65,8 +76,8 @@ def load_vars(self, vars_to_load=None, solution_number=0): p_var.set_value(g_var, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) - def get_primals(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 + def get_vars(self, vars_to_load=None, solution_id=0): + assert solution_id == 0 if self._grb_model.SolCount == 0: raise NoSolutionError() @@ -76,7 +87,8 @@ def get_primals(self, vars_to_load=None, solution_number=0): iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) return ComponentMap(iterator) - def get_duals(self, cons_to_load=None): + def get_duals(self, cons_to_load=None, solution_id=0): + assert solution_id == 0 if self._grb_model.Status != gurobipy.GRB.OPTIMAL: raise NoDualsError() @@ -96,7 +108,8 @@ def dedup(_iter): ) return {con_info[0]: dual for con_info, dual in iterator} - def get_reduced_costs(self, vars_to_load=None): + def get_reduced_costs(self, vars_to_load=None, solution_id=0): + assert solution_id == 0 if self._grb_model.Status != gurobipy.GRB.OPTIMAL: raise NoReducedCostsError() @@ -105,6 +118,9 @@ def get_reduced_costs(self, vars_to_load=None): vars_to_load = ComponentSet(vars_to_load) iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) return ComponentMap(iterator) + + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(pyomo_model=self._pyomo_model, solution_loader=self, solution_id=solution_id) class GurobiDirect(GurobiDirectBase): diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index 6eb4afa828a..a1d609c5db6 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -675,7 +675,7 @@ def _postsolve(self): status = highs.getModelStatus() results = Results() - results.solution_loader = PersistentSolutionLoader(self) + results.solution_loader = PersistentSolutionLoader(self, self._model) results.timing_info.highs_time = highs.getRunTime() self._sol = highs.getSolution() @@ -751,7 +751,7 @@ def _postsolve(self): if config.load_solutions: if has_feasible_solution: - self._load_vars() + results.solution_loader.load_solution() else: raise NoFeasibleSolutionError() timer.stop('load solution') diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 075fc998ecc..441b8eb5f61 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -109,9 +109,13 @@ def _error_check(self): ) def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: self._error_check() + if solution_id is not None: + raise ValueError('IpoptSolutionLoader does not support solution_id') if self._nl_info.scaling is None: scale_list = [1] * len(self._nl_info.variables) obj_scale = 1 @@ -430,35 +434,35 @@ def solve(self, model, **kwds) -> Results: if proven_infeasible: results = Results() results.termination_condition = TerminationCondition.provenInfeasible - results.solution_loader = SolSolutionLoader(None, None) + results.solution_loader = SolSolutionLoader(None, None, model) results.iteration_count = 0 results.timing_info.total_seconds = 0 elif len(nl_info.variables) == 0: if len(nl_info.eliminated_vars) == 0: results = Results() results.termination_condition = TerminationCondition.emptyModel - results.solution_loader = SolSolutionLoader(None, None) + results.solution_loader = SolSolutionLoader(None, None, model) else: results = Results() results.termination_condition = ( TerminationCondition.convergenceCriteriaSatisfied ) results.solution_status = SolutionStatus.optimal - results.solution_loader = SolSolutionLoader(None, nl_info=nl_info) + results.solution_loader = SolSolutionLoader(None, nl_info=nl_info, pyomo_model=model) results.iteration_count = 0 results.timing_info.total_seconds = 0 else: if os.path.isfile(basename + '.sol'): with open(basename + '.sol', 'r', encoding='utf-8') as sol_file: timer.start('parse_sol') - results = self._parse_solution(sol_file, nl_info) + results = self._parse_solution(sol_file, nl_info, model) timer.stop('parse_sol') else: results = Results() if process.returncode != 0: results.extra_info.return_code = process.returncode results.termination_condition = TerminationCondition.error - results.solution_loader = SolSolutionLoader(None, None) + results.solution_loader = SolSolutionLoader(None, None, model) else: try: results.iteration_count = parsed_output_data.pop('iters') @@ -490,19 +494,7 @@ def solve(self, model, **kwds) -> Results: if config.load_solutions: if results.solution_status == SolutionStatus.noSolution: raise NoFeasibleSolutionError() - results.solution_loader.load_vars() - if ( - hasattr(model, 'dual') - and isinstance(model.dual, Suffix) - and model.dual.import_enabled() - ): - model.dual.update(results.solution_loader.get_duals()) - if ( - hasattr(model, 'rc') - and isinstance(model.rc, Suffix) - and model.rc.import_enabled() - ): - model.rc.update(results.solution_loader.get_reduced_costs()) + results.solution_loader.load_solution() if ( results.solution_status in {SolutionStatus.feasible, SolutionStatus.optimal} @@ -665,7 +657,7 @@ def _parse_ipopt_output(self, output: Union[str, io.StringIO]) -> Dict[str, Any] return parsed_data def _parse_solution( - self, instream: io.TextIOBase, nl_info: NLWriterInfo + self, instream: io.TextIOBase, nl_info: NLWriterInfo, pyomo_model ) -> Results: results = Results() res, sol_data = parse_sol_file( @@ -673,10 +665,10 @@ def _parse_solution( ) if res.solution_status == SolutionStatus.noSolution: - res.solution_loader = SolSolutionLoader(None, None) + res.solution_loader = SolSolutionLoader(None, None, pyomo_model=pyomo_model) else: res.solution_loader = IpoptSolutionLoader( - sol_data=sol_data, nl_info=nl_info + sol_data=sol_data, nl_info=nl_info, pyomo_model=pyomo_model, ) return res diff --git a/pyomo/contrib/solver/solvers/sol_reader.py b/pyomo/contrib/solver/solvers/sol_reader.py index e580e2a72f9..7570a1ffc53 100644 --- a/pyomo/contrib/solver/solvers/sol_reader.py +++ b/pyomo/contrib/solver/solvers/sol_reader.py @@ -26,7 +26,7 @@ SolutionStatus, TerminationCondition, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes class SolFileData: @@ -49,11 +49,25 @@ class SolSolutionLoader(SolutionLoaderBase): Loader for solvers that create .sol files (e.g., ipopt) """ - def __init__(self, sol_data: SolFileData, nl_info: NLWriterInfo) -> None: + def __init__(self, sol_data: SolFileData, nl_info: NLWriterInfo, pyomo_model) -> None: self._sol_data = sol_data self._nl_info = nl_info + self._pyomo_model = pyomo_model - def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> NoReturn: + def get_number_of_solutions(self) -> int: + if self._nl_info is None: + return 0 + return 1 + + def get_solution_ids(self) -> List[Any]: + return [None] + + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) + + def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None) -> NoReturn: + if solution_id is not None: + raise ValueError(f'{self.__class__.__name__} does not support solution_id') if self._nl_info is None: raise RuntimeError( 'Solution loader does not currently have a valid solution. Please ' @@ -78,9 +92,11 @@ def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> NoRetur StaleFlagManager.mark_all_as_stale(delayed=True) - def get_primals( - self, vars_to_load: Optional[Sequence[VarData]] = None + def get_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: + if solution_id is not None: + raise ValueError(f'{self.__class__.__name__} does not support solution_id') if self._nl_info is None: raise RuntimeError( 'Solution loader does not currently have a valid solution. Please ' @@ -115,8 +131,10 @@ def get_primals( return res def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None, ) -> Dict[ConstraintData, float]: + if solution_id is not None: + raise ValueError(f'{self.__class__.__name__} does not support solution_id') if self._nl_info is None: raise RuntimeError( 'Solution loader does not currently have a valid solution. Please ' From 275d848d2c5eb09db7ac6b519794e1a0065b5869 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 17:43:11 -0600 Subject: [PATCH 21/33] run black --- pyomo/contrib/observer/component_collector.py | 5 +++- pyomo/contrib/observer/model_observer.py | 24 ++++++++-------- .../observer/tests/test_change_detector.py | 28 ++++++++++--------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index 5cbbdaf31bd..d52ec46086c 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -10,7 +10,10 @@ # ___________________________________________________________________________ from pyomo.core.expr.visitor import StreamBasedExpressionVisitor -from pyomo.core.expr.numeric_expr import ExternalFunctionExpression, NPV_ExternalFunctionExpression +from pyomo.core.expr.numeric_expr import ( + ExternalFunctionExpression, + NPV_ExternalFunctionExpression, +) from pyomo.core.base.var import VarData, ScalarVar from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.expression import ExpressionData, ScalarExpression diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index bd905e1c61d..4ab52100376 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -208,10 +208,7 @@ def update_parameters(self, params: List[ParamData]): class ModelChangeDetector: - def __init__( - self, observers: Sequence[Observer], - **kwds, - ): + def __init__(self, observers: Sequence[Observer], **kwds): """ Parameters ---------- @@ -237,13 +234,15 @@ def __init__( ) # var_id: [dict[constraints, None], dict[sos constraints, None], None or objective] self._referenced_params = ( {} - ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] self._vars_referenced_by_con = {} self._vars_referenced_by_obj = [] self._params_referenced_by_con = {} self._params_referenced_by_obj = [] self._expr_types = None - self.config: AutoUpdateConfig = AutoUpdateConfig()(value=kwds, preserve_implicit=True) + self.config: AutoUpdateConfig = AutoUpdateConfig()( + value=kwds, preserve_implicit=True + ) def set_instance(self, model): saved_config = self.config @@ -347,7 +346,10 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') sos_items = list(con.get_items()) - self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) + self._active_sos[con] = ( + [i[0] for i in sos_items], + [i[1] for i in sos_items], + ) variables = [] params = [] for v, p in sos_items: @@ -616,14 +618,14 @@ def _check_for_var_changes(self): vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) return vars_to_update, cons_to_update, update_obj - + def _check_for_param_changes(self): params_to_update = [] for pid, (p, val) in self._params.items(): if p.value != val: params_to_update.append(p) return params_to_update - + def _check_for_named_expression_changes(self): cons_to_update = [] for con, ne_list in self._named_expressions.items(): @@ -644,7 +646,7 @@ def _check_for_new_objective(self): new_obj = get_objective(self._model) if new_obj is not self._objective: update_obj = True - return new_obj, update_obj + return new_obj, update_obj def _check_for_objective_changes(self): update_obj = False @@ -717,7 +719,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if update_obj: need_to_set_objective = True timer.stop('named expressions') - + timer.start('objective') new_obj = self._objective if config.check_for_new_objective: diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index efda8a181d9..29e0de01eb9 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -6,7 +6,11 @@ import pyomo.environ as pe from pyomo.common import unittest from typing import List -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, +) from pyomo.common.collections import ComponentMap import logging @@ -31,11 +35,9 @@ def __init__(self): def check(self, expected): unittest.assertStructuredAlmostEqual( - first=expected, - second=self.counts, - places=7, + first=expected, second=self.counts, places=7 ) - + def _process(self, comps, key): for c in comps: if c not in self.counts: @@ -120,7 +122,7 @@ def test_objective(self): detector.set_instance(m) obs.check(expected) - m.obj = pe.Objective(expr=m.x**2 + m.p*m.y**2) + m.obj = pe.Objective(expr=m.x**2 + m.p * m.y**2) detector.update() expected[m.obj] = make_count_dict() expected[m.obj]['set'] += 1 @@ -131,7 +133,7 @@ def test_objective(self): expected[m.p] = make_count_dict() expected[m.p]['add'] += 1 obs.check(expected) - + m.y.setlb(0) detector.update() expected[m.y]['update'] += 1 @@ -161,7 +163,7 @@ def test_objective(self): obs.check(expected) del m.obj - m.obj = pe.Objective(expr=m.p*m.x) + m.obj = pe.Objective(expr=m.p * m.x) detector.update() expected[m.p]['add'] += 1 expected[m.y]['remove'] += 1 @@ -186,7 +188,7 @@ def test_constraints(self): obs.check(expected) m.obj = pe.Objective(expr=m.y) - m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p)**2) + m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p) ** 2) detector.update() expected[m.x] = make_count_dict() expected[m.y] = make_count_dict() @@ -208,9 +210,9 @@ def test_constraints(self): obs.pprint() expected[m.c1]['remove'] += 1 expected[m.c1]['add'] += 1 - # because x and p are only used in the - # one constraint, they get removed when - # the constraint is removed and then + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then # added again when the constraint is added expected[m.x]['update'] += 1 expected[m.x]['remove'] += 1 @@ -220,4 +222,4 @@ def test_constraints(self): obs.check(expected) def test_vars_and_params_elsewhere(self): - pass \ No newline at end of file + pass From 70ca6e72d50ac39a3c580f116dfd6c75a9b29e92 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 07:20:44 -0600 Subject: [PATCH 22/33] updating solution loader --- .../solver/solvers/gurobi/gurobi_direct.py | 2 +- .../solvers/gurobi/gurobi_direct_base.py | 2 +- .../solvers/gurobi/gurobi_persistent.py | 44 ++++++++++++++----- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index cca23315b1a..82b47ccb24b 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -193,7 +193,7 @@ def _create_solver_model(self, pyomo_model): self._gurobi_vars = x solution_loader = GurobiDirectSolutionLoader( - gurobi_model, A, x, repn.rows, repn.columns + gurobi_model, A, x, repn.rows, repn.columns, pyomo_model ) has_obj = len(repn.objectives) > 0 diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index df6bb8b5327..ae887a52fa5 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -440,7 +440,7 @@ def _postsolve(self, grb_model, solution_loader, has_obj): self.config.timer.start('load solution') if self.config.load_solutions: if grb_model.SolCount > 0: - results.solution_loader.load_vars() + results.solution_loader.load_solution() else: raise NoFeasibleSolutionError() self.config.timer.stop('load solution') diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 05acfef2b4f..8477d855a02 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -27,7 +27,7 @@ from pyomo.repn import generate_standard_repn from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import IncompatibleModelError -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( GurobiDirectBase, @@ -46,7 +46,7 @@ class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model ) -> None: super().__init__() self._solver_model = solver_model @@ -55,6 +55,7 @@ def __init__( self._con_map = con_map self._linear_cons = linear_cons self._quadratic_cons = quadratic_cons + self._pyomo_model = pyomo_model GurobiDirectBase._register_env_client() def __del__(self): @@ -75,6 +76,12 @@ def __del__(self): # interface) GurobiDirectBase._release_env_client() + def get_number_of_solutions(self) -> int: + return self._solver_model.SolCount + + def get_solution_ids(self) -> List: + return list(range(self.get_number_of_solutions())) + def load_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> None: @@ -87,7 +94,7 @@ def load_vars( solution_number=solution_id, ) - def get_primals( + def get_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> Mapping[VarData, float]: if vars_to_load is None: @@ -100,7 +107,7 @@ def get_primals( ) def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -111,7 +118,7 @@ def get_reduced_costs( ) def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=0 ) -> Dict[ConstraintData, float]: if cons_to_load is None: cons_to_load = list(self._con_map.keys()) @@ -130,13 +137,16 @@ def get_duals( quadratic_cons_to_load=quadratic_cons_to_load, ) + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) + class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model ) -> None: super().__init__( - solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model ) self._valid = True @@ -153,23 +163,35 @@ def load_vars( self._assert_solution_still_valid() return super().load_vars(vars_to_load, solution_id) - def get_primals( + def get_vars( self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_primals(vars_to_load, solution_id) def get_duals( - self, cons_to_load: Sequence[ConstraintData] | None = None + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=0, ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return super().get_duals(cons_to_load) def get_reduced_costs( - self, vars_to_load: Sequence[VarData] | None = None + self, vars_to_load: Sequence[VarData] | None = None, solution_id=0, ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_reduced_costs(vars_to_load) + + def get_number_of_solutions(self) -> int: + self._assert_solution_still_valid() + return super().get_number_of_solutions() + + def get_solution_ids(self) -> List: + self._assert_solution_still_valid() + return super().get_solution_ids() + + def load_import_suffixes(self, solution_id=None): + self._assert_solution_still_valid() + super().load_import_suffixes(solution_id) class _MutableLowerBound: @@ -380,6 +402,7 @@ def _create_solver_model(self, pyomo_model): con_map=self._pyomo_con_to_solver_con_map, linear_cons=self._linear_cons, quadratic_cons=self._quadratic_cons, + pyomo_model=pyomo_model, ) timer.stop('create gurobipy model') return self._solver_model, solution_loader, has_obj @@ -638,6 +661,7 @@ def _create_solver_model(self, pyomo_model): con_map=self._pyomo_con_to_solver_con_map, linear_cons=self._linear_cons, quadratic_cons=self._quadratic_cons, + pyomo_model=pyomo_model, ) has_obj = self._objective is not None return self._solver_model, solution_loader, has_obj From 1788ff371d52448a16b539c868aa40142c416b73 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 07:23:17 -0600 Subject: [PATCH 23/33] dont free gurobi models twice --- .../solver/solvers/gurobi/gurobi_persistent.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 05acfef2b4f..847ec958bdd 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -58,18 +58,6 @@ def __init__( GurobiDirectBase._register_env_client() def __del__(self): - if python_is_shutting_down(): - return - # Free the associated model - if self._solver_model is not None: - self._vars = None - self._var_map = None - self._con_map = None - self._linear_cons = None - self._quadratic_cons = None - # explicitly release the model - self._solver_model.dispose() - self._solver_model = None # Release the gurobi license if this is the last reference to # the environment (either through a results object or solver # interface) From 2885f42e665a6fc87c854d60984145dffd86547a Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 08:07:54 -0600 Subject: [PATCH 24/33] update solution loader --- pyomo/contrib/solver/common/base.py | 2 +- .../solver/solvers/gurobi/gurobi_direct.py | 18 +++++++------- .../solvers/gurobi/gurobi_direct_base.py | 8 +++---- .../solvers/gurobi/gurobi_persistent.py | 22 ++++++++--------- pyomo/contrib/solver/solvers/highs.py | 16 +++++++++---- pyomo/contrib/solver/solvers/ipopt.py | 2 +- .../solver/tests/solvers/test_ipopt.py | 2 +- .../solver/tests/solvers/test_solvers.py | 12 +++++----- .../contrib/solver/tests/unit/test_results.py | 11 +++++---- .../solver/tests/unit/test_solution.py | 24 ++++++++++--------- 10 files changed, 65 insertions(+), 52 deletions(-) diff --git a/pyomo/contrib/solver/common/base.py b/pyomo/contrib/solver/common/base.py index f935f3d4988..0782e577c43 100644 --- a/pyomo/contrib/solver/common/base.py +++ b/pyomo/contrib/solver/common/base.py @@ -570,7 +570,7 @@ def _solution_handler( results.solution_loader.load_import_suffixes() elif results.incumbent_objective is not None: delete_legacy_soln = False - for var, val in results.solution_loader.get_primals().items(): + for var, val in results.solution_loader.get_vars().items(): legacy_soln.variable[symbol_map.getSymbol(var)] = {'Value': val} if hasattr(model, 'dual') and model.dual.import_enabled(): for con, val in results.solution_loader.get_duals().items(): diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 82b47ccb24b..fd932f90c15 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -10,7 +10,7 @@ # ___________________________________________________________________________ import operator -from typing import List +from typing import List, Any from pyomo.common.collections import ComponentMap, ComponentSet from pyomo.common.shutdown import python_is_shutting_down @@ -63,8 +63,8 @@ def get_number_of_solutions(self) -> int: def get_solution_ids(self) -> List[Any]: return [0] - def load_vars(self, vars_to_load=None, solution_id=0): - assert solution_id == 0 + def load_vars(self, vars_to_load=None, solution_id=None): + assert solution_id == None if self._grb_model.SolCount == 0: raise NoSolutionError() @@ -76,8 +76,8 @@ def load_vars(self, vars_to_load=None, solution_id=0): p_var.set_value(g_var, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) - def get_vars(self, vars_to_load=None, solution_id=0): - assert solution_id == 0 + def get_vars(self, vars_to_load=None, solution_id=None): + assert solution_id == None if self._grb_model.SolCount == 0: raise NoSolutionError() @@ -87,8 +87,8 @@ def get_vars(self, vars_to_load=None, solution_id=0): iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) return ComponentMap(iterator) - def get_duals(self, cons_to_load=None, solution_id=0): - assert solution_id == 0 + def get_duals(self, cons_to_load=None, solution_id=None): + assert solution_id == None if self._grb_model.Status != gurobipy.GRB.OPTIMAL: raise NoDualsError() @@ -108,8 +108,8 @@ def dedup(_iter): ) return {con_info[0]: dual for con_info, dual in iterator} - def get_reduced_costs(self, vars_to_load=None, solution_id=0): - assert solution_id == 0 + def get_reduced_costs(self, vars_to_load=None, solution_id=None): + assert solution_id == None if self._grb_model.Status != gurobipy.GRB.OPTIMAL: raise NoReducedCostsError() diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index ae887a52fa5..e99d24025d5 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -99,7 +99,7 @@ def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_ return res -def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): +def _load_vars(solver_model, var_map, vars_to_load, solution_number=None): """ solver_model: gurobipy.Model var_map: Dict[int, gurobipy.Var] @@ -107,7 +107,7 @@ def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): vars_to_load: List[VarData] solution_number: int """ - for v, val in _get_primals( + for v, val in _get_vars( solver_model=solver_model, var_map=var_map, vars_to_load=vars_to_load, @@ -117,7 +117,7 @@ def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): StaleFlagManager.mark_all_as_stale(delayed=True) -def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): +def _get_vars(solver_model, var_map, vars_to_load, solution_number=None): """ solver_model: gurobipy.Model var_map: Dict[int, gurobipy.Var] @@ -128,7 +128,7 @@ def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): if solver_model.SolCount == 0: raise NoSolutionError() - if solution_number != 0: + if solution_number not in {0, None}: return _load_suboptimal_mip_solution( solver_model=solver_model, var_map=var_map, diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 752b8512128..9f19bae307f 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -33,7 +33,7 @@ GurobiDirectBase, gurobipy, _load_vars, - _get_primals, + _get_vars, _get_duals, _get_reduced_costs, ) @@ -71,7 +71,7 @@ def get_solution_ids(self) -> List: return list(range(self.get_number_of_solutions())) def load_vars( - self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> None: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -83,11 +83,11 @@ def load_vars( ) def get_vars( - self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) - return _get_primals( + return _get_vars( solver_model=self._solver_model, var_map=self._var_map, vars_to_load=vars_to_load, @@ -95,7 +95,7 @@ def get_vars( ) def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -106,7 +106,7 @@ def get_reduced_costs( ) def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=0 + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: if cons_to_load is None: cons_to_load = list(self._con_map.keys()) @@ -146,25 +146,25 @@ def _assert_solution_still_valid(self): raise RuntimeError('The results in the solver are no longer valid.') def load_vars( - self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None ) -> None: self._assert_solution_still_valid() return super().load_vars(vars_to_load, solution_id) def get_vars( - self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None ) -> Mapping[VarData, float]: self._assert_solution_still_valid() - return super().get_primals(vars_to_load, solution_id) + return super().get_vars(vars_to_load, solution_id) def get_duals( - self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=0, + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None, ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return super().get_duals(cons_to_load) def get_reduced_costs( - self, vars_to_load: Sequence[VarData] | None = None, solution_id=0, + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None, ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_reduced_costs(vars_to_load) diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index a1d609c5db6..0abf02813ab 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -758,12 +758,16 @@ def _postsolve(self): return results - def _load_vars(self, vars_to_load=None): + def _load_vars(self, vars_to_load=None, solution_id=None): + if solution_id is not None: + raise NotImplementedError('highs interface does not currently support multiple solutions') for v, val in self._get_primals(vars_to_load=vars_to_load).items(): v.set_value(val, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) - def _get_primals(self, vars_to_load=None): + def _get_primals(self, vars_to_load=None, solution_id=None): + if solution_id is not None: + raise NotImplementedError('highs interface does not currently support multiple solutions') if self._sol is None or not self._sol.value_valid: raise NoSolutionError() @@ -786,7 +790,9 @@ def _get_primals(self, vars_to_load=None): return res - def _get_reduced_costs(self, vars_to_load=None): + def _get_reduced_costs(self, vars_to_load=None, solution_id=None): + if solution_id is not None: + raise NotImplementedError('highs interface does not currently support multiple solutions') if self._sol is None or not self._sol.dual_valid: raise NoReducedCostsError() res = ComponentMap() @@ -804,7 +810,9 @@ def _get_reduced_costs(self, vars_to_load=None): return res - def _get_duals(self, cons_to_load=None): + def _get_duals(self, cons_to_load=None, solution_id=None): + if solution_id is not None: + raise NotImplementedError('highs interface does not currently support multiple solutions') if self._sol is None or not self._sol.dual_valid: raise NoDualsError() diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 441b8eb5f61..80f9775a657 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -508,7 +508,7 @@ def solve(self, model, **kwds) -> Results: nl_info.objectives[0].expr, substitution_map={ id(v): val - for v, val in results.solution_loader.get_primals().items() + for v, val in results.solution_loader.get_vars().items() }, descend_into_named_expressions=True, remove_named_expressions=True, diff --git a/pyomo/contrib/solver/tests/solvers/test_ipopt.py b/pyomo/contrib/solver/tests/solvers/test_ipopt.py index d788b66982a..f0049922d55 100644 --- a/pyomo/contrib/solver/tests/solvers/test_ipopt.py +++ b/pyomo/contrib/solver/tests/solvers/test_ipopt.py @@ -62,7 +62,7 @@ def test_custom_instantiation(self): class TestIpoptSolutionLoader(unittest.TestCase): def test_get_reduced_costs_error(self): - loader = ipopt.IpoptSolutionLoader(None, None) + loader = ipopt.IpoptSolutionLoader(None, None, None) with self.assertRaises(NoSolutionError): loader.get_reduced_costs() diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 96e2e7b2c38..9c67ed7b1e5 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -1646,12 +1646,12 @@ def test_solution_loader( m.y.value = None res.solution_loader.load_vars([m.y]) self.assertAlmostEqual(m.y.value, 1) - primals = res.solution_loader.get_primals() + primals = res.solution_loader.get_vars() self.assertIn(m.x, primals) self.assertIn(m.y, primals) self.assertAlmostEqual(primals[m.x], 1) self.assertAlmostEqual(primals[m.y], 1) - primals = res.solution_loader.get_primals([m.y]) + primals = res.solution_loader.get_vars([m.y]) self.assertNotIn(m.x, primals) self.assertIn(m.y, primals) self.assertAlmostEqual(primals[m.y], 1) @@ -2000,7 +2000,7 @@ def test_variables_elsewhere2( res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) self.assertAlmostEqual(res.incumbent_objective, 1) - sol = res.solution_loader.get_primals() + sol = res.solution_loader.get_vars() self.assertIn(m.x, sol) self.assertIn(m.y, sol) self.assertIn(m.z, sol) @@ -2010,7 +2010,7 @@ def test_variables_elsewhere2( res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) self.assertAlmostEqual(res.incumbent_objective, 0) - sol = res.solution_loader.get_primals() + sol = res.solution_loader.get_vars() self.assertIn(m.x, sol) self.assertIn(m.y, sol) self.assertNotIn(m.z, sol) @@ -2172,7 +2172,7 @@ def test_scaling(self, name: str, opt_class: Type[SolverBase], use_presolve: boo self.assertAlmostEqual(res.incumbent_objective, 1) self.assertAlmostEqual(m.x.value, 1) self.assertAlmostEqual(m.y.value, 1) - primals = res.solution_loader.get_primals() + primals = res.solution_loader.get_vars() self.assertAlmostEqual(primals[m.x], 1) self.assertAlmostEqual(primals[m.y], 1) if check_duals: @@ -2188,7 +2188,7 @@ def test_scaling(self, name: str, opt_class: Type[SolverBase], use_presolve: boo self.assertAlmostEqual(res.incumbent_objective, 2) self.assertAlmostEqual(m.x.value, 2) self.assertAlmostEqual(m.y.value, 2) - primals = res.solution_loader.get_primals() + primals = res.solution_loader.get_vars() self.assertAlmostEqual(primals[m.x], 2) self.assertAlmostEqual(primals[m.y], 2) if check_duals: diff --git a/pyomo/contrib/solver/tests/unit/test_results.py b/pyomo/contrib/solver/tests/unit/test_results.py index a818f4ff4ad..a4def8f9089 100644 --- a/pyomo/contrib/solver/tests/unit/test_results.py +++ b/pyomo/contrib/solver/tests/unit/test_results.py @@ -49,8 +49,9 @@ def __init__( self._duals = duals self._reduced_costs = reduced_costs - def get_primals( - self, vars_to_load: Optional[Sequence[VarData]] = None + def get_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: if self._primals is None: raise RuntimeError( @@ -66,7 +67,8 @@ def get_primals( return primals def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, + solution_id=None, ) -> Dict[ConstraintData, float]: if self._duals is None: raise RuntimeError( @@ -83,7 +85,8 @@ def get_duals( return duals def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: if self._reduced_costs is None: raise RuntimeError( diff --git a/pyomo/contrib/solver/tests/unit/test_solution.py b/pyomo/contrib/solver/tests/unit/test_solution.py index 0453f0e0cb2..a0fc4ac9b2f 100644 --- a/pyomo/contrib/solver/tests/unit/test_solution.py +++ b/pyomo/contrib/solver/tests/unit/test_solution.py @@ -18,7 +18,7 @@ class TestSolutionLoaderBase(unittest.TestCase): def test_member_list(self): - expected_list = ['load_vars', 'get_primals', 'get_duals', 'get_reduced_costs'] + expected_list = ['load_vars', 'get_vars', 'get_duals', 'get_reduced_costs', 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', 'load_solution'] method_list = [ method for method in dir(SolutionLoaderBase) @@ -29,18 +29,16 @@ def test_member_list(self): def test_solution_loader_base(self): self.instance = SolutionLoaderBase() with self.assertRaises(NotImplementedError): - self.instance.get_primals() - with self.assertRaises(NotImplementedError): - self.instance.get_duals() - with self.assertRaises(NotImplementedError): - self.instance.get_reduced_costs() + self.instance.get_vars() + self.assertEqual(self.instance.get_duals(), NotImplemented) + self.assertEqual(self.instance.get_reduced_costs(), NotImplemented) class TestSolSolutionLoader(unittest.TestCase): # I am currently unsure how to test this further because it relies heavily on # SolFileData and NLWriterInfo def test_member_list(self): - expected_list = ['load_vars', 'get_primals', 'get_duals', 'get_reduced_costs'] + expected_list = ['load_vars', 'get_vars', 'get_duals', 'get_reduced_costs', 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', 'load_solution'] method_list = [ method for method in dir(SolutionLoaderBase) @@ -53,10 +51,14 @@ class TestPersistentSolutionLoader(unittest.TestCase): def test_member_list(self): expected_list = [ 'load_vars', - 'get_primals', + 'get_vars', 'get_duals', 'get_reduced_costs', 'invalidate', + 'load_import_suffixes', + 'get_number_of_solutions', + 'get_solution_ids', + 'load_solution' ] method_list = [ method @@ -69,12 +71,12 @@ def test_default_initialization(self): # Realistically, a solver object should be passed into this. # However, it works with a string. It'll just error loudly if you # try to run get_primals, etc. - self.instance = PersistentSolutionLoader('ipopt') + self.instance = PersistentSolutionLoader('ipopt', None) self.assertTrue(self.instance._valid) self.assertEqual(self.instance._solver, 'ipopt') def test_invalid(self): - self.instance = PersistentSolutionLoader('ipopt') + self.instance = PersistentSolutionLoader('ipopt', None) self.instance.invalidate() with self.assertRaises(RuntimeError): - self.instance.get_primals() + self.instance.get_vars() From 23ba4d96173c363953f574498ca1827c62c3ddd7 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 09:35:47 -0600 Subject: [PATCH 25/33] typo --- pyomo/contrib/observer/tests/test_change_detector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index efda8a181d9..b88877a9f63 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -24,7 +24,7 @@ def __init__(self): super().__init__() self.counts = ComponentMap() """ - counts is be a mapping from component (e.g., variable) to another + counts is a mapping from component (e.g., variable) to another mapping from string ('add', 'remove', 'update', or 'set') to an int that indicates the number of times the corresponding method has been called """ @@ -220,4 +220,4 @@ def test_constraints(self): obs.check(expected) def test_vars_and_params_elsewhere(self): - pass \ No newline at end of file + pass From d7b991825afbbc3eb2010bb5d4823a97f1378bea Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 10:00:38 -0600 Subject: [PATCH 26/33] update observer tests --- pyomo/contrib/observer/model_observer.py | 7 +- .../observer/tests/test_change_detector.py | 158 ++++++++++++++++-- 2 files changed, 149 insertions(+), 16 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 39b832cc266..bd89d4f600d 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -366,12 +366,15 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): self._named_expressions[con] = [] self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) + for con in cons: + variables = self._vars_referenced_by_con[con] + params = self._params_referenced_by_con[con] for v in variables: self._referenced_variables[id(v)][1][con] = None for p in params: self._referenced_params[id(p)][1][con] = None - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) for obs in self._observers: obs.add_sos_constraints(cons) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 92b7bbff390..46527618ab8 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -14,7 +14,7 @@ from pyomo.core.base.param import ParamData from pyomo.core.base.sos import SOSConstraintData from pyomo.core.base.var import VarData -import pyomo.environ as pe +import pyomo.environ as pyo from pyomo.common import unittest from typing import List from pyomo.contrib.observer.model_observer import ( @@ -118,10 +118,10 @@ def update_parameters(self, params: List[ParamData]): class TestChangeDetector(unittest.TestCase): def test_objective(self): - m = pe.ConcreteModel() - m.x = pe.Var() - m.y = pe.Var() - m.p = pe.Param(mutable=True, initialize=1) + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) obs = ObserverChecker() detector = ModelChangeDetector([obs]) @@ -133,7 +133,7 @@ def test_objective(self): detector.set_instance(m) obs.check(expected) - m.obj = pe.Objective(expr=m.x**2 + m.p * m.y**2) + m.obj = pyo.Objective(expr=m.x**2 + m.p * m.y**2) detector.update() expected[m.obj] = make_count_dict() expected[m.obj]['set'] += 1 @@ -174,7 +174,7 @@ def test_objective(self): obs.check(expected) del m.obj - m.obj = pe.Objective(expr=m.p * m.x) + m.obj = pyo.Objective(expr=m.p * m.x) detector.update() expected[m.p]['add'] += 1 expected[m.y]['remove'] += 1 @@ -183,10 +183,10 @@ def test_objective(self): expected[m.obj]['set'] += 1 def test_constraints(self): - m = pe.ConcreteModel() - m.x = pe.Var() - m.y = pe.Var() - m.p = pe.Param(mutable=True, initialize=1) + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) obs = ObserverChecker() detector = ModelChangeDetector([obs]) @@ -198,8 +198,8 @@ def test_constraints(self): detector.set_instance(m) obs.check(expected) - m.obj = pe.Objective(expr=m.y) - m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p) ** 2) + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= (m.x - m.p) ** 2) detector.update() expected[m.x] = make_count_dict() expected[m.y] = make_count_dict() @@ -232,5 +232,135 @@ def test_constraints(self): expected[m.p]['add'] += 1 obs.check(expected) + def test_sos(self): + m = pyo.ConcreteModel() + m.a = pyo.Set(initialize=[1, 2, 3], ordered=True) + m.x = pyo.Var(m.a, within=pyo.Binary) + m.y = pyo.Var(within=pyo.Binary) + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.SOSConstraint(var=m.x, sos=1) + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) + detector.set_instance(m) + + expected = ComponentMap() + expected[m.obj] = make_count_dict() + for i in m.a: + expected[m.x[i]] = make_count_dict() + expected[m.y] = make_count_dict() + expected[m.c1] = make_count_dict() + expected[m.obj]['set'] += 1 + for i in m.a: + expected[m.x[i]]['add'] += 1 + expected[m.y]['add'] += 1 + expected[m.c1]['add'] += 1 + obs.check(expected) + + for i in m.a: + expected[m.x[i]]['remove'] += 1 + expected[m.c1]['remove'] += 1 + del m.c1 + detector.update() + obs.check(expected) + def test_vars_and_params_elsewhere(self): - pass + m1 = pyo.ConcreteModel() + m1.x = pyo.Var() + m1.y = pyo.Var() + m1.p = pyo.Param(mutable=True, initialize=1) + + m2 = pyo.ConcreteModel() + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) + + expected = ComponentMap() + expected[None] = make_count_dict() + expected[None]['set'] += 1 + + detector.set_instance(m2) + obs.check(expected) + + m2.obj = pyo.Objective(expr=m1.y) + m2.c1 = pyo.Constraint(expr=m1.y >= (m1.x - m1.p) ** 2) + detector.update() + expected[m1.x] = make_count_dict() + expected[m1.y] = make_count_dict() + expected[m1.p] = make_count_dict() + expected[m1.x]['add'] += 1 + expected[m1.y]['add'] += 1 + expected[m1.p]['add'] += 1 + expected[m2.c1] = make_count_dict() + expected[m2.c1]['add'] += 1 + expected[m2.obj] = make_count_dict() + expected[m2.obj]['set'] += 1 + obs.check(expected) + + # now fix a variable and make sure the + # constraint gets removed and added + m1.x.fix(1) + obs.pprint() + detector.update() + obs.pprint() + expected[m2.c1]['remove'] += 1 + expected[m2.c1]['add'] += 1 + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then + # added again when the constraint is added + expected[m1.x]['update'] += 1 + expected[m1.x]['remove'] += 1 + expected[m1.x]['add'] += 1 + expected[m1.p]['remove'] += 1 + expected[m1.p]['add'] += 1 + obs.check(expected) + + def test_named_expression(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) + + expected = ComponentMap() + expected[None] = make_count_dict() + expected[None]['set'] += 1 + + detector.set_instance(m) + obs.check(expected) + + m.obj = pyo.Objective(expr=m.y) + m.e = pyo.Expression(expr=m.x - m.p) + m.c1 = pyo.Constraint(expr=m.y >= m.e) + detector.update() + expected[m.x] = make_count_dict() + expected[m.y] = make_count_dict() + expected[m.p] = make_count_dict() + expected[m.x]['add'] += 1 + expected[m.y]['add'] += 1 + expected[m.p]['add'] += 1 + expected[m.c1] = make_count_dict() + expected[m.c1]['add'] += 1 + expected[m.obj] = make_count_dict() + expected[m.obj]['set'] += 1 + obs.check(expected) + + # now modify the named expression and make sure the + # constraint gets removed and added + m.e.expr = (m.x - m.p) ** 2 + detector.update() + expected[m.c1]['remove'] += 1 + expected[m.c1]['add'] += 1 + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then + # added again when the constraint is added + expected[m.x]['remove'] += 1 + expected[m.x]['add'] += 1 + expected[m.p]['remove'] += 1 + expected[m.p]['add'] += 1 + obs.check(expected) + From c313fe53f890bdfc000dfd2d68a76fcef8f9de8b Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 15:48:39 -0600 Subject: [PATCH 27/33] run black --- pyomo/contrib/observer/tests/test_change_detector.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 46527618ab8..f3ce6e28b5c 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -363,4 +363,3 @@ def test_named_expression(self): expected[m.p]['remove'] += 1 expected[m.p]['add'] += 1 obs.check(expected) - From d16bee5fd1f27cc7c3b55ff858ab28b7478a6975 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 18:00:06 -0600 Subject: [PATCH 28/33] adding tests for trivial constraints and fixing bugs --- pyomo/contrib/observer/model_observer.py | 2 +- .../contrib/solver/common/solution_loader.py | 30 +++++++++++ .../solvers/gurobi/gurobi_direct_base.py | 23 +++++++- .../solvers/gurobi/gurobi_persistent.py | 6 ++- .../solver/tests/solvers/test_solvers.py | 53 +++++++++++++++++++ pyomo/repn/plugins/standard_form.py | 3 +- 6 files changed, 113 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 4ab52100376..325e7a8aee6 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -614,7 +614,7 @@ def _check_for_var_changes(self): vars_to_update.append(v) elif _domain_interval != v.domain.get_interval(): vars_to_update.append(v) - elif v.value != _value: + elif v.fixed and v.value != _value: vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) return vars_to_update, cons_to_update, update_obj diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index e399d6bea55..be0ea7ad00c 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -17,6 +17,7 @@ from pyomo.core.base.var import VarData from pyomo.core.staleflag import StaleFlagManager from pyomo.core.base.suffix import Suffix +from .util import NoSolutionError def load_import_suffixes(pyomo_model, solution_loader: SolutionLoaderBase, solution_id=None): @@ -194,6 +195,35 @@ def load_import_suffixes(self, solution_id=None): return NotImplemented +class NoSolutionSolutionLoader(SolutionLoaderBase): + def __init__(self) -> None: + pass + + def get_solution_ids(self) -> List[Any]: + return [] + + def get_number_of_solutions(self) -> int: + return 0 + + def load_solution(self, solution_id=None): + raise NoSolutionError() + + def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> None: + raise NoSolutionError() + + def get_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + raise NoSolutionError() + + def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None) -> Dict[ConstraintData, float]: + raise NoSolutionError() + + def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + raise NoSolutionError() + + def load_import_suffixes(self, solution_id=None): + raise NoSolutionError() + + class PersistentSolutionLoader(SolutionLoaderBase): """ Loader for persistent solvers diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index e99d24025d5..ce77c31c6f7 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -18,7 +18,7 @@ from pyomo.common.config import ConfigValue from pyomo.common.dependencies import attempt_import from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import ApplicationError +from pyomo.common.errors import ApplicationError, InfeasibleConstraintException from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer @@ -33,6 +33,7 @@ NoReducedCostsError, NoSolutionError, ) +from pyomo.contrib.solver.common.solution_loader import NoSolutionSolutionLoader from pyomo.contrib.solver.common.results import ( Results, SolutionStatus, @@ -353,6 +354,8 @@ def solve(self, model, **kwds) -> Results: res = self._postsolve( grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj ) + except InfeasibleConstraintException: + res = self._get_infeasible_results() finally: os.chdir(orig_cwd) @@ -390,6 +393,24 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map + def _get_infeasible_results(self): + res = Results() + res.solution_loader = NoSolutionSolutionLoader() + res.solution_status = SolutionStatus.noSolution + res.termination_condition = TerminationCondition.provenInfeasible + res.incumbent_objective = None + res.objective_bound = None + res.iteration_count = None + res.timing_info.gurobi_time = None + res.solver_config = self.config + res.solver_name = self.name + res.solver_version = self.version() + if self.config.raise_exception_on_nonoptimal_result: + raise NoOptimalSolutionError() + if self.config.load_solutions: + raise NoFeasibleSolutionError() + return res + def _postsolve(self, grb_model, solution_loader, has_obj): status = grb_model.Status diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 9f19bae307f..27a61e27916 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -17,6 +17,7 @@ from pyomo.common.collections import ComponentSet, OrderedSet from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer +from pyomo.common.errors import InfeasibleConstraintException from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.base.var import VarData @@ -464,7 +465,9 @@ def _get_expr_from_pyomo_repn(self, repn): vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr(coef_list, vlist) else: - new_expr = 0.0 + # this can't just be zero in case the constraint is a + # trivial one + new_expr = gurobipy.LinExpr() if len(repn.quadratic_vars) > 0: missing_vars = {} @@ -714,6 +717,7 @@ def _add_constraints(self, cons: List[ConstraintData]): for ndx, con in enumerate(cons): lb, body, ub = con.to_bounded_expression(evaluate_bounds=False) repn = generate_standard_repn(body, quadratic=True, compute_values=False) + if len(repn.quadratic_vars) > 0: self._quadratic_cons.add(con) else: diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 9c67ed7b1e5..d154253475c 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -35,6 +35,7 @@ GurobiDirectQuadratic, GurobiPersistent, ) +from pyomo.contrib.solver.common.util import NoSolutionError, NoFeasibleSolutionError, NoOptimalSolutionError from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -1053,6 +1054,58 @@ def test_results_infeasible( ): res.solution_loader.get_reduced_costs() + @parameterized.expand(input=_load_tests(all_solvers)) + def test_trivial_constraints( + self, name: str, opt_class: Type[SolverBase], use_presolve: bool + ): + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + if any(name.startswith(i) for i in nl_solvers_set): + if use_presolve: + opt.config.writer_config.linear_presolve = True + else: + opt.config.writer_config.linear_presolve = False + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= m.x) + m.c2 = pyo.Constraint(expr=m.y >= -m.x) + m.c3 = pyo.Constraint(expr=m.x >= 0) + + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 0) + + m.x.fix(1) + opt.config.tee = True + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 1) + self.assertAlmostEqual(m.y.value, 1) + + m.x.fix(-1) + with self.assertRaises(NoOptimalSolutionError): + res = opt.solve(m) + + opt.config.load_solutions = False + opt.config.raise_exception_on_nonoptimal_result = False + res = opt.solve(m) + self.assertNotEqual(res.solution_status, SolutionStatus.optimal) + if isinstance(opt, Ipopt): + acceptable_termination_conditions = { + TerminationCondition.locallyInfeasible, + TerminationCondition.unbounded, + TerminationCondition.provenInfeasible, + } + else: + acceptable_termination_conditions = { + TerminationCondition.provenInfeasible, + TerminationCondition.infeasibleOrUnbounded, + } + self.assertIn(res.termination_condition, acceptable_termination_conditions) + self.assertIsNone(res.incumbent_objective) + @parameterized.expand(input=_load_tests(all_solvers)) def test_duals(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() diff --git a/pyomo/repn/plugins/standard_form.py b/pyomo/repn/plugins/standard_form.py index 314a1822e09..59c15910350 100644 --- a/pyomo/repn/plugins/standard_form.py +++ b/pyomo/repn/plugins/standard_form.py @@ -20,6 +20,7 @@ InEnum, document_kwargs_from_configdict, ) +from pyomo.common.errors import InfeasibleConstraintException from pyomo.common.dependencies import scipy, numpy as np from pyomo.common.enums import ObjectiveSense from pyomo.common.gc_manager import PauseGC @@ -462,7 +463,7 @@ def write(self, model): # TODO: add a (configurable) feasibility tolerance if (lb is None or lb <= offset) and (ub is None or ub >= offset): continue - raise InfeasibleError( + raise InfeasibleConstraintException( f"model contains a trivially infeasible constraint, '{con.name}'" ) From a4e2b81410b9edba643ef118f21e4ca9f3cc0b37 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 18:05:43 -0600 Subject: [PATCH 29/33] run black --- .../contrib/solver/common/solution_loader.py | 69 ++++++++----------- .../solver/solvers/gurobi/gurobi_direct.py | 13 ++-- .../solvers/gurobi/gurobi_persistent.py | 43 +++++++++--- pyomo/contrib/solver/solvers/highs.py | 16 +++-- pyomo/contrib/solver/solvers/ipopt.py | 10 +-- pyomo/contrib/solver/solvers/sol_reader.py | 17 +++-- .../contrib/solver/tests/unit/test_results.py | 9 +-- .../solver/tests/unit/test_solution.py | 24 ++++++- 8 files changed, 125 insertions(+), 76 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index e399d6bea55..3ad688d937f 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -19,7 +19,9 @@ from pyomo.core.base.suffix import Suffix -def load_import_suffixes(pyomo_model, solution_loader: SolutionLoaderBase, solution_id=None): +def load_import_suffixes( + pyomo_model, solution_loader: SolutionLoaderBase, solution_id=None +): dual_suffix = None rc_suffix = None for suffix in pyomo_model.component_objects(Suffix, descend_into=True, active=True): @@ -46,10 +48,10 @@ class SolutionLoaderBase: def get_solution_ids(self) -> List[Any]: """ - If there are multiple solutions available, this will return a - list of the solution ids which can then be used with other - methods like `load_soltuion`. If only one solution is - available, this will return [None]. If no solutions + If there are multiple solutions available, this will return a + list of the solution ids which can then be used with other + methods like `load_soltuion`. If only one solution is + available, this will return [None]. If no solutions are available, this will return None Returns @@ -58,7 +60,7 @@ def get_solution_ids(self) -> List[Any]: The identifiers for multiple solutions """ return NotImplemented - + def get_number_of_solutions(self) -> int: """ Returns @@ -75,7 +77,7 @@ def load_solution(self, solution_id=None): Parameters ---------- solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be loaded. If None, the default solution will be used. """ # this should load everything it can @@ -83,36 +85,31 @@ def load_solution(self, solution_id=None): self.load_import_suffixes(solution_id=solution_id) def load_vars( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> None: """ - Load the solution of the primal variables into the value attribute + Load the solution of the primal variables into the value attribute of the variables. Parameters ---------- vars_to_load: list - The minimum set of variables whose solution should be loaded. If - vars_to_load is None, then the solution to all primal variables - will be loaded. Even if vars_to_load is specified, the values of + The minimum set of variables whose solution should be loaded. If + vars_to_load is None, then the solution to all primal variables + will be loaded. Even if vars_to_load is specified, the values of other variables may also be loaded depending on the interface. solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be loaded. If None, the default solution will be used. """ for var, val in self.get_vars( - vars_to_load=vars_to_load, - solution_id=solution_id + vars_to_load=vars_to_load, solution_id=solution_id ).items(): var.set_value(val, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) def get_vars( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: """ Returns a ComponentMap mapping variable to var value. @@ -123,7 +120,7 @@ def get_vars( A list of the variables whose solution value should be retrieved. If vars_to_load is None, then the values for all variables will be retrieved. solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be retrieved. If None, the default solution will be used. Returns @@ -136,9 +133,7 @@ def get_vars( ) def get_duals( - self, - cons_to_load: Optional[Sequence[ConstraintData]] = None, - solution_id=None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: """ Returns a dictionary mapping constraint to dual value. @@ -149,7 +144,7 @@ def get_duals( A list of the constraints whose duals should be retrieved. If cons_to_load is None, then the duals for all constraints will be retrieved. solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be retrieved. If None, the default solution will be used. Returns @@ -160,9 +155,7 @@ def get_duals( return NotImplemented def get_reduced_costs( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: """ Returns a ComponentMap mapping variable to reduced cost. @@ -173,7 +166,7 @@ def get_reduced_costs( A list of the variables whose reduced cost should be retrieved. If vars_to_load is None, then the reduced costs for all variables will be loaded. solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be retrieved. If None, the default solution will be used. Returns @@ -182,13 +175,13 @@ def get_reduced_costs( Maps variables to reduced costs """ return NotImplemented - + def load_import_suffixes(self, solution_id=None): """ Parameters ---------- solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be loaded. If None, the default solution will be used. """ return NotImplemented @@ -211,27 +204,25 @@ def _assert_solution_still_valid(self): def get_solution_ids(self) -> List[Any]: self._assert_solution_still_valid() return super().get_solution_ids() - + def get_number_of_solutions(self) -> int: self._assert_solution_still_valid() return super().get_number_of_solutions() def get_vars(self, vars_to_load=None, solution_id=None): self._assert_solution_still_valid() - return self._solver._get_primals(vars_to_load=vars_to_load, solution_id=solution_id) + return self._solver._get_primals( + vars_to_load=vars_to_load, solution_id=solution_id + ) def get_duals( - self, - cons_to_load: Optional[Sequence[ConstraintData]] = None, - solution_id=None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return self._solver._get_duals(cons_to_load=cons_to_load) def get_reduced_costs( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return self._solver._get_reduced_costs(vars_to_load=vars_to_load) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index fd932f90c15..42ab82a1d7c 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -23,7 +23,10 @@ NoSolutionError, IncompatibleModelError, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes +from pyomo.contrib.solver.common.solution_loader import ( + SolutionLoaderBase, + load_import_suffixes, +) from .gurobi_direct_base import GurobiDirectBase, gurobipy @@ -59,7 +62,7 @@ def get_number_of_solutions(self) -> int: if self._grb_model.SolCount == 0: return 0 return 1 - + def get_solution_ids(self) -> List[Any]: return [0] @@ -118,9 +121,11 @@ def get_reduced_costs(self, vars_to_load=None, solution_id=None): vars_to_load = ComponentSet(vars_to_load) iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) return ComponentMap(iterator) - + def load_import_suffixes(self, solution_id=None): - load_import_suffixes(pyomo_model=self._pyomo_model, solution_loader=self, solution_id=solution_id) + load_import_suffixes( + pyomo_model=self._pyomo_model, solution_loader=self, solution_id=solution_id + ) class GurobiDirect(GurobiDirectBase): diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 9f19bae307f..e36e1e6db1e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -27,7 +27,10 @@ from pyomo.repn import generate_standard_repn from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import IncompatibleModelError -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes +from pyomo.contrib.solver.common.solution_loader import ( + SolutionLoaderBase, + load_import_suffixes, +) from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( GurobiDirectBase, @@ -46,7 +49,14 @@ class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model + self, + solver_model, + var_id_map, + var_map, + con_map, + linear_cons, + quadratic_cons, + pyomo_model, ) -> None: super().__init__() self._solver_model = solver_model @@ -66,7 +76,7 @@ def __del__(self): def get_number_of_solutions(self) -> int: return self._solver_model.SolCount - + def get_solution_ids(self) -> List: return list(range(self.get_number_of_solutions())) @@ -131,10 +141,23 @@ def load_import_suffixes(self, solution_id=None): class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model + self, + solver_model, + var_id_map, + var_map, + con_map, + linear_cons, + quadratic_cons, + pyomo_model, ) -> None: super().__init__( - solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model + solver_model, + var_id_map, + var_map, + con_map, + linear_cons, + quadratic_cons, + pyomo_model, ) self._valid = True @@ -158,25 +181,25 @@ def get_vars( return super().get_vars(vars_to_load, solution_id) def get_duals( - self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None, + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return super().get_duals(cons_to_load) def get_reduced_costs( - self, vars_to_load: Sequence[VarData] | None = None, solution_id=None, + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_reduced_costs(vars_to_load) - + def get_number_of_solutions(self) -> int: self._assert_solution_still_valid() return super().get_number_of_solutions() - + def get_solution_ids(self) -> List: self._assert_solution_still_valid() return super().get_solution_ids() - + def load_import_suffixes(self, solution_id=None): self._assert_solution_still_valid() super().load_import_suffixes(solution_id) diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index 0abf02813ab..87796b91b68 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -760,14 +760,18 @@ def _postsolve(self): def _load_vars(self, vars_to_load=None, solution_id=None): if solution_id is not None: - raise NotImplementedError('highs interface does not currently support multiple solutions') + raise NotImplementedError( + 'highs interface does not currently support multiple solutions' + ) for v, val in self._get_primals(vars_to_load=vars_to_load).items(): v.set_value(val, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) def _get_primals(self, vars_to_load=None, solution_id=None): if solution_id is not None: - raise NotImplementedError('highs interface does not currently support multiple solutions') + raise NotImplementedError( + 'highs interface does not currently support multiple solutions' + ) if self._sol is None or not self._sol.value_valid: raise NoSolutionError() @@ -792,7 +796,9 @@ def _get_primals(self, vars_to_load=None, solution_id=None): def _get_reduced_costs(self, vars_to_load=None, solution_id=None): if solution_id is not None: - raise NotImplementedError('highs interface does not currently support multiple solutions') + raise NotImplementedError( + 'highs interface does not currently support multiple solutions' + ) if self._sol is None or not self._sol.dual_valid: raise NoReducedCostsError() res = ComponentMap() @@ -812,7 +818,9 @@ def _get_reduced_costs(self, vars_to_load=None, solution_id=None): def _get_duals(self, cons_to_load=None, solution_id=None): if solution_id is not None: - raise NotImplementedError('highs interface does not currently support multiple solutions') + raise NotImplementedError( + 'highs interface does not currently support multiple solutions' + ) if self._sol is None or not self._sol.dual_valid: raise NoDualsError() diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 80f9775a657..bce5e9bd867 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -109,9 +109,7 @@ def _error_check(self): ) def get_reduced_costs( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: self._error_check() if solution_id is not None: @@ -448,7 +446,9 @@ def solve(self, model, **kwds) -> Results: TerminationCondition.convergenceCriteriaSatisfied ) results.solution_status = SolutionStatus.optimal - results.solution_loader = SolSolutionLoader(None, nl_info=nl_info, pyomo_model=model) + results.solution_loader = SolSolutionLoader( + None, nl_info=nl_info, pyomo_model=model + ) results.iteration_count = 0 results.timing_info.total_seconds = 0 else: @@ -668,7 +668,7 @@ def _parse_solution( res.solution_loader = SolSolutionLoader(None, None, pyomo_model=pyomo_model) else: res.solution_loader = IpoptSolutionLoader( - sol_data=sol_data, nl_info=nl_info, pyomo_model=pyomo_model, + sol_data=sol_data, nl_info=nl_info, pyomo_model=pyomo_model ) return res diff --git a/pyomo/contrib/solver/solvers/sol_reader.py b/pyomo/contrib/solver/solvers/sol_reader.py index 7570a1ffc53..f405cc85943 100644 --- a/pyomo/contrib/solver/solvers/sol_reader.py +++ b/pyomo/contrib/solver/solvers/sol_reader.py @@ -26,7 +26,10 @@ SolutionStatus, TerminationCondition, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes +from pyomo.contrib.solver.common.solution_loader import ( + SolutionLoaderBase, + load_import_suffixes, +) class SolFileData: @@ -49,7 +52,9 @@ class SolSolutionLoader(SolutionLoaderBase): Loader for solvers that create .sol files (e.g., ipopt) """ - def __init__(self, sol_data: SolFileData, nl_info: NLWriterInfo, pyomo_model) -> None: + def __init__( + self, sol_data: SolFileData, nl_info: NLWriterInfo, pyomo_model + ) -> None: self._sol_data = sol_data self._nl_info = nl_info self._pyomo_model = pyomo_model @@ -58,14 +63,16 @@ def get_number_of_solutions(self) -> int: if self._nl_info is None: return 0 return 1 - + def get_solution_ids(self) -> List[Any]: return [None] def load_import_suffixes(self, solution_id=None): load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) - def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None) -> NoReturn: + def load_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None + ) -> NoReturn: if solution_id is not None: raise ValueError(f'{self.__class__.__name__} does not support solution_id') if self._nl_info is None: @@ -131,7 +138,7 @@ def get_vars( return res def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: if solution_id is not None: raise ValueError(f'{self.__class__.__name__} does not support solution_id') diff --git a/pyomo/contrib/solver/tests/unit/test_results.py b/pyomo/contrib/solver/tests/unit/test_results.py index a4def8f9089..3dad4c523d2 100644 --- a/pyomo/contrib/solver/tests/unit/test_results.py +++ b/pyomo/contrib/solver/tests/unit/test_results.py @@ -50,8 +50,7 @@ def __init__( self._reduced_costs = reduced_costs def get_vars( - self, vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: if self._primals is None: raise RuntimeError( @@ -67,8 +66,7 @@ def get_vars( return primals def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None, - solution_id=None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: if self._duals is None: raise RuntimeError( @@ -85,8 +83,7 @@ def get_duals( return duals def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: if self._reduced_costs is None: raise RuntimeError( diff --git a/pyomo/contrib/solver/tests/unit/test_solution.py b/pyomo/contrib/solver/tests/unit/test_solution.py index a0fc4ac9b2f..79e5b39aaf6 100644 --- a/pyomo/contrib/solver/tests/unit/test_solution.py +++ b/pyomo/contrib/solver/tests/unit/test_solution.py @@ -18,7 +18,16 @@ class TestSolutionLoaderBase(unittest.TestCase): def test_member_list(self): - expected_list = ['load_vars', 'get_vars', 'get_duals', 'get_reduced_costs', 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', 'load_solution'] + expected_list = [ + 'load_vars', + 'get_vars', + 'get_duals', + 'get_reduced_costs', + 'load_import_suffixes', + 'get_number_of_solutions', + 'get_solution_ids', + 'load_solution', + ] method_list = [ method for method in dir(SolutionLoaderBase) @@ -38,7 +47,16 @@ class TestSolSolutionLoader(unittest.TestCase): # I am currently unsure how to test this further because it relies heavily on # SolFileData and NLWriterInfo def test_member_list(self): - expected_list = ['load_vars', 'get_vars', 'get_duals', 'get_reduced_costs', 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', 'load_solution'] + expected_list = [ + 'load_vars', + 'get_vars', + 'get_duals', + 'get_reduced_costs', + 'load_import_suffixes', + 'get_number_of_solutions', + 'get_solution_ids', + 'load_solution', + ] method_list = [ method for method in dir(SolutionLoaderBase) @@ -58,7 +76,7 @@ def test_member_list(self): 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', - 'load_solution' + 'load_solution', ] method_list = [ method From d25e7215f4f728ff4541060f84f9b6cd92d09229 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 18:06:51 -0600 Subject: [PATCH 30/33] run black --- .../contrib/solver/common/solution_loader.py | 30 ++++++++++++------- .../solvers/gurobi/gurobi_persistent.py | 2 +- .../solver/tests/solvers/test_solvers.py | 10 +++++-- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index b0bddeb56ba..666ea66e1e9 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -194,25 +194,33 @@ def __init__(self) -> None: def get_solution_ids(self) -> List[Any]: return [] - + def get_number_of_solutions(self) -> int: return 0 - + def load_solution(self, solution_id=None): raise NoSolutionError() - - def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> None: + + def load_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> None: raise NoSolutionError() - - def get_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + + def get_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> Mapping[VarData, float]: raise NoSolutionError() - - def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None) -> Dict[ConstraintData, float]: + + def get_duals( + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None + ) -> Dict[ConstraintData, float]: raise NoSolutionError() - - def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + + def get_reduced_costs( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> Mapping[VarData, float]: raise NoSolutionError() - + def load_import_suffixes(self, solution_id=None): raise NoSolutionError() diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index c0211604324..b9ea9a6c8e8 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -488,7 +488,7 @@ def _get_expr_from_pyomo_repn(self, repn): vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr(coef_list, vlist) else: - # this can't just be zero in case the constraint is a + # this can't just be zero in case the constraint is a # trivial one new_expr = gurobipy.LinExpr() diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index d154253475c..189b0373780 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -35,7 +35,11 @@ GurobiDirectQuadratic, GurobiPersistent, ) -from pyomo.contrib.solver.common.util import NoSolutionError, NoFeasibleSolutionError, NoOptimalSolutionError +from pyomo.contrib.solver.common.util import ( + NoSolutionError, + NoFeasibleSolutionError, + NoOptimalSolutionError, +) from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -1073,11 +1077,11 @@ def test_trivial_constraints( m.c1 = pyo.Constraint(expr=m.y >= m.x) m.c2 = pyo.Constraint(expr=m.y >= -m.x) m.c3 = pyo.Constraint(expr=m.x >= 0) - + res = opt.solve(m) self.assertAlmostEqual(m.x.value, 0) self.assertAlmostEqual(m.y.value, 0) - + m.x.fix(1) opt.config.tee = True res = opt.solve(m) From a43a38bacc66aafa6af6c7305448e3f55a3fe263 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 16 Aug 2025 09:30:56 -0600 Subject: [PATCH 31/33] forgot to inherit from PersistentSolverBase --- .../solvers/gurobi/gurobi_persistent.py | 3 +- .../solver/tests/solvers/test_solvers.py | 139 ++++++++---------- 2 files changed, 64 insertions(+), 78 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 847ec958bdd..8d16bb9082e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -28,6 +28,7 @@ from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import IncompatibleModelError from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +from pyomo.contrib.solver.common.base import PersistentSolverBase from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( GurobiDirectBase, @@ -575,7 +576,7 @@ def update_parameters(self, params: List[ParamData]): self.opt._update_parameters(params) -class GurobiPersistent(GurobiDirectQuadratic): +class GurobiPersistent(GurobiDirectQuadratic, PersistentSolverBase): _minimum_version = (7, 0, 0) def __init__(self, **kwds): diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 96e2e7b2c38..6965147d167 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -1221,55 +1221,50 @@ def test_mutable_quadratic_objective_qp( def test_fixed_vars( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): - for treat_fixed_vars_as_params in [True, False]: - opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=treat_fixed_vars_as_params) - if not opt.available(): - raise unittest.SkipTest(f'Solver {opt.name} not available.') - if any(name.startswith(i) for i in nl_solvers_set): - if use_presolve: - opt.config.writer_config.linear_presolve = True - else: - opt.config.writer_config.linear_presolve = False - m = pyo.ConcreteModel() - m.x = pyo.Var() - m.x.fix(0) - m.y = pyo.Var() - a1 = 1 - a2 = -1 - b1 = 1 - b2 = 2 - m.obj = pyo.Objective(expr=m.y) - m.c1 = pyo.Constraint(expr=m.y >= a1 * m.x + b1) - m.c2 = pyo.Constraint(expr=m.y >= a2 * m.x + b2) - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) - m.x.unfix() - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, (b2 - b1) / (a1 - a2)) - self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) - m.x.fix(0) - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) - m.x.value = 2 - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 2) - self.assertAlmostEqual(m.y.value, 3) - m.x.value = 0 - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + if any(name.startswith(i) for i in nl_solvers_set): + if use_presolve: + opt.config.writer_config.linear_presolve = True + else: + opt.config.writer_config.linear_presolve = False + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.x.fix(0) + m.y = pyo.Var() + a1 = 1 + a2 = -1 + b1 = 1 + b2 = 2 + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= a1 * m.x + b1) + m.c2 = pyo.Constraint(expr=m.y >= a2 * m.x + b2) + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) + m.x.unfix() + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, (b2 - b1) / (a1 - a2)) + self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) + m.x.fix(0) + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) + m.x.value = 2 + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 2) + self.assertAlmostEqual(m.y.value, 3) + m.x.value = 0 + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) @parameterized.expand(input=_load_tests(all_solvers)) def test_fixed_vars_2( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1313,8 +1308,6 @@ def test_fixed_vars_3( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1337,8 +1330,6 @@ def test_fixed_vars_4( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1892,10 +1883,7 @@ def test_fixed_binaries( res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 1) - if opt.is_persistent(): - opt: SolverBase = opt_class(treat_fixed_vars_as_params=False) - else: - opt = opt_class() + opt = opt_class() m.x.fix(0) res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 0) @@ -2049,33 +2037,30 @@ def test_bug_2(self, name: str, opt_class: Type[SolverBase], use_presolve: bool) This test is for a bug where an objective containing a fixed variable does not get updated properly when the variable is unfixed. """ - for fixed_var_option in [True, False]: - opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=fixed_var_option) - if not opt.available(): - raise unittest.SkipTest(f'Solver {opt.name} not available.') - if any(name.startswith(i) for i in nl_solvers_set): - if use_presolve: - opt.config.writer_config.linear_presolve = True - else: - opt.config.writer_config.linear_presolve = False - - m = pyo.ConcreteModel() - m.x = pyo.Var(bounds=(-10, 10)) - m.y = pyo.Var() - m.obj = pyo.Objective(expr=3 * m.y - m.x) - m.c = pyo.Constraint(expr=m.y >= m.x) - - m.x.fix(1) - res = opt.solve(m) - self.assertAlmostEqual(res.incumbent_objective, 2, 5) + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + if any(name.startswith(i) for i in nl_solvers_set): + if use_presolve: + opt.config.writer_config.linear_presolve = True + else: + opt.config.writer_config.linear_presolve = False - m.x.unfix() - m.x.setlb(-9) - m.x.setub(9) - res = opt.solve(m) - self.assertAlmostEqual(res.incumbent_objective, -18, 5) + m = pyo.ConcreteModel() + m.x = pyo.Var(bounds=(-10, 10)) + m.y = pyo.Var() + m.obj = pyo.Objective(expr=3 * m.y - m.x) + m.c = pyo.Constraint(expr=m.y >= m.x) + + m.x.fix(1) + res = opt.solve(m) + self.assertAlmostEqual(res.incumbent_objective, 2, 5) + + m.x.unfix() + m.x.setlb(-9) + m.x.setub(9) + res = opt.solve(m) + self.assertAlmostEqual(res.incumbent_objective, -18, 5) @parameterized.expand(input=_load_tests(nl_solvers)) def test_presolve_with_zero_coef( From e76baae7b130aa3d845e053c802a93ca5e76de26 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 16 Aug 2025 12:25:39 -0600 Subject: [PATCH 32/33] bug --- pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 8d16bb9082e..8145777ffb9 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -595,6 +595,7 @@ def __init__(self, **kwds): self._observer = _GurobiObserver(self) self._change_detector = ModelChangeDetector(observers=[self._observer]) self._constraint_ndx = 0 + self._should_update_parameters = False @property def auto_updates(self): @@ -683,6 +684,8 @@ def update(self): if self._needs_updated: self._update_gurobi_model() self._change_detector.update(timer=timer) + if self._should_update_parameters: + self._update_parameters([]) timer.stop('update') def _add_constraints(self, cons: List[ConstraintData]): @@ -915,6 +918,8 @@ def _update_variables(self, variables: List[VarData]): gurobipy_var.setAttr('lb', lb) gurobipy_var.setAttr('ub', ub) gurobipy_var.setAttr('vtype', vtype) + if var.fixed: + self._should_update_parameters = True self._needs_updated = True def _update_parameters(self, params: List[ParamData]): @@ -950,6 +955,8 @@ def _update_parameters(self, params: List[ParamData]): # parts have mutable coefficients self._solver_model.setObjective(new_gurobi_expr, sense=sense) + self._should_update_parameters = False + def _invalidate_last_results(self): if self._last_results_object is not None: self._last_results_object.solution_loader.invalidate() From c2a01777d43d47d7ea072c5b93ddc020e8b787ee Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 18 Aug 2025 09:13:12 -0600 Subject: [PATCH 33/33] bug --- pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 8145777ffb9..603e8e21800 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -432,7 +432,7 @@ def _add_variables(self, variables: List[VarData]): def _get_expr_from_pyomo_repn(self, repn): if repn.nonlinear_expr is not None: raise IncompatibleModelError( - f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' + f'GurobiDirectQuadratic only supports linear and quadratic expressions: {repn}.' ) if len(repn.linear_vars) > 0: