Skip to content

Commit 4640201

Browse files
tristantctristantc
committed
Refactor benchmark function to support multiple GAMS solvers and improve logging
1 parent d5fa9d2 commit 4640201

File tree

1 file changed

+142
-36
lines changed

1 file changed

+142
-36
lines changed

‎benchmark.py

Lines changed: 142 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@
99
from contextlib import redirect_stdout
1010

1111

12-
def benchmark(model, strategy, timelimit, result_dir, subsolver="scip"):
12+
def benchmark(
13+
model, strategy, timelimit, result_dir, subsolver="gams", solver_gams="baron"
14+
):
1315
"""Benchmark the model using the given strategy and subsolver.
1416
1517
The result files include the solver output and the JSON representation of the results.
@@ -24,6 +26,10 @@ def benchmark(model, strategy, timelimit, result_dir, subsolver="scip"):
2426
the time limit for the solver
2527
result_dir : string
2628
the directory to store the benchmark results
29+
subsolver : string
30+
the subsolver used to solve the model
31+
solver_gams : string
32+
the solver used to solve the model in GAMS
2733
2834
Returns
2935
-------
@@ -34,16 +40,26 @@ def benchmark(model, strategy, timelimit, result_dir, subsolver="scip"):
3440

3541
# Direct the solver output to a file
3642
if strategy in ["gdp.bigm", "gdp.hull"]:
37-
with open(result_dir + "/" + strategy + "_" + subsolver + ".log", "w") as f:
43+
with open(
44+
result_dir + "/" + strategy + "_" + subsolver + "_" + solver_gams + ".log",
45+
"w",
46+
) as f:
3847
with redirect_stdout(f):
3948
transformation_start_time = time.time()
4049
TransformationFactory(strategy).apply_to(model)
4150
transformation_end_time = time.time()
4251
results = SolverFactory(subsolver).solve(
4352
model,
4453
tee=True,
45-
add_options=["option reslim=3600;option threads=1;"],
54+
solver=solver_gams,
55+
add_options=[
56+
"option reslim=3600;option threads=1;option optcr=1e-2;"
57+
],
58+
# keepfiles=True,
59+
# tmpdir=os.path.join(result_dir, strategy, "nlp"),
60+
# symbolic_solver_labels=True,
4661
)
62+
results.solver.strategy = strategy
4763
results.solver.transformation_time = (
4864
transformation_end_time - transformation_start_time
4965
)
@@ -55,72 +71,162 @@ def benchmark(model, strategy, timelimit, result_dir, subsolver="scip"):
5571
"gdpopt.lbb",
5672
"gdpopt.ric",
5773
]:
58-
with open(result_dir + "/" + strategy + "_" + subsolver + ".log", "w") as f:
74+
with open(
75+
result_dir + "/" + strategy + "_" + subsolver + "_" + solver_gams + ".log",
76+
"w",
77+
) as f:
5978
with redirect_stdout(f):
6079
results = SolverFactory(strategy).solve(
6180
model,
6281
tee=True,
82+
# bound_tolerance=1e-2, # default is 1e-6
6383
nlp_solver=subsolver,
64-
nlp_solver_args=dict(add_options=["option threads=1;"]),
84+
nlp_solver_args=dict(
85+
solver=solver_gams,
86+
add_options=[
87+
"option threads=1;",
88+
'$onecho > baron.opt',
89+
'FirstLoc 1',
90+
# # 'nlpsol 9', #9: IPOPT, 6: GAMS NLP solver, default -1: Automatic NLP solver selection and switching strategy
91+
# 'optcr 1.e-2',
92+
# # 'optca 1.e-2',
93+
# #'maxiter 0', # force BARON to terminate after root node preprocessing
94+
# # 'maxiter 1', # termination after the solution of the root node
95+
# # 'maxiter 1e3',
96+
# #'numloc -1', # local searches in preprocessing from randomly generated starting points until global optimality is proved or MaxTime seconds have elapsed.
97+
# #'reslim 760',
98+
'$offecho',
99+
'GAMS_MODEL.optfile=1',
100+
"option optcr=1e-2;",
101+
],
102+
# keepfiles=True,
103+
# tmpdir=os.path.join(result_dir, strategy, "nlp"),
104+
# symbolic_solver_labels=True,
105+
# logfile=result_dir + "/" + strategy + "_" + "nlp" + ".log",
106+
tee=True,
107+
),
65108
mip_solver=subsolver,
66-
mip_solver_args=dict(add_options=["option threads=1;"]),
109+
mip_solver_args=dict(add_options=["option threads=1"], tee=True),
67110
minlp_solver=subsolver,
68-
minlp_solver_args=dict(add_options=["option threads=1;"]),
111+
minlp_solver_args=dict(
112+
solver=solver_gams,
113+
add_options=[
114+
"option threads=1;",
115+
# '$onecho > baron.opt',
116+
# # 'FirstLoc 1',
117+
# # 'nlpsol 9', #9: IPOPT, 6: GAMS NLP solver, default -1: Automatic NLP solver selection and switching strategy
118+
# 'optcr 1.e-2',
119+
# # 'optca 1.e-2',
120+
# #'maxiter 0', # force BARON to terminate after root node preprocessing
121+
# # 'maxiter 1', # termination after the solution of the root node
122+
# # 'maxiter 1e3',
123+
# #'numloc -1', # local searches in preprocessing from randomly generated starting points until global optimality is proved or MaxTime seconds have elapsed.
124+
# #'reslim 760',
125+
# '$offecho',
126+
# 'GAMS_MODEL.optfile=1',
127+
"option optcr=1e-6;",
128+
],
129+
tee=True,
130+
# keepfiles=True,
131+
# tmpdir=os.path.join(result_dir, strategy, "minlp"),
132+
# symbolic_solver_labels=True,
133+
),
69134
local_minlp_solver=subsolver,
70-
local_minlp_solver_args=dict(add_options=["option threads=1;"]),
135+
local_minlp_solver_args=dict(
136+
solver=solver_gams,
137+
add_options=[
138+
"option threads=1;",
139+
'$onecho > baron.opt',
140+
'FirstLoc 1',
141+
# # 'nlpsol 9', #9: IPOPT, 6: GAMS NLP solver, default -1: Automatic NLP solver selection and switching strategy
142+
# 'optcr 1.e-2',
143+
# # 'optca 1.e-2',
144+
# #'maxiter 0', # force BARON to terminate after root node preprocessing
145+
# # 'maxiter 1', # termination after the solution of the root node
146+
# # 'maxiter 1e3',
147+
# #'numloc -1', # local searches in preprocessing from randomly generated starting points until global optimality is proved or MaxTime seconds have elapsed.
148+
# #'reslim 760',
149+
'$offecho',
150+
'GAMS_MODEL.optfile=1',
151+
"option optcr=1e-2;",
152+
],
153+
tee=True,
154+
# keepfiles=True,
155+
# tmpdir=os.path.join(result_dir, strategy, "local_minlp"),
156+
# symbolic_solver_labels=True,
157+
),
71158
time_limit=timelimit,
72159
)
160+
# results.solver.strategy = strategy
73161
print(results)
74162

75-
with open(result_dir + "/" + strategy + "_" + subsolver + ".json", "w") as f:
163+
with open(
164+
result_dir + "/" + strategy + "_" + subsolver + "_" + solver_gams + ".json", "w"
165+
) as f:
76166
json.dump(results.json_repn(), f)
77167
return None
78168

79169

80170
if __name__ == "__main__":
81171
instance_list = [
82172
# "batch_processing",
83-
# "biofuel", # enumeration got stuck
173+
# "biofuel", # enumeration got stuck
84174
# "cstr",
85-
"disease_model",
175+
# "disease_model",
86176
"ex1_linan_2023",
87-
"gdp_col",
88-
"hda",
89-
"jobshop",
177+
# "gdp_col",
178+
# "hda",
179+
# "jobshop",
90180
# "kaibel",
91-
"med_term_purchasing",
92-
"methanol",
93-
"mod_hens",
94-
"modprodnet",
95-
"positioning",
96-
"small_batch",
97-
"spectralog",
98-
"stranded_gas",
99-
"syngas",
181+
# "med_term_purchasing",
182+
# "methanol",
183+
# "mod_hens",
184+
# "modprodnet",
185+
# "positioning",
186+
# "small_batch",
187+
# "spectralog",
188+
# "stranded_gas",
189+
# "syngas"
100190
]
101191
strategy_list = [
102-
"gdp.bigm",
103-
"gdp.hull",
192+
# "gdp.bigm",
193+
# "gdp.hull",
104194
"gdpopt.enumerate",
105-
"gdpopt.loa",
106-
"gdpopt.gloa",
107-
"gdpopt.ric",
108-
"gdpopt.lbb",
195+
# "gdpopt.loa",
196+
# "gdpopt.gloa",
197+
# "gdpopt.ric",
198+
# "gdpopt.lbb",
199+
]
200+
solver_gams_list = [
201+
'baron',
202+
# 'scip'
109203
]
110204
current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
111205
timelimit = 3600
112206

113207
for instance in instance_list:
114-
result_dir = "gdplib/" + instance + "/benchmark_result/"
208+
result_dir = "gdplib/" + instance + "/benchmark_result/" + current_time
115209
os.makedirs(result_dir, exist_ok=True)
116210

117211
print("Benchmarking instance: ", instance)
118212
model = import_module("gdplib." + instance).build_model()
119213

120214
for strategy in strategy_list:
121-
if os.path.exists(result_dir + "/" + strategy + "_" + "gams" + ".json"):
122-
continue
123-
try:
124-
benchmark(model, strategy, timelimit, result_dir, "gams")
125-
except:
126-
pass
215+
for solver_gams in solver_gams_list:
216+
if os.path.exists(
217+
result_dir
218+
+ "/"
219+
+ strategy
220+
+ "_"
221+
+ "gams"
222+
+ "_"
223+
+ solver_gams
224+
+ ".json"
225+
):
226+
continue
227+
try:
228+
benchmark(
229+
model, strategy, timelimit, result_dir, "gams", solver_gams
230+
)
231+
except:
232+
pass

0 commit comments

Comments
 (0)