Rewrite CPLEX using Clang (#316)

* Rewrite CPLEX using Clang

* Fix off-by-1 error in addmipstarts

* Fix TerminationStatus

* Add column function

* Remove JuMP from Project.toml

* Fix handling of env finalizer

* Add test for manual finalizing

* Update list of private functions and symbols

* Deprecate all existing functions

* Deprecate .inner field access

* Improve error message on failed install

* Update README

* Check user's version correctly

* Install 12100 on Travis

* Update scripts

.gitignore

@@ -5,4 +5,4 @@
 *.log
 *.cov
 Manifest.toml
-
+scripts/*.h

.travis.yml

@@ -4,8 +4,7 @@ os:
 - linux
 julia:
 - 1.0
-- 1.1
-- 1.3
+- 1
 notifications:
   email: false
 branches:

Project.toml

@@ -1,26 +1,22 @@
 name = "CPLEX"
 uuid = "a076750e-1247-5638-91d2-ce28b192dca0"
 repo = "https://github.com/jump-dev/CPLEX.jl"
-version = "0.6.6"
+version = "0.7.0"
 
 [deps]
+CEnum = "fa961155-64e5-5f13-b03f-caf6b980ea82"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
-LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee"
-MathProgBase = "fdba3010-5040-5b88-9595-932c9decdf73"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 
 [compat]
-Compat = "2, 3"
+CEnum = "0.3, 0.4"
 MathOptInterface = "0.9.14"
-MathProgBase = "~0.5.0, ~0.6, ~0.7"
 julia = "1"
 
 [extras]
-Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Compat", "Pkg", "Random", "Test"]
+test = ["Random", "Test"]

README.md

@@ -1,18 +1,30 @@
+**CPLEX.jl underwent a major rewrite between versions 0.6.6 and 0.7.0. Users of
+JuMP should see no breaking changes, but if you used the lower-level C API
+(e.g., for callbacks), you will need to update your code accordingly.**
+
+**To revert to the old API, use:**
+```julia
+import Pkg
+Pkg.add(Pkg.PackageSpec(name = "CPLEX", version = v"0.6"))
+```
+**Then restart Julia for the change to take effect.**
+
 # CPLEX.jl
 
-`CPLEX.jl` is an interface to the [IBM® ILOG® CPLEX® Optimization
-Studio](https://www.ibm.com/products/ilog-cplex-optimization-studio). It
-provides an interface to the low-level C API, as well as an implementation of
-the solver-independent
-[`MathProgBase`](https://github.com/JuliaOpt/MathProgBase.jl) and
-[`MathOptInterface`](https://github.com/jump-dev/MathOptInterface.jl) API's.
+CPLEX.jl is a wrapper for the [IBM® ILOG® CPLEX® Optimization
+Studio](https://www.ibm.com/products/ilog-cplex-optimization-studio)
 
-You cannot use `CPLEX.jl` without having purchased and installed a copy of CPLEX
+You cannot use CPLEX.jl without having purchased and installed a copy of CPLEX
 Optimization Studio from [IBM](http://www.ibm.com/). However, CPLEX is
 available for free to [academics and students](http://ibm.biz/Bdzvqw).
 
-This package is available free of charge and in no way replaces or alters any
-functionality of IBM's CPLEX Optimization Studio product.
+CPLEX.jl has two components:
+ - a thin wrapper around the complete C API
+ - an interface to [MathOptInterface](https://github.com/jump-dev/MathOptInterface.jl)
+
+The C API can be accessed via `CPLEX.CPXxx` functions, where the names and
+arguments are identical to the C API. See the [CPLEX documentation](https://www.gurobi.com/documentation/9.0/refman/c_api_details.html)
+for details.
 
 *Note: This wrapper is maintained by the JuMP community and is not
 officially supported by IBM. However, we thank IBM for providing us with a
@@ -21,42 +33,109 @@ interested in official support for CPLEX in Julia, let them know!.*
 
 ## Installation
 
-First, you must obtain a copy of the CPLEX software and a license. Then, set the
-appropriate environment variable and run `Pkg.add("CPLEX")`.
-
+First, obtain a license of CPLEX and install CPLEX solver, following the
+instructions on [IBM's website](http://www.gurobi.com). Then, set the
+`CPLEX_STUDIO_BINARIES` environment variable as appropriate and run
+`Pkg.add("CPLEX")`, then `Pkg.build("CPLEX")`. For example:
 ```julia
-# Linux
-ENV["CPLEX_STUDIO_BINARIES"] = "/path/to/cplex/bin/x86-64_linux"
-
-# OSX
-ENV["CPLEX_STUDIO_BINARIES"] = "/path/to/cplex/bin/x86-64_osx"
+# On Windows, this might be
+ENV["CPLEX_STUDIO_BINARIES"] = "C:\\Program Files\\CPLEX_Studio1210\\cplex\\bin\\x86-64_win\\"
+import Pkg
+Pkg.add("CPLEX")
+Pkg.build("CPLEX")
 
-# Windows
-ENV["CPLEX_STUDIO_BINARIES"] = "C:/IBM/CPLEX_Studio128/cplex/bin/x64_win64"
+# On OSX, this might be
+ENV["CPLEX_STUDIO_BINARIES"] = "/Applications/CPLEX_Studio1210/cplex/bin/x86-64_osx/"
+import Pkg
+Pkg.add("CPLEX")
+Pkg.build("CPLEX")
 
+# On Unix, this might be
+ENV["CPLEX_STUDIO_BINARIES"] = "/opt/CPLEX_Studio1210/cplex/bin/x86-64_linux/"
 import Pkg
 Pkg.add("CPLEX")
+Pkg.build("CPLEX")
 ```
-
-## Help! I got `LoadError: Unable to locate CPLEX installation`
-
-Which version of CPLEX are you trying to install? Currently, CPLEX.jl only
-supports 12.8, 12.9, and 12.10 given recent changes to
-[the API](https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.9.0/ilog.odms.studio.help/CPLEX/ReleaseNotes/topics/releasenotes1290/removed.html).
-
-If you want to support newer versions of CPLEX not listed above, [file an
-issue](https://github.com/jump-dev/CPLEX.jl/issues/new) with the version
-number you'd like to support. Some steps need to be taken (like checking for
-new or renamed parameters) before CPLEX.jl can support new versions.
+**Note: your path may differ. Check which folder you installed CPLEX in, and
+update the path accordingly.**
 
 ## Use with JuMP
 
-You can use CPLEX with JuMP via the `CPLEX.Optimizer()` solver.
-Set solver parameters using `set_optimizer_attribute` from `JuMP`:
+We highly recommend that you use the *CPLEX.jl* package with higher level
+packages such as [JuMP.jl](https://github.com/jump-dev/JuMP.jl).
 
+This can be done using the ``CPLEX.Optimizer`` object. Here is how to create a
+*JuMP* model that uses CPLEX as the solver.
 ```julia
+using JuMP, CPLEX
+
 model = Model(CPLEX.Optimizer)
 set_optimizer_attribute(model, "CPX_PARAM_EPINT", 1e-8)
 ```
 
-Parameters match those of the C API in the [CPLEX documentation](https://www.ibm.com/support/knowledgecenter/SSSA5P_12.9.0/ilog.odms.cplex.help/CPLEX/Parameters/topics/introListAlpha.html).
+Parameters match those of the C API in the [CPLEX documentation](https://www.ibm.com/support/knowledgecenter/SSSA5P_12.10.0/ilog.odms.cplex.help/CPLEX/Parameters/topics/introListAlpha.html).
+
+## Callbacks
+
+Here is an example using CPLEX's solver-specific callbacks.
+
+```julia
+using JuMP, CPLEX, Test
+
+model = direct_model(CPLEX.Optimizer())
+set_silent(model)
+
+# This is very, very important!!! Only use callbacks in single-threaded mode.
+MOI.set(model, MOI.NumberOfThreads(), 1)
+
+@variable(model, 0 <= x <= 2.5, Int)
+@variable(model, 0 <= y <= 2.5, Int)
+@objective(model, Max, y)
+cb_calls = Clong[]
+function my_callback_function(cb_data::CPLEX.CallbackContext, context_id::Clong)
+    # You can reference variables outside the function as normal
+    push!(cb_calls, context_id)
+    # You can select where the callback is run
+    if context_id != CPX_CALLBACKCONTEXT_CANDIDATE
+        return
+    end
+    # You can query CALLBACKINFO items
+    valueP = Ref{Cdouble}()
+    ret = CPXcallbackgetinfodbl(cb_data, CPXCALLBACKINFO_BEST_BND, valueP)
+    @info "Best bound is currently: $(valueP[])"
+    # As well as any other C API
+    x_p = Vector{Cdouble}(undef, 2)
+    obj_p = Ref{Cdouble}()
+    ret = CPXcallbackgetincumbent(cb_data, x_p, 0, 1, obj_p)
+    if ret == 0
+        @info "Objective incumbent is: $(obj_p[])"
+        @info "Incumbent solution is: $(x_p)"
+        # Use CPLEX.column to map between variable references and the 1-based
+        # column.
+        x_col = CPLEX.column(cb_data, index(x))
+        @info "x = $(x_p[x_col])"
+    else
+        # Unable to query incumbent.
+    end
+
+    # Before querying `callback_value`, you must call:
+    CPLEX.load_callback_variable_primal(cb_data, context_id)
+    x_val = callback_value(cb_data, x)
+    y_val = callback_value(cb_data, y)
+    # You can submit solver-independent MathOptInterface attributes such as
+    # lazy constraints, user-cuts, and heuristic solutions.
+    if y_val - x_val > 1 + 1e-6
+        con = @build_constraint(y - x <= 1)
+        MOI.submit(model, MOI.LazyConstraint(cb_data), con)
+    elseif y_val + x_val > 3 + 1e-6
+        con = @build_constraint(y + x <= 3)
+        MOI.submit(model, MOI.LazyConstraint(cb_data), con)
+    end
+end
+MOI.set(model, CPLEX.CallbackFunction(), my_callback_function)
+optimize!(model)
+@test termination_status(model) == MOI.OPTIMAL
+@test primal_status(model) == MOI.FEASIBLE_POINT
+@test value(x) == 1
+@test value(y) == 2
+```

deps/build.jl

@@ -1,12 +1,12 @@
 using Libdl
 
-const depsfile = joinpath(dirname(@__FILE__), "deps.jl")
-if isfile(depsfile)
-    rm(depsfile)
+const _DEPS_FILE = joinpath(dirname(@__FILE__), "deps.jl")
+if isfile(_DEPS_FILE)
+    rm(_DEPS_FILE)
 end
 
 function write_depsfile(path)
-    open(depsfile, "w") do f
+    open(_DEPS_FILE, "w") do f
         println(f, "const libcplex = \"$(escape_string(path))\"")
     end
 end
@@ -32,7 +32,7 @@ function try_local_installation()
     # the path (directly the callable library or # the CPLEX executable) or from
     # an environment variable.
     cpxvers = [
-        "128", "1280", "129", "1290", "1210", "12100"
+        "1210", "12100"
     ]
     base_env = "CPLEX_STUDIO_BINARIES"
 
@@ -56,19 +56,47 @@ function try_local_installation()
     # Perform the actual search in the potential places.
     for l in libnames
         d = Libdl.dlopen_e(l)
-        d == C_NULL && continue
+        if d == C_NULL
+            continue
+        end
         write_depsfile(Libdl.dlpath(d))
         @info("Using CPLEX found in location `$(l)`")
         return
     end
-    error(
-        "Unable to locate CPLEX installation. Note this must be downloaded " *
-        "separately. See the CPLEX.jl README for further instructions."
-    )
+    error("""
+    Unable to locate CPLEX installation. Note this must be downloaded separately.
+
+    You should set the `CPLEX_STUDIO_BINARIES` environment variable to point to
+    the install location then try again. For example (updating the path to the
+    correct location if needed):
+
+    ```
+    # On Windows, this might be
+    ENV["CPLEX_STUDIO_BINARIES"] = "C:\\\\Program Files\\\\CPLEX_Studio1210\\\\cplex\\\\bin\\\\x86-64_win\\\\"
+    import Pkg
+    Pkg.add("CPLEX")
+    Pkg.build("CPLEX")
+
+    # On OSX, this might be
+    ENV["CPLEX_STUDIO_BINARIES"] = "/Applications/CPLEX_Studio1210/cplex/bin/x86-64_osx/"
+    import Pkg
+    Pkg.add("CPLEX")
+    Pkg.build("CPLEX")
+
+    # On Unix, this might be
+    ENV["CPLEX_STUDIO_BINARIES"] = "/opt/CPLEX_Studio1210/cplex/bin/x86-64_linux/"
+    import Pkg
+    Pkg.add("CPLEX")
+    Pkg.build("CPLEX")
+    ```
+
+    See the CPLEX.jl README at https://github.com/jump-dev/CPLEX.jl for further
+    instructions.
+    """)
 end
 
 function try_travis_installation()
-    url = ENV["SECRET_CPLEX_URL"]
+    url = ENV["SECRET_CPLEX_URL_12100"]
     local_filename = joinpath(@__DIR__, "libcplex.so")
     download(url, local_filename)
     write_depsfile(local_filename)

scripts/Project.toml

@@ -0,0 +1,5 @@
+[deps]
+Clang = "40e3b903-d033-50b4-a0cc-940c62c95e31"
+
+[compat]
+Clang = "0.12"

scripts/clang.jl

@@ -0,0 +1,77 @@
+# TODO(odow):
+#
+# This script can be used to build the C interface to CPLEX. However, it
+# requires you to manually do the following steps first:
+#
+# 1) Copy cplex.h from CPLEX into this /scripts directory
+# 2) Copy cpxconst.h from CPLEX into this /scripts directory
+#
+# You should also check for an updated version of Clang.jl, and update if one is
+# available.
+
+import Clang
+
+const LIBCPX_HEADERS = [
+    joinpath(@__DIR__, "cplex.h"),
+    joinpath(@__DIR__, "cpxconst.h"),
+]
+
+const GEN_DIR = joinpath(dirname(@__DIR__), "src", "gen")
+
+wc = Clang.init(
+    headers = LIBCPX_HEADERS,
+    output_file = joinpath(GEN_DIR, "libcpx_api.jl"),
+    common_file = joinpath(GEN_DIR, "libcpx_common.jl"),
+    clang_args = String[
+        "-I" * header for header in Clang.find_std_headers()
+    ],
+    header_wrapped = (root, current) -> root == current,
+    header_library = x -> "libcplex",
+    clang_diagnostics = true,
+)
+
+run(wc)
+
+function manual_corrections_common()
+    filename = joinpath(GEN_DIR, "libcpx_common.jl")
+    lines = readlines(filename; keep = true)
+    for (i, line) in enumerate(lines)
+        if occursin("CPXINT_MAX", line)
+            lines[i] = replace(line, "= INT_MAX" => "= $(typemax(Cint))")
+        elseif occursin("CPXINT_MIN", line)
+            lines[i] = replace(line, "= INT_MIN" => "= $(typemin(Cint))")
+        elseif occursin("= version", line)
+            lines[i] = replace(line, "= version" => "= nothing")
+        elseif occursin("= NAN", line)
+            lines[i] = replace(line, "= NAN" => "= NaN")
+        elseif occursin("# Skipping Typedef: CXType_FunctionProto ", line)
+            lines[i] = replace(
+                replace(
+                    line,
+                    "# Skipping Typedef: CXType_FunctionProto " => "const "
+                ),
+                '\n' => " = Ptr{Cvoid}\n"
+            )
+        end
+    end
+    open(filename, "w") do io
+        print.(Ref(io), lines)
+    end
+end
+manual_corrections_common()
+
+function manual_corrections_api()
+    filename = joinpath(GEN_DIR, "libcpx_api.jl")
+    lines = readlines(filename; keep = true)
+    for (i, line) in enumerate(lines)
+        if occursin("Cstring", line)
+            lines[i] = replace(line, "Cstring" => "Ptr{Cchar}")
+        end
+    end
+    open(filename, "w") do io
+        print.(Ref(io), lines)
+    end
+end
+manual_corrections_api()
+
+rm(joinpath(GEN_DIR, "LibTemplate.jl"))