API Reference

abstract type AbstractPredictor end

An abstract type representing different types of prediction models.

Methods

All subtypes must implement:

• add_predictor
• build_predictor

add_predictor(
    model::JuMP.AbstractModel,
    predictor::AbstractPredictor,
    x::Vector,
)::Vector

Return a Vector representing y such that y = predictor(x).

The element type of x is deliberately unspecified. The vector x may contain any mix of scalar constants, JuMP decision variables, and scalar JuMP functions like AffExpr, QuadExpr, or NonlinearExpr.

Example

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, x[1:2]);

julia> f = MathOptAI.Affine([2.0, 3.0])
Affine(A, b) [input: 2, output: 1]

julia> y, formulation = MathOptAI.add_predictor(model, f, x);

julia> y
1-element Vector{VariableRef}:
 moai_Affine[1]

julia> formulation
Affine(A, b) [input: 2, output: 1]
├ variables [1]
│ └ moai_Affine[1]
└ constraints [1]
  └ 2 x[1] + 3 x[2] - moai_Affine[1] = 0

build_predictor(extension; kwargs...)::AbstractPredictor

A uniform interface to convert various extension types to an AbstractPredictor.

See the various extension docstrings for details.

Affine(
    A::Matrix{T},
    b::Vector{T} = zeros(T, size(A, 1)),
) where {T} <: AbstractPredictor

An AbstractPredictor that represents the relationship:

\[y = A x + b\]

Example

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, 0 <= x[i in 1:2] <= i);

julia> f = MathOptAI.Affine([2.0 3.0], [4.0])
Affine(A, b) [input: 2, output: 1]

julia> y, formulation = MathOptAI.add_predictor(model, f, x);

julia> y
1-element Vector{VariableRef}:
 moai_Affine[1]

julia> formulation
Affine(A, b) [input: 2, output: 1]
├ variables [1]
│ └ moai_Affine[1]
└ constraints [3]
  ├ moai_Affine[1] ≥ 4
  ├ moai_Affine[1] ≤ 12
  └ 2 x[1] + 3 x[2] - moai_Affine[1] = -4

julia> y, formulation =
           MathOptAI.add_predictor(model, MathOptAI.ReducedSpace(f), x);

julia> y
1-element Vector{AffExpr}:
 2 x[1] + 3 x[2] + 4

julia> formulation
ReducedSpace(Affine(A, b) [input: 2, output: 1])
├ variables [0]
└ constraints [0]

BinaryDecisionTree{K,V}(
    feat_id::Int,
    feat_value::K,
    lhs::Union{V,BinaryDecisionTree{K,V}},
    rhs::Union{V,BinaryDecisionTree{K,V}},
)

An AbstractPredictor that represents a binary decision tree.

• If x[feat_id] <= feat_value, then return lhs
• If x[feat_id] > feat_value, then return rhs

Example

To represent the tree x[1] <= 0.0 ? -1 : (x[1] <= 1.0 ? 0 : 1), do:

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, x[1:1]);

julia> f = MathOptAI.BinaryDecisionTree{Float64,Int}(
           1,
           0.0,
           -1,
           MathOptAI.BinaryDecisionTree{Float64,Int}(1, 1.0, 0, 1),
       )
BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]

julia> y, formulation = MathOptAI.add_predictor(model, f, x);

julia> y
1-element Vector{VariableRef}:
 moai_BinaryDecisionTree_value

julia> formulation
BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]
├ variables [4]
│ ├ moai_BinaryDecisionTree_value
│ ├ moai_BinaryDecisionTree_z[1]
│ ├ moai_BinaryDecisionTree_z[2]
│ └ moai_BinaryDecisionTree_z[3]
└ constraints [7]
  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] = 1
  ├ moai_BinaryDecisionTree_z[1] --> {x[1] ≤ 0}
  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≥ 0}
  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≤ 1}
  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≥ 0}
  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≥ 1}
  └ moai_BinaryDecisionTree_z[1] - moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_value = 0

GrayBox(
    output_size::Function,
    callback::Function;
    has_hessian::Bool = false,
) <: AbstractPredictor

An AbstractPredictor that represents the relationship:

\[y = f(x)\]

as a user-defined nonlinear operator.

Arguments

• output_size(x::Vector):Int: given an input vector x, return the dimension of the output vector
• callback(x::Vector)::NamedTuple -> (;value, jacobian[, hessian]): given an input vector x, return a NamedTuple that computes the primal value and Jacobian of the output value with respect to the input. jacobian[j, i] is the partial derivative of value[j] with respect to x[i].
• has_hessian: if true, the callback additionally contains a field hessian, which is an N × N × M matrix, where hessian[i, j, k] is the partial derivative of value[k] with respect to x[i] and x[j].

Example

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, x[1:2]);

julia> f = MathOptAI.GrayBox(
           x -> 2,
           x -> (value = x.^2, jacobian = [2 * x[1] 0.0; 0.0 2 * x[2]]),
       );

julia> y, formulation = MathOptAI.add_predictor(model, f, x);

julia> y
2-element Vector{VariableRef}:
 moai_GrayBox[1]
 moai_GrayBox[2]

julia> formulation
GrayBox
├ variables [2]
│ ├ moai_GrayBox[1]
│ └ moai_GrayBox[2]
└ constraints [2]
  ├ op_##330(x[1], x[2]) - moai_GrayBox[1] = 0
  └ op_##331(x[1], x[2]) - moai_GrayBox[2] = 0

julia> y, formulation =
           MathOptAI.add_predictor(model, MathOptAI.ReducedSpace(f), x);

julia> y
2-element Vector{NonlinearExpr}:
 op_##332(x[1], x[2])
 op_##333(x[1], x[2])

julia> formulation
ReducedSpace(GrayBox)
├ variables [0]
└ constraints [0]

Pipeline(layers::Vector{AbstractPredictor}) <: AbstractPredictor

An AbstractPredictor that represents the relationship:

\[y = (l_1 \circ \ldots \circ l_N)(x)\]

where $l_i$ are a list of other AbstractPredictors.

Example

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, x[1:2]);

julia> f = MathOptAI.Pipeline(
           MathOptAI.Affine([1.0 2.0], [0.0]),
           MathOptAI.ReLUQuadratic(),
       )
Pipeline with layers:
 * Affine(A, b) [input: 2, output: 1]
 * ReLUQuadratic()

julia> y, formulation = MathOptAI.add_predictor(model, f, x);

julia> y
1-element Vector{VariableRef}:
 moai_ReLU[1]

julia> formulation
Affine(A, b) [input: 2, output: 1]
├ variables [1]
│ └ moai_Affine[1]
└ constraints [1]
  └ x[1] + 2 x[2] - moai_Affine[1] = 0
ReLUQuadratic()
├ variables [2]
│ ├ moai_ReLU[1]
│ └ moai_z[1]
└ constraints [4]
  ├ moai_ReLU[1] ≥ 0
  ├ moai_z[1] ≥ 0
  ├ moai_Affine[1] - moai_ReLU[1] + moai_z[1] = 0
  └ moai_ReLU[1]*moai_z[1] = 0

PytorchModel(filename::String)

A wrapper struct for loading a PyTorch model.

The only supported file extension is .pt, where the .pt file has been created using torch.save(model, filename).

import PythonCall

Example

julia> using MathOptAI

julia> using PythonCall  #  This line is important!

julia> predictor = PytorchModel("model.pt");

Quantile{D}(distribution::D, quantiles::Vector{Float64}) where {D}

An AbstractPredictor that represents the quantiles of distribution.

Example

julia> using JuMP, Distributions, MathOptAI

julia> model = Model();

julia> @variable(model, 1 <= x <= 2);

julia> predictor = MathOptAI.Quantile([0.1, 0.9]) do x
           return Distributions.Normal(x, 3 - x)
       end
Quantile(_, [0.1, 0.9])

julia> y, formulation = MathOptAI.add_predictor(model, predictor, [x]);

julia> y
2-element Vector{VariableRef}:
 moai_quantile[1]
 moai_quantile[2]

julia> formulation
Quantile(_, [0.1, 0.9])
├ variables [2]
│ ├ moai_quantile[1]
│ └ moai_quantile[2]
└ constraints [2]
  ├ moai_quantile[1] - op_quantile_0.1(x) = 0
  └ moai_quantile[2] - op_quantile_0.9(x) = 0

ReducedSpace(predictor::AbstractPredictor)

A wrapper type for other predictors that implement a reduced-space formulation.

Example

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, x[1:2]);

julia> predictor = MathOptAI.ReducedSpace(MathOptAI.ReLU());

julia> y, formulation = MathOptAI.add_predictor(model, predictor, x);

julia> y
2-element Vector{NonlinearExpr}:
 max(0.0, x[1])
 max(0.0, x[2])

ReLU() <: AbstractPredictor

An AbstractPredictor that represents the relationship:

\[y = \max\{0, x\}\]

as a non-smooth nonlinear constraint.

Example

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, -1 <= x[i in 1:2] <= i);

julia> f = MathOptAI.ReLU()
ReLU()

julia> y, formulation = MathOptAI.add_predictor(model, f, x);

julia> y
2-element Vector{VariableRef}:
 moai_ReLU[1]
 moai_ReLU[2]

julia> formulation
ReLU()
├ variables [2]
│ ├ moai_ReLU[1]
│ └ moai_ReLU[2]
└ constraints [6]
  ├ moai_ReLU[1] ≥ 0
  ├ moai_ReLU[1] ≤ 1
  ├ moai_ReLU[2] ≥ 0
  ├ moai_ReLU[2] ≤ 2
  ├ moai_ReLU[1] - max(0.0, x[1]) = 0
  └ moai_ReLU[2] - max(0.0, x[2]) = 0

julia> y, formulation =
           MathOptAI.add_predictor(model, MathOptAI.ReducedSpace(f), x);

julia> y
2-element Vector{NonlinearExpr}:
 max(0.0, x[1])
 max(0.0, x[2])

julia> formulation
ReducedSpace(ReLU())
├ variables [0]
└ constraints [0]

ReLUBigM(M::Float64) <: AbstractPredictor

An AbstractPredictor that represents the relationship:

\[y = \max\{0, x\}\]

via the big-M MIP reformulation:

\[\begin{aligned} y \ge 0 \\ y \ge x \\ y \le M z \\ y \le x + M(1 - z) \\ z \in\{0, 1\} \end{aligned}\]

Example

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, -3 <= x[i in 1:2] <= i);

julia> f = MathOptAI.ReLUBigM(100.0)
ReLUBigM(100.0)

julia> y, formulation = MathOptAI.add_predictor(model, f, x);

julia> y
2-element Vector{VariableRef}:
 moai_ReLU[1]
 moai_ReLU[2]

julia> formulation
ReLUBigM(100.0)
├ variables [4]
│ ├ moai_ReLU[1]
│ ├ moai_ReLU[2]
│ ├ moai_z[1]
│ └ moai_z[2]
└ constraints [12]
  ├ moai_ReLU[1] ≥ 0
  ├ moai_ReLU[1] ≤ 1
  ├ moai_ReLU[2] ≥ 0
  ├ moai_ReLU[2] ≤ 2
  ├ moai_z[1] binary
  ├ -x[1] + moai_ReLU[1] ≥ 0
  ├ moai_ReLU[1] - moai_z[1] ≤ 0
  ├ -x[1] + moai_ReLU[1] + 3 moai_z[1] ≤ 3
  ├ moai_z[2] binary
  ├ -x[2] + moai_ReLU[2] ≥ 0
  ├ moai_ReLU[2] - 2 moai_z[2] ≤ 0
  └ -x[2] + moai_ReLU[2] + 3 moai_z[2] ≤ 3

ReLUQuadratic() <: AbstractPredictor

An AbstractPredictor that represents the relationship:

\[y = \max\{0, x\}\]

by the reformulation:

\[\begin{aligned} x = y - z \\ y \cdot z = 0 \\ y, z \ge 0 \end{aligned}\]

Example

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, -1 <= x[i in 1:2] <= i);

julia> f = MathOptAI.ReLUQuadratic()
ReLUQuadratic()

julia> y, formulation = MathOptAI.add_predictor(model, f, x);

julia> y
2-element Vector{VariableRef}:
 moai_ReLU[1]
 moai_ReLU[2]

julia> formulation
ReLUQuadratic()
├ variables [4]
│ ├ moai_ReLU[1]
│ ├ moai_ReLU[2]
│ ├ moai_z[1]
│ └ moai_z[2]
└ constraints [12]
  ├ moai_ReLU[1] ≥ 0
  ├ moai_ReLU[1] ≤ 1
  ├ moai_ReLU[2] ≥ 0
  ├ moai_ReLU[2] ≤ 2
  ├ moai_z[1] ≥ 0
  ├ moai_z[1] ≤ 1
  ├ moai_z[2] ≥ 0
  ├ moai_z[2] ≤ 1
  ├ x[1] - moai_ReLU[1] + moai_z[1] = 0
  ├ x[2] - moai_ReLU[2] + moai_z[2] = 0
  ├ moai_ReLU[1]*moai_z[1] = 0
  └ moai_ReLU[2]*moai_z[2] = 0

ReLUSOS1() <: AbstractPredictor

An AbstractPredictor that represents the relationship:

\[y = \max\{0, x\}\]

by the reformulation:

\[\begin{aligned} x = y - z \\ [y, z] \in SOS1 \\ y, z \ge 0 \end{aligned}\]

Example

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, -1 <= x[i in 1:2] <= i);

julia> f = MathOptAI.ReLUSOS1()
ReLUSOS1()

julia> y, formulation = MathOptAI.add_predictor(model, f, x);

julia> y
2-element Vector{VariableRef}:
 moai_ReLU[1]
 moai_ReLU[2]

julia> formulation
ReLUSOS1()
├ variables [4]
│ ├ moai_ReLU[1]
│ ├ moai_ReLU[2]
│ ├ moai_z[1]
│ └ moai_z[2]
└ constraints [10]
  ├ moai_ReLU[1] ≥ 0
  ├ moai_ReLU[1] ≤ 1
  ├ moai_ReLU[2] ≥ 0
  ├ moai_ReLU[2] ≤ 2
  ├ moai_z[1] ≤ 1
  ├ moai_z[2] ≤ 1
  ├ x[1] - moai_ReLU[1] + moai_z[1] = 0
  ├ x[2] - moai_ReLU[2] + moai_z[2] = 0
  ├ [moai_ReLU[1], moai_z[1]] ∈ MathOptInterface.SOS1{Float64}([1.0, 2.0])
  └ [moai_ReLU[2], moai_z[2]] ∈ MathOptInterface.SOS1{Float64}([1.0, 2.0])

Scale(
    scale::Vector{T},
    bias::Vector{T},
) where {T} <: AbstractPredictor

An AbstractPredictor that represents the relationship:

\[y = Diag(scale)x + bias\]

Example

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, 0 <= x[i in 1:2] <= i);

julia> f = MathOptAI.Scale([2.0, 3.0], [4.0, 5.0])
Scale(scale, bias)

julia> y, formulation = MathOptAI.add_predictor(model, f, x);

julia> y
2-element Vector{VariableRef}:
 moai_Scale[1]
 moai_Scale[2]

julia> formulation
Scale(scale, bias)
├ variables [2]
│ ├ moai_Scale[1]
│ └ moai_Scale[2]
└ constraints [6]
  ├ moai_Scale[1] ≥ 4
  ├ moai_Scale[1] ≤ 6
  ├ moai_Scale[2] ≥ 5
  ├ moai_Scale[2] ≤ 11
  ├ 2 x[1] - moai_Scale[1] = -4
  └ 3 x[2] - moai_Scale[2] = -5

julia> y, formulation =
           MathOptAI.add_predictor(model, MathOptAI.ReducedSpace(f), x);

julia> y
2-element Vector{AffExpr}:
 2 x[1] + 4
 3 x[2] + 5

julia> formulation
ReducedSpace(Scale(scale, bias))
├ variables [0]
└ constraints [0]

Sigmoid() <: AbstractPredictor

An AbstractPredictor that represents the relationship:

\[y = \frac{1}{1 + e^{-x}}\]

as a smooth nonlinear constraint.

Example

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, -1 <= x[i in 1:2] <= i);

julia> f = MathOptAI.Sigmoid()
Sigmoid()

julia> y, formulation = MathOptAI.add_predictor(model, f, x);

julia> y
2-element Vector{VariableRef}:
 moai_Sigmoid[1]
 moai_Sigmoid[2]

julia> formulation
Sigmoid()
├ variables [2]
│ ├ moai_Sigmoid[1]
│ └ moai_Sigmoid[2]
└ constraints [6]
  ├ moai_Sigmoid[1] ≥ 0.2689414213699951
  ├ moai_Sigmoid[1] ≤ 0.7310585786300049
  ├ moai_Sigmoid[2] ≥ 0.2689414213699951
  ├ moai_Sigmoid[2] ≤ 0.8807970779778823
  ├ moai_Sigmoid[1] - (1.0 / (1.0 + exp(-x[1]))) = 0
  └ moai_Sigmoid[2] - (1.0 / (1.0 + exp(-x[2]))) = 0

julia> y, formulation =
           MathOptAI.add_predictor(model, MathOptAI.ReducedSpace(f), x);

julia> y
2-element Vector{NonlinearExpr}:
 1.0 / (1.0 + exp(-x[1]))
 1.0 / (1.0 + exp(-x[2]))

julia> formulation
ReducedSpace(Sigmoid())
├ variables [0]
└ constraints [0]

SoftMax() <: AbstractPredictor

An AbstractPredictor that represents the relationship:

\[y = \frac{e^{x}}{||e^{x}||_1}\]

as a smooth nonlinear constraint.

Example

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, x[1:2]);

julia> f = MathOptAI.SoftMax()
SoftMax()

julia> y, formulation = MathOptAI.add_predictor(model, f, x);

julia> y
2-element Vector{VariableRef}:
 moai_SoftMax[1]
 moai_SoftMax[2]

julia> formulation
SoftMax()
├ variables [3]
│ ├ moai_SoftMax_denom
│ ├ moai_SoftMax[1]
│ └ moai_SoftMax[2]
└ constraints [8]
  ├ moai_SoftMax[1] ≥ 0
  ├ moai_SoftMax[1] ≤ 1
  ├ moai_SoftMax[2] ≥ 0
  ├ moai_SoftMax[2] ≤ 1
  ├ moai_SoftMax_denom ≥ 0
  ├ moai_SoftMax_denom - (0.0 + exp(x[2]) + exp(x[1])) = 0
  ├ moai_SoftMax[1] - (exp(x[1]) / moai_SoftMax_denom) = 0
  └ moai_SoftMax[2] - (exp(x[2]) / moai_SoftMax_denom) = 0

julia> y, formulation =
           MathOptAI.add_predictor(model, MathOptAI.ReducedSpace(f), x);

julia> y
2-element Vector{NonlinearExpr}:
 exp(x[1]) / moai_SoftMax_denom
 exp(x[2]) / moai_SoftMax_denom

julia> formulation
ReducedSpace(SoftMax())
├ variables [1]
│ └ moai_SoftMax_denom
└ constraints [2]
  ├ moai_SoftMax_denom ≥ 0
  └ moai_SoftMax_denom - (0.0 + exp(x[2]) + exp(x[1])) = 0

SoftPlus(; beta = 1.0) <: AbstractPredictor

An AbstractPredictor that represents the relationship:

\[y = \frac{1}{\beta} \log(1 + e^{\beta x})\]

as a smooth nonlinear constraint.

Example

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, -1 <= x[i in 1:2] <= i);

julia> f = MathOptAI.SoftPlus(; beta = 2.0)
SoftPlus(2.0)

julia> y, formulation = MathOptAI.add_predictor(model, f, x);

julia> y
2-element Vector{VariableRef}:
 moai_SoftPlus[1]
 moai_SoftPlus[2]

julia> formulation
SoftPlus(2.0)
├ variables [2]
│ ├ moai_SoftPlus[1]
│ └ moai_SoftPlus[2]
└ constraints [6]
  ├ moai_SoftPlus[1] ≥ 0.0634640055214863
  ├ moai_SoftPlus[1] ≤ 1.0634640055214863
  ├ moai_SoftPlus[2] ≥ 0.0634640055214863
  ├ moai_SoftPlus[2] ≤ 2.0090749639589047
  ├ moai_SoftPlus[1] - (log(1.0 + exp(2 x[1])) / 2.0) = 0
  └ moai_SoftPlus[2] - (log(1.0 + exp(2 x[2])) / 2.0) = 0

julia> y, formulation =
           MathOptAI.add_predictor(model, MathOptAI.ReducedSpace(f), x);

julia> y
2-element Vector{NonlinearExpr}:
 log(1.0 + exp(2 x[1])) / 2.0
 log(1.0 + exp(2 x[2])) / 2.0

julia> formulation
ReducedSpace(SoftPlus(2.0))
├ variables [0]
└ constraints [0]

Tanh() <: AbstractPredictor

An AbstractPredictor that represents the relationship:

\[y = \tanh(x)\]

as a smooth nonlinear constraint.

Example

julia> using JuMP, MathOptAI

julia> model = Model();

julia> @variable(model, -1 <= x[i in 1:2] <= i);

julia> f = MathOptAI.Tanh()
Tanh()

julia> y, formulation = MathOptAI.add_predictor(model, f, x);

julia> y
2-element Vector{VariableRef}:
 moai_Tanh[1]
 moai_Tanh[2]

julia> formulation
Tanh()
├ variables [2]
│ ├ moai_Tanh[1]
│ └ moai_Tanh[2]
└ constraints [6]
  ├ moai_Tanh[1] ≥ -0.7615941559557649
  ├ moai_Tanh[1] ≤ 0.7615941559557649
  ├ moai_Tanh[2] ≥ -0.7615941559557649
  ├ moai_Tanh[2] ≤ 0.9640275800758169
  ├ moai_Tanh[1] - tanh(x[1]) = 0
  └ moai_Tanh[2] - tanh(x[2]) = 0

julia> y, formulation =
           MathOptAI.add_predictor(model, MathOptAI.ReducedSpace(f), x);

julia> y
2-element Vector{NonlinearExpr}:
 tanh(x[1])
 tanh(x[2])

julia> formulation
ReducedSpace(Tanh())
├ variables [0]
└ constraints [0]

abstract type AbstractFormulation end

An abstract type representing different formulations.

struct Formulation{P<:AbstractPredictor} <: AbstractFormulation
    predictor::P
    variables::Vector{Any}
    constraints::Vector{Any}
end

Fields

• predictor: the predictor object used to build the formulation
• variables: a vector of new decision variables added to the model
• constraints: a vector of new constraints added to the model

Check the docstring of the predictor for an explanation of the formulation and the order of the elements in .variables and .constraints.

struct PipelineFormulation{P<:AbstractPredictor} <: AbstractFormulation
    predictor::P
    layers::Vector{Any}
end

Fields

• predictor: the predictor object used to build the formulation
• layers: the formulation associated with each of the layers in the pipeline

MathOptAI.add_predictor(
    model::JuMP.AbstractModel,
    predictor::MathOptAI.Quantile{<:AbstractGPs.PosteriorGP},
    x::Vector,
)

Add the quantiles of a trained Gaussian Process from AbstractGPs.jl to model.

Example

julia> using JuMP, MathOptAI, AbstractGPs

julia> x_data = 2π .* (0.0:0.1:1.0);

julia> y_data = sin.(x_data);

julia> fx = AbstractGPs.GP(AbstractGPs.Matern32Kernel())(x_data, 0.1);

julia> p_fx = AbstractGPs.posterior(fx, y_data);

julia> model = Model();

julia> @variable(model, 1 <= x[1:1] <= 6, start = 3);

julia> predictor = MathOptAI.Quantile(p_fx, [0.1, 0.9]);

julia> y, _ = MathOptAI.add_predictor(model, predictor, x);

julia> y
2-element Vector{VariableRef}:
 moai_quantile[1]
 moai_quantile[2]

julia> @objective(model, Max, y[2] - y[1])
moai_quantile[2] - moai_quantile[1]

MathOptAI.add_predictor(
    model::JuMP.AbstractModel,
    predictor::Union{DecisionTree.Root,DecisionTree.DecisionTreeClassifier},
    x::Vector,
)

Add a binary decision tree from DecisionTree.jl to model.

Example

julia> using JuMP, MathOptAI, DecisionTree

julia> truth(x::Vector) = x[1] <= 0.5 ? -2 : (x[2] <= 0.3 ? 3 : 4)
truth (generic function with 1 method)

julia> features = abs.(sin.((1:10) .* (3:4)'));

julia> size(features)
(10, 2)

julia> labels = truth.(Vector.(eachrow(features)));

julia> predictor = DecisionTree.build_tree(labels, features)
Decision Tree
Leaves: 3
Depth:  2

julia> model = Model();

julia> @variable(model, 0 <= x[1:2] <= 1);

julia> y, _ = MathOptAI.add_predictor(model, predictor, x);

julia> y
1-element Vector{VariableRef}:
 moai_BinaryDecisionTree_value

MathOptAI.build_predictor(predictor::DecisionTree.Root)

Convert a binary decision tree from DecisionTree.jl to a BinaryDecisionTree.

Example

julia> using MathOptAI, DecisionTree

julia> truth(x::Vector) = x[1] <= 0.5 ? -2 : (x[2] <= 0.3 ? 3 : 4)
truth (generic function with 1 method)

julia> features = abs.(sin.((1:10) .* (3:4)'));

julia> size(features)
(10, 2)

julia> labels = truth.(Vector.(eachrow(features)));

julia> tree = DecisionTree.build_tree(labels, features)
Decision Tree
Leaves: 3
Depth:  2

julia> predictor = MathOptAI.build_predictor(tree)
BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]

MathOptAI.add_predictor(
    model::JuMP.AbstractModel,
    predictor::Flux.Chain,
    x::Vector;
    config::Dict = Dict{Any,Any}(),
    reduced_space::Bool = false,
    gray_box::Bool = false,
)

Add a trained neural network from Flux.jl to model.

Supported layers

• Flux.Dense
• Flux.Scale
• Flux.softmax

Supported activation functions

• Flux.relu
• Flux.sigmoid
• Flux.softplus
• Flux.tanh

Keyword arguments

• config: a dictionary that maps supported Flux activation functions to AbstractPredictors that control how the activation functions are reformulated. For example, Flux.sigmoid => MathOptAI.Sigmoid() or Flux.relu => MathOptAI.QuadraticReLU().
• gray_box: if true, the neural network is added as a user-defined nonlinear operator, with gradients provided by Flux.withjacobian.

Example

julia> using JuMP, Flux, MathOptAI

julia> chain = Flux.Chain(Flux.Dense(1 => 16, Flux.relu), Flux.Dense(16 => 1));

julia> model = Model();

julia> @variable(model, x[1:1]);

julia> y, _ = MathOptAI.add_predictor(
           model,
           chain,
           x;
           config = Dict(Flux.relu => MathOptAI.ReLU()),
       );

julia> y
1-element Vector{VariableRef}:
 moai_Affine[1]

MathOptAI.build_predictor(
    predictor::Flux.Chain;
    config::Dict = Dict{Any,Any}(),
    gray_box::Bool = false,
    gray_box_hessian::Bool = false,
)

Convert a trained neural network from Flux.jl to a Pipeline.

Supported layers

• Flux.Dense
• Flux.Scale
• Flux.softmax

Supported activation functions

• Flux.relu
• Flux.sigmoid
• Flux.softplus
• Flux.tanh

Keyword arguments

• config: a dictionary that maps supported Flux activation functions to AbstractPredictors that control how the activation functions are reformulated. For example, Flux.sigmoid => MathOptAI.Sigmoid() or Flux.relu => MathOptAI.QuadraticReLU().
• gray_box: if true, the neural network is added as a user-defined nonlinear operator, with gradients provided by Flux.withjacobian.
• gray_box_hessian: if true, the gray box additionally computes the Hessian of the output using Flux.hessian.

Example

julia> using Flux, MathOptAI

julia> chain = Flux.Chain(Flux.Dense(1 => 16, Flux.relu), Flux.Dense(16 => 1));

julia> MathOptAI.build_predictor(
           chain;
           config = Dict(Flux.relu => MathOptAI.ReLU()),
       )
Pipeline with layers:
 * Affine(A, b) [input: 1, output: 16]
 * ReLU()
 * Affine(A, b) [input: 16, output: 1]

julia> MathOptAI.build_predictor(
           chain;
           config = Dict(Flux.relu => MathOptAI.ReLUQuadratic()),
       )
Pipeline with layers:
 * Affine(A, b) [input: 1, output: 16]
 * ReLUQuadratic()
 * Affine(A, b) [input: 16, output: 1]

MathOptAI.add_predictor(
    model::JuMP.AbstractModel,
    predictor::GLM.GeneralizedLinearModel{
        GLM.GlmResp{Vector{Float64},GLM.Bernoulli{Float64},GLM.LogitLink},
    },
    x::Vector;
    sigmoid::AbstractPredictor = MathOptAI.Sigmoid(),
    reduced_space::Bool = false,
)

Add a trained logistic regression model from GLM.jl to model.

Keyword arguments

• sigmoid: the predictor to use for the sigmoid layer.

Example

julia> using GLM, JuMP, MathOptAI

julia> X, Y = rand(10, 2), rand(Bool, 10);

julia> predictor = GLM.glm(X, Y, GLM.Bernoulli());

julia> model = Model();

julia> @variable(model, x[1:2]);

julia> y, _ = MathOptAI.add_predictor(
           model,
           predictor,
           x;
           sigmoid = MathOptAI.Sigmoid(),
       );

julia> y
1-element Vector{VariableRef}:
 moai_Sigmoid[1]

MathOptAI.add_predictor(
    model::JuMP.AbstractModel,
    predictor::GLM.LinearModel,
    x::Vector;
    reduced_space::Bool = false,
)

Add a trained linear model from GLM.jl to model.

Example

julia> using GLM, JuMP, MathOptAI

julia> X, Y = rand(10, 2), rand(10);

julia> predictor = GLM.lm(X, Y);

julia> model = Model();

julia> @variable(model, x[1:2]);

julia> y, _ = MathOptAI.add_predictor(model, predictor, x);

julia> y
1-element Vector{VariableRef}:
 moai_Affine[1]

MathOptAI.build_predictor(
    predictor::GLM.GeneralizedLinearModel{
        GLM.GlmResp{Vector{Float64},GLM.Bernoulli{Float64},GLM.LogitLink},
    };
    sigmoid::MathOptAI.AbstractPredictor = MathOptAI.Sigmoid(),
)

Convert a trained logistic model from GLM.jl to a Pipeline layer.

Keyword arguments

• sigmoid: the predictor to use for the sigmoid layer.

Example

julia> using GLM, MathOptAI

julia> X, Y = rand(10, 2), rand(Bool, 10);

julia> model = GLM.glm(X, Y, GLM.Bernoulli());

julia> predictor = MathOptAI.build_predictor(model)
Pipeline with layers:
 * Affine(A, b) [input: 2, output: 1]
 * Sigmoid()

MathOptAI.build_predictor(predictor::GLM.LinearModel)

Convert a trained linear model from GLM.jl to an Affine layer.

Example

julia> using GLM, MathOptAI

julia> X, Y = rand(10, 2), rand(10);

julia> model = GLM.lm(X, Y);

julia> predictor = MathOptAI.build_predictor(model)
Affine(A, b) [input: 2, output: 1]

MathOptAI.add_predictor(
    model::JuMP.AbstractModel,
    predictor::Tuple{<:Lux.Chain,<:NamedTuple,<:NamedTuple},
    x::Vector;
    config::Dict = Dict{Any,Any}(),
    reduced_space::Bool = false,
)

Add a trained neural network from Lux.jl to model.

Supported layers

• Lux.Dense
• Lux.Scale

Supported activation functions

• Lux.relu
• Lux.sigmoid
• Lux.softplus
• Lux.softmax
• Lux.tanh

Keyword arguments

• config: a dictionary that maps supported Lux activation functions to AbstractPredictors that control how the activation functions are reformulated. For example, Lux.sigmoid => MathOptAI.Sigmoid() or Lux.relu => MathOptAI.QuadraticReLU().

Example

julia> using JuMP, Lux, MathOptAI, Random

julia> rng = Random.MersenneTwister();

julia> chain = Lux.Chain(Lux.Dense(1 => 16, Lux.relu), Lux.Dense(16 => 1))
Chain(
    layer_1 = Dense(1 => 16, relu),     # 32 parameters
    layer_2 = Dense(16 => 1),           # 17 parameters
)         # Total: 49 parameters,
          #        plus 0 states.

julia> parameters, state = Lux.setup(rng, chain);

julia> predictor = (chain, parameters, state);

julia> model = Model();

julia> @variable(model, x[1:1]);

julia> y, _ = MathOptAI.add_predictor(
           model,
           predictor,
           x;
           config = Dict(Lux.relu => MathOptAI.ReLU()),
       );

julia> y
1-element Vector{VariableRef}:
 moai_Affine[1]

MathOptAI.build_predictor(
    predictor::Tuple{<:Lux.Chain,<:NamedTuple,<:NamedTuple};
    config::Dict = Dict{Any,Any}(),
)

Convert a trained neural network from Lux.jl to a Pipeline.

Supported layers

• Lux.Dense
• Lux.Scale

Supported activation functions

• Lux.relu
• Lux.sigmoid
• Lux.softplus
• Lux.softmax
• Lux.tanh

Keyword arguments

• config: a dictionary that maps supported Lux activation functions to AbstractPredictors that control how the activation functions are reformulated. For example, Lux.sigmoid => MathOptAI.Sigmoid() or Lux.relu => MathOptAI.QuadraticReLU().

Example

julia> using Lux, MathOptAI, Random

julia> rng = Random.MersenneTwister();

julia> chain = Lux.Chain(Lux.Dense(1 => 16, Lux.relu), Lux.Dense(16 => 1))
Chain(
    layer_1 = Dense(1 => 16, relu),     # 32 parameters
    layer_2 = Dense(16 => 1),           # 17 parameters
)         # Total: 49 parameters,
          #        plus 0 states.

julia> parameters, state = Lux.setup(rng, chain);

julia> predictor = MathOptAI.build_predictor(
           (chain, parameters, state);
           config = Dict(Lux.relu => MathOptAI.ReLU()),
       )
Pipeline with layers:
 * Affine(A, b) [input: 1, output: 16]
 * ReLU()
 * Affine(A, b) [input: 16, output: 1]

julia> MathOptAI.build_predictor(
           (chain, parameters, state);
           config = Dict(Lux.relu => MathOptAI.ReLUQuadratic()),
       )
Pipeline with layers:
 * Affine(A, b) [input: 1, output: 16]
 * ReLUQuadratic()
 * Affine(A, b) [input: 16, output: 1]

MathOptAI.add_predictor(
    model::JuMP.AbstractModel,
    predictor::MathOptAI.PytorchModel,
    x::Vector;
    config::Dict = Dict{Any,Any}(),
    reduced_space::Bool = false,
    gray_box::Bool = false,
    gray_box_hessian::Bool = false,
    gray_box_device::String = "cpu",
)

Add a trained neural network from PyTorch via PythonCall.jl to model.

Supported layers

• nn.Linear
• nn.ReLU
• nn.Sequential
• nn.Sigmoid
• nn.Softplus
• nn.Tanh

Keyword arguments

• config: a dictionary that maps Symbols to AbstractPredictors that control how the activation functions are reformulated. For example, :Sigmoid => MathOptAI.Sigmoid() or :ReLU => MathOptAI.QuadraticReLU(). The supported Symbols are :ReLU, :Sigmoid, :SoftPlus, and :Tanh.
• gray_box: if true, the neural network is added as a user-defined nonlinear operator, with gradients provided by torch.func.jacrev.
• gray_box_hessian: if true, the gray box additionally computes the Hessian of the output using torch.func.hessian.
• gray_box_device: device used to construct PyTorch tensors, e.g. "cuda" to run on an Nvidia GPU.

MathOptAI.build_predictor(
    predictor::MathOptAI.PytorchModel;
    config::Dict = Dict{Any,Any}(),
    gray_box::Bool = false,
    gray_box_hessian::Bool = false,
    gray_box_device::String = "cpu",
)

Convert a trained neural network from PyTorch via PythonCall.jl to a Pipeline.

Supported layers

• nn.Linear
• nn.ReLU
• nn.Sequential
• nn.Sigmoid
• nn.Softplus
• nn.Tanh

Keyword arguments

• config: a dictionary that maps Symbols to AbstractPredictors that control how the activation functions are reformulated. For example, :Sigmoid => MathOptAI.Sigmoid() or :ReLU => MathOptAI.QuadraticReLU(). The supported Symbols are :ReLU, :Sigmoid, :SoftPlus, and :Tanh.
• gray_box: if true, the neural network is added as a user-defined nonlinear operator, with gradients provided by torch.func.jacrev.
• gray_box_hessian: if true, the gray box additionally computes the Hessian of the output using torch.func.hessian.
• gray_box_device: device used to construct PyTorch tensors, e.g. "cuda" to run on an Nvidia GPU.

MathOptAI.add_predictor(
    model::JuMP.AbstractModel,
    predictor::StatsModels.TableRegressionModel,
    x::DataFrames.DataFrame;
    kwargs...,
)

Add a trained regression model from StatsModels.jl to model, using the DataFrame x as input.

In most cases, predictor should be a GLM.jl predictor supported by MathOptAI, but trained using @formula and a DataFrame instead of the raw matrix input.

In general, x may have some columns that are constant (Float64) and some columns that are JuMP decision variables.

Keyword arguments

All keyword arguments are passed to the corresponding add_predictor of the GLM extension.

Example

julia> using DataFrames, GLM, JuMP, MathOptAI

julia> train_df = DataFrames.DataFrame(x1 = rand(10), x2 = rand(10));

julia> train_df.y = 1.0 .* train_df.x1 + 2.0 .* train_df.x2 .+ rand(10);

julia> predictor = GLM.lm(GLM.@formula(y ~ x1 + x2), train_df);

julia> model = Model();

julia> test_df = DataFrames.DataFrame(
           x1 = rand(6),
           x2 = @variable(model, [1:6]),
       );

julia> test_df.y, _ = MathOptAI.add_predictor(model, predictor, test_df);

julia> test_df.y
6-element Vector{VariableRef}:
 moai_Affine[1]
 moai_Affine[1]
 moai_Affine[1]
 moai_Affine[1]
 moai_Affine[1]
 moai_Affine[1]