DecisionTree.jl

DecisionTree.jl is a library for fitting decision trees in Julia.

Binary decision tree regression

Here is an example:

julia> using JuMP, MathOptAI, DecisionTree
julia> truth(x::Vector) = x[1] <= 0.5 ? -2 : (x[2] <= 0.3 ? 3 : 4)truth (generic function with 1 method)
julia> features = abs.(sin.((1:10) .* (3:4)'));
julia> size(features)(10, 2)
julia> labels = truth.(Vector.(eachrow(features)));
julia> predictor = DecisionTree.build_tree(labels, features)Decision Tree
Leaves: 3
Depth:  2
julia> model = Model();
julia> @variable(model, 0 <= x[1:2] <= 1);
julia> y, formulation = MathOptAI.add_predictor(model, predictor, x);
julia> y1-element Vector{JuMP.VariableRef}:
 moai_BinaryDecisionTree_value
julia> formulationBinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]
├ variables [4]
│ ├ moai_BinaryDecisionTree_value
│ ├ moai_BinaryDecisionTree_z[1]
│ ├ moai_BinaryDecisionTree_z[2]
│ └ moai_BinaryDecisionTree_z[3]
└ constraints [7]
  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] = 1
  ├ moai_BinaryDecisionTree_z[1] --> {x[1] ≤ 0.4743457016210958}
  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≥ 0.4743457016210958}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.41966499895337794}
  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≥ 0.4743457016210958}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.41966499895337794}
  └ 2 moai_BinaryDecisionTree_z[1] - 3 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_value = 0

Random forest regression

julia> using JuMP, MathOptAI, DecisionTree
julia> truth(x::Vector) = x[1] <= 0.5 ? -2 : (x[2] <= 0.3 ? 3 : 4)truth (generic function with 1 method)
julia> features = abs.(sin.((1:10) .* (3:4)'));
julia> size(features)(10, 2)
julia> labels = truth.(Vector.(eachrow(features)));
julia> predictor = DecisionTree.build_forest(labels, features)Ensemble of Decision Trees
Trees:      10
Avg Leaves: 3.8
Avg Depth:  2.8
julia> model = Model();
julia> @variable(model, 0 <= x[1:2] <= 1);
julia> y, formulation = MathOptAI.add_predictor(model, predictor, x);
julia> y1-element Vector{JuMP.VariableRef}:
 moai_AffineCombination[1]
julia> formulationAffineCombination
├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=4, depth=2]
├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=5, depth=2]
├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]
├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]
├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]
├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=4, depth=2]
├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]
├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=4, depth=2]
├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=5, depth=2]
├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=4, depth=2]
└ 1.0 * [0.0]
├ variables [1]
│ └ moai_AffineCombination[1]
└ constraints [1]
  └ 0.1 moai_BinaryDecisionTree_value + 0.1 moai_BinaryDecisionTree_value + 0.1 moai_BinaryDecisionTree_value + 0.1 moai_BinaryDecisionTree_value + 0.1 moai_BinaryDecisionTree_value + 0.1 moai_BinaryDecisionTree_value + 0.1 moai_BinaryDecisionTree_value + 0.1 moai_BinaryDecisionTree_value + 0.1 moai_BinaryDecisionTree_value + 0.1 moai_BinaryDecisionTree_value - moai_AffineCombination[1] = 0
BinaryDecisionTree{Float64,Int64} [leaves=4, depth=2]
├ variables [5]
│ ├ moai_BinaryDecisionTree_value
│ ├ moai_BinaryDecisionTree_z[1]
│ ├ moai_BinaryDecisionTree_z[2]
│ ├ moai_BinaryDecisionTree_z[3]
│ └ moai_BinaryDecisionTree_z[4]
└ constraints [11]
  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_z[4] = 1
  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.5439997996210628}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.5439997996210628}
  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≤ 0.5233491850332456}
  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.5439997996210628}
  ├ moai_BinaryDecisionTree_z[4] --> {x[1] ≥ 0.5233491850332456}
  └ -3 moai_BinaryDecisionTree_z[1] + 2 moai_BinaryDecisionTree_z[2] + 2 moai_BinaryDecisionTree_z[3] - 4 moai_BinaryDecisionTree_z[4] + moai_BinaryDecisionTree_value = 0
BinaryDecisionTree{Float64,Int64} [leaves=5, depth=2]
├ variables [6]
│ ├ moai_BinaryDecisionTree_value
│ ├ moai_BinaryDecisionTree_z[1]
│ ├ moai_BinaryDecisionTree_z[2]
│ ├ moai_BinaryDecisionTree_z[3]
│ ├ moai_BinaryDecisionTree_z[4]
│ └ moai_BinaryDecisionTree_z[5]
└ constraints [16]
  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_z[4] + moai_BinaryDecisionTree_z[5] = 1
  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.7210756400035294}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.7210756400035294}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≤ 0.9511517486755048}
  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.7210756400035294}
  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.9511517486755048}
  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≤ 0.9905685500332488}
  ├ moai_BinaryDecisionTree_z[5] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[5] --> {x[2] ≥ 0.7210756400035294}
  ├ moai_BinaryDecisionTree_z[5] --> {x[2] ≥ 0.9511517486755048}
  ├ moai_BinaryDecisionTree_z[5] --> {x[2] ≥ 0.9905685500332488}
  └ -3 moai_BinaryDecisionTree_z[1] + 2 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + 2 moai_BinaryDecisionTree_z[4] - 4 moai_BinaryDecisionTree_z[5] + moai_BinaryDecisionTree_value = 0
BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]
├ variables [4]
│ ├ moai_BinaryDecisionTree_value
│ ├ moai_BinaryDecisionTree_z[1]
│ ├ moai_BinaryDecisionTree_z[2]
│ └ moai_BinaryDecisionTree_z[3]
└ constraints [7]
  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] = 1
  ├ moai_BinaryDecisionTree_z[1] --> {x[1] ≤ 0.4743457016210958}
  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≥ 0.4743457016210958}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.6398410850540905}
  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≥ 0.4743457016210958}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.6398410850540905}
  └ 2 moai_BinaryDecisionTree_z[1] - 3 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_value = 0
BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]
├ variables [4]
│ ├ moai_BinaryDecisionTree_value
│ ├ moai_BinaryDecisionTree_z[1]
│ ├ moai_BinaryDecisionTree_z[2]
│ └ moai_BinaryDecisionTree_z[3]
└ constraints [7]
  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] = 1
  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.5439997996210628}
  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.5439997996210628}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.5439997996210628}
  └ -3 moai_BinaryDecisionTree_z[1] + 2 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_value = 0
BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]
├ variables [4]
│ ├ moai_BinaryDecisionTree_value
│ ├ moai_BinaryDecisionTree_z[1]
│ ├ moai_BinaryDecisionTree_z[2]
│ └ moai_BinaryDecisionTree_z[3]
└ constraints [7]
  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] = 1
  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.6408430392398918}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.6408430392398918}
  └ -3 moai_BinaryDecisionTree_z[1] + 2 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_value = 0
BinaryDecisionTree{Float64,Int64} [leaves=4, depth=2]
├ variables [5]
│ ├ moai_BinaryDecisionTree_value
│ ├ moai_BinaryDecisionTree_z[1]
│ ├ moai_BinaryDecisionTree_z[2]
│ ├ moai_BinaryDecisionTree_z[3]
│ └ moai_BinaryDecisionTree_z[4]
└ constraints [11]
  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_z[4] = 1
  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.6408430392398918}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.6408430392398918}
  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≤ 0.46485166917802134}
  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.6408430392398918}
  ├ moai_BinaryDecisionTree_z[4] --> {x[1] ≥ 0.46485166917802134}
  └ -3 moai_BinaryDecisionTree_z[1] + 2 moai_BinaryDecisionTree_z[2] + 2 moai_BinaryDecisionTree_z[3] - 4 moai_BinaryDecisionTree_z[4] + moai_BinaryDecisionTree_value = 0
BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]
├ variables [4]
│ ├ moai_BinaryDecisionTree_value
│ ├ moai_BinaryDecisionTree_z[1]
│ ├ moai_BinaryDecisionTree_z[2]
│ └ moai_BinaryDecisionTree_z[3]
└ constraints [7]
  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] = 1
  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.5439997996210628}
  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.5439997996210628}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.5439997996210628}
  └ -3 moai_BinaryDecisionTree_z[1] + 2 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_value = 0
BinaryDecisionTree{Float64,Int64} [leaves=4, depth=2]
├ variables [5]
│ ├ moai_BinaryDecisionTree_value
│ ├ moai_BinaryDecisionTree_z[1]
│ ├ moai_BinaryDecisionTree_z[2]
│ ├ moai_BinaryDecisionTree_z[3]
│ └ moai_BinaryDecisionTree_z[4]
└ constraints [11]
  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_z[4] = 1
  ├ moai_BinaryDecisionTree_z[1] --> {x[1] ≤ 0.4743457016210958}
  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≥ 0.4743457016210958}
  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≤ 0.8965157834702795}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.6004242836963465}
  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≥ 0.4743457016210958}
  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≤ 0.8965157834702795}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.6004242836963465}
  ├ moai_BinaryDecisionTree_z[4] --> {x[1] ≥ 0.4743457016210958}
  ├ moai_BinaryDecisionTree_z[4] --> {x[1] ≥ 0.8965157834702795}
  └ 2 moai_BinaryDecisionTree_z[1] - 3 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] - 4 moai_BinaryDecisionTree_z[4] + moai_BinaryDecisionTree_value = 0
BinaryDecisionTree{Float64,Int64} [leaves=5, depth=2]
├ variables [6]
│ ├ moai_BinaryDecisionTree_value
│ ├ moai_BinaryDecisionTree_z[1]
│ ├ moai_BinaryDecisionTree_z[2]
│ ├ moai_BinaryDecisionTree_z[3]
│ ├ moai_BinaryDecisionTree_z[4]
│ └ moai_BinaryDecisionTree_z[5]
└ constraints [16]
  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_z[4] + moai_BinaryDecisionTree_z[5] = 1
  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.5439997996210628}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.5439997996210628}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≤ 0.831190428657276}
  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≤ 0.6541145882748094}
  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.5439997996210628}
  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≤ 0.831190428657276}
  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.6541145882748094}
  ├ moai_BinaryDecisionTree_z[5] --> {x[2] ≥ 0.4122381173327501}
  ├ moai_BinaryDecisionTree_z[5] --> {x[2] ≥ 0.5439997996210628}
  ├ moai_BinaryDecisionTree_z[5] --> {x[2] ≥ 0.831190428657276}
  └ -3 moai_BinaryDecisionTree_z[1] + 2 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + 2 moai_BinaryDecisionTree_z[4] - 4 moai_BinaryDecisionTree_z[5] + moai_BinaryDecisionTree_value = 0
BinaryDecisionTree{Float64,Int64} [leaves=4, depth=2]
├ variables [5]
│ ├ moai_BinaryDecisionTree_value
│ ├ moai_BinaryDecisionTree_z[1]
│ ├ moai_BinaryDecisionTree_z[2]
│ ├ moai_BinaryDecisionTree_z[3]
│ └ moai_BinaryDecisionTree_z[4]
└ constraints [11]
  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_z[4] = 1
  ├ moai_BinaryDecisionTree_z[1] --> {x[1] ≤ 0.39570392410849203}
  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≥ 0.39570392410849203}
  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≤ 0.87111700027134}
  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≤ 0.7434717393465864}
  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≥ 0.39570392410849203}
  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≤ 0.87111700027134}
  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≥ 0.7434717393465864}
  ├ moai_BinaryDecisionTree_z[4] --> {x[1] ≥ 0.39570392410849203}
  ├ moai_BinaryDecisionTree_z[4] --> {x[1] ≥ 0.87111700027134}
  └ 2 moai_BinaryDecisionTree_z[1] - 4 moai_BinaryDecisionTree_z[2] - 3 moai_BinaryDecisionTree_z[3] - 4 moai_BinaryDecisionTree_z[4] + moai_BinaryDecisionTree_value = 0