DecisionTree.jl

DecisionTree.jl is a library for fitting decision trees in Julia.

Binary decision tree regression

Here is an example:

julia> using JuMP, MathOptAI, DecisionTreejulia> truth(x::Vector) = x[1] <= 0.5 ? -2 : (x[2] <= 0.3 ? 3 : 4)truth (generic function with 1 method)julia> features = abs.(sin.((1:10) .* (3:4)'));julia> size(features)(10, 2)julia> labels = truth.(Vector.(eachrow(features)));julia> predictor = DecisionTree.build_tree(labels, features)Decision TreeLeaves: 3Depth:  2julia> model = Model();julia> @variable(model, 0 <= x[1:2] <= 1);julia> y, formulation = MathOptAI.add_predictor(model, predictor, x);julia> y1-element Vector{JuMP.VariableRef}: moai_BinaryDecisionTree_value[1]julia> formulationBinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]├ variables [4]│ ├ moai_BinaryDecisionTree_value[1]│ ├ moai_BinaryDecisionTree_z[1]│ ├ moai_BinaryDecisionTree_z[2]│ └ moai_BinaryDecisionTree_z[3]└ constraints [7]  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] = 1  ├ moai_BinaryDecisionTree_z[1] --> {x[1] ≤ 0.4743447016210958}  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≥ 0.4743457016210958}  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.41966399895337797}  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≥ 0.4743457016210958}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.41966499895337794}  └ 2 moai_BinaryDecisionTree_z[1] - 3 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_value[1] = 0

Random forest regression

julia> using JuMP, MathOptAI, DecisionTreejulia> truth(x::Vector) = x[1] <= 0.5 ? -2 : (x[2] <= 0.3 ? 3 : 4)truth (generic function with 1 method)julia> features = abs.(sin.((1:10) .* (3:4)'));julia> size(features)(10, 2)julia> labels = truth.(Vector.(eachrow(features)));julia> predictor = DecisionTree.build_forest(labels, features)Ensemble of Decision TreesTrees:      10Avg Leaves: 3.3Avg Depth:  2.3julia> model = Model();julia> @variable(model, 0 <= x[1:2] <= 1);julia> y, formulation = MathOptAI.add_predictor(model, predictor, x);julia> y1-element Vector{JuMP.VariableRef}: moai_AffineCombination[1]julia> formulationAffineCombination├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=2, depth=2]├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=5, depth=2]├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=4, depth=2]├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=4, depth=2]├ 0.1 * BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]└ 1.0 * [0.0]├ variables [1]│ └ moai_AffineCombination[1]└ constraints [1]  └ 0.1 moai_BinaryDecisionTree_value[1] + 0.1 moai_BinaryDecisionTree_value[1] + 0.1 moai_BinaryDecisionTree_value[1] + 0.1 moai_BinaryDecisionTree_value[1] + 0.1 moai_BinaryDecisionTree_value[1] + 0.1 moai_BinaryDecisionTree_value[1] + 0.1 moai_BinaryDecisionTree_value[1] + 0.1 moai_BinaryDecisionTree_value[1] + 0.1 moai_BinaryDecisionTree_value[1] + 0.1 moai_BinaryDecisionTree_value[1] - moai_AffineCombination[1] = 0BinaryDecisionTree{Float64,Int64} [leaves=2, depth=2]├ variables [3]│ ├ moai_BinaryDecisionTree_value[1]│ ├ moai_BinaryDecisionTree_z[1]│ └ moai_BinaryDecisionTree_z[2]└ constraints [4]  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] = 1  ├ moai_BinaryDecisionTree_z[1] --> {x[1] ≤ 0.5312021626994367}  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≥ 0.5312031626994367}  └ 2 moai_BinaryDecisionTree_z[1] - 4 moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_value[1] = 0BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]├ variables [4]│ ├ moai_BinaryDecisionTree_value[1]│ ├ moai_BinaryDecisionTree_z[1]│ ├ moai_BinaryDecisionTree_z[2]│ └ moai_BinaryDecisionTree_z[3]└ constraints [7]  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] = 1  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.403738353154152}  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≥ 0.403739353154152}  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≤ 0.6842462068231298}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.403739353154152}  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≥ 0.6842472068231298}  └ -3 moai_BinaryDecisionTree_z[1] + 2 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_value[1] = 0BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]├ variables [4]│ ├ moai_BinaryDecisionTree_value[1]│ ├ moai_BinaryDecisionTree_z[1]│ ├ moai_BinaryDecisionTree_z[2]│ └ moai_BinaryDecisionTree_z[3]└ constraints [7]  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] = 1  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.6408420392398918}  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.41223711733275015}  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.6408420392398918}  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≥ 0.4122381173327501}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.6408430392398918}  └ -3 moai_BinaryDecisionTree_z[1] + 2 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_value[1] = 0BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]├ variables [4]│ ├ moai_BinaryDecisionTree_value[1]│ ├ moai_BinaryDecisionTree_z[1]│ ├ moai_BinaryDecisionTree_z[2]│ └ moai_BinaryDecisionTree_z[3]└ constraints [7]  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] = 1  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.5439987996210628}  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≥ 0.5439997996210628}  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.9511507486755048}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.5439997996210628}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.9511517486755048}  └ 2 moai_BinaryDecisionTree_z[1] - 4 moai_BinaryDecisionTree_z[2] + 2 moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_value[1] = 0BinaryDecisionTree{Float64,Int64} [leaves=5, depth=2]├ variables [6]│ ├ moai_BinaryDecisionTree_value[1]│ ├ moai_BinaryDecisionTree_z[1]│ ├ moai_BinaryDecisionTree_z[2]│ ├ moai_BinaryDecisionTree_z[3]│ ├ moai_BinaryDecisionTree_z[4]│ └ moai_BinaryDecisionTree_z[5]└ constraints [16]  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_z[4] + moai_BinaryDecisionTree_z[5] = 1  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.41223711733275015}  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≥ 0.4122381173327501}  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.8348728730177779}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.4122381173327501}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.8348738730177779}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≤ 0.9511507486755048}  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.4122381173327501}  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.8348738730177779}  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.9511517486755048}  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≤ 0.9905675500332488}  ├ moai_BinaryDecisionTree_z[5] --> {x[2] ≥ 0.4122381173327501}  ├ moai_BinaryDecisionTree_z[5] --> {x[2] ≥ 0.8348738730177779}  ├ moai_BinaryDecisionTree_z[5] --> {x[2] ≥ 0.9511517486755048}  ├ moai_BinaryDecisionTree_z[5] --> {x[2] ≥ 0.9905685500332488}  └ -3 moai_BinaryDecisionTree_z[1] + 2 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + 2 moai_BinaryDecisionTree_z[4] - 4 moai_BinaryDecisionTree_z[5] + moai_BinaryDecisionTree_value[1] = 0BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]├ variables [4]│ ├ moai_BinaryDecisionTree_value[1]│ ├ moai_BinaryDecisionTree_z[1]│ ├ moai_BinaryDecisionTree_z[2]│ └ moai_BinaryDecisionTree_z[3]└ constraints [7]  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] = 1  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.5439987996210628}  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.41223711733275015}  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.5439987996210628}  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≥ 0.4122381173327501}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.5439997996210628}  └ -3 moai_BinaryDecisionTree_z[1] + 2 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_value[1] = 0BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]├ variables [4]│ ├ moai_BinaryDecisionTree_value[1]│ ├ moai_BinaryDecisionTree_z[1]│ ├ moai_BinaryDecisionTree_z[2]│ └ moai_BinaryDecisionTree_z[3]└ constraints [7]  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] = 1  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.41223711733275015}  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≥ 0.4122381173327501}  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≤ 0.6588474236241902}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.4122381173327501}  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≥ 0.6588484236241903}  └ -3 moai_BinaryDecisionTree_z[1] + 2 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_value[1] = 0BinaryDecisionTree{Float64,Int64} [leaves=4, depth=2]├ variables [5]│ ├ moai_BinaryDecisionTree_value[1]│ ├ moai_BinaryDecisionTree_z[1]│ ├ moai_BinaryDecisionTree_z[2]│ ├ moai_BinaryDecisionTree_z[3]│ └ moai_BinaryDecisionTree_z[4]└ constraints [11]  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_z[4] = 1  ├ moai_BinaryDecisionTree_z[1] --> {x[1] ≤ 0.4743447016210958}  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≥ 0.4743457016210958}  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≤ 0.5934293790787758}  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≥ 0.4743457016210958}  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≥ 0.5934303790787758}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≤ 0.5919245195177484}  ├ moai_BinaryDecisionTree_z[4] --> {x[1] ≥ 0.4743457016210958}  ├ moai_BinaryDecisionTree_z[4] --> {x[1] ≥ 0.5934303790787758}  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.5919255195177484}  └ 2 moai_BinaryDecisionTree_z[1] - 3 moai_BinaryDecisionTree_z[2] - 3 moai_BinaryDecisionTree_z[3] - 4 moai_BinaryDecisionTree_z[4] + moai_BinaryDecisionTree_value[1] = 0BinaryDecisionTree{Float64,Int64} [leaves=4, depth=2]├ variables [5]│ ├ moai_BinaryDecisionTree_value[1]│ ├ moai_BinaryDecisionTree_z[1]│ ├ moai_BinaryDecisionTree_z[2]│ ├ moai_BinaryDecisionTree_z[3]│ └ moai_BinaryDecisionTree_z[4]└ constraints [11]  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_z[4] = 1  ├ moai_BinaryDecisionTree_z[1] --> {x[2] ≤ 0.5223519059864967}  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≥ 0.5223529059864968}  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.831189428657276}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.5223529059864968}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.831190428657276}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≤ 0.9511507486755048}  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.5223529059864968}  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.831190428657276}  ├ moai_BinaryDecisionTree_z[4] --> {x[2] ≥ 0.9511517486755048}  └ -3 moai_BinaryDecisionTree_z[1] + 2 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + 2 moai_BinaryDecisionTree_z[4] + moai_BinaryDecisionTree_value[1] = 0BinaryDecisionTree{Float64,Int64} [leaves=3, depth=2]├ variables [4]│ ├ moai_BinaryDecisionTree_value[1]│ ├ moai_BinaryDecisionTree_z[1]│ ├ moai_BinaryDecisionTree_z[2]│ └ moai_BinaryDecisionTree_z[3]└ constraints [7]  ├ moai_BinaryDecisionTree_z[1] + moai_BinaryDecisionTree_z[2] + moai_BinaryDecisionTree_z[3] = 1  ├ moai_BinaryDecisionTree_z[1] --> {x[1] ≤ 0.515200372485301}  ├ moai_BinaryDecisionTree_z[2] --> {x[1] ≥ 0.5152013724853011}  ├ moai_BinaryDecisionTree_z[2] --> {x[2] ≤ 0.5882410751572464}  ├ moai_BinaryDecisionTree_z[3] --> {x[1] ≥ 0.5152013724853011}  ├ moai_BinaryDecisionTree_z[3] --> {x[2] ≥ 0.5882420751572465}  └ 2 moai_BinaryDecisionTree_z[1] - 3 moai_BinaryDecisionTree_z[2] - 4 moai_BinaryDecisionTree_z[3] + moai_BinaryDecisionTree_value[1] = 0