ModalAdaBoost model

Author: PasoStudio73

src/ModalDecisionTrees.jl

@@ -121,7 +121,7 @@ include("posthoc.jl")
 # Apply decision tree/forest to a dataset
 include("apply.jl")
 
-export ModalDecisionTree, ModalRandomForest
+export ModalDecisionTree, ModalRandomForest, ModalAdaBoost
 export depth
 
 export wrapdataset

src/build.jl

@@ -31,6 +31,54 @@ a random forest model on logiset `X` with labels `Y` and weights `W`.
 
 """
 
+# """$(doc_build)"""
+function build_stumps(
+    X                 :: MultiLogiset,
+    y                 :: AbstractVector{L},
+    weigths           :: Union{Nothing,AbstractVector{U},Symbol} = nothing;
+    n_iter            :: Int = 10;
+    # rng               :: Random.AbstractRNG = Random.GLOBAL_RNG,
+    kwargs...,
+) where {L<:Label,U}
+    n_y = length(y)
+    n_labels = length(unique(y))
+    base_coeff = log(n_labels - 1)
+    thresh = 1 - 1 / n_labels
+    weights = ones(n_y) / n_y
+    stumps = DTree[]
+    coeffs = Float64[]
+    # n_features = size(X, 2)
+
+    for i in 1:n_iter
+        new_stump = build_stump(X, y, weigths; impurity_importance=false, kwargs...)
+        # new_stump = MDT.build_stump( # TODO c'è anche in MDT!!!
+        #     X, y, weights; rng=DT.mk_rng(rng), impurity_importance=false
+        # )
+        # predictions = MDT.apply_tree(new_stump, X) # TODO c'è anche in MDT!!!
+        # err = DT._weighted_error(y, predictions, weights)
+        # if err >= thresh # should be better than random guess
+        #     continue
+        # end
+        # # SAMME algorithm
+        # new_coeff = log((1.0 - err) / err) + base_coeff
+        # unmatches = labels .!= predictions
+        # weights[unmatches] *= exp(new_coeff)
+        # weights /= sum(weights)
+        # push!(coeffs, new_coeff)
+        # push!(stumps, new_stump.node)
+        # if err < 1e-6
+        #     break
+        # end
+    end
+    # return (DT.Ensemble{S,T}(stumps, n_features, Float64[]), coeffs)
+
+    stumps = DTree[]
+    for i in 1:n_iter
+        push!(stump_trees, build_stump(X, y, weigths; kwargs...))
+    end
+    return stump_trees
+end
+
 """$(doc_build)"""
 function build_stump(
     X                 :: MultiLogiset,

src/interfaces/MLJ.jl

@@ -2,7 +2,7 @@
 
 module MLJInterface
 
-export ModalDecisionTree, ModalRandomForest
+export ModalDecisionTree, ModalRandomForest, ModalAdaBoost
 export depth
 export wrapdataset
 
@@ -37,12 +37,14 @@ include("MLJ/feature-importance.jl")
 
 include("MLJ/ModalDecisionTree.jl")
 include("MLJ/ModalRandomForest.jl")
+include("MLJ/ModalAdaBoost.jl")
 
 include("MLJ/docstrings.jl")
 
 const SymbolicModel = Union{
     ModalDecisionTree,
     ModalRandomForest,
+    ModalAdaBoost,
 }
 
 const TreeModel = Union{
@@ -53,6 +55,10 @@ const ForestModel = Union{
     ModalRandomForest,
 }
 
+const StumpsModel = Union{
+    ModalAdaBoost,
+}
+
 include("MLJ/downsize.jl")
 include("MLJ/clean.jl")
 
@@ -77,6 +83,8 @@ function MMI.fit(m::SymbolicModel, verbosity::Integer, X, y, var_grouping, class
             MDT.build_tree(X, y, w; get_kwargs(m, X)...)
         elseif m isa ModalRandomForest
             MDT.build_forest(X, y, w; get_kwargs(m, X)...)
+        elseif m isa ModalAdaBoost
+            MDT.build_stumps(X, y, w; get_kwargs(m, X)...)
         else
             error("Unexpected model type: $(typeof(m))")
         end

src/interfaces/MLJ/ModalAdaBoost.jl

@@ -0,0 +1,123 @@
+mutable struct ModalAdaBoost <: MMI.Probabilistic
+    ## Pruning conditions
+    max_depth               ::Union{Nothing,Int}
+    min_samples_leaf        ::Union{Nothing,Int}
+    min_purity_increase     ::Union{Nothing,Float64}
+    max_purity_at_leaf      ::Union{Nothing,Float64}
+    max_modal_depth         ::Union{Nothing,Int}
+
+    ## Logic parameters
+
+    # Relation set
+    relations               ::Union{
+        Nothing,                                            # defaults to a well-known relation set, depending on the data;
+        Symbol,                                             # one of the relation sets specified in AVAILABLE_RELATIONS;
+        Vector{<:AbstractRelation},                         # explicitly specify the relation set;
+        # Vector{<:Union{Symbol,Vector{<:AbstractRelation}}}, # MULTIMODAL CASE: specify a relation set for each modality;
+        Function                                            # A function worldtype -> relation set.
+    }
+
+    # Condition set
+    features                ::Union{
+        Nothing,                                                                   # defaults to scalar conditions (with ≥ and <) on well-known feature functions (e.g., minimum, maximum), applied to all variables;
+        Vector{<:Union{SoleData.VarFeature,Base.Callable}},                        # scalar conditions with ≥ and <, on an explicitly specified feature set (callables to be applied to each variable, or VarFeature objects);
+        Vector{<:Tuple{Base.Callable,Integer}},                                    # scalar conditions with ≥ and <, on a set of features specified as a set of callables to be applied to a set of variables each;
+        Vector{<:Tuple{TestOperator,<:Union{SoleData.VarFeature,Base.Callable}}},  # explicitly specify the pairs (test operator, feature);
+        Vector{<:SoleData.ScalarMetaCondition},                                    # explicitly specify the scalar condition set.
+    }
+    conditions              ::Union{
+        Nothing,                                                                   # defaults to scalar conditions (with ≥ and <) on well-known feature functions (e.g., minimum, maximum), applied to all variables;
+        Vector{<:Union{SoleData.VarFeature,Base.Callable}},                        # scalar conditions with ≥ and <, on an explicitly specified feature set (callables to be applied to each variable, or VarFeature objects);
+        Vector{<:Tuple{Base.Callable,Integer}},                                    # scalar conditions with ≥ and <, on a set of features specified as a set of callables to be applied to a set of variables each;
+        Vector{<:Tuple{TestOperator,<:Union{SoleData.VarFeature,Base.Callable}}},  # explicitly specify the pairs (test operator, feature);
+        Vector{<:SoleData.ScalarMetaCondition},                                    # explicitly specify the scalar condition set.
+    }
+    # Type for the extracted feature values
+    featvaltype             ::Type
+
+    # Initial conditions
+    initconditions          ::Union{
+        Nothing,                                                                   # defaults to standard conditions (e.g., start_without_world)
+        Symbol,                                                                    # one of the initial conditions specified in AVAILABLE_INITIALCONDITIONS;
+        InitialCondition,                                                          # explicitly specify an initial condition for the learning algorithm.
+    }
+
+    ## Miscellaneous
+    downsize                ::Union{Bool,NTuple{N,Integer} where N,Function}
+    force_i_variables       ::Bool
+    fixcallablenans         ::Bool
+    print_progress          ::Bool
+    rng                     ::Union{Random.AbstractRNG,Integer}
+
+    ## DecisionTree.jl parameters
+    display_depth           ::Union{Nothing,Int}
+    min_samples_split       ::Union{Nothing,Int}
+    n_subfeatures           ::Union{Nothing,Int,Float64,Function}
+    post_prune              ::Bool
+    merge_purity_threshold  ::Union{Nothing,Float64}
+    feature_importance      ::Symbol
+
+    ## AdaBoost parameters
+    n_iter                  ::Int
+end
+
+# keyword constructor
+function ModalAdaBoost(;
+    max_depth = 1,
+    min_samples_leaf = nothing,
+    min_purity_increase = nothing,
+    max_purity_at_leaf = nothing,
+    max_modal_depth = nothing,
+    #
+    relations = nothing,
+    features = nothing,
+    conditions = nothing,
+    featvaltype = Float64,
+    initconditions = nothing,
+    #
+    downsize = true,
+    force_i_variables = true,
+    fixcallablenans = false,
+    print_progress = false,
+    rng = Random.GLOBAL_RNG,
+    #
+    display_depth = nothing,
+    min_samples_split = nothing,
+    n_subfeatures = nothing,
+    post_prune = false,
+    merge_purity_threshold = nothing,
+    feature_importance = :split,
+    #
+    n_iter = 10,
+)
+    model = ModalAdaBoost(
+        max_depth,
+        min_samples_leaf,
+        min_purity_increase,
+        max_purity_at_leaf,
+        max_modal_depth,
+        #
+        relations,
+        features,
+        conditions,
+        featvaltype,
+        initconditions,
+        #
+        downsize,
+        force_i_variables,
+        fixcallablenans,
+        print_progress,
+        rng,
+        #
+        display_depth,
+        min_samples_split,
+        n_subfeatures,
+        post_prune,
+        merge_purity_threshold,
+        feature_importance,
+        n_iter,
+    )
+    message = MMI.clean!(model)
+    isempty(message) || @warn message
+    return model
+end

src/interfaces/MLJ/ModalDecisionTree.jl

@@ -116,6 +116,7 @@ function ModalDecisionTree(;
         merge_purity_threshold,
         feature_importance,
     )
+    @show model isa SymbolicModel
     message = MMI.clean!(model)
     isempty(message) || @warn message
     return model

src/interfaces/MLJ/clean.jl

@@ -29,6 +29,10 @@ function get_kwargs(m::SymbolicModel, X)
                 ntrees = m.ntrees,
                 suppress_parity_warning = true,
             )
+        elseif m isa StumpsModel
+            (;
+                n_iter = m.n_iter,
+            )
         else
             error("Unexpected model type: $(typeof(m))")
         end
@@ -38,7 +42,6 @@ end
 
 function MMI.clean!(m::SymbolicModel)
     warning = ""
-
     if m isa TreeModel
         mlj_default_min_samples_leaf = mlj_mdt_default_min_samples_leaf
         mlj_default_min_purity_increase = mlj_mdt_default_min_purity_increase
@@ -51,6 +54,12 @@ function MMI.clean!(m::SymbolicModel)
         mlj_default_n_subfeatures = mlj_mrf_default_n_subfeatures
         mlj_default_ntrees = mlj_mrf_default_ntrees
         mlj_default_sampling_fraction = mlj_mrf_default_sampling_fraction
+    elseif m isa StumpsModel
+        mlj_default_min_samples_leaf = mlj_mdt_default_min_samples_leaf
+        mlj_default_min_purity_increase = mlj_mdt_default_min_purity_increase
+        mlj_default_max_purity_at_leaf = mlj_mdt_default_max_purity_at_leaf
+        mlj_default_n_subfeatures = mlj_mdt_default_n_subfeatures
+        # TODO mlj_default_n_iter
     else
         error("Unexpected model type: $(typeof(m))")
     end

src/interfaces/MLJ/downsize.jl

@@ -8,3 +8,7 @@ end
 function make_downsizing_function(::ForestModel)
     make_downsizing_function(Val(2))
 end
+
+function make_downsizing_function(::StumpsModel)
+    make_downsizing_function(Val(1))
+end