Incorporate Youssef comments

Author: MohamedLaghdafHABIBOULLAH

paper/examples/Benchmark.jl

@@ -107,6 +107,30 @@ function run_r2n_svm!(model, x0; λ = 1.0, qn = :LBFGS, atol = 1e-3, rtol = 1e-3
     )
 end
 
+function run_LM_svm!(nls_model, x0; λ = 1.0, atol = 1e-3, rtol = 1e-3, verbose = 0)
+    reg_nls  = RegularizedNLSModel(nls_model, RootNormLhalf(λ))
+    solver   = LMSolver(reg_nls)
+    stats    = RegularizedExecutionStats(reg_nls)
+    RegularizedOptimization.solve!(solver, reg_nls, stats;
+        x = x0, atol = atol, rtol = rtol, verbose = verbose)
+    reset!(nls_model)  # Reset counters before timing
+    reg_nls  = RegularizedNLSModel(nls_model, RootNormLhalf(λ))
+    solver   = LMSolver(reg_nls)
+    t = @elapsed RegularizedOptimization.solve!(solver, reg_nls, stats;
+        x = x0, atol = atol, rtol = rtol, verbose = verbose)
+    return (
+        name      = "LM (SVM)",
+        status    = string(stats.status),
+        time      = t,
+        iters     = get(stats.solver_specific, :outer_iter, missing),
+        fevals    = neval_residual(nls_model),
+        gevals    = neval_jtprod_residual(nls_model) + neval_jprod_residual(nls_model),
+        proxcalls = get(stats.solver_specific, :prox_evals, missing),
+        solution  = stats.solution,
+        final_obj = obj(nls_model, stats.solution)
+    )
+end
+
 function bench_svm!(cfg = CFG)
     Random.seed!(cfg.SEED)
     model, nls_train, _ = RegularizedProblems.svm_train_model()
@@ -115,6 +139,7 @@ function bench_svm!(cfg = CFG)
     results = NamedTuple[]
     (:TR    in cfg.RUN_SOLVERS) && push!(results, run_tr_svm!(model, x0; λ = cfg.LAMBDA_L0, qn = cfg.QN_FOR_TR, atol = cfg.TOL, rtol = cfg.RTOL, verbose = cfg.VERBOSE_RO, sub_kwargs = cfg.SUB_KWARGS_R2N))
     (:R2N   in cfg.RUN_SOLVERS) && push!(results, run_r2n_svm!(model, x0; λ = cfg.LAMBDA_L0, qn = cfg.QN_FOR_R2N, atol = cfg.TOL, rtol = cfg.RTOL, verbose = cfg.VERBOSE_RO, sub_kwargs = cfg.SUB_KWARGS_R2N))
+    (:LM    in cfg.RUN_SOLVERS) && push!(results, run_LM_svm!(nls_train, x0; λ = cfg.LAMBDA_L0, atol = cfg.TOL, rtol = cfg.RTOL, verbose = cfg.VERBOSE_RO))
 
     # Print quick summary
     println("\n=== SVM: solver comparison ===")

paper/examples/Benchmark.tex

@@ -1,9 +1,11 @@
 \begin{tabular}{lcrrrrr}
   \hline
   \textbf{Method} & \textbf{Status} & \textbf{$t$($s$)} & \textbf{$\#f$} & \textbf{$\#\nabla f$} & \textbf{$\#prox$} & \textbf{Objective} \\\hline
-  TR (LSR1, SVM) & first\_order & 4.0193 & 347 & 291 & 4037 & 179.837 \\
-  R2N (LSR1, SVM) & first\_order & 2.5033 & 185 & 101 & 27932 & 192.493 \\
-  TR (LBFGS, NNMF) & first\_order & 0.1089 & 42 & 40 & 3160 & 976.06 \\
-  R2N (LBFGS, NNMF) & first\_order & 0.4921 & 169 & 107 & 17789 & 411.727 \\
-  LM (NNMF) & first\_order & 0.4542 & 15 & 27723 & 12320 & 131.183 \\\hline
+  TR (LSR1, SVM) & first\_order & 2.3656 & 347 & 291 & 4037 & 179.837 \\
+  R2N (LSR1, SVM) & first\_order & 0.9742 & 185 & 101 & 27932 & 192.493 \\
+  LM (SVM) & first\_order & 24.6641 & 6 & 9161 & 3644 & 202.731 \\
+  \hline
+  TR (LBFGS, NNMF) & first\_order & 0.0542 & 42 & 40 & 3160 & 976.06 \\
+  R2N (LBFGS, NNMF) & first\_order & 0.2905 & 169 & 107 & 17789 & 411.727 \\
+  LM (NNMF) & first\_order & 0.2596 & 15 & 27703 & 12320 & 131.183 \\\hline
 \end{tabular}

paper/examples/comparison-config.jl

@@ -8,7 +8,7 @@ Base.@kwdef mutable struct Config
     MAXIT_PANOC::Int          = 10000
     VERBOSE_PANOC::Bool       = false
     VERBOSE_RO::Int           = 0
-    RUN_SOLVERS::Vector{Symbol} = [:TR, :R2N]   # mutable
+    RUN_SOLVERS::Vector{Symbol} = [:LM, :TR, :R2N]   # mutable
     QN_FOR_TR::Symbol         = :LSR1
     QN_FOR_R2N::Symbol        = :LBFGS
     SUB_KWARGS_R2N::NamedTuple = (; max_iter = 200)
@@ -17,6 +17,6 @@ end
 
 # One global, constant *binding* to a mutable object = type stable & editable
 const CFG = Config(QN_FOR_R2N=:LSR1)
-const CFG2 = Config(RUN_SOLVERS = [:LM, :TR, :R2N], QN_FOR_TR = :LBFGS)
+const CFG2 = Config(QN_FOR_TR = :LBFGS)
 
 end # module

paper/paper.md

@@ -81,7 +81,7 @@ $$
 where $q$ is given, $x$ and $s$ are fixed shifts, $\chi(\cdot \mid \Delta \mathbb{B})$ is the indicator of a ball of radius $\Delta > 0$ defined by a certain norm, and $\psi(\cdot; x)$ is a model of $h$ about $x$.
 It is common to set $\psi(t + s; x) = h(x + s + t)$.
 
-These shifted operators allow us to (i) incorporate bound or trust-region constraints via the indicator, which is required for the **TR** and **TRDH** solvers, and (ii) evaluate the above **in place**, without additional allocations, which is currently not possible with ProximalOperators.jl.
+These shifted operators allow to (i) incorporate bound or trust-region constraints via the indicator, which is required for the **TR** and **TRDH** solvers, and (ii) evaluate the above in place, without additional allocations, which is currently not possible with ProximalOperators.jl.
 
 RegularizedOptimization.jl provides a consistent API to formulate optimization problems and apply different solvers.
 It integrates seamlessly with the [JuliaSmoothOptimizers](https://github.com/JuliaSmoothOptimizers)  [@jso] ecosystem, an academic organization for nonlinear optimization software development, testing, and benchmarking.
@@ -106,7 +106,7 @@ reg_nls = RegularizedNLSModel(f, h)
 RegularizedProblems.jl also provides a set of instances commonly used in data science and in the nonsmooth optimization literature, where several choices of $f$ can be paired with various nonsmooth terms $h$.
 This design makes for a convenient source of reproducible problem instances for testing and benchmarking the solvers in [RegularizedOptimization.jl](https://www.github.com/JuliaSmoothOptimizers/RegularizedOptimization.jl).
 
-## Support for exact or approximate second derivatives
+## Support for both exact and approximate Hessian
 
 In contrast with [ProximalAlgorithms.jl](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl), [RegularizedOptimization.jl](https://github.com/JuliaSmoothOptimizers/RegularizedOptimization.jl), methods such as **R2N** and **TR** methods support exact Hessians as well as several Hessian approximations of $f$.
 Hessian–vector products $v \mapsto Hv$ can be obtained via automatic differentiation through [ADNLPModels.jl](https://github.com/JuliaSmoothOptimizers/ADNLPModels.jl) or implemented manually.
@@ -175,13 +175,14 @@ The subproblem solver is **R2**.
 
 \input{examples/Benchmark.tex}
 
-- For the LM solver, gradient evaluations count $\#\nabla f$ equals the number of Jacobian–vector and adjoint-Jacobian–vector products.
+- Note that for the LM solver, gradient evaluations count $\#\nabla f$ equals the number of Jacobian–vector and adjoint-Jacobian–vector products.
 
 All methods successfully reduced the optimality measure below the specified tolerance of $10^{-4}$, and thus converged to an approximate first-order stationary point.
-However, the final objective values differ due to the nonconvexity of the problems.
+Note that, the final objective values differ due to the nonconvexity of the problems.
 
 - **SVM with $\ell^{1/2}$ penalty:** **R2N** is the fastest, requiring the fewest function and gradient evaluations compared to **TR**.
 However, it requires more proximal evaluations, but these are inexpensive.
+**LM** requires the fewest function evaluations, but many gradient evaluations, and is the slowest.
 - **NNMF with constrained $\ell_0$ penalty:** **TR** is the fastest, and requires a fewer number of function and gradient evaluations than **R2N**. **LM** is competitive in terms of function calls but incurs many Jacobian–vector products; it nevertheless achieves the lowest objective value.
 
 Additional tests (e.g., other regularizers, constraint types, and scaling dimensions) have also been conducted, and a full benchmarking campaign is currently underway.
@@ -202,6 +203,6 @@ In ongoing research, the package will be extended with algorithms that enable to
 
 The authors would like to thank Alberto Demarchi for his implementation of the Augmented Lagrangian solver.
 Mohamed Laghdaf Habiboullah is supported by an excellence FRQNT grant.
-Youssef Diouane and Dominique Orban are partially supported by an NSERC Discovery Grant.
+Youssef Diouane, Maxence Gollier and Dominique Orban are partially supported by an NSERC Discovery Grant.
 
 # References