diff --git a/Project.toml b/Project.toml index 1b84da6..b2c79d2 100644 --- a/Project.toml +++ b/Project.toml @@ -3,11 +3,13 @@ uuid = "ecbce9bc-3e5e-569d-9e29-55181f61f8d0" version = "0.3.3" [deps] +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" NaNMath = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Requires = "ae029012-a4dd-5104-9daa-d747884805df" [compat] +DataFrames = "^0.22.7" NaNMath = "0.3" Requires = "1" julia = "^1.3.0" diff --git a/src/BenchmarkProfiles.jl b/src/BenchmarkProfiles.jl index 7a61f7c..ce571b6 100644 --- a/src/BenchmarkProfiles.jl +++ b/src/BenchmarkProfiles.jl @@ -1,8 +1,10 @@ module BenchmarkProfiles +using DataFrames import NaNMath using Requires using Printf +using DataFrames export performance_ratios, performance_profile, performance_profile_data export data_ratios, data_profile diff --git a/src/performance_profiles.jl b/src/performance_profiles.jl index 44f6d12..85bfb01 100644 --- a/src/performance_profiles.jl +++ b/src/performance_profiles.jl @@ -129,3 +129,45 @@ function performance_profile(b::AbstractBackend, (x_plot, y_plot, max_ratio) = performance_profile_data(T, logscale=logscale, sampletol=sampletol, drawtol=drawtol) performance_profile_plot(b, x_plot, y_plot, max_ratio, xlabel, ylabel, labels, title, logscale; kwargs...) end +""" + performance_profile(b,stats, cost) +Produce a performance profile comparing solvers in `stats` using the `cost` function. +Inputs: +- `b :: AbstractBackend`: the backend used to produce the plot. +- `stats::Dict{Symbol,DataFrame}`: pairs of `:solver => df`; +- `cost::Function`: cost function applyed to each `df`. Should return a vector with the cost of solving the problem at each row; +- if the cost is zero for at least one problem, all costs will be shifted by 1; +- if the solver did not solve the problem, return Inf or a negative number. +Examples of cost functions: +- `cost(df) = df.elapsed_time`: Simple `elapsed_time` cost. Assumes the solver solved the problem. +- `cost(df) = (df.status .!= :first_order) * Inf + df.elapsed_time`: Takes the status of the solver into consideration. +""" +function performance_profile(b::AbstractBackend, stats::Dict{Symbol,DataFrame}, cost, args...; kwargs...) + solvers = keys(stats) + dfs = (stats[s] for s in solvers) + P = hcat([cost(df) for df in dfs]...) + performance_profile(b, P, string.(solvers), args...; kwargs...) +end + +performance_profile(b::AbstractBackend, T :: Array{Tn,2}, labels :: Vector{S}; kwargs...) where {Tn <: Number, S <: AbstractString} = + performance_profile(b, convert(Array{Float64,2}, T), convert(Vector{AbstractString}, labels); kwargs...) + +""" + performance_profile(b, stats, cost) +Produce a performance profile comparing solvers in `stats` using the `cost` function. +Inputs: +- `b::AbstractBackend`: the backend used to produce the plot. +- `stats::Dict{Symbol,DataFrame}`: pairs of `:solver => df`; +- `cost::Function`: cost function applyed to each `df`. Should return a vector with the cost of solving the problem at each row; + - if the cost is zero for at least one problem, all costs will be shifted by 1; + - if the solver did not solve the problem, return Inf or a negative number. +Examples of cost functions: +- `cost(df) = df.elapsed_time`: Simple `elapsed_time` cost. Assumes the solver solved the problem. +- `cost(df) = (df.status .!= :first_order) * Inf + df.elapsed_time`: Takes the status of the solver into consideration. +""" +function performance_profile(b::AbstractBackend, stats::Dict{Symbol,DataFrame}, cost, args...; kwargs...) + solvers = keys(stats) + dfs = (stats[s] for s in solvers) + P = hcat([cost(df) for df in dfs]...) + performance_profile(b, P, string.(solvers), args...; kwargs...) +end diff --git a/test/runtests.jl b/test/runtests.jl index ba8769d..b9879ed 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,5 @@ using BenchmarkProfiles +using DataFrames using Test @testset "No backend" begin @@ -6,24 +7,42 @@ using Test labels = ["a", "b", "c"] @test_throws ArgumentError performance_profile(PlotsBackend(), T, labels) @test_throws ArgumentError performance_profile(UnicodePlotsBackend(), T, labels) + dfs = DataFrame[] + for col in eachcol(T) + push!(dfs, DataFrame(:perf_measure => col)) + end + cost(df) = df.perf_measure + stats = Dict([:a, :b, :c] .=> dfs) + @test_throws ArgumentError performance_profile(PlotsBackend(), stats, cost) + @test_throws ArgumentError performance_profile(UnicodePlotsBackend(), stats, cost) H = rand(25, 4, 3) T = ones(10) @test_throws ArgumentError data_profile(PlotsBackend(), H, T, labels) - @test_throws ArgumentError data_profile(PlotsBackend(), H, T, labels) + @test_throws ArgumentError data_profile(UnicodePlotsBackend(), H, T, labels) end + @testset "UnicodePlots" begin using UnicodePlots T = 10 * rand(25, 3) labels = ["a", "b", "c"] profile = performance_profile(UnicodePlotsBackend(), T, labels) @test isa(profile, UnicodePlots.Plot{BrailleCanvas}) + dfs = DataFrame[] + for col in eachcol(T) + push!(dfs, DataFrame(:perf_measure => col)) + end + cost(df) = df.perf_measure + stats = Dict([:a, :b, :c] .=> dfs) + profile = performance_profile(UnicodePlotsBackend(), stats, cost) + @test isa(profile, UnicodePlots.Plot{BrailleCanvas}) H = rand(25, 4, 3) T = ones(10) profile = data_profile(UnicodePlotsBackend(), H, T, labels) @test isa(profile, UnicodePlots.Plot{BrailleCanvas}) end + if !Sys.isfreebsd() # GR_jll not available, so Plots won't install @testset "Plots" begin using Plots @@ -31,9 +50,17 @@ if !Sys.isfreebsd() # GR_jll not available, so Plots won't install labels = ["a", "b", "c"] profile = performance_profile(PlotsBackend(), T, labels, linestyles=[:solid, :dash, :dot]) @test isa(profile, Plots.Plot) + dfs = DataFrame[] + for col in eachcol(T) + push!(dfs, DataFrame(:perf_measure => col)) + end + cost(df) = df.perf_measure + stats = Dict([:a, :b, :c] .=> dfs) + profile = performance_profile(PlotsBackend(), stats, cost, linestyles=[:solid, :dash, :dot]) + @test isa(profile, Plots.Plot) H = rand(25, 4, 3) T = ones(10) - profile = data_profile(PlotsBackend(), H, T, labels) + profile = data_profile(PlotsBackend(), H, T, labels, linestyles=[:solid, :dash, :dot]) @test isa(profile, Plots.Plot) end -end +end \ No newline at end of file