Skip to content

adds Dict{Symbol,DataFrame} as input for performance_profile #54

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 15 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,13 @@ uuid = "ecbce9bc-3e5e-569d-9e29-55181f61f8d0"
version = "0.3.3"

[deps]
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
NaNMath = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
Requires = "ae029012-a4dd-5104-9daa-d747884805df"

[compat]
DataFrames = "^0.22.7"
NaNMath = "0.3"
Requires = "1"
julia = "^1.3.0"
Expand Down
2 changes: 2 additions & 0 deletions src/BenchmarkProfiles.jl
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
module BenchmarkProfiles

using DataFrames
import NaNMath
using Requires
using Printf
using DataFrames

export performance_ratios, performance_profile, performance_profile_data
export data_ratios, data_profile
Expand Down
42 changes: 42 additions & 0 deletions src/performance_profiles.jl
Original file line number Diff line number Diff line change
Expand Up @@ -129,3 +129,45 @@ function performance_profile(b::AbstractBackend,
(x_plot, y_plot, max_ratio) = performance_profile_data(T, logscale=logscale, sampletol=sampletol, drawtol=drawtol)
performance_profile_plot(b, x_plot, y_plot, max_ratio, xlabel, ylabel, labels, title, logscale; kwargs...)
end
"""
performance_profile(b,stats, cost)
Produce a performance profile comparing solvers in `stats` using the `cost` function.
Inputs:
- `b :: AbstractBackend`: the backend used to produce the plot.
- `stats::Dict{Symbol,DataFrame}`: pairs of `:solver => df`;
- `cost::Function`: cost function applyed to each `df`. Should return a vector with the cost of solving the problem at each row;
- if the cost is zero for at least one problem, all costs will be shifted by 1;
- if the solver did not solve the problem, return Inf or a negative number.
Examples of cost functions:
- `cost(df) = df.elapsed_time`: Simple `elapsed_time` cost. Assumes the solver solved the problem.
- `cost(df) = (df.status .!= :first_order) * Inf + df.elapsed_time`: Takes the status of the solver into consideration.
"""
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we use the same docstring format as in the existing performance_profile?

function performance_profile(b::AbstractBackend, stats::Dict{Symbol,DataFrame}, cost, args...; kwargs...)
solvers = keys(stats)
dfs = (stats[s] for s in solvers)
P = hcat([cost(df) for df in dfs]...)
performance_profile(b, P, string.(solvers), args...; kwargs...)
end

performance_profile(b::AbstractBackend, T :: Array{Tn,2}, labels :: Vector{S}; kwargs...) where {Tn <: Number, S <: AbstractString} =
performance_profile(b, convert(Array{Float64,2}, T), convert(Vector{AbstractString}, labels); kwargs...)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I removed this because it’s ambiguous. At some point I got a StackOverflow because of it. In addition, it’s not tested.


"""
performance_profile(b, stats, cost)
Produce a performance profile comparing solvers in `stats` using the `cost` function.
Inputs:
- `b::AbstractBackend`: the backend used to produce the plot.
- `stats::Dict{Symbol,DataFrame}`: pairs of `:solver => df`;
- `cost::Function`: cost function applyed to each `df`. Should return a vector with the cost of solving the problem at each row;
- if the cost is zero for at least one problem, all costs will be shifted by 1;
- if the solver did not solve the problem, return Inf or a negative number.
Examples of cost functions:
- `cost(df) = df.elapsed_time`: Simple `elapsed_time` cost. Assumes the solver solved the problem.
- `cost(df) = (df.status .!= :first_order) * Inf + df.elapsed_time`: Takes the status of the solver into consideration.
"""
function performance_profile(b::AbstractBackend, stats::Dict{Symbol,DataFrame}, cost, args...; kwargs...)
solvers = keys(stats)
dfs = (stats[s] for s in solvers)
P = hcat([cost(df) for df in dfs]...)
performance_profile(b, P, string.(solvers), args...; kwargs...)
end
33 changes: 30 additions & 3 deletions test/runtests.jl
Original file line number Diff line number Diff line change
@@ -1,39 +1,66 @@
using BenchmarkProfiles
using DataFrames
using Test

@testset "No backend" begin
T = 10 * rand(25, 3)
labels = ["a", "b", "c"]
@test_throws ArgumentError performance_profile(PlotsBackend(), T, labels)
@test_throws ArgumentError performance_profile(UnicodePlotsBackend(), T, labels)
dfs = DataFrame[]
for col in eachcol(T)
push!(dfs, DataFrame(:perf_measure => col))
end
cost(df) = df.perf_measure
stats = Dict([:a, :b, :c] .=> dfs)
@test_throws ArgumentError performance_profile(PlotsBackend(), stats, cost)
@test_throws ArgumentError performance_profile(UnicodePlotsBackend(), stats, cost)
H = rand(25, 4, 3)
T = ones(10)
@test_throws ArgumentError data_profile(PlotsBackend(), H, T, labels)
@test_throws ArgumentError data_profile(PlotsBackend(), H, T, labels)
@test_throws ArgumentError data_profile(UnicodePlotsBackend(), H, T, labels)
end


@testset "UnicodePlots" begin
using UnicodePlots
T = 10 * rand(25, 3)
labels = ["a", "b", "c"]
profile = performance_profile(UnicodePlotsBackend(), T, labels)
@test isa(profile, UnicodePlots.Plot{BrailleCanvas})
dfs = DataFrame[]
for col in eachcol(T)
push!(dfs, DataFrame(:perf_measure => col))
end
cost(df) = df.perf_measure
stats = Dict([:a, :b, :c] .=> dfs)
profile = performance_profile(UnicodePlotsBackend(), stats, cost)
@test isa(profile, UnicodePlots.Plot{BrailleCanvas})
H = rand(25, 4, 3)
T = ones(10)
profile = data_profile(UnicodePlotsBackend(), H, T, labels)
@test isa(profile, UnicodePlots.Plot{BrailleCanvas})
end


if !Sys.isfreebsd() # GR_jll not available, so Plots won't install
@testset "Plots" begin
using Plots
T = 10 * rand(25, 3)
labels = ["a", "b", "c"]
profile = performance_profile(PlotsBackend(), T, labels, linestyles=[:solid, :dash, :dot])
@test isa(profile, Plots.Plot)
dfs = DataFrame[]
for col in eachcol(T)
push!(dfs, DataFrame(:perf_measure => col))
end
cost(df) = df.perf_measure
stats = Dict([:a, :b, :c] .=> dfs)
profile = performance_profile(PlotsBackend(), stats, cost, linestyles=[:solid, :dash, :dot])
@test isa(profile, Plots.Plot)
H = rand(25, 4, 3)
T = ones(10)
profile = data_profile(PlotsBackend(), H, T, labels)
profile = data_profile(PlotsBackend(), H, T, labels, linestyles=[:solid, :dash, :dot])
@test isa(profile, Plots.Plot)
end
end
end