adds Dict{Symbol,DataFrame} as input for performance_profile

Project.toml

@@ -3,11 +3,13 @@ uuid = "ecbce9bc-3e5e-569d-9e29-55181f61f8d0"
 version = "0.3.3"
 
 [deps]
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 NaNMath = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 
 [compat]
+DataFrames = "^0.22.7"
 NaNMath = "0.3"
 Requires = "1"
 julia = "^1.3.0"

src/BenchmarkProfiles.jl

@@ -1,8 +1,10 @@
 module BenchmarkProfiles
 
+using DataFrames
 import NaNMath
 using Requires
 using Printf
+using DataFrames
 
 export performance_ratios, performance_profile, performance_profile_data
 export data_ratios, data_profile

src/performance_profiles.jl

@@ -129,3 +129,45 @@ function performance_profile(b::AbstractBackend,
   (x_plot, y_plot, max_ratio) = performance_profile_data(T, logscale=logscale, sampletol=sampletol, drawtol=drawtol)
   performance_profile_plot(b, x_plot, y_plot, max_ratio, xlabel, ylabel, labels, title, logscale; kwargs...)
 end
+"""
+    performance_profile(b,stats, cost)
+Produce a performance profile comparing solvers in `stats` using the `cost` function.
+Inputs:
+- `b :: AbstractBackend`: the backend used to produce the plot.
+- `stats::Dict{Symbol,DataFrame}`: pairs of `:solver => df`;
+- `cost::Function`: cost function applyed to each `df`. Should return a vector with the cost of solving the problem at each row;
+- if the cost is zero for at least one problem, all costs will be shifted by 1;
+- if the solver did not solve the problem, return Inf or a negative number.
+Examples of cost functions:
+- `cost(df) = df.elapsed_time`: Simple `elapsed_time` cost. Assumes the solver solved the problem.
+- `cost(df) = (df.status .!= :first_order) * Inf + df.elapsed_time`: Takes the status of the solver into consideration.
+"""
+function performance_profile(b::AbstractBackend, stats::Dict{Symbol,DataFrame}, cost, args...; kwargs...)
+  solvers = keys(stats)
+  dfs = (stats[s] for s in solvers)
+  P = hcat([cost(df) for df in dfs]...)
+  performance_profile(b, P, string.(solvers), args...; kwargs...)
+end
+
+performance_profile(b::AbstractBackend, T :: Array{Tn,2}, labels :: Vector{S}; kwargs...) where {Tn <: Number, S <: AbstractString} =
+  performance_profile(b, convert(Array{Float64,2}, T), convert(Vector{AbstractString}, labels); kwargs...)
+
+"""
+    performance_profile(b, stats, cost)
+Produce a performance profile comparing solvers in `stats` using the `cost` function.
+Inputs:
+- `b::AbstractBackend`: the backend used to produce the plot.
+- `stats::Dict{Symbol,DataFrame}`: pairs of `:solver => df`;
+- `cost::Function`: cost function applyed to each `df`. Should return a vector with the cost of solving the problem at each row;
+  - if the cost is zero for at least one problem, all costs will be shifted by 1;
+  - if the solver did not solve the problem, return Inf or a negative number.
+Examples of cost functions:
+- `cost(df) = df.elapsed_time`: Simple `elapsed_time` cost. Assumes the solver solved the problem.
+- `cost(df) = (df.status .!= :first_order) * Inf + df.elapsed_time`: Takes the status of the solver into consideration.
+"""
+function performance_profile(b::AbstractBackend, stats::Dict{Symbol,DataFrame}, cost, args...; kwargs...)
+  solvers = keys(stats)
+  dfs = (stats[s] for s in solvers)
+  P = hcat([cost(df) for df in dfs]...)
+  performance_profile(b, P, string.(solvers), args...; kwargs...)
+end

test/runtests.jl

@@ -1,39 +1,66 @@
 using BenchmarkProfiles
+using DataFrames
 using Test
 
 @testset "No backend" begin
   T = 10 * rand(25, 3)
   labels = ["a", "b", "c"]
   @test_throws ArgumentError performance_profile(PlotsBackend(), T, labels)
   @test_throws ArgumentError performance_profile(UnicodePlotsBackend(), T, labels)
+  dfs  = DataFrame[]
+  for col in eachcol(T)
+    push!(dfs, DataFrame(:perf_measure => col))
+  end
+  cost(df) = df.perf_measure
+  stats = Dict([:a, :b, :c] .=> dfs)
+  @test_throws ArgumentError performance_profile(PlotsBackend(), stats, cost)    
+  @test_throws ArgumentError performance_profile(UnicodePlotsBackend(), stats, cost)    
   H = rand(25, 4, 3)
   T = ones(10)
   @test_throws ArgumentError data_profile(PlotsBackend(), H, T, labels)
-  @test_throws ArgumentError data_profile(PlotsBackend(), H, T, labels)
+  @test_throws ArgumentError data_profile(UnicodePlotsBackend(), H, T, labels)
 end
 
+
 @testset "UnicodePlots" begin
   using UnicodePlots
   T = 10 * rand(25, 3)
   labels = ["a", "b", "c"]
   profile = performance_profile(UnicodePlotsBackend(), T, labels)
   @test isa(profile, UnicodePlots.Plot{BrailleCanvas})
+  dfs  = DataFrame[]
+  for col in eachcol(T)
+    push!(dfs, DataFrame(:perf_measure => col))
+  end
+  cost(df) = df.perf_measure
+  stats = Dict([:a, :b, :c] .=> dfs)
+  profile = performance_profile(UnicodePlotsBackend(), stats, cost)
+  @test isa(profile, UnicodePlots.Plot{BrailleCanvas})
   H = rand(25, 4, 3)
   T = ones(10)
   profile = data_profile(UnicodePlotsBackend(), H, T, labels)
   @test isa(profile, UnicodePlots.Plot{BrailleCanvas})
 end
 
+
 if !Sys.isfreebsd() # GR_jll not available, so Plots won't install
   @testset "Plots" begin
     using Plots
     T = 10 * rand(25, 3)
     labels = ["a", "b", "c"]
     profile = performance_profile(PlotsBackend(), T, labels, linestyles=[:solid, :dash, :dot])
     @test isa(profile, Plots.Plot)
+    dfs  = DataFrame[]
+    for col in eachcol(T)
+      push!(dfs, DataFrame(:perf_measure => col))
+    end
+    cost(df) = df.perf_measure
+    stats = Dict([:a, :b, :c] .=> dfs)
+    profile = performance_profile(PlotsBackend(), stats, cost, linestyles=[:solid, :dash, :dot])
+    @test isa(profile, Plots.Plot)
     H = rand(25, 4, 3)
     T = ones(10)
-    profile = data_profile(PlotsBackend(), H, T, labels)
+    profile = data_profile(PlotsBackend(), H, T, labels, linestyles=[:solid, :dash, :dot])
     @test isa(profile, Plots.Plot)
   end
-end
+end