Improvements to DynamicPPLBenchmarks

.github/workflows/Benchmarking.yml

@@ -0,0 +1,66 @@
+name: Benchmarking
+
+on:
+  pull_request:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up Julia
+        uses: julia-actions/setup-julia@v2
+        with:
+          version: '1'
+
+      - name: Install Dependencies
+        run: julia --project=benchmarks/ -e 'using Pkg; Pkg.instantiate()'
+
+      - name: Run Benchmarks
+        id: run_benchmarks
+        run: |
+          # Capture version info into a variable, print it, and set it as an env var for later steps
+          version_info=$(julia -e 'using InteractiveUtils; versioninfo()')
+          echo "$version_info"
+          echo "VERSION_INFO<<EOF" >> $GITHUB_ENV
+          echo "$version_info" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+          # Capture benchmark output into a variable
+          echo "Running Benchmarks..."
+          benchmark_output=$(julia --project=benchmarks benchmarks/benchmarks.jl)
+
+          # Print benchmark results directly to the workflow log
+          echo "Benchmark Results:"
+          echo "$benchmark_output"
+
+          # Set the benchmark output as an env var for later steps
+          echo "BENCHMARK_OUTPUT<<EOF" >> $GITHUB_ENV
+          echo "$benchmark_output" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+      - name: Find Existing Comment
+        uses: peter-evans/find-comment@v3
+        id: find_comment
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: github-actions[bot]
+
+      - name: Post Benchmark Results as PR Comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            ## Computer Information
+            ```
+            ${{ env.VERSION_INFO }}
+            ```
+            ## Benchmark Report
+            ${{ env.BENCHMARK_OUTPUT }}
+          comment-id: ${{ steps.find_comment.outputs.comment-id }}
+          edit-mode: replace

benchmarks/Project.toml

@@ -4,10 +4,7 @@ version = "0.1.0"
 
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
-DiffUtils = "8294860b-85a6-42f8-8c35-d911f667b5f6"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 DynamicPPL = "366bfd00-2699-11ea-058f-f148b4cae6d8"
-LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
-Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-Weave = "44d3d7a6-8a23-5bf8-98c5-b353f8df5ec9"
+PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
+TuringBenchmarking = "0db1332d-5c25-4deb-809f-459bc696f94f"

benchmarks/README.md

@@ -1,27 +1,5 @@
-To run the benchmarks, simply do:
+To run the benchmarks, simply do this from the root directory of the repository:
 
 ```sh
-julia --project -e 'using DynamicPPLBenchmarks; weave_benchmarks();'
-```
-
-```julia
-julia> @doc weave_benchmarks
-  weave_benchmarks(input="benchmarks.jmd"; kwargs...)
-
-  Weave benchmarks present in benchmarks.jmd into a single file.
-
-  Keyword arguments
-  ≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡
-
-    •  benchmarkbody: JMD-file to be rendered for each model.
-
-    •  include_commit_id=false: specify whether to include commit-id in the default name.
-
-    •  name: the name of directory in results/ to use as output directory.
-
-    •  name_old=nothing: if specified, comparisons of current run vs. the run pinted to by name_old will be included in the generated document.
-
-    •  include_typed_code=false: if true, output of code_typed for the evaluator of the model will be included in the weaved document.
-
-    •  Rest of the passed kwargs will be passed on to Weave.weave.
-```
+julia --project=benchmarks benchmarks/benchmarks.jl
+```

benchmarks/benchmarks.jl

@@ -0,0 +1,65 @@
+using DynamicPPL: @model
+using DynamicPPLBenchmarks: make_suite
+using BenchmarkTools: median, run
+using Distributions: Normal, Beta, Bernoulli
+using PrettyTables: pretty_table, PrettyTables
+
+# Define models
+@model function demo1(x)
+    m ~ Normal()
+    x ~ Normal(m, 1)
+    return (m=m, x=x)
+end
+
+@model function demo2(y)
+    p ~ Beta(1, 1)
+    N = length(y)
+    for n in 1:N
+        y[n] ~ Bernoulli(p)
+    end
+    return (; p)
+end
+
+demo1_data = randn()
+demo2_data = rand(Bool, 10)
+
+# Create model instances with the data
+demo1_instance = demo1(demo1_data)
+demo2_instance = demo2(demo2_data)
+
+# Specify the combinations to test:
+# (Model Name, model instance, VarInfo choice, AD backend)
+chosen_combinations = [
+    ("Demo1", demo1_instance, :typed, :forwarddiff),
+    ("Demo1", demo1_instance, :simple_namedtuple, :zygote),
+    ("Demo2", demo2_instance, :untyped, :reversediff),
+    ("Demo2", demo2_instance, :simple_dict, :forwarddiff),
+]
+
+results_table = Tuple{String,String,String,Float64,Float64}[]
+
+for (model_name, model, varinfo_choice, adbackend) in chosen_combinations
+    suite = make_suite(model, varinfo_choice, adbackend)
+    results = run(suite)
+
+    eval_time = median(results["AD_Benchmarking"]["evaluation"]["standard"]).time
+
+    grad_group = results["AD_Benchmarking"]["gradient"]
+    if isempty(grad_group)
+        ad_eval_time = NaN
+    else
+        grad_backend_key = first(keys(grad_group))
+        ad_eval_time = median(grad_group[grad_backend_key]["standard"]).time
+    end
+
+    push!(
+        results_table,
+        (model_name, string(adbackend), string(varinfo_choice), eval_time, ad_eval_time),
+    )
+end
+
+table_matrix = hcat(Iterators.map(collect, zip(results_table...))...)
+header = [
+    "Model", "AD Backend", "VarInfo Type", "Evaluation Time (ns)", "AD Eval Time (ns)"
+]
+pretty_table(table_matrix; header=header, tf=PrettyTables.tf_markdown)