Improvements to DynamicPPLBenchmarks (#346)

* bigboy update to benchmarks

* make models return random variables as NamedTuple as it can be useful for downstream tasks

* add benchmarking of evaluation with SimpleVarInfo with NamedTuple

* added some information about the execution environment

* added judgementtable_single

* added benchmarking of SimpleVarInfo, if present

* added ComponentArrays benchmarking for SimpleVarInfo

* formatting

* Apply suggestions from code review

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Update benchmarks/benchmarks.jmd

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* Benchmarking CI

* Julia script for benchmarking on top of current setup

* keep old results for reference

* updated benchmarking setup

* applied suggested changes

* updated benchmarks/README.md

* setup benchmarking CI

* Update benchmark models (#826)

* Update models to benchmark plus small style changes

* Make benchmark times relative. Add benchmark documentation.

* Choose whether to show linked or unlinked benchmark times

* Make table header more concise

* Make benchmarks not depend on TuringBenchmarking.jl, and run `]dev ..` (#834)

* Make benchmarks not depend on TuringBenchmarking.jl

* Make benchmarks.jl dev the local DDPPL version

* Add benchmarks compat bounds

* Use ForwardDiff with dynamic benchmark model

* Benchmarking.yml: now comments raw markdown table enclosed in triple backticks

* Benchmarking.yml: now includes the SHA of the DynamicPPL commit in Benchmark Results comment

* Benchmark more with Mooncake

* Add model dimension to benchmark table

* Add info print

* Fix type instability in benchmark model

* Remove done TODO note

* Apply suggestions from code review

Co-authored-by: Penelope Yong <penelopeysm@gmail.com>

* Fix table formatting bug

* Simplify benchmark suite code

* Use StableRNG

---------

Co-authored-by: Hong Ge <3279477+yebai@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Shravan Goswami <shravanngoswamii@gmail.com>
Co-authored-by: Markus Hauru <mhauru@turing.ac.uk>
Co-authored-by: Markus Hauru <markus@mhauru.org>
Co-authored-by: Penelope Yong <penelopeysm@gmail.com>
Co-authored-by: Shravan Goswami <123811742+shravanngoswamii@users.noreply.github.com>

Author: 6 people

.github/workflows/Benchmarking.yml

@@ -0,0 +1,76 @@
+name: Benchmarking
+
+on:
+  pull_request:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up Julia
+        uses: julia-actions/setup-julia@v2
+        with:
+          version: '1'
+
+      - name: Install Dependencies
+        run: julia --project=benchmarks/ -e 'using Pkg; Pkg.instantiate()'
+
+      - name: Run Benchmarks
+        id: run_benchmarks
+        run: |
+          # Capture version info into a variable, print it, and set it as an env var for later steps
+          version_info=$(julia -e 'using InteractiveUtils; versioninfo()')
+          echo "$version_info"
+          echo "VERSION_INFO<<EOF" >> $GITHUB_ENV
+          echo "$version_info" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+          # Capture benchmark output into a variable
+          echo "Running Benchmarks..."
+          benchmark_output=$(julia --project=benchmarks benchmarks/benchmarks.jl)
+          
+          # Print benchmark results directly to the workflow log
+          echo "Benchmark Results:"
+          echo "$benchmark_output"
+          
+          # Set the benchmark output as an env var for later steps
+          echo "BENCHMARK_OUTPUT<<EOF" >> $GITHUB_ENV
+          echo "$benchmark_output" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+          # Get the current commit SHA of DynamicPPL
+          DPPL_COMMIT_SHA=$(git rev-parse HEAD)
+          echo "DPPL_COMMIT_SHA=$DPPL_COMMIT_SHA" >> $GITHUB_ENV
+
+          COMMIT_URL="https://github.com/${{ github.repository }}/commit/$DPPL_COMMIT_SHA"
+          echo "DPPL_COMMIT_URL=$COMMIT_URL" >> $GITHUB_ENV
+
+      - name: Find Existing Comment
+        uses: peter-evans/find-comment@v3
+        id: find_comment
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: github-actions[bot]
+
+      - name: Post Benchmark Results as PR Comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            ## Benchmark Report for Commit [`${{ env.DPPL_COMMIT_SHA }}`](${{ env.DPPL_COMMIT_URL }})
+            ### Computer Information
+            ```
+            ${{ env.VERSION_INFO }}
+            ```
+            ### Benchmark Results
+            ```
+            ${{ env.BENCHMARK_OUTPUT }}
+            ```
+          comment-id: ${{ steps.find_comment.outputs.comment-id }}
+          edit-mode: replace

benchmarks/Project.toml

@@ -3,11 +3,23 @@ uuid = "d94a1522-c11e-44a7-981a-42bf5dc1a001"
 version = "0.1.0"
 
 [deps]
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
-DiffUtils = "8294860b-85a6-42f8-8c35-d911f667b5f6"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 DynamicPPL = "366bfd00-2699-11ea-058f-f148b4cae6d8"
-LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
-Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-Weave = "44d3d7a6-8a23-5bf8-98c5-b353f8df5ec9"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+LogDensityProblems = "6fdf6af0-433a-55f7-b3ed-c6c6e0b8df7c"
+Mooncake = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6"
+PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
+ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
+StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
+
+[compat]
+ADTypes = "1.14.0"
+BenchmarkTools = "1.6.0"
+Distributions = "0.25.117"
+ForwardDiff = "0.10.38"
+LogDensityProblems = "2.1.2"
+PrettyTables = "2.4.0"
+ReverseDiff = "1.15.3"

benchmarks/README.md

@@ -1,27 +1,5 @@
-To run the benchmarks, simply do:
+To run the benchmarks, run this from the root directory of the repository:
 
 ```sh
-julia --project -e 'using DynamicPPLBenchmarks; weave_benchmarks();'
-```
-
-```julia
-julia> @doc weave_benchmarks
-  weave_benchmarks(input="benchmarks.jmd"; kwargs...)
-
-  Weave benchmarks present in benchmarks.jmd into a single file.
-
-  Keyword arguments
-  ≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡
-
-    •  benchmarkbody: JMD-file to be rendered for each model.
-
-    •  include_commit_id=false: specify whether to include commit-id in the default name.
-
-    •  name: the name of directory in results/ to use as output directory.
-
-    •  name_old=nothing: if specified, comparisons of current run vs. the run pinted to by name_old will be included in the generated document.
-
-    •  include_typed_code=false: if true, output of code_typed for the evaluator of the model will be included in the weaved document.
-
-    •  Rest of the passed kwargs will be passed on to Weave.weave.
-```
+julia --project=benchmarks benchmarks/benchmarks.jl
+```

benchmarks/benchmark_body.jmd

@@ -0,0 +1,103 @@
+using Pkg
+# To ensure we benchmark the local version of DynamicPPL, dev the folder above.
+Pkg.develop(; path=joinpath(@__DIR__, ".."))
+
+using DynamicPPLBenchmarks: Models, make_suite, model_dimension
+using BenchmarkTools: @benchmark, median, run
+using PrettyTables: PrettyTables, ft_printf
+using StableRNGs: StableRNG
+
+rng = StableRNG(23)
+
+# Create DynamicPPL.Model instances to run benchmarks on.
+smorgasbord_instance = Models.smorgasbord(randn(rng, 100), randn(rng, 100))
+loop_univariate1k, multivariate1k = begin
+    data_1k = randn(rng, 1_000)
+    loop = Models.loop_univariate(length(data_1k)) | (; o=data_1k)
+    multi = Models.multivariate(length(data_1k)) | (; o=data_1k)
+    loop, multi
+end
+loop_univariate10k, multivariate10k = begin
+    data_10k = randn(rng, 10_000)
+    loop = Models.loop_univariate(length(data_10k)) | (; o=data_10k)
+    multi = Models.multivariate(length(data_10k)) | (; o=data_10k)
+    loop, multi
+end
+lda_instance = begin
+    w = [1, 2, 3, 2, 1, 1]
+    d = [1, 1, 1, 2, 2, 2]
+    Models.lda(2, d, w)
+end
+
+# Specify the combinations to test:
+# (Model Name, model instance, VarInfo choice, AD backend, linked)
+chosen_combinations = [
+    (
+        "Simple assume observe",
+        Models.simple_assume_observe(randn(rng)),
+        :typed,
+        :forwarddiff,
+        false,
+    ),
+    ("Smorgasbord", smorgasbord_instance, :typed, :forwarddiff, false),
+    ("Smorgasbord", smorgasbord_instance, :simple_namedtuple, :forwarddiff, true),
+    ("Smorgasbord", smorgasbord_instance, :untyped, :forwarddiff, true),
+    ("Smorgasbord", smorgasbord_instance, :simple_dict, :forwarddiff, true),
+    ("Smorgasbord", smorgasbord_instance, :typed, :reversediff, true),
+    ("Smorgasbord", smorgasbord_instance, :typed, :mooncake, true),
+    ("Loop univariate 1k", loop_univariate1k, :typed, :mooncake, true),
+    ("Multivariate 1k", multivariate1k, :typed, :mooncake, true),
+    ("Loop univariate 10k", loop_univariate10k, :typed, :mooncake, true),
+    ("Multivariate 10k", multivariate10k, :typed, :mooncake, true),
+    ("Dynamic", Models.dynamic(), :typed, :mooncake, true),
+    ("Submodel", Models.parent(randn(rng)), :typed, :mooncake, true),
+    ("LDA", lda_instance, :typed, :reversediff, true),
+]
+
+# Time running a model-like function that does not use DynamicPPL, as a reference point.
+# Eval timings will be relative to this.
+reference_time = begin
+    obs = randn(rng)
+    median(@benchmark Models.simple_assume_observe_non_model(obs)).time
+end
+
+results_table = Tuple{String,Int,String,String,Bool,Float64,Float64}[]
+
+for (model_name, model, varinfo_choice, adbackend, islinked) in chosen_combinations
+    @info "Running benchmark for $model_name"
+    suite = make_suite(model, varinfo_choice, adbackend, islinked)
+    results = run(suite)
+    eval_time = median(results["evaluation"]).time
+    relative_eval_time = eval_time / reference_time
+    ad_eval_time = median(results["gradient"]).time
+    relative_ad_eval_time = ad_eval_time / eval_time
+    push!(
+        results_table,
+        (
+            model_name,
+            model_dimension(model, islinked),
+            string(adbackend),
+            string(varinfo_choice),
+            islinked,
+            relative_eval_time,
+            relative_ad_eval_time,
+        ),
+    )
+end
+
+table_matrix = hcat(Iterators.map(collect, zip(results_table...))...)
+header = [
+    "Model",
+    "Dimension",
+    "AD Backend",
+    "VarInfo Type",
+    "Linked",
+    "Eval Time / Ref Time",
+    "AD Time / Eval Time",
+]
+PrettyTables.pretty_table(
+    table_matrix;
+    header=header,
+    tf=PrettyTables.tf_markdown,
+    formatters=ft_printf("%.1f", [6, 7]),
+)