feat(ci): host perf flamegraph generation

.github/workflows/compare-bench.yml

@@ -21,6 +21,16 @@ on:
           - prove
           - prove-e2e
         default: prove-e2e
+      host_flamegraph:
+        description: "Run target profiling benchmark for host flamegraph"
+        type: boolean
+        required: false
+        default: false
+      guest_flamegraph:
+        description: "Run target profiling benchmark for circuit flamegraphs"
+        type: boolean
+        required: false
+        default: false
       instance_family:
         description: "Instance family to use for benchmark"
         type: string
@@ -38,18 +48,57 @@ jobs:
       instance_family: ${{ github.event.inputs.instance_family }}
     secrets: inherit
 
-  run-target-benchmark:
-    name: "Run Target Benchmark"
+  # Only create branch because github workflow_call must run all jobs for "needs"
+  patch-target:
+    name: "Patch Target"
     uses: ./.github/workflows/update-patches.yml
     with:
       OPENVM_REV: ${{ github.event.inputs.target_rev }}
-      run_benchmark: true
-      benchmark_mode: ${{ github.event.inputs.benchmark_mode }}
+      run_benchmark: false
+    secrets: inherit
+
+  run-target-benchmark:
+    name: "Run Target Benchmark"
+    uses: ./.github/workflows/reth-benchmark.yml
+    needs: patch-target
+    with:
+      mode: ${{ github.event.inputs.benchmark_mode }}
+      instance_family: ${{ github.event.inputs.instance_family }}
+      ref: ${{ needs.patch-target.outputs.branch_name }}
+      tag: ${{ needs.patch-target.outputs.tag }}
+      profiling: none
+    secrets: inherit
+
+  target-host-flamegraph:
+    name: "Target Host Perf Flamegraph"
+    uses: ./.github/workflows/reth-benchmark.yml
+    if: ${{ github.event.inputs.host_flamegraph }}
+    needs: patch-target
+    with:
+      mode: ${{ github.event.inputs.benchmark_mode }}
+      instance_family: ${{ github.event.inputs.instance_family }}
+      ref: ${{ needs.patch-target.outputs.branch_name }}
+      tag: ${{ needs.patch-target.outputs.tag }}
+      profiling: host
+    secrets: inherit
+
+  target-guest-flamegraph:
+    name: "Target Guest Circuit Flamegraphs"
+    uses: ./.github/workflows/reth-benchmark.yml
+    if: ${{ github.event.inputs.guest_flamegraph }}
+    needs: patch-target
+    with:
+      mode: ${{ github.event.inputs.benchmark_mode }}
       instance_family: ${{ github.event.inputs.instance_family }}
+      ref: ${{ needs.patch-target.outputs.branch_name }}
+      tag: ${{ needs.patch-target.outputs.tag }}
+      profiling: guest
     secrets: inherit
 
   compare-results:
-    needs: [run-base-benchmark, run-target-benchmark]
+    needs:
+      - run-base-benchmark
+      - run-target-benchmark
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

.github/workflows/reth-benchmark.yml

@@ -50,11 +50,15 @@ on:
           - prove
           - prove-e2e
         default: prove-e2e
-      collect_metrics:
-        type: boolean
+      profiling:
+        type: choice
         required: false
-        description: Whether to collect metrics for flamegraphs
-        default: false
+        description: Profiling mode
+        options:
+          - none
+          - host # host profiling (samply, flamegraphs, etc)
+          - guest # guest profiling (circuit flamegraphs)
+        default: none
       max_segment_length:
         type: number
         required: false
@@ -106,11 +110,11 @@ on:
         required: false
         description: Running mode, one of execute, tracegen, prove, or prove-e2e
         default: prove-e2e
-      collect_metrics:
-        type: boolean
+      profiling:
+        type: string
         required: false
-        description: Whether to collect metrics for flamegraphs
-        default: false
+        description: Profiling mode (none, host, guest)
+        default: none
       max_segment_length:
         type: number
         required: false
@@ -176,7 +180,7 @@ jobs:
       - name: Set metric name
         id: set-metric-name
         run: |
-          input_json_str="${{ toJSON(github.event.inputs || inputs) }}"
+          input_json_str="${{ toJSON(inputs || github.event.inputs) }}"
           input_hash=$(echo $input_json_str | sha256sum | cut -d' ' -f1)
           METRIC_NAME=reth-$current_sha-${input_hash}
           echo "METRIC_NAME=${METRIC_NAME}" >> $GITHUB_ENV
@@ -191,12 +195,17 @@ jobs:
           TAG="${{ inputs.tag || env.current_sha }}"
           GUEST_PROFILE="release"
           HOST_PROFILE="maxperf"
-          if [[ "${{ inputs.collect_metrics || github.event.inputs.collect_metrics }}" == "true" ]]; then
+          if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "guest" ]]; then
             GUEST_PROFILE="profiling"
             HOST_PROFILE="profiling"
           fi
+          if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "host" ]]; then
+            HOST_PROFILE="profiling"
+          fi
           echo "elf_cache_key=elf-${TAG}-${arch}-${GUEST_PROFILE}" >> $GITHUB_OUTPUT
           echo "host_cache_key=host-${TAG}-${arch}-${HOST_PROFILE}" >> $GITHUB_OUTPUT
+          echo "guest_profile=${GUEST_PROFILE}" >> $GITHUB_OUTPUT
+          echo "host_profile=${HOST_PROFILE}" >> $GITHUB_OUTPUT
       - name: Load SSH key
         uses: webfactory/ssh-agent@v0.9.0
         with:
@@ -222,6 +231,8 @@ jobs:
           bash openvm/extensions/native/recursion/trusted_setup_s3.sh
           PARAMS_DIR=$(pwd)/params
           echo "PARAMS_DIR=$PARAMS_DIR" >> $GITHUB_ENV
+          OPTIONAL_ARGS="--kzg-params-dir $PARAMS_DIR --halo2-outer-k 22"
+          echo "OPTIONAL_ARGS=${OPTIONAL_ARGS}" >> $GITHUB_ENV
 
       - name: Restore Guest ELF from cache
         id: cache-guest-elf
@@ -238,36 +249,27 @@ jobs:
         if: steps.cache-guest-elf.outputs.cache-hit != 'true'
         working-directory: bin/client-eth
         run: |
-          PROFILE="release"
-          if [[ "${{ github.event.inputs.collect_metrics || inputs.collect_metrics }}" == "true" ]]; then
-            PROFILE="profiling"
-          fi
-          cargo openvm build --no-transpile --profile=$PROFILE
+          GUEST_PROFILE=${{ steps.set-cache-keys.outputs.guest_profile }}
+          cargo openvm build --no-transpile --profile=$GUEST_PROFILE
           mkdir -p ../host/elf
-          cp target/riscv32im-risc0-zkvm-elf/$PROFILE/openvm-client-eth ../host/elf/
+          cp target/riscv32im-risc0-zkvm-elf/$GUEST_PROFILE/openvm-client-eth ../host/elf/
 
       - name: Set build args
         id: set-build-args
         run: |
-          PROFILE="maxperf"
-          OPTIONAL_ARGS=""
           FEATURES="bench-metrics,nightly-features,${{ inputs.memory_allocator || github.event.inputs.memory_allocator }}"
-          if [[ "${{ inputs.collect_metrics || github.event.inputs.collect_metrics }}" == "true" ]]; then
-            OPTIONAL_ARGS="--profiling --kzg-params-dir $PARAMS_DIR"
-            PROFILE="profiling"
+          if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "guest" ]]; then
+            OPTIONAL_ARGS="${OPTIONAL_ARGS} --profiling"
             FEATURES="${FEATURES},profiling"
           fi
-          OPTIONAL_ARGS="${OPTIONAL_ARGS} --halo2-outer-k 22"
           echo "OPTIONAL_ARGS=${OPTIONAL_ARGS}" >> $GITHUB_ENV
-          echo "PROFILE=${PROFILE}" >> $GITHUB_ENV
           echo "FEATURES=${FEATURES}" >> $GITHUB_ENV
-          echo "profile=${PROFILE}" >> $GITHUB_OUTPUT
 
       - name: Restore Host Binary from cache
         id: cache-host-binary
         uses: runs-on/cache@v4
         with:
-          path: target/${{ steps.set-build-args.outputs.profile }}/openvm-reth-benchmark
+          path: target/${{ steps.set-cache-keys.outputs.host_profile }}/openvm-reth-benchmark
           key: ${{ steps.set-cache-keys.outputs.host_cache_key }}
 
       - name: Build Host Binary
@@ -287,24 +289,56 @@ jobs:
               exit 1
               ;;
           esac
+          HOST_PROFILE=${{ steps.set-cache-keys.outputs.host_profile }}
+          if [[ "$HOST_PROFILE" == "profiling" ]]; then
+              RUSTFLAGS="${RUSTFLAGS} -g -C force-frame-pointers=yes"
+          fi
           export JEMALLOC_SYS_WITH_MALLOC_CONF="retain:true,background_thread:true,metadata_thp:always,thp:always,dirty_decay_ms:-1,muzzy_decay_ms:-1,abort_conf:true"
-          RUSTFLAGS=$RUSTFLAGS cargo build --bin openvm-reth-benchmark --profile=$PROFILE --no-default-features --features=$FEATURES
+          RUSTFLAGS=$RUSTFLAGS cargo build --bin openvm-reth-benchmark --profile=$HOST_PROFILE --no-default-features --features=$FEATURES
           echo "JEMALLOC_SYS_WITH_MALLOC_CONF=${JEMALLOC_SYS_WITH_MALLOC_CONF}" >> $GITHUB_ENV
 
-      - name: Run benchmark
+      - name: Set up run benchmark script
         run: |
           mkdir -p rpc-cache
           mkdir -p .bench_metrics
           RPC_1=${{ secrets.RPC_URL_1 }}
           MODE=${{ inputs.mode || github.event.inputs.mode }}
           BLOCK_NUMBER=${{ inputs.block_number || github.event.inputs.block_number }}
-          export JEMALLOC_SYS_WITH_MALLOC_CONF=${JEMALLOC_SYS_WITH_MALLOC_CONF}
-          RUST_LOG="info,p3_=warn" OUTPUT_PATH=${METRIC_PATH} ./target/$PROFILE/openvm-reth-benchmark \
+          HOST_PROFILE=${{ steps.set-cache-keys.outputs.host_profile }}
+          cat > run_benchmark.sh <<EOF
+          ./target/$HOST_PROFILE/openvm-reth-benchmark \
             --$MODE --block-number $BLOCK_NUMBER --rpc-url $RPC_1 --cache-dir rpc-cache \
             --app-log-blowup ${{ inputs.app_log_blowup || github.event.inputs.app_log_blowup }} \
             --leaf-log-blowup ${{ inputs.leaf_log_blowup || github.event.inputs.leaf_log_blowup }} \
             --root-log-blowup ${{ inputs.root_log_blowup || github.event.inputs.root_log_blowup }} \
             --max-segment-length ${{ inputs.max_segment_length || github.event.inputs.max_segment_length }} $OPTIONAL_ARGS
+          EOF
+          chmod +x run_benchmark.sh
+
+      - name: Run benchmark
+        run: |
+          export JEMALLOC_SYS_WITH_MALLOC_CONF=${JEMALLOC_SYS_WITH_MALLOC_CONF}
+          export RUST_LOG="info,p3_=warn"
+          export OUTPUT_PATH=${METRIC_PATH}
+
+          if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "host" ]]; then
+            echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid
+            echo 0 | sudo tee /proc/sys/kernel/kptr_restrict
+
+            # install custom addr2line: https://crates.io/crates/flamegraph
+            rm -rf /tmp/addr2line
+            git clone https://github.com/gimli-rs/addr2line /tmp/addr2line
+            work_dir=$(pwd)
+            cd /tmp/addr2line
+            cargo build --release --bin addr2line --features bin
+            sudo mv target/release/addr2line /usr/local/bin/
+            cd $work_dir
+
+            perf --version
+            perf record -F 99 --call-graph dwarf -o perf.data -- ./run_benchmark.sh
+          else
+            ./run_benchmark.sh
+          fi
 
       - name: Upload metric file
         id: upload-metric-artifact
@@ -315,11 +349,7 @@ jobs:
           retention-days: 1
 
       - name: Upload Benchmark Metrics
-        run: |
-          current_sha=$(git rev-parse HEAD)
-          echo "Current SHA: $current_sha"
-          echo "current_sha=${current_sha}" >> $GITHUB_ENV
-          s5cmd cp ${METRIC_PATH} ${{ env.S3_METRICS_PATH }}/${METRIC_NAME}.json
+        run: s5cmd cp ${METRIC_PATH} ${{ env.S3_METRICS_PATH }}/${METRIC_NAME}.json
 
       - name: Generate markdown
         run: |
@@ -331,6 +361,7 @@ jobs:
 
       - uses: bencherdev/bencher@main
       - name: Upload bencher metrics
+        if: ${{ inputs.profiling || github.event.inputs.profiling || 'none' }} == 'none'
         run: |
           current_branch=$(git rev-parse --abbrev-ref HEAD)
           if [[ "$current_branch" == nightly* ]]; then
@@ -348,23 +379,21 @@ jobs:
             --adapter json \
             --file $BENCHER_METRIC_PATH
 
-      - name: Install inferno-flamegraph
-        if: ${{ inputs.collect_metrics == true || github.event.inputs.collect_metrics == true }}
-        run: cargo install inferno
-
-      - name: Generate flamegraphs
-        if: ${{ inputs.collect_metrics == true || github.event.inputs.collect_metrics == true }}
+      - name: Generate circuit flamegraphs
+        if: ${{ inputs.profiling == 'guest' || github.event.inputs.profiling == 'guest' }}
         run: |
+          cargo install inferno
+
           python3 openvm/ci/scripts/metric_unify/flamegraph.py $METRIC_PATH
-          s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.S3_FLAMEGRAPHS_PATH }}/${current_sha}/"
+          s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.S3_FLAMEGRAPHS_PATH }}/${METRIC_NAME}/"
 
           echo "" >> $MD_PATH
           echo "<details>" >> $MD_PATH
-          echo "<summary>Flamegraphs</summary>" >> $MD_PATH
+          echo "<summary>Circuit Flamegraphs</summary>" >> $MD_PATH
           echo "" >> $MD_PATH
           for file in .bench_metrics/flamegraphs/*.svg; do
             filename=$(basename "$file")
-            flamegraph_url=${{ env.S3_FLAMEGRAPHS_URL }}/${current_sha}/${filename}
+            flamegraph_url=${{ env.S3_FLAMEGRAPHS_URL }}/${METRIC_NAME}/${filename}
             echo "[![]($flamegraph_url)]($flamegraph_url)" >> $MD_PATH
           done
           rm -f .bench_metrics/flamegraphs/*.svg
@@ -392,6 +421,24 @@ jobs:
           s5cmd cp $MD_PATH $S3_MD_PATH
           echo "S3_MD_PATH=${S3_MD_PATH}" >> $GITHUB_ENV
 
+      - name: Upload host flamegraph
+        if: ${{ inputs.profiling == 'host' || github.event.inputs.profiling == 'host' }}
+        run: |
+          cargo install inferno
+
+          perf script -i perf.data | inferno-collapse-perf > stacks.folded
+          cat stacks.folded | inferno-flamegraph > flamegraph.svg
+          cat stacks.folded | inferno-flamegraph --reverse --inverted > flamegraph-reverse.svg
+
+          s5cmd cp flamegraph.svg ${{ env.S3_FLAMEGRAPHS_PATH }}/${METRIC_NAME}/host.svg
+          s5cmd cp flamegraph-reverse.svg ${{ env.S3_FLAMEGRAPHS_PATH }}/${METRIC_NAME}/host-reverse.svg
+          flamegraph_url=${{ env.S3_FLAMEGRAPHS_URL }}/${METRIC_NAME}/host.svg
+          flamegraph_reverse_url=${{ env.S3_FLAMEGRAPHS_URL }}/${METRIC_NAME}/host-reverse.svg
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Host Perf Flamegraph: $flamegraph_url" >> $GITHUB_STEP_SUMMARY
+          echo "Host Perf Reverse Flamegraph: $flamegraph_reverse_url" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
       ### Update gh-pages
       - uses: actions/checkout@v4
         with:

.github/workflows/update-openvm.yml

@@ -10,7 +10,7 @@ on:
         required: false
 
 jobs:
-  update-config:
+  update:
     name: "Update workspace and guest Cargo.toml and open a pull request"
     runs-on:
       - ubuntu-latest