diff --git a/.github/workflows/compare-bench.yml b/.github/workflows/compare-bench.yml index 138c86cdc..3079638e0 100644 --- a/.github/workflows/compare-bench.yml +++ b/.github/workflows/compare-bench.yml @@ -21,6 +21,16 @@ on: - prove - prove-e2e default: prove-e2e + host_flamegraph: + description: "Run target profiling benchmark for host flamegraph" + type: boolean + required: false + default: false + guest_flamegraph: + description: "Run target profiling benchmark for circuit flamegraphs" + type: boolean + required: false + default: false instance_family: description: "Instance family to use for benchmark" type: string @@ -38,18 +48,57 @@ jobs: instance_family: ${{ github.event.inputs.instance_family }} secrets: inherit - run-target-benchmark: - name: "Run Target Benchmark" + # Only create branch because github workflow_call must run all jobs for "needs" + patch-target: + name: "Patch Target" uses: ./.github/workflows/update-patches.yml with: OPENVM_REV: ${{ github.event.inputs.target_rev }} - run_benchmark: true - benchmark_mode: ${{ github.event.inputs.benchmark_mode }} + run_benchmark: false + secrets: inherit + + run-target-benchmark: + name: "Run Target Benchmark" + uses: ./.github/workflows/reth-benchmark.yml + needs: patch-target + with: + mode: ${{ github.event.inputs.benchmark_mode }} + instance_family: ${{ github.event.inputs.instance_family }} + ref: ${{ needs.patch-target.outputs.branch_name }} + tag: ${{ needs.patch-target.outputs.tag }} + profiling: none + secrets: inherit + + target-host-flamegraph: + name: "Target Host Perf Flamegraph" + uses: ./.github/workflows/reth-benchmark.yml + if: ${{ github.event.inputs.host_flamegraph }} + needs: patch-target + with: + mode: ${{ github.event.inputs.benchmark_mode }} + instance_family: ${{ github.event.inputs.instance_family }} + ref: ${{ needs.patch-target.outputs.branch_name }} + tag: ${{ needs.patch-target.outputs.tag }} + profiling: host + secrets: inherit + + target-guest-flamegraph: + name: "Target Guest Circuit Flamegraphs" + uses: ./.github/workflows/reth-benchmark.yml + if: ${{ github.event.inputs.guest_flamegraph }} + needs: patch-target + with: + mode: ${{ github.event.inputs.benchmark_mode }} instance_family: ${{ github.event.inputs.instance_family }} + ref: ${{ needs.patch-target.outputs.branch_name }} + tag: ${{ needs.patch-target.outputs.tag }} + profiling: guest secrets: inherit compare-results: - needs: [run-base-benchmark, run-target-benchmark] + needs: + - run-base-benchmark + - run-target-benchmark runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/reth-benchmark.yml b/.github/workflows/reth-benchmark.yml index d78bf03f3..ba0eb984b 100644 --- a/.github/workflows/reth-benchmark.yml +++ b/.github/workflows/reth-benchmark.yml @@ -50,11 +50,15 @@ on: - prove - prove-e2e default: prove-e2e - collect_metrics: - type: boolean + profiling: + type: choice required: false - description: Whether to collect metrics for flamegraphs - default: false + description: Profiling mode + options: + - none + - host # host profiling (samply, flamegraphs, etc) + - guest # guest profiling (circuit flamegraphs) + default: none max_segment_length: type: number required: false @@ -106,11 +110,11 @@ on: required: false description: Running mode, one of execute, tracegen, prove, or prove-e2e default: prove-e2e - collect_metrics: - type: boolean + profiling: + type: string required: false - description: Whether to collect metrics for flamegraphs - default: false + description: Profiling mode (none, host, guest) + default: none max_segment_length: type: number required: false @@ -176,7 +180,7 @@ jobs: - name: Set metric name id: set-metric-name run: | - input_json_str="${{ toJSON(github.event.inputs || inputs) }}" + input_json_str="${{ toJSON(inputs || github.event.inputs) }}" input_hash=$(echo $input_json_str | sha256sum | cut -d' ' -f1) METRIC_NAME=reth-$current_sha-${input_hash} echo "METRIC_NAME=${METRIC_NAME}" >> $GITHUB_ENV @@ -191,12 +195,17 @@ jobs: TAG="${{ inputs.tag || env.current_sha }}" GUEST_PROFILE="release" HOST_PROFILE="maxperf" - if [[ "${{ inputs.collect_metrics || github.event.inputs.collect_metrics }}" == "true" ]]; then + if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "guest" ]]; then GUEST_PROFILE="profiling" HOST_PROFILE="profiling" fi + if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "host" ]]; then + HOST_PROFILE="profiling" + fi echo "elf_cache_key=elf-${TAG}-${arch}-${GUEST_PROFILE}" >> $GITHUB_OUTPUT echo "host_cache_key=host-${TAG}-${arch}-${HOST_PROFILE}" >> $GITHUB_OUTPUT + echo "guest_profile=${GUEST_PROFILE}" >> $GITHUB_OUTPUT + echo "host_profile=${HOST_PROFILE}" >> $GITHUB_OUTPUT - name: Load SSH key uses: webfactory/ssh-agent@v0.9.0 with: @@ -222,6 +231,8 @@ jobs: bash openvm/extensions/native/recursion/trusted_setup_s3.sh PARAMS_DIR=$(pwd)/params echo "PARAMS_DIR=$PARAMS_DIR" >> $GITHUB_ENV + OPTIONAL_ARGS="--kzg-params-dir $PARAMS_DIR --halo2-outer-k 22" + echo "OPTIONAL_ARGS=${OPTIONAL_ARGS}" >> $GITHUB_ENV - name: Restore Guest ELF from cache id: cache-guest-elf @@ -238,36 +249,27 @@ jobs: if: steps.cache-guest-elf.outputs.cache-hit != 'true' working-directory: bin/client-eth run: | - PROFILE="release" - if [[ "${{ github.event.inputs.collect_metrics || inputs.collect_metrics }}" == "true" ]]; then - PROFILE="profiling" - fi - cargo openvm build --no-transpile --profile=$PROFILE + GUEST_PROFILE=${{ steps.set-cache-keys.outputs.guest_profile }} + cargo openvm build --no-transpile --profile=$GUEST_PROFILE mkdir -p ../host/elf - cp target/riscv32im-risc0-zkvm-elf/$PROFILE/openvm-client-eth ../host/elf/ + cp target/riscv32im-risc0-zkvm-elf/$GUEST_PROFILE/openvm-client-eth ../host/elf/ - name: Set build args id: set-build-args run: | - PROFILE="maxperf" - OPTIONAL_ARGS="" FEATURES="bench-metrics,nightly-features,${{ inputs.memory_allocator || github.event.inputs.memory_allocator }}" - if [[ "${{ inputs.collect_metrics || github.event.inputs.collect_metrics }}" == "true" ]]; then - OPTIONAL_ARGS="--profiling --kzg-params-dir $PARAMS_DIR" - PROFILE="profiling" + if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "guest" ]]; then + OPTIONAL_ARGS="${OPTIONAL_ARGS} --profiling" FEATURES="${FEATURES},profiling" fi - OPTIONAL_ARGS="${OPTIONAL_ARGS} --halo2-outer-k 22" echo "OPTIONAL_ARGS=${OPTIONAL_ARGS}" >> $GITHUB_ENV - echo "PROFILE=${PROFILE}" >> $GITHUB_ENV echo "FEATURES=${FEATURES}" >> $GITHUB_ENV - echo "profile=${PROFILE}" >> $GITHUB_OUTPUT - name: Restore Host Binary from cache id: cache-host-binary uses: runs-on/cache@v4 with: - path: target/${{ steps.set-build-args.outputs.profile }}/openvm-reth-benchmark + path: target/${{ steps.set-cache-keys.outputs.host_profile }}/openvm-reth-benchmark key: ${{ steps.set-cache-keys.outputs.host_cache_key }} - name: Build Host Binary @@ -287,24 +289,56 @@ jobs: exit 1 ;; esac + HOST_PROFILE=${{ steps.set-cache-keys.outputs.host_profile }} + if [[ "$HOST_PROFILE" == "profiling" ]]; then + RUSTFLAGS="${RUSTFLAGS} -g -C force-frame-pointers=yes" + fi export JEMALLOC_SYS_WITH_MALLOC_CONF="retain:true,background_thread:true,metadata_thp:always,thp:always,dirty_decay_ms:-1,muzzy_decay_ms:-1,abort_conf:true" - RUSTFLAGS=$RUSTFLAGS cargo build --bin openvm-reth-benchmark --profile=$PROFILE --no-default-features --features=$FEATURES + RUSTFLAGS=$RUSTFLAGS cargo build --bin openvm-reth-benchmark --profile=$HOST_PROFILE --no-default-features --features=$FEATURES echo "JEMALLOC_SYS_WITH_MALLOC_CONF=${JEMALLOC_SYS_WITH_MALLOC_CONF}" >> $GITHUB_ENV - - name: Run benchmark + - name: Set up run benchmark script run: | mkdir -p rpc-cache mkdir -p .bench_metrics RPC_1=${{ secrets.RPC_URL_1 }} MODE=${{ inputs.mode || github.event.inputs.mode }} BLOCK_NUMBER=${{ inputs.block_number || github.event.inputs.block_number }} - export JEMALLOC_SYS_WITH_MALLOC_CONF=${JEMALLOC_SYS_WITH_MALLOC_CONF} - RUST_LOG="info,p3_=warn" OUTPUT_PATH=${METRIC_PATH} ./target/$PROFILE/openvm-reth-benchmark \ + HOST_PROFILE=${{ steps.set-cache-keys.outputs.host_profile }} + cat > run_benchmark.sh <> $GITHUB_ENV - s5cmd cp ${METRIC_PATH} ${{ env.S3_METRICS_PATH }}/${METRIC_NAME}.json + run: s5cmd cp ${METRIC_PATH} ${{ env.S3_METRICS_PATH }}/${METRIC_NAME}.json - name: Generate markdown run: | @@ -331,6 +361,7 @@ jobs: - uses: bencherdev/bencher@main - name: Upload bencher metrics + if: ${{ inputs.profiling || github.event.inputs.profiling || 'none' }} == 'none' run: | current_branch=$(git rev-parse --abbrev-ref HEAD) if [[ "$current_branch" == nightly* ]]; then @@ -348,23 +379,21 @@ jobs: --adapter json \ --file $BENCHER_METRIC_PATH - - name: Install inferno-flamegraph - if: ${{ inputs.collect_metrics == true || github.event.inputs.collect_metrics == true }} - run: cargo install inferno - - - name: Generate flamegraphs - if: ${{ inputs.collect_metrics == true || github.event.inputs.collect_metrics == true }} + - name: Generate circuit flamegraphs + if: ${{ inputs.profiling == 'guest' || github.event.inputs.profiling == 'guest' }} run: | + cargo install inferno + python3 openvm/ci/scripts/metric_unify/flamegraph.py $METRIC_PATH - s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.S3_FLAMEGRAPHS_PATH }}/${current_sha}/" + s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.S3_FLAMEGRAPHS_PATH }}/${METRIC_NAME}/" echo "" >> $MD_PATH echo "
" >> $MD_PATH - echo "Flamegraphs" >> $MD_PATH + echo "Circuit Flamegraphs" >> $MD_PATH echo "" >> $MD_PATH for file in .bench_metrics/flamegraphs/*.svg; do filename=$(basename "$file") - flamegraph_url=${{ env.S3_FLAMEGRAPHS_URL }}/${current_sha}/${filename} + flamegraph_url=${{ env.S3_FLAMEGRAPHS_URL }}/${METRIC_NAME}/${filename} echo "[![]($flamegraph_url)]($flamegraph_url)" >> $MD_PATH done rm -f .bench_metrics/flamegraphs/*.svg @@ -392,6 +421,24 @@ jobs: s5cmd cp $MD_PATH $S3_MD_PATH echo "S3_MD_PATH=${S3_MD_PATH}" >> $GITHUB_ENV + - name: Upload host flamegraph + if: ${{ inputs.profiling == 'host' || github.event.inputs.profiling == 'host' }} + run: | + cargo install inferno + + perf script -i perf.data | inferno-collapse-perf > stacks.folded + cat stacks.folded | inferno-flamegraph > flamegraph.svg + cat stacks.folded | inferno-flamegraph --reverse --inverted > flamegraph-reverse.svg + + s5cmd cp flamegraph.svg ${{ env.S3_FLAMEGRAPHS_PATH }}/${METRIC_NAME}/host.svg + s5cmd cp flamegraph-reverse.svg ${{ env.S3_FLAMEGRAPHS_PATH }}/${METRIC_NAME}/host-reverse.svg + flamegraph_url=${{ env.S3_FLAMEGRAPHS_URL }}/${METRIC_NAME}/host.svg + flamegraph_reverse_url=${{ env.S3_FLAMEGRAPHS_URL }}/${METRIC_NAME}/host-reverse.svg + echo "" >> $GITHUB_STEP_SUMMARY + echo "Host Perf Flamegraph: $flamegraph_url" >> $GITHUB_STEP_SUMMARY + echo "Host Perf Reverse Flamegraph: $flamegraph_reverse_url" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + ### Update gh-pages - uses: actions/checkout@v4 with: diff --git a/.github/workflows/update-openvm.yml b/.github/workflows/update-openvm.yml index 827d83404..d24c060b2 100644 --- a/.github/workflows/update-openvm.yml +++ b/.github/workflows/update-openvm.yml @@ -10,7 +10,7 @@ on: required: false jobs: - update-config: + update: name: "Update workspace and guest Cargo.toml and open a pull request" runs-on: - ubuntu-latest diff --git a/.github/workflows/update-patches.yml b/.github/workflows/update-patches.yml index bcb1edfb2..cb1057a08 100644 --- a/.github/workflows/update-patches.yml +++ b/.github/workflows/update-patches.yml @@ -14,6 +14,16 @@ on: type: boolean required: false default: true + host_flamegraph: + description: "Run profiling benchmark for host flamegraph" + type: boolean + required: false + default: false + guest_flamegraph: + description: "Run profiling benchmark for circuit flamegraphs" + type: boolean + required: false + default: false benchmark_mode: description: "Benchmark mode (if running benchmark)" type: string @@ -41,6 +51,16 @@ on: required: false description: "Run reth benchmark after patching" default: true + host_flamegraph: + description: "Run profiling benchmark for host flamegraph" + type: boolean + required: false + default: false + guest_flamegraph: + description: "Run profiling benchmark for circuit flamegraphs" + type: boolean + required: false + default: false benchmark_mode: type: string required: false @@ -68,9 +88,15 @@ on: metric_name: description: "Name of the metric" value: ${{ jobs.run-benchmark.outputs.metric_name }} + branch_name: + description: "Name of the created branch" + value: ${{ jobs.patch.outputs.branch_name }} + tag: + description: "Tag" + value: ${{ jobs.patch.outputs.tag }} jobs: - update-config: + patch: name: "Update .cargo/config.toml and open a pull request" runs-on: ["runs-on", "runner=2cpu-linux-arm64", "run-id=${{ github.run_id }}"] @@ -192,22 +218,44 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} run-benchmark: - needs: update-config - if: ${{ github.event.inputs.run_benchmark == 'true' || github.event_name == 'schedule' || inputs.run_benchmark }} + needs: patch + if: ${{ github.event.inputs.run_benchmark == 'true' || inputs.run_benchmark || github.event_name == 'schedule' }} uses: ./.github/workflows/reth-benchmark.yml with: mode: ${{ inputs.benchmark_mode || github.event.inputs.benchmark_mode || 'prove-e2e' }} instance_family: ${{ inputs.instance_family || github.event.inputs.instance_family || 'm8g.24xlarge' }} - ref: ${{ needs.update-config.outputs.branch_name }} - tag: ${{ needs.update-config.outputs.tag }} - secrets: - GH_ACTIONS_DEPLOY_PRIVATE_KEY: ${{ secrets.GH_ACTIONS_DEPLOY_PRIVATE_KEY }} - RPC_URL_1: ${{ secrets.RPC_URL_1 }} - BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + ref: ${{ needs.patch.outputs.branch_name }} + tag: ${{ needs.patch.outputs.tag }} + profiling: none + secrets: inherit + + host-flamegraph: + needs: patch + if: ${{ github.event.inputs.host_flamegraph == 'true' || inputs.host_flamegraph || github.event_name == 'schedule' }} + uses: ./.github/workflows/reth-benchmark.yml + with: + mode: ${{ inputs.benchmark_mode || github.event.inputs.benchmark_mode || 'prove-e2e' }} + instance_family: ${{ inputs.instance_family || github.event.inputs.instance_family || 'm8g.24xlarge' }} + ref: ${{ needs.patch.outputs.branch_name }} + tag: ${{ needs.patch.outputs.tag }} + profiling: host + secrets: inherit + + guest-flamegraph: + needs: patch + if: ${{ github.event.inputs.guest_flamegraph == 'true' || inputs.guest_flamegraph || github.event_name == 'schedule' }} + uses: ./.github/workflows/reth-benchmark.yml + with: + mode: ${{ inputs.benchmark_mode || github.event.inputs.benchmark_mode || 'prove-e2e' }} + instance_family: ${{ inputs.instance_family || github.event.inputs.instance_family || 'm8g.24xlarge' }} + ref: ${{ needs.patch.outputs.branch_name }} + tag: ${{ needs.patch.outputs.tag }} + profiling: guest + secrets: inherit close-pr: - needs: [update-config, run-benchmark] - if: ${{ !inputs.manual_call }} + needs: [patch, run-benchmark] + if: ${{ always() && !inputs.manual_call }} runs-on: ubuntu-latest steps: - name: Close PR @@ -218,6 +266,6 @@ jobs: await github.rest.pulls.update({ owner: context.repo.owner, repo: context.repo.repo, - pull_number: ${{ needs.update-config.outputs.pr_number }}, + pull_number: ${{ needs.patch.outputs.pr_number }}, state: 'closed' }); diff --git a/Cargo.toml b/Cargo.toml index e1a3294ce..364873038 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,9 +109,9 @@ rustdoc.all = "warn" lto = "thin" [profile.profiling] -inherits = "bench" -debug = 2 -strip = false +inherits = "maxperf" +debug = "full" +strip = "none" [profile.maxperf] inherits = "release"