Reth Benchmark (block 22000066) #3003
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Reth Benchmark" | |
| run-name: "Reth Benchmark (block ${{ inputs.block_number || github.event.inputs.block_number }})" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| # Default runner is not big enough for this | |
| # https://aws.amazon.com/ec2/instance-types/ | |
| instance_family: | |
| type: string | |
| required: false | |
| description: The family of the instance, can be multiple ones concat with "+" e.g. r8g+r7g | |
| default: m8g.24xlarge | |
| block_number: | |
| type: number | |
| required: false | |
| description: Block number to run the benchmark on | |
| default: 21000000 | |
| app_log_blowup: | |
| type: number | |
| required: false | |
| description: Application level log blowup | |
| default: 1 | |
| leaf_log_blowup: | |
| type: number | |
| required: false | |
| description: Aggregation (leaf) level log blowup | |
| default: 1 | |
| internal_log_blowup: | |
| type: number | |
| required: false | |
| description: Internal level log blowup (only for e2e) | |
| default: 2 | |
| root_log_blowup: | |
| type: number | |
| required: false | |
| description: Root level log blowup (only for e2e) | |
| default: 3 | |
| mode: | |
| type: choice | |
| required: false | |
| description: Running mode | |
| options: | |
| - execute | |
| - tracegen | |
| - prove | |
| - prove-e2e | |
| default: prove-e2e | |
| profiling: | |
| type: choice | |
| required: false | |
| description: Profiling mode | |
| options: | |
| - none | |
| - host # host profiling (samply, flamegraphs, etc) | |
| - guest # guest profiling (circuit flamegraphs) | |
| default: none | |
| max_segment_length: | |
| type: number | |
| required: false | |
| description: Max segment length for continuations, must be larger than 524288 | |
| default: 8388508 # 2**23 - 100 | |
| max_cells_per_chip_in_segment: | |
| type: number | |
| required: false | |
| description: Max cells per chip in segment for continuations | |
| default: 1006620960 # ((2**23 - 100) * 120) | |
| workflow_call: | |
| inputs: | |
| ref: | |
| type: string | |
| required: false | |
| description: Git ref to checkout | |
| instance_family: | |
| type: string | |
| required: false | |
| description: The family of the instance, can be multiple ones concat with "+" e.g. r8g+r7g | |
| default: m8g.24xlarge | |
| memory_allocator: | |
| type: string | |
| required: false | |
| description: Memory allocator to use (mimalloc or jemalloc) | |
| default: jemalloc | |
| block_number: | |
| type: number | |
| required: false | |
| description: Block number to run the benchmark on | |
| default: 21000000 | |
| app_log_blowup: | |
| type: number | |
| required: false | |
| description: Application level log blowup | |
| default: 1 | |
| leaf_log_blowup: | |
| type: number | |
| required: false | |
| description: Aggregation (leaf) level log blowup | |
| default: 1 | |
| internal_log_blowup: | |
| type: number | |
| required: false | |
| description: Internal level log blowup (only for e2e) | |
| default: 2 | |
| root_log_blowup: | |
| type: number | |
| required: false | |
| description: Root level log blowup (only for e2e) | |
| default: 3 | |
| mode: | |
| type: string | |
| required: false | |
| description: Running mode, one of execute, tracegen, prove, or prove-e2e | |
| default: prove-e2e | |
| profiling: | |
| type: string | |
| required: false | |
| description: Profiling mode (none, host, guest) | |
| default: none | |
| max_segment_length: | |
| type: number | |
| required: false | |
| description: Max segment length for continuations, must be larger than 524288 | |
| default: 8388508 | |
| max_cells_per_chip_in_segment: | |
| type: number | |
| required: false | |
| description: Max cells per chip in segment for continuations | |
| default: 1006620960 | |
| tag: | |
| type: string | |
| required: false | |
| description: Tag for cache keys (default is commit hash) | |
| secrets: | |
| GH_ACTIONS_DEPLOY_PRIVATE_KEY: | |
| required: true | |
| RPC_URL_1: | |
| required: true | |
| BENCHER_API_TOKEN: | |
| required: true | |
| outputs: | |
| metric_name: | |
| description: "Name of the metric" | |
| value: ${{ jobs.run-reth.outputs.metric_name }} | |
| env: | |
| S3_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/results | |
| S3_METRICS_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/metrics | |
| S3_FLAMEGRAPHS_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/flamegraphs | |
| S3_FLAMEGRAPHS_URL: https://axiom-public-data-sandbox-us-east-1.s3.us-east-1.amazonaws.com/benchmark/github/flamegraphs | |
| CMD_ARGS: "" | |
| INPUT_ARGS: "" | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| BENCHER_PROJECT: openvm-reth-benchmark | |
| jobs: | |
| run-reth: | |
| name: Run Reth benchmark | |
| runs-on: | |
| - runs-on | |
| - run-id=${{ github.run_id }} | |
| - family=${{ inputs.instance_family || github.event.inputs.instance_family }} | |
| - disk=large | |
| - tag=bench-reth-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} | |
| - extras=s3-cache | |
| - image=${{ contains(inputs.instance_family || github.event.inputs.instance_family, 'g.') && 'ubuntu22-full-arm64' || 'ubuntu22-full-x64' }} | |
| outputs: | |
| metric_name: ${{ steps.set-metric-name.outputs.name }} | |
| steps: | |
| - uses: runs-on/action@v1 | |
| - name: Load SSH key | |
| uses: webfactory/ssh-agent@v0.9.0 | |
| with: | |
| ssh-private-key: | | |
| ${{ secrets.GH_ACTIONS_DEPLOY_PRIVATE_KEY }} | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.ref || github.head_ref || github.ref }} | |
| - uses: dtolnay/rust-toolchain@nightly | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| cache-on-failure: true | |
| - name: Display workflow inputs | |
| run: | | |
| if [[ "${{ inputs.manual_call }}" == "true" ]]; then | |
| echo "${{ toJSON(inputs) }}" | |
| else | |
| echo "${{ toJSON(github.event.inputs) }}" | |
| fi | |
| - name: Get current commit hash | |
| run: echo "current_sha=$(git rev-parse HEAD)" >> $GITHUB_ENV | |
| - name: Set metric name | |
| id: set-metric-name | |
| run: | | |
| run_id="${{github.run_id}}" | |
| METRIC_NAME=reth-$current_sha-${run_id} | |
| echo "METRIC_NAME=${METRIC_NAME}" >> $GITHUB_ENV | |
| METRIC_PATH=".bench_metrics/${METRIC_NAME}.json" | |
| echo "METRIC_PATH=${METRIC_PATH}" >> $GITHUB_ENV | |
| echo "name=${METRIC_NAME}" >> $GITHUB_OUTPUT | |
| - name: Set cache keys | |
| id: set-cache-keys | |
| run: | | |
| arch=$(uname -m) | |
| # Use inputs.tag if defined, otherwise use current commit SHA | |
| TAG="${{ inputs.tag || env.current_sha }}" | |
| GUEST_PROFILE="release" | |
| HOST_PROFILE="maxperf" | |
| if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "guest" ]]; then | |
| GUEST_PROFILE="profiling" | |
| HOST_PROFILE="profiling" | |
| fi | |
| if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "host" ]]; then | |
| HOST_PROFILE="profiling" | |
| fi | |
| echo "elf_cache_key=elf-${TAG}-${arch}-${GUEST_PROFILE}" >> $GITHUB_OUTPUT | |
| echo "host_cache_key=host-${TAG}-${arch}-${HOST_PROFILE}" >> $GITHUB_OUTPUT | |
| echo "guest_profile=${GUEST_PROFILE}" >> $GITHUB_OUTPUT | |
| echo "host_profile=${HOST_PROFILE}" >> $GITHUB_OUTPUT | |
| - name: Install openvm-prof | |
| run: | | |
| cargo install --git ssh://git@github.com/axiom-crypto/openvm-private.git --profile=dev --force openvm-prof | |
| - name: Checkout openvm (for scripts) | |
| run: | | |
| git clone ssh://git@github.com/axiom-crypto/openvm-private.git | |
| mv openvm-private openvm | |
| - name: Install architecture specific tools | |
| run: | | |
| rustup install nightly-2024-10-30 | |
| source openvm/ci/scripts/utils.sh | |
| install_s5cmd | |
| sudo apt update | |
| sudo apt install gnuplot | |
| - name: Setup halo2 | |
| if: ${{ (github.event.inputs.mode == 'prove-e2e') || (inputs.mode == 'prove-e2e') }} | |
| run: | | |
| bash openvm/extensions/native/recursion/trusted_setup_s3.sh | |
| PARAMS_DIR=$(pwd)/params | |
| echo "PARAMS_DIR=$PARAMS_DIR" >> $GITHUB_ENV | |
| OPTIONAL_ARGS="--kzg-params-dir $PARAMS_DIR --halo2-outer-k 22" | |
| echo "OPTIONAL_ARGS=${OPTIONAL_ARGS}" >> $GITHUB_ENV | |
| - name: Restore Guest ELF from cache | |
| id: cache-guest-elf-restore | |
| uses: runs-on/cache/restore@v4 | |
| with: | |
| path: bin/host/elf/openvm-client-eth | |
| key: ${{ steps.set-cache-keys.outputs.elf_cache_key }} | |
| - name: Install cargo-openvm | |
| if: steps.cache-guest-elf-restore.outputs.cache-hit != 'true' | |
| run: | | |
| cargo install --git ssh://git@github.com/axiom-crypto/openvm-private.git --force cargo-openvm | |
| - name: Build Guest ELF | |
| if: steps.cache-guest-elf-restore.outputs.cache-hit != 'true' | |
| working-directory: bin/client-eth | |
| run: | | |
| GUEST_PROFILE=${{ steps.set-cache-keys.outputs.guest_profile }} | |
| cargo openvm build --no-transpile --profile=$GUEST_PROFILE | |
| mkdir -p ../host/elf | |
| cp target/riscv32im-risc0-zkvm-elf/$GUEST_PROFILE/openvm-client-eth ../host/elf/ | |
| - name: Save Guest ELF to cache | |
| if: steps.cache-guest-elf-restore.outputs.cache-hit != 'true' | |
| uses: runs-on/cache/save@v4 | |
| with: | |
| path: bin/host/elf/openvm-client-eth | |
| key: ${{ steps.cache-guest-elf-restore.outputs.cache-primary-key }} | |
| - name: Set build args | |
| id: set-build-args | |
| run: | | |
| FEATURES="bench-metrics,nightly-features,${{ inputs.memory_allocator || github.event.inputs.memory_allocator || 'jemalloc' }}" | |
| if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "guest" ]]; then | |
| OPTIONAL_ARGS="${OPTIONAL_ARGS} --profiling" | |
| FEATURES="${FEATURES},profiling" | |
| fi | |
| echo "OPTIONAL_ARGS=${OPTIONAL_ARGS}" >> $GITHUB_ENV | |
| echo "FEATURES=${FEATURES}" >> $GITHUB_ENV | |
| - name: Restore Host Binary from cache | |
| id: cache-host-binary-restore | |
| uses: runs-on/cache/restore@v4 | |
| with: | |
| path: target/${{ steps.set-cache-keys.outputs.host_profile }}/openvm-reth-benchmark | |
| key: ${{ steps.set-cache-keys.outputs.host_cache_key }} | |
| - name: Build Host Binary | |
| if: steps.cache-host-binary-restore.outputs.cache-hit != 'true' | |
| run: | | |
| arch=$(uname -m) | |
| case $arch in | |
| arm64|aarch64) | |
| RUSTFLAGS="-Ctarget-cpu=native" | |
| ;; | |
| x86_64|amd64) | |
| RUSTFLAGS="-Ctarget-cpu=native -C target-feature=+avx512f" | |
| FEATURES="${FEATURES},halo2-asm" | |
| ;; | |
| *) | |
| echo "Unsupported architecture: $arch" | |
| exit 1 | |
| ;; | |
| esac | |
| HOST_PROFILE=${{ steps.set-cache-keys.outputs.host_profile }} | |
| if [[ "$HOST_PROFILE" == "profiling" ]]; then | |
| RUSTFLAGS="${RUSTFLAGS} -g -C force-frame-pointers=yes" | |
| fi | |
| export JEMALLOC_SYS_WITH_MALLOC_CONF="retain:true,background_thread:true,metadata_thp:always,thp:always,dirty_decay_ms:-1,muzzy_decay_ms:-1,abort_conf:true" | |
| RUSTFLAGS=$RUSTFLAGS cargo build --bin openvm-reth-benchmark --profile=$HOST_PROFILE --no-default-features --features=$FEATURES | |
| echo "JEMALLOC_SYS_WITH_MALLOC_CONF=${JEMALLOC_SYS_WITH_MALLOC_CONF}" >> $GITHUB_ENV | |
| - name: Save Host Binary to cache | |
| if: steps.cache-host-binary-restore.outputs.cache-hit != 'true' | |
| uses: runs-on/cache/save@v4 | |
| with: | |
| path: target/${{ steps.set-cache-keys.outputs.host_profile }}/openvm-reth-benchmark | |
| key: ${{ steps.cache-host-binary-restore.outputs.cache-primary-key }} | |
| - name: Set up run benchmark script | |
| run: | | |
| mkdir -p rpc-cache | |
| mkdir -p .bench_metrics | |
| RPC_1=${{ secrets.RPC_URL_1 }} | |
| MODE=${{ inputs.mode || github.event.inputs.mode }} | |
| BLOCK_NUMBER=${{ inputs.block_number || github.event.inputs.block_number }} | |
| HOST_PROFILE=${{ steps.set-cache-keys.outputs.host_profile }} | |
| cat > run_benchmark.sh <<EOF | |
| ./target/$HOST_PROFILE/openvm-reth-benchmark \ | |
| --$MODE --block-number $BLOCK_NUMBER --rpc-url $RPC_1 --cache-dir rpc-cache \ | |
| --app-log-blowup ${{ inputs.app_log_blowup || github.event.inputs.app_log_blowup }} \ | |
| --leaf-log-blowup ${{ inputs.leaf_log_blowup || github.event.inputs.leaf_log_blowup }} \ | |
| --root-log-blowup ${{ inputs.root_log_blowup || github.event.inputs.root_log_blowup }} \ | |
| --max-segment-length ${{ inputs.max_segment_length || github.event.inputs.max_segment_length }} \ | |
| --max-cells-per-chip-in-segment ${{ inputs.max_cells_per_chip_in_segment || github.event.inputs.max_cells_per_chip_in_segment }} \ | |
| $OPTIONAL_ARGS | |
| EOF | |
| chmod +x run_benchmark.sh | |
| - name: Run benchmark | |
| run: | | |
| export JEMALLOC_SYS_WITH_MALLOC_CONF=${JEMALLOC_SYS_WITH_MALLOC_CONF} | |
| export RUST_LOG="info,p3_=warn" | |
| export OUTPUT_PATH=${METRIC_PATH} | |
| if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "host" ]]; then | |
| echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid | |
| echo 0 | sudo tee /proc/sys/kernel/kptr_restrict | |
| # install custom addr2line: https://crates.io/crates/flamegraph | |
| rm -rf /tmp/addr2line | |
| git clone https://github.com/gimli-rs/addr2line /tmp/addr2line | |
| work_dir=$(pwd) | |
| cd /tmp/addr2line | |
| cargo build --release --bin addr2line --features bin | |
| sudo mv target/release/addr2line /usr/local/bin/ | |
| cd $work_dir | |
| perf --version | |
| perf record -F 99 --call-graph dwarf -o perf.data -- ./run_benchmark.sh | |
| else | |
| ./ci/monitor_memory.sh ./run_benchmark.sh | |
| echo "MEM_USAGE_PATH=memory_usage.png" >> $GITHUB_ENV | |
| fi | |
| - name: Upload metric file | |
| id: upload-metric-artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.set-metric-name.outputs.name }} | |
| path: ${{ env.METRIC_PATH }} | |
| overwrite: "true" | |
| retention-days: 1 | |
| - name: Upload memory usage graph | |
| if: ${{ (inputs.profiling || github.event.inputs.profiling) == 'none' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.set-metric-name.outputs.name }}-memory-usage | |
| path: ${{ env.MEM_USAGE_PATH }} | |
| retention-days: 1 | |
| - name: Upload Benchmark Metrics | |
| run: s5cmd cp ${METRIC_PATH} ${{ env.S3_METRICS_PATH }}/${METRIC_NAME}.json | |
| - name: Generate markdown | |
| run: | | |
| BENCHER_METRIC_PATH=bencher.json | |
| openvm-prof --json-paths $METRIC_PATH --output-json $BENCHER_METRIC_PATH | |
| MD_PATH=${METRIC_PATH%.json}.md | |
| echo "MD_PATH=${MD_PATH}" >> $GITHUB_ENV | |
| echo "BENCHER_METRIC_PATH=${BENCHER_METRIC_PATH}" >> $GITHUB_ENV | |
| - name: Generate circuit flamegraphs | |
| if: ${{ inputs.profiling == 'guest' || github.event.inputs.profiling == 'guest' }} | |
| run: | | |
| cargo install inferno | |
| python3 openvm/ci/scripts/metric_unify/flamegraph.py $METRIC_PATH | |
| s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.S3_FLAMEGRAPHS_PATH }}/${METRIC_NAME}/" | |
| echo "" >> $MD_PATH | |
| echo "<details>" >> $MD_PATH | |
| echo "<summary>Circuit Flamegraphs</summary>" >> $MD_PATH | |
| echo "" >> $MD_PATH | |
| for file in .bench_metrics/flamegraphs/*.svg; do | |
| filename=$(basename "$file") | |
| flamegraph_url=${{ env.S3_FLAMEGRAPHS_URL }}/${METRIC_NAME}/${filename} | |
| echo "[]($flamegraph_url)" >> $MD_PATH | |
| done | |
| rm -f .bench_metrics/flamegraphs/*.svg | |
| echo "" >> $MD_PATH | |
| echo "</details>" >> $MD_PATH | |
| echo "" >> $MD_PATH | |
| - name: Add benchmark metadata to markdown | |
| run: | | |
| COMMIT_URL=https://github.com/${{ github.repository }}/commit/${current_sha} | |
| BENCHMARK_WORKFLOW_URL=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| source openvm/ci/scripts/utils.sh | |
| add_metadata $MD_PATH \ | |
| ${{ inputs.max_segment_length || github.event.inputs.max_segment_length }} \ | |
| ${{ inputs.instance_family || github.event.inputs.instance_family }} \ | |
| ${{ inputs.memory_allocator || github.event.inputs.memory_allocator }} \ | |
| $COMMIT_URL \ | |
| $BENCHMARK_WORKFLOW_URL | |
| - name: Upload markdown | |
| run: | | |
| S3_MD_PATH="${{ env.S3_PATH }}/${METRIC_NAME}.md" | |
| s5cmd cp $MD_PATH $S3_MD_PATH | |
| echo "S3_MD_PATH=${S3_MD_PATH}" >> $GITHUB_ENV | |
| - name: Upload host flamegraph | |
| if: ${{ inputs.profiling == 'host' || github.event.inputs.profiling == 'host' }} | |
| run: | | |
| cargo install inferno | |
| perf script -i perf.data | inferno-collapse-perf > stacks.folded | |
| cat stacks.folded | inferno-flamegraph > flamegraph.svg | |
| cat stacks.folded | inferno-flamegraph --reverse --inverted > flamegraph-reverse.svg | |
| s5cmd cp flamegraph.svg ${{ env.S3_FLAMEGRAPHS_PATH }}/${METRIC_NAME}/host.svg | |
| s5cmd cp flamegraph-reverse.svg ${{ env.S3_FLAMEGRAPHS_PATH }}/${METRIC_NAME}/host-reverse.svg | |
| flamegraph_url=${{ env.S3_FLAMEGRAPHS_URL }}/${METRIC_NAME}/host.svg | |
| flamegraph_reverse_url=${{ env.S3_FLAMEGRAPHS_URL }}/${METRIC_NAME}/host-reverse.svg | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Host Perf Flamegraph: $flamegraph_url" >> $GITHUB_STEP_SUMMARY | |
| echo "Host Perf Reverse Flamegraph: $flamegraph_reverse_url" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| ### Update gh-pages | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: gh-pages | |
| - name: Set up git | |
| run: | | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --global user.name "github-actions[bot]" | |
| - name: Commit to gh-pages branch | |
| run: | | |
| GH_PAGES_PATH="benchmarks-dispatch/${{ github.head_ref || github.ref }}" | |
| echo "GH_PAGES_PATH=${GH_PAGES_PATH}" >> $GITHUB_ENV | |
| mkdir -p ${GH_PAGES_PATH} | |
| s5cmd cp $S3_MD_PATH "${GH_PAGES_PATH}/${METRIC_NAME}.md" | |
| git add ${GH_PAGES_PATH}/${METRIC_NAME}.md | |
| git commit --allow-empty -m "Update benchmark result at ${GH_PAGES_PATH}/${METRIC_NAME}.md" | |
| MAX_RETRIES=10 | |
| RETRY_DELAY=5 | |
| ATTEMPT=0 | |
| SUCCESS=false | |
| while [ $ATTEMPT -lt $MAX_RETRIES ]; do | |
| echo "Attempt $((ATTEMPT + 1)) to push of $MAX_RETRIES..." | |
| git fetch origin gh-pages | |
| git merge origin/gh-pages --no-edit | |
| if git push origin gh-pages; then | |
| SUCCESS=true | |
| break | |
| else | |
| echo "Push failed. Retrying in $RETRY_DELAY seconds..." | |
| sleep $RETRY_DELAY | |
| ATTEMPT=$((ATTEMPT + 1)) | |
| fi | |
| done | |
| if [ "$SUCCESS" = false ]; then | |
| echo "PUSH_FAILED" | |
| exit 1 | |
| fi | |
| - name: Path to result | |
| run: | | |
| echo "https://github.com/axiom-crypto/openvm-reth-benchmark/blob/gh-pages/${GH_PAGES_PATH}/${METRIC_NAME}.md" |