Skip to content
Reth Benchmark

Reth Benchmark (block 21000000) #3080

Sign in to view logs

Reth Benchmark (block 21000000)

Reth Benchmark (block 21000000) #3080

Workflow file for this run

.github/workflows/reth-benchmark.yml at 4728385
name: "Reth Benchmark"
run-name: "Reth Benchmark (block ${{ inputs.block_number || github.event.inputs.block_number }})"
on:
workflow_dispatch:
# github workflow dispatch inputs has 10 input limit...
# > you may only define up to 10 `inputs` for a `workflow_dispatch` event
inputs:
# Default runner is not big enough for this
# https://aws.amazon.com/ec2/instance-types/
instance_family:
type: string
required: false
description: The family of the instance, can be multiple ones concat with "+" e.g. r8g+r7g
default: m8g.24xlarge
block_number:
type: number
required: false
description: Block number to run the benchmark on
default: 21000000
app_log_blowup:
type: number
required: false
description: Application level log blowup
default: 1
leaf_log_blowup:
type: number
required: false
description: Aggregation (leaf) level log blowup
default: 1
internal_log_blowup:
type: number
required: false
description: Internal level log blowup
default: 2
# root_log_blowup:
# type: number
# required: false
# description: Root level log blowup (only for e2e)
# default: 3
mode:
type: choice
required: false
description: Running mode
options:
- execute
- tracegen
- prove
- prove-e2e
default: prove-e2e
profiling:
type: choice
required: false
description: Profiling mode
options:
- none
- host # host profiling (samply, flamegraphs, etc)
- guest # guest profiling (circuit flamegraphs)
default: none
max_segment_length:
type: number
required: false
description: Max segment length for continuations, must be larger than 524288
default: 8388508 # 2**23 - 100
segment_max_cells:
type: string # number might have u32 limit
required: false
description: Total main trace cells (excluding memory)
default: "1200000000" # 1.2B
# num_children_leaf:
# type: number
# required: false
# description: Number of app proofs that leaf verifier aggregates
# default: 1
num_children_internal:
type: number
required: false
description: Number of proofs that internal verifier aggregates
default: 3
workflow_call:
inputs:
ref:
type: string
required: false
description: Git ref to checkout
instance_family:
type: string
required: false
description: The family of the instance, can be multiple ones concat with "+" e.g. r8g+r7g
default: m8g.24xlarge
memory_allocator:
type: string
required: false
description: Memory allocator to use (mimalloc or jemalloc)
default: jemalloc
block_number:
type: number
required: false
description: Block number to run the benchmark on
default: 21000000
app_log_blowup:
type: number
required: false
description: Application level log blowup
default: 1
leaf_log_blowup:
type: number
required: false
description: Aggregation (leaf) level log blowup
default: 1
internal_log_blowup:
type: number
required: false
description: Internal level log blowup (only for e2e)
default: 2
root_log_blowup:
type: number
required: false
description: Root level log blowup (only for e2e)
default: 3
mode:
type: string
required: false
description: Running mode, one of execute, tracegen, prove, or prove-e2e
default: prove-e2e
profiling:
type: string
required: false
description: Profiling mode (none, host, guest)
default: none
max_segment_length:
type: number
required: false
description: Max segment length for continuations, must be larger than 524288
default: 8388508
segment_max_cells:
type: string
required: false
description: Total main trace cells (excluding memory)
default: "1200000000" # 1.2B
num_children_leaf:
type: number
required: false
description: Number of app proofs that leaf verifier aggregates
default: 1
num_children_internal:
type: number
required: false
description: Number of proofs that internal verifier aggregates
default: 3
tag:
type: string
required: false
description: Tag for cache keys (default is commit hash)
secrets:
GH_ACTIONS_DEPLOY_PRIVATE_KEY:
required: true
RPC_URL_1:
required: true
BENCHER_API_TOKEN:
required: true
outputs:
metric_name:
description: "Name of the metric"
value: ${{ jobs.run-reth.outputs.metric_name }}
env:
S3_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/results
S3_METRICS_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/metrics
S3_FLAMEGRAPHS_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/flamegraphs
S3_FLAMEGRAPHS_URL: https://axiom-public-data-sandbox-us-east-1.s3.us-east-1.amazonaws.com/benchmark/github/flamegraphs
S3_SAMPLY_PROFILE_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/samply
CMD_ARGS: ""
INPUT_ARGS: ""
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
BENCHER_PROJECT: openvm-reth-benchmark
jobs:
run-reth:
name: Run Reth benchmark
runs-on:
- runs-on
- run-id=${{ github.run_id }}
- family=${{ inputs.instance_family || github.event.inputs.instance_family }}
- disk=large
- tag=bench-reth-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
- extras=s3-cache
- image=${{ contains(inputs.instance_family || github.event.inputs.instance_family, 'g.') && 'ubuntu22-full-arm64' || 'ubuntu22-full-x64' }}
outputs:
metric_name: ${{ steps.set-metric-name.outputs.name }}
steps:
- uses: runs-on/action@v1
- uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.head_ref || github.ref }}
- uses: dtolnay/rust-toolchain@nightly
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Display workflow inputs
run: |
if [[ "${{ inputs.manual_call }}" == "true" ]]; then
echo "${{ toJSON(inputs) }}"
else
echo "${{ toJSON(github.event.inputs) }}"
fi
- name: Get current commit hash
run: echo "current_sha=$(git rev-parse HEAD)" >> $GITHUB_ENV
- name: Set metric name
id: set-metric-name
run: |
input_json_str="${{ toJSON(inputs || github.event.inputs) }}"
input_hash=$(echo $input_json_str | sha256sum | cut -d' ' -f1)
METRIC_NAME=reth-$current_sha-${input_hash}
echo "METRIC_NAME=${METRIC_NAME}" >> $GITHUB_ENV
mkdir -p .bench_metrics/${METRIC_NAME}
METRIC_PATH=".bench_metrics/${METRIC_NAME}"
echo "METRIC_PATH=${METRIC_PATH}" >> $GITHUB_ENV
echo "name=${METRIC_NAME}" >> $GITHUB_OUTPUT
- name: Set build profiles
id: set-build-profiles
run: |
GUEST_PROFILE="release"
HOST_PROFILE="maxperf"
if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "guest" ]]; then
GUEST_PROFILE="profiling"
HOST_PROFILE="profiling"
fi
if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "host" ]]; then
HOST_PROFILE="profiling"
fi
echo "guest_profile=${GUEST_PROFILE}" >> $GITHUB_OUTPUT
echo "host_profile=${HOST_PROFILE}" >> $GITHUB_OUTPUT
- name: Set build args
id: set-build-args
run: |
FEATURES="bench-metrics,nightly-features,${{ inputs.memory_allocator || github.event.inputs.memory_allocator || 'jemalloc' }}"
if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "guest" ]]; then
FEATURES="${FEATURES},profiling"
fi
arch=$(uname -m)
case $arch in
arm64|aarch64)
RUSTFLAGS="-Ctarget-cpu=native"
;;
x86_64|amd64)
RUSTFLAGS="-Ctarget-cpu=native -C target-feature=+avx512f"
FEATURES="${FEATURES},halo2-asm"
;;
*)
echo "Unsupported architecture: $arch"
exit 1
;;
esac
if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "host" ]]; then
RUSTFLAGS="${RUSTFLAGS} -C force-frame-pointers=yes"
fi
JEMALLOC_SYS_WITH_MALLOC_CONF="retain:true,background_thread:true,metadata_thp:always,thp:always,dirty_decay_ms:-1,muzzy_decay_ms:-1,abort_conf:true"
echo "RUSTFLAGS=${RUSTFLAGS}" >> $GITHUB_ENV
echo "JEMALLOC_SYS_WITH_MALLOC_CONF=${JEMALLOC_SYS_WITH_MALLOC_CONF}" >> $GITHUB_ENV
echo "FEATURES=${FEATURES}" >> $GITHUB_ENV
- name: Set cache keys
id: set-cache-keys
run: |
arch=$(uname -m)
# Use inputs.tag if defined, otherwise use current commit SHA
TAG="${{ inputs.tag || env.current_sha }}"
GUEST_PROFILE=${{ steps.set-build-profiles.outputs.guest_profile }}
echo "elf_cache_key=elf-${TAG}-${arch}-${GUEST_PROFILE}" >> $GITHUB_OUTPUT
RUSTFLAGS_HASH=$(echo "${RUSTFLAGS}" | sha256sum | cut -d' ' -f1 | head -c8)
JEMALLOC_CONF_HASH=$(echo "${JEMALLOC_SYS_WITH_MALLOC_CONF}" | sha256sum | cut -d' ' -f1 | head -c8)
FEATURES_HASH=$(echo "${FEATURES}" | sha256sum | cut -d' ' -f1 | head -c8)
HOST_PROFILE=${{ steps.set-build-profiles.outputs.host_profile }}
echo "host_cache_key=host-${TAG}-${arch}-${HOST_PROFILE}-${RUSTFLAGS_HASH}-${JEMALLOC_CONF_HASH}-${FEATURES_HASH}" >> $GITHUB_OUTPUT
- name: Load SSH key
uses: webfactory/ssh-agent@v0.9.0
with:
ssh-private-key: |
${{ secrets.GH_ACTIONS_DEPLOY_PRIVATE_KEY }}
- name: Install openvm-prof
run: |
cargo install --git https://github.com/openvm-org/openvm.git --profile=dev --force openvm-prof
- name: Checkout openvm (for scripts)
run: |
git clone --depth=1 https://github.com/openvm-org/openvm.git
- name: Install architecture specific tools
run: |
rustup install nightly-2024-10-30
source openvm/ci/scripts/utils.sh
install_s5cmd
sudo apt update
sudo apt install gnuplot
- name: Setup halo2
if: ${{ (github.event.inputs.mode == 'prove-e2e') || (inputs.mode == 'prove-e2e') }}
run: |
bash openvm/extensions/native/recursion/trusted_setup_s3.sh
PARAMS_DIR=$(pwd)/params
echo "PARAMS_DIR=$PARAMS_DIR" >> $GITHUB_ENV
OPTIONAL_ARGS="--kzg-params-dir $PARAMS_DIR --halo2-outer-k 22"
echo "OPTIONAL_ARGS=${OPTIONAL_ARGS}" >> $GITHUB_ENV
- name: Restore Guest ELF from cache
id: cache-guest-elf-restore
uses: runs-on/cache/restore@v4
with:
path: bin/host/elf/openvm-client-eth
key: ${{ steps.set-cache-keys.outputs.elf_cache_key }}
- name: Install cargo-openvm
if: steps.cache-guest-elf-restore.outputs.cache-hit != 'true'
run: |
cargo install --git https://github.com/openvm-org/openvm.git --locked --force cargo-openvm
- name: Build Guest ELF
if: steps.cache-guest-elf-restore.outputs.cache-hit != 'true'
working-directory: bin/client-eth
run: |
GUEST_PROFILE=${{ steps.set-build-profiles.outputs.guest_profile }}
RUSTFLAGS="" cargo openvm build --no-transpile --profile=$GUEST_PROFILE
mkdir -p ../host/elf
cp target/riscv32im-risc0-zkvm-elf/$GUEST_PROFILE/openvm-client-eth ../host/elf/
- name: Save Guest ELF to cache
if: steps.cache-guest-elf-restore.outputs.cache-hit != 'true'
uses: runs-on/cache/save@v4
with:
path: bin/host/elf/openvm-client-eth
key: ${{ steps.cache-guest-elf-restore.outputs.cache-primary-key }}
- name: Restore Host Binary from cache
id: cache-host-binary-restore
uses: runs-on/cache/restore@v4
with:
path: target/${{ steps.set-build-profiles.outputs.host_profile }}/openvm-reth-benchmark-bin
key: ${{ steps.set-cache-keys.outputs.host_cache_key }}
- name: Build Host Binary
if: steps.cache-host-binary-restore.outputs.cache-hit != 'true'
run: |
export JEMALLOC_SYS_WITH_MALLOC_CONF=${JEMALLOC_SYS_WITH_MALLOC_CONF}
HOST_PROFILE=${{ steps.set-build-profiles.outputs.host_profile }}
RUSTFLAGS=$RUSTFLAGS cargo build --bin openvm-reth-benchmark-bin --profile=$HOST_PROFILE --no-default-features --features=$FEATURES
- name: Save Host Binary to cache
if: steps.cache-host-binary-restore.outputs.cache-hit != 'true'
uses: runs-on/cache/save@v4
with:
path: target/${{ steps.set-build-profiles.outputs.host_profile }}/openvm-reth-benchmark-bin
key: ${{ steps.cache-host-binary-restore.outputs.cache-primary-key }}
- name: Set up run benchmark script
run: |
mkdir -p rpc-cache
mkdir -p .bench_metrics
RPC_1=${{ secrets.RPC_URL_1 }}
MODE=${{ inputs.mode || github.event.inputs.mode }}
BLOCK_NUMBER=${{ inputs.block_number || github.event.inputs.block_number }}
HOST_PROFILE=${{ steps.set-build-profiles.outputs.host_profile }}
if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "guest" ]]; then
OPTIONAL_ARGS="${OPTIONAL_ARGS} --profiling"
fi
cat > run_benchmark.sh <<EOF
./target/$HOST_PROFILE/openvm-reth-benchmark-bin \
--$MODE --block-number $BLOCK_NUMBER --rpc-url $RPC_1 --cache-dir rpc-cache \
--app-log-blowup ${{ inputs.app_log_blowup || github.event.inputs.app_log_blowup }} \
--leaf-log-blowup ${{ inputs.leaf_log_blowup || github.event.inputs.leaf_log_blowup }} \
--internal-log-blowup ${{ inputs.internal_log_blowup || github.event.inputs.internal_log_blowup || 2 }} \
--root-log-blowup ${{ inputs.root_log_blowup || github.event.inputs.root_log_blowup || 3 }} \
--max-segment-length ${{ inputs.max_segment_length || github.event.inputs.max_segment_length }} \
--segment-max-cells ${{ inputs.segment_max_cells || github.event.inputs.segment_max_cells }} \
--num-children-leaf ${{ inputs.num_children_leaf || github.event.inputs.num_children_leaf || 1 }} \
--num-children-internal ${{ inputs.num_children_internal || github.event.inputs.num_children_internal }} \
$OPTIONAL_ARGS
EOF
chmod +x run_benchmark.sh
- name: Run benchmark
run: |
export JEMALLOC_SYS_WITH_MALLOC_CONF=${JEMALLOC_SYS_WITH_MALLOC_CONF}
export RUST_LOG="info,p3_=warn"
export OUTPUT_PATH=${METRIC_PATH}/metrics.json
if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "guest" ]]; then
export GUEST_SYMBOLS_PATH=${METRIC_PATH}/guest.syms
fi
if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "host" ]]; then
echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid
echo 0 | sudo tee /proc/sys/kernel/kptr_restrict
perf --version
perf record -F 100 --call-graph=fp -g -o perf.data -- ./run_benchmark.sh
else
./ci/monitor_memory.sh ./run_benchmark.sh
echo "MEM_USAGE_PATH=memory_usage.png" >> $GITHUB_ENV
fi
- name: Upload metric artifacts
id: upload-metric-artifacts
uses: actions/upload-artifact@v4
with:
name: ${{ steps.set-metric-name.outputs.name }}
path: ${{ env.METRIC_PATH }}
retention-days: 1
- name: Upload memory usage graph
if: ${{ (inputs.profiling || github.event.inputs.profiling) == 'none' }}
uses: actions/upload-artifact@v4
with:
name: ${{ steps.set-metric-name.outputs.name }}-memory-usage
path: ${{ env.MEM_USAGE_PATH }}
retention-days: 1
- name: Upload Benchmark Metrics
run: |
s5cmd cp ${METRIC_PATH}/metrics.json ${{ env.S3_METRICS_PATH }}/${METRIC_NAME}/metrics.json
if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "guest" ]]; then
s5cmd cp ${METRIC_PATH}/guest.syms ${{ env.S3_METRICS_PATH }}/${METRIC_NAME}/guest.syms
fi
- name: Generate markdown
run: |
BENCHER_METRIC_PATH=bencher.json
openvm-prof --json-paths ${METRIC_PATH}/metrics.json --output-json $BENCHER_METRIC_PATH
MD_PATH=${METRIC_PATH}/metrics.md
echo "MD_PATH=${MD_PATH}" >> $GITHUB_ENV
echo "BENCHER_METRIC_PATH=${BENCHER_METRIC_PATH}" >> $GITHUB_ENV
- uses: bencherdev/bencher@main
- name: Upload bencher metrics
if: ${{ inputs.profiling == 'none' || github.event.inputs.profiling == 'none' }}
run: |
current_branch=$(git rev-parse --abbrev-ref HEAD)
if [[ "$current_branch" == nightly* ]]; then
BRANCH=nightly
else
BRANCH=dev
fi
echo "Bencher branch: ${BRANCH}"
bencher run \
--project ${{ env.BENCHER_PROJECT }} \
--token '${{ secrets.BENCHER_API_TOKEN }}' \
--start-point nightly \
--branch $BRANCH \
--testbed ${{ inputs.instance_family || github.event.inputs.instance_family }} \
--adapter json \
--file $BENCHER_METRIC_PATH
- name: Generate circuit flamegraphs
if: ${{ inputs.profiling == 'guest' || github.event.inputs.profiling == 'guest' }}
run: |
cargo install inferno
python3 openvm/ci/scripts/metric_unify/flamegraph.py ${METRIC_PATH}/metrics.json --guest-symbols ${METRIC_PATH}/guest.syms
s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.S3_FLAMEGRAPHS_PATH }}/${METRIC_NAME}/"
echo "" >> $MD_PATH
echo "<details>" >> $MD_PATH
echo "<summary>Circuit Flamegraphs</summary>" >> $MD_PATH
echo "" >> $MD_PATH
for file in .bench_metrics/flamegraphs/*.svg; do
filename=$(basename "$file")
flamegraph_url=${{ env.S3_FLAMEGRAPHS_URL }}/${METRIC_NAME}/${filename}
echo "[![]($flamegraph_url)]($flamegraph_url)" >> $MD_PATH
done
rm -f .bench_metrics/flamegraphs/*.svg
echo "" >> $MD_PATH
echo "</details>" >> $MD_PATH
echo "" >> $MD_PATH
- name: Add benchmark metadata to markdown
run: |
COMMIT_URL=https://github.com/${{ github.repository }}/commit/${current_sha}
BENCHMARK_WORKFLOW_URL=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
source openvm/ci/scripts/utils.sh
add_metadata $MD_PATH \
${{ inputs.max_segment_length || github.event.inputs.max_segment_length }} \
${{ inputs.instance_family || github.event.inputs.instance_family }} \
${{ inputs.memory_allocator || github.event.inputs.memory_allocator }} \
$COMMIT_URL \
$BENCHMARK_WORKFLOW_URL
- name: Upload markdown
run: |
S3_MD_PATH="${{ env.S3_PATH }}/${METRIC_NAME}.md"
s5cmd cp $MD_PATH $S3_MD_PATH
echo "S3_MD_PATH=${S3_MD_PATH}" >> $GITHUB_ENV
- name: Generate samply profile artifacts
if: ${{ inputs.profiling == 'host' || github.event.inputs.profiling == 'host' }}
run: |
cargo install --git https://github.com/mstange/samply.git samply --force
mkdir -p samply_profile
SAMPLY_PROFILE_PATH=samply_profile
samply import perf.data --unstable-presymbolicate --save-only --output $SAMPLY_PROFILE_PATH/profile.json
echo "SAMPLY_PROFILE_PATH=${SAMPLY_PROFILE_PATH}" >> $GITHUB_ENV
s5cmd cp $SAMPLY_PROFILE_PATH/profile.json ${{ env.S3_SAMPLY_PROFILE_PATH }}/${METRIC_NAME}/profile.json
s5cmd cp $SAMPLY_PROFILE_PATH/profile.syms.json ${{ env.S3_SAMPLY_PROFILE_PATH }}/${METRIC_NAME}/profile.syms.json
echo "S3_SAMPLY_PROFILE_PATH=${S3_SAMPLY_PROFILE_PATH}" >> $GITHUB_ENV
- name: Upload samply profile artifacts
if: ${{ inputs.profiling == 'host' || github.event.inputs.profiling == 'host' }}
id: upload-samply-profile-artifact
uses: actions/upload-artifact@v4
with:
name: ${{ steps.set-metric-name.outputs.name }}-samply-profile
path: ${{ env.SAMPLY_PROFILE_PATH }}
retention-days: 1
### Update gh-pages
- uses: actions/checkout@v4
with:
ref: gh-pages
- name: Set up git
run: |
git config --global user.email "github-actions[bot]@users.noreply.github.com"
git config --global user.name "github-actions[bot]"
- name: Commit to gh-pages branch
run: |
GH_PAGES_PATH="benchmarks-dispatch/${{ github.head_ref || github.ref }}"
echo "GH_PAGES_PATH=${GH_PAGES_PATH}" >> $GITHUB_ENV
mkdir -p ${GH_PAGES_PATH}
s5cmd cp $S3_MD_PATH "${GH_PAGES_PATH}/${METRIC_NAME}.md"
git add ${GH_PAGES_PATH}/${METRIC_NAME}.md
git commit --allow-empty -m "Update benchmark result at ${GH_PAGES_PATH}/${METRIC_NAME}.md"
MAX_RETRIES=10
RETRY_DELAY=5
ATTEMPT=0
SUCCESS=false
while [ $ATTEMPT -lt $MAX_RETRIES ]; do
echo "Attempt $((ATTEMPT + 1)) to push of $MAX_RETRIES..."
git fetch origin gh-pages
git merge origin/gh-pages --no-edit
if git push origin gh-pages; then
SUCCESS=true
break
else
echo "Push failed. Retrying in $RETRY_DELAY seconds..."
sleep $RETRY_DELAY
ATTEMPT=$((ATTEMPT + 1))
fi
done
if [ "$SUCCESS" = false ]; then
echo "PUSH_FAILED"
exit 1
fi
- name: Path to result
run: |
echo "https://github.com/axiom-crypto/openvm-reth-benchmark/blob/gh-pages/${GH_PAGES_PATH}/${METRIC_NAME}.md"