Reth Benchmark (block 21000000) #3071
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: "Reth Benchmark" | |
run-name: "Reth Benchmark (block ${{ inputs.block_number || github.event.inputs.block_number }})" | |
on: | |
workflow_dispatch: | |
# github workflow dispatch inputs has 10 input limit... | |
# > you may only define up to 10 `inputs` for a `workflow_dispatch` event | |
inputs: | |
# Default runner is not big enough for this | |
# https://aws.amazon.com/ec2/instance-types/ | |
instance_family: | |
type: string | |
required: false | |
description: The family of the instance, can be multiple ones concat with "+" e.g. r8g+r7g | |
default: m8g.24xlarge | |
block_number: | |
type: number | |
required: false | |
description: Block number to run the benchmark on | |
default: 21000000 | |
app_log_blowup: | |
type: number | |
required: false | |
description: Application level log blowup | |
default: 1 | |
leaf_log_blowup: | |
type: number | |
required: false | |
description: Aggregation (leaf) level log blowup | |
default: 1 | |
internal_log_blowup: | |
type: number | |
required: false | |
description: Internal level log blowup | |
default: 2 | |
# root_log_blowup: | |
# type: number | |
# required: false | |
# description: Root level log blowup (only for e2e) | |
# default: 3 | |
mode: | |
type: choice | |
required: false | |
description: Running mode | |
options: | |
- execute | |
- tracegen | |
- prove | |
- prove-e2e | |
default: prove-e2e | |
profiling: | |
type: choice | |
required: false | |
description: Profiling mode | |
options: | |
- none | |
- host # host profiling (samply, flamegraphs, etc) | |
- guest # guest profiling (circuit flamegraphs) | |
default: none | |
max_segment_length: | |
type: number | |
required: false | |
description: Max segment length for continuations, must be larger than 524288 | |
default: 8388508 # 2**23 - 100 | |
segment_max_cells: | |
type: string # number might have u32 limit | |
required: false | |
description: Total main trace cells (excluding memory) | |
default: "1200000000" # 1.2B | |
# num_children_leaf: | |
# type: number | |
# required: false | |
# description: Number of app proofs that leaf verifier aggregates | |
# default: 1 | |
num_children_internal: | |
type: number | |
required: false | |
description: Number of proofs that internal verifier aggregates | |
default: 3 | |
workflow_call: | |
inputs: | |
ref: | |
type: string | |
required: false | |
description: Git ref to checkout | |
instance_family: | |
type: string | |
required: false | |
description: The family of the instance, can be multiple ones concat with "+" e.g. r8g+r7g | |
default: m8g.24xlarge | |
memory_allocator: | |
type: string | |
required: false | |
description: Memory allocator to use (mimalloc or jemalloc) | |
default: jemalloc | |
block_number: | |
type: number | |
required: false | |
description: Block number to run the benchmark on | |
default: 21000000 | |
app_log_blowup: | |
type: number | |
required: false | |
description: Application level log blowup | |
default: 1 | |
leaf_log_blowup: | |
type: number | |
required: false | |
description: Aggregation (leaf) level log blowup | |
default: 1 | |
internal_log_blowup: | |
type: number | |
required: false | |
description: Internal level log blowup (only for e2e) | |
default: 2 | |
root_log_blowup: | |
type: number | |
required: false | |
description: Root level log blowup (only for e2e) | |
default: 3 | |
mode: | |
type: string | |
required: false | |
description: Running mode, one of execute, tracegen, prove, or prove-e2e | |
default: prove-e2e | |
profiling: | |
type: string | |
required: false | |
description: Profiling mode (none, host, guest) | |
default: none | |
max_segment_length: | |
type: number | |
required: false | |
description: Max segment length for continuations, must be larger than 524288 | |
default: 8388508 | |
segment_max_cells: | |
type: string | |
required: false | |
description: Total main trace cells (excluding memory) | |
default: "1200000000" # 1.2B | |
num_children_leaf: | |
type: number | |
required: false | |
description: Number of app proofs that leaf verifier aggregates | |
default: 1 | |
num_children_internal: | |
type: number | |
required: false | |
description: Number of proofs that internal verifier aggregates | |
default: 3 | |
tag: | |
type: string | |
required: false | |
description: Tag for cache keys (default is commit hash) | |
secrets: | |
GH_ACTIONS_DEPLOY_PRIVATE_KEY: | |
required: true | |
RPC_URL_1: | |
required: true | |
BENCHER_API_TOKEN: | |
required: true | |
outputs: | |
metric_name: | |
description: "Name of the metric" | |
value: ${{ jobs.run-reth.outputs.metric_name }} | |
env: | |
S3_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/results | |
S3_METRICS_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/metrics | |
S3_FLAMEGRAPHS_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/flamegraphs | |
S3_FLAMEGRAPHS_URL: https://axiom-public-data-sandbox-us-east-1.s3.us-east-1.amazonaws.com/benchmark/github/flamegraphs | |
S3_SAMPLY_PROFILE_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/samply | |
CMD_ARGS: "" | |
INPUT_ARGS: "" | |
CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
BENCHER_PROJECT: openvm-reth-benchmark | |
jobs: | |
run-reth: | |
name: Run Reth benchmark | |
runs-on: | |
- runs-on | |
- run-id=${{ github.run_id }} | |
- family=${{ inputs.instance_family || github.event.inputs.instance_family }} | |
- disk=large | |
- tag=bench-reth-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} | |
- extras=s3-cache | |
- image=${{ contains(inputs.instance_family || github.event.inputs.instance_family, 'g.') && 'ubuntu22-full-arm64' || 'ubuntu22-full-x64' }} | |
outputs: | |
metric_name: ${{ steps.set-metric-name.outputs.name }} | |
steps: | |
- uses: runs-on/action@v1 | |
- uses: actions/checkout@v4 | |
with: | |
ref: ${{ inputs.ref || github.head_ref || github.ref }} | |
- uses: dtolnay/rust-toolchain@nightly | |
- uses: Swatinem/rust-cache@v2 | |
with: | |
cache-on-failure: true | |
- name: Display workflow inputs | |
run: | | |
if [[ "${{ inputs.manual_call }}" == "true" ]]; then | |
echo "${{ toJSON(inputs) }}" | |
else | |
echo "${{ toJSON(github.event.inputs) }}" | |
fi | |
- name: Get current commit hash | |
run: echo "current_sha=$(git rev-parse HEAD)" >> $GITHUB_ENV | |
- name: Set metric name | |
id: set-metric-name | |
run: | | |
input_json_str="${{ toJSON(inputs || github.event.inputs) }}" | |
input_hash=$(echo $input_json_str | sha256sum | cut -d' ' -f1) | |
METRIC_NAME=reth-$current_sha-${input_hash} | |
echo "METRIC_NAME=${METRIC_NAME}" >> $GITHUB_ENV | |
METRIC_PATH=".bench_metrics/${METRIC_NAME}.json" | |
echo "METRIC_PATH=${METRIC_PATH}" >> $GITHUB_ENV | |
echo "name=${METRIC_NAME}" >> $GITHUB_OUTPUT | |
- name: Set cache keys | |
id: set-cache-keys | |
run: | | |
arch=$(uname -m) | |
# Use inputs.tag if defined, otherwise use current commit SHA | |
TAG="${{ inputs.tag || env.current_sha }}" | |
GUEST_PROFILE="release" | |
HOST_PROFILE="maxperf" | |
if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "guest" ]]; then | |
GUEST_PROFILE="profiling" | |
HOST_PROFILE="profiling" | |
fi | |
if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "host" ]]; then | |
HOST_PROFILE="profiling" | |
fi | |
echo "elf_cache_key=elf-${TAG}-${arch}-${GUEST_PROFILE}" >> $GITHUB_OUTPUT | |
echo "host_cache_key=host-${TAG}-${arch}-${HOST_PROFILE}" >> $GITHUB_OUTPUT | |
echo "guest_profile=${GUEST_PROFILE}" >> $GITHUB_OUTPUT | |
echo "host_profile=${HOST_PROFILE}" >> $GITHUB_OUTPUT | |
- name: Load SSH key | |
uses: webfactory/ssh-agent@v0.9.0 | |
with: | |
ssh-private-key: | | |
${{ secrets.GH_ACTIONS_DEPLOY_PRIVATE_KEY }} | |
- name: Install openvm-prof | |
run: | | |
cargo install --git https://github.com/openvm-org/openvm.git --profile=dev --force openvm-prof | |
- name: Checkout openvm (for scripts) | |
run: | | |
git clone --depth=1 https://github.com/openvm-org/openvm.git | |
- name: Install architecture specific tools | |
run: | | |
rustup install nightly-2024-10-30 | |
source openvm/ci/scripts/utils.sh | |
install_s5cmd | |
sudo apt update | |
sudo apt install gnuplot | |
- name: Setup halo2 | |
if: ${{ (github.event.inputs.mode == 'prove-e2e') || (inputs.mode == 'prove-e2e') }} | |
run: | | |
bash openvm/extensions/native/recursion/trusted_setup_s3.sh | |
PARAMS_DIR=$(pwd)/params | |
echo "PARAMS_DIR=$PARAMS_DIR" >> $GITHUB_ENV | |
OPTIONAL_ARGS="--kzg-params-dir $PARAMS_DIR --halo2-outer-k 22" | |
echo "OPTIONAL_ARGS=${OPTIONAL_ARGS}" >> $GITHUB_ENV | |
- name: Restore Guest ELF from cache | |
id: cache-guest-elf-restore | |
uses: runs-on/cache/restore@v4 | |
with: | |
path: bin/host/elf/openvm-client-eth | |
key: ${{ steps.set-cache-keys.outputs.elf_cache_key }} | |
- name: Install cargo-openvm | |
if: steps.cache-guest-elf-restore.outputs.cache-hit != 'true' | |
run: | | |
cargo install --git https://github.com/openvm-org/openvm.git --locked --force cargo-openvm | |
- name: Build Guest ELF | |
if: steps.cache-guest-elf-restore.outputs.cache-hit != 'true' | |
working-directory: bin/client-eth | |
run: | | |
GUEST_PROFILE=${{ steps.set-cache-keys.outputs.guest_profile }} | |
cargo openvm build --no-transpile --profile=$GUEST_PROFILE | |
mkdir -p ../host/elf | |
cp target/riscv32im-risc0-zkvm-elf/$GUEST_PROFILE/openvm-client-eth ../host/elf/ | |
- name: Save Guest ELF to cache | |
if: steps.cache-guest-elf-restore.outputs.cache-hit != 'true' | |
uses: runs-on/cache/save@v4 | |
with: | |
path: bin/host/elf/openvm-client-eth | |
key: ${{ steps.cache-guest-elf-restore.outputs.cache-primary-key }} | |
- name: Set build args | |
id: set-build-args | |
run: | | |
FEATURES="bench-metrics,nightly-features,${{ inputs.memory_allocator || github.event.inputs.memory_allocator || 'jemalloc' }}" | |
if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "guest" ]]; then | |
OPTIONAL_ARGS="${OPTIONAL_ARGS} --profiling" | |
FEATURES="${FEATURES},profiling" | |
fi | |
echo "OPTIONAL_ARGS=${OPTIONAL_ARGS}" >> $GITHUB_ENV | |
echo "FEATURES=${FEATURES}" >> $GITHUB_ENV | |
- name: Restore Host Binary from cache | |
id: cache-host-binary-restore | |
uses: runs-on/cache/restore@v4 | |
with: | |
path: target/${{ steps.set-cache-keys.outputs.host_profile }}/openvm-reth-benchmark-bin | |
key: ${{ steps.set-cache-keys.outputs.host_cache_key }} | |
- name: Build Host Binary | |
if: steps.cache-host-binary-restore.outputs.cache-hit != 'true' | |
run: | | |
arch=$(uname -m) | |
case $arch in | |
arm64|aarch64) | |
RUSTFLAGS="-Ctarget-cpu=native" | |
;; | |
x86_64|amd64) | |
RUSTFLAGS="-Ctarget-cpu=native -C target-feature=+avx512f" | |
FEATURES="${FEATURES},halo2-asm" | |
;; | |
*) | |
echo "Unsupported architecture: $arch" | |
exit 1 | |
;; | |
esac | |
HOST_PROFILE=${{ steps.set-cache-keys.outputs.host_profile }} | |
if [[ "$HOST_PROFILE" == "profiling" ]]; then | |
RUSTFLAGS="${RUSTFLAGS} -C force-frame-pointers=yes" | |
fi | |
export JEMALLOC_SYS_WITH_MALLOC_CONF="retain:false,background_thread:false,metadata_thp:always,thp:always,dirty_decay_ms:600000,muzzy_decay_ms:600000,abort_conf:true" | |
RUSTFLAGS=$RUSTFLAGS cargo build --bin openvm-reth-benchmark-bin --profile=$HOST_PROFILE --no-default-features --features=$FEATURES | |
echo "JEMALLOC_SYS_WITH_MALLOC_CONF=${JEMALLOC_SYS_WITH_MALLOC_CONF}" >> $GITHUB_ENV | |
- name: Save Host Binary to cache | |
if: steps.cache-host-binary-restore.outputs.cache-hit != 'true' | |
uses: runs-on/cache/save@v4 | |
with: | |
path: target/${{ steps.set-cache-keys.outputs.host_profile }}/openvm-reth-benchmark-bin | |
key: ${{ steps.cache-host-binary-restore.outputs.cache-primary-key }} | |
- name: Set up run benchmark script | |
run: | | |
mkdir -p rpc-cache | |
mkdir -p .bench_metrics | |
RPC_1=${{ secrets.RPC_URL_1 }} | |
MODE=${{ inputs.mode || github.event.inputs.mode }} | |
BLOCK_NUMBER=${{ inputs.block_number || github.event.inputs.block_number }} | |
HOST_PROFILE=${{ steps.set-cache-keys.outputs.host_profile }} | |
cat > run_benchmark.sh <<EOF | |
./target/$HOST_PROFILE/openvm-reth-benchmark-bin \ | |
--$MODE --block-number $BLOCK_NUMBER --rpc-url $RPC_1 --cache-dir rpc-cache \ | |
--app-log-blowup ${{ inputs.app_log_blowup || github.event.inputs.app_log_blowup }} \ | |
--leaf-log-blowup ${{ inputs.leaf_log_blowup || github.event.inputs.leaf_log_blowup }} \ | |
--internal-log-blowup ${{ inputs.internal_log_blowup || github.event.inputs.internal_log_blowup || 2 }} \ | |
--root-log-blowup ${{ inputs.root_log_blowup || github.event.inputs.root_log_blowup || 3 }} \ | |
--max-segment-length ${{ inputs.max_segment_length || github.event.inputs.max_segment_length }} \ | |
--segment-max-cells ${{ inputs.segment_max_cells || github.event.inputs.segment_max_cells }} \ | |
--num-children-leaf ${{ inputs.num_children_leaf || github.event.inputs.num_children_leaf || 1 }} \ | |
--num-children-internal ${{ inputs.num_children_internal || github.event.inputs.num_children_internal }} \ | |
$OPTIONAL_ARGS | |
EOF | |
chmod +x run_benchmark.sh | |
- name: Run benchmark | |
run: | | |
export JEMALLOC_SYS_WITH_MALLOC_CONF=${JEMALLOC_SYS_WITH_MALLOC_CONF} | |
export RUST_LOG="info,p3_=warn" | |
export OUTPUT_PATH=${METRIC_PATH} | |
if [[ "${{ inputs.profiling || github.event.inputs.profiling }}" == "host" ]]; then | |
echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid | |
echo 0 | sudo tee /proc/sys/kernel/kptr_restrict | |
perf --version | |
perf record -F 100 --call-graph=fp -g -o perf.data -- ./run_benchmark.sh | |
else | |
./ci/monitor_memory.sh ./run_benchmark.sh | |
echo "MEM_USAGE_PATH=memory_usage.png" >> $GITHUB_ENV | |
fi | |
- name: Upload metric file | |
id: upload-metric-artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: ${{ steps.set-metric-name.outputs.name }} | |
path: ${{ env.METRIC_PATH }} | |
retention-days: 1 | |
- name: Upload memory usage graph | |
if: ${{ (inputs.profiling || github.event.inputs.profiling) == 'none' }} | |
uses: actions/upload-artifact@v4 | |
with: | |
name: ${{ steps.set-metric-name.outputs.name }}-memory-usage | |
path: ${{ env.MEM_USAGE_PATH }} | |
retention-days: 1 | |
- name: Upload Benchmark Metrics | |
run: s5cmd cp ${METRIC_PATH} ${{ env.S3_METRICS_PATH }}/${METRIC_NAME}.json | |
- name: Generate markdown | |
run: | | |
BENCHER_METRIC_PATH=bencher.json | |
openvm-prof --json-paths $METRIC_PATH --output-json $BENCHER_METRIC_PATH | |
MD_PATH=${METRIC_PATH%.json}.md | |
echo "MD_PATH=${MD_PATH}" >> $GITHUB_ENV | |
echo "BENCHER_METRIC_PATH=${BENCHER_METRIC_PATH}" >> $GITHUB_ENV | |
- uses: bencherdev/bencher@main | |
- name: Upload bencher metrics | |
if: ${{ inputs.profiling == 'none' || github.event.inputs.profiling == 'none' }} | |
run: | | |
current_branch=$(git rev-parse --abbrev-ref HEAD) | |
if [[ "$current_branch" == nightly* ]]; then | |
BRANCH=nightly | |
else | |
BRANCH=dev | |
fi | |
echo "Bencher branch: ${BRANCH}" | |
bencher run \ | |
--project ${{ env.BENCHER_PROJECT }} \ | |
--token '${{ secrets.BENCHER_API_TOKEN }}' \ | |
--start-point nightly \ | |
--branch $BRANCH \ | |
--testbed ${{ inputs.instance_family || github.event.inputs.instance_family }} \ | |
--adapter json \ | |
--file $BENCHER_METRIC_PATH | |
- name: Generate circuit flamegraphs | |
if: ${{ inputs.profiling == 'guest' || github.event.inputs.profiling == 'guest' }} | |
run: | | |
cargo install inferno | |
python3 openvm/ci/scripts/metric_unify/flamegraph.py $METRIC_PATH | |
s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.S3_FLAMEGRAPHS_PATH }}/${METRIC_NAME}/" | |
echo "" >> $MD_PATH | |
echo "<details>" >> $MD_PATH | |
echo "<summary>Circuit Flamegraphs</summary>" >> $MD_PATH | |
echo "" >> $MD_PATH | |
for file in .bench_metrics/flamegraphs/*.svg; do | |
filename=$(basename "$file") | |
flamegraph_url=${{ env.S3_FLAMEGRAPHS_URL }}/${METRIC_NAME}/${filename} | |
echo "[]($flamegraph_url)" >> $MD_PATH | |
done | |
rm -f .bench_metrics/flamegraphs/*.svg | |
echo "" >> $MD_PATH | |
echo "</details>" >> $MD_PATH | |
echo "" >> $MD_PATH | |
- name: Add benchmark metadata to markdown | |
run: | | |
COMMIT_URL=https://github.com/${{ github.repository }}/commit/${current_sha} | |
BENCHMARK_WORKFLOW_URL=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
source openvm/ci/scripts/utils.sh | |
add_metadata $MD_PATH \ | |
${{ inputs.max_segment_length || github.event.inputs.max_segment_length }} \ | |
${{ inputs.instance_family || github.event.inputs.instance_family }} \ | |
${{ inputs.memory_allocator || github.event.inputs.memory_allocator }} \ | |
$COMMIT_URL \ | |
$BENCHMARK_WORKFLOW_URL | |
- name: Upload markdown | |
run: | | |
S3_MD_PATH="${{ env.S3_PATH }}/${METRIC_NAME}.md" | |
s5cmd cp $MD_PATH $S3_MD_PATH | |
echo "S3_MD_PATH=${S3_MD_PATH}" >> $GITHUB_ENV | |
- name: Generate samply profile artifacts | |
if: ${{ inputs.profiling == 'host' || github.event.inputs.profiling == 'host' }} | |
run: | | |
cargo install --git https://github.com/mstange/samply.git samply --force | |
mkdir -p samply_profile | |
SAMPLY_PROFILE_PATH=samply_profile | |
samply import perf.data --unstable-presymbolicate --save-only --output $SAMPLY_PROFILE_PATH/profile.json | |
echo "SAMPLY_PROFILE_PATH=${SAMPLY_PROFILE_PATH}" >> $GITHUB_ENV | |
s5cmd cp $SAMPLY_PROFILE_PATH/profile.json ${{ env.S3_SAMPLY_PROFILE_PATH }}/${METRIC_NAME}/profile.json | |
s5cmd cp $SAMPLY_PROFILE_PATH/profile.syms.json ${{ env.S3_SAMPLY_PROFILE_PATH }}/${METRIC_NAME}/profile.syms.json | |
echo "S3_SAMPLY_PROFILE_PATH=${S3_SAMPLY_PROFILE_PATH}" >> $GITHUB_ENV | |
- name: Upload samply profile artifacts | |
if: ${{ inputs.profiling == 'host' || github.event.inputs.profiling == 'host' }} | |
id: upload-samply-profile-artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: ${{ steps.set-metric-name.outputs.name }}-samply-profile | |
path: ${{ env.SAMPLY_PROFILE_PATH }} | |
retention-days: 1 | |
### Update gh-pages | |
- uses: actions/checkout@v4 | |
with: | |
ref: gh-pages | |
- name: Set up git | |
run: | | |
git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
git config --global user.name "github-actions[bot]" | |
- name: Commit to gh-pages branch | |
run: | | |
GH_PAGES_PATH="benchmarks-dispatch/${{ github.head_ref || github.ref }}" | |
echo "GH_PAGES_PATH=${GH_PAGES_PATH}" >> $GITHUB_ENV | |
mkdir -p ${GH_PAGES_PATH} | |
s5cmd cp $S3_MD_PATH "${GH_PAGES_PATH}/${METRIC_NAME}.md" | |
git add ${GH_PAGES_PATH}/${METRIC_NAME}.md | |
git commit --allow-empty -m "Update benchmark result at ${GH_PAGES_PATH}/${METRIC_NAME}.md" | |
MAX_RETRIES=10 | |
RETRY_DELAY=5 | |
ATTEMPT=0 | |
SUCCESS=false | |
while [ $ATTEMPT -lt $MAX_RETRIES ]; do | |
echo "Attempt $((ATTEMPT + 1)) to push of $MAX_RETRIES..." | |
git fetch origin gh-pages | |
git merge origin/gh-pages --no-edit | |
if git push origin gh-pages; then | |
SUCCESS=true | |
break | |
else | |
echo "Push failed. Retrying in $RETRY_DELAY seconds..." | |
sleep $RETRY_DELAY | |
ATTEMPT=$((ATTEMPT + 1)) | |
fi | |
done | |
if [ "$SUCCESS" = false ]; then | |
echo "PUSH_FAILED" | |
exit 1 | |
fi | |
- name: Path to result | |
run: | | |
echo "https://github.com/axiom-crypto/openvm-reth-benchmark/blob/gh-pages/${GH_PAGES_PATH}/${METRIC_NAME}.md" |