Reth Benchmark #29
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Reth Benchmark" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| # Default runner is not big enough for this | |
| # https://aws.amazon.com/ec2/instance-types/ | |
| instance_family: | |
| type: string | |
| required: false | |
| description: The family of the instance, can be multiple ones concat with "+" e.g. r8g+r7g | |
| default: m7a.48xlarge | |
| memory_allocator: | |
| type: string | |
| required: false | |
| description: Memory allocator to use (mimalloc or jemalloc) | |
| default: mimalloc | |
| block_number: | |
| type: number | |
| required: false | |
| description: Block number to run the benchmark on | |
| default: 21345144 | |
| app_log_blowup: | |
| type: number | |
| required: false | |
| description: Application level log blowup | |
| default: 2 | |
| agg_log_blowup: | |
| type: number | |
| required: false | |
| description: Aggregation (leaf) level log blowup | |
| default: 2 | |
| root_log_blowup: | |
| type: number | |
| required: false | |
| description: Root level log blowup (only for e2e) | |
| default: 3 | |
| internal_log_blowup: | |
| type: number | |
| required: false | |
| description: Internal level log blowup (only for e2e) | |
| default: 2 | |
| mode: | |
| type: choice | |
| required: false | |
| description: Running mode, one of execute, prove, or prove-e2e | |
| options: | |
| - execute | |
| - prove | |
| - prove-e2e | |
| default: prove | |
| collect_metrics: | |
| type: boolean | |
| required: false | |
| description: Whether to collect metrics for flamegraphs | |
| default: false | |
| max_segment_length: | |
| type: number | |
| required: false | |
| description: Max segment length for continuations, must be larger than 524288 | |
| default: 8388508 # 2**23 - 100 | |
| env: | |
| S3_PATH: s3://axiom-workflow-data-sandbox-us-east-1/benchmark/github/results | |
| S3_METRICS_PATH: s3://axiom-workflow-data-sandbox-us-east-1/benchmark/github/metrics | |
| PUBLIC_S3_PATH: s3://axiom-public-data-sandbox-us-east-1/benchmark/github/flamegraphs | |
| FEATURE_FLAGS: "bench-metrics,parallel,nightly-features" | |
| CMD_ARGS: "" | |
| INPUT_ARGS: "" | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| jobs: | |
| run-reth: | |
| name: Run Reth benchmark | |
| runs-on: | |
| - runs-on | |
| - run-id=${{ github.run_id }} | |
| - family=${{ inputs.instance_family }} | |
| - disk=large | |
| - tag=bench-reth-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref || github.ref }} | |
| - name: Give GitHub Actions access to axiom-crypto github org | |
| uses: webfactory/ssh-agent@v0.9.0 | |
| with: | |
| ssh-private-key: | | |
| ${{ secrets.GH_ACTIONS_DEPLOY_PRIVATE_KEY }} | |
| - uses: dtolnay/rust-toolchain@nightly | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| cache-on-failure: true | |
| - name: Checkout afs-prototype | |
| run: | | |
| git clone git@github.com:axiom-crypto/afs-prototype.git | |
| - name: Display workflow inputs | |
| run: echo "${{ toJSON(github.event.inputs) }}" | |
| - name: Setup CLI | |
| working-directory: afs-prototype/crates/cargo-axiom | |
| run: | | |
| cargo install --force --path . | |
| - name: Install architecture specific tools | |
| run: | | |
| source afs-prototype/ci/scripts/utils.sh | |
| install_s5cmd | |
| - name: Setup python | |
| run: | | |
| pip install -r ci/requirements.txt | |
| - name: Setup halo2 | |
| if: ${{ github.event.inputs.mode == 'prove-e2e' }} | |
| run: | | |
| bash afs-prototype/extensions/native/recursion/trusted_setup_s3.sh | |
| - name: Set metric name | |
| run: | | |
| input_json_str="${{ toJSON(github.event.inputs) }}" | |
| input_hash=$(echo $input_json_str | sha256sum | cut -d' ' -f1) | |
| METRIC_NAME=reth-${input_hash} | |
| echo "METRIC_NAME=${METRIC_NAME}" >> $GITHUB_ENV | |
| METRIC_PATH=".bench_metrics/${METRIC_NAME}.json" | |
| echo "METRIC_PATH=${METRIC_PATH}" >> $GITHUB_ENV | |
| - name: Run Reth | |
| run: | | |
| cd bin/client-eth | |
| cargo axiom build | |
| mkdir -p ../host/elf | |
| cp target/riscv32im-risc0-zkvm-elf/release/rsp-client-eth ../host/elf/ | |
| cd ../.. | |
| mkdir -p rpc-cache | |
| mkdir -p .bench_metrics # TODO: remove this | |
| RPC_1=${{ secrets.RPC_URL_1 }} | |
| MODE=${{ inputs.mode }} | |
| BLOCK_NUMBER=${{ inputs.block_number }} | |
| OPTIONAL_ARGS="" | |
| PROFILE="release" | |
| FEATURES="bench-metrics,${{ inputs.memory_allocator }}" | |
| if [[ "${{ inputs.collect_metrics }}" == "true" ]]; then | |
| OPTIONAL_ARGS="--collect-metrics" | |
| PROFILE="profiling" | |
| FEATURES="${FEATURES},profiling" | |
| fi | |
| cargo build --bin rsp --profile=$PROFILE --no-default-features --features=$FEATURES | |
| OUTPUT_PATH=${METRIC_PATH} RUSTFLAGS="-Ctarget-cpu=native" ./target/$PROFILE/rsp \ | |
| --$MODE --block-number $BLOCK_NUMBER --rpc-url $RPC_1 --cache-dir rpc-cache \ | |
| --app-log-blowup ${{ inputs.app_log_blowup }} \ | |
| --agg-log-blowup ${{ inputs.agg_log_blowup }} \ | |
| --root-log-blowup ${{ inputs.root_log_blowup }} \ | |
| --max-segment-length ${{ inputs.max_segment_length }} $OPTIONAL_ARGS | |
| - name: Upload Benchmark Metrics | |
| run: | | |
| current_sha=$(git rev-parse HEAD) | |
| echo "Current SHA: $current_sha" | |
| echo "current_sha=${current_sha}" >> $GITHUB_ENV | |
| s5cmd cp ${METRIC_PATH} ${{ env.S3_METRICS_PATH }}/${current_sha}-${METRIC_NAME}.json | |
| - name: Generate markdown # result path is hardcoded results.md | |
| run: | | |
| source afs-prototype/ci/scripts/utils.sh | |
| generate_markdown $METRIC_PATH $METRIC_NAME ${{ env.S3_METRICS_PATH }} "./afs-prototype" | |
| - name: Install inferno-flamegraph | |
| run: cargo install inferno | |
| - name: Generate flamegraphs | |
| run: | | |
| if [[ -f $METRIC_PATH ]]; then | |
| python3 afs-prototype/ci/scripts/metric_unify/flamegraph.py $METRIC_PATH | |
| s5cmd cp '.bench_metrics/flamegraphs/*.svg' "${{ env.PUBLIC_S3_PATH }}/${current_sha}/" | |
| echo "UPLOAD_FLAMEGRAPHS=1" >> $GITHUB_ENV | |
| fi | |
| - name: Add benchmark metadata to markdown, and upload | |
| run: | | |
| source afs-prototype/ci/scripts/utils.sh | |
| add_metadata results.md ${{ inputs.max_segment_length }} ${{ inputs.instance_family }} ${{ inputs.memory_allocator }} ${{ github.repository }} ${{ github.run_id }} | |
| - name: Summarize metrics md | |
| run: | | |
| if [[ -f $METRIC_PATH ]] && [[ ${{ inputs.mode }} == "prove-e2e" ]]; then | |
| # prepend to results.md | |
| python3 ci/summarize.py $METRIC_PATH --print > temp.md | |
| cat results.md >> temp.md | |
| rm results.md | |
| mv temp.md results.md | |
| fi | |
| - name: upload markdown | |
| run: | | |
| s5cmd cp results.md "${{ env.S3_PATH }}/${current_sha}-${METRIC_NAME}.md" | |
| ### Update gh-pages | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: gh-pages | |
| - name: Set up git | |
| run: | | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --global user.name "github-actions[bot]" | |
| - name: Commit to gh-pages branch | |
| run: | | |
| GH_PAGES_PATH="benchmarks-dispatch/${{ github.head_ref || github.ref }}" | |
| echo "GH_PAGES_PATH=${GH_PAGES_PATH}" >> $GITHUB_ENV | |
| mkdir -p ${GH_PAGES_PATH} | |
| s5cmd cp "${{ env.S3_PATH }}/${current_sha}-${METRIC_NAME}.md" "${GH_PAGES_PATH}/${METRIC_NAME}.md" | |
| git add ${GH_PAGES_PATH}/${METRIC_NAME}.md | |
| git commit --allow-empty -m "Update benchmark result at ${GH_PAGES_PATH}/${METRIC_NAME}.md" | |
| MAX_RETRIES=10 | |
| RETRY_DELAY=5 | |
| ATTEMPT=0 | |
| SUCCESS=false | |
| while [ $ATTEMPT -lt $MAX_RETRIES ]; do | |
| echo "Attempt $((ATTEMPT + 1)) to push of $MAX_RETRIES..." | |
| git fetch origin gh-pages | |
| git merge origin/gh-pages --no-edit | |
| if git push origin gh-pages; then | |
| SUCCESS=true | |
| break | |
| else | |
| echo "Push failed. Retrying in $RETRY_DELAY seconds..." | |
| sleep $RETRY_DELAY | |
| ATTEMPT=$((ATTEMPT + 1)) | |
| fi | |
| done | |
| if [ "$SUCCESS" = false ]; then | |
| echo "PUSH_FAILED" | |
| exit 1 | |
| fi | |
| - name: Path to result | |
| run: | | |
| echo "https://github.com/axiom-crypto/rsp/blob/gh-pages/${GH_PAGES_PATH}/${METRIC_NAME}.md" |