Benchmark #275
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| debug: | |
| type: boolean | |
| description: "Run with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)" | |
| required: false | |
| default: false | |
| vm: | |
| description: "Space-separated list of VMs to benchmark (openvm stwo risc0 stone sp1 jolt)" | |
| required: false | |
| default: "openvm stwo risc0 stone sp1 jolt" | |
| type: string | |
| arg-fib: | |
| description: "Fibonacci suite arguments" | |
| required: false | |
| default: "32768 65536" | |
| type: string | |
| arg-sha2: | |
| description: "SHA2 benchmark arguments" | |
| required: false | |
| default: "256 512" | |
| type: string | |
| arg-sha2-chain: | |
| description: "SHA2 chain benchmark arguments" | |
| required: false | |
| default: "8 16" | |
| type: string | |
| arg-sha3: | |
| description: "SHA3 benchmark arguments" | |
| required: false | |
| default: "256 512" | |
| type: string | |
| arg-sha3-chain: | |
| description: "SHA3 chain benchmark arguments" | |
| required: false | |
| default: "8 16" | |
| type: string | |
| arg-matmul: | |
| description: "Matrix multiplication suite arguments" | |
| required: false | |
| default: "4 8" | |
| type: string | |
| arg-ec: | |
| description: "Elliptic curve suite arguments" | |
| required: false | |
| default: "256 512" | |
| type: string | |
| machine_type: | |
| description: "Hetzner Cloud server type (e.g. cx42, ccx63)" | |
| required: false | |
| default: "cx42" | |
| type: string | |
| push: | |
| paths-ignore: | |
| - "**.md" | |
| permissions: | |
| contents: write | |
| actions: read | |
| concurrency: | |
| group: benchmark-runner | |
| cancel-in-progress: true | |
| env: | |
| FIB_ARG: ${{ startsWith(github.ref, 'refs/tags/bench-') && '32768 65536 131072 262144 524288 1048576 2097152 4194304' || inputs.arg-fib }} | |
| SHA2_ARG: ${{ startsWith(github.ref, 'refs/tags/bench-') && '256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288' || inputs.arg-sha2 }} | |
| SHA2_CHAIN_ARG: ${{ startsWith(github.ref, 'refs/tags/bench-') && '8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768' || inputs.arg-sha2-chain }} | |
| SHA3_ARG: ${{ startsWith(github.ref, 'refs/tags/bench-') && '256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288' || inputs.arg-sha3 }} | |
| SHA3_CHAIN_ARG: ${{ startsWith(github.ref, 'refs/tags/bench-') && '8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768' || inputs.arg-sha3-chain }} | |
| MATMUL_ARG: ${{ startsWith(github.ref, 'refs/tags/bench-') && '4 8 16 32 64 128' || inputs.arg-matmul }} | |
| EC_ARG: ${{ startsWith(github.ref, 'refs/tags/bench-') && '256 512 1024 2048 4096 8192 16384 32768' || inputs.arg-ec }} | |
| jobs: | |
| create-runner: | |
| name: Create Hetzner Cloud Runner | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| label: ${{ steps.create-hcloud-runner.outputs.label }} | |
| server_id: ${{ steps.create-hcloud-runner.outputs.server_id }} | |
| steps: | |
| - name: Create Runner | |
| id: create-hcloud-runner | |
| uses: Cyclenerd/hcloud-github-runner@v1 | |
| with: | |
| name: gh-benchmark-runner | |
| mode: create | |
| github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| hcloud_token: ${{ secrets.HCLOUD_TOKEN }} | |
| server_type: ${{ startsWith(github.ref, 'refs/tags/bench-') && 'ccx63' || inputs.machine_type || 'cx42' }} | |
| location: nbg1 | |
| ssh_key: 27148616 | |
| image: ubuntu-24.04 | |
| setup: | |
| name: Setup Environment | |
| timeout-minutes: 7200 | |
| needs: create-runner | |
| runs-on: ${{ needs.create-runner.outputs.label }} | |
| env: | |
| BENCH_USER: shodan | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| - name: Setup tmate session | |
| if: ${{ inputs.debug == true }} | |
| uses: mxschmitt/action-tmate@v3 | |
| - name: Create Benchmark User | |
| run: | | |
| sudo useradd -m -s /bin/bash $BENCH_USER | |
| sudo usermod -aG sudo $BENCH_USER | |
| echo "$BENCH_USER ALL=(ALL) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/$BENCH_USER | |
| sudo chown -R $BENCH_USER:$BENCH_USER . | |
| - name: Install Dependencies | |
| run: | | |
| sudo -u $BENCH_USER ./scripts/setup.sh | |
| - name: Capture Machine Information | |
| run: sudo -u $BENCH_USER just machine-info | |
| - name: Upload Machine Info Results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: result-machine-info | |
| path: ./report_info | |
| benchmark: | |
| name: Benchmark ${{ matrix.zkvm }} | |
| timeout-minutes: 7200 | |
| needs: | |
| - create-runner | |
| - setup | |
| runs-on: ${{ needs.create-runner.outputs.label }} | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| zkvm: | |
| - openvm | |
| - stwo | |
| - risc0 | |
| - stone | |
| - sp1 | |
| - jolt | |
| env: | |
| BENCH_USER: shodan | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| if: ${{ always() && (startsWith(github.ref, 'refs/tags/bench-') || contains(inputs.vm, matrix.zkvm)) }} | |
| - name: Run ${{ matrix.zkvm }} benchmark | |
| if: ${{ always() && (startsWith(github.ref, 'refs/tags/bench-') || contains(inputs.vm, matrix.zkvm)) }} | |
| run: | | |
| if [ "${{ matrix.zkvm }}" = "stone" ]; then | |
| # Stone requires special certificate setup and environment variables | |
| WORK_DIR=$(pwd) | |
| echo "${{ secrets.SHARP_CLIENT_CERT }}" > user-cert.pem | |
| echo "${{ secrets.SHARP_CLIENT_KEY }}" > user-key.pem | |
| chmod 644 user-cert.pem user-key.pem | |
| sudo -u $BENCH_USER \ | |
| SHARP_CLIENT_CERT="$WORK_DIR/user-cert.pem" \ | |
| SHARP_KEY_PATH="$WORK_DIR/user-key.pem" \ | |
| SHARP_KEY_PASSWD="${{ secrets.SHARP_CLIENT_KEY_PASSWORD }}" \ | |
| just bench-stone \ | |
| "${{ env.FIB_ARG }}" \ | |
| "${{ env.SHA2_ARG }}" \ | |
| "${{ env.SHA2_CHAIN_ARG }}" \ | |
| "${{ env.SHA3_ARG }}" \ | |
| "${{ env.SHA3_CHAIN_ARG }}" \ | |
| "${{ env.MATMUL_ARG }}" \ | |
| "${{ env.EC_ARG }}" | |
| else | |
| # Standard benchmark command for all other zkVMs | |
| sudo -u $BENCH_USER just bench-${{ matrix.zkvm }} \ | |
| "${{ env.FIB_ARG }}" \ | |
| "${{ env.SHA2_ARG }}" \ | |
| "${{ env.SHA2_CHAIN_ARG }}" \ | |
| "${{ env.SHA3_ARG }}" \ | |
| "${{ env.SHA3_CHAIN_ARG }}" \ | |
| "${{ env.MATMUL_ARG }}" \ | |
| "${{ env.EC_ARG }}" | |
| fi | |
| - name: Upload ${{ matrix.zkvm }} results | |
| if: ${{ always() && (startsWith(github.ref, 'refs/tags/bench-') || contains(inputs.vm, matrix.zkvm)) }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: result-bench-${{ matrix.zkvm }} | |
| path: ./benchmark_results | |
| merge-results: | |
| name: Merge Results | |
| needs: | |
| - setup | |
| - benchmark | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check benchmark status | |
| run: | | |
| echo "Setup job result: ${{ needs.setup.result }}" | |
| echo "Benchmark job result: ${{ needs.benchmark.result }}" | |
| if [ "${{ needs.benchmark.result }}" = "skipped" ]; then | |
| echo "All benchmarks were skipped" | |
| elif [ "${{ needs.benchmark.result }}" = "success" ]; then | |
| echo "All requested benchmarks completed successfully" | |
| else | |
| echo "Some benchmarks may have failed or been skipped" | |
| fi | |
| - name: Download all results | |
| uses: actions/download-artifact@v4 | |
| continue-on-error: true | |
| with: | |
| path: all-results | |
| pattern: "result-*" | |
| - name: Merge results and create zip | |
| run: | | |
| mkdir -p merged-results | |
| # Copy all files from all results directories if they exist | |
| if [ -d "all-results" ]; then | |
| find all-results -type f -exec cp {} merged-results/ \; 2>/dev/null || true | |
| fi | |
| # Create zip file (even if empty) | |
| cd merged-results | |
| if [ "$(ls -A .)" ]; then | |
| echo "Found $(ls | wc -l) result files" | |
| zip -r ../benchmark-results.zip . | |
| else | |
| echo "No results found - all benchmarks may have been skipped" > no-results.txt | |
| zip -r ../benchmark-results.zip no-results.txt | |
| fi | |
| cd .. | |
| - name: Upload Merged Results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: merged-benchmark-results | |
| path: merged-results/* | |
| - name: Create Release From Results | |
| if: startsWith(github.ref, 'refs/tags/bench-') | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| files: benchmark-results.zip | |
| delete-runner: | |
| name: Delete Hetzner Cloud Runner | |
| needs: | |
| - create-runner | |
| - merge-results | |
| runs-on: ubuntu-24.04 | |
| if: ${{ always() }} | |
| steps: | |
| - name: Delete runner | |
| uses: Cyclenerd/hcloud-github-runner@v1 | |
| with: | |
| mode: delete | |
| github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} | |
| hcloud_token: ${{ secrets.HCLOUD_TOKEN }} | |
| name: ${{ needs.create-runner.outputs.label }} | |
| server_id: ${{ needs.create-runner.outputs.server_id }} |