From 6e6020db1ea3ef26d8da86935988903dc004d74d Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Mon, 26 Jan 2026 15:52:54 +0000 Subject: [PATCH] chore: run proving benchmarks nightly and weekly --- .github/workflows/nightly-spartan-bench.yml | 117 +++++++++++++++++ .github/workflows/weekly-proving-bench.yml | 118 ++++++++++++++++++ bootstrap.sh | 24 ++++ ci.sh | 10 +- spartan/.gitignore | 3 +- spartan/bootstrap.sh | 20 ++- .../{prove-1tps.env => prove-n-tps-fake.env} | 4 +- spartan/environments/prove-n-tps-real.env | 44 +++++++ 8 files changed, 335 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/weekly-proving-bench.yml rename spartan/environments/{prove-1tps.env => prove-n-tps-fake.env} (94%) create mode 100644 spartan/environments/prove-n-tps-real.env diff --git a/.github/workflows/nightly-spartan-bench.yml b/.github/workflows/nightly-spartan-bench.yml index 8108c4782986..a8b290f0acb4 100644 --- a/.github/workflows/nightly-spartan-bench.yml +++ b/.github/workflows/nightly-spartan-bench.yml @@ -117,3 +117,120 @@ jobs: -H "Content-type: application/json" \ --data "$data" fi + + proving-benchmark: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + # Note cron jobs will be on 'next' by default. + + - name: Determine nightly tag + id: nightly-tag + run: | + if [[ -n "${{ inputs.nightly_tag }}" ]]; then + nightly_tag="${{ inputs.nightly_tag }}" + else + current_version=$(jq -r '."."' .release-please-manifest.json) + nightly_tag="${current_version}-nightly.$(date -u +%Y%m%d)" + fi + echo "nightly_tag=$nightly_tag" >> $GITHUB_OUTPUT + echo "Using nightly tag: $nightly_tag" + + - name: Check if Docker image exists + run: | + DOCKER_IMAGE="aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}" + echo "Checking if Docker image exists: $DOCKER_IMAGE" + if docker manifest inspect "$DOCKER_IMAGE" > /dev/null 2>&1; then + echo "Docker image exists: $DOCKER_IMAGE" + else + echo "Docker image does not exist: $DOCKER_IMAGE" + exit 1 + fi + + - name: Run proving benchmarks + timeout-minutes: 150 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + RUN_ID: ${{ github.run_id }} + AWS_SHUTDOWN_TIME: 150 + NO_SPOT: 1 + run: | + ./.github/ci3.sh network-proving-bench prove-n-tps-fake prove-n-tps-fake "aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}" + + - name: Cleanup network resources + if: always() + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + NO_SPOT: 1 + run: ./.github/ci3.sh network-teardown prove-n-tps-fake prove-n-tps-fake + + - name: Download benchmarks + if: always() + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + if ./ci.sh gh-spartan-proving-bench; then + echo "ENABLE_DEPLOY_BENCH=1" >> $GITHUB_ENV + fi + + - name: Upload benchmarks + if: always() && env.ENABLE_DEPLOY_BENCH == '1' + uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 + with: + name: Spartan + benchmark-data-dir-path: "bench/next" + tool: "customSmallerIsBetter" + output-file-path: ./bench-out/bench.json + github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + auto-push: true + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + alert-threshold: "120%" + comment-on-alert: false + fail-on-alert: false + max-items-in-chart: 100 + + - name: Notify Slack on failure + if: failure() && github.event_name != 'workflow_dispatch' + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + run: | + if [ -n "${SLACK_BOT_TOKEN}" ]; then + read -r -d '' data <" + } + EOF + curl -X POST https://slack.com/api/chat.postMessage \ + -H "Authorization: Bearer $SLACK_BOT_TOKEN" \ + -H "Content-type: application/json" \ + --data "$data" + fi + + status: + runs-on: ubuntu-latest + needs: [benchmark, proving-benchmark] + if: always() + steps: + - name: Check benchmark results + run: | + if [[ "${{ needs.benchmark.result }}" != "success" || "${{ needs.proving-benchmark.result }}" != "success" ]]; then + echo "One or more benchmark jobs failed" + echo "benchmark: ${{ needs.benchmark.result }}" + echo "proving-benchmark: ${{ needs.proving-benchmark.result }}" + exit 1 + fi + echo "All benchmark jobs succeeded" diff --git a/.github/workflows/weekly-proving-bench.yml b/.github/workflows/weekly-proving-bench.yml new file mode 100644 index 000000000000..f135b36505d7 --- /dev/null +++ b/.github/workflows/weekly-proving-bench.yml @@ -0,0 +1,118 @@ +name: Weekly Real Proving Benchmark + +on: + schedule: + - cron: "0 6 * * 1" # Every Monday at 6 AM UTC + workflow_dispatch: + inputs: + nightly_tag: + description: "Nightly tag to use (e.g., 2.3.4-nightly.20251209). Leave empty to auto-detect." + required: false + type: string + +concurrency: + group: weekly-proving-bench-${{ github.ref }} + cancel-in-progress: true + +jobs: + real-proving-benchmark: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + # Note cron jobs will be on 'next' by default. + + - name: Determine nightly tag + id: nightly-tag + run: | + if [[ -n "${{ inputs.nightly_tag }}" ]]; then + nightly_tag="${{ inputs.nightly_tag }}" + else + current_version=$(jq -r '."."' .release-please-manifest.json) + nightly_tag="${current_version}-nightly.$(date -u +%Y%m%d)" + fi + echo "nightly_tag=$nightly_tag" >> $GITHUB_OUTPUT + echo "Using nightly tag: $nightly_tag" + + - name: Check if Docker image exists + run: | + DOCKER_IMAGE="aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}" + echo "Checking if Docker image exists: $DOCKER_IMAGE" + if docker manifest inspect "$DOCKER_IMAGE" > /dev/null 2>&1; then + echo "Docker image exists: $DOCKER_IMAGE" + else + echo "Docker image does not exist: $DOCKER_IMAGE" + exit 1 + fi + + - name: Run real proving benchmarks + timeout-minutes: 150 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + RUN_ID: ${{ github.run_id }} + AWS_SHUTDOWN_TIME: 150 + NO_SPOT: 1 + run: | + ./.github/ci3.sh network-proving-bench prove-n-tps-real prove-n-tps-real "aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}" + + - name: Cleanup network resources + if: always() + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} + GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} + GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + NO_SPOT: 1 + run: ./.github/ci3.sh network-teardown prove-n-tps-real prove-n-tps-real + + - name: Download benchmarks + if: always() + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + if ./ci.sh gh-spartan-proving-bench; then + echo "ENABLE_DEPLOY_BENCH=1" >> $GITHUB_ENV + fi + + - name: Upload benchmarks + if: always() && env.ENABLE_DEPLOY_BENCH == '1' + uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 + with: + name: Spartan + benchmark-data-dir-path: "bench/next" + tool: "customSmallerIsBetter" + output-file-path: ./bench-out/bench.json + github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + auto-push: true + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + alert-threshold: "120%" + comment-on-alert: false + fail-on-alert: false + max-items-in-chart: 100 + + - name: Notify Slack on failure + if: failure() && github.event_name != 'workflow_dispatch' + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + run: | + if [ -n "${SLACK_BOT_TOKEN}" ]; then + read -r -d '' data <" + } + EOF + curl -X POST https://slack.com/api/chat.postMessage \ + -H "Authorization: Bearer $SLACK_BOT_TOKEN" \ + -H "Content-type: application/json" \ + --data "$data" + fi diff --git a/bootstrap.sh b/bootstrap.sh index fc3f9c93f103..7cad46a57714 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -653,6 +653,30 @@ case "$cmd" in bench_merge cache_upload spartan-bench-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json ;; + "ci-network-proving-bench") + # Args: [docker_image] + # Deploys network and runs proving benchmarks. Cleanup should be done separately. + export CI=1 + env_file="${1:?env_file is required}" + namespace="${2:?namespace is required}" + docker_image="${3:-}" + build + # If no docker image provided, build and push to aztecdev + if [ -z "$docker_image" ]; then + release-image/bootstrap.sh push_pr + docker_image="aztecprotocol/aztecdev:$(git rev-parse HEAD)" + fi + # Set up environment and deploy using spartan + export NAMESPACE="$namespace" + export AZTEC_DOCKER_IMAGE="$docker_image" + spartan/bootstrap.sh network_deploy "${env_file}" + # Run proving benchmarks + spartan/bootstrap.sh proving_bench "${env_file}" + rm -rf bench-out + mkdir -p bench-out + bench_merge + cache_upload spartan-proving-bench-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json + ;; "ci-network-teardown") # Args: # Tears down a deployed network. diff --git a/ci.sh b/ci.sh index d0a379a31a25..f556b7ae90c8 100755 --- a/ci.sh +++ b/ci.sh @@ -164,6 +164,14 @@ case "$cmd" in export INSTANCE_POSTFIX="n-bench" bootstrap_ec2 "./bootstrap.sh ci-network-bench $*" ;; + network-proving-bench) + # Args: [docker_image] + # Deploys network and runs proving benchmarks. + export CI_DASHBOARD="network" + export JOB_ID="x-${2:?namespace is required}-network-proving-bench" CPUS=16 + export INSTANCE_POSTFIX="n-proving-bench" + bootstrap_ec2 "./bootstrap.sh ci-network-proving-bench $*" + ;; network-teardown) # Args: export CI_DASHBOARD="network" @@ -290,7 +298,7 @@ case "$cmd" in ######################## # BENCHMARK PROCESSING # ######################## - gh-bench|gh-deploy-bench|gh-spartan-bench) + gh-bench|gh-deploy-bench|gh-spartan-bench|gh-spartan-proving-bench) cache_download ${cmd#gh-}-$(git rev-parse HEAD^{tree}).tar.gz ;; diff --git a/spartan/.gitignore b/spartan/.gitignore index ac40bd2e4129..f22abeb1e46f 100644 --- a/spartan/.gitignore +++ b/spartan/.gitignore @@ -6,7 +6,8 @@ tfplan mnemonic.tmp environments/* !environments/network-defaults.yml -!environments/prove-1tps.env +!environments/prove-n-tps-fake.env +!environments/prove-n-tps-real.env !environments/ten-tps-short-epoch.env !environments/ten-tps-long-epoch.env !environments/five-tps-short-epoch.env diff --git a/spartan/bootstrap.sh b/spartan/bootstrap.sh index 61432ed7fc0d..837fff6c1c7a 100755 --- a/spartan/bootstrap.sh +++ b/spartan/bootstrap.sh @@ -155,6 +155,12 @@ function network_bench_cmds { done } +function proving_bench_cmds { + local tps=1 + local timeout=9000 # 2.5h + echo "$hash:TIMEOUT=${timeout} TPS=${tps} BENCH_OUTPUT=bench-out/prove_n_tps.${tps}tps.bench.json $root/yarn-project/end-to-end/scripts/run_test.sh simple prove_n_tps.test.ts" +} + function network_bench { rm -rf bench-out mkdir -p bench-out @@ -167,6 +173,18 @@ function network_bench { network_bench_cmds | parallelize 1 } +function proving_bench { + rm -rf bench-out + mkdir -p bench-out + + local env_file="$1" + source_network_env $env_file + + echo_header "spartan proving bench" + gcp_auth + proving_bench_cmds | parallelize 1 +} + function ensure_eth_balances { amount="$1" # if ETHEREUM_HOST is not set, use the first RPC URL @@ -236,7 +254,7 @@ case "$cmd" in single_test $test_file ;; - network_tests|network_tests_1|network_tests_2|network_bench) + network_tests|network_tests_1|network_tests_2|network_bench|proving_bench) env_file="$1" $cmd "$env_file" ;; diff --git a/spartan/environments/prove-1tps.env b/spartan/environments/prove-n-tps-fake.env similarity index 94% rename from spartan/environments/prove-1tps.env rename to spartan/environments/prove-n-tps-fake.env index 5be8ada149c6..7b228b2db601 100644 --- a/spartan/environments/prove-1tps.env +++ b/spartan/environments/prove-n-tps-fake.env @@ -1,10 +1,10 @@ -NAMESPACE=${NAMESPACE:-prove-1tps} +NAMESPACE=${NAMESPACE:-prove-n-tps-fake} CLUSTER=aztec-gke-private GCP_REGION=us-west1-a AZTEC_EPOCH_DURATION=32 AZTEC_SLOT_DURATION=72 -AZTEC_PROOF_SUBMISSION_WINDOW=64 +AZTEC_PROOF_SUBMISSION_EPOCHS=1 AZTEC_LAG_IN_EPOCHS=1 CREATE_ETH_DEVNET=true diff --git a/spartan/environments/prove-n-tps-real.env b/spartan/environments/prove-n-tps-real.env new file mode 100644 index 000000000000..407409f9cd34 --- /dev/null +++ b/spartan/environments/prove-n-tps-real.env @@ -0,0 +1,44 @@ +NAMESPACE=${NAMESPACE:-prove-n-tps-real} +CLUSTER=aztec-gke-private +GCP_REGION=us-west1-a + +AZTEC_EPOCH_DURATION=32 +AZTEC_SLOT_DURATION=72 +AZTEC_PROOF_SUBMISSION_EPOCHS=1 +AZTEC_LAG_IN_EPOCHS=1 + +CREATE_ETH_DEVNET=true +DESTROY_NAMESPACE=true +DESTROY_AZTEC_INFRA=true +CREATE_ROLLUP_CONTRACTS=true +REDEPLOY_ROLLUP_CONTRACTS=true + +ETHEREUM_CHAIN_ID=1337 +LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk" +FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + +OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET + +VALIDATOR_REPLICAS=4 +VALIDATORS_PER_NODE=12 +PUBLISHERS_PER_VALIDATOR_KEY=1 +VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 + +REAL_VERIFIER=true + +RPC_REPLICAS=1 +RPC_INGRESS_ENABLED=false + +PROVER_REPLICAS=200 +PROVER_RESOURCE_PROFILE="prod" +PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 +PROVER_AGENT_POLL_INTERVAL_MS=10000 +PUBLISHERS_PER_PROVER=1 + +SEQ_MAX_TX_PER_BLOCK=80 +SEQ_MIN_TX_PER_BLOCK=0 +P2P_MAX_TX_POOL_SIZE=1000000000 +DEBUG_P2P_INSTRUMENT_MESSAGES=true + +PROVER_AGENT_INCLUDE_METRICS="aztec.circuit" +LOG_LEVEL=info