chore: run spartan proving tests nightly and weekly

.github/workflows/nightly-spartan-bench.yml

@@ -117,3 +117,120 @@ jobs:
               -H "Content-type: application/json" \
               --data "$data"
           fi
+
+  proving-benchmark:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        # Note cron jobs will be on 'next' by default.
+
+      - name: Determine nightly tag
+        id: nightly-tag
+        run: |
+          if [[ -n "${{ inputs.nightly_tag }}" ]]; then
+            nightly_tag="${{ inputs.nightly_tag }}"
+          else
+            current_version=$(jq -r '."."' .release-please-manifest.json)
+            nightly_tag="${current_version}-nightly.$(date -u +%Y%m%d)"
+          fi
+          echo "nightly_tag=$nightly_tag" >> $GITHUB_OUTPUT
+          echo "Using nightly tag: $nightly_tag"
+
+      - name: Check if Docker image exists
+        run: |
+          DOCKER_IMAGE="aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}"
+          echo "Checking if Docker image exists: $DOCKER_IMAGE"
+          if docker manifest inspect "$DOCKER_IMAGE" > /dev/null 2>&1; then
+            echo "Docker image exists: $DOCKER_IMAGE"
+          else
+            echo "Docker image does not exist: $DOCKER_IMAGE"
+            exit 1
+          fi
+
+      - name: Run proving benchmarks
+        timeout-minutes: 150
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          RUN_ID: ${{ github.run_id }}
+          AWS_SHUTDOWN_TIME: 150
+          NO_SPOT: 1
+        run: |
+          ./.github/ci3.sh network-proving-bench prove-n-tps-fake prove-n-tps-fake "aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}"
+
+      - name: Cleanup network resources
+        if: always()
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          NO_SPOT: 1
+        run: ./.github/ci3.sh network-teardown prove-n-tps-fake prove-n-tps-fake
+
+      - name: Download benchmarks
+        if: always()
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          if ./ci.sh gh-spartan-proving-bench; then
+            echo "ENABLE_DEPLOY_BENCH=1" >> $GITHUB_ENV
+          fi
+
+      - name: Upload benchmarks
+        if: always() && env.ENABLE_DEPLOY_BENCH == '1'
+        uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
+        with:
+          name: Spartan
+          benchmark-data-dir-path: "bench/next"
+          tool: "customSmallerIsBetter"
+          output-file-path: ./bench-out/bench.json
+          github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          auto-push: true
+          ref: ${{ github.event.workflow_run.head_sha || github.sha }}
+          alert-threshold: "120%"
+          comment-on-alert: false
+          fail-on-alert: false
+          max-items-in-chart: 100
+
+      - name: Notify Slack on failure
+        if: failure() && github.event_name != 'workflow_dispatch'
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        run: |
+          if [ -n "${SLACK_BOT_TOKEN}" ]; then
+            read -r -d '' data <<EOF || true
+            {
+              "channel": "#alerts-next-scenario",
+              "text": "Nightly Proving Benchmarks FAILED (nightly tag ${{ steps.nightly-tag.outputs.nightly_tag }}): <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
+            }
+          EOF
+            curl -X POST https://slack.com/api/chat.postMessage \
+              -H "Authorization: Bearer $SLACK_BOT_TOKEN" \
+              -H "Content-type: application/json" \
+              --data "$data"
+          fi
+
+  status:
+    runs-on: ubuntu-latest
+    needs: [benchmark, proving-benchmark]
+    if: always()
+    steps:
+      - name: Check benchmark results
+        run: |
+          if [[ "${{ needs.benchmark.result }}" != "success" || "${{ needs.proving-benchmark.result }}" != "success" ]]; then
+            echo "One or more benchmark jobs failed"
+            echo "benchmark: ${{ needs.benchmark.result }}"
+            echo "proving-benchmark: ${{ needs.proving-benchmark.result }}"
+            exit 1
+          fi
+          echo "All benchmark jobs succeeded"

.github/workflows/weekly-proving-bench.yml

@@ -0,0 +1,118 @@
+name: Weekly Real Proving Benchmark
+
+on:
+  schedule:
+    - cron: "0 6 * * 1"  # Every Monday at 6 AM UTC
+  workflow_dispatch:
+    inputs:
+      nightly_tag:
+        description: "Nightly tag to use (e.g., 2.3.4-nightly.20251209). Leave empty to auto-detect."
+        required: false
+        type: string
+
+concurrency:
+  group: weekly-proving-bench-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  real-proving-benchmark:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        # Note cron jobs will be on 'next' by default.
+
+      - name: Determine nightly tag
+        id: nightly-tag
+        run: |
+          if [[ -n "${{ inputs.nightly_tag }}" ]]; then
+            nightly_tag="${{ inputs.nightly_tag }}"
+          else
+            current_version=$(jq -r '."."' .release-please-manifest.json)
+            nightly_tag="${current_version}-nightly.$(date -u +%Y%m%d)"
+          fi
+          echo "nightly_tag=$nightly_tag" >> $GITHUB_OUTPUT
+          echo "Using nightly tag: $nightly_tag"
+
+      - name: Check if Docker image exists
+        run: |
+          DOCKER_IMAGE="aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}"
+          echo "Checking if Docker image exists: $DOCKER_IMAGE"
+          if docker manifest inspect "$DOCKER_IMAGE" > /dev/null 2>&1; then
+            echo "Docker image exists: $DOCKER_IMAGE"
+          else
+            echo "Docker image does not exist: $DOCKER_IMAGE"
+            exit 1
+          fi
+
+      - name: Run real proving benchmarks
+        timeout-minutes: 150
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          RUN_ID: ${{ github.run_id }}
+          AWS_SHUTDOWN_TIME: 150
+          NO_SPOT: 1
+        run: |
+          ./.github/ci3.sh network-proving-bench prove-n-tps-real prove-n-tps-real "aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}"
+
+      - name: Cleanup network resources
+        if: always()
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          NO_SPOT: 1
+        run: ./.github/ci3.sh network-teardown prove-n-tps-real prove-n-tps-real
+
+      - name: Download benchmarks
+        if: always()
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          if ./ci.sh gh-spartan-proving-bench; then
+            echo "ENABLE_DEPLOY_BENCH=1" >> $GITHUB_ENV
+          fi
+
+      - name: Upload benchmarks
+        if: always() && env.ENABLE_DEPLOY_BENCH == '1'
+        uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
+        with:
+          name: Spartan
+          benchmark-data-dir-path: "bench/next"
+          tool: "customSmallerIsBetter"
+          output-file-path: ./bench-out/bench.json
+          github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          auto-push: true
+          ref: ${{ github.event.workflow_run.head_sha || github.sha }}
+          alert-threshold: "120%"
+          comment-on-alert: false
+          fail-on-alert: false
+          max-items-in-chart: 100
+
+      - name: Notify Slack on failure
+        if: failure() && github.event_name != 'workflow_dispatch'
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        run: |
+          if [ -n "${SLACK_BOT_TOKEN}" ]; then
+            read -r -d '' data <<EOF || true
+            {
+              "channel": "#alerts-next-scenario",
+              "text": "Weekly Real Proving Benchmark FAILED (nightly tag ${{ steps.nightly-tag.outputs.nightly_tag }}): <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
+            }
+          EOF
+            curl -X POST https://slack.com/api/chat.postMessage \
+              -H "Authorization: Bearer $SLACK_BOT_TOKEN" \
+              -H "Content-type: application/json" \
+              --data "$data"
+          fi

bootstrap.sh

@@ -653,6 +653,30 @@ case "$cmd" in
     bench_merge
     cache_upload spartan-bench-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json
     ;;
+  "ci-network-proving-bench")
+    # Args: <env_file> <namespace> [docker_image]
+    # Deploys network and runs proving benchmarks. Cleanup should be done separately.
+    export CI=1
+    env_file="${1:?env_file is required}"
+    namespace="${2:?namespace is required}"
+    docker_image="${3:-}"
+    build
+    # If no docker image provided, build and push to aztecdev
+    if [ -z "$docker_image" ]; then
+      release-image/bootstrap.sh push_pr
+      docker_image="aztecprotocol/aztecdev:$(git rev-parse HEAD)"
+    fi
+    # Set up environment and deploy using spartan
+    export NAMESPACE="$namespace"
+    export AZTEC_DOCKER_IMAGE="$docker_image"
+    spartan/bootstrap.sh network_deploy "${env_file}"
+    # Run proving benchmarks
+    spartan/bootstrap.sh proving_bench "${env_file}"
+    rm -rf bench-out
+    mkdir -p bench-out
+    bench_merge
+    cache_upload spartan-proving-bench-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json
+    ;;
   "ci-network-teardown")
     # Args: <env_file> <namespace>
     # Tears down a deployed network.

ci.sh

@@ -164,6 +164,14 @@ case "$cmd" in
     export INSTANCE_POSTFIX="n-bench"
     bootstrap_ec2 "./bootstrap.sh ci-network-bench $*"
     ;;
+  network-proving-bench)
+    # Args: <scenario> <namespace> [docker_image]
+    # Deploys network and runs proving benchmarks.
+    export CI_DASHBOARD="network"
+    export JOB_ID="x-${2:?namespace is required}-network-proving-bench" CPUS=16
+    export INSTANCE_POSTFIX="n-proving-bench"
+    bootstrap_ec2 "./bootstrap.sh ci-network-proving-bench $*"
+    ;;
   network-teardown)
     # Args: <scenario> <namespace>
     export CI_DASHBOARD="network"
@@ -290,7 +298,7 @@ case "$cmd" in
   ########################
   # BENCHMARK PROCESSING #
   ########################
-  gh-bench|gh-deploy-bench|gh-spartan-bench)
+  gh-bench|gh-deploy-bench|gh-spartan-bench|gh-spartan-proving-bench)
     cache_download ${cmd#gh-}-$(git rev-parse HEAD^{tree}).tar.gz
     ;;
 

spartan/.gitignore

@@ -6,7 +6,8 @@ tfplan
 mnemonic.tmp
 environments/*
 !environments/network-defaults.yml
-!environments/prove-1tps.env
+!environments/prove-n-tps-fake.env
+!environments/prove-n-tps-real.env
 !environments/ten-tps-short-epoch.env
 !environments/ten-tps-long-epoch.env
 !environments/five-tps-short-epoch.env

spartan/bootstrap.sh

@@ -155,6 +155,12 @@ function network_bench_cmds {
   done
 }
 
+function proving_bench_cmds {
+  local tps=1
+  local timeout=9000  # 2.5h
+  echo "$hash:TIMEOUT=${timeout} TPS=${tps} BENCH_OUTPUT=bench-out/prove_n_tps.${tps}tps.bench.json $root/yarn-project/end-to-end/scripts/run_test.sh simple prove_n_tps.test.ts"
+}
+
 function network_bench {
   rm -rf bench-out
   mkdir -p bench-out
@@ -167,6 +173,18 @@ function network_bench {
   network_bench_cmds | parallelize 1
 }
 
+function proving_bench {
+  rm -rf bench-out
+  mkdir -p bench-out
+
+  local env_file="$1"
+  source_network_env $env_file
+
+  echo_header "spartan proving bench"
+  gcp_auth
+  proving_bench_cmds | parallelize 1
+}
+
 function ensure_eth_balances {
   amount="$1"
   # if ETHEREUM_HOST is not set, use the first RPC URL
@@ -236,7 +254,7 @@ case "$cmd" in
     single_test $test_file
     ;;
 
-  network_tests|network_tests_1|network_tests_2|network_bench)
+  network_tests|network_tests_1|network_tests_2|network_bench|proving_bench)
     env_file="$1"
     $cmd "$env_file"
     ;;

spartan/environments/prove-1tps.env → spartan/environments/prove-n-tps-fake.env

@@ -1,10 +1,10 @@
-NAMESPACE=${NAMESPACE:-prove-1tps}
+NAMESPACE=${NAMESPACE:-prove-n-tps-fake}
 CLUSTER=aztec-gke-private
 GCP_REGION=us-west1-a
 
 AZTEC_EPOCH_DURATION=32
 AZTEC_SLOT_DURATION=72
-AZTEC_PROOF_SUBMISSION_WINDOW=64
+AZTEC_PROOF_SUBMISSION_EPOCHS=1
 AZTEC_LAG_IN_EPOCHS=1
 
 CREATE_ETH_DEVNET=true