chore(ci): Run benchmarks on Earthly

.circleci/config.yml

@@ -482,37 +482,6 @@ jobs:
   # For old e2e tests see yarn-project/end-to-end/Earthfile
   # Semantics are similar to Dockerfile
 
-  # NOTE: Unlike other e2e, these will be re-enabled here as currently the logs functionality is not in the new earthfile setup
-  bench-publish-rollup:
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Benchmark"
-          command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_publish_rollup.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees
-          aztec_manifest_key: end-to-end
-    <<: *defaults_e2e_test
-
-  bench-process-history:
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Benchmark"
-          command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_process_history.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees
-          aztec_manifest_key: end-to-end
-    <<: *defaults_e2e_test
-
-  bench-tx-size:
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Benchmark"
-          command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_tx_size_fees.test.ts ENABLE_GAS=1 DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees
-          aztec_manifest_key: end-to-end
-    <<: *defaults_e2e_test
-
   e2e-join:
     docker:
       - image: cimg/base:2023.09
@@ -531,16 +500,6 @@ jobs:
           name: "Noop"
           command: echo Noop
 
-  bench-summary:
-    machine:
-      image: default
-    steps:
-      - *checkout
-      - *setup_env
-      - run:
-          name: "Assemble benchmark summary from uploaded logs"
-          command: ./scripts/ci/assemble_e2e_benchmark.sh
-
   # Deploy & release jobs.
   deploy-and-release:
     machine:
@@ -775,16 +734,5 @@ workflows:
             - e2e-join
           <<: *defaults
 
-      # Benchmark jobs.
-      - bench-publish-rollup: *e2e_test
-      - bench-process-history: *e2e_test
-      - bench-tx-size: *e2e_test
-      - bench-summary:
-          requires:
-            - bench-publish-rollup
-            - bench-process-history
-            - bench-tx-size
-          <<: *defaults
-
       # Production releases.
       - deploy-and-release: *defaults_deploy

.github/ci-setup-action/action.yml

@@ -1,14 +1,14 @@
 # Reusable setup workflow for CI tasks
 name: Setup Workflow
-description: 'Reusable setup steps'
+description: "Reusable setup steps"
 
 inputs:
   dockerhub_password:
     required: true
-    description: 'DockerHub Password'
+    description: "DockerHub Password"
   concurrency_key:
     required: false
-    description: 'Concurrency key for locking jobs'
+    description: "Concurrency key for locking jobs"
 runs:
   # define an action, runs in OS of caller
   using: composite
@@ -43,11 +43,14 @@ runs:
       with:
         # permission issue with spot runners, simpler to leave out
         use-cache: false
-        version: 'v0.8.5'
+        version: "v0.8.5"
 
     - name: Setup Env
       shell: bash
       run: ./scripts/setup_env.sh ${{ inputs.dockerhub_password }}
+      env:
+        PULL_REQUEST: "${{ github.event.pull_request.number }}"
+        BRANCH: "${{ github.ref_name }}"
 
     # As detailed in https://github.com/ben-z/gh-action-mutex
     # things do not become 'pending' in github actions, and instead just cancel one another

.github/workflows/ci.yml

@@ -11,10 +11,12 @@ on:
       runner_action:
         description: "The action to take with the self-hosted runner (start, stop, restart)."
         required: false
+
 concurrency:
   # force parallelism in master
   group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}
   cancel-in-progress: true
+
 jobs:
   setup:
     uses: ./.github/workflows/setup-runner.yml
@@ -64,10 +66,8 @@ jobs:
       matrix:
         test: ${{ fromJson( needs.build.outputs.e2e_list )}}
     steps:
-      - {
-          uses: actions/checkout@v4,
-          with: { ref: "${{ github.event.pull_request.head.sha }}" },
-        }
+      - uses: actions/checkout@v4
+        with: { ref: "${{ github.event.pull_request.head.sha }}" }
       - uses: ./.github/ci-setup-action
         with:
           dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
@@ -76,10 +76,35 @@ jobs:
       - name: Test
         working-directory: ./yarn-project/end-to-end/
         timeout-minutes: 25
-        run: earthly-ci -P --no-output +${{ matrix.test }} --e2e_mode=cache
-      # TODO
-      # - name: Upload logs
-      #   run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log
+        run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --no-output +${{ matrix.test }} --e2e_mode=cache
+
+  bench-summary:
+    needs: e2e
+    runs-on: ${{ inputs.username || github.actor }}-x86
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 100 # Downloading base benchmark from master requires access to history
+          ref: "${{ github.event.pull_request.head.sha }}"
+      - uses: ./.github/ci-setup-action
+        with:
+          dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"
+          concurrency_key: build-${{ inputs.username || github.actor }}-x86
+      - name: "Build and upload bench aggregate file"
+        working-directory: ./yarn-project/scripts
+        run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} +bench-aggregate
+      - name: "Download base benchmark"
+        if: ${{ github.event_name == 'pull_request' }}
+        run: scripts/logs/download_base_benchmark_from_s3.sh
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          BENCH_FOLDER: ./yarn-project/scripts/bench
+          PULL_REQUEST: "${{ github.event.pull_request.number }}"
+      - name: "Generate summary comment if pull request"
+        if: ${{ github.event_name == 'pull_request' }}
+        working-directory: ./yarn-project/scripts
+        run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} +bench-comment
 
   # barretenberg (prover) native and AVM (public VM) tests
   # only ran on x86 for resource reasons (memory intensive)

.gitignore

@@ -18,3 +18,7 @@ cmake-build-debug
 .DS_Store
 
 **/*.dockerignore
+
+# Earthly
+.arg
+.secret

Earthfile

@@ -32,3 +32,9 @@ bench:
 release-meta:
     COPY .release-please-manifest.json /usr/src/.release-please-manifest.json
     SAVE ARTIFACT /usr/src /usr/src
+
+scripts:
+    FROM ubuntu:lunar
+    RUN apt-get update && apt-get install -y awscli
+    COPY scripts /usr/src/scripts
+    SAVE ARTIFACT /usr/src/scripts scripts

scripts/logs/check_logs_for_benchmark.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Checks that all logs needed for assembling aggregate benchmarks have been retrieved.
+
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
+
+LOG_FOLDER="${LOG_FOLDER:-log}"
+E2E_SRC_FOLDER=/usr/src/yarn-project/end-to-end/src
+
+echo "Checking log files in $LOG_FOLDER"
+
+# Only generate the aggregated benchmark if we've managed to retrieve all the needed log files
+# If these runs were skipped due to no changes in their rebuild-patterns, then there's no need
+# to recompute the aggregated benchmark. Note that if one benchmark did run but others didn't,
+# this skips the whole aggregation. For now, that's fine because all benchmark files have the
+# same rebuild pattern rules. But if that changes, then we'd need to go up in the commit history
+# to find the latest log files for the unchanged benchmarks.
+EXPECTED_LOGS_COUNT=$(find $E2E_SRC_FOLDER -type f -name "bench*.test.ts" | wc -l)
+DOWNLOADED_LOGS_COUNT=$(find $LOG_FOLDER -type f -name "*.jsonl" | wc -l)
+if [ "$DOWNLOADED_LOGS_COUNT" -lt "$EXPECTED_LOGS_COUNT" ]; then
+  echo Found only $DOWNLOADED_LOGS_COUNT out of $EXPECTED_LOGS_COUNT benchmark log files in S3.
+  echo Files found: $(find $LOG_FOLDER -type f -name "*.jsonl")
+  exit 1
+fi
+

scripts/logs/download_base_benchmark_from_s3.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Downloads base benchmarks from S3 to compare with the current benchmarks via bench-comment
+
+[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace
+set -eu
+
+BUCKET_NAME="aztec-ci-artifacts"
+BENCH_FOLDER="${BENCH_FOLDER:-bench}"
+COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}"
+BASE_BENCHMARK_FILE_JSON="${BENCH_FOLDER}/base-benchmark.json"
+
+# If on a pull request, get the data from the most recent commit on master where it's available to generate a comment comparing them
+if [ -n "${PULL_REQUEST:-}" ]; then
+  MASTER_COMMIT_HASH=$(curl -s "https://api.github.com/repos/AztecProtocol/aztec-packages/pulls/${PULL_REQUEST##*/}" | jq -r '.base.sha')
+  MASTER_COMMIT_HASHES=($(git log $MASTER_COMMIT_HASH --format="%H" -n 50))
+
+  mkdir -p $BENCH_FOLDER
+
+  set +e
+  echo "Searching for base benchmark data starting from commit $MASTER_COMMIT_HASH"
+  for commit_hash in "${MASTER_COMMIT_HASHES[@]}"; do
+    aws s3 cp "s3://${BUCKET_NAME}/benchmarks-v1/master/$commit_hash.json" $BASE_BENCHMARK_FILE_JSON
+    if [ $? -eq 0 ]; then
+      echo "Downloaded base data from commit $commit_hash"
+      exit 0
+    fi
+  done
+  set -e
+
+  echo "No base commit data found"
+else
+  echo "Not on a pull request, skipping download of base benchmark data"
+fi

scripts/logs/download_logs_from_s3.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Downloads the log files uploaded in upload_logs_to_s3
+
+set -eu
+
+BUCKET_NAME="aztec-ci-artifacts"
+LOG_FOLDER="${LOG_FOLDER:-log}"
+COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}"
+
+echo "Downloading logs from S3 for commit $COMMIT_HASH in branch ${BRANCH:-} at pull request ${PULL_REQUEST:-none}"
+
+# Paths from upload_logs_to_s3
+if [ "${BRANCH:-}" = "master" ]; then
+  LOG_SOURCE_FOLDER="logs-v1/master/$COMMIT_HASH"
+  BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/master/$COMMIT_HASH"
+  BENCHMARK_TARGET_FILE="benchmarks-v1/master/$COMMIT_HASH.json"
+  BENCHMARK_LATEST_FILE="benchmarks-v1/latest.json"
+elif [ -n "${PULL_REQUEST:-}" ]; then
+  LOG_SOURCE_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}"
+  BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/pulls/${PULL_REQUEST##*/}"
+  BENCHMARK_TARGET_FILE="benchmarks-v1/pulls/${PULL_REQUEST##*/}.json"
+else
+  echo "Skipping benchmark run on branch ${BRANCH:-unknown}."
+  exit 0
+fi
+
+mkdir -p $LOG_FOLDER
+
+# Download benchmark log files from S3 LOG_SOURCE_FOLDER into local LOG_FOLDER
+echo "Downloading benchmark log files from $BUCKET_NAME/$LOG_SOURCE_FOLDER to $LOG_FOLDER"
+aws s3 cp "s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include 'bench*.jsonl' --recursive
+
+# Download barretenberg log files, these are direct benchmarks and separate from the above
+aws s3 cp "s3://${BUCKET_NAME}/${BARRETENBERG_BENCH_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include '*_bench.json' --recursive
+
+echo "Downloaded log files $(ls $LOG_FOLDER)"