From 542fb7f33d3efd43ca20332efe27d2b0071a7668 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Mon, 29 Apr 2024 15:47:55 -0300 Subject: [PATCH 01/32] chore(ci): Run benchmarks on Earthly --- .circleci/config.yml | 52 --------- .github/ci-setup-action/action.yml | 11 +- .github/workflows/ci.yml | 33 ++++-- scripts/ci/assemble_e2e_benchmark.sh | 101 ------------------ scripts/logs/check_logs_for_benchmark.sh | 23 ++++ .../logs/download_base_benchmark_from_s3.sh | 29 +++++ scripts/logs/download_logs_from_s3.sh | 31 ++++++ .../upload_aggregated_benchmarks_to_s3.sh | 37 +++++++ scripts/logs/upload_logs_to_s3.sh | 31 ++++++ scripts/setup_env.sh | 5 +- yarn-project/end-to-end/Earthfile | 22 ++-- yarn-project/scripts/Earthfile | 43 ++++++++ 12 files changed, 240 insertions(+), 178 deletions(-) delete mode 100755 scripts/ci/assemble_e2e_benchmark.sh create mode 100755 scripts/logs/check_logs_for_benchmark.sh create mode 100755 scripts/logs/download_base_benchmark_from_s3.sh create mode 100755 scripts/logs/download_logs_from_s3.sh create mode 100755 scripts/logs/upload_aggregated_benchmarks_to_s3.sh create mode 100755 scripts/logs/upload_logs_to_s3.sh create mode 100644 yarn-project/scripts/Earthfile diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ac64d1ae95a..70b0393270e2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -482,37 +482,6 @@ jobs: # For old e2e tests see yarn-project/end-to-end/Earthfile # Semantics are similar to Dockerfile - # NOTE: Unlike other e2e, these will be re-enabled here as currently the logs functionality is not in the new earthfile setup - bench-publish-rollup: - steps: - - *checkout - - *setup_env - - run: - name: "Benchmark" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_publish_rollup.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees - aztec_manifest_key: end-to-end - <<: *defaults_e2e_test - - bench-process-history: - steps: - - *checkout - - *setup_env - - run: - name: "Benchmark" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_process_history.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees - aztec_manifest_key: end-to-end - <<: *defaults_e2e_test - - bench-tx-size: - steps: - - *checkout - - *setup_env - - run: - name: "Benchmark" - command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_tx_size_fees.test.ts ENABLE_GAS=1 DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees - aztec_manifest_key: end-to-end - <<: *defaults_e2e_test - e2e-join: docker: - image: cimg/base:2023.09 @@ -531,16 +500,6 @@ jobs: name: "Noop" command: echo Noop - bench-summary: - machine: - image: default - steps: - - *checkout - - *setup_env - - run: - name: "Assemble benchmark summary from uploaded logs" - command: ./scripts/ci/assemble_e2e_benchmark.sh - # Deploy & release jobs. deploy-and-release: machine: @@ -775,16 +734,5 @@ workflows: - e2e-join <<: *defaults - # Benchmark jobs. - - bench-publish-rollup: *e2e_test - - bench-process-history: *e2e_test - - bench-tx-size: *e2e_test - - bench-summary: - requires: - - bench-publish-rollup - - bench-process-history - - bench-tx-size - <<: *defaults - # Production releases. - deploy-and-release: *defaults_deploy diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml index 4b1d7da6dbbd..f5d8029f6561 100644 --- a/.github/ci-setup-action/action.yml +++ b/.github/ci-setup-action/action.yml @@ -1,14 +1,14 @@ # Reusable setup workflow for CI tasks name: Setup Workflow -description: 'Reusable setup steps' +description: "Reusable setup steps" inputs: dockerhub_password: required: true - description: 'DockerHub Password' + description: "DockerHub Password" concurrency_key: required: false - description: 'Concurrency key for locking jobs' + description: "Concurrency key for locking jobs" runs: # define an action, runs in OS of caller using: composite @@ -43,11 +43,14 @@ runs: with: # permission issue with spot runners, simpler to leave out use-cache: false - version: 'v0.8.5' + version: "v0.8.5" - name: Setup Env shell: bash run: ./scripts/setup_env.sh ${{ inputs.dockerhub_password }} + env: + PULL_REQUEST: "${{ github.event.pull_request.number }}" + BRANCH: "${{ github.ref_name }}" # As detailed in https://github.com/ben-z/gh-action-mutex # things do not become 'pending' in github actions, and instead just cancel one another diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f2d4ab8783d..fcccdd4dc3b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,10 +11,12 @@ on: runner_action: description: "The action to take with the self-hosted runner (start, stop, restart)." required: false + concurrency: # force parallelism in master group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} cancel-in-progress: true + jobs: setup: uses: ./.github/workflows/setup-runner.yml @@ -64,22 +66,35 @@ jobs: matrix: test: ${{ fromJson( needs.build.outputs.e2e_list )}} steps: - - { - uses: actions/checkout@v4, - with: { ref: "${{ github.event.pull_request.head.sha }}" }, - } + - uses: actions/checkout@v4 + with: { ref: "${{ github.event.pull_request.head.sha }}" } - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" # must be globally unique for build x runner concurrency_key: e2e-${{ inputs.username || github.actor }}-x86-${{ matrix.test }} - - name: Test + - name: Test and upload logs working-directory: ./yarn-project/end-to-end/ timeout-minutes: 25 - run: earthly-ci -P --no-output +${{ matrix.test }} --e2e_mode=cache - # TODO - # - name: Upload logs - # run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + run: earthly-ci -P --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY --no-output +${{ matrix.test }} --e2e_mode=cache + + bench-summary: + needs: build + runs-on: ${{ inputs.username || github.actor }}-x86 + steps: + - uses: actions/checkout@v4, + with: { ref: "${{ github.event.pull_request.head.sha }}" } + - uses: ./.github/ci-setup-action + with: + dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_key: build-${{ inputs.username || github.actor }}-x86 + - name: "Build and upload bench aggregate file" + working-directory: ./yarn-project/scripts + run: earthly-ci -P --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY +bench-aggregate + - name: "Generate summary comment if pull request" + working-directory: ./yarn-project/scripts + run: earthly-ci -P --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_COMMENTER_GITHUB_TOKEN }} +bench-comment + if: ${{ github.event_name == 'pull_request' }} # barretenberg (prover) native and AVM (public VM) tests # only ran on x86 for resource reasons (memory intensive) diff --git a/scripts/ci/assemble_e2e_benchmark.sh b/scripts/ci/assemble_e2e_benchmark.sh deleted file mode 100755 index 90291ffa8b8e..000000000000 --- a/scripts/ci/assemble_e2e_benchmark.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -# Grabs the log files uploaded in build-system/scripts/upload_logs_to_s3 -# that contain representative benchmarks, extracts whatever metrics are interesting, -# and assembles a single file that shows the current state of the repository. - -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -BUCKET_NAME="aztec-ci-artifacts" -LOG_FOLDER="${LOG_FOLDER:-log}" -BENCH_FOLDER="${BENCH_FOLDER:-bench}" -COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" -BASE_BENCH_PATH="" -BENCHMARK_FILE_JSON="${BENCH_FOLDER}/benchmark.json" -BASE_BENCHMARK_FILE_JSON="${BENCH_FOLDER}/base-benchmark.json" - -# Paths from build-system/scripts/upload_logs_to_s3 -if [ "${CIRCLE_BRANCH:-}" = "master" ]; then - LOG_SOURCE_FOLDER="logs-v1/master/$COMMIT_HASH" - BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/master/$COMMIT_HASH" - BENCHMARK_TARGET_FILE="benchmarks-v1/master/$COMMIT_HASH.json" - BENCHMARK_LATEST_FILE="benchmarks-v1/latest.json" -elif [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then - LOG_SOURCE_FOLDER="logs-v1/pulls/${CIRCLE_PULL_REQUEST##*/}" - BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/pulls/${CIRCLE_PULL_REQUEST##*/}" - BENCHMARK_TARGET_FILE="benchmarks-v1/pulls/${CIRCLE_PULL_REQUEST##*/}.json" -elif [ -n "${CIRCLE_TAG:-}" ]; then - echo "Skipping benchmark run for ${CIRCLE_TAG} tagged release." - exit 0 -else - echo "Skipping benchmark run on branch ${CIRCLE_BRANCH:-unknown}." - exit 0 -fi - -# Download benchmark log files from S3 LOG_SOURCE_FOLDER into local LOG_FOLDER -mkdir -p $LOG_FOLDER -aws s3 cp "s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include 'bench*.jsonl' --recursive - -# Only generate the aggregated benchmark if we've managed to retrieve all the needed log files -# If these runs were skipped due to no changes in their rebuild-patterns, then there's no need -# to recompute the aggregated benchmark. Note that if one benchmark did run but others didn't, -# this skips the whole aggregation. For now, that's fine because all benchmark files have the -# same rebuild pattern rules. But if that changes, then we'd need to go up in the commit history -# to find the latest log files for the unchanged benchmarks. -EXPECTED_LOGS_COUNT=$(find yarn-project/end-to-end/src -type f -name "bench*.test.ts" | wc -l) -DOWNLOADED_LOGS_COUNT=$(find $LOG_FOLDER -type f -name "*.jsonl" | wc -l) -if [ "$DOWNLOADED_LOGS_COUNT" -lt "$EXPECTED_LOGS_COUNT" ]; then - echo Found $DOWNLOADED_LOGS_COUNT out of $EXPECTED_LOGS_COUNT benchmark log files in s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/. Exiting. - exit 0 -fi - -# Download barretenberg log files, these are direct benchmarks and separate from the above -aws s3 cp "s3://${BUCKET_NAME}/${BARRETENBERG_BENCH_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include '*_bench.json' --recursive - -# Generate the aggregated benchmark file -mkdir -p $BENCH_FOLDER -CONTAINER_BENCH_FOLDER="/usr/src/yarn-project/bench" -CONTAINER_LOG_FOLDER="/usr/src/yarn-project/log" -export DOCKER_RUN_OPTS="\ - -v $(realpath $BENCH_FOLDER):${CONTAINER_BENCH_FOLDER}:rw \ - -e BENCH_FOLDER=${CONTAINER_BENCH_FOLDER} \ - -v $(realpath $LOG_FOLDER):${CONTAINER_LOG_FOLDER}:rw \ - -e LOG_FOLDER=${CONTAINER_LOG_FOLDER} \ - -e BASE_BENCH_PATH \ - -e AZTEC_BOT_COMMENTER_GITHUB_TOKEN \ - -e CIRCLE_PULL_REQUEST" -yarn-project/scripts/run_script.sh workspace @aztec/scripts bench-aggregate -echo "generated: $BENCHMARK_FILE_JSON" - -# Upload it to master or pulls -aws s3 cp $BENCHMARK_FILE_JSON "s3://${BUCKET_NAME}/${BENCHMARK_TARGET_FILE}" - -# If on master, also update the "latest" benchmark file -if [ -n "${BENCHMARK_LATEST_FILE:-}" ]; then - aws s3 cp $BENCHMARK_FILE_JSON "s3://${BUCKET_NAME}/${BENCHMARK_LATEST_FILE}" -fi - -# If on a pull request, get the data from the most recent commit on master where it's available, -# generate a markdown comment, and post it on the pull request -if [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then - MASTER_COMMIT_HASH=$(curl -s "https://api.github.com/repos/AztecProtocol/aztec-packages/pulls/${CIRCLE_PULL_REQUEST##*/}" | jq -r '.base.sha') - MASTER_COMMIT_HASHES=($(git log $MASTER_COMMIT_HASH --format="%H" -n 50)) - - set +e - echo "Searching for base benchmark data starting from commit $MASTER_COMMIT_HASH" - for commit_hash in "${MASTER_COMMIT_HASHES[@]}"; do - aws s3 cp "s3://${BUCKET_NAME}/benchmarks-v1/master/$commit_hash.json" $BASE_BENCHMARK_FILE_JSON - if [ $? -eq 0 ]; then - echo "Downloaded base data from commit $commit_hash" - export BASE_BENCH_PATH=master/$commit_hash - break; - fi - done - set -e - - if [ -z "${BASE_BENCH_PATH:-}" ]; then - echo "No base commit data found" - fi - - (yarn-project/scripts/run_script.sh workspace @aztec/scripts bench-comment && echo "commented on pr $CIRCLE_PULL_REQUEST") || echo "failed commenting on pr" -fi diff --git a/scripts/logs/check_logs_for_benchmark.sh b/scripts/logs/check_logs_for_benchmark.sh new file mode 100755 index 000000000000..51de85c6afcd --- /dev/null +++ b/scripts/logs/check_logs_for_benchmark.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Checks that all logs needed for assembling aggregate benchmarks have been retrieved. + +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +LOG_FOLDER="${LOG_FOLDER:-log}" +E2E_SRC_FOLDER=/usr/src/yarn-project/end-to-end/src + +# Only generate the aggregated benchmark if we've managed to retrieve all the needed log files +# If these runs were skipped due to no changes in their rebuild-patterns, then there's no need +# to recompute the aggregated benchmark. Note that if one benchmark did run but others didn't, +# this skips the whole aggregation. For now, that's fine because all benchmark files have the +# same rebuild pattern rules. But if that changes, then we'd need to go up in the commit history +# to find the latest log files for the unchanged benchmarks. +EXPECTED_LOGS_COUNT=$(find $E2E_SRC_FOLDER -type f -name "bench*.test.ts" | wc -l) +DOWNLOADED_LOGS_COUNT=$(find $LOG_FOLDER -type f -name "*.jsonl" | wc -l) +if [ "$DOWNLOADED_LOGS_COUNT" -lt "$EXPECTED_LOGS_COUNT" ]; then + echo Found only $DOWNLOADED_LOGS_COUNT out of $EXPECTED_LOGS_COUNT benchmark log files in S3. + echo Files found: $(find $LOG_FOLDER -type f -name "*.jsonl") + exit 1 +fi + diff --git a/scripts/logs/download_base_benchmark_from_s3.sh b/scripts/logs/download_base_benchmark_from_s3.sh new file mode 100755 index 000000000000..2956adb19c8e --- /dev/null +++ b/scripts/logs/download_base_benchmark_from_s3.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Downloads base benchmarks from S3 to compare with the current benchmarks via bench-comment + +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +BUCKET_NAME="aztec-ci-artifacts" +BENCH_FOLDER="${BENCH_FOLDER:-bench}" +COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" +BASE_BENCHMARK_FILE_JSON="${BENCH_FOLDER}/base-benchmark.json" + +# If on a pull request, get the data from the most recent commit on master where it's available to generate a comment comparing them +if [ -n "${PULL_REQUEST:-}" ]; then + MASTER_COMMIT_HASH=$(curl -s "https://api.github.com/repos/AztecProtocol/aztec-packages/pulls/${PULL_REQUEST##*/}" | jq -r '.base.sha') + MASTER_COMMIT_HASHES=($(git log $MASTER_COMMIT_HASH --format="%H" -n 50)) + + set +e + echo "Searching for base benchmark data starting from commit $MASTER_COMMIT_HASH" + for commit_hash in "${MASTER_COMMIT_HASHES[@]}"; do + aws s3 cp "s3://${BUCKET_NAME}/benchmarks-v1/master/$commit_hash.json" $BASE_BENCHMARK_FILE_JSON + if [ $? -eq 0 ]; then + echo "Downloaded base data from commit $commit_hash" + exit 0 + fi + done + set -e + + echo "No base commit data found" +fi diff --git a/scripts/logs/download_logs_from_s3.sh b/scripts/logs/download_logs_from_s3.sh new file mode 100755 index 000000000000..8c8f057216e4 --- /dev/null +++ b/scripts/logs/download_logs_from_s3.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Downloads the log files uploaded in upload_logs_to_s3 + +set -eu + +BUCKET_NAME="aztec-ci-artifacts" +LOG_FOLDER="${LOG_FOLDER:-log}" +COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" + +# Paths from upload_logs_to_s3 +if [ "${BRANCH:-}" = "master" ]; then + LOG_SOURCE_FOLDER="logs-v1/master/$COMMIT_HASH" + BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/master/$COMMIT_HASH" + BENCHMARK_TARGET_FILE="benchmarks-v1/master/$COMMIT_HASH.json" + BENCHMARK_LATEST_FILE="benchmarks-v1/latest.json" +elif [ -n "${PULL_REQUEST:-}" ]; then + LOG_SOURCE_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" + BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/pulls/${PULL_REQUEST##*/}" + BENCHMARK_TARGET_FILE="benchmarks-v1/pulls/${PULL_REQUEST##*/}.json" +else + echo "Skipping benchmark run on branch ${BRANCH:-unknown}." + exit 0 +fi + +mkdir -p $LOG_FOLDER + +# Download benchmark log files from S3 LOG_SOURCE_FOLDER into local LOG_FOLDER +aws s3 cp "s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include 'bench*.jsonl' --recursive + +# Download barretenberg log files, these are direct benchmarks and separate from the above +aws s3 cp "s3://${BUCKET_NAME}/${BARRETENBERG_BENCH_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include '*_bench.json' --recursive diff --git a/scripts/logs/upload_aggregated_benchmarks_to_s3.sh b/scripts/logs/upload_aggregated_benchmarks_to_s3.sh new file mode 100755 index 000000000000..76b00122c89c --- /dev/null +++ b/scripts/logs/upload_aggregated_benchmarks_to_s3.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Uploads aggregated benchmark logs to S3 + +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +BUCKET_NAME="aztec-ci-artifacts" +LOG_FOLDER="${LOG_FOLDER:-log}" +BENCH_FOLDER="${BENCH_FOLDER:-bench}" +COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" +BENCHMARK_FILE_JSON="${BENCH_FOLDER}/benchmark.json" + +# Paths from upload_logs_to_s3 +if [ "${BRANCH:-}" = "master" ]; then + LOG_SOURCE_FOLDER="logs-v1/master/$COMMIT_HASH" + BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/master/$COMMIT_HASH" + BENCHMARK_TARGET_FILE="benchmarks-v1/master/$COMMIT_HASH.json" + BENCHMARK_LATEST_FILE="benchmarks-v1/latest.json" +elif [ -n "${PULL_REQUEST:-}" ]; then + LOG_SOURCE_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" + BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/pulls/${PULL_REQUEST##*/}" + BENCHMARK_TARGET_FILE="benchmarks-v1/pulls/${PULL_REQUEST##*/}.json" +elif [ -n "${CIRCLE_TAG:-}" ]; then + echo "Skipping benchmark run for ${CIRCLE_TAG} tagged release." + exit 0 +else + echo "Skipping benchmark run on branch ${BRANCH:-unknown}." + exit 0 +fi + +# Upload it to master or pulls +aws s3 cp $BENCHMARK_FILE_JSON "s3://${BUCKET_NAME}/${BENCHMARK_TARGET_FILE}" + +# If on master, also update the "latest" benchmark file +if [ -n "${BENCHMARK_LATEST_FILE:-}" ]; then + aws s3 cp $BENCHMARK_FILE_JSON "s3://${BUCKET_NAME}/${BENCHMARK_LATEST_FILE}" +fi diff --git a/scripts/logs/upload_logs_to_s3.sh b/scripts/logs/upload_logs_to_s3.sh new file mode 100755 index 000000000000..3188a80ec0fa --- /dev/null +++ b/scripts/logs/upload_logs_to_s3.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Uploads to S3 the contents of the log file mounted on the end-to-end container, +# which contains log entries with an associated event and metrics for it. +# Logs are uploaded to aztec-ci-artifacts/logs-v1/master/$COMMIT/$JOB.jsonl +# or to aztec-ci-artifacts/logs-v1/pulls/$PRNUMBER/$JOB.jsonl if on a PR + +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +LOG_FOLDER=$1 +BUCKET_NAME="aztec-ci-artifacts" +COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" + +if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then + echo "No logs in folder $LOG_FOLDER to upload" + exit 0 +fi + +# Paths used in scripts/ci/assemble_e2e_benchmark.sh +if [ "${BRANCH:-}" = "master" ]; then + TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" +elif [ -n "${PULL_REQUEST:-}" ]; then + TARGET_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" +fi + +if [ -n "${TARGET_FOLDER:-}" ]; then + aws s3 cp $LOG_FOLDER "s3://${BUCKET_NAME}/${TARGET_FOLDER}" --include "*.jsonl" --recursive +else + echo Skipping upload since no target folder was defined +fi \ No newline at end of file diff --git a/scripts/setup_env.sh b/scripts/setup_env.sh index 70ecf9d7654c..2765b5b84cad 100755 --- a/scripts/setup_env.sh +++ b/scripts/setup_env.sh @@ -1,8 +1,11 @@ #!/bin/bash +COMMIT_HASH=$(git rev-parse HEAD) + # Setup environment variables echo "Setting up environment variables..." -echo FORCE_COLOR=1 >> $GITHUB_ENV +echo "FORCE_COLOR=1" >> $GITHUB_ENV +echo "EARTHLY_BUILD_ARGS=\"PULL_REQUEST=$PULL_REQUEST,BRANCH=$BRANCH,COMMIT_HASH=$COMMIT_HASH\"" >> $GITHUB_ENV # Docker login echo "Logging in to Docker..." diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 59403a2400c8..7e2d084d5a98 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -80,11 +80,13 @@ E2E_TEST: ELSE DO +E2E_TEST_FROM_BUILD --test=$test --compose_file=$compose_file --enable_gas=$enable_gas --debug=$debug END + RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY /usr/src/scripts/logs/upload_logs_to_s3.sh ./log # Define e2e tests e2e-tests: FROM ../+end-to-end RUN yarn test ./src/e2e + RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY /usr/src/scripts/logs/upload_logs_to_s3.sh ./log flakey-e2e-tests: FROM ../+end-to-end @@ -132,16 +134,14 @@ guides-dapp-testing: # ARG e2e_mode=local # DO +E2E_TEST --test=guides/up_quick_start.test.ts --e2e_mode=$e2e_mode -# TODO hanging -# bench-publish-rollup: -# ARG e2e_mode=local -# DO +E2E_TEST --test=benchmarks/bench_publish_rollup.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml +bench-publish-rollup: + ARG e2e_mode=local + DO +E2E_TEST --test=benchmarks/bench_publish_rollup.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml -# TODO hanging -# bench-process-history: -# ARG e2e_mode=local -# DO +E2E_TEST --test=benchmarks/bench_process_history.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml +bench-process-history: + ARG e2e_mode=local + DO +E2E_TEST --test=benchmarks/bench_process_history.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml -# bench-tx-size: -# ARG e2e_mode=local -# DO +E2E_TEST --test=benchmarks/bench_tx_size_fees.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --enable_gas=1 --compose_file=./scripts/docker-compose-no-sandbox.yml +bench-tx-size: + ARG e2e_mode=local + DO +E2E_TEST --test=benchmarks/bench_tx_size_fees.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --enable_gas=1 --compose_file=./scripts/docker-compose-no-sandbox.yml diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile new file mode 100644 index 000000000000..3a4b1291824f --- /dev/null +++ b/yarn-project/scripts/Earthfile @@ -0,0 +1,43 @@ +VERSION 0.8 +FROM node:18.19.0 + +LET LOG_FOLDER=/usr/var/logs +LET BENCH_FOLDER=/usr/var/bench + +ENV LOG_FOLDER=$LOG_FOLDER +ENV BENCH_FOLDER=$BENCH_FOLDER + +# These args are loaded from the EARTHLY_BUILD_ARGS set in setup_env +ARG COMMIT_HASH +ARG PULL_REQUEST +ARG BRANCH +ENV COMMIT_HASH=$COMMIT_HASH +ENV PULL_REQUEST=$PULL_REQUEST +ENV BRANCH=$BRANCH + +base: + FROM ../+scripts-prod + RUN apt-get update && apt-get install awscli -y + +download-logs: + FROM +base + RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY ./scripts/logs/download_logs_from_s3.sh + SAVE ARTIFACT $LOG_FOLDER logs + +bench-aggregate: + FROM +base + COPY +download-logs/logs $LOG_FOLDER + WORKDIR /usr/src/yarn-project/scripts + RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY ./scripts/logs/check_logs_for_benchmark.sh \ + && (yarn bench-aggregate && ./scripts/logs/upload_aggregated_benchmarks_to_s3.sh) \ + || true + SAVE ARTIFACT $BENCH_FOLDER bench + +bench-comment: + FROM +base + COPY +bench-aggregate/bench $BENCH_FOLDER + RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY + [ -f $BENCH_FOLDER/benchmarks.json ] \ + && (./scripts/logs/download_base_benchmark_from_s3.sh && yarn bench-comment) \ + || true + \ No newline at end of file From 9dee49d1fe6ef47c21a6666668c830dc34692623 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Mon, 29 Apr 2024 16:06:32 -0300 Subject: [PATCH 02/32] Fixes --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fcccdd4dc3b8..19b5e6c8eda4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,13 +76,13 @@ jobs: - name: Test and upload logs working-directory: ./yarn-project/end-to-end/ timeout-minutes: 25 - run: earthly-ci -P --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY --no-output +${{ matrix.test }} --e2e_mode=cache + run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --no-output +${{ matrix.test }} --e2e_mode=cache bench-summary: needs: build runs-on: ${{ inputs.username || github.actor }}-x86 steps: - - uses: actions/checkout@v4, + - uses: actions/checkout@v4 with: { ref: "${{ github.event.pull_request.head.sha }}" } - uses: ./.github/ci-setup-action with: @@ -90,10 +90,10 @@ jobs: concurrency_key: build-${{ inputs.username || github.actor }}-x86 - name: "Build and upload bench aggregate file" working-directory: ./yarn-project/scripts - run: earthly-ci -P --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY +bench-aggregate + run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} +bench-aggregate - name: "Generate summary comment if pull request" working-directory: ./yarn-project/scripts - run: earthly-ci -P --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_COMMENTER_GITHUB_TOKEN }} +bench-comment + run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_COMMENTER_GITHUB_TOKEN }} +bench-comment if: ${{ github.event_name == 'pull_request' }} # barretenberg (prover) native and AVM (public VM) tests From fd8c34835dae32d6656f42459a292fa1decad0c6 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Mon, 29 Apr 2024 17:13:41 -0300 Subject: [PATCH 03/32] Fixes --- yarn-project/scripts/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index 3a4b1291824f..cddfae2a052c 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -36,7 +36,7 @@ bench-aggregate: bench-comment: FROM +base COPY +bench-aggregate/bench $BENCH_FOLDER - RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY + RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ [ -f $BENCH_FOLDER/benchmarks.json ] \ && (./scripts/logs/download_base_benchmark_from_s3.sh && yarn bench-comment) \ || true From 86d3d840a219f3f3d6c6d1d1fe6ca3788b0a4b3e Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Mon, 29 Apr 2024 17:13:56 -0300 Subject: [PATCH 04/32] Earthly doesnt support secrets when running LOCALLY --- yarn-project/end-to-end/Earthfile | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 7e2d084d5a98..81f449536d2c 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -80,7 +80,15 @@ E2E_TEST: ELSE DO +E2E_TEST_FROM_BUILD --test=$test --compose_file=$compose_file --enable_gas=$enable_gas --debug=$debug END - RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY /usr/src/scripts/logs/upload_logs_to_s3.sh ./log + +UPLOAD_LOGS: + FUNCTION + ARG e2e_mode=local + COPY ./log /usr/var/log + RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ + [ $e2e_mode == 'local' ] \ + && /usr/src/scripts/logs/upload_logs_to_s3.sh /usr/var/log \ + || true # Define e2e tests e2e-tests: @@ -95,53 +103,66 @@ flakey-e2e-tests: e2e-sandbox-example: ARG e2e_mode=local DO +E2E_TEST --test=e2e_sandbox_example.test.ts --e2e_mode=$e2e_mode + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode uniswap-trade-on-l1-from-l2: ARG e2e_mode=local DO +E2E_TEST --test=uniswap_trade_on_l1_from_l2.test.ts --e2e_mode=$e2e_mode + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode integration-l1-publisher: ARG e2e_mode=local DO +E2E_TEST --test=integration_l1_publisher.test.ts --e2e_mode=$e2e_mode + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode e2e-browser: ARG e2e_mode=local DO +E2E_TEST --test=e2e_aztec_js_browser.test.ts --e2e_mode=$e2e_mode + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode pxe: ARG e2e_mode=local DO +E2E_TEST --test=pxe.test.ts --e2e_mode=$e2e_mode + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode e2e-docs-examples: ARG e2e_mode=local DO +E2E_TEST --test=docs_examples.test.ts --e2e_mode=$e2e_mode + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode guides-writing-an-account-contract: ARG e2e_mode=local DO +E2E_TEST --test=guides/writing_an_account_contract.test.ts --e2e_mode=$e2e_mode + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode guides-dapp-testing: ARG e2e_mode=local DO +E2E_TEST --test=guides/dapp_testing.test.ts --e2e_mode=$e2e_mode + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode # TODO intermittent failure # guides-sample-dapp: -# ARG e2e_mode=local -# DO +E2E_TEST --test=sample-dapp --e2e_mode=$e2e_mode +# ARG e2e_mode=local +# DO +E2E_TEST --test=sample-dapp --e2e_mode=$e2e_mode +# DO +UPLOAD_LOGS --e2e_mode=$e2e_mode # TODO currently hangs for hour+ # guides-up-quick-start: # ARG e2e_mode=local # DO +E2E_TEST --test=guides/up_quick_start.test.ts --e2e_mode=$e2e_mode +# DO +UPLOAD_LOGS --e2e_mode=$e2e_mode bench-publish-rollup: ARG e2e_mode=local DO +E2E_TEST --test=benchmarks/bench_publish_rollup.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode bench-process-history: ARG e2e_mode=local DO +E2E_TEST --test=benchmarks/bench_process_history.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode bench-tx-size: ARG e2e_mode=local DO +E2E_TEST --test=benchmarks/bench_tx_size_fees.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --enable_gas=1 --compose_file=./scripts/docker-compose-no-sandbox.yml + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode From 48f3488e61920fb8a2c1c0d1781a98fdde6037f1 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Mon, 29 Apr 2024 17:21:28 -0300 Subject: [PATCH 05/32] "base" is a reserved target name in Earthly --- yarn-project/scripts/Earthfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index cddfae2a052c..1de7503d10fd 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -15,17 +15,17 @@ ENV COMMIT_HASH=$COMMIT_HASH ENV PULL_REQUEST=$PULL_REQUEST ENV BRANCH=$BRANCH -base: +build: FROM ../+scripts-prod RUN apt-get update && apt-get install awscli -y download-logs: - FROM +base + FROM +build RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY ./scripts/logs/download_logs_from_s3.sh SAVE ARTIFACT $LOG_FOLDER logs bench-aggregate: - FROM +base + FROM +build COPY +download-logs/logs $LOG_FOLDER WORKDIR /usr/src/yarn-project/scripts RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY ./scripts/logs/check_logs_for_benchmark.sh \ @@ -34,7 +34,7 @@ bench-aggregate: SAVE ARTIFACT $BENCH_FOLDER bench bench-comment: - FROM +base + FROM +build COPY +bench-aggregate/bench $BENCH_FOLDER RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ [ -f $BENCH_FOLDER/benchmarks.json ] \ From 537c2a624434674d0fe0a350ebf13817f17e110c Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 11:40:46 -0300 Subject: [PATCH 06/32] Fixes --- yarn-project/end-to-end/Earthfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 81f449536d2c..3282378a4860 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -35,7 +35,7 @@ E2E_TEST_FROM_CACHE: ENV DEBUG=$debug ENV AZTEC_DOCKER_TAG=$aztec_docker_tag # need a different project name for each to run in parallel - LET project_name=$(echo $test | sed 's/\./_/g') + LET project_name=$(echo $test | sed 's/[./]/_/g') IF docker compose > /dev/null 2>&1 LET CMD="docker compose" ELSE @@ -86,7 +86,7 @@ UPLOAD_LOGS: ARG e2e_mode=local COPY ./log /usr/var/log RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ - [ $e2e_mode == 'local' ] \ + [ $e2e_mode == 'cache' ] \ && /usr/src/scripts/logs/upload_logs_to_s3.sh /usr/var/log \ || true From 913870c0dbb5a84c4c9ce298d8b5675a6e21bf8d Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 12:34:57 -0300 Subject: [PATCH 07/32] Just upload logs for bench jobs --- .gitignore | 4 ++++ Earthfile | 5 +++++ scripts/logs/upload_logs_to_s3.sh | 6 ++++++ yarn-project/end-to-end/Earthfile | 23 ++++++++++------------- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index fcee2d1838e1..8e28a8bac2f9 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,7 @@ cmake-build-debug .DS_Store **/*.dockerignore + +# Earthly +.arg +.secret diff --git a/Earthfile b/Earthfile index 65abde37745a..d382d878eb18 100644 --- a/Earthfile +++ b/Earthfile @@ -32,3 +32,8 @@ bench: release-meta: COPY .release-please-manifest.json /usr/src/.release-please-manifest.json SAVE ARTIFACT /usr/src /usr/src + +scripts: + FROM ubuntu:lunar + RUN apt-get update && apt-get install -y awscli + COPY scripts /usr/src/scripts \ No newline at end of file diff --git a/scripts/logs/upload_logs_to_s3.sh b/scripts/logs/upload_logs_to_s3.sh index 3188a80ec0fa..8c1ceef5ba35 100755 --- a/scripts/logs/upload_logs_to_s3.sh +++ b/scripts/logs/upload_logs_to_s3.sh @@ -11,12 +11,18 @@ set -eu LOG_FOLDER=$1 BUCKET_NAME="aztec-ci-artifacts" COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" +E2E_MODE="${COMMIT_HASH:-local}" if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then echo "No logs in folder $LOG_FOLDER to upload" exit 0 fi +if [ "$E2E_MODE" == "local" ]; then + echo "Skipping upload since running in local mode" + exit 0 +fi + # Paths used in scripts/ci/assemble_e2e_benchmark.sh if [ "${BRANCH:-}" = "master" ]; then TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 3282378a4860..f9aa9b2572cb 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -1,5 +1,10 @@ VERSION 0.8 +ARG --global use_local_builds=false +ARG --global PULL_REQUEST +ARG --global BRANCH +ARG --global COMMIT_HASH + # requires first saving the images locally with ../+export-end-to-end # run locally and build @@ -83,18 +88,18 @@ E2E_TEST: UPLOAD_LOGS: FUNCTION - ARG e2e_mode=local + ARG e2e_mode=cache + LOCALLY + LET COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" + FROM ../../+scripts COPY ./log /usr/var/log RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ - [ $e2e_mode == 'cache' ] \ - && /usr/src/scripts/logs/upload_logs_to_s3.sh /usr/var/log \ - || true + PULL_REQUEST=$PULL_REQUEST BRANCH=$BRANCH COMMIT_HASH=$COMMIT_HASH E2E_MODE=$e2e_mode /usr/src/scripts/logs/upload_logs_to_s3.sh /usr/var/log # Define e2e tests e2e-tests: FROM ../+end-to-end RUN yarn test ./src/e2e - RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY /usr/src/scripts/logs/upload_logs_to_s3.sh ./log flakey-e2e-tests: FROM ../+end-to-end @@ -103,42 +108,34 @@ flakey-e2e-tests: e2e-sandbox-example: ARG e2e_mode=local DO +E2E_TEST --test=e2e_sandbox_example.test.ts --e2e_mode=$e2e_mode - DO +UPLOAD_LOGS --e2e_mode=$e2e_mode uniswap-trade-on-l1-from-l2: ARG e2e_mode=local DO +E2E_TEST --test=uniswap_trade_on_l1_from_l2.test.ts --e2e_mode=$e2e_mode - DO +UPLOAD_LOGS --e2e_mode=$e2e_mode integration-l1-publisher: ARG e2e_mode=local DO +E2E_TEST --test=integration_l1_publisher.test.ts --e2e_mode=$e2e_mode - DO +UPLOAD_LOGS --e2e_mode=$e2e_mode e2e-browser: ARG e2e_mode=local DO +E2E_TEST --test=e2e_aztec_js_browser.test.ts --e2e_mode=$e2e_mode - DO +UPLOAD_LOGS --e2e_mode=$e2e_mode pxe: ARG e2e_mode=local DO +E2E_TEST --test=pxe.test.ts --e2e_mode=$e2e_mode - DO +UPLOAD_LOGS --e2e_mode=$e2e_mode e2e-docs-examples: ARG e2e_mode=local DO +E2E_TEST --test=docs_examples.test.ts --e2e_mode=$e2e_mode - DO +UPLOAD_LOGS --e2e_mode=$e2e_mode guides-writing-an-account-contract: ARG e2e_mode=local DO +E2E_TEST --test=guides/writing_an_account_contract.test.ts --e2e_mode=$e2e_mode - DO +UPLOAD_LOGS --e2e_mode=$e2e_mode guides-dapp-testing: ARG e2e_mode=local DO +E2E_TEST --test=guides/dapp_testing.test.ts --e2e_mode=$e2e_mode - DO +UPLOAD_LOGS --e2e_mode=$e2e_mode # TODO intermittent failure # guides-sample-dapp: From 2ec821facb58e6c6ab5eea6aa125a59b3933af6f Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 13:46:53 -0300 Subject: [PATCH 08/32] Try to get branch and pr info into script --- scripts/logs/upload_logs_to_s3.sh | 4 +++- yarn-project/end-to-end/Earthfile | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/logs/upload_logs_to_s3.sh b/scripts/logs/upload_logs_to_s3.sh index 8c1ceef5ba35..a5e46f5a9680 100755 --- a/scripts/logs/upload_logs_to_s3.sh +++ b/scripts/logs/upload_logs_to_s3.sh @@ -11,7 +11,9 @@ set -eu LOG_FOLDER=$1 BUCKET_NAME="aztec-ci-artifacts" COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" -E2E_MODE="${COMMIT_HASH:-local}" +E2E_MODE="${E2E_MODE:-local}" + +echo "Uploading logs to S3 in mode $E2E_MODE for commit $COMMIT_HASH in branch ${BRANCH:-} at pull request ${PULL_REQUEST:-none}" if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then echo "No logs in folder $LOG_FOLDER to upload" diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index f9aa9b2572cb..1e91c67f1342 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -93,8 +93,11 @@ UPLOAD_LOGS: LET COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" FROM ../../+scripts COPY ./log /usr/var/log - RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ - PULL_REQUEST=$PULL_REQUEST BRANCH=$BRANCH COMMIT_HASH=$COMMIT_HASH E2E_MODE=$e2e_mode /usr/src/scripts/logs/upload_logs_to_s3.sh /usr/var/log + ENV PULL_REQUEST=$PULL_REQUEST + ENV BRANCH=$BRANCH + ENV COMMIT_HASH=$COMMIT_HASH + ENV E2E_MODE=$e2e_mode + RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY /usr/src/scripts/logs/upload_logs_to_s3.sh /usr/var/log # Define e2e tests e2e-tests: From e3ac4b814a331f29353061ef1e74d53353c77228 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 13:55:05 -0300 Subject: [PATCH 09/32] Try fix download logs script --- .github/workflows/ci.yml | 2 +- Earthfile | 3 ++- yarn-project/scripts/Earthfile | 10 ++++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19b5e6c8eda4..044d9f929ee4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --no-output +${{ matrix.test }} --e2e_mode=cache bench-summary: - needs: build + needs: e2e runs-on: ${{ inputs.username || github.actor }}-x86 steps: - uses: actions/checkout@v4 diff --git a/Earthfile b/Earthfile index d382d878eb18..7e3a8ae944ba 100644 --- a/Earthfile +++ b/Earthfile @@ -36,4 +36,5 @@ release-meta: scripts: FROM ubuntu:lunar RUN apt-get update && apt-get install -y awscli - COPY scripts /usr/src/scripts \ No newline at end of file + COPY scripts /usr/src/scripts + SAVE ARTIFACT /usr/src/scripts \ No newline at end of file diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index 1de7503d10fd..b69bbc207202 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -17,7 +17,9 @@ ENV BRANCH=$BRANCH build: FROM ../+scripts-prod + COPY ../../+scripts/usr/src/scripts /usr/src/scripts RUN apt-get update && apt-get install awscli -y + WORKDIR /usr/src download-logs: FROM +build @@ -27,9 +29,9 @@ download-logs: bench-aggregate: FROM +build COPY +download-logs/logs $LOG_FOLDER - WORKDIR /usr/src/yarn-project/scripts - RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY ./scripts/logs/check_logs_for_benchmark.sh \ - && (yarn bench-aggregate && ./scripts/logs/upload_aggregated_benchmarks_to_s3.sh) \ + RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ + ./scripts/logs/check_logs_for_benchmark.sh \ + && ((cd /usr/src/yarn-project/scripts && yarn bench-aggregate) && ./scripts/logs/upload_aggregated_benchmarks_to_s3.sh) \ || true SAVE ARTIFACT $BENCH_FOLDER bench @@ -38,6 +40,6 @@ bench-comment: COPY +bench-aggregate/bench $BENCH_FOLDER RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ [ -f $BENCH_FOLDER/benchmarks.json ] \ - && (./scripts/logs/download_base_benchmark_from_s3.sh && yarn bench-comment) \ + && (./scripts/logs/download_base_benchmark_from_s3.sh && (cd /usr/src/yarn-project/scripts && yarn bench-comment)) \ || true \ No newline at end of file From 6f6b5d09099a6dd88093727f817b6b1448d675a0 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 14:46:43 -0300 Subject: [PATCH 10/32] Try get PR properly into args --- scripts/setup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup_env.sh b/scripts/setup_env.sh index 2765b5b84cad..7ad729fdda9e 100755 --- a/scripts/setup_env.sh +++ b/scripts/setup_env.sh @@ -5,7 +5,7 @@ COMMIT_HASH=$(git rev-parse HEAD) # Setup environment variables echo "Setting up environment variables..." echo "FORCE_COLOR=1" >> $GITHUB_ENV -echo "EARTHLY_BUILD_ARGS=\"PULL_REQUEST=$PULL_REQUEST,BRANCH=$BRANCH,COMMIT_HASH=$COMMIT_HASH\"" >> $GITHUB_ENV +echo "EARTHLY_BUILD_ARGS=PULL_REQUEST=$PULL_REQUEST,BRANCH=$BRANCH,COMMIT_HASH=$COMMIT_HASH" >> $GITHUB_ENV # Docker login echo "Logging in to Docker..." From 6cd640a9d18cbd20bffe7adfa8a7eab1840bbacf Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 14:56:36 -0300 Subject: [PATCH 11/32] Fixes --- Earthfile | 2 +- scripts/logs/check_logs_for_benchmark.sh | 2 ++ scripts/logs/download_logs_from_s3.sh | 5 +++ yarn-project/scripts/Earthfile | 42 ++++++++++++++++-------- 4 files changed, 36 insertions(+), 15 deletions(-) diff --git a/Earthfile b/Earthfile index 7e3a8ae944ba..096ac3bac0df 100644 --- a/Earthfile +++ b/Earthfile @@ -37,4 +37,4 @@ scripts: FROM ubuntu:lunar RUN apt-get update && apt-get install -y awscli COPY scripts /usr/src/scripts - SAVE ARTIFACT /usr/src/scripts \ No newline at end of file + SAVE ARTIFACT /usr/src/scripts scripts \ No newline at end of file diff --git a/scripts/logs/check_logs_for_benchmark.sh b/scripts/logs/check_logs_for_benchmark.sh index 51de85c6afcd..69075d4160de 100755 --- a/scripts/logs/check_logs_for_benchmark.sh +++ b/scripts/logs/check_logs_for_benchmark.sh @@ -7,6 +7,8 @@ set -eu LOG_FOLDER="${LOG_FOLDER:-log}" E2E_SRC_FOLDER=/usr/src/yarn-project/end-to-end/src +echo "Checking log files in $LOG_FOLDER" + # Only generate the aggregated benchmark if we've managed to retrieve all the needed log files # If these runs were skipped due to no changes in their rebuild-patterns, then there's no need # to recompute the aggregated benchmark. Note that if one benchmark did run but others didn't, diff --git a/scripts/logs/download_logs_from_s3.sh b/scripts/logs/download_logs_from_s3.sh index 8c8f057216e4..a31979e3968c 100755 --- a/scripts/logs/download_logs_from_s3.sh +++ b/scripts/logs/download_logs_from_s3.sh @@ -7,6 +7,8 @@ BUCKET_NAME="aztec-ci-artifacts" LOG_FOLDER="${LOG_FOLDER:-log}" COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" +echo "Downloading logs from S3 for commit $COMMIT_HASH in branch ${BRANCH:-} at pull request ${PULL_REQUEST:-none}" + # Paths from upload_logs_to_s3 if [ "${BRANCH:-}" = "master" ]; then LOG_SOURCE_FOLDER="logs-v1/master/$COMMIT_HASH" @@ -25,7 +27,10 @@ fi mkdir -p $LOG_FOLDER # Download benchmark log files from S3 LOG_SOURCE_FOLDER into local LOG_FOLDER +echo "Downloading benchmark log files from $BUCKET_NAME/$LOG_SOURCE_FOLDER to $LOG_FOLDER" aws s3 cp "s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include 'bench*.jsonl' --recursive # Download barretenberg log files, these are direct benchmarks and separate from the above aws s3 cp "s3://${BUCKET_NAME}/${BARRETENBERG_BENCH_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include '*_bench.json' --recursive + +echo "Downloaded log files $(ls $LOG_FOLDER)" \ No newline at end of file diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index b69bbc207202..85b8b58737f1 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -1,33 +1,40 @@ VERSION 0.8 FROM node:18.19.0 -LET LOG_FOLDER=/usr/var/logs -LET BENCH_FOLDER=/usr/var/bench - -ENV LOG_FOLDER=$LOG_FOLDER -ENV BENCH_FOLDER=$BENCH_FOLDER +ARG --global use_local_builds=false # These args are loaded from the EARTHLY_BUILD_ARGS set in setup_env -ARG COMMIT_HASH -ARG PULL_REQUEST -ARG BRANCH -ENV COMMIT_HASH=$COMMIT_HASH -ENV PULL_REQUEST=$PULL_REQUEST -ENV BRANCH=$BRANCH +ARG --global COMMIT_HASH +ARG --global PULL_REQUEST +ARG --global BRANCH build: - FROM ../+scripts-prod - COPY ../../+scripts/usr/src/scripts /usr/src/scripts + FROM ../+scripts-prod --use_local_builds=$use_local_builds RUN apt-get update && apt-get install awscli -y + COPY ../../+scripts/scripts /usr/src/scripts WORKDIR /usr/src download-logs: FROM +build + LET LOG_FOLDER=/usr/var/logs + LET BENCH_FOLDER=/usr/var/bench + ENV LOG_FOLDER=$LOG_FOLDER + ENV BENCH_FOLDER=$BENCH_FOLDER + ENV COMMIT_HASH=$COMMIT_HASH + ENV PULL_REQUEST=$PULL_REQUEST + ENV BRANCH=$BRANCH RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY ./scripts/logs/download_logs_from_s3.sh SAVE ARTIFACT $LOG_FOLDER logs bench-aggregate: FROM +build + LET LOG_FOLDER=/usr/var/logs + LET BENCH_FOLDER=/usr/var/bench + ENV LOG_FOLDER=$LOG_FOLDER + ENV BENCH_FOLDER=$BENCH_FOLDER + ENV COMMIT_HASH=$COMMIT_HASH + ENV PULL_REQUEST=$PULL_REQUEST + ENV BRANCH=$BRANCH COPY +download-logs/logs $LOG_FOLDER RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ ./scripts/logs/check_logs_for_benchmark.sh \ @@ -38,8 +45,15 @@ bench-aggregate: bench-comment: FROM +build COPY +bench-aggregate/bench $BENCH_FOLDER + LET LOG_FOLDER=/usr/var/logs + LET BENCH_FOLDER=/usr/var/bench + ENV LOG_FOLDER=$LOG_FOLDER + ENV BENCH_FOLDER=$BENCH_FOLDER + ENV COMMIT_HASH=$COMMIT_HASH + ENV PULL_REQUEST=$PULL_REQUEST + ENV BRANCH=$BRANCH RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ - [ -f $BENCH_FOLDER/benchmarks.json ] \ + [ -f $BENCH_FOLDER/benchmark.json ] \ && (./scripts/logs/download_base_benchmark_from_s3.sh && (cd /usr/src/yarn-project/scripts && yarn bench-comment)) \ || true \ No newline at end of file From 97518fc63cfb9ebc9eec328aa68ef8926543a644 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 15:13:48 -0300 Subject: [PATCH 12/32] Try get job name into e2e test --- yarn-project/end-to-end/Earthfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 1e91c67f1342..d93f10af445b 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -18,6 +18,7 @@ E2E_TEST_LOCAL: ENV ENABLE_GAS=$enable_gas ENV TEST=$test ENV DEBUG=$debug + ENV JOB_NAME=$EARTHLY_TARGET_NAME WITH DOCKER \ --load aztecprotocol/aztec:latest=../+aztec \ --load aztecprotocol/end-to-end:latest=../+end-to-end \ @@ -39,6 +40,7 @@ E2E_TEST_FROM_CACHE: ENV TEST=$test ENV DEBUG=$debug ENV AZTEC_DOCKER_TAG=$aztec_docker_tag + ENV JOB_NAME=$EARTHLY_TARGET_NAME # need a different project name for each to run in parallel LET project_name=$(echo $test | sed 's/[./]/_/g') IF docker compose > /dev/null 2>&1 @@ -60,6 +62,7 @@ E2E_TEST_FROM_BUILD: ENV ENABLE_GAS=$enable_gas ENV TEST=$test ENV DEBUG=$debug + ENV JOB_NAME=$EARTHLY_TARGET_NAME COPY $compose_file $compose_file # For ARM, we do use WITH DOCKER as we don't have many e2e tests, but note E2E_TEST_FROM_CACHE WITH DOCKER \ From a226be114c3879b3654f9990e2a89036720d5a96 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 15:16:14 -0300 Subject: [PATCH 13/32] Create bench folder even if no benchs created --- .github/workflows/ci.yml | 2 +- yarn-project/scripts/Earthfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 044d9f929ee4..41c09acdb588 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,7 @@ jobs: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" # must be globally unique for build x runner concurrency_key: e2e-${{ inputs.username || github.actor }}-x86-${{ matrix.test }} - - name: Test and upload logs + - name: Test working-directory: ./yarn-project/end-to-end/ timeout-minutes: 25 run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --no-output +${{ matrix.test }} --e2e_mode=cache diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index 85b8b58737f1..a2c792ecaa82 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -39,7 +39,7 @@ bench-aggregate: RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ ./scripts/logs/check_logs_for_benchmark.sh \ && ((cd /usr/src/yarn-project/scripts && yarn bench-aggregate) && ./scripts/logs/upload_aggregated_benchmarks_to_s3.sh) \ - || true + || mkdir -p $BENCH_FOLDER SAVE ARTIFACT $BENCH_FOLDER bench bench-comment: From d75216aba0d3002c40e73f14572ff8f28e12136e Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 15:48:32 -0300 Subject: [PATCH 14/32] Need to declare global args before using them --- yarn-project/end-to-end/Earthfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index d93f10af445b..71c5b8a6206d 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -14,6 +14,7 @@ E2E_TEST_LOCAL: ARG compose_file=./scripts/docker-compose.yml ARG enable_gas="" ARG debug="aztec:*" + ARG EARTHLY_TARGET_NAME LOCALLY ENV ENABLE_GAS=$enable_gas ENV TEST=$test @@ -35,6 +36,7 @@ E2E_TEST_FROM_CACHE: ARG enable_gas="" ARG debug="aztec:*" ARG aztec_docker_tag + ARG EARTHLY_TARGET_NAME LOCALLY ENV ENABLE_GAS=$enable_gas ENV TEST=$test @@ -58,6 +60,7 @@ E2E_TEST_FROM_BUILD: ARG compose_file=./scripts/docker-compose.yml ARG enable_gas="" ARG debug="aztec:*" + ARG EARTHLY_TARGET_NAME FROM earthly/dind:alpine-3.19-docker-25.0.2-r0 ENV ENABLE_GAS=$enable_gas ENV TEST=$test From 36a33437c9b9e61605a53decfb78c4d43da4eb49 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 16:09:08 -0300 Subject: [PATCH 15/32] Fix bench-comment --- yarn-project/scripts/Earthfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index a2c792ecaa82..043fba9c416e 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -44,7 +44,6 @@ bench-aggregate: bench-comment: FROM +build - COPY +bench-aggregate/bench $BENCH_FOLDER LET LOG_FOLDER=/usr/var/logs LET BENCH_FOLDER=/usr/var/bench ENV LOG_FOLDER=$LOG_FOLDER @@ -52,8 +51,9 @@ bench-comment: ENV COMMIT_HASH=$COMMIT_HASH ENV PULL_REQUEST=$PULL_REQUEST ENV BRANCH=$BRANCH + COPY +bench-aggregate/bench $BENCH_FOLDER RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ [ -f $BENCH_FOLDER/benchmark.json ] \ && (./scripts/logs/download_base_benchmark_from_s3.sh && (cd /usr/src/yarn-project/scripts && yarn bench-comment)) \ - || true + || echo "No benchmark file found in $BENCH_FOLDER" \ No newline at end of file From f8fd1ae08e5aa2536e2bea7df0c381dc76905121 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 17:06:59 -0300 Subject: [PATCH 16/32] Try fix bench-comment --- yarn-project/scripts/Earthfile | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index 043fba9c416e..ce8d9f303661 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -15,6 +15,7 @@ build: WORKDIR /usr/src download-logs: + # Downloads logs for the current PR and saves them as an artifact FROM +build LET LOG_FOLDER=/usr/var/logs LET BENCH_FOLDER=/usr/var/bench @@ -27,6 +28,8 @@ download-logs: SAVE ARTIFACT $LOG_FOLDER logs bench-aggregate: + # Copies logs downloaded from download-logs and aggregates them using bench-aggregate. + # No aggregation is done if there is a log missing from the benchmark jobs. FROM +build LET LOG_FOLDER=/usr/var/logs LET BENCH_FOLDER=/usr/var/bench @@ -39,21 +42,28 @@ bench-aggregate: RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ ./scripts/logs/check_logs_for_benchmark.sh \ && ((cd /usr/src/yarn-project/scripts && yarn bench-aggregate) && ./scripts/logs/upload_aggregated_benchmarks_to_s3.sh) \ - || mkdir -p $BENCH_FOLDER + || (echo "Not all log files from benchmark jobs found"; mkdir -p $BENCH_FOLDER) SAVE ARTIFACT $BENCH_FOLDER bench bench-comment: - FROM +build - LET LOG_FOLDER=/usr/var/logs LET BENCH_FOLDER=/usr/var/bench - ENV LOG_FOLDER=$LOG_FOLDER + # Download base benchmark file locally so we have access to git to find out the right commit + LOCALLY + ENV BENCH_FOLDER=$BENCH_FOLDER + ENV COMMIT_HASH=$COMMIT_HASH + ENV PULL_REQUEST=$PULL_REQUEST + RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p $BENCH_FOLDER && ../../scripts/logs/download_base_benchmark_from_s3.sh + # Use the scripts image to run bench comment after loading the benchmark from bench-aggregate + # and the base benchmark from the local step + FROM +build ENV BENCH_FOLDER=$BENCH_FOLDER ENV COMMIT_HASH=$COMMIT_HASH ENV PULL_REQUEST=$PULL_REQUEST ENV BRANCH=$BRANCH + COPY $BENCH_FOLDER/base-benchmark.json $BENCH_FOLDER/base-benchmark.json COPY +bench-aggregate/bench $BENCH_FOLDER RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ [ -f $BENCH_FOLDER/benchmark.json ] \ - && (./scripts/logs/download_base_benchmark_from_s3.sh && (cd /usr/src/yarn-project/scripts && yarn bench-comment)) \ - || echo "No benchmark file found in $BENCH_FOLDER" + && (cd /usr/src/yarn-project/scripts && yarn bench-comment) \ + || echo "No benchmark file found in $BENCH_FOLDER" \ No newline at end of file From 5ff1719baee143f7de5f84b1376f965775b73850 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 17:07:14 -0300 Subject: [PATCH 17/32] Get to bench-summary faster for testing --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41c09acdb588..b7c5254b2002 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,8 @@ jobs: run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --no-output +${{ matrix.test }} --e2e_mode=cache bench-summary: - needs: e2e + # needs: e2e + needs: setup # Speed things up for testing runs-on: ${{ inputs.username || github.actor }}-x86 steps: - uses: actions/checkout@v4 From 1e406f6d90cb21435b8ba8b4521562412bbe792e Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 17:35:10 -0300 Subject: [PATCH 18/32] Earthly local secrets do not work --- .github/workflows/ci.yml | 7 +++++++ .../logs/download_base_benchmark_from_s3.sh | 2 ++ yarn-project/scripts/Earthfile | 19 ++++++++----------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7c5254b2002..e6c05c48b905 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,6 +92,13 @@ jobs: - name: "Build and upload bench aggregate file" working-directory: ./yarn-project/scripts run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} +bench-aggregate + - name: "Download base benchmark" + run: scripts/logs/download_base_benchmark_from_s3.sh + if: ${{ github.event_name == 'pull_request' }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + BENCH_FOLDER: ./yarn-project/scripts/bench - name: "Generate summary comment if pull request" working-directory: ./yarn-project/scripts run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_COMMENTER_GITHUB_TOKEN }} +bench-comment diff --git a/scripts/logs/download_base_benchmark_from_s3.sh b/scripts/logs/download_base_benchmark_from_s3.sh index 2956adb19c8e..966af7f43e31 100755 --- a/scripts/logs/download_base_benchmark_from_s3.sh +++ b/scripts/logs/download_base_benchmark_from_s3.sh @@ -14,6 +14,8 @@ if [ -n "${PULL_REQUEST:-}" ]; then MASTER_COMMIT_HASH=$(curl -s "https://api.github.com/repos/AztecProtocol/aztec-packages/pulls/${PULL_REQUEST##*/}" | jq -r '.base.sha') MASTER_COMMIT_HASHES=($(git log $MASTER_COMMIT_HASH --format="%H" -n 50)) + mkdir -p $BENCH_FOLDER + set +e echo "Searching for base benchmark data starting from commit $MASTER_COMMIT_HASH" for commit_hash in "${MASTER_COMMIT_HASHES[@]}"; do diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index ce8d9f303661..235821c78587 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -15,7 +15,7 @@ build: WORKDIR /usr/src download-logs: - # Downloads logs for the current PR and saves them as an artifact + # Downloads logs for the current PR and saves them as an artifact, to be consumed by bench-aggregate. FROM +build LET LOG_FOLDER=/usr/var/logs LET BENCH_FOLDER=/usr/var/bench @@ -46,21 +46,18 @@ bench-aggregate: SAVE ARTIFACT $BENCH_FOLDER bench bench-comment: - LET BENCH_FOLDER=/usr/var/bench - # Download base benchmark file locally so we have access to git to find out the right commit - LOCALLY - ENV BENCH_FOLDER=$BENCH_FOLDER - ENV COMMIT_HASH=$COMMIT_HASH - ENV PULL_REQUEST=$PULL_REQUEST - RUN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY mkdir -p $BENCH_FOLDER && ../../scripts/logs/download_base_benchmark_from_s3.sh - # Use the scripts image to run bench comment after loading the benchmark from bench-aggregate - # and the base benchmark from the local step + # Use the scripts image to run bench comment after loading the benchmark from bench-aggregate. + # Requires base-benchmark to be downloaded outside of this target. Note we cannot run it within + # an Earthly container because it needs access to the .git folder, and we cannot run it with a + # LOCALLY statement because Earthly does not support secrets when running locally (and we need) + # the AWS access keys to access S3. This step is then manually run in the ci.yml. FROM +build + LET BENCH_FOLDER=/usr/var/bench ENV BENCH_FOLDER=$BENCH_FOLDER ENV COMMIT_HASH=$COMMIT_HASH ENV PULL_REQUEST=$PULL_REQUEST ENV BRANCH=$BRANCH - COPY $BENCH_FOLDER/base-benchmark.json $BENCH_FOLDER/base-benchmark.json + COPY ./bench/base-benchmark.json $BENCH_FOLDER/base-benchmark.json COPY +bench-aggregate/bench $BENCH_FOLDER RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ [ -f $BENCH_FOLDER/benchmark.json ] \ From 78213bd16db1c2500f8b909a5972a966215611f5 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 17:47:46 -0300 Subject: [PATCH 19/32] Bypass "lock is currently held by..." so I can test bench summary fast --- .github/workflows/ci.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6c05c48b905..54ef3909efa2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: secrets: inherit build: + if: false needs: setup runs-on: ${{ inputs.username || github.actor }}-x86 outputs: @@ -59,6 +60,7 @@ jobs: # all the end-to-end integration tests for aztec e2e: + if: false needs: build runs-on: ${{ inputs.username || github.actor }}-x86 strategy: @@ -107,6 +109,7 @@ jobs: # barretenberg (prover) native and AVM (public VM) tests # only ran on x86 for resource reasons (memory intensive) bb-native-tests: + if: false needs: setup runs-on: ${{ inputs.username || github.actor }}-x86 steps: @@ -127,6 +130,7 @@ jobs: run: earthly-ci --no-output +test --hardware_concurrency=64 noir-projects: + if: false needs: setup runs-on: ${{ inputs.username || github.actor }}-x86 steps: @@ -143,6 +147,7 @@ jobs: run: earthly-ci --no-output ./noir-projects/+test yarn-project-formatting: + if: false needs: setup runs-on: ${{ github.actor }}-x86 steps: @@ -160,6 +165,7 @@ jobs: run: earthly-ci --no-output ./yarn-project/+format-check yarn-project-test: + if: false needs: noir-projects runs-on: ${{ github.actor }}-x86 steps: @@ -177,6 +183,7 @@ jobs: run: earthly-ci --no-output ./yarn-project/+test docs-preview: + if: false needs: setup runs-on: ${{ inputs.username || github.actor }}-x86 steps: @@ -194,6 +201,7 @@ jobs: # push benchmarking binaries to dockerhub registry bb-bench-binaries: + if: false needs: setup runs-on: ${{ inputs.username || github.actor }}-x86 steps: @@ -211,6 +219,7 @@ jobs: run: earthly-ci --push +bench-binaries setup-bench: + if: false uses: ./.github/workflows/setup-runner.yml needs: bb-bench-binaries with: @@ -224,6 +233,7 @@ jobs: secrets: inherit bb-bench: + if: false runs-on: ${{ inputs.username || github.actor }}-bench-x86 needs: setup-bench steps: @@ -254,7 +264,8 @@ jobs: - bb-bench - yarn-project-formatting - yarn-project-test - if: always() + # if: always() + if: false steps: - run: | echo "e2e status: ${{ needs.e2e.result }}" @@ -278,7 +289,8 @@ jobs: yarn-project-test, ] runs-on: ubuntu-latest - if: ${{ github.ref == 'refs/heads/master' && failure() }} + # if: ${{ github.ref == 'refs/heads/master' && failure() }} + if: false steps: - name: Send notification to aztec3-ci channel if workflow failed on master uses: slackapi/slack-github-action@v1.25.0 From db8133158640426531157d2f319403f4e30b1f57 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 17:52:29 -0300 Subject: [PATCH 20/32] Global args are not good for cache it seems --- yarn-project/scripts/Earthfile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index 235821c78587..ec817b992108 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -4,9 +4,6 @@ FROM node:18.19.0 ARG --global use_local_builds=false # These args are loaded from the EARTHLY_BUILD_ARGS set in setup_env -ARG --global COMMIT_HASH -ARG --global PULL_REQUEST -ARG --global BRANCH build: FROM ../+scripts-prod --use_local_builds=$use_local_builds @@ -16,6 +13,9 @@ build: download-logs: # Downloads logs for the current PR and saves them as an artifact, to be consumed by bench-aggregate. + ARG COMMIT_HASH + ARG PULL_REQUEST + ARG BRANCH FROM +build LET LOG_FOLDER=/usr/var/logs LET BENCH_FOLDER=/usr/var/bench @@ -30,6 +30,9 @@ download-logs: bench-aggregate: # Copies logs downloaded from download-logs and aggregates them using bench-aggregate. # No aggregation is done if there is a log missing from the benchmark jobs. + ARG COMMIT_HASH + ARG PULL_REQUEST + ARG BRANCH FROM +build LET LOG_FOLDER=/usr/var/logs LET BENCH_FOLDER=/usr/var/bench @@ -51,6 +54,9 @@ bench-comment: # an Earthly container because it needs access to the .git folder, and we cannot run it with a # LOCALLY statement because Earthly does not support secrets when running locally (and we need) # the AWS access keys to access S3. This step is then manually run in the ci.yml. + ARG COMMIT_HASH + ARG PULL_REQUEST + ARG BRANCH FROM +build LET BENCH_FOLDER=/usr/var/bench ENV BENCH_FOLDER=$BENCH_FOLDER From e3a4e7ce60668ac0720742a0df4b8e8045ca6f7e Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 17:54:58 -0300 Subject: [PATCH 21/32] Non-global args in end-to-end earthfile as well --- yarn-project/end-to-end/Earthfile | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 71c5b8a6206d..867cec39c6f5 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -1,9 +1,6 @@ VERSION 0.8 ARG --global use_local_builds=false -ARG --global PULL_REQUEST -ARG --global BRANCH -ARG --global COMMIT_HASH # requires first saving the images locally with ../+export-end-to-end @@ -95,6 +92,9 @@ E2E_TEST: UPLOAD_LOGS: FUNCTION ARG e2e_mode=cache + ARG PULL_REQUEST + ARG BRANCH + ARG COMMIT_HASH LOCALLY LET COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" FROM ../../+scripts @@ -150,25 +150,32 @@ guides-dapp-testing: # guides-sample-dapp: # ARG e2e_mode=local # DO +E2E_TEST --test=sample-dapp --e2e_mode=$e2e_mode -# DO +UPLOAD_LOGS --e2e_mode=$e2e_mode # TODO currently hangs for hour+ # guides-up-quick-start: # ARG e2e_mode=local # DO +E2E_TEST --test=guides/up_quick_start.test.ts --e2e_mode=$e2e_mode -# DO +UPLOAD_LOGS --e2e_mode=$e2e_mode bench-publish-rollup: ARG e2e_mode=local + ARG PULL_REQUEST + ARG BRANCH + ARG COMMIT_HASH DO +E2E_TEST --test=benchmarks/bench_publish_rollup.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml - DO +UPLOAD_LOGS --e2e_mode=$e2e_mode + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode --PULL_REQUEST=$PULL_REQUEST --BRANCH=$BRANCH --COMMIT_HASH=$COMMIT_HASH bench-process-history: ARG e2e_mode=local + ARG PULL_REQUEST + ARG BRANCH + ARG COMMIT_HASH DO +E2E_TEST --test=benchmarks/bench_process_history.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --compose_file=./scripts/docker-compose-no-sandbox.yml - DO +UPLOAD_LOGS --e2e_mode=$e2e_mode + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode --PULL_REQUEST=$PULL_REQUEST --BRANCH=$BRANCH --COMMIT_HASH=$COMMIT_HASH bench-tx-size: ARG e2e_mode=local + ARG PULL_REQUEST + ARG BRANCH + ARG COMMIT_HASH DO +E2E_TEST --test=benchmarks/bench_tx_size_fees.test.ts --debug="aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" --e2e_mode=$e2e_mode --enable_gas=1 --compose_file=./scripts/docker-compose-no-sandbox.yml - DO +UPLOAD_LOGS --e2e_mode=$e2e_mode + DO +UPLOAD_LOGS --e2e_mode=$e2e_mode --PULL_REQUEST=$PULL_REQUEST --BRANCH=$BRANCH --COMMIT_HASH=$COMMIT_HASH From 14ba1c5ea2da74d14d499287cd53c4c205d7e9e1 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 17:55:54 -0300 Subject: [PATCH 22/32] Remove references to use_local_builds --- yarn-project/end-to-end/Earthfile | 2 -- yarn-project/scripts/Earthfile | 6 +----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index 867cec39c6f5..d59b9222d0ed 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -1,7 +1,5 @@ VERSION 0.8 -ARG --global use_local_builds=false - # requires first saving the images locally with ../+export-end-to-end # run locally and build diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index ec817b992108..8ff331b0f37b 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -1,12 +1,8 @@ VERSION 0.8 FROM node:18.19.0 -ARG --global use_local_builds=false - -# These args are loaded from the EARTHLY_BUILD_ARGS set in setup_env - build: - FROM ../+scripts-prod --use_local_builds=$use_local_builds + FROM ../+scripts-prod RUN apt-get update && apt-get install awscli -y COPY ../../+scripts/scripts /usr/src/scripts WORKDIR /usr/src From be8fffa76179d1ac025858d8136c1d934aa96281 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 18:01:16 -0300 Subject: [PATCH 23/32] Place if before run in steps --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54ef3909efa2..d737084c57e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,16 +95,16 @@ jobs: working-directory: ./yarn-project/scripts run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} +bench-aggregate - name: "Download base benchmark" - run: scripts/logs/download_base_benchmark_from_s3.sh if: ${{ github.event_name == 'pull_request' }} + run: scripts/logs/download_base_benchmark_from_s3.sh env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} BENCH_FOLDER: ./yarn-project/scripts/bench - name: "Generate summary comment if pull request" + if: ${{ github.event_name == 'pull_request' }} working-directory: ./yarn-project/scripts run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_COMMENTER_GITHUB_TOKEN }} +bench-comment - if: ${{ github.event_name == 'pull_request' }} # barretenberg (prover) native and AVM (public VM) tests # only ran on x86 for resource reasons (memory intensive) From a6dff340e2b1a2d7ed10e4aae96b43f9863a3261 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 18:10:06 -0300 Subject: [PATCH 24/32] Pass pull request number to download base bench script --- .github/workflows/ci.yml | 1 + scripts/logs/download_base_benchmark_from_s3.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d737084c57e7..3b7c73385ae0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,7 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} BENCH_FOLDER: ./yarn-project/scripts/bench + PULL_REQUEST: "${{ github.event.pull_request.number }}" - name: "Generate summary comment if pull request" if: ${{ github.event_name == 'pull_request' }} working-directory: ./yarn-project/scripts diff --git a/scripts/logs/download_base_benchmark_from_s3.sh b/scripts/logs/download_base_benchmark_from_s3.sh index 966af7f43e31..dfe5d631ab47 100755 --- a/scripts/logs/download_base_benchmark_from_s3.sh +++ b/scripts/logs/download_base_benchmark_from_s3.sh @@ -28,4 +28,6 @@ if [ -n "${PULL_REQUEST:-}" ]; then set -e echo "No base commit data found" +else + echo "Not on a pull request, skipping download of base benchmark data" fi From 79622050f2d86b1ca6f1ea45b77476fcec6694ab Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 18:12:06 -0300 Subject: [PATCH 25/32] Log bench folder contents --- yarn-project/scripts/Earthfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index 8ff331b0f37b..0e70e2f5d9db 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -59,8 +59,9 @@ bench-comment: ENV COMMIT_HASH=$COMMIT_HASH ENV PULL_REQUEST=$PULL_REQUEST ENV BRANCH=$BRANCH - COPY ./bench/base-benchmark.json $BENCH_FOLDER/base-benchmark.json + COPY ./bench $BENCH_FOLDER/ COPY +bench-aggregate/bench $BENCH_FOLDER + RUN echo "Bench folder contents $(ls $BENCH_FOLDER)" RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ [ -f $BENCH_FOLDER/benchmark.json ] \ && (cd /usr/src/yarn-project/scripts && yarn bench-comment) \ From 6b2951eb7243fe400702ef7e29198612209e9364 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 18:17:10 -0300 Subject: [PATCH 26/32] Checkout history for bench summary --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b7c73385ae0..e86a0269113b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,9 @@ jobs: runs-on: ${{ inputs.username || github.actor }}-x86 steps: - uses: actions/checkout@v4 - with: { ref: "${{ github.event.pull_request.head.sha }}" } + with: + fetch-depth: 100 # Downloading base benchmark from master requires access to history + ref: "${{ github.event.pull_request.head.sha }}" - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" From f297f394313e370d7076749e84cb8b64363cd4a8 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 18:24:13 -0300 Subject: [PATCH 27/32] Bench comment expects PR_NUMBER variable --- yarn-project/scripts/Earthfile | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index 0e70e2f5d9db..db6093672cff 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -58,6 +58,7 @@ bench-comment: ENV BENCH_FOLDER=$BENCH_FOLDER ENV COMMIT_HASH=$COMMIT_HASH ENV PULL_REQUEST=$PULL_REQUEST + ENV PR_NUMBER=$PULL_REQUEST ENV BRANCH=$BRANCH COPY ./bench $BENCH_FOLDER/ COPY +bench-aggregate/bench $BENCH_FOLDER From 6dfc3b835cca6e86a07a77bf842e67a267a6cb07 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 18:34:44 -0300 Subject: [PATCH 28/32] Try fix gh creds --- yarn-project/scripts/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index db6093672cff..de1d4723e756 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -65,6 +65,6 @@ bench-comment: RUN echo "Bench folder contents $(ls $BENCH_FOLDER)" RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ [ -f $BENCH_FOLDER/benchmark.json ] \ - && (cd /usr/src/yarn-project/scripts && yarn bench-comment) \ + && (cd /usr/src/yarn-project/scripts && AZTEC_BOT_COMMENTER_GITHUB_TOKEN=$AZTEC_BOT_COMMENTER_GITHUB_TOKEN yarn bench-comment) \ || echo "No benchmark file found in $BENCH_FOLDER" \ No newline at end of file From 79e9f5b82df4930db8cb6e0a86b65942067f670c Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 18:37:11 -0300 Subject: [PATCH 29/32] Remove unnecessary secrets from script run --- yarn-project/scripts/Earthfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/scripts/Earthfile b/yarn-project/scripts/Earthfile index de1d4723e756..37aba4bda90c 100644 --- a/yarn-project/scripts/Earthfile +++ b/yarn-project/scripts/Earthfile @@ -63,7 +63,7 @@ bench-comment: COPY ./bench $BENCH_FOLDER/ COPY +bench-aggregate/bench $BENCH_FOLDER RUN echo "Bench folder contents $(ls $BENCH_FOLDER)" - RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN --secret AWS_ACCESS_KEY_ID --secret AWS_SECRET_ACCESS_KEY \ + RUN --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN \ [ -f $BENCH_FOLDER/benchmark.json ] \ && (cd /usr/src/yarn-project/scripts && AZTEC_BOT_COMMENTER_GITHUB_TOKEN=$AZTEC_BOT_COMMENTER_GITHUB_TOKEN yarn bench-comment) \ || echo "No benchmark file found in $BENCH_FOLDER" From 02978e464fd97136e75b09e4fe91516791d467f2 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 18:39:09 -0300 Subject: [PATCH 30/32] Would be nice if ga warned of missing secrets, wouldnt it? --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e86a0269113b..61cd6f3cf523 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,7 +107,7 @@ jobs: - name: "Generate summary comment if pull request" if: ${{ github.event_name == 'pull_request' }} working-directory: ./yarn-project/scripts - run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_COMMENTER_GITHUB_TOKEN }} +bench-comment + run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --secret AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} +bench-comment # barretenberg (prover) native and AVM (public VM) tests # only ran on x86 for resource reasons (memory intensive) From 9d1195a39d5854404d30d50f8c2a811b2dfd5fbb Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 18:53:15 -0300 Subject: [PATCH 31/32] Revert "Bypass "lock is currently held by..." so I can test bench summary fast" This reverts commit 5ad7dfbb22dc86f1e81171bbb2e5c5702ece1382. --- .github/workflows/ci.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61cd6f3cf523..a84d2014e2a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,6 @@ jobs: secrets: inherit build: - if: false needs: setup runs-on: ${{ inputs.username || github.actor }}-x86 outputs: @@ -60,7 +59,6 @@ jobs: # all the end-to-end integration tests for aztec e2e: - if: false needs: build runs-on: ${{ inputs.username || github.actor }}-x86 strategy: @@ -112,7 +110,6 @@ jobs: # barretenberg (prover) native and AVM (public VM) tests # only ran on x86 for resource reasons (memory intensive) bb-native-tests: - if: false needs: setup runs-on: ${{ inputs.username || github.actor }}-x86 steps: @@ -133,7 +130,6 @@ jobs: run: earthly-ci --no-output +test --hardware_concurrency=64 noir-projects: - if: false needs: setup runs-on: ${{ inputs.username || github.actor }}-x86 steps: @@ -150,7 +146,6 @@ jobs: run: earthly-ci --no-output ./noir-projects/+test yarn-project-formatting: - if: false needs: setup runs-on: ${{ github.actor }}-x86 steps: @@ -168,7 +163,6 @@ jobs: run: earthly-ci --no-output ./yarn-project/+format-check yarn-project-test: - if: false needs: noir-projects runs-on: ${{ github.actor }}-x86 steps: @@ -186,7 +180,6 @@ jobs: run: earthly-ci --no-output ./yarn-project/+test docs-preview: - if: false needs: setup runs-on: ${{ inputs.username || github.actor }}-x86 steps: @@ -204,7 +197,6 @@ jobs: # push benchmarking binaries to dockerhub registry bb-bench-binaries: - if: false needs: setup runs-on: ${{ inputs.username || github.actor }}-x86 steps: @@ -222,7 +214,6 @@ jobs: run: earthly-ci --push +bench-binaries setup-bench: - if: false uses: ./.github/workflows/setup-runner.yml needs: bb-bench-binaries with: @@ -236,7 +227,6 @@ jobs: secrets: inherit bb-bench: - if: false runs-on: ${{ inputs.username || github.actor }}-bench-x86 needs: setup-bench steps: @@ -267,8 +257,7 @@ jobs: - bb-bench - yarn-project-formatting - yarn-project-test - # if: always() - if: false + if: always() steps: - run: | echo "e2e status: ${{ needs.e2e.result }}" @@ -292,8 +281,7 @@ jobs: yarn-project-test, ] runs-on: ubuntu-latest - # if: ${{ github.ref == 'refs/heads/master' && failure() }} - if: false + if: ${{ github.ref == 'refs/heads/master' && failure() }} steps: - name: Send notification to aztec3-ci channel if workflow failed on master uses: slackapi/slack-github-action@v1.25.0 From 5991327c864803957ec8dea4f7327f6a7960f3aa Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Tue, 30 Apr 2024 18:53:34 -0300 Subject: [PATCH 32/32] Revert "Get to bench-summary faster for testing" This reverts commit 8819f22bf3244e3a15d454f4b586a266b86615ad. --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a84d2014e2a9..586902b443b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,8 +79,7 @@ jobs: run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --no-output +${{ matrix.test }} --e2e_mode=cache bench-summary: - # needs: e2e - needs: setup # Speed things up for testing + needs: e2e runs-on: ${{ inputs.username || github.actor }}-x86 steps: - uses: actions/checkout@v4