diff --git a/.circleci/config.yml b/.circleci/config.yml index 79993c70dc7e..edecd9b086df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ # The docker executor uses a custom image build in `build_image`. It's specifically streamlined for fast download # with just enough tools to execute the build system, and launch EC2 instances etc. # -# There are some `join` steps that are just noops. They are just used to produce cleaner graph rendering in CCI. +# There are some `join` steps that are just no-ops. They are just used to produce cleaner graph rendering in CCI. version: 2.1 @@ -61,12 +61,12 @@ checkout: &checkout retry git checkout FETCH_HEAD # Called setup_env to setup a bunch of global variables used throughout the rest of the build process. -# It takes the required CCI environment variables as inputs, and gives them normalised names for the rest of +# It takes the required CCI environment variables as inputs, and gives them normalized names for the rest of # the build process. This enables easy running of the build system external to CCI, as used for powerful EC2 builds. setup_env: &setup_env run: name: "Setup environment" - command: ./build-system/scripts/setup_env "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_JOB" "$CIRCLE_REPOSITORY_URL" "$CIRCLE_BRANCH" + command: ./build-system/scripts/setup_env "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_JOB" "$CIRCLE_REPOSITORY_URL" "$CIRCLE_BRANCH" "$CIRCLE_PULL_REQUEST" # This step is used to save logs from various barretenberg test to the workspace so that they can be used later to parse benchmark values out of them save_logs: &save_logs @@ -604,378 +604,345 @@ jobs: command: build end-to-end e2e-2-pxes: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_2_pxes.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_2_pxes.test.ts e2e-multiple-accounts-1-enc-key: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_multiple_accounts_1_enc_key.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_multiple_accounts_1_enc_key.test.ts e2e-deploy-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_deploy_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_deploy_contract.test.ts e2e-lending-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_lending_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_lending_contract.test.ts e2e-token-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_token_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_token_contract.test.ts e2e-sandbox-example: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_sandbox_example.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_sandbox_example.test.ts e2e-block-building: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_block_building.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_block_building.test.ts e2e-nested-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_nested_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_nested_contract.test.ts e2e-non-contract-account: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_non_contract_account.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_non_contract_account.test.ts e2e-cross-chain-messaging: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_cross_chain_messaging.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cross_chain_messaging.test.ts e2e-public-cross-chain-messaging: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_public_cross_chain_messaging.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_cross_chain_messaging.test.ts e2e-public-to-private-messaging: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_public_to_private_messaging.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_public_to_private_messaging.test.ts e2e-account-contracts: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_account_contracts.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_account_contracts.test.ts e2e-escrow-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_escrow_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_escrow_contract.test.ts e2e-pending-commitments-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_pending_commitments_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_pending_commitments_contract.test.ts e2e-ordering: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_ordering.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_ordering.test.ts uniswap-trade-on-l1-from-l2: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local uniswap_trade_on_l1_from_l2.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=uniswap_trade_on_l1_from_l2.test.ts integration-archiver-l1-to-l2: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local integration_archiver_l1_to_l2.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_archiver_l1_to_l2.test.ts integration-l1-publisher: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local integration_l1_publisher.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=integration_l1_publisher.test.ts e2e-cli: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_cli.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_cli.test.ts e2e-p2p: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_p2p_network.test.ts ./scripts/docker-compose-no-sandbox.yml - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=e2e_p2p_network.test.ts e2e-browser-sandbox: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_aztec_js_browser.test.ts ./scripts/docker-compose-browser.yml - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-browser.yml TEST=e2e_aztec_js_browser.test.ts e2e-card-game: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local e2e_card_game.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=e2e_card_game.test.ts pxe-sandbox: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local pxe_sandbox.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=pxe_sandbox.test.ts cli-docs-sandbox: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local cli_docs_sandbox.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=cli_docs_sandbox.test.ts guides-writing-an-account-contract: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local guides/writing_an_account_contract.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/writing_an_account_contract.test.ts guides-dapp-testing: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local guides/dapp_testing.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/dapp_testing.test.ts guides-sample-dapp: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local sample-dapp - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=sample-dapp guides-up-quick-start: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Test" - command: cond_run_script end-to-end ./scripts/run_tests_local guides/up_quick_start.test.ts - environment: { DEBUG: "aztec:*" } + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose.yml TEST=guides/up_quick_start.test.ts bench-publish-rollup: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Benchmark" - command: cond_run_script end-to-end ./scripts/run_tests_local benchmarks/bench_publish_rollup.test.ts ./scripts/docker-compose-no-sandbox.yml - environment: - DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_publish_rollup.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees bench-process-history: - machine: - image: ubuntu-2204:2023.07.2 - resource_class: large + docker: + - image: aztecprotocol/alpine-build-image + resource_class: small steps: - *checkout - *setup_env - run: name: "Benchmark" - command: cond_run_script end-to-end ./scripts/run_tests_local benchmarks/bench_process_history.test.ts ./scripts/docker-compose-no-sandbox.yml - environment: - DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees" + command: cond_spot_run_compose end-to-end 4 ./scripts/docker-compose-no-sandbox.yml TEST=benchmarks/bench_process_history.test.ts DEBUG=aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees build-docs: machine: diff --git a/build-system/scripts/build b/build-system/scripts/build index ff7c81d82b02..904ed2b31079 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -105,7 +105,7 @@ for PARENT_REPO in $(query_manifest dependencies $REPOSITORY); do fetch_image $PARENT_IMAGE_URI # Tag it to look like an official release as that's what we use in Dockerfiles. TAG=$ECR_DEPLOY_URL/$PARENT_REPO - retry docker tag $PARENT_IMAGE_URI $TAG + docker tag $PARENT_IMAGE_URI $TAG done COMMIT_TAG_VERSION=$(extract_tag_version $REPOSITORY false) diff --git a/build-system/scripts/cond_run_compose b/build-system/scripts/cond_run_compose new file mode 100755 index 000000000000..58eef99c71a7 --- /dev/null +++ b/build-system/scripts/cond_run_compose @@ -0,0 +1,35 @@ +#!/bin/bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +REPOSITORY=$1 +COMPOSE_FILE=$2 +shift 2 + +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +BASE_TAG=cache-$CONTENT_HASH +SUCCESS_TAG=$BASE_TAG-$JOB_NAME + +echo "Content hash: $CONTENT_HASH" + +if ! check_rebuild $SUCCESS_TAG $REPOSITORY; then + # Login to pull our ecr images with docker. + ecr_login + + # For each dependency and self, pull in the latest image and give it correct tag. + for REPO in $(query_manifest runDependencies $REPOSITORY $JOB_NAME) $REPOSITORY; do + echo "Pulling $REPO..." + REPO_IMAGE_URI=$(calculate_image_uri $REPO) + retry docker pull $REPO_IMAGE_URI + docker tag $REPO_IMAGE_URI aztecprotocol/$REPO + done + + cd $(query_manifest projectDir $REPOSITORY) + + export $@ + docker-compose -f $COMPOSE_FILE run $REPOSITORY + + upload_logs_to_s3 log + + retry tag_remote_image $REPOSITORY $BASE_TAG $SUCCESS_TAG +fi diff --git a/build-system/scripts/cond_spot_run_compose b/build-system/scripts/cond_spot_run_compose new file mode 100755 index 000000000000..b10cc9decf59 --- /dev/null +++ b/build-system/scripts/cond_spot_run_compose @@ -0,0 +1,10 @@ +#!/bin/bash +[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace +set -eu + +REPOSITORY=$1 +CPUS=$2 +shift 2 + +export TAG_POSTFIX=$JOB_NAME +cond_spot_run_script $REPOSITORY $CPUS cond_run_compose $REPOSITORY $@ diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest index edf5f62b6fd9..0358fbf89cb7 100755 --- a/build-system/scripts/query_manifest +++ b/build-system/scripts/query_manifest @@ -75,6 +75,12 @@ case "$CMD" in dependencies) yq -r ".\"$REPO\".dependencies // [] | .[]" $MANIFEST ;; + runDependencies) + echo $REPO + RUN_NAME=$3 + yq -r ".\"$REPO\".runDependencies // [] | .[]" $MANIFEST + yq -r ".\"$REPO\".run.\"$RUN_NAME\".dependencies // [] | .[]" $MANIFEST + ;; allDependencies) # Get dependencies for a given repo. Inclusive of repo itself. # If a string, attempt to execute / if exists to get dependencies, else error. @@ -94,8 +100,6 @@ case "$CMD" in ALL_DEPS+=("$1") } add_deps $REPO - # Remove ourself as a dependency. - # unset ALL_DEPS[-1] for DEP in "${ALL_DEPS[@]}"; do echo $DEP done diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index 5ad1a850e141..8aa4e0ea64f4 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -24,6 +24,8 @@ ssh -A -F $SSH_CONFIG_PATH $IP " export COMMIT_TAG=$COMMIT_TAG export JOB_NAME=$JOB_NAME export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL + export BRANCH=${BRANCH:-} + export PULL_REQUEST=${PULL_REQUEST:-} export DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD export ECR_DEPLOY_URL=$ECR_DEPLOY_URL export ECR_URL=$ECR_URL diff --git a/build-system/scripts/remote_runner b/build-system/scripts/remote_runner index 243d98e47bf5..ce1a567a1989 100755 --- a/build-system/scripts/remote_runner +++ b/build-system/scripts/remote_runner @@ -22,6 +22,6 @@ echo "Git checkout completed." BASH_ENV=/tmp/bash_env echo "Calling setup env..." -source ./build-system/scripts/setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL" +source ./build-system/scripts/setup_env "$COMMIT_HASH" "$COMMIT_TAG" "$JOB_NAME" "$GIT_REPOSITORY_URL" "$BRANCH" "$PULL_REQUEST" echo "Calling $@..." $@ diff --git a/build-system/scripts/setup_env b/build-system/scripts/setup_env index 8e436a9ca8bb..60880473baca 100755 --- a/build-system/scripts/setup_env +++ b/build-system/scripts/setup_env @@ -13,7 +13,7 @@ COMMIT_TAG=${2##*aztec-packages-} JOB_NAME=$3 GIT_REPOSITORY_URL=${4:-} BRANCH=${5:-} -COMMIT_MESSAGE=${6:-} +PULL_REQUEST=${6:-} BASH_ENV=${BASH_ENV:-} BUILD_SYSTEM_PATH=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) @@ -87,6 +87,7 @@ echo export GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL >> $BASH_ENV echo export VERSION_TAG=$VERSION_TAG >> $BASH_ENV echo export DEPLOY_TAG=$DEPLOY_TAG >> $BASH_ENV echo export BRANCH=$BRANCH >> $BASH_ENV +echo export PULL_REQUEST=$PULL_REQUEST >> $BASH_ENV # Only run the following if we're the result of a commit (i.e. not being run manually). if [ -n "$COMMIT_HASH" ]; then diff --git a/yarn-project/end-to-end/scripts/upload_logs_to_s3.sh b/build-system/scripts/upload_logs_to_s3 similarity index 83% rename from yarn-project/end-to-end/scripts/upload_logs_to_s3.sh rename to build-system/scripts/upload_logs_to_s3 index eb81df8bfac2..ede934edae59 100755 --- a/yarn-project/end-to-end/scripts/upload_logs_to_s3.sh +++ b/build-system/scripts/upload_logs_to_s3 @@ -8,8 +8,8 @@ [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace set -eu +LOG_FOLDER=$1 BUCKET_NAME="aztec-ci-artifacts" -LOG_FOLDER="${LOG_FOLDER:-log}" COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then @@ -18,10 +18,10 @@ if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then fi # Duplicated in scripts/ci/assemble_e2e_benchmark.sh -if [ "${CIRCLE_BRANCH:-}" = "master" ]; then +if [ "${BRANCH:-}" = "master" ]; then TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" -elif [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then - TARGET_FOLDER="logs-v1/pulls/${CIRCLE_PULL_REQUEST##*/}" +elif [ -n "${PULL_REQUEST:-}" ]; then + TARGET_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" fi if [ -n "${TARGET_FOLDER:-}" ]; then diff --git a/build_manifest.yml b/build_manifest.yml index dc194b390e70..6bf170c5c7b0 100644 --- a/build_manifest.yml +++ b/build_manifest.yml @@ -166,6 +166,8 @@ end-to-end: projectDir: yarn-project/end-to-end dependencies: - yarn-project + runDependencies: + - aztec-sandbox aztec-node: buildDir: yarn-project diff --git a/cspell.json b/cspell.json index b4ae2cbad6dd..00506c7af678 100644 --- a/cspell.json +++ b/cspell.json @@ -27,6 +27,8 @@ "camelcase", "cbind", "cbinds", + "cimg", + "cpus", "chainsafe", "cheatcode", "cheatcodes", @@ -37,6 +39,7 @@ "comlink", "composability", "concat", + "cond", "customizability", "danlee", "Daos", @@ -73,6 +76,7 @@ "Kademlia", "keccak", "keypairs", + "keyscan", "leveldb", "leveldown", "leveljs", @@ -132,6 +136,7 @@ "sload", "snakecase", "solhint", + "stdlib", "struct", "structs", "subrepo", @@ -178,7 +183,5 @@ "*.snap", "package.json" ], - "flagWords": [ - "anonymous" - ] + "flagWords": ["anonymous"] } diff --git a/yarn-project/end-to-end/scripts/cond_run_script.delme b/yarn-project/end-to-end/scripts/cond_run_script.delme deleted file mode 100755 index df4292891375..000000000000 --- a/yarn-project/end-to-end/scripts/cond_run_script.delme +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# Conditionally runs a script if any dependent code has changed between -# the last successful run and the present commit. -# -# It's expected to be run from the project directory, and that there be a directory called `scripts` -# containing the given named script to execute. -# -# This script is only useful if there is nothing to do in the event there is no rebuild. This is fine -# for running a suite of tests for example, but is not useful for performing a build, as even if a -# build has nothing to do, the previous images are retagged with the new commit hash for upstream jobs. -# -# Arguments are: -# 1. REPOSITORY: The project repository name in ECR. Used to determine if there are changes since last success. -# 2. SUCCESS_TAG: To track if this job needs to be run, the repository image is tagged with a success tag after a -# successful run. The script will only run if there were relevant code changes since the last successful commit. -# 3... ARGS: Script to run and args. -set -eu -set -x - -REPOSITORY=$1 -shift -SUCCESS_TAG=$1 -shift -SCRIPT_TO_RUN=$1 -shift - -CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "Content hash tag: cache-$CONTENT_HASH-$SUCCESS_TAG" -echo "Script to run is $SCRIPT_TO_RUN $@" - -if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then - "$SCRIPT_TO_RUN" "$@" - retry tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG -fi diff --git a/yarn-project/end-to-end/scripts/docker-compose-browser.yml b/yarn-project/end-to-end/scripts/docker-compose-browser.yml index 5d1e8dbc13b5..ddb5838000b6 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-browser.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-browser.yml @@ -30,7 +30,7 @@ services: end-to-end: image: aztecprotocol/end-to-end:latest environment: - BENCHMARK: true + BENCHMARK: 'true' DEBUG: ${DEBUG:-'aztec:*'} ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 @@ -38,3 +38,6 @@ services: entrypoint: ['./scripts/start_e2e_ci_browser.sh', './src/e2e_aztec_js_browser.test.ts'] volumes: - ../log:/usr/src/yarn-project/end-to-end/log:rw + depends_on: + - sandbox + - fork diff --git a/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml b/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml index bd2020709e5f..76d045ebc4bd 100644 --- a/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml +++ b/yarn-project/end-to-end/scripts/docker-compose-no-sandbox.yml @@ -15,7 +15,7 @@ services: end-to-end: image: aztecprotocol/end-to-end:latest environment: - BENCHMARK: true + BENCHMARK: 'true' DEBUG: ${DEBUG:-'aztec:*'} ETHEREUM_HOST: http://fork:8545 CHAIN_ID: 31337 @@ -28,3 +28,5 @@ services: command: ${TEST:-./src/e2e_deploy_contract.test.ts} volumes: - ../log:/usr/src/yarn-project/end-to-end/log:rw + depends_on: + - fork diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index ed5880dad89d..85c90b07cd66 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -30,7 +30,7 @@ services: end-to-end: image: aztecprotocol/end-to-end:latest environment: - BENCHMARK: true + BENCHMARK: 'true' LOG_LEVL: 'debug' DEBUG: ${DEBUG:-'aztec:*'} ETHEREUM_HOST: http://fork:8545 @@ -39,3 +39,6 @@ services: command: ${TEST:-./src/e2e_deploy_contract.test.ts} volumes: - ../log:/usr/src/yarn-project/end-to-end/log:rw + depends_on: + - sandbox + - fork diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests deleted file mode 100755 index 97b1effcc9e4..000000000000 --- a/yarn-project/end-to-end/scripts/run_tests +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# This script is used to run an e2e test in CI (see config.yml and cond_spot_run_tests). -# It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -export TEST=$1 -export COMPOSE_FILE=${2:-docker-compose.yml} - -$(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null - -export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" - -for REPO in aztec.js end-to-end aztec-sandbox; do - retry docker pull $($calculate_image_uri $REPO) - retry docker tag $($calculate_image_uri $REPO) aztecprotocol/$REPO:latest -done - -docker-compose rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end diff --git a/yarn-project/end-to-end/scripts/run_tests_local b/yarn-project/end-to-end/scripts/run_tests_local deleted file mode 100755 index 3df64ff40616..000000000000 --- a/yarn-project/end-to-end/scripts/run_tests_local +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# This script is used to run an e2e test in CI (see config.yml and cond_run_script). -# It sets a few environment variables used inside the docker-compose.yml, pulls images, and runs docker-compose. -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -export TEST=$1 -export COMPOSE_FILE=${2:-./scripts/docker-compose.yml} - -# Login to ECR. -ecr_login - -export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" - -for REPO in end-to-end aztec-sandbox; do - retry docker pull $(calculate_image_uri $REPO) - retry docker tag $(calculate_image_uri $REPO) aztecprotocol/$REPO:latest -done - -docker-compose -f $COMPOSE_FILE rm -f -docker-compose -f $COMPOSE_FILE up --exit-code-from end-to-end - -# Upload the logs with metrics to S3 -./scripts/upload_logs_to_s3.sh \ No newline at end of file