diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 0ecc960521e6..fcf79d121059 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -9,9 +9,10 @@ COMMIT_HASH=${2:-$COMMIT_HASH} REBUILD_PATTERNS=$(query_manifest rebuildPatterns $REPOSITORY) AWK_PATTERN=$(echo $REBUILD_PATTERNS | sed 's/ /|/g') +cd "$(git rev-parse --show-toplevel)" # an example line is # 100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0 yarn-project/world-state/tsconfig.json # this format is beneficial as it grabs the hashes from git efficiently # we will next filter by our rebuild patterns # then we pipe the hash portion of each file to git hash-object to produce our content hash -git ls-tree -r HEAD | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin +git ls-tree -r $COMMIT_HASH | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin diff --git a/build-system/scripts/calculate_image_uri b/build-system/scripts/calculate_image_uri new file mode 100755 index 000000000000..42bedecba2bf --- /dev/null +++ b/build-system/scripts/calculate_image_uri @@ -0,0 +1,10 @@ +#!/bin/bash +set -eu + +REPOSITORY=$1 +# Assume we might be calling this directly without build system in path +export ROOT_PATH=$(git rev-parse --show-toplevel) +export PATH="$PATH:$ROOT_PATH/build-system/scripts:$ROOT_PATH/build-system/bin" +export LD_LIBRARY_PATH="$ROOT_PATH/build-system/lib" +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 61f262307e5e..7ade4de52bde 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -9,7 +9,7 @@ shift DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "Content hash: $CONTENT_HASH" +echo "Content hash tag: cache-$CONTENT_HASH" cd $(query_manifest buildDir $REPOSITORY) diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index 154b70c8b684..d9e9aafb13c0 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -22,7 +22,7 @@ SUCCESS_TAG=$1 shift CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "Content hash: $CONTENT_HASH" +echo "Content hash tag: cache-$CONTENT_HASH-$SUCCESS_TAG" if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then init_submodules $REPOSITORY diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index c33d386f1f90..0e6417da1565 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -2,9 +2,8 @@ set -eu IP=$1 -CONTENT_HASH=$2 -FULL_PATH=$3 -shift 3 +FULL_PATH=$2 +shift 2 SSH_CONFIG_PATH=${SSH_CONFIG_PATH:-$BUILD_SYSTEM_PATH/remote/ssh_config} DIR_NAME=$(dirname $FULL_PATH) @@ -12,6 +11,7 @@ SCRIPT_NAME=$(basename $FULL_PATH) # Copy all files in script directory to spot instance. scp -F $SSH_CONFIG_PATH $DIR_NAME/* $IP:. +scp -rF $SSH_CONFIG_PATH $BUILD_SYSTEM_PATH $BUILD_SYSTEM_PATH/../.git $BUILD_SYSTEM_PATH/../build_manifest.json $IP:. # Run script on remote instance, passing environment variables. -ssh -A -F $SSH_CONFIG_PATH $IP "CONTENT_HASH=$CONTENT_HASH COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ECR_DEPLOY_URL=$ECR_DEPLOY_URL ECR_URL=$ECR_URL ./$SCRIPT_NAME $@" +ssh -A -F $SSH_CONFIG_PATH $IP "COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ECR_DEPLOY_URL=$ECR_DEPLOY_URL ECR_URL=$ECR_URL ./$SCRIPT_NAME $@" diff --git a/build-system/scripts/spot_run_script b/build-system/scripts/spot_run_script index 63f16928c857..ddf4acebc43e 100755 --- a/build-system/scripts/spot_run_script +++ b/build-system/scripts/spot_run_script @@ -23,7 +23,7 @@ IP=$(request_spot $CONTENT_HASH:$JOB_NAME $SPEC) # Run script remotely on spot instance, capturing success or failure. set +e -remote_run_script $IP $CONTENT_HASH $@ +remote_run_script $IP $@ CODE=$? # Shutdown spot. diff --git a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh index 235aa111ff4c..4037b3ad365f 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh @@ -6,7 +6,7 @@ set -eu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=barretenberg-x86_64-linux-clang-assert # use the image rebuild patterns to compute a content hash, use this to get a URI -IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) docker pull $IMAGE_URI TESTS=( diff --git a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh index b06c26036bac..6370dbebb5f4 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh @@ -4,7 +4,8 @@ set -eu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=barretenberg-x86_64-linux-clang-assert -IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) + docker pull $IMAGE_URI docker run --rm -t $IMAGE_URI /bin/sh -c "\ diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests index d67176f098fa..bab5c7f22073 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests @@ -20,7 +20,7 @@ shift # to aztec's circuits `run_tests_local` $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY="barretenberg-circuits-${ARCH}-linux-clang-builder-runner" -IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) docker pull $IMAGE_URI if [ "$ARCH" != "wasm" ]; then diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_tests b/circuits/cpp/barretenberg/cpp/scripts/run_tests index 965307ff5a59..f6ccd5a43594 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_tests @@ -14,7 +14,7 @@ $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null # use the image rebuild patterns to compute a content hash, use this to get a URI REPOSITORY=barretenberg-x86_64-linux-clang-assert -IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) docker pull $IMAGE_URI if [ -f "$TESTS" ]; then diff --git a/circuits/cpp/barretenberg/ts/scripts/run_tests b/circuits/cpp/barretenberg/ts/scripts/run_tests index cd417d0724a7..7a97ff6adefc 100755 --- a/circuits/cpp/barretenberg/ts/scripts/run_tests +++ b/circuits/cpp/barretenberg/ts/scripts/run_tests @@ -2,5 +2,7 @@ set -e $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/bb.js:cache-$CONTENT_HASH + +REPOSITORY=bb.js +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) docker run --rm $IMAGE_URI diff --git a/circuits/cpp/scripts/run_tests b/circuits/cpp/scripts/run_tests index b3fbc43f4592..b92241ad44ef 100755 --- a/circuits/cpp/scripts/run_tests +++ b/circuits/cpp/scripts/run_tests @@ -26,7 +26,7 @@ shift # arg1 (num transcripts) is not forwarded to `run_tests_local` $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY="circuits-$ARCH-linux-clang-assert" -IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) docker pull $IMAGE_URI # run tests in docker image diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index dac0f0bd921e..44c0b2e9dcac 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -14,7 +14,7 @@ fi $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$IMAGE:cache-$CONTENT_HASH" +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $IMAGE) docker pull $IMAGE_URI docker tag $IMAGE_URI aztecprotocol/canary:latest diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index e8159655bf20..88bf7d5c4b6d 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -25,10 +25,10 @@ SCRIPT_TO_RUN=$1 shift CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "Content hash: $CONTENT_HASH" +echo "Content hash tag: cache-$CONTENT_HASH-$SUCCESS_TAG" echo "Script to run is $SCRIPT_TO_RUN $@" if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then - CONTENT_HASH="$CONTENT_HASH" "$SCRIPT_TO_RUN" "$@" + "$SCRIPT_TO_RUN" "$@" tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG fi diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests index fad024b1eb2d..a41849e576fb 100755 --- a/yarn-project/end-to-end/scripts/run_tests +++ b/yarn-project/end-to-end/scripts/run_tests @@ -8,9 +8,11 @@ export COMPOSE_FILE=${2:-docker-compose.yml} $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null +BS=$(git rev-parse --show-toplevel)/build-system/scripts + for REPO in aztec.js end-to-end aztec-sandbox; do - docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH - docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest + docker pull $($BS/calculate_image_uri $REPO) + docker tag $($BS/calculate_image_uri $REPO) aztecprotocol/$REPO:latest done docker-compose rm -f