From bfa4e35b6e5cf149c2c98e486636b6ea3b91a82e Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 08:43:21 -0400 Subject: [PATCH 01/26] [TEST] ci test --- circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh index b75ec2b53b43..a18fcc1e330f 100755 --- a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh +++ b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh @@ -9,6 +9,7 @@ BB=$PWD/${BB:-../cpp/build/bin/bb} CRS_PATH=~/.bb-crs BRANCH=master + # Pull down the test vectors from the noir repo, if we don't have the folder already. if [ ! -d acir_tests ]; then if [ -n "${TEST_SRC:-}" ]; then From 7fa9fcdcff043d9f532db2f42d7093e9292a60e7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 08:47:04 -0400 Subject: [PATCH 02/26] Revert acir test --- circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh index a18fcc1e330f..b75ec2b53b43 100755 --- a/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh +++ b/circuits/cpp/barretenberg/acir_tests/run_acir_tests.sh @@ -9,7 +9,6 @@ BB=$PWD/${BB:-../cpp/build/bin/bb} CRS_PATH=~/.bb-crs BRANCH=master - # Pull down the test vectors from the noir repo, if we don't have the folder already. if [ ! -d acir_tests ]; then if [ -n "${TEST_SRC:-}" ]; then From 6373c88a2396a6dccd1e1b0993ad71009bea463a Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 08:47:43 -0400 Subject: [PATCH 03/26] Revert acir test --- circuits/cpp/barretenberg/ts/src/bigint-array/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/circuits/cpp/barretenberg/ts/src/bigint-array/index.ts b/circuits/cpp/barretenberg/ts/src/bigint-array/index.ts index c315c0758d5a..6755d959cdcd 100644 --- a/circuits/cpp/barretenberg/ts/src/bigint-array/index.ts +++ b/circuits/cpp/barretenberg/ts/src/bigint-array/index.ts @@ -10,6 +10,7 @@ export function toBigIntBE(bytes: Uint8Array) { return bigint; } + export function toBufferBE(value: bigint, byteLength = 32) { const bytes = new Uint8Array(byteLength); const view = new DataView(bytes.buffer); From f3815dcb744434fd9ddb3c6a41be9561e9df0d77 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 08:50:08 -0400 Subject: [PATCH 04/26] Revert acir test --- circuits/cpp/barretenberg/ts/src/bigint-array/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuits/cpp/barretenberg/ts/src/bigint-array/index.ts b/circuits/cpp/barretenberg/ts/src/bigint-array/index.ts index 6755d959cdcd..363af053f2b4 100644 --- a/circuits/cpp/barretenberg/ts/src/bigint-array/index.ts +++ b/circuits/cpp/barretenberg/ts/src/bigint-array/index.ts @@ -10,7 +10,7 @@ export function toBigIntBE(bytes: Uint8Array) { return bigint; } - +// test export function toBufferBE(value: bigint, byteLength = 32) { const bytes = new Uint8Array(byteLength); const view = new DataView(bytes.buffer); From f76ce97ebebd32337f84b170844eb149cd36b3df Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 08:59:54 -0400 Subject: [PATCH 05/26] Revert acir test --- build-system/scripts/cond_spot_run_build | 2 +- build-system/scripts/cond_spot_run_script | 2 +- yarn-project/end-to-end/scripts/cond_run_script | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 61f262307e5e..7ade4de52bde 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -9,7 +9,7 @@ shift DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "Content hash: $CONTENT_HASH" +echo "Content hash tag: cache-$CONTENT_HASH" cd $(query_manifest buildDir $REPOSITORY) diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index 154b70c8b684..d9e9aafb13c0 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -22,7 +22,7 @@ SUCCESS_TAG=$1 shift CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "Content hash: $CONTENT_HASH" +echo "Content hash tag: cache-$CONTENT_HASH-$SUCCESS_TAG" if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then init_submodules $REPOSITORY diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index e8159655bf20..26b3ba33c0df 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -25,7 +25,7 @@ SCRIPT_TO_RUN=$1 shift CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "Content hash: $CONTENT_HASH" +echo "Content hash tag: cache-$CONTENT_HASH-$SUCCESS_TAG" echo "Script to run is $SCRIPT_TO_RUN $@" if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then From e99e01ca9597f22c921d9bbcb52699d2e82068d3 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 09:02:23 -0400 Subject: [PATCH 06/26] Revert acir test --- circuits/cpp/barretenberg/ts/src/serialize/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/circuits/cpp/barretenberg/ts/src/serialize/index.ts b/circuits/cpp/barretenberg/ts/src/serialize/index.ts index bdd49908abec..45e9872de66f 100644 --- a/circuits/cpp/barretenberg/ts/src/serialize/index.ts +++ b/circuits/cpp/barretenberg/ts/src/serialize/index.ts @@ -1,3 +1,4 @@ export * from './buffer_reader.js'; export * from './output_type.js'; export * from './serialize.js'; +// test From 563070e9e98cee660db1c04d99eaa620a581a5e5 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 09:16:01 -0400 Subject: [PATCH 07/26] Revert acir test --- build-system/scripts/calculate_content_hash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 0ecc960521e6..ed22281f9bf8 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -8,6 +8,8 @@ COMMIT_HASH=${2:-$COMMIT_HASH} # Compute REBUILD_PATTERNS from the build manifest REBUILD_PATTERNS=$(query_manifest rebuildPatterns $REPOSITORY) +echo REBUILD $REBUILD_PATTERNS + AWK_PATTERN=$(echo $REBUILD_PATTERNS | sed 's/ /|/g') # an example line is # 100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0 yarn-project/world-state/tsconfig.json From d804fcd4c708d0ca80f7e19177d2a60ee8e3df2a Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 09:17:33 -0400 Subject: [PATCH 08/26] Revert acir test --- build-system/scripts/calculate_content_hash | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index ed22281f9bf8..28d02d973eab 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -8,12 +8,11 @@ COMMIT_HASH=${2:-$COMMIT_HASH} # Compute REBUILD_PATTERNS from the build manifest REBUILD_PATTERNS=$(query_manifest rebuildPatterns $REPOSITORY) -echo REBUILD $REBUILD_PATTERNS - AWK_PATTERN=$(echo $REBUILD_PATTERNS | sed 's/ /|/g') # an example line is # 100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0 yarn-project/world-state/tsconfig.json # this format is beneficial as it grabs the hashes from git efficiently # we will next filter by our rebuild patterns # then we pipe the hash portion of each file to git hash-object to produce our content hash -git ls-tree -r HEAD | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin +#git ls-tree -r HEAD | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin +echo $AWK_PATTERN From a7e598d698c123109dc785ce19a7e273414c979a Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 09:19:48 -0400 Subject: [PATCH 09/26] Revert acir test --- build-system/scripts/build | 2 ++ build-system/scripts/calculate_content_hash | 3 +-- build-system/scripts/cond_spot_run_build | 2 ++ build-system/scripts/cond_spot_run_script | 2 ++ build-system/scripts/spot_run_test_script | 2 ++ yarn-project/end-to-end/scripts/cond_run_script | 2 ++ 6 files changed, 11 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/build b/build-system/scripts/build index 2f59649e2738..20463a145e29 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -47,6 +47,8 @@ function fetch_image() { # Ensure ECR repository exists. retry_10 ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle +echo "REBUILD PATTERN" +query_manifest rebuildPatterns $REPOSITORY CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "Content hash: $CONTENT_HASH" diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 28d02d973eab..0ecc960521e6 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -14,5 +14,4 @@ AWK_PATTERN=$(echo $REBUILD_PATTERNS | sed 's/ /|/g') # this format is beneficial as it grabs the hashes from git efficiently # we will next filter by our rebuild patterns # then we pipe the hash portion of each file to git hash-object to produce our content hash -#git ls-tree -r HEAD | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin -echo $AWK_PATTERN +git ls-tree -r HEAD | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 7ade4de52bde..3cb11117cba2 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -8,6 +8,8 @@ SPEC=$1 shift DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) +echo "REBUILD PATTERN" +query_manifest rebuildPatterns $REPOSITORY CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "Content hash tag: cache-$CONTENT_HASH" diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index d9e9aafb13c0..74467454d447 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -21,6 +21,8 @@ shift SUCCESS_TAG=$1 shift +echo "REBUILD PATTERN" +query_manifest rebuildPatterns $REPOSITORY CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "Content hash tag: cache-$CONTENT_HASH-$SUCCESS_TAG" diff --git a/build-system/scripts/spot_run_test_script b/build-system/scripts/spot_run_test_script index 06d4d4553805..1ae08636be86 100755 --- a/build-system/scripts/spot_run_test_script +++ b/build-system/scripts/spot_run_test_script @@ -12,5 +12,7 @@ mkdir -p /tmp/test-logs set -o pipefail CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "REBUILD PATTERN" +query_manifest rebuildPatterns $REPOSITORY echo "Content hash: $CONTENT_HASH" spot_run_script $CONTENT_HASH 32 $SCRIPT_PATH $@ | tee "/tmp/test-logs/$JOB_NAME.log" diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index 26b3ba33c0df..8d577c2782b9 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -25,6 +25,8 @@ SCRIPT_TO_RUN=$1 shift CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "REBUILD PATTERN" +query_manifest rebuildPatterns $REPOSITORY echo "Content hash tag: cache-$CONTENT_HASH-$SUCCESS_TAG" echo "Script to run is $SCRIPT_TO_RUN $@" From 5e517625a7ef130f8c9da49bd0adf01fc40cb829 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 09:21:18 -0400 Subject: [PATCH 10/26] Revert acir test --- build-system/scripts/calculate_content_hash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 0ecc960521e6..73ab4d0ef8cd 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -14,4 +14,4 @@ AWK_PATTERN=$(echo $REBUILD_PATTERNS | sed 's/ /|/g') # this format is beneficial as it grabs the hashes from git efficiently # we will next filter by our rebuild patterns # then we pipe the hash portion of each file to git hash-object to produce our content hash -git ls-tree -r HEAD | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin +git ls-tree -r $COMMIT_HASH | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin From f08b783b88b82af7146532360bbf28aa5a2c23b6 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 09:23:16 -0400 Subject: [PATCH 11/26] Revert acir test --- build-system/scripts/calculate_content_hash | 1 + 1 file changed, 1 insertion(+) diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index 73ab4d0ef8cd..fcf79d121059 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -9,6 +9,7 @@ COMMIT_HASH=${2:-$COMMIT_HASH} REBUILD_PATTERNS=$(query_manifest rebuildPatterns $REPOSITORY) AWK_PATTERN=$(echo $REBUILD_PATTERNS | sed 's/ /|/g') +cd "$(git rev-parse --show-toplevel)" # an example line is # 100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0 yarn-project/world-state/tsconfig.json # this format is beneficial as it grabs the hashes from git efficiently From 606d4a88f455feb3abba9f244d4a5cfb9ad8eba4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 09:26:35 -0400 Subject: [PATCH 12/26] Revert --- build-system/scripts/build | 2 -- build-system/scripts/cond_spot_run_build | 2 -- build-system/scripts/cond_spot_run_script | 2 -- build-system/scripts/spot_run_test_script | 2 -- yarn-project/end-to-end/scripts/cond_run_script | 2 -- 5 files changed, 10 deletions(-) diff --git a/build-system/scripts/build b/build-system/scripts/build index 20463a145e29..2f59649e2738 100755 --- a/build-system/scripts/build +++ b/build-system/scripts/build @@ -47,8 +47,6 @@ function fetch_image() { # Ensure ECR repository exists. retry_10 ensure_repo $REPOSITORY $ECR_REGION refresh_lifecycle -echo "REBUILD PATTERN" -query_manifest rebuildPatterns $REPOSITORY CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "Content hash: $CONTENT_HASH" diff --git a/build-system/scripts/cond_spot_run_build b/build-system/scripts/cond_spot_run_build index 3cb11117cba2..7ade4de52bde 100755 --- a/build-system/scripts/cond_spot_run_build +++ b/build-system/scripts/cond_spot_run_build @@ -8,8 +8,6 @@ SPEC=$1 shift DOCKERFILE=$(query_manifest dockerfile $REPOSITORY) -echo "REBUILD PATTERN" -query_manifest rebuildPatterns $REPOSITORY CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "Content hash tag: cache-$CONTENT_HASH" diff --git a/build-system/scripts/cond_spot_run_script b/build-system/scripts/cond_spot_run_script index 74467454d447..d9e9aafb13c0 100755 --- a/build-system/scripts/cond_spot_run_script +++ b/build-system/scripts/cond_spot_run_script @@ -21,8 +21,6 @@ shift SUCCESS_TAG=$1 shift -echo "REBUILD PATTERN" -query_manifest rebuildPatterns $REPOSITORY CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "Content hash tag: cache-$CONTENT_HASH-$SUCCESS_TAG" diff --git a/build-system/scripts/spot_run_test_script b/build-system/scripts/spot_run_test_script index 1ae08636be86..06d4d4553805 100755 --- a/build-system/scripts/spot_run_test_script +++ b/build-system/scripts/spot_run_test_script @@ -12,7 +12,5 @@ mkdir -p /tmp/test-logs set -o pipefail CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "REBUILD PATTERN" -query_manifest rebuildPatterns $REPOSITORY echo "Content hash: $CONTENT_HASH" spot_run_script $CONTENT_HASH 32 $SCRIPT_PATH $@ | tee "/tmp/test-logs/$JOB_NAME.log" diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index 8d577c2782b9..26b3ba33c0df 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -25,8 +25,6 @@ SCRIPT_TO_RUN=$1 shift CONTENT_HASH=$(calculate_content_hash $REPOSITORY) -echo "REBUILD PATTERN" -query_manifest rebuildPatterns $REPOSITORY echo "Content hash tag: cache-$CONTENT_HASH-$SUCCESS_TAG" echo "Script to run is $SCRIPT_TO_RUN $@" From 810261988d90a57b32875abba1f4cb714f831bb8 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 09:27:09 -0400 Subject: [PATCH 13/26] Revert --- circuits/cpp/barretenberg/ts/src/bigint-array/index.ts | 1 - circuits/cpp/barretenberg/ts/src/serialize/index.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/circuits/cpp/barretenberg/ts/src/bigint-array/index.ts b/circuits/cpp/barretenberg/ts/src/bigint-array/index.ts index 363af053f2b4..c315c0758d5a 100644 --- a/circuits/cpp/barretenberg/ts/src/bigint-array/index.ts +++ b/circuits/cpp/barretenberg/ts/src/bigint-array/index.ts @@ -10,7 +10,6 @@ export function toBigIntBE(bytes: Uint8Array) { return bigint; } -// test export function toBufferBE(value: bigint, byteLength = 32) { const bytes = new Uint8Array(byteLength); const view = new DataView(bytes.buffer); diff --git a/circuits/cpp/barretenberg/ts/src/serialize/index.ts b/circuits/cpp/barretenberg/ts/src/serialize/index.ts index 45e9872de66f..bdd49908abec 100644 --- a/circuits/cpp/barretenberg/ts/src/serialize/index.ts +++ b/circuits/cpp/barretenberg/ts/src/serialize/index.ts @@ -1,4 +1,3 @@ export * from './buffer_reader.js'; export * from './output_type.js'; export * from './serialize.js'; -// test From 688441cd02c238212620952e2c427c2af7f1c092 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 10:01:59 -0400 Subject: [PATCH 14/26] See if git in end-to-end --- yarn-project/end-to-end/scripts/run_tests | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests index fad024b1eb2d..abbc6cdf7380 100755 --- a/yarn-project/end-to-end/scripts/run_tests +++ b/yarn-project/end-to-end/scripts/run_tests @@ -8,6 +8,7 @@ export COMPOSE_FILE=${2:-docker-compose.yml} $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null +git show for REPO in aztec.js end-to-end aztec-sandbox; do docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest From f83b07ef80c33fb6e03bd162f8888caa0bc4b65f Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 10:10:39 -0400 Subject: [PATCH 15/26] See if git in end-to-end --- yarn-project/end-to-end/scripts/run_tests | 1 - 1 file changed, 1 deletion(-) diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests index abbc6cdf7380..fad024b1eb2d 100755 --- a/yarn-project/end-to-end/scripts/run_tests +++ b/yarn-project/end-to-end/scripts/run_tests @@ -8,7 +8,6 @@ export COMPOSE_FILE=${2:-docker-compose.yml} $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -git show for REPO in aztec.js end-to-end aztec-sandbox; do docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest From 0784b61fb0ed3d7b595ea7912b4eada621b02ccb Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 10:23:28 -0400 Subject: [PATCH 16/26] See if git in end-to-end --- build-system/scripts/remote_run_script | 1 + yarn-project/end-to-end/scripts/run_tests | 1 + 2 files changed, 2 insertions(+) diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index c33d386f1f90..f04f7af83b1c 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -12,6 +12,7 @@ SCRIPT_NAME=$(basename $FULL_PATH) # Copy all files in script directory to spot instance. scp -F $SSH_CONFIG_PATH $DIR_NAME/* $IP:. +scp -rF $SSH_CONFIG_PATH $BUILD_SYSTEM_PATH $BUILD_SYSTEM_PATH/../.git $IP:. # Run script on remote instance, passing environment variables. ssh -A -F $SSH_CONFIG_PATH $IP "CONTENT_HASH=$CONTENT_HASH COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ECR_DEPLOY_URL=$ECR_DEPLOY_URL ECR_URL=$ECR_URL ./$SCRIPT_NAME $@" diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests index fad024b1eb2d..abbc6cdf7380 100755 --- a/yarn-project/end-to-end/scripts/run_tests +++ b/yarn-project/end-to-end/scripts/run_tests @@ -8,6 +8,7 @@ export COMPOSE_FILE=${2:-docker-compose.yml} $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null +git show for REPO in aztec.js end-to-end aztec-sandbox; do docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest From e9e42db88467590eae973145c03068def073333c Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 10:32:50 -0400 Subject: [PATCH 17/26] See if git in end-to-end --- build-system/scripts/remote_run_script | 7 +++---- yarn-project/end-to-end/scripts/cond_run_script | 2 +- yarn-project/end-to-end/scripts/run_tests | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index f04f7af83b1c..9a9b9c547ed2 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -2,9 +2,8 @@ set -eu IP=$1 -CONTENT_HASH=$2 -FULL_PATH=$3 -shift 3 +FULL_PATH=$2 +shift 2 SSH_CONFIG_PATH=${SSH_CONFIG_PATH:-$BUILD_SYSTEM_PATH/remote/ssh_config} DIR_NAME=$(dirname $FULL_PATH) @@ -15,4 +14,4 @@ scp -F $SSH_CONFIG_PATH $DIR_NAME/* $IP:. scp -rF $SSH_CONFIG_PATH $BUILD_SYSTEM_PATH $BUILD_SYSTEM_PATH/../.git $IP:. # Run script on remote instance, passing environment variables. -ssh -A -F $SSH_CONFIG_PATH $IP "CONTENT_HASH=$CONTENT_HASH COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ECR_DEPLOY_URL=$ECR_DEPLOY_URL ECR_URL=$ECR_URL ./$SCRIPT_NAME $@" +ssh -A -F $SSH_CONFIG_PATH $IP "COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ECR_DEPLOY_URL=$ECR_DEPLOY_URL ECR_URL=$ECR_URL ./$SCRIPT_NAME $@" diff --git a/yarn-project/end-to-end/scripts/cond_run_script b/yarn-project/end-to-end/scripts/cond_run_script index 26b3ba33c0df..88bf7d5c4b6d 100755 --- a/yarn-project/end-to-end/scripts/cond_run_script +++ b/yarn-project/end-to-end/scripts/cond_run_script @@ -29,6 +29,6 @@ echo "Content hash tag: cache-$CONTENT_HASH-$SUCCESS_TAG" echo "Script to run is $SCRIPT_TO_RUN $@" if ! check_rebuild cache-$CONTENT_HASH-$SUCCESS_TAG $REPOSITORY; then - CONTENT_HASH="$CONTENT_HASH" "$SCRIPT_TO_RUN" "$@" + "$SCRIPT_TO_RUN" "$@" tag_remote_image $REPOSITORY cache-$CONTENT_HASH cache-$CONTENT_HASH-$SUCCESS_TAG fi diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests index abbc6cdf7380..fad024b1eb2d 100755 --- a/yarn-project/end-to-end/scripts/run_tests +++ b/yarn-project/end-to-end/scripts/run_tests @@ -8,7 +8,6 @@ export COMPOSE_FILE=${2:-docker-compose.yml} $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -git show for REPO in aztec.js end-to-end aztec-sandbox; do docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest From 75ed1370c1ec9d5e6a4761c4fda64a7255b2ebd5 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 10:40:28 -0400 Subject: [PATCH 18/26] See if git in end-to-end --- build-system/scripts/spot_run_script | 2 +- yarn-project/end-to-end/scripts/run_tests | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/build-system/scripts/spot_run_script b/build-system/scripts/spot_run_script index 63f16928c857..ddf4acebc43e 100755 --- a/build-system/scripts/spot_run_script +++ b/build-system/scripts/spot_run_script @@ -23,7 +23,7 @@ IP=$(request_spot $CONTENT_HASH:$JOB_NAME $SPEC) # Run script remotely on spot instance, capturing success or failure. set +e -remote_run_script $IP $CONTENT_HASH $@ +remote_run_script $IP $@ CODE=$? # Shutdown spot. diff --git a/yarn-project/end-to-end/scripts/run_tests b/yarn-project/end-to-end/scripts/run_tests index fad024b1eb2d..a41849e576fb 100755 --- a/yarn-project/end-to-end/scripts/run_tests +++ b/yarn-project/end-to-end/scripts/run_tests @@ -8,9 +8,11 @@ export COMPOSE_FILE=${2:-docker-compose.yml} $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null +BS=$(git rev-parse --show-toplevel)/build-system/scripts + for REPO in aztec.js end-to-end aztec-sandbox; do - docker pull 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH - docker tag 278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPO:cache-$CONTENT_HASH aztecprotocol/$REPO:latest + docker pull $($BS/calculate_image_uri $REPO) + docker tag $($BS/calculate_image_uri $REPO) aztecprotocol/$REPO:latest done docker-compose rm -f From f3b4b85749a868312f1cdeafb9c5208b2961cbbf Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 10:44:18 -0400 Subject: [PATCH 19/26] Try to enable content hashing throughout --- circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh | 2 +- circuits/cpp/barretenberg/cpp/scripts/bin-test.sh | 3 ++- circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests | 2 +- circuits/cpp/barretenberg/cpp/scripts/run_tests | 3 +-- circuits/cpp/scripts/run_tests | 2 +- yarn-project/canary/scripts/run_tests | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh index 235aa111ff4c..4037b3ad365f 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bb-tests.sh @@ -6,7 +6,7 @@ set -eu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=barretenberg-x86_64-linux-clang-assert # use the image rebuild patterns to compute a content hash, use this to get a URI -IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) docker pull $IMAGE_URI TESTS=( diff --git a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh index b06c26036bac..6370dbebb5f4 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh +++ b/circuits/cpp/barretenberg/cpp/scripts/bin-test.sh @@ -4,7 +4,8 @@ set -eu $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY=barretenberg-x86_64-linux-clang-assert -IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) + docker pull $IMAGE_URI docker run --rm -t $IMAGE_URI /bin/sh -c "\ diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests index d67176f098fa..bab5c7f22073 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_aztec_circuits_tests @@ -20,7 +20,7 @@ shift # to aztec's circuits `run_tests_local` $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY="barretenberg-circuits-${ARCH}-linux-clang-builder-runner" -IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) docker pull $IMAGE_URI if [ "$ARCH" != "wasm" ]; then diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_tests b/circuits/cpp/barretenberg/cpp/scripts/run_tests index 965307ff5a59..2c272f96c9d5 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_tests @@ -13,8 +13,7 @@ shift $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null # use the image rebuild patterns to compute a content hash, use this to get a URI -REPOSITORY=barretenberg-x86_64-linux-clang-assert -IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) docker pull $IMAGE_URI if [ -f "$TESTS" ]; then diff --git a/circuits/cpp/scripts/run_tests b/circuits/cpp/scripts/run_tests index b3fbc43f4592..b92241ad44ef 100755 --- a/circuits/cpp/scripts/run_tests +++ b/circuits/cpp/scripts/run_tests @@ -26,7 +26,7 @@ shift # arg1 (num transcripts) is not forwarded to `run_tests_local` $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null REPOSITORY="circuits-$ARCH-linux-clang-assert" -IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) docker pull $IMAGE_URI # run tests in docker image diff --git a/yarn-project/canary/scripts/run_tests b/yarn-project/canary/scripts/run_tests index dac0f0bd921e..44c0b2e9dcac 100755 --- a/yarn-project/canary/scripts/run_tests +++ b/yarn-project/canary/scripts/run_tests @@ -14,7 +14,7 @@ fi $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -IMAGE_URI="278380418400.dkr.ecr.us-east-2.amazonaws.com/$IMAGE:cache-$CONTENT_HASH" +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $IMAGE) docker pull $IMAGE_URI docker tag $IMAGE_URI aztecprotocol/canary:latest From b40378869ec07293caca81e7164fce0a41538ee6 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 10:49:23 -0400 Subject: [PATCH 20/26] Try to enable content hashing throughout --- build-system/scripts/calculate_image_uri | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100755 build-system/scripts/calculate_image_uri diff --git a/build-system/scripts/calculate_image_uri b/build-system/scripts/calculate_image_uri new file mode 100755 index 000000000000..9d740987fc36 --- /dev/null +++ b/build-system/scripts/calculate_image_uri @@ -0,0 +1,8 @@ +!/bin/bash +set -eu + +REPOSITORY=$1 +# Assume we might be calling this directly without build system in path +export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" +CONTENT_HASH=$(calculate_content_hash $REPOSITORY) +echo "278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" From 5c8736cfed158b59557c6b4306e1da577f3c4349 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 10:52:00 -0400 Subject: [PATCH 21/26] Try to enable content hashing throughout --- circuits/cpp/barretenberg/cpp/scripts/run_tests | 1 + 1 file changed, 1 insertion(+) diff --git a/circuits/cpp/barretenberg/cpp/scripts/run_tests b/circuits/cpp/barretenberg/cpp/scripts/run_tests index 2c272f96c9d5..f6ccd5a43594 100755 --- a/circuits/cpp/barretenberg/cpp/scripts/run_tests +++ b/circuits/cpp/barretenberg/cpp/scripts/run_tests @@ -13,6 +13,7 @@ shift $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null # use the image rebuild patterns to compute a content hash, use this to get a URI +REPOSITORY=barretenberg-x86_64-linux-clang-assert IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) docker pull $IMAGE_URI From 1a37814a491f66ab3a53bcb6f39c579f83b0c220 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 10:55:18 -0400 Subject: [PATCH 22/26] Try to enable content hashing throughout --- circuits/cpp/barretenberg/ts/scripts/run_tests | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/circuits/cpp/barretenberg/ts/scripts/run_tests b/circuits/cpp/barretenberg/ts/scripts/run_tests index cd417d0724a7..7a97ff6adefc 100755 --- a/circuits/cpp/barretenberg/ts/scripts/run_tests +++ b/circuits/cpp/barretenberg/ts/scripts/run_tests @@ -2,5 +2,7 @@ set -e $(aws ecr get-login --region us-east-2 --no-include-email) 2> /dev/null -IMAGE_URI=278380418400.dkr.ecr.us-east-2.amazonaws.com/bb.js:cache-$CONTENT_HASH + +REPOSITORY=bb.js +IMAGE_URI=$($(git rev-parse --show-toplevel)/build-system/scripts/calculate_image_uri $REPOSITORY) docker run --rm $IMAGE_URI From 44f83435a9286546f2443dcfe35d89959f8c3bfe Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 10:58:31 -0400 Subject: [PATCH 23/26] Try to enable content hashing throughout --- build-system/scripts/calculate_image_uri | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build-system/scripts/calculate_image_uri b/build-system/scripts/calculate_image_uri index 9d740987fc36..b4651a66dc59 100755 --- a/build-system/scripts/calculate_image_uri +++ b/build-system/scripts/calculate_image_uri @@ -1,8 +1,9 @@ -!/bin/bash +#!/bin/bash set -eu REPOSITORY=$1 # Assume we might be calling this directly without build system in path -export PATH="$PATH:$(git rev-parse --show-toplevel)/build-system/scripts" +export ROOT_PATH=$(git rev-parse --show-toplevel) +export PATH="$PATH:$ROOT_PATH/build-system/scripts" CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" From 810b26b8e3365d74996382085eaf5da8129d0ed7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 11:00:33 -0400 Subject: [PATCH 24/26] Try to enable content hashing throughout --- build-system/scripts/calculate_image_uri | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/calculate_image_uri b/build-system/scripts/calculate_image_uri index b4651a66dc59..a2e238fad8a4 100755 --- a/build-system/scripts/calculate_image_uri +++ b/build-system/scripts/calculate_image_uri @@ -4,6 +4,6 @@ set -eu REPOSITORY=$1 # Assume we might be calling this directly without build system in path export ROOT_PATH=$(git rev-parse --show-toplevel) -export PATH="$PATH:$ROOT_PATH/build-system/scripts" +export PATH="$PATH:$ROOT_PATH/build-system/scripts:$ROOT_PATH/build-system/bin" CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" From d9906da02719e6ca0b42d8b26b79d99685b03d84 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 11:06:19 -0400 Subject: [PATCH 25/26] Try to enable content hashing throughout --- build-system/scripts/calculate_image_uri | 1 + 1 file changed, 1 insertion(+) diff --git a/build-system/scripts/calculate_image_uri b/build-system/scripts/calculate_image_uri index a2e238fad8a4..42bedecba2bf 100755 --- a/build-system/scripts/calculate_image_uri +++ b/build-system/scripts/calculate_image_uri @@ -5,5 +5,6 @@ REPOSITORY=$1 # Assume we might be calling this directly without build system in path export ROOT_PATH=$(git rev-parse --show-toplevel) export PATH="$PATH:$ROOT_PATH/build-system/scripts:$ROOT_PATH/build-system/bin" +export LD_LIBRARY_PATH="$ROOT_PATH/build-system/lib" CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH" From 143771a679859484691d8e2f0048ae4e3001752c Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 4 Sep 2023 11:08:24 -0400 Subject: [PATCH 26/26] Try to enable content hashing throughout --- build-system/scripts/remote_run_script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/remote_run_script b/build-system/scripts/remote_run_script index 9a9b9c547ed2..0e6417da1565 100755 --- a/build-system/scripts/remote_run_script +++ b/build-system/scripts/remote_run_script @@ -11,7 +11,7 @@ SCRIPT_NAME=$(basename $FULL_PATH) # Copy all files in script directory to spot instance. scp -F $SSH_CONFIG_PATH $DIR_NAME/* $IP:. -scp -rF $SSH_CONFIG_PATH $BUILD_SYSTEM_PATH $BUILD_SYSTEM_PATH/../.git $IP:. +scp -rF $SSH_CONFIG_PATH $BUILD_SYSTEM_PATH $BUILD_SYSTEM_PATH/../.git $BUILD_SYSTEM_PATH/../build_manifest.json $IP:. # Run script on remote instance, passing environment variables. ssh -A -F $SSH_CONFIG_PATH $IP "COMMIT_HASH=$COMMIT_HASH COMMIT_TAG=$COMMIT_TAG JOB_NAME=$JOB_NAME GIT_REPOSITORY_URL=$GIT_REPOSITORY_URL DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD ECR_DEPLOY_URL=$ECR_DEPLOY_URL ECR_URL=$ECR_URL ./$SCRIPT_NAME $@"