From ed4b2b867b3d455c4bc5b63ba6666f662a16dc56 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 17 Mar 2025 20:54:48 -0400 Subject: [PATCH] Revert "fix: Switch to `noir-repo` context for cache content hashing (#12784)" This reverts commit 6214c8c45707019cafa467a96ef7ee730010458c. --- CI.md | 2 +- avm-transpiler/bootstrap.sh | 2 +- boxes/bootstrap.sh | 11 +++---- ci3/cache_content_hash | 8 ++--- noir-projects/aztec-nr/bootstrap.sh | 2 +- noir-projects/bootstrap.sh | 2 +- noir-projects/noir-contracts/bootstrap.sh | 13 ++++---- .../noir-protocol-circuits/bootstrap.sh | 6 ++-- noir/.noir-repo.rebuild_patterns | 21 ------------- noir/.noir-repo.rebuild_patterns_tests | 2 -- noir/.rebuild_patterns | 28 ++++++++++++++--- noir/.rebuild_patterns_tests | 2 ++ noir/bootstrap.sh | 30 +++--------------- noir/noir-repo.patch | 31 +++---------------- yarn-project/bootstrap.sh | 9 +++--- 15 files changed, 59 insertions(+), 110 deletions(-) delete mode 100644 noir/.noir-repo.rebuild_patterns delete mode 100644 noir/.noir-repo.rebuild_patterns_tests create mode 100644 noir/.rebuild_patterns_tests diff --git a/CI.md b/CI.md index 5e1626a249bc..290b16759f42 100644 --- a/CI.md +++ b/CI.md @@ -297,7 +297,7 @@ All projects have at least a "build hash". This is computed using `cache_content Some projects will also have a "test hash". The test hash is part of the input to deciding if a test should be re-run. So this might also include files that don't make up the build hash, but are used as part of testing. -To give a concrete example, take `barretenberg/acir_tests`. Here we have a build hash that consists of what makes up `nargo` (`../../noir/.rebuild_patterns` and `../noir/.noir-repo.rebuild_patterns`, but do make use of `../../noir/bootstrap.sh hash` to compute it correctly), and the test programs themselves (`../../noir/.noir-repo.rebuild_patterns_tests`) as they are actually compiled using nargo with the results stored in the build cache. The "test hash" then additionally adds barretenbergs cpp and ts code, because both are used in the actual _running_ of the tests. +To give a concrete example, take `barretenberg/acir_tests`. Here we have a build hash that consists of what makes up `nargo` (`./../noir/.rebuild_patterns`), and the test programs themselves (`../../noir/.rebuild_patterns_tests`) as they are actually compiled using nargo with the results stored in the build cache. The "test hash" then additionally adds barretenbergs cpp and ts code, because both are used in the actual _running_ of the tests. If a test successfully runs in CI, it won't be run again unless its redis key changes. This key consists of the "test hash" and the "test command". Here's an example: diff --git a/avm-transpiler/bootstrap.sh b/avm-transpiler/bootstrap.sh index 459ffb5bb696..474f97eb35e4 100755 --- a/avm-transpiler/bootstrap.sh +++ b/avm-transpiler/bootstrap.sh @@ -4,7 +4,7 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap cmd=${1:-} -hash=$(hash_str $(../noir/bootstrap.sh hash) $(cache_content_hash .rebuild_patterns)) +hash=$(cache_content_hash ../noir/.rebuild_patterns .rebuild_patterns) export GIT_COMMIT="0000000000000000000000000000000000000000" export SOURCE_DATE_EPOCH=0 diff --git a/boxes/bootstrap.sh b/boxes/bootstrap.sh index 8318d699ab59..3b392af300c5 100755 --- a/boxes/bootstrap.sh +++ b/boxes/bootstrap.sh @@ -9,12 +9,11 @@ export NARGO=$PWD/../noir/noir-repo/target/release/nargo export AZTEC_NARGO=$PWD/../aztec-nargo/compile_then_postprocess.sh export AZTEC_BUILDER=$PWD/../yarn-project/builder/aztec-builder-dest -hash=$(hash_str \ - $(../noir/bootstrap.sh hash) \ - $(cache_content_hash \ - .rebuild_patterns \ - ../{avm-transpiler,noir-projects,l1-contracts,yarn-project}/.rebuild_patterns \ - ../barretenberg/*/.rebuild_patterns)) +hash=$(cache_content_hash \ + .rebuild_patterns \ + ../noir/.rebuild_patterns \ + ../{avm-transpiler,noir-projects,l1-contracts,yarn-project}/.rebuild_patterns \ + ../barretenberg/*/.rebuild_patterns) function build { echo_header "boxes build" diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index 70086bd15183..479dd0984a0b 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -1,8 +1,6 @@ #!/usr/bin/env bash # Calculate the content hash for an artifact that controls rebuilds. # Takes a list of rebuild pattern files (regex files that match patterns from repo root) or directories. -# The REPO_PATH env var can be used to point the script at the root of the git repository where the patterns -# are rooted, and where the git commands will be executed, which can be used to compute the hash in a sub repo. # Two special test-only environment variables can control the cache. # These are used when iterating on build infrastructure: # - AZTEC_CACHE_NO_SCRIPTS: Don't include any shell files (namely bootstrap.sh) in cache patterns. @@ -27,17 +25,17 @@ for arg in "$@"; do fi done +GREP_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$') # For sanity when iterating on build scripts as bootstrap.sh scripts are in every .rebuild_patterns. if [ -n "${AZTEC_CACHE_NO_SCRIPTS:-}" ]; then - rebuild_patterns=$(echo "$rebuild_patterns" | grep -v '.sh') + rebuild_patterns=$(echo "$rebuild_patterns" | grep -v '*\.sh') fi -GREP_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$') # Concatenate patterns with '|' and double escape backslashes for AWK filter empty lines AWK_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') # use git repo root because that is where our patterns are focused -cd "${REPO_PATH:-$(git rev-parse --show-toplevel)}" +cd "$(git rev-parse --show-toplevel)" diff="$({ git diff --name-only diff --git a/noir-projects/aztec-nr/bootstrap.sh b/noir-projects/aztec-nr/bootstrap.sh index b0a7df5608c9..dd3d0891c766 100755 --- a/noir-projects/aztec-nr/bootstrap.sh +++ b/noir-projects/aztec-nr/bootstrap.sh @@ -6,7 +6,7 @@ cmd=${1:-} export RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-16} export HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} export NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo} -hash=$(hash_str $(../../noir/bootstrap.sh hash) $(cache_content_hash "^noir-projects/aztec-nr")) +hash=$(cache_content_hash ../../noir/.rebuild_patterns "^noir-projects/aztec-nr") function build { # Being a library, aztec-nr does not technically need to be built. But we can still run nargo check to find any type diff --git a/noir-projects/bootstrap.sh b/noir-projects/bootstrap.sh index 11954d84748d..e7e245687b2f 100755 --- a/noir-projects/bootstrap.sh +++ b/noir-projects/bootstrap.sh @@ -44,7 +44,7 @@ case "$cmd" in $cmd ;; "hash") - hash_str $(../noir/bootstrap.sh hash) $(cache_content_hash .rebuild_patterns) + cache_content_hash .rebuild_patterns ../noir/.rebuild_patterns ;; *) echo_stderr "Unknown command: $cmd" diff --git a/noir-projects/noir-contracts/bootstrap.sh b/noir-projects/noir-contracts/bootstrap.sh index 6ea04a01fad2..1ae4dd733e6e 100755 --- a/noir-projects/noir-contracts/bootstrap.sh +++ b/noir-projects/noir-contracts/bootstrap.sh @@ -95,13 +95,12 @@ export -f process_function # Compute hash for a given contract. function get_contract_hash { - hash_str \ - $(../../noir/bootstrap.sh hash) \ - $(cache_content_hash \ - ../../avm-transpiler/.rebuild_patterns \ - "^noir-projects/noir-contracts/contracts/$1/" \ - "^noir-projects/aztec-nr/" \ - "^noir-projects/noir-protocol-circuits/crates/types/") + cache_content_hash \ + ../../noir/.rebuild_patterns \ + ../../avm-transpiler/.rebuild_patterns \ + "^noir-projects/noir-contracts/contracts/$1/" \ + "^noir-projects/aztec-nr/" \ + "^noir-projects/noir-protocol-circuits/crates/types/" } export -f get_contract_hash diff --git a/noir-projects/noir-protocol-circuits/bootstrap.sh b/noir-projects/noir-protocol-circuits/bootstrap.sh index 601113d3620b..3c0383e11ccb 100755 --- a/noir-projects/noir-protocol-circuits/bootstrap.sh +++ b/noir-projects/noir-protocol-circuits/bootstrap.sh @@ -4,6 +4,7 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap cmd=${1:-} project_name=$(basename "$PWD") +test_flag=$project_name-tests-$(cache_content_hash ../../noir/.rebuild_patterns "^noir-projects/$project_name") export RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-16} export HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} @@ -12,16 +13,15 @@ export PLATFORM_TAG=any export BB=${BB:-../../barretenberg/cpp/build/bin/bb} export NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo} export BB_HASH=$(cache_content_hash ../../barretenberg/cpp/.rebuild_patterns) -export NARGO_HASH=$(../../noir/bootstrap.sh hash) +export NARGO_HASH=$(cache_content_hash ../../noir/.rebuild_patterns) -test_flag=$project_name-tests-$(hash_str "$NARGO_HASH" $(cache_content_hash "^noir-projects/$project_name")) key_dir=./target/keys mkdir -p $key_dir # Hash of the entire protocol circuits. # Needed for test hash, as we presently don't have a program hash for each individual test. # Means if anything within the dir changes, the tests will rerun. -circuits_hash=$(hash_str "$NARGO_HASH" $(cache_content_hash "^noir-projects/$project_name/crates/")) +circuits_hash=$(cache_content_hash "^noir-projects/$project_name/crates/" ../../noir/.rebuild_patterns) # Circuits matching these patterns we have client-ivc keys computed, rather than ultra-honk. ivc_patterns=( diff --git a/noir/.noir-repo.rebuild_patterns b/noir/.noir-repo.rebuild_patterns deleted file mode 100644 index 1312b1dc71d1..000000000000 --- a/noir/.noir-repo.rebuild_patterns +++ /dev/null @@ -1,21 +0,0 @@ -^.yarn -^.yarnrc.yml -^acvm-repo -^compiler -^noir_stdlib -^package.json -^tooling/acvm_cli -^tooling/backend_interface -^tooling/bb_abstraction_leaks -^tooling/debugger -^tooling/lsp -^tooling/nargo -^tooling/nargo_cli -^tooling/nargo_fmt -^tooling/nargo_toml -^tooling/noir_codegen -^tooling/noir_js -^tooling/noir_js_types -^tooling/noirc_abi -^tooling/noirc_abi_wasm -^yarn.lock diff --git a/noir/.noir-repo.rebuild_patterns_tests b/noir/.noir-repo.rebuild_patterns_tests deleted file mode 100644 index 2d4698240f10..000000000000 --- a/noir/.noir-repo.rebuild_patterns_tests +++ /dev/null @@ -1,2 +0,0 @@ -^test_programs/execution_success/ -^examples/ diff --git a/noir/.rebuild_patterns b/noir/.rebuild_patterns index cf9fae813598..f1e5db6c2029 100644 --- a/noir/.rebuild_patterns +++ b/noir/.rebuild_patterns @@ -1,8 +1,28 @@ -^noir/bootstrap.sh ^noir/Dockerfile -^noir/Dockerfile.packages -^noir/noir-repo.patch ^noir/scripts/bootstrap_native.sh +^noir/scripts/test_native.sh +^noir/noir-repo/acvm-repo +^noir/noir-repo/compiler +^noir/noir-repo/noir_stdlib +^noir/noir-repo/tooling/backend_interface +^noir/noir-repo/tooling/bb_abstraction_leaks +^noir/noir-repo/tooling/debugger +^noir/noir-repo/tooling/lsp +^noir/noir-repo/tooling/nargo +^noir/noir-repo/tooling/nargo_cli +^noir/noir-repo/tooling/nargo_toml +^noir/noir-repo/tooling/nargo_fmt +^noir/noir-repo/tooling/noirc_abi +^noir/noir-repo/tooling/acvm_cli +^noir/Dockerfile.packages ^noir/scripts/bootstrap_packages.sh ^noir/scripts/test_js_packages.sh -^noir/scripts/test_native.sh +^noir/noir-repo/.yarn +^noir/noir-repo/.yarnrc.yml +^noir/noir-repo/package.json +^noir/noir-repo/yarn.lock +^noir/noir-repo/tooling/noir_codegen +^noir/noir-repo/tooling/noir_js +^noir/noir-repo/tooling/noir_js_types +^noir/noir-repo/tooling/noirc_abi_wasm +^noir/bootstrap.sh diff --git a/noir/.rebuild_patterns_tests b/noir/.rebuild_patterns_tests new file mode 100644 index 000000000000..395d58d1835a --- /dev/null +++ b/noir/.rebuild_patterns_tests @@ -0,0 +1,2 @@ +^noir/noir-repo/test_programs/execution_success/ +^noir/noir-repo/examples/ diff --git a/noir/bootstrap.sh b/noir/bootstrap.sh index 675273f9fa22..10129dad25a9 100755 --- a/noir/bootstrap.sh +++ b/noir/bootstrap.sh @@ -27,28 +27,10 @@ function noir_sync { denoise "scripts/sync.sh init && scripts/sync.sh update" } -# Calculate the content hash for caching, taking into account that `noir-repo` -# is not part of the `aztec-packages` repo itself, so the `git ls-tree` used -# by `cache_content_hash` would not take those files into account. -function noir_content_hash { - function noir_repo_content_hash { - echo $(REPO_PATH=./noir-repo cache_content_hash $@) - } - with_tests=${1:-0} - noir_hash=$(cache_content_hash .rebuild_patterns) - noir_repo_hash=$(noir_repo_content_hash .noir-repo.rebuild_patterns) - if [ "$with_tests" == "1" ]; then - noir_repo_hash_tests=$(noir_repo_content_hash .noir-repo.rebuild_patterns_tests) - else - noir_repo_hash_tests="" - fi - echo $(hash_str $noir_hash $noir_repo_hash $noir_repo_hash_tests) -} - # Builds nargo, acvm and profiler binaries. function build_native { set -euo pipefail - local hash=$(noir_content_hash) + local hash=$(cache_content_hash .rebuild_patterns) if cache_download noir-$hash.tar.gz; then return fi @@ -64,7 +46,7 @@ function build_native { # Builds js packages. function build_packages { set -euo pipefail - local hash=$(noir_content_hash) + local hash=$(cache_content_hash .rebuild_patterns) if cache_download noir-packages-$hash.tar.gz; then cd noir-repo @@ -74,7 +56,6 @@ function build_packages { cd noir-repo npm_install_deps - yarn workspaces foreach --parallel --topological-dev --verbose $js_include run build # We create a folder called packages, that contains each package as it would be published to npm, named correctly. @@ -132,7 +113,7 @@ function test { # Prints the commands to run tests, one line per test, prefixed with the appropriate content hash. function test_cmds { - local test_hash=$(noir_content_hash 1) + local test_hash=$(cache_content_hash .rebuild_patterns .rebuild_patterns_tests) cd noir-repo cargo nextest list --workspace --locked --release -Tjson-pretty 2>/dev/null | \ jq -r ' @@ -217,10 +198,7 @@ case "$cmd" in $cmd "$@" ;; "hash") - echo $(noir_content_hash) - ;; - "hash-tests") - echo $(noir_content_hash 1) + echo $(cache_content_hash .rebuild_patterns) ;; "make-patch") scripts/sync.sh make-patch diff --git a/noir/noir-repo.patch b/noir/noir-repo.patch index 673275115c3b..8024818e9b9f 100644 --- a/noir/noir-repo.patch +++ b/noir/noir-repo.patch @@ -1,7 +1,7 @@ -From b81862448ed79637e1e36cf2e78e70a60a55fdc1 Mon Sep 17 00:00:00 2001 +From f6c1dfe13a1993ab6b3e0136448bb3796625684a Mon Sep 17 00:00:00 2001 From: TomAFrench Date: Sat, 15 Mar 2025 15:36:12 +0000 -Subject: [PATCH 1/4] chore: turn on `skipLibCheck` +Subject: [PATCH 1/3] chore: turn on `skipLibCheck` --- tooling/noir_codegen/tsconfig.json | 1 + @@ -22,10 +22,10 @@ index 30dd2a7..a2712fd 100644 -- 2.43.0 -From 892ce5f3ec0ef5f1ec24f5cfc394e5e1d511f8b0 Mon Sep 17 00:00:00 2001 +From 65bce6930794adb2834ac5aba3479308159c08ba Mon Sep 17 00:00:00 2001 From: TomAFrench Date: Sat, 15 Mar 2025 18:32:24 +0000 -Subject: [PATCH 2/4] chore: delete honk test programs +Subject: [PATCH 3/3] chore: delete honk test programs --- .../double_verify_honk_proof/Nargo.toml | 6 ---- @@ -207,26 +207,3 @@ index e5f08ef..0000000 -- 2.43.0 -From fd8f444eba5db51bee2173d7c2f66bebaa693d30 Mon Sep 17 00:00:00 2001 -From: aakoshh -Date: Mon, 17 Mar 2025 12:10:58 +0000 -Subject: [PATCH 4/4] Ignore package.tgz - ---- - .gitignore | 3 +++ - 1 file changed, 3 insertions(+) - -diff --git a/.gitignore b/.gitignore -index 3349018..c93fe8e 100644 ---- a/.gitignore -+++ b/.gitignore -@@ -59,3 +59,6 @@ codegen - - mutants.out - mutants.out.old -+ -+# Artifacts created by `noir/bootstrap.sh build_packages` -+**/package.tgz --- -2.43.0 - diff --git a/yarn-project/bootstrap.sh b/yarn-project/bootstrap.sh index 1478ba5427a1..07ebff602f57 100755 --- a/yarn-project/bootstrap.sh +++ b/yarn-project/bootstrap.sh @@ -5,11 +5,10 @@ cmd=${1:-} [ -n "$cmd" ] && shift function hash { - hash_str \ - $(../noir/bootstrap.sh hash) \ - $(cache_content_hash \ - ../{avm-transpiler,noir-projects,l1-contracts,yarn-project}/.rebuild_patterns \ - ../barretenberg/*/.rebuild_patterns) + cache_content_hash \ + ../noir/.rebuild_patterns \ + ../{avm-transpiler,noir-projects,l1-contracts,yarn-project}/.rebuild_patterns \ + ../barretenberg/*/.rebuild_patterns } function compile_project {