diff --git a/CI.md b/CI.md index 290b16759f42..5e1626a249bc 100644 --- a/CI.md +++ b/CI.md @@ -297,7 +297,7 @@ All projects have at least a "build hash". This is computed using `cache_content Some projects will also have a "test hash". The test hash is part of the input to deciding if a test should be re-run. So this might also include files that don't make up the build hash, but are used as part of testing. -To give a concrete example, take `barretenberg/acir_tests`. Here we have a build hash that consists of what makes up `nargo` (`./../noir/.rebuild_patterns`), and the test programs themselves (`../../noir/.rebuild_patterns_tests`) as they are actually compiled using nargo with the results stored in the build cache. The "test hash" then additionally adds barretenbergs cpp and ts code, because both are used in the actual _running_ of the tests. +To give a concrete example, take `barretenberg/acir_tests`. Here we have a build hash that consists of what makes up `nargo` (`../../noir/.rebuild_patterns` and `../noir/.noir-repo.rebuild_patterns`, but do make use of `../../noir/bootstrap.sh hash` to compute it correctly), and the test programs themselves (`../../noir/.noir-repo.rebuild_patterns_tests`) as they are actually compiled using nargo with the results stored in the build cache. The "test hash" then additionally adds barretenbergs cpp and ts code, because both are used in the actual _running_ of the tests. If a test successfully runs in CI, it won't be run again unless its redis key changes. This key consists of the "test hash" and the "test command". Here's an example: diff --git a/avm-transpiler/bootstrap.sh b/avm-transpiler/bootstrap.sh index 474f97eb35e4..459ffb5bb696 100755 --- a/avm-transpiler/bootstrap.sh +++ b/avm-transpiler/bootstrap.sh @@ -4,7 +4,7 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap cmd=${1:-} -hash=$(cache_content_hash ../noir/.rebuild_patterns .rebuild_patterns) +hash=$(hash_str $(../noir/bootstrap.sh hash) $(cache_content_hash .rebuild_patterns)) export GIT_COMMIT="0000000000000000000000000000000000000000" export SOURCE_DATE_EPOCH=0 diff --git a/barretenberg/acir_tests/bootstrap.sh b/barretenberg/acir_tests/bootstrap.sh index 0e75889f02c6..e3787ba3487e 100755 --- a/barretenberg/acir_tests/bootstrap.sh +++ b/barretenberg/acir_tests/bootstrap.sh @@ -5,18 +5,18 @@ cmd=${1:-} export CRS_PATH=$HOME/.bb-crs export bb=$(realpath ../cpp/build/bin/bb) -tests_tar=barretenberg-acir-tests-$(cache_content_hash \ - ../../noir/.rebuild_patterns \ - ../../noir/.rebuild_patterns_tests \ +tests_tar=barretenberg-acir-tests-$(hash_str \ + $(../../noir/bootstrap.sh hash-tests) \ + $(cache_content_hash \ ../cpp/.rebuild_patterns \ - ).tar.gz + )).tar.gz -tests_hash=$(cache_content_hash \ +tests_hash=$(hash_str \ + $(../../noir/bootstrap.sh hash-tests) \ + $(cache_content_hash \ ^barretenberg/acir_tests/ \ - ../../noir/.rebuild_patterns \ - ../../noir/.rebuild_patterns_tests \ ../cpp/.rebuild_patterns \ - ../ts/.rebuild_patterns) + ../ts/.rebuild_patterns)) # Generate inputs for a given recursively verifying program. function run_proof_generation { diff --git a/boxes/bootstrap.sh b/boxes/bootstrap.sh index 3b392af300c5..8318d699ab59 100755 --- a/boxes/bootstrap.sh +++ b/boxes/bootstrap.sh @@ -9,11 +9,12 @@ export NARGO=$PWD/../noir/noir-repo/target/release/nargo export AZTEC_NARGO=$PWD/../aztec-nargo/compile_then_postprocess.sh export AZTEC_BUILDER=$PWD/../yarn-project/builder/aztec-builder-dest -hash=$(cache_content_hash \ - .rebuild_patterns \ - ../noir/.rebuild_patterns \ - ../{avm-transpiler,noir-projects,l1-contracts,yarn-project}/.rebuild_patterns \ - ../barretenberg/*/.rebuild_patterns) +hash=$(hash_str \ + $(../noir/bootstrap.sh hash) \ + $(cache_content_hash \ + .rebuild_patterns \ + ../{avm-transpiler,noir-projects,l1-contracts,yarn-project}/.rebuild_patterns \ + ../barretenberg/*/.rebuild_patterns)) function build { echo_header "boxes build" diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index 479dd0984a0b..70086bd15183 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -1,6 +1,8 @@ #!/usr/bin/env bash # Calculate the content hash for an artifact that controls rebuilds. # Takes a list of rebuild pattern files (regex files that match patterns from repo root) or directories. +# The REPO_PATH env var can be used to point the script at the root of the git repository where the patterns +# are rooted, and where the git commands will be executed, which can be used to compute the hash in a sub repo. # Two special test-only environment variables can control the cache. # These are used when iterating on build infrastructure: # - AZTEC_CACHE_NO_SCRIPTS: Don't include any shell files (namely bootstrap.sh) in cache patterns. @@ -25,17 +27,17 @@ for arg in "$@"; do fi done -GREP_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$') # For sanity when iterating on build scripts as bootstrap.sh scripts are in every .rebuild_patterns. if [ -n "${AZTEC_CACHE_NO_SCRIPTS:-}" ]; then - rebuild_patterns=$(echo "$rebuild_patterns" | grep -v '*\.sh') + rebuild_patterns=$(echo "$rebuild_patterns" | grep -v '.sh') fi +GREP_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$') # Concatenate patterns with '|' and double escape backslashes for AWK filter empty lines AWK_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') # use git repo root because that is where our patterns are focused -cd "$(git rev-parse --show-toplevel)" +cd "${REPO_PATH:-$(git rev-parse --show-toplevel)}" diff="$({ git diff --name-only diff --git a/noir-projects/aztec-nr/bootstrap.sh b/noir-projects/aztec-nr/bootstrap.sh index dd3d0891c766..b0a7df5608c9 100755 --- a/noir-projects/aztec-nr/bootstrap.sh +++ b/noir-projects/aztec-nr/bootstrap.sh @@ -6,7 +6,7 @@ cmd=${1:-} export RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-16} export HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} export NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo} -hash=$(cache_content_hash ../../noir/.rebuild_patterns "^noir-projects/aztec-nr") +hash=$(hash_str $(../../noir/bootstrap.sh hash) $(cache_content_hash "^noir-projects/aztec-nr")) function build { # Being a library, aztec-nr does not technically need to be built. But we can still run nargo check to find any type diff --git a/noir-projects/bootstrap.sh b/noir-projects/bootstrap.sh index e7e245687b2f..11954d84748d 100755 --- a/noir-projects/bootstrap.sh +++ b/noir-projects/bootstrap.sh @@ -44,7 +44,7 @@ case "$cmd" in $cmd ;; "hash") - cache_content_hash .rebuild_patterns ../noir/.rebuild_patterns + hash_str $(../noir/bootstrap.sh hash) $(cache_content_hash .rebuild_patterns) ;; *) echo_stderr "Unknown command: $cmd" diff --git a/noir-projects/noir-contracts/bootstrap.sh b/noir-projects/noir-contracts/bootstrap.sh index 1ae4dd733e6e..6ea04a01fad2 100755 --- a/noir-projects/noir-contracts/bootstrap.sh +++ b/noir-projects/noir-contracts/bootstrap.sh @@ -95,12 +95,13 @@ export -f process_function # Compute hash for a given contract. function get_contract_hash { - cache_content_hash \ - ../../noir/.rebuild_patterns \ - ../../avm-transpiler/.rebuild_patterns \ - "^noir-projects/noir-contracts/contracts/$1/" \ - "^noir-projects/aztec-nr/" \ - "^noir-projects/noir-protocol-circuits/crates/types/" + hash_str \ + $(../../noir/bootstrap.sh hash) \ + $(cache_content_hash \ + ../../avm-transpiler/.rebuild_patterns \ + "^noir-projects/noir-contracts/contracts/$1/" \ + "^noir-projects/aztec-nr/" \ + "^noir-projects/noir-protocol-circuits/crates/types/") } export -f get_contract_hash diff --git a/noir-projects/noir-protocol-circuits/bootstrap.sh b/noir-projects/noir-protocol-circuits/bootstrap.sh index 3c0383e11ccb..601113d3620b 100755 --- a/noir-projects/noir-protocol-circuits/bootstrap.sh +++ b/noir-projects/noir-protocol-circuits/bootstrap.sh @@ -4,7 +4,6 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap cmd=${1:-} project_name=$(basename "$PWD") -test_flag=$project_name-tests-$(cache_content_hash ../../noir/.rebuild_patterns "^noir-projects/$project_name") export RAYON_NUM_THREADS=${RAYON_NUM_THREADS:-16} export HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} @@ -13,15 +12,16 @@ export PLATFORM_TAG=any export BB=${BB:-../../barretenberg/cpp/build/bin/bb} export NARGO=${NARGO:-../../noir/noir-repo/target/release/nargo} export BB_HASH=$(cache_content_hash ../../barretenberg/cpp/.rebuild_patterns) -export NARGO_HASH=$(cache_content_hash ../../noir/.rebuild_patterns) +export NARGO_HASH=$(../../noir/bootstrap.sh hash) +test_flag=$project_name-tests-$(hash_str "$NARGO_HASH" $(cache_content_hash "^noir-projects/$project_name")) key_dir=./target/keys mkdir -p $key_dir # Hash of the entire protocol circuits. # Needed for test hash, as we presently don't have a program hash for each individual test. # Means if anything within the dir changes, the tests will rerun. -circuits_hash=$(cache_content_hash "^noir-projects/$project_name/crates/" ../../noir/.rebuild_patterns) +circuits_hash=$(hash_str "$NARGO_HASH" $(cache_content_hash "^noir-projects/$project_name/crates/")) # Circuits matching these patterns we have client-ivc keys computed, rather than ultra-honk. ivc_patterns=( diff --git a/noir/.noir-repo.rebuild_patterns b/noir/.noir-repo.rebuild_patterns new file mode 100644 index 000000000000..30bfdf544cd2 --- /dev/null +++ b/noir/.noir-repo.rebuild_patterns @@ -0,0 +1,21 @@ +^acvm-repo +^compiler +^noir_stdlib +^tooling/backend_interface +^tooling/bb_abstraction_leaks +^tooling/debugger +^tooling/lsp +^tooling/nargo +^tooling/nargo_cli +^tooling/nargo_toml +^tooling/nargo_fmt +^tooling/noirc_abi +^tooling/acvm_cli +^.yarn +^.yarnrc.yml +^package.json +^yarn.lock +^tooling/noir_codegen +^tooling/noir_js +^tooling/noir_js_types +^tooling/noirc_abi_wasm diff --git a/noir/.noir-repo.rebuild_patterns_tests b/noir/.noir-repo.rebuild_patterns_tests new file mode 100644 index 000000000000..d3ec0a33c158 --- /dev/null +++ b/noir/.noir-repo.rebuild_patterns_tests @@ -0,0 +1 @@ +^test_programs/execution_success/ diff --git a/noir/.rebuild_patterns b/noir/.rebuild_patterns index c9483392792c..a793d8c4242b 100644 --- a/noir/.rebuild_patterns +++ b/noir/.rebuild_patterns @@ -1,26 +1,5 @@ +^noir/bootstrap.sh ^noir/scripts/bootstrap_native.sh -^noir/scripts/test_native.sh -^noir/noir-repo/acvm-repo -^noir/noir-repo/compiler -^noir/noir-repo/noir_stdlib -^noir/noir-repo/tooling/backend_interface -^noir/noir-repo/tooling/bb_abstraction_leaks -^noir/noir-repo/tooling/debugger -^noir/noir-repo/tooling/lsp -^noir/noir-repo/tooling/nargo -^noir/noir-repo/tooling/nargo_cli -^noir/noir-repo/tooling/nargo_toml -^noir/noir-repo/tooling/nargo_fmt -^noir/noir-repo/tooling/noirc_abi -^noir/noir-repo/tooling/acvm_cli ^noir/scripts/bootstrap_packages.sh ^noir/scripts/test_js_packages.sh -^noir/noir-repo/.yarn -^noir/noir-repo/.yarnrc.yml -^noir/noir-repo/package.json -^noir/noir-repo/yarn.lock -^noir/noir-repo/tooling/noir_codegen -^noir/noir-repo/tooling/noir_js -^noir/noir-repo/tooling/noir_js_types -^noir/noir-repo/tooling/noirc_abi_wasm -^noir/bootstrap.sh +^noir/scripts/test_native.sh diff --git a/noir/.rebuild_patterns_tests b/noir/.rebuild_patterns_tests deleted file mode 100644 index 54b79db9ace4..000000000000 --- a/noir/.rebuild_patterns_tests +++ /dev/null @@ -1 +0,0 @@ -^noir/noir-repo/test_programs/execution_success/ diff --git a/noir/bootstrap.sh b/noir/bootstrap.sh index 3867e3225e69..12817257ba44 100755 --- a/noir/bootstrap.sh +++ b/noir/bootstrap.sh @@ -24,13 +24,32 @@ export RUSTFLAGS="-Dwarnings" # Update the noir-repo and compute hashes. function noir_sync { - denoise "scripts/sync.sh init && scripts/sync.sh update" + # The sync.sh script strives not to send anything to `stdout`, so as not to interfere with `test_cmds` and `hash`. + DENOISE=0 denoise "scripts/sync.sh init && scripts/sync.sh update" +} + +# Calculate the content hash for caching, taking into account that `noir-repo` +# is not part of the `aztec-packages` repo itself, so the `git ls-tree` used +# by `cache_content_hash` would not take those files into account. +function noir_content_hash { + function noir_repo_content_hash { + echo $(REPO_PATH=./noir-repo cache_content_hash $@) + } + with_tests=${1:-0} + noir_hash=$(cache_content_hash .rebuild_patterns) + noir_repo_hash=$(noir_repo_content_hash .noir-repo.rebuild_patterns) + if [ "$with_tests" == "1" ]; then + noir_repo_hash_tests=$(noir_repo_content_hash .noir-repo.rebuild_patterns_tests) + else + noir_repo_hash_tests="" + fi + echo $(hash_str $noir_hash $noir_repo_hash $noir_repo_hash_tests) } # Builds nargo, acvm and profiler binaries. function build_native { set -euo pipefail - local hash=$(cache_content_hash .rebuild_patterns) + local hash=$(noir_content_hash) if cache_download noir-$hash.tar.gz; then return fi @@ -46,7 +65,7 @@ function build_native { # Builds js packages. function build_packages { set -euo pipefail - local hash=$(cache_content_hash .rebuild_patterns) + local hash=$(noir_content_hash) if cache_download noir-packages-$hash.tar.gz; then cd noir-repo @@ -59,6 +78,7 @@ function build_packages { cd noir-repo npm_install_deps + # Hack to get around failure introduced by https://github.com/AztecProtocol/aztec-packages/pull/12371 # Tests fail with message "env: ‘mocha’: No such file or directory" yarn install @@ -90,13 +110,12 @@ function build_packages { noir-repo/tooling/noirc_abi_wasm/web } -export -f build_native build_packages +# Export functions that can be called from `parallel` in `build`, +# and all the functions they can call as well. +export -f build_native build_packages noir_content_hash function build { echo_header "noir build" - - noir_sync - # TODO: Move to build image? denoise ./noir-repo/.github/scripts/wasm-bindgen-install.sh if ! command -v cargo-binstall &>/dev/null; then @@ -119,7 +138,7 @@ function test { # Prints the commands to run tests, one line per test, prefixed with the appropriate content hash. function test_cmds { - local test_hash=$(cache_content_hash .rebuild_patterns .rebuild_patterns_tests) + local test_hash=$(noir_content_hash 1) cd noir-repo cargo nextest list --workspace --locked --release -Tjson-pretty 2>/dev/null | \ jq -r ' @@ -194,17 +213,25 @@ case "$cmd" in git clean -ffdx ;; "ci") + noir_sync build test ;; ""|"fast"|"full") + noir_sync build ;; test_cmds|build_native|build_packages|format|test|release) + noir_sync $cmd "$@" ;; "hash") - echo $(cache_content_hash .rebuild_patterns) + noir_sync + echo $(noir_content_hash) + ;; + "hash-tests") + noir_sync + echo $(noir_content_hash 1) ;; "make-patch") scripts/sync.sh make-patch diff --git a/noir/noir-repo.patch b/noir/noir-repo.patch index 2965852cd5bd..416a6f859586 100644 --- a/noir/noir-repo.patch +++ b/noir/noir-repo.patch @@ -166,7 +166,7 @@ index e5f08ef2..00000000 - ROLLUP_HONK_IDENTIFIER, - ); -} --- +-- 2.43.0 From b36a50c9cb3cf1c18ae545596a60ff0291e65389 Mon Sep 17 00:00:00 2001 @@ -190,7 +190,7 @@ index 30dd2a7e..a2712fd7 100644 }, "include": [ "src/**/*.ts" --- +-- 2.43.0 From 9ab3a9e4f931a17ca8b0dc35ade504769844e3a1 Mon Sep 17 00:00:00 2001 @@ -215,6 +215,28 @@ index 5f819990..00000000 -authors = [""] - -[dependencies] --- +-- 2.43.0 +From fd8f444eba5db51bee2173d7c2f66bebaa693d30 Mon Sep 17 00:00:00 2001 +From: aakoshh +Date: Mon, 17 Mar 2025 12:10:58 +0000 +Subject: [PATCH 4/4] Ignore package.tgz + +--- + .gitignore | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/.gitignore b/.gitignore +index 3349018..c93fe8e 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -59,3 +59,6 @@ codegen + + mutants.out + mutants.out.old ++ ++# Artifacts created by `noir/bootstrap.sh build_packages` ++**/package.tgz +-- +2.43.0 diff --git a/noir/scripts/sync-in-fixup.sh b/noir/scripts/sync-in-fixup.sh index 500003f625df..378fab2b718e 100755 --- a/noir/scripts/sync-in-fixup.sh +++ b/noir/scripts/sync-in-fixup.sh @@ -4,5 +4,5 @@ set -eu cd $(dirname $0)/../noir-repo # Remove requirement for `wasm-opt` to be installed -sed -i "s/^require_command wasm-opt/#require_command wasm-opt/" ./tooling/noirc_abi_wasm/build.sh -sed -i "s/^require_command wasm-opt/#require_command wasm-opt/" ./acvm-repo/acvm_js/build.sh +sed -i.bak "s/^require_command wasm-opt/#require_command wasm-opt/" ./tooling/noirc_abi_wasm/build.sh +sed -i.bak "s/^require_command wasm-opt/#require_command wasm-opt/" ./acvm-repo/acvm_js/build.sh diff --git a/noir/scripts/sync-out-fixup.sh b/noir/scripts/sync-out-fixup.sh index 06a6e3d9d2c1..bcdbf8371df0 100755 --- a/noir/scripts/sync-out-fixup.sh +++ b/noir/scripts/sync-out-fixup.sh @@ -4,5 +4,5 @@ set -eu cd $(dirname $0)/../noir-repo # Add requirement for `wasm-opt` to be installed -sed -i "s/^#require_command wasm-opt/require_command wasm-opt/" ./tooling/noirc_abi_wasm/build.sh -sed -i "s/^#require_command wasm-opt/require_command wasm-opt/" ./acvm-repo/acvm_js/build.sh +sed -i.bak "s/^#require_command wasm-opt/require_command wasm-opt/" ./tooling/noirc_abi_wasm/build.sh +sed -i.bak "s/^#require_command wasm-opt/require_command wasm-opt/" ./acvm-repo/acvm_js/build.sh diff --git a/noir/scripts/sync.sh b/noir/scripts/sync.sh index d3c0269f4f21..00b265290634 100755 --- a/noir/scripts/sync.sh +++ b/noir/scripts/sync.sh @@ -195,7 +195,10 @@ function latest_nightly { # Create an empty marker commit to show that patches have been applied or put in a patch file. function commit_patch_marker { - git -C noir-repo commit -m "$PATCH_COMMIT_MSG" --allow-empty --no-gpg-sign + # The output is redirected to stderr, otherwise a message like + # `[detached HEAD e4c68760f0] Noir local patch commit.` appears + # in the output, becoming part of e.g. `noir/bootstrap.sh hash` + git -C noir-repo commit -m "$PATCH_COMMIT_MSG" --allow-empty --no-gpg-sign >&2 } # Apply the fixup script and any local patch file. @@ -203,11 +206,11 @@ function patch_repo { log Applying fixup on noir-repo # Redirect the `bb` reference to the local one. scripts/sync-in-fixup.sh - git -C noir-repo add . && git -C noir-repo commit -m "$FIXUP_COMMIT_MSG" --allow-empty --no-gpg-sign + git -C noir-repo add . && git -C noir-repo commit -m "$FIXUP_COMMIT_MSG" --allow-empty --no-gpg-sign >&2 # Apply any patch file. if [ -f $NOIR_REPO_PATCH ]; then log Applying patches from $NOIR_REPO_PATCH - git -C noir-repo am ../$NOIR_REPO_PATCH + git -C noir-repo am ../$NOIR_REPO_PATCH >&2 else log "No patch file to apply" fi diff --git a/yarn-project/bootstrap.sh b/yarn-project/bootstrap.sh index 07ebff602f57..1478ba5427a1 100755 --- a/yarn-project/bootstrap.sh +++ b/yarn-project/bootstrap.sh @@ -5,10 +5,11 @@ cmd=${1:-} [ -n "$cmd" ] && shift function hash { - cache_content_hash \ - ../noir/.rebuild_patterns \ - ../{avm-transpiler,noir-projects,l1-contracts,yarn-project}/.rebuild_patterns \ - ../barretenberg/*/.rebuild_patterns + hash_str \ + $(../noir/bootstrap.sh hash) \ + $(cache_content_hash \ + ../{avm-transpiler,noir-projects,l1-contracts,yarn-project}/.rebuild_patterns \ + ../barretenberg/*/.rebuild_patterns) } function compile_project {