From 0eaf3becd435d4b99ca2034a646200ae5d835d6e Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 13 Jan 2025 14:13:07 -0500 Subject: [PATCH 01/12] abort staged content hashes --- .github/workflows/ci-mac.yml | 20 ++++-- .../commitment_schemes/ipa/ipa.hpp | 2 +- ci3/cache_content_hash | 68 +++++++++++-------- 3 files changed, 57 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci-mac.yml b/.github/workflows/ci-mac.yml index de82dcba8cef..bfb8063fcbc6 100644 --- a/.github/workflows/ci-mac.yml +++ b/.github/workflows/ci-mac.yml @@ -1,19 +1,25 @@ name: CI (Mac) + on: push: branches: - master - '*/*mac-build' + pull_request: + types: [synchronize, labeled] jobs: build-mac-intel: name: Build on Mac x86_64-apple-darwin + # Run this job if it's a direct push OR if a PR is labeled with "macos-ci". + if: > + github.event_name == 'push' || + (github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'macos-ci')) runs-on: macos-13 steps: - name: Checkout uses: actions/checkout@v3 - with: - ref: ${{ inputs.tag || env.GITHUB_REF }} - name: Create Mac Build Environment run: brew install cmake ninja llvm@16 @@ -29,12 +35,15 @@ jobs: build-mac-m1: name: Build on Mac aarch64-apple-darwin + # Same label check as above + if: > + github.event_name == 'push' || + (github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'macos-ci')) runs-on: macos-14 steps: - name: Checkout uses: actions/checkout@v3 - with: - ref: ${{ inputs.tag || env.GITHUB_REF }} - name: Create Mac Build Environment run: brew install cmake ninja @@ -48,7 +57,8 @@ jobs: build-check: name: Check builds are successful needs: [build-mac-intel, build-mac-m1] - if: ${{ always() }} + # Likewise, only run if it's triggered by push or "macos-ci" label. + if: always() runs-on: ubuntu-latest steps: - name: Report overall success diff --git a/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp b/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp index 7e83b477ef7e..ec31de7ac8d7 100644 --- a/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp +++ b/barretenberg/cpp/src/barretenberg/commitment_schemes/ipa/ipa.hpp @@ -981,4 +981,4 @@ template class IPA { } }; -} // namespace bb \ No newline at end of file +} // namespace bb diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index 833a19074419..715eb091a1b7 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -2,43 +2,57 @@ set -euo pipefail [ "${BUILD_SYSTEM_DEBUG:-}" = 1 ] && set -x -# Ensure args are set -if [[ "$#" = 0 ]]; then - echo "Error: No arguments provided." - exit 1 +if [[ "$#" -eq 0 ]]; then + echo "No arguments provided." + exit 1 fi -# If too many spurious cache misses: can be customized to pin artifacts to a specific version AZTEC_CACHE_COMMIT=${AZTEC_CACHE_COMMIT:-HEAD} PLATFORM_TAG="${PLATFORM_TAG:-${OSTYPE:-unknown}-$(uname -m)}" -rebuild_patterns=() +# Gather patterns (from direct args or from files) +patterns=() for arg in "$@"; do if [[ -f "$arg" ]]; then - rebuild_patterns+=$(cat "$arg") - rebuild_patterns+=$'\n' + patterns+=$(cat "$arg") + patterns+=$'\n' else - rebuild_patterns+="$arg"$'\n' + patterns+="$arg"$'\n' fi done -# Concatenate patterns with '|' and double escape backslashes for AWK -# filter empty lines -AWK_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') - -# use git repo root because that is where our patterns are focused -cd $(git rev-parse --show-toplevel) -# Use git ls-tree and AWK to filter files matching the rebuild patterns and extract their hashes -# Sort the hashes and compute the content hash -CONTENT_HASH=$(git ls-tree -r $AZTEC_CACHE_COMMIT | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | sort | git hash-object --stdin| cut -c1-16) - -# Check if file list was empty by comparing against the result of 'echo '' | git hash-object --stdin | cut -c1-16' -ECHO_BLANK_HASH="8b137891791fe969" -if [ "$CONTENT_HASH" = "$ECHO_BLANK_HASH" ]; then - echo "No files matched the rebuild patterns $rebuild_patterns." - echo "Awk pattern expanded: $AWK_PATTERN." - exit 1 +# Build a single pattern for grep and awk +GREP_PATTERN=$(echo "${patterns[*]}" | grep -v '^$') +AWK_PATTERN=$(echo $GREP_PATTERN | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') +# Ensure we work in the repo root +cd "$(git rev-parse --show-toplevel)" + +unstaged_diff="$(git diff --name-only | grep -E "$GREP_PATTERN" || true)" +staged_diff="$(git diff --staged --name-only | grep -E "$GREP_PATTERN" || true)" +untracked_diff="$(git ls-files --others --exclude-standard | grep -E "$GREP_PATTERN" || true)" + +# Check for untracked files +if [[ -n "$unstaged_diff" ]] || [[ -n "$staged_diff" ]] || [[ -n "$untracked_diff" ]]; then + # Signal to cache_upload and cache_download to not touch this file. + echo "disabled-cache" + exit 0 +fi + +# Calculate a content hash for matched files +CONTENT_HASH="$( + git ls-tree -r "$AZTEC_CACHE_COMMIT" \ + | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' \ + | sort \ + | git hash-object --stdin \ + | cut -c1-16 +)" + +# Check if file list was empty by comparing against the result of +# 'echo '' | git hash-object --stdin | cut -c1-16' +EMPTY_HASH="8b137891791fe969" +if [[ "$CONTENT_HASH" == "$EMPTY_HASH" ]]; then + echo "No files matched any of the provided patterns." + exit 1 fi -# important: include architecture in content hash because we target x86_64 and arm64 -echo "$CONTENT_HASH-$(echo $PLATFORM_TAG)" +echo "${CONTENT_HASH}-${PLATFORM_TAG}" From 8455bea28894bfc7ea41a48ea5d247532af33f5d Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 13 Jan 2025 14:21:32 -0500 Subject: [PATCH 02/12] propagate 'disabled-cache' --- ci3/cache_content_hash | 4 ++-- ci3/source | 9 +++++++-- ci3/source_bootstrap | 2 +- docs/bootstrap.sh | 4 ++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index 715eb091a1b7..cdf188b31abf 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -21,9 +21,9 @@ for arg in "$@"; do fi done -# Build a single pattern for grep and awk +# Build compound patterns for grep and awk GREP_PATTERN=$(echo "${patterns[*]}" | grep -v '^$') -AWK_PATTERN=$(echo $GREP_PATTERN | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') +AWK_PATTERN=$(echo "${patterns[*]}" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') # Ensure we work in the repo root cd "$(git rev-parse --show-toplevel)" diff --git a/ci3/source b/ci3/source index d1415c115628..2d8bdec0f428 100644 --- a/ci3/source +++ b/ci3/source @@ -19,7 +19,12 @@ export ci3="$root/ci3" function hash_str { set -eu - echo $1 | git hash-object --stdin | cut -c1-16 | tr -d '\n' + if [[ "$1" == *"disabled-cache"* ]]; then + # We want to propagate cache being disabled so that cache_upload and cache_download can catch this + echo "disabled-cache" + else + echo $1 | git hash-object --stdin | cut -c1-16 | tr -d '\n' + fi } function echo_stderr { @@ -28,4 +33,4 @@ function echo_stderr { export -f hash_str echo_stderr -source $ci3/source_color \ No newline at end of file +source $ci3/source_color diff --git a/ci3/source_bootstrap b/ci3/source_bootstrap index bfcc4d9c020f..36a52ccf0099 100644 --- a/ci3/source_bootstrap +++ b/ci3/source_bootstrap @@ -11,4 +11,4 @@ case "${1:-}" in ""|"fast") export USE_CACHE=${USE_CACHE:-1} ;; -esac \ No newline at end of file +esac diff --git a/docs/bootstrap.sh b/docs/bootstrap.sh index a3ef175ed515..867bbd59a505 100755 --- a/docs/bootstrap.sh +++ b/docs/bootstrap.sh @@ -3,7 +3,7 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap cmd=${1:-} # combine yarn project hash -hash=$(echo $(../yarn-project/bootstrap.sh hash) $(cache_content_hash .rebuild_patterns) | git hash-object --stdin) +hash=$(hash_str "$(../yarn-project/bootstrap.sh hash) $(cache_content_hash .rebuild_patterns)") # TODO(ci3): build command case "$cmd" in "hash") @@ -12,4 +12,4 @@ case "$cmd" in *) echo "Unknown command: $cmd" exit 1 -esac \ No newline at end of file +esac From 42376fac20b0eddd79fcf784e42c3eb887a6230c Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 13 Jan 2025 14:46:05 -0500 Subject: [PATCH 03/12] finish --- ci3/cache_download | 6 +++++- ci3/cache_download_flag | 4 ++++ ci3/cache_upload | 6 +++++- ci3/cache_upload_flag | 6 +++++- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/ci3/cache_download b/ci3/cache_download index cb47f6f35cbb..e950fcdb1f15 100755 --- a/ci3/cache_download +++ b/ci3/cache_download @@ -7,6 +7,10 @@ if [ "$#" -lt 1 ]; then exit 1 fi +if [[ "$1" == *"disabled-cache"* ]]; then + echo "Not using cache for $1 due to uncommitted changes/files." + exit 1 +fi if [ "${USE_CACHE:-0}" -lt 1 ]; then # Only download if USE_CACHE is 1 echo "Not using cache for $1 because USE_CACHE=0." @@ -26,4 +30,4 @@ else # Attempt to download and extract the cache file (curl -s -f "$S3_ENDPOINT/build-cache/$TAR_FILE" | tar -xzf - -C "$OUT_DIR" 2>/dev/null) || (echo "Cache download of $TAR_FILE failed." >&2 && exit 1) fi -echo "Cache download and extraction of $TAR_FILE complete." >&2 \ No newline at end of file +echo "Cache download and extraction of $TAR_FILE complete." >&2 diff --git a/ci3/cache_download_flag b/ci3/cache_download_flag index 4ab176ea8844..c54a2c54defd 100755 --- a/ci3/cache_download_flag +++ b/ci3/cache_download_flag @@ -7,6 +7,10 @@ if [ "$#" -lt 1 ]; then exit 1 fi +if [[ "$1" == *"disabled-cache"* ]]; then + echo "Running test $1 due to uncommitted changes/files." + exit 1 +fi if [ "${USE_CACHE:-0}" != 1 ] ; then # Don't look if CI isn't set. No need to muddle with dev runs. echo "Running test $1 because USE_CACHE is not 1." diff --git a/ci3/cache_upload b/ci3/cache_upload index fc07789ceb5c..993e46665647 100755 --- a/ci3/cache_upload +++ b/ci3/cache_upload @@ -7,6 +7,10 @@ if [ "$#" -lt 2 ]; then exit 1 fi +if [[ "$1" == *"disabled-cache"* ]]; then + echo "Skipping upload of $1 due to uncommitted changes/files." + exit 0 +fi # Name, intended to have .tar.gz ending name="$1" # Now $@ = our binary path args @@ -22,4 +26,4 @@ if tar -czf - "$@" | aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp - "s3://aztec-ci-a else echo "Cache upload of $name failed." >&2 exit 0 -fi \ No newline at end of file +fi diff --git a/ci3/cache_upload_flag b/ci3/cache_upload_flag index fb15e7f57912..0751edc649ab 100755 --- a/ci3/cache_upload_flag +++ b/ci3/cache_upload_flag @@ -7,8 +7,12 @@ if [ "$#" -lt 1 ]; then exit 1 fi +if [[ "$1" == *"disabled-cache"* ]]; then + echo "Not uploading test flag $1 due to uncommitted changes/files." + exit 0 +fi if [ "${CI:-0}" -lt 1 ]; then - # Don't upload if CI isn't set. No need to muddle with dev runs.A + # Don't upload if CI isn't set. No need to muddle with dev runs. echo "Not uploading test flag $1 because CI=0." exit 0 fi From ebda990b6477422907ffed491743d67bc29ffd1e Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 13 Jan 2025 14:48:31 -0500 Subject: [PATCH 04/12] Update cache_content_hash --- ci3/cache_content_hash | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index cdf188b31abf..587c6701dc36 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -10,20 +10,19 @@ fi AZTEC_CACHE_COMMIT=${AZTEC_CACHE_COMMIT:-HEAD} PLATFORM_TAG="${PLATFORM_TAG:-${OSTYPE:-unknown}-$(uname -m)}" -# Gather patterns (from direct args or from files) -patterns=() +rebuild_patterns=() for arg in "$@"; do if [[ -f "$arg" ]]; then - patterns+=$(cat "$arg") - patterns+=$'\n' + rebuild_patterns+=$(cat "$arg") + rebuild_patterns+=$'\n' else - patterns+="$arg"$'\n' + rebuild_patterns+="$arg"$'\n' fi done # Build compound patterns for grep and awk -GREP_PATTERN=$(echo "${patterns[*]}" | grep -v '^$') -AWK_PATTERN=$(echo "${patterns[*]}" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') +GREP_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$') +AWK_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') # Ensure we work in the repo root cd "$(git rev-parse --show-toplevel)" From ce743fd5a18390995a6c8bdfbe66374e4d2453ff Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 13 Jan 2025 14:50:37 -0500 Subject: [PATCH 05/12] Update cache_content_hash --- ci3/cache_content_hash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index 587c6701dc36..ef3c76ba9e98 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -3,7 +3,7 @@ set -euo pipefail [ "${BUILD_SYSTEM_DEBUG:-}" = 1 ] && set -x if [[ "$#" -eq 0 ]]; then - echo "No arguments provided." + echo "Error: No arguments provided." exit 1 fi From 6ea8b67e43b3c0cc8aae08c5cecdee08221b75b7 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 13 Jan 2025 14:50:59 -0500 Subject: [PATCH 06/12] Update cache_content_hash --- ci3/cache_content_hash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index ef3c76ba9e98..ac9717d31df5 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -20,8 +20,9 @@ for arg in "$@"; do fi done -# Build compound patterns for grep and awk GREP_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$') +# Concatenate patterns with '|' and double escape backslashes for AWK +# filter empty lines AWK_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') # Ensure we work in the repo root cd "$(git rev-parse --show-toplevel)" From 3e68d716a598e68d26987898558b0ad6368e6e78 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 13 Jan 2025 14:54:41 -0500 Subject: [PATCH 07/12] Update cache_content_hash --- ci3/cache_content_hash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index ac9717d31df5..3b1cd86e3814 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -24,7 +24,7 @@ GREP_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$') # Concatenate patterns with '|' and double escape backslashes for AWK # filter empty lines AWK_PATTERN=$(echo "$rebuild_patterns" | grep -v '^$' | sed 's/\\/\\\\/g' | tr '\n' '|' | sed 's/|$//') -# Ensure we work in the repo root +# use git repo root because that is where our patterns are focused cd "$(git rev-parse --show-toplevel)" unstaged_diff="$(git diff --name-only | grep -E "$GREP_PATTERN" || true)" From a783557c9661e1b60b7070e329f92e3cae843d67 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 13 Jan 2025 14:54:58 -0500 Subject: [PATCH 08/12] Update cache_content_hash --- ci3/cache_content_hash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index 3b1cd86e3814..4fa93a51921c 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -39,6 +39,8 @@ if [[ -n "$unstaged_diff" ]] || [[ -n "$staged_diff" ]] || [[ -n "$untracked_dif fi # Calculate a content hash for matched files +# Use git ls-tree and AWK to filter files matching the rebuild patterns and extract their hashes +# Sort the hashes and compute the content hash CONTENT_HASH="$( git ls-tree -r "$AZTEC_CACHE_COMMIT" \ | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' \ From 83501d5b7f4dee733e7c02fe4b4a6ca58c740da9 Mon Sep 17 00:00:00 2001 From: ludamad Date: Mon, 13 Jan 2025 14:56:09 -0500 Subject: [PATCH 09/12] Update cache_content_hash --- ci3/cache_content_hash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index 4fa93a51921c..62ee0f66b957 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -31,7 +31,7 @@ unstaged_diff="$(git diff --name-only | grep -E "$GREP_PATTERN" || true)" staged_diff="$(git diff --staged --name-only | grep -E "$GREP_PATTERN" || true)" untracked_diff="$(git ls-files --others --exclude-standard | grep -E "$GREP_PATTERN" || true)" -# Check for untracked files +# Check for uncommitted files/changes if [[ -n "$unstaged_diff" ]] || [[ -n "$staged_diff" ]] || [[ -n "$untracked_diff" ]]; then # Signal to cache_upload and cache_download to not touch this file. echo "disabled-cache" From 63d69d2f74d17ba88b21868780e42363e4674eec Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 14 Jan 2025 10:53:48 -0500 Subject: [PATCH 10/12] Update cache_content_hash --- ci3/cache_content_hash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index 62ee0f66b957..a8c4c361cbc4 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -2,7 +2,7 @@ set -euo pipefail [ "${BUILD_SYSTEM_DEBUG:-}" = 1 ] && set -x -if [[ "$#" -eq 0 ]]; then +if [[ "$#" = 0 ]]; then echo "Error: No arguments provided." exit 1 fi From 86ad9f47c3cfd2fcbe19a3fe377f1159b5d59e2b Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 14 Jan 2025 10:56:08 -0500 Subject: [PATCH 11/12] Update cache_content_hash --- ci3/cache_content_hash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index a8c4c361cbc4..bb29785f2f6e 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -53,7 +53,8 @@ CONTENT_HASH="$( # 'echo '' | git hash-object --stdin | cut -c1-16' EMPTY_HASH="8b137891791fe969" if [[ "$CONTENT_HASH" == "$EMPTY_HASH" ]]; then - echo "No files matched any of the provided patterns." + echo "No files matched the rebuild patterns $rebuild_patterns." + echo "Awk pattern expanded: $AWK_PATTERN." exit 1 fi From 076b0b37b1b37084432cd5c53efddd32b080193a Mon Sep 17 00:00:00 2001 From: ludamad Date: Tue, 14 Jan 2025 10:56:34 -0500 Subject: [PATCH 12/12] Update cache_content_hash --- ci3/cache_content_hash | 1 + 1 file changed, 1 insertion(+) diff --git a/ci3/cache_content_hash b/ci3/cache_content_hash index bb29785f2f6e..bd18b15e2de8 100755 --- a/ci3/cache_content_hash +++ b/ci3/cache_content_hash @@ -58,4 +58,5 @@ if [[ "$CONTENT_HASH" == "$EMPTY_HASH" ]]; then exit 1 fi +# important: include architecture in content hash because we target x86_64 and arm64 echo "${CONTENT_HASH}-${PLATFORM_TAG}"