chore: Bump Noir reference

.github/workflows/pull-noir.yml

@@ -1,4 +1,4 @@
-# Create a pull request from current Noir master.
+# Create a pull request from current Noir nightly.
 name: Pull from noir repo
 
 # Don't allow multiple of these running at once:
@@ -23,25 +23,23 @@ jobs:
           fetch-depth: 0
           token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
 
+      # Determine if there is already a PR to overwrite, and the current plus latest tags.
       - name: Check for existing PR
         run: |
           set -xue # print commands
           # Enable gh executable. We spread out the API requests between the github actions bot token, and aztecbot
           export GH_TOKEN="${{ secrets.GITHUB_TOKEN }}"
           # Do we have a PR active?
-          PR_URL=$(gh pr list --repo AztecProtocol/aztec-packages --head sync-noir --json url --jq ".[0].url")
+          PR_URL=$(gh pr list --repo AztecProtocol/aztec-packages --head bump-noir --json url --jq ".[0].url")
           echo "PR_URL=$PR_URL" >> $GITHUB_ENV
-          # What was our last merge on noir side?
-          # Detect our last sync commit (written by this action before pushing) with a fallback for the first time we ever do this
-          BASE_NOIR_COMMIT=$(curl https://raw.githubusercontent.com/AztecProtocol/aztec-packages/master/.noir-sync-commit)
-          if [ "$BASE_NOIR_COMMIT" = "404: Not Found" ] ; then
-            BASE_NOIR_COMMIT="50d2735825454a8638a308156d4ea23b3c4420d8"
-          fi
-          echo "BASE_NOIR_COMMIT=$BASE_NOIR_COMMIT" >> $GITHUB_ENV
-          # What was our last sync on aztec side?
-          BASE_AZTEC_COMMIT=`curl https://raw.githubusercontent.com/noir-lang/noir/master/.aztec-sync-commit`
-          echo "BASE_AZTEC_COMMIT=$BASE_AZTEC_COMMIT" >> $GITHUB_ENV
+          # Get the current Noir target
+          CURRENT_NOIR_REF=$(noir/scripts/sync.sh read-noir-repo-ref)
+          echo "CURRENT_NOIR_REF=$CURRENT_NOIR_REF" >> $GITHUB_ENV
+          # Get the latest nightly tag we can point to in Noir
+          LATEST_NOIR_NIGHTLY=$(noir/scripts/sync.sh latest-nightly)
+          echo "LATEST_NOIR_NIGHTLY=$LATEST_NOIR_NIGHTLY" >> $GITHUB_ENV
 
+      # Set PR body to be the concatenation of every message that happened in Noir since the last sync.
       - name: Generate PR body
         run: |
           # clone noir repo for manipulations, we use aztec bot token for writeability
@@ -52,7 +50,7 @@ jobs:
           function compute_commit_message() {
             cd noir-repo
             # Create a filtered git log for release-please changelog / metadata
-            RAW_MESSAGE=$(git log --pretty=format:"%s" $BASE_NOIR_COMMIT..HEAD || true)
+            RAW_MESSAGE=$(git log --pretty=format:"%s" $CURRENT_NOIR_REF..$LATEST_NOIR_NIGHTLY || true)
             # Fix Noir PR links and output message
             echo "$RAW_MESSAGE" | sed -E 's/\(#([0-9]+)\)/(https:\/\/github.com\/noir-lang\/noir\/pull\/\1)/g'
             cd ..
@@ -65,85 +63,30 @@ jobs:
           git config --global user.name AztecBot
           git config --global user.email tech@aztecprotocol.com
 
-      # We push using git subrepo (https://github.com/ingydotnet/git-subrepo)
-      # and push all Aztec commits as a single commit with metadata.
+      # Push a commit to the `bump-noir` branch that overrides the Noir ref.
       - name: Push to branch
         run: |
           set -xue # print commands
-          SUBREPO_PATH=noir/noir-repo
-          BRANCH=sync-noir
-          if [[ "$PR_URL" == "" ]]; then
-            # if no staging branch, we can overwrite
-            STAGING_BRANCH=$BRANCH
-          else
-            # otherwise we first reset our staging branch
-            STAGING_BRANCH=$BRANCH-staging
-          fi
-
-          # Get the last sync PR's last commit state
-          COMMIT_MESSAGE=$(cat .PR_BODY_MESSAGE)
-          LINES=$(echo $COMMIT_MESSAGE | wc -l)
-
-          function force_sync_staging() {
-            # reset to last noir merge
-            git checkout $STAGING_BRANCH || git checkout -b $STAGING_BRANCH
-            git reset --hard "$BASE_AZTEC_COMMIT"
-            # Reset our branch to our expected target
-            git push origin $STAGING_BRANCH --force
-            # force gitrepo to point to the right HEAD (we ignore .gitrepo contents otherwise)
-            git config --file="$SUBREPO_PATH/.gitrepo" subrepo.commit "$BASE_NOIR_COMMIT"
-            # we need to commit for git-subrepo
-            git commit -am "[$LINES changes] $COMMIT_MESSAGE"
-            if ./scripts/git-subrepo/lib/git-subrepo pull --force $SUBREPO_PATH --branch=master; then
-              # Read our actual commit sync from git subrepo, stash to file for next time
-              COMMIT=$(git config --file="$SUBREPO_PATH/.gitrepo" subrepo.commit)
-              echo "$COMMIT" > .noir-sync-commit && git add .noir-sync-commit
-              git reset --soft "$BASE_AZTEC_COMMIT"
-              # We don't really need the sync commit on our side, and don't need .gitrepo at all except just in time for the command.
-              git checkout origin/master -- noir/noir-repo/.aztec-sync-commit noir/noir-repo/.gitrepo
-              git commit -am "[$LINES changes] $COMMIT_MESSAGE"
-
-              # There's various changes which we need to make to account for CI differences so we run this script to apply them.
-              git checkout origin/master -- noir/scripts/sync-in-fixup.sh
-              noir/scripts/sync-in-fixup.sh
-              git commit -am "chore: apply sync fixes"
-
-              git push origin $STAGING_BRANCH --force
-            else
-              echo "Problems syncing noir. Needs manual attention, might be a merge conflict."
-              exit 1
-            fi
-          }
-          # merge_staging_branch: Merge our staging branch into aztec-packages.
-          function merge_staging_branch() {
-            # Fix PR branch
-            git fetch # see recent change
-            git checkout $BRANCH || git checkout -b $BRANCH
-            if ! git merge -Xtheirs origin/$STAGING_BRANCH -m "$COMMIT_MESSAGE" ; then
-              # resolve modify/delete conflicts
-              git diff --name-only --diff-filter=U | xargs git rm
-              git commit -m "$COMMIT_MESSAGE"
-            fi
-            git push origin $BRANCH
-          }
-          force_sync_staging
-          if [[ "$PR_URL" != "" ]]; then
-            merge_staging_branch
-          fi
+          BRANCH=bump-noir
+          cd noir
+          ./bootstrap.sh bump-noir-repo-ref $BRANCH $LATEST_NOIR_NIGHTLY
 
+      # Update the PR from `bump-noir` to `master` with the latest message body.
       - name: Update PR
+        env:
+          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
         run: |
           set -xue # print commands
           # Formatted for updating the PR, overrides for release-please commit message parsing
           PR_BODY="""
-          Automated pull of development from the [noir](https://github.com/noir-lang/noir) programming language, a dependency of Aztec.
+          Automated pull of nightly from the [noir](https://github.com/noir-lang/noir) programming language, a dependency of Aztec.
           BEGIN_COMMIT_OVERRIDE
           $(cat .PR_BODY_MESSAGE)
           END_COMMIT_OVERRIDE"""
           # for cross-opening PR in noir repo, we use aztecbot's token
           export GH_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
           if [[ "$PR_URL" == "" ]]; then
-            gh pr create --repo AztecProtocol/aztec-packages --title "feat: Sync from noir" --body "$PR_BODY" --base master --head sync-noir
+            gh pr create --repo AztecProtocol/aztec-packages --title "chore: Bump Noir reference" --body "$PR_BODY" --base $BRANCH_NAME --head bump-noir
           else
             echo "Updating existing PR."
             gh pr edit "$PR_URL" --body "$PR_BODY"

bootstrap.sh

@@ -54,7 +54,7 @@ function check_toolchains {
     exit 1
   fi
   # Check rustup installed.
-  local rust_version=$(yq '.toolchain.channel' ./noir/noir-repo/rust-toolchain.toml)
+  local rust_version=$(yq '.toolchain.channel' ./avm-transpiler/rust-toolchain.toml)
   if ! command -v rustup > /dev/null; then
     encourage_dev_container
     echo "Rustup not installed."
@@ -114,6 +114,8 @@ function install_hooks {
   echo "./noir-projects/precommit.sh" >>$hooks_dir/pre-commit
   echo "./yarn-project/constants/precommit.sh" >>$hooks_dir/pre-commit
   chmod +x $hooks_dir/pre-commit
+  echo "(cd noir && ./postcheckout.sh $@)" >$hooks_dir/post-checkout
+  chmod +x $hooks_dir/post-checkout
 }
 
 function test_cmds {

noir/.gitignore

@@ -1,2 +1,3 @@
 **/package.tgz
 packages
+noir-repo

noir/README.md

@@ -1,20 +1,40 @@
 # Aztec's Build of Noir
 
-We subrepo noir into the folder `noir-repo`.
+We subrepo [noir](https://github.com/noir-lang/noir) into the folder `noir-repo` during the build.
 This folder contains dockerfiles and scripts for performing our custom build of noir for the monorepo.
 
 # Syncing with the main Noir repository
 
 In order to keep aztec-packages in step with the main Noir repository we need to periodically sync between them.
 
-Syncing from aztec-packages into noir currently attempts to revert any changes in Noir since the last sync so it's recommended to always sync from Noir first to ensure that aztec-packages is up-to-date.
+Ideally there should be a one-way sync from Noir to aztec-packages, but occasionally, when the `bb` interface changes,
+or if some integration test fails and the Noir bug is fixed locally, changes have to go both ways.
 
-## Syncing from Noir to aztec-packages.
+## Syncing from Noir to aztec-packages
 
-To start the sync run [this action](https://github.com/AztecProtocol/aztec-packages/actions/workflows/pull-noir.yml) manually (click the "Run Workflow" button in the top right). aztec-bot will then open a new PR which does the initial sync, this will have merge conflicts with master which will need to be resolved.
+During the build the Noir repository is cloned or updated according to the contents of the [noir-repo-ref](./noir-repo-ref)
+file, which can be a tag, branch name or commit hash. The value can be overriden using the `NOIR_REPO_REF` environment variable,
+for example to run the integration tests in aztec-packages against a yet-to-be-released branch of Noir.
 
-## Syncing from aztec-packages to Noir.
+If the ref contains a branch, it's pulled with each build triggered by `bootstrap.sh`, but for repeatable builds it should
+point at a tag instead or commit instead, which would be updated with a regular PR opened in aztec-packages, so we can run
+the test suite before changes take effect.
+
+To start the sync run [this action](https://github.com/AztecProtocol/aztec-packages/actions/workflows/pull-noir.yml) manually (click the "Run Workflow" button in the top right). aztec-bot will then open a new PR which updates the reference. This will might merge conflicts with master which will need to be resolved.
+
+## Syncing from aztec-packages to Noir
 
 When syncing from aztec-packages to Noir it's important to check that the latest release of `bb` uses the same ACIR serialization format as the current master commit. This is because Noir uses a released version of barretenberg rather than being developed in sync with it, it's then not possible to sync if there's been serialization changes since the latest release.
 
-To start the sync run [this action](https://github.com/AztecProtocol/aztec-packages/actions/workflows/mirror-noir-subrepo.yml) manually (click the "Run Workflow" button in the top right). aztec-bot will then open a new PR in the `noir-lang/noir` repository which does the initial sync, this will have merge conflicts with master which will need to be resolved.
+When we make changes in `noir-repo` and commit them, we can check out a branch and push them back to Noir, where a PR can be opened to merge them back
+into an appropriate branch (could be `master` or some kind of integration branch). It is important to exclude the [fixup](./scripts/sync-in-fixup.sh) that the local checkout performs from the PR by running the [fixdown](./scripts/sync-out-fixup.sh) script.
+
+Syncing can be postponed by creating a few commits in `noir-repo`, but instead of opening a PR against Noir, creating a [git patch](https://git-scm.com/docs/git-format-patch) instead using, which is committed to aztec-packages and is applied to any subsequent checkout. A patch file can be made using the following command:
+
+```shell
+./bootstrap.sh make-patch
+```
+
+After this `./noir-repo.patch` should have the changes committed on top of the latest checkout, and if we commit this file to `aztec-packages` then it is automatically applied by any subsequent checkouts of `noir-repo`.
+
+To start an automated sync run [this action](https://github.com/AztecProtocol/aztec-packages/actions/workflows/mirror-noir-subrepo.yml) manually (click the "Run Workflow" button in the top right). aztec-bot will then open a new PR in the `noir-lang/noir` repository which does the initial sync, this will have merge conflicts with master which will need to be resolved.

noir/bootstrap.sh

@@ -1,9 +1,18 @@
 #!/usr/bin/env bash
 source $(git rev-parse --show-toplevel)/ci3/source_bootstrap
 
+set -eou pipefail
+
 cmd=${1:-}
 [ -n "$cmd" ] && shift
 
+# Update the noir-repo before we hash its content, unless the command is exempt.
+no_update=(clean make-patch bump-noir-repo-ref)
+if [[ -z "$cmd" || ! ${no_update[*]} =~ "$cmd" ]]; then
+  scripts/sync.sh init
+  scripts/sync.sh update
+fi
+
 export hash=$(cache_content_hash .rebuild_patterns)
 export test_hash=$(cache_content_hash .rebuild_patterns .rebuild_patterns_tests)
 
@@ -183,9 +192,27 @@ function release_commit {
   release_packages next "$CURRENT_VERSION-commit.$COMMIT_HASH"
 }
 
+# Bump the Noir repo reference on a given branch to a given ref.
+# The branch might already exist, e.g. this could be a daily job bumping the version to the
+# latest nightly, and we might have to deal with updating the patch file because the latest
+# Noir code conflicts with the contents of the patch, or we're debugging some integration
+# test failure on CI. In that case just push another commit to the branch to bump the version
+# further without losing any other commit on the branch.
+function bump_noir_repo_ref {
+  branch=$1
+  ref=$2
+  git fetch --depth 1 origin $branch || true
+  git checkout --track origin/$branch || git checkout $branch || git checkout -b $branch
+  scripts/sync.sh write-noir-repo-ref $ref
+  git add .
+  git commit -m "chore: Update noir-repo-ref to $ref" || true
+  do_or_dryrun git push --set-upstream origin $branch
+}
+
 case "$cmd" in
   "clean")
-    git clean -fdx
+    # Double `f` needed to delete the nested git repository.
+    git clean -ffdx
     ;;
   "ci")
     build
@@ -203,6 +230,12 @@ case "$cmd" in
   "hash-test")
     echo $test_hash
     ;;
+  "make-patch")
+    scripts/sync.sh make-patch
+    ;;
+  "bump-noir-repo-ref")
+    bump_noir_repo_ref $@
+    ;;
   *)
     echo "Unknown command: $cmd"
     exit 1

noir/noir-repo-ref

@@ -0,0 +1 @@
+nightly-2025-03-14