feat: Noir sync rework

.github/workflows/pull-noir.yml

@@ -1,4 +1,4 @@
-# Create a pull request from current Noir master.
+# Create a pull request from current Noir nightly.
 name: Pull from noir repo
 
 # Don't allow multiple of these running at once:
@@ -23,25 +23,23 @@ jobs:
           fetch-depth: 0
           token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
 
+      # Determine if there is already a PR to overwrite, and the current plus latest tags.
       - name: Check for existing PR
         run: |
           set -xue # print commands
           # Enable gh executable. We spread out the API requests between the github actions bot token, and aztecbot
           export GH_TOKEN="${{ secrets.GITHUB_TOKEN }}"
           # Do we have a PR active?
-          PR_URL=$(gh pr list --repo AztecProtocol/aztec-packages --head sync-noir --json url --jq ".[0].url")
+          PR_URL=$(gh pr list --repo AztecProtocol/aztec-packages --head bump-noir --json url --jq ".[0].url")
           echo "PR_URL=$PR_URL" >> $GITHUB_ENV
-          # What was our last merge on noir side?
-          # Detect our last sync commit (written by this action before pushing) with a fallback for the first time we ever do this
-          BASE_NOIR_COMMIT=$(curl https://raw.githubusercontent.com/AztecProtocol/aztec-packages/master/.noir-sync-commit)
-          if [ "$BASE_NOIR_COMMIT" = "404: Not Found" ] ; then
-            BASE_NOIR_COMMIT="50d2735825454a8638a308156d4ea23b3c4420d8"
-          fi
-          echo "BASE_NOIR_COMMIT=$BASE_NOIR_COMMIT" >> $GITHUB_ENV
-          # What was our last sync on aztec side?
-          BASE_AZTEC_COMMIT=`curl https://raw.githubusercontent.com/noir-lang/noir/master/.aztec-sync-commit`
-          echo "BASE_AZTEC_COMMIT=$BASE_AZTEC_COMMIT" >> $GITHUB_ENV
+          # Get the current Noir target
+          CURRENT_NOIR_REF=$(noir/scripts/sync.sh read-noir-repo-ref)
+          echo "CURRENT_NOIR_REF=$CURRENT_NOIR_REF" >> $GITHUB_ENV
+          # Get the latest nightly tag we can point to in Noir
+          LATEST_NOIR_NIGHTLY=$(noir/scripts/sync.sh latest-nightly)
+          echo "LATEST_NOIR_NIGHTLY=$LATEST_NOIR_NIGHTLY" >> $GITHUB_ENV
 
+      # Set PR body to be the concatenation of every message that happened in Noir since the last sync.
       - name: Generate PR body
         run: |
           # clone noir repo for manipulations, we use aztec bot token for writeability
@@ -52,7 +50,7 @@ jobs:
           function compute_commit_message() {
             cd noir-repo
             # Create a filtered git log for release-please changelog / metadata
-            RAW_MESSAGE=$(git log --pretty=format:"%s" $BASE_NOIR_COMMIT..HEAD || true)
+            RAW_MESSAGE=$(git log --pretty=format:"%s" $CURRENT_NOIR_REF..$LATEST_NOIR_NIGHTLY || true)
             # Fix Noir PR links and output message
             echo "$RAW_MESSAGE" | sed -E 's/\(#([0-9]+)\)/(https:\/\/github.com\/noir-lang\/noir\/pull\/\1)/g'
             cd ..
@@ -65,85 +63,30 @@ jobs:
           git config --global user.name AztecBot
           git config --global user.email tech@aztecprotocol.com
 
-      # We push using git subrepo (https://github.com/ingydotnet/git-subrepo)
-      # and push all Aztec commits as a single commit with metadata.
+      # Push a commit to the `bump-noir` branch that overrides the Noir ref.
       - name: Push to branch
         run: |
           set -xue # print commands
-          SUBREPO_PATH=noir/noir-repo
-          BRANCH=sync-noir
-          if [[ "$PR_URL" == "" ]]; then
-            # if no staging branch, we can overwrite
-            STAGING_BRANCH=$BRANCH
-          else
-            # otherwise we first reset our staging branch
-            STAGING_BRANCH=$BRANCH-staging
-          fi
-
-          # Get the last sync PR's last commit state
-          COMMIT_MESSAGE=$(cat .PR_BODY_MESSAGE)
-          LINES=$(echo $COMMIT_MESSAGE | wc -l)
-
-          function force_sync_staging() {
-            # reset to last noir merge
-            git checkout $STAGING_BRANCH || git checkout -b $STAGING_BRANCH
-            git reset --hard "$BASE_AZTEC_COMMIT"
-            # Reset our branch to our expected target
-            git push origin $STAGING_BRANCH --force
-            # force gitrepo to point to the right HEAD (we ignore .gitrepo contents otherwise)
-            git config --file="$SUBREPO_PATH/.gitrepo" subrepo.commit "$BASE_NOIR_COMMIT"
-            # we need to commit for git-subrepo
-            git commit -am "[$LINES changes] $COMMIT_MESSAGE"
-            if ./scripts/git-subrepo/lib/git-subrepo pull --force $SUBREPO_PATH --branch=master; then
-              # Read our actual commit sync from git subrepo, stash to file for next time
-              COMMIT=$(git config --file="$SUBREPO_PATH/.gitrepo" subrepo.commit)
-              echo "$COMMIT" > .noir-sync-commit && git add .noir-sync-commit
-              git reset --soft "$BASE_AZTEC_COMMIT"
-              # We don't really need the sync commit on our side, and don't need .gitrepo at all except just in time for the command.
-              git checkout origin/master -- noir/noir-repo/.aztec-sync-commit noir/noir-repo/.gitrepo
-              git commit -am "[$LINES changes] $COMMIT_MESSAGE"
-
-              # There's various changes which we need to make to account for CI differences so we run this script to apply them.
-              git checkout origin/master -- noir/scripts/sync-in-fixup.sh
-              noir/scripts/sync-in-fixup.sh
-              git commit -am "chore: apply sync fixes"
-
-              git push origin $STAGING_BRANCH --force
-            else
-              echo "Problems syncing noir. Needs manual attention, might be a merge conflict."
-              exit 1
-            fi
-          }
-          # merge_staging_branch: Merge our staging branch into aztec-packages.
-          function merge_staging_branch() {
-            # Fix PR branch
-            git fetch # see recent change
-            git checkout $BRANCH || git checkout -b $BRANCH
-            if ! git merge -Xtheirs origin/$STAGING_BRANCH -m "$COMMIT_MESSAGE" ; then
-              # resolve modify/delete conflicts
-              git diff --name-only --diff-filter=U | xargs git rm
-              git commit -m "$COMMIT_MESSAGE"
-            fi
-            git push origin $BRANCH
-          }
-          force_sync_staging
-          if [[ "$PR_URL" != "" ]]; then
-            merge_staging_branch
-          fi
+          BRANCH=bump-noir
+          cd noir
+          ./bootstrap.sh bump-noir-repo-ref $BRANCH $LATEST_NOIR_NIGHTLY
 
+      # Update the PR from `bump-noir` to `master` with the latest message body.
       - name: Update PR
+        env:
+          BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
         run: |
           set -xue # print commands
           # Formatted for updating the PR, overrides for release-please commit message parsing
           PR_BODY="""
-          Automated pull of development from the [noir](https://github.com/noir-lang/noir) programming language, a dependency of Aztec.
+          Automated pull of nightly from the [noir](https://github.com/noir-lang/noir) programming language, a dependency of Aztec.
           BEGIN_COMMIT_OVERRIDE
           $(cat .PR_BODY_MESSAGE)
           END_COMMIT_OVERRIDE"""
           # for cross-opening PR in noir repo, we use aztecbot's token
           export GH_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
           if [[ "$PR_URL" == "" ]]; then
-            gh pr create --repo AztecProtocol/aztec-packages --title "feat: Sync from noir" --body "$PR_BODY" --base master --head sync-noir
+            gh pr create --repo AztecProtocol/aztec-packages --title "chore: Bump Noir reference" --body "$PR_BODY" --base $BRANCH_NAME --head bump-noir
           else
             echo "Updating existing PR."
             gh pr edit "$PR_URL" --body "$PR_BODY"

barretenberg/cpp/src/barretenberg/dsl/README.md

@@ -1,16 +1,17 @@
 # Domain Specific Language
 
-This package adds support to use [ACIR](https://github.com/noir-lang/noir/tree/master/acvm-repo/acir) with barretenberg.
+This package adds support to use [ACIR](https://github.com/noir-lang/noir/blob/master/tree/master/acvm-repo/acir) with barretenberg.
 
 ## Serialization Changes
 
 There are two types of breaking serialization changes. One that alters the internal ACIR structure passed to barretenberg, and one that changes how we serialize the buffer passed to barretenberg.
 
 1. Internal Structure Change
 
-Go to the ACVM `acir` crate and re-run the [serde reflection test](../../../../../noir/noir-repo/acvm-repo/acir/src/lib.rs#L51). Remember to comment out the hash check to write the updated C++ serde file. Copy that file into the `dsl` package's [serde folder](./acir_format/serde/) where you will see an `acir.hpp` file.
+Go to the ACVM `acir` crate and re-run the [serde reflection test](https://github.com/noir-lang/noir/blob/master/acvm-repo/acir/src/lib.rs#L51). Remember to comment out the hash check to write the updated C++ serde file. Copy that file into the `dsl` package's [serde folder](./acir_format/serde/) where you will see an `acir.hpp` file.
 
 You will have to update a couple things in the new `acir.hpp`:
+
 - Replace all `throw serde::deserialization_error` with `throw_or_abort`
 - The top-level struct (such as `Program`) will still use its own namespace for its internal fields. This extra `Program::` can be removed from the top-level `struct Program { .. }` object.
 
@@ -20,14 +21,15 @@ The same can then be done for any breaking changes introduced to `witness_stack.
 
 This type of breaking change is rarely expected to happen, however, due to its nature there are multiple consumers of the pre-existing serialization you should be aware of if you ever need to make this change.
 
-A full change is when you attempt to change the object whose buffer we are actually deserializing in barretenberg. To give more detail, when [deserializing the constraint buffer](./acir_format/acir_to_constraint_buf.hpp#366) if the object for which we call `bincodeDeserialize` on changes, anything that has pre-existing ACIR using the previous object's `bincodeDeserialize` method will now fail. The serialization is once again determined by the top-level object in the [acir crate](../../../../../noir/noir-repo/acvm-repo/acir/src/circuit/mod.rs). After performing the steps outlined for an internal structure breaking change as listed in (1) above, we need to update all consumers of barretenberg.
+A full change is when you attempt to change the object whose buffer we are actually deserializing in barretenberg. To give more detail, when [deserializing the constraint buffer](./acir_format/acir_to_constraint_buf.hpp#366) if the object for which we call `bincodeDeserialize` on changes, anything that has pre-existing ACIR using the previous object's `bincodeDeserialize` method will now fail. The serialization is once again determined by the top-level object in the [acir crate](https://github.com/noir-lang/noir/blob/master/acvm-repo/acir/src/circuit/mod.rs). After performing the steps outlined for an internal structure breaking change as listed in (1) above, we need to update all consumers of barretenberg.
 
-Even if you correctly update all serialization in [acvm_js](../../../../../noir/noir-repo/acvm-repo/acvm_js/README.md) such as during [execution](../../../../../noir/noir-repo/acvm-repo/acvm_js/src/execute.rs#57), there is multiple places the `yarn-project` uses the ACIR top-level serialization. The `yarn-project` sequencer also uses the native `acvm_cli tool` that has an execute method that [expects raw byte code](../../../../../noir/noir-repo/tooling/acvm_cli/src/cli/execute_cmd.rs#63).
+Even if you correctly update all serialization in [acvm_js](https://github.com/noir-lang/noir/blob/master/acvm-repo/acvm_js/README.md) such as during [execution](https://github.com/noir-lang/noir/blob/master/acvm-repo/acvm_js/src/execute.rs#57), there is multiple places the `yarn-project` uses the ACIR top-level serialization. The `yarn-project` sequencer also uses the native `acvm_cli tool` that has an execute method that [expects raw byte code](https://github.com/noir-lang/noir/blob/master/tooling/acvm_cli/src/cli/execute_cmd.rs#63).
 
 In the context of Aztec we need to regenerate all the artifacts in [noir-projects](../../../../../noir-projects/bootstrap.sh). This regeneration assumes that we have rebuilt the compilers (Noir compiler and AVM transpiler) to use the new serialization. After regenerating these artifacts we can bootstrap the yarn-project. There are multiple packages in the yarn-project that rely on pre-computed artifacts such as `yarn-project/stdlib` and `yarn-project/protocol-contracts`.
 
 The Aztec artifacts can be individually regenerated as well using `yarn test -u`.
 You can also run the command on the relevant workspaces, which at the time are the following:
+
 ```
 yarn workspace @aztec/stdlib test -u
 yarn workspace @aztec/protocol-contracts test -u

bootstrap.sh

@@ -54,7 +54,7 @@ function check_toolchains {
     exit 1
   fi
   # Check rustup installed.
-  local rust_version=$(yq '.toolchain.channel' ./noir/noir-repo/rust-toolchain.toml)
+  local rust_version=$(yq '.toolchain.channel' ./avm-transpiler/rust-toolchain.toml)
   if ! command -v rustup > /dev/null; then
     encourage_dev_container
     echo "Rustup not installed."
@@ -114,6 +114,8 @@ function install_hooks {
   echo "./noir-projects/precommit.sh" >>$hooks_dir/pre-commit
   echo "./yarn-project/constants/precommit.sh" >>$hooks_dir/pre-commit
   chmod +x $hooks_dir/pre-commit
+  echo "(cd noir && ./postcheckout.sh $@)" >$hooks_dir/post-checkout
+  chmod +x $hooks_dir/post-checkout
 }
 
 function test_cmds {

noir/.gitignore

@@ -1,2 +1,3 @@
 **/package.tgz
 packages
+noir-repo

noir/README.md

@@ -1,20 +1,40 @@
 # Aztec's Build of Noir
 
-We subrepo noir into the folder `noir-repo`.
+We subrepo [noir](https://github.com/noir-lang/noir) into the folder `noir-repo` during the build.
 This folder contains dockerfiles and scripts for performing our custom build of noir for the monorepo.
 
 # Syncing with the main Noir repository
 
 In order to keep aztec-packages in step with the main Noir repository we need to periodically sync between them.
 
-Syncing from aztec-packages into noir currently attempts to revert any changes in Noir since the last sync so it's recommended to always sync from Noir first to ensure that aztec-packages is up-to-date.
+Ideally there should be a one-way sync from Noir to aztec-packages, but occasionally, when the `bb` interface changes,
+or if some integration test fails and the Noir bug is fixed locally, changes have to go both ways.
 
-## Syncing from Noir to aztec-packages.
+## Syncing from Noir to aztec-packages
 
-To start the sync run [this action](https://github.com/AztecProtocol/aztec-packages/actions/workflows/pull-noir.yml) manually (click the "Run Workflow" button in the top right). aztec-bot will then open a new PR which does the initial sync, this will have merge conflicts with master which will need to be resolved.
+During the build the Noir repository is cloned or updated according to the contents of the [noir-repo-ref](./noir-repo-ref)
+file, which can be a tag, branch name or commit hash. The value can be overriden using the `NOIR_REPO_REF` environment variable,
+for example to run the integration tests in aztec-packages against a yet-to-be-released branch of Noir.
 
-## Syncing from aztec-packages to Noir.
+If the ref contains a branch, it's pulled with each build triggered by `bootstrap.sh`, but for repeatable builds it should
+point at a tag instead or commit instead, which would be updated with a regular PR opened in aztec-packages, so we can run
+the test suite before changes take effect.
+
+To start the sync run [this action](https://github.com/AztecProtocol/aztec-packages/actions/workflows/pull-noir.yml) manually (click the "Run Workflow" button in the top right). aztec-bot will then open a new PR which updates the reference. This will might merge conflicts with master which will need to be resolved.
+
+## Syncing from aztec-packages to Noir
 
 When syncing from aztec-packages to Noir it's important to check that the latest release of `bb` uses the same ACIR serialization format as the current master commit. This is because Noir uses a released version of barretenberg rather than being developed in sync with it, it's then not possible to sync if there's been serialization changes since the latest release.
 
-To start the sync run [this action](https://github.com/AztecProtocol/aztec-packages/actions/workflows/mirror-noir-subrepo.yml) manually (click the "Run Workflow" button in the top right). aztec-bot will then open a new PR in the `noir-lang/noir` repository which does the initial sync, this will have merge conflicts with master which will need to be resolved.
+When we make changes in `noir-repo` and commit them, we can check out a branch and push them back to Noir, where a PR can be opened to merge them back
+into an appropriate branch (could be `master` or some kind of integration branch). It is important to exclude the [fixup](./scripts/sync-in-fixup.sh) that the local checkout performs from the PR by running the [fixdown](./scripts/sync-out-fixup.sh) script.
+
+Syncing can be postponed by creating a few commits in `noir-repo`, but instead of opening a PR against Noir, creating a [git patch](https://git-scm.com/docs/git-format-patch) instead using, which is committed to aztec-packages and is applied to any subsequent checkout. A patch file can be made using the following command:
+
+```shell
+./bootstrap.sh make-patch
+```
+
+After this `./noir-repo.patch` should have the changes committed on top of the latest checkout, and if we commit this file to `aztec-packages` then it is automatically applied by any subsequent checkouts of `noir-repo`.
+
+To start an automated sync run [this action](https://github.com/AztecProtocol/aztec-packages/actions/workflows/mirror-noir-subrepo.yml) manually (click the "Run Workflow" button in the top right). aztec-bot will then open a new PR in the `noir-lang/noir` repository which does the initial sync, this will have merge conflicts with master which will need to be resolved.

noir/bootstrap.sh

@@ -1,9 +1,18 @@
 #!/usr/bin/env bash
 source $(git rev-parse --show-toplevel)/ci3/source_bootstrap
 
+set -eou pipefail
+
 cmd=${1:-}
 [ -n "$cmd" ] && shift
 
+# Update the noir-repo before we hash its content, unless the command is exempt.
+no_update=(clean make-patch bump-noir-repo-ref)
+if [[ -z "$cmd" || ! ${no_update[*]} =~ "$cmd" ]]; then
+  scripts/sync.sh init
+  scripts/sync.sh update
+fi
+
 export hash=$(cache_content_hash .rebuild_patterns)
 export test_hash=$(cache_content_hash .rebuild_patterns .rebuild_patterns_tests)
 
@@ -183,9 +192,27 @@ function release_commit {
   release_packages next "$CURRENT_VERSION-commit.$COMMIT_HASH"
 }
 
+# Bump the Noir repo reference on a given branch to a given ref.
+# The branch might already exist, e.g. this could be a daily job bumping the version to the
+# latest nightly, and we might have to deal with updating the patch file because the latest
+# Noir code conflicts with the contents of the patch, or we're debugging some integration
+# test failure on CI. In that case just push another commit to the branch to bump the version
+# further without losing any other commit on the branch.
+function bump_noir_repo_ref {
+  branch=$1
+  ref=$2
+  git fetch --depth 1 origin $branch || true
+  git checkout --track origin/$branch || git checkout $branch || git checkout -b $branch
+  scripts/sync.sh write-noir-repo-ref $ref
+  git add .
+  git commit -m "chore: Update noir-repo-ref to $ref" || true
+  do_or_dryrun git push --set-upstream origin $branch
+}
+
 case "$cmd" in
   "clean")
-    git clean -fdx
+    # Double `f` needed to delete the nested git repository.
+    git clean -ffdx
     ;;
   "ci")
     build
@@ -203,6 +230,12 @@ case "$cmd" in
   "hash-test")
     echo $test_hash
     ;;
+  "make-patch")
+    scripts/sync.sh make-patch
+    ;;
+  "bump-noir-repo-ref")
+    bump_noir_repo_ref $@
+    ;;
   *)
     echo "Unknown command: $cmd"
     exit 1

noir/noir-repo-ref

@@ -0,0 +1 @@
+nightly-2025-03-07