diff --git a/.github/workflows/pull-noir.yml b/.github/workflows/pull-noir.yml index c7f2d70b1612..26f83b79b9be 100644 --- a/.github/workflows/pull-noir.yml +++ b/.github/workflows/pull-noir.yml @@ -1,4 +1,4 @@ -# Create a pull request from current Noir master. +# Create a pull request from current Noir nightly. name: Pull from noir repo # Don't allow multiple of these running at once: @@ -23,25 +23,23 @@ jobs: fetch-depth: 0 token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + # Determine if there is already a PR to overwrite, and the current plus latest tags. - name: Check for existing PR run: | set -xue # print commands # Enable gh executable. We spread out the API requests between the github actions bot token, and aztecbot export GH_TOKEN="${{ secrets.GITHUB_TOKEN }}" # Do we have a PR active? - PR_URL=$(gh pr list --repo AztecProtocol/aztec-packages --head sync-noir --json url --jq ".[0].url") + PR_URL=$(gh pr list --repo AztecProtocol/aztec-packages --head bump-noir --json url --jq ".[0].url") echo "PR_URL=$PR_URL" >> $GITHUB_ENV - # What was our last merge on noir side? - # Detect our last sync commit (written by this action before pushing) with a fallback for the first time we ever do this - BASE_NOIR_COMMIT=$(curl https://raw.githubusercontent.com/AztecProtocol/aztec-packages/master/.noir-sync-commit) - if [ "$BASE_NOIR_COMMIT" = "404: Not Found" ] ; then - BASE_NOIR_COMMIT="50d2735825454a8638a308156d4ea23b3c4420d8" - fi - echo "BASE_NOIR_COMMIT=$BASE_NOIR_COMMIT" >> $GITHUB_ENV - # What was our last sync on aztec side? - BASE_AZTEC_COMMIT=`curl https://raw.githubusercontent.com/noir-lang/noir/master/.aztec-sync-commit` - echo "BASE_AZTEC_COMMIT=$BASE_AZTEC_COMMIT" >> $GITHUB_ENV + # Get the current Noir target + CURRENT_NOIR_REF=$(noir/scripts/sync.sh read-noir-repo-ref) + echo "CURRENT_NOIR_REF=$CURRENT_NOIR_REF" >> $GITHUB_ENV + # Get the latest nightly tag we can point to in Noir + LATEST_NOIR_NIGHTLY=$(noir/scripts/sync.sh latest-nightly) + echo "LATEST_NOIR_NIGHTLY=$LATEST_NOIR_NIGHTLY" >> $GITHUB_ENV + # Set PR body to be the concatenation of every message that happened in Noir since the last sync. - name: Generate PR body run: | # clone noir repo for manipulations, we use aztec bot token for writeability @@ -52,7 +50,7 @@ jobs: function compute_commit_message() { cd noir-repo # Create a filtered git log for release-please changelog / metadata - RAW_MESSAGE=$(git log --pretty=format:"%s" $BASE_NOIR_COMMIT..HEAD || true) + RAW_MESSAGE=$(git log --pretty=format:"%s" $CURRENT_NOIR_REF..$LATEST_NOIR_NIGHTLY || true) # Fix Noir PR links and output message echo "$RAW_MESSAGE" | sed -E 's/\(#([0-9]+)\)/(https:\/\/github.com\/noir-lang\/noir\/pull\/\1)/g' cd .. @@ -65,85 +63,30 @@ jobs: git config --global user.name AztecBot git config --global user.email tech@aztecprotocol.com - # We push using git subrepo (https://github.com/ingydotnet/git-subrepo) - # and push all Aztec commits as a single commit with metadata. + # Push a commit to the `bump-noir` branch that overrides the Noir ref. - name: Push to branch run: | set -xue # print commands - SUBREPO_PATH=noir/noir-repo - BRANCH=sync-noir - if [[ "$PR_URL" == "" ]]; then - # if no staging branch, we can overwrite - STAGING_BRANCH=$BRANCH - else - # otherwise we first reset our staging branch - STAGING_BRANCH=$BRANCH-staging - fi - - # Get the last sync PR's last commit state - COMMIT_MESSAGE=$(cat .PR_BODY_MESSAGE) - LINES=$(echo $COMMIT_MESSAGE | wc -l) - - function force_sync_staging() { - # reset to last noir merge - git checkout $STAGING_BRANCH || git checkout -b $STAGING_BRANCH - git reset --hard "$BASE_AZTEC_COMMIT" - # Reset our branch to our expected target - git push origin $STAGING_BRANCH --force - # force gitrepo to point to the right HEAD (we ignore .gitrepo contents otherwise) - git config --file="$SUBREPO_PATH/.gitrepo" subrepo.commit "$BASE_NOIR_COMMIT" - # we need to commit for git-subrepo - git commit -am "[$LINES changes] $COMMIT_MESSAGE" - if ./scripts/git-subrepo/lib/git-subrepo pull --force $SUBREPO_PATH --branch=master; then - # Read our actual commit sync from git subrepo, stash to file for next time - COMMIT=$(git config --file="$SUBREPO_PATH/.gitrepo" subrepo.commit) - echo "$COMMIT" > .noir-sync-commit && git add .noir-sync-commit - git reset --soft "$BASE_AZTEC_COMMIT" - # We don't really need the sync commit on our side, and don't need .gitrepo at all except just in time for the command. - git checkout origin/master -- noir/noir-repo/.aztec-sync-commit noir/noir-repo/.gitrepo - git commit -am "[$LINES changes] $COMMIT_MESSAGE" - - # There's various changes which we need to make to account for CI differences so we run this script to apply them. - git checkout origin/master -- noir/scripts/sync-in-fixup.sh - noir/scripts/sync-in-fixup.sh - git commit -am "chore: apply sync fixes" - - git push origin $STAGING_BRANCH --force - else - echo "Problems syncing noir. Needs manual attention, might be a merge conflict." - exit 1 - fi - } - # merge_staging_branch: Merge our staging branch into aztec-packages. - function merge_staging_branch() { - # Fix PR branch - git fetch # see recent change - git checkout $BRANCH || git checkout -b $BRANCH - if ! git merge -Xtheirs origin/$STAGING_BRANCH -m "$COMMIT_MESSAGE" ; then - # resolve modify/delete conflicts - git diff --name-only --diff-filter=U | xargs git rm - git commit -m "$COMMIT_MESSAGE" - fi - git push origin $BRANCH - } - force_sync_staging - if [[ "$PR_URL" != "" ]]; then - merge_staging_branch - fi + BRANCH=bump-noir + cd noir + ./bootstrap.sh bump-noir-repo-ref $BRANCH $LATEST_NOIR_NIGHTLY + # Update the PR from `bump-noir` to `master` with the latest message body. - name: Update PR + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} run: | set -xue # print commands # Formatted for updating the PR, overrides for release-please commit message parsing PR_BODY=""" - Automated pull of development from the [noir](https://github.com/noir-lang/noir) programming language, a dependency of Aztec. + Automated pull of nightly from the [noir](https://github.com/noir-lang/noir) programming language, a dependency of Aztec. BEGIN_COMMIT_OVERRIDE $(cat .PR_BODY_MESSAGE) END_COMMIT_OVERRIDE""" # for cross-opening PR in noir repo, we use aztecbot's token export GH_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} if [[ "$PR_URL" == "" ]]; then - gh pr create --repo AztecProtocol/aztec-packages --title "feat: Sync from noir" --body "$PR_BODY" --base master --head sync-noir + gh pr create --repo AztecProtocol/aztec-packages --title "chore: Bump Noir reference" --body "$PR_BODY" --base $BRANCH_NAME --head bump-noir else echo "Updating existing PR." gh pr edit "$PR_URL" --body "$PR_BODY" diff --git a/barretenberg/cpp/src/barretenberg/dsl/README.md b/barretenberg/cpp/src/barretenberg/dsl/README.md index 61fc0d4d05ae..9b0ecc0cb843 100644 --- a/barretenberg/cpp/src/barretenberg/dsl/README.md +++ b/barretenberg/cpp/src/barretenberg/dsl/README.md @@ -1,6 +1,6 @@ # Domain Specific Language -This package adds support to use [ACIR](https://github.com/noir-lang/noir/tree/master/acvm-repo/acir) with barretenberg. +This package adds support to use [ACIR](https://github.com/noir-lang/noir/blob/master/tree/master/acvm-repo/acir) with barretenberg. ## Serialization Changes @@ -8,9 +8,10 @@ There are two types of breaking serialization changes. One that alters the inter 1. Internal Structure Change -Go to the ACVM `acir` crate and re-run the [serde reflection test](../../../../../noir/noir-repo/acvm-repo/acir/src/lib.rs#L51). Remember to comment out the hash check to write the updated C++ serde file. Copy that file into the `dsl` package's [serde folder](./acir_format/serde/) where you will see an `acir.hpp` file. +Go to the ACVM `acir` crate and re-run the [serde reflection test](https://github.com/noir-lang/noir/blob/master/acvm-repo/acir/src/lib.rs#L51). Remember to comment out the hash check to write the updated C++ serde file. Copy that file into the `dsl` package's [serde folder](./acir_format/serde/) where you will see an `acir.hpp` file. You will have to update a couple things in the new `acir.hpp`: + - Replace all `throw serde::deserialization_error` with `throw_or_abort` - The top-level struct (such as `Program`) will still use its own namespace for its internal fields. This extra `Program::` can be removed from the top-level `struct Program { .. }` object. @@ -20,14 +21,15 @@ The same can then be done for any breaking changes introduced to `witness_stack. This type of breaking change is rarely expected to happen, however, due to its nature there are multiple consumers of the pre-existing serialization you should be aware of if you ever need to make this change. -A full change is when you attempt to change the object whose buffer we are actually deserializing in barretenberg. To give more detail, when [deserializing the constraint buffer](./acir_format/acir_to_constraint_buf.hpp#366) if the object for which we call `bincodeDeserialize` on changes, anything that has pre-existing ACIR using the previous object's `bincodeDeserialize` method will now fail. The serialization is once again determined by the top-level object in the [acir crate](../../../../../noir/noir-repo/acvm-repo/acir/src/circuit/mod.rs). After performing the steps outlined for an internal structure breaking change as listed in (1) above, we need to update all consumers of barretenberg. +A full change is when you attempt to change the object whose buffer we are actually deserializing in barretenberg. To give more detail, when [deserializing the constraint buffer](./acir_format/acir_to_constraint_buf.hpp#366) if the object for which we call `bincodeDeserialize` on changes, anything that has pre-existing ACIR using the previous object's `bincodeDeserialize` method will now fail. The serialization is once again determined by the top-level object in the [acir crate](https://github.com/noir-lang/noir/blob/master/acvm-repo/acir/src/circuit/mod.rs). After performing the steps outlined for an internal structure breaking change as listed in (1) above, we need to update all consumers of barretenberg. -Even if you correctly update all serialization in [acvm_js](../../../../../noir/noir-repo/acvm-repo/acvm_js/README.md) such as during [execution](../../../../../noir/noir-repo/acvm-repo/acvm_js/src/execute.rs#57), there is multiple places the `yarn-project` uses the ACIR top-level serialization. The `yarn-project` sequencer also uses the native `acvm_cli tool` that has an execute method that [expects raw byte code](../../../../../noir/noir-repo/tooling/acvm_cli/src/cli/execute_cmd.rs#63). +Even if you correctly update all serialization in [acvm_js](https://github.com/noir-lang/noir/blob/master/acvm-repo/acvm_js/README.md) such as during [execution](https://github.com/noir-lang/noir/blob/master/acvm-repo/acvm_js/src/execute.rs#57), there is multiple places the `yarn-project` uses the ACIR top-level serialization. The `yarn-project` sequencer also uses the native `acvm_cli tool` that has an execute method that [expects raw byte code](https://github.com/noir-lang/noir/blob/master/tooling/acvm_cli/src/cli/execute_cmd.rs#63). In the context of Aztec we need to regenerate all the artifacts in [noir-projects](../../../../../noir-projects/bootstrap.sh). This regeneration assumes that we have rebuilt the compilers (Noir compiler and AVM transpiler) to use the new serialization. After regenerating these artifacts we can bootstrap the yarn-project. There are multiple packages in the yarn-project that rely on pre-computed artifacts such as `yarn-project/stdlib` and `yarn-project/protocol-contracts`. The Aztec artifacts can be individually regenerated as well using `yarn test -u`. You can also run the command on the relevant workspaces, which at the time are the following: + ``` yarn workspace @aztec/stdlib test -u yarn workspace @aztec/protocol-contracts test -u diff --git a/bootstrap.sh b/bootstrap.sh index 56a8022409a3..83fc08127448 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -54,7 +54,7 @@ function check_toolchains { exit 1 fi # Check rustup installed. - local rust_version=$(yq '.toolchain.channel' ./noir/noir-repo/rust-toolchain.toml) + local rust_version=$(yq '.toolchain.channel' ./avm-transpiler/rust-toolchain.toml) if ! command -v rustup > /dev/null; then encourage_dev_container echo "Rustup not installed." @@ -114,6 +114,8 @@ function install_hooks { echo "./noir-projects/precommit.sh" >>$hooks_dir/pre-commit echo "./yarn-project/constants/precommit.sh" >>$hooks_dir/pre-commit chmod +x $hooks_dir/pre-commit + echo "(cd noir && ./postcheckout.sh $@)" >$hooks_dir/post-checkout + chmod +x $hooks_dir/post-checkout } function test_cmds { diff --git a/noir/.gitignore b/noir/.gitignore index 77abdbdb12e3..e330e0935763 100644 --- a/noir/.gitignore +++ b/noir/.gitignore @@ -1,2 +1,3 @@ **/package.tgz packages +noir-repo diff --git a/noir/README.md b/noir/README.md index 25575a0383f7..0f99c668527d 100644 --- a/noir/README.md +++ b/noir/README.md @@ -1,20 +1,40 @@ # Aztec's Build of Noir -We subrepo noir into the folder `noir-repo`. +We subrepo [noir](https://github.com/noir-lang/noir) into the folder `noir-repo` during the build. This folder contains dockerfiles and scripts for performing our custom build of noir for the monorepo. # Syncing with the main Noir repository In order to keep aztec-packages in step with the main Noir repository we need to periodically sync between them. -Syncing from aztec-packages into noir currently attempts to revert any changes in Noir since the last sync so it's recommended to always sync from Noir first to ensure that aztec-packages is up-to-date. +Ideally there should be a one-way sync from Noir to aztec-packages, but occasionally, when the `bb` interface changes, +or if some integration test fails and the Noir bug is fixed locally, changes have to go both ways. -## Syncing from Noir to aztec-packages. +## Syncing from Noir to aztec-packages -To start the sync run [this action](https://github.com/AztecProtocol/aztec-packages/actions/workflows/pull-noir.yml) manually (click the "Run Workflow" button in the top right). aztec-bot will then open a new PR which does the initial sync, this will have merge conflicts with master which will need to be resolved. +During the build the Noir repository is cloned or updated according to the contents of the [noir-repo-ref](./noir-repo-ref) +file, which can be a tag, branch name or commit hash. The value can be overriden using the `NOIR_REPO_REF` environment variable, +for example to run the integration tests in aztec-packages against a yet-to-be-released branch of Noir. -## Syncing from aztec-packages to Noir. +If the ref contains a branch, it's pulled with each build triggered by `bootstrap.sh`, but for repeatable builds it should +point at a tag instead or commit instead, which would be updated with a regular PR opened in aztec-packages, so we can run +the test suite before changes take effect. + +To start the sync run [this action](https://github.com/AztecProtocol/aztec-packages/actions/workflows/pull-noir.yml) manually (click the "Run Workflow" button in the top right). aztec-bot will then open a new PR which updates the reference. This will might merge conflicts with master which will need to be resolved. + +## Syncing from aztec-packages to Noir When syncing from aztec-packages to Noir it's important to check that the latest release of `bb` uses the same ACIR serialization format as the current master commit. This is because Noir uses a released version of barretenberg rather than being developed in sync with it, it's then not possible to sync if there's been serialization changes since the latest release. -To start the sync run [this action](https://github.com/AztecProtocol/aztec-packages/actions/workflows/mirror-noir-subrepo.yml) manually (click the "Run Workflow" button in the top right). aztec-bot will then open a new PR in the `noir-lang/noir` repository which does the initial sync, this will have merge conflicts with master which will need to be resolved. +When we make changes in `noir-repo` and commit them, we can check out a branch and push them back to Noir, where a PR can be opened to merge them back +into an appropriate branch (could be `master` or some kind of integration branch). It is important to exclude the [fixup](./scripts/sync-in-fixup.sh) that the local checkout performs from the PR by running the [fixdown](./scripts/sync-out-fixup.sh) script. + +Syncing can be postponed by creating a few commits in `noir-repo`, but instead of opening a PR against Noir, creating a [git patch](https://git-scm.com/docs/git-format-patch) instead using, which is committed to aztec-packages and is applied to any subsequent checkout. A patch file can be made using the following command: + +```shell +./bootstrap.sh make-patch +``` + +After this `./noir-repo.patch` should have the changes committed on top of the latest checkout, and if we commit this file to `aztec-packages` then it is automatically applied by any subsequent checkouts of `noir-repo`. + +To start an automated sync run [this action](https://github.com/AztecProtocol/aztec-packages/actions/workflows/mirror-noir-subrepo.yml) manually (click the "Run Workflow" button in the top right). aztec-bot will then open a new PR in the `noir-lang/noir` repository which does the initial sync, this will have merge conflicts with master which will need to be resolved. diff --git a/noir/bootstrap.sh b/noir/bootstrap.sh index bc6d5b3e2106..659a67f0c14f 100755 --- a/noir/bootstrap.sh +++ b/noir/bootstrap.sh @@ -1,9 +1,18 @@ #!/usr/bin/env bash source $(git rev-parse --show-toplevel)/ci3/source_bootstrap +set -eou pipefail + cmd=${1:-} [ -n "$cmd" ] && shift +# Update the noir-repo before we hash its content, unless the command is exempt. +no_update=(clean make-patch bump-noir-repo-ref) +if [[ -z "$cmd" || ! ${no_update[*]} =~ "$cmd" ]]; then + scripts/sync.sh init + scripts/sync.sh update +fi + export hash=$(cache_content_hash .rebuild_patterns) export test_hash=$(cache_content_hash .rebuild_patterns .rebuild_patterns_tests) @@ -183,9 +192,27 @@ function release_commit { release_packages next "$CURRENT_VERSION-commit.$COMMIT_HASH" } +# Bump the Noir repo reference on a given branch to a given ref. +# The branch might already exist, e.g. this could be a daily job bumping the version to the +# latest nightly, and we might have to deal with updating the patch file because the latest +# Noir code conflicts with the contents of the patch, or we're debugging some integration +# test failure on CI. In that case just push another commit to the branch to bump the version +# further without losing any other commit on the branch. +function bump_noir_repo_ref { + branch=$1 + ref=$2 + git fetch --depth 1 origin $branch || true + git checkout --track origin/$branch || git checkout $branch || git checkout -b $branch + scripts/sync.sh write-noir-repo-ref $ref + git add . + git commit -m "chore: Update noir-repo-ref to $ref" || true + do_or_dryrun git push --set-upstream origin $branch +} + case "$cmd" in "clean") - git clean -fdx + # Double `f` needed to delete the nested git repository. + git clean -ffdx ;; "ci") build @@ -203,6 +230,12 @@ case "$cmd" in "hash-test") echo $test_hash ;; + "make-patch") + scripts/sync.sh make-patch + ;; + "bump-noir-repo-ref") + bump_noir_repo_ref $@ + ;; *) echo "Unknown command: $cmd" exit 1 diff --git a/noir/noir-repo-ref b/noir/noir-repo-ref new file mode 100644 index 000000000000..73cf7801022a --- /dev/null +++ b/noir/noir-repo-ref @@ -0,0 +1 @@ +nightly-2025-03-07 diff --git a/noir/postcheckout.sh b/noir/postcheckout.sh new file mode 100755 index 000000000000..7394debab8e9 --- /dev/null +++ b/noir/postcheckout.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Post-checkout hook for warning the user to create a patch file before running a build, +# if there are outstanding commits that could get lost if we switch to a different noir-repo checkout. +set -euo pipefail + +cd $(dirname $0) + +is_branch=$3 + +if [ "$is_branch" == "1" ] && scripts/sync.sh needs-patch; then + echo "Warning: the noir-repo has outstanding commits that need to be put in a patch file" + echo "with the './noir/bootstrap.sh make-patch' command, then committed to the appropriate branch" + echo "in aztec-packages in order to ensure they don't get lost if the noir-repo is switched." +fi diff --git a/noir/scripts/sync.sh b/noir/scripts/sync.sh new file mode 100755 index 000000000000..df0de7a8dc1f --- /dev/null +++ b/noir/scripts/sync.sh @@ -0,0 +1,401 @@ +#!/usr/bin/env bash +set -eu + +# Commands related to syncing with the noir-repo. + +NOIR_REPO_URL=https://github.com/noir-lang/noir.git +# Special message we use to indicate the commit we do after the fixup. +# This commit has changes we do *not* want to migrate back to Noir, +# they exist only to make the Noir codebase work with the projects +# in aztec-packages, rather than their released versions. +FIXUP_COMMIT_MSG="Noir local fixup commit." +# Special message we use to indicate the commit we do after applying +# any patch file committed in aztec-packages, which contains changes +# Aztec developers made to Noir. These are changes we do want to see +# migrated back to Noir eventually, after which the patch file can +# be removed. +PATCH_COMMIT_MSG="Noir local patch commit." +# There can be a patch file committed in `aztec-packages` with commits +# to be applied on top of any Noir checkout. +# The `noir/bootstrap.sh make-patch` commands takes all commits since +# the local fixup commit and compiles them into this single patch file, +# (so it will include previously applied patch commits as well as new ones), +# replacing any previous value. +# The patch commits can be pushed to Noir if they represent bugfixes; +# to do so we have to rebase on the origin and remove the fixup commit. +NOIR_REPO_PATCH=noir-repo.patch +# Certain commands such as `noir/bootstrap.sh test_cmds` are expected to print +# executable scripts, which would be corrupted by any extra logs coming from here. +NOIR_REPO_VERBOSE=${NOIR_REPO_VERBOSE:-0} + +cd $(dirname $0)/.. + +function log { + if [ "${NOIR_REPO_VERBOSE}" -eq 1 ]; then + echo $@ + fi +} + +# Read the commitish that we have to check out from the marker file +# or an env var to facilitate overriding it on CI for nightly tests. +function read_wanted_ref { + ref=${NOIR_REPO_REF:-} + if [ ! -z "$ref" ]; then + echo $ref + return + fi + cat noir-repo-ref +} + +function write_wanted_ref { + ref=$1 + if [ -z "$ref" ]; then + echo "noir-repo-ref cannot be empty" + exit 1 + fi + echo $ref > noir-repo-ref +} + +# Return the current branch name, or fail if we're not on a branch. +function branch_name { + repo_exists && git -C noir-repo symbolic-ref --short -q HEAD +} + +# Check that the repo exists +function repo_exists { + [ -d noir-repo ] && [ -d noir-repo/.git ] +} + +# Check if we are on a branch. +function is_on_branch { + test $(branch_name) +} + +# Check if we are on a detached HEAD, which means if we switch branches +# it would be difficult to recover any changes committed on this branch. +function is_detached_head { + repo_exists && ! is_on_branch +} + +# Check if noir-repo has uncommitted changes. +function has_uncommitted_changes { + ! repo_exists && return 1 + # Add any untracked files, because otherwise we might switch branches, + # apply the patch fixup, create an artifical commit and inadvertedly + # add these files to a commit they have nothing to do with. + if git -C noir-repo add . && \ + git -C noir-repo diff --quiet && \ + git -C noir-repo diff --cached --quiet ; then + return 1 # false + else + return 0 # true + fi +} + +# Check if the last commit is marked with the local patch message. +function is_last_commit_patch { + ! repo_exists && return 1 + last_msg=$(git -C noir-repo rev-list --max-count=1 --no-commit-header --format=%B HEAD) + test "$last_msg" == "$PATCH_COMMIT_MSG" +} + +# Check if we have applied the fixup in any commit in the log. +# It is possible that we checkout a branch, apply the patch, then go into noir-repo +# and work on various fixes, committing them as we go. In that case the patch won't +# be the last commit, but it doesn't have to be applied again if we switch away +# from our branch and then come back to it later. +function has_patch_commit { + ! repo_exists && return 1 + if git -C noir-repo rev-list --no-commit-header --format=%B HEAD | grep -q --max-count=1 "$PATCH_COMMIT_MSG" ; then + return 0 # true + else + return 1 # false + fi +} + +# Find the commit hash of the last applied fixup commit. +# This is a commit we don't want to include in a patch file. +function find_fixup_commit { + ! repo_exists && return 1 + fixup=$(git -C noir-repo log --oneline --no-abbrev-commit --grep "$FIXUP_COMMIT_MSG" --max-count=1 | awk '{print $1}') + [ -z "$fixup" ] && return 1 + echo $fixup +} + +# Find the commit hash of the last checkout. +function find_checkout_commit { + ! repo_exists && return 1 + fixup=$(find_fixup_commit) + [ -z "$fixup" ] && return 1 + git -C noir-repo rev-parse "$fixup~1" +} + +# Check if a ref is a tag we have locally +function has_tag { + ! repo_exists && return 1 + tag=$1 + test $(git -C noir-repo tag --list $tag) +} + +# Get the commit a tag *currently* refers to on the remote and check if we have it in our local history. +function has_tag_commit { + ! repo_exists && return 1 + tag=$1 + rev=$(git -C noir-repo ls-remote --tags origin $tag | awk '{print $1}') + if [ ! -z "$rev" ]; then + # NB `git show` would tell if we have the commit, but it would not necessarily be an ancestor. + if git -C noir-repo log --oneline --no-abbrev-commit | grep -q --max-count=1 "$rev"; then + return 0 + fi + fi + return 1 +} + +# Indicate that the `make-patch` command should be used to create a new patch file. +function needs_patch { + is_detached_head && ! is_last_commit_patch +} + +# Indicate that both the fixup and the patch has been applied. +function has_fixup_and_patch { + find_fixup_commit>/dev/null && has_patch_commit +} + +# Indicate that we have to switch to the wanted branch. +function needs_switch { + ! repo_exists && return 0 # true + want=$(read_wanted_ref) + have=false + # Are we on the wanted branch? + if is_on_branch && [ "$(branch_name)" == "$want" ]; then + have=true + fi + # Are we descending from the wanted commit? + if [ "$(find_checkout_commit)" == "$want" ]; then + have=true + fi + # Are we descending from the wanted tag? + if has_tag "$want" && has_tag_commit "$want"; then + have=true + fi + # If we're on the wanted checkout, it could be that we rebased on origin in order + # to push the branch without the fixup commit. In that case we'd need to do a fresh + # checkout to re-apply the fixup and the patches. + if [ $have == true ] && has_fixup_and_patch; then + return 1 # false + else + return 0 # true + fi +} + +# Find the latest nightly tag in the upstream repo +function latest_nightly { + git ls-remote --tags --sort -refname $NOIR_REPO_URL nightly-* | head -n 1 | awk '{split($2,a,"/"); print a[3]}' +} + +# Create an empty marker commit to show that patches have been applied or put in a patch file. +function commit_patch_marker { + git -C noir-repo commit -m "$PATCH_COMMIT_MSG" --allow-empty +} + +# Apply the fixup script and any local patch file. +function patch_repo { + log Applying fixup on noir-repo + # Redirect the `bb` reference to the local one. + scripts/sync-in-fixup.sh + git -C noir-repo add . && git -C noir-repo commit -m "$FIXUP_COMMIT_MSG" --allow-empty + # Apply any patch file. + if [ -f $NOIR_REPO_PATCH ]; then + log Applying patches from $NOIR_REPO_PATCH + git -C noir-repo am ../$NOIR_REPO_PATCH + else + log "No patch file to apply" + fi + # Create an empty marker commit to show that patches have been applied. + commit_patch_marker +} + +# Check out a tag, branch or commit. +function switch_repo { + ref=$1 + log Switching noir-repo to $ref + git -C noir-repo fetch --tags --depth 1 origin + # If we try to switch to some random commit after a branch it might not find it locally. + git -C noir-repo fetch --depth 1 origin $ref || true + # Try to check out an existing branch, or remote commit. + if git -C noir-repo checkout $ref; then + # If it's a branch we just need to pull the latest changes. + if is_on_branch; then + git -C noir-repo pull --rebase + fi + else + # If the checkout failed, then it should be a remote branch or tag + git -C noir-repo checkout --track origin/$ref + fi + # If we haven't applied the patch yet, we have to do it (again). + if ! has_patch_commit; then + patch_repo + else + log "Patches already applied" + fi +} + +# Clone the repository if it doesn't exist. +function init_repo { + url=$NOIR_REPO_URL + ref=$(read_wanted_ref) + if [ -d noir-repo ] && [ ! -d noir-repo/.git ]; then + # In all probability we just have some build leftovers after switching branches on aztec-packages, + # but play it safe and preserve them: instead of deleting `noir-repo` just re-initialize it. + git -C noir-repo init + git -C noir-repo remote add origin $url + git -C noir-repo config advice.detachedHead false + switch_repo $ref + elif [ ! -d noir-repo ]; then + log Initializing noir-repo to $ref + # If we're cloning from a tag with --depth=1, we won't be able to switch to a branch later. + # On CI we won't be switching branches, but on dev machines we can, so there make a full checkout. + depth=$([ ! -z "${CI_FULL:-}" ] && echo "--depth 1" || echo "") + # Switch off detached head warnings in the cloned repo. + advice="-c advice.detachedHead=false" + # `--branch` doesn't work for commit hashes + git clone $advice $depth --branch $ref $url noir-repo \ + || git clone $advice $url noir-repo && git -C noir-repo checkout $ref + patch_repo + fi +} + +# Bring the noir-repo in line with the commit marker. +function update_repo { + want=$(read_wanted_ref) + if ! needs_switch; then + log "noir-repo already on $want" + if is_on_branch; then + # If we're on a branch, then we there might be new commits. + # Rebasing so our local patch commit ends up on top. + # Quiet so we don't interfere with `test_cmds` + git -C noir-repo pull --rebase --quiet + return + elif ! has_tag $want; then + # We are on what we wanted and it's _not_ a tag, so it must be a commit, and we have nothing more to do. + return + elif has_tag_commit $want; then + # We checked out a tag, and it looks like it hasn't been moved to a different commit. + return + else + log "The current commit of the tag doesn't appear in our history." + fi + fi + + # We need to switch branches. + + if has_uncommitted_changes; then + echo "Error: noir-repo has uncommitted changes which could get lost if we switch to $want" + echo "Please commit these changes and consider pushing them upstream to make sure they are not lost." + exit 1 + fi + + if needs_patch; then + echo "Error: noir-repo is on a detached HEAD and the last commit is not the patch marker commit;" + echo "switching to $want could mean losing those commits." + echo "Please use the 'make-patch' command to create a $NOIR_REPO_PATCH file and commit it in aztec-packages, " + echo "so that it is re-applied after each checkout. Make sure to commit the patch on the branch where it should be." + exit 1 + fi + + switch_repo $want +} + +# Create a patch file from any outstanding commits in noir-repo +function make_patch { + if is_last_commit_patch; then + echo "The last commit is the patch commit, there is nothing new to put in a patch." + exit 0 + fi + fixup_rev=$(find_fixup_commit) + if [ -z "$fixup_rev" ]; then + echo "Could not determine the fixup commit hash, which is the commit we would like to apply patches from" + exit 1 + fi + mkdir -p patches + # The patch marker commit could be in the middle: fixup-commit from-patch-1 from-patch-2 patch-commit new-fix-1 new-fix-2 + # wo we write all patches to files, then exclude the one which is just the empty patch commit. + git -C noir-repo format-patch -o ../patches $fixup_rev..HEAD + # In theory we should be able to apply empty patches `git am --allow-empty`, but it seems to choke on them, + # so we only keep non-empty patches, which conveniently also excludes `000*-Noir-local-patch-commit.patch` as well. + rm -f $NOIR_REPO_PATCH + for patch in $(find patches -name "*.patch"); do + # --- seems to separtate the files in a patch; in an empty patch it does not appear + if cat $patch | grep -q "\-\-\-" ; then + cat $patch >> $NOIR_REPO_PATCH + fi + done + rm -rf patches + # Create an empty patch marker commit at the end to show that it is safe to switch now. + if ! is_last_commit_patch; then + commit_patch_marker + fi +} + +# Show debug information +function info { + function pad { + printf "%$2.${2#-}s" "$1"; + } + function echo_info { + echo "$(pad "$1:" -25)" $2 + } + function yesno { + $@ && echo "yes" || echo "no" + } + want=$(read_wanted_ref) + echo_info "Repo exists" $(yesno repo_exists) + echo_info "Fixup commit" $(find_fixup_commit || echo "n/a") + echo_info "Checkout commit" $(find_checkout_commit || echo "n/a") + echo_info "Wanted" $want + echo_info "Needs switch" $(yesno needs_switch) + echo_info "Needs patch" $(yesno needs_patch) + echo_info "Detached" $(yesno is_detached_head) + echo_info "On branch" $(yesno is_on_branch) + echo_info "Branch name" $(branch_name || echo "n/a") + echo_info "Has wanted tag" $(yesno has_tag $want) + echo_info "Has tag commit" $(yesno has_tag_commit $want) + echo_info "Has patch commit" $(yesno has_patch_commit) + echo_info "Last commit is patch" $(yesno is_last_commit_patch) + echo_info "Has fixup and patch" $(yesno has_fixup_and_patch) + echo_info "Has uncommitted changes" $(yesno has_uncommitted_changes) + echo_info "Latest nightly" $(latest_nightly) +} + +cmd=${1:-} +[ -n "$cmd" ] && shift + +case "$cmd" in + "init") + init_repo + ;; + "update") + update_repo + ;; + "make-patch") + make_patch + ;; + "needs-patch") + [ -d noir-repo ] && [ -d noir-repo/.git ] && needs_patch && exit 0 || exit 1 + ;; + "latest-nightly") + echo $(latest_nightly) + ;; + "read-noir-repo-ref") + echo $(read_wanted_ref) + ;; + "write-noir-repo-ref") + write_wanted_ref $1 + ;; + "info") + info + ;; + *) + echo "Unknown command: $cmd" + exit 1 +esac