diff --git a/.github/scripts/wait_for_infra.sh b/.github/scripts/wait_for_infra.sh deleted file mode 100755 index 7f44911f43dc..000000000000 --- a/.github/scripts/wait_for_infra.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -e - -INFRA=$1 -DEPLOY_TAG=$2 -API_KEY=$3 - -# When destroying and applying terraforms, they may not be -# ready for a while, as it must register with DNS etc. -# This script waits on a healthy status from the infra - a valid response to a request -# We retry every 20 seconds, and wait for a total of 5 minutes (15 times) - -if [ "$INFRA" == "mainnet-fork" ]; then - export ETHEREUM_HOSTS="https://$DEPLOY_TAG-mainnet-fork.aztec.network:8545/$API_KEY" - curl -H "Content-Type: application/json" -X POST --data '{"method":"eth_chainId","params":[],"id":49,"jsonrpc":"2.0"}' \ - --connect-timeout 30 \ - --retry 15 \ - --retry-delay 20 \ - $ETHEREUM_HOSTS -elif [ "$INFRA" == "pxe" ]; then - export PXE_URL="https://api.aztec.network/$DEPLOY_TAG/aztec-pxe/$API_KEY/status" - curl \ - --connect-timeout 30 \ - --retry 15 \ - --retry-delay 20 \ - $PXE_URL -else - echo "Invalid infra type" - exit 1 -fi diff --git a/.github/workflows/bb-sanitizers.yml b/.github/workflows/bb-sanitizers.yml deleted file mode 100644 index 02e01555c2f4..000000000000 --- a/.github/workflows/bb-sanitizers.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Checks bb (barretenberg prover library) prover with sanitizers -# Unlike most jobs uses free 4 core github runners of which we have lots of capacity (of total 1000 concurrency). -name: BB MSAN -on: -# push: -# branches: -# - master -# - "*/bb-sanitizers*" - workflow_dispatch: - inputs: - branch: - required: false - -concurrency: - # force parallelism in master - group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} - cancel-in-progress: true -jobs: - # acts as prover performance baseline - bb-baseline: - runs-on: ubuntu-latest - continue-on-error: true - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch || github.sha }} - - uses: earthly/actions-setup@v1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: "Barretenberg Baseline Performance Check" - run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=clang16 - - # memory sanitzer for prover - bb-msan-check: - runs-on: ubuntu-latest - continue-on-error: true - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch || github.sha }} - - uses: earthly/actions-setup@v1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: "Barretenberg Prover MSAN Check" - - run: earthly --no-cache ./barretenberg/cpp/+preset-msan-check || true - - # address sanitzer for prover - bb-asan-check: - runs-on: ubuntu-latest - continue-on-error: true - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch || github.sha }} - - uses: earthly/actions-setup@v1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: "Barretenberg Prover ASAN Check" - timeout-minutes: 720 # 12 hour timeout (proving) - run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=asan - - # address sanitzer for prover - bb-tsan-check: - runs-on: ubuntu-latest - continue-on-error: true - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch || github.sha }} - - uses: earthly/actions-setup@v1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: "Barretenberg Prover TSAN Check" - timeout-minutes: 720 # 12 hour timeout (proving) - run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=tsan || true - - # undefined behavior sanitzer for prover - bb-ubsan-check: - runs-on: ubuntu-latest - continue-on-error: true - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - - uses: earthly/actions-setup@v1 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: "Barretenberg Prover TSAN Check" - timeout-minutes: 720 # 12 hour timeout (proving) - run: earthly --no-cache ./barretenberg/cpp/+preset-check --preset=ubsan diff --git a/.github/workflows/ci-mac.yml b/.github/workflows/ci-mac.yml deleted file mode 100644 index 6f97da53e392..000000000000 --- a/.github/workflows/ci-mac.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: CI (Mac) - -on: - push: - branches: - - master - - '*/*mac-build' - pull_request: - types: [synchronize, labeled] - -jobs: - build-mac-intel: - name: Build on Mac x86_64-apple-darwin - # Run this job if it's a direct push OR if a PR is labeled with "macos-ci". - if: > - github.event_name == 'push' || - (github.event_name == 'pull_request' && - contains(github.event.pull_request.labels.*.name, 'macos-ci')) - runs-on: macos-13 - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Create Mac Build Environment - run: brew install cmake ninja llvm@16 - - - name: Compile Barretenberg - working-directory: barretenberg/cpp - run: | - export PATH="/usr/local/opt/llvm@16/bin:$PATH" - export LDFLAGS="-L/usr/local/opt/llvm@16/lib" - export CPPFLAGS="-I/usr/local/opt/llvm@16/include" - cmake -DCMAKE_BUILD_TYPE=RelWithAssert --preset default - cmake --build --preset default --target bb - - build-mac-m1: - name: Build on Mac aarch64-apple-darwin - # Run this job if it's a direct push OR if a PR is labeled with "macos-ci". - if: > - github.event_name == 'push' || - (github.event_name == 'pull_request' && - contains(github.event.pull_request.labels.*.name, 'macos-ci')) - runs-on: macos-14 - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Create Mac Build Environment - run: brew install cmake ninja - - - name: Compile Barretenberg - working-directory: barretenberg/cpp - run: | - cmake --preset default -DCMAKE_BUILD_TYPE=RelWithAssert - cmake --build --preset default --target bb - - build-check: - name: Check builds are successful - needs: [build-mac-intel, build-mac-m1] - if: always() - runs-on: ubuntu-latest - steps: - - name: Report overall success - env: - FAIL: ${{ contains(needs.*.result, 'failure') }} - run: | - if [[ $FAIL == true ]]; then - echo "At least one job failed, release is unsuccessful." - exit 1 - else - echo "All jobs succeeded, uploading artifacts to release." - exit 0 - fi - - - name: Send notification to aztec3-ci channel if workflow failed on master - if: ${{ failure() }} - uses: slackapi/slack-github-action@v1.25.0 - with: - payload: | - { - "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WORKFLOW_TRIGGER_URL }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 5cadd97f0133..000000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,725 +0,0 @@ -# THIS IS NO LONGER USED -# PLEASE FAMILARIZE YOURSELF WITH ci3.yml -# SEE ROOT README.md FOR DETAILS. - -name: CI -on: - # push: - # branches: - # - master - # - provernet - # - "*/ci-push*" - # merge_group: - # pull_request: - # types: - # - opened - # - reopened - # - synchronize - # - ready_for_review - # - labeled - # branches-ignore: [devnet] - workflow_dispatch: - inputs: {} - -concurrency: - # allow parallelism in master and merge queue - group: ci-${{ (contains(github.ref_name, 'gh-readonly-queue') || github.ref_name == 'master') && github.run_id || github.ref_name }} - cancel-in-progress: true - -env: - CI: 1 - DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_PASSWORD }}" - RUN_ID: ${{ github.run_id }} - RUN_ATTEMPT: ${{ github.run_attempt }} - GITHUB_TOKEN: ${{ github.token }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GITHUB_RUN_URL: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}" - GIT_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} - RUNNER_CONCURRENCY: 0 -jobs: - configure: - runs-on: ubuntu-latest - if: github.event.pull_request.draft == false - # Required permissions. - permissions: - pull-requests: read - # Set job outputs to values from filter step. - # Set username to a value derived from our github username. - outputs: - username: ${{ steps.compute_username.outputs.username }} - base-images: ${{ steps.filter.outputs.base-images }} - noir: ${{ steps.filter.outputs.noir }} - noir-projects: ${{ steps.filter.outputs.noir-projects }} - bb: ${{ steps.filter.outputs.bb }} - yarn-project: ${{ steps.filter.outputs.yarn-project }} - non-docs: ${{ steps.filter.outputs.non-docs }} - non-bb: ${{ steps.filter.outputs.non-bb }} - helm-charts: ${{ steps.filter.outputs.helm-charts }} - # Test flags: - e2e-all: ${{ github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'e2e-all') || github.event_name == 'merge_group' }} - network-all: ${{ github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'network-all') }} - e2e-list: ${{ steps.e2e_list.outputs.e2e-list }} - bench-list: ${{ steps.e2e_list.outputs.bench-list }} - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: Compute Username - id: compute_username - shell: bash - env: - REPO: "${{ github.repository }}" - BRANCH: ${{ github.event.pull_request.head.ref || github.ref_name }} - run: | - if [ "${{ github.event_name }}" == "pull_request" ]; then - instance_name="${BRANCH//\//_}" - echo "username=$instance_name" - echo "username=$instance_name" >> $GITHUB_OUTPUT - else - echo "username=${{ github.sha }}"" - echo "username=${{ github.sha }}"" >> $GITHUB_OUTPUT - fi - - name: Create Job Lists - id: e2e_list - run: | - set -eu - LABELS="" - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - LABELS=$(jq -r '.pull_request.labels | map(.name) | join(",")' "${{ github.event_path }}") - fi - echo Labels: $LABELS - set -x - echo "e2e-list=$(./scripts/ci/get_e2e_jobs.sh ${{github.ref_name}} "$LABELS")" >> $GITHUB_OUTPUT - echo "bench-list=$(./scripts/ci/get_bench_jobs.sh ${{github.ref_name}} "$LABELS")" >> $GITHUB_OUTPUT - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 - id: filter - with: - filters: | - base-images: - - 'build-images/**' - noir: - - 'noir/noir-repo/**' - bb: - - 'barretenberg/**' - noir-projects: - - 'noir-projects/**' - yarn-project: - - 'yarn-project/**' - helm-charts: - - 'spartan/**' - non-bb: - - '!(barretenberg/cpp/**)' - # don't consider AVM stuff 'core bb' - - barretenberg/cpp/pil/** - - barretenberg/cpp/src/barretenberg/vm/** - - barretenberg/cpp/src/barretenberg/**/generated/* - - barretenberg/cpp/src/barretenberg/client_ivc/client_ivc.{hpp,cpp} - - barretenberg/cpp/src/barretenberg/bb/main.cpp - non-docs: - - '!(docs/**)' - - # Used by CI build as a base image. - base-images: - needs: [configure] - if: github.event.pull_request.draft == false - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: "Build Images" - if: needs.configure.outputs.base-images == 'true' - uses: ./.github/ensure-builder - timeout-minutes: 120 - with: - runner_type: builder-x86 - username: ${{ needs.configure.outputs.username }} - run: | - sudo apt install unzip - curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" && unzip awscliv2.zip && sudo ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli --update && rm -rf aws awscliv2.zip - echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin - ./build-images/bootstrap.sh ci - - # CI for noir and bb. - ci-noir-bb: - needs: [base-images, configure] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: "CI (noir and bb)" - uses: ./.github/ensure-builder - # we place this here as ci-noir-bb is a dependency for rest of chain - if: needs.configure.outputs.noir == 'true' || needs.configure.outputs.bb == 'true' - timeout-minutes: 40 - with: - runner_type: builder-x86 - username: ${{ needs.configure.outputs.username }} - run: scripts/earthly-ci +ci-noir-bb - - bootstrap: - needs: [ci-noir-bb, configure] - runs-on: ubuntu-latest - if: needs.configure.outputs.e2e-all == 'true' || needs.configure.outputs.non-bb == 'true' - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: "Bootstrap (l1-contracts, avm-transpiler, noir-projects, yarn-project)" - uses: ./.github/ensure-builder - timeout-minutes: 40 - with: - runner_type: builder-x86 - username: ${{ needs.configure.outputs.username }} - run: scripts/earthly-ci +bootstrap - - ci-rest: - needs: [bootstrap, configure] - runs-on: ubuntu-latest - if: needs.configure.outputs.e2e-all == 'true' || (needs.configure.outputs.non-bb == 'true' && needs.configure.outputs.non-docs == 'true') - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: "CI (l1-contracts, avm-transpiler, noir-projects, yarn-project)" - uses: ./.github/ensure-builder - timeout-minutes: 80 - with: - runner_type: builder-x86 - username: ${{ needs.configure.outputs.username }} - run: scripts/earthly-ci +ci-rest - - images-e2e: - needs: [bootstrap, configure] - if: needs.configure.outputs.e2e-all == 'true' || (needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-bb == 'true') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: "E2E Images" - uses: ./.github/ensure-builder - timeout-minutes: 40 - with: - runner_type: builder-x86 - username: ${{ needs.configure.outputs.username }} - run: ./bootstrap.sh image-e2e - - docs: - needs: [bootstrap, configure] - runs-on: ubuntu-latest - if: needs.configure.outputs.e2e-all == 'true' || needs.configure.outputs.non-bb == 'true' - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: "Docs" - uses: ./.github/ensure-builder - timeout-minutes: 40 - with: - runner_type: builder-x86 - username: ${{ needs.configure.outputs.username }} - run: - scripts/earthly-ci +docs-with-cache --ENV=staging --PR=${{ github.event.number }} \ - --AZTEC_BOT_COMMENTER_GITHUB_TOKEN=${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} \ - --NETLIFY_AUTH_TOKEN=${{ secrets.NETLIFY_AUTH_TOKEN }} \ - --NETLIFY_SITE_ID=${{ secrets.NETLIFY_SITE_ID }} - - e2e: - needs: [images-e2e, configure] - if: needs.configure.outputs.e2e-all == 'true' || (needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-bb == 'true') - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - test: ${{ fromJson( needs.configure.outputs.e2e-list )}} - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: Test - timeout-minutes: 40 - run: | - until docker info &>/dev/null; do sleep 1; done - echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin - export CI=1 NO_CACHE=0 - artifact="e2e-${{matrix.test}}-$(./yarn-project/bootstrap.sh hash)" - if ci3/test_should_run "$artifact"; then - docker pull aztecprotocol/aztec:${{ env.GIT_COMMIT }} - docker pull aztecprotocol/end-to-end:${{ env.GIT_COMMIT }} - FORCE_COLOR=1 ./yarn-project/end-to-end/scripts/e2e_test.sh ${{ matrix.test }} - ci3/cache_upload_flag "$artifact" - fi - - # All the benchmarking end-to-end integration tests for aztec (not required to merge) - bench-e2e: - needs: [images-e2e, configure] - if: needs.configure.outputs.e2e-all == 'true' || needs.configure.outputs.bench-list != '[]' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - test: ${{ fromJson( needs.configure.outputs.bench-list )}} - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: Test - timeout-minutes: 40 - run: | - until docker info &>/dev/null; do sleep 1; done - echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin - export CI=1 NO_CACHE=0 - artifact="bench-${{matrix.test}}-$(./yarn-project/bootstrap.sh hash)" - if ci3/test_should_run "$artifact"; then - docker pull aztecprotocol/aztec:${{ env.GIT_COMMIT }} - docker pull aztecprotocol/end-to-end:${{ env.GIT_COMMIT }} - FORCE_COLOR=1 ./yarn-project/end-to-end/scripts/e2e_test.sh ${{ matrix.test }} - ci3/cache_upload_flag "$artifact" - fi - - name: Inspect data folder - continue-on-error: true - run: tree ./yarn-project/end-to-end/out - - name: Store benchmark result - if: github.ref == 'refs/heads/master' - continue-on-error: true - uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 - with: - name: "End-to-end Benchmark" - benchmark-data-dir-path: "dev/e2e-bench" - tool: "customSmallerIsBetter" - output-file-path: ./yarn-project/end-to-end/out/bench.json - github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - auto-push: true - alert-threshold: "150%" - comment-on-alert: true - fail-on-alert: false - alert-comment-cc-users: "@philwindle @spalladino" - max-items-in-chart: 50 - - - # Only e2e test that can't run on standard github runner - e2e-prover-full: - runs-on: ubuntu-latest - needs: [images-e2e, configure] - if: needs.configure.outputs.e2e-all == 'true' - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - uses: ./.github/ci-setup-action - - name: "Setup and Test" - uses: ./.github/ensure-tester - env: - USERNAME: ${{ needs.configure.outputs.username }} - timeout-minutes: 50 - with: - ttl: 50 - runner_type: 32core-tester-x86 - run: | - export CI=1 NO_CACHE=0 - artifact="e2e-prover-full-$(./yarn-project/bootstrap.sh hash)" - if ci3/test_should_run "$artifact"; then - docker pull aztecprotocol/aztec:${{ env.GIT_COMMIT }} - docker pull aztecprotocol/end-to-end:${{ env.GIT_COMMIT }} - FORCE_COLOR=1 ./yarn-project/end-to-end/scripts/e2e_test.sh e2e_prover_full - ci3/cache_upload_flag "$artifact" - fi - - acir-bench: - runs-on: ubuntu-latest - needs: [ci-noir-bb, configure] - if: needs.configure.outputs.e2e-all == 'true' - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - uses: ./.github/ci-setup-action - - name: "Setup and Test" - uses: ./.github/ensure-tester - env: - USERNAME: ${{ needs.configure.outputs.username }} - timeout-minutes: 30 - with: - runner_type: 16core-tester-x86 - run: | - until docker info &>/dev/null; do sleep 1; done - echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin - export CI=1 NO_CACHE=0 - if ci3/test_should_run acir-bench-$(./barretenberg/acir_tests/bootstrap.sh hash); then - scripts/earthly-ci ./barretenberg/acir_tests+bench-publish - ci3/cache_upload_flag acir-bench-$(./barretenberg/acir_tests/bootstrap.sh hash) - fi - - # barretenberg (prover) native, AVM (public VM) and Merkle tree (world state) tests - # ran on own runner for resource reasons (memory x cpu intensive) - bb-native-tests: - runs-on: ubuntu-latest - needs: [base-images, configure] - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: "Setup" - run: | - # Ensure we can SSH into the spot instances we request - mkdir -p ~/.ssh - echo ${{ secrets.BUILD_INSTANCE_SSH_KEY }} | base64 --decode > ~/.ssh/build_instance_key - chmod 600 ~/.ssh/build_instance_key - - name: "Native Prover Tests" - env: - GITHUB_LOG: 1 - CI: 1 - NO_CACHE: 0 - run: | - export BRANCH=${{ needs.configure.outputs.username }}-bb-native-tests - if ci3/test_should_run barretenberg-test-$(./barretenberg/cpp/bootstrap.sh hash); then - ci3/bootstrap_ec2 "GITHUB_RUN_URL=$GITHUB_RUN_URL ./barretenberg/cpp/bootstrap.sh ci" - fi - - kind-network-smoke: - needs: [images-e2e, configure] - if: needs.configure.outputs.e2e-all == 'true' || needs.configure.outputs.yarn-project == 'true' || needs.configure.outputs.helm-charts == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: Setup and KIND Network Test - timeout-minutes: 60 - uses: ./.github/ensure-tester - with: - username: ${{ needs.configure.outputs.username }} - runner_type: 16core-tester-x86 - ttl: 60 - run: | - until docker info &>/dev/null; do sleep 1; done - docker pull aztecprotocol/aztec:${{ env.GIT_COMMIT }} - docker pull aztecprotocol/end-to-end:${{ env.GIT_COMMIT }} - echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin - - INSTALL_METRICS=false ./spartan/scripts/test_kind.sh ./src/spartan/smoke.test.ts ci-smoke.yaml - - name: Copy Network Logs - if: always() - run: scripts/copy_from_tester spartan/scripts/logs/test_kind.log test_kind.log || true - - name: Upload Network Logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: kind-network-smoke.log - path: test_kind.log - - kind-network-test: - needs: [images-e2e, configure] - if: needs.configure.outputs.network-all == 'true' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - config: - # - name: "reorg" - # test: reorg.test.ts - # values: ci.yaml - # runner_type: 16core-tester-x86-high-memory - # timeout: 60 - - name: "transfer" - test: transfer.test.ts - values: ci.yaml - runner_type: 16core-tester-x86 - timeout: 40 - - name: "transfer-with-blob-sink" - test: transfer.test.ts - values: ci.yaml - overrides: "blobSink.enabled=true" - runner_type: 16core-tester-x86 - timeout: 40 - # - name: "gating-passive" - # test: gating-passive.test.ts - # values: ci.yaml - # runner_type: 16core-tester-x86 - # timeout: 40 - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: Setup and KIND Network Test - timeout-minutes: ${{ matrix.config.timeout }} - uses: ./.github/ensure-tester - with: - username: ${{ needs.configure.outputs.username }} - runner_type: ${{ matrix.config.runner_type }} - spot_strategy: None # use on-demand machines - ttl: ${{ matrix.config.timeout }} - run: | - until docker info &>/dev/null; do sleep 1; done - export CI=1 NO_CACHE=0 - artifact="kind-network-${{matrix.config.test}}-$(./spartan/bootstrap.sh hash)" - if ci3/test_should_run "$artifact"; then - docker pull aztecprotocol/aztec:${{ env.GIT_COMMIT }} - docker pull aztecprotocol/end-to-end:${{ env.GIT_COMMIT }} - OVERRIDES="${{ matrix.config.overrides }}" INSTALL_METRICS=false ./spartan/scripts/test_kind.sh "./src/spartan/${{ matrix.config.test }}" "${{ matrix.config.values }}" - ci3/cache_upload_flag "$artifact" - fi - - name: Copy Network Logs - if: always() - run: scripts/copy_from_tester spartan/scripts/logs/test_kind.log test_kind.log || true - - - name: Upload Network Logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: kind-network-test-${{ matrix.config.name }}.log - path: test_kind.log - - bb-bench: - runs-on: ubuntu-latest - needs: [ci-noir-bb, configure] - if: needs.configure.outputs.bb == 'true' - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: Build Bench Binaries - uses: ./.github/ensure-builder - with: - username: ${{ needs.configure.outputs.username }} - runner_type: builder-x86 - run: | - export CI=1 - export NO_CACHE=0 - # We gate the build with this - if ci3/test_should_run bb-bench-$(./barretenberg/cpp/bootstrap.sh hash); then - scripts/earthly-ci --push ./barretenberg/cpp/+bench-binaries - fi - - name: Run Bench - uses: ./.github/ensure-tester - timeout-minutes: 40 - with: - username: ${{ needs.configure.outputs.username }} - runner_type: 16core-tester-x86 - run: | - HASH=$(ci3/cache_content_hash ./barretenberg/cpp/.rebuild_patterns)-${{ github.ref }} - export CI=1 NO_CACHE=0 - if ci3/test_should_run bb-bench-$HASH; then - scripts/earthly-ci --artifact ./barretenberg/cpp/+bench/bench.json --bench_mode=cache - ci3/cache_upload_flag bb-bench-$HASH - fi - - name: Copy Bench.json - run: scripts/copy_from_tester barretenberg/cpp/bench.json bench.json || true - - name: Store benchmark result - if: github.ref == 'refs/heads/master' - continue-on-error: true - uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 - with: - name: C++ Benchmark - tool: "googlecpp" - output-file-path: bench.json - github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - auto-push: true - alert-threshold: "105%" - comment-on-alert: true - fail-on-alert: false - alert-comment-cc-users: "@ludamad @codygunton" - max-items-in-chart: 50 - - boxes-test: - needs: [ci-rest, configure] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ github.event.pull_request.head.sha }}" } - - uses: ./.github/ci-setup-action - - - name: Build Boxes - uses: ./.github/ensure-builder - timeout-minutes: 40 - with: - username: ${{ needs.configure.outputs.username }} - runner_type: builder-x86 - run: | - export CI=1 NO_CACHE=0 - if ci3/test_should_run "boxes-test-$(./boxes/bootstrap.sh hash)"; then - ./bootstrap.sh test-boxes - fi - - prover-client-test: - needs: [ci-rest, configure] - runs-on: ubuntu-latest - if: needs.configure.outputs.e2e-all == 'true' || needs.configure.outputs.noir == 'true' || needs.configure.outputs.bb == 'true' || needs.configure.outputs.noir-projects == 'true' - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ github.event.pull_request.head.sha }}" } - - uses: ./.github/ci-setup-action - - name: Test - uses: ./.github/ensure-builder - timeout-minutes: 40 - with: - username: ${{ needs.configure.outputs.username }} - runner_type: builder-x86 - run: | - export CI=1 NO_CACHE=0 - scripts/earthly-ci +prover-client-with-cache - - # testnet-installer: - # needs: [bootstrap, configure] - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # with: { ref: "${{ github.event.pull_request.head.sha }}" } - # - uses: ./.github/ci-setup-action - # - name: Testnet Installer Helper Script - # uses: ./.github/ensure-builder - # timeout-minutes: 40 - # with: - # username: ${{ needs.configure.outputs.username }} - # runner_type: builder-x86 - # run: | - # scripts/earthly-ci ./spartan/releases/testnet/+test-all - - protocol-circuits-gates-report: - needs: [ci-rest, configure] - if: github.ref_name == 'master' || (needs.configure.outputs.non-docs == 'true' && needs.configure.outputs.non-bb == 'true') - runs-on: ubuntu-latest - permissions: - pull-requests: write - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: "Noir Protocol Circuits Report" - uses: ./.github/ensure-builder - timeout-minutes: 40 - with: - username: ${{ needs.configure.outputs.username }} - runner_type: builder-x86 - run: | - scripts/earthly-ci --artifact ./noir-projects/+gates-report/gates_report.json - - name: Copy Report - run: | - scripts/copy_from_builder ./noir-projects/gates_report.json protocol_circuits_report.json - - name: Compare gates reports - id: gates_diff - uses: noir-lang/noir-gates-diff@dbe920a8dcc3370af4be4f702ca9cef29317bec1 - continue-on-error: true - timeout-minutes: 1 - with: - report: protocol_circuits_report.json - summaryQuantile: 0 # Display any diff in gate count - - - name: Add gates diff to sticky comment - if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' - uses: marocchino/sticky-pull-request-comment@v2 - continue-on-error: true - with: - # delete the comment in case changes no longer impact circuit sizes - delete: ${{ !steps.gates_diff.outputs.markdown }} - message: ${{ steps.gates_diff.outputs.markdown }} - - public-functions-size-report: - needs: [ci-rest, configure] - if: github.ref_name == 'master' || needs.configure.outputs.non-docs == 'true' - runs-on: ubuntu-latest - permissions: - pull-requests: write - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: "Aztec Public Functions Bytecode Size Report" - timeout-minutes: 40 - uses: ./.github/ensure-builder - with: - username: ${{ needs.configure.outputs.username }} - runner_type: builder-x86 - run: | - scripts/earthly-ci --artifact ./noir-projects/+public-functions-report/public_functions_report.json - - name: Copy Report - run: | - scripts/copy_from_builder noir-projects/public_functions_report.json public_functions_report.json - - name: Compare public functions bytecode size reports - id: public_functions_sizes_diff - uses: noir-lang/noir-gates-diff@dbe920a8dcc3370af4be4f702ca9cef29317bec1 - continue-on-error: true - timeout-minutes: 1 - with: - report: public_functions_report.json - header: | - # Changes to public function bytecode sizes - brillig_report: true - brillig_report_bytes: true - summaryQuantile: 0 # Display any diff in bytecode size count - - - name: Add bytecode size diff to sticky comment - if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target' - uses: marocchino/sticky-pull-request-comment@v2 - continue-on-error: true - timeout-minutes: 1 - with: - header: public_functions_size - delete: ${{ !steps.public_functions_sizes_diff.outputs.markdown }} - message: ${{ steps.public_functions_sizes_diff.outputs.markdown }} - - merge-check: - runs-on: ubuntu-latest - needs: - - configure - - base-images - - bootstrap - - ci-noir-bb - - ci-rest - - images-e2e - - docs - - e2e - - e2e-prover-full - - bb-native-tests - - kind-network-smoke - - kind-network-test - - boxes-test - # - testnet-installer - if: always() - outputs: - failure: ${{ steps.set_failed_jobs.outputs.failure }} - failed_jobs: ${{ steps.set_failed_jobs.outputs.failed_jobs }} - steps: - - name: Check for Failures and Set Output - id: set_failed_jobs - env: - NEEDS_JOBS_JSON: ${{ toJson(needs) }} - run: | - echo "Processing failed jobs..." - failed_jobs=$(echo "$NEEDS_JOBS_JSON" | jq -r 'to_entries[] | select(.value.result == "failure") | .key' | paste -sd "," -) - echo "$failed_jobs" > .failed - echo "failure=${{contains(needs.*.result, 'failure')}}" >> $GITHUB_OUTPUT - echo "failed_jobs=$failed_jobs" >> $GITHUB_OUTPUT - - - name: Report overall success (non-draft) - if: github.event.pull_request.draft == false - env: - # We treat any skipped or failing jobs as a failure for the workflow as a whole. - FAIL: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} - run: | - if [[ $FAIL == true ]]; then - echo "Jobs failed: $(cat .failed), merging not allowed." - exit 1 - else - echo "All jobs succeeded, merge allowed." - exit 0 - fi - - name: Block merge (draft) - if: github.event.pull_request.draft - run: echo "Can't merge drafts." && exit 1 - - notify: - runs-on: ubuntu-latest - needs: - - merge-check - if: github.event.pull_request.draft == false && github.ref == 'refs/heads/master' && failure() - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Get Authors of Recent Commit - id: get_authors - run: | - git fetch --depth=1 origin ${{ github.sha }} - authors=$(git log -1 --pretty=format:'%an <%ae>' ${{ github.sha }}) - echo "authors=${authors}" >> $GITHUB_OUTPUT - - - name: Send notification to aztec3-ci channel if workflow failed on master - uses: slackapi/slack-github-action@v1.25.0 - with: - payload: | - { - "text": "Master Github Actions failure", - "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", - "authors": "${{ steps.get_authors.outputs.authors }}", - "failed_jobs": "${{ needs.merge-check.outputs.failed_jobs }}" - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WORKFLOW_TRIGGER_URL2 }} diff --git a/.github/workflows/publish-bb.yml b/.github/workflows/publish-bb.yml deleted file mode 100644 index f703078c459f..000000000000 --- a/.github/workflows/publish-bb.yml +++ /dev/null @@ -1,277 +0,0 @@ -name: Publish Barretenberg -on: - workflow_dispatch: - # Allow pushing a manual nightly release - inputs: - tag: - description: The tag to build from (leave empty to build a nightly release from master) - required: false - publish: - description: Whether to publish the build artifacts - type: boolean - default: false - schedule: - # Run a nightly release at 2 AM UTC - - cron: "0 2 * * *" - -permissions: - # Necessary to upload new release artifacts - contents: write - issues: write - -jobs: - build-x86_64-linux-gnu: - name: Build on Linux GNU - runs-on: ubuntu-20.04 - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - ref: ${{ inputs.tag || env.GITHUB_REF }} - - - name: Install bleeding edge cmake - run: | - sudo apt -y remove --purge cmake - sudo snap install cmake --classic - - - name: Create Build Environment - run: | - sudo apt-get update - sudo apt-get -y install ninja-build - - - name: Install Clang16 - run: | - wget https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.0/clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz - tar -xvf clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz - sudo cp clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04/bin/* /usr/local/bin/ - sudo cp -r clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04/include/* /usr/local/include/ - sudo cp -r clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04/lib/* /usr/local/lib/ - sudo cp -r clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04/share/* /usr/local/share/ - - - name: Compile Barretenberg - run: | - cd barretenberg/cpp - - cmake --preset default -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_BUILD_TYPE=RelWithAssert -DTARGET_ARCH=westmere - cmake --build --preset default --target bb - - - name: Tar and GZip bb Binary (Ubuntu) - working-directory: barretenberg/cpp/build/bin - run: tar -cvzf barretenberg-x86_64-linux-gnu.tar.gz bb - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: release-linux - path: | - ./barretenberg/cpp/build/bin/barretenberg-x86_64-linux-gnu.tar.gz - - build-wasm-ts: - name: Build WASM and deploy to TS - runs-on: ubuntu-20.04 - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - ref: ${{ inputs.tag || env.GITHUB_REF }} - - - name: Install bleeding edge cmake - run: | - sudo apt -y remove --purge cmake - sudo snap install cmake --classic - - - name: Create Build Environment - run: | - sudo apt-get update - sudo apt-get -y install ninja-build yarn - - - name: Install Clang16 - run: | - wget https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.0/clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz - tar -xvf clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz - sudo cp clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04/bin/* /usr/local/bin/ - sudo cp -r clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04/include/* /usr/local/include/ - sudo cp -r clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04/lib/* /usr/local/lib/ - sudo cp -r clang+llvm-16.0.0-x86_64-linux-gnu-ubuntu-18.04/share/* /usr/local/share/ - - - name: Enable Corepack and Install Yarn v4.5.2 # Needed to call 'yarn build' on barretenberg/ts - run: | - corepack enable - corepack prepare yarn@4.5.2 --activate - - - name: Install WASI-SDK - run: | - cd barretenberg/cpp - - ./scripts/install-wasi-sdk.sh - - - name: Compile Typescript # Compiles bb.js and wasms - run: | - cd barretenberg/ts - yarn install && yarn && yarn build - - - name: Tar and GZip barretenberg.wasm - working-directory: barretenberg/cpp/build-wasm/bin - run: tar -cvzf barretenberg.wasm.tar.gz barretenberg.wasm - - # - name: Setup Node.js - # uses: actions/setup-node@v2 - # with: - # node-version: "18" - # registry-url: "https://registry.npmjs.org" - - # - name: Deploy Typescript to NPM - # if: github.event.inputs.tag != 'nightly' && github.event.inputs.tag != '' # Do not deploy to npm if it is a nightly build - # run: | - # cd barretenberg/ts - # yarn deploy - # env: - # NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: release-wasm - path: | - ./barretenberg/cpp/build-wasm/bin/barretenberg.wasm.tar.gz - - build-mac-intel: - name: Build on Mac x86_64-apple-darwin - runs-on: macos-13 - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ inputs.tag || env.GITHUB_REF }} - - - name: Create Mac Build Environment - run: brew install cmake ninja llvm@16 - - - name: Compile Barretenberg - working-directory: barretenberg/cpp - run: | - export PATH="/usr/local/opt/llvm@16/bin:$PATH" - export LDFLAGS="-L/usr/local/opt/llvm@16/lib" - export CPPFLAGS="-I/usr/local/opt/llvm@16/include" - cmake -DCMAKE_BUILD_TYPE=RelWithAssert --preset default - cmake --build --preset default --target bb - - - name: Package barretenberg artifact - working-directory: barretenberg/cpp/build/bin - run: | - mkdir dist - cp ./bb ./dist/bb - 7z a -ttar -so -an ./dist/* | 7z a -si ./barretenberg-x86_64-apple-darwin.tar.gz - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: barretenberg-x86_64-apple-darwin - path: ./barretenberg/cpp/build/bin/barretenberg-x86_64-apple-darwin.tar.gz - retention-days: 3 - - build-mac-m1: - name: Build on Mac aarch64-apple-darwin - runs-on: macos-14 - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ inputs.tag || env.GITHUB_REF }} - - - name: Create Mac Build Environment - run: brew install cmake ninja - - - name: Compile Barretenberg - working-directory: barretenberg/cpp - run: | - cmake --preset default -DCMAKE_BUILD_TYPE=RelWithAssert - cmake --build --preset default --target bb - - - name: Package barretenberg artifact - working-directory: barretenberg/cpp/build/bin - run: | - mkdir dist - cp ./bb ./dist/bb - 7z a -ttar -so -an ./dist/* | 7z a -si ./barretenberg-aarch64-apple-darwin.tar.gz - - - name: Upload artifact - uses: actions/upload-artifact@v4 - with: - name: barretenberg-aarch64-apple-darwin - path: ./barretenberg/cpp/build/bin/barretenberg-aarch64-apple-darwin.tar.gz - retention-days: 3 - - build-check: - name: Check builds are successful - needs: - [build-x86_64-linux-gnu, build-mac-intel, build-mac-m1, build-wasm-ts] - if: ${{ always() }} - runs-on: ubuntu-latest - steps: - - name: Report overall success - env: - FAIL: ${{ contains(needs.*.result, 'failure') }} - run: | - if [[ $FAIL == true ]]; then - echo "At least one job failed, release is unsuccessful." - exit 1 - else - echo "All jobs succeeded, uploading artifacts to release." - exit 0 - fi - - - name: Checkout - uses: actions/checkout@v3 - if: ${{ failure() }} - with: - ref: ${{ inputs.tag || env.GITHUB_REF }} - - # Raise an issue if the release failed - - name: Alert on dead links - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 - if: ${{ failure() }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WORKFLOW_NAME: ${{ github.workflow }} - WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - with: - update_existing: true - filename: .github/RELEASE_FAILED.md - - release: - name: Publish - needs: [build-check] - runs-on: ubuntu-latest - steps: - - name: Download files from Linux Runner - uses: actions/download-artifact@v4 - with: - name: release-linux - - - name: Download files for WASM - uses: actions/download-artifact@v4 - with: - name: release-wasm - - - name: Download files from x86_64 Mac Runner - uses: actions/download-artifact@v4 - with: - name: barretenberg-x86_64-apple-darwin - - - name: Download files from aarch64 Mac Runner - uses: actions/download-artifact@v4 - with: - name: barretenberg-aarch64-apple-darwin - - - name: Publish to GitHub - uses: softprops/action-gh-release@v1 - if: ${{ inputs.publish || github.event_name == 'schedule' }} - with: - tag_name: ${{ inputs.tag || 'nightly' }} # This will fail if `inputs.tag` is not a tag (e.g. testing a branch) - prerelease: true - files: | - barretenberg.wasm.tar.gz - barretenberg-x86_64-linux-gnu.tar.gz - barretenberg-x86_64-apple-darwin.tar.gz - barretenberg-aarch64-apple-darwin.tar.gz diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml deleted file mode 100644 index c6583ec45ded..000000000000 --- a/.github/workflows/publish-docs.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Publish Docs -on: - workflow_dispatch: - inputs: - tag: - description: The tag to build from. - required: true - -jobs: - setup: - uses: ./.github/workflows/setup-runner.yml - with: - username: master - runner_type: builder-x86 - secrets: inherit - - publish: - needs: setup - runs-on: master-x86 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.tag || github.ref }} - - uses: ./.github/ci-setup-action - env: - DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_PASSWORD }}" - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - with: - concurrency_key: docs-preview-${{ inputs.username || github.actor }}-x86 - - - timeout-minutes: 60 # in case of full build - run: | - earthly-ci --no-output ./docs/+deploy-prod \ - --NETLIFY_AUTH_TOKEN=${{ secrets.NETLIFY_AUTH_TOKEN }} \ - --NETLIFY_SITE_ID=${{ secrets.NETLIFY_SITE_ID }} \ - --COMMIT_TAG=${{ inputs.tag }} - - pdf: - needs: setup - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - path: aztec-packages - - name: Install Prince - run: | - curl https://www.princexml.com/download/prince-14.2-linux-generic-x86_64.tar.gz -O - tar zxf prince-14.2-linux-generic-x86_64.tar.gz - cd prince-14.2-linux-generic-x86_64 - yes "" | sudo ./install.sh - - name: Serve docs - run: | - cd aztec-packages/docs - yarn install - yarn build - yarn serve & - - name: Checkout PDF repo - uses: actions/checkout@v3 - with: - token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - repository: AztecProtocol/protocol-specs-pdf - path: protocol-specs-pdf - - name: Generate PDF - run: | - npx docusaurus-prince-pdf -u http://localhost:3000/protocol-specs/intro --output protocol-specs-pdf/protocol-specs.pdf - timeout-minutes: 4 - - name: Push to PDF repo - run: | - git config --global user.name AztecBot - git config --global user.email tech@aztecprotocol.com - cd protocol-specs-pdf - git add protocol-specs.pdf - git commit -m "chore: update protocol-specs.pdf" - git push origin main diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml deleted file mode 100644 index 9bf68adabd94..000000000000 --- a/.github/workflows/release-please.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: release-please - -# Trigger on merges to 'master' branch -on: - push: - branches: - - master - -permissions: - contents: write - pull-requests: write -env: - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - -jobs: - release-please: - name: Create Release - runs-on: ubuntu-latest - steps: - - name: Run Release Please - id: release - uses: google-github-actions/release-please-action@v3 - with: - token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - command: manifest - default-branch: ${{ env.BRANCH_NAME }} - - - name: Dispatch to publish Aztec packages workflow - uses: benc-uk/workflow-dispatch@v1 - if: ${{ steps.release.outputs.tag_name }} - with: - workflow: publish-aztec-packages.yml - ref: ${{ env.BRANCH_NAME }} - token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - inputs: '{ "tag": "${{ steps.release.outputs.tag_name }}", "publish": true }' - - - name: Dispatch to publish BB workflow - uses: benc-uk/workflow-dispatch@v1 - if: ${{ steps.release.outputs.tag_name }} - with: - workflow: publish-bb.yml - ref: ${{ env.BRANCH_NAME }} - token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - inputs: '{ "tag": "${{ steps.release.outputs.tag_name }}", "publish": true }' - - - name: Dispatch to publish docs workflow - uses: benc-uk/workflow-dispatch@v1 - if: ${{ steps.release.outputs.tag_name }} - with: - workflow: publish-docs.yml - ref: ${{ env.BRANCH_NAME }} - token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - inputs: '{ "tag": "${{ steps.release.outputs.tag_name }}" }' diff --git a/.github/workflows/rerun.yml b/.github/workflows/rerun.yml deleted file mode 100644 index 430054284125..000000000000 --- a/.github/workflows/rerun.yml +++ /dev/null @@ -1,57 +0,0 @@ -# adapted from https://github.com/orgs/community/discussions/67654 -# altered to not rerun if we are not the newest commit in the run's branch (exception for master/main) -on: - workflow_dispatch: - inputs: - run_id: - required: true - rerun_mode: - description: 'Specify rerun mode: failed or all' - required: false - default: 'failed' - -jobs: - rerun: - runs-on: ubuntu-latest - permissions: - actions: write # Needed for 'gh run rerun' - steps: - - name: Wait for run to finish - env: - GH_REPO: ${{ github.repository }} - GH_TOKEN: ${{ github.token }} - GH_DEBUG: api - run: | - gh run watch ${{ inputs.run_id }} > /dev/null 2>&1 - - - name: Rerun jobs if the commit is the latest on the branch - env: - GH_REPO: ${{ github.repository }} - GH_TOKEN: ${{ github.token }} - GH_DEBUG: api - RERUN_MODE: ${{ inputs.rerun_mode }} - run: | - RUN_ID="${{ inputs.run_id }}" - # Get the run details - RUN_INFO=$(gh run view $RUN_ID --json headSha,headBranch,event) - # Extract the commit SHA, branch name, and event type - COMMIT_SHA=$(echo "$RUN_INFO" | jq -r '.headSha') - BRANCH_NAME=$(echo "$RUN_INFO" | jq -r '.headBranch') - EVENT_TYPE=$(echo "$RUN_INFO" | jq -r '.event') - - # Get the latest commit SHA on the branch - LATEST_COMMIT_SHA=$(gh api repos/${{ github.repository }}/commits/$BRANCH_NAME --jq .sha) - - # Compare the SHAs - if [[ "$COMMIT_SHA" != "$LATEST_COMMIT_SHA" ]] && [[ "$BRANCH_NAME" != "master" && "$BRANCH_NAME" != "main" ]] ; then - echo "Commit $COMMIT_SHA is not the latest commit on branch $BRANCH_NAME (latest is $LATEST_COMMIT_SHA) and the branch is not master/main. Skipping rerun." - else - echo "Commit $COMMIT_SHA is the latest on branch $BRANCH_NAME. Proceeding with rerun." - - if [[ "$RERUN_MODE" == "all" ]]; then - gh run rerun $RUN_ID - else - gh run rerun $RUN_ID --failed - fi - fi - diff --git a/.github/workflows/setup-runner.yml b/.github/workflows/setup-runner.yml deleted file mode 100644 index 55effd4d9122..000000000000 --- a/.github/workflows/setup-runner.yml +++ /dev/null @@ -1,55 +0,0 @@ - -# Start cheap (~1/8th the cost of on demand, ~13th the cost of large GA runners) spot builders -# just for the CI job. These are specced per user and run the entire CI. -# TODO These have a persistent EBS volume that forms a fast-online docker image cache (used by Earthly), meaning -# TODO build steps that ran in previous invocations are quickly ran from cache. -name: Setup Runner and CI -on: - workflow_call: - inputs: - username: - required: true - type: string - runner_type: - required: true - type: string - secrets: - AWS_ACCESS_KEY_ID: - required: true - AWS_SECRET_ACCESS_KEY: - required: true - GH_SELF_HOSTED_RUNNER_TOKEN: - required: true - BUILD_INSTANCE_SSH_KEY: - required: true - DOCKERHUB_PASSWORD: - required: true - -concurrency: - group: ci-${{ inputs.username }}-${{ inputs.runner_type }}-${{ inputs.runner_action }} -env: - DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_PASSWORD }}" - RUN_ID: ${{ github.run_id }} - RUN_ATTEMPT: ${{ github.run_attempt }} - USERNAME: ${{ inputs.username }} - GITHUB_TOKEN: ${{ github.token }} - GH_SELF_HOSTED_RUNNER_TOKEN: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GITHUB_RUN_URL: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}" - GIT_COMMIT: ${{ github.sha }} -jobs: - start-builder: - runs-on: ubuntu-latest - # we want to avoid race conditions when making spot across multiple PRs as we only use one runner - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Start EC2 runner - uses: ./.github/ensure-builder - with: - runner_type: ${{ inputs.runner_type }} diff --git a/.github/workflows/start-spot.yml b/.github/workflows/start-spot.yml deleted file mode 100644 index ac7f3f5d955e..000000000000 --- a/.github/workflows/start-spot.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Useful if the publish npm runner is in a bad state -name: Start Legacy Runner -on: - workflow_dispatch: - inputs: - username: - description: 'Username' - required: true - runner_type: - description: 'Runner type' - default: builder-x86 - required: true -jobs: - start-build: - uses: ./.github/workflows/setup-runner.yml - with: - username: ${{ inputs.username }} - runner_type: ${{ inputs.runner_type }} - secrets: inherit diff --git a/.github/workflows/vm_full_tests.yml b/.github/workflows/vm_full_tests.yml deleted file mode 100644 index 32243dba8fed..000000000000 --- a/.github/workflows/vm_full_tests.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: AVM Full Tests - -on: - workflow_dispatch: - inputs: {} - -concurrency: - # force parallelism in master - group: avm-full-tests-${{ github.ref }} - cancel-in-progress: true - -jobs: - # barretenberg (prover) native and AVM (public VM) tests - # only ran on x86 for resource reasons (memory intensive) - avm-full-tests: - runs-on: ubuntu-latest - env: - DOCKERHUB_PASSWORD: "${{ secrets.DOCKERHUB_PASSWORD }}" - RUN_ID: ${{ github.run_id }} - RUN_ATTEMPT: ${{ github.run_attempt }} - GITHUB_TOKEN: ${{ github.token }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }} - GITHUB_RUN_URL: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}" - GIT_COMMIT: ${{ github.sha }} - steps: - - uses: actions/checkout@v4 - with: { ref: "${{ env.GIT_COMMIT }}" } - - name: "AVM Full Tests" - uses: ./.github/ensure-tester - timeout-minutes: 75 - with: - ttl: 75 - runner_type: 128core-tester-x86 - run: | - scripts/earthly-ci --no-output ./barretenberg/cpp/+vm-full-test --hardware_concurrency=64 # limit our parallelism to half our cores diff --git a/.gitignore b/.gitignore index b29367721c63..a8e334b636df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ dest node_modules .cache -scripts/.earthly .pnp.cjs .pnp.loader.mjs build/ @@ -18,12 +17,6 @@ terraform.tfstate* .graphite* .DS_Store -# Earthly -.arg -.secret -.bb_tmp - - # tmux tmux-client-*.log .supermavenignore diff --git a/PROJECT b/PROJECT deleted file mode 100644 index 5f8dfee5219d..000000000000 --- a/PROJECT +++ /dev/null @@ -1 +0,0 @@ -aztec \ No newline at end of file diff --git a/avm-transpiler/scripts/bootstrap_native.sh b/avm-transpiler/scripts/bootstrap_native.sh deleted file mode 100755 index 5b61375fff92..000000000000 --- a/avm-transpiler/scripts/bootstrap_native.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -set -eu - -cd $(dirname "$0")/.. - -# If this project has been subrepod into another project, set build data manually. -export SOURCE_DATE_EPOCH=$(date +%s) -export GIT_DIRTY=false -if [ -f ".gitrepo" ]; then - export GIT_COMMIT=$(awk '/commit =/ {print $3}' .gitrepo) -else - export GIT_COMMIT=${COMMIT_HASH:-$(git rev-parse --verify HEAD)} -fi - -# Build native. -if [ -n "${DEBUG:-}" ]; then - cargo build -else - cargo build --release -fi diff --git a/barretenberg/.github/pull_request_template.md b/barretenberg/.github/pull_request_template.md deleted file mode 100644 index 6d43dc470def..000000000000 --- a/barretenberg/.github/pull_request_template.md +++ /dev/null @@ -1,6 +0,0 @@ -:warning: :warning: :warning: DO NOT CREATE THIS PR. :warning: :warning: :warning: -You are creating a PR into the wrong repository. -Please migrate your PR to https://github.com/AztecProtocol/aztec-packages/pulls with `scripts/migrate_barretenberg_branch.sh` in the aztec-packages repo. -``` -Usage: scripts/migrate_barretenberg_branch.sh -``` diff --git a/barretenberg/.github/workflows/benchmarks.yml b/barretenberg/.github/workflows/benchmarks.yml deleted file mode 100644 index 082634b387ad..000000000000 --- a/barretenberg/.github/workflows/benchmarks.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Barretenberg Benchmarks - -# Only run on push to master -on: - push: - branches: - - master - -# This will cancel previous runs when a branch or PR is updated -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }} - cancel-in-progress: true - -jobs: - bberg-bench: - name: Barretenberg Benchmarks - runs-on: ubuntu-latest # run in linux environment - - steps: - - - name: Checkout barretenberg - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} # checkout HEAD of triggering branch - token: ${{ secrets.GITHUB_TOKEN }} - - # Only run in Linux environment for now - - name: Setup Linux environment - run: | - sudo apt update - sudo apt install libomp-dev cmake ninja-build - - - name: Download SRS elements - working-directory: cpp/srs_db - run: ./download_ignition.sh 3 # only download first 4 transcript files - - - name: Build Honk benchmarks - working-directory: cpp - run: | - cmake --preset bench - cmake --build --preset bench --target ultra_honk_bench - - - name: Run Honk benchmarks - working-directory: cpp/build-bench - run: | - bin/ultra_honk_bench --benchmark_format=json > ../src/barretenberg/benchmark/ultra_bench/bench_results.json - - # Utilize github-action-benchmark to automatically update the plots at - # https://aztecprotocol.github.io/barretenberg/dev/bench/ with new benchmark data. - # This also creates an alert if benchmarks exceed the threshold specified below. - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 - with: - name: C++ Benchmark - tool: 'googlecpp' - output-file-path: cpp/src/barretenberg/benchmark/ultra_bench/bench_results.json - # Access token to deploy GitHub Pages branch - github-token: ${{ secrets.GITHUB_TOKEN }} - # Push and deploy GitHub pages branch automatically - auto-push: true - # Enable Job Summary for PRs - summary-always: true - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '120%' # alert if bench result is 1.2x worse - comment-on-alert: true - fail-on-alert: false - alert-comment-cc-users: '@ledwards2225' - diff --git a/barretenberg/.github/workflows/noir.yml b/barretenberg/.github/workflows/noir.yml deleted file mode 100644 index d15c28100e6a..000000000000 --- a/barretenberg/.github/workflows/noir.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Noir CI - -on: - pull_request: - types: ["labeled", "opened", "synchronize", "reopened"] - -# This will cancel previous runs when a branch or PR is updated -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }} - cancel-in-progress: true - -jobs: - noir-ci: - if: ${{ contains(github.event.pull_request.labels.*.name, 'noir-ci') }} - name: Test Noir against PR - runs-on: ubuntu-latest - - steps: - - name: Checkout Noir - uses: actions/checkout@v3 - with: - repository: noir-lang/noir - # This reference can be changed to test against a different version of Noir (such as a WIP branch) - ref: master - - - uses: cachix/install-nix-action@v20 - with: - nix_path: nixpkgs=channel:nixos-22.11 - github_access_token: ${{ secrets.GITHUB_TOKEN }} - - # The GITHUB_SHA doesn't match the last commit on the PR but is instead a merge commit of the PR into master. - # As per GitHub docs: - # Note that GITHUB_SHA for this event is the last merge commit of the pull request merge branch. - - name: Update barretenberg commit in flake.lock - run: | - nix flake lock --override-input barretenberg github:AztecProtocol/barretenberg/${{ env.GITHUB_SHA }} - - - name: Build and test Noir - run: | - nix flake check -L diff --git a/barretenberg/.github/workflows/pull-request.yml b/barretenberg/.github/workflows/pull-request.yml deleted file mode 100644 index 5ceafd8ee235..000000000000 --- a/barretenberg/.github/workflows/pull-request.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Pull Request - -on: - pull_request: - types: - - opened - - reopened - - edited - - synchronize - -permissions: - pull-requests: read - -jobs: - conventional-title: - name: Validate PR title is Conventional Commit - runs-on: ubuntu-latest - steps: - - name: Check title - if: github.event_name == 'pull_request' - uses: amannn/action-semantic-pull-request@v5 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - types: | - fix - feat - chore - refactor diff --git a/barretenberg/acir_tests/run_acir_tests.sh b/barretenberg/acir_tests/run_acir_tests.sh deleted file mode 100755 index 7b53b724340f..000000000000 --- a/barretenberg/acir_tests/run_acir_tests.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env bash -# DEPRECATED: USE bootstrap.sh test -#!/usr/bin/env bash -# Env var overrides: -# BIN: to specify a different binary to test with (e.g. bb.js or bb.js-dev). -# VERBOSE: to enable logging for each test. -# RECURSIVE: to enable --recursive for each test. -set -eu - -# Catch when running in parallel -error_file="/tmp/error.$$" -pids=() - -# Handler for SIGCHLD, cleanup if child exit with error -handle_sigchild() { - for pid in "${pids[@]}"; do - # If process is no longer running - if ! kill -0 "$pid" 2>/dev/null; then - # Wait for the process and get exit status - wait "$pid" - status=$? - - # If exit status is error - if [ $status -ne 0 ]; then - # Create error file - touch "$error_file" - fi - fi - done -} -trap handle_sigchild SIGCHLD - -BIN=${BIN:-../cpp/build/bin/bb} -FLOW=${FLOW:-prove_and_verify} -HONK=${HONK:-false} -CLIENT_IVC_SKIPS=${CLIENT_IVC_SKIPS:-false} -CRS_PATH=~/.bb-crs -BRANCH=master -VERBOSE=${VERBOSE:-} -TEST_NAMES=("$@") -# We get little performance benefit over 16 cores (in fact it can be worse). -HARDWARE_CONCURRENCY=${HARDWARE_CONCURRENCY:-16} -RECURSIVE=${RECURSIVE:-false} - -FLOW_SCRIPT=$(realpath ./flows/${FLOW}.sh) - -if [ -f $BIN ]; then - BIN=$(realpath $BIN) -else - BIN=$(realpath $(which $BIN)) -fi - -export BIN CRS_PATH VERBOSE BRANCH RECURSIVE - -cd acir_tests - -# Convert them to array -# There are no issues with the tests below but as they check proper handling of dependencies or circuits that are part of a workspace -# running these require extra glue code so they are skipped for the purpose of this script -SKIP_ARRAY=(diamond_deps_0 workspace workspace_default_member) - -# TODO(https://github.com/AztecProtocol/barretenberg/issues/1108): problem regardless the proof system used -SKIP_ARRAY+=(regression_5045) - - -# if HONK is false, we should skip verify_honk_proof -if [ "$HONK" = false ]; then - # Don't run programs with Honk recursive verifier - SKIP_ARRAY+=(verify_honk_proof double_verify_honk_proof verify_rollup_honk_proof) -fi - -if [ "$HONK" = true ]; then - # Don't run programs with Plonk recursive verifier(s) - SKIP_ARRAY+=(single_verify_proof double_verify_proof double_verify_nested_proof verify_rollup_honk_proof) -fi - -if [ "$CLIENT_IVC_SKIPS" = true ]; then - # At least for now, skip folding tests that fail when run against ClientIVC. - # This is not a regression--folding was not being properly tested. - # TODO(https://github.com/AztecProtocol/barretenberg/issues/1164): Resolve this - # The reason for failure is that compile-time folding, as initially conceived, is - # only supported by ClientIVC through hacks. ClientIVC in Aztec is ultimately to be - # used through runtime folding, since the kernels that are needed are detected and - # constructed at runtime in Aztec's typescript proving interface. ClientIVC appends - # folding verifiers and does databus and Goblin merge work depending on its inputs, - # detecting which circuits are Aztec kernels. These tests may simple fail for trivial - # reasons, e.g. because the number of circuits in the stack is odd. - SKIP_ARRAY+=(fold_basic_nested_call fold_fibonacci fold_numeric_generic_poseidon ram_blowup_regression) -fi - - -function test() { - cd $1 - - set +e - start=$SECONDS - $FLOW_SCRIPT - result=$? - end=$SECONDS - duration=$((end - start)) - set -eu - - if [ $result -eq 0 ]; then - echo -e "\033[32mPASSED\033[0m ($duration s)" - else - echo -e "\033[31mFAILED\033[0m" - touch "$error_file" - exit 1 - fi - - cd .. -} - -if [ "${#TEST_NAMES[@]}" -ne 0 ]; then - for NAMED_TEST in "${TEST_NAMES[@]}"; do - echo -n "Testing $NAMED_TEST... " - test $NAMED_TEST - done -else - for TEST_NAME in $(find -maxdepth 1 -type d -not -path '.' | sed 's|^\./||'); do - echo -n "Testing $TEST_NAME... " - - if [[ " ${SKIP_ARRAY[@]} " =~ " $TEST_NAME" ]]; then - echo -e "\033[33mSKIPPED\033[0m (hardcoded to skip)" - continue - fi - - if [[ ! -f ./$TEST_NAME/target/program.json || ! -f ./$TEST_NAME/target/witness.gz ]]; then - echo -e "\033[33mSKIPPED\033[0m (uncompiled)" - continue - fi - - # If parallel flag is set, run in parallel - if [ -n "${PARALLEL:-}" ]; then - test $TEST_NAME & - else - test $TEST_NAME - fi - done -fi - -wait - -# Check for parallel errors -# If error file exists, exit with error -if [ -f "$error_file" ]; then - rm "$error_file" - echo "Error occurred in one or more child processes. Exiting..." - exit 1 -fi diff --git a/barretenberg/bootstrap_cache.sh b/barretenberg/bootstrap_cache.sh deleted file mode 100755 index 252e25f2e9ff..000000000000 --- a/barretenberg/bootstrap_cache.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -eu - -cd "$(dirname "$0")" - -# Run both tasks in the background -(cd cpp && ./bootstrap_cache.sh "$@") & -pid_cpp=$! -(cd ts && ./bootstrap_cache.sh "$@") & -pid_ts=$! - -# Wait for both processes and capture any non-zero exit codes -wait $pid_cpp || exit_code=$? -wait $pid_ts || exit_code=$? - -# Exit with the first non-zero exit code, if any -exit ${exit_code:-0} diff --git a/barretenberg/cpp/scripts/install-wasi-sdk.sh b/barretenberg/cpp/scripts/install-wasi-sdk.sh deleted file mode 100755 index 650ab723d22a..000000000000 --- a/barretenberg/cpp/scripts/install-wasi-sdk.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -set -eu - -if [[ -d ./src/wasi-sdk ]]; then - echo "WASI already installed. Skipping." - exit 0 -fi - -# Clean. -rm -rf ./src/wasi-sdk - -# Determine system. -if [[ "$OSTYPE" == "darwin"* ]]; then - OS=macos -elif [[ "$OSTYPE" == "linux-gnu" ]]; then - OS=linux -else - echo "Unknown OS: $OSTYPE" - exit 1 -fi - -# Install the webassembly toolchain. -mkdir -p src -cd ./src -curl -s -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20/wasi-sdk-20.0-$OS.tar.gz | tar zxf - -if [ "$OS" == "linux" ]; then - # Exceptional linux case that provides an llvm build that works on Ubuntu 20. - curl -s -L https://wasi-sdk.s3.eu-west-2.amazonaws.com/yamt-wasi-sdk-20.0.threads.tgz | tar zxf - -else - # For other operating systems, first download the standard release (this is to get the llvm build). - curl -s -L https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-20%2Bthreads/wasi-sdk-20.0.threads-$OS.tar.gz | tar zxf - - # Replace wasi-sysroot in wasi-sdk-20.0+threads with our custom build. - # It contains libc++ and a patch by yamt to improve thread join stability. - # Can remove once future releases are more stable. - curl -s -L https://wasi-sdk.s3.eu-west-2.amazonaws.com/yamt-wasi-sysroot-20.0.threads.tgz | tar zxf - -fi -# TODO(https://github.com/AztecProtocol/barretenberg/issues/906): in the future this should use earthly and a 'SAVE ARTIFACT wasi-sdk AS LOCAL wasi-sdk' -if [ "$(id -u)" -eq 0 ]; then - mv wasi-sdk-20.0+threads /opt/wasi-sdk -else - sudo mv wasi-sdk-20.0+threads /opt/wasi-sdk -fi diff --git a/barretenberg/ts/README.md b/barretenberg/ts/README.md index 3209bd1ed4fc..f6a0c1db0594 100644 --- a/barretenberg/ts/README.md +++ b/barretenberg/ts/README.md @@ -13,8 +13,6 @@ Note there are two independent WASM builds, one with threading enabled and one w memory flag is set within the WASM itself. If you're running in a context where you can't have shared memory, we want to fallback to single threaded performance. -The following output is from `bench_acir_tests.sh` script. - Table represents time in ms to build circuit and proof for each test on n threads. Ignores proving key construction. diff --git a/build-images/adhoc-installs.sh b/build-images/adhoc-installs.sh deleted file mode 100755 index b31d0cb5ff02..000000000000 --- a/build-images/adhoc-installs.sh +++ /dev/null @@ -1,8 +0,0 @@ -# The correct action to take in case of a missing installation is to update the build images. -# However, they will not be updated automatically going forward. -# If unable to ping Charlie or Adam to update the AMIs that have the build image (along with the build images themselves) -# this is a reasonable temporary measure. -set -eu -apt update -# for docs: -apt install -y libvips-dev diff --git a/build-images/src/Dockerfile b/build-images/src/Dockerfile index 51be3294b848..f14f82a8b837 100644 --- a/build-images/src/Dockerfile +++ b/build-images/src/Dockerfile @@ -227,10 +227,6 @@ RUN apt update && \ inotify-tools \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -# Install earthly. -RUN wget https://github.com/earthly/earthly/releases/latest/download/earthly-linux-$(dpkg --print-architecture) -O /usr/local/bin/earthly && \ - chmod +x /usr/local/bin/earthly - # Install osxcross. Requires developer to mount SDK from their mac host. COPY --from=osxcross /opt/osxcross /opt/osxcross ENV PATH="/opt/osxcross/bin:$PATH" diff --git a/ci.py b/ci.py deleted file mode 100755 index f01403878344..000000000000 --- a/ci.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 -# ubuntu: apt install python3-blessed -from blessed import Terminal -import os, json, subprocess, sys, time - -term = Terminal() -BRANCH = subprocess.run("git rev-parse --abbrev-ref HEAD", shell=True, text=True, capture_output=True).stdout.strip() -# Github actor is now just branch-derived -GITHUB_ACTOR = BRANCH.replace("/", "_") - -def main(): - selection = -1 - if len(sys.argv) >= 2: - selection = sys.argv[1] - else: - with term.fullscreen(), term.cbreak(): - print(term.home + term.clear) - while selection not in ('1', '2', '3', '4', '5', 'q'): - print(term.move_y(1) + "Please select an option:") - print("1. SSH into build machine") - print("2. SSH into bench machine") - print("3. Start/Stop spot machines") - print("4. Manage Running Jobs") - print("5. Run ci.yml manually") - print("q. Quit") - with term.location(0, term.height - 1): - selection = term.inkey() - - if selection == '1': - ssh_into_machine('x86') - elif selection == '2': - manage_spot_instances() - elif selection == '3': - manage_ci_workflows() - elif selection == '4': - call_ci_workflow() - -def ssh_into_machine(suffix): - ssh_key_path = os.path.expanduser('~/.ssh/build_instance_key') - if not os.path.exists(ssh_key_path): - print("SSH key does not exist.") - return - - # Parse the output to find the public IP address - for i in range(10): - # Command to get the instance information - cmd = f'aws ec2 describe-instances --filters "Name=instance-state-name,Values=running" "Name=tag:Name,Values=aztec-packages-{GITHUB_ACTOR}-{suffix}" --output json --region us-east-2' - print(cmd) - result = subprocess.run(cmd, shell=True, capture_output=True, text=True) - if result.returncode != 0: - print("Failed to get AWS instances:", result.stderr) - return - try: - print(result.stdout) - instances_data = json.loads(result.stdout) - instance = instances_data['Reservations'][0]['Instances'][0] - instance_ip = instance['PublicIpAddress'] - break - except (KeyError, IndexError, json.JSONDecodeError) as e: - print("Error parsing AWS CLI output, trying again:", e) - if i == 0: - print("Couldn't find spot, starting spot, and looping until we can find it") - call_spot_workflow('start') - elif i == 9: - print("Couldn't find spot even after creating it!") - sys.exit(1) - time.sleep(10) - - # SSH command using the public IP - ssh_cmd = f"ssh -o StrictHostKeychecking=no -i {ssh_key_path} ubuntu@{instance_ip} 'sudo shutdown -P 40'" - ssh_process = subprocess.Popen(ssh_cmd, shell=True) - ssh_process.wait() # Wait for the SSH session to complete - ssh_cmd = f"ssh -o StrictHostKeychecking=no -i {ssh_key_path} ubuntu@{instance_ip}" - print(f"Connecting to {instance_ip}. Note that unless a github job or 'earthly' are running, spot may terminate.") - ssh_process = subprocess.Popen(ssh_cmd, shell=True) - ssh_process.wait() # Wait for the SSH session to complete - -def call_spot_workflow(action): - subprocess.run(f'gh workflow run start-spot.yml --ref {BRANCH} --field username="{GITHUB_ACTOR}" --field action="{action}"', shell=True) - -def call_ci_workflow(): - print( - "NOTE: This is mostly useful if impersonating a GITHUB_ACTOR. Usually you rather do Manage Running Jobs and retry." - ) - subprocess.run(f'gh workflow run ci.yml --ref {BRANCH} --field username="{GITHUB_ACTOR}"', shell=True) - -def manage_spot_instances(): - call_spot_workflow(input("Enter one of 'start', 'stop', 'restart':")) - -def manage_ci_workflows(): - # Retrieve the most recent workflow run - cmd = f"gh run list --workflow=ci.yml -u {GITHUB_ACTOR} --limit 5" - result = subprocess.run(cmd, shell=True, capture_output=True, text=True) - if result.returncode != 0 or not result.stdout.strip(): - print("Failed to retrieve workflow runs or no runs found.") - return - print("Most recent CI run details:") - print(result.stdout) - - action = input("Enter action 'cancel', 'rerun', 'rerun-all', 'force-cancel' or 'view' (default)") or 'view' - print(f"\nWill perform {action}") - run_id = input(f"Enter the run ID to {action}: ") - - if action.lower() == 'cancel': - subprocess.run(f"gh run cancel {run_id}", shell=True) - if action.lower() == 'rerun': - # needed so the spot runners still work - call_spot_workflow('start') - subprocess.run(f"gh run rerun {run_id} --failed", shell=True) - elif action.lower() == 'rerun-all': - subprocess.run(f"gh run rerun {run_id}", shell=True) - elif action.lower() == 'force-cancel': - subprocess.run('gh api --method POST -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" ' + - '/repos/AztecProtocol/aztec-packages/actions/runs/' + run_id + '/force-cancel', shell=True) - else: - subprocess.run(f"gh run watch {run_id}", shell=True) - -if __name__ == "__main__": - main() - diff --git a/ci3/bootstrap_local_noninteractive b/ci3/bootstrap_local_noninteractive deleted file mode 100755 index e0a489a46b75..000000000000 --- a/ci3/bootstrap_local_noninteractive +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Launches our CI image locally and runs the bootstrap. -# This is used interim while we have CI runners that are not launched with bootstrap_ec2. -# It uses docker-in-docker as some test flows require it (e.g. e2e). -# We use a volume on /var/lib/docker as overlayfs trashes performance (in fact it just breaks). -# We mount in aws credentials to leverage the s3 cache. -# The host repository is mounted in read-only, and a clone is taken to ensure a clean start. -# If anything goes wrong during the run, the container will drop into a shell. - -root=$(git rev-parse --show-toplevel) -source $root/ci3/source -current_commit=$(git rev-parse HEAD) -cmd=${1:-"CI=1 ./bootstrap.sh fast"} - -docker run --rm \ - --privileged \ - -v bootstrap_ci_local_docker:/var/lib/docker \ - -v $root:/aztec-packages-host:ro \ - -v $HOME/.aws:/root/.aws \ - aztecprotocol/ci:2.2 bash -c " - /usr/local/share/docker-init.sh &> /dev/null - git config --global --add safe.directory /aztec-packages-host/.git - mkdir -p /root/aztec-packages && cd /root/aztec-packages - # Ensure we get a clean clone of the repo. - git init &>/dev/null - git remote add origin https://github.com/aztecprotocol/aztec-packages - git fetch --depth 1 origin $current_commit 2>/dev/null || (echo 'The commit was not pushed, run aborted.' && exit 1) - git checkout FETCH_HEAD &>/dev/null - $cmd -" diff --git a/ci3/cache_download_flag b/ci3/cache_download_flag deleted file mode 100755 index ef0137fb622e..000000000000 --- a/ci3/cache_download_flag +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -NO_CD=1 source $(git rev-parse --show-toplevel)/ci3/source - -if [ "$#" -lt 1 ]; then - echo_stderr "Usage: $0 " - exit 1 -fi - -if [[ "$1" == *"disabled-cache"* ]]; then - echo "Running test $1 due to uncommitted changes/files." - exit 1 -fi -if [ "${NO_CACHE:-0}" == 1 ] ; then - # Don't look if CI isn't set. No need to muddle with dev runs. - echo_stderr "Running test $1 because NO_CACHE=1." - exit 1 -fi - -NAME=$1 - -# Read out the build flag as it has the context of the last success. -# Extract endpoint URL if S3_BUILD_CACHE_AWS_PARAMS is set -if [[ -n "${S3_BUILD_CACHE_AWS_PARAMS:-}" ]]; then - # Write the flag file to stderr, return exit success if found. - aws $S3_BUILD_CACHE_AWS_PARAMS s3 cp "s3://aztec-ci-artifacts/build-cache/$NAME.flag" - --quiet --no-progress >&2 -else - # Default to AWS S3 URL if no custom endpoint is set. - S3_ENDPOINT="http://aztec-ci-artifacts.s3.amazonaws.com" - # Write the flag file to stderr, return exit success if found. - curl -s -f "$S3_ENDPOINT/build-cache/$NAME.flag" >&2 || (echo_stderr "Running test $NAME because cache flag not found." && exit 1) -fi diff --git a/ci3/cache_list_files b/ci3/cache_list_files deleted file mode 100755 index cb3daa8b8028..000000000000 --- a/ci3/cache_list_files +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -aws s3 cp s3://aztec-ci-artifacts/build-cache/$1 - | tar -tzf - diff --git a/ci3/cache_upload_flag b/ci3/cache_upload_flag deleted file mode 100755 index f5b6fb0c8519..000000000000 --- a/ci3/cache_upload_flag +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -NO_CD=1 source $(git rev-parse --show-toplevel)/ci3/source - -if [ "$#" -lt 1 ]; then - echo_stderr "Usage: $0 " - exit 1 -fi - -if [[ "$1" == *"disabled-cache"* ]]; then - echo "Not uploading test flag $1 due to uncommitted changes/files." - exit 0 -fi -if [ "${CI:-0}" -lt 1 ]; then - # Don't upload if CI isn't set. No need to muddle with dev runs. - echo_stderr "Not uploading test flag $1 because CI=0." - exit 0 -fi - -function on_exit() { - # Cleanup the temporary tar.gz file - rm -f .success -} -trap on_exit EXIT - -NAME="$1" - -shift 1 -if [ -n "${GITHUB_RUN_URL:-}" ]; then - CONTENT="Flag $NAME success cached: See $GITHUB_RUN_URL" -else - CONTENT="Flag $NAME success cached: Ran outside of Github Actions" -fi - -echo_stderr "$CONTENT" | aws ${S3_BUILD_CACHE_AWS_PARAMS:-} s3 cp - "s3://aztec-ci-artifacts/build-cache/$NAME.flag" --quiet --no-progress -echo_stderr "Cache upload of $NAME.flag complete." diff --git a/ci3/source_tmp b/ci3/source_tmp deleted file mode 100644 index 732f44bff21f..000000000000 --- a/ci3/source_tmp +++ /dev/null @@ -1,6 +0,0 @@ -# Injects an automatically cleaned up $TMP into the current bash environment -TMP=$(mktemp -d) -function __tmp_cleanup() { - rm -rf "$TMP" &>/dev/null || true -} -trap __tmp_cleanup EXIT \ No newline at end of file diff --git a/ci3/test_should_run b/ci3/test_should_run deleted file mode 100755 index 0bb8e95f1450..000000000000 --- a/ci3/test_should_run +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# This is called within test functions to provide an early out for running tests in CI. -source $(git rev-parse --show-toplevel)/ci3/source - -# If we're in CI and the cache flag is present, we skip running tests. -if [ "${CI:-0}" -eq 1 ] && cache_download_flag "$1"; then - exit 1 -fi \ No newline at end of file diff --git a/docker-compose.provernet.yml b/docker-compose.provernet.yml deleted file mode 100644 index e6e760cd452d..000000000000 --- a/docker-compose.provernet.yml +++ /dev/null @@ -1,201 +0,0 @@ -# Template for simulating a provernet-like setup using docker-compose, currently unused except for development or testing. -# Spins up an aztec node with a sequencer, a transactions bot, and a prover node with a separate prover agent. -# Logs latest block numbers every 10 seconds. -name: aztec-provernet -services: - # Anvil instance that serves as L1 - ethereum: - image: aztecprotocol/foundry:25f24e677a6a32a62512ad4f561995589ac2c7dc-${ARCH_TAG:-amd64} - entrypoint: "anvil --block-time 12 --silent -p 8545 --host 0.0.0.0 --chain-id 31337" - ports: - - 8545:8545 - - # Single Aztec node with a sequencer for building and publishing unproven blocks to L1 - aztec-node: - image: "aztecprotocol/${IMAGE:-aztec:master}" - ports: - - "8080:80" - environment: - LOG_LEVEL: info - ETHEREUM_HOSTS: http://ethereum:8545 - L1_CHAIN_ID: 31337 - AZTEC_PORT: 80 - DEPLOY_AZTEC_CONTRACTS: 1 - DEPLOY_AZTEC_CONTRACTS_SALT: 1 - ARCHIVER_POLLING_INTERVAL_MS: 1000 - ARCHIVER_VIEM_POLLING_INTERVAL_MS: 1000 - SEQ_VIEM_POLLING_INTERVAL_MS: 1000 - SEQ_MAX_TX_PER_BLOCK: 4 - SEQ_MIN_TX_PER_BLOCK: 1 - SEQ_MAX_SECONDS_BETWEEN_BLOCKS: 3600 - SEQ_MIN_SECONDS_BETWEEN_BLOCKS: 0 - SEQ_PUBLISHER_PRIVATE_KEY: "0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a" - VALIDATOR_PRIVATE_KEY: "0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a" - PROVER_REAL_PROOFS: "${PROVER_REAL_PROOFS:-false}" - P2P_ENABLED: false - volumes: - - ./log/aztec-node/:/usr/src/yarn-project/aztec/log:rw - healthcheck: - test: ["CMD", "curl", "-fSs", "http://127.0.0.1:80/status"] - interval: 3s - timeout: 30s - start_period: 120s - depends_on: - - ethereum - command: - - "start" - - "--node" - - "--archiver" - - "--sequencer" - - # Prover node that listens for unproven blocks on L1, generates quotes, and submits epoch proofs - # Requires one or more agents to be connected for actually generating proofs - # Fetches individual tx proofs from the aztec-node directly - aztec-prover: - image: "aztecprotocol/${IMAGE:-aztec:master}" - ports: - - "8083:80" - environment: - LOG_LEVEL: verbose - ETHEREUM_HOSTS: http://ethereum:8545 - PROVER_COORDINATION_NODE_URL: http://aztec-node - AZTEC_PORT: 80 - ARCHIVER_POLLING_INTERVAL_MS: 1000 - ARCHIVER_VIEM_POLLING_INTERVAL_MS: 1000 - PROVER_VIEM_POLLING_INTERVAL_MS: 1000 - PROVER_AGENT_COUNT: 0 - PROVER_BROKER_HOST: http://aztec-prover-broker - PROVER_PUBLISHER_PRIVATE_KEY: "0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97" - PROVER_REAL_PROOFS: "${PROVER_REAL_PROOFS:-false}" - PROVER_MINIMUM_ESCROW_AMOUNT: 1000000000 - PROVER_TARGET_ESCROW_AMOUNT: 2000000000 - PROVER_ID: "${PROVER_ID:-0x01}" - volumes: - - ./log/aztec-prover/:/usr/src/yarn-project/aztec/log:rw - depends_on: - aztec-node: - condition: service_healthy - aztec-prover-broker: - condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-fSs", "http://127.0.0.1:80/status"] - interval: 3s - timeout: 30s - start_period: 120s - command: ["start", "--prover-node", "--archiver"] - restart: on-failure:5 - - aztec-prover-broker: - image: "aztecprotocol/${IMAGE:-aztec:master}" - ports: - - "8084:80" - environment: - LOG_LEVEL: verbose - AZTEC_PORT: 80 - healthcheck: - test: ["CMD", "curl", "-fSs", "http://127.0.0.1:80/status"] - interval: 3s - timeout: 30s - start_period: 120s - command: ["start", "--prover-broker"] - restart: on-failure:5 - - # Prover agent that connects to the prover-node for fetching proving jobs and executing them - # Multiple instances can be run, or PROVER_AGENT_CONCURRENCY can be increased to run multiple workers in a single instance - aztec-prover-agent: - image: "aztecprotocol/${IMAGE:-aztec:master}" - ports: - - "8090:80" - environment: - LOG_LEVEL: verbose - ETHEREUM_HOSTS: http://ethereum:8545 - PROVER_BROKER_HOST: http://aztec-prover-broker - L1_CHAIN_ID: 31337 - AZTEC_PORT: 80 - PROVER_REAL_PROOFS: "${PROVER_REAL_PROOFS:-false}" - PROVER_TEST_DELAY_MS: "${PROVER_TEST_DELAY_MS:-0}" - BB_SKIP_CLEANUP: "${BB_SKIP_CLEANUP:-0}" # Persist tmp dirs for debugging - PROVER_ID: "${PROVER_ID:-0x01}" - volumes: - - ./log/aztec-prover-agent/:/usr/src/yarn-project/aztec/log:rw - - ./cache/bb-crs/:/root/.bb-crs:rw - - ./workdir/bb-prover/:/usr/src/yarn-project/bb:rw - depends_on: - aztec-prover-broker: - condition: service_healthy - command: ["start", "--prover-agent"] - deploy: - mode: replicated - replicas: 2 - restart: on-failure:5 - healthcheck: - test: ["CMD", "curl", "-fSs", "http://127.0.0.1:80/status"] - interval: 3s - timeout: 30s - start_period: 20s - - # Bot for keeping a steady flow of txs into the network - # Requires bootstrapping to be completed successfully - aztec-bot: - image: "aztecprotocol/${IMAGE:-aztec:master}" - ports: - - "8082:80" - environment: - LOG_LEVEL: info - ETHEREUM_HOSTS: http://ethereum:8545 - AZTEC_NODE_URL: http://aztec-node - L1_CHAIN_ID: 31337 - AZTEC_PORT: 80 - BOT_L1_PRIVATE_KEY: "0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97" - BOT_PRIVATE_KEY: 0xcafe - BOT_TX_INTERVAL_SECONDS: 300 - BOT_PRIVATE_TRANSFERS_PER_TX: 1 - BOT_PUBLIC_TRANSFERS_PER_TX: 0 - BOT_NO_WAIT_FOR_TRANSFERS: true - BOT_NO_START: false - BOT_MAX_CONSECUTIVE_ERRORS: 3 - BOT_STOP_WHEN_UNHEALTHY: true - PXE_PROVER_ENABLED: "${PROVER_REAL_PROOFS:-false}" - PROVER_REAL_PROOFS: "${PROVER_REAL_PROOFS:-false}" - BB_SKIP_CLEANUP: "${BB_SKIP_CLEANUP:-0}" # Persist tmp dirs for debugging - volumes: - - ./log/aztec-bot/:/usr/src/yarn-project/aztec/log:rw - - ./cache/bb-crs/:/root/.bb-crs:rw - - ./workdir/bb-bot/:/usr/src/yarn-project/bb:rw - depends_on: - aztec-node: - condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-fSs", "http://127.0.0.1:80/status"] - interval: 3s - timeout: 30s - start_period: 90s - restart: on-failure:5 - command: ["start", "--bot", "--pxe"] - - # Simple watcher that logs the latest block numbers every few seconds using the CLI and the bot's PXE - aztec-block-watcher: - image: "aztecprotocol/${IMAGE:-aztec:master}" - environment: - ETHEREUM_HOSTS: http://ethereum:8545 - L1_CHAIN_ID: 31337 - depends_on: - aztec-bot: - condition: service_healthy - entrypoint: "/bin/bash -c" - command: > - 'while true; do node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js block-number -u http://aztec-bot | head -n2; sleep 10; done' - restart: on-failure:5 - stop_grace_period: 1s - - # Run a CLI command against the bot's PXE or L1 - # docker compose run --rm aztec-cli block-number - aztec-cli: - image: "aztecprotocol/${IMAGE:-aztec:master}" - environment: - ETHEREUM_HOSTS: http://ethereum:8545 - PXE_URL: http://aztec-bot - L1_CHAIN_ID: 31337 - stdin_open: true - tty: true - scale: 0 diff --git a/l1-contracts/terraform/main.tf b/l1-contracts/terraform/main.tf deleted file mode 100644 index a9b9b4a3faa0..000000000000 --- a/l1-contracts/terraform/main.tf +++ /dev/null @@ -1,121 +0,0 @@ -terraform { - backend "s3" { - bucket = "aztec-terraform" - region = "eu-west-2" - } - required_providers { - aws = { - source = "hashicorp/aws" - version = "3.74.2" - } - } -} - -variable "ROLLUP_CONTRACT_ADDRESS" { - type = string - default = "" -} - -output "rollup_contract_address" { - value = var.ROLLUP_CONTRACT_ADDRESS -} - -variable "REGISTRY_CONTRACT_ADDRESS" { - type = string - default = "" -} - -output "registry_contract_address" { - value = var.REGISTRY_CONTRACT_ADDRESS -} - -variable "INBOX_CONTRACT_ADDRESS" { - type = string - default = "" -} - -output "inbox_contract_address" { - value = var.INBOX_CONTRACT_ADDRESS -} - -variable "OUTBOX_CONTRACT_ADDRESS" { - type = string - default = "" -} - -output "outbox_contract_address" { - value = var.OUTBOX_CONTRACT_ADDRESS -} - - -variable "FEE_JUICE_CONTRACT_ADDRESS" { - type = string - default = "" -} - -output "fee_juice_contract_address" { - value = var.FEE_JUICE_CONTRACT_ADDRESS -} - -variable "STAKING_ASSET_CONTRACT_ADDRESS" { - type = string - default = "" -} - -output "staking_asset_contract_address" { - value = var.STAKING_ASSET_CONTRACT_ADDRESS -} - -variable "FEE_JUICE_PORTAL_CONTRACT_ADDRESS" { - type = string - default = "" -} - -output "FEE_JUICE_PORTAL_CONTRACT_ADDRESS" { - value = var.FEE_JUICE_PORTAL_CONTRACT_ADDRESS -} - -variable "COIN_ISSUER_CONTRACT_ADDRESS" { - type = string - default = "" -} - -output "COIN_ISSUER_CONTRACT_ADDRESS" { - value = var.COIN_ISSUER_CONTRACT_ADDRESS -} - -variable "REWARD_DISTRIBUTOR_CONTRACT_ADDRESS" { - type = string - default = "" -} - -output "REWARD_DISTRIBUTOR_CONTRACT_ADDRESS" { - value = var.REWARD_DISTRIBUTOR_CONTRACT_ADDRESS -} - -variable "GOVERNANCE_PROPOSER_CONTRACT_ADDRESS" { - type = string - default = "" -} - -output "GOVERNANCE_PROPOSER_CONTRACT_ADDRESS" { - value = var.GOVERNANCE_PROPOSER_CONTRACT_ADDRESS -} - -variable "GOVERNANCE_CONTRACT_ADDRESS" { - type = string - default = "" -} - -output "GOVERNANCE_CONTRACT_ADDRESS" { - value = var.GOVERNANCE_CONTRACT_ADDRESS -} - -variable "SLASH_FACTORY_CONTRACT_ADDRESS" { - type = string - default = "" -} - -output "SLASH_FACTORY_CONTRACT_ADDRESS" { - value = var.SLASH_FACTORY_CONTRACT_ADDRESS -} diff --git a/noir-projects/noir-protocol-circuits/crates/blob/src/blob_public_inputs.nr b/noir-projects/noir-protocol-circuits/crates/blob/src/blob_public_inputs.nr index c92d67d4171e..8fc6a8db524e 100644 --- a/noir-projects/noir-protocol-circuits/crates/blob/src/blob_public_inputs.nr +++ b/noir-projects/noir-protocol-circuits/crates/blob/src/blob_public_inputs.nr @@ -6,7 +6,7 @@ use types::{ utils::reader::Reader, }; -// NB: This only exists because a nested array of [[Field; 2]; N] did not build with earthly, but was fine otherwise +// NB: This only exists because a nested array of [[Field; 2]; N] did not build with ci2.5, but was fine otherwise // For blobs, we use the compressed 48 byte BLS12 commitment to compute the challenge. We never need to operate on it, // so it's encoded as 2 fields. The first is the first 31 bytes, and the second is the next 17 bytes. pub struct BlobCommitment { diff --git a/noir-projects/scripts/check.sh b/noir-projects/scripts/check.sh deleted file mode 100755 index 81c97a8cbc16..000000000000 --- a/noir-projects/scripts/check.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -eu - -cd $(dirname "$0")/../ - -nargo check --program-dir ./aztec-nr -nargo check --program-dir ./noir-contracts -nargo check --program-dir ./noir-protocol-circuits -nargo check --program-dir ./mock-protocol-circuits diff --git a/noir/.gitignore b/noir/.gitignore index b211695f37c5..77abdbdb12e3 100644 --- a/noir/.gitignore +++ b/noir/.gitignore @@ -1,3 +1,2 @@ **/package.tgz packages -.earthly-staging \ No newline at end of file diff --git a/noir/scripts/bootstrap_native.sh b/noir/scripts/bootstrap_native.sh deleted file mode 100755 index e53331c5741f..000000000000 --- a/noir/scripts/bootstrap_native.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -set -eu - -cd $(dirname "$0")/../noir-repo - -echo Bootstrapping noir native... - -# Set build data manually. -export SOURCE_DATE_EPOCH=$(date -d "today 00:00:00" +%s) -export GIT_DIRTY=false -export GIT_COMMIT=${COMMIT_HASH:-$(git rev-parse --verify HEAD)} - -# Some of the debugger tests are a little flaky wrt to timeouts so we allow a couple of retries. -export NEXTEST_RETRIES=2 - -# Check if the 'cargo' command is available in the system -if ! command -v cargo > /dev/null; then - echo "Cargo is not installed. Please install Cargo and the Rust toolchain." - exit 1 -fi - -# Build native. -if [ -n "${DEBUG:-}" ]; then - RUSTFLAGS=-Dwarnings cargo build -else - RUSTFLAGS=-Dwarnings cargo build --release -fi - -if [ -x ../scripts/fix_incremental_ts.sh ]; then - ../scripts/fix_incremental_ts.sh -fi diff --git a/noir/scripts/bootstrap_packages.sh b/noir/scripts/bootstrap_packages.sh deleted file mode 100755 index 59ae5e27c1ea..000000000000 --- a/noir/scripts/bootstrap_packages.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -set -eu - -ROOT=$(realpath $(dirname "$0")/..) -cd $ROOT/noir-repo - -echo Bootstrapping noir js packages... - -./.github/scripts/wasm-bindgen-install.sh - -# Set build data manually. -export SOURCE_DATE_EPOCH=$(date -d "today 00:00:00" +%s) -export GIT_DIRTY=false -export GIT_COMMIT=${COMMIT_HASH:-$(git rev-parse --verify HEAD)} - -PROJECTS=( - @noir-lang/acvm_js - @noir-lang/types - @noir-lang/noirc_abi - @noir-lang/noir_codegen - @noir-lang/noir_js -) -INCLUDE=$(printf " --include %s" "${PROJECTS[@]}") - -yarn install - -yarn workspaces foreach --parallel --topological-dev --verbose $INCLUDE run build - -# We create a folder called packages, that contains each package as it would be published to npm, named correctly. -# These can be useful for testing, or portaling into other projects. -yarn workspaces foreach --parallel $INCLUDE pack - -cd $ROOT -rm -rf packages && mkdir -p packages -for PROJECT in "${PROJECTS[@]}"; do - PPATH=$(cd noir-repo && yarn workspaces list --json | jq -r "select(.name==\"$PROJECT\").location") - tar zxfv noir-repo/$PPATH/package.tgz -C packages && mv packages/package packages/${PROJECT#*/} -done - -if [ -x $ROOT/scripts/fix_incremental_ts.sh ]; then - $ROOT/scripts/fix_incremental_ts.sh -fi diff --git a/noir/scripts/test_js_packages.sh b/noir/scripts/test_js_packages.sh deleted file mode 100755 index f384aa346b4c..000000000000 --- a/noir/scripts/test_js_packages.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -eu - -cd $(dirname "$0")/../noir-repo - -./.github/scripts/wasm-bindgen-install.sh -# ./.github/scripts/playwright-install.sh - -# Set build data manually. -export SOURCE_DATE_EPOCH=$(date -d "today 00:00:00" +%s) -export GIT_DIRTY=false -export GIT_COMMIT=${COMMIT_HASH:-$(git rev-parse --verify HEAD)} - -# cargo build --release - -yarn -yarn build - -export NODE_OPTIONS=--max_old_space_size=8192 -yarn workspaces foreach \ - --parallel \ - --verbose \ - --exclude @noir-lang/root \ - --exclude @noir-lang/noir_js \ - --exclude integration-tests \ - --exclude @noir-lang/noir_wasm \ - run test - -# TOOD(ci3): Circular dependency on bb. Rethink or remove this. Should we run these? -# yarn workspaces foreach \ -# --parallel \ -# --verbose \ -# --include integration-tests \ -# --include @noir-lang/noir_wasm \ -# run test:node - -# yarn workspaces foreach \ -# --verbose \ -# --include integration-tests \ -# --include @noir-lang/noir_wasm \ -# run test:browser \ No newline at end of file diff --git a/noir/scripts/test_native.sh b/noir/scripts/test_native.sh deleted file mode 100755 index 9b6b556bf5a0..000000000000 --- a/noir/scripts/test_native.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -eu - -# Go to noir repo root. -cd $(dirname "$0")/../noir-repo - -# Set build data manually. -export SOURCE_DATE_EPOCH=$(date -d "today 00:00:00" +%s) -export GIT_DIRTY=false -export GIT_COMMIT=${COMMIT_HASH:-$(git rev-parse --verify HEAD)} - -# Check formatting of noir code. -(cd ./test_programs && ./format.sh check) - -# Check formatting of rust code. -cargo fmt --all --check - -# Linting. If local use a separate build dir to not clobber incrementals. In CI we want to save space. -[ "${CI:-0}" -eq 0 ] && args="--target-dir target/clippy" || args="" -RUSTFLAGS=-Dwarnings cargo clippy $args --workspace --locked --release - -# Install nextest. -./.github/scripts/cargo-binstall-install.sh -cargo-binstall cargo-nextest --version 0.9.67 -y --secure - -# Test. -export RAYON_NUM_THREADS=1 -jobs=$(($(nproc) / RAYON_NUM_THREADS)) -[ "${CI:-0}" -eq 0 ] && args="--target-dir target/nextest" || args="" -cargo nextest run -j$jobs $args --workspace --locked --release -E '!test(hello_world_example) & !test(simple_verifier_codegen)' diff --git a/scripts/ci/get_bench_jobs.sh b/scripts/ci/get_bench_jobs.sh deleted file mode 100755 index 1cd0c1d5cdb4..000000000000 --- a/scripts/ci/get_bench_jobs.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -set -eu - -# Enter repo root. -cd "$(dirname "$0")"/../.. - -BRANCH=$1 -# support labels with hyphens for backwards compatibility: -LABELS=$(echo $2 | sed 's/-/_/g') - -# Function to parse YAML and extract test names -get_test_names() { - yq e '.tests | keys | .[]' yarn-project/end-to-end/scripts/e2e_test_config.yml -} - -# Define the allow_list -allow_list=() - -# Add labels from input to the allow_list -IFS=',' read -r -a input_labels <<< "$LABELS" -allow_list+=("${input_labels[@]}") - -# Generate potential list of test targets on one line -test_list=$(get_test_names | grep 'bench' | xargs echo) - -# If branch is master or allow_list contains 'bench-all', return full list -if [[ "$BRANCH" == "master" ]] || [[ " ${allow_list[@]} " =~ "bench_all" ]]; then - # print as JSON list - echo "$test_list" | jq -Rc 'split(" ")' - exit 0 -fi - -# Filter the test_list to include only items in the allow_list -filtered_list=() -for item in $test_list; do - for allowed in "${allow_list[@]}"; do - if [[ "$item" == "$allowed" ]]; then - filtered_list+=("$item") - fi - done -done - -# Print the filtered list in JSON format -echo ${filtered_list[@]} | jq -Rc 'split(" ")' diff --git a/scripts/ci/get_e2e_jobs.sh b/scripts/ci/get_e2e_jobs.sh deleted file mode 100755 index f79869b3eef5..000000000000 --- a/scripts/ci/get_e2e_jobs.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash -set -eu - -# Enter repo root. -cd "$(dirname "$0")"/../.. - -BRANCH=$1 -# support labels with hyphens for backwards compatibility: -LABELS=$(echo $2 | sed 's/-/_/g') - -# Function to parse YAML and extract test names -get_test_names() { - # prover full is ran standalone - yq e '.tests | keys | .[]' yarn-project/end-to-end/scripts/e2e_test_config.yml | grep -v e2e_prover_full -} - -# Read the full list from the YAML file -full_list=$(get_test_names) - -# Define the jobs that will run on every PR -allow_list=( - "e2e_2_pxes" - "e2e_authwit" - "e2e_amm" - "e2e_avm_simulator" - "e2e_block_building" - "e2e_cross_chain_messaging" - "e2e_crowdfunding_and_claim" - "e2e_deploy_contract" - "e2e_epochs" - "e2e_fees" - "e2e_fees_failures" - "e2e_fees_gas_estimation" - "e2e_fees_private_payments" - "e2e_max_block_number" - "e2e_nested_contract" - "e2e_ordering" - "e2e_pruned_blocks" - "e2e_static_calls" - "e2e_token_bridge_tutorial_test" - "integration_l1_publisher" - "e2e_cheat_codes" - "e2e_prover_fake_proofs" - "e2e_lending_contract" - "e2e_p2p_gossip" - "kind_network_smoke" - "guides_dapp_testing" - "guides_sample_dapp" - "guides_sample_dapp_ci" - "guides_up_quick_start" - "guides_writing_an_account_contract" -) - -# Add labels from input to the allow_list, supports prefix matching -# E.g: -# e2e_p2p label will match e2e_p2p_gossip, e2e_p2p_rediscovery, e2e_p2p_reqresp etc. -# e2e_prover label will match e2e_prover_fake_proofs etc. -IFS=',' read -r -a input_labels <<< "$LABELS" -expanded_allow_list=() - -for label in "${input_labels[@]}"; do - # For each input label, find all tests that start with this prefix - matching_tests=$(echo "$full_list" | tr ' ' '\n' | grep "^${label}" || true) - - # If matching tests are found, add them to expanded_allow_list; otherwise, add the label itself - if [ -n "$matching_tests" ]; then - expanded_allow_list+=($matching_tests) - else - expanded_allow_list+=("$label") - fi -done - -# Add the input labels and expanded matches to allow_list -allow_list+=("${input_labels[@]}" "${expanded_allow_list[@]}") - -# Generate full list of targets, excluding specific entries, on one line -test_list=$(echo "${full_list[@]}" | grep -v 'base' | grep -v 'bench' | grep -v "network" | grep -v 'devnet' | xargs echo) - -# If branch is master or allow_list contains 'e2e-all', return full list -if [[ "$BRANCH" == "master" ]] || [[ " ${allow_list[@]} " =~ "e2e_all" ]]; then - # print as JSON list - echo "$test_list" | jq -Rc 'split(" ")' - exit 0 -fi - -# Filter the test_list to include only items in the allow_list -filtered_list=() -for item in $test_list; do - for allowed in "${allow_list[@]}"; do - if [[ "$item" == "$allowed" ]]; then - filtered_list+=("$item") - fi - done -done - -# Print the filtered list in JSON format -echo ${filtered_list[@]} | jq -Rc 'split(" ")' diff --git a/scripts/ci/select_runner.sh b/scripts/ci/select_runner.sh deleted file mode 100755 index 8a9460117a98..000000000000 --- a/scripts/ci/select_runner.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -# Check if the test requires a prover -requires_prover() { - local test_name=$1 - [[ $test_name == *"prover"* ]] -} - -test_name=$1 - -if requires_prover "$test_name"; then - echo "32core-tester-x86" -else - echo "8core-tester-x86" -fi diff --git a/scripts/ci/store_test_benchmark_logs b/scripts/ci/store_test_benchmark_logs deleted file mode 100755 index c9ae07fc96a4..000000000000 --- a/scripts/ci/store_test_benchmark_logs +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -# Script for storing barretenberg benchmark results. -# Uploads results to the AztecProtocol/benchmark-archive repository. - -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -e - -AZTEC_GITHUB_TOKEN=$1 - -mkdir -p /tmp/csv -export SAVED_TIMESTAMP=$(date +%s) -export HUMAN_READABLE_TIME=$(date -u -d @${SAVED_TIMESTAMP}) - -# Pick logs from test-logs and convert all information into CSV format including the current timestamp, branch, commit and tag information -for file in $(ls /tmp/test-logs); do - echo $file - cat /tmp/test-logs/$file \ - | grep "##BENCHMARK_INFO_PREFIX##" \ - | sed "s/.*##BENCHMARK_INFO_PREFIX##\(.*\)##BENCHMARK_INFO_SUFFIX##.*/\1/" \ - | sed "s/#/,/g" \ - | sed "s_\(.*\)_$SAVED_TIMESTAMP,$HUMAN_READABLE_TIME,$BRANCH,$COMMIT_HASH,$COMMIT_TAG,\1_" \ - >> /tmp/csv/new.csv -done -echo "Parsed from logs:" -cat /tmp/csv/new.csv - -# We have lots of repeated entries, no need to put them into repository. Unfortunately build times differ a bit and uniq only works with space as separator -cat /tmp/csv/new.csv \ - | sort \ - | sed "s_ _%_g" \ - | sed "s_^\(.*\),\(.*\)\$_\2 \1_" \ - | uniq -f 1 \ - | sed "s_^\(.*\) \(.*\)\$_\2,\1_" \ - | sed "s_%_ _g" >/tmp/csv/trimmed.csv - -# If there actually were any logs, update the information in the benchmark repository -if [ -s /tmp/csv/trimmed.csv ]; then - cd /tmp - - git clone --depth 1 https://$AZTEC_GITHUB_TOKEN:@github.com/AztecProtocol/benchmark-archive - - cd benchmark-archive - git config --global user.name AztecBot - git config --global user.email tech@aztecprotocol.com - cat /tmp/csv/trimmed.csv >>benchmarks.csv - git add benchmarks.csv - git commit -m "Added information from branch $BRANCH commit $COMMIT_HASH" - git push -fi diff --git a/scripts/copy_from_builder b/scripts/copy_from_builder deleted file mode 100755 index 63f65a101d3b..000000000000 --- a/scripts/copy_from_builder +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -eu - -scp -o ControlMaster=auto -o ControlPath=~/.ssh_mux_%h_%p_%r -o ControlPersist=30s -o TCPKeepAlive=no -o ServerAliveCountMax=5 -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -i "$BUILDER_SPOT_KEY" ubuntu@"$BUILDER_SPOT_IP":/home/ubuntu/run-$RUN_ID/$@ diff --git a/scripts/copy_from_tester b/scripts/copy_from_tester deleted file mode 100755 index 63fca68a4373..000000000000 --- a/scripts/copy_from_tester +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -eu - -scp -o ControlMaster=auto -o ControlPath=~/.ssh_mux_%h_%p_%r -o ControlPersist=30s -o TCPKeepAlive=no -o ServerAliveCountMax=5 -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -i "$SPOT_KEY" ubuntu@"$SPOT_IP":/home/ubuntu/run-$RUN_ID/$@ diff --git a/scripts/copy_to_tester b/scripts/copy_to_tester deleted file mode 100644 index ee5e73f27c85..000000000000 --- a/scripts/copy_to_tester +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -eu - -ABSOLUTE=${ABSOLUTE:-0} -if [ -n "${ABSOLUTE:-}" ]; then - prefix="" -else - prefix=/home/ubuntu/run-$RUN_ID/ -fi -scp -o ControlMaster=auto -o ControlPath=~/.ssh_mux_%h_%p_%r -o ControlPersist=30s -o TCPKeepAlive=no -o ServerAliveCountMax=5 -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -i "$SPOT_KEY" $1 ubuntu@"$SPOT_IP":"$prefix""$2" diff --git a/scripts/logs/download_logs_from_s3.sh b/scripts/logs/download_logs_from_s3.sh deleted file mode 100755 index a31979e3968c..000000000000 --- a/scripts/logs/download_logs_from_s3.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# Downloads the log files uploaded in upload_logs_to_s3 - -set -eu - -BUCKET_NAME="aztec-ci-artifacts" -LOG_FOLDER="${LOG_FOLDER:-log}" -COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" - -echo "Downloading logs from S3 for commit $COMMIT_HASH in branch ${BRANCH:-} at pull request ${PULL_REQUEST:-none}" - -# Paths from upload_logs_to_s3 -if [ "${BRANCH:-}" = "master" ]; then - LOG_SOURCE_FOLDER="logs-v1/master/$COMMIT_HASH" - BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/master/$COMMIT_HASH" - BENCHMARK_TARGET_FILE="benchmarks-v1/master/$COMMIT_HASH.json" - BENCHMARK_LATEST_FILE="benchmarks-v1/latest.json" -elif [ -n "${PULL_REQUEST:-}" ]; then - LOG_SOURCE_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" - BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/pulls/${PULL_REQUEST##*/}" - BENCHMARK_TARGET_FILE="benchmarks-v1/pulls/${PULL_REQUEST##*/}.json" -else - echo "Skipping benchmark run on branch ${BRANCH:-unknown}." - exit 0 -fi - -mkdir -p $LOG_FOLDER - -# Download benchmark log files from S3 LOG_SOURCE_FOLDER into local LOG_FOLDER -echo "Downloading benchmark log files from $BUCKET_NAME/$LOG_SOURCE_FOLDER to $LOG_FOLDER" -aws s3 cp "s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include 'bench*.jsonl' --recursive - -# Download barretenberg log files, these are direct benchmarks and separate from the above -aws s3 cp "s3://${BUCKET_NAME}/${BARRETENBERG_BENCH_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include '*_bench.json' --recursive - -echo "Downloaded log files $(ls $LOG_FOLDER)" \ No newline at end of file diff --git a/scripts/logs/upload_logs_to_s3.sh b/scripts/logs/upload_logs_to_s3.sh deleted file mode 100755 index fba855ab581f..000000000000 --- a/scripts/logs/upload_logs_to_s3.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# Uploads to S3 the contents of the log file mounted on the end-to-end container, -# which contains log entries with an associated event and metrics for it. -# Logs are uploaded to aztec-ci-artifacts/logs-v1/master/$COMMIT/$JOB.jsonl -# or to aztec-ci-artifacts/logs-v1/pulls/$PRNUMBER/$JOB.jsonl if on a PR - -[ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace -set -eu - -LOG_FOLDER=$1 -BUCKET_NAME="aztec-ci-artifacts" -COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" - -echo "Uploading logs to S3 for commit $COMMIT_HASH in branch ${BRANCH:-} at pull request ${PULL_REQUEST:-none}" - -if [ ! -d "$LOG_FOLDER" ] || [ -z "$(ls -A "$LOG_FOLDER")" ]; then - echo "No logs in folder $LOG_FOLDER to upload" - exit 0 -fi - -# Paths used in scripts/ci/assemble_e2e_benchmark.sh -if [ "${BRANCH:-}" = "master" ]; then - TARGET_FOLDER="logs-v1/master/$COMMIT_HASH/" -elif [ -n "${PULL_REQUEST:-}" ]; then - TARGET_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" -fi - -if [ -n "${TARGET_FOLDER:-}" ]; then - aws s3 cp $LOG_FOLDER "s3://${BUCKET_NAME}/${TARGET_FOLDER}" --include "*.jsonl" --recursive -else - echo Skipping upload since no target folder was defined -fi diff --git a/scripts/migrate_barretenberg_branch.sh b/scripts/migrate_barretenberg_branch.sh deleted file mode 100755 index 36c128a62320..000000000000 --- a/scripts/migrate_barretenberg_branch.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -set -eu - -# Usage: ./this.sh -# Script for migrating PRs from barretenberg repo to aztec-packages. -# Meant to be used from master with a branch name that exists on barretenberg repo but not aztec-packages. -# You can change the commit message after with git commit --amend if needed. - -# Display usage if not enough arguments -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -# Check for unstaged changes -if ! git diff-index --quiet HEAD --; then - echo "Error: You have unstaged changes. Please commit or stash them before running git_subrepo.sh." - exit 1 -fi - -# Check if the current branch is master -current_branch=$(git rev-parse --abbrev-ref HEAD) -if [ "$current_branch" != "master" ]; then - echo "Error: This script must be run from the 'master' branch. Current branch is '$current_branch'" - exit 1 -fi - -# Fetch the latest updates from origin -git fetch origin - -# Check if master is same as origin/master -if ! git diff --quiet master origin/master; then - echo "Error: Local 'master' branch is not up to date with 'origin/master'. Please pull the latest changes." - exit 1 -fi - -BRANCH="$1" -COMMIT_MESSAGE="$2" -SUBREPO_PATH=barretenberg # can be changed to another subrepo if useful - -SCRIPT_DIR=$(dirname "$(realpath "$0")") -cd "$SCRIPT_DIR"/.. - -echo "(branch migrate) Switching to a new branch named '$BRANCH' (this branch can't already exist)" - -# Check if branch already exists -if git rev-parse --verify "$BRANCH" >/dev/null 2>&1; then - echo "Error: Aztec branch '$BRANCH' already exists. Please delete it with 'git branch -D $BRANCH' if you are sure you don't need it." - exit 1 -fi - -git checkout -b "$BRANCH" - -echo "(branch migrate) Pulling from upstream barretenberg repo. If this doesn't work, your barretenberg branch may need to merge barretenberg master." -if ! scripts/git_subrepo.sh pull "$SUBREPO_PATH" --branch=$BRANCH; then - echo "Error: Failed to pull from upstream barretenberg repo. Check your branch name or network connection." - exit 1 -fi - -echo "(branch migrate) Automatic git data fix" -# Tosses away the .gitrepo changes, as those we only want if pulling from barretenberg master, not PRs (which will go in as aztec commits). -# because git-subrepo uses 'git rm -r', we fix up .gitmodules after as well. This is an edge-case gotcha using -# git submodules along with git-subrepo. - -git checkout HEAD^ -- "$SUBREPO_PATH"/.gitrepo .gitmodules - -if ! git commit --amend -m "$COMMIT_MESSAGE"; then - echo "Error: Failed to commit changes. Check your commit message." - exit 1 -fi -echo "(branch migrate) All done. You can now 'git push origin HEAD' and click to create a PR on aztec. Changes will then automatically go into barretenberg when merged." diff --git a/scripts/run_on_builder b/scripts/run_on_builder deleted file mode 100755 index 19e106c85115..000000000000 --- a/scripts/run_on_builder +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -set -eu - -# Enter the repo root -cd "$(dirname "$0")/.." - -# Define environment variables -ENV_VARS=" - DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD - RUN_ID=$RUN_ID - CI=$CI - RUN_ATTEMPT=$RUN_ATTEMPT - USERNAME=${USERNAME:-} - GITHUB_TOKEN=$GITHUB_TOKEN - GITHUB_RUN_URL=$GITHUB_RUN_URL - GH_SELF_HOSTED_RUNNER_TOKEN=${GH_SELF_HOSTED_RUNNER_TOKEN:-} - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY - BUILD_INSTANCE_SSH_KEY=$BUILD_INSTANCE_SSH_KEY - GIT_COMMIT=$GIT_COMMIT - AZTEC_BOT_GITHUB_TOKEN=${AZTEC_BOT_GITHUB_TOKEN:-} -" - -# Format the environment variables for the SSH command -ENV_EXPORTS=$(printf 'export %s; ' $ENV_VARS) - -ssh -o ControlMaster=auto -o ControlPath=~/.ssh_mux_%h_%p_%r -o ControlPersist=30s -o TCPKeepAlive=no -o ServerAliveCountMax=5 -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -i "$BUILDER_SPOT_KEY" ubuntu@"$BUILDER_SPOT_IP" "$ENV_EXPORTS $@" diff --git a/scripts/run_on_tester b/scripts/run_on_tester deleted file mode 100755 index 52ed7040c931..000000000000 --- a/scripts/run_on_tester +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -set -eu - -# Enter the repo root -cd "$(dirname "$0")/.." - -# Define environment variables -ENV_VARS=" - DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD - RUN_ID=$RUN_ID - CI=$CI - RUN_ATTEMPT=$RUN_ATTEMPT - USERNAME=$USERNAME - GITHUB_TOKEN=$GITHUB_TOKEN - GH_SELF_HOSTED_RUNNER_TOKEN=${GH_SELF_HOSTED_RUNNER_TOKEN:-} - GITHUB_RUN_URL=$GITHUB_RUN_URL - AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY - BUILD_INSTANCE_SSH_KEY=$BUILD_INSTANCE_SSH_KEY - GIT_COMMIT=$GIT_COMMIT - AZTEC_BOT_GITHUB_TOKEN=${AZTEC_BOT_GITHUB_TOKEN:-} -" - -# Format the environment variables for the SSH command -ENV_EXPORTS=$(printf 'export %s; ' $ENV_VARS) - -echo "Running on tester at IP: $SPOT_IP" -ssh -o ControlMaster=auto -o ControlPath=~/.ssh_mux_%h_%p_%r -o ControlPersist=30s -o TCPKeepAlive=no -o ServerAliveCountMax=5 -o ServerAliveInterval=30 -o StrictHostKeyChecking=no -i "$SPOT_KEY" ubuntu@"$SPOT_IP" "$ENV_EXPORTS $@" diff --git a/scripts/setup_env.sh b/scripts/setup_env.sh deleted file mode 100755 index c19678ca1b68..000000000000 --- a/scripts/setup_env.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -COMMIT_HASH=$(git rev-parse HEAD) - -# Setup environment variables -echo "Setting up environment variables..." -echo "FORCE_COLOR=1" >> $GITHUB_ENV -echo "EARTHLY_BUILD_ARGS=PULL_REQUEST=$PULL_REQUEST,BRANCH=$BRANCH,COMMIT_HASH=$COMMIT_HASH" >> $GITHUB_ENV - -# Docker login -echo "Logging in to Docker..." -echo $1 | docker login -u aztecprotocolci --password-stdin - -# Make earthly-ci script available -echo "PATH=$(dirname $(realpath $0)):$PATH" >> $GITHUB_ENV -echo "EARTHLY_CONFIG=$(git rev-parse --show-toplevel)/scripts/earthly-ci-config.yml" >> $GITHUB_ENV -echo ECR_REGION=us-east-2 >> $GITHUB_ENV -echo AWS_ACCOUNT=278380418400 >> $GITHUB_ENV -echo ECR_URL=278380418400.dkr.ecr.us-east-2.amazonaws.com >> $GITHUB_ENV -echo ECR_DEPLOY_REGION=eu-west-2 >> $GITHUB_ENV \ No newline at end of file diff --git a/scripts/tests/bootstrap/download_first_pass.sh b/scripts/tests/bootstrap/download_first_pass.sh deleted file mode 100755 index 279d70148bb7..000000000000 --- a/scripts/tests/bootstrap/download_first_pass.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Use ci3 script base. -echo "This file should not exist outside of bootstrap/test, this may have accidentally been committed if so!" -source "$(git rev-parse --show-toplevel)/ci3/source" -if cache_download.bkup "$@"; then - echo "Should not have found download $@" >> "$ci3/.test_failures" - exit 0 -fi -exit 1 diff --git a/scripts/tests/bootstrap/download_second_pass.sh b/scripts/tests/bootstrap/download_second_pass.sh deleted file mode 100755 index 898b77b89973..000000000000 --- a/scripts/tests/bootstrap/download_second_pass.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Use ci3 script base. -echo "This file should not exist outside of bootstrap/test, this may have accidentally been committed if so!" -source "$(git rev-parse --show-toplevel)/ci3/source" -if ! cache_download.bkup "$@"; then - echo "Should have found download $@" >> "$ci3/.test_failures" - exit 1 -fi -exit 0 diff --git a/scripts/tests/bootstrap/should_run_first_pass.sh b/scripts/tests/bootstrap/should_run_first_pass.sh deleted file mode 100755 index 0233c97fac8c..000000000000 --- a/scripts/tests/bootstrap/should_run_first_pass.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -echo "This file should not exist outside of bootstrap/test, this may have accidentally been committed if so!" -# Use ci3 script base. -source "$(git rev-parse --show-toplevel)/ci3/source" -if ! test_should_run.bkup "$@"; then - echo "Should not want to skip $@" >> "$ci3/.test_failures" - exit 1 -fi -# In the first pass, we want to run each test -exit 0 diff --git a/scripts/tests/bootstrap/should_run_second_pass.sh b/scripts/tests/bootstrap/should_run_second_pass.sh deleted file mode 100755 index fd34cc61a75b..000000000000 --- a/scripts/tests/bootstrap/should_run_second_pass.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Use ci3 script base. -echo "This file should not exist outside of bootstrap/test, this may have accidentally been committed if so!" -source "$(git rev-parse --show-toplevel)/ci3/source" -if test_should_run.bkup "$@"; then - echo "Should not want to run $@" >> "$ci3/.test_failures" -fi -# We never return true, as we don't want to run tests -exit 1 diff --git a/scripts/tests/bootstrap/test-cache b/scripts/tests/bootstrap/test-cache deleted file mode 100755 index d79ae1d85544..000000000000 --- a/scripts/tests/bootstrap/test-cache +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash -# Use the minio script base. We ensure minio is running and wipe it. -root=$(git rev-parse --show-toplevel) -source $root/ci3/source_test -set -o pipefail - -export DENOISE=${DENOISE:-1} -# Tests the various modes of bootstrap test usage. -# We mock ci3/test_should_run and ci3/cache_download for this purpose. -function run_cleanup() { - rm -f $ci3/cache_download.bkup - rm -f $ci3/test_should_run.bkup - git checkout -- $ci3/test_should_run - git checkout -- $ci3/cache_download -} -function exit_cleanup() { - run_cleanup - rm -f $ci3/.test_failures -} -trap exit_cleanup EXIT - -function run_bootstrap() { - set +e - project=$1 - cp $ci3/test_should_run $ci3/test_should_run.bkup - cp $ci3/cache_download $ci3/cache_download.bkup - # Install our mocks - cp $2 $ci3/test_should_run - cp $3 $ci3/cache_download - rm -f $ci3/.test_failures - - cd $root/$project - TEST=1 denoise ./bootstrap.sh - exit_code=$? - if [ "$exit_code" != 0 ]; then - echo "Bootstrap had a bad exit code $exit_code." - exit 1 - fi - run_cleanup -} - -function check_for_bad_asserts() { - if [ -s "$ci3/.test_failures" ]; then - echo "Failures detected:" - cat "$ci3/.test_failures" - exit 1 - fi -} - -function test_CI_0() { - CI=0 run_bootstrap $1 $root/scripts/tests/bootstrap/should_run_first_pass.sh $root/scripts/tests/bootstrap/download_first_pass.sh - check_for_bad_asserts -} - -function test_CI_1_first_pass() { - CI=1 run_bootstrap $1 $root/scripts/tests/bootstrap/should_run_first_pass.sh $root/scripts/tests/bootstrap/download_second_pass.sh - check_for_bad_asserts -} - -function test_CI_1_second_pass() { - CI=1 run_bootstrap $1 $root/scripts/tests/bootstrap/should_run_second_pass.sh $root/scripts/tests/bootstrap/download_second_pass.sh - check_for_bad_asserts -} - -PROJECTS=( - noir - barretenberg - l1-contracts - avm-transpiler - noir-projects - yarn-project - boxes -) - -for project in "${PROJECTS[@]}"; do - # Run the tests - echo "$project 1/3: CI=0 build should run and upload" - test_CI_0 $project | sed "s/^/$project 1\/3: /" || (echo "$project 1/3 failure" && exit 1) - echo "$project 1/3 success" - echo "$project 2/3: CI=1, tests should run, downloads should happen" - test_CI_1_first_pass $project | sed "s/^/$project 2\/3: /" || (echo "$project 2/3 failure" && exit 1) - echo "$project 2/3 success" - echo "$project 3/3: CI=1 tests shouldn't run, downloads should happen" - test_CI_1_second_pass $project | sed "s/^/$project 3\/3: /" || (echo "$project 3/3 failure" && exit 1) - echo "$project 3/3 success" - # We shouldn't need this cache anymore - minio_delete_cache -done - -echo "success: ./bootstrap.sh consistency tests have all passed." diff --git a/scripts/tests/test_minio b/scripts/tests/test_minio deleted file mode 100755 index 3b469c65727e..000000000000 --- a/scripts/tests/test_minio +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -# Use the minio script base. We ensure minio is running and wipe it. -source $(git rev-parse --show-toplevel)/ci3/source_test -export CI=1 -# should not exist -! cache_download cache-artifact.tar.gz -touch cache-artifact -cache_upload cache-artifact.tar.gz cache-artifact -# should not exist -cache_download cache-artifact.tar.gz diff --git a/scripts/tests/tmux_all_tests b/scripts/tests/tmux_all_tests deleted file mode 100755 index 290f10e9e39a..000000000000 --- a/scripts/tests/tmux_all_tests +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# Runs tests on each module in parallel, in a tmux split -# Use ci3 script base. -source $(git rev-parse --show-toplevel)/ci3/source -export TEST=1 -tmux_split aztec-test-all \ - "./barretenberg/bootstrap.sh test" \ - "./l1-contracts/bootstrap.sh test" \ - "./noir/bootstrap.sh test" \ - "./noir-projects/bootstrap.sh test" \ - "./yarn-packages/bootstrap.sh test" \ - "./boxes/bootstrap.sh test" diff --git a/scripts/tmux-splits b/scripts/tmux-splits deleted file mode 100755 index 6b1e28f85595..000000000000 --- a/scripts/tmux-splits +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Launches tmux with 1 window that has 2 panes running, from top to bottom: -# - aztec.js -# - a shell in end-to-end. If a test is given as an argument, the test will be run here automatically. -TEST=$1 -SESSION_NAME=work - -cd $(git rev-parse --show-toplevel) - -if tmux has-session -t $SESSION_NAME > /dev/null 2>&1; then - tmux attach -t $SESSION_NAME -else - tmux new-session -s $SESSION_NAME -d \ - split-window \; \ - select-layout even-vertical \; \ - # Give a chance for terminals to start (avoids double command echo). - sleep 3 - tmux select-pane -t 0 \; \ - send-keys 'cd yarn-project/aztec.js' C-m \; \ - select-pane -t 1 \; \ - send-keys "cd yarn-project/end-to-end && ([ -z \"$TEST\" ] || (yarn test $TEST))" C-m \; \ - attach \; -fi diff --git a/yarn-project/.gitignore b/yarn-project/.gitignore index 979675b60de4..d4379b0d490a 100644 --- a/yarn-project/.gitignore +++ b/yarn-project/.gitignore @@ -1,4 +1,3 @@ -# To be maintained along with .earthlyinore **/dest **/node_modules **/.cache