From e0a9b9005bd99f2c65e7956c317c4556d086b007 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 11 Apr 2024 18:21:14 +0000 Subject: [PATCH 01/12] naming --- .github/ci-setup-action/action.yml | 2 +- .github/workflows/ci-arm64.yml | 88 +++++++++++++++++++ .github/workflows/ci.yml | 23 ++++- ...ir_subrepo.yml => mirror-noir-subrepo.yml} | 2 +- .../{mirror_repos.yml => mirror-repos.yml} | 0 ...ase_images.yml => publish-base-images.yml} | 0 .../{pull_noir.yml => pull-noir.yml} | 11 ++- ...{release_please.yml => release-please.yml} | 0 .github/workflows/setup-runner.yml | 86 ++++++++++++++++++ 9 files changed, 205 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/ci-arm64.yml rename .github/workflows/{mirror_noir_subrepo.yml => mirror-noir-subrepo.yml} (99%) rename .github/workflows/{mirror_repos.yml => mirror-repos.yml} (100%) rename .github/workflows/{publish_base_images.yml => publish-base-images.yml} (100%) rename .github/workflows/{pull_noir.yml => pull-noir.yml} (89%) rename .github/workflows/{release_please.yml => release-please.yml} (100%) create mode 100644 .github/workflows/setup-runner.yml diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml index 879a5fadd060..add391ac4e03 100644 --- a/.github/ci-setup-action/action.yml +++ b/.github/ci-setup-action/action.yml @@ -2,7 +2,7 @@ name: Setup Workflow description: 'Reusable setup steps' -inputs: +secrets: dockerhub_password: required: true description: 'DockerHub Password' diff --git a/.github/workflows/ci-arm64.yml b/.github/workflows/ci-arm64.yml new file mode 100644 index 000000000000..8ec78948bd68 --- /dev/null +++ b/.github/workflows/ci-arm64.yml @@ -0,0 +1,88 @@ +name: CI +on: + push: + branches: [master] + workflow_dispatch: + inputs: + runner_action: + description: "The action to take with the self-hosted runner (start, stop, restart)." + required: false + just_start_spot: + description: "Should we just run spots?" + type: boolean + required: false +concurrency: + # force parallelism in master + group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} + cancel-in-progress: true +jobs: + # Start cheap (~1/8th the cost of on demand, ~13th the cost of large GA runners) spot builders + # just for the CI job. These are specced per user and run the entire CI. + # TODO These have a persistent EBS volume that forms a fast-online docker image cache (used by Earthly), meaning + # TODO build steps that ran in previous invocations are quickly ran from cache. + start-builder: + timeout-minutes: 5 + name: Start Build Runner (ARM) + runs-on: ubuntu-latest + permissions: + actions: write + strategy: + matrix: + config: + - {ec2_instance_type: m6a.32xlarge, runner_concurrency: 50, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} + - {ec2_instance_type: r6g.16xlarge, runner_concurrency: 8, ec2_ami_id: ami-0d8a9b0419ddb331a, runner_label_suffix: arm} + steps: + - name: Start EC2 runner + id: start-ec2-runner + uses: AztecProtocol/ec2-action-builder@v0.3 + with: + github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws_region: "us-east-2" + ec2_subnet_id: subnet-4cfabd25 + subaction: ${{ github.event.inputs.runner_action || (contains(github.event.head_commit.message, '[ci restart-spot]') && 'restart' || 'start') }} + # prevent reaping by mainframe spot reaper + ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' + github_action_runner_version: v2.315.0 + ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + ec2_spot_instance_strategy: BestEffort + runner_label: ${{ github.actor }}-${{ matrix.config.runner_label_suffix }} + runner_concurrency: ${{ matrix.config.runner_concurrency }} + ec2_instance_type: ${{ matrix.config.ec2_instance_type }} + ec2_ami_id: ${{ matrix.config.ec2_ami_id }} + ec2_instance_ttl: 30 # 30 minutes to reap, refreshed by job starts + + build-arm: + needs: start-builder + timeout-minutes: 25 + if: ${{ github.event.inputs.just_start_spot != 'true' }} + runs-on: ${{ github.actor }}-arm + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + # Attach our 128gb cache disk + - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-arm 128 + # prepare images locally, tagged by commit hash + - run: earthly ./yarn-project+export-end-to-end + + # all the end-to-end integration tests for aztec + e2e-arm: + needs: build-arm + runs-on: ${{ github.actor }}-arm + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + test: + - e2e-card-game + - e2e-crowdfunding-and-claim + steps: + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + - name: Test + working-directory: ./yarn-project/end-to-end/ + run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92100b135b17..3b142d549ceb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ on: type: boolean required: false concurrency: - # force parallelism + # force parallelism in master group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} cancel-in-progress: true jobs: @@ -31,7 +31,7 @@ jobs: matrix: config: - {ec2_instance_type: m6a.32xlarge, runner_concurrency: 50, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} - - {ec2_instance_type: r6g.16xlarge, runner_concurrency: 8, ec2_ami_id: ami-0d8a9b0419ddb331a, runner_label_suffix: arm} + # - {ec2_instance_type: r6g.16xlarge, runner_concurrency: 8, ec2_ami_id: ami-0d8a9b0419ddb331a, runner_label_suffix: arm} steps: - name: Start EC2 runner id: start-ec2-runner @@ -57,11 +57,24 @@ jobs: ######################### # START OF ARM PIPELINE # ######################### + start-arm-builder: + uses: ./.github/workflows/publish-bb.yml + with: + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + runner_label: ${{ github.actor }}-arm + runner_concurrency: 8 + ec2_instance_type: r6g.16xlarge + ec2_ami_id: ami-0d8a9b0419ddb331a + ec2_instance_ttl: 30 + ref_sha: ${{ github.event.pull_request.head.sha }} + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} # prevents concurrency issues with multiple (implicit) earthly bootstraps setup-arm: needs: start-builder timeout-minutes: 5 - if: ${{ github.event.inputs.just_start_spot != 'true' }} + if: ${{ github.ref_name != 'master' && github.event.inputs.just_start_spot != 'true' }} runs-on: ${{ github.actor }}-arm env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -159,6 +172,10 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + - name: Set up mutex + uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + with: + branch: gh-action-mutex-e2e-x86-${{ github.actor }}-${{ matrix.test }} - name: Test working-directory: ./yarn-project/end-to-end/ run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache diff --git a/.github/workflows/mirror_noir_subrepo.yml b/.github/workflows/mirror-noir-subrepo.yml similarity index 99% rename from .github/workflows/mirror_noir_subrepo.yml rename to .github/workflows/mirror-noir-subrepo.yml index fe768f76eba4..fb6968cc9402 100644 --- a/.github/workflows/mirror_noir_subrepo.yml +++ b/.github/workflows/mirror-noir-subrepo.yml @@ -78,7 +78,7 @@ jobs: fi BASE_NOIR_COMMIT="$LAST_PR_MERGE" - COMMIT=$(git rev-parse HEAD) + COMMIT=$(git rev-parse HEAD) COMMIT_MESSAGE=$(git log -1 --pretty=format:%B) # Fix Aztec PR links and output message COMMIT_MESSAGE=$(echo "$COMMIT_MESSAGE" | sed -E 's/\(#([0-9]+)\)/(https:\/\/github.com\/AztecProtocol\/aztec-packages\/pull\/\1)/g') diff --git a/.github/workflows/mirror_repos.yml b/.github/workflows/mirror-repos.yml similarity index 100% rename from .github/workflows/mirror_repos.yml rename to .github/workflows/mirror-repos.yml diff --git a/.github/workflows/publish_base_images.yml b/.github/workflows/publish-base-images.yml similarity index 100% rename from .github/workflows/publish_base_images.yml rename to .github/workflows/publish-base-images.yml diff --git a/.github/workflows/pull_noir.yml b/.github/workflows/pull-noir.yml similarity index 89% rename from .github/workflows/pull_noir.yml rename to .github/workflows/pull-noir.yml index c2caab755ebf..66f99ffd3b99 100644 --- a/.github/workflows/pull_noir.yml +++ b/.github/workflows/pull-noir.yml @@ -29,7 +29,11 @@ jobs: PR_URL=$(gh pr list --repo AztecProtocol/aztec-packages --head sync-noir --json url --jq ".[0].url") echo "PR_URL=$PR_URL" >> $GITHUB_ENV # What was our last merge on noir side? - BASE_NOIR_COMMIT=`gh pr list --repo=noir-lang/noir --state merged --head aztec-packages --json mergeCommit --jq=.[0].mergeCommit.oid` + # Detect our last sync commit (written by this action before pushing) with a fallback for the first time we ever do this + BASE_NOIR_COMMIT=$(curl https://raw.githubusercontent.com/AztecProtocol/aztec-packages/master/.noir-sync-commit) + if [ "$AZTEC_SYNC_COMMIT" = "404: Not Found" ] ; then + BASE_NOIR_COMMIT="50d2735825454a8638a308156d4ea23b3c4420d8" + fi echo "BASE_NOIR_COMMIT=$BASE_NOIR_COMMIT" >> $GITHUB_ENV # What was our last sync on aztec side? BASE_AZTEC_COMMIT=`curl https://raw.githubusercontent.com/noir-lang/noir/master/.aztec-sync-commit` @@ -88,6 +92,9 @@ jobs: # we need to commit for git-subrepo git commit -am "[$LINES changes] $COMMIT_MESSAGE" if ./scripts/git-subrepo/lib/git-subrepo pull --force $SUBREPO_PATH --branch=master; then + # Read our actual commit sync from git subrepo, stash to file for next time + COMMIT=$(git config --file="$SUBREPO_PATH/.gitrepo" subrepo.commit) + echo "$COMMIT" > .noir-sync-commit && git add .noir-sync-commit git reset --soft "$BASE_AZTEC_COMMIT" # We don't really need the sync commit on our side, and don't need .gitrepo at all except just in time for the command. git checkout origin/master -- noir/noir-repo/.aztec-sync-commit noir/noir-repo/.gitrepo @@ -114,7 +121,7 @@ jobs: - name: Update PR run: | set -xue # print commands - # Formatted for updating the PR, overrides for release-please commit message parsing + # Formatted for updating the PR, overrides for release-please commit message parsing PR_BODY=""" Automated pull of development from the [noir](https://github.com/noir-lang/noir) programming language, a dependency of Aztec. BEGIN_COMMIT_OVERRIDE diff --git a/.github/workflows/release_please.yml b/.github/workflows/release-please.yml similarity index 100% rename from .github/workflows/release_please.yml rename to .github/workflows/release-please.yml diff --git a/.github/workflows/setup-runner.yml b/.github/workflows/setup-runner.yml new file mode 100644 index 000000000000..9e9d4275cae8 --- /dev/null +++ b/.github/workflows/setup-runner.yml @@ -0,0 +1,86 @@ +name: Reusable Spot Instance and Setup Workflow +on: + workflow_call: + inputs: + runner_label: + required: true + type: string + runner_concurrency: + required: true + type: number + ec2_instance_type: + required: true + type: string + ec2_ami_id: + required: true + type: string + ec2_instance_ttl: + required: true + type: number + ec2_subnet_id: + default: subnet-4cfabd25 + type: string + ec2_security_group_id: + default: sg-0ccd4e5df0dcca0c9 + type: string + ec2_spot_instance_strategy: + default: BestEffort + type: string + aws_region: + default: "us-east-2" + type: string + ref_sha: + required: true + type: string + ebs_cache_size_gb: + required: true + type: string + secrets: + aws_access_key_id: + required: true + aws_secret_access_key: + required: true + github_token: + required: true + dockerhub_password: + required: true +jobs: + start-builder: + runs-on: ubuntu-latest + steps: + - name: Start EC2 runner + uses: AztecProtocol/ec2-action-builder@v0.3 + with: + github_token: ${{ inputs.github_token }} + aws_access_key_id: ${{ inputs.aws_access_key_id }} + aws_secret_access_key: ${{ inputs.aws_secret_access_key }} + aws_region: ${{ inputs.aws_region }} + ec2_subnet_id: ${{ inputs.ec2_subnet_id }} + ec2_security_group_id: ${{ inputs.ec2_security_group_id }} + ec2_spot_instance_strategy: ${{ inputs.ec2_spot_instance_strategy }} + runner_label: ${{ inputs.runner_label }} + runner_concurrency: ${{ inputs.runner_concurrency }} + ec2_instance_type: ${{ inputs.ec2_instance_type }} + ec2_ami_id: ${{ inputs.ec2_ami_id }} + ec2_instance_ttl: ${{ inputs.ec2_instance_ttl }} + ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' + + setup: + needs: start-builder + runs-on: ${{ inputs.runner_label }} + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref_sha }} + + - name: Setup CI + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ inputs.dockerhub_password }} + + - name: Attach EBS Cache Disk + run: ./scripts/attach_ebs_cache.sh ${{ inputs.runner_label }} 128 + + - name: Run Earthly Bootstrap + run: earthly bootstrap From c7dae4e3b748b5848f80d43929da8591408bbaa4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 11 Apr 2024 18:28:29 +0000 Subject: [PATCH 02/12] try arm subworkflow --- .github/workflows/ci.yml | 439 ++++++++++++++--------------- .github/workflows/setup-runner.yml | 18 +- 2 files changed, 219 insertions(+), 238 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b142d549ceb..b9713a702b7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,42 +17,42 @@ concurrency: group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} cancel-in-progress: true jobs: - # Start cheap (~1/8th the cost of on demand, ~13th the cost of large GA runners) spot builders - # just for the CI job. These are specced per user and run the entire CI. - # TODO These have a persistent EBS volume that forms a fast-online docker image cache (used by Earthly), meaning - # TODO build steps that ran in previous invocations are quickly ran from cache. - start-builder: - timeout-minutes: 5 - name: Start Build Runner (${{ matrix.config.runner_label_suffix }}) - runs-on: ubuntu-latest - permissions: - actions: write - strategy: - matrix: - config: - - {ec2_instance_type: m6a.32xlarge, runner_concurrency: 50, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} - # - {ec2_instance_type: r6g.16xlarge, runner_concurrency: 8, ec2_ami_id: ami-0d8a9b0419ddb331a, runner_label_suffix: arm} - steps: - - name: Start EC2 runner - id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v0.3 - with: - github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-east-2" - ec2_subnet_id: subnet-4cfabd25 - subaction: ${{ github.event.inputs.runner_action || (contains(github.event.head_commit.message, '[ci restart-spot]') && 'restart' || 'start') }} - # prevent reaping by mainframe spot reaper - ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' - github_action_runner_version: v2.315.0 - ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - ec2_spot_instance_strategy: BestEffort - runner_label: ${{ github.actor }}-${{ matrix.config.runner_label_suffix }} - runner_concurrency: ${{ matrix.config.runner_concurrency }} - ec2_instance_type: ${{ matrix.config.ec2_instance_type }} - ec2_ami_id: ${{ matrix.config.ec2_ami_id }} - ec2_instance_ttl: 30 # 30 minutes to reap, refreshed by job starts + # # Start cheap (~1/8th the cost of on demand, ~13th the cost of large GA runners) spot builders + # # just for the CI job. These are specced per user and run the entire CI. + # # TODO These have a persistent EBS volume that forms a fast-online docker image cache (used by Earthly), meaning + # # TODO build steps that ran in previous invocations are quickly ran from cache. + # start-builder: + # timeout-minutes: 5 + # name: Start Build Runner (${{ matrix.config.runner_label_suffix }}) + # runs-on: ubuntu-latest + # permissions: + # actions: write + # strategy: + # matrix: + # config: + # - {ec2_instance_type: m6a.32xlarge, runner_concurrency: 50, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} + # # - {ec2_instance_type: r6g.16xlarge, runner_concurrency: 8, ec2_ami_id: ami-0d8a9b0419ddb331a, runner_label_suffix: arm} + # steps: + # - name: Start EC2 runner + # id: start-ec2-runner + # uses: AztecProtocol/ec2-action-builder@v0.3 + # with: + # github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + # aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws_region: "us-east-2" + # ec2_subnet_id: subnet-4cfabd25 + # subaction: ${{ github.event.inputs.runner_action || (contains(github.event.head_commit.message, '[ci restart-spot]') && 'restart' || 'start') }} + # # prevent reaping by mainframe spot reaper + # ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' + # github_action_runner_version: v2.315.0 + # ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + # ec2_spot_instance_strategy: BestEffort + # runner_label: ${{ github.actor }}-${{ matrix.config.runner_label_suffix }} + # runner_concurrency: ${{ matrix.config.runner_concurrency }} + # ec2_instance_type: ${{ matrix.config.ec2_instance_type }} + # ec2_ami_id: ${{ matrix.config.ec2_ami_id }} + # ec2_instance_ttl: 40 # 30 minutes to reap, refreshed by job starts ######################### # START OF ARM PIPELINE # @@ -60,34 +60,15 @@ jobs: start-arm-builder: uses: ./.github/workflows/publish-bb.yml with: - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} runner_label: ${{ github.actor }}-arm runner_concurrency: 8 ec2_instance_type: r6g.16xlarge ec2_ami_id: ami-0d8a9b0419ddb331a - ec2_instance_ttl: 30 - ref_sha: ${{ github.event.pull_request.head.sha }} - dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - # prevents concurrency issues with multiple (implicit) earthly bootstraps - setup-arm: - needs: start-builder - timeout-minutes: 5 - if: ${{ github.ref_name != 'master' && github.event.inputs.just_start_spot != 'true' }} - runs-on: ${{ github.actor }}-arm - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - steps: - - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # Attach our 128gb cache disk - - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-arm 128 - - run: earthly bootstrap + ec2_instance_ttl: 40 + secrets: inherit build-arm: - needs: setup-arm + needs: start-arm-builder runs-on: ${{ github.actor }}-arm timeout-minutes: 25 env: @@ -116,184 +97,184 @@ jobs: working-directory: ./yarn-project/end-to-end/ run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - ######################### - # START OF x86 PIPELINE # - ######################### - # prevents concurrency issues with multiple (implicit) earthly bootstraps - setup-x86: - needs: start-builder - if: ${{ github.event.inputs.just_start_spot != 'true' }} - runs-on: ${{ github.actor }}-x86 - timeout-minutes: 5 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - steps: - - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # Attach our 128gb cache disk - - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-x86 128 - - run: earthly bootstrap + # ######################### + # # START OF x86 PIPELINE # + # ######################### + # # prevents concurrency issues with multiple (implicit) earthly bootstraps + # setup-x86: + # needs: start-builder + # if: ${{ github.event.inputs.just_start_spot != 'true' }} + # runs-on: ${{ github.actor }}-x86 + # timeout-minutes: 5 + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # steps: + # - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + # - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + # # Attach our 128gb cache disk + # - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-x86 128 + # - run: earthly bootstrap - build-x86: - needs: setup-x86 - runs-on: ${{ github.actor }}-x86 - timeout-minutes: 25 - outputs: - e2e_list: ${{ steps.e2e_list.outputs.list }} - steps: - - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # Only allow one memory-hunger prover test to use this runner - # As detailed in https://github.com/ben-z/gh-action-mutex - # things do not become 'pending' in github actions, and instead just cancel one another - # so we can't use the native concurrency in GA - - name: Set up mutex - uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - with: - branch: gh-action-mutex-build-x86-${{ github.actor }} - # prepare images locally, tagged by commit hash - - run: earthly ./yarn-project+export-end-to-end - # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end - # (Note ARM uses just 2 tests as a smoketest) - - name: Create list of end-to-end jobs - id: e2e_list - run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT + # build-x86: + # needs: setup-x86 + # runs-on: ${{ github.actor }}-x86 + # timeout-minutes: 25 + # outputs: + # e2e_list: ${{ steps.e2e_list.outputs.list }} + # steps: + # - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + # - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + # # Only allow one memory-hunger prover test to use this runner + # # As detailed in https://github.com/ben-z/gh-action-mutex + # # things do not become 'pending' in github actions, and instead just cancel one another + # # so we can't use the native concurrency in GA + # - name: Set up mutex + # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + # with: + # branch: gh-action-mutex-build-x86-${{ github.actor }} + # # prepare images locally, tagged by commit hash + # - run: earthly ./yarn-project+export-end-to-end + # # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end + # # (Note ARM uses just 2 tests as a smoketest) + # - name: Create list of end-to-end jobs + # id: e2e_list + # run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT - # all the end-to-end integration tests for aztec - e2e-x86: - needs: build-x86 - runs-on: ${{ github.actor }}-x86 - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} - steps: - - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - - name: Set up mutex - uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - with: - branch: gh-action-mutex-e2e-x86-${{ github.actor }}-${{ matrix.test }} - - name: Test - working-directory: ./yarn-project/end-to-end/ - run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - # TODO - # - name: Upload logs - # run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + # # all the end-to-end integration tests for aztec + # e2e-x86: + # needs: build-x86 + # runs-on: ${{ github.actor }}-x86 + # timeout-minutes: 15 + # strategy: + # fail-fast: false + # matrix: + # test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} + # steps: + # - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + # - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + # - name: Set up mutex + # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + # with: + # branch: gh-action-mutex-e2e-x86-${{ github.actor }}-${{ matrix.test }} + # - name: Test + # working-directory: ./yarn-project/end-to-end/ + # run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + # # TODO + # # - name: Upload logs + # # run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log - # barretenberg (prover) native tests - # only ran on x86 for resource reasons (memory intensive) - bb-native-tests: - needs: setup-x86 - runs-on: ${{ github.actor }}-x86 - timeout-minutes: 15 - strategy: - fail-fast: false - steps: - - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # Only allow one memory-hunger prover test to use this runner - # As detailed in https://github.com/ben-z/gh-action-mutex - # things do not become 'pending' in github actions, and instead just cancel one another - # so we can't use the native concurrency in GA - - name: Set up mutex - uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - with: - branch: gh-action-mutex-bench-${{ github.actor }} - - working-directory: ./barretenberg/cpp/ - run: earthly --no-output +test + # # barretenberg (prover) native tests + # # only ran on x86 for resource reasons (memory intensive) + # bb-native-tests: + # needs: setup-x86 + # runs-on: ${{ github.actor }}-x86 + # timeout-minutes: 15 + # strategy: + # fail-fast: false + # steps: + # - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + # - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + # # Only allow one memory-hunger prover test to use this runner + # # As detailed in https://github.com/ben-z/gh-action-mutex + # # things do not become 'pending' in github actions, and instead just cancel one another + # # so we can't use the native concurrency in GA + # - name: Set up mutex + # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + # with: + # branch: gh-action-mutex-bench-${{ github.actor }} + # - working-directory: ./barretenberg/cpp/ + # run: earthly --no-output +test - # push benchmarking binaries to dockerhub registry - bb-bench-binaries: - runs-on: ${{ github.actor }}-x86 - timeout-minutes: 15 - needs: setup-x86 - steps: - - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - - name: Build and Push Binaries - if: ${{ github.event.inputs.just_start_spot != 'true' }} - working-directory: ./barretenberg/cpp/ - run: earthly --push +bench-binaries + # # push benchmarking binaries to dockerhub registry + # bb-bench-binaries: + # runs-on: ${{ github.actor }}-x86 + # timeout-minutes: 15 + # needs: setup-x86 + # steps: + # - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + # - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + # - name: Build and Push Binaries + # if: ${{ github.event.inputs.just_start_spot != 'true' }} + # working-directory: ./barretenberg/cpp/ + # run: earthly --push +bench-binaries - start-bb-bench-runner: - timeout-minutes: 5 - # We wait for binaries to be done for kickoff - needs: bb-bench-binaries - name: Start Bench Runner - runs-on: ubuntu-latest - permissions: - actions: write - steps: - - name: Start EC2 runner - id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v0.3 - with: - github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-east-2" - ec2_subnet_id: subnet-4cfabd25 - subaction: ${{ github.event.inputs.runner_action || 'start' }} - # prevent reaping by mainframe spot reaper - ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' - github_action_runner_version: v2.315.0 - ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - ec2_spot_instance_strategy: BestEffort - runner_label: ${{ github.actor }}-bench-x86 - runner_concurrency: 1 - ec2_instance_type: m7a.4xlarge - ec2_ami_id: ami-04d8422a9ba4de80f - ec2_instance_ttl: 10 # 10 minutes to reap, refreshed by job starts - # try if spot variance too high, uses on-demand: - # ec2_spot_instance_strategy: none + # start-bb-bench-runner: + # timeout-minutes: 5 + # # We wait for binaries to be done for kickoff + # needs: bb-bench-binaries + # name: Start Bench Runner + # runs-on: ubuntu-latest + # permissions: + # actions: write + # steps: + # - name: Start EC2 runner + # id: start-ec2-runner + # uses: AztecProtocol/ec2-action-builder@v0.3 + # with: + # github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + # aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + # aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # aws_region: "us-east-2" + # ec2_subnet_id: subnet-4cfabd25 + # subaction: ${{ github.event.inputs.runner_action || 'start' }} + # # prevent reaping by mainframe spot reaper + # ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' + # github_action_runner_version: v2.315.0 + # ec2_security_group_id: sg-0ccd4e5df0dcca0c9 + # ec2_spot_instance_strategy: BestEffort + # runner_label: ${{ github.actor }}-bench-x86 + # runner_concurrency: 1 + # ec2_instance_type: m7a.4xlarge + # ec2_ami_id: ami-04d8422a9ba4de80f + # ec2_instance_ttl: 10 # 10 minutes to reap, refreshed by job starts + # # try if spot variance too high, uses on-demand: + # # ec2_spot_instance_strategy: none - setup-bb-bench: - runs-on: ${{ github.actor }}-bench-x86 - needs: start-bb-bench-runner - timeout-minutes: 5 - if: ${{ github.event.inputs.just_start_spot != 'true' }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - steps: - - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # Attach our 328gb cache disk - - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-bench-x86 32 - - run: earthly bootstrap + # setup-bb-bench: + # runs-on: ${{ github.actor }}-bench-x86 + # needs: start-bb-bench-runner + # timeout-minutes: 5 + # if: ${{ github.event.inputs.just_start_spot != 'true' }} + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # steps: + # - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + # - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + # # Attach our 328gb cache disk + # - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-bench-x86 32 + # - run: earthly bootstrap - bb-bench: - runs-on: ${{ github.actor }}-bench-x86 - needs: setup-bb-bench - timeout-minutes: 15 - steps: - - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # Use bench_mode=cache to read the pushed build above - - name: Client IVC Bench - working-directory: ./barretenberg/cpp/ - run: earthly --no-output +bench-client-ivc --bench_mode=cache + # bb-bench: + # runs-on: ${{ github.actor }}-bench-x86 + # needs: setup-bb-bench + # timeout-minutes: 15 + # steps: + # - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + # - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + # # Use bench_mode=cache to read the pushed build above + # - name: Client IVC Bench + # working-directory: ./barretenberg/cpp/ + # run: earthly --no-output +bench-client-ivc --bench_mode=cache - - name: Ultrahonk Bench - working-directory: ./barretenberg/cpp/ - run: earthly --no-output +bench-ultra-honk --bench_mode=cache + # - name: Ultrahonk Bench + # working-directory: ./barretenberg/cpp/ + # run: earthly --no-output +bench-ultra-honk --bench_mode=cache - # # Post actions, deploy and summarize logs - # aztec-bench-summary: - # runs-on: ${{ github.actor }} - # # IMPORTANT security flaw if we don't need 'check-run-condition' - # needs: e2e-x86 - # concurrency: - # group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 - # cancel-in-progress: true - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # ref: ${{ github.event.pull_request.head.sha }} + # # # Post actions, deploy and summarize logs + # # aztec-bench-summary: + # # runs-on: ${{ github.actor }} + # # # IMPORTANT security flaw if we don't need 'check-run-condition' + # # needs: e2e-x86 + # # concurrency: + # # group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + # # cancel-in-progress: true + # # steps: + # # - name: Checkout + # # uses: actions/checkout@v4 + # # with: + # # ref: ${{ github.event.pull_request.head.sha }} - # - name: "Assemble benchmark summary from uploaded logs" - # command: ./scripts/ci/assemble_e2e_benchmark_earthly.sh + # # - name: "Assemble benchmark summary from uploaded logs" + # # command: ./scripts/ci/assemble_e2e_benchmark_earthly.sh diff --git a/.github/workflows/setup-runner.yml b/.github/workflows/setup-runner.yml index 9e9d4275cae8..1770ebe71eaf 100644 --- a/.github/workflows/setup-runner.yml +++ b/.github/workflows/setup-runner.yml @@ -36,13 +36,13 @@ on: required: true type: string secrets: - aws_access_key_id: + AWS_ACCESS_KEY_ID: required: true - aws_secret_access_key: + AWS_SECRET_ACCESS_KEY: required: true - github_token: + GH_SELF_HOSTED_RUNNER_TOKEN: required: true - dockerhub_password: + DOCKERHUB_PASSWORD: required: true jobs: start-builder: @@ -51,9 +51,9 @@ jobs: - name: Start EC2 runner uses: AztecProtocol/ec2-action-builder@v0.3 with: - github_token: ${{ inputs.github_token }} - aws_access_key_id: ${{ inputs.aws_access_key_id }} - aws_secret_access_key: ${{ inputs.aws_secret_access_key }} + github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} + aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws_region: ${{ inputs.aws_region }} ec2_subnet_id: ${{ inputs.ec2_subnet_id }} ec2_security_group_id: ${{ inputs.ec2_security_group_id }} @@ -67,7 +67,7 @@ jobs: setup: needs: start-builder - runs-on: ${{ inputs.runner_label }} + runs-on: ${{ github.event.pull_request.head.sha }} steps: - name: Checkout Repository uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: - name: Setup CI uses: ./.github/ci-setup-action with: - dockerhub_password: ${{ inputs.dockerhub_password }} + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Attach EBS Cache Disk run: ./scripts/attach_ebs_cache.sh ${{ inputs.runner_label }} 128 From 3e267acd1a4285f57a241f65e891002195f89e11 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 11 Apr 2024 18:31:25 +0000 Subject: [PATCH 03/12] fix --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9713a702b7d..51a97d829918 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: # START OF ARM PIPELINE # ######################### start-arm-builder: - uses: ./.github/workflows/publish-bb.yml + uses: ./.github/workflows/setup-runner.yml with: runner_label: ${{ github.actor }}-arm runner_concurrency: 8 From 3aa547de652ef4dc87e4aa9a4013c340b09f8f8c Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 11 Apr 2024 18:31:56 +0000 Subject: [PATCH 04/12] ref sha --- .github/workflows/setup-runner.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/setup-runner.yml b/.github/workflows/setup-runner.yml index 1770ebe71eaf..deff2055f0ed 100644 --- a/.github/workflows/setup-runner.yml +++ b/.github/workflows/setup-runner.yml @@ -29,9 +29,6 @@ on: aws_region: default: "us-east-2" type: string - ref_sha: - required: true - type: string ebs_cache_size_gb: required: true type: string From 086c54b5e8af5254e1b96b38fc8979073a30048e Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 11 Apr 2024 18:32:48 +0000 Subject: [PATCH 05/12] add missing param --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51a97d829918..a38c51856d05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,7 @@ jobs: uses: ./.github/workflows/setup-runner.yml with: runner_label: ${{ github.actor }}-arm + ebs_cache_size_gb: 128 runner_concurrency: 8 ec2_instance_type: r6g.16xlarge ec2_ami_id: ami-0d8a9b0419ddb331a From 52933acc87b27a11e3f635918c2fa189832502a1 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 11 Apr 2024 19:01:09 +0000 Subject: [PATCH 06/12] better CI concurrency --- .github/ci-setup-action/action.yml | 14 +- .github/workflows/ci-arm64.yml | 76 +++---- .github/workflows/ci.yml | 328 +++++++++-------------------- .github/workflows/setup-runner.yml | 9 +- 4 files changed, 148 insertions(+), 279 deletions(-) diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml index add391ac4e03..f5f09d90051d 100644 --- a/.github/ci-setup-action/action.yml +++ b/.github/ci-setup-action/action.yml @@ -2,10 +2,13 @@ name: Setup Workflow description: 'Reusable setup steps' -secrets: +inputs: dockerhub_password: required: true description: 'DockerHub Password' + concurrency_key: + required: true + description: 'Concurrency key for locking jobs' runs: # define an action, runs in OS of caller using: composite @@ -56,3 +59,12 @@ runs: else echo "Docker daemon already configured." fi + # As detailed in https://github.com/ben-z/gh-action-mutex + # things do not become 'pending' in github actions, and instead just cancel one another + # so we can't use the native concurrency in GA + - name: Limit concurrency + uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + if: ${{ inputs.concurrency_key != "" }} + with: + repository: AztecProtocol/git-metadata + branch: gh-actions-mutex-${{ inputs.concurrency_key }} diff --git a/.github/workflows/ci-arm64.yml b/.github/workflows/ci-arm64.yml index 8ec78948bd68..807a5e2262ce 100644 --- a/.github/workflows/ci-arm64.yml +++ b/.github/workflows/ci-arm64.yml @@ -12,67 +12,40 @@ on: type: boolean required: false concurrency: - # force parallelism in master + # force parallelism in master, cancelling in branches (only relevant to workflow_dispatch) group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} cancel-in-progress: true jobs: - # Start cheap (~1/8th the cost of on demand, ~13th the cost of large GA runners) spot builders - # just for the CI job. These are specced per user and run the entire CI. - # TODO These have a persistent EBS volume that forms a fast-online docker image cache (used by Earthly), meaning - # TODO build steps that ran in previous invocations are quickly ran from cache. - start-builder: - timeout-minutes: 5 - name: Start Build Runner (ARM) - runs-on: ubuntu-latest - permissions: - actions: write - strategy: - matrix: - config: - - {ec2_instance_type: m6a.32xlarge, runner_concurrency: 50, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} - - {ec2_instance_type: r6g.16xlarge, runner_concurrency: 8, ec2_ami_id: ami-0d8a9b0419ddb331a, runner_label_suffix: arm} - steps: - - name: Start EC2 runner - id: start-ec2-runner - uses: AztecProtocol/ec2-action-builder@v0.3 - with: - github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: "us-east-2" - ec2_subnet_id: subnet-4cfabd25 - subaction: ${{ github.event.inputs.runner_action || (contains(github.event.head_commit.message, '[ci restart-spot]') && 'restart' || 'start') }} - # prevent reaping by mainframe spot reaper - ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' - github_action_runner_version: v2.315.0 - ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - ec2_spot_instance_strategy: BestEffort - runner_label: ${{ github.actor }}-${{ matrix.config.runner_label_suffix }} - runner_concurrency: ${{ matrix.config.runner_concurrency }} - ec2_instance_type: ${{ matrix.config.ec2_instance_type }} - ec2_ami_id: ${{ matrix.config.ec2_ami_id }} - ec2_instance_ttl: 30 # 30 minutes to reap, refreshed by job starts + start-arm-builder: + uses: ./.github/workflows/setup-runner.yml + with: + runner_label: master-arm + ebs_cache_size_gb: 128 + runner_concurrency: 8 + ec2_instance_type: r6g.16xlarge + ec2_ami_id: ami-0d8a9b0419ddb331a + ec2_instance_ttl: 40 # refreshed by jobs + secrets: inherit build-arm: - needs: start-builder - timeout-minutes: 25 - if: ${{ github.event.inputs.just_start_spot != 'true' }} - runs-on: ${{ github.actor }}-arm - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + needs: start-arm-builder + runs-on: master-arm + timeout-minutes: 40 steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # Attach our 128gb cache disk - - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-arm 128 + # Ensure that CI only ever is running one build at a time + - name: Set up mutex + uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + with: + branch: gh-action-mutex-build-arm-master-${{ matrix.test }} # prepare images locally, tagged by commit hash - run: earthly ./yarn-project+export-end-to-end # all the end-to-end integration tests for aztec e2e-arm: needs: build-arm - runs-on: ${{ github.actor }}-arm + runs-on: master-arm timeout-minutes: 15 strategy: fail-fast: false @@ -83,6 +56,15 @@ jobs: steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + - uses: ./.github/ci-setup-action + with: + dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_key: e2e-x86-${{ github.actor }}-${{ matrix.test }} + # Ensure that a test only ever is running one e2e at a time + - name: Set up mutex + uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + with: + branch: gh-action-mutex-e2e-arm-master-${{ matrix.test }} - name: Test working-directory: ./yarn-project/end-to-end/ run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a38c51856d05..c788c471a342 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,265 +17,135 @@ concurrency: group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} cancel-in-progress: true jobs: - # # Start cheap (~1/8th the cost of on demand, ~13th the cost of large GA runners) spot builders - # # just for the CI job. These are specced per user and run the entire CI. - # # TODO These have a persistent EBS volume that forms a fast-online docker image cache (used by Earthly), meaning - # # TODO build steps that ran in previous invocations are quickly ran from cache. - # start-builder: - # timeout-minutes: 5 - # name: Start Build Runner (${{ matrix.config.runner_label_suffix }}) - # runs-on: ubuntu-latest - # permissions: - # actions: write - # strategy: - # matrix: - # config: - # - {ec2_instance_type: m6a.32xlarge, runner_concurrency: 50, ec2_ami_id: ami-04d8422a9ba4de80f, runner_label_suffix: x86} - # # - {ec2_instance_type: r6g.16xlarge, runner_concurrency: 8, ec2_ami_id: ami-0d8a9b0419ddb331a, runner_label_suffix: arm} - # steps: - # - name: Start EC2 runner - # id: start-ec2-runner - # uses: AztecProtocol/ec2-action-builder@v0.3 - # with: - # github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - # aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws_region: "us-east-2" - # ec2_subnet_id: subnet-4cfabd25 - # subaction: ${{ github.event.inputs.runner_action || (contains(github.event.head_commit.message, '[ci restart-spot]') && 'restart' || 'start') }} - # # prevent reaping by mainframe spot reaper - # ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' - # github_action_runner_version: v2.315.0 - # ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - # ec2_spot_instance_strategy: BestEffort - # runner_label: ${{ github.actor }}-${{ matrix.config.runner_label_suffix }} - # runner_concurrency: ${{ matrix.config.runner_concurrency }} - # ec2_instance_type: ${{ matrix.config.ec2_instance_type }} - # ec2_ami_id: ${{ matrix.config.ec2_ami_id }} - # ec2_instance_ttl: 40 # 30 minutes to reap, refreshed by job starts - - ######################### - # START OF ARM PIPELINE # - ######################### - start-arm-builder: + setup: uses: ./.github/workflows/setup-runner.yml with: - runner_label: ${{ github.actor }}-arm + runner_label: ${{ github.actor }}-x86 ebs_cache_size_gb: 128 runner_concurrency: 8 ec2_instance_type: r6g.16xlarge ec2_ami_id: ami-0d8a9b0419ddb331a - ec2_instance_ttl: 40 + ec2_instance_ttl: 40 # refreshed by jobs secrets: inherit - build-arm: - needs: start-arm-builder - runs-on: ${{ github.actor }}-arm - timeout-minutes: 25 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + build: + needs: setup + runs-on: ${{ github.actor }}-x86 + timeout-minutes: 40 + outputs: + e2e_list: ${{ steps.e2e_list.outputs.list }} steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + - uses: ./.github/ci-setup-action + with: + dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_key: build-${{ github.actor }}-x86 # prepare images locally, tagged by commit hash - run: earthly ./yarn-project+export-end-to-end + # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end + # (Note ARM uses just 2 tests as a smoketest) + - name: Create list of end-to-end jobs + id: e2e_list + run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT # all the end-to-end integration tests for aztec - e2e-arm: - needs: build-arm - runs-on: ${{ github.actor }}-arm + e2e: + needs: build + runs-on: ${{ github.actor }}-x86 timeout-minutes: 15 strategy: fail-fast: false matrix: - test: - - e2e-card-game - - e2e-crowdfunding-and-claim + test: ${{ fromJson( needs.build.outputs.e2e_list )}} steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + - uses: ./.github/ci-setup-action + with: + dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_key: e2e-${{ github.actor }}-x86-${{ matrix.test }} - name: Test working-directory: ./yarn-project/end-to-end/ run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache + # TODO + # - name: Upload logs + # run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log - # ######################### - # # START OF x86 PIPELINE # - # ######################### - # # prevents concurrency issues with multiple (implicit) earthly bootstraps - # setup-x86: - # needs: start-builder - # if: ${{ github.event.inputs.just_start_spot != 'true' }} - # runs-on: ${{ github.actor }}-x86 - # timeout-minutes: 5 - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # steps: - # - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - # - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # # Attach our 128gb cache disk - # - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-x86 128 - # - run: earthly bootstrap + # barretenberg (prover) native tests + # only ran on x86 for resource reasons (memory intensive) + bb-native-tests: + needs: setup + runs-on: ${{ github.actor }}-x86 + timeout-minutes: 15 + strategy: + fail-fast: false + steps: + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + # Only allow one memory-hunger prover test to use this runner + - uses: ./.github/ci-setup-action + with: + dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_key: bb-native-tests-${{ github.actor }}-x86 + - working-directory: ./barretenberg/cpp/ + run: earthly --no-output +test - # build-x86: - # needs: setup-x86 - # runs-on: ${{ github.actor }}-x86 - # timeout-minutes: 25 - # outputs: - # e2e_list: ${{ steps.e2e_list.outputs.list }} - # steps: - # - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - # - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # # Only allow one memory-hunger prover test to use this runner - # # As detailed in https://github.com/ben-z/gh-action-mutex - # # things do not become 'pending' in github actions, and instead just cancel one another - # # so we can't use the native concurrency in GA - # - name: Set up mutex - # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - # with: - # branch: gh-action-mutex-build-x86-${{ github.actor }} - # # prepare images locally, tagged by commit hash - # - run: earthly ./yarn-project+export-end-to-end - # # We base our e2e list used in e2e-x86 off the targets in ./yarn-project/end-to-end - # # (Note ARM uses just 2 tests as a smoketest) - # - name: Create list of end-to-end jobs - # id: e2e_list - # run: echo "list=$(earthly ls ./yarn-project/end-to-end | grep -v '+base' | sed 's/+//' | jq -R . | jq -cs .)" >> $GITHUB_OUTPUT + # push benchmarking binaries to dockerhub registry + bb-bench-binaries: + needs: setup + runs-on: ${{ github.actor }}-x86 + timeout-minutes: 15 + steps: + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - uses: ./.github/ci-setup-action + with: + dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_key: bb-bench-binaries-${{ github.actor }} + - name: Build and Push Binaries + if: ${{ github.event.inputs.just_start_spot != 'true' }} + working-directory: ./barretenberg/cpp/ + run: earthly --push +bench-binaries - # # all the end-to-end integration tests for aztec - # e2e-x86: - # needs: build-x86 - # runs-on: ${{ github.actor }}-x86 - # timeout-minutes: 15 - # strategy: - # fail-fast: false - # matrix: - # test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} - # steps: - # - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - # - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # - name: Set up mutex - # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - # with: - # branch: gh-action-mutex-e2e-x86-${{ github.actor }}-${{ matrix.test }} - # - name: Test - # working-directory: ./yarn-project/end-to-end/ - # run: earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache - # # TODO - # # - name: Upload logs - # # run: BRANCH=${{ github.ref_name }} PULL_REQUEST=${{ github.event.number }} scripts/ci/upload_logs_to_s3 ./yarn-project/end-to-end/log + setup-bench: + uses: ./.github/workflows/setup-runner.yml + needs: bb-bench-binaries + with: + runner_label: ${{ github.actor }}-bench-x86 + ebs_cache_size_gb: 32 + runner_concurrency: 1 + ec2_instance_type: r6g.4xlarge + ec2_ami_id: ami-0d8a9b0419ddb331a + ec2_instance_ttl: 15 # refreshed by jobs + secrets: inherit - # # barretenberg (prover) native tests - # # only ran on x86 for resource reasons (memory intensive) - # bb-native-tests: - # needs: setup-x86 - # runs-on: ${{ github.actor }}-x86 - # timeout-minutes: 15 - # strategy: - # fail-fast: false - # steps: - # - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - # - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # # Only allow one memory-hunger prover test to use this runner - # # As detailed in https://github.com/ben-z/gh-action-mutex - # # things do not become 'pending' in github actions, and instead just cancel one another - # # so we can't use the native concurrency in GA - # - name: Set up mutex - # uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - # with: - # branch: gh-action-mutex-bench-${{ github.actor }} - # - working-directory: ./barretenberg/cpp/ - # run: earthly --no-output +test + bb-bench: + runs-on: ${{ github.actor }}-bench-x86 + needs: setup-bench + timeout-minutes: 15 + steps: + - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} + - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + # Use bench_mode=cache to read the pushed build above + - name: Client IVC Bench + working-directory: ./barretenberg/cpp/ + run: earthly --no-output +bench-client-ivc --bench_mode=cache - # # push benchmarking binaries to dockerhub registry - # bb-bench-binaries: - # runs-on: ${{ github.actor }}-x86 - # timeout-minutes: 15 - # needs: setup-x86 - # steps: - # - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - # - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # - name: Build and Push Binaries - # if: ${{ github.event.inputs.just_start_spot != 'true' }} - # working-directory: ./barretenberg/cpp/ - # run: earthly --push +bench-binaries + - name: Ultrahonk Bench + working-directory: ./barretenberg/cpp/ + run: earthly --no-output +bench-ulAtra-honk --bench_mode=cache - # start-bb-bench-runner: - # timeout-minutes: 5 - # # We wait for binaries to be done for kickoff - # needs: bb-bench-binaries - # name: Start Bench Runner - # runs-on: ubuntu-latest - # permissions: - # actions: write + # # Post actions, deploy and summarize logs + # aztec-bench-summary: + # runs-on: ${{ github.actor }} + # # IMPORTANT security flaw if we don't need 'check-run-condition' + # needs: e2e-x86 + # concurrency: + # group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + # cancel-in-progress: true # steps: - # - name: Start EC2 runner - # id: start-ec2-runner - # uses: AztecProtocol/ec2-action-builder@v0.3 + # - name: Checkout + # uses: actions/checkout@v4 # with: - # github_token: ${{ secrets.GH_SELF_HOSTED_RUNNER_TOKEN }} - # aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - # aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # aws_region: "us-east-2" - # ec2_subnet_id: subnet-4cfabd25 - # subaction: ${{ github.event.inputs.runner_action || 'start' }} - # # prevent reaping by mainframe spot reaper - # ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]' - # github_action_runner_version: v2.315.0 - # ec2_security_group_id: sg-0ccd4e5df0dcca0c9 - # ec2_spot_instance_strategy: BestEffort - # runner_label: ${{ github.actor }}-bench-x86 - # runner_concurrency: 1 - # ec2_instance_type: m7a.4xlarge - # ec2_ami_id: ami-04d8422a9ba4de80f - # ec2_instance_ttl: 10 # 10 minutes to reap, refreshed by job starts - # # try if spot variance too high, uses on-demand: - # # ec2_spot_instance_strategy: none - - # setup-bb-bench: - # runs-on: ${{ github.actor }}-bench-x86 - # needs: start-bb-bench-runner - # timeout-minutes: 5 - # if: ${{ github.event.inputs.just_start_spot != 'true' }} - # env: - # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - # steps: - # - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - # - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # # Attach our 328gb cache disk - # - run: ./scripts/attach_ebs_cache.sh ${{ github.actor }}-bench-x86 32 - # - run: earthly bootstrap - - # bb-bench: - # runs-on: ${{ github.actor }}-bench-x86 - # needs: setup-bb-bench - # timeout-minutes: 15 - # steps: - # - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - # - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # # Use bench_mode=cache to read the pushed build above - # - name: Client IVC Bench - # working-directory: ./barretenberg/cpp/ - # run: earthly --no-output +bench-client-ivc --bench_mode=cache - - # - name: Ultrahonk Bench - # working-directory: ./barretenberg/cpp/ - # run: earthly --no-output +bench-ultra-honk --bench_mode=cache + # ref: ${{ github.event.pull_request.head.sha }} - # # # Post actions, deploy and summarize logs - # # aztec-bench-summary: - # # runs-on: ${{ github.actor }} - # # # IMPORTANT security flaw if we don't need 'check-run-condition' - # # needs: e2e-x86 - # # concurrency: - # # group: aztec-bench-summary-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 - # # cancel-in-progress: true - # # steps: - # # - name: Checkout - # # uses: actions/checkout@v4 - # # with: - # # ref: ${{ github.event.pull_request.head.sha }} + # - name: "Assemble benchmark summary from uploaded logs" + # command: ./scripts/ci/assemble_e2e_benchmark_earthly.sh - # # - name: "Assemble benchmark summary from uploaded logs" - # # command: ./scripts/ci/assemble_e2e_benchmark_earthly.sh diff --git a/.github/workflows/setup-runner.yml b/.github/workflows/setup-runner.yml index deff2055f0ed..d5607d8e3089 100644 --- a/.github/workflows/setup-runner.yml +++ b/.github/workflows/setup-runner.yml @@ -1,3 +1,8 @@ + +# Start cheap (~1/8th the cost of on demand, ~13th the cost of large GA runners) spot builders +# just for the CI job. These are specced per user and run the entire CI. +# TODO These have a persistent EBS volume that forms a fast-online docker image cache (used by Earthly), meaning +# TODO build steps that ran in previous invocations are quickly ran from cache. name: Reusable Spot Instance and Setup Workflow on: workflow_call: @@ -64,12 +69,12 @@ jobs: setup: needs: start-builder - runs-on: ${{ github.event.pull_request.head.sha }} + runs-on: ${{ inputs.runner_label }} steps: - name: Checkout Repository uses: actions/checkout@v4 with: - ref: ${{ inputs.ref_sha }} + ref: ${{ github.event.pull_request.head.sha }} - name: Setup CI uses: ./.github/ci-setup-action From 028a92c3438ce60620ab31e87cad2609c47ae734 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 11 Apr 2024 19:04:43 +0000 Subject: [PATCH 07/12] renames --- .../workflows/{ci-arm64.yml => ci-arm.yml} | 21 +++++++++---------- .github/workflows/ci.yml | 13 ++++++++++-- 2 files changed, 21 insertions(+), 13 deletions(-) rename .github/workflows/{ci-arm64.yml => ci-arm.yml} (80%) diff --git a/.github/workflows/ci-arm64.yml b/.github/workflows/ci-arm.yml similarity index 80% rename from .github/workflows/ci-arm64.yml rename to .github/workflows/ci-arm.yml index 807a5e2262ce..99d387ce67d6 100644 --- a/.github/workflows/ci-arm64.yml +++ b/.github/workflows/ci-arm.yml @@ -16,7 +16,7 @@ concurrency: group: ci-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} cancel-in-progress: true jobs: - start-arm-builder: + setup: uses: ./.github/workflows/setup-runner.yml with: runner_label: master-arm @@ -27,24 +27,23 @@ jobs: ec2_instance_ttl: 40 # refreshed by jobs secrets: inherit - build-arm: - needs: start-arm-builder + build: + needs: setup runs-on: master-arm timeout-minutes: 40 steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - # Ensure that CI only ever is running one build at a time - - name: Set up mutex - uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + - uses: ./.github/ci-setup-action with: - branch: gh-action-mutex-build-arm-master-${{ matrix.test }} + dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + # must be globally unique for build x runner + concurrency_key: build-master-arm # prepare images locally, tagged by commit hash - run: earthly ./yarn-project+export-end-to-end # all the end-to-end integration tests for aztec - e2e-arm: - needs: build-arm + e2e: + needs: build runs-on: master-arm timeout-minutes: 15 strategy: @@ -59,7 +58,7 @@ jobs: - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" - concurrency_key: e2e-x86-${{ github.actor }}-${{ matrix.test }} + concurrency_key: e2e-master-arm # Ensure that a test only ever is running one e2e at a time - name: Set up mutex uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c788c471a342..951c7fb0aa17 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,7 @@ jobs: - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + # must be globally unique for build x runner concurrency_key: build-${{ github.actor }}-x86 # prepare images locally, tagged by commit hash - run: earthly ./yarn-project+export-end-to-end @@ -62,6 +63,7 @@ jobs: - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + # must be globally unique for build x runner concurrency_key: e2e-${{ github.actor }}-x86-${{ matrix.test }} - name: Test working-directory: ./yarn-project/end-to-end/ @@ -84,6 +86,7 @@ jobs: - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + # must be globally unique for build x runner concurrency_key: bb-native-tests-${{ github.actor }}-x86 - working-directory: ./barretenberg/cpp/ run: earthly --no-output +test @@ -98,7 +101,8 @@ jobs: - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" - concurrency_key: bb-bench-binaries-${{ github.actor }} + # must be globally unique for build x runner + concurrency_key: bb-bench-binaries-${{ github.actor }}-x86 - name: Build and Push Binaries if: ${{ github.event.inputs.just_start_spot != 'true' }} working-directory: ./barretenberg/cpp/ @@ -122,7 +126,12 @@ jobs: timeout-minutes: 15 steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} + - uses: ./.github/ci-setup-action + with: + dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + # must be globally unique for build x runner + # technically not needed as we only make one GA runner, but a backup + concurrency_key: bb-bench-${{ github.actor }}-bench-x86 # Use bench_mode=cache to read the pushed build above - name: Client IVC Bench working-directory: ./barretenberg/cpp/ From ae7e4ae636e0d034aaf048ba6811767c541b9722 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 11 Apr 2024 19:05:31 +0000 Subject: [PATCH 08/12] simpler if --- .github/ci-setup-action/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml index f5f09d90051d..5d37a01fd47d 100644 --- a/.github/ci-setup-action/action.yml +++ b/.github/ci-setup-action/action.yml @@ -64,7 +64,7 @@ runs: # so we can't use the native concurrency in GA - name: Limit concurrency uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 - if: ${{ inputs.concurrency_key != "" }} + if: ${{ inputs.concurrency_key }} with: repository: AztecProtocol/git-metadata branch: gh-actions-mutex-${{ inputs.concurrency_key }} From 065e9225b30ba1b4d8186f481be37a2d486549e5 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 11 Apr 2024 19:05:56 +0000 Subject: [PATCH 09/12] fix --- .github/ci-setup-action/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml index 5d37a01fd47d..c332bdd88b62 100644 --- a/.github/ci-setup-action/action.yml +++ b/.github/ci-setup-action/action.yml @@ -7,7 +7,7 @@ inputs: required: true description: 'DockerHub Password' concurrency_key: - required: true + required: false description: 'Concurrency key for locking jobs' runs: # define an action, runs in OS of caller From 97dcf765614c786f95209b5194f21fe307ef23ce Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 11 Apr 2024 19:08:19 +0000 Subject: [PATCH 10/12] creds --- .github/workflows/setup-runner.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/setup-runner.yml b/.github/workflows/setup-runner.yml index d5607d8e3089..6874e8ba0132 100644 --- a/.github/workflows/setup-runner.yml +++ b/.github/workflows/setup-runner.yml @@ -82,6 +82,9 @@ jobs: dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} - name: Attach EBS Cache Disk + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_ACCESS_SECRET_KEY: ${{ secrets.AWS_ACCESS_SECRET_KEY }} run: ./scripts/attach_ebs_cache.sh ${{ inputs.runner_label }} 128 - name: Run Earthly Bootstrap From 4db7cf7ea5ba7262aca3cdef40b705c0b5b91ed4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 11 Apr 2024 19:09:30 +0000 Subject: [PATCH 11/12] creds --- .github/workflows/setup-runner.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/setup-runner.yml b/.github/workflows/setup-runner.yml index 6874e8ba0132..8bfc3f155271 100644 --- a/.github/workflows/setup-runner.yml +++ b/.github/workflows/setup-runner.yml @@ -84,7 +84,7 @@ jobs: - name: Attach EBS Cache Disk env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_ACCESS_SECRET_KEY: ${{ secrets.AWS_ACCESS_SECRET_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: ./scripts/attach_ebs_cache.sh ${{ inputs.runner_label }} 128 - name: Run Earthly Bootstrap From 309ed96ef7f831a82a352ac6732e446a953965cf Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 11 Apr 2024 19:18:15 +0000 Subject: [PATCH 12/12] customization --- .github/ci-setup-action/action.yml | 4 ++++ .github/workflows/ci-arm.yml | 3 ++- .github/workflows/ci.yml | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml index c332bdd88b62..f0f9d78a8329 100644 --- a/.github/ci-setup-action/action.yml +++ b/.github/ci-setup-action/action.yml @@ -9,6 +9,9 @@ inputs: concurrency_key: required: false description: 'Concurrency key for locking jobs' + concurrency_token: + required: false + description: 'Must be provided with concurrency key. GH token used to lock this job.' runs: # define an action, runs in OS of caller using: composite @@ -66,5 +69,6 @@ runs: uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 if: ${{ inputs.concurrency_key }} with: + repo-token: ${{ inputs.concurrency_token }} repository: AztecProtocol/git-metadata branch: gh-actions-mutex-${{ inputs.concurrency_key }} diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 99d387ce67d6..ec0b39b38eab 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -36,6 +36,7 @@ jobs: - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_token: "${{ secrets.AZTEC_GITHUB_TOKEN }}" # must be globally unique for build x runner concurrency_key: build-master-arm # prepare images locally, tagged by commit hash @@ -54,10 +55,10 @@ jobs: - e2e-crowdfunding-and-claim steps: - {uses: actions/checkout@v4, with: { ref: "${{ github.event.pull_request.head.sha }}"}} - - {uses: ./.github/ci-setup-action, with: { dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}"}} - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_token: "${{ secrets.AZTEC_GITHUB_TOKEN }}" concurrency_key: e2e-master-arm # Ensure that a test only ever is running one e2e at a time - name: Set up mutex diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 951c7fb0aa17..e123dfd965ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,7 @@ jobs: - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_token: "${{ secrets.AZTEC_GITHUB_TOKEN }}" # must be globally unique for build x runner concurrency_key: build-${{ github.actor }}-x86 # prepare images locally, tagged by commit hash @@ -63,6 +64,7 @@ jobs: - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_token: "${{ secrets.AZTEC_GITHUB_TOKEN }}" # must be globally unique for build x runner concurrency_key: e2e-${{ github.actor }}-x86-${{ matrix.test }} - name: Test @@ -86,6 +88,7 @@ jobs: - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_token: "${{ secrets.AZTEC_GITHUB_TOKEN }}" # must be globally unique for build x runner concurrency_key: bb-native-tests-${{ github.actor }}-x86 - working-directory: ./barretenberg/cpp/ @@ -101,6 +104,7 @@ jobs: - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_token: "${{ secrets.AZTEC_GITHUB_TOKEN }}" # must be globally unique for build x runner concurrency_key: bb-bench-binaries-${{ github.actor }}-x86 - name: Build and Push Binaries @@ -129,6 +133,7 @@ jobs: - uses: ./.github/ci-setup-action with: dockerhub_password: "${{ secrets.DOCKERHUB_PASSWORD }}" + concurrency_token: "${{ secrets.AZTEC_GITHUB_TOKEN }}" # must be globally unique for build x runner # technically not needed as we only make one GA runner, but a backup concurrency_key: bb-bench-${{ github.actor }}-bench-x86