From da44da3a5854293ac1c5a8c1835c5fb61b385136 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 13 May 2024 18:37:42 +0000
Subject: [PATCH 01/11] better spot copy times

---
 .github/ensure-tester-with-images/action.yml | 5 +++--
 scripts/ci/attach_ebs_cache.sh               | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/ensure-tester-with-images/action.yml b/.github/ensure-tester-with-images/action.yml
index 23c773caf1ef..36ac7b00c3c1 100644
--- a/.github/ensure-tester-with-images/action.yml
+++ b/.github/ensure-tester-with-images/action.yml
@@ -60,8 +60,9 @@ runs:
           export BUILDER_SPOT_IP=${{ env.BUILDER_SPOT_IP }}
           export BUILDER_SPOT_KEY=~/.ssh/build_instance_key
           scripts/run_on_builder "
-            flock ${{ env.IMAGE_KEY }}.lock bash -c '! [ -f ${{ env.IMAGE_KEY }}.brotli ] && docker save ${{ inputs.builder_images_to_copy }} | brotli -2 > ${{ env.IMAGE_KEY }}.brotli'
-            cat ${{ env.IMAGE_KEY }}.brotli
+            sudo mkdir -p /var/lib/docker/images
+            sudo flock /var/lib/docker/images/${{ env.IMAGE_KEY }}.lock bash -c '! [ -f /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli ] && docker save ${{ inputs.builder_images_to_copy }} | brotli -2 > /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli'
+            sudo cat /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli
           " | brotli --decompress | docker load
 
     - name: Test
diff --git a/scripts/ci/attach_ebs_cache.sh b/scripts/ci/attach_ebs_cache.sh
index b97e59202819..0304ddc05d08 100755
--- a/scripts/ci/attach_ebs_cache.sh
+++ b/scripts/ci/attach_ebs_cache.sh
@@ -153,6 +153,7 @@ fi
 mkdir -p /var/lib/docker
 mount $BLKDEVICE /var/lib/docker
 service docker restart
+rm -rf /var/lib/docker/images
 # important: everything (except earthly ls) should go through earthly-ci
 scripts/earthly-ci bootstrap
 touch /home/ubuntu/.setup-complete
\ No newline at end of file

From 80b7959feadb4fb87a525006015fe3a237244197 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 13 May 2024 18:48:03 +0000
Subject: [PATCH 02/11] fix hash on master

---
 .github/workflows/ci.yml       | 45 +++++++++++++++++-----------------
 scripts/ci/attach_ebs_cache.sh |  1 +
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 57c0e4301108..34f0a0d170a5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,6 +24,7 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+  GIT_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
   # kludge until we move away from runners
   WAIT_FOR_RUNNERS: false
 
@@ -42,7 +43,7 @@ jobs:
       bench_list: ${{ steps.bench_list.outputs.list }}
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: build-${{ inputs.username || github.actor }}-x86
@@ -69,7 +70,7 @@ jobs:
         test: ${{ fromJson( needs.build.outputs.e2e_list )}}
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
       - name: Setup and Test
         timeout-minutes: 40
@@ -79,7 +80,7 @@ jobs:
           builder_type: builder-x86
           # these are copied to the tester and expected by the earthly command below
           # if they fail to copy, it will try to build them on the tester and fail
-          builder_images_to_copy: aztecprotocol/aztec:${{ github.event.pull_request.head.sha }} aztecprotocol/end-to-end:${{ github.event.pull_request.head.sha }}
+          builder_images_to_copy: aztecprotocol/aztec:${{ env.GITHUB_COMMIT }} aztecprotocol/end-to-end:${{ env.GITHUB_COMMIT }}
           # command to produce the images in case they don't exist
           builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
           run: |
@@ -98,7 +99,7 @@ jobs:
         test: ${{ fromJson( needs.build.outputs.bench_list )}}
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
       - name: Setup and Test
         uses: ./.github/ensure-tester-with-images
@@ -108,7 +109,7 @@ jobs:
           builder_type: builder-x86
           # these are copied to the tester and expected by the earthly command below
           # if they fail to copy, it will try to build them on the tester and fail
-          builder_images_to_copy: aztecprotocol/aztec:${{ github.event.pull_request.head.sha }} aztecprotocol/end-to-end:${{ github.event.pull_request.head.sha }}
+          builder_images_to_copy: aztecprotocol/aztec:${{ env.GITHUB_COMMIT }} aztecprotocol/end-to-end:${{ env.GITHUB_COMMIT }}
           # command to produce the images in case they don't exist
           builder_command: cd yarn-project/end-to-end/ && ../../scripts/earthly-ci +${{ matrix.test }}
           run: |
@@ -128,7 +129,7 @@ jobs:
   # #     - uses: actions/checkout@v4
   # #       with:
   # #         fetch-depth: 100 # Downloading base benchmark from master requires access to history
-  # #         ref: "${{ github.event.pull_request.head.sha }}"
+  # #         ref: "${{ env.GITHUB_COMMIT }}"
   # #     - uses: ./.github/ci-setup-action
   # #       with:
   # #         concurrency_key: build-${{ inputs.username || github.actor }}-x86
@@ -153,7 +154,7 @@ jobs:
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: noir-format-${{ inputs.username || github.actor }}-x86
@@ -173,7 +174,7 @@ jobs:
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       # Only allow one memory-hunger prover test to use this runner
       - uses: ./.github/ci-setup-action
         with:
@@ -190,7 +191,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: bb-js-test-${{ github.actor }}-x86
@@ -204,7 +205,7 @@ jobs:
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: noir-${{ inputs.username || github.actor }}-x86
@@ -216,7 +217,7 @@ jobs:
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: noir-packages-${{ inputs.username || github.actor }}-x86
@@ -228,7 +229,7 @@ jobs:
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: noir-projects-${{ inputs.username || github.actor }}-x86
@@ -241,7 +242,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       # Only allow one memory-hunger prover test to use this runner
       - uses: ./.github/ci-setup-action
         with:
@@ -256,7 +257,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       # Only allow one memory-hunger prover test to use this runner
       - uses: ./.github/ci-setup-action
         with:
@@ -270,7 +271,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: prover-client-test-${{ github.actor }}-x86
@@ -283,7 +284,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: build-acir-tests-${{ github.actor }}-x86
@@ -296,7 +297,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: barretenberg-acir-tests-bb-${{ github.actor }}-x86
@@ -310,7 +311,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: barretenberg-acir-tests-sol-${{ github.actor }}-x86
@@ -324,7 +325,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: barretenberg-acir-tests-bb-js-${{ github.actor }}-x86
@@ -339,7 +340,7 @@ jobs:
     if: github.event.number
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: docs-preview-${{ inputs.username || github.actor }}-x86
@@ -352,7 +353,7 @@ jobs:
     needs: setup
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       - uses: ./.github/ci-setup-action
       - name: Build Bench Binaries
         uses: ./.github/ensure-builder
@@ -378,7 +379,7 @@ jobs:
       pull-requests: write
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GITHUB_COMMIT }}" }
       # Only allow one memory-hunger prover test to use this runner
       - uses: ./.github/ci-setup-action
         with:
diff --git a/scripts/ci/attach_ebs_cache.sh b/scripts/ci/attach_ebs_cache.sh
index 0304ddc05d08..06c08a3c6382 100755
--- a/scripts/ci/attach_ebs_cache.sh
+++ b/scripts/ci/attach_ebs_cache.sh
@@ -153,6 +153,7 @@ fi
 mkdir -p /var/lib/docker
 mount $BLKDEVICE /var/lib/docker
 service docker restart
+# clear our images cache
 rm -rf /var/lib/docker/images
 # important: everything (except earthly ls) should go through earthly-ci
 scripts/earthly-ci bootstrap

From 0baef4b117ef08af19d99a8075412b1016e19e25 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 13 May 2024 19:06:26 +0000
Subject: [PATCH 03/11] better string

---
 .github/ensure-tester-with-images/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/ensure-tester-with-images/action.yml b/.github/ensure-tester-with-images/action.yml
index 36ac7b00c3c1..92031f71e365 100644
--- a/.github/ensure-tester-with-images/action.yml
+++ b/.github/ensure-tester-with-images/action.yml
@@ -61,7 +61,7 @@ runs:
           export BUILDER_SPOT_KEY=~/.ssh/build_instance_key
           scripts/run_on_builder "
             sudo mkdir -p /var/lib/docker/images
-            sudo flock /var/lib/docker/images/${{ env.IMAGE_KEY }}.lock bash -c '! [ -f /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli ] && docker save ${{ inputs.builder_images_to_copy }} | brotli -2 > /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli'
+            sudo flock /var/lib/docker/images/${{ env.IMAGE_KEY }}.lock bash -c '! [ -f /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli ] && docker save ${{ inputs.builder_images_to_copy }} | brotli -2 > /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli.tmp && mv /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli'
             sudo cat /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli
           " | brotli --decompress | docker load
 

From 056968716d7a474bc284ad3f7f715b8d8aad19c5 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 13 May 2024 19:07:35 +0000
Subject: [PATCH 04/11] fix

---
 .github/ensure-tester-with-images/action.yml | 6 +++---
 scripts/ci/attach_ebs_cache.sh               | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/ensure-tester-with-images/action.yml b/.github/ensure-tester-with-images/action.yml
index 92031f71e365..f359016b5c36 100644
--- a/.github/ensure-tester-with-images/action.yml
+++ b/.github/ensure-tester-with-images/action.yml
@@ -60,9 +60,9 @@ runs:
           export BUILDER_SPOT_IP=${{ env.BUILDER_SPOT_IP }}
           export BUILDER_SPOT_KEY=~/.ssh/build_instance_key
           scripts/run_on_builder "
-            sudo mkdir -p /var/lib/docker/images
-            sudo flock /var/lib/docker/images/${{ env.IMAGE_KEY }}.lock bash -c '! [ -f /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli ] && docker save ${{ inputs.builder_images_to_copy }} | brotli -2 > /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli.tmp && mv /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli'
-            sudo cat /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli
+            sudo mkdir -p /var/lib/docker/tmp
+            sudo flock /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.lock bash -c '! [ -f /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli ] && docker save ${{ inputs.builder_images_to_copy }} | brotli -2 > /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli.tmp && mv /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli'
+            sudo cat /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli
           " | brotli --decompress | docker load
 
     - name: Test
diff --git a/scripts/ci/attach_ebs_cache.sh b/scripts/ci/attach_ebs_cache.sh
index 06c08a3c6382..640341814d99 100755
--- a/scripts/ci/attach_ebs_cache.sh
+++ b/scripts/ci/attach_ebs_cache.sh
@@ -153,8 +153,8 @@ fi
 mkdir -p /var/lib/docker
 mount $BLKDEVICE /var/lib/docker
 service docker restart
-# clear our images cache
-rm -rf /var/lib/docker/images
+# clear our images temp folder
+rm -rf /var/lib/docker/tmp
 # important: everything (except earthly ls) should go through earthly-ci
 scripts/earthly-ci bootstrap
 touch /home/ubuntu/.setup-complete
\ No newline at end of file

From a0969482cb47fb80de8ffdca8e8608802abbdec3 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 13 May 2024 19:08:10 +0000
Subject: [PATCH 05/11] fix

---
 .github/ensure-tester-with-images/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/ensure-tester-with-images/action.yml b/.github/ensure-tester-with-images/action.yml
index f359016b5c36..3bd04eb38310 100644
--- a/.github/ensure-tester-with-images/action.yml
+++ b/.github/ensure-tester-with-images/action.yml
@@ -61,7 +61,7 @@ runs:
           export BUILDER_SPOT_KEY=~/.ssh/build_instance_key
           scripts/run_on_builder "
             sudo mkdir -p /var/lib/docker/tmp
-            sudo flock /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.lock bash -c '! [ -f /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli ] && docker save ${{ inputs.builder_images_to_copy }} | brotli -2 > /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli.tmp && mv /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli'
+            sudo flock /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.lock bash -c '! [ -f /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli ] && docker save ${{ inputs.builder_images_to_copy }} | brotli -2 > /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli.tmp && mv /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli.tmp /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli'
             sudo cat /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli
           " | brotli --decompress | docker load
 

From a6c26850fc4a763765b47287328772bdc275278d Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 13 May 2024 19:31:47 +0000
Subject: [PATCH 06/11] fix env

---
 .github/workflows/ci.yml | 46 ++++++++++++++++++++--------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 34f0a0d170a5..24f87227e818 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -30,7 +30,7 @@ env:
 
 jobs:
   setup:
-    uses: ./.github/workflows/setup-runner.yml
+    uses: ./.github/workflows/setup-runner.ymlA
     with:
       username: ${{ inputs.username || github.actor }}
       runner_type: builder-x86
@@ -43,7 +43,7 @@ jobs:
       bench_list: ${{ steps.bench_list.outputs.list }}
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: build-${{ inputs.username || github.actor }}-x86
@@ -70,7 +70,7 @@ jobs:
         test: ${{ fromJson( needs.build.outputs.e2e_list )}}
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
       - name: Setup and Test
         timeout-minutes: 40
@@ -80,7 +80,7 @@ jobs:
           builder_type: builder-x86
           # these are copied to the tester and expected by the earthly command below
           # if they fail to copy, it will try to build them on the tester and fail
-          builder_images_to_copy: aztecprotocol/aztec:${{ env.GITHUB_COMMIT }} aztecprotocol/end-to-end:${{ env.GITHUB_COMMIT }}
+          builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
           # command to produce the images in case they don't exist
           builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
           run: |
@@ -99,7 +99,7 @@ jobs:
         test: ${{ fromJson( needs.build.outputs.bench_list )}}
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
       - name: Setup and Test
         uses: ./.github/ensure-tester-with-images
@@ -109,7 +109,7 @@ jobs:
           builder_type: builder-x86
           # these are copied to the tester and expected by the earthly command below
           # if they fail to copy, it will try to build them on the tester and fail
-          builder_images_to_copy: aztecprotocol/aztec:${{ env.GITHUB_COMMIT }} aztecprotocol/end-to-end:${{ env.GITHUB_COMMIT }}
+          builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
           # command to produce the images in case they don't exist
           builder_command: cd yarn-project/end-to-end/ && ../../scripts/earthly-ci +${{ matrix.test }}
           run: |
@@ -129,7 +129,7 @@ jobs:
   # #     - uses: actions/checkout@v4
   # #       with:
   # #         fetch-depth: 100 # Downloading base benchmark from master requires access to history
-  # #         ref: "${{ env.GITHUB_COMMIT }}"
+  # #         ref: "${{ env.GIT_COMMIT }}"
   # #     - uses: ./.github/ci-setup-action
   # #       with:
   # #         concurrency_key: build-${{ inputs.username || github.actor }}-x86
@@ -154,7 +154,7 @@ jobs:
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: noir-format-${{ inputs.username || github.actor }}-x86
@@ -174,7 +174,7 @@ jobs:
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       # Only allow one memory-hunger prover test to use this runner
       - uses: ./.github/ci-setup-action
         with:
@@ -191,7 +191,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: bb-js-test-${{ github.actor }}-x86
@@ -205,7 +205,7 @@ jobs:
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: noir-${{ inputs.username || github.actor }}-x86
@@ -217,7 +217,7 @@ jobs:
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: noir-packages-${{ inputs.username || github.actor }}-x86
@@ -229,7 +229,7 @@ jobs:
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: noir-projects-${{ inputs.username || github.actor }}-x86
@@ -242,7 +242,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       # Only allow one memory-hunger prover test to use this runner
       - uses: ./.github/ci-setup-action
         with:
@@ -257,7 +257,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       # Only allow one memory-hunger prover test to use this runner
       - uses: ./.github/ci-setup-action
         with:
@@ -271,7 +271,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: prover-client-test-${{ github.actor }}-x86
@@ -284,7 +284,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: build-acir-tests-${{ github.actor }}-x86
@@ -297,7 +297,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: barretenberg-acir-tests-bb-${{ github.actor }}-x86
@@ -311,7 +311,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: barretenberg-acir-tests-sol-${{ github.actor }}-x86
@@ -325,7 +325,7 @@ jobs:
     runs-on: ${{ github.actor }}-x86
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: barretenberg-acir-tests-bb-js-${{ github.actor }}-x86
@@ -340,7 +340,7 @@ jobs:
     if: github.event.number
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
         with:
           concurrency_key: docs-preview-${{ inputs.username || github.actor }}-x86
@@ -353,7 +353,7 @@ jobs:
     needs: setup
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
       - name: Build Bench Binaries
         uses: ./.github/ensure-builder
@@ -379,7 +379,7 @@ jobs:
       pull-requests: write
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ env.GITHUB_COMMIT }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
       # Only allow one memory-hunger prover test to use this runner
       - uses: ./.github/ci-setup-action
         with:

From 2b59382982155fcfd15b99a23140d0c3ba53e948 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 13 May 2024 19:33:47 +0000
Subject: [PATCH 07/11] fix

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 24f87227e818..70315ae27927 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -30,7 +30,7 @@ env:
 
 jobs:
   setup:
-    uses: ./.github/workflows/setup-runner.ymlA
+    uses: ./.github/workflows/setup-runner.yml
     with:
       username: ${{ inputs.username || github.actor }}
       runner_type: builder-x86

From 708a21af93c9cb93e16536d9661dfa384b3973d5 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 13 May 2024 19:37:37 +0000
Subject: [PATCH 08/11] fix

---
 .github/ensure-tester-with-images/action.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/ensure-tester-with-images/action.yml b/.github/ensure-tester-with-images/action.yml
index 3bd04eb38310..010acf9119b0 100644
--- a/.github/ensure-tester-with-images/action.yml
+++ b/.github/ensure-tester-with-images/action.yml
@@ -61,7 +61,12 @@ runs:
           export BUILDER_SPOT_KEY=~/.ssh/build_instance_key
           scripts/run_on_builder "
             sudo mkdir -p /var/lib/docker/tmp
-            sudo flock /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.lock bash -c '! [ -f /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli ] && docker save ${{ inputs.builder_images_to_copy }} | brotli -2 > /var/lib/docker/images/${{ env.IMAGE_KEY }}.brotli.tmp && mv /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli.tmp /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli'
+
+            sudo flock /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.lock bash -c '
+              if ! [ -f /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli ] ; then
+                docker save aztecprotocol/aztec:${{ env.IMAGE_KEY }} aztecprotocol/end-to-end:${{ env.IMAGE_KEY }} | brotli -2 > /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli.tmp
+                mv /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli.tmp /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli
+              fi'
             sudo cat /var/lib/docker/tmp/${{ env.IMAGE_KEY }}.brotli
           " | brotli --decompress | docker load
 

From 0dd254329e1d5c886ac9b461dbc63b7fcb896f89 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 13 May 2024 20:34:12 +0000
Subject: [PATCH 09/11] better retry loop

---
 .github/spot-runner-action/dist/index.js | 41 +++++++++------------
 .github/spot-runner-action/src/ec2.ts    |  7 ++--
 .github/spot-runner-action/src/main.ts   | 46 +++++++++++-------------
 3 files changed, 42 insertions(+), 52 deletions(-)

diff --git a/.github/spot-runner-action/dist/index.js b/.github/spot-runner-action/dist/index.js
index 3404bc2f3824..69852cd1c297 100644
--- a/.github/spot-runner-action/dist/index.js
+++ b/.github/spot-runner-action/dist/index.js
@@ -337,8 +337,9 @@ class Ec2Instance {
             const fleet = yield client.createFleet(createFleetRequest).promise();
             if (fleet.Errors && fleet.Errors.length > 0) {
                 for (const error of fleet.Errors) {
-                    if (error.ErrorCode === "RequestLimitExceeded") {
-                        return "RequestLimitExceeded";
+                    if (error.ErrorCode === "RequestLimitExceeded" ||
+                        error.ErrorCode === "InsufficientInstanceCapacity") {
+                        return error.ErrorCode;
                     }
                 }
                 core.error(JSON.stringify(fleet.Errors, null, 2));
@@ -732,29 +733,21 @@ function requestAndWaitForSpot(config) {
             // 6 * 10000ms = 1 minute per strategy, unless we hit RequestLimitExceeded, then we do exponential backoff
             // TODO make longer lived spot request?
             for (let i = 0; i < 6; i++) {
-                try {
-                    // Start instance
-                    instanceId =
-                        yield ec2Client.requestMachine(
-                        // we fallback to on-demand
-                        ec2Strategy.toLocaleLowerCase() === "none");
-                    // let's exit, only loop on InsufficientInstanceCapacity
-                    if (instanceId !== "RequestLimitExceeded") {
-                        break;
-                    }
+                // Start instance
+                const instanceIdOrError = yield ec2Client.requestMachine(
+                // we fallback to on-demand
+                ec2Strategy.toLocaleLowerCase() === "none");
+                // let's exit, only loop on InsufficientInstanceCapacity
+                if (instanceIdOrError !== "RequestLimitExceeded" &&
+                    instanceIdOrError !== "InsufficientInstanceCapacity") {
+                    core.info("Failed to create instance due to " +
+                        instanceIdOrError +
+                        " , waiting 10 seconds and trying again.");
+                    backoff += 1;
                 }
-                catch (error) {
-                    // TODO is this still the relevant error?
-                    if ((error === null || error === void 0 ? void 0 : error.code) &&
-                        error.code === "InsufficientInstanceCapacity" &&
-                        ec2SpotStrategies.length > 0 &&
-                        ec2Strategy.toLocaleLowerCase() != "none") {
-                        core.info("Failed to create instance due to 'InsufficientInstanceCapacity', waiting 10 seconds and trying again.");
-                        // we loop after 10 seconds
-                    }
-                    else {
-                        throw error;
-                    }
+                else {
+                    instanceId = instanceIdOrError;
+                    break;
                 }
                 // wait 10 seconds
                 yield new Promise((r) => setTimeout(r, 10000 * Math.pow(2, backoff)));
diff --git a/.github/spot-runner-action/src/ec2.ts b/.github/spot-runner-action/src/ec2.ts
index a00ca587b6cc..41c91bcdd4d8 100644
--- a/.github/spot-runner-action/src/ec2.ts
+++ b/.github/spot-runner-action/src/ec2.ts
@@ -256,8 +256,11 @@ export class Ec2Instance {
     const fleet = await client.createFleet(createFleetRequest).promise();
     if (fleet.Errors && fleet.Errors.length > 0) {
       for (const error of fleet.Errors) {
-        if (error.ErrorCode === "RequestLimitExceeded") {
-          return "RequestLimitExceeded";
+        if (
+          error.ErrorCode === "RequestLimitExceeded" ||
+          error.ErrorCode === "InsufficientInstanceCapacity"
+        ) {
+          return error.ErrorCode;
         }
       }
       core.error(JSON.stringify(fleet.Errors, null, 2));
diff --git a/.github/spot-runner-action/src/main.ts b/.github/spot-runner-action/src/main.ts
index b5db5bb376c5..b03ef175ab79 100644
--- a/.github/spot-runner-action/src/main.ts
+++ b/.github/spot-runner-action/src/main.ts
@@ -66,32 +66,26 @@ async function requestAndWaitForSpot(config: ActionConfig): Promise<string> {
     // 6 * 10000ms = 1 minute per strategy, unless we hit RequestLimitExceeded, then we do exponential backoff
     // TODO make longer lived spot request?
     for (let i = 0; i < 6; i++) {
-      try {
-        // Start instance
-        instanceId =
-          await ec2Client.requestMachine(
-            // we fallback to on-demand
-            ec2Strategy.toLocaleLowerCase() === "none"
-          );
-        // let's exit, only loop on InsufficientInstanceCapacity
-        if (instanceId !== "RequestLimitExceeded") {
-          break;
-        }
-      } catch (error) {
-        // TODO is this still the relevant error?
-        if (
-          error?.code &&
-          error.code === "InsufficientInstanceCapacity" &&
-          ec2SpotStrategies.length > 0 &&
-          ec2Strategy.toLocaleLowerCase() != "none"
-        ) {
-          core.info(
-            "Failed to create instance due to 'InsufficientInstanceCapacity', waiting 10 seconds and trying again."
-          );
-          // we loop after 10 seconds
-        } else {
-          throw error;
-        }
+      // Start instance
+      const instanceIdOrError =
+        await ec2Client.requestMachine(
+          // we fallback to on-demand
+          ec2Strategy.toLocaleLowerCase() === "none"
+        );
+      // let's exit, only loop on InsufficientInstanceCapacity
+      if (
+        instanceIdOrError !== "RequestLimitExceeded" &&
+        instanceIdOrError !== "InsufficientInstanceCapacity"
+      ) {
+        core.info(
+          "Failed to create instance due to " +
+            instanceIdOrError +
+            " , waiting 10 seconds and trying again."
+        );
+        backoff += 1;
+      } else {
+        instanceId = instanceIdOrError;
+        break;
       }
       // wait 10 seconds
       await new Promise((r) => setTimeout(r, 10000 * 2 ** backoff));

From 76629d6cec56631199b403a067a51554bd51d97c Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 13 May 2024 20:36:21 +0000
Subject: [PATCH 10/11] fix

---
 .github/spot-runner-action/dist/index.js | 4 ++--
 .github/spot-runner-action/src/main.ts   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/spot-runner-action/dist/index.js b/.github/spot-runner-action/dist/index.js
index 69852cd1c297..666658bd34c3 100644
--- a/.github/spot-runner-action/dist/index.js
+++ b/.github/spot-runner-action/dist/index.js
@@ -738,8 +738,8 @@ function requestAndWaitForSpot(config) {
                 // we fallback to on-demand
                 ec2Strategy.toLocaleLowerCase() === "none");
                 // let's exit, only loop on InsufficientInstanceCapacity
-                if (instanceIdOrError !== "RequestLimitExceeded" &&
-                    instanceIdOrError !== "InsufficientInstanceCapacity") {
+                if (instanceIdOrError === "RequestLimitExceeded" ||
+                    instanceIdOrError === "InsufficientInstanceCapacity") {
                     core.info("Failed to create instance due to " +
                         instanceIdOrError +
                         " , waiting 10 seconds and trying again.");
diff --git a/.github/spot-runner-action/src/main.ts b/.github/spot-runner-action/src/main.ts
index b03ef175ab79..b48c33ab5979 100644
--- a/.github/spot-runner-action/src/main.ts
+++ b/.github/spot-runner-action/src/main.ts
@@ -74,8 +74,8 @@ async function requestAndWaitForSpot(config: ActionConfig): Promise<string> {
         );
       // let's exit, only loop on InsufficientInstanceCapacity
       if (
-        instanceIdOrError !== "RequestLimitExceeded" &&
-        instanceIdOrError !== "InsufficientInstanceCapacity"
+        instanceIdOrError === "RequestLimitExceeded" ||
+        instanceIdOrError === "InsufficientInstanceCapacity"
       ) {
         core.info(
           "Failed to create instance due to " +

From 14d931f6bf33b95168ca59e27dbc1dce4abf0c92 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 13 May 2024 20:37:12 +0000
Subject: [PATCH 11/11] fix

---
 .github/spot-runner-action/src/main.ts | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/spot-runner-action/src/main.ts b/.github/spot-runner-action/src/main.ts
index b48c33ab5979..01397bcfd50f 100644
--- a/.github/spot-runner-action/src/main.ts
+++ b/.github/spot-runner-action/src/main.ts
@@ -61,10 +61,8 @@ async function requestAndWaitForSpot(config: ActionConfig): Promise<string> {
 
   let instanceId = "";
   for (const ec2Strategy of ec2SpotStrategies) {
-    let backoff = 1;
+    let backoff = 0;
     core.info(`Starting instance with ${ec2Strategy} strategy`);
-    // 6 * 10000ms = 1 minute per strategy, unless we hit RequestLimitExceeded, then we do exponential backoff
-    // TODO make longer lived spot request?
     for (let i = 0; i < 6; i++) {
       // Start instance
       const instanceIdOrError =
@@ -77,12 +75,12 @@ async function requestAndWaitForSpot(config: ActionConfig): Promise<string> {
         instanceIdOrError === "RequestLimitExceeded" ||
         instanceIdOrError === "InsufficientInstanceCapacity"
       ) {
+        backoff += 1;
         core.info(
           "Failed to create instance due to " +
             instanceIdOrError +
-            " , waiting 10 seconds and trying again."
+            " , waiting " + 10000 * 2 ** backoff + " seconds and trying again."
         );
-        backoff += 1;
       } else {
         instanceId = instanceIdOrError;
         break;