chore(ci): lock aztec-packages commit in CI

.github/benchmark_projects.yml

@@ -0,0 +1,100 @@
+define: &AZ_COMMIT 7b7d960d51027b549ec9627ba2822fb87100a022
+
+projects:
+  private-kernel-inner:
+    repo: AztecProtocol/aztec-packages
+    ref: *AZ_COMMIT
+    path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner
+    num_runs: 5
+    compilation-timeout: 2.5
+    execution-timeout: 0.08
+    compilation-memory-limit: 300
+    execution-memory-limit: 250
+  private-kernel-tail:
+    repo: AztecProtocol/aztec-packages
+    ref: *AZ_COMMIT
+    path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail 
+    num_runs: 5
+    timeout: 4
+    compilation-timeout: 1.2
+    execution-timeout: 0.02
+    compilation-memory-limit: 250
+    execution-memory-limit: 200
+  private-kernel-reset:
+    repo: AztecProtocol/aztec-packages
+    ref: *AZ_COMMIT
+    path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset
+    num_runs: 5
+    timeout: 250
+    compilation-timeout: 7
+    execution-timeout: 0.35
+    compilation-memory-limit: 750
+    execution-memory-limit: 300
+  rollup-base-private:
+    repo: AztecProtocol/aztec-packages
+    ref: *AZ_COMMIT
+    path: noir-projects/noir-protocol-circuits/crates/rollup-base-private
+    num_runs: 5
+    timeout: 15
+    compilation-timeout: 10
+    execution-timeout: 0.5
+    compilation-memory-limit: 1100
+    execution-memory-limit: 500
+  rollup-base-public:
+    repo: AztecProtocol/aztec-packages
+    ref: *AZ_COMMIT
+    path: noir-projects/noir-protocol-circuits/crates/rollup-base-public
+    num_runs: 5
+    timeout: 15
+    compilation-timeout: 8
+    execution-timeout: 0.4
+    compilation-memory-limit: 1000
+    execution-memory-limit: 500
+  rollup-block-root-empty:
+    repo: AztecProtocol/aztec-packages
+    ref: *AZ_COMMIT
+    path: noir-projects/noir-protocol-circuits/crates/rollup-block-root-empty
+    cannot_execute: true
+    num_runs: 5
+    timeout: 60
+    compilation-timeout: 1.5
+    compilation-memory-limit: 400
+  rollup-block-root-single-tx:
+    repo: AztecProtocol/aztec-packages
+    ref: *AZ_COMMIT
+    path: noir-projects/noir-protocol-circuits/crates/rollup-block-root-single-tx
+    cannot_execute: true
+    num_runs: 1
+    timeout: 60
+    compilation-timeout: 80
+    compilation-memory-limit: 5500
+  rollup-block-root:
+    repo: AztecProtocol/aztec-packages
+    ref: *AZ_COMMIT
+    path: noir-projects/noir-protocol-circuits/crates/rollup-block-root
+    num_runs: 1
+    timeout: 60
+    compilation-timeout: 80
+    execution-timeout: 35
+    compilation-memory-limit: 5500
+    execution-memory-limit: 1200
+  rollup-merge:
+    repo: AztecProtocol/aztec-packages
+    ref: *AZ_COMMIT
+    path: noir-projects/noir-protocol-circuits/crates/rollup-merge
+    num_runs: 5
+    timeout: 300
+    compilation-timeout: 1.5
+    execution-timeout: 0.01
+    compilation-memory-limit: 400
+    execution-memory-limit: 400
+  rollup-root:
+    repo: AztecProtocol/aztec-packages
+    ref: *AZ_COMMIT
+    path: noir-projects/noir-protocol-circuits/crates/rollup-root
+    num_runs: 5
+    timeout: 300
+    compilation-timeout: 2
+    execution-timeout: 0.6
+    compilation-memory-limit: 450
+    execution-memory-limit: 400

.github/workflows/reports.yml

@@ -7,6 +7,23 @@ on:
   pull_request:
 
 jobs:
+  benchmark-projects-list:
+    name: Load benchmark projects list
+    runs-on: ubuntu-22.04
+    outputs:
+      projects: ${{ steps.get_bench_projects.outputs.projects }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build list of projects
+        id: get_bench_projects
+        run: |
+          PROJECTS=$(yq ./.github/benchmark_projects.yml -o json | jq -c '.projects | map(.)')
+          echo "projects=$PROJECTS"
+          echo "projects=$PROJECTS" >> $GITHUB_OUTPUT
+
   build-nargo:
     runs-on: ubuntu-22.04
 
@@ -259,25 +276,13 @@ jobs:
           overwrite: true
 
   external_repo_compilation_and_execution_report:
-    needs: [build-nargo]
+    needs: [build-nargo, benchmark-projects-list]
     runs-on: ubuntu-22.04
     timeout-minutes: 15
     strategy:
       fail-fast: false
       matrix:
-        include:
-          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts, cannot_execute: true }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner, num_runs: 5 }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail, num_runs: 5  }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset, num_runs: 5 }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private, num_runs: 5 }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public, num_runs: 5 }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-merge, num_runs: 5 }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-root-empty, num_runs: 5, cannot_execute: true }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-root-single-tx, num_runs: 1, flags: "--skip-brillig-constraints-check --skip-underconstrained-check", cannot_execute: true }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-root, num_runs: 1, flags: "--skip-brillig-constraints-check --skip-underconstrained-check" }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-merge, num_runs: 5 }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-root, num_runs: 5 }
+        project: ${{ fromJson( needs.benchmark-projects-list.outputs.projects )}}
 
     name: External repo compilation and execution reports - ${{ matrix.project.repo }}/${{ matrix.project.path }}
     steps:
@@ -322,7 +327,7 @@ jobs:
       - name: Check compilation time limit
         run: |
           TIME=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/compilation_report.json)
-          TIME_LIMIT=80
+          TIME_LIMIT=${{ matrix.project.compilation-timeout }}
           if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "$TIME_LIMIT"; then
             # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause.
             echo "Failing due to compilation exceeding timeout..."
@@ -343,7 +348,7 @@ jobs:
         if: ${{ !matrix.project.cannot_execute }}
         run: |
           TIME=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/execution_report.json)
-          TIME_LIMIT=60
+          TIME_LIMIT=${{ matrix.project.execution-timeout }}
           if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "$TIME_LIMIT"; then
             # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause.
             echo "Failing due to execution exceeding timeout..."
@@ -388,28 +393,15 @@ jobs:
           overwrite: true
 
   external_repo_memory_report:
-    needs: [build-nargo]
+    needs: [build-nargo, benchmark-projects-list]
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
-        include:
-          # TODO: Bring this report back under a flag. The `noir-contracts` report takes just under 30 minutes.
-          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail  }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-merge }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-root-empty, cannot_execute: true }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-root-single-tx, flags: "--skip-brillig-constraints-check --skip-underconstrained-check", cannot_execute: true }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-root, flags: "--skip-brillig-constraints-check --skip-underconstrained-check" }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-merge }
-          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-root }
-
-    name: External repo memory report - ${{ matrix.project.repo }}/${{ matrix.project.path }}
+        include: ${{ fromJson( needs.benchmark-projects-list.outputs.projects )}}
+
+    name: External repo memory report - ${{ matrix.repo }}/${{ matrix.path }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -426,25 +418,25 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          repository: ${{ matrix.project.repo }}
+          repository: ${{ matrix.repo }}
           path: test-repo
-          ref: ${{ matrix.project.ref }}
+          ref: ${{ matrix.ref }}
 
       - name: Generate compilation memory report
-        working-directory: ./test-repo/${{ matrix.project.path }}
+        working-directory: ./test-repo/${{ matrix.path }}
         run: |
           mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh
           mv /home/runner/work/noir/noir/scripts/test_programs/parse_memory.sh ./parse_memory.sh
           ./memory_report.sh 1
           # Rename the memory report as the execution report is about to write to the same file
           cp memory_report.json compilation_memory_report.json
         env:
-          FLAGS: ${{ matrix.project.flags }}
+          FLAGS: ${{ matrix.flags }}
 
       - name: Check compilation memory limit
         run: |
-          MEMORY=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/compilation_memory_report.json)
-          MEMORY_LIMIT=6000
+          MEMORY=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/compilation_memory_report.json)
+          MEMORY_LIMIT=${{ matrix.compilation-memory-limit }}
           if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$MEMORY" "$MEMORY_LIMIT"; then
             # Don't bump this limit without understanding why this has happened and confirming that you're not the cause.
             echo "Failing due to compilation exceeding memory limit..."
@@ -454,16 +446,16 @@ jobs:
           fi
 
       - name: Generate execution memory report
-        working-directory: ./test-repo/${{ matrix.project.path }}
-        if: ${{ !matrix.project.cannot_execute }}
+        working-directory: ./test-repo/${{ matrix.path }}
+        if: ${{ !matrix.cannot_execute }}
         run: |
           ./memory_report.sh 1 1
 
       - name: Check execution memory limit
-        if: ${{ !matrix.project.cannot_execute }}
+        if: ${{ !matrix.cannot_execute }}
         run: |
-          MEMORY=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/memory_report.json)
-          MEMORY_LIMIT=1300
+          MEMORY=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/memory_report.json)
+          MEMORY_LIMIT=${{ matrix.execution-memory-limit }}
           if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$MEMORY" "$MEMORY_LIMIT"; then
             # Don't bump this limit without understanding why this has happened and confirming that you're not the cause.
             echo "Failing due to execution exceeding memory limit..."
@@ -476,9 +468,9 @@ jobs:
         id: compilation_mem_report
         shell: bash
         run: |
-          PACKAGE_NAME=${{ matrix.project.path }}
+          PACKAGE_NAME=${{ matrix.path }}
           PACKAGE_NAME=$(basename $PACKAGE_NAME)
-          mv ./test-repo/${{ matrix.project.path }}/compilation_memory_report.json ./memory_report_$PACKAGE_NAME.json
+          mv ./test-repo/${{ matrix.path }}/compilation_memory_report.json ./memory_report_$PACKAGE_NAME.json
           echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
 
       - name: Upload compilation memory report
@@ -491,11 +483,11 @@ jobs:
 
       - name: Move execution report
         id: execution_mem_report
-        if: ${{ !matrix.project.cannot_execute }}
+        if: ${{ !matrix.cannot_execute }}
         run: |
-          PACKAGE_NAME=${{ matrix.project.path }}
+          PACKAGE_NAME=${{ matrix.path }}
           PACKAGE_NAME=$(basename $PACKAGE_NAME)
-          mv ./test-repo/${{ matrix.project.path }}/memory_report.json ./memory_report_$PACKAGE_NAME.json
+          mv ./test-repo/${{ matrix.path }}/memory_report.json ./memory_report_$PACKAGE_NAME.json
           echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
 
       - name: Upload execution memory report

.github/workflows/test-js-packages.yml

@@ -492,9 +492,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        project: ${{ fromJson( needs.critical-library-list.outputs.libraries )}}
+        include: ${{ fromJson( needs.critical-library-list.outputs.libraries )}}
 
-    name: Check external repo - ${{ matrix.project.repo }}/${{ matrix.project.path }}
+    name: Check external repo - ${{ matrix.repo }}/${{ matrix.path }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -504,9 +504,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          repository: ${{ matrix.project.repo }}
+          repository: ${{ matrix.repo }}
           path: test-repo
-          ref: ${{ matrix.project.ref }}
+          ref: ${{ matrix.ref }}
 
       - name: Download nargo binary
         uses: ./noir-repo/.github/actions/download-nargo
@@ -520,14 +520,14 @@ jobs:
 
       - name: Run nargo test
         id: test_report
-        working-directory: ./test-repo/${{ matrix.project.path }}
+        working-directory: ./test-repo/${{ matrix.path }}
         run: |
-          output_file=${{ github.workspace }}/noir-repo/.github/critical_libraries_status/${{ matrix.project.repo }}/${{ matrix.project.path }}.actual.jsonl
+          output_file=${{ github.workspace }}/noir-repo/.github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.actual.jsonl
           BEFORE=$SECONDS
-          nargo test --silence-warnings --skip-brillig-constraints-check --format json ${{ matrix.project.nargo_args }} | tee $output_file
+          nargo test --silence-warnings --skip-brillig-constraints-check --format json ${{ matrix.nargo_args }} | tee $output_file
           TIME=$(($SECONDS-$BEFORE))
 
-          NAME=${{ matrix.project.repo }}/${{ matrix.project.path }}
+          NAME=${{ matrix.repo }}/${{ matrix.path }}
           # Replace any slashes with underscores
           NAME=${NAME//\//_}
           TEST_REPORT_NAME=test_report_$NAME
@@ -544,25 +544,25 @@ jobs:
 
       - name: Check test time limit
         run: |
-          TIME=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/${{ steps.test_report.outputs.test_report_name }}.json)
-          if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "${{ matrix.project.timeout }}"; then
+          TIME=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/${{ steps.test_report.outputs.test_report_name }}.json)
+          if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "${{ matrix.timeout }}"; then
             # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause.
             echo "Failing due to test suite exceeding timeout..."
-            echo "Timeout: ${{ matrix.project.timeout }}"
+            echo "Timeout: ${{ matrix.timeout }}"
             echo "Test suite took: $TIME".
             exit 1
           fi
 
       - name: Compare test results
         working-directory: ./noir-repo
-        run: .github/scripts/check_test_results.sh .github/critical_libraries_status/${{ matrix.project.repo }}/${{ matrix.project.path }}.failures.jsonl .github/critical_libraries_status/${{ matrix.project.repo }}/${{ matrix.project.path }}.actual.jsonl
+        run: .github/scripts/check_test_results.sh .github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.failures.jsonl .github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.actual.jsonl
 
       - name: Upload test report
-        if: ${{ matrix.project.timeout > 10 }} # We want to avoid recording benchmarking for a ton of tiny libraries, these should be covered with aggressive timeouts
+        if: ${{ matrix.timeout > 10 }} # We want to avoid recording benchmarking for a ton of tiny libraries, these should be covered with aggressive timeouts
         uses: actions/upload-artifact@v4
         with:
           name: ${{ steps.test_report.outputs.test_report_name }}
-          path: ./test-repo/${{ matrix.project.path }}/${{ steps.test_report.outputs.test_report_name }}.json
+          path: ./test-repo/${{ matrix.path }}/${{ steps.test_report.outputs.test_report_name }}.json
           retention-days: 3
           overwrite: true