diff --git a/.github/benchmark_projects.yml b/.github/benchmark_projects.yml new file mode 100644 index 00000000000..80982adc46b --- /dev/null +++ b/.github/benchmark_projects.yml @@ -0,0 +1,100 @@ +define: &AZ_COMMIT 7b7d960d51027b549ec9627ba2822fb87100a022 + +projects: + private-kernel-inner: + repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT + path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner + num_runs: 5 + compilation-timeout: 2.5 + execution-timeout: 0.08 + compilation-memory-limit: 300 + execution-memory-limit: 250 + private-kernel-tail: + repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT + path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail + num_runs: 5 + timeout: 4 + compilation-timeout: 1.2 + execution-timeout: 0.02 + compilation-memory-limit: 250 + execution-memory-limit: 200 + private-kernel-reset: + repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT + path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset + num_runs: 5 + timeout: 250 + compilation-timeout: 7 + execution-timeout: 0.35 + compilation-memory-limit: 750 + execution-memory-limit: 300 + rollup-base-private: + repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT + path: noir-projects/noir-protocol-circuits/crates/rollup-base-private + num_runs: 5 + timeout: 15 + compilation-timeout: 10 + execution-timeout: 0.5 + compilation-memory-limit: 1100 + execution-memory-limit: 500 + rollup-base-public: + repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT + path: noir-projects/noir-protocol-circuits/crates/rollup-base-public + num_runs: 5 + timeout: 15 + compilation-timeout: 8 + execution-timeout: 0.4 + compilation-memory-limit: 1000 + execution-memory-limit: 500 + rollup-block-root-empty: + repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT + path: noir-projects/noir-protocol-circuits/crates/rollup-block-root-empty + cannot_execute: true + num_runs: 5 + timeout: 60 + compilation-timeout: 1.5 + compilation-memory-limit: 400 + rollup-block-root-single-tx: + repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT + path: noir-projects/noir-protocol-circuits/crates/rollup-block-root-single-tx + cannot_execute: true + num_runs: 1 + timeout: 60 + compilation-timeout: 80 + compilation-memory-limit: 5500 + rollup-block-root: + repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT + path: noir-projects/noir-protocol-circuits/crates/rollup-block-root + num_runs: 1 + timeout: 60 + compilation-timeout: 80 + execution-timeout: 35 + compilation-memory-limit: 5500 + execution-memory-limit: 1200 + rollup-merge: + repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT + path: noir-projects/noir-protocol-circuits/crates/rollup-merge + num_runs: 5 + timeout: 300 + compilation-timeout: 1.5 + execution-timeout: 0.01 + compilation-memory-limit: 400 + execution-memory-limit: 400 + rollup-root: + repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT + path: noir-projects/noir-protocol-circuits/crates/rollup-root + num_runs: 5 + timeout: 300 + compilation-timeout: 2 + execution-timeout: 0.6 + compilation-memory-limit: 450 + execution-memory-limit: 400 diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 74967b901c8..63cf0e937d1 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -7,6 +7,23 @@ on: pull_request: jobs: + benchmark-projects-list: + name: Load benchmark projects list + runs-on: ubuntu-22.04 + outputs: + projects: ${{ steps.get_bench_projects.outputs.projects }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build list of projects + id: get_bench_projects + run: | + PROJECTS=$(yq ./.github/benchmark_projects.yml -o json | jq -c '.projects | map(.)') + echo "projects=$PROJECTS" + echo "projects=$PROJECTS" >> $GITHUB_OUTPUT + build-nargo: runs-on: ubuntu-22.04 @@ -259,25 +276,13 @@ jobs: overwrite: true external_repo_compilation_and_execution_report: - needs: [build-nargo] + needs: [build-nargo, benchmark-projects-list] runs-on: ubuntu-22.04 timeout-minutes: 15 strategy: fail-fast: false matrix: - include: - # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts, cannot_execute: true } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner, num_runs: 5 } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail, num_runs: 5 } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset, num_runs: 5 } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private, num_runs: 5 } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public, num_runs: 5 } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-merge, num_runs: 5 } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-root-empty, num_runs: 5, cannot_execute: true } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-root-single-tx, num_runs: 1, flags: "--skip-brillig-constraints-check --skip-underconstrained-check", cannot_execute: true } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-root, num_runs: 1, flags: "--skip-brillig-constraints-check --skip-underconstrained-check" } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-merge, num_runs: 5 } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-root, num_runs: 5 } + project: ${{ fromJson( needs.benchmark-projects-list.outputs.projects )}} name: External repo compilation and execution reports - ${{ matrix.project.repo }}/${{ matrix.project.path }} steps: @@ -322,7 +327,7 @@ jobs: - name: Check compilation time limit run: | TIME=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/compilation_report.json) - TIME_LIMIT=80 + TIME_LIMIT=${{ matrix.project.compilation-timeout }} if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "$TIME_LIMIT"; then # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause. echo "Failing due to compilation exceeding timeout..." @@ -343,7 +348,7 @@ jobs: if: ${{ !matrix.project.cannot_execute }} run: | TIME=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/execution_report.json) - TIME_LIMIT=60 + TIME_LIMIT=${{ matrix.project.execution-timeout }} if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "$TIME_LIMIT"; then # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause. echo "Failing due to execution exceeding timeout..." @@ -388,28 +393,15 @@ jobs: overwrite: true external_repo_memory_report: - needs: [build-nargo] + needs: [build-nargo, benchmark-projects-list] runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: fail-fast: false matrix: - include: - # TODO: Bring this report back under a flag. The `noir-contracts` report takes just under 30 minutes. - # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-merge } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-root-empty, cannot_execute: true } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-root-single-tx, flags: "--skip-brillig-constraints-check --skip-underconstrained-check", cannot_execute: true } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-block-root, flags: "--skip-brillig-constraints-check --skip-underconstrained-check" } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-merge } - - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-root } - - name: External repo memory report - ${{ matrix.project.repo }}/${{ matrix.project.path }} + include: ${{ fromJson( needs.benchmark-projects-list.outputs.projects )}} + + name: External repo memory report - ${{ matrix.repo }}/${{ matrix.path }} steps: - uses: actions/checkout@v4 with: @@ -426,12 +418,12 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - repository: ${{ matrix.project.repo }} + repository: ${{ matrix.repo }} path: test-repo - ref: ${{ matrix.project.ref }} + ref: ${{ matrix.ref }} - name: Generate compilation memory report - working-directory: ./test-repo/${{ matrix.project.path }} + working-directory: ./test-repo/${{ matrix.path }} run: | mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh mv /home/runner/work/noir/noir/scripts/test_programs/parse_memory.sh ./parse_memory.sh @@ -439,12 +431,12 @@ jobs: # Rename the memory report as the execution report is about to write to the same file cp memory_report.json compilation_memory_report.json env: - FLAGS: ${{ matrix.project.flags }} + FLAGS: ${{ matrix.flags }} - name: Check compilation memory limit run: | - MEMORY=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/compilation_memory_report.json) - MEMORY_LIMIT=6000 + MEMORY=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/compilation_memory_report.json) + MEMORY_LIMIT=${{ matrix.compilation-memory-limit }} if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$MEMORY" "$MEMORY_LIMIT"; then # Don't bump this limit without understanding why this has happened and confirming that you're not the cause. echo "Failing due to compilation exceeding memory limit..." @@ -454,16 +446,16 @@ jobs: fi - name: Generate execution memory report - working-directory: ./test-repo/${{ matrix.project.path }} - if: ${{ !matrix.project.cannot_execute }} + working-directory: ./test-repo/${{ matrix.path }} + if: ${{ !matrix.cannot_execute }} run: | ./memory_report.sh 1 1 - name: Check execution memory limit - if: ${{ !matrix.project.cannot_execute }} + if: ${{ !matrix.cannot_execute }} run: | - MEMORY=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/memory_report.json) - MEMORY_LIMIT=1300 + MEMORY=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/memory_report.json) + MEMORY_LIMIT=${{ matrix.execution-memory-limit }} if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$MEMORY" "$MEMORY_LIMIT"; then # Don't bump this limit without understanding why this has happened and confirming that you're not the cause. echo "Failing due to execution exceeding memory limit..." @@ -476,9 +468,9 @@ jobs: id: compilation_mem_report shell: bash run: | - PACKAGE_NAME=${{ matrix.project.path }} + PACKAGE_NAME=${{ matrix.path }} PACKAGE_NAME=$(basename $PACKAGE_NAME) - mv ./test-repo/${{ matrix.project.path }}/compilation_memory_report.json ./memory_report_$PACKAGE_NAME.json + mv ./test-repo/${{ matrix.path }}/compilation_memory_report.json ./memory_report_$PACKAGE_NAME.json echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT - name: Upload compilation memory report @@ -491,11 +483,11 @@ jobs: - name: Move execution report id: execution_mem_report - if: ${{ !matrix.project.cannot_execute }} + if: ${{ !matrix.cannot_execute }} run: | - PACKAGE_NAME=${{ matrix.project.path }} + PACKAGE_NAME=${{ matrix.path }} PACKAGE_NAME=$(basename $PACKAGE_NAME) - mv ./test-repo/${{ matrix.project.path }}/memory_report.json ./memory_report_$PACKAGE_NAME.json + mv ./test-repo/${{ matrix.path }}/memory_report.json ./memory_report_$PACKAGE_NAME.json echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT - name: Upload execution memory report diff --git a/.github/workflows/test-js-packages.yml b/.github/workflows/test-js-packages.yml index 2e0f226b0a2..cfc97d10146 100644 --- a/.github/workflows/test-js-packages.yml +++ b/.github/workflows/test-js-packages.yml @@ -492,9 +492,9 @@ jobs: strategy: fail-fast: false matrix: - project: ${{ fromJson( needs.critical-library-list.outputs.libraries )}} + include: ${{ fromJson( needs.critical-library-list.outputs.libraries )}} - name: Check external repo - ${{ matrix.project.repo }}/${{ matrix.project.path }} + name: Check external repo - ${{ matrix.repo }}/${{ matrix.path }} steps: - name: Checkout uses: actions/checkout@v4 @@ -504,9 +504,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - repository: ${{ matrix.project.repo }} + repository: ${{ matrix.repo }} path: test-repo - ref: ${{ matrix.project.ref }} + ref: ${{ matrix.ref }} - name: Download nargo binary uses: ./noir-repo/.github/actions/download-nargo @@ -520,14 +520,14 @@ jobs: - name: Run nargo test id: test_report - working-directory: ./test-repo/${{ matrix.project.path }} + working-directory: ./test-repo/${{ matrix.path }} run: | - output_file=${{ github.workspace }}/noir-repo/.github/critical_libraries_status/${{ matrix.project.repo }}/${{ matrix.project.path }}.actual.jsonl + output_file=${{ github.workspace }}/noir-repo/.github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.actual.jsonl BEFORE=$SECONDS - nargo test --silence-warnings --skip-brillig-constraints-check --format json ${{ matrix.project.nargo_args }} | tee $output_file + nargo test --silence-warnings --skip-brillig-constraints-check --format json ${{ matrix.nargo_args }} | tee $output_file TIME=$(($SECONDS-$BEFORE)) - NAME=${{ matrix.project.repo }}/${{ matrix.project.path }} + NAME=${{ matrix.repo }}/${{ matrix.path }} # Replace any slashes with underscores NAME=${NAME//\//_} TEST_REPORT_NAME=test_report_$NAME @@ -544,25 +544,25 @@ jobs: - name: Check test time limit run: | - TIME=$(jq '.[0].value' ./test-repo/${{ matrix.project.path }}/${{ steps.test_report.outputs.test_report_name }}.json) - if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "${{ matrix.project.timeout }}"; then + TIME=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/${{ steps.test_report.outputs.test_report_name }}.json) + if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "${{ matrix.timeout }}"; then # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause. echo "Failing due to test suite exceeding timeout..." - echo "Timeout: ${{ matrix.project.timeout }}" + echo "Timeout: ${{ matrix.timeout }}" echo "Test suite took: $TIME". exit 1 fi - name: Compare test results working-directory: ./noir-repo - run: .github/scripts/check_test_results.sh .github/critical_libraries_status/${{ matrix.project.repo }}/${{ matrix.project.path }}.failures.jsonl .github/critical_libraries_status/${{ matrix.project.repo }}/${{ matrix.project.path }}.actual.jsonl + run: .github/scripts/check_test_results.sh .github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.failures.jsonl .github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.actual.jsonl - name: Upload test report - if: ${{ matrix.project.timeout > 10 }} # We want to avoid recording benchmarking for a ton of tiny libraries, these should be covered with aggressive timeouts + if: ${{ matrix.timeout > 10 }} # We want to avoid recording benchmarking for a ton of tiny libraries, these should be covered with aggressive timeouts uses: actions/upload-artifact@v4 with: name: ${{ steps.test_report.outputs.test_report_name }} - path: ./test-repo/${{ matrix.project.path }}/${{ steps.test_report.outputs.test_report_name }}.json + path: ./test-repo/${{ matrix.path }}/${{ steps.test_report.outputs.test_report_name }}.json retention-days: 3 overwrite: true diff --git a/EXTERNAL_NOIR_LIBRARIES.yml b/EXTERNAL_NOIR_LIBRARIES.yml index 2d41e8c5ec5..a0a6acc2d35 100644 --- a/EXTERNAL_NOIR_LIBRARIES.yml +++ b/EXTERNAL_NOIR_LIBRARIES.yml @@ -1,3 +1,4 @@ +define: &AZ_COMMIT 7b7d960d51027b549ec9627ba2822fb87100a022 libraries: noir_check_shuffle: @@ -47,34 +48,42 @@ libraries: timeout: 3 aztec_nr: repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT path: noir-projects/aztec-nr timeout: 60 noir_contracts: repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT path: noir-projects/noir-contracts timeout: 80 blob: repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT path: noir-projects/noir-protocol-circuits/crates/blob timeout: 70 protocol_circuits_parity_lib: repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT path: noir-projects/noir-protocol-circuits/crates/parity-lib timeout: 4 protocol_circuits_private_kernel_lib: repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT path: noir-projects/noir-protocol-circuits/crates/private-kernel-lib timeout: 250 protocol_circuits_reset_kernel_lib: repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT path: noir-projects/noir-protocol-circuits/crates/reset-kernel-lib timeout: 15 protocol_circuits_types: repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT path: noir-projects/noir-protocol-circuits/crates/types timeout: 60 protocol_circuits_rollup_lib: repo: AztecProtocol/aztec-packages + ref: *AZ_COMMIT path: noir-projects/noir-protocol-circuits/crates/rollup-lib timeout: 300 # Use 1 test threads for rollup-lib because each test requires a lot of memory, and multiple ones in parallel exceed the maximum memory limit.