From 2ab29931bfc8852abb34c087f4bbf3a4bee0eec4 Mon Sep 17 00:00:00 2001 From: TomAFrench Date: Wed, 9 Apr 2025 20:10:35 +0000 Subject: [PATCH 1/6] chore: add `teddav/tdd.nr` to external repo checks --- .../teddav/tdd.nr/.failures.jsonl | 2 + .github/scripts/run-external-repo-tests.sh | 39 ++++++++++++++ .github/workflows/test-js-packages.yml | 51 +++++++++---------- EXTERNAL_NOIR_LIBRARIES.yml | 24 +++++++++ 4 files changed, 88 insertions(+), 28 deletions(-) create mode 100644 .github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl create mode 100755 .github/scripts/run-external-repo-tests.sh diff --git a/.github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl b/.github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl new file mode 100644 index 00000000000..92e5d190b04 --- /dev/null +++ b/.github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl @@ -0,0 +1,2 @@ +{"suite":"tdd.nr","name":"tests::test_public_keys"} +{"suite":"tdd.nr","name":"tests::test_verify_signature"} \ No newline at end of file diff --git a/.github/scripts/run-external-repo-tests.sh b/.github/scripts/run-external-repo-tests.sh new file mode 100755 index 00000000000..1725e4b529a --- /dev/null +++ b/.github/scripts/run-external-repo-tests.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -eu + +NARGO=${NARGO:-nargo} + +# Make output paths absolute to ensure they don't get written inside of temp directory. +OUTPUT_FILE=$(realpath -m $OUTPUT_FILE) +BENCHMARK_FILE=$(realpath -m $BENCHMARK_FILE) + +mkdir -p $(dirname $OUTPUT_FILE) +mkdir -p $(dirname $BENCHMARK_FILE) + +if [ -z "${CI:-}" ]; then + # Sadly we cannot use depth=1 clones here as we need to be able to checkout + # commit hashes as well as branches/releases + TMP_DIR=$(mktemp -d) + trap "rm -rf $TMP_DIR" EXIT + git clone $REPO $TMP_DIR + git -C $TMP_DIR -c advice.detachedHead=false checkout $TAG +fi + +REPO_DIR=${REPO_DIR:-$TMP_DIR} + +cd $REPO_DIR/$PROJECT_PATH + +set +e +sed -i '/^compiler_version/d' {Nargo.toml,./**/Nargo.toml} +set -e + +BEFORE=$SECONDS +$NARGO test --silence-warnings --skip-brillig-constraints-check --format json $NARGO_ARGS | tee $OUTPUT_FILE +TIME=$(($SECONDS-$BEFORE)) + +if [ ! -s $OUTPUT_FILE ]; then +# The file is empty so we delete it to signal that `nargo test` failed before it could run any tests +rm -f $OUTPUT_FILE +fi + +jq --null-input "[{ name: \"$NAME\", value: (\"$TIME\" | tonumber), unit: \"s\" }]" > $BENCHMARK_FILE \ No newline at end of file diff --git a/.github/workflows/test-js-packages.yml b/.github/workflows/test-js-packages.yml index 67d79c28975..99209d9370e 100644 --- a/.github/workflows/test-js-packages.yml +++ b/.github/workflows/test-js-packages.yml @@ -541,6 +541,9 @@ jobs: include: ${{ fromJson( needs.critical-library-list.outputs.libraries )}} name: Check external repo - ${{ matrix.repo }}/${{ matrix.path }} + env: + OUTPUT_FILE: ${{ github.workspace }}/.github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.actual.jsonl + steps: - name: Matrix Info run: | @@ -551,8 +554,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - with: - path: noir-repo - name: Checkout uses: actions/checkout@v4 @@ -562,40 +563,34 @@ jobs: ref: ${{ matrix.ref }} - name: Download nargo binary - uses: ./noir-repo/.github/actions/download-nargo + uses: ./.github/actions/download-nargo - - name: Remove requirements on compiler version - working-directory: ./test-repo + - name: Generate paths + id: test_report_name run: | - # Github actions seems to not expand "**" in globs by default. - shopt -s globstar - sed -i '/^compiler_version/d' ./**/Nargo.toml - - - name: Run nargo test - id: test_report - working-directory: ./test-repo/${{ matrix.path }} - run: | - BEFORE=$SECONDS - nargo test --silence-warnings --skip-brillig-constraints-check --format json ${{ matrix.nargo_args }} | tee $OUTPUT_FILE - TIME=$(($SECONDS-$BEFORE)) - - if [ ! -s $OUTPUT_FILE ]; then - # The file is empty so we delete it to signal that `nargo test` failed before it could run any tests - rm -f $OUTPUT_FILE - fi - - NAME=${{ matrix.repo }}/${{ matrix.path }} # Replace any slashes with underscores NAME=${NAME//\//_} TEST_REPORT_NAME=test_report_$NAME echo "test_report_name=$TEST_REPORT_NAME" >> $GITHUB_OUTPUT + env: + NAME: ${{ matrix.repo }}/${{ matrix.path }} + + - name: Run nargo test + id: test_report + run: | + # Github actions seems to not expand "**" in globs by default. + shopt -s globstar - jq --null-input "[{ name: \"$NAME\", value: (\"$TIME\" | tonumber), unit: \"s\" }]" > $TEST_REPORT_NAME.json + .github/scripts/run-external-repo-tests.sh env: + REPO_DIR: ./test-repo + PROJECT_PATH: ${{ matrix.path }} NARGO_IGNORE_TEST_FAILURES_FROM_FOREIGN_CALLS: true - OUTPUT_FILE: ${{ github.workspace }}/noir-repo/.github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.actual.jsonl + NARGO_ARGS: ${{ matrix.nargo_args }} + NAME: ${{ steps.test_report_name.outputs.test_report_name }} + BENCHMARK_FILE: ${{ steps.test_report_name.outputs.test_report_name }}.json + - name: Compare test results - working-directory: ./noir-repo run: .github/scripts/check_test_results.sh .github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.failures.jsonl .github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.actual.jsonl - name: Upload test report @@ -603,13 +598,13 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ steps.test_report.outputs.test_report_name }} - path: ./test-repo/${{ matrix.path }}/${{ steps.test_report.outputs.test_report_name }}.json + path: ${{ steps.test_report_name.outputs.test_report_name }}.json retention-days: 3 overwrite: true - name: Check test time limit run: | - TIME=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/${{ steps.test_report.outputs.test_report_name }}.json) + TIME=$(jq '.[0].value' $OUTPUT_FILE) if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "${{ matrix.timeout }}"; then # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause. echo "Failing due to test suite exceeding timeout..." diff --git a/EXTERNAL_NOIR_LIBRARIES.yml b/EXTERNAL_NOIR_LIBRARIES.yml index 637138027c5..f3a4d93b4c2 100644 --- a/EXTERNAL_NOIR_LIBRARIES.yml +++ b/EXTERNAL_NOIR_LIBRARIES.yml @@ -2,74 +2,92 @@ define: &AZ_COMMIT c199f2c4ebd3e1045ebab317e1e5f76a0d9a2627 libraries: noir_check_shuffle: repo: noir-lang/noir_check_shuffle + ref: master timeout: 2 critical: true ec: repo: noir-lang/ec + ref: master timeout: 2 critical: true eddsa: repo: noir-lang/eddsa + ref: master timeout: 3 critical: true mimc: repo: noir-lang/mimc + ref: master timeout: 2 critical: true schnorr: repo: noir-lang/schnorr + ref: master timeout: 2 critical: true noir_sort: repo: noir-lang/noir_sort + ref: main timeout: 2 critical: true noir-edwards: repo: noir-lang/noir-edwards + ref: main timeout: 2 critical: true noir-bignum: repo: noir-lang/noir-bignum + ref: main timeout: 90 critical: true noir_bigcurve: repo: noir-lang/noir_bigcurve + ref: main timeout: 250 critical: true noir_base64: repo: noir-lang/noir_base64 + ref: main timeout: 5 critical: true noir_string_search: repo: noir-lang/noir_string_search + ref: master timeout: 2 critical: true sparse_array: repo: noir-lang/sparse_array + ref: master timeout: 2 critical: true noir_rsa: repo: noir-lang/noir_rsa + ref: main timeout: 4 critical: true noir_json_parser: repo: noir-lang/noir_json_parser + ref: main timeout: 12 critical: true sha256: repo: noir-lang/sha256 + ref: main timeout: 3 critical: true sha512: repo: noir-lang/sha512 + ref: main timeout: 30 critical: true keccak256: repo: noir-lang/keccak256 + ref: main timeout: 3 critical: true poseidon: repo: noir-lang/poseidon + ref: master timeout: 3 critical: true aztec_nr: @@ -144,3 +162,9 @@ libraries: path: lib timeout: 20 critical: false + tdd: + repo: teddav/tdd.nr + ref: 03b5c2b02b6f7ffdc8735e0ae104abc26b073541 + timeout: 20 + critical: false + From e2cab894fc035b8a676f56a6445df74a8a54cf85 Mon Sep 17 00:00:00 2001 From: TomAFrench Date: Wed, 9 Apr 2025 21:20:58 +0000 Subject: [PATCH 2/6] . --- .github/workflows/test-js-packages.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-js-packages.yml b/.github/workflows/test-js-packages.yml index 99209d9370e..a47a8ad687d 100644 --- a/.github/workflows/test-js-packages.yml +++ b/.github/workflows/test-js-packages.yml @@ -576,7 +576,6 @@ jobs: NAME: ${{ matrix.repo }}/${{ matrix.path }} - name: Run nargo test - id: test_report run: | # Github actions seems to not expand "**" in globs by default. shopt -s globstar @@ -597,7 +596,7 @@ jobs: if: ${{ matrix.timeout > 10 }} # We want to avoid recording benchmarking for a ton of tiny libraries, these should be covered with aggressive timeouts uses: actions/upload-artifact@v4 with: - name: ${{ steps.test_report.outputs.test_report_name }} + name: ${{ steps.test_report_name.outputs.test_report_name }} path: ${{ steps.test_report_name.outputs.test_report_name }}.json retention-days: 3 overwrite: true From 02230f3c997df98c54afec10d8e10801a537aad5 Mon Sep 17 00:00:00 2001 From: TomAFrench Date: Wed, 9 Apr 2025 21:25:32 +0000 Subject: [PATCH 3/6] . --- .github/workflows/test-js-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-js-packages.yml b/.github/workflows/test-js-packages.yml index a47a8ad687d..b3c1faefa03 100644 --- a/.github/workflows/test-js-packages.yml +++ b/.github/workflows/test-js-packages.yml @@ -603,7 +603,7 @@ jobs: - name: Check test time limit run: | - TIME=$(jq '.[0].value' $OUTPUT_FILE) + TIME=$(jq '.[0].value' ${{ steps.test_report_name.outputs.test_report_name }}.json) if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "${{ matrix.timeout }}"; then # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause. echo "Failing due to test suite exceeding timeout..." From aae9c0702eeb0e4586fe553bf24a043ae953da66 Mon Sep 17 00:00:00 2001 From: TomAFrench Date: Wed, 9 Apr 2025 21:30:41 +0000 Subject: [PATCH 4/6] . --- .github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl b/.github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl index 92e5d190b04..86a43b46756 100644 --- a/.github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl +++ b/.github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl @@ -1,2 +1,2 @@ {"suite":"tdd.nr","name":"tests::test_public_keys"} -{"suite":"tdd.nr","name":"tests::test_verify_signature"} \ No newline at end of file +{"suite":"tdd.nr","name":"tests::test_verify_signature"} From 36b5655febc61a1a03814e5105fea8b52c5a0b6f Mon Sep 17 00:00:00 2001 From: TomAFrench Date: Wed, 9 Apr 2025 21:38:37 +0000 Subject: [PATCH 5/6] . --- EXTERNAL_NOIR_LIBRARIES.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/EXTERNAL_NOIR_LIBRARIES.yml b/EXTERNAL_NOIR_LIBRARIES.yml index f3a4d93b4c2..bdfcf84964c 100644 --- a/EXTERNAL_NOIR_LIBRARIES.yml +++ b/EXTERNAL_NOIR_LIBRARIES.yml @@ -2,92 +2,74 @@ define: &AZ_COMMIT c199f2c4ebd3e1045ebab317e1e5f76a0d9a2627 libraries: noir_check_shuffle: repo: noir-lang/noir_check_shuffle - ref: master timeout: 2 critical: true ec: repo: noir-lang/ec - ref: master timeout: 2 critical: true eddsa: repo: noir-lang/eddsa - ref: master timeout: 3 critical: true mimc: repo: noir-lang/mimc - ref: master timeout: 2 critical: true schnorr: repo: noir-lang/schnorr - ref: master timeout: 2 critical: true noir_sort: repo: noir-lang/noir_sort - ref: main timeout: 2 critical: true noir-edwards: repo: noir-lang/noir-edwards - ref: main timeout: 2 critical: true noir-bignum: repo: noir-lang/noir-bignum - ref: main timeout: 90 critical: true noir_bigcurve: repo: noir-lang/noir_bigcurve - ref: main timeout: 250 critical: true noir_base64: repo: noir-lang/noir_base64 - ref: main timeout: 5 critical: true noir_string_search: repo: noir-lang/noir_string_search - ref: master timeout: 2 critical: true sparse_array: repo: noir-lang/sparse_array - ref: master timeout: 2 critical: true noir_rsa: repo: noir-lang/noir_rsa - ref: main timeout: 4 critical: true noir_json_parser: repo: noir-lang/noir_json_parser - ref: main timeout: 12 critical: true sha256: repo: noir-lang/sha256 - ref: main timeout: 3 critical: true sha512: repo: noir-lang/sha512 - ref: main timeout: 30 critical: true keccak256: repo: noir-lang/keccak256 - ref: main timeout: 3 critical: true poseidon: repo: noir-lang/poseidon - ref: master timeout: 3 critical: true aztec_nr: From f235054396bdc935ec2ef350636dc27ce9d69800 Mon Sep 17 00:00:00 2001 From: TomAFrench Date: Thu, 10 Apr 2025 10:29:50 +0000 Subject: [PATCH 6/6] . --- .github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl | 2 -- EXTERNAL_NOIR_LIBRARIES.yml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl b/.github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl index 86a43b46756..e69de29bb2d 100644 --- a/.github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl +++ b/.github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl @@ -1,2 +0,0 @@ -{"suite":"tdd.nr","name":"tests::test_public_keys"} -{"suite":"tdd.nr","name":"tests::test_verify_signature"} diff --git a/EXTERNAL_NOIR_LIBRARIES.yml b/EXTERNAL_NOIR_LIBRARIES.yml index bdfcf84964c..fcfbb4a3aad 100644 --- a/EXTERNAL_NOIR_LIBRARIES.yml +++ b/EXTERNAL_NOIR_LIBRARIES.yml @@ -146,7 +146,7 @@ libraries: critical: false tdd: repo: teddav/tdd.nr - ref: 03b5c2b02b6f7ffdc8735e0ae104abc26b073541 + ref: 4bebfb361f2e5175d23376645310eecca96749a5 timeout: 20 critical: false