diff --git a/.github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl b/.github/critical_libraries_status/teddav/tdd.nr/.failures.jsonl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/.github/scripts/run-external-repo-tests.sh b/.github/scripts/run-external-repo-tests.sh new file mode 100755 index 00000000000..1725e4b529a --- /dev/null +++ b/.github/scripts/run-external-repo-tests.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -eu + +NARGO=${NARGO:-nargo} + +# Make output paths absolute to ensure they don't get written inside of temp directory. +OUTPUT_FILE=$(realpath -m $OUTPUT_FILE) +BENCHMARK_FILE=$(realpath -m $BENCHMARK_FILE) + +mkdir -p $(dirname $OUTPUT_FILE) +mkdir -p $(dirname $BENCHMARK_FILE) + +if [ -z "${CI:-}" ]; then + # Sadly we cannot use depth=1 clones here as we need to be able to checkout + # commit hashes as well as branches/releases + TMP_DIR=$(mktemp -d) + trap "rm -rf $TMP_DIR" EXIT + git clone $REPO $TMP_DIR + git -C $TMP_DIR -c advice.detachedHead=false checkout $TAG +fi + +REPO_DIR=${REPO_DIR:-$TMP_DIR} + +cd $REPO_DIR/$PROJECT_PATH + +set +e +sed -i '/^compiler_version/d' {Nargo.toml,./**/Nargo.toml} +set -e + +BEFORE=$SECONDS +$NARGO test --silence-warnings --skip-brillig-constraints-check --format json $NARGO_ARGS | tee $OUTPUT_FILE +TIME=$(($SECONDS-$BEFORE)) + +if [ ! -s $OUTPUT_FILE ]; then +# The file is empty so we delete it to signal that `nargo test` failed before it could run any tests +rm -f $OUTPUT_FILE +fi + +jq --null-input "[{ name: \"$NAME\", value: (\"$TIME\" | tonumber), unit: \"s\" }]" > $BENCHMARK_FILE \ No newline at end of file diff --git a/.github/workflows/test-js-packages.yml b/.github/workflows/test-js-packages.yml index 67d79c28975..b3c1faefa03 100644 --- a/.github/workflows/test-js-packages.yml +++ b/.github/workflows/test-js-packages.yml @@ -541,6 +541,9 @@ jobs: include: ${{ fromJson( needs.critical-library-list.outputs.libraries )}} name: Check external repo - ${{ matrix.repo }}/${{ matrix.path }} + env: + OUTPUT_FILE: ${{ github.workspace }}/.github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.actual.jsonl + steps: - name: Matrix Info run: | @@ -551,8 +554,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - with: - path: noir-repo - name: Checkout uses: actions/checkout@v4 @@ -562,54 +563,47 @@ jobs: ref: ${{ matrix.ref }} - name: Download nargo binary - uses: ./noir-repo/.github/actions/download-nargo + uses: ./.github/actions/download-nargo - - name: Remove requirements on compiler version - working-directory: ./test-repo + - name: Generate paths + id: test_report_name run: | - # Github actions seems to not expand "**" in globs by default. - shopt -s globstar - sed -i '/^compiler_version/d' ./**/Nargo.toml - - - name: Run nargo test - id: test_report - working-directory: ./test-repo/${{ matrix.path }} - run: | - BEFORE=$SECONDS - nargo test --silence-warnings --skip-brillig-constraints-check --format json ${{ matrix.nargo_args }} | tee $OUTPUT_FILE - TIME=$(($SECONDS-$BEFORE)) - - if [ ! -s $OUTPUT_FILE ]; then - # The file is empty so we delete it to signal that `nargo test` failed before it could run any tests - rm -f $OUTPUT_FILE - fi - - NAME=${{ matrix.repo }}/${{ matrix.path }} # Replace any slashes with underscores NAME=${NAME//\//_} TEST_REPORT_NAME=test_report_$NAME echo "test_report_name=$TEST_REPORT_NAME" >> $GITHUB_OUTPUT + env: + NAME: ${{ matrix.repo }}/${{ matrix.path }} - jq --null-input "[{ name: \"$NAME\", value: (\"$TIME\" | tonumber), unit: \"s\" }]" > $TEST_REPORT_NAME.json + - name: Run nargo test + run: | + # Github actions seems to not expand "**" in globs by default. + shopt -s globstar + + .github/scripts/run-external-repo-tests.sh env: + REPO_DIR: ./test-repo + PROJECT_PATH: ${{ matrix.path }} NARGO_IGNORE_TEST_FAILURES_FROM_FOREIGN_CALLS: true - OUTPUT_FILE: ${{ github.workspace }}/noir-repo/.github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.actual.jsonl + NARGO_ARGS: ${{ matrix.nargo_args }} + NAME: ${{ steps.test_report_name.outputs.test_report_name }} + BENCHMARK_FILE: ${{ steps.test_report_name.outputs.test_report_name }}.json + - name: Compare test results - working-directory: ./noir-repo run: .github/scripts/check_test_results.sh .github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.failures.jsonl .github/critical_libraries_status/${{ matrix.repo }}/${{ matrix.path }}.actual.jsonl - name: Upload test report if: ${{ matrix.timeout > 10 }} # We want to avoid recording benchmarking for a ton of tiny libraries, these should be covered with aggressive timeouts uses: actions/upload-artifact@v4 with: - name: ${{ steps.test_report.outputs.test_report_name }} - path: ./test-repo/${{ matrix.path }}/${{ steps.test_report.outputs.test_report_name }}.json + name: ${{ steps.test_report_name.outputs.test_report_name }} + path: ${{ steps.test_report_name.outputs.test_report_name }}.json retention-days: 3 overwrite: true - name: Check test time limit run: | - TIME=$(jq '.[0].value' ./test-repo/${{ matrix.path }}/${{ steps.test_report.outputs.test_report_name }}.json) + TIME=$(jq '.[0].value' ${{ steps.test_report_name.outputs.test_report_name }}.json) if awk 'BEGIN{exit !(ARGV[1]>ARGV[2])}' "$TIME" "${{ matrix.timeout }}"; then # Don't bump this timeout without understanding why this has happened and confirming that you're not the cause. echo "Failing due to test suite exceeding timeout..." diff --git a/EXTERNAL_NOIR_LIBRARIES.yml b/EXTERNAL_NOIR_LIBRARIES.yml index 637138027c5..fcfbb4a3aad 100644 --- a/EXTERNAL_NOIR_LIBRARIES.yml +++ b/EXTERNAL_NOIR_LIBRARIES.yml @@ -144,3 +144,9 @@ libraries: path: lib timeout: 20 critical: false + tdd: + repo: teddav/tdd.nr + ref: 4bebfb361f2e5175d23376645310eecca96749a5 + timeout: 20 + critical: false +