feat: Fast node_modules caching in CI.

.devcontainer/scripts/onCreateCommand.sh

@@ -3,7 +3,7 @@
 TYPE=$1
 NAME=$2
 
-curl -s install.aztec.network | NON_INTERACTIVE=1 bash -s
+curl -s https://install.aztec.network | NON_INTERACTIVE=1 bash -s
 docker compose -f $HOME/.aztec/docker-compose.sandbox.yml pull
 
 if ! grep -q "PXE_URL" ~/.bashrc; then

.test_patterns.yml

@@ -12,6 +12,7 @@ names:
   - adam: &adam "U04BM8H25NJ"
   - alex: &alex "U05QWV669JB"
   - charlie: &charlie "UKUMA5J7K"
+  - grego: &grego "U0689QRCE9L"
   - lasse: &lasse "U03E5SYLY3Z"
   - leila: &leila "UBLTU1NJ3"
   - luke: &luke "U03JYU7AQET"
@@ -36,7 +37,7 @@ tests:
       - *luke
 
   # Sumcheck is failing for some reason
-  - regex: "barretenberg/acir_tests/run_test.sh ram_blowup_regression"
+  - regex: "barretenberg/acir_tests/scripts/run_test.sh ram_blowup_regression"
     skip: true
     owners:
       - *tom
@@ -154,8 +155,12 @@ tests:
     owners:
       - *alex
 
-  # kind tests
+  # boxes
+  - regex: "boxes/scripts/run_test.sh vite"
+    owners:
+      - *grego
 
+  # kind tests
   - regex: "spartan/bootstrap.sh test-kind-4epochs"
     owners:
       - *leila

aztec-up/bin/aztec-nargo

@@ -15,6 +15,7 @@ if [ -t 0 ]; then
 fi
 
 docker run ${args:-} \
+  --user $(id -u):$(id -g) \
   -v $HOME:$HOME \
   -e HOME=$HOME \
   --workdir="$PWD" \

aztec-up/bootstrap.sh

@@ -11,6 +11,18 @@ function build_dind_image {
   denoise "docker build -t aztecprotocol/dind ."
 }
 
+function update_manifest {
+  # We update the manifest to point to the latest arch specific images, pushed above.
+  local image=aztecprotocol/dind:latest
+  # Remove any old local manifest if present.
+  docker manifest rm $image || true
+  # Create new manifest and push.
+  docker manifest create $image \
+    --amend aztecprotocol/dind:latest-amd64 \
+    --amend aztecprotocol/dind:latest-arm64
+  docker manifest push $image
+}
+
 function test_cmds {
   echo "$hash aztec-up/scripts/run_test.sh basic_install"
   echo "$hash aztec-up/scripts/run_test.sh counter_contract"
@@ -52,9 +64,8 @@ function release {
 
 case "$cmd" in
   ""|"full"|"fast")
-    build_dind_image
     ;;
-  test_cmds|test|release)
+  test_cmds|test|release|build_dind_image|update_manifest)
     $cmd
     ;;
   *)

barretenberg/acir_tests/.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

barretenberg/acir_tests/bootstrap.sh

@@ -3,6 +3,7 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap
 
 cmd=${1:-}
 export CRS_PATH=$HOME/.bb-crs
+export bb=$(realpath ../cpp/build/bin/bb)
 
 tests_tar=barretenberg-acir-tests-$(cache_content_hash \
     ../../noir/.rebuild_patterns \
@@ -18,53 +19,64 @@ tests_hash=$(cache_content_hash \
     ../ts/.rebuild_patterns)
 
 # Generate inputs for a given recursively verifying program.
-function run_proof_generation() {
-    local program=$1
-    local outdir=$(mktemp -d)
-    trap "rm -rf $outdir" EXIT
-    local adjustment=16
-    local ipa_accumulation_flag=""
-
-    # Adjust settings based on program type
-    if [[ $program == *"rollup"* ]]; then
-        adjustment=26
-        ipa_accumulation_flag="--ipa_accumulation"
-    fi
-    local prove_cmd="$bb prove --scheme ultra_honk --init_kzg_accumulator $ipa_accumulation_flag --output_format fields --write_vk -o $outdir -b ./target/program.json -w ./target/witness.gz"
-    echo_stderr "$prove_cmd"
-    dump_fail "$prove_cmd"
-
-    local vk_fields=$(cat "$outdir/vk_fields.json")
-    local proof_fields=$(cat "$outdir/proof_fields.json")
-    local num_inner_public_inputs=$(( 16#$(echo "$vk_fields" | jq -r '.[1] | ltrimstr("0x")') - adjustment ))
-
-    echo "num_inner_public_inputs for $program = $num_inner_public_inputs"
-
-    generate_toml "$program" "$vk_fields" "$proof_fields" "$num_inner_public_inputs"
+function run_proof_generation {
+  local program=$1
+  local outdir=$(mktemp -d)
+  trap "rm -rf $outdir" EXIT
+  local adjustment=16
+  local ipa_accumulation_flag=""
+
+  cd ./acir_tests/assert_statement
+
+  # Adjust settings based on program type
+  if [[ $program == *"rollup"* ]]; then
+      adjustment=26
+      ipa_accumulation_flag="--ipa_accumulation"
+  fi
+  local prove_cmd="$bb prove --scheme ultra_honk --init_kzg_accumulator $ipa_accumulation_flag --output_format fields --write_vk -o $outdir -b ./target/program.json -w ./target/witness.gz"
+  echo_stderr "$prove_cmd"
+  dump_fail "$prove_cmd"
+
+  local vk_fields=$(cat "$outdir/vk_fields.json")
+  local proof_fields=$(cat "$outdir/proof_fields.json")
+  local num_inner_public_inputs=$(( 16#$(echo "$vk_fields" | jq -r '.[1] | ltrimstr("0x")') - adjustment ))
+
+  echo "num_inner_public_inputs for $program = $num_inner_public_inputs"
+
+  generate_toml "$program" "$vk_fields" "$proof_fields" "$num_inner_public_inputs"
 }
 
-function generate_toml() {
-    local program=$1
-    local vk_fields=$2
-    local proof_fields=$3
-    local num_inner_public_inputs=$4
-    local output_file="../$program/Prover.toml"
-    local key_hash="0x0000000000000000000000000000000000000000000000000000000000000000"
-
-    jq -nr \
-        --arg key_hash "$key_hash" \
-        --argjson vkf "$vk_fields" \
-        --argjson prooff "$proof_fields" \
-        --argjson num_inner_public_inputs "$num_inner_public_inputs" \
-        '[
-          "key_hash = \($key_hash)",
-          "proof = [\($prooff | .[$num_inner_public_inputs:] | map("\"" + . + "\"") | join(", "))]",
-          "public_inputs = [\($prooff | .[:$num_inner_public_inputs] | map("\"" + . + "\"") | join(", "))]",
-          "verification_key = [\($vkf | map("\"" + . + "\"") | join(", "))]"
-          '"$( [[ $program == *"double"* ]] && echo ',"proof_b = [\($prooff | .[$num_inner_public_inputs:] | map("\"" + . + "\"") | join(", "))]"' )"'
-        ] | join("\n")' > "$output_file"
+function generate_toml {
+  local program=$1
+  local vk_fields=$2
+  local proof_fields=$3
+  local num_inner_public_inputs=$4
+  local output_file="../$program/Prover.toml"
+  local key_hash="0x0000000000000000000000000000000000000000000000000000000000000000"
+
+  jq -nr \
+      --arg key_hash "$key_hash" \
+      --argjson vkf "$vk_fields" \
+      --argjson prooff "$proof_fields" \
+      --argjson num_inner_public_inputs "$num_inner_public_inputs" \
+      '[
+        "key_hash = \($key_hash)",
+        "proof = [\($prooff | .[$num_inner_public_inputs:] | map("\"" + . + "\"") | join(", "))]",
+        "public_inputs = [\($prooff | .[:$num_inner_public_inputs] | map("\"" + . + "\"") | join(", "))]",
+        "verification_key = [\($vkf | map("\"" + . + "\"") | join(", "))]"
+        '"$( [[ $program == *"double"* ]] && echo ',"proof_b = [\($prooff | .[$num_inner_public_inputs:] | map("\"" + . + "\"") | join(", "))]"' )"'
+      ] | join("\n")' > "$output_file"
 }
 
+function regenerate_recursive_inputs {
+  local program=$1
+  # Compile the assert_statement test as it's used for the recursive tests.
+  COMPILE=2 ./scripts/run_test.sh assert_statement
+  parallel 'run_proof_generation {}' ::: $(ls internal_test_programs)
+}
+
+export -f regenerate_recursive_inputs run_proof_generation generate_toml
+
 function build {
   echo_header "acir_tests build"
 
@@ -74,36 +86,31 @@ function build {
     cp -R ../../noir/noir-repo/test_programs/execution_success acir_tests
     # Running these requires extra gluecode so they're skipped.
     rm -rf acir_tests/{diamond_deps_0,workspace,workspace_default_member,regression_7323}
-
-    # COMPILE=2 only compiles the test.
-    denoise "parallel --joblog joblog.txt --line-buffered 'COMPILE=2 ./run_test.sh \$(basename {})' ::: ./acir_tests/*"
-
+    # Merge the internal test programs with the acir tests.
     cp -R ./internal_test_programs/* acir_tests
-    echo "Regenerating verify_honk_proof, double_verify_honk_proof, verify_rollup_honk_proof recursive inputs."
-    local bb=$(realpath ../cpp/build/bin/bb)
-    cd ./acir_tests/assert_statement
-    for program in verify_honk_proof double_verify_honk_proof verify_rollup_honk_proof; do
-      echo $program
-      run_proof_generation "$program"
-    done
-    cd ../..
 
+    # Generates the Prover.toml files for the recursive tests from the assert_statement test.
+    denoise regenerate_recursive_inputs
 
-    local internal_tests=($(ls -d internal_test_programs/* | xargs -n1 basename))
-    local internal_tests_string=$(echo ${internal_tests[@]} | awk -v OFS="," '{$1=$1;print}')
-    denoise "parallel --joblog joblog.txt --line-buffered 'COMPILE=2 ./run_test.sh \$(basename {})' ::: ./acir_tests/{$internal_tests_string}"
+    # COMPILE=2 only compiles the test.
+    denoise "parallel --joblog joblog.txt --line-buffered 'COMPILE=2 ./scripts/run_test.sh \$(basename {})' ::: ./acir_tests/*"
 
     cache_upload $tests_tar acir_tests
   fi
 
+  npm_install_deps
+  # TODO: Check if still needed.
+  denoise "cd browser-test-app && yarn add --dev @aztec/bb.js@portal:../../ts"
+
   # TODO: Revisit. Update yarn.lock so it can be committed.
   # Be lenient about bb.js hash changing, even if we try to minimize the occurrences.
-  denoise "cd browser-test-app && yarn add --dev @aztec/bb.js@portal:../../ts && yarn"
-  denoise "cd headless-test && yarn"
-  denoise "cd sol-test && yarn"
+  # denoise "cd browser-test-app && yarn add --dev @aztec/bb.js@portal:../../ts && yarn"
+  # denoise "cd headless-test && yarn"
+  # denoise "cd sol-test && yarn"
+
   # TODO: Revist. The md5sum of everything is the same after each yarn call.
   # Yet seemingly yarn's content hash will churn unless we reset timestamps
-  find {headless-test,browser-test-app} -exec touch -t 197001010000 {} + 2>/dev/null || true
+  # find {headless-test,browser-test-app} -exec touch -t 197001010000 {} + 2>/dev/null || true
 
   denoise "cd browser-test-app && yarn build"
 }
@@ -127,22 +134,22 @@ function test_cmds_internal {
   local honk_tests=$(find ./acir_tests -maxdepth 1 -mindepth 1 -type d | \
     grep -vE 'single_verify_proof|double_verify_proof|double_verify_nested_proof|verify_rollup_honk_proof|fold')
 
-  local run_test=$(realpath --relative-to=$root ./run_test.sh)
-  local run_test_browser=$(realpath --relative-to=$root ./run_test_browser.sh)
+  local run_test=$(realpath --relative-to=$root ./scripts/run_test.sh)
+  local run_test_browser=$(realpath --relative-to=$root ./scripts/run_test_browser.sh)
   local bbjs_bin="../ts/dest/node/main.js"
 
   # barretenberg-acir-tests-sol:
-  echo FLOW=sol $run_test assert_statement
-  echo FLOW=sol $run_test double_verify_proof
-  echo FLOW=sol $run_test double_verify_nested_proof
-  echo FLOW=sol_honk $run_test assert_statement
-  echo FLOW=sol_honk $run_test 1_mul
-  echo FLOW=sol_honk $run_test slices
-  echo FLOW=sol_honk $run_test verify_honk_proof
-  echo FLOW=sol_honk_zk $run_test assert_statement
-  echo FLOW=sol_honk_zk $run_test 1_mul
-  echo FLOW=sol_honk_zk $run_test slices
-  echo FLOW=sol_honk_zk $run_test verify_honk_proof
+  echo "docker_isolate 'FLOW=sol $run_test assert_statement'"
+  echo "docker_isolate 'FLOW=sol $run_test double_verify_proof'"
+  echo "docker_isolate 'FLOW=sol $run_test double_verify_nested_proof'"
+  echo "docker_isolate 'FLOW=sol_honk $run_test assert_statement'"
+  echo "docker_isolate 'FLOW=sol_honk $run_test 1_mul'"
+  echo "docker_isolate 'FLOW=sol_honk $run_test slices'"
+  echo "docker_isolate 'FLOW=sol_honk $run_test verify_honk_proof'"
+  echo "docker_isolate 'FLOW=sol_honk_zk $run_test assert_statement'"
+  echo "docker_isolate 'FLOW=sol_honk_zk $run_test 1_mul'"
+  echo "docker_isolate 'FLOW=sol_honk_zk $run_test slices'"
+  echo "docker_isolate 'FLOW=sol_honk_zk $run_test verify_honk_proof'"
 
   # barretenberg-acir-tests-bb.js:
   # Browser tests.
@@ -183,7 +190,8 @@ function test_cmds_internal {
 }
 
 function ultra_honk_wasm_memory {
-  VERBOSE=1 BIN=../ts/dest/node/main.js SYS=ultra_honk_deprecated FLOW=prove_then_verify ./run_test.sh verify_honk_proof &> ./bench-out/ultra_honk_rec_wasm_memory.txt
+  VERBOSE=1 BIN=../ts/dest/node/main.js SYS=ultra_honk_deprecated FLOW=prove_then_verify \
+    ./scripts/run_test.sh verify_honk_proof &> ./bench-out/ultra_honk_rec_wasm_memory.txt
 }
 
 function run_benchmark {

barretenberg/acir_tests/headless-test/bb.js.browser

@@ -1,4 +1,4 @@
 #!/bin/sh
 SCRIPT_PATH=$(dirname $(realpath $0))
 export TS_NODE_PROJECT="$SCRIPT_PATH/tsconfig.json"
-NODE_OPTIONS="--loader $SCRIPT_PATH/node_modules/ts-node/esm/transpile-only.mjs --no-warnings" node $SCRIPT_PATH/src/index.ts $@
+NODE_OPTIONS="--loader $SCRIPT_PATH/../node_modules/ts-node/esm/transpile-only.mjs --no-warnings" node $SCRIPT_PATH/src/index.ts $@