feat: merge-train/ci

.github/ci3.sh

@@ -10,7 +10,7 @@ set -euo pipefail
 CI_MODE="${1:?CI_MODE must be provided as first argument}"
 shift
 
-NO_CD=1 source $(git rev-parse --show-toplevel)/ci3/source
+NO_CD=1 source $(git rev-parse --show-toplevel)/ci3/source_base
 
 function setup_environment {
   echo_header "Setup"

.github/ci3_labels_to_env.sh

@@ -50,8 +50,8 @@ function main {
       if [ -n "$pr_number" ]; then
         local head_branch
         head_branch=$(GH_TOKEN="$GITHUB_TOKEN" gh pr view "$pr_number" --json headRefName -q '.headRefName' 2>/dev/null || true)
-        if [ "$head_branch" == "merge-train/spartan" ]; then
-          echo "Merge-train/spartan PR detected, using merge-queue-heavy mode" >&2
+        if [ "$head_branch" == "merge-train/spartan" ] || [ "$head_branch" == "merge-train/ci" ]; then
+          echo "Merge-train PR ($head_branch) detected, using merge-queue-heavy mode" >&2
           ci_mode="merge-queue-heavy"
         fi
       fi

.github/local_workflow.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Runs a github workflow locally.
 #

.github/workflows/merge-train-create-pr.yml

@@ -47,7 +47,7 @@ jobs:
 
               # Create PR with ci-no-squash label
               labels="ci-no-squash"
-              if [[ "$branch" == "merge-train/spartan" ]]; then
+              if [[ "$branch" == "merge-train/spartan" || "$branch" == "merge-train/ci" ]]; then
                 labels="$labels,ci-full-no-test-cache"
               fi
               gh pr create --base "$base_branch" --head "$branch" \

Makefile

@@ -188,7 +188,7 @@ bb-docs:
 	$(call build,$@,barretenberg/docs)
 
 # BB Solidity - Solidity verifier contracts
-bb-sol: bb-cpp-native
+bb-sol: bb-cpp-native bb-crs
 	$(call build,$@,barretenberg/sol)
 
 #==============================================================================

aztec-up/bootstrap.sh

@@ -4,6 +4,12 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap
 hash=$(hash_str $(cache_content_hash ^aztec-up/) $(../yarn-project/bootstrap.sh hash))
 
 function build {
+  # Noop if user doesn't have docker.
+  if ! command -v docker &>/dev/null; then
+    echo "Docker not installed. Skipping..."
+    return
+  fi
+
   # Create versions.json so we know what to install.
   ../bootstrap.sh versions > ./bin/0.0.1/versions
   echo "Versions:"
@@ -147,8 +153,9 @@ function prep_test_mac {
   fi
 
   # Cleanup background processes on exit.
-  local pids=()
-  trap 'kill "${pids[@]}" &>/dev/null || true' EXIT
+  # Note: can't use local - the trap fires after function scope is gone.
+  _bg_pids=()
+  trap 'kill "${_bg_pids[@]}" &>/dev/null || true' EXIT
 
   # Start Verdaccio in offline mode (no uplinks), bound to all interfaces.
   cat > /tmp/verdaccio-mac-test.yaml <<EOF
@@ -170,13 +177,13 @@ logs: { type: stdout, format: pretty, level: warn }
 EOF
 
   verdaccio --config /tmp/verdaccio-mac-test.yaml --listen 0.0.0.0:$verdaccio_port &>/dev/null &
-  pids+=($!)
+  _bg_pids+=($!)
   while ! nc -z localhost $verdaccio_port &>/dev/null; do sleep 1; done
   echo "Verdaccio running on 0.0.0.0:$verdaccio_port"
 
   # Serve bin/ directory over HTTP to mimic S3-hosted install scripts.
   python3 -m http.server $http_port --directory ./bin --bind 0.0.0.0 &>/dev/null &
-  pids+=($!)
+  _bg_pids+=($!)
   while ! nc -z localhost $http_port &>/dev/null; do sleep 1; done
   echo "HTTP server running on 0.0.0.0:$http_port (serving ./bin/)"
 }
@@ -235,13 +242,20 @@ export -f install_on_mac_vm launch_and_install_on_mac_vm
 
 # Assumes a macos vm is already running.
 # Starts services, and runs install script on the mac vm via ssh.
-function test_mac {
-  echo_header "aztec-up test_mac"
+function test_on_mac_vm {
   local mac_name="${1:?Mac vm name (e.g. 14)}"
+  echo_header "aztec-up test_on_mac_vm"
   prep_test_mac
   install_on_mac_vm $mac_name
 }
 
+function test_mac {
+  local mac_name="${1:?Mac vm name (e.g. 14)}"
+  echo_header "aztec-up test_mac"
+  prep_test_mac
+  launch_and_install_on_mac_vm $mac_name
+}
+
 # Starts services, launches a mac vm for each version and runs install script via ssh.
 function test_macs {
   echo_header "aztec-up test_macs"

barretenberg/cpp/CMakePresets.json

@@ -35,13 +35,13 @@
     },
     {
       "name": "clang20",
-      "displayName": "Build with Zig (glibc 2.35)",
-      "description": "Build with Zig-wrapped Clang targeting glibc 2.35 (Ubuntu 22.04+)",
+      "displayName": "Build with Zig (glibc 2.35 on Linux)",
+      "description": "Build with Zig-wrapped Clang. On Linux, pins glibc 2.35 (Ubuntu 22.04+). On macOS, uses native target.",
       "inherits": "default",
       "binaryDir": "build",
       "environment": {
-        "CC": "zig cc -target native-linux-gnu.2.35",
-        "CXX": "zig c++ -target native-linux-gnu.2.35"
+        "CC": "${sourceDir}/scripts/zig-cc.sh",
+        "CXX": "${sourceDir}/scripts/zig-c++.sh"
       },
       "cacheVariables": {
         "CMAKE_AR": "${sourceDir}/scripts/zig-ar.sh",

barretenberg/cpp/bootstrap.sh

@@ -35,6 +35,13 @@ function inject_version {
   # Version starts immediately after the sentinel
   local version_offset=$((sentinel_offset + ${#sentinel}))
   printf "$version\0" | dd of="$binary" bs=1 seek=$version_offset conv=notrunc 2>/dev/null
+
+  # Re-sign after modifying the binary.
+  if [[ "$(os)" == "macos" ]]; then
+    codesign -s - -f "$binary" 2>/dev/null || true
+  elif llvm-objdump --macho --private-header "$binary" &>/dev/null; then
+    ldid -S "$binary"
+  fi
 }
 
 # Define build commands for each preset
@@ -55,6 +62,7 @@ function build_preset() {
 function build_native_objects {
   set -eu
   if ! cache_exists barretenberg-$native_preset-$hash.zst; then
+    (flock -x 200 && cd src/barretenberg/nodejs_module && yarn --immutable) 200>/tmp/bb-yarn.lock
     cmake --preset "$native_preset"
     targets=$(cmake --build --preset "$native_preset" --target help | awk -F: '$1 ~ /(_objects|_tests|_bench|_gen|.a)$/ && $1 !~ /^cmake_/{print $1}' | tr '\n' ' ')
     cmake --build --preset "$native_preset" --target $targets nodejs_module
@@ -87,6 +95,7 @@ function build_cross_objects {
   set -eu
   target=$1
   if ! cache_exists barretenberg-$target-$hash.zst; then
+    (flock -x 200 && cd src/barretenberg/nodejs_module && yarn --immutable) 200>/tmp/bb-yarn.lock
     build_preset zig-$target --target barretenberg nodejs_module vm2_stub circuit_checker honk
   fi
 }
@@ -96,17 +105,13 @@ function build_cross_objects {
 function build_cross {
   set -eu
   target=$1
-  is_macos=${2:-false}
   if ! cache_download barretenberg-$target-$hash.zst; then
+    (flock -x 200 && cd src/barretenberg/nodejs_module && yarn --immutable) 200>/tmp/bb-yarn.lock
     build_preset zig-$target --target bb --target nodejs_module --target bb-external
     cache_upload barretenberg-$target-$hash.zst build-zig-$target/{bin,lib}
   fi
   # Always inject version (even for cached binaries) to ensure correct version on release
   inject_version build-zig-$target/bin/bb
-  # Code sign for macOS after version injection (must be last modification to binary)
-  if [ "$is_macos" == "true" ]; then
-    ldid -S build-zig-$target/bin/bb
-  fi
 }
 
 # Build static library (.a) for iOS using Zig cross-compilation from Linux.
@@ -294,8 +299,6 @@ function build {
     rm -rf build*
   fi
 
-  (cd src/barretenberg/nodejs_module && yarn --frozen-lockfile --prefer-offline)
-
   if semver check "$REF_NAME" && [[ "$(arch)" == "amd64" ]]; then
     # Download mobile SDKs before parallel builds (shared across presets)
     bash scripts/download-ios-sdk.sh
@@ -306,8 +309,8 @@ function build {
       "build_wasm" \
       "build_wasm_threads" \
       "build_cross arm64-linux" \
-      "build_cross amd64-macos true" \
-      "build_cross arm64-macos true" \
+      "build_cross amd64-macos" \
+      "build_cross arm64-macos" \
       "build_ios zig-arm64-ios" \
       "build_ios zig-arm64-ios-sim" \
       "build_android zig-arm64-android" \
@@ -325,7 +328,7 @@ function build {
     if [ "$(arch)" == "amd64" ] && [ "$CI_FULL" -eq 1 ]; then
       bash scripts/download-ios-sdk.sh
       bash scripts/download-android-sysroot.sh
-      builds+=("build_cross arm64-macos true" build_smt_verification "build_ios zig-arm64-ios" "build_ios zig-arm64-ios-sim" "build_android zig-arm64-android" "build_android zig-x86_64-android")
+      builds+=("build_cross arm64-macos" build_smt_verification "build_ios zig-arm64-ios" "build_ios zig-arm64-ios-sim" "build_android zig-arm64-android" "build_android zig-x86_64-android")
     fi
     parallel --line-buffered --tag --halt now,fail=1 "denoise {}" ::: "${builds[@]}"
   fi

barretenberg/cpp/cmake/avm-transpiler.cmake

@@ -20,3 +20,7 @@ add_definitions(-DENABLE_AVM_TRANSPILER)
 
 message(STATUS "avm-transpiler library: ${AVM_TRANSPILER_LIB}")
 message(STATUS "avm-transpiler include: ${AVM_TRANSPILER_INCLUDE}")
+
+if(APPLE AND NOT CMAKE_CROSSCOMPILING)
+    target_link_libraries(avm_transpiler INTERFACE "-framework CoreFoundation")
+endif()

barretenberg/cpp/cmake/module.cmake

@@ -204,7 +204,7 @@ function(barretenberg_module_with_sources MODULE_NAME)
         endif()
         if(NOT WASM)
             # Currently haven't found a way to easily wrap the calls in wasmtime when run from ctest.
-            gtest_discover_tests(${MODULE_NAME}_tests WORKING_DIRECTORY ${CMAKE_BINARY_DIR} TEST_FILTER -*_SKIP_CI*)
+            gtest_discover_tests(${MODULE_NAME}_tests WORKING_DIRECTORY ${CMAKE_BINARY_DIR} TEST_FILTER -*_SKIP_CI* DISCOVERY_TIMEOUT 30)
         endif()
     endif()
 

barretenberg/cpp/scripts/audit/generate_audit_status_headers.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # --- Setup Paths --------------------------------------------------------------
 

barretenberg/cpp/scripts/audit/run_dashboard_server.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 
 # Resolve script directory

barretenberg/cpp/scripts/bench_cpu_scaling_local.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # CPU scaling benchmark that runs benchmarks locally
 # This script runs a command multiple times with different HARDWARE_CONCURRENCY values
@@ -70,12 +70,12 @@ extract_bench_time() {
     # Extract time from JSON file using grep and sed
     # JSON format is: {"benchmark_name": time_in_nanoseconds, ...}
     local time_ns=""
-    
+
     if [ -f "$json_file" ]; then
         # Extract the value for the specific benchmark name from JSON
         time_ns=$(grep -oP "\"${bench_name//\\/\\\\}\":\s*\K\d+" "$json_file" 2>/dev/null | head -1)
     fi
-    
+
     # If JSON extraction failed, try to extract from log file (fallback)
     if [ -z "$time_ns" ] && [ -f "${json_file%/bench.json}/output.log" ]; then
         local log_file="${json_file%/bench.json}/output.log"
@@ -123,7 +123,7 @@ for cpu_count in "${CPU_COUNTS[@]}"; do
     # Execute the command locally with HARDWARE_CONCURRENCY environment variable
     # Add --bench_out flag to get JSON output
     HARDWARE_CONCURRENCY=$cpu_count eval "$COMMAND --bench_out $bench_json_file" 2>&1 | tee "$log_file"
-    
+
     end_time=$(date +%s.%N)
     wall_time=$(awk -v e="$end_time" -v s="$start_time" 'BEGIN{printf "%.2f", e-s}')
 
@@ -133,14 +133,14 @@ for cpu_count in "${CPU_COUNTS[@]}"; do
     if [ -z "$bench_time_ns" ] || [ "$bench_time_ns" = "0" ]; then
         echo -e "${RED}Warning: Could not extract timing for '$BENCH_NAME' from JSON${NC}"
         echo -e "${YELLOW}Check the JSON file: $bench_json_file${NC}"
-        
+
         # Show what's in the JSON file for debugging
         if [ -f "$bench_json_file" ]; then
             echo -e "${YELLOW}JSON content (first 500 chars):${NC}"
             head -c 500 "$bench_json_file"
             echo ""
         fi
-        
+
         echo "CPUs: $cpu_count - No timing data found" >> "$RESULTS_FILE"
         continue
     fi
@@ -278,4 +278,4 @@ if [ "${#ALL_SPEEDUPS[@]}" -gt 1 ]; then
     fi
 fi
 
-echo ""
+echo ""

barretenberg/cpp/scripts/bench_cpu_scaling_remote.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # CPU scaling benchmark wrapper that uses benchmark_remote.sh properly
 # This script runs a command multiple times with different HARDWARE_CONCURRENCY values
@@ -81,12 +81,12 @@ extract_bench_time() {
     # Extract time from JSON file using grep and sed
     # JSON format is: {"benchmark_name": time_in_nanoseconds, ...}
     local time_ns=""
-    
+
     if [ -f "$json_file" ]; then
         # Extract the value for the specific benchmark name from JSON
         time_ns=$(grep -oP "\"${bench_name//\\/\\\\}\":\s*\K\d+" "$json_file" 2>/dev/null | head -1)
     fi
-    
+
     # If JSON extraction failed, try to extract from log file (fallback)
     if [ -z "$time_ns" ] && [ -f "${json_file%/bench.json}/output.log" ]; then
         local log_file="${json_file%/bench.json}/output.log"
@@ -135,10 +135,10 @@ for cpu_count in "${CPU_COUNTS[@]}"; do
     # Use tee to show output in real-time AND save to log file
     bench_json_file="$run_dir/bench.json"
     ./scripts/benchmark_remote.sh bb "HARDWARE_CONCURRENCY=$cpu_count $COMMAND --bench_out /tmp/bench_${cpu_count}.json" 2>&1 | tee "$log_file"
-    
+
     # Retrieve the JSON file from remote
     ssh $BB_SSH_KEY $BB_SSH_INSTANCE "cat /tmp/bench_${cpu_count}.json" > "$bench_json_file" 2>/dev/null
-    
+
     # Clean up the remote benchmark file after retrieval
     ssh $BB_SSH_KEY $BB_SSH_INSTANCE "rm -f /tmp/bench_${cpu_count}.json" 2>/dev/null
 
@@ -151,14 +151,14 @@ for cpu_count in "${CPU_COUNTS[@]}"; do
     if [ -z "$bench_time_ns" ] || [ "$bench_time_ns" = "0" ]; then
         echo -e "${RED}Warning: Could not extract timing for '$BENCH_NAME' from JSON${NC}"
         echo -e "${YELLOW}Check the JSON file: $bench_json_file${NC}"
-        
+
         # Show what's in the JSON file for debugging
         if [ -f "$bench_json_file" ]; then
             echo -e "${YELLOW}JSON content (first 500 chars):${NC}"
             head -c 500 "$bench_json_file"
             echo ""
         fi
-        
+
         echo "CPUs: $cpu_count - No timing data found" >> "$RESULTS_FILE"
         continue
     fi

barretenberg/cpp/scripts/test_chonk_standalone_vks_havent_changed.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 source $(git rev-parse --show-toplevel)/ci3/source
 
 # export bb as it is needed when using exported functions

barretenberg/cpp/scripts/update_acir.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 source $(git rev-parse --show-toplevel)/ci3/source
 
 # This script replaces std::array<T, N> with std::shared_ptr<std::array<T, N>>

barretenberg/cpp/scripts/zig-ar.sh

@@ -1,2 +1,2 @@
-#!/bin/bash
+#!/usr/bin/env bash
 exec zig ar "$@"

barretenberg/cpp/scripts/zig-c++.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Wrapper for zig c++ that pins glibc 2.35 on Linux (Ubuntu 22.04+ compat)
+# and uses native target on macOS.
+if [[ "$(uname -s)" == "Linux" ]]; then
+  exec zig c++ -target native-linux-gnu.2.35 "$@"
+else
+  exec zig c++ "$@"
+fi

barretenberg/cpp/scripts/zig-cc.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Wrapper for zig cc that pins glibc 2.35 on Linux (Ubuntu 22.04+ compat)
+# and uses native target on macOS.
+if [[ "$(uname -s)" == "Linux" ]]; then
+  exec zig cc -target native-linux-gnu.2.35 "$@"
+else
+  exec zig cc "$@"
+fi

barretenberg/cpp/scripts/zig-ranlib.sh

@@ -1,2 +1,2 @@
-#!/bin/bash
+#!/usr/bin/env bash
 exec zig ranlib "$@"

barretenberg/cpp/src/barretenberg/avm_fuzzer/run_fuzzer.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Script to run AVM fuzzers with default parameters
 # Usage: ./run_fuzzer.sh <command> <fuzzer_type> [options]

barretenberg/cpp/src/barretenberg/bb/CMakeLists.txt

@@ -20,7 +20,7 @@ if (NOT(FUZZING))
     )
     # Link avm_transpiler when library is provided
     if(AVM_TRANSPILER_LIB)
-        target_link_libraries(bb PRIVATE ${AVM_TRANSPILER_LIB})
+        target_link_libraries(bb PRIVATE avm_transpiler)
     endif()
     if(NOT WASM)
         target_link_libraries(bb PRIVATE ipc)
@@ -60,7 +60,7 @@ if (NOT(FUZZING))
         )
         # Link avm_transpiler when library is provided
         if(AVM_TRANSPILER_LIB)
-            target_link_libraries(bb-avm PRIVATE ${AVM_TRANSPILER_LIB})
+            target_link_libraries(bb-avm PRIVATE avm_transpiler)
         endif()
         if(NOT WASM)
             target_link_libraries(bb-avm PRIVATE ipc)