feat: merge-train/spartan

.claude/skills/updating-changelog/SKILL.md

@@ -14,7 +14,7 @@ Read `.release-please-manifest.json` to get the version (e.g., `{"." : "4.0.0"}`
 **Target files:**
 
 - Aztec contract developers: `docs/docs-developers/docs/resources/migration_notes.md`
-- Node operators and Ethereum contract developers: `docs/docs-network/reference/changelog/v{major}.md`
+- Node operators and Ethereum contract developers: `docs/docs-operate/operators/reference/changelog/v{major}.md`
 
 ### 2. Analyze Branch Changes
 
@@ -60,7 +60,7 @@ Explanation of what changed.
 
 ## Node Operator Changelog Format
 
-**File:** `docs/docs-network/reference/changelog/v{major}.md`
+**File:** `docs/docs-operate/operators/reference/changelog/v{major}.md`
 
 **Breaking changes:**
 ````markdown

.github/workflows/ci3.yml

@@ -94,6 +94,7 @@ jobs:
           PR_COMMITS: ${{ github.event.pull_request.commits }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
           GITHUB_REF_NAME: ${{ github.ref_name }}
+          GITHUB_ACTOR: ${{ github.actor }}
         # NOTE: $CI_MODE is set in the Determine CI Mode step.
         run: ./.github/ci3.sh $CI_MODE
 

.github/workflows/weekly-proving-bench.yml

@@ -47,7 +47,7 @@ jobs:
           fi
 
       - name: Run real proving benchmarks
-        timeout-minutes: 150
+        timeout-minutes: 180
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -57,7 +57,7 @@ jobs:
           GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
           RUN_ID: ${{ github.run_id }}
-          AWS_SHUTDOWN_TIME: 150
+          AWS_SHUTDOWN_TIME: 180
           NO_SPOT: 1
         run: |
           ./.github/ci3.sh network-proving-bench prove-n-tps-real prove-n-tps-real "aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}"

.test_patterns.yml

@@ -309,6 +309,11 @@ tests:
     owners:
       - *palla
 
+  - regex: "src/e2e_block_building.test.ts"
+    error_regex: "✕ processes txs until hitting timetable"
+    owners:
+      - *palla
+
   # http://ci.aztec-labs.com/e8228a36afda93b8
   # Test passed but there was an error on stopping
   - regex: "playground/scripts/run_test.sh"

bootstrap.sh

@@ -233,11 +233,14 @@ function start_txes {
   export TOKIO_WORKER_THREADS=1
 
   # Starting txe servers with incrementing port numbers.
+  # Base port is below the Linux ephemeral range (32768-60999) to avoid conflicts.
+  local txe_base_port=14730
   for i in $(seq 0 $((NUM_TXES-1))); do
-    port=$((45730 + i))
+    port=$((txe_base_port + i))
     existing_pid=$(lsof -ti :$port || true)
     if [ -n "$existing_pid" ]; then
       echo "Killing existing process $existing_pid on port: $port"
+      check_port $port
       kill -9 $existing_pid &>/dev/null || true
       while kill -0 $existing_pid &>/dev/null; do sleep 0.1; done
     fi
@@ -248,8 +251,12 @@ function start_txes {
   echo "Waiting for TXE's to start..."
   for i in $(seq 0 $((NUM_TXES-1))); do
       local j=0
-      while ! nc -z 127.0.0.1 $((45730 + i)) &>/dev/null; do
-        [ $j == 60 ] && echo_stderr "TXE $i took too long to start. Exiting." && exit 1
+      while ! nc -z 127.0.0.1 $((txe_base_port + i)) &>/dev/null; do
+        if [ $j == 60 ]; then
+          echo_stderr "TXE $i failed to start on port $((txe_base_port + i)) after 60s."
+          check_port $((txe_base_port + i))
+          exit 1
+        fi
         sleep 1
         j=$((j+1))
       done

ci3/aws_request_instance_type

@@ -88,6 +88,21 @@ fi
 
 echo "Instance id: $iid"
 
+tags="Key=Name,Value=$name Key=Group,Value=build-instance"
+[ -n "${GITHUB_ACTOR:-}" ] && tags+=" Key=GithubActor,Value=$GITHUB_ACTOR"
+[ -n "${CI_MODE:-}" ] && tags+=" Key=CICommand,Value=$CI_MODE"
+[ -n "${CI_DASHBOARD:-}" ] && tags+=" Key=Dashboard,Value=$CI_DASHBOARD"
+if [ "${UNSAFE_AWS_KEEP_ALIVE:-0}" -eq 1 ]; then
+  echo_stderr "You have set UNSAFE_AWS_KEEP_ALIVE=1, so the instance will not be terminated after 1.5 hours by the reaper script. Make sure you shut the machine down when done."
+  tags+=" Key=Keep-Alive,Value=true"
+fi
+aws ec2 create-tags --resources $iid --tags $tags
+
+# Record the instance type so callers can pass it downstream (e.g. into Docker).
+echo $instance_type > $state_dir/instance_type
+# Record whether this is spot or on-demand.
+[ -f "$sir_path" ] && echo spot > $state_dir/spot || echo ondemand > $state_dir/spot
+
 while [ -z "${ip:-}" ]; do
   sleep 1
   ip=$(aws ec2 describe-instances \

ci3/bootstrap_ec2

@@ -89,6 +89,8 @@ if [[ -f "$state_dir/sir" ]]; then
   sir=$(cat $state_dir/sir)
 fi
 iid=$(cat $state_dir/iid)
+export EC2_INSTANCE_TYPE=$(cat $state_dir/instance_type 2>/dev/null || echo "unknown")
+export EC2_SPOT=$(cat $state_dir/spot 2>/dev/null || echo "unknown")
 
 # If AWS credentials are not set, try to load them from ~/.aws/build_instance_credentials.
 if [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then
@@ -191,16 +193,6 @@ container_script=$(
       log_ci_run FAILED \$ci_log_id
       merge_train_failure_slack_notify \$ci_log_id
       release_canary_slack_notify \$ci_log_id
-      ci_failed_data=\$(jq -n \\
-        --arg status "failed" \\
-        --arg log_id "\$ci_log_id" \\
-        --arg ref_name "\${TARGET_BRANCH:-\$REF_NAME}" \\
-        --arg commit_hash "\$COMMIT_HASH" \\
-        --arg commit_author "\$COMMIT_AUTHOR" \\
-        --arg commit_msg "\$COMMIT_MSG" \\
-        --argjson exit_code "\$code" \\
-        '{status: \$status, log_id: \$log_id, ref_name: \$ref_name, commit_hash: \$commit_hash, commit_author: \$commit_author, commit_msg: \$commit_msg, exit_code: \$exit_code, timestamp: now | todate}')
-      redis_publish "ci:run:failed" "\$ci_failed_data"
       ;;
   esac
   exit \$code
@@ -330,6 +322,9 @@ function run {
         -e AWS_TOKEN=\$aws_token \
         -e NAMESPACE=${NAMESPACE:-} \
         -e NETWORK=${NETWORK:-} \
+        -e GITHUB_ACTOR=${GITHUB_ACTOR:-} \
+        -e EC2_INSTANCE_TYPE=${EC2_INSTANCE_TYPE:-unknown} \
+        -e EC2_SPOT=${EC2_SPOT:-unknown} \
         --pids-limit=65536 \
         --shm-size=2g \
         aztecprotocol/devbox:3.0 bash -c $(printf '%q' "$container_script")

ci3/check_port

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Check if a port is free. If taken, print the process tree of the holder.
+# Usage: check_port <port>
+# Exit code: 0 if free, 1 if taken.
+set -eu
+
+port="${1:?Usage: check_port <port>}"
+
+pid=$(lsof -ti :"$port" 2>/dev/null | head -1 || true)
+if [ -z "$pid" ]; then
+  exit 0
+fi
+
+echo "Port $port is taken by PID $pid:" >&2
+# Show the command line of the process.
+ps -p "$pid" -o pid,ppid,user,args --no-headers >&2 || true
+# Show the process tree rooted at this PID.
+pstree -apls "$pid" >&2 2>/dev/null || pstree -p "$pid" >&2 2>/dev/null || true
+exit 1

ci3/ci-metrics/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.12
+
+RUN apt update && apt install -y jq redis-tools && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY requirements.txt requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt gunicorn
+RUN git config --global --add safe.directory /aztec-packages
+COPY . .
+EXPOSE 8081
+CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:8081", "app:app"]