feat: Add parent log link to top of CI logs.

.test_patterns.yml

@@ -24,6 +24,7 @@ names:
   - tom: &tom "U04LLT331NK"
   - sergei: &sergei "U05RR81391B"
   - spyros: &spyros "U03FCET0DQE"
+  - jan: &jan "U03FGA0BWNR"
 
 tests:
   # barretenberg
@@ -52,7 +53,10 @@ tests:
     error_regex: "verified: false"
     owners:
       - *sergei
-
+  - regex: "barretenberg/acir_tests/scripts/run_test_browser.sh"
+    error_regex: "Failed to fetch"
+    owners:
+      - *adam
   # Sumcheck is failing for some reason
   - regex: "barretenberg/acir_tests/scripts/run_test.sh ram_blowup_regression"
     skip: true
@@ -69,13 +73,13 @@ tests:
     skip: true
     owners:
       - *charlie
-
   # Sometimes see this on ARM. But not when run on it's own...
   - regex: "tests::debug_ram_blowup_regression"
     skip: true
     owners:
       - *charlie
 
+  # e2e tests
   - regex: "simple e2e_p2p"
     error_regex: "could not listen on any available address"
     owners:
@@ -84,6 +88,10 @@ tests:
     error_regex: "TimeoutError"
     owners:
       - *sean
+  - regex: "simple e2e_p2p/reex"
+    error_regex: "Received promise resolved instead of rejected"
+    owners:
+      - *sean
   - regex: "simple e2e_p2p/slashing"
     skip: true
     owners:
@@ -92,14 +100,15 @@ tests:
     error_regex: "TimeoutError: Timeout awaiting blocks mined"
     owners:
       - *palla
-
   - regex: "simple e2e_fees/private_payments"
     owners:
       - *phil
-
-  - regex: "simple e2e_cheat_codes"
+  # http://ci.aztec-labs.com/7a5104e136578d64
+  # The "loop that wastes 5m" thing.
+  - regex: "simple e2e_fees/failures"
+    error_regex: "Exceeded timeout"
     owners:
-      - *lasse
+      - *palla
   - regex: "simple e2e_lending_contract"
     error_regex: "✕ Depositing"
     owners:
@@ -116,11 +125,10 @@ tests:
     error_regex: "✕ Someone else can mint funds to me on my behalf"
     owners:
       - *lasse
-
   - regex: "simple e2e_block_building"
     error_regex: "Cannot read properties of undefined"
     owners:
-      - *palla
+      - *jan
   - regex: "simple e2e_block_building"
     error_regex: "Could not retrieve body for block"
     owners:
@@ -129,24 +137,12 @@ tests:
     error_regex: "BlockOutOfRangeError"
     owners:
       - *palla
-  - regex: "prover-client/src/test/bb_prover_parity.test.ts"
-    owners:
-      - *palla
-  - regex: "prover-client/src/proving_broker/broker_prover_facade.test.ts"
-    owners:
-      - *palla
-  - regex: "prover-client/src/orchestrator/orchestrator_errors.test.ts"
-    owners:
-      - *palla
 
   # yarn-project tests
   - regex: "p2p/src/services/discv5/discv5_service.test.ts"
     error_regex: "Timeout: Failed to connect to"
     owners:
       - *sean
-  - regex: "p2p/src/services/reqresp/reqresp.test.ts"
-    owners:
-      - *sean
   - regex: "p2p/src/client/p2p_client.test.ts"
     error_regex: "Exceeded timeout of 120000"
     owners:
@@ -155,12 +151,6 @@ tests:
     error_regex: "Timeout waiting for worker"
     owners:
       - *sean
-  - regex: "sequencer-client/src/slasher/slasher_client.test.ts"
-    owners:
-      - *lasse
-  - regex: "prover-client/src/test/bb_prover_full_rollup.test.ts"
-    owners:
-      - *palla
   - regex: "yarn-project/kv-store"
     error_regex: "Could not import your test module"
     owners:
@@ -193,26 +183,25 @@ tests:
     error_regex: "job smoke-aztec-network-setup-l2-contracts failed: BackoffLimitExceeded"
     owners:
       - *mitch
-
+  - regex: "spartan/bootstrap.sh test-kind-smoke"
+    error_regex: "timeout: sending signal TERM to command"
+    owners:
+      - *mitch
   - regex: "spartan/bootstrap.sh test-kind-4epochs"
     owners:
-      - *leila
-
+      - *mitch
   - regex: "spartan/bootstrap.sh test-kind-upgrade-rollup-version"
     owners:
-      - *adam
-
+      - *mitch
   - regex: "spartan/bootstrap.sh test-cli-upgrade-with-lock"
     owners:
-      - *adam
-
+      - *mitch
   - regex: "spartan/bootstrap.sh test-kind-transfer"
     owners:
-      - *adam
-
+      - *mitch
   - regex: "spartan/bootstrap.sh test-prod-deployment"
     owners:
-      - *adam
+      - *mitch
 
   # aztec-up
   - regex: "aztec-up/scripts/run_test.sh"
@@ -222,11 +211,7 @@ tests:
 
   # Slack testing.
   - regex: "nonsense to match"
-    owners:
-      - *charlie
-
-  - regex: "nonsense to match"
+    error_regex: "something else"
     owners:
       - *charlie
       - *adam
-    error_regex: "something else"

ci.sh

@@ -71,15 +71,18 @@ function tail_live_instance {
 case "$cmd" in
   "ec2")
     # Spin up ec2 instance and ci bootstrap with shell on failure.
-    bootstrap_ec2 "./bootstrap.sh ci"
+    bootstrap_ec2
     ;;
   "ec2-no-cache")
-    # Same as ec2, but disable the build and test cache.
-    bootstrap_ec2 "NO_CACHE=1 USE_TEST_CACHE=0 ./bootstrap.sh ci"
+    # Disable the build and test cache.
+    export NO_CACHE=1
+    export USE_TEST_CACHE=0
+    bootstrap_ec2
     ;;
   "ec2-test")
-    # Same as ec2, but don't use the test cache.
-    bootstrap_ec2 "USE_TEST_CACHE=0 ./bootstrap.sh ci"
+    # Can use the build cache, but don't use the test cache.
+    export USE_TEST_CACHE=0
+    bootstrap_ec2
     ;;
   "ec2-shell")
     # Spin up ec2 instance, clone, and drop into shell.
@@ -89,9 +92,11 @@ case "$cmd" in
   "ec2-grind")
     # Same as ec2-test but repeat it over arg1 instances.
     export DENOISE=1
+    export CI_FULL=1
+    export USE_TEST_CACHE=0
     num=${1:-5}
     seq 0 $((num - 1)) | parallel --tag --line-buffered \
-      'INSTANCE_POSTFIX=$USER_{} bootstrap_ec2 "USE_TEST_CACHE=0 ./bootstrap.sh ci" 2>&1 | cache_log "Grind {}"'
+      'INSTANCE_POSTFIX=${USER}_{} bootstrap_ec2 2>&1 | cache_log "Grind {}"'
     ;;
   "local")
     # Create container with clone of local repo and bootstrap.

ci3/bootstrap_ec2

@@ -6,6 +6,9 @@ cmd=${1:-"./bootstrap.sh ci"}
 arch=${ARCH:-amd64}
 NO_TERMINATE=${NO_TERMINATE:-0}
 
+# We're always "in CI" if we're running on a remote instance.
+export CI=1
+
 if [ "$arch" == "arm64" ]; then
   cores=64
   # Allow for 90 minutes as ARM has less cores.
@@ -128,25 +131,26 @@ container_script=$(cat <<EOF
   source ci3/source_refname
   source ci3/source_redis
   ci_log_id=\$(log_ci_run)
+  export PARENT_LOG_URL=http://ci.aztec-labs.com/\$ci_log_id
 
   # Heartbeat.
   while true; do redis_cli SETEX hb-\$ci_log_id 60 1 &>/dev/null || true; sleep 30; done &
 
-  echo "env: REF_NAME=\$REF_NAME COMMIT_HASH=\$COMMIT_HASH CURRENT_VERSION=\$CURRENT_VERSION CI_FULL=\$CI_FULL DRY_RUN=\${DRY_RUN:-0}"
-  if semver check "\$REF_NAME"; then
-    echo "Performing a release because \$REF_NAME is a semver."
-  fi
-
   function run {
+    echo "env: REF_NAME=\$REF_NAME COMMIT_HASH=\$COMMIT_HASH CURRENT_VERSION=\$CURRENT_VERSION CI=\$CI CI_FULL=\$CI_FULL DRY_RUN=\${DRY_RUN:-0}"
+    if semver check "\$REF_NAME"; then
+      echo "Performing a release because \$REF_NAME is a semver."
+    fi
+
     set +e
     set -x
     $cmd
     local code=\${PIPESTATUS[0]}
     set +x
-    sudo dmesg 2>&1 | ci3/cache_log 'dmesg'
-    sudo cat /tmp/netfile | ci3/cache_log 'netfile'
-    sudo cat /tmp/cpufile | ci3/cache_log 'cpufile'
-    sudo cat /tmp/memfile | ci3/cache_log 'memfile'
+    sudo dmesg 2>&1 | cache_log 'dmesg'
+    sudo cat /tmp/netfile | cache_log 'netfile'
+    sudo cat /tmp/cpufile | cache_log 'cpufile'
+    sudo cat /tmp/memfile | cache_log 'memfile'
     return \$code
   }
   export -f run
@@ -213,7 +217,8 @@ ssh ${ssh_args:-} -F $ci3/aws/build_instance_ssh_config ubuntu@$ip "
     -v /mnt/bb-crs:/home/aztec-dev/.bb-crs:ro \
     -v /tmp:/tmp \
     -v /dev/kmsg:/dev/kmsg \
-    -e USE_TEST_CACHE=1 \
+    -e NO_CACHE=${NO_CACHE:-} \
+    -e USE_TEST_CACHE=${USE_TEST_CACHE:-1} \
     -e CI_REDIS='ci-redis-tiered.lzka0i.ng.0001.use2.cache.amazonaws.com' \
     -e SSH_CONNECTION=' ' \
     -e LOCAL_USER_ID=\$(id -u) \

ci3/cache_log

@@ -11,7 +11,7 @@ key=${2:-$(uuid)}
 outfile=/tmp/$key
 dup=${DUP:-0}
 trap 'rm -f $outfile' EXIT
-touch $outfile
+echo -e "Parent Log: ${PARENT_LOG_URL:-none}\n" >$outfile
 
 function publish_log {
   cat $outfile | redis_setexz $key $CI_REDIS_EXPIRE
@@ -22,7 +22,10 @@ function live_publish_log {
     if [ $(( $(date +%s) - $(stat -c %Y "$outfile") )) -le 5 ]; then
       publish_log
     fi
-    sleep 5
+    for i in {1..5}; do
+      [ ! -f $outfile ] && break
+      sleep 1
+    done
   done
 }
 
@@ -31,17 +34,19 @@ if [ "$CI_REDIS_AVAILABLE" -eq 1 ]; then
   echo_stderr -e "$log_str"
   live_publish_log &
   if [ "$dup" -eq 1 ]; then
-    tee $outfile
+    redact | tee -a $outfile
+    # Print a second line if the log is long enough.
+    # This is to avoid having to scroll up to see the log id.
     if [ "$(cat $outfile | wc -l)" -ge 16 ]; then
       echo_stderr -e "$log_str"
     fi
   else
-    cat >$outfile
+    redact | cat >>$outfile
   fi
   publish_log
 else
   if [ "$dup" -eq 1 ]; then
-    cat
+    redact | cat
   else
     cat >/dev/null
   fi

ci3/denoise

@@ -39,10 +39,11 @@ function publish_log {
   fi
 
   {
+    echo "Parent Log: ${PARENT_LOG_URL:-none}"
     echo "Command: $cmd"
-    echo "   Date: $(date)"
-    echo " Status: $status"
-    echo "   Took: ${time}"
+    echo "Date: $(date)"
+    echo "Status: $status"
+    echo "Took: ${time}"
     echo
 
     cat $outfile
@@ -68,7 +69,7 @@ fi
 set +e
 echo -e "Executing: $cmd ${log_info:-}"
 echo -n "   0 "
-bash -c "$cmd" 2>&1 | while IFS= read -r line; do
+PARENT_LOG_URL=$url bash -c "$cmd" 2>&1 | redact | while IFS= read -r line; do
   ((dot_count++))
   [ $realtime -eq 1 ] && printf "."
   if [[ "$dots_per_line" -gt 0 && $((dot_count % dots_per_line)) -eq 0 ]]; then

ci3/redact

@@ -0,0 +1,12 @@
+#!/bin/bash
+sed_expr=""
+for var in $(compgen -e); do
+  if [[ "$var" =~ (KEY|TOKEN|PASSWORD|SECRET|MNEMONIC) ]]; then
+    val="${!var}"
+    if [[ -n "$val" ]]; then
+      esc=$(printf '%s\n' "$val" | sed 's/[][\/.^$*]/\\&/g')
+      sed_expr+="s/${esc}/***/g;"
+    fi
+  fi
+done
+sed -u "$sed_expr"

ci3/run_test_cmd

@@ -33,6 +33,7 @@ export VERBOSE=1
 tmp_file=/tmp/$key
 # Print test metadata header.
 cat <<EOF >$tmp_file
+Parent Log: ${PARENT_LOG_URL:-none}
 Command: $cmd
 Commit: https://github.com/AztecProtocol/aztec-packages/commit/$COMMIT_HASH
 Env: REF_NAME=$REF_NAME CURRENT_VERSION=$CURRENT_VERSION CI_FULL=$CI_FULL
@@ -80,7 +81,7 @@ else
   timeout -v 600s bash -c "set -euo pipefail; $test_cmd 2>&1 | add_timestamps" &>> $tmp_file &
 fi
 timeout_pid=$!
-trap "kill -- -$timeout_pid" SIGTERM SIGINT
+trap "kill -- -$timeout_pid &>/dev/null; exit" SIGTERM SIGINT
 wait $timeout_pid
 code=$?
 
@@ -124,7 +125,7 @@ function pass {
 
 # Show FAILED and exit with error code.
 function fail {
-  local line="${red}FAILED${reset}${log_info:-}: $test_cmd (${SECONDS}s) (code: $code)"
+  local line="${red}FAILED${reset}${log_info:-}: $test_cmd (${SECONDS}s) (code: $code)${1:-}"
   track_test $test_hash "$line"
   track_test "failed_tests" "$line"
   echo -e "$line"
@@ -173,7 +174,8 @@ function get_owners {
     # If error_regex is present, check the log content.
     if [ -n "$error_regex" ]; then
       # Strip color codes from the log file.
-      if sed 's/\x1B\[[0-9;]*[a-zA-Z]//g' $tmp_file | grep -qE "$error_regex"; then
+      # Do *not* use a pipe here. grep -q will exit immediately on first match and break the pipe.
+      if grep -qE "$error_regex" <(sed 's/\x1B\[[0-9;]*[a-zA-Z]//g' "$tmp_file"); then
         echo "$entry" | jq -r '.owners[]'
       fi
     else

spartan/bootstrap.sh

@@ -55,14 +55,14 @@ function test_cmds {
   fi
   # Note: commands that start with 'timeout ...' override the default timeout.
   # TODO figure out why these take long sometimes.
-  echo "$hash timeout -v 20m ./spartan/bootstrap.sh test-kind-smoke"
+  echo "$hash ./spartan/bootstrap.sh test-kind-smoke"
   if [ "$CI_FULL" -eq 1 ]; then
     # echo "$hash timeout -v 20m ./spartan/bootstrap.sh test-kind-transfer"
     # TODO(#12791) re-enable
     # echo "$hash timeout -v 30m ./spartan/bootstrap.sh test-kind-4epochs"
     # echo "$hash timeout -v 30m ./spartan/bootstrap.sh test-kind-upgrade-rollup-version"
     # echo "$hash timeout -v 30m ./spartan/bootstrap.sh test-prod-deployment"
-    echo "$hash timeout -v 30m ./spartan/bootstrap.sh test-cli-upgrade-with-lock"
+    echo "$hash ./spartan/bootstrap.sh test-cli-upgrade-with-lock"
   fi
 
   if [ "$CI_NIGHTLY" -eq 1 ]; then