diff --git a/.test_patterns.yml b/.test_patterns.yml index f985e6660a8d..03c54b2e57ab 100644 --- a/.test_patterns.yml +++ b/.test_patterns.yml @@ -24,6 +24,7 @@ names: - tom: &tom "U04LLT331NK" - sergei: &sergei "U05RR81391B" - spyros: &spyros "U03FCET0DQE" + - jan: &jan "U03FGA0BWNR" tests: # barretenberg @@ -52,7 +53,10 @@ tests: error_regex: "verified: false" owners: - *sergei - + - regex: "barretenberg/acir_tests/scripts/run_test_browser.sh" + error_regex: "Failed to fetch" + owners: + - *adam # Sumcheck is failing for some reason - regex: "barretenberg/acir_tests/scripts/run_test.sh ram_blowup_regression" skip: true @@ -69,13 +73,13 @@ tests: skip: true owners: - *charlie - # Sometimes see this on ARM. But not when run on it's own... - regex: "tests::debug_ram_blowup_regression" skip: true owners: - *charlie + # e2e tests - regex: "simple e2e_p2p" error_regex: "could not listen on any available address" owners: @@ -84,6 +88,10 @@ tests: error_regex: "TimeoutError" owners: - *sean + - regex: "simple e2e_p2p/reex" + error_regex: "Received promise resolved instead of rejected" + owners: + - *sean - regex: "simple e2e_p2p/slashing" skip: true owners: @@ -92,14 +100,15 @@ tests: error_regex: "TimeoutError: Timeout awaiting blocks mined" owners: - *palla - - regex: "simple e2e_fees/private_payments" owners: - *phil - - - regex: "simple e2e_cheat_codes" + # http://ci.aztec-labs.com/7a5104e136578d64 + # The "loop that wastes 5m" thing. + - regex: "simple e2e_fees/failures" + error_regex: "Exceeded timeout" owners: - - *lasse + - *palla - regex: "simple e2e_lending_contract" error_regex: "✕ Depositing" owners: @@ -116,11 +125,10 @@ tests: error_regex: "✕ Someone else can mint funds to me on my behalf" owners: - *lasse - - regex: "simple e2e_block_building" error_regex: "Cannot read properties of undefined" owners: - - *palla + - *jan - regex: "simple e2e_block_building" error_regex: "Could not retrieve body for block" owners: @@ -129,24 +137,12 @@ tests: error_regex: "BlockOutOfRangeError" owners: - *palla - - regex: "prover-client/src/test/bb_prover_parity.test.ts" - owners: - - *palla - - regex: "prover-client/src/proving_broker/broker_prover_facade.test.ts" - owners: - - *palla - - regex: "prover-client/src/orchestrator/orchestrator_errors.test.ts" - owners: - - *palla # yarn-project tests - regex: "p2p/src/services/discv5/discv5_service.test.ts" error_regex: "Timeout: Failed to connect to" owners: - *sean - - regex: "p2p/src/services/reqresp/reqresp.test.ts" - owners: - - *sean - regex: "p2p/src/client/p2p_client.test.ts" error_regex: "Exceeded timeout of 120000" owners: @@ -155,12 +151,6 @@ tests: error_regex: "Timeout waiting for worker" owners: - *sean - - regex: "sequencer-client/src/slasher/slasher_client.test.ts" - owners: - - *lasse - - regex: "prover-client/src/test/bb_prover_full_rollup.test.ts" - owners: - - *palla - regex: "yarn-project/kv-store" error_regex: "Could not import your test module" owners: @@ -193,26 +183,25 @@ tests: error_regex: "job smoke-aztec-network-setup-l2-contracts failed: BackoffLimitExceeded" owners: - *mitch - + - regex: "spartan/bootstrap.sh test-kind-smoke" + error_regex: "timeout: sending signal TERM to command" + owners: + - *mitch - regex: "spartan/bootstrap.sh test-kind-4epochs" owners: - - *leila - + - *mitch - regex: "spartan/bootstrap.sh test-kind-upgrade-rollup-version" owners: - - *adam - + - *mitch - regex: "spartan/bootstrap.sh test-cli-upgrade-with-lock" owners: - - *adam - + - *mitch - regex: "spartan/bootstrap.sh test-kind-transfer" owners: - - *adam - + - *mitch - regex: "spartan/bootstrap.sh test-prod-deployment" owners: - - *adam + - *mitch # aztec-up - regex: "aztec-up/scripts/run_test.sh" @@ -222,11 +211,7 @@ tests: # Slack testing. - regex: "nonsense to match" - owners: - - *charlie - - - regex: "nonsense to match" + error_regex: "something else" owners: - *charlie - *adam - error_regex: "something else" diff --git a/ci.sh b/ci.sh index bf5472efbda4..24b83a5eda5f 100755 --- a/ci.sh +++ b/ci.sh @@ -71,15 +71,18 @@ function tail_live_instance { case "$cmd" in "ec2") # Spin up ec2 instance and ci bootstrap with shell on failure. - bootstrap_ec2 "./bootstrap.sh ci" + bootstrap_ec2 ;; "ec2-no-cache") - # Same as ec2, but disable the build and test cache. - bootstrap_ec2 "NO_CACHE=1 USE_TEST_CACHE=0 ./bootstrap.sh ci" + # Disable the build and test cache. + export NO_CACHE=1 + export USE_TEST_CACHE=0 + bootstrap_ec2 ;; "ec2-test") - # Same as ec2, but don't use the test cache. - bootstrap_ec2 "USE_TEST_CACHE=0 ./bootstrap.sh ci" + # Can use the build cache, but don't use the test cache. + export USE_TEST_CACHE=0 + bootstrap_ec2 ;; "ec2-shell") # Spin up ec2 instance, clone, and drop into shell. @@ -89,9 +92,11 @@ case "$cmd" in "ec2-grind") # Same as ec2-test but repeat it over arg1 instances. export DENOISE=1 + export CI_FULL=1 + export USE_TEST_CACHE=0 num=${1:-5} seq 0 $((num - 1)) | parallel --tag --line-buffered \ - 'INSTANCE_POSTFIX=$USER_{} bootstrap_ec2 "USE_TEST_CACHE=0 ./bootstrap.sh ci" 2>&1 | cache_log "Grind {}"' + 'INSTANCE_POSTFIX=${USER}_{} bootstrap_ec2 2>&1 | cache_log "Grind {}"' ;; "local") # Create container with clone of local repo and bootstrap. diff --git a/ci3/bootstrap_ec2 b/ci3/bootstrap_ec2 index d2a09b190216..89371feea751 100755 --- a/ci3/bootstrap_ec2 +++ b/ci3/bootstrap_ec2 @@ -6,6 +6,9 @@ cmd=${1:-"./bootstrap.sh ci"} arch=${ARCH:-amd64} NO_TERMINATE=${NO_TERMINATE:-0} +# We're always "in CI" if we're running on a remote instance. +export CI=1 + if [ "$arch" == "arm64" ]; then cores=64 # Allow for 90 minutes as ARM has less cores. @@ -128,25 +131,26 @@ container_script=$(cat </dev/null || true; sleep 30; done & - echo "env: REF_NAME=\$REF_NAME COMMIT_HASH=\$COMMIT_HASH CURRENT_VERSION=\$CURRENT_VERSION CI_FULL=\$CI_FULL DRY_RUN=\${DRY_RUN:-0}" - if semver check "\$REF_NAME"; then - echo "Performing a release because \$REF_NAME is a semver." - fi - function run { + echo "env: REF_NAME=\$REF_NAME COMMIT_HASH=\$COMMIT_HASH CURRENT_VERSION=\$CURRENT_VERSION CI=\$CI CI_FULL=\$CI_FULL DRY_RUN=\${DRY_RUN:-0}" + if semver check "\$REF_NAME"; then + echo "Performing a release because \$REF_NAME is a semver." + fi + set +e set -x $cmd local code=\${PIPESTATUS[0]} set +x - sudo dmesg 2>&1 | ci3/cache_log 'dmesg' - sudo cat /tmp/netfile | ci3/cache_log 'netfile' - sudo cat /tmp/cpufile | ci3/cache_log 'cpufile' - sudo cat /tmp/memfile | ci3/cache_log 'memfile' + sudo dmesg 2>&1 | cache_log 'dmesg' + sudo cat /tmp/netfile | cache_log 'netfile' + sudo cat /tmp/cpufile | cache_log 'cpufile' + sudo cat /tmp/memfile | cache_log 'memfile' return \$code } export -f run @@ -213,7 +217,8 @@ ssh ${ssh_args:-} -F $ci3/aws/build_instance_ssh_config ubuntu@$ip " -v /mnt/bb-crs:/home/aztec-dev/.bb-crs:ro \ -v /tmp:/tmp \ -v /dev/kmsg:/dev/kmsg \ - -e USE_TEST_CACHE=1 \ + -e NO_CACHE=${NO_CACHE:-} \ + -e USE_TEST_CACHE=${USE_TEST_CACHE:-1} \ -e CI_REDIS='ci-redis-tiered.lzka0i.ng.0001.use2.cache.amazonaws.com' \ -e SSH_CONNECTION=' ' \ -e LOCAL_USER_ID=\$(id -u) \ diff --git a/ci3/cache_log b/ci3/cache_log index 2b82f5d986bc..b41ed18f9e04 100755 --- a/ci3/cache_log +++ b/ci3/cache_log @@ -11,7 +11,7 @@ key=${2:-$(uuid)} outfile=/tmp/$key dup=${DUP:-0} trap 'rm -f $outfile' EXIT -touch $outfile +echo -e "Parent Log: ${PARENT_LOG_URL:-none}\n" >$outfile function publish_log { cat $outfile | redis_setexz $key $CI_REDIS_EXPIRE @@ -22,7 +22,10 @@ function live_publish_log { if [ $(( $(date +%s) - $(stat -c %Y "$outfile") )) -le 5 ]; then publish_log fi - sleep 5 + for i in {1..5}; do + [ ! -f $outfile ] && break + sleep 1 + done done } @@ -31,17 +34,19 @@ if [ "$CI_REDIS_AVAILABLE" -eq 1 ]; then echo_stderr -e "$log_str" live_publish_log & if [ "$dup" -eq 1 ]; then - tee $outfile + redact | tee -a $outfile + # Print a second line if the log is long enough. + # This is to avoid having to scroll up to see the log id. if [ "$(cat $outfile | wc -l)" -ge 16 ]; then echo_stderr -e "$log_str" fi else - cat >$outfile + redact | cat >>$outfile fi publish_log else if [ "$dup" -eq 1 ]; then - cat + redact | cat else cat >/dev/null fi diff --git a/ci3/denoise b/ci3/denoise index 876a0da438e8..546c7ef571d2 100755 --- a/ci3/denoise +++ b/ci3/denoise @@ -39,10 +39,11 @@ function publish_log { fi { + echo "Parent Log: ${PARENT_LOG_URL:-none}" echo "Command: $cmd" - echo " Date: $(date)" - echo " Status: $status" - echo " Took: ${time}" + echo "Date: $(date)" + echo "Status: $status" + echo "Took: ${time}" echo cat $outfile @@ -68,7 +69,7 @@ fi set +e echo -e "Executing: $cmd ${log_info:-}" echo -n " 0 " -bash -c "$cmd" 2>&1 | while IFS= read -r line; do +PARENT_LOG_URL=$url bash -c "$cmd" 2>&1 | redact | while IFS= read -r line; do ((dot_count++)) [ $realtime -eq 1 ] && printf "." if [[ "$dots_per_line" -gt 0 && $((dot_count % dots_per_line)) -eq 0 ]]; then diff --git a/ci3/redact b/ci3/redact new file mode 100755 index 000000000000..c4898247d88d --- /dev/null +++ b/ci3/redact @@ -0,0 +1,12 @@ +#!/bin/bash +sed_expr="" +for var in $(compgen -e); do + if [[ "$var" =~ (KEY|TOKEN|PASSWORD|SECRET|MNEMONIC) ]]; then + val="${!var}" + if [[ -n "$val" ]]; then + esc=$(printf '%s\n' "$val" | sed 's/[][\/.^$*]/\\&/g') + sed_expr+="s/${esc}/***/g;" + fi + fi +done +sed -u "$sed_expr" diff --git a/ci3/run_test_cmd b/ci3/run_test_cmd index 50ad0cee6eb4..29aee562945f 100755 --- a/ci3/run_test_cmd +++ b/ci3/run_test_cmd @@ -33,6 +33,7 @@ export VERBOSE=1 tmp_file=/tmp/$key # Print test metadata header. cat <$tmp_file +Parent Log: ${PARENT_LOG_URL:-none} Command: $cmd Commit: https://github.com/AztecProtocol/aztec-packages/commit/$COMMIT_HASH Env: REF_NAME=$REF_NAME CURRENT_VERSION=$CURRENT_VERSION CI_FULL=$CI_FULL @@ -80,7 +81,7 @@ else timeout -v 600s bash -c "set -euo pipefail; $test_cmd 2>&1 | add_timestamps" &>> $tmp_file & fi timeout_pid=$! -trap "kill -- -$timeout_pid" SIGTERM SIGINT +trap "kill -- -$timeout_pid &>/dev/null; exit" SIGTERM SIGINT wait $timeout_pid code=$? @@ -124,7 +125,7 @@ function pass { # Show FAILED and exit with error code. function fail { - local line="${red}FAILED${reset}${log_info:-}: $test_cmd (${SECONDS}s) (code: $code)" + local line="${red}FAILED${reset}${log_info:-}: $test_cmd (${SECONDS}s) (code: $code)${1:-}" track_test $test_hash "$line" track_test "failed_tests" "$line" echo -e "$line" @@ -173,7 +174,8 @@ function get_owners { # If error_regex is present, check the log content. if [ -n "$error_regex" ]; then # Strip color codes from the log file. - if sed 's/\x1B\[[0-9;]*[a-zA-Z]//g' $tmp_file | grep -qE "$error_regex"; then + # Do *not* use a pipe here. grep -q will exit immediately on first match and break the pipe. + if grep -qE "$error_regex" <(sed 's/\x1B\[[0-9;]*[a-zA-Z]//g' "$tmp_file"); then echo "$entry" | jq -r '.owners[]' fi else diff --git a/spartan/bootstrap.sh b/spartan/bootstrap.sh index 6a9ec63a2ddb..cfc98b12ed23 100755 --- a/spartan/bootstrap.sh +++ b/spartan/bootstrap.sh @@ -55,14 +55,14 @@ function test_cmds { fi # Note: commands that start with 'timeout ...' override the default timeout. # TODO figure out why these take long sometimes. - echo "$hash timeout -v 20m ./spartan/bootstrap.sh test-kind-smoke" + echo "$hash ./spartan/bootstrap.sh test-kind-smoke" if [ "$CI_FULL" -eq 1 ]; then # echo "$hash timeout -v 20m ./spartan/bootstrap.sh test-kind-transfer" # TODO(#12791) re-enable # echo "$hash timeout -v 30m ./spartan/bootstrap.sh test-kind-4epochs" # echo "$hash timeout -v 30m ./spartan/bootstrap.sh test-kind-upgrade-rollup-version" # echo "$hash timeout -v 30m ./spartan/bootstrap.sh test-prod-deployment" - echo "$hash timeout -v 30m ./spartan/bootstrap.sh test-cli-upgrade-with-lock" + echo "$hash ./spartan/bootstrap.sh test-cli-upgrade-with-lock" fi if [ "$CI_NIGHTLY" -eq 1 ]; then