diff --git a/.circleci/continue/main.yml b/.circleci/continue/main.yml index 240863018284f..efde2837b5ec1 100644 --- a/.circleci/continue/main.yml +++ b/.circleci/continue/main.yml @@ -952,7 +952,7 @@ jobs: if [ "$SKIP_SLOW_TESTS" = "false" ]; then TIMEOUT="45m" fi - gotestsum --format=testname --junitfile=../tmp/test-results/cannon-64.xml --jsonfile=../tmp/testlogs/log-64.json \ + ../ops/scripts/gotestsum-split.sh --format=testname --junitfile=../tmp/test-results/cannon-64.xml --jsonfile=../tmp/testlogs/log-64.json \ -- -timeout=$TIMEOUT -parallel=$(nproc) -coverpkg=github.com/ethereum-optimism/optimism/cannon/... -coverprofile=coverage-64.out ./... working_directory: cannon - codecov/upload: @@ -1920,12 +1920,7 @@ jobs: path: ./tmp/test-results - run: name: Compress test logs - command: | - if [ -n "$CIRCLE_NODE_TOTAL" ] && [ "$CIRCLE_NODE_TOTAL" -gt 1 ]; then - tar -czf testlogs-${CIRCLE_NODE_INDEX}-of-${CIRCLE_NODE_TOTAL}.tar.gz -C ./tmp testlogs - else - tar -czf testlogs.tar.gz -C ./tmp testlogs - fi + command: tar -czf testlogs.tar.gz -C ./tmp testlogs when: always - run: name: Clean up op-deployer artifacts @@ -1933,7 +1928,7 @@ jobs: rm -rf ~/.op-deployer/* when: always - store_artifacts: - path: testlogs*.tar.gz + path: testlogs.tar.gz when: always - when: condition: "<>" diff --git a/justfile b/justfile index 8dab027f3226c..07376fdeb0299 100644 --- a/justfile +++ b/justfile @@ -277,7 +277,7 @@ _go-tests-ci-internal go_test_flags="": PARALLEL_PACKAGES=$(echo "$ALL_PACKAGES" | tr ' ' '\n' | awk -v idx="$NODE_INDEX" -v total="$NODE_TOTAL" 'NR % total == idx' | tr '\n' ' ') if [ -n "$PARALLEL_PACKAGES" ]; then echo "Node $NODE_INDEX/$NODE_TOTAL running packages: $PARALLEL_PACKAGES" - gotestsum --format=testname \ + ./ops/scripts/gotestsum-split.sh --format=testname \ --junitfile=./tmp/test-results/results-"$NODE_INDEX".xml \ --jsonfile=./tmp/testlogs/log-"$NODE_INDEX".json \ --rerun-fails=3 \ @@ -289,7 +289,7 @@ _go-tests-ci-internal go_test_flags="": exit 1 fi else - gotestsum --format=testname \ + ./ops/scripts/gotestsum-split.sh --format=testname \ --junitfile=./tmp/test-results/results.xml \ --jsonfile=./tmp/testlogs/log.json \ --rerun-fails=3 \ @@ -327,7 +327,7 @@ go-tests-fraud-proofs-ci: export MAINNET_RPC_URL="https://ci-mainnet-l1-archive.optimism.io" export NAT_INTEROP_LOADTEST_TARGET=10 export NAT_INTEROP_LOADTEST_TIMEOUT=30s - gotestsum --format=testname \ + ./ops/scripts/gotestsum-split.sh --format=testname \ --junitfile=./tmp/test-results/results.xml \ --jsonfile=./tmp/testlogs/log.json \ --rerun-fails=3 \ diff --git a/op-e2e/justfile b/op-e2e/justfile index 2fe97ed1098e4..b8ccd54fe773b 100644 --- a/op-e2e/justfile +++ b/op-e2e/justfile @@ -6,7 +6,7 @@ JUNIT_FILE := env('JUNIT_FILE', '') JSON_LOG_FILE := env('JSON_LOG_FILE', '') _go_test := if JUNIT_FILE != "" { - "OP_TESTLOG_DISABLE_COLOR=true OP_E2E_DISABLE_PARALLEL=false gotestsum --format=testname --junitfile=" + JUNIT_FILE + " --jsonfile=" + JSON_LOG_FILE + " -- -failfast" + "OP_TESTLOG_DISABLE_COLOR=true OP_E2E_DISABLE_PARALLEL=false ../ops/scripts/gotestsum-split.sh --format=testname --junitfile=" + JUNIT_FILE + " --jsonfile=" + JSON_LOG_FILE + " -- -failfast" } else { "go test" } diff --git a/ops/scripts/gotestsum-split.sh b/ops/scripts/gotestsum-split.sh new file mode 100755 index 0000000000000..e801f837a3a73 --- /dev/null +++ b/ops/scripts/gotestsum-split.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# gotestsum-split.sh — Drop-in gotestsum wrapper that splits JSON logs per test. +# +# Usage: gotestsum-split.sh [gotestsum args...] +# +# Drop-in replacement for gotestsum. Passes all arguments through, then splits +# the --jsonfile output into per-test log files via split-test-logs.sh. +# +# If --jsonfile is not provided, the wrapper adds one automatically using a +# default path (tmp/testlogs/log.json relative to cwd), ensuring per-test +# logs are always generated. +# +# Preserves gotestsum's exit code so the split runs even on test failure +# (when per-test logs are most useful for debugging). + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Scan args for --jsonfile. +json_file="" +next_is_jsonfile=false +for arg in "$@"; do + if $next_is_jsonfile; then + json_file="$arg" + break + fi + case "$arg" in + --jsonfile=*) json_file="${arg#--jsonfile=}" ; break ;; + --jsonfile) next_is_jsonfile=true ;; + esac +done + +# If --jsonfile wasn't provided, add one automatically. +if [ -z "$json_file" ]; then + json_file="tmp/testlogs/log.json" + mkdir -p "$(dirname "$json_file")" + set -- --jsonfile="$json_file" "$@" +fi + +gotestsum "$@" +GOTESTSUM_EXIT=$? + +"$SCRIPT_DIR/split-test-logs.sh" "$json_file" +SPLIT_EXIT=$? + +if [ $SPLIT_EXIT -ne 0 ]; then + exit $SPLIT_EXIT +fi +exit $GOTESTSUM_EXIT diff --git a/ops/scripts/split-test-logs.sh b/ops/scripts/split-test-logs.sh new file mode 100755 index 0000000000000..9ff875b470527 --- /dev/null +++ b/ops/scripts/split-test-logs.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# split-test-logs.sh — Split a gotestsum JSON log file into per-test log files. +# +# Usage: split-test-logs.sh +# +# Reads a gotestsum --jsonfile output and writes one file per test containing +# that test's output lines. Output goes to a "per-test" sibling directory next +# to the JSON file, organized as /.log. +# +# Fails if the jsonfile doesn't exist or python3 is not available. + +set -euo pipefail + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +json_file="$1" + +if [ ! -f "$json_file" ]; then + echo "Error: JSON log file not found: $json_file" >&2 + exit 1 +fi + +if ! command -v python3 &>/dev/null; then + echo "Error: python3 is required but not found" >&2 + exit 1 +fi + +output_dir="$(dirname "$json_file")/per-test" + +python3 - "$json_file" "$output_dir" <<'PYEOF' +import json, os, sys, re + +json_file = sys.argv[1] +output_dir = sys.argv[2] + +handles = {} + +def get_handle(path): + if path not in handles: + os.makedirs(os.path.dirname(path), exist_ok=True) + handles[path] = open(path, "w") + return handles[path] + +def sanitize(name): + return re.sub(r'[<>:"|?*]', '_', name) + +count = 0 +with open(json_file) as f: + for line in f: + line = line.strip() + if not line: + continue + try: + ev = json.loads(line) + except json.JSONDecodeError: + continue + + test = ev.get("Test") + action = ev.get("Action") + output = ev.get("Output") + package = ev.get("Package", "") + + if not test or action != "output" or output is None: + continue + + pkg_dir = sanitize(package.replace("/", ".")) + test_name = sanitize(test) + file_path = os.path.join(output_dir, pkg_dir, f"{test_name}.log") + + fh = get_handle(file_path) + fh.write(output) + count += 1 + +for fh in handles.values(): + fh.close() + +print(f"Split {count} output lines across {len(handles)} test files in {output_dir}") +PYEOF diff --git a/rust/kona/tests/justfile b/rust/kona/tests/justfile index deca5e61719c2..6139e49d132c9 100644 --- a/rust/kona/tests/justfile +++ b/rust/kona/tests/justfile @@ -132,7 +132,7 @@ test-e2e-sysgo-run BINARY="node" GO_PKG_NAME="node/common" DEVNET="simple-kona" # Run the test with count=1 to avoid caching the test results. cd {{SOURCE}} mkdir -p ./tmp/test-results ./tmp/testlogs - gotestsum --format=testname \ + {{SOURCE}}/../../../ops/scripts/gotestsum-split.sh --format=testname \ --junitfile=./tmp/test-results/results.xml \ --jsonfile=./tmp/testlogs/log.json \ -- -count=1 -timeout 40m ./$GO_PKG_NAME $FILTER @@ -225,11 +225,14 @@ action-tests-single-run test_name='Test_ProgramAction' parallel="0" *args='': --jsonfile=./tmp/testlogs/log-$NODE_INDEX.json \ -- -count=1 -parallel=$PARALLEL -coverprofile=coverage-$NODE_INDEX.out -timeout=60m -run '^{}$' \ < /tmp/tests_shard.txt + # xargs calls gotestsum multiple times appending to the same jsonfile, + # so split once at the end rather than wrapping each call. + {{SOURCE}}/../../../ops/scripts/split-test-logs.sh ./tmp/testlogs/log-$NODE_INDEX.json exit 0 fi - gotestsum --format=testname \ + {{SOURCE}}/../../../ops/scripts/gotestsum-split.sh --format=testname \ --junitfile=./tmp/test-results/results.xml \ --jsonfile=./tmp/testlogs/log.json \ -- -count=1 -parallel=$PARALLEL -coverprofile=coverage.out -timeout=60m -run "{{test_name}}" @@ -248,7 +251,7 @@ action-tests-interop-run test_name='TestInteropFaultProofs' *args='': # https://github.com/gotestyourself/gotestsum/blob/b4b13345fee56744d80016a20b760d3599c13504/testjson/format.go#L442-L444 echo "Running action tests for the client program on the native target" - cd {{SOURCE}}/../../op-e2e/actions/interop && GITHUB_ACTIONS=false gotestsum --format=short-verbose -- -count=1 -timeout 60m -run "{{test_name}}" {{args}} + cd {{SOURCE}}/../../op-e2e/actions/interop && GITHUB_ACTIONS=false {{SOURCE}}/../../../ops/scripts/gotestsum-split.sh --format=short-verbose -- -count=1 -timeout 60m -run "{{test_name}}" {{args}} update-packages: #!/bin/bash diff --git a/rust/op-reth/crates/tests/justfile b/rust/op-reth/crates/tests/justfile index 832c6f0d97b6c..9d21f80cbd996 100644 --- a/rust/op-reth/crates/tests/justfile +++ b/rust/op-reth/crates/tests/justfile @@ -72,4 +72,5 @@ test-e2e-sysgo: build unzip-contract-artifacts build-contracts export OP_RETH_EXEC_PATH="{{SOURCE_DIR}}/../../../target/debug/op-reth" export OP_DEVSTACK_PROOF_SEQUENCER_EL="{{OP_DEVSTACK_PROOF_SEQUENCER_EL}}" export OP_DEVSTACK_PROOF_VALIDATOR_EL="{{OP_DEVSTACK_PROOF_VALIDATOR_EL}}" - go test -count=1 -timeout 40m -v ./{{GO_PKG_NAME}} + {{SOURCE_DIR}}/../../../../ops/scripts/gotestsum-split.sh --format=testname \ + -- -count=1 -timeout 40m ./{{GO_PKG_NAME}} diff --git a/rust/op-reth/tests/justfile b/rust/op-reth/tests/justfile index a19114c5ce893..9beb5b8b3a6d8 100644 --- a/rust/op-reth/tests/justfile +++ b/rust/op-reth/tests/justfile @@ -42,7 +42,7 @@ test-e2e-sysgo: build-contracts export OP_RETH_EXEC_PATH="{{SOURCE_DIR}}/../../target/release/op-reth" export OP_DEVSTACK_PROOF_SEQUENCER_EL="{{OP_DEVSTACK_PROOF_SEQUENCER_EL}}" export OP_DEVSTACK_PROOF_VALIDATOR_EL="{{OP_DEVSTACK_PROOF_VALIDATOR_EL}}" - gotestsum --format=testname \ + {{SOURCE_DIR}}/../../../ops/scripts/gotestsum-split.sh --format=testname \ --junitfile=tmp/test-results/results.xml \ --jsonfile=tmp/testlogs/log.json \ -- -count=1 -timeout 40m ./{{GO_PKG_NAME}}