feat: split gotestsum logs into per-test files

.circleci/continue/main.yml

@@ -952,7 +952,7 @@ jobs:
             if [ "$SKIP_SLOW_TESTS" = "false" ]; then
               TIMEOUT="45m"
             fi
-            gotestsum --format=testname --junitfile=../tmp/test-results/cannon-64.xml --jsonfile=../tmp/testlogs/log-64.json \
+            ../ops/scripts/gotestsum-split.sh --format=testname --junitfile=../tmp/test-results/cannon-64.xml --jsonfile=../tmp/testlogs/log-64.json \
             -- -timeout=$TIMEOUT -parallel=$(nproc) -coverpkg=github.com/ethereum-optimism/optimism/cannon/... -coverprofile=coverage-64.out ./...
           working_directory: cannon
       - codecov/upload:
@@ -1920,20 +1920,15 @@ jobs:
           path: ./tmp/test-results
       - run:
           name: Compress test logs
-          command: |
-            if [ -n "$CIRCLE_NODE_TOTAL" ] && [ "$CIRCLE_NODE_TOTAL" -gt 1 ]; then
-              tar -czf testlogs-${CIRCLE_NODE_INDEX}-of-${CIRCLE_NODE_TOTAL}.tar.gz -C ./tmp testlogs
-            else
-              tar -czf testlogs.tar.gz -C ./tmp testlogs
-            fi
+          command: tar -czf testlogs.tar.gz -C ./tmp testlogs
           when: always
       - run:
           name: Clean up op-deployer artifacts
           command: |
             rm -rf ~/.op-deployer/*
           when: always
       - store_artifacts:
-          path: testlogs*.tar.gz
+          path: testlogs.tar.gz
           when: always
       - when:
           condition: "<<parameters.notify>>"

justfile

@@ -277,7 +277,7 @@ _go-tests-ci-internal go_test_flags="":
       PARALLEL_PACKAGES=$(echo "$ALL_PACKAGES" | tr ' ' '\n' | awk -v idx="$NODE_INDEX" -v total="$NODE_TOTAL" 'NR % total == idx' | tr '\n' ' ')
       if [ -n "$PARALLEL_PACKAGES" ]; then
           echo "Node $NODE_INDEX/$NODE_TOTAL running packages: $PARALLEL_PACKAGES"
-          gotestsum --format=testname \
+          ./ops/scripts/gotestsum-split.sh --format=testname \
               --junitfile=./tmp/test-results/results-"$NODE_INDEX".xml \
               --jsonfile=./tmp/testlogs/log-"$NODE_INDEX".json \
               --rerun-fails=3 \
@@ -289,7 +289,7 @@ _go-tests-ci-internal go_test_flags="":
           exit 1
       fi
   else
-      gotestsum --format=testname \
+      ./ops/scripts/gotestsum-split.sh --format=testname \
           --junitfile=./tmp/test-results/results.xml \
           --jsonfile=./tmp/testlogs/log.json \
           --rerun-fails=3 \
@@ -327,7 +327,7 @@ go-tests-fraud-proofs-ci:
   export MAINNET_RPC_URL="https://ci-mainnet-l1-archive.optimism.io"
   export NAT_INTEROP_LOADTEST_TARGET=10
   export NAT_INTEROP_LOADTEST_TIMEOUT=30s
-  gotestsum --format=testname \
+  ./ops/scripts/gotestsum-split.sh --format=testname \
       --junitfile=./tmp/test-results/results.xml \
       --jsonfile=./tmp/testlogs/log.json \
       --rerun-fails=3 \

op-e2e/justfile

@@ -6,7 +6,7 @@ JUNIT_FILE := env('JUNIT_FILE', '')
 JSON_LOG_FILE := env('JSON_LOG_FILE', '')
 
 _go_test := if JUNIT_FILE != "" {
-    "OP_TESTLOG_DISABLE_COLOR=true OP_E2E_DISABLE_PARALLEL=false gotestsum --format=testname --junitfile=" + JUNIT_FILE + " --jsonfile=" + JSON_LOG_FILE + " -- -failfast"
+    "OP_TESTLOG_DISABLE_COLOR=true OP_E2E_DISABLE_PARALLEL=false ../ops/scripts/gotestsum-split.sh --format=testname --junitfile=" + JUNIT_FILE + " --jsonfile=" + JSON_LOG_FILE + " -- -failfast"
 } else {
     "go test"
 }

ops/scripts/gotestsum-split.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# gotestsum-split.sh — Drop-in gotestsum wrapper that splits JSON logs per test.
+#
+# Usage: gotestsum-split.sh [gotestsum args...]
+#
+# Drop-in replacement for gotestsum. Passes all arguments through, then splits
+# the --jsonfile output into per-test log files via split-test-logs.sh.
+#
+# If --jsonfile is not provided, the wrapper adds one automatically using a
+# default path (tmp/testlogs/log.json relative to cwd), ensuring per-test
+# logs are always generated.
+#
+# Preserves gotestsum's exit code so the split runs even on test failure
+# (when per-test logs are most useful for debugging).
+
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+# Scan args for --jsonfile.
+json_file=""
+next_is_jsonfile=false
+for arg in "$@"; do
+  if $next_is_jsonfile; then
+    json_file="$arg"
+    break
+  fi
+  case "$arg" in
+    --jsonfile=*) json_file="${arg#--jsonfile=}" ; break ;;
+    --jsonfile)   next_is_jsonfile=true ;;
+  esac
+done
+
+# If --jsonfile wasn't provided, add one automatically.
+if [ -z "$json_file" ]; then
+  json_file="tmp/testlogs/log.json"
+  mkdir -p "$(dirname "$json_file")"
+  set -- --jsonfile="$json_file" "$@"
+fi
+
+gotestsum "$@"
+GOTESTSUM_EXIT=$?
+
+"$SCRIPT_DIR/split-test-logs.sh" "$json_file"
+SPLIT_EXIT=$?
+
+if [ $SPLIT_EXIT -ne 0 ]; then
+  exit $SPLIT_EXIT
+fi
+exit $GOTESTSUM_EXIT

ops/scripts/split-test-logs.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+# split-test-logs.sh — Split a gotestsum JSON log file into per-test log files.
+#
+# Usage: split-test-logs.sh <jsonfile>
+#
+# Reads a gotestsum --jsonfile output and writes one file per test containing
+# that test's output lines. Output goes to a "per-test" sibling directory next
+# to the JSON file, organized as <package>/<TestName>.log.
+#
+# Fails if the jsonfile doesn't exist or python3 is not available.
+
+set -euo pipefail
+
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 <jsonfile>" >&2
+  exit 1
+fi
+
+json_file="$1"
+
+if [ ! -f "$json_file" ]; then
+  echo "Error: JSON log file not found: $json_file" >&2
+  exit 1
+fi
+
+if ! command -v python3 &>/dev/null; then
+  echo "Error: python3 is required but not found" >&2
+  exit 1
+fi
+
+output_dir="$(dirname "$json_file")/per-test"
+
+python3 - "$json_file" "$output_dir" <<'PYEOF'
+import json, os, sys, re
+
+json_file = sys.argv[1]
+output_dir = sys.argv[2]
+
+handles = {}
+
+def get_handle(path):
+    if path not in handles:
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        handles[path] = open(path, "w")
+    return handles[path]
+
+def sanitize(name):
+    return re.sub(r'[<>:"|?*]', '_', name)
+
+count = 0
+with open(json_file) as f:
+    for line in f:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            ev = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+
+        test = ev.get("Test")
+        action = ev.get("Action")
+        output = ev.get("Output")
+        package = ev.get("Package", "")
+
+        if not test or action != "output" or output is None:
+            continue
+
+        pkg_dir = sanitize(package.replace("/", "."))
+        test_name = sanitize(test)
+        file_path = os.path.join(output_dir, pkg_dir, f"{test_name}.log")
+
+        fh = get_handle(file_path)
+        fh.write(output)
+        count += 1
+
+for fh in handles.values():
+    fh.close()
+
+print(f"Split {count} output lines across {len(handles)} test files in {output_dir}")
+PYEOF

rust/kona/tests/justfile

@@ -132,7 +132,7 @@ test-e2e-sysgo-run BINARY="node" GO_PKG_NAME="node/common" DEVNET="simple-kona"
     # Run the test with count=1 to avoid caching the test results.
     cd {{SOURCE}}
     mkdir -p ./tmp/test-results ./tmp/testlogs
-    gotestsum --format=testname \
+    {{SOURCE}}/../../../ops/scripts/gotestsum-split.sh --format=testname \
         --junitfile=./tmp/test-results/results.xml \
         --jsonfile=./tmp/testlogs/log.json \
         -- -count=1 -timeout 40m ./$GO_PKG_NAME $FILTER
@@ -225,11 +225,14 @@ action-tests-single-run test_name='Test_ProgramAction' parallel="0" *args='':
       --jsonfile=./tmp/testlogs/log-$NODE_INDEX.json \
       -- -count=1 -parallel=$PARALLEL -coverprofile=coverage-$NODE_INDEX.out -timeout=60m -run '^{}$' \
       < /tmp/tests_shard.txt
+    # xargs calls gotestsum multiple times appending to the same jsonfile,
+    # so split once at the end rather than wrapping each call.
+    {{SOURCE}}/../../../ops/scripts/split-test-logs.sh ./tmp/testlogs/log-$NODE_INDEX.json
 
     exit 0
   fi
 
-  gotestsum --format=testname \
+  {{SOURCE}}/../../../ops/scripts/gotestsum-split.sh --format=testname \
     --junitfile=./tmp/test-results/results.xml \
     --jsonfile=./tmp/testlogs/log.json \
     -- -count=1 -parallel=$PARALLEL -coverprofile=coverage.out -timeout=60m -run "{{test_name}}"
@@ -248,7 +251,7 @@ action-tests-interop-run test_name='TestInteropFaultProofs' *args='':
   # https://github.com/gotestyourself/gotestsum/blob/b4b13345fee56744d80016a20b760d3599c13504/testjson/format.go#L442-L444
   echo "Running action tests for the client program on the native target"
 
-  cd {{SOURCE}}/../../op-e2e/actions/interop && GITHUB_ACTIONS=false gotestsum --format=short-verbose -- -count=1 -timeout 60m -run "{{test_name}}" {{args}}
+  cd {{SOURCE}}/../../op-e2e/actions/interop && GITHUB_ACTIONS=false {{SOURCE}}/../../../ops/scripts/gotestsum-split.sh --format=short-verbose -- -count=1 -timeout 60m -run "{{test_name}}" {{args}}
 
 update-packages:
     #!/bin/bash

rust/op-reth/crates/tests/justfile

@@ -72,4 +72,5 @@ test-e2e-sysgo: build unzip-contract-artifacts build-contracts
   export OP_RETH_EXEC_PATH="{{SOURCE_DIR}}/../../../target/debug/op-reth"
   export OP_DEVSTACK_PROOF_SEQUENCER_EL="{{OP_DEVSTACK_PROOF_SEQUENCER_EL}}"
   export OP_DEVSTACK_PROOF_VALIDATOR_EL="{{OP_DEVSTACK_PROOF_VALIDATOR_EL}}"
-  go test -count=1 -timeout 40m -v ./{{GO_PKG_NAME}}
+  {{SOURCE_DIR}}/../../../../ops/scripts/gotestsum-split.sh --format=testname \
+    -- -count=1 -timeout 40m ./{{GO_PKG_NAME}}

rust/op-reth/tests/justfile

@@ -42,7 +42,7 @@ test-e2e-sysgo: build-contracts
   export OP_RETH_EXEC_PATH="{{SOURCE_DIR}}/../../target/release/op-reth"
   export OP_DEVSTACK_PROOF_SEQUENCER_EL="{{OP_DEVSTACK_PROOF_SEQUENCER_EL}}"
   export OP_DEVSTACK_PROOF_VALIDATOR_EL="{{OP_DEVSTACK_PROOF_VALIDATOR_EL}}"
-  gotestsum --format=testname \
+  {{SOURCE_DIR}}/../../../ops/scripts/gotestsum-split.sh --format=testname \
     --junitfile=tmp/test-results/results.xml \
     --jsonfile=tmp/testlogs/log.json \
     -- -count=1 -timeout 40m ./{{GO_PKG_NAME}}