From 0ffd986edc5e5b86217584703e1c9339776a3c93 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Tue, 11 Aug 2026 21:49:06 -0500 Subject: [PATCH] fix(evals): separate stdout and stderr capture in the eval harness The eval harness sent stderr into the same file as stdout. A stderr diagnostic line then broke the JSON parse for --json cases. The CLI correctly writes a line like "[error] Unknown output type from daemon: ..." to stderr on some turns. That line is correct CLI behavior, not a CLI bug. The extra line in the stdout file broke stdout_json_envelope_valid and produced a false eval failure. A controlled A/B run proved this: two of three failures for one case came from this exact issue, and one of the two "failed" runs had in fact behaved correctly. This change writes stdout and stderr to two separate files, in both run_prompt (single-turn) and run_prompt_resume (multi-turn --resume). An audit of every assertion helper (stdout_contains, stdout_json_*, and all direct $STDOUT_FILE reads) and every case assertion function found none that reads stderr text from the old shared file. So this change does not alter any case result; it only removes a source of false failures. The archive step now copies the stderr file next to the stdout file for each run, in evals/runs//stdout/. A failed run stays diagnosable with both streams present. Verification: bash -n evals/run-evals.sh passes. dotnet slopwatch analyze finds no new issue. The harness has no self-test or lint mode. A live eval run needs a real model endpoint, so no live eval happened for this change. --- evals/run-evals.sh | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/evals/run-evals.sh b/evals/run-evals.sh index 6a0338215..a15b86e4b 100755 --- a/evals/run-evals.sh +++ b/evals/run-evals.sh @@ -84,6 +84,7 @@ RESULTS_DB="" # Per-prompt state (set by run_prompt, read by assertion helpers) STDOUT_FILE="" +STDERR_FILE="" DAEMON_LOG_LINES_BEFORE=0 # ─── Prerequisites ──────────────────────────────────────────────────────────── @@ -136,7 +137,7 @@ check_prerequisites() { fi # Eval-owned temp roots: everything under $EVAL_HOME is torn down by the - # EXIT trap. $TMPDIR_EVAL holds per-prompt stdout captures. + # EXIT trap. $TMPDIR_EVAL holds per-prompt stdout and stderr captures. EVAL_HOME=$(mktemp -d -t netclaw-eval-home-XXXXXX) TMPDIR_EVAL=$(mktemp -d -t netclaw-eval-tmp-XXXXXX) RESULTS_DIR="$EVAL_HOME/evals" @@ -198,7 +199,7 @@ cleanup_eval_env() { if [[ -n "${EVAL_CONTAINER_NAME:-}" ]]; then docker stop "$EVAL_CONTAINER_NAME" >/dev/null 2>&1 || true fi - # TMPDIR_EVAL only holds host-owned per-prompt stdout captures, so a + # TMPDIR_EVAL only holds host-owned per-prompt stdout/stderr captures, so a # plain rm always succeeds — no force_rmrf fallback needed. if [[ -n "${TMPDIR_EVAL:-}" && -d "$TMPDIR_EVAL" ]]; then rm -rf "$TMPDIR_EVAL" @@ -238,10 +239,14 @@ archive_eval_run() { cp "$RESULTS_DB" "$archive_dir/results.db" 2>/dev/null || true fi - # Copy stdout captures + # Copy stdout and stderr captures. Both land in the same archive_dir/stdout + # directory — the stdout_*/stderr_* filename prefix already tells them apart, + # and keeping stderr alongside stdout means a failing run's diagnostics + # (denied approvals, daemon errors) are always one directory away. if [[ -n "${TMPDIR_EVAL:-}" && -d "$TMPDIR_EVAL" ]]; then mkdir -p "$archive_dir/stdout" cp "$TMPDIR_EVAL"/stdout_*.txt "$archive_dir/stdout/" 2>/dev/null || true + cp "$TMPDIR_EVAL"/stderr_*.txt "$archive_dir/stdout/" 2>/dev/null || true fi # Write run metadata, including the immutable image identity so before/after @@ -762,7 +767,10 @@ check_daemon_alive() { run_prompt() { local prompt="$1" local output_format="${2:-text}" - STDOUT_FILE="$TMPDIR_EVAL/stdout_$(date +%s%N).txt" + local ts + ts="$(date +%s%N)" + STDOUT_FILE="$TMPDIR_EVAL/stdout_${ts}.txt" + STDERR_FILE="$TMPDIR_EVAL/stderr_${ts}.txt" # Record daemon log position before the prompt (the daemon writes to a # daily-rotating file at /root/.netclaw/logs/daemon-YYYY-MM-DD.log, and @@ -780,10 +788,15 @@ run_prompt() { output_args+=(--json) fi + # Stdout and stderr go to separate files. A merged capture corrupts + # --json assertions: the CLI writes legitimate diagnostics (for example + # "[error] Unknown output type from daemon: ...") to stderr, and a + # trailing diagnostic line breaks jq's parse of the JSON envelope on + # stdout, producing a false eval failure rather than a real one. NETCLAW_DAEMON_ENDPOINT="http://127.0.0.1:$EVAL_PORT" \ NETCLAW_HOME="$EVAL_HOME" \ timeout "$PROMPT_TIMEOUT" "$NETCLAW_BIN" chat -p "${output_args[@]}" "$prompt" \ - > "$STDOUT_FILE" 2>&1 || true + > "$STDOUT_FILE" 2> "$STDERR_FILE" || true # Brief pause for daemon log flush sleep 2 @@ -792,24 +805,35 @@ run_prompt() { ## Runs a prompt against an existing (or new) named session via `chat -p --resume`. ## Appends output to a per-turn file AND the shared STDOUT_FILE so existing ## assertion helpers (stdout_contains, etc.) see the full concatenated output. +## Stderr is captured the same way, into a separate shared STDERR_FILE, so +## stdout stays pure for assertions (see run_prompt for why that matters). ## Args: session_id, prompt run_prompt_resume() { local session_id="$1" local prompt="$2" - local turn_file="$TMPDIR_EVAL/stdout_$(date +%s%N)_turn.txt" + local ts + ts="$(date +%s%N)" + local turn_file="$TMPDIR_EVAL/stdout_${ts}_turn.txt" + local turn_stderr_file="$TMPDIR_EVAL/stderr_${ts}_turn.txt" if [[ ! -x "$NETCLAW_BIN" ]]; then echo "ERROR: eval CLI disappeared during the run: $NETCLAW_BIN" >&2 exit 2 fi - # First call in a multi-turn case: open a fresh shared STDOUT_FILE. + # First call in a multi-turn case: open fresh shared STDOUT_FILE/STDERR_FILE. if [[ -z "${MULTI_TURN_STDOUT_FILE:-}" ]]; then MULTI_TURN_STDOUT_FILE="$TMPDIR_EVAL/stdout_$(date +%s%N)_multi.txt" : > "$MULTI_TURN_STDOUT_FILE" fi STDOUT_FILE="$MULTI_TURN_STDOUT_FILE" + if [[ -z "${MULTI_TURN_STDERR_FILE:-}" ]]; then + MULTI_TURN_STDERR_FILE="$TMPDIR_EVAL/stderr_$(date +%s%N)_multi.txt" + : > "$MULTI_TURN_STDERR_FILE" + fi + STDERR_FILE="$MULTI_TURN_STDERR_FILE" + if [[ -f "$DAEMON_LOG" ]]; then DAEMON_LOG_LINES_BEFORE=$(wc -l < "$DAEMON_LOG") else @@ -819,10 +843,11 @@ run_prompt_resume() { NETCLAW_DAEMON_ENDPOINT="http://127.0.0.1:$EVAL_PORT" \ NETCLAW_HOME="$EVAL_HOME" \ timeout "$PROMPT_TIMEOUT" "$NETCLAW_BIN" chat -p --resume "$session_id" "$prompt" \ - > "$turn_file" 2>&1 || true + > "$turn_file" 2> "$turn_stderr_file" || true - # Append this turn's output to the shared file so assertions see all turns. + # Append this turn's output to the shared files so assertions see all turns. cat "$turn_file" >> "$STDOUT_FILE" + cat "$turn_stderr_file" >> "$STDERR_FILE" # Per-turn metrics — read the usage line from this turn's file only. LAST_TURN_USAGE_LINE=$(grep -ao '\[usage\].*' "$turn_file" 2>/dev/null | tail -1 || echo "") @@ -857,6 +882,7 @@ run_multi_turn_case() { # Fresh session per run so runs don't pollute each other. local session_id="eval/${case_name}-run${run}-$$" MULTI_TURN_STDOUT_FILE="" + MULTI_TURN_STDERR_FILE="" local setup_fn="setup_${case_name}" if declare -f "$setup_fn" >/dev/null 2>&1; then