Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 142 additions & 0 deletions scripts/post-fix-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,148 @@ run_precommit_retry_test "precommit-passes-with-unstaged" \
run_precommit_retry_test "precommit-retry-passes-but-left-unstaged" \
"1" "yes" "0" "blocked:retry-left-unstaged" "yes"

# ---------------------------------------------------------------------------
# Test helper — reimplements the FULLSEND_VALIDATED_ITERATION_DIR selection
# logic from post-fix.sh section 5. Given an env var value and a set of files
# on disk, returns which result file would be selected.
#
# Mirrors the three-branch logic: expected filename → result.json fallback →
# fail closed with error (no silent rescan).
# ---------------------------------------------------------------------------
resolve_fix_result() {
local validated_dir="$1" # value of FULLSEND_VALIDATED_ITERATION_DIR ("" = unset)
local run_dir="$2" # directory containing iteration-*/output/

if [ -n "${validated_dir}" ]; then
if [ -f "${validated_dir}/fix-result.json" ]; then
echo "${validated_dir}/fix-result.json"
elif [ -f "${validated_dir}/result.json" ]; then
echo "${validated_dir}/result.json"
else
echo "error:neither-filename"
fi
else
local result=""
for dir in "${run_dir}"/iteration-*/output; do
if [ -f "${dir}/fix-result.json" ]; then
result="${dir}/fix-result.json"
fi
done
if [ -z "${result}" ]; then
echo "error:not-found"
else
echo "${result}"
fi
fi
}

RESOLVE_TMPDIR="$(mktemp -d)"

run_resolve_test() {
local test_name="$1"
local setup_fn="$2"
local expected="$3"

local run_dir="${RESOLVE_TMPDIR}/${test_name}"
local validated_dir="${run_dir}/validated-output"
mkdir -p "${run_dir}"

# Let the setup function create the directory structure.
${setup_fn} "${run_dir}" "${validated_dir}"

local actual
actual="$(resolve_fix_result "${validated_dir}" "${run_dir}")"

if [ "${actual}" != "${expected}" ]; then
echo "FAIL: ${test_name}"
echo " expected: '${expected}'"
echo " actual: '${actual}'"
FAILURES=$((FAILURES + 1))
return
fi

echo "PASS: ${test_name}"
}

run_resolve_test_unset() {
local test_name="$1"
local setup_fn="$2"
local expected="$3"

local run_dir="${RESOLVE_TMPDIR}/${test_name}"
mkdir -p "${run_dir}"

${setup_fn} "${run_dir}" ""

local actual
actual="$(resolve_fix_result "" "${run_dir}")"

if [ "${actual}" != "${expected}" ]; then
echo "FAIL: ${test_name}"
echo " expected: '${expected}'"
echo " actual: '${actual}'"
FAILURES=$((FAILURES + 1))
return
fi

echo "PASS: ${test_name}"
}

# Setup: validated dir has fix-result.json
setup_fix_expected() {
local run_dir="$1"
local validated_dir="$2"
mkdir -p "${validated_dir}"
echo '{}' > "${validated_dir}/fix-result.json"
# Also place a file in iteration-2 to verify it's NOT used.
mkdir -p "${run_dir}/iteration-2/output"
echo '{}' > "${run_dir}/iteration-2/output/fix-result.json"
}

# Setup: validated dir has only result.json
setup_fix_fallback() {
local run_dir="$1"
local validated_dir="$2"
mkdir -p "${validated_dir}"
echo '{}' > "${validated_dir}/result.json"
}

# Setup: validated dir has neither filename
setup_fix_neither() {
local run_dir="$1"
local validated_dir="$2"
mkdir -p "${validated_dir}"
}

# Setup: env var unset, iteration dirs present (backward compat)
setup_fix_iteration_scan() {
local run_dir="$1"
mkdir -p "${run_dir}/iteration-1/output"
mkdir -p "${run_dir}/iteration-2/output"
echo '{}' > "${run_dir}/iteration-1/output/fix-result.json"
echo '{}' > "${run_dir}/iteration-2/output/fix-result.json"
}

# --- FULLSEND_VALIDATED_ITERATION_DIR test cases ---

run_resolve_test "fix-validated-dir-expected-filename" \
setup_fix_expected \
"${RESOLVE_TMPDIR}/fix-validated-dir-expected-filename/validated-output/fix-result.json"

run_resolve_test "fix-validated-dir-fallback-filename" \
setup_fix_fallback \
"${RESOLVE_TMPDIR}/fix-validated-dir-fallback-filename/validated-output/result.json"

run_resolve_test "fix-validated-dir-neither-filename" \
setup_fix_neither \
"error:neither-filename"
Comment thread
waynesun09 marked this conversation as resolved.
Resolved

run_resolve_test_unset "fix-unset-falls-back-to-scan" \
setup_fix_iteration_scan \
"${RESOLVE_TMPDIR}/fix-unset-falls-back-to-scan/iteration-2/output/fix-result.json"

rm -rf "${RESOLVE_TMPDIR}"

# --- Summary ---

echo ""
Expand Down
31 changes: 21 additions & 10 deletions scripts/post-fix.sh
Original file line number Diff line number Diff line change
Expand Up @@ -817,18 +817,29 @@ if [ ! -f "${PROCESS_SCRIPT}" ]; then
fi
fi

# Find fix-result.json in the output directory.
# Find fix-result.json — prefer the validated iteration when set.
# RUN_DIR is the original cwd (runDir = <outputBase>/<sandboxName>), saved
# before we cd'd into REPO_DIR. The agent writes its structured output to
# iteration-<N>/output/fix-result.json within runDir. Uses glob order
# (naturally ascending iteration numbers) to find the last iteration,
# matching the pattern in post-triage.sh.
RESULT_FILE=""
for dir in "${RUN_DIR}"/iteration-*/output; do
if [ -f "${dir}/fix-result.json" ]; then
RESULT_FILE="${dir}/fix-result.json"
fi
done
# iteration-<N>/output/fix-result.json within runDir.
if [ -n "${FULLSEND_VALIDATED_ITERATION_DIR:-}" ]; then
if [ -f "${FULLSEND_VALIDATED_ITERATION_DIR}/fix-result.json" ]; then
RESULT_FILE="${FULLSEND_VALIDATED_ITERATION_DIR}/fix-result.json"
elif [ -f "${FULLSEND_VALIDATED_ITERATION_DIR}/result.json" ]; then
RESULT_FILE="${FULLSEND_VALIDATED_ITERATION_DIR}/result.json"
else
Comment thread
waynesun09 marked this conversation as resolved.
Resolved
gha_echo error "FULLSEND_VALIDATED_ITERATION_DIR is set but contains neither fix-result.json nor result.json"
RESULT_FILE=""
Comment thread
waynesun09 marked this conversation as resolved.
Resolved
fi
else
# Backward compatibility: scan iteration-N/ subdirectories for the last
# iteration's output (glob order = naturally ascending iteration numbers).
RESULT_FILE=""
for dir in "${RUN_DIR}"/iteration-*/output; do
if [ -f "${dir}/fix-result.json" ]; then
RESULT_FILE="${dir}/fix-result.json"
fi
done
fi

if [ -z "${RESULT_FILE}" ] || [ ! -f "${RESULT_FILE}" ]; then
gha_echo warning "No fix-result.json found — skipping summary comment"
Expand Down
29 changes: 20 additions & 9 deletions scripts/post-fix.src.sh
Original file line number Diff line number Diff line change
Expand Up @@ -370,18 +370,29 @@ if [ ! -f "${PROCESS_SCRIPT}" ]; then
fi
fi

# Find fix-result.json in the output directory.
# Find fix-result.json — prefer the validated iteration when set.
# RUN_DIR is the original cwd (runDir = <outputBase>/<sandboxName>), saved
# before we cd'd into REPO_DIR. The agent writes its structured output to
# iteration-<N>/output/fix-result.json within runDir. Uses glob order
# (naturally ascending iteration numbers) to find the last iteration,
# matching the pattern in post-triage.sh.
RESULT_FILE=""
for dir in "${RUN_DIR}"/iteration-*/output; do
if [ -f "${dir}/fix-result.json" ]; then
RESULT_FILE="${dir}/fix-result.json"
# iteration-<N>/output/fix-result.json within runDir.
if [ -n "${FULLSEND_VALIDATED_ITERATION_DIR:-}" ]; then
if [ -f "${FULLSEND_VALIDATED_ITERATION_DIR}/fix-result.json" ]; then
RESULT_FILE="${FULLSEND_VALIDATED_ITERATION_DIR}/fix-result.json"
elif [ -f "${FULLSEND_VALIDATED_ITERATION_DIR}/result.json" ]; then
RESULT_FILE="${FULLSEND_VALIDATED_ITERATION_DIR}/result.json"
else
gha_echo error "FULLSEND_VALIDATED_ITERATION_DIR is set but contains neither fix-result.json nor result.json"
RESULT_FILE=""
fi
done
else
# Backward compatibility: scan iteration-N/ subdirectories for the last
# iteration's output (glob order = naturally ascending iteration numbers).
RESULT_FILE=""
for dir in "${RUN_DIR}"/iteration-*/output; do
if [ -f "${dir}/fix-result.json" ]; then
RESULT_FILE="${dir}/fix-result.json"
fi
done
fi

if [ -z "${RESULT_FILE}" ] || [ ! -f "${RESULT_FILE}" ]; then
gha_echo warning "No fix-result.json found — skipping summary comment"
Expand Down
82 changes: 82 additions & 0 deletions scripts/post-prioritize-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -636,6 +636,88 @@ export GITHUB_CSMA_MAX_ATTEMPTS=3
run_test_failure_stderr "exit0-rate-limit-exhausted" "100" "rate limit exceeded" "exit0-ratelimit"
unset GITHUB_CSMA_MAX_ATTEMPTS

# ---------------------------------------------------------------------------
# FULLSEND_VALIDATED_ITERATION_DIR tests
# Verify that when FULLSEND_VALIDATED_ITERATION_DIR is set, the script reads
# from that directory instead of scanning iteration-*/output.
# ---------------------------------------------------------------------------

run_validated_dir_test() {
local test_name="$1"
local validated_dir_file="$2" # "agent-result.json", "result.json", or "none"
local expected_stdout="$3"
local expect_failure="${4:-false}"

local run_dir="${TEST_TMPDIR}/run-${test_name}"
local validated_dir="${run_dir}/validated-output"
mkdir -p "${validated_dir}"

# Place the fixture in the validated dir under the specified filename.
if [[ "${validated_dir_file}" != "none" ]]; then
echo "${FIXTURE_JSON}" > "${validated_dir}/${validated_dir_file}"
fi

# Also place a DIFFERENT result in iteration-2 to ensure the script does
# NOT fall back to scanning when the validated dir is set.
mkdir -p "${run_dir}/iteration-2/output"
echo '{"reach":0,"impact":0,"confidence":0,"effort":1,"reasoning":{"reach":"wrong","impact":"wrong","confidence":"wrong","effort":"wrong"}}' \
> "${run_dir}/iteration-2/output/agent-result.json"

: > "${GH_LOG}"
rm -f "${GH_FAIL_COUNT}"
unset GH_CSMA_FAIL_UNTIL GH_CSMA_FAIL_MODE 2>/dev/null || true

local exit_code=0
(
cd "${run_dir}"
export FULLSEND_VALIDATED_ITERATION_DIR="${validated_dir}"
bash "${POST_SCRIPT}"
) > "${TEST_TMPDIR}/stdout-${test_name}.log" 2>&1 || exit_code=$?

if [[ "${expect_failure}" == "true" ]]; then
if [[ ${exit_code} -eq 0 ]]; then
echo "FAIL: ${test_name} — expected failure but got success"
FAILURES=$((FAILURES + 1))
return
fi
echo "PASS: ${test_name} (expected failure)"
return
fi

if [[ ${exit_code} -ne 0 ]]; then
echo "FAIL: ${test_name} — exit code ${exit_code}"
cat "${TEST_TMPDIR}/stdout-${test_name}.log"
FAILURES=$((FAILURES + 1))
return
fi

if [[ -n "${expected_stdout}" ]] && ! grep -qF -- "${expected_stdout}" "${TEST_TMPDIR}/stdout-${test_name}.log"; then
echo "FAIL: ${test_name} — expected stdout '${expected_stdout}' not found"
echo "Actual stdout:"
cat "${TEST_TMPDIR}/stdout-${test_name}.log"
FAILURES=$((FAILURES + 1))
return
fi

echo "PASS: ${test_name}"
}

# Validated dir has agent-result.json → used
run_validated_dir_test "validated-dir-expected-filename" \
"agent-result.json" \
"Reading RICE result from: ${TEST_TMPDIR}/run-validated-dir-expected-filename/validated-output/agent-result.json"

# Validated dir has only result.json → used as fallback
run_validated_dir_test "validated-dir-fallback-filename" \
"result.json" \
"Reading RICE result from: ${TEST_TMPDIR}/run-validated-dir-fallback-filename/validated-output/result.json"

# Validated dir has neither filename → fails closed
run_validated_dir_test "validated-dir-neither-filename" \
"none" \
"" \
"true"

if [[ ${FAILURES} -gt 0 ]]; then
echo ""
echo "${FAILURES} test(s) failed."
Expand Down
24 changes: 18 additions & 6 deletions scripts/post-prioritize.sh
Original file line number Diff line number Diff line change
Expand Up @@ -345,13 +345,25 @@ if [[ ! "${GITHUB_ISSUE_URL}" =~ ^https://github\.com/[a-zA-Z0-9._-]+/[a-zA-Z0-9
exit 1
fi

# Find the result JSON from the last iteration.
RESULT_FILE=""
for dir in iteration-*/output; do
if [[ -f "${dir}/agent-result.json" ]]; then
RESULT_FILE="${dir}/agent-result.json"
# Find the result JSON — prefer the validated iteration when set.
if [[ -n "${FULLSEND_VALIDATED_ITERATION_DIR:-}" ]]; then
if [[ -f "${FULLSEND_VALIDATED_ITERATION_DIR}/agent-result.json" ]]; then
RESULT_FILE="${FULLSEND_VALIDATED_ITERATION_DIR}/agent-result.json"
elif [[ -f "${FULLSEND_VALIDATED_ITERATION_DIR}/result.json" ]]; then
RESULT_FILE="${FULLSEND_VALIDATED_ITERATION_DIR}/result.json"
else
echo "ERROR: FULLSEND_VALIDATED_ITERATION_DIR is set but contains neither agent-result.json nor result.json" >&2
exit 1
fi
done
else
# Backward compatibility: scan iteration-N/ subdirectories for the last one's output.
RESULT_FILE=""
for dir in iteration-*/output; do
if [[ -f "${dir}/agent-result.json" ]]; then
RESULT_FILE="${dir}/agent-result.json"
fi
done
fi

if [[ -z "${RESULT_FILE}" ]]; then
echo "ERROR: agent-result.json not found in any iteration output directory" >&2
Expand Down
24 changes: 18 additions & 6 deletions scripts/post-prioritize.src.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,25 @@ if [[ ! "${GITHUB_ISSUE_URL}" =~ ^https://github\.com/[a-zA-Z0-9._-]+/[a-zA-Z0-9
exit 1
fi

# Find the result JSON from the last iteration.
RESULT_FILE=""
for dir in iteration-*/output; do
if [[ -f "${dir}/agent-result.json" ]]; then
RESULT_FILE="${dir}/agent-result.json"
# Find the result JSON — prefer the validated iteration when set.
if [[ -n "${FULLSEND_VALIDATED_ITERATION_DIR:-}" ]]; then
if [[ -f "${FULLSEND_VALIDATED_ITERATION_DIR}/agent-result.json" ]]; then
RESULT_FILE="${FULLSEND_VALIDATED_ITERATION_DIR}/agent-result.json"
elif [[ -f "${FULLSEND_VALIDATED_ITERATION_DIR}/result.json" ]]; then
RESULT_FILE="${FULLSEND_VALIDATED_ITERATION_DIR}/result.json"
else
echo "ERROR: FULLSEND_VALIDATED_ITERATION_DIR is set but contains neither agent-result.json nor result.json" >&2
exit 1
fi
done
else
# Backward compatibility: scan iteration-N/ subdirectories for the last one's output.
RESULT_FILE=""
for dir in iteration-*/output; do
if [[ -f "${dir}/agent-result.json" ]]; then
RESULT_FILE="${dir}/agent-result.json"
fi
done
fi

if [[ -z "${RESULT_FILE}" ]]; then
echo "ERROR: agent-result.json not found in any iteration output directory" >&2
Expand Down
Loading
Loading