Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions agents/code.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ the review agent — if the triage was wrong, your code will fail review.

## Structured output

You MUST produce a JSON file at `$FULLSEND_OUTPUT_DIR/code-result.json`
You MUST produce a JSON file at `$FULLSEND_OUTPUT_DIR/agent-result.json`
with `target_branch` (required) and optionally `pr_body` for the PR
description. The `code-implementation` skill describes the schema and
the exact steps where you write each field. The post-script reads this
Expand All @@ -91,7 +91,7 @@ file, the validation loop rejects the run and retries.
After writing the file, validate it before exiting:

```bash
fullsend-check-output "${FULLSEND_OUTPUT_DIR}/code-result.json"
fullsend-check-output "${FULLSEND_OUTPUT_DIR}/agent-result.json"
```

If validation fails, read the error output, fix the JSON file, and
Expand Down
6 changes: 3 additions & 3 deletions agents/fix.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ triggers. When triggered by a human (username doesn't end in `[bot]`), the
`HUMAN_INSTRUCTION` environment variable contains the instruction text.

**Important:** `TRIGGER_SOURCE` is a GitHub username — not the value you
write to `fix-result.json`. The `trigger_source` field in structured output
write to `agent-result.json`. The `trigger_source` field in structured output
must be normalized to `"bot"` or `"human"` (the schema enum). Map it:
if the username ends in `[bot]`, use `"bot"`; otherwise use `"human"`.

Expand Down Expand Up @@ -133,7 +133,7 @@ asks for it.

## Structured output

You MUST produce a JSON file at `$FULLSEND_OUTPUT_DIR/fix-result.json` that
You MUST produce a JSON file at `$FULLSEND_OUTPUT_DIR/agent-result.json` that
documents your actions on every review finding. The `fix-review` skill
describes the schema. The post-script reads this file to post a summary
comment on the PR. Without this file, the post-script cannot communicate
Expand All @@ -142,7 +142,7 @@ your work back to the reviewer.
After writing the file, validate it before exiting:

```bash
fullsend-check-output "${FULLSEND_OUTPUT_DIR}/fix-result.json"
fullsend-check-output "${FULLSEND_OUTPUT_DIR}/agent-result.json"
```

If validation fails, read the error output, fix the JSON file, and
Expand Down
1 change: 0 additions & 1 deletion harness/code.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ validation_loop:
# These are expanded from the runner environment and NEVER enter the sandbox.
runner_env:
CODE_ALLOWED_TARGET_BRANCHES: "${CODE_ALLOWED_TARGET_BRANCHES}"
FULLSEND_OUTPUT_FILE: code-result.json

timeout_minutes: 35

Expand Down
1 change: 0 additions & 1 deletion harness/fix.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ env:
FIX_ITERATION: "${FIX_ITERATION}"
REVIEW_BODY_FILE: "${REVIEW_BODY_FILE}"
PRE_AGENT_HEAD: "${PRE_AGENT_HEAD}"
FULLSEND_OUTPUT_FILE: fix-result.json
sandbox:
MAX_RETRIES: "1"
TIMEOUT_SECONDS: "1500"
Expand Down
38 changes: 12 additions & 26 deletions scripts/post-code-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -913,7 +913,7 @@ run_precommit_retry_test "precommit-retry-passes-but-left-unstaged" \
# value and a set of files on disk, returns which result file (if any) would
# be selected.
#
# Mirrors the three-branch logic: expected filename -> result.json fallback
# Mirrors the two-branch logic: expected filename (agent-result.json)
# -> no silent rescan (degrades to empty, matching this script's existing
# soft-fallback-to-default-branch behavior rather than a hard failure).
# ---------------------------------------------------------------------------
Expand All @@ -922,18 +922,16 @@ resolve_code_result() {
local run_dir="$2" # directory containing iteration-*/output/

if [ -n "${validated_dir}" ]; then
if [ -f "${validated_dir}/code-result.json" ]; then
echo "${validated_dir}/code-result.json"
elif [ -f "${validated_dir}/result.json" ]; then
echo "${validated_dir}/result.json"
if [ -f "${validated_dir}/agent-result.json" ]; then
echo "${validated_dir}/agent-result.json"
else
echo ""
fi
else
local result=""
for dir in "${run_dir}"/iteration-*/output; do
if [ -f "${dir}/code-result.json" ]; then
result="${dir}/code-result.json"
if [ -f "${dir}/agent-result.json" ]; then
result="${dir}/agent-result.json"
fi
done
echo "${result}"
Expand Down Expand Up @@ -991,23 +989,15 @@ run_resolve_code_test_unset() {
echo "PASS: ${test_name}"
}

# Setup: validated dir has code-result.json
# Setup: validated dir has agent-result.json
setup_code_expected() {
local run_dir="$1"
local validated_dir="$2"
mkdir -p "${validated_dir}"
echo '{}' > "${validated_dir}/code-result.json"
echo '{}' > "${validated_dir}/agent-result.json"
# Also place a file in iteration-2 to verify it's NOT used.
mkdir -p "${run_dir}/iteration-2/output"
echo '{}' > "${run_dir}/iteration-2/output/code-result.json"
}

# Setup: validated dir has only result.json
setup_code_fallback() {
local run_dir="$1"
local validated_dir="$2"
mkdir -p "${validated_dir}"
echo '{}' > "${validated_dir}/result.json"
echo '{}' > "${run_dir}/iteration-2/output/agent-result.json"
}

# Setup: validated dir has neither filename
Expand All @@ -1022,25 +1012,21 @@ setup_code_iteration_scan() {
local run_dir="$1"
mkdir -p "${run_dir}/iteration-1/output"
mkdir -p "${run_dir}/iteration-2/output"
echo '{}' > "${run_dir}/iteration-1/output/code-result.json"
echo '{}' > "${run_dir}/iteration-2/output/code-result.json"
echo '{}' > "${run_dir}/iteration-1/output/agent-result.json"
echo '{}' > "${run_dir}/iteration-2/output/agent-result.json"
}

run_resolve_code_test "code-validated-dir-expected-filename" \
setup_code_expected \
"${RESOLVE_TMPDIR}/code-validated-dir-expected-filename/validated-output/code-result.json"

run_resolve_code_test "code-validated-dir-fallback-filename" \
setup_code_fallback \
"${RESOLVE_TMPDIR}/code-validated-dir-fallback-filename/validated-output/result.json"
"${RESOLVE_TMPDIR}/code-validated-dir-expected-filename/validated-output/agent-result.json"

run_resolve_code_test "code-validated-dir-neither-filename-degrades-to-empty" \
setup_code_neither \
""

run_resolve_code_test_unset "code-unset-falls-back-to-scan" \
setup_code_iteration_scan \
"${RESOLVE_TMPDIR}/code-unset-falls-back-to-scan/iteration-2/output/code-result.json"
"${RESOLVE_TMPDIR}/code-unset-falls-back-to-scan/iteration-2/output/agent-result.json"

rm -rf "${RESOLVE_TMPDIR}"

Expand Down
25 changes: 9 additions & 16 deletions scripts/post-code.sh
Original file line number Diff line number Diff line change
Expand Up @@ -700,7 +700,7 @@ fi
# Resolve target branch (ADR 0053)
#
# Priority: agent output > allowed-list validation > auto-detect default
# The agent writes its chosen branch to code-result.json. The post-script
# The agent writes its chosen branch to agent-result.json. The post-script
# validates it against CODE_ALLOWED_TARGET_BRANCHES (comma-separated list
# or "*" for any). When unset, only the auto-detected default branch is
# allowed. Falls back to "main" if the API call fails.
Expand All @@ -712,20 +712,13 @@ AGENT_TARGET=""
# guard) is applied here; the value is trusted from the external harness.
# If the trust model changes, add a realpath prefix check.
if [ -n "${FULLSEND_VALIDATED_ITERATION_DIR:-}" ]; then
if [ -f "${FULLSEND_VALIDATED_ITERATION_DIR}/code-result.json" ]; then
RESULT_FILE="${FULLSEND_VALIDATED_ITERATION_DIR}/code-result.json"
elif [ -f "${FULLSEND_VALIDATED_ITERATION_DIR}/result.json" ]; then
# NOTE: This fallback is currently unreachable in production.
# validate-output-schema.sh only accepts result.json when _output_file is
# "agent-result.json" (the default). code.yaml sets FULLSEND_OUTPUT_FILE
# to "code-result.json", so a bare result.json will never become the
# validated iteration's output. Kept as defensive code.
RESULT_FILE="${FULLSEND_VALIDATED_ITERATION_DIR}/result.json"
if [ -f "${FULLSEND_VALIDATED_ITERATION_DIR}/agent-result.json" ]; then
RESULT_FILE="${FULLSEND_VALIDATED_ITERATION_DIR}/agent-result.json"
else
# No silent rescan: an env var pointing at a dir with neither filename
# must not fall back to scanning other iterations, which could pick up
# a different (possibly invalid) iteration's output. Degrade to no
# result the same as the "nothing found" case below — this script
# No silent rescan: an env var pointing at a dir without the expected
# filename must not fall back to scanning other iterations, which could
# pick up a different (possibly invalid) iteration's output. Degrade to
# no result the same as the "nothing found" case below — this script
# already falls back to the auto-detected default branch when
# RESULT_FILE is empty, so this isn't a hard failure.
RESULT_FILE=""
Expand All @@ -735,8 +728,8 @@ else
# iteration's output (glob order = naturally ascending iteration numbers).
RESULT_FILE=""
for dir in "${RUN_DIR}"/iteration-*/output; do
if [ -f "${dir}/code-result.json" ]; then
RESULT_FILE="${dir}/code-result.json"
if [ -f "${dir}/agent-result.json" ]; then
RESULT_FILE="${dir}/agent-result.json"
fi
done
fi
Expand Down
25 changes: 9 additions & 16 deletions scripts/post-code.src.sh
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ fi
# Resolve target branch (ADR 0053)
#
# Priority: agent output > allowed-list validation > auto-detect default
# The agent writes its chosen branch to code-result.json. The post-script
# The agent writes its chosen branch to agent-result.json. The post-script
# validates it against CODE_ALLOWED_TARGET_BRANCHES (comma-separated list
# or "*" for any). When unset, only the auto-detected default branch is
# allowed. Falls back to "main" if the API call fails.
Expand All @@ -82,20 +82,13 @@ AGENT_TARGET=""
# guard) is applied here; the value is trusted from the external harness.
# If the trust model changes, add a realpath prefix check.
if [ -n "${FULLSEND_VALIDATED_ITERATION_DIR:-}" ]; then
if [ -f "${FULLSEND_VALIDATED_ITERATION_DIR}/code-result.json" ]; then
RESULT_FILE="${FULLSEND_VALIDATED_ITERATION_DIR}/code-result.json"
elif [ -f "${FULLSEND_VALIDATED_ITERATION_DIR}/result.json" ]; then
# NOTE: This fallback is currently unreachable in production.
# validate-output-schema.sh only accepts result.json when _output_file is
# "agent-result.json" (the default). code.yaml sets FULLSEND_OUTPUT_FILE
# to "code-result.json", so a bare result.json will never become the
# validated iteration's output. Kept as defensive code.
RESULT_FILE="${FULLSEND_VALIDATED_ITERATION_DIR}/result.json"
if [ -f "${FULLSEND_VALIDATED_ITERATION_DIR}/agent-result.json" ]; then
RESULT_FILE="${FULLSEND_VALIDATED_ITERATION_DIR}/agent-result.json"
else
# No silent rescan: an env var pointing at a dir with neither filename
# must not fall back to scanning other iterations, which could pick up
# a different (possibly invalid) iteration's output. Degrade to no
# result the same as the "nothing found" case below — this script
# No silent rescan: an env var pointing at a dir without the expected
# filename must not fall back to scanning other iterations, which could
# pick up a different (possibly invalid) iteration's output. Degrade to
# no result the same as the "nothing found" case below — this script
# already falls back to the auto-detected default branch when
# RESULT_FILE is empty, so this isn't a hard failure.
RESULT_FILE=""
Expand All @@ -105,8 +98,8 @@ else
# iteration's output (glob order = naturally ascending iteration numbers).
RESULT_FILE=""
for dir in "${RUN_DIR}"/iteration-*/output; do
if [ -f "${dir}/code-result.json" ]; then
RESULT_FILE="${dir}/code-result.json"
if [ -f "${dir}/agent-result.json" ]; then
RESULT_FILE="${dir}/agent-result.json"
fi
done
fi
Expand Down
40 changes: 13 additions & 27 deletions scripts/post-fix-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -192,18 +192,16 @@ resolve_fix_result() {
local run_dir="$2" # directory containing iteration-*/output/

if [ -n "${validated_dir}" ]; then
if [ -f "${validated_dir}/fix-result.json" ]; then
echo "${validated_dir}/fix-result.json"
elif [ -f "${validated_dir}/result.json" ]; then
echo "${validated_dir}/result.json"
if [ -f "${validated_dir}/agent-result.json" ]; then
echo "${validated_dir}/agent-result.json"
else
echo "error:neither-filename"
fi
else
local result=""
for dir in "${run_dir}"/iteration-*/output; do
if [ -f "${dir}/fix-result.json" ]; then
result="${dir}/fix-result.json"
if [ -f "${dir}/agent-result.json" ]; then
result="${dir}/agent-result.json"
fi
done
if [ -z "${result}" ]; then
Expand Down Expand Up @@ -266,23 +264,15 @@ run_resolve_test_unset() {
echo "PASS: ${test_name}"
}

# Setup: validated dir has fix-result.json
# Setup: validated dir has agent-result.json
setup_fix_expected() {
local run_dir="$1"
local validated_dir="$2"
mkdir -p "${validated_dir}"
echo '{}' > "${validated_dir}/fix-result.json"
echo '{}' > "${validated_dir}/agent-result.json"
# Also place a file in iteration-2 to verify it's NOT used.
mkdir -p "${run_dir}/iteration-2/output"
echo '{}' > "${run_dir}/iteration-2/output/fix-result.json"
}

# Setup: validated dir has only result.json
setup_fix_fallback() {
local run_dir="$1"
local validated_dir="$2"
mkdir -p "${validated_dir}"
echo '{}' > "${validated_dir}/result.json"
echo '{}' > "${run_dir}/iteration-2/output/agent-result.json"
}

# Setup: validated dir has neither filename
Expand All @@ -297,34 +287,30 @@ setup_fix_iteration_scan() {
local run_dir="$1"
mkdir -p "${run_dir}/iteration-1/output"
mkdir -p "${run_dir}/iteration-2/output"
echo '{}' > "${run_dir}/iteration-1/output/fix-result.json"
echo '{}' > "${run_dir}/iteration-2/output/fix-result.json"
echo '{}' > "${run_dir}/iteration-1/output/agent-result.json"
echo '{}' > "${run_dir}/iteration-2/output/agent-result.json"
}

# --- FULLSEND_VALIDATED_ITERATION_DIR test cases ---

run_resolve_test "fix-validated-dir-expected-filename" \
setup_fix_expected \
"${RESOLVE_TMPDIR}/fix-validated-dir-expected-filename/validated-output/fix-result.json"

run_resolve_test "fix-validated-dir-fallback-filename" \
setup_fix_fallback \
"${RESOLVE_TMPDIR}/fix-validated-dir-fallback-filename/validated-output/result.json"
"${RESOLVE_TMPDIR}/fix-validated-dir-expected-filename/validated-output/agent-result.json"

run_resolve_test "fix-validated-dir-neither-filename" \
setup_fix_neither \
"error:neither-filename"

run_resolve_test_unset "fix-unset-falls-back-to-scan" \
setup_fix_iteration_scan \
"${RESOLVE_TMPDIR}/fix-unset-falls-back-to-scan/iteration-2/output/fix-result.json"
"${RESOLVE_TMPDIR}/fix-unset-falls-back-to-scan/iteration-2/output/agent-result.json"

rm -rf "${RESOLVE_TMPDIR}"

# ---------------------------------------------------------------------------
# Integration test — run the REAL post-fix.sh to verify that it exits non-zero
# when FULLSEND_VALIDATED_ITERATION_DIR is set but contains neither
# fix-result.json nor result.json. This catches the fail-open bug that the
# when FULLSEND_VALIDATED_ITERATION_DIR is set but does not contain
# agent-result.json. This catches the fail-open bug that the
# isolated reimplementation tests above cannot detect.
#
# Strategy: initialize a bare git repo on the main branch so NO_PUSH=true,
Expand Down
Loading
Loading