Skip to content

test(e2e): tolerate preapproved scope upgrades #5406

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cv merged 3 commits into from
Jun 14, 2026
Merged

test(e2e): tolerate preapproved scope upgrades #5406

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5fc521d
test(e2e): tolerate preapproved scope upgrades
cv Jun 14, 2026
9d1c506
test(e2e): keep gateway proof for preapproved scopes
cv Jun 14, 2026
d5c7ee7
test(e2e): validate scope-upgrade modes early
cv Jun 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 108 additions & 68 deletions test/e2e/test-issue-4462-scope-upgrade-approval.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,16 @@ section() {

info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; }

finish_success() {
section "Summary"
echo ""
printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m\n' \
"$TOTAL" "$PASS" "$FAIL"
echo ""
echo "$1"
exit 0
}

if [ -d /workspace ] && [ -f /workspace/install.sh ]; then
REPO="/workspace"
elif [ -f "$(cd "${SCRIPT_DIR}/../.." && pwd)/install.sh" ]; then
Expand All @@ -70,6 +80,13 @@ E2E_DIR="${SCRIPT_DIR}"
SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-issue-4462-scope-upgrade}"
OPENSHELL_BIN="${NEMOCLAW_OPENSHELL_BIN:-openshell}"
TEST_MODE="${NEMOCLAW_4462_MODE:-approval}"
case "$TEST_MODE" in
approval | legacy-repro) ;;
*)
fail "Unknown NEMOCLAW_4462_MODE=${TEST_MODE}; expected approval or legacy-repro"
exit 1
;;
esac
INSTALL_LOG="${NEMOCLAW_4462_INSTALL_LOG:-/tmp/nemoclaw-e2e-issue-4462-scope-upgrade-install.log}"
APPROVAL_LOG="${NEMOCLAW_4462_APPROVAL_LOG:-/tmp/nemoclaw-issue-4462-scope-upgrade-approval.log}"
AGENT_LOG="${NEMOCLAW_4462_AGENT_LOG:-/tmp/nemoclaw-issue-4462-scope-upgrade-agent.log}"
Expand All @@ -90,6 +107,14 @@ AUTO_PAIR_FAST_DEADLINE_SECS="${NEMOCLAW_4462_AUTO_PAIR_FAST_DEADLINE_SECS:-${NE
AUTO_PAIR_DEADLINE_SECS="${NEMOCLAW_4462_AUTO_PAIR_DEADLINE_SECS:-${NEMOCLAW_AUTO_PAIR_DEADLINE_SECS:-$AUTO_PAIR_DEADLINE_DEFAULT}}"
AUTO_PAIR_SLOW_INTERVAL_SECS="${NEMOCLAW_4462_AUTO_PAIR_SLOW_INTERVAL_SECS:-${NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS:-$AUTO_PAIR_SLOW_INTERVAL_DEFAULT}}"
AUTO_PAIR_RUN_TIMEOUT_SECS="${NEMOCLAW_4462_AUTO_PAIR_RUN_TIMEOUT_SECS:-${NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS:-$AUTO_PAIR_RUN_TIMEOUT_DEFAULT}}"
# Current onboard finalization may warm up and approve the CLI operator.read/write
# scope-upgrade before this E2E inspects the intermediate low-scope state. That
# is an acceptable final authorization state only if operator.admin is absent;
# the script must still prove the final agent turn stays on the gateway path.
# In legacy-repro mode, a preapproved state leaves no pending upgrade to
# characterize, so the final result calls that out explicitly instead of
# claiming the legacy gateway-pinned approve behavior was exercised.
SCOPE_UPGRADE_ALREADY_SATISFIED=0

# shellcheck source=test/e2e/lib/sandbox-teardown.sh
. "${E2E_DIR}/lib/sandbox-teardown.sh"
Expand Down Expand Up @@ -864,8 +889,15 @@ else
if [ -n "$paired_without_write" ]; then
pass "CLI device is already paired with low scope (${paired_without_write})"
else
fail "No pending or paired low-scope CLI device found after devices list: ${summary}"
exit 1
paired_with_agent_scopes=$(printf '%s' "$state" | select_cli_paired_with_agent_scopes 2>/dev/null) || paired_with_agent_scopes=""
paired_with_admin=$(printf '%s' "$state" | select_cli_paired_with_admin 2>/dev/null) || paired_with_admin=""
if [ -n "$paired_with_agent_scopes" ] && [ -z "$paired_with_admin" ]; then
pass "CLI device already has operator.read/operator.write without operator.admin (${paired_with_agent_scopes})"
SCOPE_UPGRADE_ALREADY_SATISFIED=1
else
fail "No pending or paired low-scope CLI device found after devices list: ${summary}"
exit 1
fi
fi
fi

Expand All @@ -874,12 +906,16 @@ state="$(device_state_json 2>&1)" || {
exit 1
}
printf '=== state after initial approval ===\n%s\n' "$state" >>"$STATE_LOG"
paired_without_write=$(printf '%s' "$state" | select_cli_paired_without_write 2>/dev/null) || paired_without_write=""
if [ -n "$paired_without_write" ]; then
pass "CLI device is paired with operator.pairing but not operator.write"
if [ "$SCOPE_UPGRADE_ALREADY_SATISFIED" = "1" ]; then
pass "initial approval check skipped because CLI scope-upgrade is already satisfied"
else
fail "Initial approval did not leave a low-scope CLI device: $(printf '%s' "$state" | summarize_device_state)"
exit 1
paired_without_write=$(printf '%s' "$state" | select_cli_paired_without_write 2>/dev/null) || paired_without_write=""
if [ -n "$paired_without_write" ]; then
pass "CLI device is paired with operator.pairing but not operator.write"
else
fail "Initial approval did not leave a low-scope CLI device: $(printf '%s' "$state" | summarize_device_state)"
exit 1
fi
fi

gateway_list=$(sandbox_exec_sh_script 60 '
Expand All @@ -898,14 +934,17 @@ else
exit 1
fi

if [ "$TEST_MODE" = "legacy-repro" ]; then
if [ "$TEST_MODE" = "legacy-repro" ] && [ "$SCOPE_UPGRADE_ALREADY_SATISFIED" != "1" ]; then
wait_for_auto_pair_watcher_inactive || exit 1
fi

section "Phase 4: Trigger and approve CLI scope upgrade"

info "Triggering agent operator.write scope upgrade"
trigger_output=$(sandbox_exec_sh_script 120 '
if [ "$SCOPE_UPGRADE_ALREADY_SATISFIED" = "1" ]; then
info "Skipping trigger/approval because CLI operator.read/operator.write scopes were already approved"
else
info "Triggering agent operator.write scope upgrade"
trigger_output=$(sandbox_exec_sh_script 120 '
set -u
# shellcheck source=/dev/null
. /tmp/nemoclaw-proxy-env.sh
Expand All @@ -921,61 +960,56 @@ set -e
printf "__TRIGGER_AGENT_RC__=%s\n" "$agent_rc"
exit 0
' 2>&1)
printf '=== trigger agent output ===\n%s\n' "$trigger_output" >>"$AGENT_LOG"

scope_request_id=""
auto_approved_device=""
for _attempt in 1 2 3 4 5; do
state="$(device_state_json 2>&1)" || state=""
if [ -n "$state" ]; then
printf '=== state while waiting for scope upgrade ===\n%s\n' "$state" >>"$STATE_LOG"
scope_request_id=$(printf '%s' "$state" | select_cli_request scope-upgrade 2>/dev/null) || scope_request_id=""
auto_approved_device=$(printf '%s' "$state" | select_cli_paired_with_agent_scopes 2>/dev/null) || auto_approved_device=""
fi
[ -n "$scope_request_id" ] && break
if [ "$TEST_MODE" = "approval" ] && [ -n "$auto_approved_device" ]; then
break
fi
sleep 2
done

if [ -z "$scope_request_id" ] && [ "$TEST_MODE" = "legacy-repro" ]; then
scope_request_id=$(printf '%s' "$trigger_output" | extract_scope_request_id_from_output) || scope_request_id=""
fi
printf '=== trigger agent output ===\n%s\n' "$trigger_output" >>"$AGENT_LOG"

scope_request_id=""
auto_approved_device=""
for _attempt in 1 2 3 4 5; do
state="$(device_state_json 2>&1)" || state=""
if [ -n "$state" ]; then
printf '=== state while waiting for scope upgrade ===\n%s\n' "$state" >>"$STATE_LOG"
scope_request_id=$(printf '%s' "$state" | select_cli_request scope-upgrade 2>/dev/null) || scope_request_id=""
auto_approved_device=$(printf '%s' "$state" | select_cli_paired_with_agent_scopes 2>/dev/null) || auto_approved_device=""
fi
[ -n "$scope_request_id" ] && break
if [ "$TEST_MODE" = "approval" ] && [ -n "$auto_approved_device" ]; then
break
fi
sleep 2
done

if [ -n "$scope_request_id" ]; then
pass "pending CLI scope-upgrade request exists (${scope_request_id})"
elif [ "$TEST_MODE" = "approval" ] && [ -n "$auto_approved_device" ]; then
pass "auto-pair watcher approved the CLI scope upgrade before pending inspection (${auto_approved_device})"
else
fail "No pending CLI scope-upgrade request appeared after agent trigger. State: $(printf '%s' "${state:-{}}" | summarize_device_state 2>/dev/null || true). Trigger: ${trigger_output:0:500}"
exit 1
fi
if [ -z "$scope_request_id" ] && [ "$TEST_MODE" = "legacy-repro" ]; then
scope_request_id=$(printf '%s' "$trigger_output" | extract_scope_request_id_from_output) || scope_request_id=""
fi

if [ "$TEST_MODE" = "legacy-repro" ]; then
legacy_gateway_pinned_approval_characterization "$scope_request_id" || exit 1
section "Summary"
echo ""
printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m\n' \
"$TOTAL" "$PASS" "$FAIL"
echo ""
if [ "$FAIL" -gt 0 ]; then
echo "RESULT: FAILED - ${FAIL} test(s) failed"
if [ -n "$scope_request_id" ]; then
pass "pending CLI scope-upgrade request exists (${scope_request_id})"
elif [ "$TEST_MODE" = "approval" ] && [ -n "$auto_approved_device" ]; then
pass "auto-pair watcher approved the CLI scope upgrade before pending inspection (${auto_approved_device})"
else
fail "No pending CLI scope-upgrade request appeared after agent trigger. State: $(printf '%s' "${state:-{}}" | summarize_device_state 2>/dev/null || true). Trigger: ${trigger_output:0:500}"
exit 1
fi
echo "RESULT: PASSED - #4462 legacy gateway-pinned approval behavior characterized and final state handled"
exit 0
fi

if [ "$TEST_MODE" != "approval" ]; then
fail "Unknown NEMOCLAW_4462_MODE=${TEST_MODE}; expected approval or legacy-repro"
exit 1
fi
if [ "$TEST_MODE" = "legacy-repro" ]; then
legacy_gateway_pinned_approval_characterization "$scope_request_id" || exit 1
if [ "$FAIL" -gt 0 ]; then
section "Summary"
echo ""
printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m\n' \
"$TOTAL" "$PASS" "$FAIL"
echo ""
echo "RESULT: FAILED - ${FAIL} test(s) failed"
exit 1
fi
finish_success "RESULT: PASSED - #4462 legacy gateway-pinned approval behavior characterized and final state handled"
fi

if [ -n "$scope_request_id" ]; then
approve_request "$scope_request_id" "CLI scope upgrade" 1 || exit 1
else
info "Skipping manual scope-upgrade approval because the auto-pair watcher already granted it"
if [ -n "$scope_request_id" ]; then
approve_request "$scope_request_id" "CLI scope upgrade" 1 || exit 1
else
info "Skipping manual scope-upgrade approval because the auto-pair watcher already granted it"
fi
fi

state="$(device_state_json 2>&1)" || {
Expand All @@ -998,7 +1032,11 @@ if [ -n "$paired_with_admin" ]; then
fail "Unexpected operator.admin approval for CLI device (${paired_with_admin})"
exit 1
fi
pass "scope-upgrade approval grants the CLI device operator.write and operator.read without approving operator.admin"
if [ "$SCOPE_UPGRADE_ALREADY_SATISFIED" = "1" ]; then
pass "preapproved CLI scope-upgrade state has operator.write/operator.read without operator.admin"
else
pass "scope-upgrade approval grants the CLI device operator.write and operator.read without approving operator.admin"
fi

section "Phase 5: Verify agent stays on gateway path"

Expand Down Expand Up @@ -1043,16 +1081,18 @@ fi

pass "approved agent output contains no fallback or pairing markers"

section "Summary"
echo ""
printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m\n' \
"$TOTAL" "$PASS" "$FAIL"
echo ""

if [ "$FAIL" -gt 0 ]; then
section "Summary"
echo ""
printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m\n' \
"$TOTAL" "$PASS" "$FAIL"
echo ""
echo "RESULT: FAILED - ${FAIL} test(s) failed"
exit 1
fi

echo "RESULT: PASSED - #4462 CLI scope-upgrade approval stays on the gateway path"
exit 0
if [ "$TEST_MODE" = "legacy-repro" ] && [ "$SCOPE_UPGRADE_ALREADY_SATISFIED" = "1" ]; then
finish_success "RESULT: PASSED - #4462 legacy gateway-pinned approval characterization skipped because scope-upgrade was already satisfied; final gateway path verified"
fi

finish_success "RESULT: PASSED - #4462 CLI scope-upgrade approval stays on the gateway path"
Loading