fix(start): drop slow-mode polling on late allowlisted scope upgrades

.github/workflows/nightly-e2e.yaml

@@ -15,10 +15,10 @@
 #   openclaw-discord-pairing-e2e
 #                            Validates hermetic Discord pairing request approval across
 #                            gateway and connect-shell OpenClaw state roots (#4061).
-#   issue-4462-scope-upgrade-approval-e2e
+#   cli-scope-upgrade-approval-e2e
 #                            Validates real CLI scope-upgrade approval and confirms
 #                            the approved agent run stays on gateway mode (#4462).
-#   issue-4462-gateway-pinned-approval-characterization-e2e
+#   cli-scope-upgrade-legacy-repro-e2e
 #                            Characterizes legacy gateway-pinned scope approval
 #                            against a real sandbox, then recovers with the fix.
 #   messaging-compatible-endpoint-e2e
@@ -124,8 +124,8 @@ on:
           docs-validation-e2e, messaging-providers-e2e, openclaw-slack-pairing-e2e,
           openclaw-tui-chat-correlation-e2e, issue-4434-tui-unreachable-inference-e2e,
           issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e,
-          issue-4462-scope-upgrade-approval-e2e,
-          issue-4462-gateway-pinned-approval-characterization-e2e,
+          cli-scope-upgrade-approval-e2e,
+          cli-scope-upgrade-legacy-repro-e2e,
           messaging-compatible-endpoint-e2e, sessions-agents-cli-e2e,
           channels-add-remove-e2e, channels-stop-start-openclaw-e2e,
           channels-stop-start-hermes-e2e, brave-search-e2e,
@@ -624,39 +624,39 @@ jobs:
   # Positive proof: in a real sandbox, accept either a visible pending CLI
   # scope upgrade or the fixed watcher's immediate approval, then confirm
   # openclaw agent still uses the gateway path rather than embedded fallback.
-  issue-4462-scope-upgrade-approval-e2e:
+  cli-scope-upgrade-approval-e2e:
     if: >-
       github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
        inputs.jobs == '' ||
-       contains(format(',{0},', inputs.jobs), ',issue-4462-scope-upgrade-approval-e2e,'))
+       contains(format(',{0},', inputs.jobs), ',cli-scope-upgrade-approval-e2e,'))
     uses: ./.github/workflows/e2e-script.yaml
     with:
       ref: ${{ inputs.target_ref || github.ref }}
-      script: test/e2e/test-issue-4462-scope-upgrade-approval.sh
+      script: test/e2e/test-cli-scope-upgrade-approval.sh
       timeout_minutes: 60
       artifact_name: "issue-4462-scope-upgrade-approval-logs"
       artifact_path: |
         /tmp/nemoclaw-e2e-issue-4462-scope-upgrade-install.log
         /tmp/nemoclaw-issue-4462-scope-upgrade-approval.log
         /tmp/nemoclaw-issue-4462-scope-upgrade-agent.log
         /tmp/nemoclaw-issue-4462-scope-upgrade-state.log
-      env_json: '{"NEMOCLAW_4462_MODE":"approval","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AUTO_PAIR_DEADLINE_SECS":"30","NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS":"3","NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS":"600","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-issue-4462-scope-upgrade"}'
+      env_json: '{"NEMOCLAW_4462_MODE":"approval","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AUTO_PAIR_DEADLINE_SECS":"30","NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS":"3","NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS":"5","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-issue-4462-scope-upgrade"}'
       nvidia_api_key: true
       github_token: false
     secrets: *nightly-e2e-default-secrets
   # ── OpenClaw Gateway-Pinned Approval Characterization (#4462) ──
   # Diagnostic proof: in a real sandbox, wait for the fixed watcher to exit,
   # force the legacy gateway-pinned approve path, record the observed
   # OpenClaw outcome, and recover through the fixed proxy-env guard if needed.
-  issue-4462-gateway-pinned-approval-characterization-e2e:
+  cli-scope-upgrade-legacy-repro-e2e:
     if: >-
       github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
        inputs.jobs == '' ||
-       contains(format(',{0},', inputs.jobs), ',issue-4462-gateway-pinned-approval-characterization-e2e,'))
+       contains(format(',{0},', inputs.jobs), ',cli-scope-upgrade-legacy-repro-e2e,'))
     uses: ./.github/workflows/e2e-script.yaml
     with:
       ref: ${{ inputs.target_ref || github.ref }}
-      script: test/e2e/test-issue-4462-scope-upgrade-approval.sh
+      script: test/e2e/test-cli-scope-upgrade-approval.sh
       timeout_minutes: 60
       artifact_name: "issue-4462-gateway-pinned-approval-characterization-logs"
       artifact_path: |
@@ -2336,8 +2336,8 @@ jobs:
         issue-4434-tui-unreachable-inference-e2e,
         issue-3600-gpu-proof-optional-e2e,
         openclaw-discord-pairing-e2e,
-        issue-4462-scope-upgrade-approval-e2e,
-        issue-4462-gateway-pinned-approval-characterization-e2e,
+        cli-scope-upgrade-approval-e2e,
+        cli-scope-upgrade-legacy-repro-e2e,
         messaging-compatible-endpoint-e2e,
         sessions-agents-cli-e2e,
         channels-add-remove-e2e,
@@ -2454,8 +2454,8 @@ jobs:
         issue-4434-tui-unreachable-inference-e2e,
         issue-3600-gpu-proof-optional-e2e,
         openclaw-discord-pairing-e2e,
-        issue-4462-scope-upgrade-approval-e2e,
-        issue-4462-gateway-pinned-approval-characterization-e2e,
+        cli-scope-upgrade-approval-e2e,
+        cli-scope-upgrade-legacy-repro-e2e,
         messaging-compatible-endpoint-e2e,
         sessions-agents-cli-e2e,
         channels-add-remove-e2e,
@@ -2641,8 +2641,8 @@ jobs:
         issue-4434-tui-unreachable-inference-e2e,
         issue-3600-gpu-proof-optional-e2e,
         openclaw-discord-pairing-e2e,
-        issue-4462-scope-upgrade-approval-e2e,
-        issue-4462-gateway-pinned-approval-characterization-e2e,
+        cli-scope-upgrade-approval-e2e,
+        cli-scope-upgrade-legacy-repro-e2e,
         messaging-compatible-endpoint-e2e,
         sessions-agents-cli-e2e,
         channels-add-remove-e2e,

scripts/nemoclaw-start.sh

@@ -1906,7 +1906,29 @@ def _env_seconds(name, default):
 # embedded mode. Defaults: 8h total, 30s slow-mode cadence.
 FAST_DEADLINE = time.time() + _env_seconds('NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS', 600)
 DEADLINE = time.time() + _env_seconds('NEMOCLAW_AUTO_PAIR_DEADLINE_SECS', 28800)
-SLOW_INTERVAL = _env_seconds('NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS', 30)
+# After convergence the watcher polls at SLOW_INTERVAL. A late allowlisted
+# scope upgrade — e.g. `openclaw tui` or `openclaw agent` invoked after the
+# watcher entered slow mode — can wait up to SLOW_INTERVAL before being
+# approved, which is longer than the OpenClaw client's tolerance for `scope
+# upgrade pending approval` and forces a fallback to embedded mode. The
+# default sits well below typical client-side wait windows; raise it through
+# NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS when the gateway connect handler is
+# load-sensitive. When the watcher successfully approves a fresh allowlisted
+# request during slow mode it also bumps a bounded fast-reentry counter
+# (NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS) that drops polling back to 1s for
+# the next few iterations, so cascading upgrades and transient approve
+# failures both clear before the OpenClaw client gives up. The counter is
+# only bumped on the rising edge for each requestId (tracked in
+# FAST_REENTRY_BUMPED_REQUEST_IDS and garbage-collected against the live
+# pending list), so a sticky failing request cannot pin the watcher in fast
+# polling. This is a polling-cadence fix only — non-allowlisted scopes such
+# as `operator.admin` are still rejected by the device approval policy, and
+# requests that need them must be approved through a separate operator path.
+SLOW_INTERVAL = _env_seconds('NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS', 5)
+FAST_REENTRY_POLLS = int(_env_seconds('NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS', 5))
+FAST_REENTRY_INTERVAL = _env_seconds('NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS', 1)
+FAST_REENTRY_REMAINING = 0
+FAST_REENTRY_BUMPED_REQUEST_IDS = set()
 QUIET_POLLS = 0
 APPROVED = 0
 SLOW_MODE = False
@@ -1948,15 +1970,33 @@ def run(*args, strip_gateway_env=False):
         print(f'[auto-pair] timeout calling {args[1] if len(args) > 1 else "openclaw"} {args[2] if len(args) > 2 else ""}'.rstrip())
         return 124, out.strip(), err.strip()
 
+
+def sleep_for_next_poll(default_seconds):
+    # Apply the bounded fast-reentry override before the caller's default
+    # sleep so a recent allowlisted approval (which bumps the remaining
+    # counter) drops polling to FAST_REENTRY_INTERVAL for the next few
+    # iterations. Mutates the global counter so callers do not need to
+    # thread the state through. The override is floored by the caller's
+    # default so it never increases the inter-poll latency (e.g. when the
+    # default is already tighter than FAST_REENTRY_INTERVAL during a
+    # bounded retry pass in fast mode).
+    global FAST_REENTRY_REMAINING
+    if FAST_REENTRY_REMAINING > 0:
+        FAST_REENTRY_REMAINING -= 1
+        time.sleep(min(FAST_REENTRY_INTERVAL, default_seconds))
+        return
+    time.sleep(default_seconds)
+
+
 while time.time() < DEADLINE:
     rc, out, err = run(OPENCLAW, 'devices', 'list', '--json')
     if rc != 0 or not out:
-        time.sleep(SLOW_INTERVAL if SLOW_MODE else 1)
+        sleep_for_next_poll(SLOW_INTERVAL if SLOW_MODE else 1)
         continue
     try:
         data = json.loads(out)
     except Exception:
-        time.sleep(SLOW_INTERVAL if SLOW_MODE else 1)
+        sleep_for_next_poll(SLOW_INTERVAL if SLOW_MODE else 1)
         continue
 
     pending = data.get('pending') or []
@@ -1975,11 +2015,16 @@ while time.time() < DEADLINE:
 
     if pending:
         QUIET_POLLS = 0
+        attempted_request_ids = set()
+        pending_request_ids = set()
         for device in pending:
             if not isinstance(device, dict):
                 continue
             request_id = device.get('requestId')
-            if not request_id or request_id in HANDLED:
+            if not request_id:
+                continue
+            pending_request_ids.add(request_id)
+            if request_id in HANDLED:
                 continue
             decision = approval_request_decision(device)
             client_id = decision['client_id']
@@ -1997,6 +2042,7 @@ while time.time() < DEADLINE:
                 scopes = decision['scopes']
                 print(f'[auto-pair] rejected disallowed scopes={sorted(scopes)} client={client_id} mode={client_mode}')
                 continue
+            attempted_request_ids.add(request_id)
             arc, aout, aerr = run(
                 OPENCLAW, 'devices', 'approve', request_id, '--json', strip_gateway_env=True,
             )
@@ -2013,7 +2059,25 @@ while time.time() < DEADLINE:
                 print(f'[auto-pair] approved request={request_id} client={client_id} mode={client_mode}')
             elif aout or aerr:
                 print(f'[auto-pair] approve failed request={request_id}: {(aerr or aout)[:400]}')
-        time.sleep(SLOW_INTERVAL if SLOW_MODE else 1)
+        # Drop previously-bumped requestIds that the gateway no longer reports
+        # as pending so a future re-appearance of the same id (very unlikely,
+        # but kept robust) can bump again. The set is otherwise small and
+        # never crosses out of the watcher process.
+        FAST_REENTRY_BUMPED_REQUEST_IDS.intersection_update(pending_request_ids)
+        # Fast-reentry is armed on the rising edge per requestId — once for
+        # each freshly-observed allowlisted attempt. A sticky pending request
+        # that fails approval repeatedly therefore stops bumping the counter
+        # after the first attempt, so it cannot keep the watcher in fast
+        # polling for the rest of DEADLINE; the next slow-cadence poll
+        # decides whether to retry. Cascading approvals from new ids still
+        # bump as they appear, which is the case the override targets.
+        new_attempted_ids = attempted_request_ids - FAST_REENTRY_BUMPED_REQUEST_IDS
+        if new_attempted_ids and FAST_REENTRY_POLLS > 0:
+            FAST_REENTRY_REMAINING = FAST_REENTRY_POLLS
+            FAST_REENTRY_BUMPED_REQUEST_IDS.update(new_attempted_ids)
+            mode_label = 'slow' if SLOW_MODE else 'fast'
+            print(f'[auto-pair] fast-reentry bumped polls={FAST_REENTRY_POLLS} approved={APPROVED} mode={mode_label}')
+        sleep_for_next_poll(SLOW_INTERVAL if SLOW_MODE else 1)
         continue
 
     QUIET_POLLS += 1
@@ -2046,13 +2110,15 @@ while time.time() < DEADLINE:
     # 5s in fast mode once anything is paired/approved, and SLOW_INTERVAL
     # (default 30s) after convergence. Slow-mode keepalive lets late CLI
     # scope upgrades get approved through the rest of DEADLINE without
-    # hammering the gateway.
+    # hammering the gateway. The bounded fast-reentry counter (bumped above
+    # when an allowlisted upgrade was attempted) overrides whichever tier
+    # is selected here so the next few polls catch cascading upgrades.
     if SLOW_MODE:
-        time.sleep(SLOW_INTERVAL)
+        sleep_for_next_poll(SLOW_INTERVAL)
     elif APPROVED > 0 or paired:
-        time.sleep(5)
+        sleep_for_next_poll(5)
     else:
-        time.sleep(1)
+        sleep_for_next_poll(1)
 else:
     print(f'[auto-pair] watcher deadline reached approvals={APPROVED}')
 PYAUTOPAIR

test/e2e-script-workflow.test.ts

@@ -136,6 +136,7 @@ const LEGACY_E2E_SHELL_ALLOWLIST = [
   "test/e2e/test-brave-search-e2e.sh",
   "test/e2e/test-channels-add-remove.sh",
   "test/e2e/test-channels-stop-start.sh",
+  "test/e2e/test-cli-scope-upgrade-approval.sh",
   "test/e2e/test-cloud-inference-e2e.sh",
   "test/e2e/test-cloud-onboard-e2e.sh",
   "test/e2e/test-common-egress-agent-e2e.sh",
@@ -162,7 +163,6 @@ const LEGACY_E2E_SHELL_ALLOWLIST = [
   "test/e2e/test-inference-routing.sh",
   "test/e2e/test-issue-2478-crash-loop-recovery.sh",
   "test/e2e/test-issue-4434-tui-unreachable-inference.sh",
-  "test/e2e/test-issue-4462-scope-upgrade-approval.sh",
   "test/e2e/test-jetson-nvmap-gpu.sh",
   "test/e2e/test-kimi-inference-compat.sh",
   "test/e2e/test-launchable-smoke.sh",
@@ -214,6 +214,7 @@ const NIGHTLY_E2E_SCRIPT_ALLOWLIST = [
   "test/e2e/test-brave-search-e2e.sh",
   "test/e2e/test-channels-add-remove.sh",
   "test/e2e/test-channels-stop-start.sh",
+  "test/e2e/test-cli-scope-upgrade-approval.sh",
   "test/e2e/test-cloud-inference-e2e.sh",
   "test/e2e/test-cloud-onboard-e2e.sh",
   "test/e2e/test-common-egress-agent-e2e.sh",
@@ -235,7 +236,6 @@ const NIGHTLY_E2E_SCRIPT_ALLOWLIST = [
   "test/e2e/test-inference-routing.sh",
   "test/e2e/test-issue-2478-crash-loop-recovery.sh",
   "test/e2e/test-issue-4434-tui-unreachable-inference.sh",
-  "test/e2e/test-issue-4462-scope-upgrade-approval.sh",
   "test/e2e/test-jetson-nvmap-gpu.sh",
   "test/e2e/test-kimi-inference-compat.sh",
   "test/e2e/test-launchable-smoke.sh",

test/e2e/test-issue-4462-scope-upgrade-approval.sh → test/e2e/test-cli-scope-upgrade-approval.sh

@@ -78,7 +78,7 @@ INSTALL_TIMEOUT_SECONDS="${NEMOCLAW_E2E_INSTALL_TIMEOUT_SECONDS:-1800}"
 
 AUTO_PAIR_FAST_DEADLINE_DEFAULT="3"
 AUTO_PAIR_DEADLINE_DEFAULT="30"
-AUTO_PAIR_SLOW_INTERVAL_DEFAULT="600"
+AUTO_PAIR_SLOW_INTERVAL_DEFAULT="5"
 AUTO_PAIR_RUN_TIMEOUT_DEFAULT="10"
 if [ "$TEST_MODE" = "legacy-repro" ]; then
   AUTO_PAIR_FAST_DEADLINE_DEFAULT="1"
@@ -858,7 +858,7 @@ info "$summary"
 initial_request_id=$(printf '%s' "$state" | select_cli_request new 2>/dev/null) || initial_request_id=""
 if [ -n "$initial_request_id" ]; then
   pass "pending low-scope CLI pairing request exists (${initial_request_id})"
-  approve_request "$initial_request_id" "initial CLI pairing" || exit 1
+  approve_request "$initial_request_id" "initial CLI pairing" 1 || exit 1
 else
   paired_without_write=$(printf '%s' "$state" | select_cli_paired_without_write 2>/dev/null) || paired_without_write=""
   if [ -n "$paired_without_write" ]; then
@@ -1043,6 +1043,48 @@ fi
 
 pass "approved agent output contains no fallback or pairing markers"
 
+section "Phase 6: Verify watcher emits slow-mode fast-reentry instrumentation"
+
+# Once the watcher converges (fast-deadline elapsed or paired devices seen
+# for several quiet polls) it transitions to slow-mode keepalive. A late
+# allowlisted scope upgrade arriving after that point must drop polling
+# back to a bounded fast-reentry window so the OpenClaw client does not
+# time out and fall back to embedded mode. Phases 3-5 leave at least one
+# paired device in the sandbox and the in-sandbox watcher runs with a
+# tight SLOW_INTERVAL (AUTO_PAIR_SLOW_INTERVAL_DEFAULT), so the watcher
+# is expected to observe one of the pending allowlisted requests before
+# the explicit approve_request wins the race and to record the
+# fast-reentry marker. Both the slow-mode transition and the marker are
+# asserted strictly here. The explicit approve_request calls earlier in
+# the test tolerate watcher-wins via allow_already_approved=1, so the
+# watcher catching the request first does not destabilise prior phases.
+
+auto_pair_log_snapshot=$(sandbox_exec_sh_script 20 '
+set -u
+cat /tmp/auto-pair.log 2>/dev/null || true
+' 2>&1)
+printf '=== /tmp/auto-pair.log snapshot ===\n%s\n' "$auto_pair_log_snapshot" >>"$STATE_LOG"
+
+slow_mode_observed=0
+if grep -F '[auto-pair] fast-mode deadline reached; switching to slow-mode' <<<"$auto_pair_log_snapshot" >/dev/null \
+  || grep -F '[auto-pair] browser pairing converged; entering slow-mode' <<<"$auto_pair_log_snapshot" >/dev/null \
+  || grep -F '[auto-pair] devices paired ' <<<"$auto_pair_log_snapshot" >/dev/null \
+  || grep -F '[auto-pair] non-browser pairing converged; entering slow-mode' <<<"$auto_pair_log_snapshot" >/dev/null; then
+  slow_mode_observed=1
+fi
+
+if [ "$slow_mode_observed" -eq 1 ]; then
+  pass "watcher reached slow-mode keepalive"
+else
+  fail "watcher did not record any slow-mode transition during the test window"
+fi
+
+if grep -F '[auto-pair] fast-reentry bumped' <<<"$auto_pair_log_snapshot" >/dev/null; then
+  pass "watcher logged fast-reentry marker on at least one allowlisted approval attempt"
+else
+  fail "watcher did not log any fast-reentry marker during the test window"
+fi
+
 section "Summary"
 echo ""
 printf '  Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m\n' \