fix(start): drop slow-mode polling on late allowlisted scope upgrades

.github/workflows/nightly-e2e.yaml

@@ -15,10 +15,10 @@
 #   openclaw-discord-pairing-e2e
 #                            Validates hermetic Discord pairing request approval across
 #                            gateway and connect-shell OpenClaw state roots (#4061).
-#   issue-4462-scope-upgrade-approval-e2e
+#   cli-scope-upgrade-approval-e2e
 #                            Validates real CLI scope-upgrade approval and confirms
-#                            the approved agent run stays on gateway mode (#4462).
-#   issue-4462-gateway-pinned-approval-characterization-e2e
+#                            the approved agent run stays on gateway mode.
+#   cli-scope-upgrade-legacy-repro-e2e
 #                            Characterizes legacy gateway-pinned scope approval
 #                            against a real sandbox, then recovers with the fix.
 #   messaging-compatible-endpoint-e2e
@@ -124,8 +124,8 @@ on:
           docs-validation-e2e, messaging-providers-e2e, openclaw-slack-pairing-e2e,
           openclaw-tui-chat-correlation-e2e, issue-4434-tui-unreachable-inference-e2e,
           issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e,
-          issue-4462-scope-upgrade-approval-e2e,
-          issue-4462-gateway-pinned-approval-characterization-e2e,
+          cli-scope-upgrade-approval-e2e,
+          cli-scope-upgrade-legacy-repro-e2e,
           messaging-compatible-endpoint-e2e, sessions-agents-cli-e2e,
           channels-add-remove-e2e, channels-stop-start-openclaw-e2e,
           channels-stop-start-hermes-e2e, brave-search-e2e,
@@ -631,51 +631,51 @@ jobs:
       nvidia_api_key: true
       github_token: true
     secrets: *nightly-e2e-default-secrets
-  # ── OpenClaw Scope-Upgrade Approval E2E (#4462) ────────────────
+  # ── OpenClaw Scope-Upgrade Approval E2E ────────────────
   # Positive proof: in a real sandbox, accept either a visible pending CLI
   # scope upgrade or the fixed watcher's immediate approval, then confirm
   # openclaw agent still uses the gateway path rather than embedded fallback.
-  issue-4462-scope-upgrade-approval-e2e:
+  cli-scope-upgrade-approval-e2e:
     if: >-
       github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
        inputs.jobs == '' ||
-       contains(format(',{0},', inputs.jobs), ',issue-4462-scope-upgrade-approval-e2e,'))
+       contains(format(',{0},', inputs.jobs), ',cli-scope-upgrade-approval-e2e,'))
     uses: ./.github/workflows/e2e-script.yaml
     with:
       ref: ${{ inputs.target_ref || github.ref }}
       script: test/e2e/test-issue-4462-scope-upgrade-approval.sh
       timeout_minutes: 60
-      artifact_name: "issue-4462-scope-upgrade-approval-logs"
+      artifact_name: "cli-scope-upgrade-approval-logs"
       artifact_path: |
-        /tmp/nemoclaw-e2e-issue-4462-scope-upgrade-install.log
-        /tmp/nemoclaw-issue-4462-scope-upgrade-approval.log
-        /tmp/nemoclaw-issue-4462-scope-upgrade-agent.log
-        /tmp/nemoclaw-issue-4462-scope-upgrade-state.log
-      env_json: '{"NEMOCLAW_4462_MODE":"approval","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AUTO_PAIR_DEADLINE_SECS":"30","NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS":"3","NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS":"600","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-issue-4462-scope-upgrade"}'
+        /tmp/nemoclaw-e2e-cli-scope-upgrade-install.log
+        /tmp/nemoclaw-cli-scope-upgrade-approval.log
+        /tmp/nemoclaw-cli-scope-upgrade-agent.log
+        /tmp/nemoclaw-cli-scope-upgrade-state.log
+      env_json: '{"NEMOCLAW_CLI_SCOPE_MODE":"approval","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AUTO_PAIR_DEADLINE_SECS":"30","NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS":"3","NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS":"5","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-cli-scope-upgrade"}'
       nvidia_api_key: true
       github_token: false
     secrets: *nightly-e2e-default-secrets
-  # ── OpenClaw Gateway-Pinned Approval Characterization (#4462) ──
+  # ── OpenClaw Gateway-Pinned Approval Characterization ──
   # Diagnostic proof: in a real sandbox, wait for the fixed watcher to exit,
   # force the legacy gateway-pinned approve path, record the observed
   # OpenClaw outcome, and recover through the fixed proxy-env guard if needed.
-  issue-4462-gateway-pinned-approval-characterization-e2e:
+  cli-scope-upgrade-legacy-repro-e2e:
     if: >-
       github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
        inputs.jobs == '' ||
-       contains(format(',{0},', inputs.jobs), ',issue-4462-gateway-pinned-approval-characterization-e2e,'))
+       contains(format(',{0},', inputs.jobs), ',cli-scope-upgrade-legacy-repro-e2e,'))
     uses: ./.github/workflows/e2e-script.yaml
     with:
       ref: ${{ inputs.target_ref || github.ref }}
       script: test/e2e/test-issue-4462-scope-upgrade-approval.sh
       timeout_minutes: 60
-      artifact_name: "issue-4462-gateway-pinned-approval-characterization-logs"
+      artifact_name: "cli-scope-upgrade-legacy-repro-logs"
       artifact_path: |
-        /tmp/nemoclaw-e2e-issue-4462-scope-upgrade-repro-install.log
-        /tmp/nemoclaw-issue-4462-scope-upgrade-repro-approval.log
-        /tmp/nemoclaw-issue-4462-scope-upgrade-repro-agent.log
-        /tmp/nemoclaw-issue-4462-scope-upgrade-repro-state.log
-      env_json: '{"NEMOCLAW_4462_AGENT_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-agent.log","NEMOCLAW_4462_APPROVAL_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-approval.log","NEMOCLAW_4462_AUTO_PAIR_DEADLINE_SECS":"12","NEMOCLAW_4462_AUTO_PAIR_FAST_DEADLINE_SECS":"1","NEMOCLAW_4462_AUTO_PAIR_RUN_TIMEOUT_SECS":"2","NEMOCLAW_4462_AUTO_PAIR_SLOW_INTERVAL_SECS":"1","NEMOCLAW_4462_INSTALL_LOG":"/tmp/nemoclaw-e2e-issue-4462-scope-upgrade-repro-install.log","NEMOCLAW_4462_MODE":"legacy-repro","NEMOCLAW_4462_STATE_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-state.log","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-issue-4462-scope-upgrade-repro"}'
+        /tmp/nemoclaw-e2e-cli-scope-upgrade-repro-install.log
+        /tmp/nemoclaw-cli-scope-upgrade-repro-approval.log
+        /tmp/nemoclaw-cli-scope-upgrade-repro-agent.log
+        /tmp/nemoclaw-cli-scope-upgrade-repro-state.log
+      env_json: '{"NEMOCLAW_CLI_SCOPE_AGENT_LOG":"/tmp/nemoclaw-cli-scope-upgrade-repro-agent.log","NEMOCLAW_CLI_SCOPE_APPROVAL_LOG":"/tmp/nemoclaw-cli-scope-upgrade-repro-approval.log","NEMOCLAW_CLI_SCOPE_AUTO_PAIR_DEADLINE_SECS":"12","NEMOCLAW_CLI_SCOPE_AUTO_PAIR_FAST_DEADLINE_SECS":"1","NEMOCLAW_CLI_SCOPE_AUTO_PAIR_RUN_TIMEOUT_SECS":"2","NEMOCLAW_CLI_SCOPE_AUTO_PAIR_SLOW_INTERVAL_SECS":"1","NEMOCLAW_CLI_SCOPE_INSTALL_LOG":"/tmp/nemoclaw-e2e-cli-scope-upgrade-repro-install.log","NEMOCLAW_CLI_SCOPE_MODE":"legacy-repro","NEMOCLAW_CLI_SCOPE_STATE_LOG":"/tmp/nemoclaw-cli-scope-upgrade-repro-state.log","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-cli-scope-upgrade-repro"}'
       nvidia_api_key: true
       github_token: false
     secrets: *nightly-e2e-default-secrets
@@ -2483,8 +2483,8 @@ jobs:
         issue-4434-tui-unreachable-inference-e2e,
         issue-3600-gpu-proof-optional-e2e,
         openclaw-discord-pairing-e2e,
-        issue-4462-scope-upgrade-approval-e2e,
-        issue-4462-gateway-pinned-approval-characterization-e2e,
+        cli-scope-upgrade-approval-e2e,
+        cli-scope-upgrade-legacy-repro-e2e,
         messaging-compatible-endpoint-e2e,
         sessions-agents-cli-e2e,
         channels-add-remove-e2e,
@@ -2601,8 +2601,8 @@ jobs:
         issue-4434-tui-unreachable-inference-e2e,
         issue-3600-gpu-proof-optional-e2e,
         openclaw-discord-pairing-e2e,
-        issue-4462-scope-upgrade-approval-e2e,
-        issue-4462-gateway-pinned-approval-characterization-e2e,
+        cli-scope-upgrade-approval-e2e,
+        cli-scope-upgrade-legacy-repro-e2e,
         messaging-compatible-endpoint-e2e,
         sessions-agents-cli-e2e,
         channels-add-remove-e2e,
@@ -2788,8 +2788,8 @@ jobs:
         issue-4434-tui-unreachable-inference-e2e,
         issue-3600-gpu-proof-optional-e2e,
         openclaw-discord-pairing-e2e,
-        issue-4462-scope-upgrade-approval-e2e,
-        issue-4462-gateway-pinned-approval-characterization-e2e,
+        cli-scope-upgrade-approval-e2e,
+        cli-scope-upgrade-legacy-repro-e2e,
         messaging-compatible-endpoint-e2e,
         sessions-agents-cli-e2e,
         channels-add-remove-e2e,

scripts/nemoclaw-start.sh

@@ -2197,14 +2197,16 @@ start_auto_pair() {
   if [ "$(id -u)" -eq 0 ]; then
     run_prefix=("${STEP_DOWN_PREFIX_SANDBOX[@]}")
   fi
-  OPENCLAW_BIN="$OPENCLAW" nohup "${run_prefix[@]}" python3 - <<'PYAUTOPAIR' >>/tmp/auto-pair.log 2>&1 &
+  OPENCLAW_BIN="$OPENCLAW" nohup "${run_prefix[@]}" python3 -u - <<'PYAUTOPAIR' >>/tmp/auto-pair.log 2>&1 &
 import json
 import importlib.util
 import os
 import stat
 import subprocess
 import time
 
+print('[auto-pair] watcher started', flush=True)
+
 APPROVAL_POLICY_FILE = '/usr/local/lib/nemoclaw/openclaw_device_approval_policy.py'
 
 
@@ -2251,7 +2253,29 @@ def _env_seconds(name, default):
 # embedded mode. Defaults: 8h total, 30s slow-mode cadence.
 FAST_DEADLINE = time.time() + _env_seconds('NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS', 600)
 DEADLINE = time.time() + _env_seconds('NEMOCLAW_AUTO_PAIR_DEADLINE_SECS', 28800)
-SLOW_INTERVAL = _env_seconds('NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS', 30)
+# After convergence the watcher polls at SLOW_INTERVAL. A late allowlisted
+# scope upgrade — e.g. `openclaw tui` or `openclaw agent` invoked after the
+# watcher entered slow mode — can wait up to SLOW_INTERVAL before being
+# approved, which is longer than the OpenClaw client's tolerance for `scope
+# upgrade pending approval` and forces a fallback to embedded mode. The
+# default sits well below typical client-side wait windows; raise it through
+# NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS when the gateway connect handler is
+# load-sensitive. When the watcher successfully approves a fresh allowlisted
+# request during slow mode it also bumps a bounded fast-reentry counter
+# (NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS) that drops polling back to 1s for
+# the next few iterations, so cascading upgrades and transient approve
+# failures both clear before the OpenClaw client gives up. The counter is
+# only bumped on the rising edge for each requestId (tracked in
+# FAST_REENTRY_BUMPED_REQUEST_IDS and garbage-collected against the live
+# pending list), so a sticky failing request cannot pin the watcher in fast
+# polling. This is a polling-cadence fix only — non-allowlisted scopes such
+# as `operator.admin` are still rejected by the device approval policy, and
+# requests that need them must be approved through a separate operator path.
+SLOW_INTERVAL = _env_seconds('NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS', 5)
+FAST_REENTRY_POLLS = int(_env_seconds('NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS', 5))
+FAST_REENTRY_INTERVAL = _env_seconds('NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS', 1)
+FAST_REENTRY_REMAINING = 0
+FAST_REENTRY_BUMPED_REQUEST_IDS = set()
 QUIET_POLLS = 0
 APPROVED = 0
 SLOW_MODE = False
@@ -2293,15 +2317,33 @@ def run(*args, strip_gateway_env=False):
         print(f'[auto-pair] timeout calling {args[1] if len(args) > 1 else "openclaw"} {args[2] if len(args) > 2 else ""}'.rstrip())
         return 124, out.strip(), err.strip()
 
+
+def sleep_for_next_poll(default_seconds):
+    # Apply the bounded fast-reentry override before the caller's default
+    # sleep so a recent allowlisted approval (which bumps the remaining
+    # counter) drops polling to FAST_REENTRY_INTERVAL for the next few
+    # iterations. Mutates the global counter so callers do not need to
+    # thread the state through. The override is floored by the caller's
+    # default so it never increases the inter-poll latency (e.g. when the
+    # default is already tighter than FAST_REENTRY_INTERVAL during a
+    # bounded retry pass in fast mode).
+    global FAST_REENTRY_REMAINING
+    if FAST_REENTRY_REMAINING > 0:
+        FAST_REENTRY_REMAINING -= 1
+        time.sleep(min(FAST_REENTRY_INTERVAL, default_seconds))
+        return
+    time.sleep(default_seconds)
+
+
 while time.time() < DEADLINE:
     rc, out, err = run(OPENCLAW, 'devices', 'list', '--json')
     if rc != 0 or not out:
-        time.sleep(SLOW_INTERVAL if SLOW_MODE else 1)
+        sleep_for_next_poll(SLOW_INTERVAL if SLOW_MODE else 1)
         continue
     try:
         data = json.loads(out)
     except Exception:
-        time.sleep(SLOW_INTERVAL if SLOW_MODE else 1)
+        sleep_for_next_poll(SLOW_INTERVAL if SLOW_MODE else 1)
         continue
 
     pending = data.get('pending') or []
@@ -2320,11 +2362,16 @@ while time.time() < DEADLINE:
 
     if pending:
         QUIET_POLLS = 0
+        attempted_request_ids = set()
+        pending_request_ids = set()
         for device in pending:
             if not isinstance(device, dict):
                 continue
             request_id = device.get('requestId')
-            if not request_id or request_id in HANDLED:
+            if not request_id:
+                continue
+            pending_request_ids.add(request_id)
+            if request_id in HANDLED:
                 continue
             decision = approval_request_decision(device)
             client_id = decision['client_id']
@@ -2342,6 +2389,7 @@ while time.time() < DEADLINE:
                 scopes = decision['scopes']
                 print(f'[auto-pair] rejected disallowed scopes={sorted(scopes)} client={client_id} mode={client_mode}')
                 continue
+            attempted_request_ids.add(request_id)
             arc, aout, aerr = run(
                 OPENCLAW, 'devices', 'approve', request_id, '--json', strip_gateway_env=True,
             )
@@ -2371,7 +2419,25 @@ while time.time() < DEADLINE:
                     print(f'[auto-pair] approve failed request={request_id}: {(aerr or aout)[:400]}')
             elif aout or aerr:
                 print(f'[auto-pair] approve failed request={request_id}: {(aerr or aout)[:400]}')
-        time.sleep(SLOW_INTERVAL if SLOW_MODE else 1)
+        # Drop previously-bumped requestIds that the gateway no longer reports
+        # as pending so a future re-appearance of the same id (very unlikely,
+        # but kept robust) can bump again. The set is otherwise small and
+        # never crosses out of the watcher process.
+        FAST_REENTRY_BUMPED_REQUEST_IDS.intersection_update(pending_request_ids)
+        # Fast-reentry is armed on the rising edge per requestId — once for
+        # each freshly-observed allowlisted attempt. A sticky pending request
+        # that fails approval repeatedly therefore stops bumping the counter
+        # after the first attempt, so it cannot keep the watcher in fast
+        # polling for the rest of DEADLINE; the next slow-cadence poll
+        # decides whether to retry. Cascading approvals from new ids still
+        # bump as they appear, which is the case the override targets.
+        new_attempted_ids = attempted_request_ids - FAST_REENTRY_BUMPED_REQUEST_IDS
+        if new_attempted_ids and FAST_REENTRY_POLLS > 0:
+            FAST_REENTRY_REMAINING = FAST_REENTRY_POLLS
+            FAST_REENTRY_BUMPED_REQUEST_IDS.update(new_attempted_ids)
+            mode_label = 'slow' if SLOW_MODE else 'fast'
+            print(f'[auto-pair] fast-reentry bumped polls={FAST_REENTRY_POLLS} approved={APPROVED} mode={mode_label}')
+        sleep_for_next_poll(SLOW_INTERVAL if SLOW_MODE else 1)
         continue
 
     QUIET_POLLS += 1
@@ -2404,13 +2470,15 @@ while time.time() < DEADLINE:
     # 5s in fast mode once anything is paired/approved, and SLOW_INTERVAL
     # (default 30s) after convergence. Slow-mode keepalive lets late CLI
     # scope upgrades get approved through the rest of DEADLINE without
-    # hammering the gateway.
+    # hammering the gateway. The bounded fast-reentry counter (bumped above
+    # when an allowlisted upgrade was attempted) overrides whichever tier
+    # is selected here so the next few polls catch cascading upgrades.
     if SLOW_MODE:
-        time.sleep(SLOW_INTERVAL)
+        sleep_for_next_poll(SLOW_INTERVAL)
     elif APPROVED > 0 or paired:
-        time.sleep(5)
+        sleep_for_next_poll(5)
     else:
-        time.sleep(1)
+        sleep_for_next_poll(1)
 else:
     print(f'[auto-pair] watcher deadline reached approvals={APPROVED}')
 PYAUTOPAIR

test/e2e-script-workflow.test.ts

@@ -1043,4 +1043,49 @@ describe("E2E reusable workflow contract", () => {
     );
     expect(script).toContain("entrypoint start log: ${entrypoint_start_log_stat}");
   });
+
+  it("secret-bearing scope-upgrade e2e verifies the Ollama tarball sha256 before privileged extract", () => {
+    const job = nightlyWorkflow.jobs["cli-scope-upgrade-approval-e2e"];
+    expect(job).toBeDefined();
+    expect(String(job.with?.nvidia_api_key)).toBe("true");
+    expect(job.with?.script).toBe("test/e2e/test-issue-4462-scope-upgrade-approval.sh");
+
+    const script = readFileSync(
+      new URL("./e2e/test-issue-4462-scope-upgrade-approval.sh", import.meta.url),
+      "utf8",
+    );
+
+    expect(script).toMatch(/OLLAMA_PINNED_VERSION_DEFAULT="\d+\.\d+\.\d+"/);
+    expect(script).toMatch(/OLLAMA_PINNED_SHA256_DEFAULT="[0-9a-f]{64}"/);
+    expect(script).toContain(
+      'fail "Ollama install requires NEMOCLAW_CLI_SCOPE_OLLAMA_SHA256 when NEMOCLAW_CLI_SCOPE_OLLAMA_VERSION overrides the pinned default',
+    );
+    expect(script).toContain('if [ "$computed_sha" != "$OLLAMA_PINNED_SHA256" ]; then');
+    expect(script).toContain('sudo tar -C /usr/local -xzf "${install_tmp}/ollama.tgz"');
+    expect(script).not.toContain("Skipping Ollama tarball sha256 verification");
+  });
+
+  it("default Phase 7 lane does not claim full #5343 qwen3.5 model coverage", () => {
+    const script = readFileSync(
+      new URL("./e2e/test-issue-4462-scope-upgrade-approval.sh", import.meta.url),
+      "utf8",
+    );
+
+    const defaultModelMatch = script.match(
+      /OLLAMA_TWO_PROVIDER_MODEL="\$\{NEMOCLAW_CLI_SCOPE_OLLAMA_MODEL:-([^}]+)\}"/,
+    );
+    const specModelMatch = script.match(/OLLAMA_SPEC_MODEL_5343="([^"]+)"/);
+    expect(defaultModelMatch?.[1]).toBeDefined();
+    expect(specModelMatch?.[1]).toBe("qwen3.5:9b");
+    expect(defaultModelMatch?.[1]).not.toBe(specModelMatch?.[1]);
+
+    expect(script).toContain(
+      'section "Phase 7 (CPU-substitute lane): Verify two-sandbox concurrent differing-provider gateway-backed agent turns"',
+    );
+    expect(script).toContain("Phase 7 CPU-lane substitute: using ${OLLAMA_TWO_PROVIDER_MODEL}");
+    expect(script).toContain(
+      'if [ "$OLLAMA_TWO_PROVIDER_MODEL" = "$OLLAMA_SPEC_MODEL_5343" ]; then',
+    );
+    expect(script).toContain("substituting for GPU-only spec model");
+  });
 });