diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index a566b3556aa..b2483e7937e 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -2197,7 +2197,7 @@ start_auto_pair() { if [ "$(id -u)" -eq 0 ]; then run_prefix=("${STEP_DOWN_PREFIX_SANDBOX[@]}") fi - OPENCLAW_BIN="$OPENCLAW" nohup "${run_prefix[@]}" python3 - <<'PYAUTOPAIR' >>/tmp/auto-pair.log 2>&1 & + OPENCLAW_BIN="$OPENCLAW" nohup "${run_prefix[@]}" python3 -u - <<'PYAUTOPAIR' >>/tmp/auto-pair.log 2>&1 & import json import importlib.util import os @@ -2205,6 +2205,8 @@ import stat import subprocess import time +print('[auto-pair] watcher started', flush=True) + APPROVAL_POLICY_FILE = '/usr/local/lib/nemoclaw/openclaw_device_approval_policy.py' @@ -2251,7 +2253,55 @@ def _env_seconds(name, default): # embedded mode. Defaults: 8h total, 30s slow-mode cadence. FAST_DEADLINE = time.time() + _env_seconds('NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS', 600) DEADLINE = time.time() + _env_seconds('NEMOCLAW_AUTO_PAIR_DEADLINE_SECS', 28800) -SLOW_INTERVAL = _env_seconds('NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS', 30) +# After convergence the watcher polls at SLOW_INTERVAL. A late allowlisted +# scope upgrade — e.g. `openclaw tui` or `openclaw agent` invoked after the +# watcher entered slow mode — can wait up to SLOW_INTERVAL before being +# approved, which is longer than the OpenClaw client's tolerance for `scope +# upgrade pending approval` and forces a fallback to embedded mode. The +# default sits well below typical client-side wait windows; raise it through +# NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS when the gateway connect handler is +# load-sensitive. When the watcher successfully approves a fresh allowlisted +# request during slow mode it also bumps a bounded fast-reentry counter +# (NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS) that drops polling back to 1s for +# the next few iterations, so cascading upgrades and transient approve +# failures both clear before the OpenClaw client gives up. The counter is +# only bumped on the rising edge for each requestId (tracked in +# FAST_REENTRY_BUMPED_REQUEST_IDS and garbage-collected against the live +# pending list), so a sticky failing request cannot pin the watcher in fast +# polling. This is a polling-cadence fix only — non-allowlisted scopes such +# as `operator.admin` are still rejected by the device approval policy, and +# requests that need them must be approved through a separate operator path. +SLOW_INTERVAL = _env_seconds('NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS', 5) +# SOURCE_OF_TRUTH_REVIEW (auto-pair slow-mode cadence default 30s → 5s): +# +# * Source boundary: the single SLOW_INTERVAL global above is the only +# steady-state inter-poll wait for the in-sandbox auto-pair watcher +# after browser pairing converges. The watcher's faster pre-converge +# cadence (1s) is unaffected. +# * Invalid state at the old default: a late +# `openclaw tui` / `openclaw agent` allowlisted scope upgrade lands +# inside a 30s window and waits up to one full SLOW_INTERVAL before +# the watcher polls. Two sibling sandboxes onboarded back-to-back +# each hit this window and both fall back to embedded mode (#5343). +# * Source-fix constraint: the 5s default is a bounded 6x increase in +# steady-state `openclaw devices list --json` calls per sandbox — at +# most one extra call per 5s vs. per 30s, which the gateway connect +# handler tolerates easily; the bounded fast-reentry counter above +# keeps cascading upgrades from exceeding this cadence. +# * Migration: operators who relied on the old cadence (load-sensitive +# gateways, large multi-sandbox deployments) can restore it by +# exporting NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS=30 in the sandbox +# environment; the PR body calls this out under "Changes" too. +# * Regression test: test/nemoclaw-start.test.ts's late-CLI fixture +# covers the new default deterministically; #5343 Phase 5 covers it +# end to end. +# * Removal condition: when OpenClaw signals scope-upgrade requests via +# a push channel rather than a poll, the cadence becomes irrelevant +# and the variable retires. +FAST_REENTRY_POLLS = int(_env_seconds('NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS', 5)) +FAST_REENTRY_INTERVAL = _env_seconds('NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS', 1) +FAST_REENTRY_REMAINING = 0 +FAST_REENTRY_BUMPED_REQUEST_IDS = set() QUIET_POLLS = 0 APPROVED = 0 SLOW_MODE = False @@ -2293,15 +2343,39 @@ def run(*args, strip_gateway_env=False): print(f'[auto-pair] timeout calling {args[1] if len(args) > 1 else "openclaw"} {args[2] if len(args) > 2 else ""}'.rstrip()) return 124, out.strip(), err.strip() + +def sleep_for_next_poll(default_seconds, productive=True): + # Apply the bounded fast-reentry override before the caller's default + # sleep so a recent allowlisted approval (which bumps the remaining + # counter) drops polling to FAST_REENTRY_INTERVAL for the next few + # iterations. Mutates the global counter so callers do not need to + # thread the state through. The override is floored by the caller's + # default so it never increases the inter-poll latency (e.g. when the + # default is already tighter than FAST_REENTRY_INTERVAL during a + # bounded retry pass in fast mode). + # + # Error-path callers pass productive=False so a string of gateway + # errors or JSON-parse failures after a fast-reentry bump does not + # silently drain the bounded window before a productive poll observes + # the cascading upgrades. + global FAST_REENTRY_REMAINING + if FAST_REENTRY_REMAINING > 0: + if productive: + FAST_REENTRY_REMAINING -= 1 + time.sleep(min(FAST_REENTRY_INTERVAL, default_seconds)) + return + time.sleep(default_seconds) + + while time.time() < DEADLINE: rc, out, err = run(OPENCLAW, 'devices', 'list', '--json') if rc != 0 or not out: - time.sleep(SLOW_INTERVAL if SLOW_MODE else 1) + sleep_for_next_poll(SLOW_INTERVAL if SLOW_MODE else 1, productive=False) continue try: data = json.loads(out) except Exception: - time.sleep(SLOW_INTERVAL if SLOW_MODE else 1) + sleep_for_next_poll(SLOW_INTERVAL if SLOW_MODE else 1, productive=False) continue pending = data.get('pending') or [] @@ -2320,11 +2394,16 @@ while time.time() < DEADLINE: if pending: QUIET_POLLS = 0 + attempted_request_ids = set() + pending_request_ids = set() for device in pending: if not isinstance(device, dict): continue request_id = device.get('requestId') - if not request_id or request_id in HANDLED: + if not request_id: + continue + pending_request_ids.add(request_id) + if request_id in HANDLED: continue decision = approval_request_decision(device) client_id = decision['client_id'] @@ -2342,6 +2421,7 @@ while time.time() < DEADLINE: scopes = decision['scopes'] print(f'[auto-pair] rejected disallowed scopes={sorted(scopes)} client={client_id} mode={client_mode}') continue + attempted_request_ids.add(request_id) arc, aout, aerr = run( OPENCLAW, 'devices', 'approve', request_id, '--json', strip_gateway_env=True, ) @@ -2371,7 +2451,30 @@ while time.time() < DEADLINE: print(f'[auto-pair] approve failed request={request_id}: {(aerr or aout)[:400]}') elif aout or aerr: print(f'[auto-pair] approve failed request={request_id}: {(aerr or aout)[:400]}') - time.sleep(SLOW_INTERVAL if SLOW_MODE else 1) + # Drop previously-bumped requestIds that the gateway no longer reports + # as pending so a future re-appearance of the same id (very unlikely, + # but kept robust) can bump again. The set is otherwise small and + # never crosses out of the watcher process. + FAST_REENTRY_BUMPED_REQUEST_IDS.intersection_update(pending_request_ids) + # Fast-reentry is armed on the rising edge per requestId — once for + # each freshly-observed allowlisted attempt. A sticky pending request + # that fails approval repeatedly therefore stops bumping the counter + # after the first attempt, so it cannot keep the watcher in fast + # polling for the rest of DEADLINE; the next slow-cadence poll + # decides whether to retry. Cascading approvals from new ids still + # bump as they appear, which is the case the override targets. + new_attempted_ids = attempted_request_ids - FAST_REENTRY_BUMPED_REQUEST_IDS + # Bump in fast mode too: the cadence override is a no-op there + # (min(FAST_REENTRY_INTERVAL=1, default=1) = 1) but the requestId + # is still recorded in FAST_REENTRY_BUMPED_REQUEST_IDS so the same + # sticky id cannot re-arm the counter later when the watcher + # transitions into slow mode. + if new_attempted_ids and FAST_REENTRY_POLLS > 0: + FAST_REENTRY_REMAINING = FAST_REENTRY_POLLS + FAST_REENTRY_BUMPED_REQUEST_IDS.update(new_attempted_ids) + mode_label = 'slow' if SLOW_MODE else 'fast' + print(f'[auto-pair] fast-reentry bumped polls={FAST_REENTRY_POLLS} approved={APPROVED} mode={mode_label}') + sleep_for_next_poll(SLOW_INTERVAL if SLOW_MODE else 1) continue QUIET_POLLS += 1 @@ -2402,15 +2505,17 @@ while time.time() < DEADLINE: # Back off polling: 1s in fast mode while waiting for first pairing, # 5s in fast mode once anything is paired/approved, and SLOW_INTERVAL - # (default 30s) after convergence. Slow-mode keepalive lets late CLI + # (default 5s) after convergence. Slow-mode keepalive lets late CLI # scope upgrades get approved through the rest of DEADLINE without - # hammering the gateway. + # hammering the gateway. The bounded fast-reentry counter (bumped above + # when an allowlisted upgrade was attempted) overrides whichever tier + # is selected here so the next few polls catch cascading upgrades. if SLOW_MODE: - time.sleep(SLOW_INTERVAL) + sleep_for_next_poll(SLOW_INTERVAL) elif APPROVED > 0 or paired: - time.sleep(5) + sleep_for_next_poll(5) else: - time.sleep(1) + sleep_for_next_poll(1) else: print(f'[auto-pair] watcher deadline reached approvals={APPROVED}') PYAUTOPAIR @@ -3730,6 +3835,13 @@ gateway_pid_is_openclaw_gateway() { printf '%s' "$cmdline" | grep -qE 'openclaw([ -]gateway| gateway run|$)' } +# Positive integer guard used by the gateway watchdog env validation. Extracted +# so a regression test can exercise the regex against trailing-non-digit and +# zero/garbage inputs without spinning up the whole watcher. +gateway_watchdog_positive_int_ok() { + [[ "$1" =~ ^[1-9][0-9]*$ ]] +} + start_gateway_serving_watchdog() { ( local interval refused_threshold armed=0 refused_streak=0 pid last_pid="" rc msg @@ -3738,20 +3850,16 @@ start_gateway_serving_watchdog() { # Both knobs must be positive integers: a zero/garbage interval would # busy-loop the probe, and a zero threshold would kill on the first # refusal. Fall back to the defaults rather than trusting bad input. - case "$interval" in - [1-9] | [1-9][0-9]*) ;; - *) - echo "[gateway-watchdog] invalid NEMOCLAW_GATEWAY_WATCHDOG_INTERVAL_SECONDS='${interval}'; defaulting to 30" >&2 - interval=30 - ;; - esac - case "$refused_threshold" in - [1-9] | [1-9][0-9]*) ;; - *) - echo "[gateway-watchdog] invalid NEMOCLAW_GATEWAY_WATCHDOG_REFUSED_THRESHOLD='${refused_threshold}'; defaulting to 4" >&2 - refused_threshold=4 - ;; - esac + # gateway_watchdog_positive_int_ok uses regex (=~), not glob, so trailing + # non-digit input like "12x" or "30abc" is rejected, not coerced. + if ! gateway_watchdog_positive_int_ok "$interval"; then + echo "[gateway-watchdog] invalid NEMOCLAW_GATEWAY_WATCHDOG_INTERVAL_SECONDS='${interval}'; defaulting to 30" >&2 + interval=30 + fi + if ! gateway_watchdog_positive_int_ok "$refused_threshold"; then + echo "[gateway-watchdog] invalid NEMOCLAW_GATEWAY_WATCHDOG_REFUSED_THRESHOLD='${refused_threshold}'; defaulting to 4" >&2 + refused_threshold=4 + fi [ -n "${_DASHBOARD_PORT:-}" ] || exit 0 while :; do sleep "$interval" diff --git a/src/lib/actions/inference-set.test.ts b/src/lib/actions/inference-set.test.ts index b18e18fd6be..3b03c1922a3 100644 --- a/src/lib/actions/inference-set.test.ts +++ b/src/lib/actions/inference-set.test.ts @@ -244,6 +244,11 @@ describe("patchOpenClawInferenceConfig", () => { it("is a no-op when OpenClaw already matches the requested route", () => { const config: ConfigObject = { + _nemoclaw_upstream: { + provider: "nvidia-prod", + model: "nvidia/model-a", + base_url: "https://inference.local/v1", + }, agents: { defaults: { model: { primary: "inference/nvidia/model-a" } } }, models: { mode: "merge", diff --git a/test/e2e/lib/read-host-registry.py b/test/e2e/lib/read-host-registry.py new file mode 100755 index 00000000000..38b5292879e --- /dev/null +++ b/test/e2e/lib/read-host-registry.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Read the NemoClaw host-side sandbox registry and emit provider/model JSON. + +SOURCE_OF_TRUTH_REVIEW (Phase 7 / #5343 differing-providers): + +- Source boundary: ``~/.nemoclaw/sandboxes.json`` written by every + ``nemoclaw onboard`` / ``nemoclaw inference-set``. This file is the only + host-side record of which provider and model each sandbox was configured + with; the in-sandbox OpenClaw config flattens every managed route to + ``providerKey="inference"`` via ``patchOpenClawInferenceConfig`` and is + therefore insufficient to distinguish "sandbox A on NVIDIA Cloud" from + "sandbox B on Ollama-local" — that is what makes Phase 7's + differing-providers assertion meaningful. +- Invalid state: the registry file is missing, unreadable, malformed JSON, + or has no entry for the named sandbox. +- Source-fix constraint: this script never writes to the registry; it only + reads. Anything else that needs provider/model intent must come through + this single reader so a schema drift in the host registry surfaces in + one place. +- Regression test: ``test/ollama-pinned-install.test.ts`` covers the + shell-side caller; the in-sandbox effective route uses a separate reader + (``read-openclaw-route.py``) that runs inside the sandbox itself. +- Removal condition: when NemoClaw exposes a stable read-only API for + per-sandbox effective inference metadata, this reader becomes a wrapper + around that API and the JSON-on-disk path is dropped. + +Exit codes: 0 on success; 2 if the registry file is unreadable or invalid; +3 if the named sandbox is not registered (fail-closed for Phase 7's +two-sandbox contract). +""" + +import json +import os +import sys + + +def main() -> int: + sandbox_name = sys.argv[1] + registry_file = os.path.join( + os.environ.get("HOME", "/tmp"), + ".nemoclaw", + "sandboxes.json", + ) + try: + with open(registry_file, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError) as exc: + sys.stderr.write(f"registry-read-failed: {exc}\n") + return 2 + + entries = data.get("sandboxes") or {} + if sandbox_name not in entries: + sys.stderr.write( + f"registry-missing-sandbox: {sandbox_name!r} not registered in {registry_file}\n", + ) + return 3 + entry = entries.get(sandbox_name) or {} + provider = str(entry.get("provider") or "").strip() + model = str(entry.get("model") or "").strip() + print(json.dumps({"provider": provider, "model": model}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/e2e/lib/redact-device-state.py b/test/e2e/lib/redact-device-state.py new file mode 100755 index 00000000000..659b1020147 --- /dev/null +++ b/test/e2e/lib/redact-device-state.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Redact secret-shaped fields and values from device-state JSON. + +Reads a JSON document on stdin, walks dicts and lists, and replaces any field +whose key matches the secret-name shape with [REDACTED]. String values whose +content matches the secret-value shape (JWT, GitHub PAT, OpenAI/NVIDIA/HF +keys, AWS access keys, Slack tokens) are also replaced. Writes the redacted +JSON to stdout. Preserves request IDs, device IDs, client modes, and scope +lists used for diagnosis. +""" + +import json +import re +import sys + +SECRET_FIELD_RE = re.compile( + r"(?:^|[._-])(token|tokens|secret|secrets|credential|credentials|" + r"authorization|authorisation|auth|password|passwd|apikey|api_key|" + r"access_key|refresh|cookie|cookies|header|headers|bearer)(?:$|[._-])", + re.IGNORECASE, +) +SECRET_VALUE_RE = re.compile( + r"^(?:eyJ[A-Za-z0-9_-]{6,}|gh[pousr]_[A-Za-z0-9]{16,}|" + r"github_pat_[A-Za-z0-9_]{20,}|sk-[A-Za-z0-9_-]{12,}|" + r"nvapi-[A-Za-z0-9._-]{12,}|hf_[A-Za-z0-9]{16,}|" + r"AKIA[0-9A-Z]{12,}|ASIA[0-9A-Z]{12,}|xox[abprs]-[A-Za-z0-9-]{8,})" +) +REDACTED = "[REDACTED]" + + +def redact(value): + if isinstance(value, dict): + clean = {} + for key, item in value.items(): + if isinstance(key, str) and SECRET_FIELD_RE.search(key): + clean[key] = REDACTED + else: + clean[key] = redact(item) + return clean + if isinstance(value, list): + return [redact(item) for item in value] + if isinstance(value, str) and SECRET_VALUE_RE.match(value): + return REDACTED + return value + + +def main() -> int: + try: + raw = sys.stdin.read() + doc = json.loads(raw) if raw.strip() else {} + except json.JSONDecodeError as exc: + sys.stderr.write(f"redact-device-state: invalid JSON on stdin: {exc}\n") + return 1 + json.dump(redact(doc), sys.stdout, sort_keys=True) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/e2e/lib/redact-text.py b/test/e2e/lib/redact-text.py new file mode 100755 index 00000000000..caa392affea --- /dev/null +++ b/test/e2e/lib/redact-text.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Redact token-shaped substrings from arbitrary text streams. + +Reads text on stdin, replaces matches for known token shapes (JWT, GitHub PAT, +OpenAI/NVIDIA/HF keys, AWS access keys, Slack tokens, and HTTP Authorization / +Bearer headers and `token=` / `apikey=` style query parameters) with +``[REDACTED]``, then writes the scrubbed text to stdout. Used to scrub +diagnostic log excerpts (gateway, auto-pair) before they are appended to +secret-bearing E2E artefacts. + +Secret-shape catalogue extensibility +------------------------------------ + +Adding a new token shape: + +1. Add a branch to ``TOKEN_VALUE_RE`` (whole-token matches; substituted + wholesale) OR to ``AUTH_HEADER_RE`` / ``BEARER_RE`` / ``QUERY_PARAM_RE`` + (matches keep the structural prefix and redact only the value). +2. Keep the branch as conservative as the existing entries — anchor with + ``\\b`` or a structural prefix so the pattern never matches inside + ordinary hyphenated text. The ``\\bsk-`` branch is the canonical + left-bounded shape; copy that form for new vendor prefixes. +3. Add a positive test (the shape is redacted) AND a regression test (a + near-miss is preserved) in ``test/redact-text.test.ts``. Both must run + before merge; ``test/redact-text.test.ts`` is the single home for + redactor unit coverage. +4. Document the new shape in the file-level paragraph above so reviewers + know the catalogue surface without reading the regex. + +This module intentionally has no external dependencies and no error +paths beyond the regex substitution; stdout is a deterministic function +of stdin. Callers that need a fail-closed marker (e.g. on a transient +subprocess failure) wrap the invocation in +``redact_text_for_log_or_marker`` in +``test/e2e/test-issue-4462-scope-upgrade-approval.sh``. +""" + +import re +import sys + +TOKEN_VALUE_RE = re.compile( + r"eyJ[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_.-]+" + r"|gh[pousr]_[A-Za-z0-9]{16,}" + r"|github_pat_[A-Za-z0-9_]{20,}" + r"|\bsk-[A-Za-z0-9_-]{12,}" + r"|nvapi-[A-Za-z0-9._-]{12,}" + r"|hf_[A-Za-z0-9]{16,}" + r"|AKIA[0-9A-Z]{12,}" + r"|ASIA[0-9A-Z]{12,}" + r"|xox[abprs]-[A-Za-z0-9-]{8,}" +) +AUTH_HEADER_RE = re.compile( + r"(?i)((?:authorization|authorisation|x-api-key|api-key|x-auth-token|" + r"x-nvidia-api-key|x-openrouter-api-key|cookie|set-cookie)" + r"\s*[:=]\s*(?:Bearer|Token|Basic)?\s*)([^\s,;'\"]+)" +) +BEARER_RE = re.compile(r"(?i)(\b(?:Bearer|Token)\s+)([^\s,;'\"]+)") +QUERY_PARAM_RE = re.compile( + r"(?i)(\b(?:token|api[_-]?key|access[_-]?token|refresh[_-]?token|" + r"client[_-]?secret|password|passwd|secret)\s*=\s*)([^\s&'\"]+)" +) +REDACTED = "[REDACTED]" + + +def redact_line(line: str) -> str: + cleaned = TOKEN_VALUE_RE.sub(REDACTED, line) + cleaned = AUTH_HEADER_RE.sub(r"\1" + REDACTED, cleaned) + cleaned = BEARER_RE.sub(r"\1" + REDACTED, cleaned) + cleaned = QUERY_PARAM_RE.sub(r"\1" + REDACTED, cleaned) + return cleaned + + +def main() -> int: + text = sys.stdin.read() + sys.stdout.write("".join(redact_line(line) for line in text.splitlines(keepends=True))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/gateway-watchdog-validation.test.ts b/test/gateway-watchdog-validation.test.ts new file mode 100644 index 00000000000..2502be24577 --- /dev/null +++ b/test/gateway-watchdog-validation.test.ts @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const START_SCRIPT = path.resolve(HERE, "..", "scripts", "nemoclaw-start.sh"); + +function requireNonNegative(value: number, message: string): number { + return value >= 0 + ? value + : (() => { + throw new Error(message); + })(); +} + +function extractShellFunction(scriptPath: string, name: string): string { + const body = readFileSync(scriptPath, "utf8"); + const startMarker = `${name}() {`; + const start = requireNonNegative( + body.indexOf(startMarker), + `function ${name} not found in ${scriptPath}`, + ); + const lines = body.slice(start).split("\n"); + const endIndex = requireNonNegative( + lines.findIndex((line, index) => index > 0 && line === "}"), + `function ${name} missing closing brace in ${scriptPath}`, + ); + return lines.slice(0, endIndex + 1).join("\n"); +} + +function runGuard(value: string): number { + const functionBody = extractShellFunction(START_SCRIPT, "gateway_watchdog_positive_int_ok"); + const harness = ` +${functionBody} +gateway_watchdog_positive_int_ok "$1" +`; + const result = spawnSync("bash", ["-c", harness, "bash", value], { + encoding: "utf-8", + timeout: 10_000, + }); + return result.status ?? -1; +} + +describe("gateway watchdog numeric env guard", () => { + it.each([ + ["1", 0], + ["12", 0], + ["30", 0], + ["999", 0], + ])("accepts positive integer %s", (input, expected) => { + expect(runGuard(input)).toBe(expected); + }); + + it.each([ + ["", 1], + ["0", 1], + ["00", 1], + ["12x", 1], + ["30abc", 1], + ["-5", 1], + [" 5 ", 1], + ["5.0", 1], + ["one", 1], + ])("rejects non-positive-integer %j", (input, expected) => { + expect(runGuard(input)).toBe(expected); + }); +}); diff --git a/test/nemoclaw-start-gateway-health.test.ts b/test/nemoclaw-start-gateway-health.test.ts index 4f98d6ea608..6da39fc95a3 100644 --- a/test/nemoclaw-start-gateway-health.test.ts +++ b/test/nemoclaw-start-gateway-health.test.ts @@ -54,6 +54,7 @@ function watchdogFunctions(): string { safeTmpHelpers(src), extractShellFunction(src, "record_gateway_pid"), extractShellFunction(src, "gateway_pid_is_openclaw_gateway"), + extractShellFunction(src, "gateway_watchdog_positive_int_ok"), extractShellFunction(src, "start_gateway_serving_watchdog"), ].join("\n"); } @@ -566,6 +567,7 @@ describe("gateway launch wiring (#4710)", () => { ), extractShellFunction(src, "record_gateway_pid"), extractShellFunction(src, "gateway_pid_is_openclaw_gateway"), + extractShellFunction(src, "gateway_watchdog_positive_int_ok"), extractShellFunction(src, "start_gateway_serving_watchdog"), ].join("\n"); diff --git a/test/nemoclaw-start.test.ts b/test/nemoclaw-start.test.ts index 192325648d0..6878aae19d9 100644 --- a/test/nemoclaw-start.test.ts +++ b/test/nemoclaw-start.test.ts @@ -1756,43 +1756,35 @@ describe("nemoclaw-start auto-pair slow-mode keepalive (#4263)", () => { return autoPairPythonScript(src); } - it("approves late CLI scope upgrades after browser pairing converges", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-auto-pair-slow-")); + // Shared late-CLI poll timeline: + // 1-2: first-time browser pairing request pending. + // 3-6: browser paired, nothing pending (watcher converges to slow mode). + // 7-10: late CLI scope upgrade arrives. + // 11+: cli paired alongside browser. + function setupLateCliFixture(prefix: string): { + tmpDir: string; + fakeOpenclaw: string; + approveLog: string; + } { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); const fakeOpenclaw = path.join(tmpDir, "openclaw"); const stateFile = path.join(tmpDir, "list-count"); const approveLog = path.join(tmpDir, "approvals.log"); - - // Poll timeline: - // 1-2: first-time browser pairing request pending. - // 3-6: browser paired, nothing pending (watcher converges to slow mode). - // 7-10: late CLI scope upgrade arrives — must still get approved. - // 11+: cli paired alongside browser. + const browserClient = { clientId: "openclaw-control-ui", clientMode: "webchat" }; + const cliClient = { clientId: "openclaw-cli", clientMode: "cli" }; const initialPending = JSON.stringify({ - pending: [ - { - requestId: "browser-pair", - clientId: "openclaw-control-ui", - clientMode: "webchat", - }, - ], + pending: [{ requestId: "browser-pair", ...browserClient }], paired: [], }); - const browserPaired = JSON.stringify({ - pending: [], - paired: [{ clientId: "openclaw-control-ui", clientMode: "webchat" }], - }); + const browserPaired = JSON.stringify({ pending: [], paired: [browserClient] }); const lateCli = JSON.stringify({ - pending: [{ requestId: "late-cli", clientId: "openclaw-cli", clientMode: "cli" }], - paired: [{ clientId: "openclaw-control-ui", clientMode: "webchat" }], - }); - const allPaired = JSON.stringify({ - pending: [], - paired: [ - { clientId: "openclaw-control-ui", clientMode: "webchat" }, - { clientId: "openclaw-cli", clientMode: "cli" }, + pending: [ + { requestId: "late-cli", ...cliClient }, + { requestId: "late-cli-b", ...cliClient }, ], + paired: [browserClient], }); - + const allPaired = JSON.stringify({ pending: [], paired: [browserClient, cliClient] }); fs.writeFileSync( fakeOpenclaw, `#!/usr/bin/env bash @@ -1801,15 +1793,10 @@ if [ "\${1:-}" = "devices" ] && [ "\${2:-}" = "list" ]; then count="$(cat ${JSON.stringify(stateFile)} 2>/dev/null || echo 0)" count=$((count + 1)) echo "$count" > ${JSON.stringify(stateFile)} - if [ "$count" -le 2 ]; then - printf '%s\n' ${JSON.stringify(initialPending)} - elif [ "$count" -le 6 ]; then - printf '%s\n' ${JSON.stringify(browserPaired)} - elif [ "$count" -le 10 ]; then - printf '%s\n' ${JSON.stringify(lateCli)} - else - printf '%s\n' ${JSON.stringify(allPaired)} - fi + if [ "$count" -le 2 ]; then printf '%s\n' ${JSON.stringify(initialPending)} + elif [ "$count" -le 6 ]; then printf '%s\n' ${JSON.stringify(browserPaired)} + elif [ "$count" -le 10 ]; then printf '%s\n' ${JSON.stringify(lateCli)} + else printf '%s\n' ${JSON.stringify(allPaired)}; fi exit 0 fi if [ "\${1:-}" = "devices" ] && [ "\${2:-}" = "approve" ]; then @@ -1822,20 +1809,23 @@ exit 2 `, { mode: 0o755 }, ); + return { tmpDir, fakeOpenclaw, approveLog }; + } + it("approves concurrent late CLI scope upgrades after browser pairing converges and drops back to fast cadence", () => { + const { tmpDir, fakeOpenclaw, approveLog } = setupLateCliFixture("nemoclaw-auto-pair-slow-"); try { const run = spawnSync("python3", ["-c", buildAutoPairScript()], { encoding: "utf-8", env: { ...process.env, OPENCLAW_BIN: fakeOpenclaw, - // Short deadline so the test terminates promptly. time.sleep is - // monkey-patched out, so wall-clock matters only for the DEADLINE - // check; 5s gives the loop ~tens of iterations through every - // branch before exiting. + // SLOW_INTERVAL > FAST_REENTRY_INTERVAL exposes any regression. NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS: "600", NEMOCLAW_AUTO_PAIR_DEADLINE_SECS: "5", - NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS: "1", + NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS: "5", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "3", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "1", }, timeout: 30_000, }); @@ -1846,20 +1836,25 @@ exit 2 expect(run.stdout).toContain( "[auto-pair] browser pairing converged; entering slow-mode approvals=1", ); - // Critical: the late CLI scope upgrade is approved AFTER convergence. + // Concurrent late wave — proxy for two sibling sandboxes' upgrades. expect(run.stdout).toContain( "[auto-pair] approved request=late-cli client=openclaw-cli mode=cli", ); - // Deadline-based exit message (instead of an early convergence break). - expect(run.stdout).toContain("watcher deadline reached approvals=2"); - // The watcher MUST NOT print the old early-exit messages. - expect(run.stdout).not.toContain("browser pairing converged approvals="); - expect(run.stdout).not.toContain("devices paired ("); - expect(run.stdout).not.toContain("non-browser pairing converged approvals="); - // Both allowlisted approvals should have been recorded. + expect(run.stdout).toContain( + "[auto-pair] approved request=late-cli-b client=openclaw-cli mode=cli", + ); + expect(run.stdout).toContain("watcher deadline reached approvals=3"); + // Single marker per poll wave, transition after convergence. + expect(run.stdout).toContain("[auto-pair] fast-reentry bumped polls=3 approved=3 mode=slow"); + const convergedAt = run.stdout.indexOf("browser pairing converged"); + const bumpedAt = run.stdout.indexOf("fast-reentry bumped polls=3 approved=3 mode=slow"); + expect(bumpedAt).toBeGreaterThan(convergedAt); + const slowMarkerRe = /fast-reentry bumped polls=3 approved=3 mode=slow/g; + expect(run.stdout.match(slowMarkerRe)?.length).toBe(1); expect(fs.readFileSync(approveLog, "utf-8").trim().split("\n")).toEqual([ "browser-pair", "late-cli", + "late-cli-b", ]); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); @@ -2305,7 +2300,7 @@ exit 2 } }, 40_000); - it("retries a non-zero approve failure without counting it as approved", () => { + it("retries a non-zero approve failure without counting it as approved or re-arming fast-reentry", () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-auto-pair-afail-")); const fakeOpenclaw = path.join(tmpDir, "openclaw"); const stateFile = path.join(tmpDir, "approve-count"); @@ -2358,6 +2353,8 @@ exit 2 NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS: "600", NEMOCLAW_AUTO_PAIR_DEADLINE_SECS: "1", NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS: "1", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS: "3", + NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS: "1", }, timeout: 20_000, }); @@ -2371,6 +2368,9 @@ exit 2 expect(run.stdout).toContain("watcher deadline reached approvals=1"); expect(fs.readFileSync(stateFile, "utf-8").trim()).toBe("2"); expect(fs.readFileSync(approveLog, "utf-8").trim().split("\n")).toEqual(["retry-cli"]); + const markerRe = /fast-reentry bumped polls=3 /g; + expect(run.stdout.match(markerRe)?.length).toBe(1); + expect(run.stdout).toContain("[auto-pair] fast-reentry bumped polls=3 approved=0 mode=fast"); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } diff --git a/test/redact-device-state.test.ts b/test/redact-device-state.test.ts new file mode 100644 index 00000000000..aacddd7cae3 --- /dev/null +++ b/test/redact-device-state.test.ts @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const REDACTOR = path.resolve(HERE, "e2e/lib/redact-device-state.py"); +const REDACTED = "[REDACTED]"; + +function runRedactor(input: unknown): { rc: number; stdout: string; stderr: string; doc: unknown } { + const result = spawnSync("python3", [REDACTOR], { + input: JSON.stringify(input), + encoding: "utf-8", + timeout: 20_000, + }); + const stdout = result.stdout ?? ""; + const stderr = result.stderr ?? ""; + const doc: unknown = + result.status === 0 && stdout.trim().length > 0 ? JSON.parse(stdout) : undefined; + return { rc: result.status ?? -1, stdout, stderr, doc }; +} + +describe("device-state JSON redactor", () => { + it("redacts nested token, header, auth, credential fields while preserving diagnostic identifiers", () => { + const input = { + pending: [ + { + requestId: "req-abc-123", + deviceId: "dev-cli-007", + clientMode: "cli", + clientId: "openclaw-cli", + scopes: ["operator.read", "operator.write"], + requestedScopes: ["operator.read", "operator.write"], + tokens: { + operator: { value: "secret-operator-token", expiresAt: 9_999_999_999 }, + }, + headers: { Authorization: "Bearer raw-bearer-token" }, + credentials: { apiKey: "credential-leak" }, + }, + ], + paired: [ + { + deviceId: "dev-cli-008", + clientMode: "cli", + approvedScopes: ["operator.read"], + auth: { primary: { secret: "do-not-leak" } }, + notes: "device pairing approved manually", + }, + ], + paths: { + pending: "/sandbox/.openclaw/devices/pending.json", + paired: "/sandbox/.openclaw/devices/paired.json", + }, + }; + + const result = runRedactor(input); + expect(result.rc).toBe(0); + const doc = result.doc as typeof input; + + const pending = doc.pending[0]!; + expect(pending.requestId).toBe("req-abc-123"); + expect(pending.deviceId).toBe("dev-cli-007"); + expect(pending.clientMode).toBe("cli"); + expect(pending.clientId).toBe("openclaw-cli"); + expect(pending.scopes).toEqual(["operator.read", "operator.write"]); + expect(pending.requestedScopes).toEqual(["operator.read", "operator.write"]); + expect(pending.tokens).toBe(REDACTED); + expect(pending.headers).toBe(REDACTED); + expect(pending.credentials).toBe(REDACTED); + + const paired = doc.paired[0]!; + expect(paired.deviceId).toBe("dev-cli-008"); + expect(paired.approvedScopes).toEqual(["operator.read"]); + expect(paired.auth).toBe(REDACTED); + expect(paired.notes).toBe("device pairing approved manually"); + + expect(doc.paths.pending).toBe("/sandbox/.openclaw/devices/pending.json"); + expect(doc.paths.paired).toBe("/sandbox/.openclaw/devices/paired.json"); + expect(result.stdout).not.toContain("secret-operator-token"); + expect(result.stdout).not.toContain("raw-bearer-token"); + expect(result.stdout).not.toContain("credential-leak"); + expect(result.stdout).not.toContain("do-not-leak"); + }); + + it("redacts dotted nvapi values and other token-shaped strings under non-secret-shaped fields", () => { + const input = { + pending: [ + { + deviceId: "dev-cli-009", + clientMode: "cli", + scopes: ["operator.pairing"], + providerKey: "nvapi-abc.def_ghi-jkl-mnopqrstu", + extra: "sk-projXYZ1234567890abcd", + githubToken: "ghp_aaaaaaaaaaaaaaaaaa11", + githubPat: "github_pat_abcdefghijklmnopqrstu", + hfToken: "hf_aaaaaaaaaaaaaaaaaa", + slackBot: "xoxb-1111-2222-aaaaa", + jwtNote: "eyJabcdefg.payload.signature123", + awsKey: "AKIAABCDEFGHIJKLMNOP", + plainText: "nothing to redact here", + }, + ], + paired: [], + }; + + const result = runRedactor(input); + expect(result.rc).toBe(0); + const entry = (result.doc as typeof input).pending[0]!; + + expect(entry.deviceId).toBe("dev-cli-009"); + expect(entry.scopes).toEqual(["operator.pairing"]); + expect(entry.providerKey).toBe(REDACTED); + expect(entry.extra).toBe(REDACTED); + expect(entry.githubToken).toBe(REDACTED); + expect(entry.githubPat).toBe(REDACTED); + expect(entry.hfToken).toBe(REDACTED); + expect(entry.slackBot).toBe(REDACTED); + expect(entry.jwtNote).toBe(REDACTED); + expect(entry.awsKey).toBe(REDACTED); + expect(entry.plainText).toBe("nothing to redact here"); + + expect(result.stdout).not.toContain("nvapi-abc.def_ghi"); + expect(result.stdout).not.toContain("sk-projXYZ"); + expect(result.stdout).not.toContain("ghp_aaaaa"); + expect(result.stdout).not.toContain("github_pat_abcdefg"); + expect(result.stdout).not.toContain("hf_aaaaa"); + expect(result.stdout).not.toContain("xoxb-1111"); + expect(result.stdout).not.toContain("eyJabcdefg"); + expect(result.stdout).not.toContain("AKIAABCDEFG"); + }); + + it("preserves an empty document and rejects invalid JSON", () => { + const empty = runRedactor({}); + expect(empty.rc).toBe(0); + expect(empty.doc).toEqual({}); + + const invalid = spawnSync("python3", [REDACTOR], { + input: "not-json", + encoding: "utf-8", + timeout: 20_000, + }); + expect(invalid.status).toBe(1); + expect(invalid.stderr).toContain("invalid JSON"); + }); +}); diff --git a/test/redact-text.test.ts b/test/redact-text.test.ts new file mode 100644 index 00000000000..3819421b1d2 --- /dev/null +++ b/test/redact-text.test.ts @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const TEXT_REDACTOR = path.resolve(HERE, "e2e/lib/redact-text.py"); +const REDACTED = "[REDACTED]"; + +function runTextRedactor(input: string): { rc: number; stdout: string; stderr: string } { + const result = spawnSync("python3", [TEXT_REDACTOR], { + input, + encoding: "utf-8", + timeout: 20_000, + }); + return { + rc: result.status ?? -1, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + }; +} + +describe("scope-upgrade diagnostic text redactor", () => { + it("scrubs token-shaped substrings from raw gateway and auto-pair log excerpts", () => { + const input = [ + "Authorization: Bearer nvapi-abc.def_ghi-jkl-mnopqrstu", + "Cookie: session=eyJabcdefg.payload.signature123", + "github-token=ghp_aaaaaaaaaaaaaaaaaa11", + "X-API-Key: sk-projXYZ1234567890abcd", + "request: token=github_pat_abcdefghijklmnopqrstu", + "huggingface key hf_aaaaaaaaaaaaaaaaaa logged", + "aws AKIAABCDEFGHIJKLMNOP", + "slack xoxb-1111-2222-aaaaa", + "plain gateway connect: ok", + "", + ].join("\n"); + + const result = runTextRedactor(input); + expect(result.rc).toBe(0); + expect(result.stdout).not.toContain("nvapi-abc.def_ghi"); + expect(result.stdout).not.toContain("eyJabcdefg.payload"); + expect(result.stdout).not.toContain("ghp_aaaaa"); + expect(result.stdout).not.toContain("sk-projXYZ"); + expect(result.stdout).not.toContain("github_pat_abcdefg"); + expect(result.stdout).not.toContain("hf_aaaaa"); + expect(result.stdout).not.toContain("AKIAABCDEFG"); + expect(result.stdout).not.toContain("xoxb-1111"); + expect(result.stdout).toContain("plain gateway connect: ok"); + expect(result.stdout).toContain(REDACTED); + }); + + it("preserves structural prefixes while substituting only the secret value", () => { + const result = runTextRedactor("Authorization: Bearer raw-bearer-token\n"); + expect(result.rc).toBe(0); + expect(result.stdout).toContain("Authorization:"); + expect(result.stdout).toContain("Bearer "); + expect(result.stdout).not.toContain("raw-bearer-token"); + expect(result.stdout).toContain(REDACTED); + }); + + it("passes through input free of token-shaped substrings unchanged", () => { + const input = "ls -la /tmp/auto-pair.log\nslow-mode keepalive transition observed\n"; + const result = runTextRedactor(input); + expect(result.rc).toBe(0); + expect(result.stdout).toBe(input); + }); + + it("preserves ordinary hyphenated diagnostic text containing sk-", () => { + const input = "task-management-system-deployment completed without fallback\n"; + const result = runTextRedactor(input); + expect(result.rc).toBe(0); + expect(result.stdout).toBe(input); + }); + + it("returns success on empty input", () => { + const result = runTextRedactor(""); + expect(result.rc).toBe(0); + expect(result.stdout).toBe(""); + expect(result.stderr).toBe(""); + }); + + it("handles input without a trailing newline", () => { + const result = runTextRedactor("plain text without newline"); + expect(result.rc).toBe(0); + expect(result.stdout).toBe("plain text without newline"); + }); + + it("redacts multiple shapes on the same line", () => { + const result = runTextRedactor( + "trace: Bearer nvapi-abc.def_ghi-jkl-mnopqrstu while X-API-Key=sk-projXYZ1234567890abcd\n", + ); + expect(result.rc).toBe(0); + expect(result.stdout).not.toContain("nvapi-abc.def_ghi"); + expect(result.stdout).not.toContain("sk-projXYZ"); + expect(result.stdout).toContain("Bearer "); + expect(result.stdout).toContain("X-API-Key"); + const redactedCount = (result.stdout.match(/\[REDACTED\]/g) ?? []).length; + expect(redactedCount).toBeGreaterThanOrEqual(2); + }); + + it("preserves newline structure across long multi-line input", () => { + const lines = Array.from({ length: 64 }, (_, i) => + i % 8 === 0 ? `line ${i} nvapi-secret-value-${i}-padded-12345` : `line ${i} plain diagnostic`, + ); + const input = `${lines.join("\n")}\n`; + const result = runTextRedactor(input); + expect(result.rc).toBe(0); + expect(result.stdout.split("\n").length).toBe(lines.length + 1); + expect(result.stdout).not.toMatch(/nvapi-secret-value-\d+-padded/); + expect(result.stdout).toMatch(/line 1 plain diagnostic/); + }); +});