Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions scripts/nemoclaw-start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -304,16 +304,15 @@ apply_model_override() {
local max_tokens="${NEMOCLAW_MAX_TOKENS:-}"
local reasoning="${NEMOCLAW_REASONING:-}"

# Validate numeric values
if [ -n "$context_window" ] && ! printf '%s' "$context_window" | grep -qE '^[0-9]+$'; then
# Validate supplemental override values before relaxing or writing config.
if [ -n "$context_window" ] && ! printf '%s' "$context_window" | grep -qE '^[1-9][0-9]*$'; then
printf '[SECURITY] NEMOCLAW_CONTEXT_WINDOW must be a positive integer, got "%s" — skipping override\n' "$context_window" >&2
return 0
fi
if [ -n "$max_tokens" ] && ! printf '%s' "$max_tokens" | grep -qE '^[0-9]+$'; then
if [ -n "$max_tokens" ] && ! printf '%s' "$max_tokens" | grep -qE '^[1-9][0-9]*$'; then
printf '[SECURITY] NEMOCLAW_MAX_TOKENS must be a positive integer, got "%s" — skipping override\n' "$max_tokens" >&2
return 0
fi
# Validate reasoning is true/false
if [ -n "$reasoning" ]; then
case "$reasoning" in
true | false) ;;
Expand Down
25 changes: 22 additions & 3 deletions src/lib/agent-runtime.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ describe("buildRecoveryScript", () => {
const script = buildRecoveryScript(agent, 19000);
expect(script).toContain("GATEWAY_CMD_BIN='custom-launch'");
expect(script).toContain('command -v "$GATEWAY_CMD_BIN" >/dev/null 2>&1');
expect(script).toContain("_GATEWAY_PROC_PATTERN='[c]ustom-launch([ -]gateway| gateway run|$)'");
expect(script).toContain("nohup custom-launch --mode recovery --port 19000");
});

Expand Down Expand Up @@ -111,19 +112,37 @@ describe("buildRecoveryScript", () => {
expect(script).toContain("or ciao preload");
});

it("stops stale launcher and gateway processes before relaunch", () => {
const script = buildRecoveryScript(minimalAgent, 19000);
expect(script).toContain("_GATEWAY_PROC_PATTERN='[t]est-agent([ -]gateway| gateway run|$)'");
expect(script).toContain('pkill -TERM -f "$_GATEWAY_PROC_PATTERN"');
expect(script).toContain('pkill -KILL -f "$_GATEWAY_PROC_PATTERN"');
expect(script).toContain("GATEWAY_STALE_PROCESSES");
});

it("sources proxy-env.sh BEFORE launching the gateway binary", () => {
const script = buildRecoveryScript(minimalAgent, 19000);
expect(script).not.toBeNull();
const sourceIdx = script!.indexOf("/tmp/nemoclaw-proxy-env.sh");
const launchIdx = script!.indexOf("gateway run");
const staleStopIdx = script!.indexOf('pkill -TERM -f "$_GATEWAY_PROC_PATTERN"');
const sourceIdx = script!.indexOf("then . /tmp/nemoclaw-proxy-env.sh");
const launchIdx = script!.indexOf("nohup");
expect(staleStopIdx).toBeGreaterThanOrEqual(0);
expect(sourceIdx).toBeGreaterThanOrEqual(0);
expect(launchIdx).toBeGreaterThanOrEqual(0);
expect(staleStopIdx).toBeLessThan(sourceIdx);
expect(sourceIdx).toBeLessThan(launchIdx);
});

it("fails recovery when an existing proxy-env.sh does not install required guards", () => {
const script = buildRecoveryScript(minimalAgent, 19000);
expect(script).toContain('if [ "$_PE_MISSING" = "0" ]');
expect(script).toContain("refusing unguarded gateway relaunch");
expect(script).toContain('echo "$_E" >> /tmp/gateway.log; exit 1');
});

it("writes the warning to gateway.log so it persists for sysadmin tail", () => {
const script = buildRecoveryScript(minimalAgent, 19000);
// Both warnings must end up in /tmp/gateway.log, not just stderr —
// The missing-env warning must end up in /tmp/gateway.log, not just stderr —
// executeSandboxCommand silently discards stderr from the recovery
// script, so a warning that only goes to stderr is invisible to
// anyone debugging a crash-loop. (#2478)
Expand Down
52 changes: 30 additions & 22 deletions src/lib/agent-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,20 @@ export function getHealthProbeUrl(agent: AgentDefinition | null): string {
return agent.healthProbe?.url || `http://127.0.0.1:${DASHBOARD_PORT}/`;
}

function escapeEre(value: string): string {
return value.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&");
}

function escapeCharClass(value: string): string {
return value.replace(/[\\\]\[\^\-]/g, "\\$&");
}

function selfSafeGatewayProcessPattern(binaryName: string): string {
const [first = "", ...rest] = Array.from(binaryName);
if (!first) return "";
return `[${escapeCharClass(first)}]${escapeEre(rest.join(""))}([ -]gateway| gateway run|$)`;
}

/**
* Build the recovery shell script for a non-OpenClaw agent.
* Returns the script string, or null if agent is null (use existing inline
Expand All @@ -56,13 +70,16 @@ export function getHealthProbeUrl(agent: AgentDefinition | null): string {
export function buildRecoveryScript(agent: AgentDefinition | null, port: number): string | null {
if (!agent) return null;

const probeUrl = getHealthProbeUrl(agent);
const binaryPath = agent.binary_path || "/usr/local/bin/openclaw";
const binaryName = binaryPath.split("/").pop() ?? "openclaw";
const defaultGatewayCommand = `${binaryName} gateway run`;
const configuredGatewayCommand = agent.gateway_command?.trim() || defaultGatewayCommand;
const usesValidatedBinary = configuredGatewayCommand === defaultGatewayCommand;
const customGatewayExecutable = configuredGatewayCommand.split(/\s+/)[0] ?? binaryName;
const gatewayExecutableName = usesValidatedBinary
? binaryName
: (customGatewayExecutable.split("/").pop() ?? customGatewayExecutable);
const staleGatewayPattern = selfSafeGatewayProcessPattern(gatewayExecutableName);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
const validationSteps = usesValidatedBinary
? [
`AGENT_BIN=${shellQuote(binaryPath)}; if [ ! -x "$AGENT_BIN" ]; then AGENT_BIN="$(command -v ${shellQuote(binaryName)})"; fi;`,
Expand All @@ -83,34 +100,25 @@ export function buildRecoveryScript(agent: AgentDefinition | null, port: number)
const isHermes = agent.name === "hermes";
const hermesHome = isHermes ? "export HERMES_HOME=/sandbox/.hermes; " : "";

// Source /tmp/nemoclaw-proxy-env.sh explicitly before launching. That file
// Source /tmp/nemoclaw-proxy-env.sh immediately before launching. That file
// is the single source of truth for NODE_OPTIONS preload guards (safety-net,
// ciao networkInterfaces, slack, http-proxy, ws-proxy, nemotron). On the
// first start it reaches the gateway transitively via .bashrc, but on
// gateway respawn (laptop sleep, health-monitor restart) silent sourcing
// failures left guards out of NODE_OPTIONS and the gateway crash-looped
// on the next library error (#2478). Source it explicitly, log when the
// file is missing, and warn when the safety-net preload is not in the
// resulting NODE_OPTIONS so future regressions stay observable instead
// of silently regressing into a crash loop.
// Source proxy-env.sh and check NODE_OPTIONS first, but defer warning
// emission until AFTER touch+chmod gateway.log so warnings land in the
// fresh log a sysadmin would tail. Writing to stderr alone hides them
// because the recovery script's stderr is captured by executeSandboxCommand
// (returned to nemoclaw status, not displayed). Routing them through
// /tmp/gateway.log makes the diagnostic discoverable for both real users
// and the #2478 e2e regression test.
// ciao networkInterfaces, slack, http-proxy, ws-proxy, nemotron). Recovery
// also stops stale launcher/gateway processes that may have respawned
// between the health probe and relaunch. A missing env file remains warning-
// only; a present env file that does not install required guards is a hard
// failure because launching would create an unguarded gateway.
return [
"if [ -r /tmp/nemoclaw-proxy-env.sh ]; then . /tmp/nemoclaw-proxy-env.sh; _PE_MISSING=0; else _PE_MISSING=1; fi;",
"[ -f ~/.bashrc ] && . ~/.bashrc;",
'case "${NODE_OPTIONS:-}" in *nemoclaw-sandbox-safety-net*) _SN_MISSING=0 ;; *) _SN_MISSING=1 ;; esac; case "${NODE_OPTIONS:-}" in *nemoclaw-ciao-network-guard*) _CIAO_MISSING=0 ;; *) _CIAO_MISSING=1 ;; esac; if [ "$_SN_MISSING" = "0" ] && [ "$_CIAO_MISSING" = "0" ]; then _GUARDS_MISSING=0; else _GUARDS_MISSING=1; fi;',
hermesHome,
`if curl -sf --max-time 3 ${shellQuote(probeUrl)} > /dev/null 2>&1; then echo ALREADY_RUNNING; exit 0; fi;`,
"rm -f /tmp/gateway.log;",
"touch /tmp/gateway.log; chmod 600 /tmp/gateway.log;",
'[ "$_PE_MISSING" = "1" ] && { _W="[gateway-recovery] WARNING: /tmp/nemoclaw-proxy-env.sh missing — gateway launching without library guards (#2478)"; echo "$_W" >&2; echo "$_W" >> /tmp/gateway.log; };',
'[ "$_GUARDS_MISSING" = "1" ] && { _W="[gateway-recovery] WARNING: NODE_OPTIONS missing safety-net preload or ciao preload — gateway may crash on unhandled library errors (#2478)"; echo "$_W" >&2; echo "$_W" >> /tmp/gateway.log; };',
`_GATEWAY_PROC_PATTERN=${shellQuote(staleGatewayPattern)};`,
'if [ -n "$_GATEWAY_PROC_PATTERN" ]; then pkill -TERM -f "$_GATEWAY_PROC_PATTERN" 2>/dev/null || true; for _i in 1 2 3 4 5; do pgrep -f "$_GATEWAY_PROC_PATTERN" >/dev/null 2>&1 || break; sleep 1; done; pkill -KILL -f "$_GATEWAY_PROC_PATTERN" 2>/dev/null || true; for _i in 1 2 3 4 5; do pgrep -f "$_GATEWAY_PROC_PATTERN" >/dev/null 2>&1 || break; sleep 1; done; if pgrep -f "$_GATEWAY_PROC_PATTERN" >/dev/null 2>&1; then echo GATEWAY_STALE_PROCESSES; exit 1; fi; fi;',
...validationSteps,
"if [ -r /tmp/nemoclaw-proxy-env.sh ]; then . /tmp/nemoclaw-proxy-env.sh; _PE_MISSING=0; else _PE_MISSING=1; fi;",
'if [ "$_PE_MISSING" = "0" ]; then case "${NODE_OPTIONS:-}" in *nemoclaw-sandbox-safety-net*) _SN_MISSING=0 ;; *) _SN_MISSING=1 ;; esac; case "${NODE_OPTIONS:-}" in *nemoclaw-ciao-network-guard*) _CIAO_MISSING=0 ;; *) _CIAO_MISSING=1 ;; esac; if [ "$_SN_MISSING" = "0" ] && [ "$_CIAO_MISSING" = "0" ]; then _GUARDS_MISSING=0; else _GUARDS_MISSING=1; fi; else _GUARDS_MISSING=0; fi;',
'[ "$_PE_MISSING" = "1" ] && { _W="[gateway-recovery] WARNING: /tmp/nemoclaw-proxy-env.sh missing — gateway launching without library guards (#2478)"; echo "$_W" >&2; echo "$_W" >> /tmp/gateway.log; };',
'[ "$_PE_MISSING" = "0" ] && [ "$_GUARDS_MISSING" = "1" ] && { _E="[gateway-recovery] ERROR: /tmp/nemoclaw-proxy-env.sh present but NODE_OPTIONS missing safety-net preload or ciao preload — refusing unguarded gateway relaunch (#2478)"; echo "$_E" >&2; echo "$_E" >> /tmp/gateway.log; exit 1; };',
launchCommand,
"GPID=$!; sleep 2;",
'if kill -0 "$GPID" 2>/dev/null; then echo "GATEWAY_PID=$GPID"; else echo GATEWAY_FAILED; cat /tmp/gateway.log 2>/dev/null | tail -5; fi',
Expand Down
20 changes: 5 additions & 15 deletions src/nemoclaw.ts
Original file line number Diff line number Diff line change
Expand Up @@ -294,28 +294,18 @@ function recoverSandboxProcesses(sandboxName: string): boolean {
const script =
agentScript ||
[
// Source /tmp/nemoclaw-proxy-env.sh explicitly so NODE_OPTIONS preload
// guards (safety-net, ciao, slack, …) survive gateway respawn. Without
// this, library errors crash-loop the gateway because the original
// .bashrc-only path silently failed when the env file was unreadable
// or the shell did not source ~/.bashrc. See #2478. Mirrors the
// hardened block in src/lib/agent-runtime.ts:buildRecoveryScript.
// Defer warning emission until AFTER touch+chmod gateway.log so
// warnings land in the persistent log a sysadmin would tail. Stderr
// alone hides them because executeSandboxCommand captures stderr
// without surfacing it. Mirrors src/lib/agent-runtime.ts.
"if [ -r /tmp/nemoclaw-proxy-env.sh ]; then . /tmp/nemoclaw-proxy-env.sh; _PE_MISSING=0; else _PE_MISSING=1; fi;",
"[ -f ~/.bashrc ] && . ~/.bashrc;",
'case "${NODE_OPTIONS:-}" in *nemoclaw-sandbox-safety-net*) _SN_MISSING=0 ;; *) _SN_MISSING=1 ;; esac; case "${NODE_OPTIONS:-}" in *nemoclaw-ciao-network-guard*) _CIAO_MISSING=0 ;; *) _CIAO_MISSING=1 ;; esac; if [ "$_SN_MISSING" = "0" ] && [ "$_CIAO_MISSING" = "0" ]; then _GUARDS_MISSING=0; else _GUARDS_MISSING=1; fi;',
`if curl -sf --max-time 3 http://127.0.0.1:${DASHBOARD_PORT}/ > /dev/null 2>&1; then echo ALREADY_RUNNING; exit 0; fi;`,
"rm -rf /tmp/openclaw-*/gateway.*.lock 2>/dev/null;",
"rm -f /tmp/gateway.log /tmp/auto-pair.log;",
"touch /tmp/gateway.log; chmod 600 /tmp/gateway.log;",
"touch /tmp/auto-pair.log; chmod 600 /tmp/auto-pair.log;",
'[ "$_PE_MISSING" = "1" ] && { _W="[gateway-recovery] WARNING: /tmp/nemoclaw-proxy-env.sh missing — gateway launching without library guards (#2478)"; echo "$_W" >&2; echo "$_W" >> /tmp/gateway.log; };',
'[ "$_GUARDS_MISSING" = "1" ] && { _W="[gateway-recovery] WARNING: NODE_OPTIONS missing safety-net preload or ciao preload — gateway may crash on unhandled library errors (#2478)"; echo "$_W" >&2; echo "$_W" >> /tmp/gateway.log; };',
'pkill -TERM -f "[o]penclaw([ -]gateway| gateway run|$)" 2>/dev/null || true; for _i in 1 2 3 4 5; do pgrep -f "[o]penclaw([ -]gateway| gateway run|$)" >/dev/null 2>&1 || break; sleep 1; done; pkill -KILL -f "[o]penclaw([ -]gateway| gateway run|$)" 2>/dev/null || true; for _i in 1 2 3 4 5; do pgrep -f "[o]penclaw([ -]gateway| gateway run|$)" >/dev/null 2>&1 || break; sleep 1; done; if pgrep -f "[o]penclaw([ -]gateway| gateway run|$)" >/dev/null 2>&1; then echo GATEWAY_STALE_PROCESSES; exit 1; fi;',
'OPENCLAW="$(command -v openclaw)";',
'if [ -z "$OPENCLAW" ]; then echo OPENCLAW_MISSING; exit 1; fi;',
"if [ -r /tmp/nemoclaw-proxy-env.sh ]; then . /tmp/nemoclaw-proxy-env.sh; _PE_MISSING=0; else _PE_MISSING=1; fi;",
'if [ "$_PE_MISSING" = "0" ]; then case "${NODE_OPTIONS:-}" in *nemoclaw-sandbox-safety-net*) _SN_MISSING=0 ;; *) _SN_MISSING=1 ;; esac; case "${NODE_OPTIONS:-}" in *nemoclaw-ciao-network-guard*) _CIAO_MISSING=0 ;; *) _CIAO_MISSING=1 ;; esac; if [ "$_SN_MISSING" = "0" ] && [ "$_CIAO_MISSING" = "0" ]; then _GUARDS_MISSING=0; else _GUARDS_MISSING=1; fi; else _GUARDS_MISSING=0; fi;',
'[ "$_PE_MISSING" = "1" ] && { _W="[gateway-recovery] WARNING: /tmp/nemoclaw-proxy-env.sh missing — gateway launching without library guards (#2478)"; echo "$_W" >&2; echo "$_W" >> /tmp/gateway.log; };',
'[ "$_PE_MISSING" = "0" ] && [ "$_GUARDS_MISSING" = "1" ] && { _E="[gateway-recovery] ERROR: /tmp/nemoclaw-proxy-env.sh present but NODE_OPTIONS missing safety-net preload or ciao preload — refusing unguarded gateway relaunch (#2478)"; echo "$_E" >&2; echo "$_E" >> /tmp/gateway.log; exit 1; };',
// Append rather than truncate so [gateway-recovery] WARNING lines
// written above survive past the launch. (#2478)
`nohup "$OPENCLAW" gateway run --port ${DASHBOARD_PORT} >> /tmp/gateway.log 2>&1 &`,
Expand Down
58 changes: 46 additions & 12 deletions test/nemoclaw-start.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -694,19 +694,53 @@ describe("runtime model override (#759)", () => {
expect(hash).toContain("openclaw.json");
});

it("ignores invalid numeric and API overrides without mutating config", () => {
const { result, config } = runApplyModelOverride({
NEMOCLAW_MODEL_OVERRIDE: "new-model",
NEMOCLAW_INFERENCE_API_OVERRIDE: "unexpected-api",
NEMOCLAW_CONTEXT_WINDOW: "not-a-number",
});
it("treats invalid supplemental overrides as atomic no-ops", () => {
const cases = [
{
env: { NEMOCLAW_CONTEXT_WINDOW: "not-a-number" },
message: "NEMOCLAW_CONTEXT_WINDOW must be a positive integer",
},
{
env: { NEMOCLAW_CONTEXT_WINDOW: "0" },
message: "NEMOCLAW_CONTEXT_WINDOW must be a positive integer",
},
{
env: { NEMOCLAW_MAX_TOKENS: "not-a-number" },
message: "NEMOCLAW_MAX_TOKENS must be a positive integer",
},
{
env: { NEMOCLAW_MAX_TOKENS: "0" },
message: "NEMOCLAW_MAX_TOKENS must be a positive integer",
},
{
env: { NEMOCLAW_REASONING: "maybe" },
message: 'NEMOCLAW_REASONING must be "true" or "false"',
},
{
env: { NEMOCLAW_INFERENCE_API_OVERRIDE: "unexpected-api" },
message: 'must be "openai-completions" or "anthropic-messages"',
},
];

expect(result.status).toBe(0);
expect(`${result.stdout}${result.stderr}`).toContain(
'must be "openai-completions" or "anthropic-messages"',
);
expect(config.agents.defaults.model.primary).toBe("old-model");
expect(config.models.providers.inference.models[0].contextWindow).toBe(1024);
for (const { env, message } of cases) {
const { result, config, hash } = runApplyModelOverride({
NEMOCLAW_MODEL_OVERRIDE: "new-model",
...env,
});

expect(result.status).toBe(0);
expect(`${result.stdout}${result.stderr}`).toContain(message);
expect(config.agents.defaults.model.primary).toBe("old-model");
expect(config.models.providers.inference.api).toBe("openai-completions");
expect(config.models.providers.inference.models[0]).toMatchObject({
id: "old-model",
name: "old-model",
contextWindow: 1024,
maxTokens: 128,
reasoning: false,
});
expect(hash).toBe("oldhash\n");
}
});
});

Expand Down
Loading