Skip to content
Merged
50 changes: 46 additions & 4 deletions scripts/nemoclaw-start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4119,9 +4119,7 @@ start_gateway_serving_watchdog() {
fi
msg="[gateway-watchdog] CRITICAL: gateway pid $pid is alive but dropped its HTTP listener on port ${_DASHBOARD_PORT} ($refused_streak consecutive refused probes); killing it so the respawn loop can relaunch (#4710)"
echo "$msg" >&2
# _NEMOCLAW_GATEWAY_LOG is a test seam; production always appends to
# /tmp/gateway.log alongside the gateway's own output.
echo "$msg" >>"${_NEMOCLAW_GATEWAY_LOG:-/tmp/gateway.log}" 2>/dev/null || true
append_openclaw_gateway_log_line "$msg" || true
record_gateway_watchdog_kill "$tracked_identity"
kill -TERM "$pid" 2>/dev/null || true
for _ in 1 2 3 4 5 6 7 8 9 10; do
Expand Down Expand Up @@ -4467,11 +4465,55 @@ openclaw_runtime_guard_chain_complete() {
done
}

append_openclaw_gateway_log_line() {
local log_file="/tmp/gateway.log"
local line="$1"
python3 -I - "$log_file" "$line" <<'PYAPPEND'
import os
import stat
import sys

# Source boundary: production startup owns /tmp/gateway.log creation through
# _nemoclaw_safe_create_tmp_file before any PID 1 recovery path runs. This
# permanent defensive append policy never creates the log, never honors an
# inherited alternate-path environment variable, and refuses link/swap targets
# before writing recovery breadcrumbs.
path = sys.argv[1]
line = sys.argv[2].replace("\r", " ").replace("\n", " ")
flags = os.O_WRONLY | os.O_APPEND | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
try:
before = os.lstat(path)
if not stat.S_ISREG(before.st_mode):
print(f"[SECURITY] refusing unsafe gateway log path: {path}", file=sys.stderr)
raise SystemExit(1)
fd = os.open(path, flags)
except FileNotFoundError:
# Production pre-creates /tmp/gateway.log with _nemoclaw_safe_create_tmp_file.
# Do not create it here from a PID 1/root recovery path.
raise SystemExit(0)
except OSError as exc:
print(f"[SECURITY] refusing unsafe gateway log path: {path}: {exc}", file=sys.stderr)
raise SystemExit(1)
try:
current = os.fstat(fd)
if not stat.S_ISREG(current.st_mode) or (current.st_dev, current.st_ino) != (before.st_dev, before.st_ino):
print(f"[SECURITY] refusing replaced gateway log path: {path}", file=sys.stderr)
raise SystemExit(1)
os.write(fd, (line + "\n").encode("utf-8"))
finally:
os.close(fd)
PYAPPEND
}

restore_openclaw_runtime_guard_chain() {
if ! openclaw_runtime_guard_chain_complete; then
local _guard_warn="[gateway-recovery] WARNING: /tmp guard chain missing or unsafe - restoring library guards from packaged preloads (#2478/#2701)"
echo "$_guard_warn" >&2
echo "$_guard_warn" >>"${_NEMOCLAW_GATEWAY_LOG:-/tmp/gateway.log}" 2>/dev/null || true
local _append_rc=0
append_openclaw_gateway_log_line "$_guard_warn" || _append_rc=$?
if [ "$_append_rc" -ne 0 ] && [ "$_append_rc" -ne 1 ]; then
return "$_append_rc"
fi
fi

# Preserve startup ordering: immutable core preloads first, then the
Expand Down
199 changes: 195 additions & 4 deletions test/nemoclaw-start-guard-recovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,68 @@ function extractShellFunction(source: string, name: string): string {
return `${name}() {${body.slice(0, closing?.index ?? 0)}\n}`;
}

function extractGatewayLogAppendFunction(
source: string,
gatewayLog: string,
{ replaceAfterLstat = false }: { replaceAfterLstat?: boolean } = {},
): string {
const functionSource = extractShellFunction(source, "append_openclaw_gateway_log_line").replace(
' local log_file="/tmp/gateway.log"',
` local log_file=${JSON.stringify(gatewayLog)}`,
);
return replaceAfterLstat
? functionSource.replace(
" fd = os.open(path, flags)",
[
' replacement = f"{path}.replacement"',
' with open(replacement, "w", encoding="utf-8") as handle:',
' handle.write("replacement\\n")',
" os.replace(replacement, path)",
" fd = os.open(path, flags)",
].join("\n"),
)
: functionSource;
}

type Harness = {
eventLog: string;
gatewayLog: string;
result: SpawnSyncReturns<string>;
sensitiveTarget?: string;
sources: Record<string, string>;
targets: Record<string, string>;
tmpDir: string;
};

function runRecoveryHarness({ missingCiaoSource = false } = {}): Harness {
type RecoveryHarnessOptions = {
gatewayLogKind?: "regular" | "symlink" | "directory" | "missing";
missingCiaoSource?: boolean;
};

function runRecoveryHarness({
gatewayLogKind = "regular",
missingCiaoSource = false,
}: RecoveryHarnessOptions = {}): Harness {
const source = fs.readFileSync(START_SCRIPT, "utf8");
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-guard-recovery-"));
const eventLog = path.join(tmpDir, "events.log");
const gatewayLog = path.join(tmpDir, "gateway.log");
let sensitiveTarget: string | undefined;
switch (gatewayLogKind) {
case "regular":
fs.writeFileSync(gatewayLog, "", { mode: 0o644 });
break;
case "symlink":
sensitiveTarget = path.join(tmpDir, "sensitive.log");
fs.writeFileSync(sensitiveTarget, "do-not-touch\n", { mode: 0o600 });
fs.symlinkSync(sensitiveTarget, gatewayLog);
break;
case "directory":
fs.mkdirSync(gatewayLog);
break;
case "missing":
break;
}
const sources = {
safety: path.join(tmpDir, "source-safety.js"),
proxy: path.join(tmpDir, "source-proxy.js"),
Expand Down Expand Up @@ -59,6 +109,7 @@ function runRecoveryHarness({ missingCiaoSource = false } = {}): Harness {
const script = [
"set -uo pipefail",
`EVENT_LOG=${JSON.stringify(eventLog)}`,
`_NEMOCLAW_GATEWAY_LOG=${JSON.stringify(gatewayLog)}`,
'NODE_OPTIONS=""',
"NODE_USE_ENV_PROXY=1",
`_SANDBOX_SAFETY_NET=${JSON.stringify(targets.safety)}`,
Expand Down Expand Up @@ -100,6 +151,7 @@ function runRecoveryHarness({ missingCiaoSource = false } = {}): Harness {
extractShellFunction(source, "append_node_require_once"),
extractShellFunction(source, "install_core_runtime_preloads"),
extractShellFunction(source, "openclaw_runtime_guard_chain_complete"),
extractGatewayLogAppendFunction(source, gatewayLog),
extractShellFunction(source, "restore_openclaw_runtime_guard_chain"),
extractShellFunction(source, "prepare_openclaw_gateway_restart"),
"rc=0; prepare_openclaw_gateway_restart || rc=$?",
Expand All @@ -112,7 +164,15 @@ function runRecoveryHarness({ missingCiaoSource = false } = {}): Harness {
env: { ...process.env, RUN_TWICE: missingCiaoSource ? "0" : "1" },
timeout: 10_000,
});
return { eventLog, result, sources, targets, tmpDir };
return {
eventLog,
gatewayLog,
result,
sensitiveTarget,
sources,
targets,
tmpDir,
};
}

describe("OpenClaw PID 1 guard-chain recovery", () => {
Expand All @@ -122,6 +182,9 @@ describe("OpenClaw PID 1 guard-chain recovery", () => {
expect(harness.result.status, harness.result.stderr).toBe(0);
expect(harness.result.stdout).toContain("rc:0\n");
expect(harness.result.stderr.match(/restoring library guards/g)).toHaveLength(1);
expect(
fs.readFileSync(harness.gatewayLog, "utf8").match(/restoring library guards/g),
).toHaveLength(1);

const onePass = [
"guard:preflight-restart",
Expand Down Expand Up @@ -157,6 +220,132 @@ describe("OpenClaw PID 1 guard-chain recovery", () => {
}
});

it("does not write guard-chain warnings through an unsafe gateway log symlink", () => {
const harness = runRecoveryHarness({ gatewayLogKind: "symlink" });
try {
expect(harness.result.status, harness.result.stderr).toBe(0);
expect(harness.result.stderr).toContain("refusing unsafe gateway log path");
expect(harness.result.stderr).toContain("restoring library guards");
expect(harness.sensitiveTarget).toBeDefined();
expect(fs.readlinkSync(harness.gatewayLog)).toBe(harness.sensitiveTarget);
expect(fs.readFileSync(harness.sensitiveTarget ?? "", "utf8")).toBe("do-not-touch\n");
} finally {
fs.rmSync(harness.tmpDir, { recursive: true, force: true });
}
});

it("does not write guard-chain warnings to a non-regular gateway log target", () => {
const harness = runRecoveryHarness({ gatewayLogKind: "directory" });
try {
expect(harness.result.status, harness.result.stderr).toBe(0);
expect(harness.result.stderr).toContain("refusing unsafe gateway log path");
expect(fs.statSync(harness.gatewayLog).isDirectory()).toBe(true);
expect(fs.readdirSync(harness.gatewayLog)).toHaveLength(0);
} finally {
fs.rmSync(harness.tmpDir, { recursive: true, force: true });
}
});

it("does not create a missing gateway log from guard-chain recovery", () => {
const harness = runRecoveryHarness({ gatewayLogKind: "missing" });
try {
expect(harness.result.status, harness.result.stderr).toBe(0);
expect(harness.result.stderr).toContain("restoring library guards");
expect(fs.existsSync(harness.gatewayLog)).toBe(false);
} finally {
fs.rmSync(harness.tmpDir, { recursive: true, force: true });
}
});

it("refuses a gateway log path replaced between validation and append", () => {
const source = fs.readFileSync(START_SCRIPT, "utf8");
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-guard-replaced-"));
const gatewayLog = path.join(tmpDir, "gateway.log");
try {
fs.writeFileSync(gatewayLog, "original\n", { mode: 0o644 });
const script = [
"set -uo pipefail",
`_NEMOCLAW_GATEWAY_LOG=${JSON.stringify(gatewayLog)}`,
extractGatewayLogAppendFunction(source, gatewayLog, { replaceAfterLstat: true }),
"rc=0; append_openclaw_gateway_log_line 'safe-line' || rc=$?",
'printf "rc:%s\\n" "$rc"',
].join("\n");

const result = spawnSync("bash", ["--noprofile", "--norc", "-c", script], {
encoding: "utf8",
timeout: 5000,
});

expect(result.status, result.stderr).toBe(0);
expect(result.stdout).toBe("rc:1\n");
expect(result.stderr).toContain("refusing replaced gateway log path");
expect(fs.readFileSync(gatewayLog, "utf8")).toBe("replacement\n");
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});

it("ignores inherited non-canonical gateway log environment", () => {
const source = fs.readFileSync(START_SCRIPT, "utf8");
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-guard-contract-"));
const gatewayLog = path.join(tmpDir, "gateway.log");
const inheritedTarget = path.join(tmpDir, "inherited.log");
try {
fs.writeFileSync(gatewayLog, "", { mode: 0o644 });
fs.writeFileSync(inheritedTarget, "existing\n", { mode: 0o644 });
const script = [
"set -uo pipefail",
`_NEMOCLAW_GATEWAY_LOG=${JSON.stringify(inheritedTarget)}`,
"_NEMOCLAW_GATEWAY_LOG_TEST_MODE=1",
extractGatewayLogAppendFunction(source, gatewayLog),
"append_openclaw_gateway_log_line 'safe-line'",
].join("\n");

const result = spawnSync("bash", ["--noprofile", "--norc", "-c", script], {
encoding: "utf8",
timeout: 5000,
});

expect(result.status, result.stderr).toBe(0);
expect(fs.readFileSync(gatewayLog, "utf8")).toBe("safe-line\n");
expect(fs.readFileSync(inheritedTarget, "utf8")).toBe("existing\n");
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});

it("routes watchdog gateway-log breadcrumbs through the safe append helper", () => {
const source = fs.readFileSync(START_SCRIPT, "utf8");

expect(source).not.toContain('>>"${_NEMOCLAW_GATEWAY_LOG:-/tmp/gateway.log}"');
expect(source).toContain('append_openclaw_gateway_log_line "$msg" || true');
});

it("sanitizes gateway log lines before appending", () => {
const source = fs.readFileSync(START_SCRIPT, "utf8");
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-guard-sanitize-"));
const gatewayLog = path.join(tmpDir, "gateway.log");
try {
fs.writeFileSync(gatewayLog, "", { mode: 0o644 });
const script = [
"set -uo pipefail",
`_NEMOCLAW_GATEWAY_LOG=${JSON.stringify(gatewayLog)}`,
extractGatewayLogAppendFunction(source, gatewayLog),
"append_openclaw_gateway_log_line $'first\\nsecond'",
].join("\n");

const result = spawnSync("bash", ["--noprofile", "--norc", "-c", script], {
encoding: "utf8",
timeout: 5000,
});

expect(result.status, result.stderr).toBe(0);
expect(fs.readFileSync(gatewayLog, "utf8")).toBe("first second\n");
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});

it("fails closed as preload-missing before validation when a packaged guard is absent", () => {
const harness = runRecoveryHarness({ missingCiaoSource: true });
try {
Expand All @@ -180,13 +369,14 @@ describe("OpenClaw PID 1 guard-chain recovery", () => {
// #5874 moved recovery to a docker-IPC path where the warning was written to
// PID 1 stderr only; the live `issue-2478-crash-loop-recovery` E2E polls
// /tmp/gateway.log and went red. That target does not run on PR CI, so this
// mocked unit pins the file write (via the _NEMOCLAW_GATEWAY_LOG seam) in the
// PR gate to keep a refactor from silently regressing to stderr-only.
// mocked unit pins the file write through an extracted helper in the PR gate
// to keep a refactor from silently regressing to stderr-only.
it("mirrors the guard-chain restore warning into the gateway log file", () => {
const source = fs.readFileSync(START_SCRIPT, "utf8");
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-guard-warn-"));
const gatewayLog = path.join(tmpDir, "gateway.log");
try {
fs.writeFileSync(gatewayLog, "", { mode: 0o644 });
const script = [
"set -uo pipefail",
`_NEMOCLAW_GATEWAY_LOG=${JSON.stringify(gatewayLog)}`,
Expand All @@ -199,6 +389,7 @@ describe("OpenClaw PID 1 guard-chain recovery", () => {
"verify_messaging_runtime_secret_scans() { return 0; }",
"write_runtime_shell_env() { return 0; }",
"validate_nemoclaw_tmp_permissions() { return 0; }",
extractGatewayLogAppendFunction(source, gatewayLog),
extractShellFunction(source, "restore_openclaw_runtime_guard_chain"),
"rc=0; restore_openclaw_runtime_guard_chain || rc=$?",
'printf "rc:%s\\n" "$rc"',
Expand Down
52 changes: 52 additions & 0 deletions test/nemoclaw-start-reconcile.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,58 @@ describe("agent identity reconciliation with provider (#3175)", () => {

// ── Gateway-as-source-of-truth path (the #3175 user-reported repro) ──

it("preserves an explicit model override when the live gateway reports a conflicting model", () => {
const initial = {
agents: { defaults: { model: { primary: "anthropic/claude-sonnet-4-6" } } },
models: {
providers: {
inference: {
api: "openai-completions",
models: [{ id: "anthropic/claude-sonnet-4-6", name: "anthropic/claude-sonnet-4-6" }],
},
},
},
};
const { result, config, hash } = runReconcile(initial, {
env: { NEMOCLAW_MODEL_OVERRIDE: "anthropic/claude-sonnet-4-6" },
gatewayModel: "nvidia/nemotron-3-super-120b-a12b",
});

expect(result.status).toBe(0);
expect(config).toEqual(initial);
expect(hash).toBe("oldhash\n");
});

it("still reconciles from the live gateway when no explicit model override is set", () => {
const { result, config, hash } = runReconcile(
{
agents: { defaults: { model: { primary: "inference/nvidia-routed" } } },
models: {
providers: {
inference: {
api: "openai-completions",
models: [{ id: "nvidia-routed", name: "inference/nvidia-routed" }],
},
},
},
},
{ gatewayModel: "nvidia/nemotron-3-super-120b-a12b" },
);

expect(result.status).toBe(0);
expect(config.agents.defaults.model.primary).toBe(
"inference/nvidia/nemotron-3-super-120b-a12b",
);
expect(config.models.providers.inference.models[0].name).toBe(
"inference/nvidia/nemotron-3-super-120b-a12b",
);
expect(config.models.providers.inference.models[0].id).toBe(
"nvidia/nemotron-3-super-120b-a12b",
);
expect(hash).not.toBe("oldhash\n");
expect(hash).toContain("openclaw.json");
});

it("patches primary AND models[0] to the live gateway model when both file fields are stale", () => {
const { result, config, hash } = runReconcile(
{
Expand Down
Loading