diff --git a/bin/lib/onboard.js b/bin/lib/onboard.js index fa5150a980..9635d516ac 100644 --- a/bin/lib/onboard.js +++ b/bin/lib/onboard.js @@ -38,7 +38,7 @@ const registry = require("./registry"); const nim = require("./nim"); const onboardSession = require("./onboard-session"); const policies = require("./policies"); -const { checkPortAvailable } = require("./preflight"); +const { checkPortAvailable, ensureSwap, getMemoryInfo } = require("./preflight"); const EXPERIMENTAL = process.env.NEMOCLAW_EXPERIMENTAL === "1"; const USE_COLOR = !process.env.NO_COLOR && !!process.stdout.isTTY; const DIM = USE_COLOR ? "\x1b[2m" : ""; @@ -1582,6 +1582,45 @@ async function preflight() { console.log(" ⓘ No GPU detected — will use cloud inference"); } + // Memory / swap check (Linux only) + if (process.platform === "linux") { + const mem = getMemoryInfo(); + if (mem) { + if (mem.totalMB < 12000) { + console.log(` ⚠ Low memory detected (${mem.totalRamMB} MB RAM + ${mem.totalSwapMB} MB swap = ${mem.totalMB} MB total)`); + + let proceedWithSwap = false; + if (!isNonInteractive()) { + const answer = await prompt( + " Create a 4 GB swap file to prevent OOM during sandbox build? (requires sudo) [y/N]: " + ); + proceedWithSwap = answer && answer.toLowerCase().startsWith("y"); + } + + if (!proceedWithSwap) { + console.log(" ⓘ Skipping swap creation. Sandbox build may fail with OOM on this system."); + } else { + console.log(" Creating 4 GB swap file to prevent OOM during sandbox build..."); + const swapResult = ensureSwap(12000); + if (swapResult.ok && swapResult.swapCreated) { + console.log(" ✓ Swap file created and activated"); + } else if (swapResult.ok) { + if (swapResult.reason) { + console.log(` ⓘ ${swapResult.reason} — existing swap should help prevent OOM`); + } else { + console.log(` ✓ Memory OK: ${mem.totalRamMB} MB RAM + ${mem.totalSwapMB} MB swap`); + } + } else { + console.log(` ⚠ Could not create swap: ${swapResult.reason}`); + console.log(" Sandbox creation may fail with OOM on low-memory systems."); + } + } + } else { + console.log(` ✓ Memory OK: ${mem.totalRamMB} MB RAM + ${mem.totalSwapMB} MB swap`); + } + } + } + return gpu; } diff --git a/bin/lib/preflight.js b/bin/lib/preflight.js index 007eb7c4f3..caa0a76df6 100644 --- a/bin/lib/preflight.js +++ b/bin/lib/preflight.js @@ -3,7 +3,10 @@ // // Preflight checks for NemoClaw onboarding. +const fs = require("fs"); const net = require("net"); +const os = require("os"); +const path = require("path"); const { runCapture } = require("./runner"); async function probePortAvailability(port, opts = {}) { @@ -105,4 +108,214 @@ async function checkPortAvailable(port, opts) { return probePortAvailability(p, o); } -module.exports = { checkPortAvailable, probePortAvailability }; +/** + * Read system memory info (RAM + swap). + * + * On Linux, parses /proc/meminfo. On macOS, uses sysctl. + * Returns null on unsupported platforms or read errors. + * + * opts.meminfoContent — inject fake /proc/meminfo for testing + * opts.platform — override process.platform for testing + * + * Returns: + * { totalRamMB: number, totalSwapMB: number, totalMB: number } + */ +function getMemoryInfo(opts) { + const o = opts || {}; + const platform = o.platform || process.platform; + + if (platform === "linux") { + let content; + if (typeof o.meminfoContent === "string") { + content = o.meminfoContent; + } else { + try { + content = fs.readFileSync("/proc/meminfo", "utf-8"); + } catch { + return null; + } + } + + const parseKB = (key) => { + const match = content.match(new RegExp(`^${key}:\\s+(\\d+)`, "m")); + return match ? parseInt(match[1], 10) : 0; + }; + + const totalRamKB = parseKB("MemTotal"); + const totalSwapKB = parseKB("SwapTotal"); + const totalRamMB = Math.floor(totalRamKB / 1024); + const totalSwapMB = Math.floor(totalSwapKB / 1024); + return { totalRamMB, totalSwapMB, totalMB: totalRamMB + totalSwapMB }; + } + + if (platform === "darwin") { + try { + const memBytes = parseInt( + runCapture("sysctl -n hw.memsize", { ignoreError: true }), + 10 + ); + if (!memBytes || isNaN(memBytes)) return null; + const totalRamMB = Math.floor(memBytes / 1024 / 1024); + // macOS does not use traditional swap files in the same way + return { totalRamMB, totalSwapMB: 0, totalMB: totalRamMB }; + } catch { + return null; + } + } + + return null; +} + +/** + * Ensure the system has enough memory (RAM + swap) for sandbox operations. + * + * If total memory is below minTotalMB and no swap file exists, attempts to + * create a 4 GB swap file via sudo to prevent OOM kills during sandbox image push. + * + * opts.memoryInfo — inject mock getMemoryInfo() result for testing + * opts.platform — override process.platform for testing + * opts.dryRun — if true, skip actual swap creation (for testing) + * + * Returns: + * { ok: true, totalMB, swapCreated: boolean } + * { ok: false, reason: string } + */ +function ensureSwap(minTotalMB, opts = {}) { + const o = { + platform: process.platform, + memoryInfo: null, + swapfileExists: fs.existsSync("/swapfile"), + dryRun: false, + interactive: process.stdout.isTTY && !process.env.NEMOCLAW_NON_INTERACTIVE, + getMemoryInfoImpl: getMemoryInfo, + ...opts, + }; + const threshold = minTotalMB ?? 12000; + const platform = o.platform; + + if (platform !== "linux") { + return { ok: true, totalMB: 0, swapCreated: false }; + } + + const mem = o.memoryInfo ?? o.getMemoryInfoImpl({ platform }); + if (!mem) { + return { ok: false, reason: "could not read memory info" }; + } + + if (mem.totalMB >= threshold) { + return { ok: true, totalMB: mem.totalMB, swapCreated: false }; + } + + if (!o.dryRun) { + const swapfileExists = (() => { + try { + fs.accessSync("/swapfile"); + return true; + } catch { + return false; + } + })(); + + if (swapfileExists) { + const swaps = (() => { + try { + return fs.readFileSync("/proc/swaps", "utf-8"); + } catch { + return ""; + } + })(); + + if (swaps.includes("/swapfile")) { + // Active swap — nothing to do + return { + ok: true, + totalMB: mem.totalMB, + swapCreated: false, + reason: "/swapfile already exists", + }; + } + // File exists but isn't active — re-activate rather than overwrite + try { + runCapture("sudo swapon /swapfile", { ignoreError: false }); + return { ok: true, totalMB: mem.totalMB + 4096, swapCreated: true }; + } catch (err) { + return { + ok: false, + reason: `found orphaned /swapfile but could not activate it: ${err.message}`, + }; + } + } + // No swapfile at all — fall through to creation + } else { + // In dry-run mode, simulate the check + if (o.swapfileExists) { + return { + ok: true, + totalMB: mem.totalMB, + swapCreated: false, + reason: "/swapfile already exists", + }; + } + } + + // Bail if disk is too small for a 4 GB swap file + if (!o.dryRun) { + try { + const dfOut = runCapture("df / --output=avail -k 2>/dev/null | tail -1", { ignoreError: true }); + const freeKB = parseInt((dfOut || "").trim(), 10); + if (!isNaN(freeKB) && freeKB < 5000000) { + return { + ok: false, + reason: `insufficient disk space (${Math.floor(freeKB / 1024)} MB free, need ~5 GB) to create swap file`, + }; + } + } catch { + // df unavailable — let dd fail naturally if out of space + } + } + + if (o.dryRun) { + return { ok: true, totalMB: mem.totalMB, swapCreated: true }; + } + + // Create 4 GB swap file + try { + runCapture("sudo dd if=/dev/zero of=/swapfile bs=1M count=4096 status=none", { ignoreError: false }); + runCapture("sudo chmod 600 /swapfile", { ignoreError: false }); + runCapture("sudo mkswap /swapfile", { ignoreError: false }); + runCapture("sudo swapon /swapfile", { ignoreError: false }); + runCapture( + "grep -q '/swapfile' /etc/fstab || echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab", + { ignoreError: false } + ); + + const nemoclawDir = path.join(os.homedir(), ".nemoclaw"); + if (!fs.existsSync(nemoclawDir)) { + runCapture(`mkdir -p ${nemoclawDir}`, { ignoreError: true }); + } + try { + fs.writeFileSync(path.join(nemoclawDir, "managed_swap"), "/swapfile"); + } catch { + } + + return { ok: true, totalMB: mem.totalMB + 4096, swapCreated: true }; + } catch (err) { + // Attempt cleanup of partial state + try { + runCapture("sudo swapoff /swapfile 2>/dev/null || true", { ignoreError: true }); + runCapture("sudo rm -f /swapfile", { ignoreError: true }); + } catch { + // Best effort cleanup + } + + return { + ok: false, + reason: `swap creation failed: ${err.message}. Create swap manually:\n` + + " sudo dd if=/dev/zero of=/swapfile bs=1M count=4096 status=none && sudo chmod 600 /swapfile && " + + "sudo mkswap /swapfile && sudo swapon /swapfile", + }; + } +} + +module.exports = { checkPortAvailable, probePortAvailability, getMemoryInfo, ensureSwap }; + diff --git a/docs/reference/troubleshooting.md b/docs/reference/troubleshooting.md index 1a0ba5d7e1..da61ee93e0 100644 --- a/docs/reference/troubleshooting.md +++ b/docs/reference/troubleshooting.md @@ -141,6 +141,22 @@ If neither is found, verify that Colima is running: $ colima status ``` +### Sandbox creation killed by OOM (exit 137) + +On systems with 8 GB RAM or less and no swap configured, the sandbox image push can exhaust available memory and get killed by the Linux OOM killer (exit code 137). + +NemoClaw automatically detects low memory during onboarding and prompts to create a 4 GB swap file. +If this automatic step fails or you are using a custom setup flow, create swap manually before running `nemoclaw onboard`: + +```console +$ sudo dd if=/dev/zero of=/swapfile bs=1M count=4096 status=none +$ sudo chmod 600 /swapfile +$ sudo mkswap /swapfile +$ sudo swapon /swapfile +$ echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab +$ nemoclaw onboard +``` + ## Runtime ### Reconnect after a host reboot diff --git a/scripts/setup.sh b/scripts/setup.sh index 34d60600d3..0908d3ea2e 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -188,7 +188,28 @@ fi info "Setting inference route to nvidia-nim / Nemotron 3 Super..." openshell inference set --no-verify --provider nvidia-nim --model nvidia/nemotron-3-super-120b-a12b >/dev/null 2>&1 -# 5. Build and create sandbox +# 5. Swap check — prevent OOM during sandbox image push (Linux only) +if [ "$(uname -s)" = "Linux" ]; then + MIN_TOTAL_MB=12000 + total_ram_mb=$(awk '/MemTotal/{printf "%d", $2/1024}' /proc/meminfo 2>/dev/null || echo 0) + total_swap_mb=$(awk '/SwapTotal/{printf "%d", $2/1024}' /proc/meminfo 2>/dev/null || echo 0) + total_mb=$((total_ram_mb + total_swap_mb)) + if [ "$total_mb" -lt "$MIN_TOTAL_MB" ] && [ ! -f /swapfile ]; then + # Bail if disk can't fit a 4 GB swap file + free_disk_kb=$(df / --output=avail -k 2>/dev/null | tail -1 | tr -d ' ') + if [ -n "$free_disk_kb" ] && [ "$free_disk_kb" -lt 5000000 ]; then + warn "Insufficient disk space ($((free_disk_kb / 1024)) MB free, need ~5 GB) to create swap file. Skipping." + else + warn "Low memory detected (${total_mb} MB). Sandbox creation may fail with OOM." + warn "Consider manually creating a swap file:" + warn " sudo dd if=/dev/zero of=/swapfile bs=1M count=4096 status=none && sudo chmod 600 /swapfile && sudo mkswap /swapfile && sudo swapon /swapfile" + fi + elif [ "$total_mb" -ge "$MIN_TOTAL_MB" ]; then + info "Memory OK: ${total_ram_mb} MB RAM + ${total_swap_mb} MB swap" + fi +fi + +# 6. Build and create sandbox info "Deleting old ${SANDBOX_NAME} sandbox (if any)..." openshell sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 || true diff --git a/test/preflight.test.js b/test/preflight.test.js index 90e1f4d9be..948c61cdd2 100644 --- a/test/preflight.test.js +++ b/test/preflight.test.js @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { assert, describe, expect, it } from "vitest"; import { checkPortAvailable } from "../bin/lib/preflight"; @@ -114,3 +114,104 @@ describe("checkPortAvailable", () => { expect(result.ok).toBe(true); }); }); + +describe("getMemoryInfo", () => { + const { getMemoryInfo } = require("../bin/lib/preflight"); + + it("parses valid /proc/meminfo content", () => { + const meminfoContent = [ + "MemTotal: 8152056 kB", + "MemFree: 1234567 kB", + "MemAvailable: 4567890 kB", + "SwapTotal: 4194300 kB", + "SwapFree: 4194300 kB", + ].join("\n"); + + const result = getMemoryInfo({ meminfoContent, platform: "linux" }); + assert.equal(result.totalRamMB, Math.floor(8152056 / 1024)); + assert.equal(result.totalSwapMB, Math.floor(4194300 / 1024)); + assert.equal(result.totalMB, result.totalRamMB + result.totalSwapMB); + }); + + it("returns correct values when swap is zero", () => { + const meminfoContent = [ + "MemTotal: 8152056 kB", + "MemFree: 1234567 kB", + "SwapTotal: 0 kB", + "SwapFree: 0 kB", + ].join("\n"); + + const result = getMemoryInfo({ meminfoContent, platform: "linux" }); + assert.equal(result.totalRamMB, Math.floor(8152056 / 1024)); + assert.equal(result.totalSwapMB, 0); + assert.equal(result.totalMB, result.totalRamMB); + }); + + it("returns null on unsupported platforms", () => { + const result = getMemoryInfo({ platform: "win32" }); + assert.equal(result, null); + }); + + it("handles malformed /proc/meminfo gracefully", () => { + const result = getMemoryInfo({ meminfoContent: "garbage data\nno fields here", platform: "linux" }); + assert.equal(result.totalRamMB, 0); + assert.equal(result.totalSwapMB, 0); + assert.equal(result.totalMB, 0); + }); +}); + +describe("ensureSwap", () => { + const { ensureSwap } = require("../bin/lib/preflight"); + + it("returns ok when total memory already exceeds threshold", () => { + const result = ensureSwap(6144, { + platform: "linux", + memoryInfo: { totalRamMB: 8000, totalSwapMB: 0, totalMB: 8000 }, + }); + assert.equal(result.ok, true); + assert.equal(result.swapCreated, false); + assert.equal(result.totalMB, 8000); + }); + + it("reports swap would be created in dry-run mode when below threshold", () => { + const result = ensureSwap(6144, { + platform: "linux", + memoryInfo: { totalRamMB: 4000, totalSwapMB: 0, totalMB: 4000 }, + dryRun: true, + swapfileExists: false, + }); + assert.equal(result.ok, true); + assert.equal(result.swapCreated, true); + }); + + it("skips swap creation when /swapfile already exists (dry-run)", () => { + const result = ensureSwap(6144, { + platform: "linux", + memoryInfo: { totalRamMB: 4000, totalSwapMB: 0, totalMB: 4000 }, + dryRun: true, + swapfileExists: true, + }); + assert.equal(result.ok, true); + assert.equal(result.swapCreated, false); + assert.match(result.reason, /swapfile already exists/); + }); + + it("skips on non-Linux platforms", () => { + const result = ensureSwap(6144, { + platform: "darwin", + memoryInfo: { totalRamMB: 4000, totalSwapMB: 0, totalMB: 4000 }, + }); + assert.equal(result.ok, true); + assert.equal(result.swapCreated, false); + }); + + it("returns error when memory info is unavailable", () => { + const result = ensureSwap(6144, { + platform: "linux", + memoryInfo: null, + getMemoryInfoImpl: () => null, + }); + assert.equal(result.ok, false); + assert.match(result.reason, /could not read memory info/); + }); +}); diff --git a/uninstall.sh b/uninstall.sh index c8bf4d4d6c..a5a4950a57 100755 --- a/uninstall.sh +++ b/uninstall.sh @@ -493,6 +493,46 @@ remove_optional_ollama_models() { done } +remove_nemoclaw_swap() { + if [ ! -f /swapfile ]; then + info "No /swapfile found; skipping swap cleanup." + return 0 + fi + + if [ ! -f "$NEMOCLAW_STATE_DIR/managed_swap" ]; then + warn "No NemoClaw-managed swap marker found, skipping swap cleanup." + return 0 + fi + + local swap_file + swap_file=$(cat "$NEMOCLAW_STATE_DIR/managed_swap" 2>/dev/null || echo "") + if [ "$swap_file" != "/swapfile" ]; then + warn "Marker file does not point to /swapfile, skipping swap cleanup." + return 0 + fi + + if [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ] || [ ! -t 0 ]; then + warn "Skipping swap cleanup in non-interactive mode (requires sudo)." + return 0 + fi + + info "Deactivating and removing /swapfile..." + sudo swapoff /swapfile 2>/dev/null || true + sudo rm -f /swapfile + + if [ -f /swapfile ]; then + warn "Failed to remove /swapfile. Please remove it manually." + return 1 + fi + + # Clean fstab entry + if grep -q '/swapfile' /etc/fstab 2>/dev/null; then + sudo sed -i '\|^/swapfile[[:space:]]|d' /etc/fstab + info "Removed /swapfile entry from /etc/fstab" + fi + info "Swap file removed" +} + remove_runtime_temp_artifacts() { remove_glob_paths "${TMP_ROOT}/nemoclaw-create-*.log" remove_glob_paths "${TMP_ROOT}/nemoclaw-tg-ssh-*.conf" @@ -545,6 +585,10 @@ main() { remove_optional_ollama_models step 6 "State and binaries" + info "Removing NemoClaw-managed swap file..." + remove_nemoclaw_swap + + info "Removing runtime temp artifacts..." remove_runtime_temp_artifacts remove_openshell_binary remove_nemoclaw_state