Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 40 additions & 1 deletion bin/lib/onboard.js
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ const registry = require("./registry");
const nim = require("./nim");
const onboardSession = require("./onboard-session");
const policies = require("./policies");
const { checkPortAvailable } = require("./preflight");
const { checkPortAvailable, ensureSwap, getMemoryInfo } = require("./preflight");
const EXPERIMENTAL = process.env.NEMOCLAW_EXPERIMENTAL === "1";
const USE_COLOR = !process.env.NO_COLOR && !!process.stdout.isTTY;
const DIM = USE_COLOR ? "\x1b[2m" : "";
Expand Down Expand Up @@ -1582,6 +1582,45 @@ async function preflight() {
console.log(" ⓘ No GPU detected — will use cloud inference");
}

// Memory / swap check (Linux only)
if (process.platform === "linux") {
const mem = getMemoryInfo();
if (mem) {
if (mem.totalMB < 12000) {
console.log(` ⚠ Low memory detected (${mem.totalRamMB} MB RAM + ${mem.totalSwapMB} MB swap = ${mem.totalMB} MB total)`);

let proceedWithSwap = false;
if (!isNonInteractive()) {
const answer = await prompt(
" Create a 4 GB swap file to prevent OOM during sandbox build? (requires sudo) [y/N]: "
);
proceedWithSwap = answer && answer.toLowerCase().startsWith("y");
}

if (!proceedWithSwap) {
console.log(" ⓘ Skipping swap creation. Sandbox build may fail with OOM on this system.");
} else {
console.log(" Creating 4 GB swap file to prevent OOM during sandbox build...");
const swapResult = ensureSwap(12000);
if (swapResult.ok && swapResult.swapCreated) {
console.log(" ✓ Swap file created and activated");
} else if (swapResult.ok) {
if (swapResult.reason) {
console.log(` ⓘ ${swapResult.reason} — existing swap should help prevent OOM`);
} else {
console.log(` ✓ Memory OK: ${mem.totalRamMB} MB RAM + ${mem.totalSwapMB} MB swap`);
}
} else {
console.log(` ⚠ Could not create swap: ${swapResult.reason}`);
console.log(" Sandbox creation may fail with OOM on low-memory systems.");
}
}
} else {
console.log(` ✓ Memory OK: ${mem.totalRamMB} MB RAM + ${mem.totalSwapMB} MB swap`);
}
}
}

return gpu;
}

Expand Down
215 changes: 214 additions & 1 deletion bin/lib/preflight.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@
//
// Preflight checks for NemoClaw onboarding.

const fs = require("fs");
const net = require("net");
const os = require("os");
const path = require("path");
const { runCapture } = require("./runner");

async function probePortAvailability(port, opts = {}) {
Expand Down Expand Up @@ -105,4 +108,214 @@ async function checkPortAvailable(port, opts) {
return probePortAvailability(p, o);
}

module.exports = { checkPortAvailable, probePortAvailability };
/**
* Read system memory info (RAM + swap).
*
* On Linux, parses /proc/meminfo. On macOS, uses sysctl.
* Returns null on unsupported platforms or read errors.
*
* opts.meminfoContent — inject fake /proc/meminfo for testing
* opts.platform — override process.platform for testing
*
* Returns:
* { totalRamMB: number, totalSwapMB: number, totalMB: number }
*/
function getMemoryInfo(opts) {
const o = opts || {};
const platform = o.platform || process.platform;

if (platform === "linux") {
let content;
if (typeof o.meminfoContent === "string") {
content = o.meminfoContent;
} else {
try {
content = fs.readFileSync("/proc/meminfo", "utf-8");
} catch {
return null;
}
}

const parseKB = (key) => {
const match = content.match(new RegExp(`^${key}:\\s+(\\d+)`, "m"));
return match ? parseInt(match[1], 10) : 0;
};

const totalRamKB = parseKB("MemTotal");
const totalSwapKB = parseKB("SwapTotal");
const totalRamMB = Math.floor(totalRamKB / 1024);
const totalSwapMB = Math.floor(totalSwapKB / 1024);
return { totalRamMB, totalSwapMB, totalMB: totalRamMB + totalSwapMB };
}

if (platform === "darwin") {
try {
const memBytes = parseInt(
runCapture("sysctl -n hw.memsize", { ignoreError: true }),
10
);
if (!memBytes || isNaN(memBytes)) return null;
const totalRamMB = Math.floor(memBytes / 1024 / 1024);
// macOS does not use traditional swap files in the same way
return { totalRamMB, totalSwapMB: 0, totalMB: totalRamMB };
} catch {
return null;
}
}

return null;
}

/**
* Ensure the system has enough memory (RAM + swap) for sandbox operations.
*
* If total memory is below minTotalMB and no swap file exists, attempts to
* create a 4 GB swap file via sudo to prevent OOM kills during sandbox image push.
*
* opts.memoryInfo — inject mock getMemoryInfo() result for testing
* opts.platform — override process.platform for testing
* opts.dryRun — if true, skip actual swap creation (for testing)
*
* Returns:
* { ok: true, totalMB, swapCreated: boolean }
* { ok: false, reason: string }
*/
function ensureSwap(minTotalMB, opts = {}) {
const o = {
platform: process.platform,
memoryInfo: null,
swapfileExists: fs.existsSync("/swapfile"),
dryRun: false,
interactive: process.stdout.isTTY && !process.env.NEMOCLAW_NON_INTERACTIVE,
getMemoryInfoImpl: getMemoryInfo,
...opts,
};
const threshold = minTotalMB ?? 12000;
const platform = o.platform;

if (platform !== "linux") {
return { ok: true, totalMB: 0, swapCreated: false };
}

const mem = o.memoryInfo ?? o.getMemoryInfoImpl({ platform });
if (!mem) {
return { ok: false, reason: "could not read memory info" };
}

if (mem.totalMB >= threshold) {
return { ok: true, totalMB: mem.totalMB, swapCreated: false };
}

if (!o.dryRun) {
const swapfileExists = (() => {
try {
fs.accessSync("/swapfile");
return true;
} catch {
return false;
}
})();

if (swapfileExists) {
const swaps = (() => {
try {
return fs.readFileSync("/proc/swaps", "utf-8");
} catch {
return "";
}
})();

if (swaps.includes("/swapfile")) {
// Active swap — nothing to do
return {
ok: true,
totalMB: mem.totalMB,
swapCreated: false,
reason: "/swapfile already exists",
};
}
// File exists but isn't active — re-activate rather than overwrite
try {
runCapture("sudo swapon /swapfile", { ignoreError: false });
return { ok: true, totalMB: mem.totalMB + 4096, swapCreated: true };
} catch (err) {
return {
ok: false,
reason: `found orphaned /swapfile but could not activate it: ${err.message}`,
};
}
}
// No swapfile at all — fall through to creation
} else {
// In dry-run mode, simulate the check
if (o.swapfileExists) {
return {
ok: true,
totalMB: mem.totalMB,
swapCreated: false,
reason: "/swapfile already exists",
};
}
}

// Bail if disk is too small for a 4 GB swap file
if (!o.dryRun) {
try {
const dfOut = runCapture("df / --output=avail -k 2>/dev/null | tail -1", { ignoreError: true });
const freeKB = parseInt((dfOut || "").trim(), 10);
if (!isNaN(freeKB) && freeKB < 5000000) {
return {
ok: false,
reason: `insufficient disk space (${Math.floor(freeKB / 1024)} MB free, need ~5 GB) to create swap file`,
};
}
} catch {
// df unavailable — let dd fail naturally if out of space
}
}

if (o.dryRun) {
return { ok: true, totalMB: mem.totalMB, swapCreated: true };
}

// Create 4 GB swap file
try {
runCapture("sudo dd if=/dev/zero of=/swapfile bs=1M count=4096 status=none", { ignoreError: false });
runCapture("sudo chmod 600 /swapfile", { ignoreError: false });
runCapture("sudo mkswap /swapfile", { ignoreError: false });
runCapture("sudo swapon /swapfile", { ignoreError: false });
runCapture(
"grep -q '/swapfile' /etc/fstab || echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab",
{ ignoreError: false }
);

const nemoclawDir = path.join(os.homedir(), ".nemoclaw");
if (!fs.existsSync(nemoclawDir)) {
runCapture(`mkdir -p ${nemoclawDir}`, { ignoreError: true });
}
try {
fs.writeFileSync(path.join(nemoclawDir, "managed_swap"), "/swapfile");
} catch {
}

return { ok: true, totalMB: mem.totalMB + 4096, swapCreated: true };
} catch (err) {
// Attempt cleanup of partial state
try {
runCapture("sudo swapoff /swapfile 2>/dev/null || true", { ignoreError: true });
runCapture("sudo rm -f /swapfile", { ignoreError: true });
} catch {
// Best effort cleanup
}

return {
ok: false,
reason: `swap creation failed: ${err.message}. Create swap manually:\n` +
" sudo dd if=/dev/zero of=/swapfile bs=1M count=4096 status=none && sudo chmod 600 /swapfile && " +
"sudo mkswap /swapfile && sudo swapon /swapfile",
};
}
}

module.exports = { checkPortAvailable, probePortAvailability, getMemoryInfo, ensureSwap };

16 changes: 16 additions & 0 deletions docs/reference/troubleshooting.md
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,22 @@ If neither is found, verify that Colima is running:
$ colima status
```

### Sandbox creation killed by OOM (exit 137)

On systems with 8 GB RAM or less and no swap configured, the sandbox image push can exhaust available memory and get killed by the Linux OOM killer (exit code 137).

NemoClaw automatically detects low memory during onboarding and prompts to create a 4 GB swap file.
If this automatic step fails or you are using a custom setup flow, create swap manually before running `nemoclaw onboard`:

```console
$ sudo dd if=/dev/zero of=/swapfile bs=1M count=4096 status=none
$ sudo chmod 600 /swapfile
$ sudo mkswap /swapfile
$ sudo swapon /swapfile
$ echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab
$ nemoclaw onboard
```

## Runtime

### Reconnect after a host reboot
Expand Down
23 changes: 22 additions & 1 deletion scripts/setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,28 @@ fi
info "Setting inference route to nvidia-nim / Nemotron 3 Super..."
openshell inference set --no-verify --provider nvidia-nim --model nvidia/nemotron-3-super-120b-a12b >/dev/null 2>&1

# 5. Build and create sandbox
# 5. Swap check — prevent OOM during sandbox image push (Linux only)
if [ "$(uname -s)" = "Linux" ]; then
MIN_TOTAL_MB=12000
total_ram_mb=$(awk '/MemTotal/{printf "%d", $2/1024}' /proc/meminfo 2>/dev/null || echo 0)
total_swap_mb=$(awk '/SwapTotal/{printf "%d", $2/1024}' /proc/meminfo 2>/dev/null || echo 0)
total_mb=$((total_ram_mb + total_swap_mb))
if [ "$total_mb" -lt "$MIN_TOTAL_MB" ] && [ ! -f /swapfile ]; then
# Bail if disk can't fit a 4 GB swap file
free_disk_kb=$(df / --output=avail -k 2>/dev/null | tail -1 | tr -d ' ')
if [ -n "$free_disk_kb" ] && [ "$free_disk_kb" -lt 5000000 ]; then
warn "Insufficient disk space ($((free_disk_kb / 1024)) MB free, need ~5 GB) to create swap file. Skipping."
else
warn "Low memory detected (${total_mb} MB). Sandbox creation may fail with OOM."
warn "Consider manually creating a swap file:"
warn " sudo dd if=/dev/zero of=/swapfile bs=1M count=4096 status=none && sudo chmod 600 /swapfile && sudo mkswap /swapfile && sudo swapon /swapfile"
fi
elif [ "$total_mb" -ge "$MIN_TOTAL_MB" ]; then
info "Memory OK: ${total_ram_mb} MB RAM + ${total_swap_mb} MB swap"
fi
fi

# 6. Build and create sandbox
info "Deleting old ${SANDBOX_NAME} sandbox (if any)..."
openshell sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 || true

Expand Down
Loading