Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 126 additions & 37 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,8 @@ const {
const onboardProviders = require("./onboard/providers");
const hermesProviderAuth = require("./hermes-provider-auth");

const OLLAMA_SYSTEMD_OVERRIDE_PATH = "/etc/systemd/system/ollama.service.d/override.conf";

type RemoteProviderConfigEntry = {
label: string;
providerName: string;
Expand Down Expand Up @@ -6643,6 +6645,108 @@ async function selectAndValidateOllamaModel(
}
}

type OllamaLoopbackSystemdOverrideState = "not-applicable" | "ready" | "failed";

function ensureOllamaLoopbackSystemdOverride(): OllamaLoopbackSystemdOverrideState {
if (process.platform !== "linux" || isWsl()) return "not-applicable";

const hasOllamaSystemdUnit = !!runCapture(
[
"sh",
"-c",
"command -v systemctl >/dev/null && [ -d /run/systemd/system ] && systemctl list-unit-files ollama.service --no-legend 2>/dev/null | head -n1",
],
{ ignoreError: true },
).trim();
if (!hasOllamaSystemdUnit) return "not-applicable";

console.log(" Configuring Ollama systemd loopback override...");
console.log(
` Applying an Ollama systemd override (OLLAMA_HOST=127.0.0.1:${OLLAMA_PORT}). ` +
"The next steps use sudo to write the drop-in, reload systemd, and restart the service; " +
"you may be prompted for your password.",
);
const sudoPrefix = isNonInteractive() || !process.stdin.isTTY ? "sudo -n" : "sudo";
const existingDropInResult = runShell(
`if [ -e ${shellQuote(OLLAMA_SYSTEMD_OVERRIDE_PATH)} ]; then ${sudoPrefix} cat ${shellQuote(OLLAMA_SYSTEMD_OVERRIDE_PATH)}; fi`,
{ ignoreError: true, suppressOutput: true, timeout: 30_000 },
);
if (existingDropInResult.error || existingDropInResult.status !== 0) {
console.error(" Failed to inspect existing Ollama systemd override.");
console.error(" Refusing to continue because preserving existing Ollama settings is required.");
process.exit(1);
}
const existingDropIn = String(existingDropInResult.stdout || "");
const dropInBody = mergeOllamaLoopbackSystemdOverride(existingDropIn);
const tmpDropIn = secureTempFile("nemoclaw-ollama-override", ".conf");
let overrideFailed = false;
try {
fs.writeFileSync(tmpDropIn, dropInBody, { mode: 0o644 });
const overrideResult = runShell(
[
"set -e",
`pre_state=$(${sudoPrefix} systemctl show ollama --property=ActiveEnterTimestampMonotonic --property=MainPID --value 2>/dev/null | tr '\\n' ' ')`,
`${sudoPrefix} install -D -m 0644 ${shellQuote(tmpDropIn)} ${shellQuote(OLLAMA_SYSTEMD_OVERRIDE_PATH)}`,
`${sudoPrefix} systemctl daemon-reload`,
`${sudoPrefix} systemctl --no-block restart ollama`,
"for _ in $(seq 1 30); do",
` current_state=$(${sudoPrefix} systemctl show ollama --property=ActiveEnterTimestampMonotonic --property=MainPID --value 2>/dev/null | tr '\\n' ' ')`,
` if [ "$current_state" != "$pre_state" ] && ${sudoPrefix} systemctl is-active --quiet ollama; then exit 0; fi`,
" sleep 1",
"done",
"exit 1",
].join("\n"),
{ ignoreError: true, timeout: 45_000 },
);
if (overrideResult.error || overrideResult.status !== 0) {
overrideFailed = true;
}
} finally {
cleanupTempDir(tmpDropIn, "nemoclaw-ollama-override");
}
if (overrideFailed) {
console.error(" Failed to apply Ollama systemd loopback override.");
console.error(" Refusing to continue with a potentially non-loopback Ollama bind.");
process.exit(1);
}

// The restart may briefly drop Ollama. Clear the cached successful probe so
// the readiness loop checks the daemon that systemd just restarted.
resetOllamaHostCache();
for (let i = 0; i < 10; i++) {
if (findReachableOllamaHost()) return "ready";
sleep(1);
}
return "failed";
}

function mergeOllamaLoopbackSystemdOverride(existingDropIn: string): string {
const desiredLine = `Environment="OLLAMA_HOST=127.0.0.1:${OLLAMA_PORT}"`;
const lines = existingDropIn.trimEnd().length > 0 ? existingDropIn.trimEnd().split(/\r?\n/) : [];
const serviceStart = lines.findIndex((line) => /^\s*\[Service\]\s*(?:[#;].*)?$/.test(line));
if (serviceStart === -1) {
return [...lines, ...(lines.length > 0 ? [""] : []), "[Service]", desiredLine].join("\n") + "\n";
}

let serviceEnd = lines.length;
for (let i = serviceStart + 1; i < lines.length; i++) {
if (/^\s*\[[^\]]+\]\s*(?:[#;].*)?$/.test(lines[i])) {
serviceEnd = i;
break;
}
}

const hostLines = lines
.slice(serviceStart + 1, serviceEnd)
.filter((line) => !/^\s*[#;]/.test(line) && /\bOLLAMA_HOST=/.test(line));
if (hostLines.at(-1)?.trim() === desiredLine) {
return lines.join("\n") + "\n";
}

lines.splice(serviceEnd, 0, desiredLine);
return lines.join("\n") + "\n";
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

async function setupNim(
gpu: ReturnType<typeof nim.detectGpu>,
sandboxName: string | null = null,
Expand Down Expand Up @@ -7544,7 +7648,17 @@ async function setupNim(
break;
} else if (selected.key === "ollama") {
if (!checkOllamaPortsOrWarn()) continue selectionLoop;
if (!ollamaRunning) {
let ollamaReady = ollamaRunning;
const overrideState = ensureOllamaLoopbackSystemdOverride();
if (overrideState === "ready") {
ollamaReady = true;
} else if (overrideState === "failed") {
console.error(
" Ollama systemd restart did not recover after applying the loopback override.",
);
process.exit(1);
}
if (!ollamaReady) {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
console.log(" Starting Ollama...");
// Keep raw Ollama loopback-only. Non-WSL containers reach it through
// the authenticated proxy on OLLAMA_PROXY_PORT.
Expand Down Expand Up @@ -7694,48 +7808,23 @@ async function setupNim(
"It uses sudo for those steps; you may be prompted for your password.",
);
runShell("set -o pipefail; curl -fsSL https://ollama.com/install.sh | sh");
// install.sh only creates ollama.service when systemctl is present.
// On non-systemd Linux (Alpine/OpenRC, container images, etc.) the
// installer just lays down the binary and we have to start it
// ourselves.
const hasOllamaSystemdUnit = !!runCapture(
[
"sh",
"-c",
"command -v systemctl >/dev/null && systemctl list-unit-files ollama.service --no-legend 2>/dev/null | head -n1",
],
{ ignoreError: true },
).trim();
// Give the just-started ollama.service a moment to bind port
// 11434 before we probe or apply the systemd drop-in override.
sleep(2);
// Linux native + systemd: force a loopback-only OLLAMA_HOST drop-in
// and let systemd own the daemon (avoids racing the installer's
// daemon with our own `ollama serve`). This also repairs older
// NemoClaw-created overrides that exposed raw Ollama on all interfaces.
// WSL keeps Ollama's default binding. No-systemd / daemon failed to
// start: manual launch with the loopback binding.
if (!isWsl() && hasOllamaSystemdUnit) {
console.log(" Configuring Ollama systemd loopback override...");
console.log(
` Applying an Ollama systemd override (OLLAMA_HOST=127.0.0.1:${OLLAMA_PORT}). ` +
"The next steps use sudo to write the drop-in, reload systemd, and restart the service; " +
"you may be prompted for your password.",
);
const dropInBody = `[Service]\nEnvironment="OLLAMA_HOST=127.0.0.1:${OLLAMA_PORT}"\n`;
const tmpDropIn = secureTempFile("nemoclaw-ollama-override", ".conf");
fs.writeFileSync(tmpDropIn, dropInBody, { mode: 0o644 });
const overrideResult = runShell(
`sudo install -D -m 0644 ${shellQuote(tmpDropIn)} ${shellQuote("/etc/systemd/system/ollama.service.d/override.conf")} && sudo systemctl daemon-reload && sudo systemctl restart ollama`,
{ ignoreError: true },
// WSL and non-systemd Linux fall back to a manual loopback launch.
const overrideState = ensureOllamaLoopbackSystemdOverride();
if (overrideState === "failed") {
console.error(
" Ollama systemd restart did not recover after applying the loopback override.",
);
cleanupTempDir(tmpDropIn, "nemoclaw-ollama-override");
if (overrideResult.error || overrideResult.status !== 0) {
console.error(" Failed to apply Ollama systemd loopback override.");
console.error(" Refusing to continue with a potentially non-loopback Ollama bind.");
process.exit(1);
}
waitUntil(() => Boolean(findReachableOllamaHost()), 10, 1000);
process.exit(1);
}
// Fall back to manual start if systemd path failed or isn't present.
if (!findReachableOllamaHost()) {
// Fall back to manual start only when systemd is unavailable.
if (overrideState === "not-applicable" && !findReachableOllamaHost()) {
console.log(" Starting Ollama...");
const ollamaEnv = isWsl() ? "" : `OLLAMA_HOST=127.0.0.1:${OLLAMA_PORT} `;
runShell(`${ollamaEnv}ollama serve > /dev/null 2>&1 &`, { ignoreError: true });
Expand Down
Loading
Loading