Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
c6414af
refactor(cli): replace race-condition sleeps with readiness probes
ksapru Apr 24, 2026
1961db8
fix(cli): bubble up errors when policy preset wait loop times out, fi…
ksapru Apr 24, 2026
ef67b43
fix(cli): abort onboarding if readiness probes timeout
ksapru Apr 24, 2026
d3c5e5c
fix(cli): add strict readiness check
ksapru Apr 24, 2026
ddf29ab
refactor(cli): extract dashboard delivery chain into contract/health/…
jyaunches Apr 24, 2026
d0f5083
fix(sandbox): ensure procps survives hardening and stale base images …
ericksoa Apr 24, 2026
93db521
refactor(types): tighten repo-wide type boundaries (#2422)
cv Apr 24, 2026
29d43b3
feat(cli): typed command registry as single source of truth (#2388) (…
jyaunches Apr 24, 2026
47b4e61
fix(onboard): validate Slack bot token format before saving (#2130)
latenighthackathon Apr 24, 2026
3308d99
refactor(types): consolidate duplicated errno and JSON type helpers (…
jyaunches Apr 24, 2026
8b55f6c
fix(test): satisfy strict typecheck on Slack token validation tests (…
cjagwani Apr 24, 2026
94bae3f
ci: gate every PR on tsc-cli regardless of touched paths (#2461)
cjagwani Apr 24, 2026
9830309
fix(security): validate symlinks in snapshot create and rollback path…
ericksoa Apr 24, 2026
d49c1d5
fix(onboard): always inject NEMOCLAW_DASHBOARD_PORT into sandbox env …
prekshivyas Apr 24, 2026
fa27a62
docs: reorganize Get Started and add Deployment Topology diagram (#2445)
miyoungc Apr 24, 2026
1547947
chore(test): apply shfmt formatting to E2E scripts (#2359)
ericksoa Apr 25, 2026
13d7a77
fix(test): move installer integration tests to CI-only vitest project…
ericksoa Apr 25, 2026
9d35df4
docs: catch up release prep for 0.0.25 (#2463)
miyoungc Apr 25, 2026
f0cbbb1
fix(cli): restore pre-#2398 gateway recovery (fixes E2E hangs) (#2471)
ericksoa Apr 25, 2026
f2ddb99
fix(install): refuse to auto-resume a failed onboarding session (#2437)
laitingsheng Apr 25, 2026
1e69692
fix(cli): resolve merge conflicts and improve onboarding readiness pr…
ksapru Apr 25, 2026
9507a85
refactor(cli): extract streamGatewayStart to reduce onboard.ts size
ksapru May 12, 2026
98b0b30
Merge current main into fix/replace-race-condition-sleeps
ericksoa May 13, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -535,7 +535,7 @@ $ nemoclaw my-assistant snapshot restore 2026-04-21T07-35-55-987Z
$ nemoclaw my-assistant snapshot restore v3 --to my-assistant-clone
```

### `openshell term`
## `openshell term`

Open the OpenShell TUI to monitor sandbox activity and approve network egress requests.
Run this on the host where the sandbox is running.
Expand Down
95 changes: 75 additions & 20 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ const {
const LOCAL_INFERENCE_PROVIDERS = ["ollama-local", "vllm-local"];
const {
sleepSeconds,
waitForPort,
waitForHttp,
waitUntil,
} = require("./wait");
const { inferContainerRuntime, isWsl, shouldPatchCoredns } = require("./platform");
const { resolveOpenshell } = require("./resolve-openshell");
Expand Down Expand Up @@ -2058,7 +2061,10 @@ function startOllamaAuthProxy(): boolean {
// If the user backs out to a different provider, the token stays in memory
// only and is discarded.
const pid = spawnOllamaAuthProxy(ollamaProxyToken);
sleep(1);
if (!waitForPort(OLLAMA_PROXY_PORT, 2)) {
console.error(` Error: Ollama auth proxy did not become ready on :${OLLAMA_PROXY_PORT} within timeout.`);
return false;
}
if (!isOllamaProxyProcess(pid)) {
console.error(` Error: Ollama auth proxy failed to start on :${OLLAMA_PROXY_PORT}`);
console.error(` Containers will not be able to reach Ollama without the proxy.`);
Expand Down Expand Up @@ -2089,7 +2095,11 @@ function ensureOllamaAuthProxy(): void {
killStaleProxy();
ollamaProxyToken = token;
spawnOllamaAuthProxy(token);
sleep(1);
if (!waitForPort(OLLAMA_PROXY_PORT, 2)) {
console.error(` Error: Ollama auth proxy did not become ready on :${OLLAMA_PROXY_PORT} within timeout.`);
// Void function, return early
return;
}
}

function getOllamaProxyToken(): string | null {
Expand Down Expand Up @@ -3328,8 +3338,30 @@ async function startGatewayWithOptions(_gpu, { exitOnFailure = true } = {}) {
run(["bash", path.join(SCRIPTS, "fix-coredns.sh"), GATEWAY_NAME], {
ignoreError: true,
});
const corednsReady = waitUntil(() => {
const check = runCaptureOpenshell(
[
"doctor",
"exec",
"--",
"kubectl",
"get",
"pods",
"-n",
"kube-system",
"-l",
"k8s-app=kube-dns",
"-o",
'jsonpath={range .items[*]}{.status.phase}{" "}{range .status.containerStatuses[*]}{.ready}{" "}{end}{end}',
],
{ ignoreError: true },
);
return check.includes("Running") && !check.includes("false") && check.includes("true");
}, 10);
if (!corednsReady) {
console.warn(" CoreDNS did not report ready within timeout; continuing may cause DNS flakiness.");
}
}
sleep(5);
runOpenshell(["gateway", "select", GATEWAY_NAME], { ignoreError: true });
process.env.OPENSHELL_GATEWAY = GATEWAY_NAME;
}
Expand Down Expand Up @@ -4853,7 +4885,11 @@ async function setupNim(gpu) {
// Shell required: backgrounding (&), env var prefix, output redirection.
const ollamaEnv = isWsl() ? "" : `OLLAMA_HOST=0.0.0.0:${OLLAMA_PORT} `;
run(`${ollamaEnv}ollama serve > /dev/null 2>&1 &`, { ignoreError: true });
sleep(2);
if (!waitForHttp(`http://127.0.0.1:${OLLAMA_PORT}/`, 10)) {
console.error(` Ollama did not become ready on :${OLLAMA_PORT} within timeout.`);
if (isNonInteractive()) process.exit(1);
continue selectionLoop;
}
if (!isWsl()) printOllamaExposureWarning();
}
if (isWsl()) {
Expand Down Expand Up @@ -4930,7 +4966,11 @@ async function setupNim(gpu) {
run(`OLLAMA_HOST=0.0.0.0:${OLLAMA_PORT} ollama serve > /dev/null 2>&1 &`, {
ignoreError: true,
});
sleep(2);
if (!waitForHttp(`http://127.0.0.1:${OLLAMA_PORT}/`, 10)) {
console.error(` Ollama did not become ready on :${OLLAMA_PORT} within timeout.`);
if (isNonInteractive()) process.exit(1);
continue selectionLoop;
}
if (!startOllamaAuthProxy()) {
process.exit(1);
}
Expand Down Expand Up @@ -5622,17 +5662,22 @@ async function _setupPolicies(sandboxName, options = {}) {
}
note(` [non-interactive] Applying policy presets: ${selectedPresets.join(", ")}`);
for (const name of selectedPresets) {
for (let attempt = 0; attempt < 3; attempt += 1) {
let lastErr: Error | null = null;
const success = waitUntil(() => {
try {
policies.applyPreset(sandboxName, name);
break;
return true;
} catch (err) {
const message = err && err.message ? err.message : String(err);
if (!message.includes("sandbox not found") || attempt === 2) {
lastErr = err as Error;
const message = err && (err as any).message ? (err as any).message : String(err);
if (!message.includes("sandbox not found")) {
throw err;
}
sleep(2);
return false;
}
}, 10, 2000);
if (!success && lastErr) {
throw lastErr;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
}
} else {
Expand Down Expand Up @@ -6289,25 +6334,31 @@ function syncPresetSelection(
const newlySelected = target.filter((name) => !appliedSet.has(name));

for (const name of deselected) {
for (let attempt = 0; attempt < 3; attempt += 1) {
let lastErr: Error | null = null;
const success = waitUntil(() => {
try {
if (!policies.removePreset(sandboxName, name)) {
throw new Error(`Failed to remove preset '${name}'.`);
}
break;
return true;
} catch (err) {
const message = err && err.message ? err.message : String(err);
if (!message.includes("sandbox not found") || attempt === 2) {
lastErr = err as Error;
const message = err && (err as any).message ? (err as any).message : String(err);
if (!message.includes("sandbox not found")) {
throw err;
}
sleep(2);
return false;
}
}, 10, 2000);
if (!success && lastErr) {
throw lastErr;
}
}

for (const name of newlySelected) {
const options = accessByName ? { access: accessByName[name] } : undefined;
for (let attempt = 0; attempt < 3; attempt += 1) {
let lastErr: Error | null = null;
const success = waitUntil(() => {
try {
// applyPreset returns false (without throwing) on some error paths —
// e.g. unknown preset, malformed YAML. Treat that as a failure so
Expand All @@ -6316,14 +6367,18 @@ function syncPresetSelection(
if (!policies.applyPreset(sandboxName, name, options)) {
throw new Error(`Failed to apply preset '${name}'.`);
}
break;
return true;
} catch (err) {
const message = err && err.message ? err.message : String(err);
if (!message.includes("sandbox not found") || attempt === 2) {
lastErr = err as Error;
const message = err && (err as any).message ? (err as any).message : String(err);
if (!message.includes("sandbox not found")) {
throw err;
}
sleep(2);
return false;
}
}, 10, 2000);
if (!success && lastErr) {
throw lastErr;
}
}
}
Expand Down
42 changes: 42 additions & 0 deletions src/lib/wait.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,45 @@ export function sleepMs(ms: number): void {
export function sleepSeconds(seconds: number): void {
sleepMs(seconds * 1000);
}

/**
* Synchronously wait until a condition is met.
*/
export function waitUntil(conditionFn: () => boolean, timeoutSeconds = 10, pollIntervalMs = 250): boolean {
const start = Date.now();
while (Date.now() - start < timeoutSeconds * 1000) {
if (conditionFn()) return true;
sleepMs(pollIntervalMs);
}
return false;
}

/**
* Synchronously wait for a TCP port to become reachable on localhost.
*/
export function waitForPort(port: number, timeoutSeconds = 5): boolean {
const { spawnSync } = require("child_process");
return waitUntil(() => {
try {
const result = spawnSync("nc", ["-z", "127.0.0.1", String(port)], { stdio: "ignore" });
return result.status === 0;
} catch {
return false;
}
}, timeoutSeconds, 200);
}

/**
* Synchronously wait for an HTTP endpoint to return a success status code.
*/
export function waitForHttp(url: string, timeoutSeconds = 5): boolean {
const { spawnSync } = require("child_process");
return waitUntil(() => {
try {
const result = spawnSync("curl", ["-sf", "--connect-timeout", "1", "--max-time", "1", url], { stdio: "ignore" });
return result.status === 0;
} catch {
return false;
}
}, timeoutSeconds, 200);
}
2 changes: 1 addition & 1 deletion test/onboard.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4135,7 +4135,7 @@ console.log(JSON.stringify({ exists: providerExistsInGateway("nonexistent") }));
assert.equal(payload.exists, false);
});

it("continues once the sandbox is Ready even if the create stream never closes", async () => {
it("continues once the sandbox is Ready even if the create stream never closes", { timeout: 10000 }, async () => {
const repoRoot = path.join(import.meta.dirname, "..");
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-create-ready-"));
const fakeBin = path.join(tmpDir, "bin");
Expand Down