From ffbee34d6c1d00fefa74c05bca6c1154c3bb9904 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 9 Jul 2026 10:47:18 +0700 Subject: [PATCH 1/2] fix(onboard): format local BuildKit stage output --- src/lib/onboard/sandbox-prebuild.test.ts | 60 +++++++++++++- src/lib/onboard/sandbox-prebuild.ts | 101 ++++++++++++++++++++++- 2 files changed, 155 insertions(+), 6 deletions(-) diff --git a/src/lib/onboard/sandbox-prebuild.test.ts b/src/lib/onboard/sandbox-prebuild.test.ts index 74aaa63f404..75b79363ae6 100644 --- a/src/lib/onboard/sandbox-prebuild.test.ts +++ b/src/lib/onboard/sandbox-prebuild.test.ts @@ -313,6 +313,8 @@ describe("sandbox BuildKit prebuild", () => { it("uses the argv-based Docker helper and returns the local image on success", async () => { const { buildCtx, createArgs, dockerfile } = createBuildContext(); const buildImage = vi.fn(async () => 0); + const resolvedBuildCtx = fs.realpathSync(buildCtx); + const resolvedDockerfile = fs.realpathSync(dockerfile); const result = await prebuildSandboxImageIfEligible({ buildCtx, buildId: BUILD_ID, @@ -332,12 +334,13 @@ describe("sandbox BuildKit prebuild", () => { "-t", "nemoclaw-sandbox-local:alpha-1234567890", "-f", - dockerfile, - buildCtx, + resolvedDockerfile, + resolvedBuildCtx, ], expect.objectContaining({ env: expect.objectContaining({ DOCKER_BUILDKIT: "1" }), - stdio: "inherit", + onOutput: expect.any(Function), + stdio: "pipe", }), ); expect(result).toEqual({ @@ -346,6 +349,57 @@ describe("sandbox BuildKit prebuild", () => { }); }); + it("formats local BuildKit output as plain stage lines", async () => { + const { buildCtx, createArgs } = createBuildContext(); + const buildImage = vi.fn(async (_args, options) => { + options.onOutput( + [ + '#0 building with "default" instance using docker driver', + "#1 [internal] load build definition from Dockerfile", + "#1 transferring dockerfile: 95.23kB done", + '#1 WARN: SecretsUsedInArgOrEnv: Do not use ARG or ENV instructions for sensitive data (ARG "NEMOCLAW_PROVIDER_KEY") (line 819)', + "#1 DONE 0.0s", + "#2 [stage-2 1/61] FROM ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:abc123", + "#2 CACHED", + "#67 [stage-2 52/61] RUN openclaw doctor --fix --non-interactive", + "#67 0.942 OPENCLAW", + "#67 DONE 5.8s", + "", + ].join("\n"), + ); + return 0; + }); + const log = vi.fn(); + + await expect( + prebuildSandboxImageIfEligible({ + buildCtx, + buildId: BUILD_ID, + origin: "generated", + createArgs, + sandboxName: "alpha", + dockerDriverGateway: true, + env: {}, + buildImage, + log, + }), + ).resolves.toMatchObject({ + imageRef: "nemoclaw-sandbox-local:alpha-1234567890", + }); + + const lines = log.mock.calls.map((call) => call[0] as string); + expect(lines).toContain(" [internal] load build definition from Dockerfile"); + expect(lines.some((line) => line.includes("WARN: SecretsUsedInArgOrEnv"))).toBe(true); + expect(lines).toContain( + " [stage-2 1/61] FROM ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:abc123", + ); + expect(lines).toContain(" [stage-2 52/61] RUN openclaw doctor --fix --non-interactive"); + expect(lines.some((line) => line.includes("#0 building"))).toBe(false); + expect(lines.some((line) => line.includes("CACHED"))).toBe(false); + expect(lines.some((line) => line.includes("DONE"))).toBe(false); + expect(lines.some((line) => line.includes("OPENCLAW"))).toBe(false); + }); + it.each([ ["nonzero result", async () => 1], ["missing exit status", async () => null], diff --git a/src/lib/onboard/sandbox-prebuild.ts b/src/lib/onboard/sandbox-prebuild.ts index 0fb2c6928c9..bb31a0d069a 100644 --- a/src/lib/onboard/sandbox-prebuild.ts +++ b/src/lib/onboard/sandbox-prebuild.ts @@ -23,6 +23,12 @@ const DOCKER_ENV_NAMES = [ "DOCKER_CONTEXT", "DOCKER_TLS_VERIFY", ] as const; +const BUILDKIT_STEP_RE = /^#(?\d+)\s+(?\[[^\]]+\]\s+.+)$/; +const BUILDKIT_STATUS_RE = /^#(?\d+)\s+(?DONE|CACHED)\b/; +const BUILDKIT_STEP_OUTPUT_RE = /^#(?\d+)\s+[\d.]+\s+/; +const BUILDKIT_WARNING_RE = /^#\d+\s+(?WARN:\s+.+)$/; +const BUILDKIT_ERROR_RE = /^#(?\d+)\s+(?ERROR:\s+.+)$/; +const MAX_FAILURE_OUTPUT_LINES = 80; export interface SandboxPrebuildInput { buildCtx: string; @@ -34,7 +40,11 @@ export interface SandboxPrebuildInput { env?: NodeJS.ProcessEnv; buildImage?: ( args: readonly string[], - options: { env: NodeJS.ProcessEnv; stdio: "inherit" }, + options: { + env: NodeJS.ProcessEnv; + onOutput: (chunk: Buffer | string) => void; + stdio: "pipe"; + }, ) => Promise; log?: (message: string) => void; } @@ -125,6 +135,77 @@ export function resolveSandboxPrebuildEnabled( return !env.VITEST && env.NODE_ENV !== "test"; } +function trimDisplayLine(line: string): string { + const width = Math.max(60, Number(process.stdout.columns || 120)); + const maxLen = Math.max(40, width - 4); + if (line.length <= maxLen) return line; + return `${line.slice(0, Math.max(0, maxLen - 3))}...`; +} + +function createBuildKitStageLogger(log: (message: string) => void) { + const rawLines: string[] = []; + const printed = new Set(); + let pending = ""; + let lastPrinted = ""; + + function print(line: string): void { + const display = trimDisplayLine(` ${line}`); + if (display === lastPrinted) return; + log(display); + lastPrinted = display; + } + + function recordLine(rawLine: string): void { + const line = rawLine.replace(/\r/g, "").trimEnd(); + if (!line) return; + rawLines.push(line); + + const step = line.match(BUILDKIT_STEP_RE); + if (step?.groups) { + const key = `${step.groups.id}:${step.groups.detail}`; + if (!printed.has(key)) { + printed.add(key); + print(step.groups.detail); + } + return; + } + + const warning = line.match(BUILDKIT_WARNING_RE); + if (warning?.groups) { + print(warning.groups.warning); + return; + } + + const error = line.match(BUILDKIT_ERROR_RE); + if (error?.groups) { + print(error.groups.error); + return; + } + + if (BUILDKIT_STATUS_RE.test(line) || BUILDKIT_STEP_OUTPUT_RE.test(line)) { + return; + } + } + + return { + onOutput(chunk: Buffer | string): void { + pending += chunk.toString(); + const lines = pending.split("\n"); + pending = lines.pop() ?? ""; + for (const line of lines) recordLine(line); + }, + flush(): void { + if (!pending) return; + const line = pending; + pending = ""; + recordLine(line); + }, + failureTail(): string[] { + return rawLines.slice(-MAX_FAILURE_OUTPUT_LINES); + }, + }; +} + export function sandboxLocalImageRef(sandboxName: string, buildId: string): string { const sanitize = (value: string) => value @@ -189,13 +270,17 @@ export async function prebuildSandboxImageIfEligible( input.buildImage ?? ((args, options) => new Promise((resolve, reject) => { - const child = dockerSpawn(args, { ...options, shell: false }); + const { onOutput, ...spawnOptions } = options; + const child = dockerSpawn(args, { ...spawnOptions, shell: false }); + child.stdout?.on("data", onOutput); + child.stderr?.on("data", onOutput); child.once("error", reject); child.once("close", resolve); })); log(" Building sandbox image with BuildKit (skips the slower in-gateway builder)..."); let status: number | null; + const buildOutput = createBuildKitStageLogger(log); try { status = await buildImage( [ @@ -209,10 +294,13 @@ export async function prebuildSandboxImageIfEligible( ], { env: { ...dockerBuildSubprocessEnv(), DOCKER_BUILDKIT: "1" }, - stdio: "inherit", + onOutput: buildOutput.onOutput, + stdio: "pipe", }, ); + buildOutput.flush(); } catch (error) { + buildOutput.flush(); const detail = error instanceof Error ? error.message : String(error); log(` Local BuildKit build could not start (${detail}); using the gateway builder instead.`); return { createArgs, imageRef: null }; @@ -221,6 +309,13 @@ export async function prebuildSandboxImageIfEligible( if (status !== 0) { const detail = status === null ? " without an exit status" : ` (exit ${status})`; log(` Local BuildKit build failed${detail}; using the gateway builder instead.`); + const tail = buildOutput.failureTail(); + if (tail.length > 0) { + log(" Local BuildKit output (last lines):"); + for (const line of tail) { + log(trimDisplayLine(` ${line}`)); + } + } return { createArgs, imageRef: null }; } From 798988eb0e98dde7ce7dbbd7dd393e417229c1cb Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 9 Jul 2026 12:27:40 +0700 Subject: [PATCH 2/2] fix(onboard): avoid parsing BuildKit progress --- src/lib/onboard/sandbox-prebuild.test.ts | 55 +---------- src/lib/onboard/sandbox-prebuild.ts | 111 +---------------------- 2 files changed, 5 insertions(+), 161 deletions(-) diff --git a/src/lib/onboard/sandbox-prebuild.test.ts b/src/lib/onboard/sandbox-prebuild.test.ts index 75b79363ae6..96cf387f537 100644 --- a/src/lib/onboard/sandbox-prebuild.test.ts +++ b/src/lib/onboard/sandbox-prebuild.test.ts @@ -330,7 +330,6 @@ describe("sandbox BuildKit prebuild", () => { expect(buildImage).toHaveBeenCalledWith( [ "build", - "--progress=plain", "-t", "nemoclaw-sandbox-local:alpha-1234567890", "-f", @@ -339,8 +338,7 @@ describe("sandbox BuildKit prebuild", () => { ], expect.objectContaining({ env: expect.objectContaining({ DOCKER_BUILDKIT: "1" }), - onOutput: expect.any(Function), - stdio: "pipe", + stdio: "inherit", }), ); expect(result).toEqual({ @@ -349,57 +347,6 @@ describe("sandbox BuildKit prebuild", () => { }); }); - it("formats local BuildKit output as plain stage lines", async () => { - const { buildCtx, createArgs } = createBuildContext(); - const buildImage = vi.fn(async (_args, options) => { - options.onOutput( - [ - '#0 building with "default" instance using docker driver', - "#1 [internal] load build definition from Dockerfile", - "#1 transferring dockerfile: 95.23kB done", - '#1 WARN: SecretsUsedInArgOrEnv: Do not use ARG or ENV instructions for sensitive data (ARG "NEMOCLAW_PROVIDER_KEY") (line 819)', - "#1 DONE 0.0s", - "#2 [stage-2 1/61] FROM ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:abc123", - "#2 CACHED", - "#67 [stage-2 52/61] RUN openclaw doctor --fix --non-interactive", - "#67 0.942 OPENCLAW", - "#67 DONE 5.8s", - "", - ].join("\n"), - ); - return 0; - }); - const log = vi.fn(); - - await expect( - prebuildSandboxImageIfEligible({ - buildCtx, - buildId: BUILD_ID, - origin: "generated", - createArgs, - sandboxName: "alpha", - dockerDriverGateway: true, - env: {}, - buildImage, - log, - }), - ).resolves.toMatchObject({ - imageRef: "nemoclaw-sandbox-local:alpha-1234567890", - }); - - const lines = log.mock.calls.map((call) => call[0] as string); - expect(lines).toContain(" [internal] load build definition from Dockerfile"); - expect(lines.some((line) => line.includes("WARN: SecretsUsedInArgOrEnv"))).toBe(true); - expect(lines).toContain( - " [stage-2 1/61] FROM ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:abc123", - ); - expect(lines).toContain(" [stage-2 52/61] RUN openclaw doctor --fix --non-interactive"); - expect(lines.some((line) => line.includes("#0 building"))).toBe(false); - expect(lines.some((line) => line.includes("CACHED"))).toBe(false); - expect(lines.some((line) => line.includes("DONE"))).toBe(false); - expect(lines.some((line) => line.includes("OPENCLAW"))).toBe(false); - }); - it.each([ ["nonzero result", async () => 1], ["missing exit status", async () => null], diff --git a/src/lib/onboard/sandbox-prebuild.ts b/src/lib/onboard/sandbox-prebuild.ts index bb31a0d069a..44dac578c63 100644 --- a/src/lib/onboard/sandbox-prebuild.ts +++ b/src/lib/onboard/sandbox-prebuild.ts @@ -23,12 +23,6 @@ const DOCKER_ENV_NAMES = [ "DOCKER_CONTEXT", "DOCKER_TLS_VERIFY", ] as const; -const BUILDKIT_STEP_RE = /^#(?\d+)\s+(?\[[^\]]+\]\s+.+)$/; -const BUILDKIT_STATUS_RE = /^#(?\d+)\s+(?DONE|CACHED)\b/; -const BUILDKIT_STEP_OUTPUT_RE = /^#(?\d+)\s+[\d.]+\s+/; -const BUILDKIT_WARNING_RE = /^#\d+\s+(?WARN:\s+.+)$/; -const BUILDKIT_ERROR_RE = /^#(?\d+)\s+(?ERROR:\s+.+)$/; -const MAX_FAILURE_OUTPUT_LINES = 80; export interface SandboxPrebuildInput { buildCtx: string; @@ -40,11 +34,7 @@ export interface SandboxPrebuildInput { env?: NodeJS.ProcessEnv; buildImage?: ( args: readonly string[], - options: { - env: NodeJS.ProcessEnv; - onOutput: (chunk: Buffer | string) => void; - stdio: "pipe"; - }, + options: { env: NodeJS.ProcessEnv; stdio: "inherit" }, ) => Promise; log?: (message: string) => void; } @@ -135,77 +125,6 @@ export function resolveSandboxPrebuildEnabled( return !env.VITEST && env.NODE_ENV !== "test"; } -function trimDisplayLine(line: string): string { - const width = Math.max(60, Number(process.stdout.columns || 120)); - const maxLen = Math.max(40, width - 4); - if (line.length <= maxLen) return line; - return `${line.slice(0, Math.max(0, maxLen - 3))}...`; -} - -function createBuildKitStageLogger(log: (message: string) => void) { - const rawLines: string[] = []; - const printed = new Set(); - let pending = ""; - let lastPrinted = ""; - - function print(line: string): void { - const display = trimDisplayLine(` ${line}`); - if (display === lastPrinted) return; - log(display); - lastPrinted = display; - } - - function recordLine(rawLine: string): void { - const line = rawLine.replace(/\r/g, "").trimEnd(); - if (!line) return; - rawLines.push(line); - - const step = line.match(BUILDKIT_STEP_RE); - if (step?.groups) { - const key = `${step.groups.id}:${step.groups.detail}`; - if (!printed.has(key)) { - printed.add(key); - print(step.groups.detail); - } - return; - } - - const warning = line.match(BUILDKIT_WARNING_RE); - if (warning?.groups) { - print(warning.groups.warning); - return; - } - - const error = line.match(BUILDKIT_ERROR_RE); - if (error?.groups) { - print(error.groups.error); - return; - } - - if (BUILDKIT_STATUS_RE.test(line) || BUILDKIT_STEP_OUTPUT_RE.test(line)) { - return; - } - } - - return { - onOutput(chunk: Buffer | string): void { - pending += chunk.toString(); - const lines = pending.split("\n"); - pending = lines.pop() ?? ""; - for (const line of lines) recordLine(line); - }, - flush(): void { - if (!pending) return; - const line = pending; - pending = ""; - recordLine(line); - }, - failureTail(): string[] { - return rawLines.slice(-MAX_FAILURE_OUTPUT_LINES); - }, - }; -} - export function sandboxLocalImageRef(sandboxName: string, buildId: string): string { const sanitize = (value: string) => value @@ -270,37 +189,22 @@ export async function prebuildSandboxImageIfEligible( input.buildImage ?? ((args, options) => new Promise((resolve, reject) => { - const { onOutput, ...spawnOptions } = options; - const child = dockerSpawn(args, { ...spawnOptions, shell: false }); - child.stdout?.on("data", onOutput); - child.stderr?.on("data", onOutput); + const child = dockerSpawn(args, { ...options, shell: false }); child.once("error", reject); child.once("close", resolve); })); log(" Building sandbox image with BuildKit (skips the slower in-gateway builder)..."); let status: number | null; - const buildOutput = createBuildKitStageLogger(log); try { status = await buildImage( - [ - "build", - "--progress=plain", - "-t", - imageRef, - "-f", - trustedContext.dockerfile, - trustedContext.buildCtx, - ], + ["build", "-t", imageRef, "-f", trustedContext.dockerfile, trustedContext.buildCtx], { env: { ...dockerBuildSubprocessEnv(), DOCKER_BUILDKIT: "1" }, - onOutput: buildOutput.onOutput, - stdio: "pipe", + stdio: "inherit", }, ); - buildOutput.flush(); } catch (error) { - buildOutput.flush(); const detail = error instanceof Error ? error.message : String(error); log(` Local BuildKit build could not start (${detail}); using the gateway builder instead.`); return { createArgs, imageRef: null }; @@ -309,13 +213,6 @@ export async function prebuildSandboxImageIfEligible( if (status !== 0) { const detail = status === null ? " without an exit status" : ` (exit ${status})`; log(` Local BuildKit build failed${detail}; using the gateway builder instead.`); - const tail = buildOutput.failureTail(); - if (tail.length > 0) { - log(" Local BuildKit output (last lines):"); - for (const line of tail) { - log(trimDisplayLine(` ${line}`)); - } - } return { createArgs, imageRef: null }; }