Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 85 additions & 0 deletions .github/workflows/nightly-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ on:
Valid: cloud-e2e, cloud-onboard-e2e, cloud-inference-e2e,
skill-agent-e2e, docs-validation-e2e, messaging-providers-e2e,
openclaw-slack-pairing-e2e,
openclaw-tui-chat-correlation-e2e,
openclaw-discord-pairing-e2e,
messaging-compatible-endpoint-e2e,
kimi-inference-compat-e2e,
Expand Down Expand Up @@ -395,6 +396,90 @@ jobs:
path: /tmp/nemoclaw-e2e-openclaw-slack-pairing-install.log
if-no-files-found: ignore

# ── OpenClaw TUI Chat Correlation E2E (#2603/#3145) ───────────
# Creates a fresh OpenClaw sandbox, then runs the live gateway/webchat
# correlation harness against rapid sequential sends.
openclaw-tui-chat-correlation-e2e:
if: >-
github.repository == 'NVIDIA/NemoClaw' &&
(github.event_name != 'workflow_dispatch' ||
inputs.jobs == '' ||
contains(format(',{0},', inputs.jobs), ',openclaw-tui-chat-correlation-e2e,'))
runs-on: ubuntu-latest
timeout-minutes: 75
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ inputs.target_ref || github.ref }}

- name: Setup Node.js
uses: actions/setup-node@v6
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
with:
node-version: "22"

- name: Install test dependencies
run: npm ci --include=dev

- name: Run OpenClaw TUI chat correlation E2E test
env:
NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_RECREATE_SANDBOX: "1"
NEMOCLAW_SANDBOX_NAME: "e2e-openclaw-tui-correlation"
GITHUB_TOKEN: ${{ github.token }}
run: bash test/e2e/test-openclaw-tui-chat-correlation.sh

- name: Upload install log on failure
if: failure()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: install-log-openclaw-tui-chat-correlation
path: /tmp/nemoclaw-e2e-openclaw-tui-correlation-install.log
if-no-files-found: ignore

# ── DGX Station GPU optional proof validation (#3600) ──────────
# CI cannot emulate GB300, but this guards the release-blocker mitigation:
# optional direct GPU proofs must not abort onboard before the fatal throw.
issue-3600-gpu-proof-optional-e2e:
if: >-
github.repository == 'NVIDIA/NemoClaw' &&
(github.event_name != 'workflow_dispatch' ||
inputs.jobs == '' ||
contains(format(',{0},', inputs.jobs), ',issue-3600-gpu-proof-optional-e2e,'))
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ inputs.target_ref || github.ref }}

- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: "22"

- name: Verify optional GPU proof cannot abort onboard
run: |
node - <<'NODE'
const fs = require('node:fs');
const source = fs.readFileSync('src/lib/onboard.ts', 'utf8');
const start = source.indexOf('function verifyDirectSandboxGpu');
const end = source.indexOf('\n}\n\nfunction upsertMessagingProviders', start) + 2;
if (start < 0 || end < 2) {
throw new Error('verifyDirectSandboxGpu function not found');
}
const fn = source.slice(start, end);
const optional = fn.indexOf('if (proof.optional === true) return;');
const throwing = fn.indexOf('throw new Error(`GPU proof failed:');
if (!(optional >= 0 && throwing >= 0 && optional < throwing)) {
throw new Error('optional GPU proof handling does not precede fatal throw');
}
console.log('optional-before-throw=true');
NODE

# ── OpenClaw Discord Pairing E2E (#4061) ──────────────────────
# Hermetic Discord Gateway placeholder rewrite proof, then connect-shell
# `openclaw pairing approve discord <code>` against shared state.
Expand Down
20 changes: 19 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,9 @@ ENV NPM_CONFIG_AUDIT=false \
NPM_CONFIG_FETCH_TIMEOUT=300000
RUN npm ci --omit=dev
COPY scripts/patch-openclaw-tool-catalog.js /usr/local/lib/nemoclaw/patch-openclaw-tool-catalog.js
RUN chmod 755 /usr/local/lib/nemoclaw/patch-openclaw-tool-catalog.js
COPY scripts/patch-openclaw-chat-send.js /usr/local/lib/nemoclaw/patch-openclaw-chat-send.js
RUN chmod 755 /usr/local/lib/nemoclaw/patch-openclaw-tool-catalog.js \
/usr/local/lib/nemoclaw/patch-openclaw-chat-send.js

# Upgrade OpenClaw if the base image is stale.
#
Expand Down Expand Up @@ -290,6 +292,22 @@ RUN set -eu; \
if grep -REq --include='*.js' 'DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS = (1e4|15e3)' "$OC_DIST"; then echo "ERROR: Patch 5 left a short handshake-timeout constant" >&2; exit 1; fi; \
if ! grep -REq --include='*.js' 'DEFAULT_PREAUTH_HANDSHAKE_TIMEOUT_MS = 6e4' "$OC_DIST"; then echo "ERROR: Patch 5 did not find patched 6e4 constant" >&2; exit 1; fi

# Patch OpenClaw chat.send gateway behavior for OpenClaw 2026.5.x.
#
# OpenClaw can accept rapid TUI/WebChat chat.send requests and then emit a
# terminal chat event with state="final" but no assistant message for the later
# submitted run. That makes clients treat the turn as complete even though no
# visible reply was delivered. The shim also correlates real agent run IDs back
# to the submitted chat.send run ID when OpenClaw starts an internal run with a
# different ID, carries that submitted ID through queued follow-up turns, and
# adds the submitted run ID as the transcript idempotency key.
#
# Removal criteria: drop when upstream OpenClaw fixes openclaw/openclaw#70164
# and openclaw/openclaw#50298, or when NemoClaw no longer ships OpenClaw 2026.5.x.
# hadolint ignore=DL3059
RUN node /usr/local/lib/nemoclaw/patch-openclaw-chat-send.js \
/usr/local/lib/node_modules/openclaw/dist

# Patch OpenClaw's pinned 2026.5.18 compiled selection runtime to expose a
# compact searchable tool catalog to the model while preserving the full
# effective tool set behind tool_call. NEMOCLAW_TOOL_CATALOG=0 disables this
Expand Down
249 changes: 249 additions & 0 deletions scripts/patch-openclaw-chat-send.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,249 @@
#!/usr/bin/env node
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

/*
* Temporary NemoClaw compatibility shim for OpenClaw 2026.5.x chat.send
* gateway behavior. Remove this when upstream OpenClaw preserves submitted
* chat.send run lineage and stops emitting empty terminal chat events.
*/

const fs = require("node:fs");
const path = require("node:path");

const distDir = process.argv[2];
if (!distDir) {
console.error("Usage: patch-openclaw-chat-send.js <openclaw-dist-dir>");
process.exit(2);
}

function fail(message) {
console.error(`ERROR: ${message}`);
process.exit(1);
}

function listJsFiles(dir) {
return fs
.readdirSync(dir, { withFileTypes: true })
.filter((entry) => entry.isFile() && entry.name.endsWith(".js"))
.map((entry) => path.join(dir, entry.name));
}

function patchChatSendFile(file) {
let source = fs.readFileSync(file, "utf8");
const original = source;

if (!source.includes("nemoclaw: correlate chat.send run ids")) {
const next = source.replace(
/(onAgentRunStart: \(runId\) => \{\n)(\s*)agentRunStarted = true;/,
(_match, prefix, indent) =>
`${prefix}${indent}agentRunStarted = true;\n` +
`${indent}if (runId && runId !== clientRunId) context.addChatRun(runId, { sessionKey, clientRunId }); ` +
`// nemoclaw: correlate chat.send run ids (#2603, #3145)`,
);
if (next === source) {
fail(`OpenClaw chat.send run-start shape not recognized in ${file}`);
}
source = next;
}

if (!source.includes("idempotencyKey: clientRunId")) {
let inserted = false;
source = source.replace(
/(createIfMissing: true,\n)(\s*)(ttsSupplement: ttsSupplementMarker,)/g,
(match, prefix, indent, ttsLine, offset) => {
const preceding = source.slice(Math.max(0, offset - 300), offset);
if (preceding.includes("idempotencyKey:")) return match;
inserted = true;
return `${prefix}${indent}idempotencyKey: clientRunId,\n${indent}${ttsLine}`;
},
);
if (!inserted) {
fail(`OpenClaw chat.send transcript append shape not recognized in ${file}`);
}
}

if (!source.includes("suppressing empty final event")) {
const next = source.replace(
/\n(\s*)broadcastChatFinal\(\{\n(\s*)context,\n\s*runId: clientRunId,\n\s*sessionKey,\n\s*message\n\s*\}\);/,
(_match, outerIndent, innerIndent) =>
`\n${outerIndent}if (message) broadcastChatFinal({\n` +
`${innerIndent}context,\n` +
`${innerIndent}runId: clientRunId,\n` +
`${innerIndent}sessionKey,\n` +
`${innerIndent}message\n` +
`${outerIndent}}); else context.logGateway.warn("webchat chat.send completed without visible assistant reply; suppressing empty final event (nemoclaw #2603/#3145)");`,
);
if (next === source) {
fail(`OpenClaw chat.send empty-final shape not recognized in ${file}`);
}
source = next;
}

if (source !== original) {
fs.writeFileSync(file, source);
return true;
}
return false;
}

function patchFollowupRunnerFile(file) {
let source = fs.readFileSync(file, "utf8");
const original = source;

if (
source.includes(
"const runId = opts?.runId ?? crypto.randomUUID(); // nemoclaw: preserve chat.send run ids in followup queue",
)
) {
source = source.replace(
"const runId = opts?.runId ?? crypto.randomUUID(); // nemoclaw: preserve chat.send run ids in followup queue",
"const runId = queued.runId ?? opts?.runId ?? crypto.randomUUID(); // nemoclaw: preserve chat.send run ids in followup queue",
);
}

if (!source.includes("preserve chat.send run ids in followup queue")) {
const next = source.replace(
/(replyOperation = createReplyOperation\(\{\n\s*sessionId: run\.sessionId,\n\s*sessionKey: replySessionKey \?\? "",\n\s*resetTriggered: false,\n\s*upstreamAbortSignal: queued\.abortSignal \?\? opts\?\.abortSignal\n\s*\}\);\n\s*)const runId = crypto\.randomUUID\(\);/,
(_match, prefix) =>
`${prefix}const runId = queued.runId ?? opts?.runId ?? crypto.randomUUID(); ` +
`// nemoclaw: preserve chat.send run ids in followup queue (#2603, #3145)`,
);
if (next === source) {
fail(`OpenClaw followup runner run-id shape not recognized in ${file}`);
}
source = next;
}

if (source !== original) {
fs.writeFileSync(file, source);
return true;
}
return false;
}

function patchGetReplyFile(file) {
let source = fs.readFileSync(file, "utf8");
const original = source;

if (!source.includes("carry chat.send run id into queued followup")) {
const next = source.replace(
/(const followupRun = \{\n)(\s*)prompt: queuedBody,/,
(_match, prefix, indent) =>
`${prefix}${indent}runId: opts?.runId, ` +
`// nemoclaw: carry chat.send run id into queued followup (#2603, #3145)\n` +
`${indent}prompt: queuedBody,`,
);
if (next === source) {
fail(`OpenClaw get-reply followup run shape not recognized in ${file}`);
}
source = next;
}

if (!source.includes("force webchat chat.send queued turns")) {
if (source.includes("const resolvedQueue = useFastReplyRuntime ? {")) {
source = source.replace(
"const resolvedQueue = useFastReplyRuntime ? {",
"let resolvedQueue = useFastReplyRuntime ? {",
);
} else if (!source.includes("let resolvedQueue = useFastReplyRuntime ? {")) {
fail(`OpenClaw get-reply queue settings shape not recognized in ${file}`);
}

const next = source.replace(
/\n(\s*)const piRuntime = useFastReplyRuntime \? null : await traceRunPhase\("reply\.load_pi_runtime", \(\) => loadPiEmbeddedRuntime\(\)\);/,
(_match, indent) =>
`\n${indent}if (opts?.runId && sessionCtx.Provider === "webchat" && resolvedQueue.mode === "steer") resolvedQueue = {\n` +
`${indent}\t...resolvedQueue,\n` +
`${indent}\tmode: "followup",\n` +
`${indent}\tdebounceMs: 0\n` +
`${indent}}; // nemoclaw: force webchat chat.send queued turns to keep per-message replies (#2603, #3145)\n` +
`${indent}const piRuntime = useFastReplyRuntime ? null : await traceRunPhase("reply.load_pi_runtime", () => loadPiEmbeddedRuntime());`,
);
if (next === source) {
fail(`OpenClaw get-reply pi runtime shape not recognized in ${file}`);
}
source = next;
}

if (source !== original) {
fs.writeFileSync(file, source);
return true;
}
return false;
}

const candidates = listJsFiles(distDir).filter((file) => {
const source = fs.readFileSync(file, "utf8");
return source.includes('"chat.send"') && source.includes("onAgentRunStart");
});

if (candidates.length !== 1) {
fail(`expected exactly one OpenClaw chat.send runtime file, found ${candidates.length}`);
}

const chatFile = candidates[0];
patchChatSendFile(chatFile);

const getReplyCandidates = listJsFiles(distDir).filter((file) => {
const source = fs.readFileSync(file, "utf8");
return (
source.includes("resolveQueueSettings") &&
(source.includes("const followupRun = {") ||
source.includes("carry chat.send run id into queued followup"))
);
});

if (getReplyCandidates.length !== 1) {
fail(`expected exactly one OpenClaw get-reply runtime file, found ${getReplyCandidates.length}`);
}

const getReplyFile = getReplyCandidates[0];
patchGetReplyFile(getReplyFile);

const followupCandidates = listJsFiles(distDir).filter((file) => {
const source = fs.readFileSync(file, "utf8");
return (
source.includes("function createFollowupRunner") &&
source.includes("replyOperation = createReplyOperation") &&
(source.includes("const runId = crypto.randomUUID();") ||
source.includes("preserve chat.send run ids in followup queue"))
);
});

if (followupCandidates.length !== 1) {
fail(
`expected exactly one OpenClaw followup runner runtime file, found ${followupCandidates.length}`,
);
}

const followupFile = followupCandidates[0];
patchFollowupRunnerFile(followupFile);

const patched = fs.readFileSync(chatFile, "utf8");
if (!patched.includes("nemoclaw: correlate chat.send run ids")) {
fail("chat.send run-id correlation patch did not apply");
}
if (!patched.includes("idempotencyKey: clientRunId")) {
fail("chat.send transcript idempotency patch did not apply");
}
if (!patched.includes("suppressing empty final event")) {
fail("chat.send empty-final suppression patch did not apply");
}

const patchedGetReply = fs.readFileSync(getReplyFile, "utf8");
if (!patchedGetReply.includes("carry chat.send run id into queued followup")) {
fail("get-reply queued run-id patch did not apply");
}
if (!patchedGetReply.includes("force webchat chat.send queued turns")) {
fail("get-reply webchat queue mode patch did not apply");
}

const patchedFollowup = fs.readFileSync(followupFile, "utf8");
if (!patchedFollowup.includes("preserve chat.send run ids in followup queue")) {
fail("followup runner run-id patch did not apply");
}

console.log(
`INFO: patched OpenClaw chat.send compatibility in ${path.basename(chatFile)}, ${path.basename(getReplyFile)}, and ${path.basename(followupFile)}`,
);
4 changes: 4 additions & 0 deletions src/lib/sandbox/build-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,10 @@ function stageOptimizedSandboxBuildContext(
path.join(rootDir, "scripts", "patch-openclaw-tool-catalog.js"),
path.join(stagedScriptsDir, "patch-openclaw-tool-catalog.js"),
);
fs.copyFileSync(
path.join(rootDir, "scripts", "patch-openclaw-chat-send.js"),
path.join(stagedScriptsDir, "patch-openclaw-chat-send.js"),
);

return { buildCtx, stagedDockerfile };
}
Expand Down
Loading
Loading