diff --git a/src/lib/actions/inference-set-gateway-restart.ts b/src/lib/actions/inference-set-gateway-restart.ts index 216e253a55e..3316079f0dc 100644 --- a/src/lib/actions/inference-set-gateway-restart.ts +++ b/src/lib/actions/inference-set-gateway-restart.ts @@ -6,12 +6,113 @@ import type { ConfigObject } from "../security/credential-filter"; import type { ShieldsAuditEntry } from "../shields/audit"; import { type InferenceApi, readOpenClawPrimaryRouteApi } from "./inference-route-api"; import { InferenceSetError } from "./inference-set-error"; +import { + runPortableOpenClawPairingApproval, + runPortableOpenClawPairingRequestProducer, + type PortableOpenClawPairingApprovalReceipt, +} from "./sandbox/auto-pair-approval"; import type { GatewayRestartResult } from "./sandbox/gateway-restart"; +import { + observeOpenClawPairingSettlement, + type OpenClawPairingSettlementObservation, +} from "./sandbox/launch-readiness/openclaw-pairing-qualification"; + +export type InferenceSetOpenClawPairingTarget = { + readonly sandboxName: string; + readonly gatewayName: string; + readonly openclawVersion: string; + readonly stateDirectory: string; +}; + +export type InferenceSetOpenClawPairingFailureLayer = + | "initial-state-unavailable" + | "final-state-unavailable" + | "final-state-unsettled" + | "pairing-operation-failed" + | "pairing-target-unavailable" + | `approval-${Exclude}`; + +export type InferenceSetOpenClawPairingResult = + | { readonly ok: true } + | { readonly ok: false; readonly failureLayer: InferenceSetOpenClawPairingFailureLayer }; + +export type InferenceSetOpenClawPairingDeps = { + observePairing: ( + target: InferenceSetOpenClawPairingTarget, + ) => OpenClawPairingSettlementObservation; + publishScopeRequest: (target: InferenceSetOpenClawPairingTarget) => void; + approveScopeRequest: ( + target: InferenceSetOpenClawPairingTarget, + deviceIdentitySha256: string, + ) => PortableOpenClawPairingApprovalReceipt; +}; + +const defaultOpenClawPairingDeps: InferenceSetOpenClawPairingDeps = { + observePairing: (target) => + observeOpenClawPairingSettlement( + target.sandboxName, + target.gatewayName, + target.openclawVersion, + target.stateDirectory, + ), + publishScopeRequest: (target) => + runPortableOpenClawPairingRequestProducer(target.sandboxName, target.gatewayName), + approveScopeRequest: (target, deviceIdentitySha256) => + runPortableOpenClawPairingApproval( + target.sandboxName, + target.gatewayName, + deviceIdentitySha256, + ), +}; + +/** + * Reconcile the local OpenClaw CLI device after an inference route change. + * + * The state observer accepts only the exact operator pairing/read/write projection. + * The request producer runs only for a pairing-only device. The approval helper then + * binds one allowlisted request to the observed device identity. A final state read, + * not command output, decides whether the inference switch can report success. + */ +export function settleInferenceSetOpenClawPairing( + target: InferenceSetOpenClawPairingTarget, + deps: InferenceSetOpenClawPairingDeps = defaultOpenClawPairingDeps, +): InferenceSetOpenClawPairingResult { + let initial: OpenClawPairingSettlementObservation; + try { + initial = deps.observePairing(target); + } catch { + return { ok: false, failureLayer: "initial-state-unavailable" }; + } + if (initial.state === "settled") return { ok: true }; + + let approval: PortableOpenClawPairingApprovalReceipt; + try { + deps.publishScopeRequest(target); + approval = deps.approveScopeRequest(target, initial.deviceIdentitySha256); + } catch { + return { ok: false, failureLayer: "pairing-operation-failed" }; + } + + let final: OpenClawPairingSettlementObservation; + try { + final = deps.observePairing(target); + } catch { + return { ok: false, failureLayer: "final-state-unavailable" }; + } + if (final.state === "settled") return { ok: true }; + return { + ok: false, + failureLayer: approval === "approved" ? "final-state-unsettled" : `approval-${approval}`, + }; +} export interface InferenceGatewayRestartDeps { appendAuditEntry: (entry: ShieldsAuditEntry) => void; log: (message: string) => void; restartSandboxGateway: (sandboxName: string) => GatewayRestartResult; + settleOpenClawPairing: ( + target: InferenceSetOpenClawPairingTarget, + ) => InferenceSetOpenClawPairingResult; } interface InferenceResultForGateway { @@ -33,17 +134,25 @@ interface InferenceResultForGateway { export interface InferenceMutation { result: T; openClawGatewayRestartRequired: boolean; + openClawPairing: + | { readonly state: "not-required" } + | { readonly state: "required"; readonly target: InferenceSetOpenClawPairingTarget } + | { readonly state: "target-unavailable" }; } -// SOURCE_OF_TRUTH_REVIEW (cross-family OpenClaw restart; gateway regression -// #4504, OpenClaw 2026.6.10 adopted in #5595): that version hot-reloads model +// SOURCE_OF_TRUTH_REVIEW (OpenClaw post-switch convergence; gateway regressions +// #4504 and #9527): OpenClaw 2026.6.10 adopted in #5595 hot-reloads model // identity but retains request shaping when the API family changes. NemoClaw -// therefore restarts only after the route, config, and integrity hash commit, -// and outside the config transition lock. Unit coverage proves restart, -// no-restart, redaction, audit-failure, and post-commit recovery behavior; -// openclaw-inference-switch live coverage proves gateway health and forwarding. -// Remove this coordination when the minimum supported OpenClaw hot-reloads -// request shaping across API-family changes, keeping the tests until then. +// restarts only after the route, config, and integrity hash commit. Every +// changed OpenClaw route then requires exact local device-scope convergence +// before the command reports success. Both operations run outside the config +// transition lock and inside the sandbox lifecycle lock. Unit coverage proves +// restart, no-restart, scope convergence, redaction, audit-failure, and +// post-commit recovery behavior. The openclaw-inference-switch live target +// proves gateway health and forwarding. Remove the restart when the minimum +// supported OpenClaw hot-reloads request shaping across API-family changes. +// Remove pairing settlement when OpenClaw no longer requires a separate +// allowlisted device-scope upgrade after a route change. export function defaultInferenceGatewayRestart(sandboxName: string): GatewayRestartResult { const recovery: typeof import("./sandbox/process-recovery") = require("./sandbox/process-recovery"); @@ -78,18 +187,21 @@ export function finalizeInferenceMutation( agentName: string; configChanged: boolean; nextApi: string; + openClawPairingTarget?: InferenceSetOpenClawPairingTarget; previousApi: InferenceApi | null; result: T; }, deps: Pick, ): InferenceMutation { - const { agentName, configChanged, nextApi, previousApi, result } = options; + const { agentName, configChanged, nextApi, openClawPairingTarget, previousApi, result } = options; const openClawGatewayRestartRequired = agentName === "openclaw" && configChanged && result.inSandboxConfigSynced && previousApi !== null && previousApi !== nextApi; + const openClawPairingConvergenceRequired = + agentName === "openclaw" && configChanged && result.inSandboxConfigSynced; const auditEntry: ShieldsAuditEntry = { action: "inference_set", @@ -99,11 +211,13 @@ export function finalizeInferenceMutation( !result.inSandboxConfigSynced ? " (in-sandbox sync incomplete)" : openClawGatewayRestartRequired - ? " (gateway restart pending)" - : "" + ? " (gateway restart and pairing convergence pending)" + : openClawPairingConvergenceRequired + ? " (pairing convergence pending)" + : "" }`, }; - if (openClawGatewayRestartRequired) { + if (openClawGatewayRestartRequired || openClawPairingConvergenceRequired) { appendPostCommitInferenceAudit(deps, auditEntry); } else { deps.appendAuditEntry(auditEntry); @@ -112,7 +226,12 @@ export function finalizeInferenceMutation( // A Hermes switch whose Web Dashboard profile did not converge is not fully // applied, so withhold the success line (the caller already warned) (#6893). const hermesDashboardStale = agentName === "hermes" && result.dashboardConverged === false; - if (result.inSandboxConfigSynced && !openClawGatewayRestartRequired && !hermesDashboardStale) { + if ( + result.inSandboxConfigSynced && + !openClawGatewayRestartRequired && + !openClawPairingConvergenceRequired && + !hermesDashboardStale + ) { deps.log( agentName === "hermes" ? ` Inference route synced for '${result.sandboxName}': ${result.model}` @@ -120,43 +239,79 @@ export function finalizeInferenceMutation( ); } - return { result, openClawGatewayRestartRequired }; + return { + result, + openClawGatewayRestartRequired, + openClawPairing: !openClawPairingConvergenceRequired + ? { state: "not-required" } + : openClawPairingTarget + ? { state: "required", target: openClawPairingTarget } + : { state: "target-unavailable" }, + }; } -export function completeInferenceGatewayRestart( +export function completeInferencePostCommit( mutation: InferenceMutation, deps: InferenceGatewayRestartDeps, ): void { - if (!mutation.openClawGatewayRestartRequired) return; - const { result } = mutation; - deps.log( - ` Restarting the OpenClaw gateway in '${result.sandboxName}' to apply the new inference API family...`, - ); - let restartFailure: string | null = null; - try { - const restart = deps.restartSandboxGateway(result.sandboxName); - if (!restart.ok) restartFailure = restart.failureLayer; - } catch { - restartFailure = "restart exception"; + if (mutation.openClawGatewayRestartRequired) { + deps.log( + ` Restarting the OpenClaw gateway in '${result.sandboxName}' to apply the new inference API family...`, + ); + let restartFailure: string | null = null; + try { + const restart = deps.restartSandboxGateway(result.sandboxName); + if (!restart.ok) restartFailure = restart.failureLayer; + } catch { + restartFailure = "restart exception"; + } + if (restartFailure) { + appendPostCommitInferenceAudit(deps, { + action: "inference_set", + sandbox: result.sandboxName, + timestamp: new Date().toISOString(), + reason: `inference set openclaw:${result.provider}:${result.model} (config committed; gateway restart failed: ${restartFailure})`, + }); + throw new InferenceSetError( + `Inference route and config were updated for '${result.sandboxName}', but the managed OpenClaw gateway restart/recovery did not complete successfully. ` + + `The committed route was not rolled back. Retry with '${CLI_NAME} ${result.sandboxName} gateway restart'.`, + ); + } + } + const pairingMutation = mutation.openClawPairing; + if (pairingMutation.state === "not-required") return; + let pairing: InferenceSetOpenClawPairingResult; + if (pairingMutation.state === "target-unavailable") { + pairing = { ok: false, failureLayer: "pairing-target-unavailable" }; + } else { + try { + pairing = deps.settleOpenClawPairing(pairingMutation.target); + } catch { + pairing = { ok: false, failureLayer: "pairing-operation-failed" }; + } } - if (restartFailure) { + if (!pairing.ok) { appendPostCommitInferenceAudit(deps, { action: "inference_set", sandbox: result.sandboxName, timestamp: new Date().toISOString(), - reason: `inference set openclaw:${result.provider}:${result.model} (config committed; gateway restart failed: ${restartFailure})`, + reason: `inference set openclaw:${result.provider}:${result.model} (config committed; ${ + mutation.openClawGatewayRestartRequired ? "gateway restart completed; " : "" + }pairing convergence failed: ${pairing.failureLayer})`, }); throw new InferenceSetError( - `Inference route and config were updated for '${result.sandboxName}', but the managed OpenClaw gateway restart/recovery did not complete successfully. ` + - `The committed route was not rolled back. Retry with '${CLI_NAME} ${result.sandboxName} gateway restart'.`, + `Inference route and config were updated for '${result.sandboxName}', but OpenClaw gateway pairing did not converge (${pairing.failureLayer}). ` + + `The committed route was not rolled back. Run '${CLI_NAME} ${result.sandboxName} doctor --fix', then retry the agent turn.`, ); } appendPostCommitInferenceAudit(deps, { action: "inference_set", sandbox: result.sandboxName, timestamp: new Date().toISOString(), - reason: `inference set openclaw:${result.provider}:${result.model} (gateway restart completed)`, + reason: `inference set openclaw:${result.provider}:${result.model} (${ + mutation.openClawGatewayRestartRequired ? "gateway restart and " : "" + }pairing convergence completed)`, }); deps.log(` Inference route synced for '${result.sandboxName}': ${result.primaryModelRef}`); } diff --git a/src/lib/actions/inference-set-openclaw-gateway-restart.test.ts b/src/lib/actions/inference-set-openclaw-gateway-restart.test.ts index 7090f1d9be3..12d5a5abe0b 100644 --- a/src/lib/actions/inference-set-openclaw-gateway-restart.test.ts +++ b/src/lib/actions/inference-set-openclaw-gateway-restart.test.ts @@ -92,14 +92,20 @@ describe("runInferenceSet OpenClaw gateway restart", () => { }); expect(deps.calls.restartSandboxGateway).toHaveBeenCalledOnce(); expect(deps.calls.restartSandboxGateway).toHaveBeenCalledWith("alpha"); + expect(deps.calls.settleOpenClawPairing).toHaveBeenCalledWith({ + sandboxName: "alpha", + gatewayName: "nemoclaw", + openclawVersion: "", + stateDirectory: "/sandbox/.openclaw", + }); const auditReasons = deps.calls.appendAuditEntry.mock.calls.map(([entry]) => String(entry.reason), ); expect(auditReasons).toContain( - "inference set openclaw:compatible-anthropic-endpoint:claude-sonnet-proxy (gateway restart pending)", + "inference set openclaw:compatible-anthropic-endpoint:claude-sonnet-proxy (gateway restart and pairing convergence pending)", ); expect(auditReasons).toContain( - "inference set openclaw:compatible-anthropic-endpoint:claude-sonnet-proxy (gateway restart completed)", + "inference set openclaw:compatible-anthropic-endpoint:claude-sonnet-proxy (gateway restart and pairing convergence completed)", ); expect(deps.calls.log).toHaveBeenCalledWith( " Inference route synced for 'alpha': anthropic/claude-sonnet-proxy", @@ -108,10 +114,12 @@ describe("runInferenceSet OpenClaw gateway restart", () => { " Warning: could not record the post-commit inference audit entry for 'alpha'.", ); const restartOrder = deps.calls.restartSandboxGateway.mock.invocationCallOrder[0] ?? 0; + const pairingOrder = deps.calls.settleOpenClawPairing.mock.invocationCallOrder[0] ?? 0; expect(deps.calls.writeSandboxConfig.mock.invocationCallOrder[0]).toBeLessThan(restartOrder); expect(deps.calls.recomputeSandboxConfigHash.mock.invocationCallOrder[0]).toBeLessThan( restartOrder, ); + expect(restartOrder).toBeLessThan(pairingOrder); }); it("does not restart OpenClaw when the requested route is already current (#4504)", async () => { @@ -146,6 +154,7 @@ describe("runInferenceSet OpenClaw gateway restart", () => { expect(result.configChanged).toBe(false); expect(result.inSandboxConfigSynced).toBe(true); expect(deps.calls.restartSandboxGateway).not.toHaveBeenCalled(); + expect(deps.calls.settleOpenClawPairing).not.toHaveBeenCalled(); }); it("reports a post-commit restart failure without rolling state back (#4504)", async () => { @@ -191,6 +200,7 @@ describe("runInferenceSet OpenClaw gateway restart", () => { ); expect(deps.calls.restartSandboxGateway).toHaveBeenCalledWith("alpha"); + expect(deps.calls.settleOpenClawPairing).not.toHaveBeenCalled(); expect(deps.calls.writeSandboxConfig).toHaveBeenCalledOnce(); expect(deps.calls.recomputeSandboxConfigHash).toHaveBeenCalledOnce(); expect(deps.calls.updateSandbox.mock.calls.at(-1)).toEqual([ @@ -205,7 +215,7 @@ describe("runInferenceSet OpenClaw gateway restart", () => { String(entry.reason), ); expect(auditReasons).toContain( - "inference set openclaw:compatible-anthropic-endpoint:claude-sonnet-proxy (gateway restart pending)", + "inference set openclaw:compatible-anthropic-endpoint:claude-sonnet-proxy (gateway restart and pairing convergence pending)", ); expect(auditReasons).toContain( "inference set openclaw:compatible-anthropic-endpoint:claude-sonnet-proxy (config committed; gateway restart failed: health timeout)", @@ -216,6 +226,113 @@ describe("runInferenceSet OpenClaw gateway restart", () => { ); }); + it("fails a committed cross-family switch when pairing does not converge (#9527)", async () => { + const config: ConfigObject = { + agents: { defaults: { model: { primary: "openai/nvidia/model-a" } } }, + models: { + providers: { + openai: { + baseUrl: "https://inference.local/v1", + api: "openai-completions", + models: [{ id: "nvidia/model-a", name: "openai/nvidia/model-a" }], + }, + }, + }, + }; + const deps = createDeps({ + config, + session: baseSession({ + provider: "compatible-anthropic-endpoint", + model: "claude-sonnet-proxy", + endpointUrl: "https://anthropic-compatible.example/v1", + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + preferredInferenceApi: "anthropic-messages", + }), + settleOpenClawPairing: () => ({ + ok: false, + failureLayer: "approval-rejected", + }), + }); + + await expect( + runInferenceSet( + { + provider: "compatible-anthropic-endpoint", + model: "claude-sonnet-proxy", + noVerify: true, + }, + deps, + ), + ).rejects.toThrow( + "OpenClaw gateway pairing did not converge (approval-rejected). The committed route was not rolled back.", + ); + + expect(deps.calls.restartSandboxGateway).toHaveBeenCalledOnce(); + expect(deps.calls.settleOpenClawPairing).toHaveBeenCalledOnce(); + expect(deps.calls.writeSandboxConfig).toHaveBeenCalledOnce(); + expect(deps.calls.recomputeSandboxConfigHash).toHaveBeenCalledOnce(); + expect(deps.calls.updateSandbox.mock.calls.at(-1)).toEqual([ + "alpha", + expect.objectContaining({ + provider: "compatible-anthropic-endpoint", + model: "claude-sonnet-proxy", + }), + ]); + expect(deps.calls.log.mock.calls.map(([line]) => String(line)).join("\n")).not.toContain( + "Inference route synced", + ); + expect(deps.calls.appendAuditEntry.mock.calls.map(([entry]) => String(entry.reason))).toContain( + "inference set openclaw:compatible-anthropic-endpoint:claude-sonnet-proxy (config committed; gateway restart completed; pairing convergence failed: approval-rejected)", + ); + }); + + it("does not expose pairing command output from a post-commit failure (#9527)", async () => { + const config: ConfigObject = { + agents: { defaults: { model: { primary: "openai/nvidia/model-a" } } }, + models: { + providers: { + openai: { + baseUrl: "https://inference.local/v1", + api: "openai-completions", + models: [{ id: "nvidia/model-a", name: "openai/nvidia/model-a" }], + }, + }, + }, + }; + const deps = createDeps({ + config, + session: baseSession({ + provider: "compatible-anthropic-endpoint", + model: "claude-sonnet-proxy", + endpointUrl: "https://anthropic-compatible.example/v1", + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + preferredInferenceApi: "anthropic-messages", + }), + settleOpenClawPairing: () => { + throw new Error("token=do-not-report"); + }, + }); + + const failure = await runInferenceSet( + { + provider: "compatible-anthropic-endpoint", + model: "claude-sonnet-proxy", + noVerify: true, + }, + deps, + ).catch((error: unknown) => error); + + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain("pairing-operation-failed"); + expect((failure as Error).message).not.toContain("do-not-report"); + expect( + deps.calls.appendAuditEntry.mock.calls.map(([entry]) => String(entry.reason)).join("\n"), + ).toContain("pairing convergence failed: pairing-operation-failed"); + expect( + deps.calls.appendAuditEntry.mock.calls.map(([entry]) => String(entry.reason)).join("\n"), + ).not.toContain("do-not-report"); + }); + it("restarts when leaving a legacy Anthropic route without provider.api (#4504)", async () => { const config: ConfigObject = { agents: { defaults: { model: { primary: "anthropic/claude-sonnet-proxy" } } }, diff --git a/src/lib/actions/inference-set-openclaw-pairing.test.ts b/src/lib/actions/inference-set-openclaw-pairing.test.ts new file mode 100644 index 00000000000..4a8e71ca0e8 --- /dev/null +++ b/src/lib/actions/inference-set-openclaw-pairing.test.ts @@ -0,0 +1,205 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + completeInferencePostCommit, + finalizeInferenceMutation, + type InferenceSetOpenClawPairingDeps, + type InferenceSetOpenClawPairingTarget, + settleInferenceSetOpenClawPairing, +} from "./inference-set-gateway-restart"; + +const TARGET: InferenceSetOpenClawPairingTarget = { + sandboxName: "alpha", + gatewayName: "nemoclaw-8080", + openclawVersion: "2026.7.1", + stateDirectory: "/sandbox/.openclaw", +}; +const DEVICE_IDENTITY_SHA256 = "a".repeat(64); + +function observation(state: "settled" | "pairing-only") { + return { state, deviceIdentitySha256: DEVICE_IDENTITY_SHA256 } as const; +} + +function pairingDeps( + options: { + observePairing?: InferenceSetOpenClawPairingDeps["observePairing"]; + approval?: ReturnType; + } = {}, +): InferenceSetOpenClawPairingDeps { + return { + observePairing: vi.fn(options.observePairing ?? (() => observation("settled"))), + publishScopeRequest: vi.fn(), + approveScopeRequest: vi.fn(() => options.approval ?? "approved"), + }; +} + +describe("settleInferenceSetOpenClawPairing", () => { + it("accepts exact settled scope state without publishing a request (#9527)", () => { + const deps = pairingDeps(); + + expect(settleInferenceSetOpenClawPairing(TARGET, deps)).toEqual({ ok: true }); + expect(deps.publishScopeRequest).not.toHaveBeenCalled(); + expect(deps.approveScopeRequest).not.toHaveBeenCalled(); + }); + + it("approves one device-bound request and requires final settled state (#9527)", () => { + const order: string[] = []; + const deps = pairingDeps(); + vi.mocked(deps.observePairing).mockImplementation(() => { + order.push("observe"); + const state = order.length === 1 ? "pairing-only" : "settled"; + return { state, deviceIdentitySha256: DEVICE_IDENTITY_SHA256 }; + }); + vi.mocked(deps.publishScopeRequest).mockImplementation(() => order.push("publish")); + vi.mocked(deps.approveScopeRequest).mockImplementation(() => { + order.push("approve"); + return "approved"; + }); + + expect(settleInferenceSetOpenClawPairing(TARGET, deps)).toEqual({ ok: true }); + expect(deps.publishScopeRequest).toHaveBeenCalledWith(TARGET); + expect(deps.approveScopeRequest).toHaveBeenCalledWith(TARGET, DEVICE_IDENTITY_SHA256); + expect(order).toEqual(["observe", "publish", "approve", "observe"]); + }); + + it("accepts an ambiguous approval only when final state is settled (#9527)", () => { + const deps = pairingDeps({ + observePairing: vi + .fn() + .mockReturnValueOnce(observation("pairing-only")) + .mockReturnValueOnce(observation("settled")), + approval: "ambiguous", + }); + + expect(settleInferenceSetOpenClawPairing(TARGET, deps)).toEqual({ ok: true }); + }); + + it("rejects an ambiguous approval when final state stays pairing-only (#9527)", () => { + const deps = pairingDeps({ + observePairing: vi.fn(() => observation("pairing-only")), + approval: "ambiguous", + }); + + expect(settleInferenceSetOpenClawPairing(TARGET, deps)).toEqual({ + ok: false, + failureLayer: "approval-ambiguous", + }); + }); + + it("rejects unavailable initial state without exposing observer output (#9527)", () => { + const deps = pairingDeps({ + observePairing: vi.fn(() => { + throw new Error("token=do-not-report"); + }), + }); + + const result = settleInferenceSetOpenClawPairing(TARGET, deps); + + expect(result).toEqual({ ok: false, failureLayer: "initial-state-unavailable" }); + expect(JSON.stringify(result)).not.toContain("do-not-report"); + expect(deps.publishScopeRequest).not.toHaveBeenCalled(); + expect(deps.approveScopeRequest).not.toHaveBeenCalled(); + }); + + it("reports approval-rejected when scope state stays pairing-only (#9527)", () => { + const deps = pairingDeps({ + observePairing: vi.fn(() => observation("pairing-only")), + approval: "rejected", + }); + + expect(settleInferenceSetOpenClawPairing(TARGET, deps)).toEqual({ + ok: false, + failureLayer: "approval-rejected", + }); + }); + + it("rejects approved scope state that does not settle (#9527)", () => { + const deps = pairingDeps({ + observePairing: vi.fn(() => observation("pairing-only")), + }); + + expect(settleInferenceSetOpenClawPairing(TARGET, deps)).toEqual({ + ok: false, + failureLayer: "final-state-unsettled", + }); + }); + + it("collapses request publication errors into a credential-free classification (#9527)", () => { + const deps = pairingDeps({ + observePairing: vi.fn(() => observation("pairing-only")), + }); + vi.mocked(deps.publishScopeRequest).mockImplementation(() => { + throw new Error("credential=do-not-report"); + }); + + const result = settleInferenceSetOpenClawPairing(TARGET, deps); + + expect(result).toEqual({ ok: false, failureLayer: "pairing-operation-failed" }); + expect(JSON.stringify(result)).not.toContain("do-not-report"); + expect(deps.approveScopeRequest).not.toHaveBeenCalled(); + }); + + it("rejects unavailable final state after one approval attempt (#9527)", () => { + const deps = pairingDeps({ + observePairing: vi + .fn() + .mockReturnValueOnce(observation("pairing-only")) + .mockImplementationOnce(() => { + throw new Error("raw paired state"); + }), + }); + + expect(settleInferenceSetOpenClawPairing(TARGET, deps)).toEqual({ + ok: false, + failureLayer: "final-state-unavailable", + }); + expect(deps.approveScopeRequest).toHaveBeenCalledOnce(); + }); + + it("fails closed when required convergence has no pairing target (#9527)", () => { + const appendAuditEntry = vi.fn(); + const log = vi.fn(); + const settleOpenClawPairing = vi.fn(() => ({ ok: true }) as const); + const mutation = finalizeInferenceMutation( + { + agentName: "openclaw", + configChanged: true, + nextApi: "openai-completions", + previousApi: "openai-completions", + result: { + sandboxName: "alpha", + provider: "nvidia-prod", + model: "nvidia/model-b", + primaryModelRef: "inference/nvidia/model-b", + inSandboxConfigSynced: true, + }, + }, + { appendAuditEntry, log }, + ); + + expect(() => + completeInferencePostCommit(mutation, { + appendAuditEntry, + log, + restartSandboxGateway: vi.fn( + () => + ({ + ok: true, + restarted: true, + healthPassed: true, + forwardRecovered: true, + }) as const, + ), + settleOpenClawPairing, + }), + ).toThrow("OpenClaw gateway pairing did not converge (pairing-target-unavailable)"); + expect(settleOpenClawPairing).not.toHaveBeenCalled(); + expect(log.mock.calls.flat().join("\n")).not.toContain("Inference route synced"); + expect(appendAuditEntry.mock.calls.map(([entry]) => String(entry.reason))).toContain( + "inference set openclaw:nvidia-prod:nvidia/model-b (config committed; pairing convergence failed: pairing-target-unavailable)", + ); + }); +}); diff --git a/src/lib/actions/inference-set-openclaw-run.test.ts b/src/lib/actions/inference-set-openclaw-run.test.ts index e18890f8b1f..f756f2b32b4 100644 --- a/src/lib/actions/inference-set-openclaw-run.test.ts +++ b/src/lib/actions/inference-set-openclaw-run.test.ts @@ -7,7 +7,7 @@ import { runInferenceSet } from "./inference-set"; import { baseSession, createDeps, OPENCLAW_TARGET } from "./inference-set.test-support"; describe("runInferenceSet OpenClaw routing", () => { - it("updates OpenShell, OpenClaw config, registry, and the matching onboard session", async () => { + it("completes a same-API switch and pairing when audit persistence fails (#9527)", async () => { const config: ConfigObject = { agents: { defaults: { model: { primary: "inference/moonshotai/kimi-k2.6" } } }, models: { @@ -20,6 +20,9 @@ describe("runInferenceSet OpenClaw routing", () => { }, }; const deps = createDeps({ config, session: baseSession() }); + deps.calls.appendAuditEntry.mockImplementationOnce(() => { + throw new Error("audit storage unavailable"); + }); const result = await runInferenceSet( { @@ -78,7 +81,16 @@ describe("runInferenceSet OpenClaw routing", () => { expect.objectContaining({ action: "inference_set", sandbox: "alpha", - reason: "inference set openclaw:nvidia-prod:nvidia/nemotron-3-super-120b-a12b", + reason: + "inference set openclaw:nvidia-prod:nvidia/nemotron-3-super-120b-a12b (pairing convergence pending)", + }), + ); + expect(deps.calls.appendAuditEntry).toHaveBeenCalledWith( + expect.objectContaining({ + action: "inference_set", + sandbox: "alpha", + reason: + "inference set openclaw:nvidia-prod:nvidia/nemotron-3-super-120b-a12b (pairing convergence completed)", }), ); expect(result).toMatchObject({ @@ -91,6 +103,15 @@ describe("runInferenceSet OpenClaw routing", () => { inSandboxConfigSynced: true, }); expect(deps.calls.restartSandboxGateway).not.toHaveBeenCalled(); + expect(deps.calls.settleOpenClawPairing).toHaveBeenCalledWith({ + sandboxName: "alpha", + gatewayName: "nemoclaw", + openclawVersion: "", + stateDirectory: "/sandbox/.openclaw", + }); + expect(deps.calls.log).toHaveBeenCalledWith( + " Warning: could not record the post-commit inference audit entry for 'alpha'.", + ); }); it("preserves same-provider Bedrock Runtime adapter routing for OpenClaw switches", async () => { diff --git a/src/lib/actions/inference-set.test-support.ts b/src/lib/actions/inference-set.test-support.ts index 59cf82a96f0..ed132e5d1e8 100644 --- a/src/lib/actions/inference-set.test-support.ts +++ b/src/lib/actions/inference-set.test-support.ts @@ -138,6 +138,7 @@ export function createDeps(options: { probeSandboxRoute?: InferenceSetDeps["probeSandboxRoute"]; updateSandbox?: InferenceSetDeps["updateSandbox"]; restartSandboxGateway?: InferenceSetDeps["restartSandboxGateway"]; + settleOpenClawPairing?: InferenceSetDeps["settleOpenClawPairing"]; seedHermesDashboardConfigResult?: "converged" | "absent" | "failed"; withGatewayRouteMutationLock?: InferenceSetDeps["withGatewayRouteMutationLock"]; }): InferenceSetDeps & { @@ -162,6 +163,7 @@ export function createDeps(options: { probeSandboxRoute: ReturnType; sleep: ReturnType; restartSandboxGateway: ReturnType; + settleOpenClawPairing: ReturnType; withGatewayRouteMutationLock: ReturnType; }; getSession: () => Session | null; @@ -232,6 +234,7 @@ export function createDeps(options: { forwardRecovered: true, })), ), + settleOpenClawPairing: vi.fn(options.settleOpenClawPairing ?? (() => ({ ok: true }) as const)), withGatewayRouteMutationLock: vi.fn( options.withGatewayRouteMutationLock ?? (async (_gatewayName: string, operation: () => Promise | unknown) => @@ -272,6 +275,7 @@ export function createDeps(options: { withGatewayRouteMutationLock: calls.withGatewayRouteMutationLock as InferenceSetDeps["withGatewayRouteMutationLock"], restartSandboxGateway: calls.restartSandboxGateway, + settleOpenClawPairing: calls.settleOpenClawPairing, calls, getSession: () => session, }; diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index cbcc30ee79f..64869ca5022 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -63,12 +63,13 @@ import { openshellReportsProviderNotFound, } from "./inference-set-error"; import { - completeInferenceGatewayRestart, + completeInferencePostCommit, defaultInferenceGatewayRestart, finalizeInferenceMutation, type InferenceGatewayRestartDeps, type InferenceMutation, readPreviousOpenClawInferenceApi, + settleInferenceSetOpenClawPairing, } from "./inference-set-gateway-restart"; import { type InferenceSetSandboxRouteProbe, @@ -280,6 +281,7 @@ function defaultDeps(): InferenceSetDeps { sleep: sleepInferenceSetRouteConvergence, withGatewayRouteMutationLock, restartSandboxGateway: defaultInferenceGatewayRestart, + settleOpenClawPairing: settleInferenceSetOpenClawPairing, isSandboxConfigMutable: (sandboxName) => { const { isShieldsDown }: typeof import("../shields") = require("../shields"); return isShieldsDown(sandboxName, true); @@ -1401,6 +1403,15 @@ async function runInferenceSetWithoutHostLock( agentName, configChanged: patched.changed, nextApi: patched.route.inferenceApi, + openClawPairingTarget: + agentName === "openclaw" + ? { + sandboxName, + gatewayName: expectedGatewayName, + openclawVersion: entry.agentVersion ?? "", + stateDirectory: target.configDir, + } + : undefined, previousApi: previousOpenClawInferenceApi, result: { sandboxName, @@ -1501,10 +1512,11 @@ export async function runInferenceSet( ), ), ); - // Release the config transition lock before the managed restart reacquires - // it, but retain the outer sandbox lifecycle lock so another process cannot - // destroy/recreate this name between the committed write and restart. - completeInferenceGatewayRestart(mutation, deps); + // Release the config transition lock before post-commit gateway work + // reacquires its own route state. Retain the outer sandbox lifecycle lock + // so another process cannot replace this sandbox between the committed + // write, an optional restart, and device-scope convergence. + completeInferencePostCommit(mutation, deps); if (mutation.result.dashboardConverged === false) { throw new InferenceSetError( `Inference route and main Hermes config were updated for '${mutation.result.sandboxName}', ` + diff --git a/src/lib/actions/sandbox/launch-readiness/openclaw-pairing-qualification.test.ts b/src/lib/actions/sandbox/launch-readiness/openclaw-pairing-qualification.test.ts index 8c066fd741a..cc274a31a32 100644 --- a/src/lib/actions/sandbox/launch-readiness/openclaw-pairing-qualification.test.ts +++ b/src/lib/actions/sandbox/launch-readiness/openclaw-pairing-qualification.test.ts @@ -219,6 +219,24 @@ describe("OpenClaw launch-readiness pairing qualification", () => { expect(JSON.stringify([settled, pairingOnly])).not.toContain(publicKey); }); + it("observes settlement without version provenance but keeps qualification version-bound (#9527)", () => { + const deps = { + getOpenshellBinary: () => "openshell", + readApprovalPolicy: () => POLICY, + spawnSync: localScriptSpawn as typeof spawnSync, + }; + + expect( + observeOpenClawPairingSettlement("alpha", "nemoclaw-8080", "", stateDirectory, deps), + ).toEqual({ + state: "settled", + deviceIdentitySha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }); + expect(() => + observeOpenClawPairingQualification("alpha", "nemoclaw-8080", "", stateDirectory, deps), + ).toThrow("OpenClaw pairing qualification is unavailable"); + }); + it("accepts the exact canonical Ed25519 public-key PEM representation (#9207)", () => { const identityPath = path.join(stateDirectory, "identity", "device.json"); writeJson(identityPath, { diff --git a/src/lib/actions/sandbox/launch-readiness/openclaw-pairing-qualification.ts b/src/lib/actions/sandbox/launch-readiness/openclaw-pairing-qualification.ts index 3870d818d86..3c837902899 100644 --- a/src/lib/actions/sandbox/launch-readiness/openclaw-pairing-qualification.ts +++ b/src/lib/actions/sandbox/launch-readiness/openclaw-pairing-qualification.ts @@ -580,7 +580,11 @@ function runOpenClawPairingObservation( ): { readonly output: string; readonly policy: string } { const approvalPolicy = (execDeps?.readApprovalPolicy ?? readAutoPairApprovalPolicyModule)(); const normalizedVersion = openclawVersion.trim(); - if (!approvalPolicy || !normalizedVersion || normalizedVersion.length > 128) { + if ( + !approvalPolicy || + normalizedVersion.length > 128 || + (mode === "qualification" && !normalizedVersion) + ) { throw new OpenClawPairingQualificationError(); } const approvalPolicyModuleB64 = Buffer.from(approvalPolicy, "utf8").toString("base64");