fix(maintenance): surface gateway schema preflight on host-process gateway drift

src/lib/actions/gateway-drift-preflight.test.ts

@@ -23,6 +23,15 @@ const driftIssue: OpenShellStateRpcIssue = {
   },
 };
 
+const hostProcessDriftIssue: OpenShellStateRpcIssue = {
+  kind: "host_process_drift",
+  drift: {
+    gatewayBin: "/home/u/.local/bin/openshell-gateway",
+    currentVersion: "0.0.43",
+    expectedVersion: "0.0.44",
+  },
+};
+
 function mockExit() {
   return vi.spyOn(process, "exit").mockImplementation(((code?: string | number | null) => {
     throw new Error(`process.exit(${code ?? 0})`);
@@ -128,6 +137,34 @@ describe("gateway drift preflight for maintenance actions", () => {
     expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled();
   });
 
+  it("backup-all fails before sandbox list on host-process gateway binary drift", async () => {
+    detectPreflightIssueSpy.mockReturnValue(hostProcessDriftIssue);
+
+    await expect(backupAll()).rejects.toThrow("process.exit(1)");
+
+    expect(printIssueSpy).toHaveBeenCalledWith(
+      hostProcessDriftIssue,
+      expect.objectContaining({ command: "nemoclaw backup-all" }),
+    );
+    expect(captureOpenshellSpy).not.toHaveBeenCalled();
+    expect(backupSandboxStateSpy).not.toHaveBeenCalled();
+    expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled();
+  });
+
+  it("upgrade-sandboxes fails before sandbox list on host-process gateway binary drift", async () => {
+    detectPreflightIssueSpy.mockReturnValue(hostProcessDriftIssue);
+
+    await expect(upgradeSandboxes({ check: true })).rejects.toThrow("process.exit(1)");
+
+    expect(printIssueSpy).toHaveBeenCalledWith(
+      hostProcessDriftIssue,
+      expect.objectContaining({ command: "nemoclaw upgrade-sandboxes" }),
+    );
+    expect(captureOpenshellSpy).not.toHaveBeenCalled();
+    expect(classifyUpgradeableSandboxesSpy).not.toHaveBeenCalled();
+    expect(recoverNamedGatewayRuntimeSpy).not.toHaveBeenCalled();
+  });
+
   it("backup-all recovers the named gateway and retries the sandbox list before backing up", async () => {
     captureOpenshellSpy
       .mockReturnValueOnce({ status: 1, output: "client error (Connect): Connection refused" })

src/lib/actions/sandbox/status.ts

@@ -72,7 +72,7 @@ export interface SandboxStatusReport {
   phase: string | null;
   gatewayState: string;
   inferenceHealth: ProviderHealthStatus | null;
-  rpcIssue: { kind: "image_drift" | "protobuf_mismatch" } | null;
+  rpcIssue: { kind: "image_drift" | "host_process_drift" | "protobuf_mismatch" } | null;
   hostGpuDetected: boolean;
   sandboxGpuEnabled: boolean;
   sandboxGpuMode: string | null;

src/lib/adapters/openshell/gateway-drift.test.ts

@@ -10,6 +10,7 @@ import {
   detectOpenShellStateRpcResultIssue,
   formatOpenShellStateRpcIssue,
   getGatewayClusterImageDrift,
+  getGatewayHostProcessDrift,
   parseGatewayClusterImageVersion,
 } from "../../../../dist/lib/adapters/openshell/gateway-drift";
 
@@ -52,6 +53,10 @@ describe("OpenShell gateway drift preflight", () => {
         deps: {
           getInstalledOpenshellVersion: () => "0.0.37",
           getGatewayClusterImageRef: () => "ghcr.io/nvidia/openshell/cluster:0.0.37",
+          // An active cluster gateway at the installed version: no drift of
+          // either kind. isGatewayClusterActive keeps the host-process probe
+          // from falling through to real system state in this unit test.
+          isGatewayClusterActive: () => true,
         },
       }),
     ).toBeNull();
@@ -186,6 +191,154 @@ describe("OpenShell gateway drift preflight", () => {
     expect(getGatewayClusterImageDrift()).toBeNull();
   });
 
+  it("detects host-process gateway binary drift when no cluster container exists", () => {
+    const drift = getGatewayHostProcessDrift({
+      deps: {
+        getInstalledOpenshellVersion: () => "0.0.44",
+        getGatewayClusterImageRef: () => null,
+        getHostProcessGatewayRuntime: () => ({
+          gatewayBin: "/home/u/.local/bin/openshell-gateway",
+          runningVersion: "0.0.43",
+        }),
+      },
+    });
+
+    expect(drift).toEqual({
+      gatewayBin: "/home/u/.local/bin/openshell-gateway",
+      currentVersion: "0.0.43",
+      expectedVersion: "0.0.44",
+    });
+  });
+
+  it("does not flag a matching host-process gateway binary", () => {
+    expect(
+      getGatewayHostProcessDrift({
+        deps: {
+          getInstalledOpenshellVersion: () => "0.0.44",
+          getGatewayClusterImageRef: () => null,
+          getHostProcessGatewayRuntime: () => ({
+            gatewayBin: "/home/u/.local/bin/openshell-gateway",
+            runningVersion: "0.0.44",
+          }),
+        },
+      }),
+    ).toBeNull();
+  });
+
+  it("does not probe host-process drift while an active cluster gateway is present", () => {
+    let runtimeProbed = false;
+    expect(
+      getGatewayHostProcessDrift({
+        deps: {
+          getInstalledOpenshellVersion: () => "0.0.44",
+          getGatewayClusterImageRef: () => "ghcr.io/nvidia/openshell/cluster:0.0.44",
+          isGatewayClusterActive: () => true,
+          getHostProcessGatewayRuntime: () => {
+            runtimeProbed = true;
+            return { gatewayBin: "/x", runningVersion: "0.0.43" };
+          },
+        },
+      }),
+    ).toBeNull();
+    expect(runtimeProbed).toBe(false);
+  });
+
+  it("detects host-process drift when a leftover cluster container exists but is not active", () => {
+    const drift = getGatewayHostProcessDrift({
+      deps: {
+        getInstalledOpenshellVersion: () => "0.0.44",
+        // A stopped/leftover cluster container still returns an image ref...
+        getGatewayClusterImageRef: () => "ghcr.io/nvidia/openshell/cluster:0.0.36",
+        // ...but it is not the active gateway, so it must not mask host drift.
+        isGatewayClusterActive: () => false,
+        getHostProcessGatewayRuntime: () => ({
+          gatewayBin: "/home/u/.local/bin/openshell-gateway",
+          runningVersion: "0.0.43",
+        }),
+      },
+    });
+
+    expect(drift).toMatchObject({ currentVersion: "0.0.43", expectedVersion: "0.0.44" });
+  });
+
+  it("returns null when the host-process gateway version cannot be probed", () => {
+    expect(
+      getGatewayHostProcessDrift({
+        deps: {
+          getInstalledOpenshellVersion: () => "0.0.44",
+          getGatewayClusterImageRef: () => null,
+          getHostProcessGatewayRuntime: () => ({
+            gatewayBin: "/home/u/.local/bin/openshell-gateway",
+            runningVersion: null,
+          }),
+        },
+      }),
+    ).toBeNull();
+  });
+
+  it("surfaces host-process drift as a preflight issue when cluster image drift is absent", () => {
+    const issue = detectOpenShellStateRpcPreflightIssue({
+      deps: {
+        getInstalledOpenshellVersion: () => "0.0.44",
+        getGatewayClusterImageRef: () => null,
+        getHostProcessGatewayRuntime: () => ({
+          gatewayBin: "/home/u/.local/bin/openshell-gateway",
+          runningVersion: "0.0.43",
+        }),
+      },
+    });
+
+    expect(issue).toMatchObject({
+      kind: "host_process_drift",
+      drift: { currentVersion: "0.0.43", expectedVersion: "0.0.44" },
+    });
+  });
+
+  it("formats host-process drift with a running gateway binary line and preflight phase", () => {
+    const lines = formatOpenShellStateRpcIssue(
+      {
+        kind: "host_process_drift",
+        drift: {
+          gatewayBin: "/home/u/.local/bin/openshell-gateway",
+          currentVersion: "0.0.43",
+          expectedVersion: "0.0.44",
+        },
+      },
+      { action: "backing up registered sandboxes", command: "nemoclaw backup-all" },
+    );
+
+    const joined = lines.join("\n");
+    expect(joined).toContain(
+      "OpenShell gateway schema preflight failed before backing up registered sandboxes.",
+    );
+    expect(joined).toContain("Installed OpenShell: 0.0.44");
+    expect(joined).toContain(
+      "Running gateway binary: /home/u/.local/bin/openshell-gateway (0.0.43)",
+    );
+    expect(joined).toContain("No sandbox data was changed.");
+    expect(joined).toContain("nemoclaw backup-all");
+    // Host-process gateways have no cluster container; do not reference its volumes.
+    expect(joined).not.toContain("openshell-cluster-nemoclaw");
+    expect(joined).not.toContain("Running gateway image");
+  });
+
+  it("ignores host-process gateway drift when the Vitest sentinel is set", () => {
+    expect(process.env.VITEST).toBe("true");
+    expect(process.env.NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT).toBe("1");
+
+    // Injected deps opt back into detection; the bare call stays disabled.
+    expect(
+      getGatewayHostProcessDrift({
+        deps: {
+          getInstalledOpenshellVersion: () => "0.0.44",
+          getGatewayClusterImageRef: () => null,
+          getHostProcessGatewayRuntime: () => ({ gatewayBin: "/x", runningVersion: "0.0.43" }),
+        },
+      }),
+    ).not.toBeNull();
+    expect(getGatewayHostProcessDrift()).toBeNull();
+  });
+
   it("classifies protobuf invalid-wire output as an unsafe OpenShell state result", () => {
     const issue = detectOpenShellStateRpcResultIssue(
       {
@@ -210,22 +363,57 @@ describe("OpenShell gateway drift preflight", () => {
     );
   });
 
-  it("formats runtime protobuf mismatches with gateway-specific recovery guidance", () => {
+  it("attaches host-process drift to a protobuf mismatch when there is no cluster container", () => {
+    const issue = detectOpenShellStateRpcResultIssue(
+      {
+        status: 1,
+        output:
+          'Error: status: Internal, message: "Sandbox.metadata: SandboxResponse.sandbox: invalid wire type value: 6"',
+      },
+      {
+        deps: {
+          getInstalledOpenshellVersion: () => "0.0.44",
+          getGatewayClusterImageRef: () => null,
+          getHostProcessGatewayRuntime: () => ({
+            gatewayBin: "/home/u/.local/bin/openshell-gateway",
+            runningVersion: "0.0.43",
+          }),
+        },
+      },
+    );
+
+    expect(issue?.kind).toBe("protobuf_mismatch");
+    expect(issue?.drift).toMatchObject({
+      gatewayBin: "/home/u/.local/bin/openshell-gateway",
+      currentVersion: "0.0.43",
+      expectedVersion: "0.0.44",
+    });
+    const joined = formatOpenShellStateRpcIssue(issue!).join("\n");
+    expect(joined).toContain(
+      "Running gateway binary: /home/u/.local/bin/openshell-gateway (0.0.43)",
+    );
+  });
+
+  it("formats runtime protobuf mismatches with gateway-neutral recovery guidance when drift is unknown", () => {
     const lines = formatOpenShellStateRpcIssue(
       {
         kind: "protobuf_mismatch",
         output: "Sandbox.metadata: invalid wire type value: 6",
       },
       {
         action: "querying live sandboxes",
-        gatewayName: "custom-gw",
       },
     );
 
+    const joined = lines.join("\n");
     expect(lines).toContain(
       "  OpenShell gateway/schema mismatch was detected while querying live sandboxes.",
     );
-    expect(lines.join("\n")).toContain("openshell-cluster-custom-gw");
-    expect(lines.join("\n")).not.toContain("schema preflight failed before");
+    expect(joined).toContain("preserve sandbox state first");
+    expect(joined).toContain("No sandbox data was changed.");
+    // Driver is unknown when no drift was resolved: do not name a cluster container.
+    expect(joined).not.toContain("openshell-cluster");
+    expect(joined).not.toContain("Docker volumes");
+    expect(joined).not.toContain("schema preflight failed before");
   });
 });