fix(onboard): probe registered extra providers exactly

src/lib/onboard/extra-provider-reconciliation.test.ts

@@ -1,94 +1,256 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { describe, expect, it, vi } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 import { reconcileRegisteredExtraProviders } from "./extra-provider-reconciliation";
 
-type RunResult = { status: number | null; stderr?: string; stdout?: string | Buffer };
+type ProbeResult = {
+  status: number | null;
+  output?: string | Buffer | null;
+  stdout?: string | Buffer | null;
+  stderr?: string | Buffer | null;
+};
+
+afterEach(() => {
+  vi.unstubAllEnvs();
+});
 
 describe("reconcileRegisteredExtraProviders", () => {
-  it("keeps every user-owned extra whose exact name appears in one scoped gateway list", () => {
-    const runOpenshell = vi.fn(
-      (_args: string[]): RunResult => ({
-        status: 0,
-        stdout: Buffer.from(
-          " tavily-search\nbrave-search\ncustom-provider\nmy-slack-bridge\nunrelated\n",
-        ),
+  it("does not probe the gateway when no extra provider is recorded (#6501)", () => {
+    const runOpenshell = vi.fn((): ProbeResult => ({ status: 0 }));
+
+    expect(
+      reconcileRegisteredExtraProviders("nemoclaw", {
+        listExtraProviders: () => [],
+        runOpenshell,
       }),
+    ).toEqual([]);
+    expect(runOpenshell).not.toHaveBeenCalled();
+  });
+
+  it("probes beyond the first hundred records and omits an exact stale provider (#6501)", () => {
+    const recorded = Object.freeze(
+      Array.from({ length: 128 }, (_value, index) => `custom-provider-${index}`),
     );
-    const recorded = ["tavily-search", "brave-search", "custom-provider", "my-slack-bridge"];
+    const calls: Array<{ args: string[]; options: Record<string, unknown> | undefined }> = [];
+    const runOpenshell = vi.fn((args: string[], options?: Record<string, unknown>): ProbeResult => {
+      calls.push({ args, options });
+      if (args.at(-1) === "custom-provider-127") {
+        return { status: 1, stderr: "Error: provider 'custom-provider-127' not found" };
+      }
+      return { status: 0, stdout: "" };
+    });
 
-    const result = reconcileRegisteredExtraProviders("nemoclaw", {
+    const reconciled = reconcileRegisteredExtraProviders("nemoclaw", {
+      listExtraProviders: () => [...recorded],
       runOpenshell,
-      listExtraProviders: () => recorded,
     });
 
-    expect(result).toEqual(recorded);
-    expect(runOpenshell).toHaveBeenCalledOnce();
-    expect(runOpenshell).toHaveBeenCalledWith(["provider", "list", "-g", "nemoclaw", "--names"], {
-      ignoreError: true,
-      stdio: ["ignore", "pipe", "pipe"],
-      suppressOutput: true,
+    expect(reconciled).toEqual(recorded.slice(0, -1));
+    expect(calls).toHaveLength(recorded.length);
+    expect(calls.every(({ args }) => args[0] === "provider" && args[1] === "get")).toBe(true);
+    expect(calls.some(({ args }) => args.includes("list") || args.includes("--names"))).toBe(false);
+    expect(calls[0]).toEqual({
+      args: ["provider", "get", "-g", "nemoclaw", "custom-provider-0"],
+      options: {
+        ignoreError: true,
+        maxBuffer: 64 * 1024,
+        stdio: ["ignore", "pipe", "pipe"],
+        suppressOutput: true,
+        timeout: 5_000,
+      },
     });
   });
 
-  it("omits a stale tavily record only from the current create plan (#6501)", () => {
-    const recorded = ["tavily-search", "custom-provider"];
+  it("keeps healthy custom providers even when successful get output is empty (#6501)", () => {
+    const recorded = Object.freeze(["custom-provider", "my-slack-bridge"]);
+    const runOpenshell = vi.fn((): ProbeResult => ({ status: 0, stdout: "", stderr: "" }));
 
-    const result = reconcileRegisteredExtraProviders("nemoclaw", {
-      runOpenshell: vi.fn(() => ({ status: 0, stdout: "custom-provider\n" })),
-      listExtraProviders: () => recorded,
+    const reconciled = reconcileRegisteredExtraProviders("nemoclaw", {
+      listExtraProviders: () => [...recorded],
+      runOpenshell,
     });
 
-    expect(result).toEqual(["custom-provider"]);
-    expect(recorded).toEqual(["tavily-search", "custom-provider"]);
+    expect(reconciled).toEqual(recorded);
+    expect(recorded).toEqual(["custom-provider", "my-slack-bridge"]);
+    expect(runOpenshell).toHaveBeenCalledTimes(2);
   });
 
-  it("matches complete names instead of prefixes or provider-name heuristics", () => {
-    const result = reconcileRegisteredExtraProviders("nemoclaw", {
-      runOpenshell: vi.fn(() => ({
-        status: 0,
-        stdout: "tavily-search-backup\ncustom-provider-v2\n",
-      })),
-      listExtraProviders: () => ["tavily-search", "custom-provider"],
+  it("omits only the entry with an exact provider-specific not-found diagnostic (#6501)", () => {
+    const recorded = Object.freeze([
+      "healthy-provider",
+      "stale-provider",
+      "indeterminate-provider",
+    ]);
+    const runOpenshell = vi.fn((args: string[]): ProbeResult => {
+      const providerName = args.at(-1);
+      if (providerName === "stale-provider") {
+        return {
+          status: 1,
+          stderr: Buffer.from("Error: provider 'stale-provider' not found\n"),
+        };
+      }
+      if (providerName === "indeterminate-provider") {
+        return { status: 1, stderr: "Error: provider 'some-other-provider' not found" };
+      }
+      return { status: 0 };
     });
 
-    expect(result).toEqual([]);
+    const reconciled = reconcileRegisteredExtraProviders("nemoclaw", {
+      listExtraProviders: () => [...recorded],
+      runOpenshell,
+    });
+
+    expect(reconciled).toEqual(["healthy-provider", "indeterminate-provider"]);
+    expect(recorded).toEqual(["healthy-provider", "stale-provider", "indeterminate-provider"]);
   });
 
-  it("preserves every recorded extra when the gateway list exits nonzero", () => {
-    const recorded = ["tavily-search", "brave-search", "custom-provider"];
+  it("accepts the exact gRPC not-found ordering without using broad name heuristics (#6501)", () => {
+    const runOpenshell = vi.fn(
+      (): ProbeResult => ({
+        status: 1,
+        stderr: 'rpc error: NotFound: provider "stale-provider"',
+      }),
+    );
+
+    expect(
+      reconcileRegisteredExtraProviders("nemoclaw", {
+        listExtraProviders: () => ["stale-provider"],
+        runOpenshell,
+      }),
+    ).toEqual([]);
+  });
 
-    const result = reconcileRegisteredExtraProviders("nemoclaw", {
-      runOpenshell: vi.fn(() => ({ status: 1, stderr: "gateway unavailable" })),
-      listExtraProviders: () => recorded,
+  it("compares captured names byte-for-byte while matching keywords case-insensitively (#6501)", () => {
+    const runOpenshell = vi.fn((args: string[]): ProbeResult => {
+      if (args.at(-1) === "ProviderA") {
+        return { status: 1, stderr: "Error: provider 'providera' not found" };
+      }
+      return { status: 1, stderr: "ERROR: PROVIDER 'ProviderB' NOT FOUND" };
     });
 
-    expect(result).toEqual(recorded);
+    expect(
+      reconcileRegisteredExtraProviders("nemoclaw", {
+        listExtraProviders: () => ["ProviderA", "ProviderB"],
+        runOpenshell,
+      }),
+    ).toEqual(["ProviderA"]);
   });
 
-  it("preserves every recorded extra when the gateway list throws", () => {
-    const recorded = ["tavily-search", "my-slack-bridge"];
+  it("preserves providers when a missing-gateway diagnostic merely mentions them (#6501)", () => {
+    const diagnostics = [
+      "Error: gateway 'nemoclaw' not found while checking provider 'custom-provider'",
+      "Error: gateway not found: provider 'custom-provider' was requested",
+      'Error: status: NotFound, message: "gateway not found"; requested provider "custom-provider"',
+      "Error: gateway 'nemoclaw' not found; provider 'custom-provider' not found during lookup",
+      "Unknown gateway 'nemoclaw' while checking provider 'custom-provider'",
+      "No such gateway 'nemoclaw'; provider 'custom-provider' not found",
+      "Transport closed while checking provider 'custom-provider' not found",
+      "transport error\nError: provider 'custom-provider' not found",
+      "authentication failed\nError: provider 'custom-provider' not found",
+    ];
 
-    const result = reconcileRegisteredExtraProviders("nemoclaw", {
-      runOpenshell: vi.fn(() => {
-        throw new Error("spawn failed");
-      }),
-      listExtraProviders: () => recorded,
+    for (const diagnostic of diagnostics) {
+      const runOpenshell = vi.fn((): ProbeResult => ({ status: 1, stderr: diagnostic }));
+      expect(
+        reconcileRegisteredExtraProviders("nemoclaw", {
+          listExtraProviders: () => ["custom-provider"],
+          runOpenshell,
+        }),
+        diagnostic,
+      ).toEqual(["custom-provider"]);
+    }
+  });
+
+  it("fails open for thrown, timed-out, and otherwise ambiguous probes (#6501)", () => {
+    const runOpenshell = vi.fn((args: string[]): ProbeResult => {
+      switch (args.at(-1)) {
+        case "thrown-provider":
+          throw new Error("gateway process unavailable");
+        case "timed-out-provider":
+          return {
+            status: null,
+            stderr: "operation timed out: provider 'timed-out-provider' not found",
+          };
+        case "nonstandard-exit-provider":
+          return {
+            status: 7,
+            stderr: "provider 'nonstandard-exit-provider' not found",
+          };
+        case "ambiguous-provider":
+          return { status: 1, stderr: "provider lookup failed" };
+        case "unavailable-provider":
+          return {
+            status: 1,
+            stderr:
+              "Error: status: Unavailable, message: \"provider 'unavailable-provider' not found\"",
+          };
+        default:
+          return { status: 0 };
+      }
     });
+    const recorded = [
+      "thrown-provider",
+      "timed-out-provider",
+      "nonstandard-exit-provider",
+      "ambiguous-provider",
+      "unavailable-provider",
+    ];
 
-    expect(result).toEqual(recorded);
+    expect(
+      reconcileRegisteredExtraProviders("nemoclaw", {
+        listExtraProviders: () => [...recorded],
+        runOpenshell,
+      }),
+    ).toEqual(recorded);
   });
 
-  it("does not query the gateway when no extras are recorded", () => {
-    const runOpenshell = vi.fn();
+  it("bounds aggregate probe latency and preserves names left after the deadline (#6501)", () => {
+    let now = 0;
+    const timeouts: number[] = [];
+    const runOpenshell = vi.fn(
+      (_args: string[], options?: Record<string, unknown>): ProbeResult => {
+        const timeout = Number(options?.timeout);
+        timeouts.push(timeout);
+        now += timeout;
+        return { status: null, stderr: "provider process timed out" };
+      },
+    );
+    const recorded = ["provider-1", "provider-2", "provider-3", "provider-4", "provider-5"];
 
     expect(
       reconcileRegisteredExtraProviders("nemoclaw", {
+        listExtraProviders: () => [...recorded],
+        nowMs: () => now,
         runOpenshell,
-        listExtraProviders: () => [],
       }),
-    ).toEqual([]);
+    ).toEqual(recorded);
+    expect(runOpenshell).toHaveBeenCalledTimes(3);
+    expect(timeouts).toEqual([5_000, 5_000, 5_000]);
+  });
+
+  it("enforces gateway containment before making any nonempty provider probe (#6501)", () => {
+    vi.stubEnv("OPENSHELL_GATEWAY_ENDPOINT", "https://other.example.test");
+    const runOpenshell = vi.fn((): ProbeResult => ({ status: 0 }));
+
+    expect(() =>
+      reconcileRegisteredExtraProviders("nemoclaw", {
+        listExtraProviders: () => ["custom-provider"],
+        runOpenshell,
+      }),
+    ).toThrow(/OPENSHELL_GATEWAY_ENDPOINT is set/);
+    expect(runOpenshell).not.toHaveBeenCalled();
+  });
+
+  it("requires a gateway name when recorded providers need reconciliation (#6501)", () => {
+    const runOpenshell = vi.fn((): ProbeResult => ({ status: 0 }));
+
+    expect(() =>
+      reconcileRegisteredExtraProviders("", {
+        listExtraProviders: () => ["custom-provider"],
+        runOpenshell,
+      }),
+    ).toThrow("OpenShell gateway name is required.");
     expect(runOpenshell).not.toHaveBeenCalled();
   });
 });