diff --git a/assistant/src/__tests__/cross-provider-web-search.test.ts b/assistant/src/__tests__/cross-provider-web-search.test.ts index 2b64b10d0a4..79231decc92 100644 --- a/assistant/src/__tests__/cross-provider-web-search.test.ts +++ b/assistant/src/__tests__/cross-provider-web-search.test.ts @@ -312,6 +312,90 @@ describe("Cross-Provider Web Search — OpenAI (Responses API)", () => { }); }); +// --------------------------------------------------------------------------- +// OpenAI Responses API — native web search tool mapping +// --------------------------------------------------------------------------- + +describe("Cross-Provider Web Search — OpenAI (Responses API, native mode)", () => { + beforeEach(() => { + lastOpenAIResponsesParams = null; + }); + + test("maps web_search to native web_search_preview tool when useNativeWebSearch is enabled", async () => { + const provider = new OpenAIResponsesProvider("sk-test", "gpt-4o", { + useNativeWebSearch: true, + }); + + const tools = [ + { + name: "file_read", + description: "Read a file", + input_schema: { + type: "object", + properties: { path: { type: "string" } }, + }, + }, + { + name: "web_search", + description: "Search the web", + input_schema: { + type: "object", + properties: { query: { type: "string" } }, + }, + }, + ]; + + await provider.sendMessage([userMsg("Search for something")], tools); + + const sentTools = lastOpenAIResponsesParams!.tools as Array< + Record + >; + expect(sentTools).toHaveLength(2); + // Non-web-search tools stay as function tools + expect(sentTools[0]).toMatchObject({ type: "function", name: "file_read" }); + // web_search is replaced with native hosted tool + expect(sentTools[1]).toEqual({ type: "web_search_preview" }); + }); + + test("still degrades web search history blocks in native mode", async () => { + const provider = new OpenAIResponsesProvider("sk-test", "gpt-4o", { + useNativeWebSearch: true, + }); + await provider.sendMessage(webSearchConversation()); + + const input = lastOpenAIResponsesParams!.input as Array<{ + type: string; + role?: string; + content?: Array<{ type: string; text?: string }>; + }>; + + // server_tool_use in assistant history is still degraded to text placeholder + const assistantItems = input.filter( + (item) => item.type === "message" && item.role === "assistant", + ); + const hasWebSearchPlaceholder = assistantItems.some((item) => + item.content?.some( + (part) => + part.type === "output_text" && + part.text?.includes("[Web search: web_search]"), + ), + ); + expect(hasWebSearchPlaceholder).toBe(true); + + // web_search_tool_result in user history is still degraded to text placeholder + const userItems = input.filter( + (item) => item.type === "message" && item.role === "user", + ); + const hasWebSearchResult = userItems.some((item) => + item.content?.some( + (part) => + part.type === "input_text" && part.text === "[Web search results]", + ), + ); + expect(hasWebSearchResult).toBe(true); + }); +}); + // --------------------------------------------------------------------------- // OpenAI Chat Completions compatibility provider tests // --------------------------------------------------------------------------- diff --git a/assistant/src/__tests__/llm-context-normalization.test.ts b/assistant/src/__tests__/llm-context-normalization.test.ts index a4aa8300122..14e1da553fb 100644 --- a/assistant/src/__tests__/llm-context-normalization.test.ts +++ b/assistant/src/__tests__/llm-context-normalization.test.ts @@ -1601,4 +1601,125 @@ describe("normalizeLlmContextPayloads", () => { expect(normalized).toEqual({}); }); + + test("normalizes Responses API web_search_call output as a tool_use section", () => { + const normalized = normalizeLlmContextPayloads({ + createdAt: 1_742_400_000_030, + requestPayload: { + model: "gpt-5.4", + instructions: "Search the web when needed.", + input: [ + { + role: "user", + content: [{ type: "input_text", text: "What is the weather?" }], + type: "message", + }, + ], + tools: [{ type: "web_search_preview" }], + }, + responsePayload: { + model: "gpt-5.4", + output: [ + { + type: "web_search_call", + id: "ws_abc", + status: "completed", + }, + { + type: "message", + role: "assistant", + content: [ + { type: "output_text", text: "It is sunny in Boston today." }, + ], + }, + ], + usage: { input_tokens: 30, output_tokens: 15 }, + status: "completed", + }, + }); + + expect(normalized.summary).toEqual({ + provider: "openai", + model: "gpt-5.4", + inputTokens: 30, + outputTokens: 15, + stopReason: "stop", + requestMessageCount: 1, + requestToolCount: 1, + responseMessageCount: 1, + responseToolCallCount: 1, + responsePreview: "It is sunny in Boston today.", + toolCallNames: ["web_search"], + }); + expect(normalized.responseSections).toEqual([ + { + kind: "tool_use", + label: "Response tool call 1", + role: "assistant", + toolName: "web_search", + data: { id: "ws_abc", status: "completed" }, + text: "[Web search: completed]", + }, + { + kind: "message", + label: "Assistant response", + role: "assistant", + text: "It is sunny in Boston today.", + }, + ]); + }); + + test("normalizes Responses API response with only web_search_call (no message)", () => { + const normalized = normalizeLlmContextPayloads({ + createdAt: 1_742_400_000_031, + requestPayload: { + model: "gpt-5.4", + instructions: "Search the web.", + input: [ + { + role: "user", + content: [{ type: "input_text", text: "Find latest news" }], + type: "message", + }, + ], + tools: [{ type: "web_search_preview" }], + }, + responsePayload: { + model: "gpt-5.4", + output: [ + { + type: "web_search_call", + id: "ws_only", + status: "searching", + }, + ], + usage: { input_tokens: 20, output_tokens: 5 }, + status: "incomplete", + }, + }); + + expect(normalized.summary).toEqual({ + provider: "openai", + model: "gpt-5.4", + inputTokens: 20, + outputTokens: 5, + stopReason: "incomplete", + requestMessageCount: 1, + requestToolCount: 1, + responseMessageCount: 1, + responseToolCallCount: 1, + responsePreview: undefined, + toolCallNames: ["web_search"], + }); + expect(normalized.responseSections).toEqual([ + { + kind: "tool_use", + label: "Response tool call 1", + role: "assistant", + toolName: "web_search", + data: { id: "ws_only", status: "searching" }, + text: "[Web search: searching]", + }, + ]); + }); }); diff --git a/assistant/src/__tests__/managed-proxy-context.test.ts b/assistant/src/__tests__/managed-proxy-context.test.ts index 1726060fccb..6e7b5ed7ad9 100644 --- a/assistant/src/__tests__/managed-proxy-context.test.ts +++ b/assistant/src/__tests__/managed-proxy-context.test.ts @@ -112,10 +112,12 @@ describe("buildManagedBaseUrl", () => { expect(await buildManagedBaseUrl("gemini")).toBe( "https://platform.example.com/v1/runtime-proxy/gemini", ); + expect(await buildManagedBaseUrl("openai")).toBe( + "https://platform.example.com/v1/runtime-proxy/openai", + ); }); test("returns undefined for non-managed providers", async () => { - expect(await buildManagedBaseUrl("openai")).toBeUndefined(); expect(await buildManagedBaseUrl("fireworks")).toBeUndefined(); expect(await buildManagedBaseUrl("openrouter")).toBeUndefined(); expect(await buildManagedBaseUrl("ollama")).toBeUndefined(); @@ -130,6 +132,7 @@ describe("buildManagedBaseUrl", () => { mockAssistantApiKey = null; expect(await buildManagedBaseUrl("anthropic")).toBeUndefined(); expect(await buildManagedBaseUrl("gemini")).toBeUndefined(); + expect(await buildManagedBaseUrl("openai")).toBeUndefined(); }); }); @@ -142,7 +145,7 @@ describe("managedFallbackEnabledFor", () => { test("returns true only for managed fallback providers with prerequisites", async () => { expect(await managedFallbackEnabledFor("anthropic")).toBe(true); expect(await managedFallbackEnabledFor("gemini")).toBe(true); - expect(await managedFallbackEnabledFor("openai")).toBe(false); + expect(await managedFallbackEnabledFor("openai")).toBe(true); }); test("returns false for non-managed provider", async () => { @@ -158,5 +161,6 @@ describe("managedFallbackEnabledFor", () => { mockAssistantApiKey = null; expect(await managedFallbackEnabledFor("anthropic")).toBe(false); expect(await managedFallbackEnabledFor("gemini")).toBe(false); + expect(await managedFallbackEnabledFor("openai")).toBe(false); }); }); diff --git a/assistant/src/__tests__/openai-responses-provider.test.ts b/assistant/src/__tests__/openai-responses-provider.test.ts index 4a7848ba48d..bcc5c481f66 100644 --- a/assistant/src/__tests__/openai-responses-provider.test.ts +++ b/assistant/src/__tests__/openai-responses-provider.test.ts @@ -118,6 +118,16 @@ function functionCallArgsDoneEvent( }; } +function webSearchCallAddedEvent(itemId: string): FakeStreamEvent { + return { + type: "response.output_item.added", + item: { + type: "web_search_call", + id: itemId, + }, + }; +} + function completedEvent( inputTokens: number, outputTokens: number, @@ -1116,3 +1126,376 @@ describe("OpenAIResponsesProvider", () => { expect(result.stopReason).toBe("incomplete"); }); }); + +// --------------------------------------------------------------------------- +// Native web search tool mapping +// --------------------------------------------------------------------------- + +describe("OpenAIResponsesProvider — Native Web Search", () => { + const webSearchTool: ToolDefinition = { + name: "web_search", + description: "Search the web", + input_schema: { + type: "object", + properties: { query: { type: "string" } }, + }, + }; + + const fileReadTool: ToolDefinition = { + name: "file_read", + description: "Read a file", + input_schema: { + type: "object", + properties: { path: { type: "string" } }, + required: ["path"], + }, + }; + + beforeEach(() => { + fakeStreamEvents = []; + lastStreamParams = null; + lastStreamOptions = null; + lastConstructorOptions = null; + shouldThrow = null; + }); + + test("maps web_search to native web_search_preview tool when useNativeWebSearch is enabled", async () => { + const nativeProvider = new OpenAIResponsesProvider("sk-test", "gpt-5.2", { + useNativeWebSearch: true, + }); + fakeStreamEvents = [textDeltaEvent("OK"), completedEvent(10, 2)]; + + await nativeProvider.sendMessage( + [{ role: "user", content: [{ type: "text", text: "Search for cats" }] }], + [webSearchTool], + ); + + const sentTools = lastStreamParams!.tools as Array>; + expect(sentTools).toHaveLength(1); + expect(sentTools[0]).toEqual({ type: "web_search_preview" }); + }); + + test("keeps web_search as function tool when useNativeWebSearch is disabled", async () => { + const nonNativeProvider = new OpenAIResponsesProvider("sk-test", "gpt-5.2"); + fakeStreamEvents = [textDeltaEvent("OK"), completedEvent(10, 2)]; + + await nonNativeProvider.sendMessage( + [{ role: "user", content: [{ type: "text", text: "Search for cats" }] }], + [webSearchTool], + ); + + const sentTools = lastStreamParams!.tools as Array>; + expect(sentTools).toHaveLength(1); + expect(sentTools[0]).toEqual({ + type: "function", + name: "web_search", + description: "Search the web", + parameters: { + type: "object", + properties: { query: { type: "string" } }, + }, + strict: null, + }); + }); + + test("mixes native web_search_preview with regular function tools", async () => { + const nativeProvider = new OpenAIResponsesProvider("sk-test", "gpt-5.2", { + useNativeWebSearch: true, + }); + fakeStreamEvents = [textDeltaEvent("OK"), completedEvent(10, 2)]; + + await nativeProvider.sendMessage( + [{ role: "user", content: [{ type: "text", text: "Search and read" }] }], + [fileReadTool, webSearchTool], + ); + + const sentTools = lastStreamParams!.tools as Array>; + expect(sentTools).toHaveLength(2); + // Non-web-search tools remain as function tools + expect(sentTools[0]).toEqual({ + type: "function", + name: "file_read", + description: "Read a file", + parameters: { + type: "object", + properties: { path: { type: "string" } }, + required: ["path"], + }, + strict: null, + }); + // web_search is mapped to native tool + expect(sentTools[1]).toEqual({ type: "web_search_preview" }); + }); + + test("sends all tools as function tools when no web_search is present and native mode is on", async () => { + const nativeProvider = new OpenAIResponsesProvider("sk-test", "gpt-5.2", { + useNativeWebSearch: true, + }); + fakeStreamEvents = [textDeltaEvent("OK"), completedEvent(10, 2)]; + + await nativeProvider.sendMessage( + [{ role: "user", content: [{ type: "text", text: "Read file" }] }], + [fileReadTool], + ); + + const sentTools = lastStreamParams!.tools as Array>; + expect(sentTools).toHaveLength(1); + expect(sentTools[0]).toEqual({ + type: "function", + name: "file_read", + description: "Read a file", + parameters: { + type: "object", + properties: { path: { type: "string" } }, + required: ["path"], + }, + strict: null, + }); + }); + + // ----------------------------------------------------------------------- + // web_search_call stream event handling + // ----------------------------------------------------------------------- + + test("emits server_tool_start when web_search_call output item is added", async () => { + const nativeProvider = new OpenAIResponsesProvider("sk-test", "gpt-5.2", { + useNativeWebSearch: true, + }); + fakeStreamEvents = [ + webSearchCallAddedEvent("ws_call_1"), + textDeltaEvent("Search results here."), + completedEvent(50, 30), + ]; + + const events: ProviderEvent[] = []; + await nativeProvider.sendMessage( + [{ role: "user", content: [{ type: "text", text: "Search for cats" }] }], + [webSearchTool], + undefined, + { onEvent: (e) => events.push(e) }, + ); + + const startEvents = events.filter((e) => e.type === "server_tool_start"); + expect(startEvents).toHaveLength(1); + expect(startEvents[0]).toEqual({ + type: "server_tool_start", + name: "web_search", + toolUseId: "ws_call_1", + input: {}, + }); + }); + + test("emits server_tool_complete on response.completed for tracked web search calls", async () => { + const nativeProvider = new OpenAIResponsesProvider("sk-test", "gpt-5.2", { + useNativeWebSearch: true, + }); + fakeStreamEvents = [ + webSearchCallAddedEvent("ws_call_1"), + textDeltaEvent("Answer with citations."), + completedEvent(50, 30), + ]; + + const events: ProviderEvent[] = []; + await nativeProvider.sendMessage( + [{ role: "user", content: [{ type: "text", text: "Search for dogs" }] }], + [webSearchTool], + undefined, + { onEvent: (e) => events.push(e) }, + ); + + const completeEvents = events.filter( + (e) => e.type === "server_tool_complete", + ); + expect(completeEvents).toHaveLength(1); + expect(completeEvents[0]).toEqual({ + type: "server_tool_complete", + toolUseId: "ws_call_1", + isError: false, + }); + }); + + test("emits server_tool_complete for multiple web search calls", async () => { + const nativeProvider = new OpenAIResponsesProvider("sk-test", "gpt-5.2", { + useNativeWebSearch: true, + }); + fakeStreamEvents = [ + webSearchCallAddedEvent("ws_call_1"), + webSearchCallAddedEvent("ws_call_2"), + textDeltaEvent("Combined results."), + completedEvent(80, 50), + ]; + + const events: ProviderEvent[] = []; + await nativeProvider.sendMessage( + [{ role: "user", content: [{ type: "text", text: "Search multiple" }] }], + [webSearchTool], + undefined, + { onEvent: (e) => events.push(e) }, + ); + + const startEvents = events.filter((e) => e.type === "server_tool_start"); + expect(startEvents).toHaveLength(2); + expect(startEvents[0]).toEqual({ + type: "server_tool_start", + name: "web_search", + toolUseId: "ws_call_1", + input: {}, + }); + expect(startEvents[1]).toEqual({ + type: "server_tool_start", + name: "web_search", + toolUseId: "ws_call_2", + input: {}, + }); + + const completeEvents = events.filter( + (e) => e.type === "server_tool_complete", + ); + expect(completeEvents).toHaveLength(2); + expect(completeEvents[0]).toEqual({ + type: "server_tool_complete", + toolUseId: "ws_call_1", + isError: false, + }); + expect(completeEvents[1]).toEqual({ + type: "server_tool_complete", + toolUseId: "ws_call_2", + isError: false, + }); + }); + + test("does not emit server_tool events for non-web-search output items", async () => { + const nativeProvider = new OpenAIResponsesProvider("sk-test", "gpt-5.2", { + useNativeWebSearch: true, + }); + fakeStreamEvents = [ + functionCallAddedEvent("call_1", "file_read"), + functionCallArgsDeltaEvent('{"path":"/tmp/a"}', "call_1"), + functionCallArgsDoneEvent("call_1", "file_read", '{"path":"/tmp/a"}'), + completedEvent(20, 10), + ]; + + const events: ProviderEvent[] = []; + await nativeProvider.sendMessage( + [{ role: "user", content: [{ type: "text", text: "Read file" }] }], + [fileReadTool], + undefined, + { onEvent: (e) => events.push(e) }, + ); + + const serverToolEvents = events.filter( + (e) => + e.type === "server_tool_start" || e.type === "server_tool_complete", + ); + expect(serverToolEvents).toHaveLength(0); + }); + + // ----------------------------------------------------------------------- + // server_tool_use content blocks in ProviderResponse + // ----------------------------------------------------------------------- + + test("includes paired server_tool_use + web_search_tool_result content blocks for web search calls", async () => { + const nativeProvider = new OpenAIResponsesProvider("sk-test", "gpt-5.2", { + useNativeWebSearch: true, + }); + fakeStreamEvents = [ + webSearchCallAddedEvent("ws_call_1"), + textDeltaEvent("Here are the results."), + completedEvent(50, 30), + ]; + + const result = await nativeProvider.sendMessage( + [{ role: "user", content: [{ type: "text", text: "Search for cats" }] }], + [webSearchTool], + ); + + // server_tool_use + web_search_tool_result pair should appear before text + expect(result.content).toHaveLength(3); + expect(result.content[0]).toEqual({ + type: "server_tool_use", + id: "ws_call_1", + name: "web_search", + input: {}, + }); + expect(result.content[1]).toEqual({ + type: "web_search_tool_result", + tool_use_id: "ws_call_1", + content: [], + }); + expect(result.content[2]).toEqual({ + type: "text", + text: "Here are the results.", + }); + }); + + test("includes paired server_tool_use + web_search_tool_result for multiple web search calls", async () => { + const nativeProvider = new OpenAIResponsesProvider("sk-test", "gpt-5.2", { + useNativeWebSearch: true, + }); + fakeStreamEvents = [ + webSearchCallAddedEvent("ws_call_1"), + webSearchCallAddedEvent("ws_call_2"), + textDeltaEvent("Combined search results."), + completedEvent(80, 50), + ]; + + const result = await nativeProvider.sendMessage( + [ + { + role: "user", + content: [{ type: "text", text: "Search two things" }], + }, + ], + [webSearchTool], + ); + + expect(result.content).toHaveLength(5); + expect(result.content[0]).toEqual({ + type: "server_tool_use", + id: "ws_call_1", + name: "web_search", + input: {}, + }); + expect(result.content[1]).toEqual({ + type: "web_search_tool_result", + tool_use_id: "ws_call_1", + content: [], + }); + expect(result.content[2]).toEqual({ + type: "server_tool_use", + id: "ws_call_2", + name: "web_search", + input: {}, + }); + expect(result.content[3]).toEqual({ + type: "web_search_tool_result", + tool_use_id: "ws_call_2", + content: [], + }); + expect(result.content[4]).toEqual({ + type: "text", + text: "Combined search results.", + }); + }); + + test("does not include server_tool_use blocks when no web search calls occur", async () => { + const nativeProvider = new OpenAIResponsesProvider("sk-test", "gpt-5.2", { + useNativeWebSearch: true, + }); + fakeStreamEvents = [ + textDeltaEvent("No search needed."), + completedEvent(10, 5), + ]; + + const result = await nativeProvider.sendMessage( + [{ role: "user", content: [{ type: "text", text: "Hello" }] }], + [webSearchTool], + ); + + expect(result.content).toHaveLength(1); + expect(result.content[0]).toEqual({ + type: "text", + text: "No search needed.", + }); + }); +}); diff --git a/assistant/src/__tests__/provider-managed-proxy-integration.test.ts b/assistant/src/__tests__/provider-managed-proxy-integration.test.ts index 2581be09cd3..89aea943147 100644 --- a/assistant/src/__tests__/provider-managed-proxy-integration.test.ts +++ b/assistant/src/__tests__/provider-managed-proxy-integration.test.ts @@ -90,7 +90,7 @@ const DIRECT_OR_MANAGED_PROVIDER_KEYS: string[] = [ "fireworks", "openrouter", ]; -const MANAGED_FALLBACK_PROVIDERS: string[] = ["anthropic", "gemini"]; +const MANAGED_FALLBACK_PROVIDERS: string[] = ["anthropic", "gemini", "openai"]; function enableManagedProxy() { mockPlatformBaseUrl = PLATFORM_BASE; @@ -172,14 +172,18 @@ describe("managed proxy integration — credential precedence", () => { }, ); - test("managed bootstrap registers anthropic and gemini only", async () => { + test("managed bootstrap registers anthropic, openai, and gemini", async () => { enableManagedProxy(); mockProviderKeys = {}; await initializeProviders(makeProvidersConfig("anthropic", "test-model")); - expect(listProviders()).toEqual(["anthropic", "gemini"]); + expect(listProviders()).toEqual( + expect.arrayContaining(["anthropic", "openai", "gemini"]), + ); + expect(listProviders()).toHaveLength(3); expect(getProviderRoutingSource("anthropic")).toBe("managed-proxy"); + expect(getProviderRoutingSource("openai")).toBe("managed-proxy"); expect(getProviderRoutingSource("gemini")).toBe("managed-proxy"); - for (const p of ["openai", "fireworks", "openrouter"]) { + for (const p of ["fireworks", "openrouter"]) { expect(getProviderRoutingSource(p)).toBeUndefined(); } }); @@ -205,6 +209,23 @@ describe("managed proxy integration — credential precedence", () => { expect(baseURL).toContain("/v1/runtime-proxy/anthropic"); }); + test("managed openai uses openai proxy path", async () => { + enableManagedProxy(); + mockProviderKeys = {}; + await initializeProviders(makeProvidersConfig("openai", "gpt-4o")); + + const provider = getProvider("openai"); + + // Unwrap RetryProvider → OpenAIResponsesProvider to inspect the OpenAI + // SDK client's baseURL. + const retryInner = (provider as any).inner; + const openaiClient = (retryInner as any).client; + + expect(openaiClient).toBeDefined(); + const baseURL: string = openaiClient.baseURL; + expect(baseURL).toContain("/v1/runtime-proxy/openai"); + }); + test("managed gemini uses gemini proxy path", async () => { enableManagedProxy(); mockProviderKeys = {}; @@ -242,16 +263,18 @@ describe("managed proxy integration — credential precedence", () => { }); describe("mixed: some user keys + managed fallback fills gaps", () => { - test("user key for anthropic routes direct and managed fallback only fills gemini", async () => { + test("user key for anthropic routes direct and managed fallback fills openai and gemini", async () => { enableManagedProxy(); setUserKeysFor("anthropic"); await initializeProviders(makeProvidersConfig("anthropic", "test-model")); const registered = listProviders(); expect(registered).toContain("anthropic"); expect(getProviderRoutingSource("anthropic")).toBe("user-key"); + expect(registered).toContain("openai"); + expect(getProviderRoutingSource("openai")).toBe("managed-proxy"); expect(registered).toContain("gemini"); expect(getProviderRoutingSource("gemini")).toBe("managed-proxy"); - for (const p of ["openai", "fireworks", "openrouter"]) { + for (const p of ["fireworks", "openrouter"]) { expect(registered).not.toContain(p); expect(getProviderRoutingSource(p)).toBeUndefined(); } @@ -268,6 +291,7 @@ describe("managed proxy integration — credential precedence", () => { expect(getProviderRoutingSource("anthropic")).toBe("managed-proxy"); expect(registered).toContain("gemini"); expect(getProviderRoutingSource("gemini")).toBe("managed-proxy"); + // OpenAI has a user key so it's user-key, not managed-proxy for (const p of ["fireworks", "openrouter"]) { expect(registered).not.toContain(p); expect(getProviderRoutingSource(p)).toBeUndefined(); @@ -307,8 +331,8 @@ describe("managed proxy integration — ollama exclusion", () => { }); describe("managed proxy integration — constants integrity", () => { - test("anthropic and gemini have metadata with managed=true and a proxyPath", () => { - for (const provider of ["anthropic", "gemini"]) { + test("anthropic, openai, and gemini have metadata with managed=true and a proxyPath", () => { + for (const provider of ["anthropic", "openai", "gemini"]) { const meta = MANAGED_PROVIDER_META[provider]; expect(meta).toBeDefined(); expect(meta.managed).toBe(true); @@ -329,8 +353,14 @@ describe("managed proxy integration — constants integrity", () => { ); }); - test("openai-compatible providers are not managed proxy capable", () => { - for (const provider of ["openai", "fireworks", "openrouter"]) { + test("openai routes through openai proxy path", () => { + expect(MANAGED_PROVIDER_META.openai.proxyPath).toBe( + "/v1/runtime-proxy/openai", + ); + }); + + test("fireworks and openrouter are not managed proxy capable", () => { + for (const provider of ["fireworks", "openrouter"]) { expect(MANAGED_PROVIDER_META[provider].managed).toBe(false); expect(MANAGED_PROVIDER_META[provider].proxyPath).toBeUndefined(); } diff --git a/assistant/src/__tests__/secret-routes-managed-proxy.test.ts b/assistant/src/__tests__/secret-routes-managed-proxy.test.ts index 327cb00925f..97f358b855f 100644 --- a/assistant/src/__tests__/secret-routes-managed-proxy.test.ts +++ b/assistant/src/__tests__/secret-routes-managed-proxy.test.ts @@ -11,7 +11,7 @@ let providerRefreshCalls = 0; const PLATFORM_BASE_URL = "https://platform.example.com"; const ASSISTANT_API_KEY_PATH = credentialKey("vellum", "assistant_api_key"); const PLATFORM_BASE_URL_PATH = credentialKey("vellum", "platform_base_url"); -const MANAGED_PROVIDERS = ["anthropic", "gemini"] as const; +const MANAGED_PROVIDERS = ["anthropic", "openai", "gemini"] as const; let platformBaseUrlOverride: string | undefined; diff --git a/assistant/src/providers/managed-proxy/constants.ts b/assistant/src/providers/managed-proxy/constants.ts index fa532411e66..7082b5cf8f8 100644 --- a/assistant/src/providers/managed-proxy/constants.ts +++ b/assistant/src/providers/managed-proxy/constants.ts @@ -28,7 +28,8 @@ export interface ManagedProviderMeta { export const MANAGED_PROVIDER_META: Record = { openai: { name: "openai", - managed: false, + managed: true, + proxyPath: "/v1/runtime-proxy/openai", }, anthropic: { name: "anthropic", diff --git a/assistant/src/providers/openai/responses-provider.ts b/assistant/src/providers/openai/responses-provider.ts index f6f49231181..017c7e79d8f 100644 --- a/assistant/src/providers/openai/responses-provider.ts +++ b/assistant/src/providers/openai/responses-provider.ts @@ -20,6 +20,7 @@ export interface OpenAIResponsesProviderOptions { providerName?: string; providerLabel?: string; streamTimeoutMs?: number; + useNativeWebSearch?: boolean; } /** Map our internal effort values to the Responses API reasoning.effort parameter. @@ -77,6 +78,7 @@ export class OpenAIResponsesProvider implements Provider { private client: OpenAI; private model: string; private streamTimeoutMs: number; + private useNativeWebSearch: boolean; constructor( apiKey: string, @@ -91,6 +93,7 @@ export class OpenAIResponsesProvider implements Provider { }); this.model = model; this.streamTimeoutMs = options.streamTimeoutMs ?? 1_800_000; + this.useNativeWebSearch = options.useNativeWebSearch ?? false; } async sendMessage( @@ -133,13 +136,31 @@ export class OpenAIResponsesProvider implements Provider { } if (tools && tools.length > 0) { - params.tools = tools.map((t) => ({ - type: "function" as const, - name: t.name, - description: t.description, - parameters: t.input_schema, - strict: null, - })); + if ( + this.useNativeWebSearch && + tools.some((t) => t.name === "web_search") + ) { + const otherTools = tools.filter((t) => t.name !== "web_search"); + const mappedOther = otherTools.map((t) => ({ + type: "function" as const, + name: t.name, + description: t.description, + parameters: t.input_schema, + strict: null, + })); + const webSearchTool = { + type: "web_search_preview" as const, + }; + params.tools = [...mappedOther, webSearchTool]; + } else { + params.tools = tools.map((t) => ({ + type: "function" as const, + name: t.name, + description: t.description, + parameters: t.input_schema, + strict: null, + })); + } } const { signal: timeoutSignal, cleanup: cleanupTimeout } = @@ -154,6 +175,8 @@ export class OpenAIResponsesProvider implements Provider { >(); // Maps item_id → callId so we can look up tool calls from delta events. const itemIdToCallId = new Map(); + // Track web search call item IDs so we can emit server_tool_complete. + const webSearchCallIds: string[] = []; let finishReason = "unknown"; let responseModel = modelOverride ?? this.model; let inputTokens = 0; @@ -197,6 +220,15 @@ export class OpenAIResponsesProvider implements Provider { args: "", }); itemIdToCallId.set(itemId, callId); + } else if (item?.type === "web_search_call") { + const toolUseId = item.id ?? ""; + webSearchCallIds.push(toolUseId); + onEvent?.({ + type: "server_tool_start", + name: "web_search", + toolUseId, + input: {}, + }); } break; } @@ -250,6 +282,14 @@ export class OpenAIResponsesProvider implements Provider { } finishReason = response.status ?? "completed"; } + // Emit server_tool_complete for any web search calls that were started. + for (const toolUseId of webSearchCallIds) { + onEvent?.({ + type: "server_tool_complete", + toolUseId, + isError: false, + }); + } break; } } @@ -258,8 +298,28 @@ export class OpenAIResponsesProvider implements Provider { cleanupTimeout(); } - // Build content blocks + // Build content blocks. + // Inject server_tool_use + web_search_tool_result pairs before text so + // conversation history matches the shape Anthropic produces for native + // web search. The paired result block prevents repairHistory() from + // treating completed searches as interrupted (which would inject a + // synthetic web_search_tool_result_error and corrupt history). OpenAI + // weaves search results into the text output, so the result content is + // an empty array — the actual results are in the text block that follows. const content: ContentBlock[] = []; + for (const toolUseId of webSearchCallIds) { + content.push({ + type: "server_tool_use", + id: toolUseId, + name: "web_search", + input: {}, + }); + content.push({ + type: "web_search_tool_result", + tool_use_id: toolUseId, + content: [], + }); + } if (contentText) { content.push({ type: "text", text: contentText }); } diff --git a/assistant/src/providers/registry.ts b/assistant/src/providers/registry.ts index 52273b9bb2c..b0290aa851c 100644 --- a/assistant/src/providers/registry.ts +++ b/assistant/src/providers/registry.ts @@ -175,6 +175,7 @@ export async function initializeProviders( "openai", new RetryProvider( new OpenAIResponsesProvider(openaiCreds.apiKey, model, { + useNativeWebSearch, streamTimeoutMs, ...(openaiCreds.baseURL ? { baseURL: openaiCreds.baseURL } : {}), }), diff --git a/assistant/src/runtime/routes/llm-context-normalization.ts b/assistant/src/runtime/routes/llm-context-normalization.ts index 09d90997fbf..708fbf68980 100644 --- a/assistant/src/runtime/routes/llm-context-normalization.ts +++ b/assistant/src/runtime/routes/llm-context-normalization.ts @@ -400,7 +400,8 @@ function normalizeOpenAiResponsesResponsePayload( output.some( (item) => asString(item.type) === "message" || - asString(item.type) === "function_call", + asString(item.type) === "function_call" || + asString(item.type) === "web_search_call", ); if (!hasResponsesSignal) { return null; @@ -444,6 +445,23 @@ function normalizeOpenAiResponsesResponsePayload( }; toolCallSections.push(section); responseSections.push(section); + continue; + } + + if (itemType === "web_search_call") { + toolCallIndex++; + const status = asString(item.status); + const section: LlmContextSection = { + kind: "tool_use", + label: `Response tool call ${toolCallIndex}`, + role: "assistant", + toolName: "web_search", + data: omitRecordKeys(item, ["type"]) ?? undefined, + text: status ? `[Web search: ${status}]` : "[Web search]", + }; + toolCallSections.push(section); + responseSections.push(section); + continue; } } @@ -1037,6 +1055,10 @@ function extractOpenAiResponsesRequestToolNames(tools: unknown): string[] { if (asString(tool.type) === "function" && asString(tool.name)) { return asString(tool.name); } + // Native web search tool: { type: "web_search_preview" } + if (asString(tool.type) === "web_search_preview") { + return "web_search"; + } return undefined; }) .filter((name): name is string => typeof name === "string"); diff --git a/clients/macos/vellum-assistant/Features/Settings/InferenceServiceCard.swift b/clients/macos/vellum-assistant/Features/Settings/InferenceServiceCard.swift index 338914202a3..d6fd647146b 100644 --- a/clients/macos/vellum-assistant/Features/Settings/InferenceServiceCard.swift +++ b/clients/macos/vellum-assistant/Features/Settings/InferenceServiceCard.swift @@ -4,9 +4,9 @@ import VellumAssistantShared /// Card for the inference service with Managed/Your Own mode toggle. /// /// Shows different content based on mode and auth state: -/// - **Managed + logged in**: Model picker, Save button +/// - **Managed + logged in**: Provider picker (managed-capable only), model picker, Save button /// - **Managed + not logged in**: Empty state prompting login -/// - **Your Own**: Provider picker, API key field, model picker, Save + Reset buttons +/// - **Your Own**: Provider picker (all), API key field, model picker, Save + Reset buttons @MainActor struct InferenceServiceCard: View { @ObservedObject var store: SettingsStore @@ -51,20 +51,32 @@ struct InferenceServiceCard: View { authManager.isAuthenticated } - /// True when changing inference mode would invalidate the current web search config. + /// True when changing inference mode/provider would invalidate the current web search config. private var wouldInvalidateWebSearch: Bool { let modeChanging = draftMode != store.inferenceMode - guard modeChanging else { return false } + let providerChanging = draftProvider != store.selectedInferenceProvider + guard modeChanging || providerChanging else { return false } // Switching to Your Own inference while web search is Managed // (managed web search requires managed inference). - if draftMode == "your-own" && store.webSearchMode == "managed" { + if modeChanging && draftMode == "your-own" && store.webSearchMode == "managed" { return true } - // Switching to Managed inference while web search uses Provider Native - // (Provider Native requires Your Own inference). - if draftMode == "managed" && store.webSearchProvider == "inference-provider-native" { - return true + // Switching to Managed inference while web search uses Provider Native — + // only invalidate when the resulting provider cannot support native web search. + // Skip when web search is in managed mode (webSearchProvider is stale). + if draftMode == "managed" && store.webSearchMode == "your-own" && store.webSearchProvider == "inference-provider-native" { + if !store.isNativeWebSearchCapable(draftProvider) { + return true + } + } + // Switching providers while web search uses Provider Native — invalidate + // when the new provider cannot support native web search. + // Skip when web search is in managed mode (webSearchProvider is stale). + if providerChanging && store.webSearchMode == "your-own" && store.webSearchProvider == "inference-provider-native" { + if !store.isNativeWebSearchCapable(draftProvider) { + return true + } } return false } @@ -82,26 +94,26 @@ struct InferenceServiceCard: View { let modeChanged = draftMode != store.inferenceMode let hasNewKey = draftMode == "your-own" && !apiKeyText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty let modelChanged = draftModel != initialModel - let effectiveDraftProvider = draftMode == "managed" ? "anthropic" : draftProvider - let providerChanged = effectiveDraftProvider != initialProvider + let providerChanged = draftProvider != initialProvider return modeChanged || hasNewKey || modelChanged || providerChanged } var body: some View { ServiceModeCard( title: "Inference", - subtitle: draftMode == "managed" - ? "Configure which model to use to power your assistant" - : "Configure which LLM provider and model to use to power your assistant", + subtitle: "Configure which LLM provider and model to use to power your assistant", draftMode: $draftMode, managedContent: { if isLoggedIn { - PickerWithInlineSave( - hasChanges: hasChanges, - isSaving: store.apiKeySaving, - onSave: { save() } - ) { - modelPicker + VStack(alignment: .leading, spacing: VSpacing.sm) { + managedProviderPicker + PickerWithInlineSave( + hasChanges: hasChanges, + isSaving: store.apiKeySaving, + onSave: { save() } + ) { + modelPicker + } } } else { managedLoginPrompt @@ -157,13 +169,15 @@ struct InferenceServiceCard: View { // Symmetric case: if the user is authenticated and the mode is // still the default "your-own", switch to "managed" so signed-in // users get managed inference out of the box — but only when the - // provider requires an API key and the user hasn't configured one. - // Providers like Ollama that don't use keys (apiKeyPlaceholder is - // nil) are left alone since the user intentionally set up a local - // provider. + // provider is managed-capable, requires an API key, and the user + // hasn't configured one. Providers like Ollama that don't use keys + // (apiKeyPlaceholder is nil) or non-managed providers (fireworks, + // openrouter) are left alone since the user intentionally set up + // that provider. let providerRequiresKey = store.dynamicProviderApiKeyPlaceholder(draftProvider) != nil let hasLocalKey = APIKeyManager.getKey(for: draftProvider) != nil - if isLoggedIn && draftMode == "your-own" && providerRequiresKey && !hasLocalKey { + let providerIsManagedCapable = store.isManagedCapable(draftProvider) + if isLoggedIn && draftMode == "your-own" && providerIsManagedCapable && providerRequiresKey && !hasLocalKey { draftMode = "managed" store.setInferenceMode("managed") } @@ -190,12 +204,13 @@ struct InferenceServiceCard: View { // mode that onAppear may have temporarily overridden. draftMode = "managed" } else if isAuthenticated && store.inferenceMode == "your-own" { - // When a user signs in and has no BYO key for a key-based - // provider, default to managed. Keyless providers (e.g. Ollama) - // are left in your-own mode. + // When a user signs in and has no BYO key for a managed-capable, + // key-based provider, default to managed. Keyless providers + // (e.g. Ollama) and non-managed providers are left in your-own mode. let requiresKey = store.dynamicProviderApiKeyPlaceholder(draftProvider) != nil let hasLocalKey = APIKeyManager.getKey(for: draftProvider) != nil - if requiresKey && !hasLocalKey { + let isManagedCapable = store.isManagedCapable(draftProvider) + if isManagedCapable && requiresKey && !hasLocalKey { draftMode = "managed" store.setInferenceMode("managed") } @@ -256,11 +271,19 @@ struct InferenceServiceCard: View { } .onChange(of: draftMode) { _, newMode in if newMode == "managed" { - let anthropicModels = store.dynamicProviderModels("anthropic") - let isCurrentModelAnthropic = anthropicModels.contains { $0.id == draftModel } - if !isCurrentModelAnthropic { - let defaultModel = store.dynamicProviderDefaultModel("anthropic") - draftModel = defaultModel.isEmpty ? "claude-opus-4-7" : defaultModel + // When switching to managed mode, fall back to a managed-capable + // provider if the current one does not support managed routing. + if !store.isManagedCapable(draftProvider) { + draftProvider = "anthropic" + } + // Validate the model against the selected managed provider's catalog. + let managedModels = store.dynamicProviderModels(draftProvider) + let isCurrentModelValid = managedModels.contains { $0.id == draftModel } + if !isCurrentModelValid { + let defaultModel = store.dynamicProviderDefaultModel(draftProvider) + draftModel = defaultModel.isEmpty + ? (managedModels.first?.id ?? "") + : defaultModel } } else if newMode == "your-own" { let providerModels = store.dynamicProviderModels(draftProvider) @@ -281,7 +304,7 @@ struct InferenceServiceCard: View { Button("Continue") { performSave() } } message: { Text( - "Changing your inference mode will also update your Web Search settings." + "Changing your inference settings will also update your Web Search settings." + " You'll need to review and save them below." ) } @@ -327,6 +350,22 @@ struct InferenceServiceCard: View { } } + /// Provider picker filtered to managed-capable providers, shown in managed mode. + private var managedProviderPicker: some View { + VStack(alignment: .leading, spacing: VSpacing.sm) { + Text("Provider") + .font(VFont.labelDefault) + .foregroundStyle(VColor.contentSecondary) + VDropdown( + placeholder: "Select a provider\u{2026}", + selection: $draftProvider, + options: store.managedCapableProviders.map { entry in + (label: entry.displayName, value: entry.id) + } + ) + } + } + // MARK: - API Key Field private var apiKeyField: some View { @@ -356,11 +395,10 @@ struct InferenceServiceCard: View { /// Per-provider catalog model dropdown. private var providerModelPicker: some View { - let provider = draftMode == "managed" ? "anthropic" : draftProvider - return VDropdown( + VDropdown( placeholder: "Select a model\u{2026}", selection: $draftModel, - options: store.dynamicProviderModels(provider).map { model in + options: store.dynamicProviderModels(draftProvider).map { model in (label: model.displayName, value: model.id) } ) @@ -390,16 +428,10 @@ struct InferenceServiceCard: View { // changed — switching between managed and your-own implies a // provider change even if the resolved provider ID happens to // match initialProvider (ensures config stays consistent). - let persistProvider = draftMode == "managed" ? "anthropic" : draftProvider - let providerChanged = persistProvider != initialProvider || modeChanged - let pendingProvider = providerChanged ? store.setInferenceProvider(persistProvider) : nil + let providerChanged = draftProvider != initialProvider || modeChanged + let pendingProvider = providerChanged ? store.setInferenceProvider(draftProvider) : nil if providerChanged { - initialProvider = persistProvider - } - // Normalize draftProvider to match what was persisted so hasChanges - // (which compares draftProvider against initialProvider) stays in sync. - if draftProvider != persistProvider { - draftProvider = persistProvider + initialProvider = draftProvider } // Persist API key if entered and in your-own mode. @@ -420,12 +452,12 @@ struct InferenceServiceCard: View { // daemon's read-modify-write cycle for the model doesn't overwrite them. store.selectedModel = draftModel let capturedModel = draftModel - let saveProvider = draftMode == "managed" ? "anthropic" : draftProvider + let capturedProvider = draftProvider let forceSend = modeChanged Task { if let pendingMode { _ = await pendingMode.value } if let pendingProvider { _ = await pendingProvider.value } - store.setModel(capturedModel, provider: saveProvider, force: forceSend) + store.setModel(capturedModel, provider: capturedProvider, force: forceSend) } initialModel = draftModel } diff --git a/clients/macos/vellum-assistant/Features/Settings/SettingsStore.swift b/clients/macos/vellum-assistant/Features/Settings/SettingsStore.swift index 584fdaf8b20..579ef94d84b 100644 --- a/clients/macos/vellum-assistant/Features/Settings/SettingsStore.swift +++ b/clients/macos/vellum-assistant/Features/Settings/SettingsStore.swift @@ -1063,6 +1063,36 @@ public final class SettingsStore: ObservableObject { providerCatalog.first { $0.id == provider }?.apiKeyPlaceholder } + // MARK: - Provider Capability Helpers + + /// Provider IDs that support managed proxy routing (i.e., can be used in managed mode). + /// Mirrors the `MANAGED_PROVIDER_META` table in the backend. + private static let managedCapableProviderIds: Set = ["anthropic", "openai", "gemini"] + + /// Provider IDs that support native web search (inference-provider-native). + /// Anthropic and OpenAI pass `useNativeWebSearch` to their providers; others do not. + private static let nativeWebSearchCapableProviderIds: Set = ["anthropic", "openai"] + + /// Returns the catalog entries for providers that support managed proxy routing. + var managedCapableProviders: [ProviderCatalogEntry] { + providerCatalog.filter { Self.managedCapableProviderIds.contains($0.id) } + } + + /// Returns the catalog entries for providers that support native web search. + var nativeWebSearchCapableProviders: [ProviderCatalogEntry] { + providerCatalog.filter { Self.nativeWebSearchCapableProviderIds.contains($0.id) } + } + + /// Whether a given provider supports managed proxy routing. + func isManagedCapable(_ provider: String) -> Bool { + Self.managedCapableProviderIds.contains(provider) + } + + /// Whether a given provider supports native web search. + func isNativeWebSearchCapable(_ provider: String) -> Bool { + Self.nativeWebSearchCapableProviderIds.contains(provider) + } + // MARK: - Embedding Config Actions func refreshEmbeddingConfig() { diff --git a/clients/macos/vellum-assistant/Features/Settings/WebSearchServiceCard.swift b/clients/macos/vellum-assistant/Features/Settings/WebSearchServiceCard.swift index 8315ba0500a..d246baf28cc 100644 --- a/clients/macos/vellum-assistant/Features/Settings/WebSearchServiceCard.swift +++ b/clients/macos/vellum-assistant/Features/Settings/WebSearchServiceCard.swift @@ -7,8 +7,9 @@ import VellumAssistantShared /// - **Managed + Managed inference + logged in**: Message that web search is included. /// - **Managed + Managed inference + not logged in**: Login prompt. /// - **Managed + Your Own inference**: Message that managed web search is not yet available. -/// - **Your Own + Your Own inference**: Provider picker (Provider Native, Perplexity, Brave) + API key. -/// - **Your Own + Managed inference**: Provider picker (Perplexity, Brave only) + API key. +/// - **Your Own**: Provider picker + API key. Provider Native is available whenever the +/// inference provider supports native web search (e.g. Anthropic, OpenAI), regardless of +/// inference mode. Perplexity and Brave are always available as key-based alternatives. @MainActor struct WebSearchServiceCard: View { @ObservedObject var store: SettingsStore @@ -44,10 +45,11 @@ struct WebSearchServiceCard: View { authManager.isAuthenticated } - /// The available providers depend on the current inference mode. - /// Provider Native requires Your Own inference (it uses the user's own API key). + /// The available providers depend on the current inference provider's capabilities. + /// Provider Native is available whenever the inference provider supports native web search + /// (e.g. Anthropic, OpenAI), regardless of whether inference is managed or your-own. private var availableProviders: [String] { - store.inferenceMode == "your-own" + store.isNativeWebSearchCapable(store.selectedInferenceProvider) ? ["inference-provider-native", "perplexity", "brave"] : ["perplexity", "brave"] } @@ -156,7 +158,16 @@ struct WebSearchServiceCard: View { draftMode = "your-own" } if newValue == "managed" && draftProvider == "inference-provider-native" { - // Provider Native requires Your Own inference. + // Only auto-correct when the managed provider lacks native web search support. + if !store.isNativeWebSearchCapable(store.selectedInferenceProvider) { + draftProvider = "perplexity" + } + } + } + .onChange(of: store.selectedInferenceProvider) { _, newProvider in + // Auto-correct when the inference provider changes to one that + // does not support native web search while provider-native is selected. + if draftProvider == "inference-provider-native" && !store.isNativeWebSearchCapable(newProvider) { draftProvider = "perplexity" } } diff --git a/clients/macos/vellum-assistantTests/SettingsStoreManagedInferenceSelectionTests.swift b/clients/macos/vellum-assistantTests/SettingsStoreManagedInferenceSelectionTests.swift new file mode 100644 index 00000000000..c971e8e3a43 --- /dev/null +++ b/clients/macos/vellum-assistantTests/SettingsStoreManagedInferenceSelectionTests.swift @@ -0,0 +1,243 @@ +import XCTest +@testable import VellumAssistantLib +@testable import VellumAssistantShared + +/// Tests for SettingsStore provider capability helpers and managed-mode +/// provider selection behavior. +@MainActor +final class SettingsStoreManagedInferenceSelectionTests: XCTestCase { + + private var store: SettingsStore! + + override func setUp() { + super.setUp() + store = SettingsStore(settingsClient: MockSettingsClient()) + } + + override func tearDown() { + store = nil + super.tearDown() + } + + // MARK: - isManagedCapable + + func testAnthropicIsManagedCapable() { + XCTAssertTrue(store.isManagedCapable("anthropic")) + } + + func testOpenAIIsManagedCapable() { + XCTAssertTrue(store.isManagedCapable("openai")) + } + + func testGeminiIsManagedCapable() { + XCTAssertTrue(store.isManagedCapable("gemini")) + } + + func testOllamaIsNotManagedCapable() { + XCTAssertFalse(store.isManagedCapable("ollama")) + } + + func testFireworksIsNotManagedCapable() { + XCTAssertFalse(store.isManagedCapable("fireworks")) + } + + func testOpenRouterIsNotManagedCapable() { + XCTAssertFalse(store.isManagedCapable("openrouter")) + } + + func testUnknownProviderIsNotManagedCapable() { + XCTAssertFalse(store.isManagedCapable("unknown-provider")) + } + + // MARK: - isNativeWebSearchCapable + + func testAnthropicIsNativeWebSearchCapable() { + XCTAssertTrue(store.isNativeWebSearchCapable("anthropic")) + } + + func testOpenAIIsNativeWebSearchCapable() { + XCTAssertTrue(store.isNativeWebSearchCapable("openai")) + } + + func testGeminiIsNotNativeWebSearchCapable() { + XCTAssertFalse(store.isNativeWebSearchCapable("gemini")) + } + + func testOllamaIsNotNativeWebSearchCapable() { + XCTAssertFalse(store.isNativeWebSearchCapable("ollama")) + } + + // MARK: - managedCapableProviders + + func testManagedCapableProvidersContainsExpectedEntries() { + let ids = store.managedCapableProviders.map(\.id) + XCTAssertTrue(ids.contains("anthropic"), "expected anthropic in managed-capable providers") + XCTAssertTrue(ids.contains("openai"), "expected openai in managed-capable providers") + XCTAssertTrue(ids.contains("gemini"), "expected gemini in managed-capable providers") + } + + func testManagedCapableProvidersExcludesNonManagedEntries() { + let ids = store.managedCapableProviders.map(\.id) + XCTAssertFalse(ids.contains("ollama"), "ollama should not be in managed-capable providers") + XCTAssertFalse(ids.contains("fireworks"), "fireworks should not be in managed-capable providers") + XCTAssertFalse(ids.contains("openrouter"), "openrouter should not be in managed-capable providers") + } + + // MARK: - nativeWebSearchCapableProviders + + func testNativeWebSearchCapableProvidersContainsExpectedEntries() { + let ids = store.nativeWebSearchCapableProviders.map(\.id) + XCTAssertTrue(ids.contains("anthropic"), "expected anthropic in native-web-search-capable providers") + XCTAssertTrue(ids.contains("openai"), "expected openai in native-web-search-capable providers") + } + + func testNativeWebSearchCapableProvidersExcludesOthers() { + let ids = store.nativeWebSearchCapableProviders.map(\.id) + XCTAssertFalse(ids.contains("gemini"), "gemini should not be in native-web-search-capable providers") + XCTAssertFalse(ids.contains("ollama"), "ollama should not be in native-web-search-capable providers") + } + + // MARK: - Managed Provider Persistence + + func testManagedModeCanPersistOpenAIAsProvider() { + let mockClient = MockSettingsClient() + mockClient.patchConfigResponse = true + let testStore = SettingsStore(settingsClient: mockClient) + + // Simulate selecting OpenAI in managed mode + testStore.selectedInferenceProvider = "openai" + testStore.inferenceMode = "managed" + + // Persist the provider selection + _ = testStore.setInferenceProvider("openai") + + // Wait for the async patch to be captured + let predicate = NSPredicate { _, _ in + mockClient.patchConfigCalls.count >= 1 + } + let expectation = XCTNSPredicateExpectation(predicate: predicate, object: nil) + wait(for: [expectation], timeout: 2.0) + + // Verify the patched provider is "openai", not "anthropic" + let providerPatches = mockClient.patchConfigCalls.compactMap { call -> String? in + guard let services = call["services"] as? [String: Any], + let inference = services["inference"] as? [String: Any], + let provider = inference["provider"] as? String else { + return nil + } + return provider + } + XCTAssertTrue(providerPatches.contains("openai"), + "expected openai to be persisted as the inference provider, got: \(providerPatches)") + } + + func testManagedModeCanPersistGeminiAsProvider() { + let mockClient = MockSettingsClient() + mockClient.patchConfigResponse = true + let testStore = SettingsStore(settingsClient: mockClient) + + testStore.selectedInferenceProvider = "gemini" + testStore.inferenceMode = "managed" + _ = testStore.setInferenceProvider("gemini") + + let predicate = NSPredicate { _, _ in + mockClient.patchConfigCalls.count >= 1 + } + let expectation = XCTNSPredicateExpectation(predicate: predicate, object: nil) + wait(for: [expectation], timeout: 2.0) + + let providerPatches = mockClient.patchConfigCalls.compactMap { call -> String? in + guard let services = call["services"] as? [String: Any], + let inference = services["inference"] as? [String: Any], + let provider = inference["provider"] as? String else { + return nil + } + return provider + } + XCTAssertTrue(providerPatches.contains("gemini"), + "expected gemini to be persisted as the inference provider, got: \(providerPatches)") + } + + // MARK: - Managed Provider + Native Web Search Capability Gating + + func testManagedOpenAIPlusProviderNativeIsValid() { + // OpenAI is both managed-capable and native-web-search-capable, + // so managed inference + inference-provider-native should be allowed. + XCTAssertTrue(store.isManagedCapable("openai")) + XCTAssertTrue(store.isNativeWebSearchCapable("openai")) + } + + func testManagedAnthropicPlusProviderNativeIsValid() { + // Anthropic is both managed-capable and native-web-search-capable. + XCTAssertTrue(store.isManagedCapable("anthropic")) + XCTAssertTrue(store.isNativeWebSearchCapable("anthropic")) + } + + func testManagedGeminiPlusProviderNativeIsInvalid() { + // Gemini is managed-capable but NOT native-web-search-capable, + // so managed Gemini + inference-provider-native should be rejected. + XCTAssertTrue(store.isManagedCapable("gemini")) + XCTAssertFalse(store.isNativeWebSearchCapable("gemini")) + } + + func testManagedOpenAIProviderNativeWebSearchCanBePersisted() { + let mockClient = MockSettingsClient() + mockClient.patchConfigResponse = true + let testStore = SettingsStore(settingsClient: mockClient) + + // Configure managed OpenAI inference + provider-native web search + testStore.selectedInferenceProvider = "openai" + testStore.inferenceMode = "managed" + testStore.webSearchProvider = "inference-provider-native" + + // Persist the web search provider + testStore.setWebSearchProvider("inference-provider-native") + + let predicate = NSPredicate { _, _ in + mockClient.patchConfigCalls.count >= 1 + } + let expectation = XCTNSPredicateExpectation(predicate: predicate, object: nil) + wait(for: [expectation], timeout: 2.0) + + // Verify inference-provider-native was persisted (not rewritten to perplexity) + let webSearchPatches = mockClient.patchConfigCalls.compactMap { call -> String? in + guard let services = call["services"] as? [String: Any], + let webSearch = services["web-search"] as? [String: Any], + let provider = webSearch["provider"] as? String else { + return nil + } + return provider + } + XCTAssertTrue(webSearchPatches.contains("inference-provider-native"), + "expected inference-provider-native to be persisted with managed OpenAI, got: \(webSearchPatches)") + } + + func testNonNativeWebSearchCapableProviderFallsBackToPerplexity() { + // When the inference provider doesn't support native web search, + // isNativeWebSearchCapable should return false, indicating the UI + // should enforce fallback to perplexity or brave. + XCTAssertFalse(store.isNativeWebSearchCapable("gemini")) + XCTAssertFalse(store.isNativeWebSearchCapable("ollama")) + XCTAssertFalse(store.isNativeWebSearchCapable("fireworks")) + XCTAssertFalse(store.isNativeWebSearchCapable("openrouter")) + } + + // MARK: - Model Validation Against Selected Provider + + func testOpenAIModelsAreAvailableForOpenAIProvider() { + let models = store.dynamicProviderModels("openai") + XCTAssertFalse(models.isEmpty, "expected OpenAI to have models in the default catalog") + // Verify these are OpenAI models (not Anthropic) + let modelIds = models.map(\.id) + XCTAssertTrue(modelIds.allSatisfy { !$0.hasPrefix("claude-") }, + "OpenAI models should not contain claude model IDs") + } + + func testAnthropicModelsAreAvailableForAnthropicProvider() { + let models = store.dynamicProviderModels("anthropic") + XCTAssertFalse(models.isEmpty, "expected Anthropic to have models in the default catalog") + let modelIds = models.map(\.id) + XCTAssertTrue(modelIds.allSatisfy { $0.hasPrefix("claude-") }, + "Anthropic models should all be claude models") + } +}