diff --git a/docs/api-reference/veryfront/agent.md b/docs/api-reference/veryfront/agent.md index 20c3bf6d21..59863929ff 100644 --- a/docs/api-reference/veryfront/agent.md +++ b/docs/api-reference/veryfront/agent.md @@ -761,7 +761,7 @@ Input delivered to a hosted agent-service detached execution callback. | `executeHostedDurableChatRun` | Execute hosted durable chat run. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/durable-chat-run-start.ts#L232) | | `executeHostedDurableChildFork` | Execute hosted durable child fork. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/durable-child-fork-execution.ts#L807) | | `executeHostedLocalChildInvoke` | Execute hosted local child invoke. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/durable-child-fork-execution.ts#L414) | -| `expandAllowedRemoteToolNames` | Normalize allowed remote tool names without adding undeclared provider-native tools. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/provider-native-tool-inventory.ts#L62) | +| `expandAllowedRemoteToolNames` | Normalize allowed remote tool names without adding undeclared provider-native tools. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/provider-native-tool-inventory.ts#L82) | | `expandHostedChildRequestedTools` | Expand hosted child requested tools helper. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/child-requested-tools.ts#L87) | | `extractChatMessageMetadata` | Extract chat message metadata. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/chat/chat-ui-message-helpers.ts#L302) | | `extractLatestUserText` | Extract latest user text. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/artifacts/default-research-artifact-support.ts#L68) | @@ -803,7 +803,7 @@ Input delivered to a hosted agent-service detached execution callback. | `getConversationRun` | Return conversation run. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/conversation/durable.ts#L1065) | | `getConversationRunEventJsonByteLength` | Return conversation run event JSON byte length. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/conversation/run-event-normalization.ts#L27) | | `getEmptyHostedFinalizedMessageTerminalError` | Error shape for get empty hosted finalized message terminal. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/stream-terminal-error.ts#L113) | -| `getForkRuntimeAllowedToolNames` | Return fork runtime allowed tool names. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/provider-native-tool-inventory.ts#L69) | +| `getForkRuntimeAllowedToolNames` | Return fork runtime allowed tool names. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/provider-native-tool-inventory.ts#L89) | | `getForwardedHostedModelId` | Return forwarded hosted model ID. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/runtime-request-config.ts#L86) | | `getForwardedHostedRuntimeOverrides` | Return forwarded hosted runtime overrides. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/runtime-request-config.ts#L96) | | `getHostedChildWrittenArtifactPath` | Return hosted child written artifact path. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/hosted/child-artifact-support.ts#L112) | @@ -814,7 +814,7 @@ Input delivered to a hosted agent-service detached execution callback. | `getMaxForkRuntimeStepCount` | Return max fork runtime step count. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/streaming/fork-runtime-stream.ts#L445) | | `getProjectAgentRuntimeAgentIdCandidates` | Return project agent runtime agent ID candidates. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/project/agent-runtime.ts#L223) | | `getProjectSteeringMutation` | Return project steering mutation. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/project/steering-mutation.ts#L112) | -| `getProviderNativeToolNames` | Return provider native tool names. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/provider-native-tool-inventory.ts#L48) | +| `getProviderNativeToolNames` | Return provider native tool names. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/provider-native-tool-inventory.ts#L53) | | `getProviderToolProfile` | Return provider tool profile. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/provider-tool-compat.ts#L59) | | `getRuntimeAgentMarkdownDefinition` | Definition for get runtime agent markdown. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/agent-markdown-adapter.ts#L45) | | `getRuntimeProjectFile` | Return runtime project file. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/project-files-client.ts#L404) | @@ -1719,7 +1719,7 @@ Input delivered to a hosted agent-service detached execution callback. | `ProjectSteeringMutationInput` | Input payload for project steering mutation. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/project/steering-mutation.ts#L29) | | `ProjectSteeringMutationResult` | Result returned from project steering mutation. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/project/steering-mutation.ts#L38) | | `ProjectSteeringPaths` | Public API contract for project steering paths. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/project/steering-mutation.ts#L23) | -| `ProviderNativeToolInventoryOptions` | Options accepted by provider native tool inventory. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/provider-native-tool-inventory.ts#L11) | +| `ProviderNativeToolInventoryOptions` | Options accepted by provider native tool inventory. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/provider-native-tool-inventory.ts#L16) | | `ProviderToolCompatOptions` | Options accepted by provider tool compat. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/provider-tool-compat.ts#L20) | | `ProviderToolCompatProvider` | Public API contract for provider tool compat provider. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/provider-tool-compat.ts#L5) | | `ProviderToolProfile` | Public API contract for provider tool profile. | [source](https://github.com/veryfront/veryfront-code/blob/main/src/agent/runtime/provider-tool-compat.ts#L13) | diff --git a/docs/architecture/28-model-driven-tool-discovery.md b/docs/architecture/28-model-driven-tool-discovery.md index 046f697d2f..7ded305206 100644 --- a/docs/architecture/28-model-driven-tool-discovery.md +++ b/docs/architecture/28-model-driven-tool-discovery.md @@ -38,11 +38,13 @@ example. Read it in [`chat-runtime-tool-assembly.ts`](../../src/agent/hosted/chat-runtime-tool-assembly.ts). - **deferred**: the agent has no explicit `tools` binding. The model initially - sees only `tool_search` plus whichever bootstrap tools (`form_input`, - `load_skill`) the run actually authorizes. Bootstrap tools are filtered - against the authorized set, so a run that authorizes neither exposes - `tool_search` alone. That is why the measurement below reports one initially - exposed tool. + sees only `tool_search` plus the `load_skill` bootstrap tool when the run + authorizes it. `form_input` remains authorized but deferred until a search + loads it. Bootstrap tools are filtered against the authorized set, so a run + that does not authorize `load_skill` exposes `tool_search` alone. That is why + the measurement below reports one initially exposed tool. Its deterministic + fixture authorizes exactly 64 generated tools and does not authorize + `load_skill`. - **eager**: the agent declares a binding. The bound set is exposed directly and `selectProviderCompatibleToolNames` still applies, so a binding larger than the provider cap is still truncated in alphabetical order after local tools @@ -78,14 +80,19 @@ mechanism. constants in `tool-exposure.ts`. - Matching schemas are loaded into `ToolExposureState.loadedToolNames` and are callable from the next model step. +- Configured provider-native tools supported by the selected model enter the + authorized catalog as schema-free name and description records. A matching + search attaches the provider's native schema on the next model step. The + runtime never treats that record as a local executable tool. ## Authorization: two independent gates 1. **Discovery**: `tool_search` searches only the authorized catalog. -2. **Execution**: unchanged. `prepareExecution` in - `project-scoped-remote-tools.ts` re-checks allowance at call time via - `isRemoteToolNameAllowed`, so a schema reaching a request is not sufficient - to execute. +2. **Execution**: `prepareExecution` in + `project-scoped-remote-tools.ts` re-checks remote allowance at call time via + `isRemoteToolNameAllowed`. Provider-native exposure is intersected with the + current configured provider tools and selected model support before each + step. A schema reaching an earlier request is not sufficient to execute. ## Durability and resume diff --git a/docs/guides/agents.md b/docs/guides/agents.md index c5c8f3c465..22fdb20dcc 100644 --- a/docs/guides/agents.md +++ b/docs/guides/agents.md @@ -154,11 +154,15 @@ const assistant = agent({ ``` The framework `tool_search` fallback is provider-neutral. It searches the -authorized `tools` catalog and does not search `providerTools`. Search ranks an -exact tool name first, followed by normalized substrings in the tool name, -description, and input parameter descriptions. It returns at most five names -and descriptions. Results never include schemas, and `tool_search` has no -pagination options. +authorized `tools` catalog and configured `providerTools` that the selected +model supports. Provider-native entries contain only a name and description +until a search loads them. The runtime attaches the provider's native schema on +the next model step. + +Search ranks an exact tool name first, followed by normalized substrings in the +tool name, description, and input parameter descriptions. It returns at most +five names and descriptions. Results never include schemas, and `tool_search` +has no pagination options. Loading a schema never authorizes a tool. The runtime rechecks authorization before execution. It also filters restored loaded-tool state against the @@ -168,8 +172,9 @@ You can use deferred loading with a direct provider and its API key without Veryfront Cloud. Hosted durable runs additionally require the Veryfront API durable run-event contract. The hosted runtime stores loaded-tool state in a private checkpoint and waits for that checkpoint before continuing. Private -checkpoint data does not appear in public messages or replay. Provider-native -tool search and provider replay are not part of this feature. +checkpoint data does not appear in public messages or replay. Configured, +supported provider-native tools use the same private exposure checkpoint. +Provider replay is not part of this feature. See [Tools](./tools.md#how-agents-use-tools) for the search and execution flow. diff --git a/docs/guides/tools.md b/docs/guides/tools.md index 82364baa77..4d235698d4 100644 --- a/docs/guides/tools.md +++ b/docs/guides/tools.md @@ -66,11 +66,12 @@ Use this pattern to verify the tool contract before giving the tool to an agent. ## How agents use tools An explicit tool map authorizes only the selected tools and sends those schemas -to the model immediately. Use `tools: true` for a broad authorized scope: the -framework initially sends only bootstrap schemas and `tool_search`, then loads -matching schemas for the next model step. Omit `tools` to expose no project -tools. Search results contain names and descriptions, not input or output -schemas. +to the model immediately. Use `tools: true` for a broad authorized scope. The +framework initially sends only the `load_skill` bootstrap schema when it is +authorized, plus `tool_search`, then loads matching schemas for the next model +step. `form_input` is authorized but deferred until it is needed. Omit `tools` +to expose no project tools. Search results contain names and descriptions, not +input or output schemas. ```ts // agents/assistant.ts @@ -106,11 +107,15 @@ has an aggregate schema-work budget. A malformed, cyclic, or over-budget schema cannot abort the search; that tool simply cannot match by parameter description. Name and description matching remains available for other healthy tools. -The search only loads schemas from the current authorized `tools` catalog. It -does not search or load provider-native `providerTools`. The runtime reapplies -the current authorization policy before executing a tool and when restoring a -loaded-tool checkpoint, so a previously loaded name cannot restore a removed -permission. +The search loads schemas from the current authorized `tools` catalog. It also +loads configured `providerTools` that the selected model supports. A +provider-native search entry contains only its name and description. The +runtime attaches the provider's native schema on the next model step and never +executes it as a local tool. + +The runtime reapplies the current authorization policy, configured provider +tools, and model support before execution and checkpoint restoration, so a +previously loaded name cannot restore a removed permission. Deferred loading works with direct provider model strings and provider API keys. Veryfront Cloud is not required for that path. Hosted durable execution @@ -147,8 +152,9 @@ Agent config separates tools by execution boundary: Use `tools` for functions you define in the project. Do not add provider-native tools or skill loader tools to `tools`. -Use `providerTools` for provider-executed capabilities. Framework -`tool_search` does not search this catalog: +Use `providerTools` for provider-executed capabilities. In deferred mode, +`tool_search` can load a configured capability when the selected model supports +it: ```ts // agents/researcher.ts diff --git a/src/agent/hosted/chat-runtime-tool-assembly.test.ts b/src/agent/hosted/chat-runtime-tool-assembly.test.ts index c7a4967e7b..fe1ba625b8 100644 --- a/src/agent/hosted/chat-runtime-tool-assembly.test.ts +++ b/src/agent/hosted/chat-runtime-tool-assembly.test.ts @@ -162,7 +162,11 @@ Deno.test("prepareHostedChatRuntimeToolAssembly defers an omitted allowed tools sourceIntegrationPolicy: unrestrictedSourceIntegrationPolicy, taskContext, instructions: "Base instructions", - localTools: { sleep: localTool("Sleep") }, + localTools: { + form_input: localTool("Form input"), + load_skill: localTool("Load skill"), + sleep: localTool("Sleep"), + }, apiUrl: "https://api.example.com", apiMcpUrl: "https://api.example.com/mcp", createRemoteToolSource: remoteSourceFromConfig, @@ -170,7 +174,13 @@ Deno.test("prepareHostedChatRuntimeToolAssembly defers an omitted allowed tools }); assertEquals(toolAssembly.toolLoadingMode, "deferred"); - assertEquals(taskContext.availableToolNames, ["tool_search"]); + assertEquals(toolAssembly.availableToolNames, [ + "create_file", + "form_input", + "load_skill", + "sleep", + ]); + assertEquals(taskContext.availableToolNames, ["load_skill", "tool_search"]); }); Deno.test("prepareHostedChatRuntimeToolAssembly defers an unrestricted tools true catalog", async () => { diff --git a/src/agent/hosted/chat-runtime-tool-assembly.ts b/src/agent/hosted/chat-runtime-tool-assembly.ts index 6e071cf8a4..d49e014a0b 100644 --- a/src/agent/hosted/chat-runtime-tool-assembly.ts +++ b/src/agent/hosted/chat-runtime-tool-assembly.ts @@ -381,9 +381,7 @@ export async function prepareHostedChatRuntimeToolAssembly< const compatibleRemoteToolNames = toolLoadingMode === "deferred" ? remoteToolNames : remoteToolNames.filter((toolName) => compatibleToolNames.has(toolName)); - const bootstrapToolNames = availableToolNames.filter((toolName) => - toolName === "form_input" || toolName === "load_skill" - ); + const bootstrapToolNames = availableToolNames.filter((toolName) => toolName === "load_skill"); const hasDeferredTools = availableToolNames.length > bootstrapToolNames.length; const modelVisibleToolNames = toolLoadingMode === "deferred" ? [ diff --git a/src/agent/hosted/default-project-steering-refresh.test.ts b/src/agent/hosted/default-project-steering-refresh.test.ts index 01052ec062..c88052da38 100644 --- a/src/agent/hosted/default-project-steering-refresh.test.ts +++ b/src/agent/hosted/default-project-steering-refresh.test.ts @@ -51,7 +51,7 @@ function createRefreshInput( sourceIntegrationPolicy: normalizeSourceIntegrationPolicy(undefined), runtimeTools: {}, remoteToolSources: [], - localToolNames: ["load_skill", "sleep"], + localToolNames: ["form_input", "load_skill", "sleep"], remoteToolNames: [], providerToolNames: [], availableToolNames: [], diff --git a/src/agent/hosted/default-project-steering-refresh.ts b/src/agent/hosted/default-project-steering-refresh.ts index 90ff1960fe..d914bb52f6 100644 --- a/src/agent/hosted/default-project-steering-refresh.ts +++ b/src/agent/hosted/default-project-steering-refresh.ts @@ -275,9 +275,7 @@ export function createDefaultHostedProjectSteeringRefresh( model: input.taskContext.model, requiredToolNames: input.toolAssembly.localToolNames, }); - const bootstrapToolNames = toolNames.filter((toolName) => - toolName === "form_input" || toolName === "load_skill" - ); + const bootstrapToolNames = toolNames.filter((toolName) => toolName === "load_skill"); const hasDeferredTools = toolNames.length > bootstrapToolNames.length; const modelVisibleToolNames = input.toolAssembly.toolLoadingMode === "deferred" ? [ diff --git a/src/agent/runtime/agent-runtime-step.test.ts b/src/agent/runtime/agent-runtime-step.test.ts index 7d5ad07f05..9da9fb0876 100644 --- a/src/agent/runtime/agent-runtime-step.test.ts +++ b/src/agent/runtime/agent-runtime-step.test.ts @@ -59,15 +59,16 @@ describe("agent/runtime-step", () => { assertEquals( prepared.tools.map((tool) => tool.name), - ["form_input", "get_release", "load_skill", "tool_search"], + ["get_release", "load_skill", "tool_search"], ); assertEquals( prepared.toolExposurePlan.deferred.map((tool) => tool.name), - ["create_release"], + ["create_release", "form_input"], ); }); - it("keeps provider-native tools in prompt inventory but outside tool_search authorization", async () => { + it("keeps provider-native tools authorized but deferred until tool_search loads them", async () => { + const state = createToolExposureState(); const prepared = await prepareAgentRuntimeStep({ agentId: "agent_1", activeSkillToolAvailability: undefined, @@ -88,10 +89,41 @@ describe("agent/runtime-step", () => { step: 0, systemPrompt: "Base", toolContextBase: undefined, + toolExposureState: state, }); assertEquals(prepared.systemPrompt.includes("- web_search"), true); - assertEquals(prepared.toolExposurePlan.authorized.map((tool) => tool.name), ["create_release"]); + assertEquals(prepared.tools.map((tool) => tool.name), ["tool_search"]); + assertEquals( + prepared.toolExposurePlan.authorized.map((tool) => tool.name), + ["create_release", "web_search"], + ); + assertEquals( + prepared.toolExposurePlan.deferred.map((tool) => tool.name), + ["create_release", "web_search"], + ); + + state.loadedToolNames.add("web_search"); + const loaded = await prepareAgentRuntimeStep({ + agentId: "agent_1", + activeSkillToolAvailability: undefined, + allowedRemoteToolNames: undefined, + config: { model: "anthropic/claude-opus-4-6", system: "Base", tools: true } as AgentConfig, + forwardedRemoteToolDefinitions: undefined, + getAvailableTools: async () => [toolDefinition("create_release")], + supportsToolCalling: true, + messages: [], + mode: "generate", + providerToolNames: ["web_search"], + remoteToolSources: undefined, + resolveRuntimeState: async () => ({ systemPrompt: prepared.systemPrompt }), + runtimeContext: undefined, + step: 1, + systemPrompt: prepared.systemPrompt, + toolContextBase: undefined, + toolExposureState: state, + }); + assertEquals(loaded.tools.map((tool) => tool.name), ["tool_search", "web_search"]); }); it("preserves a typed empty integration catalog without adding a warning", async () => { diff --git a/src/agent/runtime/agent-runtime-step.ts b/src/agent/runtime/agent-runtime-step.ts index fab5a1b2f0..3ef4cfc6ce 100644 --- a/src/agent/runtime/agent-runtime-step.ts +++ b/src/agent/runtime/agent-runtime-step.ts @@ -23,6 +23,7 @@ import { withRuntimeToolInventory, } from "./tool-inventory.ts"; import { getProviderToolProfile } from "./provider-tool-compat.ts"; +import { createProviderNativeToolExposureDefinitions } from "./provider-native-tool-inventory.ts"; export type AgentRuntimeStepMode = "generate" | "stream"; @@ -218,6 +219,14 @@ export async function prepareAgentRuntimeStep( integrationToolDiscovery = { status: "ok", tools: usableForwardedTools }; } } + const existingToolNames = new Set(tools.map((tool) => tool.name)); + tools = [ + ...tools, + ...createProviderNativeToolExposureDefinitions({ + model: input.effectiveModel ?? input.config.model, + toolNames: input.providerToolNames ?? [], + }).filter((tool) => !existingToolNames.has(tool.name)), + ]; const toolExposureState = input.toolExposureState ?? createToolExposureState(); if (input.toolExposureCheckpoint) { const restoredState = restoreToolExposureState(input.toolExposureCheckpoint, tools); @@ -237,9 +246,7 @@ export async function prepareAgentRuntimeStep( ? flattenSystemInstructions( withRuntimeToolInventory( baseSystemPrompt, - [...toolExposurePlan.visible.map((tool) => tool.name), ...(input.providerToolNames ?? [])] - .filter((name, index, names) => names.indexOf(name) === index) - .sort(), + toolExposurePlan.visible.map((tool) => tool.name), // Naming what is deferred is the difference between a tool the model // can seek and one it has no reason to believe exists. toolExposurePlan.deferred.map((tool) => ({ diff --git a/src/agent/runtime/index.ts b/src/agent/runtime/index.ts index 670d98e832..4c99e5e5ec 100644 --- a/src/agent/runtime/index.ts +++ b/src/agent/runtime/index.ts @@ -1181,8 +1181,9 @@ export class AgentRuntime { "tool.catalog.deferred_count": preparedStep.toolExposurePlan.deferred.length, "tool.loading.path": "framework-fallback", }); + const visibleToolNames = new Set(tools.map((tool) => tool.name)); const stepProviderTools = supportsToolCalling && !agentWriteFinalResponseToolGuardEnabled - ? providerTools + ? providerTools.filter((toolName) => visibleToolNames.has(toolName)) : []; const temperature = this.resolveTemperature( @@ -1783,8 +1784,9 @@ export class AgentRuntime { "tool.catalog.deferred_count": preparedStep.toolExposurePlan.deferred.length, "tool.loading.path": "framework-fallback", }); + const visibleToolNames = new Set(tools.map((tool) => tool.name)); const stepProviderTools = supportsToolCalling && !agentWriteFinalResponseToolGuardEnabled - ? providerTools + ? providerTools.filter((toolName) => visibleToolNames.has(toolName)) : []; const runtimeTools = convertToolsToRuntimeTools(tools, { diff --git a/src/agent/runtime/provider-native-tool-inventory.test.ts b/src/agent/runtime/provider-native-tool-inventory.test.ts index 0993d875c3..43941257ae 100644 --- a/src/agent/runtime/provider-native-tool-inventory.test.ts +++ b/src/agent/runtime/provider-native-tool-inventory.test.ts @@ -3,6 +3,7 @@ import { assertEquals } from "#veryfront/testing/assert.ts"; import { describe, it } from "#veryfront/testing/bdd.ts"; import type { HostToolSet } from "#veryfront/tool"; import { + createProviderNativeToolExposureDefinitions, expandAllowedRemoteToolNames, getForkRuntimeAllowedToolNames, getProviderNativeToolNames, @@ -46,6 +47,27 @@ describe("provider-native-tool-inventory", () => { assertEquals(getProviderNativeToolNames({ model: "google/gemini-3.5-flash" }), []); }); + it("creates deterministic schema-free search entries only for configured supported tools", () => { + assertEquals( + createProviderNativeToolExposureDefinitions({ + model: "anthropic/claude-sonnet-4-6", + toolNames: ["web_search", "unknown", "web_fetch", "web_search"], + }), + [ + { + name: "web_fetch", + description: "Fetch and read the contents of a web page.", + parameters: { type: "object", properties: {} }, + }, + { + name: "web_search", + description: "Search the web for current information.", + parameters: { type: "object", properties: {} }, + }, + ], + ); + }); + it("preserves a fork/runtime allowlist without adding undeclared provider-native tools", () => { assertEquals( expandAllowedRemoteToolNames({ diff --git a/src/agent/runtime/provider-native-tool-inventory.ts b/src/agent/runtime/provider-native-tool-inventory.ts index f7538b5358..f138b97a7b 100644 --- a/src/agent/runtime/provider-native-tool-inventory.ts +++ b/src/agent/runtime/provider-native-tool-inventory.ts @@ -1,4 +1,4 @@ -import type { HostToolSet } from "#veryfront/tool"; +import type { HostToolSet, ToolDefinition } from "#veryfront/tool"; const ANTHROPIC_PROVIDER_NATIVE_TOOL_NAMES = [ "web_fetch", @@ -7,6 +7,11 @@ const ANTHROPIC_PROVIDER_NATIVE_TOOL_NAMES = [ const OPENAI_PROVIDER_NATIVE_TOOL_NAMES = ["web_search"] as const; +const PROVIDER_NATIVE_TOOL_DESCRIPTIONS: Readonly> = { + web_fetch: "Fetch and read the contents of a web page.", + web_search: "Search the web for current information.", +}; + /** Options accepted by provider native tool inventory. */ export interface ProviderNativeToolInventoryOptions { model?: string; @@ -58,6 +63,21 @@ export function getProviderNativeToolNames( } } +/** Create schema-free search entries for configured provider-native tools. */ +export function createProviderNativeToolExposureDefinitions( + options: ProviderNativeToolInventoryOptions & { toolNames: readonly string[] }, +): ToolDefinition[] { + const supported = new Set(getProviderNativeToolNames(options)); + return [...new Set(options.toolNames)] + .filter((toolName) => supported.has(toolName)) + .sort() + .map((toolName) => ({ + name: toolName, + description: PROVIDER_NATIVE_TOOL_DESCRIPTIONS[toolName] ?? "Provider-native tool.", + parameters: { type: "object", properties: {} }, + })); +} + /** Normalize allowed remote tool names without adding undeclared provider-native tools. */ export function expandAllowedRemoteToolNames( options: ExpandAllowedRemoteToolNamesOptions, diff --git a/src/agent/runtime/tool-exposure-runtime.test.ts b/src/agent/runtime/tool-exposure-runtime.test.ts index 0f02b9dc81..d39f63e3fa 100644 --- a/src/agent/runtime/tool-exposure-runtime.test.ts +++ b/src/agent/runtime/tool-exposure-runtime.test.ts @@ -22,14 +22,25 @@ function toolNames(options: unknown): string[] { : Object.keys((value as Record | undefined) ?? {}).sort(); } +function systemPrompt(options: unknown): string { + const prompt = (options as { prompt?: Array<{ role?: string; content?: unknown }> }).prompt; + if (!Array.isArray(prompt)) return ""; + return prompt + .filter((entry) => entry?.role === "system" && typeof entry.content === "string") + .map((entry) => entry.content as string) + .join("\n"); +} + it("deferred generate searches, exposes on the next step, and executes once", async () => { const observedTools: string[][] = []; + const observedSystems: string[] = []; let step = 0; const model: ModelRuntime = { provider: "hosted", modelId: "hosted/deferred-tools", async doGenerate(options: unknown) { observedTools.push(toolNames(options)); + observedSystems.push(systemPrompt(options)); step++; if (step === 1) { return { @@ -102,16 +113,102 @@ it("deferred generate searches, exposes on the next step, and executes once", as const response = await assistant.generate({ input: "Read the release marker" }); - assertEquals(observedTools[0], ["form_input", "load_skill", "tool_search"]); + assertEquals(observedTools[0], ["load_skill", "tool_search"]); assertEquals(observedTools[1], [ - "form_input", "load_skill", "read_release_marker", + "tool_search", ]); + assertEquals((observedSystems[0] ?? "").includes("form_input"), false); + assertEquals((observedSystems[0] ?? "").includes("read_release_marker"), false); + assertEquals((observedSystems[1] ?? "").includes("read_release_marker"), false); assertEquals(executionCount, 1); assertEquals(response.text, "Release marker marker-1"); }); +it("deferred generate activates and executes provider-native web search on demand", async () => { + const observedTools: string[][] = []; + const observedSystems: string[] = []; + let step = 0; + const model: ModelRuntime = { + provider: "anthropic", + modelId: "claude-sonnet-4-6", + async doGenerate(options: unknown) { + observedTools.push(toolNames(options)); + observedSystems.push(systemPrompt(options)); + step++; + if (step === 1) { + return { + content: [{ + type: "tool-call", + toolCallId: "search-native-1", + toolName: "tool_search", + input: JSON.stringify({ query: "web_search" }), + }], + finishReason: "tool-calls", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + }; + } + if (step === 2) { + return { + content: [ + { + type: "tool-call", + toolCallId: "web-search-1", + toolName: "web_search", + input: JSON.stringify({ query: "Veryfront" }), + }, + { + type: "tool-result", + toolCallId: "web-search-1", + toolName: "web_search", + result: { results: [{ title: "Veryfront" }] }, + providerExecuted: true, + }, + ], + finishReason: "tool-calls", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + }; + } + return { + content: [{ type: "text", text: "Found Veryfront." }], + finishReason: "stop", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + }; + }, + async doStream() { + return { stream: new ReadableStream() }; + }, + }; + const assistant = agent( + { + id: "deferred-provider-native-test", + model: "anthropic/claude-sonnet-4-6", + system: "Use web search when current information is required.", + skills: false, + tools: {}, + providerTools: ["web_search", "web_fetch"], + maxSteps: 3, + resolveModelTransport: () => ({ model }), + __vfToolLoadingMode: "deferred", + } as AgentConfig & RuntimeToolFilterConfig, + ); + + const response = await assistant.generate({ input: "Search the web for Veryfront" }); + + assertEquals(observedTools[0], ["tool_search"]); + assertEquals(observedTools[1], ["tool_search", "web_search"]); + assertEquals(observedTools[2], ["tool_search", "web_search"]); + assertEquals((observedSystems[0] ?? "").includes("web_search"), false); + assertEquals((observedSystems[0] ?? "").includes("web_fetch"), false); + assertEquals((observedSystems[1] ?? "").includes("web_search"), false); + assertEquals(response.toolCalls.map((call) => [call.name, call.status]), [ + ["tool_search", "completed"], + ["web_search", "completed"], + ]); + assertEquals(response.text, "Found Veryfront."); +}); + it("deferred generate rejects a guessed tool that was not exposed", async () => { let step = 0; let executionCount = 0; diff --git a/src/agent/runtime/tool-exposure.test.ts b/src/agent/runtime/tool-exposure.test.ts index 2d8eca2f1a..de1a81e323 100644 --- a/src/agent/runtime/tool-exposure.test.ts +++ b/src/agent/runtime/tool-exposure.test.ts @@ -69,15 +69,15 @@ it("tool exposure plans eager and deferred visibility deterministically", () => }); assertEquals( deferred.visible.map((tool) => tool.name), - ["form_input", "load_skill", TOOL_SEARCH_TOOL_NAME], + ["load_skill", TOOL_SEARCH_TOOL_NAME], ); assertEquals( deferred.deferred.map((tool) => tool.name), - ["archive_release", "create_release", "get_release"], + ["archive_release", "create_release", "form_input", "get_release"], ); }); -it("deferred exposure omits tool_search when only bootstrap tools are authorized", () => { +it("deferred exposure keeps form_input searchable and omits search for load_skill alone", () => { const deferred = createToolExposurePlan({ authorized: [ definition("form_input", "Ask the user for structured input"), @@ -89,9 +89,17 @@ it("deferred exposure omits tool_search when only bootstrap tools are authorized assertEquals( deferred.visible.map((tool) => tool.name), - ["form_input", "load_skill"], + ["load_skill", TOOL_SEARCH_TOOL_NAME], ); - assertEquals(deferred.deferred, []); + assertEquals(deferred.deferred.map((tool) => tool.name), ["form_input"]); + + const loadSkillOnly = createToolExposurePlan({ + authorized: [definition("load_skill", "Load a configured skill")], + mode: "deferred", + state: createToolExposureState(), + }); + assertEquals(loadSkillOnly.visible.map((tool) => tool.name), ["load_skill"]); + assertEquals(loadSkillOnly.deferred, []); }); it("deferred exposure keeps injected tool_search in visible ASCII order", () => { @@ -478,10 +486,43 @@ it("authorized search matches load for the next step", () => { const next = createToolExposurePlan({ authorized: catalog, mode: "deferred", state }); assertEquals( next.visible.map((tool) => tool.name), - ["form_input", "get_release", "load_skill", TOOL_SEARCH_TOOL_NAME], + ["get_release", "load_skill", TOOL_SEARCH_TOOL_NAME], ); }); +it("form_input activation survives a private checkpoint and current-authorization restore", () => { + const authorized = [ + definition("form_input", "Ask the user for structured input"), + definition("load_skill", "Load a configured skill"), + ]; + const state = createToolExposureState(); + const initial = createToolExposurePlan({ + authorized, + mode: "deferred", + state, + }); + assertEquals(initial.visible.map((tool) => tool.name), ["load_skill", TOOL_SEARCH_TOOL_NAME]); + + const search = searchToolExposure({ + query: "form_input", + authorized: initial.deferred, + state, + }); + assertEquals(search.matches.map((match) => match.name), ["form_input"]); + + const restored = restoreToolExposureState( + createToolExposureCheckpoint(authorized, state), + authorized, + ); + const resumed = createToolExposurePlan({ + authorized, + mode: "deferred", + state: restored, + }); + assertEquals(resumed.visible.map((tool) => tool.name), ["form_input", "load_skill"]); + assertEquals(resumed.deferred, []); +}); + it("deferred exposure reserves bootstrap and search inside the provider tool budget", () => { const remoteCatalog = Array.from( { length: 130 }, @@ -504,11 +545,11 @@ it("deferred exposure reserves bootstrap and search inside the provider tool bud }); assertEquals(plan.visible.length, 128); - assertEquals(plan.visible.some((tool) => tool.name === "form_input"), true); + assertEquals(plan.visible.some((tool) => tool.name === "form_input"), false); assertEquals(plan.visible.some((tool) => tool.name === "load_skill"), true); assertEquals(plan.visible.some((tool) => tool.name === TOOL_SEARCH_TOOL_NAME), true); - assertEquals(plan.maxLoadedTools, 125); - assertEquals(state.loadedToolNames.size, 125); + assertEquals(plan.maxLoadedTools, 126); + assertEquals(state.loadedToolNames.size, 126); assertEquals(state.loadedToolNames.has("catalog_tool_000"), false); assertEquals(state.loadedToolNames.has("catalog_tool_129"), true); }); @@ -523,7 +564,10 @@ it("deferred exposure uses the full provider budget once the exact-fit catalog i definition("load_skill", "Load a configured skill"), ...remoteCatalog, ]; - const state = createToolExposureState(remoteCatalog.map((tool) => tool.name)); + const state = createToolExposureState([ + "form_input", + ...remoteCatalog.map((tool) => tool.name), + ]); const plan = createToolExposurePlan({ authorized, @@ -535,8 +579,8 @@ it("deferred exposure uses the full provider budget once the exact-fit catalog i assertEquals(plan.visible.length, 128); assertEquals(plan.visible.some((tool) => tool.name === TOOL_SEARCH_TOOL_NAME), false); assertEquals(plan.deferred, []); - assertEquals(plan.maxLoadedTools, 126); - assertEquals(state.loadedToolNames.size, 126); + assertEquals(plan.maxLoadedTools, 127); + assertEquals(state.loadedToolNames.size, 127); }); it("exact-fit deferred exposure loads the final schema without exceeding the provider budget", () => { @@ -549,7 +593,10 @@ it("exact-fit deferred exposure loads the final schema without exceeding the pro definition("load_skill", "Load a configured skill"), ...remoteCatalog, ]; - const state = createToolExposureState(remoteCatalog.slice(0, 125).map((tool) => tool.name)); + const state = createToolExposureState([ + "form_input", + ...remoteCatalog.slice(0, 125).map((tool) => tool.name), + ]); const searchStep = createToolExposurePlan({ authorized, @@ -559,7 +606,7 @@ it("exact-fit deferred exposure loads the final schema without exceeding the pro }); assertEquals(searchStep.visible.length, 128); assertEquals(searchStep.deferred.map((tool) => tool.name), ["catalog_tool_125"]); - assertEquals(searchStep.maxLoadedTools, 126); + assertEquals(searchStep.maxLoadedTools, 127); const search = searchToolExposure({ query: "catalog_tool_125", @@ -568,7 +615,7 @@ it("exact-fit deferred exposure loads the final schema without exceeding the pro maxLoadedTools: searchStep.maxLoadedTools, }); assertEquals(search.loadedCount, 1); - assertEquals(state.loadedToolNames.size, 126); + assertEquals(state.loadedToolNames.size, 127); const loadedStep = createToolExposurePlan({ authorized, @@ -593,6 +640,7 @@ it("deferred exposure prunes revoked and bootstrap names before budget eviction" retained.name, "revoked_tool", "form_input", + "load_skill", ]); const plan = createToolExposurePlan({ @@ -602,7 +650,7 @@ it("deferred exposure prunes revoked and bootstrap names before budget eviction" maxVisibleTools: 4, }); - assertEquals([...state.loadedToolNames], [retained.name]); + assertEquals([...state.loadedToolNames], [retained.name, "form_input"]); assertEquals( plan.visible.map((tool) => tool.name), ["form_input", "load_skill", retained.name, TOOL_SEARCH_TOOL_NAME], diff --git a/src/agent/runtime/tool-exposure.ts b/src/agent/runtime/tool-exposure.ts index 9fccc63227..a339584bad 100644 --- a/src/agent/runtime/tool-exposure.ts +++ b/src/agent/runtime/tool-exposure.ts @@ -5,7 +5,7 @@ import { isOwnDataPropertyDescriptor } from "./data-property-descriptor.ts"; /** Framework-owned model-facing tool used to load authorized schemas. */ export const TOOL_SEARCH_TOOL_NAME = "tool_search"; -const DEFAULT_BOOTSTRAP_TOOL_NAMES = new Set(["form_input", "load_skill"]); +const DEFAULT_BOOTSTRAP_TOOL_NAMES = new Set(["load_skill"]); const TOOL_SEARCH_RESULT_LIMIT = 5; const TOOL_SEARCH_QUERY_MAX_BYTES = 256; const TOOL_SEARCH_CANDIDATE_LIMIT = 4_096; diff --git a/tests/chat/agent-runtime-streaming.test.ts b/tests/chat/agent-runtime-streaming.test.ts index 42e7c19740..cea74bca76 100644 --- a/tests/chat/agent-runtime-streaming.test.ts +++ b/tests/chat/agent-runtime-streaming.test.ts @@ -131,11 +131,11 @@ it("deferred respond searches, exposes on the next step, and executes once", asy ); const body = await response.text(); - assertEquals(observedTools[0], ["form_input", "load_skill", "tool_search"]); + assertEquals(observedTools[0], ["load_skill", "tool_search"]); assertEquals(observedTools[1], [ - "form_input", "load_skill", "read_release_marker", + "tool_search", ]); assertEquals(executionCount, 1); assert(body.includes("marker-1"), "respond should stream the final marker"); diff --git a/tests/docs/guide-content.test.ts b/tests/docs/guide-content.test.ts index bd3cc83b91..6e2af891ba 100644 --- a/tests/docs/guide-content.test.ts +++ b/tests/docs/guide-content.test.ts @@ -78,6 +78,22 @@ describe("guide content contracts", () => { ); }); + it("documents deferred form and provider-native tool discovery", async () => { + const agentsGuide = await Deno.readTextFile("docs/guides/agents.md"); + const toolsGuide = await Deno.readTextFile("docs/guides/tools.md"); + const architecture = await Deno.readTextFile( + "docs/architecture/28-model-driven-tool-discovery.md", + ); + const docs = [agentsGuide, toolsGuide, architecture].join("\n"); + + assertStringIncludes(toolsGuide, "`form_input` is authorized but deferred"); + assertStringIncludes(docs, "configured `providerTools`"); + assertStringIncludes(docs, "provider's native schema on the next model step"); + assertEquals(docs.includes("does not search or load provider-native `providerTools`"), false); + assertEquals(docs.includes("does not search `providerTools`"), false); + assertEquals(docs.includes("bootstrap tools (`form_input`"), false); + }); + it("documents the exact-run writer capability migration", async () => { const guide = await Deno.readTextFile( "docs/guides/agent-service-runtime.md",