Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
200 changes: 198 additions & 2 deletions src/agent/runtime/tool-exposure.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -210,20 +210,23 @@ it("tool search normalizes ASCII case and underscores as spaces", () => {
);
});

it("tool search falls back to deterministic whitespace terms after a phrase miss", () => {
it("tool search scores fallback terms by selectivity after a phrase miss", () => {
const fileCatalog = [
definition("create_file", "Create a project file"),
definition("update_file", "Update a project file"),
definition("sandbox_write_file", "Write only inside the sandbox filesystem"),
];

// `file` matches every candidate and therefore says nothing about which one the
// caller meant; `create` and `update` each match exactly one. `sandbox_write_file`
// matches no term but `file`, so returning it would be filler, not a result.
assertEquals(
searchToolExposure({
query: "create_file update_file project file",
authorized: fileCatalog,
state: createToolExposureState(),
}).matches.map((match) => match.name),
["create_file", "update_file", "sandbox_write_file"],
["create_file", "update_file"],
);
});

Expand Down Expand Up @@ -865,3 +868,196 @@ it("tool_search is reserved only when deferred framework search is injected", ()
}
assert(message.includes("reserved"));
});

const VERYFRONT_LIST_TOOL_NAMES = [
"list_accessible_agents",
"list_agent_runs",
"list_agent_templates",
"list_agent_tool_references",
"list_agent_workers",
"list_agents",
"list_child_agent_runs",
"list_child_agent_runs_by_parent_conversation",
"list_eval_runs",
"list_evals",
"list_external_files",
"list_files",
"list_input_requests",
"list_integrations",
"list_models",
"list_projects",
"list_prompts",
"list_releases",
"list_resources",
"list_sandbox_background_commands",
"list_sandbox_sessions",
"list_schedules",
"list_skills",
"list_tasks",
"list_tools",
"list_uploads",
"list_workflows",
];

const CATALOG_NAMESPACE_DESCRIPTION =
"Get detailed configuration, available tool IDs, and input schemas for an integration. " +
"Use this for integration tool IDs in these namespaces: confluence, github, jira, salesforce.";

function integrationDiscoveryCatalog(): ToolDefinition[] {
return [
...VERYFRONT_LIST_TOOL_NAMES.map((name) =>
definition(name, `List ${name.slice("list_".length).replaceAll("_", " ")}`)
),
definition("get_integration", CATALOG_NAMESPACE_DESCRIPTION, "integration namespace name"),
];
}

it("tool search ranks a rare description term above a common name term", () => {
const matches = searchToolExposure({
query: "list github issues",
authorized: integrationDiscoveryCatalog(),
state: createToolExposureState(),
}).matches.map((match) => match.name);

assertEquals(matches[0], "get_integration");
assertEquals(matches.filter((name) => name.startsWith("list_agent")), []);
});

it("tool search resolves a canonical integration tool id to its namespace", () => {
// Every one of these word-splits onto a platform tool that would otherwise win:
// `list_projects`, `list_...`, and the `search_*` family all share the generic
// half of the id. Resolving the namespace is what keeps the wrong tool out.
for (
const query of [
"jira__list_projects",
"jira__list_comments",
"jira__list_sites",
"jira__search_users",
"github__list_repos",
]
) {
assertEquals(
searchToolExposure({
query,
authorized: integrationDiscoveryCatalog(),
state: createToolExposureState(),
}).matches.map((match) => match.name),
["get_integration"],
`canonical id ${query} must resolve to its namespace`,
);
}
});

it("tool search prefers an authorized integration tool over its namespace catalog entry", () => {
const result = searchToolExposure({
query: "jira__list_projects",
authorized: [
...integrationDiscoveryCatalog(),
definition("jira__list_projects", "List Jira projects on a site"),
],
state: createToolExposureState(),
});

assertEquals(result.matches[0]?.name, "jira__list_projects");
});

it("tool search does not let a same-named local tool satisfy a canonical id", () => {
// `jira__list_projects` and the local id `jira_list_projects` normalize to the
// same text, and the registry permits any local id without `__`. A phrase match
// must not hand back the local tool for a canonical query.
const result = searchToolExposure({
query: "jira__list_projects",
authorized: [
...integrationDiscoveryCatalog(),
definition("jira_list_projects", "A project-local tool that is not the Jira integration"),
],
state: createToolExposureState(),
});

assertEquals(result.matches.map((match) => match.name), ["get_integration"]);
});

it("tool search keeps a malformed namespace-shaped query off local tools", () => {
// `jira__list__projects` is not a valid canonical id, but `__` is reserved for
// the integration namespace and local ids may not contain it. Normalization
// collapses it onto the local `jira_list_projects`, which must not win.
const authorized = [
...integrationDiscoveryCatalog(),
definition("jira_list_projects", "A project-local tool that is not the Jira integration"),
];

for (const query of ["jira__list__projects", "JIRA__LIST_PROJECTS", "jira__list_projects "]) {
assertEquals(
searchToolExposure({ query, authorized, state: createToolExposureState() }).matches.map((
match,
) => match.name),
["get_integration"],
`namespace-shaped query ${JSON.stringify(query)} must not resolve to a local tool`,
);
}
});

it("tool search resolves a namespace that itself contains an underscore", () => {
// The grammar permits `_` inside a namespace segment, but candidate text has
// underscores rewritten to spaces, so an un-normalized namespace never matches.
assertEquals(
searchToolExposure({
query: "foo_bar__list_items",
authorized: [
definition("get_integration", "Tool ids and schemas. Namespaces: foo_bar, jira."),
definition("list_items", "List items in this project"),
],
state: createToolExposureState(),
}).matches.map((match) => match.name),
["get_integration"],
);
});

it("tool search accepts canonical ids with the authorization layer's segment grammar", () => {
// The authoritative grammar allows consecutive and trailing separators; search
// must not be stricter, or it disagrees with authorization about what an id is.
assertEquals(
searchToolExposure({
query: "github__list-issues-",
authorized: integrationDiscoveryCatalog(),
state: createToolExposureState(),
}).matches.map((match) => match.name),
["get_integration"],
);
});

it("tool search matches a full-coverage candidate in a single-tool catalog", () => {
// With one candidate every term matches everything, so no term can clear the
// selectivity floor. Full term coverage is its own evidence: there is no
// competing candidate for the floor to protect against.
assertEquals(
searchToolExposure({
query: "create project",
authorized: [definition("create_file", "Create a project file")],
state: createToolExposureState(),
}).matches.map((match) => match.name),
["create_file"],
);
});

it("tool search reports a miss when no candidate matches a selective term", () => {
assertEquals(
searchToolExposure({
query: "list spreadsheet macros",
authorized: integrationDiscoveryCatalog(),
state: createToolExposureState(),
}),
{ matches: [], resultCount: 0, loadedCount: 0, miss: true },
);
});

it("tool search still resolves a bare platform tool id to its exact name match", () => {
assertEquals(
searchToolExposure({
query: "list_projects",
authorized: integrationDiscoveryCatalog(),
state: createToolExposureState(),
}).matches[0]?.name,
"list_projects",
);
});
Loading
Loading