diff --git a/packages/worker/src/mcp/tools/search-format.node.test.ts b/packages/worker/src/mcp/tools/search-format.node.test.ts index 9e7c34c7fa..f15b144ffa 100644 --- a/packages/worker/src/mcp/tools/search-format.node.test.ts +++ b/packages/worker/src/mcp/tools/search-format.node.test.ts @@ -98,6 +98,7 @@ test('search markdown and entity detail formatting preserve structured behavior' { type: 'value', id: 'user:preferred_repo', + entityRef: 'user:preferred_repo:value', name: 'preferred_repo', title: 'preferred_repo', description: 'Preferred repository owner/name.', @@ -108,6 +109,7 @@ test('search markdown and entity detail formatting preserve structured behavior' { type: 'connector', id: 'github', + entityRef: 'github:connector', name: 'github', title: 'github', description: 'GitHub OAuth connector config', diff --git a/packages/worker/src/mcp/tools/search-format.ts b/packages/worker/src/mcp/tools/search-format.ts index 148f3aed84..fe535580be 100644 --- a/packages/worker/src/mcp/tools/search-format.ts +++ b/packages/worker/src/mcp/tools/search-format.ts @@ -20,6 +20,37 @@ export type SearchResultStructuredContent = { matches: Array offline: boolean warnings: Array + telemetry?: { + intent: { + task: string + confidence: number + entityCount: number + actionCount: number + constraintCount: number + topEntities: Array<{ + type: string + id: string + confidence: number + }> + } + candidateCounts: Partial< + Record< + 'capability' | 'package' | 'value' | 'connector' | 'secret', + number + > + > + topResultTypes: Array< + 'capability' | 'package' | 'value' | 'connector' | 'secret' + > + trimmedMatchCount?: number + responseTrimmed?: boolean + } + phaseTimings?: { + queryUnderstandingMs: number + candidateGenerationMs: number + rerankingMs: number + formattingMs?: number + } memories?: { surfaced: Array<{ id: string @@ -54,6 +85,7 @@ export type SlimSearchMatch = | { type: 'capability' id: string + entityRef: string title: string description: string usage: string @@ -61,6 +93,7 @@ export type SlimSearchMatch = | { type: 'package' id: string + entityRef: string packageId: string kodyId: string title: string @@ -73,6 +106,7 @@ export type SlimSearchMatch = | { type: 'secret' id: string + entityRef: string title: string description: string usage: string @@ -80,6 +114,7 @@ export type SlimSearchMatch = | { type: 'value' id: string + entityRef: string name: string title: string description: string @@ -90,6 +125,7 @@ export type SlimSearchMatch = | { type: 'connector' id: string + entityRef: string name: string title: string description: string @@ -340,7 +376,7 @@ export function formatSearchMarkdown(input: { '- Built-in capabilities — `execute` with `import { codemode } from "kody:runtime"`', '- Persisted values — `codemode.value_get({ name, scope })` or `codemode.value_list({ scope })`', '- Saved connectors — `codemode.connector_get({ name })` or `codemode.connector_list({})`', - '- Saved packages — import from `kody:@package-id/export-name`, edit with `repo_*`, and open package apps with `open_generated_ui({ package_id })` when the package declares `kody.app`', + '- Saved packages — import from `kody:@package-id/export-name`, edit with `repo_*`, and open package apps with `open_generated_ui({ kody_id })` or `open_generated_ui({ package_id })` when the package declares `kody.app`', '- Secrets — placeholders in execute-time fetches or `codemode.secret_list` (never paste raw secrets in chat or embed `{{secret:...}}` literally into visible content such as comments, prompts, or issue bodies)', ) } @@ -395,6 +431,8 @@ function formatMatchBlock(match: SearchMatch, baseUrl: string) { `## Capability — \`${match.name}\``, '', 'description' in match ? match.description : '', + '', + `**Entity:** \`${match.name}:capability\``, ] } if (match.type === 'package') { @@ -406,6 +444,7 @@ function formatMatchBlock(match: SearchMatch, baseUrl: string) { '', match.description, '', + `**Entity:** \`${match.kodyId}:package\``, `**Tags:** ${match.tags.length > 0 ? match.tags.map((tag) => `\`${tag}\``).join(', ') : 'none'}`, `**Has app:** ${match.hasApp ? 'yes' : 'no'}`, ...(hostedUrl ? [`**Hosted URL:** \`${hostedUrl}\``] : []), @@ -426,11 +465,18 @@ function formatMatchBlock(match: SearchMatch, baseUrl: string) { '', match.description, '', + `**Entity:** \`${match.connectorName}:connector\``, `**Flow:** \`${match.flow}\``, `**API base URL:** ${match.apiBaseUrl ? `\`${match.apiBaseUrl}\`` : 'none'}`, ] } - return [`## Secret — \`${match.name}\``, '', match.description] + return [ + `## Secret — \`${match.name}\``, + '', + match.description, + '', + `**Entity:** \`${match.name}:secret\``, + ] } export function toSlimStructuredMatches(input: { @@ -442,6 +488,7 @@ export function toSlimStructuredMatches(input: { return { type: 'capability', id: match.name, + entityRef: `${match.name}:capability`, title: match.name, description: 'description' in match && typeof match.description === 'string' @@ -454,12 +501,13 @@ export function toSlimStructuredMatches(input: { return { type: 'package', id: match.kodyId, + entityRef: `${match.kodyId}:package`, packageId: match.packageId, kodyId: match.kodyId, title: match.title, description: match.description, usage: match.hasApp - ? `open_generated_ui({ package_id: "${match.packageId}" })` + ? `open_generated_ui({ kody_id: "${match.kodyId}" })` : `import entry from "${buildPackageImportSpecifier(match.kodyId, '.')}"`, tags: match.tags, hasApp: match.hasApp, @@ -472,6 +520,7 @@ export function toSlimStructuredMatches(input: { return { type: 'value', id: match.valueId, + entityRef: `${match.valueId}:value`, name: match.name, title: match.name, description: match.description, @@ -484,6 +533,7 @@ export function toSlimStructuredMatches(input: { return { type: 'connector', id: match.connectorName, + entityRef: `${match.connectorName}:connector`, name: match.connectorName, title: match.title, description: match.description, @@ -496,6 +546,7 @@ export function toSlimStructuredMatches(input: { return { type: 'secret', id: match.name, + entityRef: `${match.name}:secret`, title: match.name, description: match.description, usage: `{{secret:${match.name}|scope=user}}`, @@ -639,7 +690,7 @@ export function formatEntityDetailMarkdown(detail: SearchEntityDetail) { title: detail.title, description: detail.description, usage: detail.record.hasApp - ? `open_generated_ui({ package_id: "${detail.record.id}" })` + ? `open_generated_ui({ kody_id: "${detail.record.kodyId}" })` : `import entry from "${buildPackageImportSpecifier(detail.record.kodyId, '.')}"`, packageId: detail.record.id, kodyId: detail.record.kodyId, diff --git a/packages/worker/src/mcp/tools/search-handler.node.test.ts b/packages/worker/src/mcp/tools/search-handler.node.test.ts index d53a466b2e..2e63c2c628 100644 --- a/packages/worker/src/mcp/tools/search-handler.node.test.ts +++ b/packages/worker/src/mcp/tools/search-handler.node.test.ts @@ -7,7 +7,10 @@ const mockModule = vi.hoisted(() => ({ name: 'search_docs', description: 'Search docs capability', domain: 'meta', + keywords: [], + inputFields: [], requiredInputFields: [], + outputFields: [], readOnly: true, idempotent: true, destructive: false, @@ -120,16 +123,14 @@ test('search tool includes timing metadata in success responses', async () => { }) expect(response.isError).toBeUndefined() - expect(response.structuredContent).toEqual( - expect.objectContaining({ - conversationId: 'conv-search', - timing: { - startedAt: expect.any(String), - endedAt: expect.any(String), - durationMs: 12, - }, - }), - ) + expect(response.structuredContent).toMatchObject({ + conversationId: 'conv-search', + timing: { + startedAt: expect.any(String), + endedAt: expect.any(String), + }, + }) + expect(response.structuredContent.timing.durationMs).toBeGreaterThanOrEqual(0) }) test('search tool includes timing metadata in validation errors', async () => { diff --git a/packages/worker/src/mcp/tools/search.ts b/packages/worker/src/mcp/tools/search.ts index 926b8c0516..10738cc054 100644 --- a/packages/worker/src/mcp/tools/search.ts +++ b/packages/worker/src/mcp/tools/search.ts @@ -8,9 +8,10 @@ import { } from '#mcp/capabilities/values/connector-shared.ts' import { getCapabilityRegistryForContext } from '#mcp/capabilities/registry.ts' import { + buildCapabilityEmbedText, + blendLexicalAndVectorScore, cosineSimilarity, deterministicEmbedding, - hybridSearchScore, isCapabilitySearchOffline, lexicalScore, } from '#mcp/capabilities/capability-search.ts' @@ -57,6 +58,13 @@ import { } from './search-format.ts' import { finishToolTiming, startToolTiming } from './tool-timing.ts' import { prependToolMetadataContent } from './tool-response-content.ts' +import { + type SearchIntent, + type SearchableEntityDescriptor, + extractSearchTokens, + normalizeSearchText, + understandSearchQuery, +} from './understand-search-query.ts' const charsPerToken = 4 const maxTokens = 6_000 @@ -76,82 +84,513 @@ export type OptionalSearchRowsResult = { warnings: Array } -export function searchUnified(input: { - env: Env - query: string - limit: number +export type SearchScoreComponents = { + base: number + lexical: number + vector: number + entityMatch: number + actionMatch: number + taskAffinity: number + appAvailability: number + constraint: number + final: number +} + +type SearchCandidate = { + match: SearchMatch + type: SearchMatch['type'] + id: string + title: string + searchFields: Array + scoreComponents: SearchScoreComponents +} + +export type SearchTelemetry = { + intent: { + task: SearchIntent['task']['name'] + confidence: number + entityCount: number + actionCount: number + constraintCount: number + topEntities: Array<{ + type: string + id: string + confidence: number + }> + } + candidateCounts: Partial> + topResultTypes: Array + trimmedMatchCount?: number + responseTrimmed?: boolean +} + +type SearchPhaseTimings = { + queryUnderstandingMs: number + candidateGenerationMs: number + rerankingMs: number + formattingMs?: number +} + +type SearchUnifiedResult = { + matches: Array + offline: boolean + intent: SearchIntent + telemetry: SearchTelemetry + phaseTimings: SearchPhaseTimings +} + +function buildSearchableEntityDescriptors(input: { registry: Awaited> optionalRows: Pick< OptionalSearchRowsResult, 'packageRows' | 'userSecretRows' | 'userValueRows' > -}): { matches: Array; offline: boolean } { - const offline = isCapabilitySearchOffline(input.env) - const query = input.query.trim() - if (!query) { - return { - matches: [], - offline, - } +}): Array { + const descriptors: Array = [] + + for (const spec of Object.values(input.registry.capabilitySpecs)) { + descriptors.push({ + type: 'capability', + id: spec.name, + title: spec.name, + primaryAliases: [spec.name], + secondaryAliases: [ + spec.domain, + spec.description, + ...(spec.keywords ?? []), + ], + tertiaryAliases: [ + ...(spec.inputFields ?? []), + ...(spec.outputFields ?? []), + ], + }) } - const limit = Math.max(1, input.limit) - const queryEmbedding = deterministicEmbedding(query) - const capabilityMatches = Object.values(input.registry.capabilitySpecs) - .map((spec) => ({ - type: 'capability' as const, - name: spec.name, - description: spec.description, - score: lexicalScore(query, `${spec.name}\n${spec.description}`), - })) - .filter((match) => match.score > 0) - const packageMatches = input.optionalRows.packageRows - .map((entry) => { - const doc = [ - entry.record.name, - entry.record.kodyId, + for (const entry of input.optionalRows.packageRows) { + descriptors.push({ + type: 'package', + id: entry.record.kodyId, + title: entry.record.name, + primaryAliases: [entry.record.kodyId, entry.record.name], + secondaryAliases: [ entry.record.description, - entry.record.tags.join(' '), entry.record.searchText ?? '', + ...entry.record.tags, + ], + tertiaryAliases: [ + ...entry.projection.exports, + ...entry.projection.jobs.map((job) => job.name), + ...(entry.record.hasApp ? ['app', 'ui', 'remote'] : []), + ], + }) + } + + for (const row of input.optionalRows.userValueRows) { + const connectorName = parseConnectorValueName(row.name) + if (connectorName) { + const config = parseConnectorConfig( + parseConnectorJson(row.value), + connectorName, + ) + if (!config) continue + descriptors.push({ + type: 'connector', + id: connectorName, + title: connectorName, + primaryAliases: [connectorName], + secondaryAliases: [ + row.description, + config.apiBaseUrl ?? '', + config.tokenUrl, + config.flow, + ], + tertiaryAliases: config.requiredHosts ?? [], + }) + continue + } + + descriptors.push({ + type: 'value', + id: buildValueEntityId(row), + title: row.name, + primaryAliases: [row.name], + secondaryAliases: [row.description, row.scope], + tertiaryAliases: [row.value], + }) + } + + for (const row of input.optionalRows.userSecretRows) { + descriptors.push({ + type: 'secret', + id: row.name, + title: row.name, + primaryAliases: [row.name], + secondaryAliases: [row.description], + }) + } + + return descriptors +} + +function buildCandidateTelemetry(input: { + intent: SearchIntent + candidates: Array + matches: Array + trimmedMatchCount?: number +}): SearchTelemetry { + const candidateCounts = input.candidates.reduce( + (counts, candidate) => { + counts[candidate.type] = (counts[candidate.type] ?? 0) + 1 + return counts + }, + {} as Partial>, + ) + + return { + intent: { + task: input.intent.task.name, + confidence: input.intent.confidence, + entityCount: input.intent.entities.length, + actionCount: input.intent.actions.length, + constraintCount: input.intent.constraints.length, + topEntities: input.intent.entities.slice(0, 3).map((entity) => ({ + type: entity.type, + id: entity.id, + confidence: entity.confidence, + })), + }, + candidateCounts, + topResultTypes: input.matches.slice(0, 5).map((match) => match.type), + ...(input.trimmedMatchCount !== undefined + ? { + trimmedMatchCount: input.trimmedMatchCount, + responseTrimmed: input.trimmedMatchCount > 0, + } + : {}), + } +} + +function buildCandidateBaseScore(input: { + lexical: number + vector?: number +}): SearchScoreComponents { + const vector = input.vector ?? 0 + return { + base: + input.vector === undefined + ? input.lexical + : blendLexicalAndVectorScore(input.lexical, vector), + lexical: input.lexical, + vector, + entityMatch: 0, + actionMatch: 0, + taskAffinity: 0, + appAvailability: 0, + constraint: 0, + final: + input.vector === undefined + ? input.lexical + : blendLexicalAndVectorScore(input.lexical, vector), + } +} + +function scoreMatchedTerms( + fields: ReadonlyArray, + matchedTerms: ReadonlyArray, +): number { + if (matchedTerms.length === 0 || fields.length === 0) return 0 + const fieldTokens = new Set() + for (const field of fields) { + for (const token of extractSearchTokens(field)) { + fieldTokens.add(token) + } + } + let matched = 0 + for (const term of matchedTerms) { + if (fieldTokens.has(term)) matched += 1 + } + return matched / Math.max(1, matchedTerms.length) +} + +function scoreConstraintBoost( + fields: ReadonlyArray, + intent: SearchIntent, +): number { + if (intent.constraints.length === 0) return 0 + const normalizedFields = fields.map((field) => normalizeSearchText(field)) + let score = 0 + for (const constraint of intent.constraints) { + if ( + normalizedFields.some((field) => + field.includes(normalizeSearchText(constraint.value)), + ) + ) { + score += 0.08 + } + } + return score +} + +function scoreTaskAffinity( + candidate: SearchCandidate, + intent: SearchIntent, +): Pick< + SearchScoreComponents, + 'taskAffinity' | 'actionMatch' | 'appAvailability' | 'constraint' +> { + const taskConfidenceWeight = Math.min( + 1, + Math.max(0.2, intent.task.confidence), + ) + const matchedActionTerms = intent.actions.flatMap( + (action) => action.matchedTerms, + ) + const actionMatch = + scoreMatchedTerms(candidate.searchFields, matchedActionTerms) * + 0.25 * + taskConfidenceWeight + const constraint = + scoreConstraintBoost(candidate.searchFields, intent) * taskConfidenceWeight + + let taskAffinity = 0 + let appAvailability = 0 + + switch (intent.task.name) { + case 'operate': + if (candidate.type === 'package') { + taskAffinity += 0.16 + if ('hasApp' in candidate.match && candidate.match.hasApp) { + appAvailability += 0.12 + } + } + if (candidate.type === 'connector') taskAffinity += 0.08 + if (candidate.type === 'capability') taskAffinity -= 0.04 + break + case 'setup': + if (candidate.type === 'connector') taskAffinity += 0.18 + if (candidate.type === 'value') taskAffinity += 0.06 + if (candidate.type === 'capability') taskAffinity += 0.05 + break + case 'inspect': + if (candidate.type === 'value' || candidate.type === 'connector') { + taskAffinity += 0.12 + } + if (candidate.type === 'package') taskAffinity += 0.05 + break + case 'learn': + if (candidate.type === 'capability') taskAffinity += 0.16 + if (candidate.type === 'package') taskAffinity -= 0.02 + break + case 'debug': + if (candidate.type === 'connector') taskAffinity += 0.16 + if (candidate.type === 'capability') taskAffinity += 0.06 + if (candidate.type === 'value') taskAffinity += 0.04 + break + case 'unknown': + break + } + + return { + taskAffinity: taskAffinity * taskConfidenceWeight, + actionMatch, + appAvailability, + constraint, + } +} + +function rerankCandidates(input: { + candidates: Array + intent: SearchIntent + limit: number +}): Array { + const entityConfidenceByKey = new Map( + input.intent.entities.map((entity) => [ + `${entity.type}:${entity.id}`, + entity.confidence, + ]), + ) + const rerankWeight = Math.min(1, Math.max(0.25, input.intent.confidence)) + + const reranked = input.candidates.map((candidate) => { + const entityMatch = + (entityConfidenceByKey.get(`${candidate.type}:${candidate.id}`) ?? 0) * + 0.45 * + rerankWeight + const taskSignals = scoreTaskAffinity(candidate, input.intent) + const final = + candidate.scoreComponents.base + + entityMatch + + taskSignals.actionMatch + + taskSignals.taskAffinity + + taskSignals.appAvailability + + taskSignals.constraint + + return { + ...candidate, + scoreComponents: { + ...candidate.scoreComponents, + entityMatch, + actionMatch: taskSignals.actionMatch, + taskAffinity: taskSignals.taskAffinity, + appAvailability: taskSignals.appAvailability, + constraint: taskSignals.constraint, + final, + }, + } + }) + + return reranked + .filter((candidate) => candidate.scoreComponents.final > 0) + .sort((left, right) => { + if (right.scoreComponents.final !== left.scoreComponents.final) { + return right.scoreComponents.final - left.scoreComponents.final + } + return left.title.localeCompare(right.title) + }) + .slice(0, input.limit) +} + +function buildCapabilityCandidates(input: { + query: string + registry: Awaited> + queryEmbedding: ReadonlyArray +}): Array { + return Object.values(input.registry.capabilitySpecs) + .map((spec) => { + const document = buildCapabilityEmbedText(spec) + const lexical = lexicalScore(input.query, document) + const vector = cosineSimilarity( + input.queryEmbedding, + deterministicEmbedding(document), + ) + return { + match: { + type: 'capability' as const, + name: spec.name, + description: spec.description, + }, + type: 'capability' as const, + id: spec.name, + title: spec.name, + searchFields: [ + spec.name, + spec.domain, + spec.description, + ...(spec.keywords ?? []), + ...(spec.inputFields ?? []), + ...(spec.outputFields ?? []), + ], + scoreComponents: buildCandidateBaseScore({ + lexical, + vector, + }), + } + }) + .filter((candidate) => candidate.scoreComponents.base > 0) +} + +function buildPackageCandidates(input: { + query: string + rows: Array + queryEmbedding: ReadonlyArray +}): Array { + return input.rows + .map((entry) => { + const tags = Array.isArray(entry.projection.tags) + ? entry.projection.tags + : [] + const exports = Array.isArray(entry.projection.exports) + ? entry.projection.exports + : [] + const jobs = Array.isArray(entry.projection.jobs) + ? entry.projection.jobs + : [] + const document = [ + entry.projection.kodyId, + entry.projection.name, + entry.projection.description, + tags.join(' '), + entry.projection.searchText ?? '', + exports.join(' '), + jobs.map((job) => job.name).join(' '), ].join('\n') - const lexical = lexicalScore(query, doc) + const lexical = lexicalScore(input.query, document) const vector = cosineSimilarity( - queryEmbedding, - deterministicEmbedding(doc), + input.queryEmbedding, + deterministicEmbedding(document), ) return { + match: { + type: 'package' as const, + packageId: entry.record.id, + kodyId: entry.record.kodyId, + name: entry.record.name, + title: entry.record.name, + description: entry.record.description, + tags: entry.record.tags, + hasApp: entry.record.hasApp, + }, type: 'package' as const, - packageId: entry.record.id, - kodyId: entry.record.kodyId, - name: entry.record.name, + id: entry.record.kodyId, title: entry.record.name, - description: entry.record.description, - tags: entry.record.tags, - hasApp: entry.record.hasApp, - score: hybridSearchScore(lexical, vector), + searchFields: [ + entry.record.kodyId, + entry.record.name, + entry.record.description, + entry.record.searchText ?? '', + ...entry.record.tags, + ...exports, + ...jobs.map((job) => job.name), + ...(entry.record.hasApp ? ['app', 'ui', 'remote'] : []), + ], + scoreComponents: buildCandidateBaseScore({ + lexical, + vector, + }), } }) - .filter((match) => match.score > 0) - const valueMatches = input.optionalRows.userValueRows + .filter((candidate) => candidate.scoreComponents.base > 0) +} + +function buildValueCandidates(input: { + query: string + rows: Array +}): Array { + return input.rows .flatMap((row) => { if (parseConnectorValueName(row.name)) return [] + const lexical = lexicalScore( + input.query, + [row.name, row.description, row.scope, row.value].join('\n'), + ) return [ { + match: { + type: 'value' as const, + valueId: buildValueEntityId(row), + name: row.name, + description: describeValue(row), + scope: row.scope, + appId: row.appId, + }, type: 'value' as const, - valueId: buildValueEntityId(row), - name: row.name, - description: describeValue(row), - scope: row.scope, - appId: row.appId, - score: lexicalScore( - query, - [row.name, row.description, row.scope, row.value].join('\n'), - ), - }, + id: buildValueEntityId(row), + title: row.name, + searchFields: [row.name, row.description, row.scope, row.value], + scoreComponents: buildCandidateBaseScore({ + lexical, + }), + } satisfies SearchCandidate, ] }) - .filter((match) => match.score > 0) - const connectorMatches = input.optionalRows.userValueRows + .filter((candidate) => candidate.scoreComponents.base > 0) +} + +function buildConnectorCandidates(input: { + query: string + rows: Array +}): Array { + return input.rows .flatMap((row) => { const connectorName = parseConnectorValueName(row.name) if (!connectorName) return [] @@ -160,51 +599,187 @@ export function searchUnified(input: { connectorName, ) if (!config) return [] + const lexical = lexicalScore( + input.query, + [ + connectorName, + row.description, + config.tokenUrl, + config.apiBaseUrl ?? '', + config.flow, + ...(config.requiredHosts ?? []), + ].join('\n'), + ) return [ { + match: { + type: 'connector' as const, + connectorName, + title: connectorName, + description: + row.description.trim() || + `Saved OAuth connector configuration (${config.flow} flow).`, + flow: config.flow, + apiBaseUrl: config.apiBaseUrl ?? null, + requiredHosts: config.requiredHosts ?? [], + }, type: 'connector' as const, - connectorName, + id: connectorName, title: connectorName, - description: - row.description.trim() || - `Saved OAuth connector configuration (${config.flow} flow).`, - flow: config.flow, - apiBaseUrl: config.apiBaseUrl ?? null, - requiredHosts: config.requiredHosts ?? [], - score: lexicalScore( - query, - [ - connectorName, - row.description, - config.tokenUrl, - config.apiBaseUrl ?? '', - ].join('\n'), - ), - }, + searchFields: [ + connectorName, + row.description, + config.flow, + config.apiBaseUrl ?? '', + config.tokenUrl, + ...(config.requiredHosts ?? []), + ], + scoreComponents: buildCandidateBaseScore({ + lexical, + }), + } satisfies SearchCandidate, ] }) - .filter((match) => match.score > 0) - const secretMatches = input.optionalRows.userSecretRows - .map((row) => ({ - type: 'secret' as const, - name: row.name, - description: row.description, - score: lexicalScore(query, `${row.name}\n${row.description}`), - })) - .filter((match) => match.score > 0) + .filter((candidate) => candidate.scoreComponents.base > 0) +} + +function buildSecretCandidates(input: { + query: string + rows: Array +}): Array { + return input.rows + .map((row) => { + const lexical = lexicalScore( + input.query, + `${row.name}\n${row.description}`, + ) + return { + match: { + type: 'secret' as const, + name: row.name, + description: row.description, + }, + type: 'secret' as const, + id: row.name, + title: row.name, + searchFields: [row.name, row.description], + scoreComponents: buildCandidateBaseScore({ + lexical, + }), + } + }) + .filter((candidate) => candidate.scoreComponents.base > 0) +} + +export function searchUnified(input: { + env: Env + query: string + limit: number + registry: Awaited> + optionalRows: Pick< + OptionalSearchRowsResult, + 'packageRows' | 'userSecretRows' | 'userValueRows' + > +}): SearchUnifiedResult { + const offline = isCapabilitySearchOffline(input.env) + const query = input.query.trim() + if (!query) { + const queryUnderstandingStart = performance.now() + const emptyIntent = understandSearchQuery({ + query, + entities: [], + }) + const queryUnderstandingMs = Math.max( + 0, + Math.round(performance.now() - queryUnderstandingStart), + ) + return { + matches: [], + offline, + intent: emptyIntent, + telemetry: buildCandidateTelemetry({ + intent: emptyIntent, + candidates: [], + matches: [], + }), + phaseTimings: { + queryUnderstandingMs, + candidateGenerationMs: 0, + rerankingMs: 0, + }, + } + } + + const limit = Math.max(1, input.limit) + const entityDescriptors = buildSearchableEntityDescriptors({ + registry: input.registry, + optionalRows: input.optionalRows, + }) + const queryUnderstandingStart = performance.now() + const intent = understandSearchQuery({ + query, + entities: entityDescriptors, + }) + const queryUnderstandingMs = Math.max( + 0, + Math.round(performance.now() - queryUnderstandingStart), + ) + const candidateGenerationStart = performance.now() + const queryEmbedding = deterministicEmbedding(intent.normalizedQuery) + const candidates = [ + ...buildCapabilityCandidates({ + query: intent.normalizedQuery, + registry: input.registry, + queryEmbedding, + }), + ...buildPackageCandidates({ + query: intent.normalizedQuery, + rows: input.optionalRows.packageRows, + queryEmbedding, + }), + ...buildValueCandidates({ + query: intent.normalizedQuery, + rows: input.optionalRows.userValueRows, + }), + ...buildConnectorCandidates({ + query: intent.normalizedQuery, + rows: input.optionalRows.userValueRows, + }), + ...buildSecretCandidates({ + query: intent.normalizedQuery, + rows: input.optionalRows.userSecretRows, + }), + ] + const candidateGenerationMs = Math.max( + 0, + Math.round(performance.now() - candidateGenerationStart), + ) + const rerankingStart = performance.now() + const reranked = rerankCandidates({ + candidates, + intent, + limit, + }) + const matches = reranked.map((candidate) => candidate.match) + const rerankingMs = Math.max( + 0, + Math.round(performance.now() - rerankingStart), + ) return { - matches: [ - ...capabilityMatches, - ...packageMatches, - ...valueMatches, - ...connectorMatches, - ...secretMatches, - ] - .sort((left, right) => right.score - left.score) - .slice(0, limit) - .map(({ score: _score, ...match }) => match), + matches, offline, + intent, + telemetry: buildCandidateTelemetry({ + intent, + candidates, + matches, + }), + phaseTimings: { + queryUnderstandingMs, + candidateGenerationMs, + rerankingMs, + }, } } @@ -215,48 +790,22 @@ export async function searchPackages(input: { limit: number rows: Array }): Promise<{ matches: Array; offline: boolean }> { - const query = input.query.trim() - const offline = isCapabilitySearchOffline(input.env) - if (!query || input.rows.length === 0) { - return { matches: [], offline } - } - const queryEmbedding = deterministicEmbedding(query) - const ranked = input.rows - .map((row) => { - const document = [ - row.projection.kodyId, - row.projection.name, - row.projection.description, - row.projection.tags.join(' '), - row.projection.searchText ?? '', - row.projection.exports.join(' '), - row.projection.jobs.map((job) => job.name).join(' '), - ].join('\n') - const lexical = lexicalScore(query, document) - const vector = cosineSimilarity( - queryEmbedding, - deterministicEmbedding(document), - ) - return { - row, - score: lexical + vector, - } - }) - .sort((left, right) => right.score - left.score) - .slice(0, Math.max(1, Math.min(input.limit, input.rows.length))) - .map(({ row }) => ({ - type: 'package' as const, - packageId: row.record.id, - kodyId: row.record.kodyId, - name: row.record.name, - title: row.record.name, - description: row.record.description, - tags: row.record.tags, - hasApp: row.record.hasApp, - })) + const result = searchUnified({ + env: input.env, + query: input.query, + limit: input.limit, + registry: { + capabilitySpecs: {}, + } as Awaited>, + optionalRows: { + packageRows: input.rows, + userSecretRows: [], + userValueRows: [], + }, + }) return { - matches: ranked, - offline, + matches: result.matches, + offline: result.offline, } } @@ -331,7 +880,7 @@ empty ranked list. capabilities. Types and fields: see response. Packages: \`package_list\`, \`package_get\`, and \`repo_*\` for editing/publishing. -Open package apps with \`open_generated_ui({ package_id })\` or hosted package URLs. +Open package apps with \`open_generated_ui({ kody_id })\` (recommended) or hosted package URLs. Secrets: never raw in results; use \`codemode.secret_list\` during execute and UI for missing values. Persisted values use \`codemode.value_get\` / \`codemode.value_list\`. Connectors @@ -349,7 +898,7 @@ Example arguments: - \`{ "entity": "kody_official_guide:capability" }\` - \`{ "entity": "user:preferred_org:value" }\` - \`{ "entity": "github:connector" }\` -- To open a saved package app: \`open_generated_ui({ "package_id": "" })\` +- To open a saved package app: \`open_generated_ui({ "kody_id": "" })\` https://github.com/kentcdodds/kody/blob/main/docs/use/search.md `.trim(), @@ -705,6 +1254,22 @@ export async function registerSearchTool(agent: McpRegistrationAgent) { const userId = callerContext.user?.userId ?? null if (!args.query && !args.entity) { const timing = finishToolTiming(timingStart) + logMcpEvent({ + category: 'mcp', + tool: 'search', + toolName: 'search', + outcome: 'failure', + durationMs: timing.durationMs, + baseUrl, + hasUser, + sandboxError: false, + errorName: 'ValidationError', + errorMessage: 'Provide either "query" or "entity".', + message: 'Search request missing both query and entity.', + context: { + failurePhase: 'validation_error', + }, + }) return { content: prependToolMetadataContent(conversationId, [ { @@ -768,10 +1333,7 @@ export async function registerSearchTool(agent: McpRegistrationAgent) { const outcome: | { mode: 'list' - result: { - matches: Array - offline: boolean - } + result: SearchUnifiedResult } | { mode: 'entity' @@ -904,6 +1466,7 @@ export async function registerSearchTool(agent: McpRegistrationAgent) { ], }) } + const formattingStartMs = performance.now() const { payload: trimmedPayload, serialized } = applyMaxResponseSize( payload, maxResponseSize, @@ -921,9 +1484,29 @@ export async function registerSearchTool(agent: McpRegistrationAgent) { }), (value) => value.matches.length, ) + const trimmedMatchCount = Math.max( + 0, + outcome.result.matches.length - trimmedPayload.matches.length, + ) + const formattingMs = Math.max( + 0, + Math.round(performance.now() - formattingStartMs), + ) const result: SearchResultStructuredContent = { offline: trimmedPayload.offline, warnings: trimmedPayload.warnings, + telemetry: { + ...outcome.result.telemetry, + topResultTypes: trimmedPayload.matches + .slice(0, 5) + .map((match) => match.type), + trimmedMatchCount, + responseTrimmed: trimmedMatchCount > 0, + }, + phaseTimings: { + ...outcome.result.phaseTimings, + formattingMs, + }, ...(trimmedPayload.memories ? { memories: trimmedPayload.memories, @@ -951,6 +1534,21 @@ export async function registerSearchTool(agent: McpRegistrationAgent) { durationMs: timing.durationMs, baseUrl, hasUser, + message: 'Search completed successfully.', + context: { + task: outcome.result.intent.task.name, + intentConfidence: outcome.result.intent.confidence, + entityCount: outcome.result.intent.entities.length, + actionCount: outcome.result.intent.actions.length, + constraintCount: outcome.result.intent.constraints.length, + candidateCounts: outcome.result.telemetry.candidateCounts, + topResultTypes: result.telemetry?.topResultTypes ?? [], + responseTrimmed: result.telemetry?.responseTrimmed ?? false, + trimmedMatchCount, + offline: outcome.result.offline, + warningsCount: warnings.length, + phaseTimings: result.phaseTimings, + }, }) return { content: prependToolMetadataContent(conversationId, [ diff --git a/packages/worker/src/mcp/tools/understand-search-query.ts b/packages/worker/src/mcp/tools/understand-search-query.ts new file mode 100644 index 0000000000..c1e916f8e8 --- /dev/null +++ b/packages/worker/src/mcp/tools/understand-search-query.ts @@ -0,0 +1,394 @@ +import { type SearchEntityType } from './search-format.ts' + +const searchTokenPattern = /[a-z0-9]+/g +const trivialSearchStopwords = new Set([ + 'a', + 'an', + 'and', + 'for', + 'from', + 'i', + 'in', + 'into', + 'is', + 'me', + 'my', + 'of', + 'on', + 'or', + 'please', + 'the', + 'to', + 'with', +]) + +const taskLexicon = { + operate: [ + 'open', + 'play', + 'pause', + 'resume', + 'start', + 'stop', + 'run', + 'launch', + 'set', + 'turn', + 'skip', + 'queue', + 'volume', + 'remote', + ], + setup: [ + 'connect', + 'setup', + 'configure', + 'config', + 'oauth', + 'login', + 'auth', + 'authenticate', + 'secret', + 'token', + 'install', + ], + inspect: [ + 'list', + 'show', + 'status', + 'state', + 'details', + 'inspect', + 'view', + 'current', + ], + learn: [ + 'what', + 'how', + 'guide', + 'guides', + 'docs', + 'documentation', + 'explain', + 'usage', + 'schema', + ], + debug: [ + 'debug', + 'why', + 'error', + 'errors', + 'fail', + 'failed', + 'failing', + 'broken', + 'troubleshoot', + 'diagnose', + ], +} as const + +const deviceConstraintTerms = new Set([ + 'laptop', + 'computer', + 'desktop', + 'phone', + 'tablet', + 'browser', + 'tv', + 'app', +]) + +const stateConstraintTerms = new Set([ + 'active', + 'paused', + 'playing', + 'running', + 'connected', + 'disconnected', + 'current', + 'latest', +]) + +export type SearchTask = + | 'operate' + | 'setup' + | 'inspect' + | 'learn' + | 'debug' + | 'unknown' + +export type SearchConstraint = { + kind: 'device' | 'state' + value: string +} + +export type SearchIntentAction = { + name: SearchTask + matchedTerms: Array + confidence: number +} + +export type SearchIntentEntity = { + type: SearchEntityType + id: string + title: string + matchedTerms: Array + confidence: number +} + +export type SearchIntent = { + normalizedQuery: string + tokens: Array + meaningfulTokens: Array + phrases: Array + task: { + name: SearchTask + confidence: number + } + actions: Array + entities: Array + constraints: Array + confidence: number +} + +export type SearchableEntityDescriptor = { + type: SearchEntityType + id: string + title: string + primaryAliases: Array + secondaryAliases?: Array + tertiaryAliases?: Array +} + +export function normalizeSearchText(text: string): string { + return text + .normalize('NFKD') + .replace(/[\u0300-\u036f]/g, '') + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .replace(/[_./:-]+/g, ' ') + .toLowerCase() +} + +export function extractSearchTokens(text: string): Array { + return normalizeSearchText(text).match(searchTokenPattern) ?? [] +} + +export function extractMeaningfulSearchTokens(text: string): Array { + const tokens = extractSearchTokens(text) + const meaningful: Array = [] + const seen = new Set() + for (const token of tokens) { + if (token.length < 2) continue + if (trivialSearchStopwords.has(token)) continue + if (seen.has(token)) continue + seen.add(token) + meaningful.push(token) + } + return meaningful +} + +export function buildSearchPhrases( + tokens: ReadonlyArray, +): Array { + if (tokens.length < 2) return [] + const phrases: Array = [] + for (let index = 0; index < tokens.length - 1; index += 1) { + const first = tokens[index] + const second = tokens[index + 1] + if (!first || !second) continue + phrases.push(`${first} ${second}`) + } + return phrases +} + +function collectMatchedTerms( + queryTokens: ReadonlyArray, + phrases: ReadonlyArray, + fields: ReadonlyArray, +) { + const fieldTokens = new Set() + const normalizedFields = fields.map((field) => normalizeSearchText(field)) + for (const field of fields) { + for (const token of extractSearchTokens(field)) { + fieldTokens.add(token) + } + } + + const matchedTerms = queryTokens.filter((token) => fieldTokens.has(token)) + const matchedPhrases = phrases.filter((phrase) => + normalizedFields.some((field) => field.includes(phrase)), + ) + + return { + matchedTerms, + matchedPhrases, + } +} + +function scoreEntityDescriptor( + descriptor: SearchableEntityDescriptor, + queryTokens: ReadonlyArray, + phrases: ReadonlyArray, +) { + if (queryTokens.length === 0) { + return { + score: 0, + matchedTerms: [] as Array, + } + } + + const primary = collectMatchedTerms( + queryTokens, + phrases, + descriptor.primaryAliases, + ) + const secondary = collectMatchedTerms( + queryTokens, + phrases, + descriptor.secondaryAliases ?? [], + ) + const tertiary = collectMatchedTerms( + queryTokens, + phrases, + descriptor.tertiaryAliases ?? [], + ) + + const primaryScore = + primary.matchedTerms.length / Math.max(1, queryTokens.length) + + primary.matchedPhrases.length * 0.2 + const secondaryScore = + secondary.matchedTerms.length / Math.max(1, queryTokens.length) + + secondary.matchedPhrases.length * 0.15 + const tertiaryScore = + tertiary.matchedTerms.length / Math.max(1, queryTokens.length) + + tertiary.matchedPhrases.length * 0.1 + + const score = primaryScore * 0.75 + secondaryScore * 0.4 + tertiaryScore * 0.2 + const matchedTerms = Array.from( + new Set([ + ...primary.matchedTerms, + ...secondary.matchedTerms, + ...tertiary.matchedTerms, + ]), + ) + + return { score, matchedTerms } +} + +function inferTaskSignals( + meaningfulTokens: ReadonlyArray, + normalizedQuery: string, +) { + const tokenSet = new Set(meaningfulTokens) + const results = ( + Object.entries(taskLexicon) as Array<[SearchTask, ReadonlyArray]> + ).map(([task, terms]) => { + const matchedTerms = terms.filter( + (term) => + tokenSet.has(term) || + (term.includes(' ') && normalizedQuery.includes(term)) || + (term === 'setup' && normalizedQuery.includes('set up')), + ) + const setupBoost = + terms.includes('setup') && normalizedQuery.includes('set up') ? 1 : 0 + const score = + (matchedTerms.length + setupBoost) / + Math.max(1, Math.min(3, meaningfulTokens.length)) + return { + task, + matchedTerms, + score, + } + }) + + results.sort((left, right) => right.score - left.score) + const strongest = results[0] + if (!strongest || strongest.score <= 0) { + return { + task: { name: 'unknown' as const, confidence: 0 }, + actions: [] as Array, + } + } + + const actions = results + .filter((result) => result.score > 0) + .map((result) => ({ + name: result.task, + matchedTerms: result.matchedTerms, + confidence: Math.min(1, result.score), + })) + + return { + task: { + name: strongest.task, + confidence: Math.min(1, strongest.score), + }, + actions, + } +} + +function inferConstraints( + meaningfulTokens: ReadonlyArray, +): Array { + const constraints: Array = [] + for (const token of meaningfulTokens) { + if (deviceConstraintTerms.has(token)) { + constraints.push({ kind: 'device', value: token }) + continue + } + if (stateConstraintTerms.has(token)) { + constraints.push({ kind: 'state', value: token }) + } + } + return constraints +} + +export function understandSearchQuery(input: { + query: string + entities: Array +}): SearchIntent { + const normalizedQuery = normalizeSearchText(input.query).trim() + const tokens = extractSearchTokens(normalizedQuery) + const meaningfulTokens = extractMeaningfulSearchTokens(normalizedQuery) + const phrases = buildSearchPhrases(meaningfulTokens) + const { task, actions } = inferTaskSignals(meaningfulTokens, normalizedQuery) + const entities = input.entities + .map((descriptor) => { + const { score, matchedTerms } = scoreEntityDescriptor( + descriptor, + meaningfulTokens, + phrases, + ) + return { + type: descriptor.type, + id: descriptor.id, + title: descriptor.title, + matchedTerms, + confidence: Math.min(1, score), + } satisfies SearchIntentEntity + }) + .filter((entity) => entity.confidence > 0.15) + .sort((left, right) => right.confidence - left.confidence) + .slice(0, 8) + const constraints = inferConstraints(meaningfulTokens) + const confidenceInputs = [ + task.confidence, + entities[0]?.confidence ?? 0, + actions[0]?.confidence ?? 0, + ] + const confidence = + confidenceInputs.reduce((sum, value) => sum + value, 0) / + confidenceInputs.length + + return { + normalizedQuery, + tokens, + meaningfulTokens, + phrases, + task, + actions, + entities, + constraints, + confidence, + } +}