Add package retriever hooks

docs/contributing/packages-and-manifests.md

@@ -19,6 +19,7 @@ Use `package.json` as the canonical source of truth for saved package metadata.
 - `kody.tags` — search tags
 - `kody.app` — optional hosted package app config
 - `kody.jobs` — optional package-owned schedules
+- `kody.retrievers` — optional package-owned search/context retrievers
 
 The package manifest is `package.json`.
 
@@ -30,6 +31,7 @@ Think in terms of:
 - package exports
 - package apps
 - package-owned jobs
+- package-owned retrievers
 
 The top-level saved identity is the package.
 
@@ -67,6 +69,46 @@ Jobs belong to packages.
 
 Jobs are not their own top-level saved primitive.
 
+## Package-owned retrievers
+
+Retrievers let packages return user-owned documents or facts to Kody search and
+automatic context retrieval without promoting those records to durable memory.
+
+- Define retrievers under `package.json#kody.retrievers`
+- Each retriever names a package export, display name, description, and one or
+  more scopes: `search`, `context`
+- Package metadata remains the source of truth; runtime discovery uses derived
+  KV manifest and scope indexes that are rebuilt on package refresh
+- Retriever exports run read-only against the package storage bucket
+
+Example:
+
+```json
+{
+	"kody": {
+		"retrievers": {
+			"personal-inbox": {
+				"export": "./search",
+				"name": "Personal Inbox",
+				"description": "Searches saved notes and snippets.",
+				"scopes": ["search"],
+				"timeoutMs": 1000,
+				"maxResults": 5
+			}
+		}
+	}
+}
+```
+
+Retriever exports receive `params` with `query`, `scope`, `memoryContext`,
+`limit`, and `conversationId`, and return `{ "results": [...] }` where each
+result has `id`, `title`, `summary`, optional `details`, optional `score`,
+optional `source`, optional `url`, and optional `metadata`.
+
+The runtime validates retriever output before surfacing it. A retriever may
+return at most 20 results; payloads with more than 20 results are rejected.
+Retriever implementations should truncate or paginate before returning.
+
 ## Repo-backed workflow
 
 Package source is edited and published through the repo session capabilities.

packages/worker/src/agent-turn/tools.ts

@@ -15,6 +15,21 @@ import { listUserSecretsForSearch } from '#mcp/secrets/service.ts'
 import { listSavedPackagesByUserId } from '#worker/package-registry/repo.ts'
 import { listValues } from '#mcp/values/service.ts'
 import { runModuleWithRegistry } from '#mcp/run-codemode-registry.ts'
+import * as Sentry from '@sentry/cloudflare'
+
+function toRetrieverWarning(error: unknown) {
+	console.error(
+		JSON.stringify({
+			message: 'package retriever search unavailable',
+		}),
+	)
+	Sentry.captureException(error, {
+		tags: {
+			scope: 'agent-turn.package-retrievers',
+		},
+	})
+	return 'Package retrievers are temporarily unavailable.'
+}
 
 const defaultSearchLimit = 15
 const defaultMaxResponseSize = 4_000
@@ -45,7 +60,31 @@ export async function createAgentTurnToolSet(input: {
 			}),
 			execute: async (args) => {
 				const userId = input.callerContext.user?.userId ?? null
-				const [registry, optionalRows] = await Promise.all([
+				const retrieverSearchPromise = (async () => {
+					if (!userId) return { results: [], warnings: [] }
+					try {
+						const { runPackageRetrievers } =
+							await import('#worker/package-retrievers/service.ts')
+						return await runPackageRetrievers({
+							env: input.env,
+							baseUrl: input.callerContext.baseUrl,
+							userId,
+							scope: 'search',
+							query: args.query,
+							memoryContext: resolveSearchMemoryContext({
+								query: args.query,
+								memoryContext: input.memoryContext ?? undefined,
+							}),
+							conversationId: input.conversationId,
+						})
+					} catch (error) {
+						return {
+							results: [],
+							warnings: [toRetrieverWarning(error)],
+						}
+					}
+				})()
+				const [registry, optionalRows, retrieverSearch] = await Promise.all([
 					getCapabilityRegistryForContext({
 						env: input.env,
 						callerContext: input.callerContext,
@@ -81,13 +120,15 @@ export async function createAgentTurnToolSet(input: {
 								},
 							}),
 					}),
+					retrieverSearchPromise,
 				])
 				const result = await searchUnified({
 					env: input.env,
 					query: args.query,
 					limit: args.limit ?? defaultSearchLimit,
 					registry,
 					optionalRows,
+					retrieverResults: retrieverSearch.results,
 				})
 				const remoteConnectorStatuses = await loadDownRemoteConnectorStatuses({
 					env: input.env,
@@ -104,13 +145,15 @@ export async function createAgentTurnToolSet(input: {
 				})
 				return {
 					offline: result.offline,
-					warnings: optionalRows.warnings,
+					warnings: [...optionalRows.warnings, ...retrieverSearch.warnings],
 					guidance: result.guidance,
 					memories: memoryToolContext
 						? {
 								surfaced: memoryToolContext.memories,
 								suppressedCount: memoryToolContext.suppressedCount,
 								retrievalQuery: memoryToolContext.retrievalQuery,
+								retrieverResults: memoryToolContext.retrieverResults,
+								retrieverWarnings: memoryToolContext.retrieverWarnings,
 							}
 						: undefined,
 					remoteConnectorStatuses: remoteConnectorStatuses.map((status) => ({
@@ -124,6 +167,7 @@ export async function createAgentTurnToolSet(input: {
 						matches: result.matches,
 						baseUrl: input.callerContext.baseUrl,
 					}),
+					retrieverResults: retrieverSearch.results,
 				}
 			},
 		}),

packages/worker/src/mcp/tools/markdown-safety.ts

@@ -0,0 +1,14 @@
+const markdownSpecialChars = /[\\`*_{}[\]()#+\-.!|>]/g
+
+export function escapeMarkdownText(value: string) {
+	return value.replace(markdownSpecialChars, '\\$&').replace(/\r?\n/g, ' ')
+}
+
+export function formatMarkdownInlineCode(value: string) {
+	if (!value.includes('`')) return `\`${value}\``
+	const backtickRuns = value.match(/`+/g) ?? []
+	const delimiter = '`'.repeat(
+		Math.max(...backtickRuns.map((run) => run.length + 1)),
+	)
+	return `${delimiter} ${value} ${delimiter}`
+}

packages/worker/src/mcp/tools/memory-tool-context.node.test.ts

@@ -0,0 +1,170 @@
+import { expect, test, vi } from 'vitest'
+
+const mockModule = vi.hoisted(() => ({
+	surfaceRelevantMemories: vi.fn(),
+	runPackageRetrievers: vi.fn(),
+}))
+
+vi.mock('#mcp/memory/service.ts', () => ({
+	surfaceRelevantMemories: (...args: Array<unknown>) =>
+		mockModule.surfaceRelevantMemories(...args),
+}))
+
+vi.mock('#worker/package-retrievers/service.ts', () => ({
+	runPackageRetrievers: (...args: Array<unknown>) =>
+		mockModule.runPackageRetrievers(...args),
+}))
+
+const { loadRelevantMemoriesForTool } = await import('./memory-tool-context.ts')
+const { formatSurfacedMemoriesMarkdown } =
+	await import('./memory-tool-context.ts')
+
+function setupMemoryContextMocks() {
+	mockModule.surfaceRelevantMemories.mockReset()
+	mockModule.runPackageRetrievers.mockReset()
+	mockModule.surfaceRelevantMemories.mockResolvedValue({
+		memories: [],
+		suppressedCount: 0,
+		retrievalQuery: 'sprinkler instructions',
+	})
+	mockModule.runPackageRetrievers.mockResolvedValue({
+		results: [
+			{
+				id: 'note-1',
+				title: '## Sprinkler controller',
+				summary: '```ignore\nHold next and back for setup mode.\n```',
+				packageId: 'package-1',
+				kodyId: 'personal-inbox',
+				retrieverKey: 'notes',
+				retrieverName: 'Personal notes',
+			},
+		],
+		warnings: [
+			'Package retriever "personal-inbox/notes" failed and was skipped.',
+		],
+	})
+}
+
+test('loadRelevantMemoriesForTool returns context retriever results alongside memories', async () => {
+	setupMemoryContextMocks()
+	const result = await loadRelevantMemoriesForTool({
+		env: { APP_DB: {}, AI: {} } as Env,
+		callerContext: {
+			baseUrl: 'https://heykody.dev',
+			user: {
+				userId: 'user-1',
+				email: 'user@example.com',
+				displayName: 'User',
+			},
+			storageContext: null,
+			homeConnectorId: null,
+			remoteConnectors: null,
+			repoContext: null,
+		},
+		conversationId: 'conversation-1',
+		memoryContext: {
+			query: 'sprinkler instructions',
+		},
+	})
+
+	expect(mockModule.runPackageRetrievers).toHaveBeenCalledWith(
+		expect.objectContaining({
+			baseUrl: 'https://heykody.dev',
+			userId: 'user-1',
+			scope: 'context',
+			query: 'sprinkler instructions',
+			maxProviders: 3,
+		}),
+	)
+	expect(result?.memories).toEqual([])
+	expect(result?.retrieverResults).toEqual([
+		expect.objectContaining({
+			id: 'note-1',
+			kodyId: 'personal-inbox',
+			retrieverKey: 'notes',
+		}),
+	])
+	expect(result?.retrieverWarnings).toEqual([
+		'Package retriever "personal-inbox/notes" failed and was skipped.',
+	])
+})
+
+test('loadRelevantMemoriesForTool keeps memories when context retrievers fail', async () => {
+	setupMemoryContextMocks()
+	mockModule.surfaceRelevantMemories.mockResolvedValue({
+		memories: [
+			{
+				id: 'memory-1',
+				category: 'workflow',
+				status: 'active',
+				subject: 'Sprinkler setup',
+				summary: 'Sprinkler instructions are stored in notes.',
+				details: '',
+				tags: ['sprinkler'],
+				sourceUris: [],
+				updatedAt: '2026-04-28T00:00:00.000Z',
+			},
+		],
+		suppressedCount: 0,
+		retrievalQuery: 'sprinkler instructions',
+	})
+	mockModule.runPackageRetrievers.mockRejectedValue(
+		new Error('retriever unavailable'),
+	)
+
+	const result = await loadRelevantMemoriesForTool({
+		env: { APP_DB: {}, AI: {} } as Env,
+		callerContext: {
+			baseUrl: 'https://heykody.dev',
+			user: {
+				userId: 'user-1',
+				email: 'user@example.com',
+				displayName: 'User',
+			},
+			storageContext: null,
+			homeConnectorId: null,
+			remoteConnectors: null,
+			repoContext: null,
+		},
+		conversationId: 'conversation-1',
+		memoryContext: {
+			query: 'sprinkler instructions',
+		},
+	})
+
+	expect(result?.memories).toEqual([
+		expect.objectContaining({
+			id: 'memory-1',
+			subject: 'Sprinkler setup',
+		}),
+	])
+	expect(result?.retrieverResults).toEqual([])
+	expect(result?.retrieverWarnings).toEqual([])
+})
+
+test('formatSurfacedMemoriesMarkdown omits empty memories heading for retriever-only context', () => {
+	const [content] = formatSurfacedMemoriesMarkdown({
+		memories: [],
+		retrieverResults: [
+			{
+				id: 'note-1',
+				title: 'Sprinkler controller',
+				summary: 'Hold next and back for setup mode.',
+				packageId: 'package-1',
+				kodyId: 'personal-inbox',
+				retrieverKey: 'notes',
+				retrieverName: 'Personal notes',
+			},
+		],
+		retrieverWarnings: [],
+		suppressedCount: 0,
+		retrievalQuery: 'sprinkler instructions',
+	})
+
+	expect(content?.type).toBe('text')
+	expect(content?.text).not.toContain('## Relevant memories')
+	expect(content?.text?.startsWith('## Relevant retriever results')).toBe(true)
+	expect(content?.text).toContain('## Relevant retriever results')
+	expect(content?.text).not.toContain('## Sprinkler controller')
+	expect(content?.text).not.toContain('```ignore')
+})