Implement repo-backed source foundation

packages/shared/src/chat.ts

@@ -47,6 +47,19 @@ export const mcpStorageContextSchema = object({
 	storageId: optional(nullable(string())),
 })
 
+export const mcpRepoContextSchema = object({
+	sourceId: optional(nullable(string())),
+	repoId: optional(nullable(string())),
+	sessionId: optional(nullable(string())),
+	sessionRepoId: optional(nullable(string())),
+	baseCommit: optional(nullable(string())),
+	manifestPath: optional(nullable(string())),
+	sourceRoot: optional(nullable(string())),
+	publishedCommit: optional(nullable(string())),
+	entityKind: optional(nullable(string())),
+	entityId: optional(nullable(string())),
+})
+
 const remoteConnectorRefSchema = object({
 	kind: remoteConnectorKindFieldSchema,
 	instanceId: remoteConnectorInstanceIdFieldSchema,
@@ -58,10 +71,12 @@ export const mcpCallerContextSchema = object({
 	homeConnectorId: optional(nullable(string())),
 	remoteConnectors: optional(nullable(array(remoteConnectorRefSchema))),
 	storageContext: optional(nullable(mcpStorageContextSchema)),
+	repoContext: optional(nullable(mcpRepoContextSchema)),
 })
 
 export type McpUserContext = InferOutput<typeof mcpUserContextSchema>
 export type McpStorageContext = InferOutput<typeof mcpStorageContextSchema>
+export type McpRepoContext = InferOutput<typeof mcpRepoContextSchema>
 export type McpCallerContext = InferOutput<typeof mcpCallerContextSchema>
 
 export const chatAgentPropsSchema = object({

packages/worker/migrations/0023-entity-sources.sql

@@ -0,0 +1,22 @@
+CREATE TABLE IF NOT EXISTS entity_sources (
+	id TEXT PRIMARY KEY,
+	user_id TEXT NOT NULL,
+	entity_kind TEXT NOT NULL,
+	entity_id TEXT NOT NULL,
+	repo_id TEXT NOT NULL,
+	published_commit TEXT,
+	indexed_commit TEXT,
+	manifest_path TEXT NOT NULL DEFAULT 'kody.json',
+	source_root TEXT NOT NULL DEFAULT '/',
+	created_at TEXT NOT NULL,
+	updated_at TEXT NOT NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_entity_sources_user_entity
+ON entity_sources(user_id, entity_kind, entity_id);
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_entity_sources_repo_id
+ON entity_sources(repo_id);
+
+CREATE INDEX IF NOT EXISTS idx_entity_sources_user_id
+ON entity_sources(user_id);

packages/worker/migrations/0024-repo-sessions-and-source-columns.sql

@@ -0,0 +1,49 @@
+CREATE TABLE IF NOT EXISTS repo_sessions (
+	id TEXT PRIMARY KEY,
+	user_id TEXT NOT NULL,
+	source_id TEXT NOT NULL,
+	session_repo_id TEXT NOT NULL,
+	session_repo_name TEXT NOT NULL,
+	session_repo_namespace TEXT NOT NULL DEFAULT 'default',
+	base_commit TEXT NOT NULL,
+	source_root TEXT NOT NULL DEFAULT '/',
+	conversation_id TEXT,
+	status TEXT NOT NULL DEFAULT 'active',
+	expires_at TEXT,
+	last_checkpoint_at TEXT,
+	last_checkpoint_commit TEXT,
+	last_check_run_id TEXT,
+	last_check_tree_hash TEXT,
+	created_at TEXT NOT NULL,
+	updated_at TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_repo_sessions_user_id
+ON repo_sessions(user_id);
+
+CREATE INDEX IF NOT EXISTS idx_repo_sessions_source_id
+ON repo_sessions(source_id);
+
+CREATE INDEX IF NOT EXISTS idx_repo_sessions_conversation_id
+ON repo_sessions(conversation_id);
+
+ALTER TABLE mcp_skills
+ADD COLUMN source_id TEXT;
+
+CREATE INDEX IF NOT EXISTS idx_mcp_skills_source_id
+ON mcp_skills(source_id);
+
+ALTER TABLE ui_artifacts
+ADD COLUMN source_id TEXT;
+
+CREATE INDEX IF NOT EXISTS idx_ui_artifacts_source_id
+ON ui_artifacts(source_id);
+
+ALTER TABLE jobs
+ADD COLUMN source_id TEXT;
+
+ALTER TABLE jobs
+ADD COLUMN published_commit TEXT;
+
+CREATE INDEX IF NOT EXISTS idx_jobs_source_id
+ON jobs(source_id);

packages/worker/package.json

@@ -8,7 +8,9 @@
   "dependencies": {
     "@cloudflare/ai-chat": "^0.4.4",
     "@cloudflare/codemode": "^0.3.4",
+    "@cloudflare/shell": "^0.3.2",
     "@cloudflare/think": "^0.2.4",
+    "@cloudflare/worker-bundler": "^0.1.1",
     "@cloudflare/workers-oauth-provider": "^0.4.0",
     "@epic-web/invariant": "^1.0.0",
     "@kody-internal/shared": "file:../shared",

packages/worker/src/agent-turn/tools.ts

@@ -19,6 +19,7 @@ import { slugifySkillCollectionName } from '#mcp/skills/skill-collections.ts'
 import { listUiArtifactsByUserId } from '#mcp/ui-artifacts-repo.ts'
 import { listValues } from '#mcp/values/service.ts'
 import { runCodemodeWithRegistry } from '#mcp/run-codemode-registry.ts'
+import { listJobs } from '#worker/jobs/service.ts'
 
 const defaultSearchLimit = 15
 const defaultMaxResponseSize = 4_000
@@ -61,6 +62,11 @@ export async function createAgentTurnToolSet(input: {
 						listUiArtifactsByUserId(input.env.APP_DB, userId!, {
 							hidden: false,
 						}),
+					loadJobs: () =>
+						listJobs({
+							env: input.env,
+							userId: userId!,
+						}),
 					loadUserSecrets: () =>
 						listUserSecretsForSearch({
 							env: input.env,
@@ -97,6 +103,7 @@ export async function createAgentTurnToolSet(input: {
 					userId,
 					skillRows: optionalRows.skillRows,
 					uiArtifactRows: optionalRows.uiArtifactRows,
+					jobRows: optionalRows.jobRows,
 					userSecretRows: optionalRows.userSecretRows,
 					userValueRows: optionalRows.userValueRows,
 					appSecretsByAppId,

packages/worker/src/app/handlers/saved-ui.ts

@@ -6,6 +6,7 @@ import { renderHostedSavedUiHtml } from '#app/saved-ui-hosted-html.ts'
 import { type routes } from '#app/routes.ts'
 import { createGeneratedUiAppSession } from '#mcp/generated-ui-app-session.ts'
 import { getUiArtifactByOwnerIds } from '#mcp/ui-artifacts-repo.ts'
+import { resolveSavedAppSource } from '#worker/repo/app-source.ts'
 
 async function getSavedUiForRequest(
 	request: Request,
@@ -47,8 +48,24 @@ export function createSavedUiPageHandler(env: Env) {
 				appId: artifact.id,
 				homeConnectorId: null,
 			})
-			const html = renderHostedSavedUiHtml({
+			const resolvedArtifact = await resolveSavedAppSource({
+				env,
+				baseUrl,
 				artifact,
+			})
+			const html = renderHostedSavedUiHtml({
+				artifact: {
+					...artifact,
+					title: resolvedArtifact.title,
+					description: resolvedArtifact.description,
+					hidden: resolvedArtifact.hidden,
+					parameters: resolvedArtifact.parameters
+						? JSON.stringify(resolvedArtifact.parameters)
+						: null,
+					clientCode: resolvedArtifact.clientCode,
+					serverCode: resolvedArtifact.serverCode,
+					serverCodeId: resolvedArtifact.serverCodeId,
+				},
 				appSession,
 				appBaseUrl: baseUrl,
 			})

packages/worker/src/env-schema.ts

@@ -202,6 +202,7 @@ export const EnvSchema = object({
 	CLOUDFLARE_API_TOKEN: optionalNonEmptyStringSchema,
 	CLOUDFLARE_API_BASE_URL: optionalUrlStringSchema,
 	CAPABILITY_REINDEX_SECRET: optionalNonEmptyStringSchema,
+	JOB_REINDEX_SECRET: optionalNonEmptyStringSchema,
 	HOME_CONNECTOR_SHARED_SECRET: optionalNonEmptyStringSchema,
 	REMOTE_CONNECTOR_SECRETS: optionalRemoteConnectorSecretsSchema,
 })

packages/worker/src/index.ts

@@ -8,6 +8,7 @@ import { AppFacetBridge, AppRunner } from './mcp/app-runner.ts'
 import { JobManager } from './jobs/manager-do.ts'
 import { StorageRunner } from './storage-runner.ts'
 import { AgentTurnRunner } from './agent-turn/runner-do.ts'
+import { RepoSession } from './repo/repo-session-do.ts'
 import { chatAgentBasePath } from '@kody-internal/shared/chat-routes.ts'
 import { getWorkerSentryOptions } from './sentry-options.ts'
 import { handleRequest } from '#app/handler.ts'
@@ -34,6 +35,7 @@ import {
 import { readGeneratedUiAppBackendSession } from './mcp/generated-ui-app-auth.ts'
 import { withCors } from './utils.ts'
 import { handleCapabilityReindexRequest } from './capability-maintenance.ts'
+import { handleJobReindexRequest } from './job-maintenance.ts'
 import { handleMemoryReindexRequest } from './memory-maintenance.ts'
 import { handleSkillReindexRequest } from './skill-maintenance.ts'
 import { handleUiArtifactReindexRequest } from './ui-artifact-maintenance.ts'
@@ -46,6 +48,7 @@ import {
 export {
 	ChatAgent,
 	AgentTurnRunner,
+	RepoSession,
 	CodemodeFetchGateway,
 	HomeConnectorSession,
 	HomeMCP,
@@ -115,6 +118,10 @@ const appHandler = withCors({
 			return handleUiArtifactReindexRequest(request, env)
 		}
 
+		if (url.pathname === '/__maintenance/reindex-jobs') {
+			return handleJobReindexRequest(request, env)
+		}
+
 		if (url.pathname === oauthPaths.authorize) {
 			return handleAuthorizeRequest(request, env)
 		}

packages/worker/src/job-maintenance.ts

@@ -0,0 +1,31 @@
+import { reindexJobVectors } from './jobs/job-reindex.ts'
+
+export async function handleJobReindexRequest(
+	request: Request,
+	env: Env,
+): Promise<Response> {
+	if (request.method !== 'POST') {
+		return new Response('Method Not Allowed', { status: 405 })
+	}
+
+	const secret = env.JOB_REINDEX_SECRET?.trim()
+	if (!secret) {
+		return new Response('Job reindex is not configured', { status: 503 })
+	}
+
+	const auth = request.headers.get('Authorization')?.trim()
+	const token = auth?.startsWith('Bearer ')
+		? auth.slice('Bearer '.length).trim()
+		: null
+	if (token !== secret) {
+		return new Response('Unauthorized', { status: 401 })
+	}
+
+	try {
+		const { upserted } = await reindexJobVectors(env)
+		return Response.json({ ok: true, upserted })
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error)
+		return Response.json({ ok: false, error: message }, { status: 500 })
+	}
+}

packages/worker/src/jobs/job-reindex.ts

@@ -0,0 +1,72 @@
+import {
+	getCapabilityVectorIndex,
+	isCapabilitySearchOffline,
+} from '#mcp/capabilities/capability-search.ts'
+import { buildJobEmbedText } from '#mcp/jobs-embed.ts'
+import { jobVectorId } from '#mcp/jobs-vectorize.ts'
+import { listJobs } from './service.ts'
+
+const upsertBatchSize = 16
+
+type JobEmbeddingBatchResult = { data?: Array<Array<number>> }
+
+export async function reindexJobVectors(
+	env: Env,
+): Promise<{ upserted: number }> {
+	const index = getCapabilityVectorIndex(env)
+	if (!index) {
+		throw new Error('CAPABILITY_VECTOR_INDEX binding is not configured')
+	}
+	if (isCapabilitySearchOffline(env)) {
+		return { upserted: 0 }
+	}
+
+	const rows = await env.APP_DB.prepare(
+		`SELECT DISTINCT user_id FROM jobs WHERE source_id IS NOT NULL OR code IS NOT NULL`,
+	).all<{ user_id: string }>()
+	const userIds = (rows.results ?? []).map((row) => row.user_id).filter(Boolean)
+	if (userIds.length === 0) return { upserted: 0 }
+
+	const jobs = (
+		await Promise.all(
+			userIds.map((userId) =>
+				listJobs({
+					env,
+					userId,
+				}).then((userJobs) => userJobs.map((job) => ({ userId, job }))),
+			),
+		)
+	).flat()
+	if (jobs.length === 0) return { upserted: 0 }
+
+	let upserted = 0
+	for (let offset = 0; offset < jobs.length; offset += upsertBatchSize) {
+		const batch = jobs.slice(offset, offset + upsertBatchSize)
+		const texts = batch.map(({ job }) =>
+			buildJobEmbedText({
+				name: job.name,
+				scheduleSummary: job.scheduleSummary,
+				sourceId: job.sourceId,
+				publishedCommit: job.publishedCommit,
+			}),
+		)
+		const result = (await env.AI.run('@cf/baai/bge-small-en-v1.5', {
+			text: texts,
+			pooling: 'mean',
+		})) as JobEmbeddingBatchResult
+		const vectors = result.data
+		if (!vectors || vectors.length !== batch.length) {
+			throw new Error('Workers AI embedding batch size mismatch for jobs')
+		}
+		await index.upsert(
+			batch.map(({ userId, job }, index_) => ({
+				id: jobVectorId(job.id),
+				values: vectors[index_]!,
+				metadata: { kind: 'job', userId },
+			})),
+		)
+		upserted += batch.length
+	}
+
+	return { upserted }
+}