Keep production Vectorize reindex under the deploy timeout

.github/workflows/deploy.yml

@@ -261,7 +261,7 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     name: 🚀 Deploy to production
-    timeout-minutes: 12
+    timeout-minutes: 20
     needs:
       - sha-guard
       - deploy-jobs-worker
@@ -684,20 +684,43 @@ jobs:
             exit 0
           fi
           base="${DEPLOY_URL%/}"
-          echo "POST ${base}/__maintenance/reindex-capabilities"
-          status="$(curl --silent --show-error --location --max-time 120 \
-            -X POST \
-            -H "Authorization: Bearer ${CAPABILITY_REINDEX_SECRET}" \
-            -H "Accept: application/json" \
-            --output reindex-response.json \
-            --write-out "%{http_code}" \
-            "${base}/__maintenance/reindex-capabilities")"
-          cat reindex-response.json
-          echo
-          if [ "$status" -lt 200 ] || [ "$status" -ge 300 ]; then
-            echo "Capability reindex failed with HTTP ${status}." >&2
-            exit 1
-          fi
+          max_sweeps=8
+          sweep=1
+          payload='{}'
+          while true; do
+            echo "POST ${base}/__maintenance/reindex-capabilities (sweep ${sweep})"
+            status="$(curl --silent --show-error --location --max-time 120 \
+              -X POST \
+              -H "Authorization: Bearer ${CAPABILITY_REINDEX_SECRET}" \
+              -H "Accept: application/json" \
+              -H "Content-Type: application/json" \
+              --data "${payload}" \
+              --output reindex-response.json \
+              --write-out "%{http_code}" \
+              "${base}/__maintenance/reindex-capabilities")"
+            cat reindex-response.json
+            echo
+            if [ "$status" -lt 200 ] || [ "$status" -ge 300 ]; then
+              echo "Capability reindex failed with HTTP ${status}." >&2
+              exit 1
+            fi
+            complete="$(jq -r '.complete // false' reindex-response.json)"
+            if [ "$complete" = "true" ]; then
+              echo "Capability reindex complete after ${sweep} sweep(s)."
+              break
+            fi
+            if [ "$sweep" -ge "$max_sweeps" ]; then
+              echo "Capability reindex did not finish after ${max_sweeps} sweeps." >&2
+              exit 1
+            fi
+            cursor="$(jq -c '.cursor // empty' reindex-response.json)"
+            if [ -z "$cursor" ] || [ "$cursor" = "null" ]; then
+              echo "Capability reindex is incomplete but returned no cursor." >&2
+              exit 1
+            fi
+            payload="$(jq -nc --argjson cursor "$cursor" '{cursor: $cursor}')"
+            sweep=$((sweep + 1))
+          done
 
   deploy-status-worker:
     runs-on: ubuntu-latest

docs/contributing/adding-capabilities.md

@@ -400,7 +400,9 @@ the domain modules.
    keywords, or schemas, run the guarded
    `POST /__maintenance/reindex-capabilities` maintenance endpoint so Vectorize
    has fresh Workers AI embeddings. The endpoint also rebuilds memory, job, and
-   saved-package vectors with user-scoped metadata.
+   saved-package vectors with user-scoped metadata. Each POST is time-budgeted;
+   if the JSON response has `complete: false`, POST again with
+   `{ "cursor": <response.cursor> }` until `complete` is true.
 
 Example (assuming `example` exists in `capabilityDomainNames`):
 

docs/contributing/architecture/data-storage.md

@@ -1348,7 +1348,10 @@ deterministic so upserts and deletes target the same vector.
 Search paths query only per-account namespaces plus the reserved builtin
 namespace. Vector rows are derived from D1 and can be rebuilt with the bounded
 `POST /__maintenance/reindex-capabilities` sweep, which keyset-pages memory,
-job, and saved-package rows and rebuilds builtins in their reserved namespace.
+job, and saved-package rows, rebuilds builtins in their reserved namespace, and
+returns before a request-time budget. An incomplete sweep includes
+`complete: false` and a `cursor` so the caller can POST again until `complete`
+is true. Production deploy CI loops that endpoint after each production ship.
 
 ### `entity_sources` and package import contracts
 

docs/contributing/environment-variables.md

@@ -282,10 +282,11 @@ Worker secrets:
   endpoints. Use the reindex endpoint after changing the embedding model,
   pooling, or Vectorize index dimensions; it rebuilds built-in capability,
   memory, job, and saved-package vectors with per-user `userId` metadata on
-  user-owned rows. Use the re-encrypt endpoint to rewrite remaining pre-AAD
-  (2-part) secret ciphertexts to v2 without rotating `SECRET_STORE_KEY` (see
-  [Secret rotation](./secret-rotation.md)). Local dev uses offline search while
-  `WRANGLER_IS_LOCAL_DEV` is set or the binding is missing.
+  user-owned rows. Each call is time-budgeted and may return `complete: false`
+  plus a `cursor` to resume. Use the re-encrypt endpoint to rewrite remaining
+  pre-AAD (2-part) secret ciphertexts to v2 without rotating `SECRET_STORE_KEY`
+  (see [Secret rotation](./secret-rotation.md)). Local dev uses offline search
+  while `WRANGLER_IS_LOCAL_DEV` is set or the binding is missing.
 - **`JOB_REINDEX_SECRET`** — optional Worker secret; bearer token for
   `POST /__maintenance/reindex-jobs` when you want a jobs-only Vectorize rebuild
   (without the full capability/memory/package reindex that

docs/contributing/setup-manifest.md

@@ -681,10 +681,12 @@ How to get/set each value:
   - Generate a long random secret (for example `openssl rand -hex 32`), store it
     as the repository secret `CAPABILITY_REINDEX_SECRET`, and let the deploy
     workflow sync it to the Worker. After each production deploy, CI POSTs to
-    `/__maintenance/reindex-capabilities` with `Authorization: Bearer …` to
-    refresh built-in capability, memory, job, and saved-package embeddings. Run
-    the same POST manually after changing the embedding model, pooling, or
-    Vectorize index dimensions so existing rows are rebuilt with compatible
+    `/__maintenance/reindex-capabilities` with `Authorization: Bearer …` and
+    loops on the returned `cursor` until `complete` is true (each POST is
+    bounded to about 70 seconds; CI follows the cursor for up to 8 sweeps),
+    refreshing built-in capability, memory, job, and saved-package embeddings.
+    Run the same POST loop manually after changing the embedding model, pooling,
+    or Vectorize index dimensions so existing rows are rebuilt with compatible
     vectors. The same bearer authenticates
     `POST /__maintenance/reencrypt-secrets` (see
     [Secret rotation](./secret-rotation.md)). Local and preview environments can

packages/worker/src/capability-maintenance.node.test.ts

@@ -47,22 +47,33 @@ function resetMocks() {
 	mockModule.reindexSavedPackageVectors.mockReset()
 }
 
-function createReindexRequest() {
+function completeStep(upserted: number) {
+	return { upserted, complete: true, afterId: null }
+}
+
+function createReindexRequest(
+	body?: Record<string, unknown>,
+	input: { method?: string } = {},
+) {
 	return new Request(
 		'https://kody.example.com/__maintenance/reindex-capabilities',
 		{
-			method: 'POST',
-			headers: { Authorization: 'Bearer secret' },
+			method: input.method ?? 'POST',
+			headers: {
+				Authorization: 'Bearer secret',
+				'Content-Type': 'application/json',
+			},
+			body: body === undefined ? undefined : JSON.stringify(body),
 		},
 	)
 }
 
-test('capability reindex maintenance route rebuilds every vector kind', async () => {
+test('capability reindex maintenance route rebuilds every vector kind and resumes incomplete sweeps', async () => {
 	resetMocks()
-	mockModule.reindexCapabilityVectors.mockResolvedValue({ upserted: 3 })
-	mockModule.reindexMemoryVectors.mockResolvedValue({ upserted: 2 })
-	mockModule.reindexJobVectors.mockResolvedValue({ upserted: 1 })
-	mockModule.reindexSavedPackageVectors.mockResolvedValue({ upserted: 4 })
+	mockModule.reindexCapabilityVectors.mockResolvedValue(completeStep(3))
+	mockModule.reindexMemoryVectors.mockResolvedValue(completeStep(2))
+	mockModule.reindexJobVectors.mockResolvedValue(completeStep(1))
+	mockModule.reindexSavedPackageVectors.mockResolvedValue(completeStep(4))
 	const env = {
 		CAPABILITY_REINDEX_SECRET: 'secret',
 	} as Env
@@ -75,10 +86,11 @@ test('capability reindex maintenance route rebuilds every vector kind', async ()
 	expect(response.status).toBe(200)
 	await expect(response.json()).resolves.toEqual({
 		ok: true,
-		capabilities: { upserted: 3 },
-		memories: { upserted: 2 },
-		jobs: { upserted: 1 },
-		packages: { upserted: 4 },
+		complete: true,
+		capabilities: completeStep(3),
+		memories: completeStep(2),
+		jobs: completeStep(1),
+		packages: completeStep(4),
 	})
 	expect(mockModule.reindexCapabilityVectors).toHaveBeenCalledWith(
 		env,
@@ -87,21 +99,94 @@ test('capability reindex maintenance route rebuilds every vector kind', async ()
 				name: 'example_capability',
 			}),
 		}),
+		{
+			afterId: null,
+			deadlineMs: expect.any(Number),
+		},
 	)
-	expect(mockModule.reindexMemoryVectors).toHaveBeenCalledWith(env)
-	expect(mockModule.reindexJobVectors).toHaveBeenCalledWith(env)
+	expect(mockModule.reindexMemoryVectors).toHaveBeenCalledWith(env, {
+		afterId: null,
+		deadlineMs: expect.any(Number),
+	})
+	expect(mockModule.reindexJobVectors).toHaveBeenCalledWith(env, {
+		afterId: null,
+		deadlineMs: expect.any(Number),
+	})
 	expect(mockModule.reindexSavedPackageVectors).toHaveBeenCalledWith(env, {
 		baseUrl: 'https://kody.example.com',
+		afterId: null,
+		deadlineMs: expect.any(Number),
+	})
+
+	resetMocks()
+	mockModule.reindexCapabilityVectors.mockResolvedValue(completeStep(3))
+	mockModule.reindexMemoryVectors.mockResolvedValue({
+		upserted: 8,
+		complete: false,
+		afterId: 'memory-8',
+	})
+	const incompleteResponse = await handleCapabilityReindexRequest(
+		createReindexRequest({ timeBudgetMs: 5_000 }),
+		env,
+	)
+	expect(incompleteResponse.status).toBe(200)
+	await expect(incompleteResponse.json()).resolves.toEqual({
+		ok: true,
+		complete: false,
+		cursor: { phase: 'memories', afterId: 'memory-8' },
+		capabilities: completeStep(3),
+		memories: { upserted: 8, complete: false, afterId: 'memory-8' },
+		jobs: completeStep(0),
+		packages: completeStep(0),
+	})
+	expect(mockModule.reindexJobVectors).not.toHaveBeenCalled()
+	expect(mockModule.reindexSavedPackageVectors).not.toHaveBeenCalled()
+
+	resetMocks()
+	mockModule.reindexMemoryVectors.mockResolvedValue(completeStep(2))
+	mockModule.reindexJobVectors.mockResolvedValue(completeStep(1))
+	mockModule.reindexSavedPackageVectors.mockResolvedValue(completeStep(4))
+	const resumeResponse = await handleCapabilityReindexRequest(
+		createReindexRequest({
+			cursor: { phase: 'memories', afterId: 'memory-8' },
+		}),
+		env,
+	)
+	expect(resumeResponse.status).toBe(200)
+	await expect(resumeResponse.json()).resolves.toEqual({
+		ok: true,
+		complete: true,
+		capabilities: completeStep(0),
+		memories: completeStep(2),
+		jobs: completeStep(1),
+		packages: completeStep(4),
+	})
+	expect(mockModule.reindexCapabilityVectors).not.toHaveBeenCalled()
+	expect(mockModule.reindexMemoryVectors).toHaveBeenCalledWith(env, {
+		afterId: 'memory-8',
+		deadlineMs: expect.any(Number),
+	})
+
+	const invalidCursorResponse = await handleCapabilityReindexRequest(
+		createReindexRequest({ cursor: { phase: 'nope', afterId: null } }),
+		env,
+	)
+	expect(invalidCursorResponse.status).toBe(400)
+	await expect(invalidCursorResponse.json()).resolves.toEqual({
+		ok: false,
+		error: 'cursor.phase must be capabilities, memories, jobs, or packages.',
 	})
 })
 
 test('capability reindex maintenance route attempts every vector kind before reporting failures', async () => {
 	resetMocks()
-	mockModule.reindexCapabilityVectors.mockResolvedValue({ upserted: 3 })
+	mockModule.reindexCapabilityVectors.mockResolvedValue(completeStep(3))
 	mockModule.reindexMemoryVectors.mockRejectedValue(new Error('memory failed'))
-	mockModule.reindexJobVectors.mockResolvedValue({ upserted: 1 })
+	mockModule.reindexJobVectors.mockResolvedValue(completeStep(1))
 	mockModule.reindexSavedPackageVectors.mockResolvedValue({
 		upserted: 4,
+		complete: true,
+		afterId: null,
 		failed: 1,
 		error: '1 saved package vector(s) failed to reindex',
 	})
@@ -117,11 +202,19 @@ test('capability reindex maintenance route attempts every vector kind before rep
 	expect(response.status).toBe(500)
 	await expect(response.json()).resolves.toEqual({
 		ok: false,
-		capabilities: { upserted: 3 },
-		memories: { upserted: 0, error: 'memory failed' },
-		jobs: { upserted: 1 },
+		complete: true,
+		capabilities: completeStep(3),
+		memories: {
+			upserted: 0,
+			complete: false,
+			afterId: null,
+			error: 'memory failed',
+		},
+		jobs: completeStep(1),
 		packages: {
 			upserted: 4,
+			complete: true,
+			afterId: null,
 			failed: 1,
 			error: '1 saved package vector(s) failed to reindex',
 		},