fix(ci): docs-sync bot — no errors, no timeouts, no lost PRs

.github/docs-sync/edit.mjs

@@ -6,20 +6,22 @@
  * Batching bounds each `kilo run` context (a replay window can yield dozens
  * of docs-worthy PRs with large diffs). Each batch gets its own CLI session
  * and writes its own summary file; results are merged into
- * docs-sync-out/edit-summary.json. A batch that fails is skipped with a
- * warning — its PRs show up in the rolling PR body as skipped, so nothing
- * fails silently.
+ * docs-sync-out/edit-summary.json. A batch that fails or is deferred by the
+ * wall-clock budget is recorded as action "pending" so the watermark holds
+ * back and the next run re-collects those PRs.
  *
  * Env: EDIT_MODEL (provider/model), KILO_API_KEY + KILO_ORG_ID (set by workflow; read natively by the kilo provider).
+ * Budgets: EDIT_BUDGET_MINUTES (default 50), EDIT_BATCH_TIMEOUT_MINUTES (default 15).
+ * Test hook: DOCS_SYNC_BACKOFF_MS replaces every retry wait when set.
  */
 
-import { execFileSync } from "node:child_process"
 import fs from "node:fs"
 import path from "node:path"
 import { fileURLToPath } from "node:url"
+import { backoffMsForAttempt, deadline, remainingMs, runKilo, sleepSync } from "./lib.mjs"
 
 const BATCH_SIZE = 5
-const ATTEMPTS = 2
+const ATTEMPTS = 3
 const OUT_DIR = "docs-sync-out"
 export const SUMMARY_FILE = ".docs-sync-summary.json"
 
@@ -28,6 +30,10 @@ const basePrompt = fs.readFileSync(path.join(HERE, "edit-prompt.md"), "utf8")
 const model = process.env.EDIT_MODEL
 if (!model) throw new Error("EDIT_MODEL is required")
 
+const EDIT_BUDGET_MINUTES = Number(process.env.EDIT_BUDGET_MINUTES) || 50
+const EDIT_BATCH_TIMEOUT_MINUTES = Number(process.env.EDIT_BATCH_TIMEOUT_MINUTES) || 15
+const BATCH_TIMEOUT_MS = EDIT_BATCH_TIMEOUT_MINUTES * 60 * 1000
+
 const worthy = JSON.parse(fs.readFileSync(`${OUT_DIR}/worthy.json`, "utf8"))
 const triage = JSON.parse(fs.readFileSync(`${OUT_DIR}/triage.json`, "utf8"))
 const priority = new Map(triage.map((e) => [e.url, e]))
@@ -36,7 +42,18 @@ const ordered = [...worthy].sort((a, b) => {
   return (rank[priority.get(a.url)?.priority] ?? 1) - (rank[priority.get(b.url)?.priority] ?? 1)
 })
 
-function editBatch(batch, index) {
+/** @type {Map<string, string>} url → pending cause for failed/deferred batches */
+const pendingCauses = new Map()
+
+function formatCause(result) {
+  const bits = []
+  if (result.timedOut) bits.push("timed out")
+  if (result.exitCode !== null && result.exitCode !== undefined) bits.push(`exit ${result.exitCode}`)
+  if (result.stderrTail) bits.push(result.stderrTail.replaceAll("\n", " ").slice(0, 200))
+  return bits.join("; ") || "no diagnostic"
+}
+
+function editBatch(batch, index, budgetDeadline) {
   const batchFile = `${OUT_DIR}/edit-batch-${index}.json`
   const triageFile = `${OUT_DIR}/edit-batch-triage-${index}.json`
   const summaryFile = `${OUT_DIR}/edit-summary-${index}.json`
@@ -54,31 +71,59 @@ function editBatch(batch, index) {
 
 Batch specifics for this run: the PRs to handle are in the attached ${batchFile} (full details) and ${triageFile} (triage verdicts). Handle ONLY the PRs in these batch files. When finished, write your per-PR results in the summary JSON format described above to the file \`${summaryFile}\` (path relative to the repository root).`
 
+  let lastCause = "edit pass failed"
   for (let attempt = 1; attempt <= ATTEMPTS; attempt++) {
-    try {
-      // Message positional first: --file is multi-value and would otherwise
-      // consume a trailing message as a file path ("File not found").
-      execFileSync(
-        "kilo",
-        ["run", prompt, "-m", model, "--variant", "high", "--dir", process.cwd(), "-f", batchFile, "-f", triageFile],
-        // stdout streams live to the Actions log; stderr is piped so failure
-        // warnings can include the tail of the actual CLI error.
-        { encoding: "utf8", maxBuffer: 32 * 1024 * 1024, timeout: 25 * 60 * 1000, stdio: ["ignore", "inherit", "pipe"] },
+    const left = remainingMs(budgetDeadline)
+    if (left < BATCH_TIMEOUT_MS) {
+      lastCause = `edit budget exhausted before batch ${index} attempt ${attempt} (${Math.ceil(left / 1000)}s left, need ${EDIT_BATCH_TIMEOUT_MINUTES}m)`
+      console.warn(
+        `batch ${index}: stopping retries — remaining budget cannot fit another ${EDIT_BATCH_TIMEOUT_MINUTES}m attempt`,
       )
-      if (fs.existsSync(summaryFile)) return true
-      // Tolerate the agent dropping the docs-sync-out/ prefix.
-      const alt = path.basename(summaryFile)
-      if (fs.existsSync(alt)) {
-        fs.renameSync(alt, summaryFile)
-        return true
+      break
+    }
+
+    const result = runKilo({
+      args: ["run", prompt, "-m", model, "--variant", "high", "--dir", process.cwd(), "-f", batchFile, "-f", triageFile],
+      timeoutMs: Math.min(BATCH_TIMEOUT_MS, left),
+      streamStdout: true,
+      label: `edit batch ${index} attempt ${attempt}`,
+    })
+
+    if (fs.existsSync(summaryFile)) return true
+    // Tolerate the agent dropping the docs-sync-out/ prefix.
+    const alt = path.basename(summaryFile)
+    if (fs.existsSync(alt)) {
+      fs.renameSync(alt, summaryFile)
+      return true
+    }
+
+    // Exit 0 is not success: missing summary is a failure logged WITH the
+    // captured stderrTail and exit code on every attempt.
+    const cause = formatCause(result)
+    lastCause = `edit batch ${index}: ${cause}`
+    console.warn(
+      `batch ${index} attempt ${attempt}: summary file ${summaryFile} not produced` +
+        ` (exit ${result.exitCode}${result.timedOut ? ", timed out" : ""})` +
+        (result.stderrTail ? `\nstderr tail:\n${result.stderrTail}` : "\nstderr tail: (empty)"),
+    )
+
+    if (attempt < ATTEMPTS) {
+      const wait = backoffMsForAttempt(attempt)
+      // Skip the wait when the remaining budget cannot fit another attempt.
+      const afterWait = remainingMs(budgetDeadline) - wait
+      if (wait > 0 && afterWait >= BATCH_TIMEOUT_MS) {
+        console.warn(`batch ${index}: backing off ${wait / 1000}s before attempt ${attempt + 1}`)
+        sleepSync(wait)
+      } else if (wait > 0) {
+        console.warn(
+          `batch ${index}: skipping backoff — remaining budget cannot fit attempt ${attempt + 1} after wait`,
+        )
       }
-      console.warn(`batch ${index} attempt ${attempt}: summary file ${summaryFile} not produced`)
-    } catch (err) {
-      const stderr = String(err.stderr ?? "").trim().split("\n").slice(-5).join("\n")
-      console.warn(`batch ${index} attempt ${attempt}: kilo run failed: ${stderr || err.message}`)
     }
   }
-  console.warn(`::warning::edit batch ${index} failed after ${ATTEMPTS} attempts; ${batch.length} PRs skipped`)
+
+  console.warn(`::warning::edit batch ${index} failed after up to ${ATTEMPTS} attempts; ${batch.length} PRs pending`)
+  for (const d of batch) pendingCauses.set(d.url, lastCause)
   return false
 }
 
@@ -88,12 +133,34 @@ for (let i = 0; i < ordered.length; i += BATCH_SIZE) {
 }
 console.log(`editing docs for ${ordered.length} PRs in ${batches.length} batches of up to ${BATCH_SIZE}`)
 
+const budgetDeadline = deadline(EDIT_BUDGET_MINUTES)
+let deferredFrom = -1
+
 for (let i = 0; i < batches.length; i++) {
-  editBatch(batches[i], i)
+  const left = remainingMs(budgetDeadline)
+  if (left < BATCH_TIMEOUT_MS) {
+    deferredFrom = i
+    const deferredPrs = batches.slice(i).reduce((n, b) => n + b.length, 0)
+    console.warn(
+      `stopping edit pass before batch ${i}: remaining budget (${Math.ceil(left / 1000)}s) cannot fit a ${EDIT_BATCH_TIMEOUT_MINUTES}m batch; deferring ${deferredPrs} PRs`,
+    )
+    const cause = `edit budget exhausted before batch ${i} (${Math.ceil(left / 1000)}s left)`
+    for (let j = i; j < batches.length; j++) {
+      for (const d of batches[j]) pendingCauses.set(d.url, cause)
+    }
+    break
+  }
+  editBatch(batches[i], i, budgetDeadline)
+}
+
+if (deferredFrom >= 0) {
+  console.warn(
+    `edit pass deferred ${batches.slice(deferredFrom).reduce((n, b) => n + b.length, 0)} PRs due to wall-clock budget`,
+  )
 }
 
 // Merge batch summaries. Coverage: every worthy PR gets an entry so the PR
-// body accounts for it; failed batches show up as skipped.
+// body accounts for it; failed/deferred batches show up as pending (not skipped).
 const merged = []
 const seen = new Set()
 for (let i = 0; i < batches.length; i++) {
@@ -108,15 +175,24 @@ for (let i = 0; i < batches.length; i++) {
     const url = String(e?.url ?? "")
     if (!url.startsWith("http") || seen.has(url)) continue
     seen.add(url)
-    merged.push({ pr: Number(e.pr) || 0, url, action: String(e.action ?? "skipped"), reason: String(e.reason ?? "") })
+    merged.push({
+      pr: Number(e.pr) || 0,
+      url,
+      action: String(e.action ?? "skipped"),
+      reason: String(e.reason ?? ""),
+    })
   }
 }
 for (const d of ordered) {
   if (seen.has(d.url)) continue
-  merged.push({ pr: d.number, url: d.url, action: "skipped", reason: "edit pass failed or timed out for this PR" })
+  const cause = pendingCauses.get(d.url) || "edit pass failed or timed out for this PR"
+  merged.push({ pr: d.number, url: d.url, action: "pending", reason: cause })
 }
 
 // upsert-pr.mjs consumes the merged summary from the repo root; the file is
 // removed there before committing so it never lands in the docs PR.
 fs.writeFileSync(SUMMARY_FILE, JSON.stringify(merged, null, 2))
-console.log(`edit pass complete: ${merged.filter((e) => e.action !== "skipped").length} changed, ${merged.filter((e) => e.action === "skipped").length} skipped`)
+const changed = merged.filter((e) => e.action !== "skipped" && e.action !== "pending").length
+const skipped = merged.filter((e) => e.action === "skipped").length
+const pending = merged.filter((e) => e.action === "pending").length
+console.log(`edit pass complete: ${changed} changed, ${skipped} skipped, ${pending} pending`)

.github/docs-sync/lib.mjs

@@ -6,6 +6,7 @@
  * gh CLI.
  */
 
+import { spawnSync } from "node:child_process"
 import fs from "node:fs"
 
 const API = "https://api.github.com"
@@ -111,3 +112,97 @@ export function appendSummary(markdown) {
   const summary = process.env.GITHUB_STEP_SUMMARY
   if (summary) fs.appendFileSync(summary, markdown + "\n")
 }
+
+/**
+ * Absolute deadline timestamp (ms since epoch) for a wall-clock budget.
+ * Used by triage/edit to stop before the job timeout rather than silently
+ * truncating.
+ */
+export function deadline(minutes) {
+  return Date.now() + Number(minutes) * 60 * 1000
+}
+
+/** Remaining milliseconds until a deadline; never negative. */
+export function remainingMs(deadlineMs) {
+  return Math.max(0, Number(deadlineMs) - Date.now())
+}
+
+/**
+ * Backoff schedule between kilo-run attempts. Production waits 60s then 300s
+ * (observed outage lasted ~11 min; batch 8 recovered on attempt 2). When
+ * DOCS_SYNC_BACKOFF_MS is set it replaces EVERY wait (`0` disables waiting);
+ * the workflow never sets it — only selftests do.
+ */
+export function backoffMsForAttempt(attempt) {
+  // attempt is 1-based; wait happens after attempt N before attempt N+1.
+  const override = process.env.DOCS_SYNC_BACKOFF_MS
+  if (override !== undefined && override !== "") {
+    const n = Number(override)
+    return Number.isFinite(n) && n >= 0 ? n : 0
+  }
+  // After attempt 1 → 60s; after attempt 2 → 300s; nothing after the last.
+  if (attempt === 1) return 60_000
+  if (attempt === 2) return 300_000
+  return 0
+}
+
+/**
+ * Blocking sleep used between kilo-run retries. Prefer this over async sleep
+ * so edit/triage stay synchronous around spawnSync.
+ */
+export function sleepSync(ms) {
+  const n = Number(ms)
+  if (!Number.isFinite(n) || n <= 0) return
+  const end = Date.now() + n
+  // Atomics.wait is the portable Node sync sleep (no busy loop).
+  const sab = new SharedArrayBuffer(4)
+  const view = new Int32Array(sab)
+  while (Date.now() < end) {
+    const left = end - Date.now()
+    if (left <= 0) break
+    Atomics.wait(view, 0, 0, Math.min(left, 2_147_483_647))
+  }
+}
+
+const STDERR_TAIL_LINES = 20
+const STDERR_TAIL_CHARS = 4_000
+
+function tailText(text, { lines = STDERR_TAIL_LINES, chars = STDERR_TAIL_CHARS } = {}) {
+  const s = String(text ?? "").trim()
+  if (!s) return ""
+  const lastLines = s.split("\n").slice(-lines).join("\n")
+  return lastLines.length > chars ? lastLines.slice(-chars) : lastLines
+}
+
+/**
+ * Run `kilo` via spawnSync so stderr is always recoverable — including when
+ * the child exits 0 after writing a diagnostic (execFileSync cannot return
+ * piped stderr on exit 0; that path lost every diagnostic on run 30122603016).
+ *
+ * streamStdout:true → inherit fd 1 (edit live log); false → capture stdout
+ * (triage parses it). stderr is always buffered.
+ */
+export function runKilo({ args, timeoutMs, streamStdout = false, label = "kilo" }) {
+  const result = spawnSync("kilo", args, {
+    encoding: "utf8",
+    maxBuffer: 32 * 1024 * 1024,
+    timeout: timeoutMs,
+    stdio: ["ignore", streamStdout ? "inherit" : "pipe", "pipe"],
+  })
+
+  const timedOut = Boolean(result.error && result.error.code === "ETIMEDOUT")
+  const exitCode =
+    typeof result.status === "number" ? result.status : timedOut ? null : result.status === null ? null : result.status
+  const stderrTail = tailText(result.stderr)
+  const stdout = streamStdout ? "" : String(result.stdout ?? "")
+  // ok is "process finished without OS-level failure". Callers still treat a
+  // missing summary / unparseable output as failure even when ok is true —
+  // exit 0 is not success for the docs-sync bot.
+  const ok = !result.error && result.status === 0
+
+  if (result.error && !timedOut) {
+    console.warn(`${label}: spawn error: ${result.error.message}`)
+  }
+
+  return { ok, stdout, stderrTail, exitCode, timedOut }
+}