From 869b9b38a644ca1610e7c0fa56b6c2a500fdf727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Mon, 27 Jul 2026 22:30:23 +0200 Subject: [PATCH 1/4] fix(ci): configure git identity before the docs-sync merge and classify merge failures --- .github/docs-sync/prepare-branch.mjs | 110 +++++++++++++++++++-------- .github/docs-sync/upsert-pr.mjs | 4 +- .github/docs-sync/watermark.mjs | 84 +++++++++++++------- .github/workflows/docs-sync.yml | 20 +++++ 4 files changed, 157 insertions(+), 61 deletions(-) diff --git a/.github/docs-sync/prepare-branch.mjs b/.github/docs-sync/prepare-branch.mjs index 2b57932c442..2710832ccf3 100644 --- a/.github/docs-sync/prepare-branch.mjs +++ b/.github/docs-sync/prepare-branch.mjs @@ -6,56 +6,100 @@ * origin/main (preserves any human commits on the branch) * - otherwise -> fresh branch from origin/main (bot force-pushes later) * - * Outputs: branch, mode (update|fresh), pr_number (empty when fresh). + * Outputs: branch, mode (update|fresh|conflict), pr_number (empty when fresh). */ import { execFileSync } from "node:child_process" +import { pathToFileURL } from "node:url" import { api, appendOutput, repo, searchIssues } from "./lib.mjs" export const DEFAULT_BRANCH = "docs/auto-sync" -const git = (args) => execFileSync("git", args, { stdio: ["ignore", "pipe", "inherit"] }).toString().trim() +const defaultGit = (args) => execFileSync("git", args, { stdio: ["ignore", "pipe", "inherit"] }).toString().trim() -const prs = await searchIssues(`repo:${repo()} is:pr is:open label:auto-docs sort:created-desc`, { maxPages: 1 }) - -let mode = "fresh" -let prNumber = "" -let branch = DEFAULT_BRANCH - -if (prs.length > 0) { - const pr = await api(`/repos/${repo()}/pulls/${prs[0].number}`) - branch = pr.head?.ref ?? DEFAULT_BRANCH - prNumber = String(pr.number) - git(["fetch", "origin", "main", branch]) - git(["checkout", branch]) +/** + * Merge origin/main into the current branch. On a genuine conflict, abort the + * merge, switch to a dated fallback branch from origin/main, and return + * mode=conflict so human commits on the rolling branch stay untouched. Any + * other merge failure (missing identity, corrupt ref, fetch issues) is + * rethrown so the job fails loudly. + */ +export function mergeOrFallback({ branch, git = defaultGit }) { try { git(["merge", "origin/main", "--no-edit"]) - mode = "update" - } catch { + return { branch, mode: "update" } + } catch (err) { + // Conflict ⇔ unmerged index entries (or MERGE_HEAD still present). + // Identity failures and similar abort before a merge is started, so + // merge --abort would itself fail — those must rethrow. + let unmerged = "" + try { + unmerged = git(["ls-files", "--unmerged"]) + } catch { + // ls-files itself failing is not a conflict signal + } + let mergeInProgress = false + try { + git(["rev-parse", "-q", "--verify", "MERGE_HEAD"]) + mergeInProgress = true + } catch { + mergeInProgress = false + } + const isConflict = unmerged.length > 0 || mergeInProgress + if (!isConflict) throw err + console.warn(`merge of origin/main into ${branch} conflicted.`) - console.warn("Leaving the conflicted branch untouched so human commits are preserved; continuing on a fresh dated branch.") + console.warn( + "Leaving the conflicted branch untouched so human commits are preserved; continuing on a fresh dated branch.", + ) git(["merge", "--abort"]) - branch = `${DEFAULT_BRANCH}-${new Date().toISOString().slice(0, 10)}` + const fallback = `${DEFAULT_BRANCH}-${new Date().toISOString().slice(0, 10)}` + try { + git(["fetch", "origin", `+refs/heads/${fallback}:refs/remotes/origin/${fallback}`]) + } catch { + console.log(`dated branch ${fallback} does not exist on origin yet; will create it on push`) + } + git(["checkout", "-B", fallback, "origin/main"]) + return { branch: fallback, mode: "conflict" } + } +} + +async function main() { + const git = defaultGit + const prs = await searchIssues(`repo:${repo()} is:pr is:open label:auto-docs sort:created-desc`, { maxPages: 1 }) + + let mode = "fresh" + let prNumber = "" + let branch = DEFAULT_BRANCH + + if (prs.length > 0) { + const pr = await api(`/repos/${repo()}/pulls/${prs[0].number}`) + branch = pr.head?.ref ?? DEFAULT_BRANCH + prNumber = String(pr.number) + git(["fetch", "origin", "main", branch]) + git(["checkout", branch]) + ;({ branch, mode } = mergeOrFallback({ branch, git })) + } else { + // Keep the remote-tracking ref current so the later --force-with-lease + // push (stale branch left over from a merged/closed PR) is safe. try { git(["fetch", "origin", `+refs/heads/${branch}:refs/remotes/origin/${branch}`]) } catch { - console.log(`dated branch ${branch} does not exist on origin yet; will create it on push`) + console.log(`branch ${branch} does not exist on origin yet; will create it on push`) } git(["checkout", "-B", branch, "origin/main"]) - mode = "conflict" - } -} else { - // Keep the remote-tracking ref current so the later --force-with-lease - // push (stale branch left over from a merged/closed PR) is safe. - try { - git(["fetch", "origin", `+refs/heads/${branch}:refs/remotes/origin/${branch}`]) - } catch { - console.log(`branch ${branch} does not exist on origin yet; will create it on push`) } - git(["checkout", "-B", branch, "origin/main"]) + + appendOutput("branch", branch) + appendOutput("mode", mode) + appendOutput("pr_number", prNumber) + console.log(`branch ${branch} ready (mode=${mode}, pr=${prNumber || "none"})`) } -appendOutput("branch", branch) -appendOutput("mode", mode) -appendOutput("pr_number", prNumber) -console.log(`branch ${branch} ready (mode=${mode}, pr=${prNumber || "none"})`) +const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href +if (isMain) { + main().catch((err) => { + console.error(err) + process.exit(1) + }) +} diff --git a/.github/docs-sync/upsert-pr.mjs b/.github/docs-sync/upsert-pr.mjs index e39551f307c..529112fea18 100644 --- a/.github/docs-sync/upsert-pr.mjs +++ b/.github/docs-sync/upsert-pr.mjs @@ -121,8 +121,8 @@ async function main() { return } - git(["config", "user.name", "github-actions[bot]"]) - git(["config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com"]) + // Git identity is configured once in docs-sync.yml (Configure git identity) + // before any commit-creating step, including prepare-branch's merge. git(["add", DOCS_PATH]) git(["commit", "-m", `docs: sync with merged PRs (${date})`]) diff --git a/.github/docs-sync/watermark.mjs b/.github/docs-sync/watermark.mjs index cc4c9690168..32986fef67f 100644 --- a/.github/docs-sync/watermark.mjs +++ b/.github/docs-sync/watermark.mjs @@ -7,9 +7,10 @@ * * Priority: workflow_dispatch input `since` > latest open bot PR marker > * last merged bot PR marker > 72h ago. Hard cap: never look back more than - * 14 days. + * 14 days — unless the human explicitly requested a window via INPUT_SINCE. */ +import { pathToFileURL } from "node:url" import { appendOutput, appendSummary, repo, searchIssues } from "./lib.mjs" const FALLBACK_HOURS = 72 @@ -42,34 +43,65 @@ async function findWatermark() { return null } -const now = new Date() -let since - -const input = (process.env.INPUT_SINCE ?? "").trim() -if (input) { - since = new Date(input) - if (Number.isNaN(since.getTime())) { - throw new Error(`Invalid INPUT_SINCE: ${input}`) +/** + * Apply the 14-day lookback cap. When `explicit` is true (dispatch override), + * the cap is skipped so a human-requested recovery window is not silently + * shortened. Returns `{ since, clamped }`. + */ +export function applyCap(since, now, { explicit = false } = {}) { + if (explicit) { + console.log(`14-day cap skipped: INPUT_SINCE was set explicitly (${since.toISOString()})`) + return { since, clamped: false } } - console.log(`watermark from dispatch input: ${since.toISOString()}`) -} else { - since = - (await findWatermark()) ?? new Date(now.getTime() - FALLBACK_HOURS * 3600 * 1000) + const cap = new Date(now.getTime() - CAP_DAYS * 24 * 3600 * 1000) + if (since < cap) { + const from = since.toISOString() + const to = cap.toISOString() + console.warn( + `::warning::docs-sync watermark clamped from ${from} to ${to} (${CAP_DAYS}-day cap). ` + + `Anything still uncovered before ${to} is abandoned and needs a human.`, + ) + return { since: cap, clamped: true } + } + return { since, clamped: false } } -// A forged, edited, or malformed marker in the future would silently match -// nothing in the merged:>= search; clamp it loudly. -if (since > now) { - console.warn(`watermark ${since.toISOString()} is in the future, clamping to now`) - since = now -} +async function main() { + const now = new Date() + let since + let explicit = false + + const input = (process.env.INPUT_SINCE ?? "").trim() + if (input) { + since = new Date(input) + if (Number.isNaN(since.getTime())) { + throw new Error(`Invalid INPUT_SINCE: ${input}`) + } + explicit = true + console.log(`watermark from dispatch input: ${since.toISOString()}`) + } else { + since = (await findWatermark()) ?? new Date(now.getTime() - FALLBACK_HOURS * 3600 * 1000) + } + + // A forged, edited, or malformed marker in the future would silently match + // nothing in the merged:>= search; clamp it loudly. + if (since > now) { + console.warn(`watermark ${since.toISOString()} is in the future, clamping to now`) + since = now + } + + ;({ since } = applyCap(since, now, { explicit })) -const cap = new Date(now.getTime() - CAP_DAYS * 24 * 3600 * 1000) -if (since < cap) { - console.log(`watermark ${since.toISOString()} older than ${CAP_DAYS}d cap, clamping`) - since = cap + appendOutput("since", since.toISOString()) + appendOutput("now", now.toISOString()) + appendOutput("since_override", explicit ? "true" : "false") + appendSummary(`### docs-sync watermark\n\n- since: \`${since.toISOString()}\`\n- now: \`${now.toISOString()}\`\n`) } -appendOutput("since", since.toISOString()) -appendOutput("now", now.toISOString()) -appendSummary(`### docs-sync watermark\n\n- since: \`${since.toISOString()}\`\n- now: \`${now.toISOString()}\`\n`) +const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href +if (isMain) { + main().catch((err) => { + console.error(err) + process.exit(1) + }) +} diff --git a/.github/workflows/docs-sync.yml b/.github/workflows/docs-sync.yml index ffc803c17c3..3f60f996cb2 100644 --- a/.github/workflows/docs-sync.yml +++ b/.github/workflows/docs-sync.yml @@ -40,6 +40,7 @@ jobs: sync: if: github.repository == 'Kilo-Org/kilocode' runs-on: blacksmith-4vcpu-ubuntu-2404 + # Budget: 3 setup/collect + 35 triage + 50 edit + 2 verify + 10 fix + 2 upsert = 102 min, 18-minute reserve. timeout-minutes: 120 env: # Both are required: without KILO_ORG_ID the gateway bills the key @@ -52,6 +53,11 @@ jobs: with: fetch-depth: 0 # prepare-branch merges main into the rolling branch + - name: Configure git identity + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + - name: Setup Node uses: actions/setup-node@v6 with: @@ -79,6 +85,8 @@ jobs: - name: Triage merged PRs (LLM, chunked) id: triage if: steps.collect.outputs.count != '0' + env: + SINCE_OVERRIDE: ${{ steps.wm.outputs.since_override }} run: node .github/docs-sync/triage.mjs - name: Filter docs-worthy PRs @@ -104,8 +112,18 @@ jobs: GH_TOKEN: ${{ github.token }} run: node .github/docs-sync/prepare-branch.mjs + # After prepare-branch checks out the rolling branch and merges main, the + # worktree holds main's scripts. Restore the dispatched ref's copies so a + # branch-dispatch AC9 run actually exercises the fixed code. git restore + # (not checkout) leaves them unstaged so upsert-pr's bare commit won't + # include them in the docs PR. + - name: Restore docs-sync scripts from the dispatched ref + if: (steps.worthy.outputs.count || '0') != '0' && inputs.dry_run != true + run: git restore --source=${{ github.sha }} -- .github/docs-sync + - name: Update docs (Kilo CLI, batched) if: (steps.worthy.outputs.count || '0') != '0' && inputs.dry_run != true + continue-on-error: true run: node .github/docs-sync/edit.mjs - name: Verify docs build and tests @@ -122,6 +140,7 @@ jobs: id: fix if: steps.verify.outcome == 'failure' continue-on-error: true + timeout-minutes: 10 env: NEXT_PUBLIC_POSTHOG_KEY: ${{ secrets.POSTHOG_API_KEY }} run: | @@ -150,6 +169,7 @@ jobs: GH_TOKEN: ${{ github.token }} PROCESSED_THROUGH: ${{ steps.wm.outputs.now }} SINCE: ${{ steps.wm.outputs.since }} + SINCE_OVERRIDE: ${{ steps.wm.outputs.since_override }} BRANCH: ${{ steps.prep.outputs.branch }} PREP_MODE: ${{ steps.prep.outputs.mode }} PR_NUMBER: ${{ steps.prep.outputs.pr_number }} From d58af35740c271f624d263a188c9feda053db450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Mon, 27 Jul 2026 22:58:25 +0200 Subject: [PATCH 2/4] fix(ci): hold the docs-sync watermark back until every PR has an outcome --- .github/docs-sync/edit.mjs | 138 +++++++++++++---- .github/docs-sync/lib.mjs | 95 ++++++++++++ .github/docs-sync/triage.mjs | 162 ++++++++++++++------ .github/docs-sync/upsert-pr.mjs | 256 ++++++++++++++++++++++++++++++-- 4 files changed, 565 insertions(+), 86 deletions(-) diff --git a/.github/docs-sync/edit.mjs b/.github/docs-sync/edit.mjs index 7da66bca642..777c3a2929c 100644 --- a/.github/docs-sync/edit.mjs +++ b/.github/docs-sync/edit.mjs @@ -6,20 +6,22 @@ * Batching bounds each `kilo run` context (a replay window can yield dozens * of docs-worthy PRs with large diffs). Each batch gets its own CLI session * and writes its own summary file; results are merged into - * docs-sync-out/edit-summary.json. A batch that fails is skipped with a - * warning — its PRs show up in the rolling PR body as skipped, so nothing - * fails silently. + * docs-sync-out/edit-summary.json. A batch that fails or is deferred by the + * wall-clock budget is recorded as action "pending" so the watermark holds + * back and the next run re-collects those PRs. * * Env: EDIT_MODEL (provider/model), KILO_API_KEY + KILO_ORG_ID (set by workflow; read natively by the kilo provider). + * Budgets: EDIT_BUDGET_MINUTES (default 50), EDIT_BATCH_TIMEOUT_MINUTES (default 15). + * Test hook: DOCS_SYNC_BACKOFF_MS replaces every retry wait when set. */ -import { execFileSync } from "node:child_process" import fs from "node:fs" import path from "node:path" import { fileURLToPath } from "node:url" +import { backoffMsForAttempt, deadline, remainingMs, runKilo, sleepSync } from "./lib.mjs" const BATCH_SIZE = 5 -const ATTEMPTS = 2 +const ATTEMPTS = 3 const OUT_DIR = "docs-sync-out" export const SUMMARY_FILE = ".docs-sync-summary.json" @@ -28,6 +30,10 @@ const basePrompt = fs.readFileSync(path.join(HERE, "edit-prompt.md"), "utf8") const model = process.env.EDIT_MODEL if (!model) throw new Error("EDIT_MODEL is required") +const EDIT_BUDGET_MINUTES = Number(process.env.EDIT_BUDGET_MINUTES) || 50 +const EDIT_BATCH_TIMEOUT_MINUTES = Number(process.env.EDIT_BATCH_TIMEOUT_MINUTES) || 15 +const BATCH_TIMEOUT_MS = EDIT_BATCH_TIMEOUT_MINUTES * 60 * 1000 + const worthy = JSON.parse(fs.readFileSync(`${OUT_DIR}/worthy.json`, "utf8")) const triage = JSON.parse(fs.readFileSync(`${OUT_DIR}/triage.json`, "utf8")) const priority = new Map(triage.map((e) => [e.url, e])) @@ -36,7 +42,18 @@ const ordered = [...worthy].sort((a, b) => { return (rank[priority.get(a.url)?.priority] ?? 1) - (rank[priority.get(b.url)?.priority] ?? 1) }) -function editBatch(batch, index) { +/** @type {Map} url → pending cause for failed/deferred batches */ +const pendingCauses = new Map() + +function formatCause(result) { + const bits = [] + if (result.timedOut) bits.push("timed out") + if (result.exitCode !== null && result.exitCode !== undefined) bits.push(`exit ${result.exitCode}`) + if (result.stderrTail) bits.push(result.stderrTail.replaceAll("\n", " ").slice(0, 200)) + return bits.join("; ") || "no diagnostic" +} + +function editBatch(batch, index, budgetDeadline) { const batchFile = `${OUT_DIR}/edit-batch-${index}.json` const triageFile = `${OUT_DIR}/edit-batch-triage-${index}.json` const summaryFile = `${OUT_DIR}/edit-summary-${index}.json` @@ -54,31 +71,59 @@ function editBatch(batch, index) { Batch specifics for this run: the PRs to handle are in the attached ${batchFile} (full details) and ${triageFile} (triage verdicts). Handle ONLY the PRs in these batch files. When finished, write your per-PR results in the summary JSON format described above to the file \`${summaryFile}\` (path relative to the repository root).` + let lastCause = "edit pass failed" for (let attempt = 1; attempt <= ATTEMPTS; attempt++) { - try { - // Message positional first: --file is multi-value and would otherwise - // consume a trailing message as a file path ("File not found"). - execFileSync( - "kilo", - ["run", prompt, "-m", model, "--variant", "high", "--dir", process.cwd(), "-f", batchFile, "-f", triageFile], - // stdout streams live to the Actions log; stderr is piped so failure - // warnings can include the tail of the actual CLI error. - { encoding: "utf8", maxBuffer: 32 * 1024 * 1024, timeout: 25 * 60 * 1000, stdio: ["ignore", "inherit", "pipe"] }, + const left = remainingMs(budgetDeadline) + if (left < BATCH_TIMEOUT_MS) { + lastCause = `edit budget exhausted before batch ${index} attempt ${attempt} (${Math.ceil(left / 1000)}s left, need ${EDIT_BATCH_TIMEOUT_MINUTES}m)` + console.warn( + `batch ${index}: stopping retries — remaining budget cannot fit another ${EDIT_BATCH_TIMEOUT_MINUTES}m attempt`, ) - if (fs.existsSync(summaryFile)) return true - // Tolerate the agent dropping the docs-sync-out/ prefix. - const alt = path.basename(summaryFile) - if (fs.existsSync(alt)) { - fs.renameSync(alt, summaryFile) - return true + break + } + + const result = runKilo({ + args: ["run", prompt, "-m", model, "--variant", "high", "--dir", process.cwd(), "-f", batchFile, "-f", triageFile], + timeoutMs: Math.min(BATCH_TIMEOUT_MS, left), + streamStdout: true, + label: `edit batch ${index} attempt ${attempt}`, + }) + + if (fs.existsSync(summaryFile)) return true + // Tolerate the agent dropping the docs-sync-out/ prefix. + const alt = path.basename(summaryFile) + if (fs.existsSync(alt)) { + fs.renameSync(alt, summaryFile) + return true + } + + // Exit 0 is not success: missing summary is a failure logged WITH the + // captured stderrTail and exit code on every attempt. + const cause = formatCause(result) + lastCause = `edit batch ${index}: ${cause}` + console.warn( + `batch ${index} attempt ${attempt}: summary file ${summaryFile} not produced` + + ` (exit ${result.exitCode}${result.timedOut ? ", timed out" : ""})` + + (result.stderrTail ? `\nstderr tail:\n${result.stderrTail}` : "\nstderr tail: (empty)"), + ) + + if (attempt < ATTEMPTS) { + const wait = backoffMsForAttempt(attempt) + // Skip the wait when the remaining budget cannot fit another attempt. + const afterWait = remainingMs(budgetDeadline) - wait + if (wait > 0 && afterWait >= BATCH_TIMEOUT_MS) { + console.warn(`batch ${index}: backing off ${wait / 1000}s before attempt ${attempt + 1}`) + sleepSync(wait) + } else if (wait > 0) { + console.warn( + `batch ${index}: skipping backoff — remaining budget cannot fit attempt ${attempt + 1} after wait`, + ) } - console.warn(`batch ${index} attempt ${attempt}: summary file ${summaryFile} not produced`) - } catch (err) { - const stderr = String(err.stderr ?? "").trim().split("\n").slice(-5).join("\n") - console.warn(`batch ${index} attempt ${attempt}: kilo run failed: ${stderr || err.message}`) } } - console.warn(`::warning::edit batch ${index} failed after ${ATTEMPTS} attempts; ${batch.length} PRs skipped`) + + console.warn(`::warning::edit batch ${index} failed after up to ${ATTEMPTS} attempts; ${batch.length} PRs pending`) + for (const d of batch) pendingCauses.set(d.url, lastCause) return false } @@ -88,12 +133,34 @@ for (let i = 0; i < ordered.length; i += BATCH_SIZE) { } console.log(`editing docs for ${ordered.length} PRs in ${batches.length} batches of up to ${BATCH_SIZE}`) +const budgetDeadline = deadline(EDIT_BUDGET_MINUTES) +let deferredFrom = -1 + for (let i = 0; i < batches.length; i++) { - editBatch(batches[i], i) + const left = remainingMs(budgetDeadline) + if (left < BATCH_TIMEOUT_MS) { + deferredFrom = i + const deferredPrs = batches.slice(i).reduce((n, b) => n + b.length, 0) + console.warn( + `stopping edit pass before batch ${i}: remaining budget (${Math.ceil(left / 1000)}s) cannot fit a ${EDIT_BATCH_TIMEOUT_MINUTES}m batch; deferring ${deferredPrs} PRs`, + ) + const cause = `edit budget exhausted before batch ${i} (${Math.ceil(left / 1000)}s left)` + for (let j = i; j < batches.length; j++) { + for (const d of batches[j]) pendingCauses.set(d.url, cause) + } + break + } + editBatch(batches[i], i, budgetDeadline) +} + +if (deferredFrom >= 0) { + console.warn( + `edit pass deferred ${batches.slice(deferredFrom).reduce((n, b) => n + b.length, 0)} PRs due to wall-clock budget`, + ) } // Merge batch summaries. Coverage: every worthy PR gets an entry so the PR -// body accounts for it; failed batches show up as skipped. +// body accounts for it; failed/deferred batches show up as pending (not skipped). const merged = [] const seen = new Set() for (let i = 0; i < batches.length; i++) { @@ -108,15 +175,24 @@ for (let i = 0; i < batches.length; i++) { const url = String(e?.url ?? "") if (!url.startsWith("http") || seen.has(url)) continue seen.add(url) - merged.push({ pr: Number(e.pr) || 0, url, action: String(e.action ?? "skipped"), reason: String(e.reason ?? "") }) + merged.push({ + pr: Number(e.pr) || 0, + url, + action: String(e.action ?? "skipped"), + reason: String(e.reason ?? ""), + }) } } for (const d of ordered) { if (seen.has(d.url)) continue - merged.push({ pr: d.number, url: d.url, action: "skipped", reason: "edit pass failed or timed out for this PR" }) + const cause = pendingCauses.get(d.url) || "edit pass failed or timed out for this PR" + merged.push({ pr: d.number, url: d.url, action: "pending", reason: cause }) } // upsert-pr.mjs consumes the merged summary from the repo root; the file is // removed there before committing so it never lands in the docs PR. fs.writeFileSync(SUMMARY_FILE, JSON.stringify(merged, null, 2)) -console.log(`edit pass complete: ${merged.filter((e) => e.action !== "skipped").length} changed, ${merged.filter((e) => e.action === "skipped").length} skipped`) +const changed = merged.filter((e) => e.action !== "skipped" && e.action !== "pending").length +const skipped = merged.filter((e) => e.action === "skipped").length +const pending = merged.filter((e) => e.action === "pending").length +console.log(`edit pass complete: ${changed} changed, ${skipped} skipped, ${pending} pending`) diff --git a/.github/docs-sync/lib.mjs b/.github/docs-sync/lib.mjs index 56dfb7b9226..67c13354d88 100644 --- a/.github/docs-sync/lib.mjs +++ b/.github/docs-sync/lib.mjs @@ -6,6 +6,7 @@ * gh CLI. */ +import { spawnSync } from "node:child_process" import fs from "node:fs" const API = "https://api.github.com" @@ -111,3 +112,97 @@ export function appendSummary(markdown) { const summary = process.env.GITHUB_STEP_SUMMARY if (summary) fs.appendFileSync(summary, markdown + "\n") } + +/** + * Absolute deadline timestamp (ms since epoch) for a wall-clock budget. + * Used by triage/edit to stop before the job timeout rather than silently + * truncating. + */ +export function deadline(minutes) { + return Date.now() + Number(minutes) * 60 * 1000 +} + +/** Remaining milliseconds until a deadline; never negative. */ +export function remainingMs(deadlineMs) { + return Math.max(0, Number(deadlineMs) - Date.now()) +} + +/** + * Backoff schedule between kilo-run attempts. Production waits 60s then 300s + * (observed outage lasted ~11 min; batch 8 recovered on attempt 2). When + * DOCS_SYNC_BACKOFF_MS is set it replaces EVERY wait (`0` disables waiting); + * the workflow never sets it — only selftests do. + */ +export function backoffMsForAttempt(attempt) { + // attempt is 1-based; wait happens after attempt N before attempt N+1. + const override = process.env.DOCS_SYNC_BACKOFF_MS + if (override !== undefined && override !== "") { + const n = Number(override) + return Number.isFinite(n) && n >= 0 ? n : 0 + } + // After attempt 1 → 60s; after attempt 2 → 300s; nothing after the last. + if (attempt === 1) return 60_000 + if (attempt === 2) return 300_000 + return 0 +} + +/** + * Blocking sleep used between kilo-run retries. Prefer this over async sleep + * so edit/triage stay synchronous around spawnSync. + */ +export function sleepSync(ms) { + const n = Number(ms) + if (!Number.isFinite(n) || n <= 0) return + const end = Date.now() + n + // Atomics.wait is the portable Node sync sleep (no busy loop). + const sab = new SharedArrayBuffer(4) + const view = new Int32Array(sab) + while (Date.now() < end) { + const left = end - Date.now() + if (left <= 0) break + Atomics.wait(view, 0, 0, Math.min(left, 2_147_483_647)) + } +} + +const STDERR_TAIL_LINES = 20 +const STDERR_TAIL_CHARS = 4_000 + +function tailText(text, { lines = STDERR_TAIL_LINES, chars = STDERR_TAIL_CHARS } = {}) { + const s = String(text ?? "").trim() + if (!s) return "" + const lastLines = s.split("\n").slice(-lines).join("\n") + return lastLines.length > chars ? lastLines.slice(-chars) : lastLines +} + +/** + * Run `kilo` via spawnSync so stderr is always recoverable — including when + * the child exits 0 after writing a diagnostic (execFileSync cannot return + * piped stderr on exit 0; that path lost every diagnostic on run 30122603016). + * + * streamStdout:true → inherit fd 1 (edit live log); false → capture stdout + * (triage parses it). stderr is always buffered. + */ +export function runKilo({ args, timeoutMs, streamStdout = false, label = "kilo" }) { + const result = spawnSync("kilo", args, { + encoding: "utf8", + maxBuffer: 32 * 1024 * 1024, + timeout: timeoutMs, + stdio: ["ignore", streamStdout ? "inherit" : "pipe", "pipe"], + }) + + const timedOut = Boolean(result.error && result.error.code === "ETIMEDOUT") + const exitCode = + typeof result.status === "number" ? result.status : timedOut ? null : result.status === null ? null : result.status + const stderrTail = tailText(result.stderr) + const stdout = streamStdout ? "" : String(result.stdout ?? "") + // ok is "process finished without OS-level failure". Callers still treat a + // missing summary / unparseable output as failure even when ok is true — + // exit 0 is not success for the docs-sync bot. + const ok = !result.error && result.status === 0 + + if (result.error && !timedOut) { + console.warn(`${label}: spawn error: ${result.error.message}`) + } + + return { ok, stdout, stderrTail, exitCode, timedOut } +} diff --git a/.github/docs-sync/triage.mjs b/.github/docs-sync/triage.mjs index a3a56831166..eb075980b51 100644 --- a/.github/docs-sync/triage.mjs +++ b/.github/docs-sync/triage.mjs @@ -6,52 +6,82 @@ * A daily window holds ~30-50 PRs; a replay can hold several hundred. A * single triage call over that volume truncates its JSON output, so the * digest is split into chunks of CHUNK_SIZE and each chunk is triaged with - * its own `kilo run` call. A chunk that fails twice is degraded to - * "unclassified" entries (docs_worthy=false) instead of failing the run — - * the PR body then shows those PRs as skipped, visible to reviewers. + * its own `kilo run` call. A chunk that fails, is only partially classified, + * or is deferred by the wall-clock budget is marked pending:true (still + * docs_worthy:false so filter-worthy excludes it) so the watermark holds + * back and the next run re-collects those PRs. * * Env: TRIAGE_MODEL (provider/model), KILO_API_KEY + KILO_ORG_ID (gateway auth, set by * the workflow; the kilo provider reads them natively). Reads the prompt from triage-prompt.md next to this script. + * Budget: TRIAGE_BUDGET_MINUTES (default 35). Test hook: DOCS_SYNC_BACKOFF_MS. */ -import { execFileSync } from "node:child_process" import fs from "node:fs" import path from "node:path" import { fileURLToPath } from "node:url" import { parseTriageEntries } from "./extract-json.mjs" +import { appendSummary, backoffMsForAttempt, deadline, remainingMs, runKilo, sleepSync } from "./lib.mjs" const CHUNK_SIZE = 25 -const ATTEMPTS = 2 +const ATTEMPTS = 3 const OUT_DIR = "docs-sync-out" +const CHUNK_TIMEOUT_MS = 10 * 60 * 1000 const HERE = path.dirname(fileURLToPath(import.meta.url)) const prompt = fs.readFileSync(path.join(HERE, "triage-prompt.md"), "utf8") const model = process.env.TRIAGE_MODEL if (!model) throw new Error("TRIAGE_MODEL is required") +const TRIAGE_BUDGET_MINUTES = Number(process.env.TRIAGE_BUDGET_MINUTES) || 35 + const digest = JSON.parse(fs.readFileSync(`${OUT_DIR}/digest.json`, "utf8")) -function triageChunk(chunk, index) { +function formatCause(result) { + const bits = [] + if (result.timedOut) bits.push("timed out") + if (result.exitCode !== null && result.exitCode !== undefined) bits.push(`exit ${result.exitCode}`) + if (result.stderrTail) bits.push(result.stderrTail.replaceAll("\n", " ").slice(0, 200)) + return bits.join("; ") || "no diagnostic" +} + +function pendingEntry(d, reason) { + return { + pr: d.number, + url: d.url, + docs_worthy: false, + pending: true, + reason, + target_sections: [], + priority: "medium", + } +} + +function triageChunk(chunk, index, budgetDeadline) { const chunkFile = `${OUT_DIR}/triage-chunk-${index}.json` fs.writeFileSync(chunkFile, JSON.stringify(chunk, null, 2)) + let lastCause = "triage failed to classify this PR" for (let attempt = 1; attempt <= ATTEMPTS; attempt++) { - let raw - try { - // Message positional first: --file is multi-value and would otherwise - // consume a trailing message as a file path ("File not found"). - raw = execFileSync( - "kilo", - ["run", prompt, "-m", model, "--dir", process.cwd(), "-f", chunkFile], - { encoding: "utf8", maxBuffer: 32 * 1024 * 1024, timeout: 10 * 60 * 1000, stdio: ["ignore", "pipe", "pipe"] }, + const left = remainingMs(budgetDeadline) + if (left < CHUNK_TIMEOUT_MS) { + lastCause = `triage budget exhausted before chunk ${index} attempt ${attempt}` + console.warn( + `chunk ${index}: stopping retries — remaining budget cannot fit another ${CHUNK_TIMEOUT_MS / 60000}m attempt`, ) - } catch (err) { - const stderr = String(err.stderr ?? "").trim().split("\n").slice(-5).join("\n") - console.warn(`chunk ${index} attempt ${attempt}: kilo run failed: ${stderr || err.message}`) - continue + break } - fs.writeFileSync(`${OUT_DIR}/triage-raw-${index}.txt`, raw) - const entries = parseTriageEntries(raw) + + const result = runKilo({ + args: ["run", prompt, "-m", model, "--dir", process.cwd(), "-f", chunkFile], + timeoutMs: Math.min(CHUNK_TIMEOUT_MS, left), + streamStdout: false, + label: `triage chunk ${index} attempt ${attempt}`, + }) + + const raw = result.stdout + if (raw) fs.writeFileSync(`${OUT_DIR}/triage-raw-${index}.txt`, raw) + + const entries = raw ? parseTriageEntries(raw) : null if (entries) { // An entry for a PR outside this chunk must not win the shared dedupe // against the chunk that actually owns it — drop foreign entries. @@ -62,18 +92,35 @@ function triageChunk(chunk, index) { } if (owned.length > 0) return owned } - console.warn(`chunk ${index} attempt ${attempt}: no valid JSON in output`) + + // Exit 0 is not success: unparseable output is a failure logged WITH + // the captured stderrTail and exit code on every attempt. + const cause = formatCause(result) + lastCause = `triage chunk ${index}: ${cause}` + console.warn( + `chunk ${index} attempt ${attempt}: no valid JSON in output` + + ` (exit ${result.exitCode}${result.timedOut ? ", timed out" : ""})` + + (result.stderrTail ? `\nstderr tail:\n${result.stderrTail}` : "\nstderr tail: (empty)"), + ) + + if (attempt < ATTEMPTS) { + const wait = backoffMsForAttempt(attempt) + const afterWait = remainingMs(budgetDeadline) - wait + if (wait > 0 && afterWait >= CHUNK_TIMEOUT_MS) { + console.warn(`chunk ${index}: backing off ${wait / 1000}s before attempt ${attempt + 1}`) + sleepSync(wait) + } else if (wait > 0) { + console.warn( + `chunk ${index}: skipping backoff — remaining budget cannot fit attempt ${attempt + 1} after wait`, + ) + } + } } - console.warn(`::warning::chunk ${index} failed triage after ${ATTEMPTS} attempts; marking ${chunk.length} PRs unclassified`) - return chunk.map((d) => ({ - pr: d.number, - url: d.url, - docs_worthy: false, - reason: "triage failed to classify this PR", - target_sections: [], - priority: "medium", - })) + console.warn( + `::warning::chunk ${index} failed triage after up to ${ATTEMPTS} attempts; marking ${chunk.length} PRs pending`, + ) + return chunk.map((d) => pendingEntry(d, lastCause.includes("triage failed") ? lastCause : `triage failed to classify this PR (${lastCause})`)) } const chunks = [] @@ -82,30 +129,61 @@ for (let i = 0; i < digest.length; i += CHUNK_SIZE) { } console.log(`triaging ${digest.length} PRs in ${chunks.length} chunks of up to ${CHUNK_SIZE}`) +const budgetDeadline = deadline(TRIAGE_BUDGET_MINUTES) const merged = [] const seen = new Set() + for (let i = 0; i < chunks.length; i++) { - for (const e of triageChunk(chunks[i], i)) { + const left = remainingMs(budgetDeadline) + if (left < CHUNK_TIMEOUT_MS) { + const deferredPrs = chunks.slice(i).reduce((n, c) => n + c.length, 0) + console.warn( + `stopping triage before chunk ${i}: remaining budget (${Math.ceil(left / 1000)}s) cannot fit a ${CHUNK_TIMEOUT_MS / 60000}m chunk; deferring ${deferredPrs} PRs`, + ) + const cause = `triage budget exhausted before chunk ${i} (${Math.ceil(left / 1000)}s left)` + for (let j = i; j < chunks.length; j++) { + for (const d of chunks[j]) { + if (seen.has(d.url)) continue + seen.add(d.url) + merged.push(pendingEntry(d, cause)) + } + } + break + } + + for (const e of triageChunk(chunks[i], i, budgetDeadline)) { if (seen.has(e.url)) continue seen.add(e.url) merged.push(e) } } -// Coverage: every digest PR gets a triage entry so the PR body's skipped -// table is complete. Unclassified defaults to not-docs-worthy (conservative). +// Coverage: every digest PR gets a triage entry. Partial-chunk backfill and +// any other missing URL are pending:true — not a genuine "not worthy" verdict. for (const d of digest) { if (seen.has(d.url)) continue - merged.push({ - pr: d.number, - url: d.url, - docs_worthy: false, - reason: "not classified by triage", - target_sections: [], - priority: "medium", - }) + merged.push(pendingEntry(d, "not classified by triage")) } fs.writeFileSync(`${OUT_DIR}/triage.json`, JSON.stringify(merged, null, 2)) const worthy = merged.filter((e) => e.docs_worthy).length -console.log(`triage complete: ${merged.length} entries, ${worthy} docs-worthy`) +const pending = merged.filter((e) => e.pending === true) +console.log(`triage complete: ${merged.length} entries, ${worthy} docs-worthy, ${pending.length} pending`) + +// Upsert is gated off when worthy == 0, so triage emits its own Step Summary +// listing every PR it marked pending:true and why. +if (pending.length > 0) { + const lines = pending.map((e) => `- [${e.url}] ${e.reason}`) + appendSummary( + `### docs-sync: triage pending (will retry)\n\n${pending.length} PR(s) were not classified and will be re-collected on the next run:\n\n${lines.join("\n")}`, + ) +} + +// Replay warning (S2j): warn IFF since-override AND something pending AND +// docs-worthy count is 0 (Upsert is gated off, so noDiffReport never runs). +const sinceOverride = process.env.SINCE_OVERRIDE === "true" +if (sinceOverride && pending.length > 0 && worthy === 0) { + console.warn( + "::warning::docs-sync since-override replay left uncovered PRs and wrote no PR body (worthy=0); re-run the override — the watermark was not held back in the body", + ) +} diff --git a/.github/docs-sync/upsert-pr.mjs b/.github/docs-sync/upsert-pr.mjs index 529112fea18..1ef25b75964 100644 --- a/.github/docs-sync/upsert-pr.mjs +++ b/.github/docs-sync/upsert-pr.mjs @@ -8,6 +8,14 @@ * diff exceeds the file cap or verification failed. The PR body carries * marker-delimited sections so later runs can append rows, plus a * machine-readable processed-through watermark. + * + * Watermark invariant: processed-through never moves past a PR that has no + * terminal outcome. Terminal := action !== "pending" (a deliberate agent + * "skipped" IS terminal). Uncovered PRs hold the marker at earliest + * merged_at − 1 ms so collect's merged:>=since re-collects them next run. + * Three review rounds found four independent defects in a queue-based + * alternative (unreachable gate, empty-PR creation, cap-overflow loss, + * draft-state corruption); a held-back watermark has none of those modes. */ import { execFileSync } from "node:child_process" @@ -17,6 +25,7 @@ import { pathToFileURL } from "node:url" const BRANCH = process.env.BRANCH || "docs/auto-sync" const FILE_CAP = 15 const ROW_CAP = 150 +const PENDING_DISPLAY_CAP = 60 const SUMMARY_FILE = ".docs-sync-summary.json" const DOCS_PATH = "packages/kilo-docs" @@ -42,6 +51,11 @@ function skippedRow(e) { return `| [${shortRef(e.url)}](${clean(e.url)}) | ${reason} |` } +function pendingRow(e) { + const reason = clean(e.reason ?? e.cause ?? "").replaceAll("|", "\\|").replaceAll("\n", " ") + return `| [${shortRef(e.url)}](${clean(e.url)}) | ${reason} |` +} + export function extractSectionRows(body, name) { const m = String(body ?? "").match( new RegExp(`([\\s\\S]*?)`), @@ -50,7 +64,14 @@ export function extractSectionRows(body, name) { return m[1] .split("\n") .map((l) => l.trim()) - .filter((l) => l.startsWith("|") && !l.startsWith("| ---") && !/^\|\s*Docs change/.test(l) && !/^\|\s*PR\s*\|/.test(l)) + .filter( + (l) => + l.startsWith("|") && + !l.startsWith("| ---") && + !/^\|\s*Docs change/.test(l) && + !/^\|\s*PR\s*\|/.test(l) && + !/^\|\s*Why\s*\|/.test(l), + ) } function section(name, header, rows) { @@ -58,7 +79,12 @@ function section(name, header, rows) { return `\n${body}\n` } -export function renderBody({ date, since, through, changesRows, skippedRows, verified, draftReasons, note }) { +export function renderBody({ date, since, through, changesRows, pendingRows, skippedRows, verified, draftReasons, note }) { + const pendingDisplay = + pendingRows.length > PENDING_DISPLAY_CAP + ? [...pendingRows.slice(0, PENDING_DISPLAY_CAP), `| +${pendingRows.length - PENDING_DISPLAY_CAP} more | |`] + : pendingRows + return `## Automated docs sync — ${date} This PR keeps kilo.ai/docs in sync with features merged to [Kilo-Org/cloud](https://github.com/Kilo-Org/cloud) and [Kilo-Org/kilocode](https://github.com/Kilo-Org/kilocode). Every change below links to the merged PR it documents. @@ -70,6 +96,10 @@ ${note ? `- ${note}\n` : ""}${draftReasons.length > 0 ? `- Draft because: ${draf ${section("changes", "| Docs change | Source |", changesRows)} +### Pending — will retry + +${section("pending", "| PR | Why |", pendingDisplay)} + ### Considered, no docs change needed ${section("skipped", "| PR | Reason |", skippedRows)} @@ -100,24 +130,210 @@ function readJson(path, fallback) { } } +/** + * Uncovered = (worthy URLs with no summary row) ∪ (summary action "pending") + * ∪ (triage entries with pending: true). A worthy PR is covered iff it has a + * summary row whose action !== "pending" and carries no triage pending flag. + */ +export function computeUncovered({ worthy, summary, triage }) { + const worthyList = Array.isArray(worthy) ? worthy : [] + const summaryList = Array.isArray(summary) ? summary : [] + const triageList = Array.isArray(triage) ? triage : [] + + const summaryByUrl = new Map() + for (const e of summaryList) { + if (e?.url) summaryByUrl.set(e.url, e) + } + + const triagePendingByUrl = new Map() + for (const e of triageList) { + if (e?.url && e.pending === true) triagePendingByUrl.set(e.url, e) + } + + /** @type {Map} */ + const out = new Map() + + for (const w of worthyList) { + const url = w?.url + if (!url) continue + const row = summaryByUrl.get(url) + if (!row) { + out.set(url, { + url, + pr: w.number ?? w.pr, + reason: "no edit summary row (edit pass did not cover this PR)", + }) + continue + } + if (row.action === "pending") { + out.set(url, { + url, + pr: row.pr ?? w.number ?? w.pr, + reason: row.reason || "edit pass pending", + }) + } + } + + // Summary pending rows for URLs not in worthy (defensive). + for (const row of summaryList) { + if (row?.action === "pending" && row.url && !out.has(row.url)) { + out.set(row.url, { + url: row.url, + pr: row.pr, + reason: row.reason || "edit pass pending", + }) + } + } + + for (const [url, e] of triagePendingByUrl) { + if (out.has(url)) continue + out.set(url, { + url, + pr: e.pr, + reason: e.reason || "triage pending", + }) + } + + return [...out.values()] +} + +/** + * processed-through = now when uncovered is empty; otherwise earliest + * merged_at among uncovered PRs minus 1 ms (from digest-full.json). + * When uncovered is non-empty but no merged_at resolves, hold at + * `fallback` (the run's window start / SINCE): every uncovered PR was + * collected via merged:>=since, so holding there re-collects all of them. + * Never advance past unresolved uncovered PRs (Defect-B permanent-loss). + */ +export function computeProcessedThrough({ uncovered, digest, now, fallback }) { + const nowIso = typeof now === "string" ? now : new Date(now).toISOString() + if (!uncovered || uncovered.length === 0) return nowIso + + const digestList = Array.isArray(digest) ? digest : [] + const byUrl = new Map(digestList.filter((d) => d?.url).map((d) => [d.url, d])) + + let earliest = null + for (const u of uncovered) { + const d = byUrl.get(u.url) + const mergedAt = d?.merged_at + if (!mergedAt) continue + const t = Date.parse(mergedAt) + if (!Number.isFinite(t)) continue + if (earliest === null || t < earliest) earliest = t + } + + if (earliest === null) { + // digest-full missing/corrupt while uncovered is non-empty: hold at + // window start so collect's merged:>=since re-collects every PR. + // Never use now−1ms — that strands uncovered PRs permanently. + const fallbackMs = fallback == null ? NaN : Date.parse(fallback) + if (!Number.isFinite(fallbackMs)) { + throw new Error( + `docs-sync: cannot resolve merged_at for ${uncovered.length} uncovered PR(s) and fallback/SINCE is missing or unparseable; refusing to advance processed-through`, + ) + } + const fallbackIso = new Date(fallbackMs).toISOString() + console.warn( + `::warning::docs-sync: merge times for ${uncovered.length} uncovered PR(s) could not be resolved; holding watermark at window start ${fallbackIso}`, + ) + return fallbackIso + } + + return new Date(earliest - 1).toISOString() +} + +/** + * Route summary + triage into the three body sections. + * changesRows = action neither skipped nor pending + * pendingRows = uncovered from computeUncovered + * skippedRows = action === "skipped" ∪ triage docs_worthy false && !pending + */ +export function routeRows({ summary, triage, uncovered }) { + const summaryList = Array.isArray(summary) ? summary : [] + const triageList = Array.isArray(triage) ? triage : [] + const uncoveredList = Array.isArray(uncovered) ? uncovered : [] + + const changesEntries = summaryList.filter((e) => e.action !== "skipped" && e.action !== "pending") + const skippedEntries = [ + ...triageList.filter((e) => e.docs_worthy === false && e.pending !== true), + ...summaryList.filter((e) => e.action === "skipped"), + ] + + return { + changesRows: changesEntries.map(changeRow), + pendingRows: uncoveredList.map(pendingRow), + skippedRows: skippedEntries.map(skippedRow), + } +} + +/** + * Drop pre-existing Considered rows whose reason contains any of the three + * legacy failure literals (substring match — live rows carry longer strings). + * Genuine no-doc-needed rows are untouched. + */ +export function dropLegacySkipped(rows) { + const list = Array.isArray(rows) ? rows : [] + const needles = ["edit pass failed or timed out", "triage failed to classify", "not classified by triage"] + return list.filter((row) => { + const s = String(row ?? "") + return !needles.some((n) => s.includes(n)) + }) +} + +/** + * No-diff early-return report. Returns summary markdown and an optional + * replay warning. Warns IFF sinceOverride && uncovered non-empty (no commit + * happened — that is the caller's situation). + */ +export function noDiffReport({ uncovered, sinceOverride }) { + const list = Array.isArray(uncovered) ? uncovered : [] + const lines = + list.length === 0 + ? ["The agent found nothing worth documenting in this window."] + : [ + `No packages/kilo-docs diff was produced, but ${list.length} PR(s) remain uncovered and will be re-collected on the next scheduled run:`, + "", + ...list.map((u) => `- [${u.url}] ${u.reason || "uncovered"}`), + ] + + const summary = `### docs-sync: no docs changes\n\n${lines.join("\n")}` + + let warning = null + if (sinceOverride && list.length > 0) { + warning = + "docs-sync since-override replay left uncovered PRs and wrote no PR body (no docs commit); re-run the override — the watermark was not held back in the body" + } + + return { summary, warning } +} + async function main() { const { api, appendOutput, appendSummary, repo } = await import("./lib.mjs") - const through = process.env.PROCESSED_THROUGH ?? new Date().toISOString() + const now = process.env.PROCESSED_THROUGH ?? new Date().toISOString() const since = process.env.SINCE ?? "unknown" + const sinceOverride = process.env.SINCE_OVERRIDE === "true" const mode = ["update", "conflict"].includes(process.env.PREP_MODE) ? process.env.PREP_MODE : "fresh" const existingPr = process.env.PR_NUMBER || "" const verified = process.env.VERIFIED === "true" - const date = through.slice(0, 10) + const date = now.slice(0, 10) // The agent's run summary is consumed here and never committed. const agentSummary = readJson(SUMMARY_FILE, []) fs.rmSync(SUMMARY_FILE, { force: true }) const triage = readJson("docs-sync-out/triage.json", []) + const worthy = readJson("docs-sync-out/worthy.json", []) + const digest = readJson("docs-sync-out/digest-full.json", []) + + // Order matters: compute uncovered BEFORE the no-diff early return so + // noDiffReport can name every held-back PR. + const uncovered = computeUncovered({ worthy, summary: agentSummary, triage }) if (git(["status", "--porcelain", "--", DOCS_PATH]) === "") { console.log("no packages/kilo-docs changes produced; nothing to commit") - appendSummary("### docs-sync: no docs changes\n\nThe agent found nothing worth documenting in this window.") + const { summary, warning } = noDiffReport({ uncovered, sinceOverride }) + appendSummary(summary) + if (warning) console.warn(`::warning::${warning}`) return } @@ -126,6 +342,10 @@ async function main() { git(["add", DOCS_PATH]) git(["commit", "-m", `docs: sync with merged PRs (${date})`]) + // Watermark: now when fully covered; else earliest uncovered merged_at − 1ms. + // Pass SINCE as fallback so missing digest-full cannot strand uncovered PRs. + const through = computeProcessedThrough({ uncovered, digest, now, fallback: since }) + // The draft cap bounds the cumulative PR diff, not just this run's commit. const changedFiles = git(["diff", "--name-only", "origin/main...HEAD", "--", DOCS_PATH]) .split("\n") @@ -151,25 +371,33 @@ async function main() { git(mode === "update" ? ["push", "origin", `HEAD:${BRANCH}`] : ["push", "--force-with-lease", "origin", `HEAD:${BRANCH}`]) - const changesNew = agentSummary.filter((e) => e.action !== "skipped").map(changeRow) - const skippedNew = [ - ...triage.filter((e) => e.docs_worthy === false), - ...agentSummary.filter((e) => e.action === "skipped"), - ].map(skippedRow) + const { changesRows: changesNew, pendingRows: pendingNew, skippedRows: skippedNew } = routeRows({ + summary: agentSummary, + triage, + uncovered, + }) let oldChanges = [] let oldSkipped = [] + let oldPending = [] if (mode === "update" && existingPr) { const pr = await api(`/repos/${repo()}/pulls/${existingPr}`) oldChanges = extractSectionRows(pr.body, "changes") - oldSkipped = extractSectionRows(pr.body, "skipped") + oldSkipped = dropLegacySkipped(extractSectionRows(pr.body, "skipped")) + oldPending = extractSectionRows(pr.body, "pending") } + // Pending is replaced each run (informational only); do not merge legacy + // pending rows — uncovered is recomputed fresh. oldPending is read only so + // extractSectionRows stays exercised; discarded deliberately. + void oldPending + const body = renderBody({ date, since, through, changesRows: mergeRows(oldChanges, changesNew), + pendingRows: pendingNew, skippedRows: mergeRows(oldSkipped, skippedNew), verified, draftReasons, @@ -228,8 +456,10 @@ async function main() { } appendOutput("pr_url", prUrl) - appendSummary(`### docs-sync PR\n\n- ${prUrl}\n- changed files: ${changedFiles.length}\n- draft: ${draft}\n`) - console.log(`PR ${prNumber}: ${prUrl} (draft=${draft}, files=${changedFiles.length})`) + appendSummary( + `### docs-sync PR\n\n- ${prUrl}\n- changed files: ${changedFiles.length}\n- draft: ${draft}\n- uncovered: ${uncovered.length}\n- processed-through: ${through}\n`, + ) + console.log(`PR ${prNumber}: ${prUrl} (draft=${draft}, files=${changedFiles.length}, uncovered=${uncovered.length}, through=${through})`) } const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href From 3316b3b55ab579f026ee5b9a6c8f8aeb1ccd5195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Mon, 27 Jul 2026 23:10:13 +0200 Subject: [PATCH 3/4] test(ci): self-check for the docs-sync failure paths --- .github/docs-sync/selftest.mjs | 964 ++++++++++++++++++++++++++++++++ .github/workflows/docs-sync.yml | 37 +- 2 files changed, 997 insertions(+), 4 deletions(-) create mode 100644 .github/docs-sync/selftest.mjs diff --git a/.github/docs-sync/selftest.mjs b/.github/docs-sync/selftest.mjs new file mode 100644 index 00000000000..fe2cb1faec3 --- /dev/null +++ b/.github/docs-sync/selftest.mjs @@ -0,0 +1,964 @@ +// kilocode_change - new file + +/** + * Offline self-check for the docs-sync failure paths (S4). + * Plain node:assert, no network, no LLM, no new dependency. + * Run: node .github/docs-sync/selftest.mjs + */ + +import assert from "node:assert/strict" +import { execFileSync, spawnSync } from "node:child_process" +import fs from "node:fs" +import os from "node:os" +import path from "node:path" +import { fileURLToPath } from "node:url" + +import { mergeOrFallback, DEFAULT_BRANCH } from "./prepare-branch.mjs" +import { applyCap } from "./watermark.mjs" +import { + computeUncovered, + computeProcessedThrough, + routeRows, + dropLegacySkipped, + noDiffReport, + renderBody, + extractSectionRows, +} from "./upsert-pr.mjs" + +const HERE = path.dirname(fileURLToPath(import.meta.url)) +const EDIT_SCRIPT = path.join(HERE, "edit.mjs") +const TRIAGE_SCRIPT = path.join(HERE, "triage.mjs") +const COLLECT_SCRIPT = path.join(HERE, "collect.mjs") + +const temps = [] + +function mktemp(prefix) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)) + temps.push(dir) + return dir +} + +function cleanup() { + for (const dir of temps.splice(0)) { + try { + fs.rmSync(dir, { recursive: true, force: true }) + } catch { + // best-effort + } + } +} + +function writeExecutable(filePath, body) { + fs.writeFileSync(filePath, body, { mode: 0o755 }) +} + +function makeStubKiloDir({ mode, callLog, stderrText = "event stream disconnected" }) { + const dir = mktemp("docs-sync-kilo-") + const kiloPath = path.join(dir, "kilo") + // mode: "stderr-exit0" | "record" | "partial-triage" | "mixed-triage" + const script = `#!/usr/bin/env node +const fs = require("node:fs"); +const path = require("node:path"); +const mode = ${JSON.stringify(mode)}; +const callLog = ${JSON.stringify(callLog ?? "")}; +const stderrText = ${JSON.stringify(stderrText)}; +if (callLog) { + fs.appendFileSync(callLog, JSON.stringify({ argv: process.argv.slice(2), cwd: process.cwd() }) + "\\n"); +} +if (mode === "stderr-exit0") { + process.stderr.write(stderrText + "\\n"); + process.exit(0); +} +if (mode === "record") { + process.stderr.write("recorded\\n"); + process.exit(0); +} +// Parse -f chunk/batch file from args for triage stubs +const args = process.argv.slice(2); +const fIdx = args.indexOf("-f"); +const fileArg = fIdx >= 0 ? args[fIdx + 1] : null; +let chunk = []; +if (fileArg && fs.existsSync(fileArg)) { + try { chunk = JSON.parse(fs.readFileSync(fileArg, "utf8")); } catch { chunk = []; } +} +if (mode === "partial-triage") { + // Classify only a proper subset (first URL) of the chunk. + const owned = chunk.slice(0, Math.max(0, chunk.length - 1)); + const entries = owned.map((d) => ({ + pr: d.number, + url: d.url, + docs_worthy: false, + reason: "genuine not worthy", + target_sections: [], + priority: "medium", + })); + if (entries.length === 0 && chunk.length > 0) { + // single-PR chunk: still leave one missing by emitting empty-ish foreign-only + process.stdout.write("[]\\n"); + } else { + process.stdout.write(JSON.stringify(entries) + "\\n"); + } + process.exit(0); +} +if (mode === "mixed-triage") { + // Half docs_worthy true, half fail (no output for second half — but we return + // only some entries so backfill marks the rest pending). Actually: return + // docs_worthy:true for first half of chunk URLs so worthy > 0. + const half = Math.ceil(chunk.length / 2); + const entries = chunk.slice(0, half).map((d) => ({ + pr: d.number, + url: d.url, + docs_worthy: true, + reason: "needs docs", + target_sections: ["overview"], + priority: "high", + })); + process.stdout.write(JSON.stringify(entries) + "\\n"); + process.exit(0); +} +process.stderr.write("unknown stub mode\\n"); +process.exit(1); +` + writeExecutable(kiloPath, script) + return dir +} + +function gitIn(cwd, args, env = {}) { + return execFileSync("git", args, { + cwd, + env: { ...process.env, ...env }, + stdio: ["ignore", "pipe", "pipe"], + encoding: "utf8", + }).toString().trim() +} + +function makeGitRunner(cwd, env = {}) { + return (args) => gitIn(cwd, args, env) +} + +function initRepoWithIdentity(dir) { + gitIn(dir, ["init", "-b", "main"]) + gitIn(dir, ["config", "user.name", "docs-sync-selftest"]) + gitIn(dir, ["config", "user.email", "docs-sync-selftest@example.com"]) + gitIn(dir, ["config", "commit.gpgsign", "false"]) +} + +// --------------------------------------------------------------------------- +// Case 1 — Defect A: mergeOrFallback +// --------------------------------------------------------------------------- +function case1_mergeOrFallback() { + console.log("case 1: Defect A (mergeOrFallback)") + + // 1a — identity configured + clean merge → mode=update + { + const dir = mktemp("docs-sync-merge-clean-") + initRepoWithIdentity(dir) + fs.writeFileSync(path.join(dir, "a.txt"), "base\n") + gitIn(dir, ["add", "a.txt"]) + gitIn(dir, ["commit", "-m", "base"]) + gitIn(dir, ["checkout", "-b", DEFAULT_BRANCH]) + fs.writeFileSync(path.join(dir, "b.txt"), "on branch\n") + gitIn(dir, ["add", "b.txt"]) + gitIn(dir, ["commit", "-m", "branch commit"]) + // Advance main without conflict + gitIn(dir, ["checkout", "main"]) + fs.writeFileSync(path.join(dir, "c.txt"), "on main\n") + gitIn(dir, ["add", "c.txt"]) + gitIn(dir, ["commit", "-m", "main advance"]) + gitIn(dir, ["update-ref", "refs/remotes/origin/main", "main"]) + gitIn(dir, ["checkout", DEFAULT_BRANCH]) + + const result = mergeOrFallback({ branch: DEFAULT_BRANCH, git: makeGitRunner(dir) }) + assert.equal(result.mode, "update") + assert.equal(result.branch, DEFAULT_BRANCH) + // merge brought c.txt in + assert.ok(fs.existsSync(path.join(dir, "c.txt"))) + } + + // 1b — genuine conflict → mode=conflict, abort succeeds, original branch untouched + { + const dir = mktemp("docs-sync-merge-conflict-") + initRepoWithIdentity(dir) + fs.writeFileSync(path.join(dir, "conflict.txt"), "base\n") + gitIn(dir, ["add", "conflict.txt"]) + gitIn(dir, ["commit", "-m", "base"]) + const baseSha = gitIn(dir, ["rev-parse", "HEAD"]) + + gitIn(dir, ["checkout", "-b", DEFAULT_BRANCH]) + fs.writeFileSync(path.join(dir, "conflict.txt"), "branch side\n") + gitIn(dir, ["add", "conflict.txt"]) + gitIn(dir, ["commit", "-m", "branch edit"]) + const branchShaBefore = gitIn(dir, ["rev-parse", "HEAD"]) + + gitIn(dir, ["checkout", "main"]) + fs.writeFileSync(path.join(dir, "conflict.txt"), "main side\n") + gitIn(dir, ["add", "conflict.txt"]) + gitIn(dir, ["commit", "-m", "main edit"]) + gitIn(dir, ["update-ref", "refs/remotes/origin/main", "main"]) + gitIn(dir, ["checkout", DEFAULT_BRANCH]) + + const result = mergeOrFallback({ branch: DEFAULT_BRANCH, git: makeGitRunner(dir) }) + assert.equal(result.mode, "conflict") + assert.ok(result.branch.startsWith(`${DEFAULT_BRANCH}-`)) + // Original rolling branch tip unchanged + const branchShaAfter = gitIn(dir, ["rev-parse", DEFAULT_BRANCH]) + assert.equal(branchShaAfter, branchShaBefore) + // No merge in progress + let mergeHead = true + try { + gitIn(dir, ["rev-parse", "-q", "--verify", "MERGE_HEAD"]) + } catch { + mergeHead = false + } + assert.equal(mergeHead, false) + void baseSha + } + + // 1c — identity-less / non-conflict merge failure → throws (does not fake conflict) + { + const dir = mktemp("docs-sync-merge-noid-") + initRepoWithIdentity(dir) + fs.writeFileSync(path.join(dir, "a.txt"), "base\n") + gitIn(dir, ["add", "a.txt"]) + gitIn(dir, ["commit", "-m", "base"]) + gitIn(dir, ["checkout", "-b", DEFAULT_BRANCH]) + fs.writeFileSync(path.join(dir, "b.txt"), "branch\n") + gitIn(dir, ["add", "b.txt"]) + gitIn(dir, ["commit", "-m", "branch"]) + gitIn(dir, ["checkout", "main"]) + fs.writeFileSync(path.join(dir, "c.txt"), "main\n") + gitIn(dir, ["add", "c.txt"]) + gitIn(dir, ["commit", "-m", "main"]) + gitIn(dir, ["update-ref", "refs/remotes/origin/main", "main"]) + gitIn(dir, ["checkout", DEFAULT_BRANCH]) + + // Strip identity so merge cannot create a commit + gitIn(dir, ["config", "--unset", "user.name"]) + gitIn(dir, ["config", "--unset", "user.email"]) + + const env = { + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_SYSTEM: "/dev/null", + GIT_CONFIG_NOSYSTEM: "1", + } + const git = (args) => + execFileSync("git", ["-c", "user.useConfigOnly=true", ...args], { + cwd: dir, + env: { ...process.env, ...env }, + stdio: ["ignore", "pipe", "pipe"], + encoding: "utf8", + }).toString().trim() + + assert.throws(() => mergeOrFallback({ branch: DEFAULT_BRANCH, git }), (err) => { + // Must throw the original merge error, not a merge --abort failure + const msg = String(err?.stderr ?? err?.message ?? err) + assert.ok(!/no merge to abort/i.test(msg), `should not reach merge --abort: ${msg}`) + return true + }) + } +} + +// --------------------------------------------------------------------------- +// Helpers to run edit.mjs / triage.mjs as child processes +// --------------------------------------------------------------------------- +function setupEditCwd(worthy, triage) { + const cwd = mktemp("docs-sync-edit-") + fs.mkdirSync(path.join(cwd, "docs-sync-out"), { recursive: true }) + fs.writeFileSync(path.join(cwd, "docs-sync-out", "worthy.json"), JSON.stringify(worthy, null, 2)) + fs.writeFileSync(path.join(cwd, "docs-sync-out", "triage.json"), JSON.stringify(triage, null, 2)) + return cwd +} + +function setupTriageCwd(digest) { + const cwd = mktemp("docs-sync-triage-") + fs.mkdirSync(path.join(cwd, "docs-sync-out"), { recursive: true }) + fs.writeFileSync(path.join(cwd, "docs-sync-out", "digest.json"), JSON.stringify(digest, null, 2)) + return cwd +} + +function runNodeScript(scriptPath, { cwd, env = {}, kiloDir }) { + const pathEnv = [kiloDir, process.env.PATH].filter(Boolean).join(path.delimiter) + const result = spawnSync(process.execPath, [scriptPath], { + cwd, + env: { + ...process.env, + ...env, + PATH: pathEnv, + DOCS_SYNC_BACKOFF_MS: env.DOCS_SYNC_BACKOFF_MS ?? "0", + }, + encoding: "utf8", + timeout: 60_000, + }) + return { + status: result.status, + stdout: String(result.stdout ?? ""), + stderr: String(result.stderr ?? ""), + output: `${result.stdout ?? ""}${result.stderr ?? ""}`, + error: result.error, + } +} + +function samplePr(n, { merged_at, repo = "Kilo-Org/cloud" } = {}) { + return { + repo, + number: n, + title: `feat: sample ${n}`, + url: `https://github.com/${repo}/pull/${n}`, + author: "dev", + merged_at: merged_at ?? "2026-07-20T12:00:00.000Z", + labels: [], + body: "body", + files: [], + files_total: 1, + patch_excerpt: "", + } +} + +// --------------------------------------------------------------------------- +// Case 2 — Defect B: edit.mjs with stub kilo (exit 0 + stderr) +// --------------------------------------------------------------------------- +function case2_defectB() { + console.log("case 2: Defect B (edit.mjs stderr-on-exit-0)") + + const prs = [1, 2, 3, 4, 5].map((n) => samplePr(n)) + const worthy = prs + const triage = prs.map((p) => ({ + pr: p.number, + url: p.url, + docs_worthy: true, + reason: "needs docs", + target_sections: ["overview"], + priority: "high", + })) + const cwd = setupEditCwd(worthy, triage) + const stderrText = "event stream disconnected DIAG-CASE2" + const kiloDir = makeStubKiloDir({ mode: "stderr-exit0", stderrText }) + + const started = Date.now() + const result = runNodeScript(EDIT_SCRIPT, { + cwd, + kiloDir, + env: { + EDIT_MODEL: "test/model", + DOCS_SYNC_BACKOFF_MS: "0", + // Enough budget for 3 attempts × tiny timeout + EDIT_BUDGET_MINUTES: "5", + EDIT_BATCH_TIMEOUT_MINUTES: "1", + }, + }) + const elapsed = Date.now() - started + + assert.equal(result.status, 0, `edit.mjs exit: ${result.output}`) + // Backoff collapsed — 3 attempts without 60s+300s waits + assert.ok(elapsed < 15_000, `backoff should collapse with DOCS_SYNC_BACKOFF_MS=0; elapsed=${elapsed}ms`) + + assert.match(result.output, /stderr tail:/) + assert.match(result.output, /DIAG-CASE2|event stream disconnected/) + assert.match(result.output, /attempt 1/) + assert.match(result.output, /attempt 2/) + // 3 attempts + assert.match(result.output, /attempt 3|failed after up to 3 attempts/) + + const summary = JSON.parse(fs.readFileSync(path.join(cwd, ".docs-sync-summary.json"), "utf8")) + assert.equal(summary.length, 5) + for (const e of summary) { + assert.equal(e.action, "pending", `expected pending, got ${JSON.stringify(e)}`) + } + + const uncovered = computeUncovered({ worthy, summary, triage }) + assert.equal(uncovered.length, 5) + for (const u of uncovered) { + assert.ok(u.reason, "uncovered reason present") + } +} + +// --------------------------------------------------------------------------- +// Case 3 — watermark invariant +// --------------------------------------------------------------------------- +function case3_watermark() { + console.log("case 3: watermark invariant") + + const now = "2026-07-27T12:00:00.000Z" + const nowMs = Date.parse(now) + + const prA = samplePr(10, { merged_at: "2026-07-20T10:00:00.000Z" }) + const prB = samplePr(11, { merged_at: "2026-07-22T15:30:00.000Z" }) + const prC = samplePr(12, { merged_at: "2026-07-25T08:00:00.000Z" }) + const digest = [prA, prB, prC] + + // all covered → processed-through === now + { + const worthy = [prA, prB] + const summary = [ + { pr: 10, url: prA.url, action: "updated packages/kilo-docs/pages/x.md", reason: "" }, + { pr: 11, url: prB.url, action: "skipped", reason: "already documented" }, + ] + const triage = [ + { pr: 10, url: prA.url, docs_worthy: true, pending: false, reason: "ok" }, + { pr: 11, url: prB.url, docs_worthy: true, pending: false, reason: "ok" }, + ] + const uncovered = computeUncovered({ worthy, summary, triage }) + assert.equal(uncovered.length, 0) + const through = computeProcessedThrough({ uncovered, digest, now }) + assert.equal(through, now) + } + + // one uncovered → merged_at − 1 ms, strictly < now + { + const worthy = [prA, prB] + const summary = [ + { pr: 10, url: prA.url, action: "updated x", reason: "" }, + { pr: 11, url: prB.url, action: "pending", reason: "edit batch 0: exit 0" }, + ] + const uncovered = computeUncovered({ worthy, summary, triage: [] }) + assert.equal(uncovered.length, 1) + assert.equal(uncovered[0].url, prB.url) + const through = computeProcessedThrough({ uncovered, digest, now }) + const expected = new Date(Date.parse(prB.merged_at) - 1).toISOString() + assert.equal(through, expected) + assert.ok(Date.parse(through) < nowMs) + } + + // several uncovered → earliest merge time wins + { + const worthy = [prA, prB, prC] + const summary = [ + { pr: 10, url: prA.url, action: "pending", reason: "fail" }, + { pr: 12, url: prC.url, action: "pending", reason: "fail" }, + ] + // prB missing from summary entirely + const uncovered = computeUncovered({ worthy, summary, triage: [] }) + assert.ok(uncovered.length >= 2) + const through = computeProcessedThrough({ uncovered, digest, now }) + // earliest among A, B, C that are uncovered — A is earliest + const times = uncovered + .map((u) => digest.find((d) => d.url === u.url)?.merged_at) + .filter(Boolean) + .map((t) => Date.parse(t)) + const earliest = Math.min(...times) + assert.equal(through, new Date(earliest - 1).toISOString()) + } + + // summary missing/truncated while worthy non-empty → every worthy URL held back + { + const worthy = [prA, prB] + const uncovered = computeUncovered({ worthy, summary: [], triage: [] }) + assert.equal(uncovered.length, 2) + const through = computeProcessedThrough({ uncovered, digest, now }) + assert.equal(through, new Date(Date.parse(prA.merged_at) - 1).toISOString()) + } + + // noDiffReport three arms + { + const uncovered = [{ url: prA.url, reason: "edit batch failed" }] + const arm1 = noDiffReport({ uncovered, sinceOverride: true }) + assert.ok(arm1.summary.includes(prA.url)) + assert.ok(arm1.warning, "override + uncovered → warning present") + + const arm2 = noDiffReport({ uncovered: [], sinceOverride: true }) + assert.equal(arm2.warning, null, "override + empty uncovered → warning absent") + + const arm3 = noDiffReport({ uncovered, sinceOverride: false }) + assert.equal(arm3.warning, null, "scheduled + uncovered → warning absent") + } + + // triage pending:true backfill rows land in uncovered (consumption) + { + const triage = [ + { + pr: 99, + url: "https://github.com/Kilo-Org/cloud/pull/99", + docs_worthy: false, + pending: true, + reason: "not classified by triage", + }, + ] + const uncovered = computeUncovered({ worthy: [], summary: [], triage }) + assert.equal(uncovered.length, 1) + assert.equal(uncovered[0].url, triage[0].url) + const through = computeProcessedThrough({ + uncovered, + digest: [{ url: triage[0].url, merged_at: "2026-07-21T00:00:00.000Z" }], + now, + }) + assert.equal(through, new Date(Date.parse("2026-07-21T00:00:00.000Z") - 1).toISOString()) + } + + // fallback field (post-plan repair) + { + const uncovered = [{ url: "https://github.com/Kilo-Org/cloud/pull/50", reason: "missing" }] + const fallback = "2026-07-17T00:00:00.000Z" + // unresolved merged_at + parseable fallback → hold at fallback, warn + const prevWarn = console.warn + const warnings = [] + console.warn = (...a) => warnings.push(a.join(" ")) + try { + const through = computeProcessedThrough({ uncovered, digest: [], now, fallback }) + assert.equal(through, new Date(fallback).toISOString()) + assert.ok(Date.parse(through) < nowMs) + assert.ok(warnings.some((w) => w.includes("::warning::"))) + } finally { + console.warn = prevWarn + } + + // unresolved + unparseable/missing fallback → throws + assert.throws(() => computeProcessedThrough({ uncovered, digest: [], now }), /fallback|SINCE|refusing/i) + assert.throws( + () => computeProcessedThrough({ uncovered, digest: [], now, fallback: "not-a-date" }), + /fallback|SINCE|refusing/i, + ) + + // resolved merged_at ignores fallback + const throughResolved = computeProcessedThrough({ + uncovered: [{ url: prA.url, reason: "x" }], + digest: [prA], + now, + fallback: "2020-01-01T00:00:00.000Z", + }) + assert.equal(throughResolved, new Date(Date.parse(prA.merged_at) - 1).toISOString()) + + // empty uncovered ignores fallback + const throughEmpty = computeProcessedThrough({ + uncovered: [], + digest: [], + now, + fallback: "2020-01-01T00:00:00.000Z", + }) + assert.equal(throughEmpty, now) + } +} + +// --------------------------------------------------------------------------- +// Case 4 — routing and round trip +// --------------------------------------------------------------------------- +function case4_routing() { + console.log("case 4: routing and round trip") + + const summary = [ + { pr: 1, url: "https://github.com/Kilo-Org/cloud/pull/1", action: "updated pages/a.md", reason: "" }, + { pr: 2, url: "https://github.com/Kilo-Org/cloud/pull/2", action: "skipped", reason: "already documented" }, + { pr: 3, url: "https://github.com/Kilo-Org/cloud/pull/3", action: "pending", reason: "edit batch 1: exit 0" }, + ] + const triage = [ + { + pr: 4, + url: "https://github.com/Kilo-Org/cloud/pull/4", + docs_worthy: false, + pending: false, + reason: "chore only", + }, + { + pr: 5, + url: "https://github.com/Kilo-Org/cloud/pull/5", + docs_worthy: false, + pending: true, + reason: "triage failed to classify this PR", + }, + ] + const worthy = [ + { number: 1, url: summary[0].url }, + { number: 2, url: summary[1].url }, + { number: 3, url: summary[2].url }, + ] + const uncovered = computeUncovered({ worthy, summary, triage }) + const { changesRows, pendingRows, skippedRows } = routeRows({ summary, triage, uncovered }) + + // pending appears in neither Changes nor Considered + const changesText = changesRows.join("\n") + const skippedText = skippedRows.join("\n") + assert.ok(changesText.includes("pull/1"), "success in Changes") + assert.ok(!changesText.includes("pull/3"), "pending must not be in Changes") + assert.ok(!changesText.includes("pull/5"), "triage-pending must not be in Changes") + assert.ok(skippedText.includes("pull/2"), "genuine skipped in Considered") + assert.ok(skippedText.includes("pull/4"), "genuine not-worthy in Considered") + assert.ok(!skippedText.includes("pull/3"), "pending must not be in Considered") + assert.ok(!skippedText.includes("pull/5"), "triage-pending must not be in Considered") + assert.ok(pendingRows.some((r) => r.includes("pull/3"))) + assert.ok(pendingRows.some((r) => r.includes("pull/5"))) + + // round-trip renderBody → extractSectionRows + const through = "2026-07-20T09:59:59.999Z" + const body = renderBody({ + date: "2026-07-27", + since: "2026-07-17T00:00:00.000Z", + through, + changesRows, + pendingRows, + skippedRows, + verified: true, + draftReasons: [], + note: "", + }) + assert.ok(body.includes(``)) + const extChanges = extractSectionRows(body, "changes") + const extPending = extractSectionRows(body, "pending") + const extSkipped = extractSectionRows(body, "skipped") + assert.deepEqual(extChanges, changesRows) + assert.deepEqual(extPending, pendingRows) + assert.deepEqual(extSkipped, skippedRows) + + // clean() prevents marker forgery in agent-generated row strings + { + const forgedRows = routeRows({ + summary: [ + { + pr: 9, + url: "https://github.com/Kilo-Org/cloud/pull/9", + action: "skipped", + reason: "x injection", + }, + ], + triage: [], + uncovered: [], + }) + assert.ok( + !forgedRows.skippedRows[0].includes(""), + "clean() must strip --> from reasons", + ) + const forgedBody = renderBody({ + date: "2026-07-27", + since: "s", + through: "t", + changesRows: [], + pendingRows: [], + skippedRows: forgedRows.skippedRows, + verified: true, + draftReasons: [], + note: "", + }) + // Exactly one real section end marker — the forged sequences were stripped + assert.equal((forgedBody.match(//g) || []).length, 1) + const extracted = extractSectionRows(forgedBody, "skipped") + assert.equal(extracted.length, 1) + assert.ok(extracted[0].includes("injection")) + } + + const legacyRows = [ + "| [Kilo-Org/cloud#1](https://github.com/Kilo-Org/cloud/pull/1) | edit pass failed or timed out for this PR |", + "| [Kilo-Org/cloud#2](https://github.com/Kilo-Org/cloud/pull/2) | triage failed to classify this PR |", + "| [Kilo-Org/cloud#3](https://github.com/Kilo-Org/cloud/pull/3) | not classified by triage |", + "| [Kilo-Org/cloud#4](https://github.com/Kilo-Org/cloud/pull/4) | already covered by existing docs |", + ] + const kept = dropLegacySkipped(legacyRows) + assert.equal(kept.length, 1) + assert.ok(kept[0].includes("pull/4")) + assert.ok(!kept.some((r) => r.includes("edit pass failed"))) + assert.ok(!kept.some((r) => r.includes("triage failed to classify"))) + assert.ok(!kept.some((r) => r.includes("not classified by triage"))) +} + +// --------------------------------------------------------------------------- +// Case 5 — re-collection window +// --------------------------------------------------------------------------- +function case5_recollection() { + console.log("case 5: re-collection closes the loop") + + const collectSrc = fs.readFileSync(COLLECT_SCRIPT, "utf8") + // Query template must use merged:>= + assert.ok( + /merged:>=\$\{since\.toISOString\(\)\}/.test(collectSrc) || /merged:>=/.test(collectSrc), + "collect.mjs must search merged:>=since", + ) + assert.match(collectSrc, /merged:>=/) + + const mergedAt = "2026-07-22T15:30:00.000Z" + const uncovered = [{ url: "https://github.com/Kilo-Org/cloud/pull/11", reason: "pending" }] + const digest = [{ url: uncovered[0].url, merged_at: mergedAt }] + const now = "2026-07-27T12:00:00.000Z" + const since = computeProcessedThrough({ uncovered, digest, now }) + // held-back since is strictly before the uncovered PR's merged_at + assert.ok(Date.parse(since) < Date.parse(mergedAt), `since ${since} must be < merged_at ${mergedAt}`) + // And the query window merged:>=since therefore includes that PR + assert.ok(Date.parse(mergedAt) >= Date.parse(since)) +} + +// --------------------------------------------------------------------------- +// Case 6 — budgets +// --------------------------------------------------------------------------- +function case6_budgets() { + console.log("case 6: budgets") + + // --- edit budget --- + { + // 12 PRs = 3 batches of 5; budget too small for even one batch unit + const prs = Array.from({ length: 12 }, (_, i) => samplePr(100 + i)) + const worthy = prs + const triage = prs.map((p) => ({ + pr: p.number, + url: p.url, + docs_worthy: true, + reason: "needs docs", + target_sections: [], + priority: "medium", + })) + const cwd = setupEditCwd(worthy, triage) + const callLog = path.join(cwd, "kilo-calls.log") + const kiloDir = makeStubKiloDir({ mode: "record", callLog }) + + // EDIT_BUDGET_MINUTES must be positive (0 falls through to default 50). + // BATCH_TIMEOUT default would be 15m; set both tiny so left < BATCH_TIMEOUT immediately. + const result = runNodeScript(EDIT_SCRIPT, { + cwd, + kiloDir, + env: { + EDIT_MODEL: "test/model", + DOCS_SYNC_BACKOFF_MS: "0", + EDIT_BUDGET_MINUTES: "0.0001", + EDIT_BATCH_TIMEOUT_MINUTES: "15", + }, + }) + assert.equal(result.status, 0, result.output) + assert.match(result.output, /deferring \d+ PRs/) + assert.match(result.output, /deferred \d+ PRs due to wall-clock budget/) + + const calls = fs.existsSync(callLog) ? fs.readFileSync(callLog, "utf8").trim() : "" + const callCount = calls ? calls.split("\n").filter(Boolean).length : 0 + assert.equal(callCount, 0, `kilo must not be invoked for deferred edit batches; got ${callCount}`) + + const summary = JSON.parse(fs.readFileSync(path.join(cwd, ".docs-sync-summary.json"), "utf8")) + assert.ok(summary.every((e) => e.action === "pending")) + const uncovered = computeUncovered({ worthy, summary, triage }) + assert.equal(uncovered.length, 12) + assert.ok(summary.every((e) => e.action !== "skipped")) + } + + // --- triage budget --- + { + // CHUNK_SIZE=25; 30 PRs = 2 chunks; budget too small for a 10m chunk + const digest = Array.from({ length: 30 }, (_, i) => samplePr(200 + i)) + const cwd = setupTriageCwd(digest) + const callLog = path.join(cwd, "kilo-calls.log") + const kiloDir = makeStubKiloDir({ mode: "record", callLog }) + const summaryFile = path.join(cwd, "step-summary.md") + fs.writeFileSync(summaryFile, "") + + const result = runNodeScript(TRIAGE_SCRIPT, { + cwd, + kiloDir, + env: { + TRIAGE_MODEL: "test/model", + DOCS_SYNC_BACKOFF_MS: "0", + TRIAGE_BUDGET_MINUTES: "0.0001", + GITHUB_STEP_SUMMARY: summaryFile, + }, + }) + assert.equal(result.status, 0, result.output) + assert.match(result.output, /deferring \d+ PRs/) + + const calls = fs.existsSync(callLog) ? fs.readFileSync(callLog, "utf8").trim() : "" + const callCount = calls ? calls.split("\n").filter(Boolean).length : 0 + assert.equal(callCount, 0, `kilo must not be invoked for deferred triage chunks; got ${callCount}`) + + const triage = JSON.parse(fs.readFileSync(path.join(cwd, "docs-sync-out", "triage.json"), "utf8")) + assert.equal(triage.length, 30) + assert.ok(triage.every((e) => e.pending === true)) + assert.ok(triage.every((e) => e.docs_worthy === false)) + const uncovered = computeUncovered({ worthy: [], summary: [], triage }) + assert.equal(uncovered.length, 30) + } +} + +// --------------------------------------------------------------------------- +// Case 7 — applyCap both arms +// --------------------------------------------------------------------------- +function case7_cap() { + console.log("case 7: applyCap") + + const now = new Date("2026-07-27T12:00:00.000Z") + const old = new Date("2026-06-01T00:00:00.000Z") + + const prevLog = console.log + const prevWarn = console.warn + const logs = [] + const warnings = [] + console.log = (...a) => logs.push(a.join(" ")) + console.warn = (...a) => warnings.push(a.join(" ")) + try { + // explicit:false + older than 14 days → clamped AND reported + const a = applyCap(old, now, { explicit: false }) + assert.equal(a.clamped, true) + assert.ok(a.since.getTime() > old.getTime()) + const cap = new Date(now.getTime() - 14 * 24 * 3600 * 1000) + assert.equal(a.since.toISOString(), cap.toISOString()) + assert.ok(warnings.some((w) => w.includes("::warning::") && w.includes("clamped"))) + + // explicit:true + older than 14 days → unchanged, skip reported + logs.length = 0 + warnings.length = 0 + const b = applyCap(old, now, { explicit: true }) + assert.equal(b.clamped, false) + assert.equal(b.since.toISOString(), old.toISOString()) + assert.ok(logs.some((l) => /cap skipped|INPUT_SINCE/i.test(l))) + } finally { + console.log = prevLog + console.warn = prevWarn + } +} + +// --------------------------------------------------------------------------- +// Case 8 — triage.mjs outputs +// --------------------------------------------------------------------------- +function case8_triage() { + console.log("case 8: triage pass outputs") + + // 8a Run A: SINCE_OVERRIDE=true + everything pending → warning present + { + const digest = [samplePr(301), samplePr(302), samplePr(303)] + const cwd = setupTriageCwd(digest) + const kiloDir = makeStubKiloDir({ mode: "stderr-exit0", stderrText: "stream end before idle" }) + const summaryFile = path.join(cwd, "step-summary.md") + fs.writeFileSync(summaryFile, "") + + const result = runNodeScript(TRIAGE_SCRIPT, { + cwd, + kiloDir, + env: { + TRIAGE_MODEL: "test/model", + DOCS_SYNC_BACKOFF_MS: "0", + TRIAGE_BUDGET_MINUTES: "30", + SINCE_OVERRIDE: "true", + GITHUB_STEP_SUMMARY: summaryFile, + }, + }) + assert.equal(result.status, 0, result.output) + const triage = JSON.parse(fs.readFileSync(path.join(cwd, "docs-sync-out", "triage.json"), "utf8")) + assert.equal(triage.length, 3) + assert.ok(triage.every((e) => e.pending === true)) + const summary = fs.readFileSync(summaryFile, "utf8") + assert.match(summary, /triage pending/) + for (const d of digest) { + assert.ok(summary.includes(d.url), `summary lists ${d.url}`) + } + assert.match(result.output, /::warning::.*since-override/) + } + + // 8a Run B: SINCE_OVERRIDE unset → warning absent + { + const digest = [samplePr(311), samplePr(312)] + const cwd = setupTriageCwd(digest) + const kiloDir = makeStubKiloDir({ mode: "stderr-exit0", stderrText: "stream end" }) + const summaryFile = path.join(cwd, "step-summary.md") + fs.writeFileSync(summaryFile, "") + + const result = runNodeScript(TRIAGE_SCRIPT, { + cwd, + kiloDir, + env: { + TRIAGE_MODEL: "test/model", + DOCS_SYNC_BACKOFF_MS: "0", + TRIAGE_BUDGET_MINUTES: "30", + GITHUB_STEP_SUMMARY: summaryFile, + }, + }) + assert.equal(result.status, 0, result.output) + const triage = JSON.parse(fs.readFileSync(path.join(cwd, "docs-sync-out", "triage.json"), "utf8")) + assert.ok(triage.every((e) => e.pending === true)) + assert.ok(fs.readFileSync(summaryFile, "utf8").includes("triage pending")) + assert.ok(!/::warning::.*since-override/.test(result.output), "override warning must be absent when unset") + } + + // 8a Run C: SINCE_OVERRIDE=true with MIXED stub (worthy > 0) → warning ABSENT + { + const digest = [samplePr(321), samplePr(322), samplePr(323), samplePr(324)] + const cwd = setupTriageCwd(digest) + const kiloDir = makeStubKiloDir({ mode: "mixed-triage" }) + const summaryFile = path.join(cwd, "step-summary.md") + fs.writeFileSync(summaryFile, "") + + const result = runNodeScript(TRIAGE_SCRIPT, { + cwd, + kiloDir, + env: { + TRIAGE_MODEL: "test/model", + DOCS_SYNC_BACKOFF_MS: "0", + TRIAGE_BUDGET_MINUTES: "30", + SINCE_OVERRIDE: "true", + GITHUB_STEP_SUMMARY: summaryFile, + }, + }) + assert.equal(result.status, 0, result.output) + const triage = JSON.parse(fs.readFileSync(path.join(cwd, "docs-sync-out", "triage.json"), "utf8")) + const worthy = triage.filter((e) => e.docs_worthy === true).length + const pending = triage.filter((e) => e.pending === true).length + assert.ok(worthy > 0, "mixed stub must produce worthy > 0") + assert.ok(pending > 0, "mixed stub must leave some pending") + assert.ok( + !/::warning::.*since-override/.test(result.output), + "override warning must be ABSENT when worthy > 0 (Upsert will run)", + ) + } + + // 8b — partial classification → missing URLs pending:true + computeUncovered + { + // One chunk of 4 PRs; stub classifies first 3 only + const digest = [samplePr(401), samplePr(402), samplePr(403), samplePr(404)] + const cwd = setupTriageCwd(digest) + const kiloDir = makeStubKiloDir({ mode: "partial-triage" }) + const summaryFile = path.join(cwd, "step-summary.md") + fs.writeFileSync(summaryFile, "") + + const result = runNodeScript(TRIAGE_SCRIPT, { + cwd, + kiloDir, + env: { + TRIAGE_MODEL: "test/model", + DOCS_SYNC_BACKOFF_MS: "0", + TRIAGE_BUDGET_MINUTES: "30", + GITHUB_STEP_SUMMARY: summaryFile, + }, + }) + assert.equal(result.status, 0, result.output) + const triage = JSON.parse(fs.readFileSync(path.join(cwd, "docs-sync-out", "triage.json"), "utf8")) + assert.equal(triage.length, 4) + const missing = triage.filter((e) => e.reason === "not classified by triage") + assert.ok(missing.length >= 1, "backfill must mark unclassified URLs") + assert.ok(missing.every((e) => e.pending === true)) + const uncovered = computeUncovered({ worthy: [], summary: [], triage }) + for (const m of missing) { + assert.ok( + uncovered.some((u) => u.url === m.url), + `${m.url} must appear in computeUncovered`, + ) + } + } +} + +// --------------------------------------------------------------------------- +// main +// --------------------------------------------------------------------------- +function main() { + const cases = [ + case1_mergeOrFallback, + case2_defectB, + case3_watermark, + case4_routing, + case5_recollection, + case6_budgets, + case7_cap, + case8_triage, + ] + let failed = 0 + for (const fn of cases) { + try { + fn() + console.log(` ok: ${fn.name}`) + } catch (err) { + failed++ + console.error(` FAIL: ${fn.name}`) + console.error(err) + } finally { + cleanup() + } + } + if (failed > 0) { + console.error(`\nselftest: ${failed} case(s) failed`) + process.exit(1) + } + console.log("\nselftest: all cases passed") +} + +main() diff --git a/.github/workflows/docs-sync.yml b/.github/workflows/docs-sync.yml index 3f60f996cb2..66a7799423d 100644 --- a/.github/workflows/docs-sync.yml +++ b/.github/workflows/docs-sync.yml @@ -5,9 +5,13 @@ name: docs-sync # triages them for docs relevance, runs Kilo CLI headless to update # packages/kilo-docs, and maintains one rolling PR for human review. # -# Security posture: scheduled/manual only, checks out main, never executes -# code from PR branches. State is derived from the bot's own PRs (watermark -# marker in the PR body), so missed or failed runs self-heal on the next run. +# Security posture: scheduled/manual runs check out the dispatched ref and may +# push/comment with write permissions and org secrets. PR runs (paths-limited to +# this workflow and .github/docs-sync/**) execute branch code only in a +# read-only, secretless `selftest` job that never pushes, comments, or calls an +# LLM. `pull_request` (not `pull_request_target`) keeps fork tokens read-only. +# State is derived from the bot's own PRs (watermark marker in the PR body), so +# missed or failed runs self-heal on the next run. on: schedule: @@ -22,6 +26,10 @@ on: description: "Collect + triage only, no edits, no PR" type: boolean default: false + pull_request: + paths: + - ".github/docs-sync/**" + - ".github/workflows/docs-sync.yml" permissions: contents: write # push the rolling branch, create the auto-docs label @@ -37,9 +45,27 @@ env: EDIT_MODEL: ${{ vars.DOCS_SYNC_EDIT_MODEL || 'kilo/moonshotai/kimi-k3' }} jobs: - sync: + selftest: if: github.repository == 'Kilo-Org/kilocode' runs-on: blacksmith-4vcpu-ubuntu-2404 + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: "24" + package-manager-cache: false + + - name: Run docs-sync selftest + run: node .github/docs-sync/selftest.mjs + + sync: + if: github.repository == 'Kilo-Org/kilocode' && github.event_name != 'pull_request' + runs-on: blacksmith-4vcpu-ubuntu-2404 # Budget: 3 setup/collect + 35 triage + 50 edit + 2 verify + 10 fix + 2 upsert = 102 min, 18-minute reserve. timeout-minutes: 120 env: @@ -64,6 +90,9 @@ jobs: node-version: "24" package-manager-cache: false + - name: Run docs-sync selftest + run: node .github/docs-sync/selftest.mjs + - name: Install Kilo CLI run: | npm install -g @kilocode/cli From eeb19d905b88a9b829819d394e23cfc29d5888db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= Date: Tue, 28 Jul 2026 00:46:12 +0200 Subject: [PATCH 4/4] fix(ci): isolate PR selftest concurrency from the daily docs-sync run --- .github/workflows/docs-sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-sync.yml b/.github/workflows/docs-sync.yml index 66a7799423d..5a32e876986 100644 --- a/.github/workflows/docs-sync.yml +++ b/.github/workflows/docs-sync.yml @@ -37,7 +37,7 @@ permissions: issues: write # comment on the rolling PR concurrency: - group: docs-sync + group: ${{ github.event_name == 'pull_request' && format('docs-sync-pr-{0}', github.event.pull_request.number) || 'docs-sync' }} cancel-in-progress: false env: