From 4431875e862be461f40f781897871ff59398f91f Mon Sep 17 00:00:00 2001 From: Yury Zialionka Date: Tue, 11 Aug 2026 17:05:11 -0600 Subject: [PATCH] fix(ci): reduce Windows CLI test contention Windows CLI unit tests ran 4-at-a-time on the 4-vCPU runner (custom test-runner default = min(4, cpus)), oversubscribing CPU so heavy real-server test files blew their per-test timeouts. - Cap KILO_TEST_CONCURRENCY=2 on Windows (each file gets ~2 vCPU). - Grow Windows shards 4 -> 6 to absorb the lower per-shard parallelism. - Shard by observed duration instead of file size, so the two heaviest files no longer stack in one shard. - Raise the Windows per-file kill deadline to 600s (KILO_TEST_FILE_TIMEOUT); the heaviest file runs ~270s, only ~30s under the old 300s default. Scoped to the @kilocode/cli custom runner on Windows only. Linux/macOS unchanged; the non-CLI/core suite runs via plain 'bun test' and is not affected (separate lever). --- .github/workflows/test.yml | 16 +++++- packages/opencode/script/test-runner.ts | 65 ++++++++++++++++++++++--- 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d57ca436e4b..e252eb78f5a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,7 +64,10 @@ jobs: exit 0 fi echo 'general=true' >> "$GITHUB_OUTPUT" - echo 'settings=[{"os":"linux","index":1,"total":2,"host":"blacksmith-4vcpu-ubuntu-2404","run":true,"packages":true},{"os":"linux","index":2,"total":2,"host":"blacksmith-4vcpu-ubuntu-2404","run":true,"packages":false},{"os":"macos","index":1,"total":1,"host":"macos-15","run":true,"packages":true},{"os":"windows","index":1,"total":4,"host":"blacksmith-4vcpu-windows-2025","run":true,"packages":true},{"os":"windows","index":2,"total":4,"host":"blacksmith-4vcpu-windows-2025","run":true,"packages":false},{"os":"windows","index":3,"total":4,"host":"blacksmith-4vcpu-windows-2025","run":true,"packages":false},{"os":"windows","index":4,"total":4,"host":"blacksmith-4vcpu-windows-2025","run":true,"packages":false}]' >> "$GITHUB_OUTPUT" + # kilocode_change - Windows is 6 shards (was 4): CLI tests now run at KILO_TEST_CONCURRENCY=2 + # instead of the default 4 to cut CPU contention on the 4-vCPU runner; more shards keep + # per-shard wall-clock within the job timeout despite the lower per-shard parallelism. + echo 'settings=[{"os":"linux","index":1,"total":2,"host":"blacksmith-4vcpu-ubuntu-2404","run":true,"packages":true},{"os":"linux","index":2,"total":2,"host":"blacksmith-4vcpu-ubuntu-2404","run":true,"packages":false},{"os":"macos","index":1,"total":1,"host":"macos-15","run":true,"packages":true},{"os":"windows","index":1,"total":6,"host":"blacksmith-4vcpu-windows-2025","run":true,"packages":true},{"os":"windows","index":2,"total":6,"host":"blacksmith-4vcpu-windows-2025","run":true,"packages":false},{"os":"windows","index":3,"total":6,"host":"blacksmith-4vcpu-windows-2025","run":true,"packages":false},{"os":"windows","index":4,"total":6,"host":"blacksmith-4vcpu-windows-2025","run":true,"packages":false},{"os":"windows","index":5,"total":6,"host":"blacksmith-4vcpu-windows-2025","run":true,"packages":false},{"os":"windows","index":6,"total":6,"host":"blacksmith-4vcpu-windows-2025","run":true,"packages":false}]' >> "$GITHUB_OUTPUT" # kilocode_change end unit: # kilocode_change start @@ -172,6 +175,17 @@ jobs: KILO_EXPERIMENTAL_DISABLE_FILEWATCHER: "true" # kilocode_change - was Windows-only; the CLI now starts a watcher per instance, too heavy/racy for unit tests. Watcher tests opt back in. KILO_TEST_PROFILE: ${{ matrix.settings.os == 'macos' && 'darwin' || '' }} KILO_TEST_SHARD: ${{ format('{0}/{1}', matrix.settings.index, matrix.settings.total) }} + # kilocode_change - cap parallelism on the 4-vCPU Windows runner. At the default + # min(4, cpus)=4, four heavy real-server test files share 4 vCPUs (~1 each) and blow + # their per-test timeouts; 2 gives each process real CPU headroom. Windows grows to + # 6 shards to absorb the lower per-shard parallelism. Linux/macOS (not timeout + # offenders; macOS is a single unsharded job) keep the default. + KILO_TEST_CONCURRENCY: ${{ matrix.settings.os == 'windows' && '2' || '' }} + # kilocode_change - raise the per-file kill deadline on Windows only. Heavy real-server + # files run ~230-270s serially there (vs ~40s on macOS/Linux), leaving only ~30s under + # the 300s default; 600s gives healthy-but-slow files real margin without masking hangs + # elsewhere (Linux/macOS keep the 300s default). + KILO_TEST_FILE_TIMEOUT: ${{ matrix.settings.os == 'windows' && '600000' || '' }} # kilocode_change end # kilocode_change start diff --git a/packages/opencode/script/test-runner.ts b/packages/opencode/script/test-runner.ts index c27ce536102..a2cbfe643ca 100644 --- a/packages/opencode/script/test-runner.ts +++ b/packages/opencode/script/test-runner.ts @@ -29,9 +29,9 @@ if (argv.includes("--help") || argv.includes("-h")) { "", "Options:", " --ci Enable JUnit XML output to .artifacts/unit/junit.xml", - " --concurrency Max parallel processes (default: min(4, CPU count))", + " --concurrency Max parallel processes (default: min(4, CPU count), env: KILO_TEST_CONCURRENCY)", " --timeout Per-test timeout passed to bun test (default: 60000)", - " --file-timeout Per-file process timeout (default: 300000)", + " --file-timeout Per-file process timeout (default: 300000, env: KILO_TEST_FILE_TIMEOUT)", " --retries Extra attempts for failing files (default: 1)", " --profile Run a curated test profile (env: KILO_TEST_PROFILE)", " --shard Run one balanced file shard (env: KILO_TEST_SHARD)", @@ -73,9 +73,39 @@ const dots = !verbose && (ci || argv.includes("--dots")) // Cap concurrency at 4 even on bigger runners: the bottleneck is shared // resources (ports, global filesystem like ~/.local/share/kilo), not CPU. // Eight parallel processes was triggering port/FS races, not going faster. -const concurrency = opt("concurrency", Math.min(4, os.cpus().length)) +// kilocode_change start - allow CI to lower concurrency via env. On the 4-vCPU +// Windows runner, the default (min(4, cpus)=4) oversubscribes: 4 heavy real-server +// test files share 4 vCPUs (~1 each) and blow their per-test timeouts. +// `KILO_TEST_CONCURRENCY` lets the workflow throttle Windows without affecting the +// local default. An explicit `--concurrency` flag wins. +const concurrencyEnv = (() => { + const raw = process.env.KILO_TEST_CONCURRENCY?.trim() + if (!raw) return undefined + const value = Number(raw) + if (!Number.isSafeInteger(value) || value < 1) { + console.error(`Invalid KILO_TEST_CONCURRENCY "${raw}"; expected a positive integer`) + process.exit(2) + } + return value +})() +const concurrency = opt("concurrency", concurrencyEnv ?? Math.min(4, os.cpus().length)) +// kilocode_change end const timeout = opt("timeout", 60000) -const deadline = opt("file-timeout", 300000) +// kilocode_change start - allow CI to raise the per-file kill deadline via env. On Windows, +// heavy real-server files (e.g. config-overlay) legitimately run ~270s serially, only ~30s +// under the 300s default; raising it there prevents a slow-but-healthy run from being killed. +const fileTimeoutEnv = (() => { + const raw = process.env.KILO_TEST_FILE_TIMEOUT?.trim() + if (!raw) return undefined + const value = Number(raw) + if (!Number.isSafeInteger(value) || value < 1) { + console.error(`Invalid KILO_TEST_FILE_TIMEOUT "${raw}"; expected a positive integer (ms)`) + process.exit(2) + } + return value +})() +const deadline = opt("file-timeout", fileTimeoutEnv ?? 300000) +// kilocode_change end const retries = opt("retries", 1) const flag = text("profile") const env = process.env.KILO_TEST_PROFILE?.trim() || undefined @@ -156,7 +186,28 @@ if (shard && shard.total > candidates.length) { console.error(`Test shard count ${shard.total} exceeds selected file count ${candidates.length}`) process.exit(2) } -const weight = (file: string) => Bun.file(path.join(root, "test", file)).size +// kilocode_change start - shard by estimated DURATION, not file size. File size is a poor +// proxy: run-process.test.ts is ~7 KB but ~230s, while config-overlay is the single slowest +// file — under size-weighting both landed in the same shard, stacking the two heaviest files. +// DURATION_HINTS are max observed per-file durations (ms) from real Windows CI runs; the LPT +// splitter places the highest-weight files first, so hinted heavy files get spread across +// distinct shards. Unhinted files fall back to size (a fine proxy among the fast majority); +// hint values (tens of thousands of ms) dominate byte sizes, so heavy files always sort first. +// Refresh these from observed CI durations when the suite changes materially. +const DURATION_HINTS: Record = { + "kilocode/server/config-overlay.test.ts": 270_000, + "cli/run/run-process.test.ts": 233_000, + "snapshot/snapshot.test.ts": 165_000, + "session/prompt.test.ts": 128_000, + "tool/shell.test.ts": 95_000, + "kilocode/background-process.test.ts": 94_000, + "provider/provider.test.ts": 90_000, + "kilocode/indexing-startup.test.ts": 88_000, + "kilocode/daemon.test.ts": 65_000, + "tool/task.test.ts": 64_000, +} +const weight = (file: string) => DURATION_HINTS[file] ?? Bun.file(path.join(root, "test", file)).size +// kilocode_change end const files = shard ? TestShard.split(candidates, weight, shard.total)[shard.index - 1] : candidates if (files.length === 0) { @@ -189,9 +240,7 @@ const xmldir = ci ? path.join(os.tmpdir(), `opencode-junit-${process.pid}`) : "" if (ci) await fs.mkdir(xmldir, { recursive: true }) // kilocode_change start const supplied = process.env[TestCli.ENV] -const built = supplied - ? { binary: supplied, dir: undefined } - : { binary: await TestCli.build(root), dir: undefined } +const built = supplied ? { binary: supplied, dir: undefined } : { binary: await TestCli.build(root), dir: undefined } async function cleanBinary() { if (!built.dir) return