diff --git a/ci/cli-test-timing-hints.json b/ci/cli-test-timing-hints.json index 20da6f8badd..40b2ffac005 100644 --- a/ci/cli-test-timing-hints.json +++ b/ci/cli-test-timing-hints.json @@ -1,106 +1,232 @@ { - "schemaVersion": 1, + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0", + "schemaVersion": 2, "defaultDurationMs": 5000, - "source": { - "runId": 29461162042, - "headSha": "2f53a162657b26f4c2ae538c5baab0e8a78ffdb5", - "recordedAt": "2026-07-16T00:29:23Z" - }, + "sources": [ + { + "runId": 32538808045, + "artifactId": 9467034647, + "headSha": "9577b175338d6cf1ead335452ade76470f1593a9", + "recordedAt": "2026-08-22T00:40:44Z" + }, + { + "runId": 32541609216, + "artifactId": 9467388387, + "headSha": "1080ecce4fd4d0366e546b3e92a25c3ec158af61", + "recordedAt": "2026-08-22T01:03:19Z" + }, + { + "runId": 32542347175, + "artifactId": 9467589302, + "headSha": "bb686324dd2ce19f3708c6900b3d22110e198662", + "recordedAt": "2026-08-22T01:16:45Z" + } + ], "files": { - "src/lib/actions/sandbox/doctor-flow.test.ts": 10845, - "src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts": 12841, - "src/lib/actions/sandbox/mcp-bridge-status-state.test.ts": 5515, - "src/lib/shields/flow.test.ts": 9935, - "src/lib/tunnel/services.test.ts": 8161, - "test/channels-remove-full-teardown.test.ts": 6803, - "test/cli/connect-recovery.test.ts": 6627, - "test/cli/debug-command.test.ts": 12502, - "test/cli/destroy-gateway-cleanup.test.ts": 9411, - "test/cli/dispatch-basics.test.ts": 6395, - "test/cli/docker-outage.test.ts": 5393, - "test/cli/doctor-gateway-token.test.ts": 13392, - "test/cli/helpers.test.ts": 5768, - "test/cli/list-inference.test.ts": 9021, - "test/cli/list-share-live-inference.test.ts": 9127, - "test/cli/logs.test.ts": 9613, - "test/cli/maintenance-command.test.ts": 6112, - "test/cli/onboard-compatibility.test.ts": 8286, - "test/cli/sandbox-mutations.test.ts": 10933, - "test/cli/sandbox-status-json.test.ts": 18427, - "test/cli/sandbox-status-text.test.ts": 9429, - "test/cli/snapshot-shields.test.ts": 6540, - "test/cli/tunnel-command.test.ts": 9079, - "test/dcode-session-supervisor.test.ts": 7504, - "test/dcode-wrapper-identity.test.ts": 8230, - "test/dependency-upgrade-skill.test.ts": 16586, - "test/e2e-recommendations.test.ts": 26617, - "test/e2e/support/dockerhub-auth-workflow-boundary.test.ts": 5493, - "test/e2e/support/e2e-live-target-gating.test.ts": 14703, - "test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts": 7331, - "test/e2e/support/e2e-workflow-trace.test.ts": 5566, - "test/e2e/support/hermes-workflow-boundary.test.ts": 5434, - "test/e2e/support/shared-e2e-workflow-boundary.test.ts": 5043, - "test/exit-code-user-error-surfaces.test.ts": 6568, - "test/gateway-state-reconcile-2276.test.ts": 6022, - "test/hermes-restart-config-seal-write-lock.test.ts": 23034, - "test/hermes-runtime-api-key.test.ts": 8983, - "test/install-onboard-yes.test.ts": 7188, - "test/install-openshell-upgrade-prompt.test.ts": 19253, - "test/installer-hash-check.test.ts": 33005, - "test/internal-cli.test.ts": 6868, - "test/langchain-deepagents-code-direct-module-patch.test.ts": 16907, - "test/langchain-deepagents-code-image-credentials.test.ts": 22629, - "test/langchain-deepagents-code-nemotron-profile-plugin.test.ts": 5206, - "test/list-command-helper-uses.test.ts": 7153, - "test/local-credential-helper-session.test.ts": 5104, - "test/mcp-add-crash-consistency.test.ts": 15394, - "test/mcp-bridge-destroy-marker-recovery.test.ts": 6229, - "test/mcp-bridge-servers.test.ts": 15163, - "test/nemo-deepagents-alias.test.ts": 11578, - "test/nemoclaw-start-gateway-health.test.ts": 5691, - "test/nemoclaw-start.test.ts": 28215, - "test/nemohermes-alias.test.ts": 9628, - "test/ollama-auth-proxy-handler.test.ts": 5518, - "test/ollama-proxy-recovery.test.ts": 5533, - "test/onboard-custom-dockerfile.test.ts": 11854, - "test/onboard-fsm-live-slices.test.ts": 8745, - "test/onboard-messaging.test.ts": 23525, - "test/onboard-ollama-autostart.test.ts": 5274, - "test/onboard-prompt-default-case.test.ts": 9136, - "test/onboard-sandbox-build.test.ts": 5352, - "test/onboard-sandbox-recreation.test.ts": 20130, - "test/onboard-selection-vllm.test.ts": 5005, - "test/onboard-selection.test.ts": 26638, - "test/openclaw-chat-send-patch.test.ts": 7378, - "test/openclaw-config-guard.test.ts": 17693, - "test/openclaw-device-self-approval-patch.test.ts": 11677, - "test/plugin-vitest-project.test.ts": 5205, - "test/policy-add-remove-session-sync.test.ts": 6200, - "test/policy-channel-agent-resolution.test.ts": 5566, - "test/pr-review-advisor-ledger-tools.test.ts": 6997, - "test/pr-review-advisor-security-boundaries.test.ts": 6470, - "test/pr-review-advisor-workflow-boundary.test.ts": 9110, - "test/pr-workflow-contract.test.ts": 15130, - "test/reboot-identity-drift.test.ts": 10286, - "test/rebuild-credential-preflight.test.ts": 19588, - "test/recover-port-forward.test.ts": 6524, - "test/release-latest-tag.test.ts": 26289, - "test/repro-2201.test.ts": 19110, - "test/repro-2666-silent-list-status.test.ts": 8412, - "test/sandbox-connect-inference/auto-pair-approval.test.ts": 12500, - "test/sandbox-connect-inference/route-swap-repair.test.ts": 19663, - "test/sandbox-sessions-export-cli.test.ts": 9559, - "test/sandbox-stuck-recovery.test.ts": 5346, - "test/skills/check-gates-compliance.test.ts": 10109, - "test/skills/triage-runtime.test.ts": 6584, - "test/snapshot-gateway-guard.test.ts": 7223, - "test/snapshot-openclaw-managed-extensions.test.ts": 13184, - "test/snapshot-restore-existing-dest.test.ts": 11332, - "test/snapshot.test.ts": 16652, - "test/source-shape-scanner.test.ts": 60577, - "test/state-dir-guard.test.ts": 6168, - "test/telegram-diagnostics.test.ts": 5021, - "test/uninstall-prompt-pty.test.ts": 7258, - "test/uninstall.test.ts": 7866 + "src/lib/actions/sandbox/auto-pair-approval.test.ts": 10074, + "src/lib/actions/sandbox/destroy-flow.test.ts": 25594, + "src/lib/actions/sandbox/doctor-flow.test.ts": 24265, + "src/lib/actions/sandbox/mcp-bridge-adapter-openclaw.test.ts": 8470, + "src/lib/actions/sandbox/mcp-bridge-input-targets.test.ts": 5667, + "src/lib/actions/sandbox/mcp-bridge-status-removal.test.ts": 7973, + "src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts": 34602, + "src/lib/actions/sandbox/mcp-bridge-status-state.test.ts": 10485, + "src/lib/actions/sandbox/start.test.ts": 21550, + "src/lib/actions/uninstall/bedrock-runtime-adapter-cleanup.test.ts": 5619, + "src/lib/actions/uninstall/hermes-portable-uninstall.test.ts": 20347, + "src/lib/actions/uninstall/portable-runtime-cleanup.test.ts": 13026, + "src/lib/actions/uninstall/run-plan-portable-runtime.test.ts": 5406, + "src/lib/agent/base-image-hermes-resolution.test.ts": 5182, + "src/lib/agent/base-image.test.ts": 9684, + "src/lib/inference/serving/catalog.test.ts": 20421, + "src/lib/onboard/compatible-endpoint-smoke.test.ts": 5086, + "src/lib/onboard/exit-step-failure.test.ts": 5262, + "src/lib/onboard/experimental/hermes-portable-build-context.test.ts": 7152, + "src/lib/onboard/gateway-host-runtime.test.ts": 14070, + "src/lib/onboard/runtime-provider/persisted-engine-lifecycle.test.ts": 11857, + "src/lib/onboard/setup-nim-flow.test.ts": 5286, + "src/lib/shields/flow.test.ts": 10379, + "src/lib/shields/hermes-runtime-state-mutation.test.ts": 8873, + "src/lib/shields/index.test.ts": 9645, + "src/lib/shields/legacy-hermes-compat.test.ts": 10641, + "src/lib/shields/mcp-policy-transition.test.ts": 7366, + "src/lib/shields/openclaw-transition.test.ts": 10802, + "src/lib/shields/policy-transition.test.ts": 11035, + "src/lib/shields/timer.test.ts": 5759, + "src/lib/state/onboard-session-station-express.test.ts": 11308, + "src/lib/state/onboard-session.test.ts": 6062, + "src/lib/state/portable-uninstall-retirement.test.ts": 27072, + "src/lib/tunnel/services.test.ts": 5573, + "test/brev-launchable-e2e.test.ts": 34518, + "test/channels-remove-full-teardown.test.ts": 9727, + "test/cli-oclif-compatibility.test.ts": 7416, + "test/cli/connect-recovery.test.ts": 14143, + "test/cli/credentials-command.test.ts": 6511, + "test/cli/debug-command.test.ts": 27452, + "test/cli/destroy-gateway-cleanup.test.ts": 20186, + "test/cli/dispatch-basics.test.ts": 10094, + "test/cli/docker-outage.test.ts": 12355, + "test/cli/doctor-gateway-token.test.ts": 19537, + "test/cli/helpers.test.ts": 7927, + "test/cli/launch-routing.test.ts": 13502, + "test/cli/list-inference.test.ts": 25333, + "test/cli/list-share-live-inference.test.ts": 26393, + "test/cli/logs-documented-invocations.test.ts": 18011, + "test/cli/logs.test.ts": 17679, + "test/cli/maintenance-command.test.ts": 10074, + "test/cli/onboard-compatibility.test.ts": 13249, + "test/cli/sandbox-mutations.test.ts": 22796, + "test/cli/sandbox-skill-install.test.ts": 5955, + "test/cli/sandbox-status-json.test.ts": 31057, + "test/cli/sandbox-status-text.test.ts": 14597, + "test/cli/snapshot-shields.test.ts": 14050, + "test/cli/tunnel-command.test.ts": 16882, + "test/credentials.test.ts": 6328, + "test/dcode-session-supervisor.test.ts": 7644, + "test/dcode-wrapper-identity.test.ts": 11176, + "test/deepagents-code-tui-startup-check.test.ts": 28222, + "test/dependency-upgrade-skill-security.test.ts": 5353, + "test/dependency-upgrade-skill.test.ts": 19850, + "test/dns-proxy.test.ts": 5827, + "test/e2e-recommendations.test.ts": 45421, + "test/e2e/support/base-image-publication-workflow-boundary.test.ts": 5429, + "test/e2e/support/cli-artifact-workflow-boundary.test.ts": 5392, + "test/e2e/support/dcode-profile-import-gate-workflow-boundary.test.ts": 6533, + "test/e2e/support/dockerhub-auth-workflow-boundary.test.ts": 8284, + "test/e2e/support/e2e-live-target-gating.test.ts": 27808, + "test/e2e/support/e2e-operations-workflow-boundary.test.ts": 26042, + "test/e2e/support/e2e-progress-outcome.test.ts": 11907, + "test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts": 7706, + "test/e2e/support/e2e-semantic-phase-check.test.ts": 8397, + "test/e2e/support/e2e-workflow-trace.test.ts": 6200, + "test/e2e/support/hermes-workflow-boundary.test.ts": 7516, + "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts": 5436, + "test/e2e/support/launch-agent-turn.test.ts": 62646, + "test/e2e/support/live-test-outcome-invocation.test.ts": 5281, + "test/e2e/support/mcp-bridge-runtime-compatibility-cli.test.ts": 5746, + "test/e2e/support/mcp-workflow-boundary.test.ts": 10250, + "test/e2e/support/podman-cpu-lifecycle-helpers.test.ts": 6203, + "test/e2e/support/portable-profile-systemctl-shim.test.ts": 10124, + "test/e2e/support/shared-e2e-workflow-boundary.test.ts": 5690, + "test/e2e/support/workflow-plan.test.ts": 39886, + "test/effective-policy-contracts.test.ts": 7870, + "test/exit-code-user-error-surfaces.test.ts": 13250, + "test/gateway-drift-preflight.test.ts": 7324, + "test/gateway-serving-watchdog.test.ts": 12304, + "test/gateway-state-reconcile-2276.test.ts": 29961, + "test/generate-managed-startup-profile-fixture.test.ts": 5664, + "test/hermes-kanban-snapshot.test.ts": 10528, + "test/hermes-provider-foundation.test.ts": 5353, + "test/hermes-runtime-api-key.test.ts": 9872, + "test/hermes-state-ledger-snapshot.test.ts": 5742, + "test/hermes-tool-gateway-broker.test.ts": 8095, + "test/install-gateway-state-root.test.ts": 5494, + "test/install-onboard-yes.test.ts": 11335, + "test/install-openshell-gateway-service.test.ts": 5873, + "test/install-openshell-upgrade-prompt.test.ts": 77471, + "test/installer-hash-check.test.ts": 41961, + "test/installer-sandbox-build-trust.test.ts": 6681, + "test/installer-supervisor-manifest-trust.test.ts": 6603, + "test/internal-cli.test.ts": 16451, + "test/langchain-deepagents-code-config.test.ts": 9774, + "test/langchain-deepagents-code-direct-module-patch.test.ts": 31234, + "test/langchain-deepagents-code-image-credentials.test.ts": 31365, + "test/langchain-deepagents-code-image.test.ts": 17504, + "test/langchain-deepagents-code-nemotron-profile-plugin.test.ts": 7530, + "test/layer-import-boundaries.test.ts": 5292, + "test/list-command-helper-uses.test.ts": 7294, + "test/local-credential-helper-session.test.ts": 5247, + "test/managed-bootstrap-trampoline.test.ts": 14561, + "test/mcp-add-crash-consistency.test.ts": 43147, + "test/mcp-bridge-destroy-marker-recovery.test.ts": 14345, + "test/mcp-bridge-servers.test.ts": 15199, + "test/mcp-policy-key-ownership.test.ts": 9878, + "test/mcp-policy-transition.test.ts": 5011, + "test/mcp-provider-ownership.test.ts": 9896, + "test/mcp-tool-discovery-image-contract.test.ts": 8489, + "test/nemo-deepagents-alias.test.ts": 17997, + "test/nemoclaw-start.test.ts": 28600, + "test/nemohermes-alias.test.ts": 14670, + "test/ollama-auth-proxy-handler.test.ts": 6722, + "test/ollama-proxy-recovery.test.ts": 34833, + "test/ollama-proxy-startup.test.ts": 8094, + "test/onboard-brave-validation.test.ts": 8395, + "test/onboard-custom-dockerfile.test.ts": 19813, + "test/onboard-dashboard.test.ts": 8568, + "test/onboard-endpoint-url-rejection.test.ts": 18559, + "test/onboard-exit-handler.test.ts": 8716, + "test/onboard-extra-provider-reconciliation.test.ts": 12683, + "test/onboard-finalization-dashboard-forward.test.ts": 5857, + "test/onboard-fsm-live-slices.test.ts": 20855, + "test/onboard-inference-reconciliation.test.ts": 10281, + "test/onboard-installer-restore-intent.test.ts": 14131, + "test/onboard-lifecycle.test.ts": 6967, + "test/onboard-managed-image-buildless-e2e.test.ts": 14383, + "test/onboard-messaging.test.ts": 72008, + "test/onboard-ollama-autostart.test.ts": 17749, + "test/onboard-ollama-context-floor.test.ts": 5807, + "test/onboard-prepared-build-context.test.ts": 8435, + "test/onboard-prompt-default-case.test.ts": 14894, + "test/onboard-provider-key-bridge.test.ts": 7265, + "test/onboard-remote-recreate-credential-reuse.test.ts": 5737, + "test/onboard-reservation-recreate.test.ts": 18690, + "test/onboard-sandbox-build.test.ts": 28225, + "test/onboard-sandbox-recreation.test.ts": 68947, + "test/onboard-selection-vllm.test.ts": 13182, + "test/onboard-selection.test.ts": 50371, + "test/onboard-terminal-dashboard.test.ts": 11927, + "test/onboard.test.ts": 7899, + "test/openclaw-chat-send-patch.test.ts": 8126, + "test/openclaw-config-guard.test.ts": 22936, + "test/openclaw-config-snapshot.test.ts": 5147, + "test/openclaw-device-self-approval-patch.test.ts": 36649, + "test/openclaw-device-stored-auth-patch.test.ts": 8297, + "test/openclaw-gemini-inference-compat-runtime.test.ts": 36915, + "test/openclaw-integrity-pin-base.test.ts": 10104, + "test/plugin-vitest-project.test.ts": 7766, + "test/policies.test.ts": 5377, + "test/policy-add-remove-session-sync.test.ts": 10393, + "test/policy-channel-agent-resolution.test.ts": 8083, + "test/policy-explain-cli.test.ts": 5249, + "test/policy-openclaw-npm-compatibility.test.ts": 14295, + "test/policy-preset-noop-disclosure.test.ts": 9796, + "test/policy-tiers-onboard-restricted-stale-otel.test.ts": 5808, + "test/policy-tiers-onboard.test.ts": 8804, + "test/portable-cpu-delegation-docs.test.ts": 39159, + "test/portable-policy-failure-finality.test.ts": 8392, + "test/pr-review-advisor-security-boundaries.test.ts": 24108, + "test/pr-review-advisor-submission-tools.test.ts": 5422, + "test/pr-review-advisor-writing-guide.test.ts": 8835, + "test/reboot-identity-drift.test.ts": 13632, + "test/rebuild-credential-hydration.test.ts": 9487, + "test/rebuild-credential-preflight.test.ts": 22966, + "test/recover-port-forward.test.ts": 12821, + "test/release-latest-tag.test.ts": 20264, + "test/repro-2666-silent-list-status.test.ts": 12380, + "test/retire-release-label.test.ts": 5642, + "test/sandbox-agent-surface-parity.test.ts": 6276, + "test/sandbox-build-context.test.ts": 33488, + "test/sandbox-connect-inference/auto-pair-approval.test.ts": 20974, + "test/sandbox-connect-inference/route-swap-repair.test.ts": 5822, + "test/sandbox-download-upload-cli.test.ts": 6982, + "test/sandbox-sessions-admin-agent-cli.test.ts": 7935, + "test/sandbox-sessions-export-cli.test.ts": 20387, + "test/sandbox-stuck-recovery.test.ts": 8760, + "test/skills/check-gates-actions-evidence.test.ts": 21898, + "test/skills/check-gates-compliance.test.ts": 22684, + "test/snapshot-gateway-guard.test.ts": 17301, + "test/snapshot-openclaw-managed-extensions.test.ts": 18060, + "test/snapshot-restore-existing-dest.test.ts": 17653, + "test/snapshot-stale-directory-restore.test.ts": 7384, + "test/snapshot.test.ts": 23861, + "test/source-architecture.test.ts": 5535, + "test/source-require-loader.test.ts": 5155, + "test/state-dir-guard.test.ts": 10477, + "test/strict-tool-call-probe.test.ts": 31346, + "test/telegram-diagnostics.test.ts": 5127, + "test/uninstall-prompt-pty.test.ts": 17156, + "test/uninstall.test.ts": 14144, + "test/update.test.ts": 10077, + "test/wechat-runtime-audit-workflow.test.ts": 26307 } } diff --git a/test/cli-coverage-sequencer.test.ts b/test/cli-coverage-sequencer.test.ts index d7599b4a387..c7d864235f1 100644 --- a/test/cli-coverage-sequencer.test.ts +++ b/test/cli-coverage-sequencer.test.ts @@ -7,6 +7,10 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import type { TestSpecification, Vitest } from "vitest/node"; +import { + discoverVitestCandidates, + expectedProjectForTestPath, +} from "../scripts/checks/vitest-project-overlap.mts"; import { assignStableShards, CliCoverageSequencer, @@ -44,38 +48,20 @@ function sequencer(index: number, count: number): CliCoverageSequencer { } as unknown as Vitest); } -function representativeCliCoverageEntries(): WeightedShardEntry[] { - const measured = Object.entries(cliTestTimingHints.files).map(([file, weightMs]) => { - const projectName = file.startsWith("src/") - ? "cli" - : file.startsWith("test/e2e/support/") - ? "e2e-support" - : "integration"; - return { key: `${projectName}:${file}`, weightMs, value: file }; +function currentCliCoverageEntries(): WeightedShardEntry[] { + const coverageProjects = new Set(["cli", "integration", "e2e-support"]); + return [...discoverVitestCandidates()].flatMap((file) => { + const projectName = expectedProjectForTestPath(file); + return projectName && coverageProjects.has(projectName) + ? [ + { + key: `${projectName}:${file}`, + weightMs: timingWeightForPath(file), + value: file, + }, + ] + : []; }); - const projectSizes = { cli: 832, integration: 512, "e2e-support": 116 } as const; - const ordinary = (Object.keys(projectSizes) as (keyof typeof projectSizes)[]).flatMap( - (projectName) => { - const measuredCount = measured.filter((entry) => - entry.key.startsWith(`${projectName}:`), - ).length; - return Array.from({ length: projectSizes[projectName] - measuredCount }, (_, index) => { - const file = - projectName === "cli" - ? `src/lib/fixture-${index}.test.ts` - : projectName === "e2e-support" - ? `test/e2e/support/fixture-${index}.test.ts` - : `test/fixture-${index}.test.ts`; - return { - key: `${projectName}:${file}`, - weightMs: cliTestTimingHints.defaultDurationMs, - value: file, - }; - }); - }, - ); - - return [...measured, ...ordinary]; } describe("stable CLI coverage sharding", () => { @@ -129,22 +115,36 @@ describe("stable CLI coverage sharding", () => { ); expect(Object.fromEntries(owners)).toEqual({ - "cli:src/lib/example.test.ts": 5, - "e2e-support:test/e2e/support/example.test.ts": 5, + "cli:src/lib/example.test.ts": 6, + "e2e-support:test/e2e/support/example.test.ts": 8, "integration:test/hermes-restart-config-seal-write-lock.test.ts": 6, - "integration:test/local-credential-helper-fields.test.ts": 5, - "integration:test/regular-0.test.ts": 4, + "integration:test/local-credential-helper-fields.test.ts": 7, + "integration:test/regular-0.test.ts": 6, }); }); - it("keeps a representative test roster balanced across the eight CI shards", () => { - const shards = assignStableShards(representativeCliCoverageEntries(), 8); + it("keeps the current test roster balanced across the twelve CI shards (#6237)", () => { + const shards = assignStableShards(currentCliCoverageEntries(), 12); const weights = shards.map((shard) => shard.totalWeightMs); const averageWeight = weights.reduce((total, weight) => total + weight, 0) / weights.length; expect(Math.max(...weights)).toBeLessThanOrEqual(averageWeight * 1.05); }); + it("balances the serialized integration lane across the twelve CI shards (#6237)", () => { + const integrationEntries = currentCliCoverageEntries().filter((entry) => + entry.key.startsWith("integration:"), + ); + expect(integrationEntries.length).toBeGreaterThan(0); + + const weights = assignStableShards(integrationEntries, 12).map( + (shard) => shard.totalWeightMs, + ); + const averageWeight = weights.reduce((total, weight) => total + weight, 0) / weights.length; + + expect(Math.max(...weights)).toBeLessThanOrEqual(averageWeight * 1.1); + }); + it("uses stable sharding only for projects owned by the CLI coverage matrix", () => { expect(shouldUseCliCoverageSharding(["cli", "integration", "e2e-support"])).toBe(true); expect(shouldUseCliCoverageSharding(["integration"])).toBe(true); @@ -179,6 +179,26 @@ describe("stable CLI coverage sharding", () => { const files = Object.keys(cliTestTimingHints.files); expect(cliTestTimingHints.defaultDurationMs).toBe(5_000); + expect(cliTestTimingHints.sources).toEqual([ + { + runId: 32538808045, + artifactId: 9467034647, + headSha: "9577b175338d6cf1ead335452ade76470f1593a9", + recordedAt: "2026-08-22T00:40:44Z", + }, + { + runId: 32541609216, + artifactId: 9467388387, + headSha: "1080ecce4fd4d0366e546b3e92a25c3ec158af61", + recordedAt: "2026-08-22T01:03:19Z", + }, + { + runId: 32542347175, + artifactId: 9467589302, + headSha: "bb686324dd2ce19f3708c6900b3d22110e198662", + recordedAt: "2026-08-22T01:16:45Z", + }, + ]); expect(files).toEqual([...files].sort()); expect(files.length).toBeGreaterThan(50); files.forEach((file) => { @@ -189,7 +209,7 @@ describe("stable CLI coverage sharding", () => { }); it("rejects malformed timing hint manifests", () => { - expect(() => parseCliTestTimingHints({ schemaVersion: 2 })).toThrow(/schemaVersion 1/u); + expect(() => parseCliTestTimingHints({ schemaVersion: 1 })).toThrow(/schemaVersion 2/u); expect(() => parseCliTestTimingHints({ ...cliTestTimingHints, @@ -197,4 +217,30 @@ describe("stable CLI coverage sharding", () => { }), ).toThrow(/Invalid CLI test timing hint/u); }); + + it.each([ + { name: "no sources", sources: [] }, + { + name: "a source without an artifact ID", + sources: [ + { + runId: 32538808045, + headSha: "9577b175338d6cf1ead335452ade76470f1593a9", + recordedAt: "2026-08-22T00:40:44Z", + }, + ], + }, + { + name: "a non-positive artifact ID", + sources: [{ ...cliTestTimingHints.sources[0], artifactId: 0 }], + }, + { + name: "a non-integer artifact ID", + sources: [{ ...cliTestTimingHints.sources[0], artifactId: 1.5 }], + }, + ])("rejects timing hint manifests with $name", ({ sources }) => { + expect(() => parseCliTestTimingHints({ ...cliTestTimingHints, sources })).toThrow( + /timing hints? (?:require source metadata|source)/u, + ); + }); }); diff --git a/test/helpers/cli-coverage-sequencer.ts b/test/helpers/cli-coverage-sequencer.ts index f85aa25158d..1d979dc7751 100644 --- a/test/helpers/cli-coverage-sequencer.ts +++ b/test/helpers/cli-coverage-sequencer.ts @@ -9,14 +9,15 @@ import { BaseSequencer, type TestSpecification } from "vitest/node"; interface TimingHintSource { runId: number; + artifactId: number; headSha: string; recordedAt: string; } export interface CliTestTimingHints { - schemaVersion: 1; + schemaVersion: 2; defaultDurationMs: number; - source: TimingHintSource; + sources: readonly TimingHintSource[]; files: Readonly>; } @@ -35,11 +36,14 @@ export interface WeightedShard { // E2E-support is hermetic and shares the same installed dependencies and CLI // build as the CLI coverage projects, so the coverage matrix owns it too. const cliCoverageProjects = new Set(["cli", "integration", "e2e-support"]); -// Changing either salt intentionally remaps that lane's tests. These values -// are calibrated against the timing-hint source profile, then kept fixed so +// Changing a salt intentionally remaps that lane's tests. These values are +// calibrated against the timing-hint source profile, then kept fixed so // ordinary roster changes preserve ownership between profile refreshes. -const stableShardSalt = "6096"; -const e2eSupportShardSalt = "2045"; +// Integration coverage is serialized, so it needs an independent salt instead +// of relying on combined weight from the parallel CLI and E2E-support lanes. +const stableShardSalt = "7257"; +const integrationShardSalt = "28320"; +const e2eSupportShardSalt = "13930"; // Only measured outliers are stored; new and ordinary files share the // conservative fallback used to estimate each stable shard's load. const timingHintsUrl = new URL("../../ci/cli-test-timing-hints.json", import.meta.url); @@ -49,26 +53,32 @@ function isRecord(value: unknown): value is Record { } export function parseCliTestTimingHints(value: unknown): CliTestTimingHints { - if (!isRecord(value) || value.schemaVersion !== 1) { - throw new Error("CLI test timing hints must use schemaVersion 1"); + if (!isRecord(value) || value.schemaVersion !== 2) { + throw new Error("CLI test timing hints must use schemaVersion 2"); } if (!Number.isSafeInteger(value.defaultDurationMs) || Number(value.defaultDurationMs) <= 0) { throw new Error("CLI test timing hints require a positive integer defaultDurationMs"); } - if (!isRecord(value.source)) { + if (!Array.isArray(value.sources) || value.sources.length === 0) { throw new Error("CLI test timing hints require source metadata"); } - - const { runId, headSha, recordedAt } = value.source; - if (!Number.isSafeInteger(runId) || Number(runId) <= 0) { - throw new Error("CLI test timing hint source requires a positive runId"); - } - if (typeof headSha !== "string" || !/^[0-9a-f]{40}$/u.test(headSha)) { - throw new Error("CLI test timing hint source requires a full commit SHA"); - } - if (typeof recordedAt !== "string" || Number.isNaN(Date.parse(recordedAt))) { - throw new Error("CLI test timing hint source requires an ISO timestamp"); - } + const sources = value.sources.map((source) => { + if (!isRecord(source)) throw new Error("Invalid CLI test timing hint source"); + const { runId, artifactId, headSha, recordedAt } = source; + if ( + !Number.isSafeInteger(runId) || + Number(runId) <= 0 || + !Number.isSafeInteger(artifactId) || + Number(artifactId) <= 0 || + typeof headSha !== "string" || + !/^[0-9a-f]{40}$/u.test(headSha) || + typeof recordedAt !== "string" || + Number.isNaN(Date.parse(recordedAt)) + ) { + throw new Error("Invalid CLI test timing hint source"); + } + return { runId: Number(runId), artifactId: Number(artifactId), headSha, recordedAt }; + }); if (!isRecord(value.files)) { throw new Error("CLI test timing hints require a files map"); } @@ -91,9 +101,9 @@ export function parseCliTestTimingHints(value: unknown): CliTestTimingHints { } return { - schemaVersion: 1, + schemaVersion: 2, defaultDurationMs, - source: { runId: Number(runId), headSha, recordedAt }, + sources, files, }; } @@ -138,7 +148,11 @@ export function assignStableShards( // removing, or renaming another test cannot move existing files between the // long-lived coverage shards and change which source maps are merged together. for (const entry of ranked) { - const salt = entry.key.startsWith("e2e-support:") ? e2eSupportShardSalt : stableShardSalt; + const salt = entry.key.startsWith("integration:") + ? integrationShardSalt + : entry.key.startsWith("e2e-support:") + ? e2eSupportShardSalt + : stableShardSalt; const digest = createHash("sha256").update(`${salt}:${entry.key}`).digest(); const target = shards[digest.readUInt32BE(0) % shardCount]; if (!target) throw new Error("Stable shard allocation requires at least one shard"); diff --git a/test/helpers/integration-project-scheduling.ts b/test/helpers/integration-project-scheduling.ts index d7b3f279afd..1f07eed3c93 100644 --- a/test/helpers/integration-project-scheduling.ts +++ b/test/helpers/integration-project-scheduling.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 const LOCAL_INTEGRATION_WORKER_CAP = 4; +const CLI_COVERAGE_SHARD_WORKER_CAP = 2; interface IntegrationProjectSchedulingContext { isCi: boolean; @@ -10,6 +11,30 @@ interface IntegrationProjectSchedulingContext { availableParallelism?: number; } +interface CliCoverageShardSchedulingContext { + isCi: boolean; + cliShard: string | undefined; + cliShardCount: string | undefined; +} + +function parsePositiveInteger(rawValue: string | undefined): number | null { + if (!rawValue || !/^\d+$/u.test(rawValue)) return null; + const parsed = Number(rawValue); + return Number.isSafeInteger(parsed) && parsed >= 1 ? parsed : null; +} + +export function resolveCliCoverageShardScheduling({ + isCi, + cliShard, + cliShardCount, +}: CliCoverageShardSchedulingContext) { + const shard = parsePositiveInteger(cliShard); + const shardCount = parsePositiveInteger(cliShardCount); + return isCi && shard !== null && shardCount !== null && shard <= shardCount + ? { maxWorkers: CLI_COVERAGE_SHARD_WORKER_CAP } + : {}; +} + function parseWorkerCount(rawValue: string, availableWorkers: number): number { if (/^\d+$/.test(rawValue)) { const parsed = Number(rawValue); diff --git a/test/vitest-integration-scheduling.test.ts b/test/vitest-integration-scheduling.test.ts index f5fba078019..bd6925c8bb8 100644 --- a/test/vitest-integration-scheduling.test.ts +++ b/test/vitest-integration-scheduling.test.ts @@ -3,7 +3,35 @@ import { describe, expect, it } from "vitest"; -import { resolveIntegrationProjectScheduling } from "./helpers/integration-project-scheduling"; +import { + resolveCliCoverageShardScheduling, + resolveIntegrationProjectScheduling, +} from "./helpers/integration-project-scheduling"; + +describe("CLI coverage shard scheduling", () => { + it("uses two workers for a validated CI shard (#6237)", () => { + expect( + resolveCliCoverageShardScheduling({ + isCi: true, + cliShard: "2", + cliShardCount: "12", + }), + ).toEqual({ maxWorkers: 2 }); + }); + + it.each([ + ["local shard", false, "1", "12"], + ["missing shard", true, undefined, "12"], + ["missing shard count", true, "1", undefined], + ["invalid shard", true, "x", "12"], + ["invalid shard count", true, "1", "0"], + ["out-of-range shard", true, "13", "12"], + ])("does not constrain $0", (_name, isCi, cliShard, cliShardCount) => { + expect( + resolveCliCoverageShardScheduling({ isCi, cliShard, cliShardCount }), + ).toEqual({}); + }); +}); describe("integration project scheduling", () => { it("parallelizes the canonical local full-suite run (#6245)", () => { diff --git a/vitest.config.ts b/vitest.config.ts index 5674220983b..640d347069b 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -9,7 +9,10 @@ import { defineConfig, defineProject } from "vitest/config"; import pluginVitestProjectOptions from "./nemoclaw/vitest.project"; import { shouldRunLiveE2E } from "./test/e2e/fixtures/live-project-gate.ts"; import { CliCoverageSequencer } from "./test/helpers/cli-coverage-sequencer"; -import { resolveIntegrationProjectScheduling } from "./test/helpers/integration-project-scheduling"; +import { + resolveCliCoverageShardScheduling, + resolveIntegrationProjectScheduling, +} from "./test/helpers/integration-project-scheduling"; import { sourceLoaderNodeOptions } from "./test/helpers/source-loader-options"; import { testTimeout } from "./test/helpers/timeouts"; import { resolveVitestCoverageThresholds } from "./test/helpers/vitest-coverage-thresholds"; @@ -95,6 +98,15 @@ const controlledNonLiveEnv = { const fixtureUmaskSetup = "test/helpers/normalize-fixture-umask.ts"; const isolatedTestStateSetup = "test/helpers/isolate-test-state.ts"; const pluginVitestProject = defineProject(pluginVitestProjectOptions); +// Pull-request jobs execute the base branch's trusted composite action, so an +// action change in a PR cannot constrain that PR's own Vitest workers. Apply a +// bounded cap from the validated shard environment instead; this is shared by the +// trusted PR action and the main-branch action. +const cliCoverageShardScheduling = resolveCliCoverageShardScheduling({ + isCi, + cliShard: process.env.CLI_SHARD, + cliShardCount: process.env.CLI_SHARD_COUNT, +}); const integrationProjectScheduling = resolveIntegrationProjectScheduling({ isCi, npmLifecycleEvent: process.env.npm_lifecycle_event, @@ -104,6 +116,7 @@ const integrationProjectScheduling = resolveIntegrationProjectScheduling({ export default defineConfig({ test: { + ...cliCoverageShardScheduling, globalSetup: "test/helpers/vitest-temp-root.ts", tags: [ {