diff --git a/.github/workflows/windows-baseline.yml b/.github/workflows/windows-baseline.yml index cd490757b4..924262bd55 100644 --- a/.github/workflows/windows-baseline.yml +++ b/.github/workflows/windows-baseline.yml @@ -181,9 +181,10 @@ jobs: # Full packages/storage test:dist is ~10 minutes on windows-latest and # mostly duplicates the Linux unit lane. Baseline keeps process/path/ - # lock/crash-sensitive gates here; crash-recovery patterns stay in the - # dedicated step below. Bump concurrency carefully — several suites spawn - # child processes and fight for disk under high fan-out. + # lock-sensitive gates here. Release-blocking crash evidence belongs to + # windows-recovery.yml, so this diagnostic lane does not duplicate it. + # Bump concurrency carefully — several suites spawn child processes and + # fight for disk under high fan-out. - id: storage name: Run Windows storage path and lock gates if: always() && steps.build.outcome == 'success' && needs.changes.outputs.windows_storage == 'true' @@ -194,8 +195,6 @@ jobs: $storageGates = @( 'packages/storage/dist/__tests__/root-authority.test.js', 'packages/storage/dist/__tests__/sqlite-recovery-concurrency.test.js', - 'packages/storage/dist/__tests__/sqlite-runtime-crash.test.js', - 'packages/storage/dist/__tests__/sqlite-long-term-memory-crash.test.js', 'packages/storage/dist/__tests__/artifact-writer-lock.test.js', 'packages/storage/dist/__tests__/write-queue.test.js', 'packages/storage/dist/__tests__/marker-file.test.js', @@ -208,18 +207,6 @@ jobs: 2>&1 | Tee-Object -FilePath "$env:WINDOWS_BASELINE_LOG_DIR/storage.log" exit $LASTEXITCODE - - id: managed_workspace_crash - name: Run managed workspace crash recovery gates - if: always() && steps.build.outcome == 'success' && needs.changes.outputs.windows_storage == 'true' - continue-on-error: true - shell: pwsh - run: | - $env:MAKA_STORAGE_STRESS = '1' - node.exe --test --test-concurrency=1 --test-name-pattern="real process crash|real crash|real-process crash|crash after baseline ref publication" packages/storage/dist/__tests__/managed-workspace-baseline.test.js packages/storage/dist/__tests__/git-workspace-service.test.js *> "$env:WINDOWS_BASELINE_LOG_DIR/managed-workspace-crash.log" - $exitCode = $LASTEXITCODE - Get-Content "$env:WINDOWS_BASELINE_LOG_DIR/managed-workspace-crash.log" - exit $exitCode - # Full suite is evidence-only: nightly + manual dispatch. Do not run on # ordinary PRs — Linux unit already covers the package, and the full # Windows wall clock is ~10 minutes. @@ -338,7 +325,6 @@ jobs: | CLI / Electron smoke | ${{ steps.smoke.outcome }} | | Storage path/lock gates | ${{ steps.storage.outcome }} | | Runtime PTY input | ${{ steps.runtime_pty_input.outcome }} | - | Managed workspace crash recovery | ${{ steps.managed_workspace_crash.outcome }} | | Full storage suite (nightly/manual) | ${{ steps.storage_full.outcome }} | | Residual processes | ${{ steps.processes.outcome }} | diff --git a/.github/workflows/windows-recovery.yml b/.github/workflows/windows-recovery.yml new file mode 100644 index 0000000000..1884907028 --- /dev/null +++ b/.github/workflows/windows-recovery.yml @@ -0,0 +1,81 @@ +name: Windows recovery + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +concurrency: + group: windows-recovery-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + windows_recovery: + name: windows_recovery + runs-on: windows-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + cache: npm + + - name: Install dependencies + run: npm.cmd ci + + - name: Build test artifacts + run: npm.cmd run build:test + + - name: Verify SQLite crash recovery + shell: pwsh + run: | + node.exe --test --test-concurrency=1 ` + packages/storage/dist/__tests__/sqlite-runtime-crash.test.js ` + packages/storage/dist/__tests__/sqlite-long-term-memory-crash.test.js + + - name: Verify Runtime continuation recovery + shell: pwsh + run: | + node.exe --test --test-concurrency=1 ` + packages/runtime/dist/__tests__/runtime-resume-crash.test.js ` + packages/runtime/dist/__tests__/runtime-continuation-crash.test.js + + - name: Verify Runtime Host owner-death recovery + shell: pwsh + run: | + node.exe --test --test-reporter=tap --test-concurrency=1 ` + --test-name-pattern="owner death|a killed Host is recovered exactly once" ` + packages/runtime-host/dist/__tests__/artifact-two-client-uds.test.js ` + packages/runtime-host/dist/__tests__/execution-host-queue.test.js ` + 2>&1 | Tee-Object -FilePath "$env:RUNNER_TEMP/runtime-host-recovery.tap" + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { exit $exitCode } + $output = Get-Content "$env:RUNNER_TEMP/runtime-host-recovery.tap" + if ($output -notcontains '# tests 2' -or $output -notcontains '# pass 2' -or $output -notcontains '# skipped 0') { + Write-Error 'Runtime Host recovery gate did not run exactly two passing tests' + exit 1 + } + + - name: Verify managed workspace crash recovery + shell: pwsh + run: | + $env:MAKA_STORAGE_STRESS = '1' + node.exe --test --test-reporter=tap --test-concurrency=1 ` + --test-name-pattern="real process crash|real crash|real-process crash|crash after baseline ref publication" ` + packages/storage/dist/__tests__/managed-workspace-baseline.test.js ` + packages/storage/dist/__tests__/git-workspace-service.test.js ` + 2>&1 | Tee-Object -FilePath "$env:RUNNER_TEMP/managed-workspace-recovery.tap" + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { exit $exitCode } + $output = Get-Content "$env:RUNNER_TEMP/managed-workspace-recovery.tap" + if ($output -notcontains '# tests 12' -or $output -notcontains '# pass 12' -or $output -notcontains '# skipped 0') { + Write-Error 'Managed workspace recovery gate did not run exactly twelve passing tests' + exit 1 + } diff --git a/package.json b/package.json index 374a2d701c..a2add78e72 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "test:dist": "npm run test:scripts:full && node scripts/run-workspace-tests-parallel.mjs --concurrency=3", "test:dist:serial": "npm run test:scripts:full && node scripts/run-workspace-tests-parallel.mjs --serial", "test:fast": "npm run build:test && npm run test:scripts && node scripts/run-workspace-tests-parallel.mjs --concurrency=3", - "test:scripts": "node --test scripts/electron-builder-config.test.mjs scripts/install-electron-with-retry.test.mjs scripts/sync-model-metadata.test.mjs scripts/fixture-env.test.mjs scripts/electron-lifecycle.test.mjs scripts/check-story-annotations.test.mjs scripts/check-dead-css.test.mjs scripts/check-astryx-alignment.test.mjs scripts/check-astryx-surface-inventory.test.mjs scripts/build-astryx-theme.test.mjs scripts/ci-test-plan.test.mjs scripts/run-headless-tests.test.mjs scripts/run-workspace-tests-parallel.test.mjs scripts/storybook-visual-smoke.test.mjs scripts/cu-e2e-scenarios.test.mjs scripts/cu-report-sanitize.test.mjs scripts/cu-trace-analyse.test.mjs scripts/prepare-maka-cu-provenance.test.mjs scripts/prepare-bundled-git.test.mjs scripts/prepare-bundled-git-source.test.mjs scripts/bundled-skill-catalog.test.mjs scripts/windows-test-inventory.test.mjs scripts/windows-smoke.test.mjs scripts/windows-baseline-workflow.test.mjs scripts/code-mode-build-order.test.mjs scripts/cli-build-order.test.mjs scripts/dependency-audit-workflow.test.mjs apps/desktop/scripts/dev-app-runtime.test.mjs", + "test:scripts": "node --test scripts/electron-builder-config.test.mjs scripts/install-electron-with-retry.test.mjs scripts/sync-model-metadata.test.mjs scripts/fixture-env.test.mjs scripts/electron-lifecycle.test.mjs scripts/check-story-annotations.test.mjs scripts/check-dead-css.test.mjs scripts/check-astryx-alignment.test.mjs scripts/check-astryx-surface-inventory.test.mjs scripts/build-astryx-theme.test.mjs scripts/ci-test-plan.test.mjs scripts/run-headless-tests.test.mjs scripts/run-workspace-tests-parallel.test.mjs scripts/storybook-visual-smoke.test.mjs scripts/cu-e2e-scenarios.test.mjs scripts/cu-report-sanitize.test.mjs scripts/cu-trace-analyse.test.mjs scripts/prepare-maka-cu-provenance.test.mjs scripts/prepare-bundled-git.test.mjs scripts/prepare-bundled-git-source.test.mjs scripts/bundled-skill-catalog.test.mjs scripts/windows-test-inventory.test.mjs scripts/windows-smoke.test.mjs scripts/windows-baseline-workflow.test.mjs scripts/windows-recovery-workflow.test.mjs scripts/code-mode-build-order.test.mjs scripts/cli-build-order.test.mjs scripts/dependency-audit-workflow.test.mjs apps/desktop/scripts/dev-app-runtime.test.mjs", "test:scripts:extended": "node --test scripts/cu-provider-matrix.test.mjs scripts/cu-process-restart-harness.test.mjs scripts/cu-real-model-launcher.test.mjs scripts/macos-arm64-release.test.mjs scripts/windows-x64-release.test.mjs scripts/measure-session-bundle.test.mjs", "test:scripts:full": "npm run test:scripts && npm run test:scripts:extended", "dev": "npm --workspace @maka/desktop run dev:hmr --", diff --git a/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts b/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts index 96a2929a0c..aa71a4939d 100644 --- a/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts +++ b/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts @@ -17,6 +17,7 @@ import { FakeBackend } from '../fake-backend.js'; import { terminateChildProcessTree } from '../process-tree-terminator.js'; const CRASH_CHILD_ENV = 'MAKA_RUNTIME_CONTINUATION_CRASH_CHILD'; +const CRASH_HARNESS_TIMEOUT_MS = process.platform === 'win32' ? 120_000 : 60_000; const FAILPOINTS: readonly RuntimeContinuationFailpoint[] = [ 'after_continuation_claim_committed', 'after_run_created', @@ -30,7 +31,7 @@ if (process.env[CRASH_CHILD_ENV] === '1') { } else { describe('runtime resume phase 1 process crash harness', () => { test('reopens and repairs every committed continuation prefix after SIGKILL', { - timeout: 60_000, + timeout: CRASH_HARNESS_TIMEOUT_MS, }, async () => { const root = await mkdtemp(join(tmpdir(), 'maka-runtime-continuation-crash-')); try { diff --git a/packages/runtime/src/__tests__/runtime-resume-crash.test.ts b/packages/runtime/src/__tests__/runtime-resume-crash.test.ts index cd5b30a0d6..020d797ffb 100644 --- a/packages/runtime/src/__tests__/runtime-resume-crash.test.ts +++ b/packages/runtime/src/__tests__/runtime-resume-crash.test.ts @@ -18,13 +18,14 @@ import { import { terminateChildProcessTree } from '../process-tree-terminator.js'; const CRASH_CHILD_ENV = 'MAKA_RUNTIME_RESUME_CRASH_CHILD'; +const CRASH_HARNESS_TIMEOUT_MS = process.platform === 'win32' ? 120_000 : 60_000; if (process.env[CRASH_CHILD_ENV] === '1') { await runCrashChild(); } else { describe('runtime resume phase 0 process crash harness', () => { test('reopens every fully committed P0-P11 ledger prefix after SIGKILL', { - timeout: 60_000, + timeout: CRASH_HARNESS_TIMEOUT_MS, }, async () => { const root = await mkdtemp(join(tmpdir(), 'maka-runtime-resume-crash-')); try { diff --git a/scripts/windows-baseline-workflow.test.mjs b/scripts/windows-baseline-workflow.test.mjs index 4c0f1343ae..018895ad6b 100644 --- a/scripts/windows-baseline-workflow.test.mjs +++ b/scripts/windows-baseline-workflow.test.mjs @@ -79,7 +79,6 @@ test('Windows baseline workflow keeps its non-blocking evidence contract', async 'smoke', 'runtime_pty_input', 'storage', - 'managed_workspace_crash', 'storage_full', 'processes', ]); @@ -100,14 +99,13 @@ test('Windows baseline workflow keeps its non-blocking evidence contract', async 'packages/storage/dist/__tests__/sqlite-recovery-concurrency.test.js', 'packages/storage/dist/__tests__/managed-workspace-owner.test.js', 'node.exe --test --test-force-exit --test-timeout=15000 --test-reporter=tap --test-concurrency=1 --test-name-pattern="semantic text and Enter actions|terminal mode parsed before the control cut" packages/runtime/dist/__tests__/shell-run-manager.test.js', - 'node.exe --test --test-concurrency=1 --test-name-pattern="real process crash|real crash|real-process crash|crash after baseline ref publication" packages/storage/dist/__tests__/managed-workspace-baseline.test.js packages/storage/dist/__tests__/git-workspace-service.test.js', // Full suite only via storage_full (nightly / manual), not the PR storage step. 'node.exe scripts/run-workspace-tests-parallel.mjs --concurrency=1 --workspaces=packages/storage', ]) { assert.ok(workflow.includes(command), command); } assert.match(workflow, /needs\.changes\.outputs\.windows_storage_full == 'true'/u); - assert.equal(workflow.match(/needs\.changes\.outputs\.windows_storage == 'true'/gu)?.length, 2); + assert.equal(workflow.match(/needs\.changes\.outputs\.windows_storage == 'true'/gu)?.length, 1); assert.equal(workflow.match(/needs\.changes\.outputs\.windows_runtime == 'true'/gu)?.length, 1); assert.match(workflow, /Runtime PTY input gate did not run exactly two passing tests/u); assert.match(workflow, /name: Capture full storage suite baseline/u); @@ -130,8 +128,14 @@ test('Windows baseline workflow keeps its non-blocking evidence contract', async assert.match(workflow, /\$exitCode = \$LASTEXITCODE/u); assert.match(workflow, /Tee-Object -FilePath "\$env:WINDOWS_BASELINE_LOG_DIR\/storage\.log"/u); assert.match(workflow, /name: Run Windows storage path and lock gates/u); - assert.match(workflow, /\$env:MAKA_STORAGE_STRESS = '1'/u); - assert.match(workflow, /managed-workspace-crash\.log/u); + for (const blockingRecoveryArtifact of [ + 'sqlite-runtime-crash.test.js', + 'sqlite-long-term-memory-crash.test.js', + 'managed-workspace-baseline.test.js', + 'git-workspace-service.test.js', + ]) { + assert.ok(!workflow.includes(blockingRecoveryArtifact), blockingRecoveryArtifact); + } // Pin-short-comment contract: the workflow must pin upload-artifact to a // full SHA and annotate it with the exact version it resolves to. The major // is intentionally not asserted — dependabot may bump it — but the diff --git a/scripts/windows-recovery-workflow.test.mjs b/scripts/windows-recovery-workflow.test.mjs new file mode 100644 index 0000000000..11035d4bf3 --- /dev/null +++ b/scripts/windows-recovery-workflow.test.mjs @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import test from 'node:test'; + +const workflowUrl = new URL('../.github/workflows/windows-recovery.yml', import.meta.url); + +test('Windows recovery workflow is a bounded release-blocking evidence gate', async () => { + const workflow = await readFile(workflowUrl, 'utf8'); + + assert.match(workflow, /^\s+windows_recovery:$/mu); + assert.match(workflow, /^\s+name: windows_recovery$/mu); + assert.match(workflow, /^\s+runs-on: windows-latest$/mu); + assert.match(workflow, /^\s+timeout-minutes: 30$/mu); + assert.doesNotMatch(workflow, /continue-on-error/u); + + for (const command of ['npm.cmd ci', 'npm.cmd run build:test']) { + assert.ok(workflow.includes(command), command); + } + + for (const testArtifact of [ + 'packages/storage/dist/__tests__/sqlite-runtime-crash.test.js', + 'packages/storage/dist/__tests__/sqlite-long-term-memory-crash.test.js', + 'packages/runtime/dist/__tests__/runtime-resume-crash.test.js', + 'packages/runtime/dist/__tests__/runtime-continuation-crash.test.js', + 'packages/runtime-host/dist/__tests__/artifact-two-client-uds.test.js', + 'packages/runtime-host/dist/__tests__/execution-host-queue.test.js', + 'packages/storage/dist/__tests__/managed-workspace-baseline.test.js', + 'packages/storage/dist/__tests__/git-workspace-service.test.js', + ]) { + assert.ok(workflow.includes(testArtifact), testArtifact); + } + + assert.match(workflow, /\$env:MAKA_STORAGE_STRESS = '1'/u); + assert.match(workflow, /owner death\|a killed Host is recovered exactly once/u); + assert.match(workflow, /real process crash\|real crash\|real-process crash/u); + assert.equal(workflow.match(/--test-reporter=tap/gu)?.length, 2); + for (const evidence of [ + '# tests 2', + '# pass 2', + '# tests 12', + '# pass 12', + '# skipped 0', + 'Runtime Host recovery gate did not run exactly two passing tests', + 'Managed workspace recovery gate did not run exactly twelve passing tests', + ]) { + assert.ok(workflow.includes(evidence), evidence); + } +});