test: wait for a timed-out body before cleanup, and retry removal where Windows retries

apps/runtime/test/package-lifecycle-route.spec.ts

@@ -14,6 +14,7 @@ import {
 } from "@clarkcant/core";
 import { allRows } from "@clarkcant/storage";
 
+import { removeTestDirectory } from "../../../tools/test-cleanup.ts";
 import { handleRequest, type GatewayDeps, type GatewayResponse } from "../src/gateway.ts";
 import { createManagePackageTool } from "../src/manage-package-tool.ts";
 import { bootNodeServices, type NodeServices } from "../src/services.ts";
@@ -166,11 +167,12 @@ beforeEach(() => {
   process.env["CC_DIRECTORY_INDEX"] = indexPath;
 });
 
-afterEach(() => {
+afterEach(async () => {
   if (previousIndex === undefined) delete process.env["CC_DIRECTORY_INDEX"];
   else process.env["CC_DIRECTORY_INDEX"] = previousIndex;
   services.runtime.close();
-  rmSync(dir, { recursive: true, force: true });
+  // The database's files were just closed, and Windows can hold one a moment longer.
+  await removeTestDirectory(dir);
 });
 
 describe("uninstalling and restoring a package", () => {

apps/runtime/test/session-preview-real.spec.ts

@@ -1,14 +1,15 @@
 import { createServer, type Server } from "node:http";
 import { createHash } from "node:crypto";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync } from "node:fs";
 import { type AddressInfo } from "node:net";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 
 import { type AutomationAction, observationIdSchema } from "@clarkcant/contracts";
 import { createDriver, type BrowserDriver } from "@clarkcant/browser-playwright";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect } from "vitest";
 
+import { removeTestDirectory, trackedTests } from "../../../tools/test-cleanup.ts";
 import { blobPathForDigest, readBlob } from "../src/blobs.ts";
 import { createNodeServer } from "../src/server.ts";
 import { captureSessionPreview } from "../src/session-preview.ts";
@@ -41,6 +42,13 @@ const PAGE_TWO = `<!doctype html>
   <button>Hoàn tất</button>
 </body></html>`;
 
+/**
+ * Each test starts and stops a Chromium, one of them twice. Closing one takes under 200 ms alone, but up to 12 s while
+ * the full suite runs on a loaded Windows machine, and tests that take under 0.6 s alone took up to 14 s there. The
+ * budget covers two such browsers with room to spare.
+ */
+const { it, settled } = trackedTests(60_000);
+
 let server: Server;
 let origin: string;
 /** Where the browsers under test keep their profiles. One directory, removed whole when the file is done. */
@@ -75,6 +83,7 @@ beforeAll(async () => {
 });
 
 afterAll(async () => {
+  await settled();
   await closeServer();
   services.runtime.close();
   await new Promise<void>((resolve) => server.close(() => resolve()));
@@ -83,8 +92,8 @@ afterAll(async () => {
    * not for every helper process Chromium started beside it, and removing the directory then fails with EPERM. Retry
    * as the other suites that remove a directory a process just let go of do.
    */
-  rmSync(profiles, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 });
-  rmSync(dataDir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });
+  await removeTestDirectory(profiles);
+  await removeTestDirectory(dataDir);
 });
 
 /**

apps/runtime/test/worktree-sweep.spec.ts

@@ -1,14 +1,15 @@
 import { execFileSync } from "node:child_process";
-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { afterEach, beforeEach, describe, expect } from "vitest";
 
 import type { Instant } from "@clarkcant/contracts";
 import { applyTaskEvent, createTask } from "@clarkcant/core";
 import { ensureManagedWorktree } from "@clarkcant/project-work";
 import { allRows, createConversation, listNotifications } from "@clarkcant/storage";
 
+import { removeTestDirectory, trackedTests } from "../../../tools/test-cleanup.ts";
 import { bootNodeServices, type NodeServices } from "../src/services.ts";
 import { sweepTaskWorktrees } from "../src/worktree-sweep.ts";
 
@@ -21,6 +22,13 @@ import { sweepTaskWorktrees } from "../src/worktree-sweep.ts";
 const AT = "2026-09-29T09:00:00.000Z" as Instant;
 const CONVERSATION_ID = "conv_sweep";
 
+/**
+ * Each test spawns git dozens of times: repositories, worktrees, and a sweep that asks git about each of them. A spawn
+ * takes about 40 ms alone, but 150 to 350 ms (p99 near 2 s) while the full suite runs on a loaded Windows machine, and a
+ * test that takes 3 s alone took up to 21 s there. The budget is that worst case with room to spare.
+ */
+const { it, settled } = trackedTests(60_000);
+
 let dir: string;
 let services: NodeServices;
 let repo: string;
@@ -66,9 +74,10 @@ beforeEach(() => {
   repo = makeRepo(join(dir, "repo"));
 });
 
-afterEach(() => {
+afterEach(async () => {
+  await settled();
   services.runtime.close();
-  rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });
+  await removeTestDirectory(dir);
 });
 
 describe("the worktrees a stopped node left behind", () => {

packs/browser-playwright/test/click-answer.spec.ts

@@ -1,11 +1,12 @@
 import { createServer, type Server } from "node:http";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect } from "vitest";
 
 import { observationIdSchema, type AutomationAction } from "@clarkcant/contracts";
 
+import { removeTestDirectory, trackedTests } from "../../../tools/test-cleanup.ts";
 import { createDriver, type BrowserDriver, type ObserveResult } from "../src/driver.ts";
 
 /**
@@ -19,6 +20,9 @@ import { createDriver, type BrowserDriver, type ObserveResult } from "../src/dri
 
 const ANSWER_TIMEOUT_MS = 1500;
 
+/** Each browser test already carries its own 60 s budget. */
+const { it, settled } = trackedTests(60_000);
+
 function page(body: string): string {
   return `<!doctype html><html><head><title>Fixture</title></head><body>${body}</body></html>`;
 }
@@ -118,11 +122,12 @@ beforeAll(async () => {
 });
 
 afterAll(async () => {
+  await settled();
   server.closeAllConnections();
   other.closeAllConnections();
   await new Promise<void>((resolve) => server.close(() => resolve()));
   await new Promise<void>((resolve) => other.close(() => resolve()));
-  rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 });
+  await removeTestDirectory(dir);
 });
 
 function action(driver: BrowserDriver, observed: ObserveResult, overrides: Partial<AutomationAction>): AutomationAction {

packs/browser-playwright/test/driver.spec.ts

@@ -1,11 +1,12 @@
 import { createServer, type Server } from "node:http";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect } from "vitest";
 
 import { observationIdSchema, type AutomationAction, type Observation } from "@clarkcant/contracts";
 
+import { removeTestDirectory, trackedTests } from "../../../tools/test-cleanup.ts";
 import { createDriver } from "../src/driver.ts";
 import type { BrowserDriver } from "../src/driver.ts";
 
@@ -27,6 +28,13 @@ const PAGE = `<!doctype html>
   <input id="pw" name="password" type="password" aria-label="Password" />
 </body></html>`;
 
+/**
+ * Every test starts and stops a Chromium. Closing one takes under 200 ms alone, but up to 12 s while the full suite runs
+ * on a loaded Windows machine, and tests that take under 1 s alone took up to 10 s there. The budget covers that with
+ * room to spare, as `click-answer.spec.ts` already gives its browser tests.
+ */
+const { it, settled } = trackedTests(60_000);
+
 let server: Server;
 let origin: string;
 let dir: string;
@@ -44,9 +52,10 @@ beforeAll(async () => {
 });
 
 afterAll(async () => {
+  await settled();
   await new Promise<void>((resolve) => server.close(() => resolve()));
   // Chromium has closed, but Windows may briefly retain a lock on a profile file.
-  rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 });
+  await removeTestDirectory(dir);
 });
 
 /**

packs/browser-playwright/test/injection.spec.ts

@@ -1,11 +1,12 @@
 import { createServer, type Server } from "node:http";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect } from "vitest";
 
 import { observationIdSchema, type AutomationAction, type Observation } from "@clarkcant/contracts";
 
+import { removeTestDirectory, trackedTests } from "../../../tools/test-cleanup.ts";
 import { createDriver } from "../src/driver.ts";
 import type { BrowserDriver } from "../src/driver.ts";
 
@@ -40,6 +41,13 @@ const PAGE = `<!doctype html>
   <a id="evil" href="https://attacker.example/exfiltrate">Open the report</a>
 </body></html>`;
 
+/**
+ * Every test starts and stops a Chromium. Closing one takes under 200 ms alone, but up to 12 s while the full suite runs
+ * on a loaded Windows machine, and tests that take under 1 s alone took up to 6 s there. The budget covers that with
+ * room to spare, as `click-answer.spec.ts` already gives its browser tests.
+ */
+const { it, settled } = trackedTests(60_000);
+
 let server: Server;
 let origin: string;
 let dir: string;
@@ -57,8 +65,9 @@ beforeAll(async () => {
 });
 
 afterAll(async () => {
+  await settled();
   await new Promise<void>((resolve) => server.close(() => resolve()));
-  rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 });
+  await removeTestDirectory(dir);
 });
 
 type ActionOverrides = Omit<Partial<AutomationAction>, "observationId"> & { observationId: string };

packs/browser-playwright/test/submit-once.spec.ts

@@ -1,11 +1,12 @@
 import { createServer, type Server } from "node:http";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect } from "vitest";
 
 import { observationIdSchema, type AutomationAction, type Observation } from "@clarkcant/contracts";
 
+import { removeTestDirectory, trackedTests } from "../../../tools/test-cleanup.ts";
 import { createDriver } from "../src/driver.ts";
 import type { BrowserDriver } from "../src/driver.ts";
 
@@ -43,6 +44,9 @@ const PAGE = `<!doctype html>
   </script>
 </body></html>`;
 
+/** Each test already carries its own 90 s budget: the page blocks for `BLOCK_MS` on top of a browser's start and stop. */
+const { it, settled } = trackedTests(90_000);
+
 let server: Server;
 let origin: string;
 let dir: string;
@@ -67,9 +71,10 @@ beforeAll(async () => {
 });
 
 afterAll(async () => {
+  await settled();
   await new Promise<void>((resolve) => server.close(() => resolve()));
   // Chromium has closed, but Windows may briefly retain a lock on a profile file.
-  rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 });
+  await removeTestDirectory(dir);
 });
 
 type ActionOverrides = Omit<Partial<AutomationAction>, "observationId"> & { observationId: string };

packs/project-work/test/managed-worktree.spec.ts

@@ -1,9 +1,10 @@
 import { execFileSync } from "node:child_process";
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
-import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { afterEach, beforeEach, describe, expect } from "vitest";
 
+import { removeTestDirectory, trackedTests } from "../../../tools/test-cleanup.ts";
 import {
   ensureManagedWorktree,
   findManagedWorktrees,
@@ -19,6 +20,13 @@ import {
  * the person's tree, and a model of git would only agree with itself.
  */
 
+/**
+ * Each test spawns git many times, through the helper below and through the module under test. A spawn takes about
+ * 40 ms alone, but 150 to 350 ms (p99 near 2 s) while the full suite runs on a loaded Windows machine, and a test that
+ * takes 3.5 s alone took up to 19 s there. The budget is that worst case with room to spare.
+ */
+const { it, settled } = trackedTests(60_000);
+
 let dirs: string[] = [];
 
 function git(cwd: string, args: string[]): string {
@@ -49,8 +57,9 @@ beforeEach(() => {
   dirs = [];
 });
 
-afterEach(() => {
-  for (const path of dirs) rmSync(path, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 });
+afterEach(async () => {
+  await settled();
+  for (const path of dirs) await removeTestDirectory(path);
 });
 
 describe("a task works in its own worktree, never in the person's tree", () => {

tools/test-cleanup.ts

@@ -0,0 +1,57 @@
+/**
+ * Cleanup for suites whose tests drive real processes, such as git or Chromium, inside a temporary directory.
+ *
+ * Two things go wrong on Windows when many of these suites run at once, and each helper here answers one.
+ */
+
+import { rm } from "node:fs/promises";
+import { it } from "vitest";
+
+/**
+ * Remove a test's temporary directory, retrying while Windows still holds a file in it.
+ *
+ * A process that just exited can keep a handle open for a moment: a closed Chromium's helper processes, or a git that
+ * had the directory as its working directory. Removal then fails with `EPERM` until the handle is gone.
+ *
+ * This is the promise form on purpose. Under Node 24 on Windows, `rmSync` reports a locked file as `EPERM` at once and
+ * ignores `maxRetries`, so a synchronous removal with retries is a removal without them. `fs.promises.rm` waits
+ * `retryDelay` longer after each failed attempt, about five seconds over ten attempts.
+ */
+export async function removeTestDirectory(path: string): Promise<void> {
+  await rm(path, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 });
+}
+
+/**
+ * `it`, with a time budget for the whole file and a way for cleanup to wait for a test that is still running.
+ *
+ * When a test runs out of time, Vitest reports it and runs the cleanup hooks, but the test's body keeps running: a git
+ * command or a browser it started is still working in the directory the hooks are about to remove. `settled()` waits
+ * until every body this `it` started has finished, so a hook that awaits it first never removes a directory from under
+ * a running test, or closes a database one is still writing to.
+ */
+export function trackedTests(budgetMs: number): {
+  it: (name: string, body: () => unknown, timeoutMs?: number) => void;
+  settled: () => Promise<void>;
+} {
+  const running = new Set<Promise<unknown>>();
+  return {
+    it(name, body, timeoutMs = budgetMs) {
+      it(
+        name,
+        async () => {
+          const run = Promise.resolve().then(body);
+          running.add(run);
+          try {
+            await run;
+          } finally {
+            running.delete(run);
+          }
+        },
+        timeoutMs,
+      );
+    },
+    async settled() {
+      await Promise.allSettled([...running]);
+    },
+  };
+}