Fully test the isolated mutation run state

scripts/mutation/isolation-state.ts

@@ -13,7 +13,7 @@ import {
   resolve,
   SEPARATOR,
 } from "@std/path";
-import { openLockFile } from "#scripts/lock-file.ts";
+import { openLockFile, withFileLock } from "#scripts/lock-file.ts";
 import { rethrowUnlessNotFound } from "#scripts/not-found.ts";
 import { projectRoot } from "#scripts/project-root.ts";
 import { denoExitCode } from "./child-process.ts";
@@ -44,7 +44,6 @@ const SKIPPED_TOP_LEVEL_NAMES = new Set([
   "cov",
   "cov_profile",
   "dist",
-  "docs-output",
   "misc",
   "node_modules",
   "undefined",
@@ -65,7 +64,6 @@ const SKIPPED_FILE_NAMES = new Set([
   ".test-junit.xml",
   "bunny-script.ts",
   "bunny-script.ts.map",
-  "tickets.db",
 ]);
 
 export type MutationRunStatus =
@@ -157,19 +155,16 @@ const copyDirectory = async (
     if (entry.isDirectory) {
       await copyDirectory(fromRoot, toRoot, childPath);
     } else {
-      await Deno.mkdir(dirname(to), { recursive: true });
       await Deno.copyFile(from, to);
     }
   }
 };
 
-export const copyMutationSnapshot = async (
+/** Copy a checkout into a snapshot, leaving out everything a run must not carry. */
+export const copyMutationSnapshot = (
   fromRoot: string,
   toRoot: string,
-): Promise<void> => {
-  await Deno.mkdir(toRoot, { recursive: true });
-  await copyDirectory(fromRoot, toRoot);
-};
+): Promise<void> => copyDirectory(fromRoot, toRoot);
 
 const compactIso = (iso: string): string =>
   iso
@@ -452,19 +447,13 @@ export const runLockIsHeld = async (
   return (await lockProbeExitCode(path, timeoutMs)) === LOCK_HELD_EXIT_CODE;
 };
 
+/** Hold a run's own lock, making its folder first so there is one to lock. */
 export const withMutationRunLock = async <Result>(
   runRootPath: string,
   run: () => Promise<Result>,
 ): Promise<Result> => {
   await Deno.mkdir(runRootPath, { recursive: true });
-  const file = await openLockFile(join(runRootPath, MUTATION_RUN_LOCK_FILE));
-  try {
-    await file.lock(true);
-    return await run();
-  } finally {
-    await file.unlock();
-    file.close();
-  }
+  return await withFileLock(join(runRootPath, MUTATION_RUN_LOCK_FILE), run);
 };
 
 export const selectedRuns = (

test/scripts/mutation/isolation-state/commands.test.ts

@@ -18,7 +18,9 @@ describe("mutation isolation command parsing and listing", () => {
       message: "Mutation source and test globs are required.",
     });
     expect(parseIsolationCommand(["--help"])).toEqual({ kind: "help" });
+    expect(parseIsolationCommand(["-h"])).toEqual({ kind: "help" });
     expect(parseIsolationCommand(["--list"])).toEqual({ kind: "list" });
+    expect(parseIsolationCommand(["list"])).toEqual({ kind: "list" });
     expect(parseIsolationCommand(["kill", "run-1", "--force"])).toEqual({
       force: true,
       kind: "kill",
@@ -32,6 +34,10 @@ describe("mutation isolation command parsing and listing", () => {
       kind: "clean",
       target: "finished",
     });
+    expect(parseIsolationCommand(["--clean", "all"])).toEqual({
+      kind: "clean",
+      target: "all",
+    });
     expect(parseIsolationCommand(["clean"])).toEqual({
       kind: "invalid",
       message: "A run id, all, or finished is required for --clean.",
@@ -73,6 +79,14 @@ describe("mutation isolation command parsing and listing", () => {
     ).toEqual(["mutation-passed", "mutation-failed", "mutation-interrupted"]);
     expect(selectedRuns(records, "mutation-running")).toEqual([running]);
     expect(selectedRuns(records, "mutation-runn")).toEqual([running]);
+    // A whole id wins outright, even when it also starts another run's id.
+    const longer = markRunning(
+      newRunRecord("mutation-running-again", ["src/f.ts"], "/repo"),
+      12,
+    );
+    expect(selectedRuns([...records, longer], "mutation-running")).toEqual([
+      running,
+    ]);
     expect(selectedRuns(records, "mutation-")).toEqual([]);
     expect(selectedRuns(records, "missing")).toEqual([]);
   });
@@ -92,6 +106,16 @@ describe("mutation isolation command parsing and listing", () => {
     ).toEqual([
       "mutation-running running pid=10 exit=- work=.mutation-runs/mutation-running/work args=src/a.ts",
     ]);
+    // Several arguments are spaced out the way they were typed.
+    const twoArgs = markRunning(
+      newRunRecord("mutation-two", ["src/a.ts", "test/a.test.ts"], "/repo"),
+      11,
+    );
+    expect(
+      formatRunList([twoArgs], new Set(["mutation-two"]), "/repo"),
+    ).toEqual([
+      "mutation-two running pid=11 exit=- work=.mutation-runs/mutation-two/work args=src/a.ts test/a.test.ts",
+    ]);
     expect(formatRunList([running], new Set(), "/repo")).toEqual([
       "mutation-running stale pid=10 exit=- work=.mutation-runs/mutation-running/work args=src/a.ts",
     ]);

test/scripts/mutation/isolation-state/env.test.ts

@@ -0,0 +1,38 @@
+import { expect } from "@std/expect";
+import { describe, it as test } from "@std/testing/bdd";
+import {
+  MUTATION_RUN_ID_ENV,
+  MUTATION_RUN_ROOT_ENV,
+  MUTATION_SNAPSHOT_CHILD_ENV,
+  MUTATION_WORK_ROOT_ENV,
+} from "#scripts/mutation/isolation-state.ts";
+
+/** How a run tells its child which snapshot it is working in. */
+const HANDOVER_NAMES = [
+  MUTATION_SNAPSHOT_CHILD_ENV,
+  MUTATION_RUN_ID_ENV,
+  MUTATION_RUN_ROOT_ENV,
+  MUTATION_WORK_ROOT_ENV,
+];
+
+describe("what a run tells its child", () => {
+  test("uses a separate name for each thing it passes down", () => {
+    expect(new Set(HANDOVER_NAMES).size).toBe(HANDOVER_NAMES.length);
+  });
+
+  for (const name of HANDOVER_NAMES) {
+    test(`can put ${name || "an unnamed value"} in an environment`, () => {
+      // A nameless or "=" bearing variable cannot be set at all, so the child
+      // would be started without it and quietly behave as if it were not in a
+      // snapshot. Deno is the judge here, not a rule written out again.
+      expect(() => Deno.env.set(name, "1")).not.toThrow();
+      Deno.env.delete(name);
+    });
+  }
+
+  test("keeps them under our own prefix, away from anyone else's", () => {
+    for (const name of HANDOVER_NAMES) {
+      expect(name.startsWith("TICKETS_MUTATION_")).toBe(true);
+    }
+  });
+});

test/scripts/mutation/isolation-state/lock.test.ts

@@ -0,0 +1,109 @@
+import { join } from "node:path";
+import { expect } from "@std/expect";
+import { describe, it as test } from "@std/testing/bdd";
+import {
+  MUTATION_RUN_LOCK_FILE,
+  runLockIsHeld,
+  withMutationRunLock,
+} from "#scripts/mutation/isolation-state.ts";
+import { withTempDir } from "#test/scripts/mutation/isolation-helpers.ts";
+import { pathExists } from "#test-utils/files.ts";
+
+/**
+ * Long enough for the operating system to hand over a lock it was willing to
+ * hand over. Without this pause a lock that excludes nobody still looks like
+ * it is working, because the second holder has not been let in yet either.
+ */
+const LONG_ENOUGH_TO_BE_LET_IN_MS = 30;
+
+const pause = (milliseconds: number): Promise<void> =>
+  new Promise((resolve) => setTimeout(resolve, milliseconds));
+
+describe("the lock that keeps two runs out of one folder", () => {
+  test("makes the run folder it is asked to lock", async () => {
+    await withTempDir(async (root) => {
+      const runFolder = join(root, ".mutation-runs", "mutation-new");
+
+      expect(
+        await withMutationRunLock(runFolder, () => Promise.resolve(7)),
+      ).toBe(7);
+
+      expect(await pathExists(join(runFolder, MUTATION_RUN_LOCK_FILE))).toBe(
+        true,
+      );
+    });
+  });
+
+  test("keeps a second run out until the first one is done", async () => {
+    await withTempDir(async (root) => {
+      const runFolder = join(root, ".mutation-runs", "mutation-shared");
+      const order: string[] = [];
+      const firstInside = Promise.withResolvers<void>();
+      const releaseFirst = Promise.withResolvers<void>();
+
+      const first = withMutationRunLock(runFolder, async () => {
+        order.push("first in");
+        firstInside.resolve();
+        await releaseFirst.promise;
+        order.push("first out");
+      });
+      await firstInside.promise;
+
+      const second = withMutationRunLock(runFolder, () => {
+        order.push("second in");
+        return Promise.resolve();
+      });
+      await pause(LONG_ENOUGH_TO_BE_LET_IN_MS);
+
+      expect(order).toEqual(["first in"]);
+
+      releaseFirst.resolve();
+      await Promise.all([first, second]);
+
+      expect(order).toEqual(["first in", "first out", "second in"]);
+    });
+  });
+
+  test("hands the lock back when the work inside it fails", async () => {
+    await withTempDir(async (root) => {
+      const runFolder = join(root, ".mutation-runs", "mutation-failing");
+
+      await expect(
+        withMutationRunLock(runFolder, () => Promise.reject(new Error("boom"))),
+      ).rejects.toThrow("boom");
+
+      // A run that could not take the lock back would hang here instead.
+      expect(
+        await withMutationRunLock(runFolder, () => Promise.resolve("ok")),
+      ).toBe("ok");
+    });
+  });
+
+  test("reports a run nobody is holding as not held", async () => {
+    await withTempDir(async (root) => {
+      const record = { root: join(root, ".mutation-runs", "mutation-idle") };
+      await Deno.mkdir(record.root, { recursive: true });
+
+      // No timeout given, so this also checks the standard one is long enough
+      // to tell "free" from "held" rather than calling everything held.
+      expect(await runLockIsHeld(record)).toBe(false);
+    });
+  });
+
+  test("reports a run somebody is holding as held", async () => {
+    await withTempDir(async (root) => {
+      const record = { root: join(root, ".mutation-runs", "mutation-busy") };
+      const asked = Promise.withResolvers<boolean>();
+      const release = Promise.withResolvers<void>();
+
+      const holding = withMutationRunLock(record.root, async () => {
+        asked.resolve(await runLockIsHeld(record));
+        await release.promise;
+      });
+
+      expect(await asked.promise).toBe(true);
+      release.resolve();
+      await holding;
+    });
+  });
+});

test/scripts/mutation/isolation-state/paths.test.ts

@@ -10,6 +10,21 @@ import { withTempDir } from "#test/scripts/mutation/isolation-helpers.ts";
 import { pathExists } from "#test-utils/files.ts";
 
 describe("mutation isolation paths", () => {
+  test("copies into a snapshot folder that is already there", async () => {
+    await withTempDir(async (dir) => {
+      const source = join(dir, "source");
+      const snapshot = join(dir, "snapshot");
+      await Deno.mkdir(source, { recursive: true });
+      await Deno.writeTextFile(join(source, "kept.ts"), "export {};\n");
+      // The run's lock makes this folder before the copy starts.
+      await Deno.mkdir(snapshot, { recursive: true });
+
+      await copyMutationSnapshot(source, snapshot);
+
+      expect(await pathExists(join(snapshot, "kept.ts"))).toBe(true);
+    });
+  });
+
   test("copies source-like files and skips git, reports, secrets, dbs, and generated assets", async () => {
     await withTempDir(async (dir) => {
       const source = join(dir, "source");

test/scripts/mutation/isolation-state/records.test.ts

@@ -69,6 +69,38 @@ describe("mutation isolation run records", () => {
     expect(runStartedRecently(stale, now)).toBe(false);
   });
 
+  test("treats a record whose time makes no sense as not started recently", () => {
+    const broken = markRunning(
+      newRunRecord("broken", [], "/repo", "not a time"),
+      3,
+      "not a time",
+    );
+
+    expect(
+      runStartedRecently(broken, new Date("2026-07-10T12:00:00.000Z")),
+    ).toBe(false);
+  });
+
+  test("counts a run stamped a moment after the epoch", () => {
+    const justAfterEpoch = markRunning(
+      newRunRecord("epoch", [], "/repo", "1970-01-01T00:00:00.001Z"),
+      4,
+      "1970-01-01T00:00:00.001Z",
+    );
+
+    expect(runStartedRecently(justAfterEpoch, new Date(1), 1000)).toBe(true);
+  });
+
+  test("gives each run a plain id with an eight-letter tail", () => {
+    const id = createRunId();
+    const tail = id.split("-").at(-1);
+
+    // Two runs started in the same second must still land in their own folder.
+    expect(tail).toHaveLength(8);
+    expect(id.startsWith("mutation-")).toBe(true);
+    expect(createRunId()).not.toBe(id);
+  });
+
   test("writes, reads, sorts, and ignores broken records", async () => {
     await withTempDir(async (root) => {
       expect(await readRunRecords(root)).toEqual([]);
@@ -92,6 +124,21 @@ describe("mutation isolation run records", () => {
     });
   });
 
+  test("writes the record so a person can read it", async () => {
+    await withTempDir(async (root) => {
+      const record = newRunRecord("readable", ["src/a.ts"], root);
+      await writeRunRecord(record);
+
+      const written = await Deno.readTextFile(
+        join(root, ".mutation-runs", "readable", "run.json"),
+      );
+
+      // Indented and one line per field, so `cat run.json` is worth doing.
+      expect(written.split("\n")[1]).toBe('  "args": [');
+      expect(written.endsWith("}\n")).toBe(true);
+    });
+  });
+
   test("reads records from the current run directory", async () => {
     await withTempDir(async (root) => {
       const { id, record } = await writeMovedRunRecord(root);