diff --git a/scripts/mutation/isolation-state.ts b/scripts/mutation/isolation-state.ts index d39fc73627..73dd2359ea 100644 --- a/scripts/mutation/isolation-state.ts +++ b/scripts/mutation/isolation-state.ts @@ -13,7 +13,7 @@ import { resolve, SEPARATOR, } from "@std/path"; -import { openLockFile } from "#scripts/lock-file.ts"; +import { openLockFile, withFileLock } from "#scripts/lock-file.ts"; import { rethrowUnlessNotFound } from "#scripts/not-found.ts"; import { projectRoot } from "#scripts/project-root.ts"; import { denoExitCode } from "./child-process.ts"; @@ -44,7 +44,6 @@ const SKIPPED_TOP_LEVEL_NAMES = new Set([ "cov", "cov_profile", "dist", - "docs-output", "misc", "node_modules", "undefined", @@ -65,7 +64,6 @@ const SKIPPED_FILE_NAMES = new Set([ ".test-junit.xml", "bunny-script.ts", "bunny-script.ts.map", - "tickets.db", ]); export type MutationRunStatus = @@ -157,19 +155,16 @@ const copyDirectory = async ( if (entry.isDirectory) { await copyDirectory(fromRoot, toRoot, childPath); } else { - await Deno.mkdir(dirname(to), { recursive: true }); await Deno.copyFile(from, to); } } }; -export const copyMutationSnapshot = async ( +/** Copy a checkout into a snapshot, leaving out everything a run must not carry. */ +export const copyMutationSnapshot = ( fromRoot: string, toRoot: string, -): Promise => { - await Deno.mkdir(toRoot, { recursive: true }); - await copyDirectory(fromRoot, toRoot); -}; +): Promise => copyDirectory(fromRoot, toRoot); const compactIso = (iso: string): string => iso @@ -452,19 +447,13 @@ export const runLockIsHeld = async ( return (await lockProbeExitCode(path, timeoutMs)) === LOCK_HELD_EXIT_CODE; }; +/** Hold a run's own lock, making its folder first so there is one to lock. */ export const withMutationRunLock = async ( runRootPath: string, run: () => Promise, ): Promise => { await Deno.mkdir(runRootPath, { recursive: true }); - const file = await openLockFile(join(runRootPath, MUTATION_RUN_LOCK_FILE)); - try { - await file.lock(true); - return await run(); - } finally { - await file.unlock(); - file.close(); - } + return await withFileLock(join(runRootPath, MUTATION_RUN_LOCK_FILE), run); }; export const selectedRuns = ( diff --git a/test/scripts/mutation/isolation-state/commands.test.ts b/test/scripts/mutation/isolation-state/commands.test.ts index 6de224358d..850027ff52 100644 --- a/test/scripts/mutation/isolation-state/commands.test.ts +++ b/test/scripts/mutation/isolation-state/commands.test.ts @@ -18,7 +18,9 @@ describe("mutation isolation command parsing and listing", () => { message: "Mutation source and test globs are required.", }); expect(parseIsolationCommand(["--help"])).toEqual({ kind: "help" }); + expect(parseIsolationCommand(["-h"])).toEqual({ kind: "help" }); expect(parseIsolationCommand(["--list"])).toEqual({ kind: "list" }); + expect(parseIsolationCommand(["list"])).toEqual({ kind: "list" }); expect(parseIsolationCommand(["kill", "run-1", "--force"])).toEqual({ force: true, kind: "kill", @@ -32,6 +34,10 @@ describe("mutation isolation command parsing and listing", () => { kind: "clean", target: "finished", }); + expect(parseIsolationCommand(["--clean", "all"])).toEqual({ + kind: "clean", + target: "all", + }); expect(parseIsolationCommand(["clean"])).toEqual({ kind: "invalid", message: "A run id, all, or finished is required for --clean.", @@ -73,6 +79,14 @@ describe("mutation isolation command parsing and listing", () => { ).toEqual(["mutation-passed", "mutation-failed", "mutation-interrupted"]); expect(selectedRuns(records, "mutation-running")).toEqual([running]); expect(selectedRuns(records, "mutation-runn")).toEqual([running]); + // A whole id wins outright, even when it also starts another run's id. + const longer = markRunning( + newRunRecord("mutation-running-again", ["src/f.ts"], "/repo"), + 12, + ); + expect(selectedRuns([...records, longer], "mutation-running")).toEqual([ + running, + ]); expect(selectedRuns(records, "mutation-")).toEqual([]); expect(selectedRuns(records, "missing")).toEqual([]); }); @@ -92,6 +106,16 @@ describe("mutation isolation command parsing and listing", () => { ).toEqual([ "mutation-running running pid=10 exit=- work=.mutation-runs/mutation-running/work args=src/a.ts", ]); + // Several arguments are spaced out the way they were typed. + const twoArgs = markRunning( + newRunRecord("mutation-two", ["src/a.ts", "test/a.test.ts"], "/repo"), + 11, + ); + expect( + formatRunList([twoArgs], new Set(["mutation-two"]), "/repo"), + ).toEqual([ + "mutation-two running pid=11 exit=- work=.mutation-runs/mutation-two/work args=src/a.ts test/a.test.ts", + ]); expect(formatRunList([running], new Set(), "/repo")).toEqual([ "mutation-running stale pid=10 exit=- work=.mutation-runs/mutation-running/work args=src/a.ts", ]); diff --git a/test/scripts/mutation/isolation-state/env.test.ts b/test/scripts/mutation/isolation-state/env.test.ts new file mode 100644 index 0000000000..0f2819a4ef --- /dev/null +++ b/test/scripts/mutation/isolation-state/env.test.ts @@ -0,0 +1,38 @@ +import { expect } from "@std/expect"; +import { describe, it as test } from "@std/testing/bdd"; +import { + MUTATION_RUN_ID_ENV, + MUTATION_RUN_ROOT_ENV, + MUTATION_SNAPSHOT_CHILD_ENV, + MUTATION_WORK_ROOT_ENV, +} from "#scripts/mutation/isolation-state.ts"; + +/** How a run tells its child which snapshot it is working in. */ +const HANDOVER_NAMES = [ + MUTATION_SNAPSHOT_CHILD_ENV, + MUTATION_RUN_ID_ENV, + MUTATION_RUN_ROOT_ENV, + MUTATION_WORK_ROOT_ENV, +]; + +describe("what a run tells its child", () => { + test("uses a separate name for each thing it passes down", () => { + expect(new Set(HANDOVER_NAMES).size).toBe(HANDOVER_NAMES.length); + }); + + for (const name of HANDOVER_NAMES) { + test(`can put ${name || "an unnamed value"} in an environment`, () => { + // A nameless or "=" bearing variable cannot be set at all, so the child + // would be started without it and quietly behave as if it were not in a + // snapshot. Deno is the judge here, not a rule written out again. + expect(() => Deno.env.set(name, "1")).not.toThrow(); + Deno.env.delete(name); + }); + } + + test("keeps them under our own prefix, away from anyone else's", () => { + for (const name of HANDOVER_NAMES) { + expect(name.startsWith("TICKETS_MUTATION_")).toBe(true); + } + }); +}); diff --git a/test/scripts/mutation/isolation-state/lock.test.ts b/test/scripts/mutation/isolation-state/lock.test.ts new file mode 100644 index 0000000000..bc05c6c62c --- /dev/null +++ b/test/scripts/mutation/isolation-state/lock.test.ts @@ -0,0 +1,109 @@ +import { join } from "node:path"; +import { expect } from "@std/expect"; +import { describe, it as test } from "@std/testing/bdd"; +import { + MUTATION_RUN_LOCK_FILE, + runLockIsHeld, + withMutationRunLock, +} from "#scripts/mutation/isolation-state.ts"; +import { withTempDir } from "#test/scripts/mutation/isolation-helpers.ts"; +import { pathExists } from "#test-utils/files.ts"; + +/** + * Long enough for the operating system to hand over a lock it was willing to + * hand over. Without this pause a lock that excludes nobody still looks like + * it is working, because the second holder has not been let in yet either. + */ +const LONG_ENOUGH_TO_BE_LET_IN_MS = 30; + +const pause = (milliseconds: number): Promise => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + +describe("the lock that keeps two runs out of one folder", () => { + test("makes the run folder it is asked to lock", async () => { + await withTempDir(async (root) => { + const runFolder = join(root, ".mutation-runs", "mutation-new"); + + expect( + await withMutationRunLock(runFolder, () => Promise.resolve(7)), + ).toBe(7); + + expect(await pathExists(join(runFolder, MUTATION_RUN_LOCK_FILE))).toBe( + true, + ); + }); + }); + + test("keeps a second run out until the first one is done", async () => { + await withTempDir(async (root) => { + const runFolder = join(root, ".mutation-runs", "mutation-shared"); + const order: string[] = []; + const firstInside = Promise.withResolvers(); + const releaseFirst = Promise.withResolvers(); + + const first = withMutationRunLock(runFolder, async () => { + order.push("first in"); + firstInside.resolve(); + await releaseFirst.promise; + order.push("first out"); + }); + await firstInside.promise; + + const second = withMutationRunLock(runFolder, () => { + order.push("second in"); + return Promise.resolve(); + }); + await pause(LONG_ENOUGH_TO_BE_LET_IN_MS); + + expect(order).toEqual(["first in"]); + + releaseFirst.resolve(); + await Promise.all([first, second]); + + expect(order).toEqual(["first in", "first out", "second in"]); + }); + }); + + test("hands the lock back when the work inside it fails", async () => { + await withTempDir(async (root) => { + const runFolder = join(root, ".mutation-runs", "mutation-failing"); + + await expect( + withMutationRunLock(runFolder, () => Promise.reject(new Error("boom"))), + ).rejects.toThrow("boom"); + + // A run that could not take the lock back would hang here instead. + expect( + await withMutationRunLock(runFolder, () => Promise.resolve("ok")), + ).toBe("ok"); + }); + }); + + test("reports a run nobody is holding as not held", async () => { + await withTempDir(async (root) => { + const record = { root: join(root, ".mutation-runs", "mutation-idle") }; + await Deno.mkdir(record.root, { recursive: true }); + + // No timeout given, so this also checks the standard one is long enough + // to tell "free" from "held" rather than calling everything held. + expect(await runLockIsHeld(record)).toBe(false); + }); + }); + + test("reports a run somebody is holding as held", async () => { + await withTempDir(async (root) => { + const record = { root: join(root, ".mutation-runs", "mutation-busy") }; + const asked = Promise.withResolvers(); + const release = Promise.withResolvers(); + + const holding = withMutationRunLock(record.root, async () => { + asked.resolve(await runLockIsHeld(record)); + await release.promise; + }); + + expect(await asked.promise).toBe(true); + release.resolve(); + await holding; + }); + }); +}); diff --git a/test/scripts/mutation/isolation-state/paths.test.ts b/test/scripts/mutation/isolation-state/paths.test.ts index 553c5d4d76..326614b180 100644 --- a/test/scripts/mutation/isolation-state/paths.test.ts +++ b/test/scripts/mutation/isolation-state/paths.test.ts @@ -10,6 +10,21 @@ import { withTempDir } from "#test/scripts/mutation/isolation-helpers.ts"; import { pathExists } from "#test-utils/files.ts"; describe("mutation isolation paths", () => { + test("copies into a snapshot folder that is already there", async () => { + await withTempDir(async (dir) => { + const source = join(dir, "source"); + const snapshot = join(dir, "snapshot"); + await Deno.mkdir(source, { recursive: true }); + await Deno.writeTextFile(join(source, "kept.ts"), "export {};\n"); + // The run's lock makes this folder before the copy starts. + await Deno.mkdir(snapshot, { recursive: true }); + + await copyMutationSnapshot(source, snapshot); + + expect(await pathExists(join(snapshot, "kept.ts"))).toBe(true); + }); + }); + test("copies source-like files and skips git, reports, secrets, dbs, and generated assets", async () => { await withTempDir(async (dir) => { const source = join(dir, "source"); diff --git a/test/scripts/mutation/isolation-state/records.test.ts b/test/scripts/mutation/isolation-state/records.test.ts index 5422efac69..846ea09b0a 100644 --- a/test/scripts/mutation/isolation-state/records.test.ts +++ b/test/scripts/mutation/isolation-state/records.test.ts @@ -69,6 +69,38 @@ describe("mutation isolation run records", () => { expect(runStartedRecently(stale, now)).toBe(false); }); + test("treats a record whose time makes no sense as not started recently", () => { + const broken = markRunning( + newRunRecord("broken", [], "/repo", "not a time"), + 3, + "not a time", + ); + + expect( + runStartedRecently(broken, new Date("2026-07-10T12:00:00.000Z")), + ).toBe(false); + }); + + test("counts a run stamped a moment after the epoch", () => { + const justAfterEpoch = markRunning( + newRunRecord("epoch", [], "/repo", "1970-01-01T00:00:00.001Z"), + 4, + "1970-01-01T00:00:00.001Z", + ); + + expect(runStartedRecently(justAfterEpoch, new Date(1), 1000)).toBe(true); + }); + + test("gives each run a plain id with an eight-letter tail", () => { + const id = createRunId(); + const tail = id.split("-").at(-1); + + // Two runs started in the same second must still land in their own folder. + expect(tail).toHaveLength(8); + expect(id.startsWith("mutation-")).toBe(true); + expect(createRunId()).not.toBe(id); + }); + test("writes, reads, sorts, and ignores broken records", async () => { await withTempDir(async (root) => { expect(await readRunRecords(root)).toEqual([]); @@ -92,6 +124,21 @@ describe("mutation isolation run records", () => { }); }); + test("writes the record so a person can read it", async () => { + await withTempDir(async (root) => { + const record = newRunRecord("readable", ["src/a.ts"], root); + await writeRunRecord(record); + + const written = await Deno.readTextFile( + join(root, ".mutation-runs", "readable", "run.json"), + ); + + // Indented and one line per field, so `cat run.json` is worth doing. + expect(written.split("\n")[1]).toBe(' "args": ['); + expect(written.endsWith("}\n")).toBe(true); + }); + }); + test("reads records from the current run directory", async () => { await withTempDir(async (root) => { const { id, record } = await writeMovedRunRecord(root); diff --git a/test/scripts/mutation/isolation-state/skips.test.ts b/test/scripts/mutation/isolation-state/skips.test.ts new file mode 100644 index 0000000000..3495a46c8b --- /dev/null +++ b/test/scripts/mutation/isolation-state/skips.test.ts @@ -0,0 +1,102 @@ +import { expect } from "@std/expect"; +import { describe, it as test } from "@std/testing/bdd"; +import { shouldCopySnapshotPath } from "#scripts/mutation/isolation-state.ts"; + +/** + * Every folder a snapshot must leave behind. Each one is either huge, rebuilt + * on demand, private to the checkout, or a run folder that would nest copies + * of itself. They are listed here so dropping one from the copier fails a test + * rather than silently doubling how long every mutation run takes. + */ +const SKIPPED_FOLDERS = [ + ".agents", + ".claude", + ".codex", + ".deno", + ".deno-cache", + ".deno_cache", + ".direnv", + ".do", + ".git", + ".i18n-work", + ".local-data", + ".mutation-runs", + ".pi-worktrees", + "cov", + "cov_profile", + "dist", + "misc", + "node_modules", + // Folders literally named for a value that went missing somewhere. + "undefined", + "null", +]; + +/** Folders skipped by how their name starts, so numbered variants go too. */ +const SKIPPED_FOLDER_PREFIXES = ["coverage", ".jscpd", "docs-output"]; + +/** Files that belong to one checkout only, or that a snapshot rebuilds. */ +const SKIPPED_FILES = [ + ".build-tag", + ".db-key", + ".env", + ".static-assets-cache.json", + ".static-assets-build.lock", + ".test-junit.xml", + "bunny-script.ts", + "bunny-script.ts.map", +]; + +describe("what a mutation snapshot leaves behind", () => { + for (const folder of SKIPPED_FOLDERS) { + test(`leaves the ${folder} folder behind`, () => { + expect(shouldCopySnapshotPath(`${folder}/inside.ts`)).toBe(false); + }); + } + + for (const prefix of SKIPPED_FOLDER_PREFIXES) { + test(`leaves behind folders whose name starts with ${prefix}`, () => { + expect(shouldCopySnapshotPath(`${prefix}-2/inside.ts`)).toBe(false); + }); + } + + for (const file of SKIPPED_FILES) { + test(`leaves the ${file} file behind`, () => { + expect(shouldCopySnapshotPath(file)).toBe(false); + // Also wherever else it turns up, not just at the top. + expect(shouldCopySnapshotPath(`sub/${file}`)).toBe(false); + }); + } + + test("copies an ordinary source file", () => { + expect(shouldCopySnapshotPath("src/shared/dates.ts")).toBe(true); + }); + + test("only skips a folder name when it is the top folder", () => { + // "dist" nested under something else is somebody's ordinary folder. + expect(shouldCopySnapshotPath("a/dist/notes.md")).toBe(true); + }); + + test("leaves every kind of database file behind", () => { + expect(shouldCopySnapshotPath("local.db")).toBe(false); + expect(shouldCopySnapshotPath("local.db-shm")).toBe(false); + expect(shouldCopySnapshotPath("local.db-wal")).toBe(false); + }); + + test("leaves built browser assets behind", () => { + expect(shouldCopySnapshotPath("src/ui/static/admin.js")).toBe(false); + expect(shouldCopySnapshotPath("src/ui/static/style.css")).toBe(false); + }); + + test("copies a script that is built but does not live in the assets folder", () => { + expect(shouldCopySnapshotPath("src/ui/client/admin.js")).toBe(true); + }); + + test("copies a file in the assets folder that is not a built asset", () => { + expect(shouldCopySnapshotPath("src/ui/static/notes.txt")).toBe(true); + }); + + test("copies anything it cannot name", () => { + expect(shouldCopySnapshotPath("")).toBe(true); + }); +});