Fully test the spec runner

scripts/mutation/equivalent-mutants.txt

@@ -977,3 +977,7 @@ scripts/process.ts:62:12  = → +=   # the handle starts at 0, so adding to it a
 
 # Isolated mutation runs: the exit code before any path has set one.
 scripts/mutation/isolation.ts:197:18  1 → 0   # every path overwrites this before it is returned: the lock always runs its callback, which either sets 130 on an interrupt or hands back a child whose status sets the code, and any throw is caught and sets it there
+
+# Cucumber runs: telling Cucumber to be strict, when we already reject the
+# statuses strictness is about.
+scripts/specs/run.ts:88:17  true → false   # strict only changes how Cucumber treats undefined and pending steps, and messageIssues rejects exactly those two statuses (REJECTED_STATUSES in scripts/specs/messages.ts), so a run containing either fails on the issues whatever strict says, and a run containing neither is unaffected by it

scripts/specs/paths.ts

@@ -1,4 +1,15 @@
+import { join } from "node:path";
 import { inProjectFolders, relativeToProject } from "#scripts/path.ts";
+import { projectRoot } from "#scripts/project-root.ts";
+
+/** Where a run leaves its reports for anyone who wants to read them after. */
+export const SPEC_REPORT_DIR = join(projectRoot, "reports");
+
+/** The files a run loads before it starts: the world it needs, then the steps. */
+export const SPEC_SUPPORT_GLOBS = [
+  "test/specs/support/**/*.ts",
+  "test/specs/steps/**/*.ts",
+];
 
 export const isFeaturePath = (path: string): boolean =>
   relativeToProject(path).endsWith(".feature");

scripts/specs/run.ts

@@ -14,6 +14,7 @@ import { readSpecCatalog } from "./catalog.ts";
 import { messageIssues } from "./messages.ts";
 import { shouldCheckUnusedSteps } from "./options.ts";
 import { specWorkerCount } from "./parallel.ts";
+import { SPEC_REPORT_DIR, SPEC_SUPPORT_GLOBS } from "./paths.ts";
 import { selectSpecCases } from "./selection.ts";
 import type { SpecCatalog } from "./types.ts";
 
@@ -51,14 +52,12 @@ interface SpecRunControls {
   parallel?: number;
 }
 
-const REPORT_DIR = join(projectRoot, "reports");
-const DEFAULT_SUPPORT = [
-  "test/specs/support/**/*.ts",
-  "test/specs/steps/**/*.ts",
-];
+/** A tag no Feature carries, so asking for it selects nothing at all. */
+const NO_MATCHING_CASE_TAG = "@__tickets_no_matching_case__";
+
 const DEFAULT_ENVIRONMENT: SpecRunEnvironment = {
-  reportDir: REPORT_DIR,
-  support: DEFAULT_SUPPORT,
+  reportDir: SPEC_REPORT_DIR,
+  support: SPEC_SUPPORT_GLOBS,
 };
 const withProcessEnvironment = environmentTasks(processEnvironment);
 const reportFormats = (reports: boolean, reportDir: string): string[] =>
@@ -117,25 +116,27 @@ export const runSpecs = async (
   environment: SpecRunEnvironment = DEFAULT_ENVIRONMENT,
   controls: SpecRunControls = {},
 ): Promise<SpecRunSummary> => {
-  const paths = options.paths ?? ["specs"];
-  if (options.reports ?? true) await prepareReports(environment.reportDir);
+  // Only a missing value takes the default, so an explicit one always wins.
+  const { paths = ["specs"], reports = true, tags = "" } = options;
+  if (reports) await prepareReports(environment.reportDir);
   const catalog = await readSpecCatalog(paths);
   await controls.beforeRun?.(catalog);
-  const selectedPaths = selectSpecCases(catalog, options.tags ?? "");
+  const selectedPaths = selectSpecCases(catalog, tags);
+  const {
+    parallel = specWorkerCount(
+      selectedPaths.length,
+      Deno.env.get("DENO_JOBS"),
+      navigator.hardwareConcurrency,
+    ),
+  } = controls;
   const complete: CompleteRunSpecsOptions = {
     enforceUnused: shouldCheckUnusedSteps(options),
-    parallel:
-      controls.parallel ??
-      specWorkerCount(
-        selectedPaths.length,
-        Deno.env.get("DENO_JOBS"),
-        navigator.hardwareConcurrency,
-      ),
+    parallel,
     paths: selectedPaths.length > 0 ? selectedPaths : paths,
-    reports: options.reports ?? true,
+    reports,
     tags:
       selectedPaths.length === 0 && options.tags !== undefined
-        ? "@__tickets_no_matching_case__"
+        ? NO_MATCHING_CASE_TAG
         : "",
   };
   const messages: Envelope[] = [];

test/scripts/specs/fixtures/failing.steps.ts

@@ -0,0 +1,6 @@
+import { Given } from "@cucumber/cucumber";
+
+/** A step that always fails, for tests about what a failed run does next. */
+Given("a selected example runs", () => {
+  throw new Error("this example was meant to fail");
+});

test/scripts/specs/paths.test.ts

@@ -1,7 +1,14 @@
+import { join } from "node:path";
 import { expect } from "@std/expect";
 import { describe, it as test } from "@std/testing/bdd";
+import { relativeToProject } from "#scripts/path.ts";
 import { projectRoot } from "#scripts/project-root.ts";
-import { isFeaturePath, isSpecPath } from "#scripts/specs/paths.ts";
+import {
+  isFeaturePath,
+  isSpecPath,
+  SPEC_REPORT_DIR,
+  SPEC_SUPPORT_GLOBS,
+} from "#scripts/specs/paths.ts";
 
 describe("Cucumber paths", () => {
   test("recognises project-relative and absolute spec paths", () => {
@@ -16,3 +23,39 @@ describe("Cucumber paths", () => {
     expect(isFeaturePath("specs/payments/example.feature.md")).toBe(false);
   });
 });
+
+/** The folder a support glob looks inside. */
+const folderOf = (glob: string): string =>
+  join(projectRoot, glob.slice(0, glob.indexOf("/**")));
+
+/** How many TypeScript files live in a folder, however deep. */
+const countScripts = async (folder: string): Promise<number> => {
+  let found = 0;
+  for await (const entry of Deno.readDir(folder)) {
+    if (entry.isDirectory)
+      found += await countScripts(join(folder, entry.name));
+    else if (entry.name.endsWith(".ts")) found += 1;
+  }
+  return found;
+};
+
+describe("where a spec run reads and writes", () => {
+  test("keeps its reports in the project's own reports folder", () => {
+    expect(relativeToProject(SPEC_REPORT_DIR)).toBe("reports");
+  });
+
+  for (const glob of SPEC_SUPPORT_GLOBS) {
+    test(`finds files to load for ${glob}`, async () => {
+      // A glob pointing at nothing would leave a run with no steps at all, and
+      // every scenario would fail for want of anything to do.
+      expect(await countScripts(folderOf(glob))).toBeGreaterThan(0);
+    });
+  }
+
+  test("loads the world before the steps that lean on it", () => {
+    const [world, steps] = SPEC_SUPPORT_GLOBS;
+
+    expect(world).toContain("support");
+    expect(steps).toContain("steps");
+  });
+});

test/scripts/specs/run-execution.test.ts → test/scripts/specs/run/execution.test.ts

@@ -1,71 +1,64 @@
 import type { Envelope } from "@cucumber/messages";
 import { expect } from "@std/expect";
 import { describe, it as test } from "@std/testing/bdd";
-import { runSpecs, type SpecRunEnvironment } from "#scripts/specs/run.ts";
+import { stub } from "@std/testing/mock";
+import { runSpecs } from "#scripts/specs/run.ts";
+import type { SpecCatalog } from "#scripts/specs/types.ts";
 import { CONCURRENT_ROWS } from "#test/scripts/specs/fixtures/concurrent.steps.ts";
 import {
   requiredSpecRunsPath,
   SPEC_RUNS_PATH_ENV,
 } from "#test/scripts/specs/fixtures/record-step.ts";
 import { withEnv } from "#test-utils/env.ts";
-
-interface OutlineFixture {
-  directory: string;
-  environment: SpecRunEnvironment;
-  featurePath: string;
-  runsPath: string;
-}
-
-const createOutlineFixture = async (
-  support: string,
-): Promise<OutlineFixture> => {
-  const directory = await Deno.makeTempDir();
-  const featurePath = `${directory}/outline.feature`;
-  const runsPath = `${directory}/runs.txt`;
-  await Deno.writeTextFile(
-    featurePath,
-    `
-@story:payments.outline-selection
-@owner:payments @risk:high
-@actor:customer @edition:managed
-Feature: Select a payment example
-  A stable case id selects one example from a Scenario Outline.
-
-  @rule:payments.outline-selection-rule
-  Rule: One example is selected
-    Only the requested example is run.
-
-    Scenario Outline: Payment result <label>
-      Given a selected example runs
-
-      Examples:
-        | case_id                  | label  |
-        | payment.selection-first  | first  |
-        | payment.selection-second | second |
-`,
-  );
-  return {
-    directory,
-    environment: {
-      env: { [SPEC_RUNS_PATH_ENV]: runsPath },
-      reportDir: `${directory}/reports`,
-      support: [support],
-    },
-    featurePath,
-    runsPath,
-  };
-};
-
-const removeOutlineFixture = async (fixture: OutlineFixture): Promise<void> => {
-  await Deno.remove(fixture.directory, { recursive: true });
-};
+import {
+  createOutlineFixture,
+  type OutlineFixture,
+  removeOutlineFixture,
+} from "./outline-fixture.ts";
 
 const readMessages = async (reportDir: string): Promise<Envelope[]> =>
   (await Deno.readTextFile(`${reportDir}/cucumber.ndjson`))
     .trim()
     .split("\n")
     .map((line) => JSON.parse(line));
 
+const WORKING_STEPS = "test/scripts/specs/fixtures/selected.steps.ts";
+
+/** Run the throwaway Feature, with whatever is passed layered on top. */
+const runOutline = async (
+  fixture: OutlineFixture,
+  options: Parameters<typeof runSpecs>[0],
+  support: string[] = [WORKING_STEPS],
+): Promise<{ success: boolean }> =>
+  await runSpecs(
+    { paths: [fixture.featurePath], ...options },
+    { reportDir: fixture.environment.reportDir, support },
+    { env: { [SPEC_RUNS_PATH_ENV]: fixture.runsPath }, parallel: 0 },
+  );
+
+const withOutline = async (
+  body: (fixture: OutlineFixture) => Promise<void>,
+): Promise<void> => {
+  const fixture = await createOutlineFixture(WORKING_STEPS);
+  try {
+    await body(fixture);
+  } finally {
+    await removeOutlineFixture(fixture);
+  }
+};
+
+/** Everything the run complained about while it ran. */
+const complaintsFrom = async (
+  run: () => Promise<unknown>,
+): Promise<string[]> => {
+  const complaints: string[] = [];
+  using _error = stub(console, "error", (...parts: unknown[]) => {
+    complaints.push(parts.map(String).join(" "));
+  });
+  await run();
+  return complaints;
+};
+
 describe("Cucumber execution", () => {
   test("requires the path used to record selected examples", () => {
     using _env = withEnv({ [SPEC_RUNS_PATH_ENV]: undefined });
@@ -208,4 +201,91 @@ describe("Cucumber execution", () => {
       await Deno.remove(directory, { recursive: true });
     }
   });
+
+  test("selects nothing at all when the asked-for case is not in the catalog", async () => {
+    await withOutline(async (fixture) => {
+      // The Feature would pass if it ran, so only a filter that keeps every
+      // case out can make this fail.
+      expect(
+        await runOutline(fixture, { tags: "@case:not.in-catalog" }),
+      ).toEqual({ success: false });
+
+      // Nothing was tried at all, rather than tried and failed.
+      const messages = await readMessages(fixture.environment.reportDir);
+      expect(messages.filter(({ testCase }) => testCase)).toEqual([]);
+    });
+  });
+
+  test("fails a run whose steps nobody has written", async () => {
+    await withOutline(async (fixture) => {
+      expect(
+        await runOutline(
+          fixture,
+          { reports: false, tags: "@case:payment.selection-first" },
+          [],
+        ),
+      ).toEqual({ success: false });
+    });
+  });
+
+  test("runs a failing case once instead of trying it again", async () => {
+    await withOutline(async (fixture) => {
+      const complaints = await complaintsFrom(() =>
+        runOutline(fixture, { tags: "@case:payment.selection-first" }, [
+          "test/scripts/specs/fixtures/failing.steps.ts",
+        ]),
+      );
+
+      // A second attempt would be reported as a retry, which we never allow.
+      expect(complaints).toEqual([]);
+      const messages = await readMessages(fixture.environment.reportDir);
+      expect(
+        messages.filter(({ testCaseStarted }) => testCaseStarted),
+      ).toHaveLength(1);
+    });
+  });
+
+  test("makes the whole reports folder, however deep it is", async () => {
+    const directory = await Deno.makeTempDir();
+    try {
+      const reportDir = `${directory}/nested/deeper/reports`;
+
+      await expect(
+        runSpecs({ paths: ["specs/owners.json"] }, { reportDir, support: [] }),
+      ).rejects.toThrow("No Cucumber Feature files found");
+
+      expect((await Deno.stat(reportDir)).isDirectory).toBe(true);
+    } finally {
+      await Deno.remove(directory, { recursive: true });
+    }
+  });
+
+  test("reads the specs folder when asked for no paths in particular", async () => {
+    const directory = await Deno.makeTempDir();
+    let catalogued: SpecCatalog | undefined;
+    try {
+      await expect(
+        runSpecs(
+          { reports: false },
+          { reportDir: `${directory}/reports`, support: [] },
+          {
+            beforeRun: (catalog) => {
+              catalogued = catalog;
+              // Stop before Cucumber runs: the catalog is all this checks.
+              throw new Error("read the catalog");
+            },
+          },
+        ),
+      ).rejects.toThrow("read the catalog");
+
+      expect(catalogued?.stories.length).toBeGreaterThan(0);
+      // Everything it found came from the specs folder, not from wherever the
+      // command happened to be run.
+      expect(
+        catalogued?.stories.every((story) => story.uri.startsWith("specs/")),
+      ).toBe(true);
+    } finally {
+      await Deno.remove(directory, { recursive: true });
+    }
+  });
 });