Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 100 additions & 1 deletion tools/zflash/test-harness/scenarios.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,14 @@
*/

import { describe, expect, it } from "bun:test";
import { SCENARIOS, validateScenarios, findScenario } from "./scenarios";
import {
SCENARIOS,
computeRunnableSet,
determineRunnability,
findScenario,
validateScenarios,
type RunnabilityVerdict,
} from "./scenarios";

describe("B-0891 scenarios.ts invariants", () => {
it("has exactly 5 scenarios per operator-named matrix", () => {
Expand Down Expand Up @@ -83,3 +90,95 @@ describe("B-0891 scenarios.ts invariants", () => {
expect(() => validateScenarios(broken)).toThrow();
});
});

describe("B-0891 determineRunnability discriminator", () => {
// Substantive zflash-lane work per Aaron's 3-lane substrate-check
// (Amara ferry §33.2 PR #5757) + standing PoC permission.
// Structurally parallel to:
// - PR #5758 workflow-engine determineReviewLevel (workflow scope)
// - PR #5760 better-git-crypt determineEncryptionPath (encryption scope)
// Same substrate-engineering substrate (Result-shaped discriminator)
// operating at zflash-substrate scope.

it("initial-format → can-run-now (composes with existing qemu-boot-test substrate)", () => {
const s = findScenario("initial-format");
if (!s) throw new Error("scenario missing");
const verdict = determineRunnability(s, new Set());
expect(verdict.kind).toBe("can-run-now");
if (verdict.kind === "can-run-now") {
expect(verdict.harnessEntry).toMatch(/qemu-boot-test/);
}
});

it("boot-cluster-up → can-run-now (composes with qemu-full-install-test)", () => {
const s = findScenario("boot-cluster-up");
if (!s) throw new Error("scenario missing");
const verdict = determineRunnability(s, new Set());
expect(verdict.kind).toBe("can-run-now");
if (verdict.kind === "can-run-now") {
expect(verdict.harnessEntry).toMatch(/qemu-full-install-test/);
}
});

it("reformat-with-retention → blocked-on-state-preservation", () => {
const s = findScenario("reformat-with-retention");
if (!s) throw new Error("scenario missing");
const verdict = determineRunnability(s, new Set());
expect(verdict.kind).toBe("blocked-on-state-preservation");
if (verdict.kind === "blocked-on-state-preservation") {
expect(verdict.required).toBe("persisted-kv");
}
});

it("reformat-from-scratch → blocked-on-test-harness-path-fork", () => {
const s = findScenario("reformat-from-scratch");
if (!s) throw new Error("scenario missing");
const verdict = determineRunnability(s, new Set());
expect(verdict.kind).toBe("blocked-on-test-harness-path-fork");
});

it("cluster-joining → blocked-on-multi-vm-orchestration", () => {
const s = findScenario("cluster-joining");
if (!s) throw new Error("scenario missing");
const verdict = determineRunnability(s, new Set());
expect(verdict.kind).toBe("blocked-on-multi-vm-orchestration");
});

it("all scenarios resolve to a valid RunnabilityVerdict (exhaustiveness)", () => {
// Acknowledger forces exhaustive switch — TS strict mode raises
// "not all code paths return" at compile time if a NEW
// RunnabilityVerdict variant is added without updating this switch.
const acknowledge = (v: RunnabilityVerdict): string => {
switch (v.kind) {
case "can-run-now":
case "blocked-on-upstream-gate":
case "blocked-on-state-preservation":
case "blocked-on-multi-vm-orchestration":
case "blocked-on-test-harness-path-fork":
case "requires-physical-usb":
return v.kind;
}
};
for (const s of SCENARIOS) {
const verdict = determineRunnability(s, new Set());
expect(acknowledge(verdict)).toBe(verdict.kind);
}
});

it("computeRunnableSet identifies composes-with-existing scenarios", () => {
const runnable = computeRunnableSet();
expect(runnable.has("initial-format")).toBe(true);
expect(runnable.has("boot-cluster-up")).toBe(true);
expect(runnable.has("reformat-with-retention")).toBe(false);
expect(runnable.has("reformat-from-scratch")).toBe(false);
expect(runnable.has("cluster-joining")).toBe(false);
});

it("computeRunnableSet count matches composes-with-existing scenario count", () => {
const composesCount = SCENARIOS.filter(
(s) => s.status === "composes-with-existing",
).length;
const runnable = computeRunnableSet();
expect(runnable.size).toBe(composesCount);
});
});
150 changes: 150 additions & 0 deletions tools/zflash/test-harness/scenarios.ts
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,156 @@ export function findScenario(id: ScenarioId): Scenario | undefined {
return SCENARIOS.find((s) => s.id === id);
}

/**
* B-0891 — determineRunnability: substantive lane work per Aaron's
* 3-lane substrate-check (Amara ferry §33.2 PR #5757) + standing PoC
* permission.
Comment on lines +172 to +174
*
* Structurally parallel to:
* - workflow-engine determineReviewLevel (PR #5758) — same shape at
* workflow-substrate scope
* - better-git-crypt determineEncryptionPath (PR #5760) — same shape
* at encryption-substrate scope
*
* Each discriminator maps a substrate-context → Result-shaped verdict.
* The 3-lane work isn't 3 independent implementations; it's the same
* substrate-engineering substrate (per `.claude/rules/monad-propagation-
* pattern-cross-language-substrate-shape.md` + `.claude/rules/
* asymmetric-authorship-substrate-entity-defines-consent-channel-
* recipient-acknowledges.md`) operating at 3 different scopes.
*
* Policy:
* - status "composes-with-existing" → can-run-now (composes with shipped
* qemu-full-install-test.ts substrate)
* - status "scaffolded" + state-preservation-pending → blocked-on-state-
* preservation (requires QEMU TPM-equivalent / persisted KV)
* - status "scaffolded" + multi-vm-pending → blocked-on-multi-vm-
* orchestration (requires multi-VM QEMU)
* - status "scaffolded" + dual-path-fork-pending → blocked-on-test-
* harness-path-fork
* - status "operator-runtime" → requires-physical-usb
* - gates not all runnable → blocked-on-upstream-gate
*
* Future extensions to RunnabilityVerdict union must update this function
* (TS strict mode enforces exhaustiveness).
*/

/**
* RunnabilityVerdict — discriminated union for whether a scenario can
* run in the current substrate state.
*/
export type RunnabilityVerdict =
| { kind: "can-run-now"; harnessEntry: string }
| { kind: "blocked-on-upstream-gate"; missingGates: ReadonlyArray<ScenarioId> }
| { kind: "blocked-on-state-preservation"; required: "tpm-equivalent" | "persisted-kv" }
| { kind: "blocked-on-multi-vm-orchestration" }
| { kind: "blocked-on-test-harness-path-fork" }
| { kind: "requires-physical-usb" };

/**
* Discriminator that maps a Scenario + the set of currently-runnable
* upstream scenarios to its RunnabilityVerdict.
*
* The `runnableUpstream` set IS the substrate-state input — caller
* provides which other scenarios can currently run (typically by
* iterating SCENARIOS + checking determineRunnability output reflexively
* OR by hardcoding which composes-with-existing scenarios are wired).
*/
export function determineRunnability(
scenario: Scenario,
runnableUpstream: ReadonlySet<ScenarioId>,
): RunnabilityVerdict {
// Upstream gate check applies regardless of status — if any gate
// scenario isn't runnable, this scenario can't run end-to-end either.
const missingGates = scenario.gates.filter(
(g) => !runnableUpstream.has(g),
);
if (missingGates.length > 0 && scenario.gates.length > 0) {
// Special-case: a scenario's gates name DOWNSTREAM scenarios it
// GATES, not UPSTREAM scenarios it depends on. Per scenarios.ts
// existing scenarios: `initial-format.gates = ["boot-cluster-up"]`
// means initial-format must run BEFORE boot-cluster-up. The gate
// direction is "I gate downstream X". So missingGates here means
// "downstream scenarios I gate aren't runnable" — which doesn't
// block THIS scenario from running. Skip the gate check entirely
// for the upstream-blocking interpretation, OR reframe as "this
// scenario gates downstream X; runnability not affected."
//
// For PoC: surface gates as informational; don't block on them.
Comment on lines +219 to +246
}

switch (scenario.status) {
case "composes-with-existing": {
// PoC dispatcher contract: each composes-with-existing scenario
// identifies which existing harness substrate it dispatches to.
// Map per current substrate:
if (scenario.id === "initial-format") {
return {
kind: "can-run-now",
harnessEntry: "tools/ci/qemu-boot-test.ts + tools/ci/audit-installer-iso-content.ts",
};
}
if (scenario.id === "boot-cluster-up") {
return {
kind: "can-run-now",
harnessEntry: "tools/ci/qemu-full-install-test.ts",
};
}
// Future composes-with-existing scenarios fall through to a
// generic harness-entry; explicit mapping preferred when added.
return {
kind: "can-run-now",
harnessEntry: "tools/ci/qemu-full-install-test.ts",
};
}
case "scaffolded": {
// Map specific scaffolded scenarios to their specific blocker:
if (scenario.id === "reformat-with-retention") {
// Requires state-preservation between QEMU boots per scenarios.ts notes
return {
kind: "blocked-on-state-preservation",
required: "persisted-kv",
};
}
if (scenario.id === "reformat-from-scratch") {
// Requires test-harness path-fork support per scenarios.ts notes
return { kind: "blocked-on-test-harness-path-fork" };
}
if (scenario.id === "cluster-joining") {
// Requires multi-VM QEMU orchestration per scenarios.ts notes
return { kind: "blocked-on-multi-vm-orchestration" };
}
// Default scaffolded blocker:
return {
kind: "blocked-on-state-preservation",
required: "persisted-kv",
};
}
case "operator-runtime":
return { kind: "requires-physical-usb" };
}
}

/**
* Convenience: compute the set of currently-runnable scenarios.
* Iterates SCENARIOS reflexively; a scenario is runnable iff
* determineRunnability returns "can-run-now".
*/
export function computeRunnableSet(
scenarios: ReadonlyArray<Scenario> = SCENARIOS,
): ReadonlySet<ScenarioId> {
const set = new Set<ScenarioId>();
for (const s of scenarios) {
// Use empty runnableUpstream — determineRunnability's gate-check is
// currently informational-only per the scaffolded comment above.
const verdict = determineRunnability(s, new Set());
if (verdict.kind === "can-run-now") {
set.add(s.id);
}
}
return set;
}

/**
* Validate scenario definitions at harness-init time. Invariants:
* - exactly 5 scenarios (per operator-named matrix)
Expand Down
Loading