test(e2e): extend migration inventory to scenario runner internals

test/e2e-scenario/docs/MIGRATION.md

@@ -84,18 +84,22 @@ that owns the work instead.
 
 The one repo-local exception is the machine-readable deletion gate inventory at
 `test/e2e-scenario/migration/legacy-inventory.json`. Keep that file focused on
-script-level migration state that prevents accidental legacy E2E deletion. It
+deletion-readiness evidence that prevents accidental legacy E2E deletion. It
 must cover every direct legacy shell entrypoint under `test/e2e/test-*.sh`,
-plus any explicitly retained bridge entrypoints such as Brev. It is not a
-progress dashboard or owner queue:
+plus any explicitly retained bridge entrypoints such as Brev. It also tracks
+coarse internal legacy runner surfaces such as the YAML/bash scenario workers,
+validation suites, TypeScript shell-runner orchestrators, and runtime helper
+libraries so those surfaces cannot be removed without #4357 evidence. It is not
+a progress dashboard or owner queue:
 
 - `not-migrated`: legacy coverage still has no equivalent Vitest scenario.
 - `bridge-probe`: coverage is temporarily represented by a bridge path.
 - `covered`: equivalent Vitest live scenario coverage exists.
 - `retired`: maintainers agreed the legacy coverage is no longer required.
 
-Do not set `deletionReady: true` unless the entry is `covered` or `retired` and
-the deletion approval is recorded through #4357.
+Do not set `deletionReady: true` on a script entry or internal surface unless
+the record is `covered` or `retired` and the deletion approval is recorded
+through #4357.
 
 After #4357 completes final legacy E2E reconciliation, remove the inventory if
 there are no remaining legacy entrypoints to guard. If maintainers keep it, keep

test/e2e-scenario/docs/README.md

@@ -203,9 +203,12 @@ is tracked in #4941.
 The narrow repo-local exception is
 `test/e2e-scenario/migration/legacy-inventory.json`, a machine-readable deletion
 gate for direct legacy `test/e2e/test-*.sh` entrypoints and explicit bridge
-entrypoints. It should prevent accidental deletions, not become a parallel
-status table. Remove it after #4357 completes final legacy E2E reconciliation,
-or keep it only as an audit artifact if maintainers still need that record.
+entrypoints. It also tracks coarse internal legacy runner surfaces such as
+scenario shell workers, validation suites, shell-runner orchestrators, and
+runtime helper libraries so they cannot be removed without #4357 evidence. It
+should prevent accidental deletions, not become a parallel status table. Remove
+it after #4357 completes final legacy E2E reconciliation, or keep it only as an
+audit artifact if maintainers still need that record.
 
 The old workflow-level parity report has been removed. Use scenario framework
 tests, the coverage report, PR review, and the audit issues to decide what to

test/e2e-scenario/framework-tests/e2e-migration-inventory.test.ts

@@ -10,6 +10,14 @@ const INVENTORY_PATH = path.resolve(import.meta.dirname, "../migration/legacy-in
 const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
 const LEGACY_E2E_DIR = path.join(REPO_ROOT, "test/e2e");
 const EXPECTED_STATUS_VALUES = ["not-migrated", "bridge-probe", "covered", "retired"] as const;
+const INTERNAL_SURFACE_ROOTS = [
+  "test/e2e-scenario/nemoclaw_scenarios",
+  "test/e2e-scenario/onboarding_assertions",
+  "test/e2e-scenario/runtime/lib",
+  "test/e2e-scenario/runtime/reports",
+  "test/e2e-scenario/scenarios/orchestrators",
+  "test/e2e-scenario/validation_suites",
+] as const;
 
 type MigrationStatus = "not-migrated" | "bridge-probe" | "covered" | "retired";
 
@@ -26,13 +34,29 @@ interface LegacyInventoryEntry {
   notes: string;
 }
 
+interface LegacyInternalSurface {
+  id: string;
+  paths: string[];
+  domain: string;
+  ownerIssue: string;
+  status: MigrationStatus;
+  replacementSurface: string;
+  targetVitestScenarios: string[];
+  bridgeProbes: string[];
+  retiredReason: string;
+  deletionReady: boolean;
+  deletionApprovalIssue?: string;
+  notes: string;
+}
+
 interface LegacyInventory {
   version: number;
   statusValues: MigrationStatus[];
   deletionReadiness: {
     requires: string[];
   };
   entries: LegacyInventoryEntry[];
+  internalSurfaces: LegacyInternalSurface[];
 }
 
 function loadInventory(): LegacyInventory {
@@ -54,16 +78,86 @@ function listLegacyShellEntrypoints(): string[] {
     .sort();
 }
 
+function listRepoFilesUnder(repoRelativeDir: string): string[] {
+  const absoluteDir = path.join(REPO_ROOT, repoRelativeDir);
+  const files: string[] = [];
+  const visit = (dir: string) => {
+    for (const dirent of fs.readdirSync(dir, { withFileTypes: true })) {
+      const absolutePath = path.join(dir, dirent.name);
+      if (dirent.isDirectory()) {
+        visit(absolutePath);
+      } else if (dirent.isFile()) {
+        files.push(path.relative(REPO_ROOT, absolutePath).split(path.sep).join("/"));
+      }
+    }
+  };
+  visit(absoluteDir);
+  return files.sort();
+}
+
+function isCoveredByInventoryPath(filePath: string, inventoryPath: string): boolean {
+  return filePath === inventoryPath || filePath.startsWith(`${inventoryPath}/`);
+}
+
+function expectPathListIsRepoRelative(paths: readonly string[]) {
+  expect(paths.length).toBeGreaterThan(0);
+  for (const repoRelativePath of paths) {
+    expect(repoRelativePath).not.toBe("");
+    expect(repoPathExists(repoRelativePath)).toBe(true);
+  }
+}
+
+function expectMigrationRecordDeletionGate(
+  record: Pick<
+    LegacyInventoryEntry | LegacyInternalSurface,
+    | "status"
+    | "targetVitestScenarios"
+    | "bridgeProbes"
+    | "retiredReason"
+    | "deletionReady"
+    | "deletionApprovalIssue"
+  >,
+) {
+  if (record.status === "covered") {
+    expect(record.targetVitestScenarios.length).toBeGreaterThan(0);
+    for (const scenario of record.targetVitestScenarios) {
+      expect(scenario).toMatch(/^test\/e2e-scenario\/live\/.+\.test\.ts$/);
+      expect(repoPathExists(scenario)).toBe(true);
+    }
+  }
+
+  if (record.status === "bridge-probe") {
+    expect(record.bridgeProbes.length).toBeGreaterThan(0);
+    for (const probe of record.bridgeProbes) {
+      expect(repoPathExists(probe)).toBe(true);
+    }
+  }
+
+  if (record.status === "retired") {
+    expect(record.retiredReason).not.toBe("");
+  }
+
+  if (record.deletionReady) {
+    expect(["covered", "retired"]).toContain(record.status);
+    expect(record.deletionApprovalIssue).toBe("#4357");
+    expect(
+      record.status === "retired" ? record.retiredReason : record.targetVitestScenarios.length,
+    ).toBeTruthy();
+  }
+}
+
 describe("E2E migration inventory deletion gates", () => {
   it("uses a constrained migration vocabulary with owning issues", () => {
     const inventory = loadInventory();
     const statuses = new Set(inventory.statusValues);
     const legacyScripts = new Set<string>();
+    const internalSurfaceIds = new Set<string>();
 
     expect(inventory.version).toBe(1);
     expect(inventory.statusValues).toEqual([...EXPECTED_STATUS_VALUES]);
     expect(inventory.deletionReadiness.requires.length).toBeGreaterThan(0);
     expect(inventory.entries.length).toBeGreaterThan(0);
+    expect(inventory.internalSurfaces.length).toBeGreaterThan(0);
 
     for (const entry of inventory.entries) {
       expect(statuses.has(entry.status)).toBe(true);
@@ -75,6 +169,18 @@ describe("E2E migration inventory deletion gates", () => {
       expect(entry.ownerIssue).toMatch(/^#(?:3588|434[7-9]|435[0-7]|4941)$/);
       expect(entry.notes).not.toBe("");
     }
+
+    for (const surface of inventory.internalSurfaces) {
+      expect(statuses.has(surface.status)).toBe(true);
+      expect(surface.id).toMatch(/^[a-z0-9-]+$/);
+      expect(internalSurfaceIds.has(surface.id)).toBe(false);
+      internalSurfaceIds.add(surface.id);
+      expectPathListIsRepoRelative(surface.paths);
+      expect(surface.domain).not.toBe("");
+      expect(surface.ownerIssue).toMatch(/^#(?:3588|434[7-9]|435[0-7]|4941)$/);
+      expect(surface.replacementSurface).not.toBe("");
+      expect(surface.notes).not.toBe("");
+    }
   });
 
   it("covers every current direct legacy shell entrypoint", () => {
@@ -87,36 +193,30 @@ describe("E2E migration inventory deletion gates", () => {
     expect(inventoriedShellScripts).toEqual(listLegacyShellEntrypoints());
   });
 
-  it("requires coverage, retirement evidence, and #4357 approval before deletion", () => {
+  it("covers legacy scenario runner internal surfaces by path", () => {
     const inventory = loadInventory();
+    const surfacePaths = inventory.internalSurfaces.flatMap((surface) => surface.paths);
 
-    for (const entry of inventory.entries) {
-      if (entry.status === "covered") {
-        expect(entry.targetVitestScenarios.length).toBeGreaterThan(0);
-        for (const scenario of entry.targetVitestScenarios) {
-          expect(scenario).toMatch(/^test\/e2e-scenario\/live\/.+\.test\.ts$/);
-          expect(repoPathExists(scenario)).toBe(true);
-        }
+    for (const root of INTERNAL_SURFACE_ROOTS) {
+      const files = listRepoFilesUnder(root);
+      expect(files.length).toBeGreaterThan(0);
+      for (const file of files) {
+        expect(
+          surfacePaths.some((surfacePath) => isCoveredByInventoryPath(file, surfacePath)),
+        ).toBe(true);
       }
+    }
+  });
 
-      if (entry.status === "bridge-probe") {
-        expect(entry.bridgeProbes.length).toBeGreaterThan(0);
-        for (const probe of entry.bridgeProbes) {
-          expect(repoPathExists(probe)).toBe(true);
-        }
-      }
+  it("requires coverage, retirement evidence, and #4357 approval before deletion", () => {
+    const inventory = loadInventory();
 
-      if (entry.status === "retired") {
-        expect(entry.retiredReason).not.toBe("");
-      }
+    for (const entry of inventory.entries) {
+      expectMigrationRecordDeletionGate(entry);
+    }
 
-      if (entry.deletionReady) {
-        expect(["covered", "retired"]).toContain(entry.status);
-        expect(entry.deletionApprovalIssue).toBe("#4357");
-        expect(
-          entry.status === "retired" ? entry.retiredReason : entry.targetVitestScenarios.length,
-        ).toBeTruthy();
-      }
+    for (const surface of inventory.internalSurfaces) {
+      expectMigrationRecordDeletionGate(surface);
     }
   });
 });

test/e2e-scenario/migration/legacy-inventory.json

@@ -816,5 +816,75 @@
       "deletionReady": false,
       "notes": "Already uses Vitest, but still dispatches legacy remote shell suites; keep as a bridge until remote execution uses shared fixtures."
     }
+  ],
+  "internalSurfaces": [
+    {
+      "id": "typed-shell-orchestrators",
+      "paths": ["test/e2e-scenario/scenarios/orchestrators"],
+      "domain": "scenario-runner",
+      "ownerIssue": "#4357",
+      "status": "not-migrated",
+      "replacementSurface": "test/e2e-scenario/framework/phases",
+      "targetVitestScenarios": [],
+      "bridgeProbes": [],
+      "retiredReason": "",
+      "deletionReady": false,
+      "notes": "Retire after the registry-driven Vitest runner owns phase ordering, expected-failure matching, cleanup, redaction, and artifact evidence."
+    },
+    {
+      "id": "legacy-bash-scenario-workers",
+      "paths": ["test/e2e-scenario/nemoclaw_scenarios"],
+      "domain": "scenario-runner",
+      "ownerIssue": "#4357",
+      "status": "not-migrated",
+      "replacementSurface": "test/e2e-scenario/framework/phases",
+      "targetVitestScenarios": [],
+      "bridgeProbes": [],
+      "retiredReason": "",
+      "deletionReady": false,
+      "notes": "Install, onboarding, lifecycle, probe, fixture, and context workers are bridge adapters until equivalent Vitest fixtures or CI setup actions own those phases."
+    },
+    {
+      "id": "legacy-onboarding-assertion-workers",
+      "paths": ["test/e2e-scenario/onboarding_assertions"],
+      "domain": "smoke-onboarding",
+      "ownerIssue": "#4348",
+      "status": "not-migrated",
+      "replacementSurface": "test/e2e-scenario/framework/phases/onboarding.ts",
+      "targetVitestScenarios": [],
+      "bridgeProbes": [],
+      "retiredReason": "",
+      "deletionReady": false,
+      "notes": "Retire after onboarding phase fixtures emit equivalent pass/fail evidence for base install and preflight assertions."
+    },
+    {
+      "id": "legacy-validation-suites",
+      "paths": ["test/e2e-scenario/validation_suites"],
+      "domain": "runtime-suites",
+      "ownerIssue": "#4357",
+      "status": "not-migrated",
+      "replacementSurface": "test/e2e-scenario/framework/phases",
+      "targetVitestScenarios": [],
+      "bridgeProbes": [],
+      "retiredReason": "",
+      "deletionReady": false,
+      "notes": "Runtime suite assertions migrate one family at a time into typed Vitest runtime helpers before this shell suite tree can be removed."
+    },
+    {
+      "id": "legacy-runtime-helper-libraries",
+      "paths": [
+        "test/e2e-scenario/runtime/lib",
+        "test/e2e-scenario/runtime/reports"
+      ],
+      "domain": "scenario-runner",
+      "ownerIssue": "#4357",
+      "status": "not-migrated",
+      "replacementSurface": "test/e2e-scenario/framework",
+      "targetVitestScenarios": [],
+      "bridgeProbes": [],
+      "retiredReason": "",
+      "deletionReady": false,
+      "notes": "Runtime helper libraries stay only while bridge shell workers need shared environment, logging, context, teardown, or report rendering behavior."
+    }
   ]
 }