Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2570,6 +2570,7 @@ jobs:
${{ steps.workspace.outputs.work_dir }}/cleanup.json

live:
name: ${{ matrix.label }}
needs: [base-image-publication, generate-matrix]
if: ${{ needs.generate-matrix.outputs.matrix != '[]' }}
runs-on: ${{ matrix.runner }}
Expand Down Expand Up @@ -2683,7 +2684,7 @@ jobs:
TARGET_ID: ${{ matrix.id }}
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/registry-targets.test.ts --selector "^${TARGET_ID}$"
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/registry-targets.test.ts --selector "^${TARGET_ID}:"

# The sanitizer reads raw traces only after checking the workflow-owned
# runner-temp path, then writes the timing-only file into upload roots.
Expand Down
2 changes: 1 addition & 1 deletion src/lib/onboard/dockerfile-patch.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -707,7 +707,7 @@ describe("dockerfile patch helpers", () => {
{ channelId: "telegram", active: true },
],
agentRender: [
{ agent: "openclaw", channelId: "discord", target: "config", path: ["discord"] },
{ agent: "openclaw", channelId: "discord", target: "openclaw.json", path: ["discord"] },
],
});
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-dockerfile-plan-"));
Expand Down
9 changes: 8 additions & 1 deletion test/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,12 @@ Each execution row declares three coverage fields:
- `observableOutcome` names the behavior that produces the evidence. Catalogue targets use their outcome-oriented `displayName` as this value.
- `environmentOrInferenceEndpoint` names the host boundary or inference endpoint that distinguishes the evidence.

Typed registry tests derive each human-readable execution title as
`<observableOutcome> [<agentRuntime>; <environmentOrInferenceEndpoint>]`.
Typed registry tests prefix that title with the stable target ID. Workflows use
the ID prefix for selection, while the semantic tuple makes the test purpose and
evidence boundary visible in Vitest and GitHub Actions.

Keep coverage metadata with the execution owner:

- Catalogue targets declare it in `tools/e2e/target-catalogue.mts`.
Expand All @@ -365,7 +371,8 @@ multiple rows. `tools/e2e/workflow-plan.mts` composes and validates these source
Do not add a separate hand-maintained execution list.

The default coverage matrix excludes explicit-only jobs and inert typed-registry declarations.
The rendered report lists those categories separately.
The rendered report lists those categories separately and inventories every typed declaration,
including declarations that have no executable matrix cell.
Explicit-only rows keep their coverage dimensions but do not join the default release matrix.
Inert declarations report unresolved coverage fields and the missing executable ownership.

Expand Down
9 changes: 5 additions & 4 deletions test/e2e/docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,11 @@ dispatch shell validation suites.

## Selecting One Target

`.github/workflows/e2e.yaml` runs one matrix target by passing its id through
`TARGET_ID` and selecting the matching test with `-t "^${TARGET_ID}$"`. The
selector performs the restriction; `TARGET_ID` alone does not limit which
targets run.
`.github/workflows/e2e.yaml` runs one matrix target by passing its ID through
`TARGET_ID`. The workflow selects the test title with the stable
`-t "^${TARGET_ID}:"` prefix. The title suffix contains the observable outcome,
agent runtime, and environment or inference endpoint. The selector performs the
restriction; `TARGET_ID` alone does not limit which targets run.

The `generate-matrix` job resolves dispatch input through `requireTargets`, so
an unknown id fails there before any target job starts.
Expand Down
8 changes: 4 additions & 4 deletions test/e2e/live/registry-targets.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import {
readRegistrySandboxEntry,
} from "../fixtures/phases/index.ts";
import { listTargets, requireTargets } from "../registry/registry.ts";
import { liveTargetSupport, liveTargetTestName } from "../registry/runtime-support.ts";
import { liveTargetSupport, liveTargetTestTitle } from "../registry/runtime-support.ts";
import { cloudExperimentalChecksForOnboarding } from "./cloud-experimental-check-list.ts";
import { runE2eCloudExperimentalChecks } from "./cloud-experimental-checks.ts";
import {
Expand Down Expand Up @@ -42,7 +42,7 @@ const E2E_CLOUD_EXPERIMENTAL_CHECKS_DIR = path.join(
);
process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT;

// The workflow filters by target ID via `-t "^${TARGET_ID}$"`.
// The workflow filters by the stable target ID prefix via `-t "^${TARGET_ID}:"`.
// When that env is set, surface the structured `[not wired]` reason for the
// targeted unsupported target at module load so the job log/summary
// captures it before Vitest reports the skipped test by ID.
Expand Down Expand Up @@ -77,15 +77,15 @@ for (const [targetIndex, target] of listTargets().entries()) {
console.warn(`[not wired] ${target.id}: ${support.reasons.join("; ")}`);
}
test.skip(
liveTargetTestName(target),
liveTargetTestTitle(target, support),
{ meta: { e2ePhases: REGISTRY_TARGET_PHASES } },
() => {},
);
continue;
}

test(
liveTargetTestName(target),
liveTargetTestTitle(target, support),
{ meta: { e2ePhases: REGISTRY_TARGET_PHASES } },
async ({
artifacts,
Expand Down
90 changes: 37 additions & 53 deletions test/e2e/registry/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,16 @@
import { realpathSync } from "node:fs";
import { fileURLToPath } from "node:url";

import {
type E2eAgentRuntime,
validateE2eExecutionMetadata,
} from "../../../tools/e2e/execution-coverage.mts";
import type { E2eExecutionMetadata } from "../../../tools/e2e/execution-coverage.mts";

import { listTargets, requireTargets } from "./registry.ts";
import { resolveRunnerForTarget } from "./runner-routing.ts";
import { type LiveTargetSupport, liveTargetSupport } from "./runtime-support.ts";
import {
liveTargetExecutionCoverage,
type LiveTargetSupport,
liveTargetSupport,
liveTargetTestTitle,
} from "./runtime-support.ts";
import type { TargetDefinition } from "./types.ts";

interface Args {
Expand All @@ -20,12 +22,13 @@ interface Args {
targets: string[];
}

export interface LiveTargetMatrixEntry {
export interface LiveTargetInventoryEntry extends E2eExecutionMetadata {
id: string;
agentRuntime: E2eAgentRuntime;
observableOutcome: string;
environmentOrInferenceEndpoint: string;
unresolvedReason: string;
supported: boolean;
supportReasons: string[];
}

export interface LiveTargetMatrixEntry extends LiveTargetInventoryEntry {
runner: string;
label: string;
platform: string;
Expand All @@ -35,8 +38,6 @@ export interface LiveTargetMatrixEntry {
expectedStateId: string;
suites: string[];
requiredSecrets: string[];
supported: boolean;
supportReasons: string[];
pendingRuntimeSuites: string[];
}

Expand Down Expand Up @@ -80,67 +81,50 @@ function printList() {
}
}

function buildLabel(target: TargetDefinition): string {
const platform = target.environment?.platform ?? "unknown-platform";
const suites = target.suiteIds ?? [];
if (target.expectedFailure) {
const cls = target.expectedFailure.errorClass ?? "expected-failure";
return `${platform} · ${target.id} · expect-fail:${cls}`;
}
if (suites.length === 0) {
return `${platform} · ${target.id}`;
}
if (suites.length <= 3) {
return `${platform} · ${target.id} · ${suites.join("+")}`;
}
return `${platform} · ${target.id} · ${suites.length} suites`;
}

function liveMatrixEntry(
target: TargetDefinition,
support: LiveTargetSupport,
): LiveTargetMatrixEntry {
const { runner } = resolveRunnerForTarget(target);
if (support.supported && !target.executionCoverage) {
throw new Error(
`Executable typed E2E target ${target.id} requires execution coverage metadata`,
);
}
const executionCoverage = validateE2eExecutionMetadata(
target.executionCoverage ?? {
agentRuntime: "unresolved",
observableOutcome: "unresolved",
environmentOrInferenceEndpoint: "unresolved",
unresolvedReason: "This typed registry declaration has no executable owner",
},
`Typed E2E target ${target.id}`,
);
return {
id: target.id,
...executionCoverage,
...liveTargetInventoryEntry(target, support),
runner,
label: buildLabel(target),
label: liveTargetTestTitle(target, support),
platform: target.environment?.platform ?? "unknown",
install: target.environment?.install ?? "unknown",
runtime: target.environment?.runtime ?? "unknown",
onboarding: target.environment?.onboarding ?? "unknown",
expectedStateId: target.expectedStateId ?? "",
suites: target.suiteIds ?? [],
requiredSecrets: target.requiredSecrets ?? [],
pendingRuntimeSuites: support.pendingRuntimeSuites,
};
}

export function liveTargetInventoryEntry(
target: TargetDefinition,
support = liveTargetSupport(target),
): LiveTargetInventoryEntry {
return {
id: target.id,
...liveTargetExecutionCoverage(target, support),
supported: support.supported,
supportReasons: support.reasons,
pendingRuntimeSuites: support.pendingRuntimeSuites,
};
}

export function buildLiveTargetInventory(): LiveTargetInventoryEntry[] {
return listTargets().map((target) => liveTargetInventoryEntry(target));
}

export function buildLiveTargetMatrix(ids: string[] = []): LiveTargetMatrixEntry[] {
const targetSupport = (ids.length > 0 ? requireTargets(ids) : listTargets()).map((target) => ({
target,
support: liveTargetSupport(target),
}));
const liveEntries =
ids.length > 0 ? targetSupport : targetSupport.filter(({ support }) => support.supported);
return liveEntries.map(({ target, support }) => liveMatrixEntry(target, support));
if (ids.length === 0) {
return listTargets().flatMap((target) => {
const support = liveTargetSupport(target);
return support.supported ? [liveMatrixEntry(target, support)] : [];
});
}
return requireTargets(ids).map((target) => liveMatrixEntry(target, liveTargetSupport(target)));
}

function emitLiveMatrix(ids: string[]) {
Expand Down
47 changes: 36 additions & 11 deletions test/e2e/registry/runtime-support.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import {
e2eExecutionTitle,
type E2eExecutionMetadata,
validateE2eExecutionMetadata,
} from "../../../tools/e2e/execution-coverage.mts";
import type { TargetDefinition } from "./types.ts";

const SUPPORTED_PLATFORMS = new Set(["ubuntu-local"]);
Expand All @@ -25,17 +30,6 @@ export interface LiveTargetSupport {
pendingRuntimeSuites: string[];
}

/**
* Canonical name under which a target is registered with Vitest in the
* live registry-targets test file. The workflow filters by exact ID via
* `-t "^${TARGET_ID}$"`, so both supported and unsupported targets MUST
* be registered under this exact name. Skip reasons are surfaced via the
* job log instead of the test name suffix.
*/
export function liveTargetTestName(target: TargetDefinition): string {
return target.id;
}

export function liveTargetSupport(target: TargetDefinition): LiveTargetSupport {
const reasons: string[] = [];
const environment = target.environment;
Expand Down Expand Up @@ -71,3 +65,34 @@ export function liveTargetSupport(target: TargetDefinition): LiveTargetSupport {
pendingRuntimeSuites: target.suiteIds ?? [],
};
}

export function liveTargetExecutionCoverage(
target: TargetDefinition,
support = liveTargetSupport(target),
): E2eExecutionMetadata {
if (support.supported && !target.executionCoverage) {
throw new Error(
`Executable typed E2E target ${target.id} requires execution coverage metadata`,
);
}
return validateE2eExecutionMetadata(
target.executionCoverage ?? {
agentRuntime: "unresolved",
observableOutcome: "unresolved",
environmentOrInferenceEndpoint: "unresolved",
unresolvedReason: "This typed registry declaration has no executable owner",
},
`Typed E2E target ${target.id}`,
);
}

/**
* The stable target ID remains the workflow selector. The semantic tuple
* supplies the human-visible Vitest title without changing target identity.
*/
export function liveTargetTestTitle(
target: TargetDefinition,
support = liveTargetSupport(target),
): string {
return `${target.id}: ${e2eExecutionTitle(liveTargetExecutionCoverage(target, support))}`;
}
15 changes: 15 additions & 0 deletions test/e2e/support/e2e-live-registry-discovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest";

import { buildLiveTargetRunPlan } from "../live/run-plan.ts";
import { target } from "../registry/builder.ts";
import { liveTargetInventoryEntry } from "../registry/run.ts";
import { liveTargetSupport } from "../registry/runtime-support.ts";
import type { TargetDefinition, TargetEnvironment } from "../registry/types.ts";

Expand Down Expand Up @@ -79,6 +80,20 @@ describe("live target registry discovery support", () => {
});
});

it("reports a missing-environment declaration without resolving a runner (#9167)", () => {
const declaration = target("synthetic-no-environment").expectedState("synthetic-ready").build();

expect(liveTargetInventoryEntry(declaration)).toEqual({
id: declaration.id,
agentRuntime: "unresolved",
observableOutcome: "unresolved",
environmentOrInferenceEndpoint: "unresolved",
unresolvedReason: "This typed registry declaration has no executable owner",
supported: false,
supportReasons: ["missing environment"],
});
});

it("compiles a run plan from synthetic target behavior", () => {
const registered = syntheticTarget();

Expand Down
Loading
Loading