Skip to content

test(e2e): assert the generate matrix covers every declared tool target #2116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dyoshikawa merged 1 commit into from
Jul 6, 2026
Merged

test(e2e): assert the generate matrix covers every declared tool target #2116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 113 additions & 90 deletions src/e2e/e2e-commands.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,60 +3,110 @@ import { join } from "node:path";
import { describe, expect, it } from "vitest";

import { RULESYNC_COMMANDS_RELATIVE_DIR_PATH } from "../constants/rulesync-paths.js";
import { CommandsProcessor } from "../features/commands/commands-processor.js";
import { readFileContent, writeFileContent } from "../utils/file.js";
import {
assertGenerateMatrixCoversTargets,
runGenerate,
runImport,
useGlobalTestDirectories,
useTestDirectory,
} from "./e2e-helper.js";

const commandsGenerateTargets = [
{ target: "claudecode", outputPath: join(".claude", "commands", "review-pr.md") },
{ target: "cursor", outputPath: join(".cursor", "commands", "review-pr.md") },
{ target: "augmentcode", outputPath: join(".augment", "commands", "review-pr.md") },
{ target: "copilot", outputPath: join(".github", "prompts", "review-pr.prompt.md") },
{ target: "opencode", outputPath: join(".opencode", "commands", "review-pr.md") },
{ target: "cline", outputPath: join(".clinerules", "workflows", "review-pr.md") },
{ target: "kilo", outputPath: join(".kilo", "commands", "review-pr.md") },
{ target: "roo", outputPath: join(".roo", "commands", "review-pr.md") },
{ target: "kiro", outputPath: join(".kiro", "prompts", "review-pr.md") },
{ target: "kiro-cli", outputPath: join(".kiro", "prompts", "review-pr.md") },
{ target: "kiro-ide", outputPath: join(".kiro", "prompts", "review-pr.md") },
{ target: "antigravity-ide", outputPath: join(".agents", "workflows", "review-pr.md") },
{ target: "antigravity-cli", outputPath: join(".agents", "workflows", "review-pr.md") },
{ target: "junie", outputPath: join(".junie", "commands", "review-pr.md") },
{ target: "takt", outputPath: join(".takt", "facets", "instructions", "review-pr.md") },
{ target: "pi", outputPath: join(".pi", "prompts", "review-pr.md") },
{ target: "devin", outputPath: join(".devin", "workflows", "review-pr.md") },
{ target: "factorydroid", outputPath: join(".factory", "commands", "review-pr.md") },
{ target: "goose", outputPath: join(".goose", "recipes", "review-pr.yaml") },
{ target: "qwencode", outputPath: join(".qwen", "commands", "review-pr.md") },
{ target: "reasonix", outputPath: join(".reasonix", "commands", "review-pr.md") },
{ target: "rovodev", outputPath: join(".rovodev", "prompts", "review-pr.md") },
] as const;

const commandsGlobalTargets = [
{ target: "claudecode", outputPath: join(".claude", "commands", "review-pr.md") },
{ target: "cursor", outputPath: join(".cursor", "commands", "review-pr.md") },
{ target: "augmentcode", outputPath: join(".augment", "commands", "review-pr.md") },
{ target: "opencode", outputPath: join(".config", "opencode", "commands", "review-pr.md") },
{ target: "codexcli", outputPath: join(".codex", "prompts", "review-pr.md") },
{ target: "cline", outputPath: join("Documents", "Cline", "Workflows", "review-pr.md") },
{ target: "kilo", outputPath: join(".config", "kilo", "commands", "review-pr.md") },
{ target: "junie", outputPath: join(".junie", "commands", "review-pr.md") },
{ target: "kiro-cli", outputPath: join(".kiro", "prompts", "review-pr.md") },
{
target: "antigravity-ide",
outputPath: join(".gemini", "antigravity", "global_workflows", "review-pr.md"),
},
{
target: "antigravity-cli",
outputPath: join(".gemini", "antigravity-cli", "global_workflows", "review-pr.md"),
},
{
target: "takt",
outputPath: join(".takt", "facets", "instructions", "review-pr.md"),
},
{ target: "pi", outputPath: join(".pi", "agent", "prompts", "review-pr.md") },
{
target: "devin",
outputPath: join(".codeium", "windsurf", "global_workflows", "review-pr.md"),
},
{ target: "factorydroid", outputPath: join(".factory", "commands", "review-pr.md") },
{ target: "goose", outputPath: join(".config", "goose", "recipes", "review-pr.yaml") },
{ target: "qwencode", outputPath: join(".qwen", "commands", "review-pr.md") },
// Hermes Agent has no project-scoped command location; commands are emitted
// as Hermes skills under ~/.hermes/skills/<slug>/SKILL.md (global only).
{ target: "hermesagent", outputPath: join(".hermes", "skills", "review-pr", "SKILL.md") },
{ target: "reasonix", outputPath: join(".reasonix", "commands", "review-pr.md") },
{ target: "rovodev", outputPath: join(".rovodev", "prompts", "review-pr.md") },
] as const;

describe("E2E: commands", () => {
const { getTestDir } = useTestDirectory();

it.each([
{ target: "claudecode", outputPath: join(".claude", "commands", "review-pr.md") },
{ target: "cursor", outputPath: join(".cursor", "commands", "review-pr.md") },
{ target: "augmentcode", outputPath: join(".augment", "commands", "review-pr.md") },
{ target: "copilot", outputPath: join(".github", "prompts", "review-pr.prompt.md") },
{ target: "opencode", outputPath: join(".opencode", "commands", "review-pr.md") },
{ target: "cline", outputPath: join(".clinerules", "workflows", "review-pr.md") },
{ target: "kilo", outputPath: join(".kilo", "commands", "review-pr.md") },
{ target: "roo", outputPath: join(".roo", "commands", "review-pr.md") },
{ target: "kiro", outputPath: join(".kiro", "prompts", "review-pr.md") },
{ target: "antigravity-ide", outputPath: join(".agents", "workflows", "review-pr.md") },
{ target: "antigravity-cli", outputPath: join(".agents", "workflows", "review-pr.md") },
{ target: "junie", outputPath: join(".junie", "commands", "review-pr.md") },
{ target: "takt", outputPath: join(".takt", "facets", "instructions", "review-pr.md") },
{ target: "pi", outputPath: join(".pi", "prompts", "review-pr.md") },
{ target: "devin", outputPath: join(".devin", "workflows", "review-pr.md") },
{ target: "factorydroid", outputPath: join(".factory", "commands", "review-pr.md") },
{ target: "goose", outputPath: join(".goose", "recipes", "review-pr.yaml") },
{ target: "qwencode", outputPath: join(".qwen", "commands", "review-pr.md") },
{ target: "reasonix", outputPath: join(".reasonix", "commands", "review-pr.md") },
{ target: "rovodev", outputPath: join(".rovodev", "prompts", "review-pr.md") },
])("should generate $target commands", async ({ target, outputPath }) => {
const testDir = getTestDir();
it("generate matrix must cover every native commands tool target", () => {
assertGenerateMatrixCoversTargets({
processor: CommandsProcessor,
testedTargets: commandsGenerateTargets.map((e) => e.target),
});
});

// Setup: Create .rulesync/commands/review-pr.md
const commandContent = `---
it.each(commandsGenerateTargets)(
"should generate $target commands",
async ({ target, outputPath }) => {
const testDir = getTestDir();

const commandContent = `---
description: "Review a pull request"
targets: ["*"]
---
Check the PR diff and provide feedback.
`;
await writeFileContent(
join(testDir, RULESYNC_COMMANDS_RELATIVE_DIR_PATH, "review-pr.md"),
commandContent,
);
await writeFileContent(
join(testDir, RULESYNC_COMMANDS_RELATIVE_DIR_PATH, "review-pr.md"),
commandContent,
);

// Execute: Generate commands for the target
await runGenerate({ target, features: "commands" });
await runGenerate({ target, features: "commands" });

// Verify that the expected output file was generated
const generatedContent = await readFileContent(join(testDir, outputPath));
expect(generatedContent).toContain("Check the PR diff and provide feedback.");
});
const generatedContent = await readFileContent(join(testDir, outputPath));
expect(generatedContent).toContain("Check the PR diff and provide feedback.");
},
);

it("should generate a rovodev prompts.yml manifest alongside the content file", async () => {
const testDir = getTestDir();
Expand Down Expand Up @@ -212,70 +262,43 @@ describe("E2E: commands (import)", () => {
describe("E2E: commands (global mode)", () => {
const { getProjectDir, getHomeDir } = useGlobalTestDirectories();

it.each([
{ target: "claudecode", outputPath: join(".claude", "commands", "review-pr.md") },
{ target: "cursor", outputPath: join(".cursor", "commands", "review-pr.md") },
{ target: "augmentcode", outputPath: join(".augment", "commands", "review-pr.md") },
{ target: "opencode", outputPath: join(".config", "opencode", "commands", "review-pr.md") },
{ target: "codexcli", outputPath: join(".codex", "prompts", "review-pr.md") },
{ target: "cline", outputPath: join("Documents", "Cline", "Workflows", "review-pr.md") },
{ target: "kilo", outputPath: join(".config", "kilo", "commands", "review-pr.md") },
{ target: "junie", outputPath: join(".junie", "commands", "review-pr.md") },
{ target: "kiro-cli", outputPath: join(".kiro", "prompts", "review-pr.md") },
{
target: "antigravity-ide",
outputPath: join(".gemini", "antigravity", "global_workflows", "review-pr.md"),
},
{
target: "antigravity-cli",
outputPath: join(".gemini", "antigravity-cli", "global_workflows", "review-pr.md"),
},
{
target: "takt",
outputPath: join(".takt", "facets", "instructions", "review-pr.md"),
},
{ target: "pi", outputPath: join(".pi", "agent", "prompts", "review-pr.md") },
{
target: "devin",
outputPath: join(".codeium", "windsurf", "global_workflows", "review-pr.md"),
},
{ target: "factorydroid", outputPath: join(".factory", "commands", "review-pr.md") },
{ target: "goose", outputPath: join(".config", "goose", "recipes", "review-pr.yaml") },
{ target: "qwencode", outputPath: join(".qwen", "commands", "review-pr.md") },
// Hermes Agent has no project-scoped command location; commands are emitted
// as Hermes skills under ~/.hermes/skills/<slug>/SKILL.md (global only).
{ target: "hermesagent", outputPath: join(".hermes", "skills", "review-pr", "SKILL.md") },
{ target: "reasonix", outputPath: join(".reasonix", "commands", "review-pr.md") },
{ target: "rovodev", outputPath: join(".rovodev", "prompts", "review-pr.md") },
])("should generate $target commands in home directory", async ({ target, outputPath }) => {
const projectDir = getProjectDir();
const homeDir = getHomeDir();
it("global matrix must cover every native global commands tool target", () => {
assertGenerateMatrixCoversTargets({
processor: CommandsProcessor,
testedTargets: commandsGlobalTargets.map((e) => e.target),
global: true,
});
});

// Setup: Create .rulesync/commands/review-pr.md with root: true
const commandContent = `---
it.each(commandsGlobalTargets)(
"should generate $target commands in home directory",
async ({ target, outputPath }) => {
const projectDir = getProjectDir();
const homeDir = getHomeDir();

const commandContent = `---
root: true
description: "Review a pull request"
targets: ["*"]
---
Check the PR diff and provide feedback.
`;
await writeFileContent(
join(projectDir, RULESYNC_COMMANDS_RELATIVE_DIR_PATH, "review-pr.md"),
commandContent,
);
await writeFileContent(
join(projectDir, RULESYNC_COMMANDS_RELATIVE_DIR_PATH, "review-pr.md"),
commandContent,
);

// Execute: Generate commands in global mode with HOME pointed to temp dir
await runGenerate({
target,
features: "commands",
global: true,
env: { HOME_DIR: homeDir },
});
await runGenerate({
target,
features: "commands",
global: true,
env: { HOME_DIR: homeDir },
});

// Verify that the expected output file was generated
const generatedContent = await readFileContent(join(homeDir, outputPath));
expect(generatedContent).toContain("Check the PR diff and provide feedback.");
});
const generatedContent = await readFileContent(join(homeDir, outputPath));
expect(generatedContent).toContain("Check the PR diff and provide feedback.");
},
);

it("should ignore non-root commands in global mode", async () => {
const projectDir = getProjectDir();
Expand Down
56 changes: 55 additions & 1 deletion src/e2e/e2e-helper.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@ import { execFile } from "node:child_process";
import { join, resolve, sep } from "node:path";
import { promisify } from "node:util";

import { afterEach, beforeEach } from "vitest";
import { afterEach, beforeEach, expect } from "vitest";

import { setupTestDirectory } from "../test-utils/test-directories.js";
import type { ToolTarget } from "../types/tool-targets.js";

// Save original working directory
const originalCwd = process.cwd();
Expand Down Expand Up @@ -159,3 +160,56 @@ export function useGlobalTestDirectories(): {
getHomeDir: () => homeDir,
};
}

type ProcessorTargets = {
getToolTargets(options?: { global?: boolean; importOnly?: boolean }): ToolTarget[];
};

/**
* Asserts that an e2e generate/global happy-path matrix stays in lock-step with
* the tool targets a feature processor actually declares, so adding a tool to a
* processor without wiring it into the matrix (or dropping one) fails CI instead
* of silently eroding coverage.
*
* The declared targets (from `getToolTargets`) must partition exactly into
* `testedTargets` (tools with an entry in the matrix `it.each` dictionary) and
* `untested` (tools intentionally excluded from this matrix — e.g. tools whose
* output only exists in another scope, or that merge into a shared file). Every
* excluded tool must be listed explicitly with a reason so the omission is a
* conscious decision rather than an accidental gap.
*
* `-legacy` targets are dropped from the comparison: they are duplicate aliases
* that the same tables/generators exclude, and are never exercised end-to-end.
*
* Mirrors the "derive from the implementation, fail on drift" idiom already used
* by the TOOL_DISPLAY completeness check and the gitignore derivation.
*/
export function assertGenerateMatrixCoversTargets({
processor,
testedTargets,
untested = [],
global = false,
}: {
processor: ProcessorTargets;
testedTargets: readonly string[];
untested?: readonly string[];
global?: boolean;
}): void {
const declared = processor
.getToolTargets({ global })
.filter((target) => !target.endsWith("-legacy"));
const declaredSet = new Set<string>(declared);

const stray = [...testedTargets, ...untested].filter((t) => !declaredSet.has(t)).toSorted();
expect(
stray,
`These matrix/untested entries are not declared by the processor (stale or mistyped): ${stray.join(", ")}`,
).toEqual([]);

const covered = new Set<string>([...testedTargets, ...untested]);
const uncovered = declared.filter((t) => !covered.has(t)).toSorted();
expect(
uncovered,
`These tools are declared by the processor but missing from the e2e matrix. Add each to the matrix dictionary, or to the \`untested\` list with a reason: ${uncovered.join(", ")}`,
).toEqual([]);
}
Loading
Loading