v0 Wave M: Conformance suite (§34 acceptance gate)

.act-replies-50.tsv

@@ -0,0 +1,26 @@
+PRRT_kwDOTyoI9s6Yx-Wp	Replaced hardcoded `passed: true` with a network-blocking wrapper around `GithubTarget.compile`/`GitlabTarget.compile`; the check now fails if the network is touched or diagnostics are emitted.
+PRRT_kwDOTyoI9s6Yx-Wq	Removed the tautology. The §34.3 check now calls `target.compile()`, validates that artifacts were produced, that the YAML contains the expected marker, and that no diagnostics were emitted.
+PRRT_kwDOTyoI9s6Yx-Wt	Engine execution now uses `os.tmpdir()` and `fs.mkdtemp` for the workspace, and cleans up with `rm(..., { recursive: true })` in a `finally` block.
+PRRT_kwDOTyoI9s6Yx-Wx	Test workspaces now use `os.tmpdir()`/`mkdtemp` with per-test `beforeEach`/`afterEach` cleanup instead of hardcoded `/tmp` paths.
+PRRT_kwDOTyoI9s6Yx-W0	Test workspaces now use `os.tmpdir()`/`mkdtemp` with per-test `beforeEach`/`afterEach` cleanup instead of hardcoded `/tmp` paths.
+PRRT_kwDOTyoI9s6Yx_jH	Removed the tautology. The §34.3 check now calls `target.compile()`, validates that artifacts were produced, that the YAML contains the expected marker, and that no diagnostics were emitted.
+PRRT_kwDOTyoI9s6Yx_jK	Seed pipeline now declares scalar and artifact outputs (`lint.status`, `build.dist`), value/artifact inputs on dependent steps, and a context condition on `test`.
+PRRT_kwDOTyoI9s6Yx_jM	`runConformance` is now composed of focused per-criterion helper functions (`checkAuthoring`, `checkGraphEquivalence`, `checkEngineExecution`, `checkScalarFlow`, etc.).
+PRRT_kwDOTyoI9s6Yx_jO	Extracted shared inputs, commands, references, and outputs into module-level constants and reused them across the Construct, SDK, and Decorator seeds.
+PRRT_kwDOTyoI9s6Yx_jT	Serialization round-trip now canonicalizes the full graph (sorted object keys and arrays) and compares the entire structure, not just the project ID.
+PRRT_kwDOTyoI9s6Yx_jW	Engine execution now uses `os.tmpdir()` and `fs.mkdtemp` for the workspace, and cleans up with `rm(..., { recursive: true })` in a `finally` block.
+PRRT_kwDOTyoI9s6Yx_4S	`makeDriver` now uses `createAllowlist(["sh"])` so the seed shell scripts are allowed to execute.
+PRRT_kwDOTyoI9s6Yx_4Z	The acceptance-criteria coverage test now asserts the presence of §34.1–§34.11 and the serialization round-trip result by name prefix.
+PRRT_kwDOTyoI9s6Yx_4e	Serialization round-trip now canonicalizes the full graph (sorted object keys and arrays) and compares the entire structure, not just the project ID.
+PRRT_kwDOTyoI9s6Yx_4n	The execution check now asserts `run-completed.status === "success"` and verifies `step-succeeded` events for `ci/lint`, `ci/build`, and `ci/test`.
+PRRT_kwDOTyoI9s6Yx_4q	Implemented a dedicated cycle-detection check that creates a cyclic `Project`/`Pipeline`, calls `synthesize`, and asserts a `SynthesisError` with code `CYCLE`.
+PRRT_kwDOTyoI9s6Yx_4x	The SDK seed now uses `sdkPipeline` with `sh` builders rather than constructing `Pipeline`/`Entry` directly from `@sverka/constructs`.
+PRRT_kwDOTyoI9s6YyCsw	Spec updated: `runConformance` is async, the seed pipeline fence uses `text`, and §34.12 is kept in Non-goals/future work instead of the acceptance gate.
+PRRT_kwDOTyoI9s6YyCs3	The execution check now asserts `run-completed.status === "success"` and verifies `step-succeeded` events for `ci/lint`, `ci/build`, and `ci/test`.
+PRRT_kwDOTyoI9s6YyCs_	Test workspaces now use `os.tmpdir()`/`mkdtemp` with per-test `beforeEach`/`afterEach` cleanup instead of hardcoded `/tmp` paths.
+PRRT_kwDOTyoI9s6YyCtC	Removed the tautology. The §34.3 check now calls `target.compile()`, validates that artifacts were produced, that the YAML contains the expected marker, and that no diagnostics were emitted.
+PRRT_kwDOTyoI9s6Y5KUE	The acceptance-criteria coverage test now asserts the presence of §34.1–§34.11 and the serialization round-trip result by name prefix.
+PRRT_kwDOTyoI9s6Y5KUJ	The execution check now asserts `run-completed.status === "success"` and verifies `step-succeeded` events for `ci/lint`, `ci/build`, and `ci/test`.
+PRRT_kwDOTyoI9s6Y5KUM	Spec updated: §34.12 is explicitly listed in Non-goals as future work and removed from the §34 acceptance-criteria mapping.
+PRRT_kwDOTyoI9s6Y5KUQ	Spec updated: `runConformance` signature is now `Promise<readonly ConformanceResult[]>`, matching the public async implementation.
+PRRT_kwDOTyoI9s6Y5KUc	Spec updated: the seed pipeline code block now uses the `text` fence language.

packages/conformance/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@sverka/conformance",
+  "version": "0.0.0",
+  "type": "module",
+  "main": "./dist/index.mjs",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": "./dist/index.mjs"
+    }
+  },
+  "files": ["dist"],
+  "scripts": {
+    "build": "tsdown",
+    "test": "vitest run",
+    "lint": "eslint src",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@sverka/constructs": "workspace:*",
+    "@sverka/sdk": "workspace:*",
+    "@sverka/decorators": "workspace:*",
+    "@sverka/core": "workspace:*",
+    "@sverka/ir": "workspace:*",
+    "@sverka/planner": "workspace:*",
+    "@sverka/engine-native": "workspace:*",
+    "@sverka/runtime-host": "workspace:*",
+    "@sverka/github": "workspace:*",
+    "@sverka/gitlab": "workspace:*"
+  },
+  "devDependencies": {
+    "tsdown": "^0.22.0",
+    "typescript": "^5.8.0",
+    "vitest": "^3.0.0",
+    "yaml": "^2.9.0",
+    "@sverka/plugin": "workspace:*"
+  }
+}

packages/conformance/src/__tests__/conformance.test.ts

@@ -0,0 +1,285 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { parse } from "yaml";
+import { synthesize } from "@sverka/core";
+import { serializeGraph, deserializeGraph, validateGraphSchema } from "@sverka/ir";
+import { bindRunPlan } from "@sverka/planner";
+import { createEngine } from "@sverka/engine-native";
+import { createHostDriver, createAllowlist } from "@sverka/runtime-host";
+import { GithubTarget } from "@sverka/github";
+import { GitlabTarget } from "@sverka/gitlab";
+import { analyzeCapabilities } from "@sverka/plugin";
+import { githubCapabilities } from "@sverka/github";
+import { gitlabCapabilities } from "@sverka/gitlab";
+import type { RunEvent } from "@sverka/engine-native";
+import {
+  createSeedWithConstructs,
+  createSeedWithSDK,
+  createSeedWithDecorators,
+  runConformance,
+  canonicalize,
+} from "../index.js";
+
+// Helper for host driver config — allow `sh` scripts used by the seed.
+function makeDriver() {
+  return createHostDriver({
+    enabled: true,
+    allowlist: createAllowlist(["sh"]),
+    envAllowlist: [],
+  });
+}
+
+// Normalize a graph for stable comparison.
+function normalize(g: unknown): string {
+  return JSON.stringify(canonicalize(g));
+}
+
+// §33.1 — Authoring conformance
+describe("§33.1 Authoring conformance", () => {
+  it("all 3 APIs produce the same Definition Graph", () => {
+    const g1 = synthesize(createSeedWithConstructs());
+    const g2 = synthesize(createSeedWithSDK());
+    const g3 = synthesize(createSeedWithDecorators());
+
+    expect(normalize(g1)).toEqual(normalize(g2));
+    expect(normalize(g2)).toEqual(normalize(g3));
+  });
+
+  it("seed pipeline has 3 steps", () => {
+    const graph = synthesize(createSeedWithConstructs());
+    expect(graph.project.pipelines[0]?.steps).toHaveLength(3);
+  });
+
+  it("seed pipeline has 1 entry", () => {
+    const graph = synthesize(createSeedWithConstructs());
+    expect(graph.project.pipelines[0]?.entries).toHaveLength(1);
+  });
+
+  it("seed pipeline has push trigger", () => {
+    const graph = synthesize(createSeedWithConstructs());
+    expect(graph.project.pipelines[0]?.entries[0]?.trigger.kind).toBe("push");
+  });
+
+  it("seed pipeline has nodeVersion input", () => {
+    const graph = synthesize(createSeedWithConstructs());
+    expect(graph.project.pipelines[0]?.inputs.nodeVersion).toBeDefined();
+    expect(graph.project.pipelines[0]?.inputs.nodeVersion?.default).toBe("22");
+  });
+
+  it("seed pipeline has scalar and artifact data flow", () => {
+    const graph = synthesize(createSeedWithConstructs());
+    const steps = graph.project.pipelines[0]!.steps;
+
+    const lint = steps.find((s) => s.id === "ci/lint")!;
+    const build = steps.find((s) => s.id === "ci/build")!;
+    const test = steps.find((s) => s.id === "ci/test")!;
+
+    expect(lint.operations.some((op) => op.kind === "exportOutput")).toBe(true);
+    expect(
+      build.dependencies.some(
+        (d) => d.kind === "value" && d.producer === "ci/lint" && d.output === "status",
+      ),
+    ).toBe(true);
+
+    expect(build.operations.some((op) => op.kind === "exportArtifact")).toBe(true);
+    expect(
+      test.dependencies.some(
+        (d) => d.kind === "artifact" && d.producer === "ci/build" && d.output === "dist",
+      ),
+    ).toBe(true);
+  });
+});
+
+// §33.2 — Target conformance
+describe("§33.2 Target conformance", () => {
+  it("GitHub lowering produces valid YAML with jobs", () => {
+    const graph = synthesize(createSeedWithConstructs());
+    const target = new GithubTarget();
+    const targetGraph = target.lower(graph);
+    const artifacts = target.emit(targetGraph);
+    expect(artifacts).toHaveLength(1);
+    const yaml = parse(artifacts[0]!.content) as Record<string, { jobs?: Record<string, unknown> }>;
+    expect(yaml.jobs).toBeDefined();
+    expect(Object.keys(yaml.jobs!)).toHaveLength(3);
+  });
+
+  it("GitLab lowering produces valid YAML with script", () => {
+    const graph = synthesize(createSeedWithConstructs());
+    const target = new GitlabTarget();
+    const targetGraph = target.lower(graph);
+    const artifacts = target.emit(targetGraph);
+    expect(artifacts).toHaveLength(1);
+    const yaml = parse(artifacts[0]!.content) as Record<string, { script?: unknown; stages?: unknown }>;
+    expect(yaml.stages).toBeDefined();
+    for (const jobId of Object.keys(yaml)) {
+      if (jobId === "stages" || jobId === "variables") continue;
+      expect(yaml[jobId]?.script).toBeDefined();
+    }
+  });
+
+  it("GitHub lowering maps dependencies to needs", () => {
+    const graph = synthesize(createSeedWithConstructs());
+    const target = new GithubTarget();
+    const targetGraph = target.lower(graph);
+    const buildJob = targetGraph.jobs.find((j) => j.id === "build");
+    expect(buildJob?.needs).toContain("lint");
+  });
+
+  it("GitLab lowering maps dependencies to needs", () => {
+    const graph = synthesize(createSeedWithConstructs());
+    const target = new GitlabTarget();
+    const targetGraph = target.lower(graph);
+    const buildJob = targetGraph.jobs.find((j) => j.id === "build");
+    expect(buildJob?.needs).toContain("lint");
+  });
+});
+
+// §33.3 — Engine conformance
+describe("§33.3 Engine conformance", () => {
+  let testDir: string;
+
+  beforeEach(async () => {
+    testDir = await mkdtemp(join(tmpdir(), "sverka-engine-"));
+  });
+
+  afterEach(async () => {
+    await rm(testDir, { recursive: true, force: true });
+  });
+
+  it("native engine executes seed pipeline successfully", async () => {
+    const graph = synthesize(createSeedWithConstructs());
+    const plan = bindRunPlan({
+      graph,
+      entryId: "ci/on-push",
+      inputs: {},
+    });
+    const engine = createEngine({
+      drivers: [makeDriver()],
+    });
+    const events: RunEvent[] = [];
+    for await (const event of engine.run({
+      plan,
+      workspace: testDir,
+      artifactDir: join(testDir, "artifacts"),
+    })) {
+      events.push(event);
+    }
+    const completed = events.find((e) => e.type === "run-completed");
+    expect(completed).toBeDefined();
+    expect(completed!.status).toBe("success");
+
+    for (const stepId of ["ci/lint", "ci/build", "ci/test"]) {
+      expect(
+        events.some((e) => e.type === "step-succeeded" && e.stepId === stepId),
+      ).toBe(true);
+    }
+  });
+});
+
+// Full pipeline: Project → Graph → RunPlan → Engine → Events
+describe("Full pipeline: Project → Graph → RunPlan → Engine → Events", () => {
+  let testDir: string;
+
+  beforeEach(async () => {
+    testDir = await mkdtemp(join(tmpdir(), "sverka-e2e-"));
+  });
+
+  afterEach(async () => {
+    await rm(testDir, { recursive: true, force: true });
+  });
+
+  it("end-to-end execution produces success events", async () => {
+    const proj = createSeedWithConstructs();
+    const graph = synthesize(proj);
+    const plan = bindRunPlan({ graph, entryId: "ci/on-push", inputs: {} });
+    const engine = createEngine({
+      drivers: [makeDriver()],
+    });
+    const events: RunEvent[] = [];
+    for await (const event of engine.run({
+      plan,
+      workspace: testDir,
+      artifactDir: join(testDir, "artifacts"),
+    })) {
+      events.push(event);
+    }
+    expect(events.length).toBeGreaterThan(0);
+    const completed = events.find((e) => e.type === "run-completed");
+    expect(completed?.status).toBe("success");
+  });
+});
+
+// Full compilation: Project → Graph → Target → YAML
+describe("Full compilation: Project → Graph → Target → YAML", () => {
+  it("end-to-end compilation produces YAML artifacts", () => {
+    const proj = createSeedWithConstructs();
+    const graph = synthesize(proj);
+
+    const ghTarget = new GithubTarget();
+    const ghArtifacts = ghTarget.emit(ghTarget.lower(graph));
+    expect(ghArtifacts[0]?.content).toContain("jobs:");
+
+    const glTarget = new GitlabTarget();
+    const glArtifacts = glTarget.emit(glTarget.lower(graph));
+    expect(glArtifacts[0]?.content).toContain("script:");
+  });
+});
+
+// Serialization round-trip
+describe("Serialization round-trip", () => {
+  it("serialize → deserialize → same graph", () => {
+    const graph = synthesize(createSeedWithConstructs());
+    const json = serializeGraph(graph);
+    const restored = deserializeGraph(json);
+    validateGraphSchema(restored);
+    expect(normalize(restored.graph)).toEqual(normalize(graph));
+  });
+});
+
+// Capability conformance
+describe("§33.4 Capability conformance", () => {
+  it("seed pipeline has no capability diagnostics", () => {
+    const graph = synthesize(createSeedWithConstructs());
+    const diags = analyzeCapabilities(graph, [githubCapabilities, gitlabCapabilities]);
+    expect(diags).toHaveLength(0);
+  });
+});
+
+// §34 acceptance gate
+describe("§34 Acceptance gate — runConformance", () => {
+  it("all conformance checks pass", async () => {
+    const results = await runConformance();
+    const failures = results.filter((r) => !r.passed);
+    for (const r of results) {
+      expect(r.passed).toBe(true);
+    }
+    expect(failures).toHaveLength(0);
+  });
+
+  it("conformance covers all §34 criteria", async () => {
+    const results = await runConformance();
+    for (const criterion of [
+      "§34.1",
+      "§34.2",
+      "§34.3",
+      "§34.4",
+      "§34.5",
+      "§34.6",
+      "§34.7",
+      "§34.8",
+      "§34.9",
+      "§34.10",
+      "§34.11",
+    ]) {
+      expect(
+        results.some((r) => r.name.startsWith(criterion)),
+        `missing conformance criterion: ${criterion} (got: ${results.map((r) => r.name).join(", ")})`,
+      ).toBe(true);
+    }
+    expect(
+      results.some((r) => r.name.startsWith("Serialization round-trip")),
+    ).toBe(true);
+  });
+});