Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
321 changes: 321 additions & 0 deletions engdocs/architecture/wave-05-runtime-docker-plan.md

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion packages/core/project.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
"test": {
"executor": "nx:run-commands",
"options": {
"command": "bun run vitest run --passWithNoTests",
"command": "bun run vitest run",
"cwd": "packages/core"
}
},
Expand Down
16 changes: 9 additions & 7 deletions packages/core/src/__tests__/composables/workflow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,16 @@ describe("workflow()", () => {
const d = run({ command: "d" });
const wf = workflow("mixed", parallel(a, b), pipeline(c, d));
const result = await wf.plan(makePlanRuntime());
const ids = result.operations.map((o) => o.id);
expect(ids).toContain("run:a");
expect(ids).toContain("run:b");
expect(ids).toContain("run:c");
expect(ids).toContain("run:d");
const byCmd = new Map(result.operations.map((o) => [o.command, o]));
const aSpec = byCmd.get("a")!;
const bSpec = byCmd.get("b")!;
const cSpec = byCmd.get("c")!;
const dSpec = byCmd.get("d")!;
for (const s of [aSpec, bSpec, cSpec, dSpec]) {
expect(s.id).toMatch(/^op-[0-9a-f]{64}$/);
}
// d depends on c
const dSpec = result.operations.find((o) => o.id === "run:d")!;
expect(dSpec.dependsOn).toEqual(["run:c"]);
expect(dSpec.dependsOn).toEqual([cSpec.id]);
});

it("empty workflow plans to zero operations", async () => {
Expand Down
81 changes: 46 additions & 35 deletions packages/core/src/__tests__/dag.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,44 +6,56 @@ import { workflow } from "../composables/workflow.js";
import { CompositionError } from "../errors.js";
import { makePlanRuntime } from "./helpers/runtime.js";

const OP_ID_RE = /^op-[0-9a-f]{64}$/;

describe("DAG validation", () => {
it("rejects a cycle with node ids in context", async () => {
// User-provided dependsOn string ids forming a cycle: x→y→x.
// User-provided spec.id aliases referenced in dependsOn form a cycle: x→y→x.
// (Predecessor-ref cycles are structurally impossible with immutable
// after(); cycles arise from explicit dependsOn ids.)
// after(); cycles arise from explicit dependsOn ids. spec.id values are
// resolved to op- ids during edge resolution, so the cycle is detected on
// the content-addressed ids.)
const x = run({ id: "x", command: "x", dependsOn: ["y"] });
const y = run({ id: "y", command: "y", dependsOn: ["x"] });
const wf = workflow("cyclic", x, y);
const promise = wf.plan(makePlanRuntime());
await expect(promise).rejects.toThrow(CompositionError);
await expect(wf.plan(makePlanRuntime())).rejects.toThrow(CompositionError);
try {
await promise;
await wf.plan(makePlanRuntime());
} catch (err) {
if (!(err instanceof CompositionError)) throw err;
expect(err.context).toBeDefined();
expect(err.context?.["cycle"]).toBeDefined();
const ce = err as CompositionError;
expect(ce.context).toBeDefined();
expect(ce.context?.["cycle"]).toBeDefined();
}
});

it("rejects a dependsOn id that matches no operation", async () => {
const a = run({ id: "a", command: "a", dependsOn: ["ghost"] });
const wf = workflow("unknown-dep", a);
const promise = wf.plan(makePlanRuntime());
await expect(promise).rejects.toThrow(CompositionError);
await expect(promise).rejects.toThrow(/unknown id 'ghost'/);
});

it("rejects duplicate user-provided ids", async () => {
it("rejects duplicate user-provided spec.id aliases", async () => {
// spec.id is folded into the hash (as userId) and also used as a
// dependsOn alias; duplicate aliases are rejected because they would make
// edge resolution ambiguous.
const a = run({ id: "dup", command: "a" });
const b = run({ id: "dup", command: "b" });
const wf = workflow("dup-ids", a, b);
const promise = wf.plan(makePlanRuntime());
await expect(promise).rejects.toThrow(CompositionError);
await expect(wf.plan(makePlanRuntime())).rejects.toThrow(CompositionError);
try {
await wf.plan(makePlanRuntime());
} catch (err) {
const ce = err as CompositionError;
expect(ce.context?.["id"]).toBe("dup");
}
});

it("rejects true duplicate operations (identical kind/name/context)", async () => {
// Two operations with identical content produce the same content-addressed
// op- id; the planner detects the collision and raises CompositionError.
const a = run({ command: "echo" });
const b = run({ command: "echo" });
const wf = workflow("true-dups", a, b);
await expect(wf.plan(makePlanRuntime())).rejects.toThrow(CompositionError);
try {
await promise;
await wf.plan(makePlanRuntime());
} catch (err) {
if (!(err instanceof CompositionError)) throw err;
expect(err.context?.["id"]).toBe("dup");
const ce = err as CompositionError;
expect(ce.context?.["id"]).toMatch(OP_ID_RE);
}
});

Expand All @@ -53,27 +65,26 @@ describe("DAG validation", () => {
const c = run({ command: "c" });
const wf = workflow("linear", pipeline(a, b, c));
const result = await wf.plan(makePlanRuntime());
const byCmd = new Map(result.operations.map((o) => [o.command, o]));
const aSpec = byCmd.get("a")!;
const bSpec = byCmd.get("b")!;
const cSpec = byCmd.get("c")!;
const ids = result.operations.map((o) => o.id);
const aIdx = ids.indexOf("run:a");
const bIdx = ids.indexOf("run:b");
const cIdx = ids.indexOf("run:c");
expect(aIdx).toBeGreaterThanOrEqual(0);
expect(bIdx).toBeGreaterThan(aIdx);
expect(cIdx).toBeGreaterThan(bIdx);
// b depends on a, c depends on b
const bSpec = result.operations.find((o) => o.id === "run:b")!;
const cSpec = result.operations.find((o) => o.id === "run:c")!;
expect(bSpec.dependsOn).toEqual(["run:a"]);
expect(cSpec.dependsOn).toEqual(["run:b"]);
expect(ids.indexOf(aSpec.id)).toBeGreaterThanOrEqual(0);
expect(ids.indexOf(bSpec.id)).toBeGreaterThan(ids.indexOf(aSpec.id));
expect(ids.indexOf(cSpec.id)).toBeGreaterThan(ids.indexOf(bSpec.id));
expect(bSpec.dependsOn).toEqual([aSpec.id]);
expect(cSpec.dependsOn).toEqual([bSpec.id]);
});

it("parallel siblings have no inter-sibling dependsOn", async () => {
const a = run({ command: "a" });
const b = run({ command: "b" });
const wf = workflow("parallel", a, b);
const result = await wf.plan(makePlanRuntime());
const aSpec = result.operations.find((o) => o.id === "run:a")!;
const bSpec = result.operations.find((o) => o.id === "run:b")!;
const byCmd = new Map(result.operations.map((o) => [o.command, o]));
const aSpec = byCmd.get("a")!;
const bSpec = byCmd.get("b")!;
expect(aSpec.dependsOn ?? []).toEqual([]);
expect(bSpec.dependsOn ?? []).toEqual([]);
});
Expand Down
73 changes: 73 additions & 0 deletions packages/core/src/__tests__/ids.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import { describe, it, expect } from "vitest";
import { createHash } from "node:crypto";
import { computeOperationId } from "../internal/ids.js";
import { canonicalStringify } from "../internal/canonical.js";

const OP_ID_RE = /^op-[0-9a-f]{64}$/;

describe("computeOperationId", () => {
it("is prefixed with op- followed by 64 hex chars", () => {
const id = computeOperationId("run", "build", {});
expect(id).toMatch(OP_ID_RE);
});

it("is deterministic for identical inputs", () => {
expect(computeOperationId("run", "build", { os: "linux" })).toBe(
computeOperationId("run", "build", { os: "linux" }),
);
});

it("is independent of context key insertion order", () => {
const a = computeOperationId("run", "build", { os: "linux", arch: "x64" });
const b = computeOperationId("run", "build", { arch: "x64", os: "linux" });
expect(a).toBe(b);
});

it("differs across kinds", () => {
expect(computeOperationId("run", "build", {})).not.toBe(
computeOperationId("check", "build", {}),
);
});

it("differs across names", () => {
expect(computeOperationId("run", "build", {})).not.toBe(
computeOperationId("run", "test", {}),
);
});

it("differs across context values", () => {
expect(computeOperationId("run", "build", { os: "linux" })).not.toBe(
computeOperationId("run", "build", { os: "macos" }),
);
});

it("differs when context values differ only by type", () => {
expect(computeOperationId("run", "build", { target: 1 })).not.toBe(
computeOperationId("run", "build", { target: "1" }),
);
});

it("matrix expansion produces distinct, deterministic ids", () => {
const combos = [
{ os: "linux", arch: "x64" },
{ os: "linux", arch: "arm64" },
{ os: "macos", arch: "x64" },
{ os: "macos", arch: "arm64" },
];
const ids = combos.map((c) => computeOperationId("run", "build", c));
expect(new Set(ids).size).toBe(ids.length);
expect(combos.map((c) => computeOperationId("run", "build", c))).toEqual(ids);
});

it("matches a manual sha256 over the canonical JSON of {kind,name,context}", () => {
const kind = "run";
const name = "build";
const context = { os: "linux", arch: "x64" };
const expected =
"op-" +
createHash("sha256")
.update(canonicalStringify({ kind, name, context }), "utf8")
.digest("hex");
expect(computeOperationId(kind, name, context)).toBe(expected);
});
});
6 changes: 5 additions & 1 deletion packages/core/src/__tests__/laziness.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,11 @@ describe("laziness: composables perform no I/O", () => {
const fetchSpy = vi.spyOn(globalThis, "fetch");
const a = run({ command: "a" });
const b = run({ command: "b" });
const wf = workflow("ci", parallel(a, b), pipeline(a, b));
const c = run({ command: "c" });
const d = run({ command: "d" });
// Distinct ops per root — content-addressed ids reject true duplicates,
// so we don't reuse a/b across roots (this test checks I/O, not dedup).
const wf = workflow("ci", parallel(a, b), pipeline(c, d));
await wf.plan(makePlanRuntime());
expect(childProcessMock.spawn).not.toHaveBeenCalled();
expect(fsMock.writeFileSync).not.toHaveBeenCalled();
Expand Down
71 changes: 31 additions & 40 deletions packages/core/src/__tests__/matrix.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,37 +2,41 @@ import { describe, it, expect } from "vitest";
import { run } from "../composables/run.js";
import { matrix } from "../composables/matrix.js";
import { workflow } from "../composables/workflow.js";
import { CompositionError } from "../errors.js";
import { makePlanRuntime } from "./helpers/runtime.js";
import { computeOperationId } from "../internal/ids.js";

const OP_ID_RE = /^op-[0-9a-f]{64}$/;

describe("matrix expansion", () => {
it("produces one node per value with distinct ids and MATRIX_<DIM> env", async () => {
it("produces one node per value with distinct op- ids and MATRIX_<DIM> env", async () => {
const op = matrix({ node: ["20", "24"] }, run({ command: "test" }));
const wf = workflow("matrix-1d", op);
const result = await wf.plan(makePlanRuntime());
expect(result.operations).toHaveLength(2);
const ids = result.operations.map((o) => o.id).sort();
expect(ids).toEqual(["run:test[node=s:20]", "run:test[node=s:24]"]);
const ids = result.operations.map((o) => o.id);
for (const id of ids) expect(id).toMatch(OP_ID_RE);
expect(new Set(ids).size).toBe(ids.length); // distinct
// ids are content-addressed over {kind, name, context:{node, command}}
const expected = ["20", "24"].map((v) =>
computeOperationId("run", "test", { node: v, command: "test" }),
);
expect(ids.sort()).toEqual([...expected].sort());
const envs = result.operations.map((o) => o.env?.MATRIX_NODE).sort();
expect(envs).toEqual(["20", "24"]);
});

it("multi-dimension cartesian product with joined id suffix", async () => {
it("multi-dimension cartesian product with distinct op- ids", async () => {
const op = matrix({ node: ["20", "24"], os: ["linux", "macos"] }, run({ command: "test" }));
const wf = workflow("matrix-2d", op);
const result = await wf.plan(makePlanRuntime());
expect(result.operations).toHaveLength(4);
const ids = result.operations.map((o) => o.id).sort();
expect(ids).toEqual([
"run:test[node=s:20,os=s:linux]",
"run:test[node=s:20,os=s:macos]",
"run:test[node=s:24,os=s:linux]",
"run:test[node=s:24,os=s:macos]",
]);
const envs = result.operations
.map((spec) => `${spec.env?.MATRIX_NODE}/${spec.env?.MATRIX_OS}`)
.sort();
expect(envs).toEqual(["20/linux", "20/macos", "24/linux", "24/macos"]);
const ids = result.operations.map((o) => o.id);
for (const id of ids) expect(id).toMatch(OP_ID_RE);
expect(new Set(ids).size).toBe(ids.length);
for (const spec of result.operations) {
expect(spec.env?.MATRIX_NODE).toBeDefined();
expect(spec.env?.MATRIX_OS).toBeDefined();
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
});

it("children inherit predecessors from the template", async () => {
Expand All @@ -41,10 +45,11 @@ describe("matrix expansion", () => {
const matrixed = matrix({ node: ["20", "24"] }, test).after(build);
const wf = workflow("matrix-deps", matrixed);
const result = await wf.plan(makePlanRuntime());
const children = result.operations.filter((spec) => spec.id.startsWith("run:test["));
expect(children).toHaveLength(2);
for (const spec of children) {
expect(spec.dependsOn).toEqual(["run:build"]);
const buildSpec = result.operations.find((o) => o.command === "build")!;
const testSpecs = result.operations.filter((o) => o.command === "test");
expect(testSpecs.length).toBe(2);
for (const spec of testSpecs) {
expect(spec.dependsOn).toEqual([buildSpec.id]);
}
});

Expand All @@ -64,32 +69,18 @@ describe("matrix expansion", () => {
expect(result.operations).toHaveLength(3);
});

it("duplicate matrix values raise CompositionError", async () => {
it("rejects duplicate matrix values that would produce duplicate ids", async () => {
const op = matrix({ node: ["20", "20"] }, run({ command: "test" }));
const wf = workflow("matrix-dup", op);
await expect(wf.plan(makePlanRuntime())).rejects.toThrow(CompositionError);
});

it("delimiter characters in values are escaped in ids", async () => {
const op = matrix({ key: ["a,b=c"] }, run({ command: "test" }));
const wf = workflow("matrix-escape", op);
const result = await wf.plan(makePlanRuntime());
expect(result.operations).toHaveLength(1);
// The comma and equals in the value should be escaped, not treated as
// dimension boundaries.
const id = result.operations[0]!.id;
expect(id).toContain("a\\,b\\=c");
await expect(wf.plan(makePlanRuntime())).rejects.toThrow("duplicate operation id");
});

it("number and string values with the same text produce distinct ids", async () => {
const op = matrix({ v: [1, "1", true] }, run({ command: "test" }));
it("distinguishes values with identical text but different types", async () => {
const op = matrix({ node: [1, "1", true] as unknown[] }, run({ command: "test" }));
const wf = workflow("matrix-types", op);
const result = await wf.plan(makePlanRuntime());
expect(result.operations).toHaveLength(3);
expect(result.operations.map((o) => o.id).sort()).toEqual([
"run:test[v=b:true]",
"run:test[v=n:1]",
"run:test[v=s:1]",
]);
const ids = result.operations.map((o) => o.id);
expect(new Set(ids).size).toBe(3);
});
});
25 changes: 24 additions & 1 deletion packages/core/src/__tests__/public-api.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,31 @@ describe("public API surface", () => {
// named exports of the public barrel.
const publicNames = Object.keys(api);
const internalLeaked = publicNames.filter((n) =>
["createNode", "asNode", "withSpec", "mergeSpecs", "concatDedupe", "planWorkflow", "assignId", "evaluateCondition"].includes(n),
[
"createNode",
"asNode",
"withSpec",
"mergeSpecs",
"concatDedupe",
"planWorkflow",
"evaluateCondition",
].includes(n),
);
expect(internalLeaked).toEqual([]);
});

it("exports computeOperationId (content-addressed id primitive, ADR-006)", () => {
// computeOperationId is a pure, dependency-free primitive that any
// consumer (planner, compiler, ir) can use to recompute an operation id
// from its content. It is part of the stable public contract.
expect(typeof api.computeOperationId).toBe("function");
expect(api.computeOperationId("run", "build", {})).toMatch(/^op-[0-9a-f]{64}$/);
});

it("exports canonicalStringify (canonical JSON primitive, ADR-006)", () => {
// canonicalStringify is the stable serialization primitive shared by
// computeOperationId and the ir package's serializePlan/computePlanId.
expect(typeof api.canonicalStringify).toBe("function");
expect(api.canonicalStringify({ b: 1, a: 2 })).toBe('{"a":2,"b":1}');
});
});
Loading