Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions .coderabbit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@
# Review instructions are the conventions themselves — the ones a test cannot check.

language: 'en-US'
# Hard limit: 250 characters. Over it the whole file fails to parse and CodeRabbit silently
# reviews with its defaults, so every instruction below is dropped too.
tone_instructions: >-
Terse and concrete. Lead with the rule, cite the file that states it, skip nitpicks Biome already
catches. The primary user of this codebase is an AI agent: block anything that makes a failure
unreadable to one — a bare Error, a missing fix command, a command without --json.
Terse and concrete. Lead with the rule, cite the file that states it, skip nitpicks Biome
catches. The primary user here is an AI agent: block whatever makes a failure unreadable to
one — a bare Error, a missing fix, a command without --json.
early_access: true
enable_free_tier: true

Expand Down
59 changes: 53 additions & 6 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ jobs:
timeout-minutes: 5
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
Expand All @@ -44,6 +46,8 @@ jobs:
timeout-minutes: 8
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
Expand All @@ -61,6 +65,8 @@ jobs:
timeout-minutes: 5
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
Expand All @@ -73,6 +79,8 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
Expand All @@ -83,12 +91,9 @@ jobs:
restore-keys: bun-${{ runner.os }}-
- run: bun install --frozen-lockfile
- name: bun test
# `bun run test`, not bare `bun test`: CI must run the repo's own gate, not a
# different command. The script's ignore patterns drop the `e2e/` suites, which
# bind a real socket and are opt-in behind ULTIMATE_TEST_ALLOW_NET=1, and the
# dummy app's contract/live/job/eval suites, which request fixtures nothing in
# the repo registers (#9). Bare `bun test` picked both up, which is why `main`
# has been red since 2026-07-30 and every PR inherited it.
# No opt-in exclusions: the framework's e2e/contract/live/job/eval suites all run, every
# time. The reference app's suites are not here — they run in `reference-app-verify`,
# against the app's own preload and fixtures.
run: bun run test

# The whole gate, exactly as `x verify` runs it. Green here means shippable.
Expand All @@ -98,6 +103,8 @@ jobs:
needs: [lint, typecheck, boundaries, test]
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
Expand All @@ -109,3 +116,43 @@ jobs:
- run: bun install --frozen-lockfile
- name: x verify
run: bun run scripts/verify.ts --json

# The reference app runs the same 15-step gate a generated app would. Not in `verify`'s
# `needs`: the framework gate and the app gate are two different contracts, checked
# independently — a red app doesn't hide a green framework or vice versa. That is also why
# the app is absent from the `typecheck` and `test` jobs above: its typecheck and its suites
# are steps 1 and 6-11 of the gate below, so running them twice only decides which job's
# redness you notice first.
#
# Advisory, not blocking. As of 2026-08 Postly is written against the finished API — the fluent
# `action` surface, `defineMail`/`defineService`, the `page`/`subscribe`/`evaluate` fixtures —
# and the framework has not shipped it yet, so every failure here is drift the roadmap already
# owns rather than a regression this PR introduced. Blocking on it would make red the resting
# state, which is how a gate stops being read. `continue-on-error` comes off at milestone 9 in
# docs/idea/14-roadmap.md — the last of those surfaces to land — when app and framework agree.
reference-app-verify:
runs-on: ubuntu-latest
timeout-minutes: 12
steps:
- uses: actions/checkout@v7
Comment thread
coderabbitai[bot] marked this conversation as resolved.
with:
persist-credentials: false
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
- uses: actions/cache@v6
with:
path: ~/.bun/install/cache
key: bun-${{ runner.os }}-${{ hashFiles('bun.lock') }}
restore-keys: bun-${{ runner.os }}-
- run: bun install --frozen-lockfile
- name: x verify (examples/dummy)
# `continue-on-error` on the STEP, not the job: on the job it leaves the check itself red
# for as long as the drift above lasts, and a check that is always red is a check nobody
# reads — the exact failure mode this PR exists to remove. On the step the run is marked
# with a warning, and the whole 15-step table, every failing step and every fix line stay
# in the log. Human render, not `--json`: the reader of an advisory result is a person or
# an agent scanning for which step went red, not a parser.
continue-on-error: true
working-directory: examples/dummy
run: bun run ../../packages/cli/src/bin.ts verify
2 changes: 2 additions & 0 deletions .github/workflows/pages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ jobs:
timeout-minutes: 10
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false

- uses: oven-sh/setup-bun@v2
with:
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ jobs:
timeout-minutes: 20
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false

# No `registry-url:` on purpose. It writes an .npmrc containing
# //registry.npmjs.org/:_authToken=${NODE_AUTH_TOKEN}
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/wiki.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ jobs:
timeout-minutes: 5
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false

- name: detect wiki sources
id: wiki
Expand Down
4 changes: 2 additions & 2 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ CLI binary: `x`. npm scope: `@ultimat3`. Import paths: `@ultimat3/<pkg>`.
| Task | Command |
|---|---|
| install | `bun install` |
| **the gate** | `bun run verify` — typecheck + lint + boundaries + tests. Green = shippable. |
| **the gate** | `bun run verify` — `x verify` at the repo root: typecheck, lint, boundaries, sizes, shape, every test type, drift, contracts, budgets, manifest. Green = shippable. |
| typecheck | `bun run typecheck` |
| lint | `bun run lint` · fix: `bun run lint:fix` |
| test (all) | `bun run test` — bare `bun test` also collects the opt-in `e2e/` and dummy-app fixture suites (#9) |
| test (all) | `bun run test` — every framework suite, opt-in ones included. The reference app is gated separately: `cd examples/dummy && bun run ../../packages/cli/src/bin.ts verify` |
| test (one file) | `bun test packages/core/src/errors.test.ts` |
| test (one name) | `bun test -t 'formats the fix line'` |
| import boundaries | `bun run boundaries` |
Expand Down
20 changes: 14 additions & 6 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,25 +18,33 @@ embedded Postgres (PGlite), in-process events, and S3 to a local directory.
| `bin/setup` | fresh clone to running |
| `bin/dev <args>` | run the `x` CLI from source: `bin/dev verify --json` |
| `bin/check` | the CI gate, locally |
| `bun run test` | every package's tests (bare `bun test` also picks up the opt-in suites — see #9) |
| `bun run test` | every package's tests, opt-in suites included; `examples/` is gated by its own `x verify` |
| `bun run scripts/help.ts` | the full script catalogue |

## ✅ The gate

**`bun run scripts/verify.ts` — green means shippable.** CI runs exactly this. There is no second
checklist and no CI-only step.

It **is** `x verify`, run at the repo root: one step list, defined once in
`packages/cli/src/cmd-verify.ts`, so a contributor and a user see the same steps. A step that has
nothing to check here is reported as skipped (`-`), never as passed.

| Step | Fails on |
|---|---|
| `typecheck` | any type error; `any` is banned, and a cast is not a fix |
| `lint` | Biome: formatting, `any`, unused, default exports |
| `boundaries` | a tier violation (see below) |
| `test` | any failing test; a flake is a failure |
| `boundaries` | a tier violation (see below) or an app surface violation |
| `filesize` | a file over 500 lines |
| `package-shape` | a package missing `README.md`, `CLAUDE.md`, `tsconfig.json`, `src/index.ts` |
| `manifest` | the framework manifest cannot be generated |

Narrow it while iterating: `bun run scripts/verify.ts --only lint,boundaries --json`.
| `unit` | any failing test that is not one of the typed suites below; a flake is a failure |
| `contract` `live` `job` `e2e` `eval` | any failing `*.<type>.test.ts` suite (or any test under `e2e/`) |
| `drift` | an app schema that no migration recorded |
| `contract-diff` | a breaking change to a published action without a version bump |
| `budgets` | per-route JS bytes or LCP over the declared limit |
| `manifest` | the manifest differs from what the code produces, or cannot be generated |

There is no `--only` and no `--skip`: "green" has to mean the same thing for everyone (axiom 5).

## 📦 Import tiers (a build error, not a preference)

Expand Down
3 changes: 3 additions & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion bunfig.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,8 @@ production = false

[test]
root = "."
# Frozen clock, seeded RNG, sealed network. See packages/testing.
# Frozen clock, seeded RNG, sealed network, framework fixtures. See packages/testing.
# Only the framework's preload: `examples/dummy` is a separate project with its own bunfig,
# its own fixtures (`seed`, `actorFor`) and its own gate. Registering the app's fixtures here
# would load the app's entity graph and mail registry into every framework test.
preload = ["./scripts/test-setup.ts"]
4 changes: 3 additions & 1 deletion docs/architecture/14-testing-internals.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ X_TEST_NETWORK_EGRESS: unmocked request in a test
fix: http.mock('POST https://api.stripe.com/v1/charges', { status: 200, body: {...} })
```

The trap is installed at the fetch/socket layer, so it catches SDKs and transitive dependencies, not just direct `fetch` calls. There is no allowlist flag; a genuine integration test declares `network: 'live'` on the file and is excluded from `x verify`'s default set.
The trap is installed at the fetch/socket layer, so it catches SDKs and transitive dependencies, not just direct `fetch` calls. A server the test itself started is not egress — `start()` announces its socket to core, so a request back to that port passes. There is no allowlist API a file can call; the one opt-out is `ULTIMATE_TEST_ALLOW_NET=1` in the environment, reserved for a deliberate live integration, so no test can quietly unseal the network for itself.

## The six test types

Expand Down Expand Up @@ -120,6 +120,8 @@ $ x verify
| 1 | typecheck | any error; `any` is banned by lint, not tolerated by a cast | fastest signal, and everything downstream assumes types hold |
| 2 | lint (Biome) | formatting, `any`, default exports, bare `Error`, raw hex, hardcoded strings | seconds, and it catches the cross-cutting rules before an expensive test run |
| 3 | **import boundaries** | `site/` → `app/`, routes → DB, services → HTTP, framework tier violations | an import-scan pass; a boundary break invalidates the bundle-graph assumptions the later budget check depends on |
| 3a | file size | a source file over 500 lines | a file read; one file, one job is cheapest to check before anything runs |
| 3b | package shape | a workspace package missing `README.md`, `CLAUDE.md`, `tsconfig.json`, `src/index.ts` | four `stat` calls, and every later step assumes the package is navigable |
| 4 | unit tests | any failure | no DB, so still cheap; fails fast on logic |
| 5 | contract, live, job tests | any failure; a flake **is** a failure | needs cloned databases — first genuinely expensive step |
| 6 | **migration drift** | schema ≠ migrations ≠ catalog, or an irreversible migration without a marker | after tests, because tests are what would have exercised the new column |
Expand Down
2 changes: 1 addition & 1 deletion docs/architecture/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,4 +77,4 @@ A convention that isn't a build error doesn't exist. What actually fails, and wh
| SEO + i18n | missing meta, duplicate meta, missing key in any locale | [`09`](./09-rendering-internals.md), [`10`](./10-cross-cutting.md) |
| manifest freshness | `x.manifest.json` / `openapi.json` differ from the code | [`11`](./11-ai-surface.md) |

One command runs all of them: `bun run verify` in this repo, `x verify` in a generated app. There is no `--skip`.
One command runs all of them, from one step list: `bun run verify` in this repo *is* `x verify` run at the repo root — same steps, same report, same exit code. There is no `--only` and no `--skip`.
3 changes: 3 additions & 0 deletions examples/dummy/CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@ Feature slice: `apps/web/app/<feature>/{entity,repo,service,actions,mutator,live
- `idempotencyKey` on every job is required by the type. Keys derive from `input` only.
- Tests sit next to their source: `<file>.test.ts` (unit), `.contract.test.ts`, `.live.test.ts`,
`.job.test.ts`, `.e2e.test.ts`, `.eval.test.ts`.
- Test fixtures come from `scripts/test-setup.ts`, the one preload in `bunfig.toml`. `clock`,
`mail` and `runJobs` are the framework's; `seed` and `actorFor` are Postly's. A test that
destructures anything else fails with `X_TEST_FIXTURE_UNKNOWN` — register it there, once.

## Boundaries (build errors, not lint warnings)

Expand Down
12 changes: 12 additions & 0 deletions examples/dummy/bunfig.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Bun runtime + test config for Postly. https://bun.sh/docs/runtime/bunfig
#
# One preload, two jobs: `scripts/test-setup.ts` pulls in @ultimat3/testing's preload (frozen
# clock, seeded RNG, sealed network, matchers) and registers the fixtures the app owns.

[install]
exact = false
production = false

[test]
root = "."
preload = ["./scripts/test-setup.ts"]
1 change: 1 addition & 0 deletions examples/dummy/packages/db/migrations/0001_init.hash
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
81a4b14eea7ebdea
110 changes: 110 additions & 0 deletions examples/dummy/scripts/test-setup.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
// Postly's test preload: the fixtures the APP owns. `clock`, `mail` and `runJobs` arrive with
// the framework's own preload, imported below — an app registers only what the framework cannot
// know, which here is the seed graph and how a member becomes an actor.
//
// [test]
// preload = ["./scripts/test-setup.ts"]
//
// The framework is imported by relative path rather than by `@ultimat3/*`: this directory is not
// a workspace member yet (issue #9), and a preload runs before anything else, so it must not
// depend on workspace symlinks. A generated app writes `@ultimat3/testing` here. For the same
// reason `scripts/` is not in tsconfig's `include` — a composite project cannot reach across
// into another one's sources. Both go away when the app joins the workspace.

import { assert, userActor } from '../../../packages/core/src/index';
import type { Driver, EntityCore, Repo, Seed } from '../../../packages/entity/src/index';
import { memoryDriver, seedId } from '../../../packages/entity/src/index';
import { defineFixtures } from '../../../packages/testing/src/index';
import '../../../packages/testing/src/preload';

/** Every seeded row carries an id; the rest of the columns are the entity's business. */
export interface SeedRow {
readonly id: string;
readonly [column: string]: unknown;
}

export interface SeedHandle {
/** `pick({ draft: 'post:draft-money' })` — seed labels in, rows out, aliased at the call site. */
pick<M extends Readonly<Record<string, string>>>(
labels: M,
): Promise<{ readonly [K in keyof M]: SeedRow }>;
}

const idOf = (row: unknown): string | undefined => {
const value = (row as { readonly id?: unknown }).id;
return typeof value === 'string' ? value : undefined;
};

/**
* Rows are captured on the way in rather than read back out: a tenant-scoped entity refuses an
* unscoped read, so a fixture would have to name the org before it could fetch the org. Insert
* still runs `$parse` and the invariants, so seeding still tests the schema.
*/
const capturingDriver = (rows: Map<string, SeedRow>): Driver => {
const base = memoryDriver();
return {
repo: <Row>(entity: EntityCore<Row>): Repo<Row> => {
const inner = base.repo(entity);
return {
...inner,
insert: async (values, options) => {
const row = await inner.insert(values, options);
const id = idOf(row);
if (id !== undefined) rows.set(id, row as SeedRow);
return row;
},
};
},
};
};

/** A fresh graph per call: two tests must never see each other's writes. */
const handleFor = (seed: Seed): SeedHandle => {
const rows = new Map<string, SeedRow>();
const ready = seed.run({ driver: capturingDriver(rows) });

return {
pick: async <M extends Readonly<Record<string, string>>>(labels: M) => {
await ready;
const picked: Record<string, SeedRow> = {};
for (const [alias, label] of Object.entries(labels)) {
const row = rows.get(seedId(label));
assert(
row !== undefined,
`seed "${seed.name}" has no row labelled "${label}"`,
`add it to packages/db/seeds/${seed.name}.ts with id: id('${label}')`,
);
picked[alias] = row;
}
return picked as { readonly [K in keyof M]: SeedRow };
},
};
};

/** Imported on demand, so a test that never seeds never loads the entity graph. */
const createSeed = async (): Promise<(name: string) => SeedHandle> => {
const { dev } = await import('../packages/db/seeds/dev');
const seeds: Readonly<Record<string, Seed>> = { dev };
return (name) => {
const seed = seeds[name];
assert(
seed !== undefined,
`no seed named "${name}" — known seeds: ${Object.keys(seeds).join(', ')}`,
'x db seed --list, then use one of the names it prints',
);
return handleFor(seed);
};
};

/** A member row is the actor: same org, and its membership role is the authz role. */
const actorFor = (member: SeedRow) =>
userActor({
id: String(member['userId'] ?? member.id),
orgId: String(member['orgId']),
roles: [String(member['role'])],
});

defineFixtures({
seed: createSeed,
actorFor: () => actorFor,
});
Loading
Loading