Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions web/scripts/build-devbox-freestyle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
*/
import { Freestyle } from "freestyle";
import { fileURLToPath } from "node:url";
import { VM_GUEST_MODEL_PLANE_ENV_PATH, renderVmGuestModelPlaneEnvFile, vmGuestModelPlaneEnv } from "../services/coderouter/vmGuestEnv";
import {
CMUX_TUI_SESSION,
cmuxTuiInstallCommand,
Expand Down Expand Up @@ -453,6 +454,12 @@ try {
`mkdir -p /usr/local/share/blesh/state.d && chmod a+rwxt /usr/local/share/blesh/state.d && for h in ${WORK_HOME} /root /etc/skel; do mkdir -p "$h/.cache" "$h/.local/state" && touch "$h/.cache/motd.legal-displayed"; done && chown -R ${WORK_USER}:${WORK_USER} ${WORK_HOME} && [ "$(find ${WORK_HOME} -not -user ${WORK_USER} | wc -l)" = 0 ] && ${interactiveShellProbe(1)} && ${interactiveShellProbe(2)} && sudo -n -u ${WORK_USER} env -i HOME=${WORK_HOME} USER=${WORK_USER} TERM=xterm-256color bash -c 'tmux -L bake new-session -d -s ghost -x 100 -y 24 && sleep 2 && tmux -L bake send-keys -t ghost cl && sleep 2 && tmux -L bake capture-pane -pt ghost | grep -o "claude --dangerously-skip-permissions" | head -1; rc=$?; tmux -L bake kill-server 2>/dev/null; exit $rc' && [ "$(find ${WORK_HOME} -not -user ${WORK_USER} | wc -l)" = 0 ] && echo home-hygiene-ok`,
);

// The model-plane env is the same bytes for every machine (an alias host the
// edge routes per deployment), so it is baked and create writes nothing. It
// goes in after every layer that opens a login shell: once it exists, any
// shell materializes the harness configs, and the image must carry none.
await vm.fs.writeFile(VM_GUEST_MODEL_PLANE_ENV_PATH, renderVmGuestModelPlaneEnvFile(vmGuestModelPlaneEnv()), { mode: 0o644 });
await step("model-plane-env", `sh -n ${VM_GUEST_MODEL_PLANE_ENV_PATH} && grep -q "^export OPENAI_BASE_URL='https://" ${VM_GUEST_MODEL_PLANE_ENV_PATH} && ! grep -q crt_ ${VM_GUEST_MODEL_PLANE_ENV_PATH} && env -i HOME=/tmp/mp-check bash -c '. /etc/cmux/agent-config.sh; echo $OPENAI_BASE_URL' | grep -q '^https://' && rm -rf /tmp/mp-check && echo model-plane-env-baked`);
// Stamp last: its presence tells the driver and the verifier every layer
// above baked successfully, and which layers the image carries.
await step(
Expand Down
6 changes: 4 additions & 2 deletions web/scripts/verify-devbox-image.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
// The devbox freestyle bake targets the public platform (see
// build-devbox-freestyle.ts), the same platform the shipped driver speaks.
import { Freestyle } from "freestyle";
import { DEFAULT_VM_EDGE_ALIAS_DOMAIN } from "../services/coderouter/vmGuestEnv";
import path from "node:path";
import {
CMUX_TUI_SESSION,
Expand Down Expand Up @@ -89,7 +90,7 @@ const CHECKS: readonly string[] = [
// header: the edge injects it) and persists every var 0600; the
// unreachable config endpoint writes no opencode config; the image ships
// no pre-generated config for root.
"rm -rf /tmp/cmux-agent-config-verify && env HOME=/tmp/cmux-agent-config-verify OPENAI_BASE_URL=https://example.invalid/v1 OPENAI_API_KEY=cmux-vm-edge-placeholder CMUX_CODEROUTER_URL=https://example.invalid ANTHROPIC_BASE_URL=https://example.invalid ANTHROPIC_API_KEY=cmux-vm-edge-placeholder CMUX_VM_ID=vm-check bash -lc 'true' && grep -q 'model_provider = \"cmux\"' /tmp/cmux-agent-config-verify/.codex/config.toml && grep -q 'wire_api = \"responses\"' /tmp/cmux-agent-config-verify/.codex/config.toml && grep -q \"export OPENAI_API_KEY='cmux-vm-edge-placeholder'\" /tmp/cmux-agent-config-verify/.config/cmux/model-plane.env && grep -q \"export CMUX_VM_ID='vm-check'\" /tmp/cmux-agent-config-verify/.config/cmux/model-plane.env && [ \"$(stat -c %a /tmp/cmux-agent-config-verify/.config/cmux/model-plane.env)\" = \"600\" ] && grep -qF '\"apiKey\": \"e30.' /tmp/cmux-agent-config-verify/.pi/agent/models.json && ! grep -q x-coderouter-route-token /tmp/cmux-agent-config-verify/.pi/agent/models.json && ! grep -q crt_ /tmp/cmux-agent-config-verify/.pi/agent/models.json && test ! -e /tmp/cmux-agent-config-verify/.config/opencode/opencode.json && rm -rf /tmp/cmux-agent-config-verify && test ! -e /root/.codex/config.toml && test ! -e /root/.pi/agent/models.json && test ! -e /root/.config/opencode/opencode.json && echo agent-config-ok",
`rm -rf /tmp/cmux-agent-config-verify && env HOME=/tmp/cmux-agent-config-verify OPENAI_BASE_URL=https://example.invalid/v1 OPENAI_API_KEY=cmux-vm-edge-placeholder CMUX_CODEROUTER_URL=https://example.invalid ANTHROPIC_BASE_URL=https://example.invalid ANTHROPIC_API_KEY=cmux-vm-edge-placeholder CMUX_VM_ID=vm-check bash -lc 'true' && grep -q 'model_provider = "cmux"' /tmp/cmux-agent-config-verify/.codex/config.toml && grep -q 'wire_api = "responses"' /tmp/cmux-agent-config-verify/.codex/config.toml && grep -q "export OPENAI_API_KEY='cmux-vm-edge-placeholder'" /tmp/cmux-agent-config-verify/.config/cmux/model-plane.env && grep -q "export CMUX_VM_ID='vm-check'" /tmp/cmux-agent-config-verify/.config/cmux/model-plane.env && [ "$(stat -c %a /tmp/cmux-agent-config-verify/.config/cmux/model-plane.env)" = "600" ] && grep -qF '"apiKey": "e30.' /tmp/cmux-agent-config-verify/.pi/agent/models.json && ! grep -q x-coderouter-route-token /tmp/cmux-agent-config-verify/.pi/agent/models.json && ! grep -q crt_ /tmp/cmux-agent-config-verify/.pi/agent/models.json && test ! -e /tmp/cmux-agent-config-verify/.config/opencode/opencode.json && rm -rf /tmp/cmux-agent-config-verify && grep -q 'base_url = "https://' /root/.codex/config.toml && grep -qF "${DEFAULT_VM_EDGE_ALIAS_DOMAIN}/v1" /root/.codex/config.toml && ! grep -q crt_ /root/.codex/config.toml && ! grep -q crt_ /root/.pi/agent/models.json && test ! -e /root/.config/opencode/opencode.json && echo agent-config-ok`,
"grep -q cleanupPeriodDays /etc/claude-code/managed-settings.json && echo claude-retention-ok",
"whoami; nproc; free -m | sed -n 2p; df -h / | tail -1",
];
Expand All @@ -112,7 +113,8 @@ const DAEMON_CHECKS: readonly string[] = [
`test -s ${REMOTE_IDENTITY} && echo daemon-identity-present`,
`test "$(cat /etc/cmux/daemon-instance-id)" = "$(${INSTANCE_ID})" && echo daemon-identity-bound-to-this-instance`,
`test -s /etc/cmux/bake-instance-id && test "$(cat /etc/cmux/bake-instance-id)" != "$(${INSTANCE_ID})" && echo builder-instance-differs`,
"test -d /root/.config/cmux && echo model-plane-env-dir-baked",
// The static model-plane env is baked; a shell with no boot env sources it.
`test -s /etc/cmux/model-plane.env && grep -q "^export OPENAI_BASE_URL='https://" /etc/cmux/model-plane.env && ! grep -q crt_ /etc/cmux/model-plane.env && env -i HOME=/tmp/mp-verify bash -c '. /etc/cmux/agent-config.sh; printf %s "$OPENAI_BASE_URL"' | grep -q '^https://' && rm -rf /tmp/mp-verify && echo model-plane-env-baked`,
"systemctl is-active cmux-tui-daemon >/dev/null && echo systemd-supervisor-active",
];

Expand Down
3 changes: 2 additions & 1 deletion web/services/coderouter/routeTokenAuth.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ export const VM_ID_HEADER = "x-cmux-vm-id";
* token the edge injects. Never matches the `crt_` token grammar, so it can
* never be mistaken for a token by any verifier.
*/
export const VM_PLACEHOLDER_API_KEY = "cmux-vm-edge-placeholder";
import { VM_PLACEHOLDER_API_KEY } from "./vmGuestEnv";
export { VM_PLACEHOLDER_API_KEY };

export type RouteTokenIdentity = {
readonly teamId: string;
Expand Down
63 changes: 63 additions & 0 deletions web/services/coderouter/vmGuestEnv.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// The model-plane env every Cloud VM guest gets: where the coding agents
// inside the machine send model traffic. It is the same bytes for every
// machine in every environment, so the bake writes it into the snapshot
// (/etc/cmux/model-plane.env) and create writes nothing into the guest.
//
// How it stays static: the guest dials one alias host name. The machine's TLS
// edge rule (services/coderouter/vmModelPlane.ts) terminates that name inside
// the platform's edge, forwards to this deployment's real API host, and adds
// the machine's route token header. Prod, staging, and previews differ only in
// the rule's destination, never in what the guest sees. This module must stay
// free of database and framework imports: the bake script runs it on a laptop.

/** The public placeholder credential agents send; the edge swaps in the real token. */
export const VM_PLACEHOLDER_API_KEY = "cmux-vm-edge-placeholder";

export const VM_EDGE_ALIAS_DOMAIN_ENV = "CMUX_VM_EDGE_ALIAS_DOMAIN";
export const DEFAULT_VM_EDGE_ALIAS_DOMAIN = "coderouter.cmux.internal";

const HOST_NAME = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$/;

/** The host name the guest dials for model traffic; overridable for a deployment, never per machine. */
export function vmEdgeAliasDomain(raw: string | undefined = process.env[VM_EDGE_ALIAS_DOMAIN_ENV]): string {
const value = raw?.trim().toLowerCase();
if (!value) return DEFAULT_VM_EDGE_ALIAS_DOMAIN;
if (!HOST_NAME.test(value)) {
throw new Error(`${VM_EDGE_ALIAS_DOMAIN_ENV} must be a bare host name, got ${JSON.stringify(raw)}`);
}
return value;
}

/** The variables agent-config.sh persists and every harness reads. No token, no per-machine value. */
export function vmGuestModelPlaneEnv(alias: string = vmEdgeAliasDomain()): Record<string, string> {
const origin = `https://${alias}`;
return {
OPENAI_BASE_URL: `${origin}/v1`,
OPENAI_API_KEY: VM_PLACEHOLDER_API_KEY,
CMUX_CODEROUTER_URL: origin,
ANTHROPIC_BASE_URL: origin,
ANTHROPIC_API_KEY: VM_PLACEHOLDER_API_KEY,
};
}

const ENV_NAME = /^[A-Z_][A-Z0-9_]*$/;
const ROUTE_TOKEN_GRAMMAR = /\bcrt_[A-Za-z0-9._-]+/;

/**
* The file agent-config.sh sources (same format it writes itself from a boot
* env). Refuses a route token anywhere in the values: the guest never holds one.
*/
export function renderVmGuestModelPlaneEnvFile(env: Readonly<Record<string, string>>): string {
const quote = (value: string) => `'${value.replace(/'/g, `'\\''`)}'`;
const lines = ["# generated by cmux; the same for every machine; managed, do not edit"];
for (const [key, value] of Object.entries(env)) {
if (!ENV_NAME.test(key)) throw new Error(`model-plane env key ${JSON.stringify(key)} is not a shell identifier`);
if (ROUTE_TOKEN_GRAMMAR.test(value)) throw new Error(`model-plane env ${key} carries a route token; the guest never holds one`);
if (value === "") continue;
lines.push(`export ${key}=${quote(value)}`);
}
return `${lines.join("\n")}\n`;
}

/** Where the bake puts the file; agent-config.sh falls back to it when no boot env and no per-home file exist. */
export const VM_GUEST_MODEL_PLANE_ENV_PATH = "/etc/cmux/model-plane.env";
23 changes: 7 additions & 16 deletions web/services/coderouter/vmModelPlane.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
// unwired machine (no env, no rule, still no secret). Never set it in
// production. Tokens never rotate; destroy revokes them.
import { issueRouteToken, revokeRouteTokensForVm } from "./repository";
import { ROUTE_TOKEN_HEADER, VM_ID_HEADER, VM_PLACEHOLDER_API_KEY } from "./routeTokenAuth";
import { ROUTE_TOKEN_HEADER, VM_ID_HEADER } from "./routeTokenAuth";
import { vmEdgeAliasDomain } from "./vmGuestEnv";
import type { VmEdgeRule } from "../vms/drivers/types";

export const VM_ROUTE_TOKEN_LABEL = "vm";
Expand All @@ -29,8 +30,6 @@ export type VmModelPlaneInput = {
};

export type VmModelPlaneProvision = {
/** Guest env: base URLs, placeholder keys, the VM id. Never a token. */
readonly envs: Record<string, string>;
/** Edge header injection for the coderouter host. Holds the token. */
readonly edgeRules: readonly VmEdgeRule[];
};
Expand Down Expand Up @@ -131,17 +130,6 @@ export function coderouterEdgeOrigin(raw: string | undefined): string {
return url.origin;
}

/** The guest env for one origin and VM id. No token anywhere. */
export function vmModelPlaneEnvs(origin: string, cloudVmId: string): Record<string, string> {
return {
OPENAI_BASE_URL: `${origin}/v1`,
OPENAI_API_KEY: VM_PLACEHOLDER_API_KEY,
CMUX_CODEROUTER_URL: origin,
ANTHROPIC_BASE_URL: origin,
ANTHROPIC_API_KEY: VM_PLACEHOLDER_API_KEY,
CMUX_VM_ID: cloudVmId,
};
}

/**
* Mint the machine's route token and build its edge rule and env. Throws
Expand Down Expand Up @@ -169,11 +157,14 @@ export async function provisionVmModelPlane(
} catch (err) {
throw new VmModelPlaneUnavailableError(`coderouter route token issue failed: ${errorMessage(err)}`, err);
}
// The guest dials the alias; the edge terminates it and forwards to this
// deployment's API host with the machine's token. The guest env is static
// and baked (services/coderouter/vmGuestEnv.ts), so nothing is written here.
return {
envs: vmModelPlaneEnvs(origin, input.cloudVmId),
edgeRules: [
{
domain: new URL(origin).hostname,
domain: vmEdgeAliasDomain(),
destinationHost: new URL(origin).hostname,
headers: {
...edgeOriginHeaders(dependencies),
[ROUTE_TOKEN_HEADER]: token,
Expand Down
46 changes: 25 additions & 21 deletions web/services/vms/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -375,8 +375,10 @@ the public platform `api.freestyle.sh`): chatmux-devbox tool parity (mise node/p
uv, gh, devtools, pinned coding agents, ble.sh, half-life prompt, seeded history). Machines
run no cmuxd-remote: the **cmux-tui remote daemon is the machine's only session daemon**,
and the bake installs the pinned static-musl build at `/root/.cmux/bin/cmux-tui` with
`sha256sum -c` verification. A create is `vms.create`, the grow-only resize, and one
model-plane env file write; the baked supervisor starts the daemon with a fresh identity
`sha256sum -c` verification. A create is one `vms.create` (firewall, VPC, and the coderouter
TLS rule inline); a size-less image also gets the grow-only resize. Nothing is written into
the guest: the model-plane env is baked (see "Model plane"). The baked supervisor starts the
daemon with a fresh identity
within a second of resume (the snapshot is a memory image, so the identity is keyed on
the platform instance id). Attach heals a daemon that is not listening, reinstalling only
when the binary is missing or behind the manifest pin. The daemon runs as root with
Expand All @@ -403,15 +405,15 @@ route at the stable public IPv6. The daemon binds dual-stack (`[::]:1337`), re-a
every attach-time heal, which is also what makes the VPC address reachable.
The Noise handshake encrypts and authenticates the session end to end, so carrier TLS is not
required; the route token exists only for the lease ledger. Creates take no ports field and
no create-time env, so the coderouter model-plane vars are delivered by writing the persisted
`/root/.config/cmux/model-plane.env` (0600) that `/etc/cmux/agent-config.sh` already sources.
That file carries base URLs, a placeholder key, and the VM id only; the credential is
no create-time env; the guest's model-plane env is the same for every machine and baked at
`/etc/cmux/model-plane.env`, which `/etc/cmux/agent-config.sh` sources when no boot env and no
per-home file exist. It carries alias base URLs and a placeholder key only; the credential is
edge-injected (see "Model plane" below).

Every guest command is run with `linuxUser: "root"`. The 0.2 API's default is *not* root but
"the account holding uid 1000, or root in an image with no such account", and the devbox image
ships a uid-1000 user — leaving it unset would silently move the daemon, its install, and the
model-plane write off the root layout they are baked around.
ships a uid-1000 user — leaving it unset would silently move the daemon and its install off
the root layout they are baked around.

`POST /api/vm/[id]/attach-endpoint` with
`{"transport":"cmux-remote","clientCapabilities":[...]}` returns
Expand Down Expand Up @@ -446,20 +448,22 @@ No coderouter secret ever lands in a guest. `createVm`/`restoreVm` take a `model
provisioner (`services/vms/modelPlaneGateway.ts` adapting
`services/coderouter/vmModelPlane.ts`). After the `cloud_vms` row exists and before the
provider call, it mints one route token bound to the row id (`coderouter_route_tokens.vm_id`)
and returns guest env (`OPENAI_BASE_URL`, `ANTHROPIC_BASE_URL`, `CMUX_CODEROUTER_URL`,
placeholder `OPENAI_API_KEY`/`ANTHROPIC_API_KEY`, `CMUX_VM_ID`) plus one edge rule for the
coderouter host with headers `x-coderouter-route-token` and `x-cmux-vm-id`. The Freestyle
driver passes the rule inline as `tls.rules` on the create; the platform steers the host to
its edge and installs its CA in the guest at boot, and the edge injects (and overwrites) those
headers on every request. coderouter rejects a bound token whose request carries a different
VM id. Rules created after boot never reach a running guest, so the rule is never added later.

The guest dials the real name, `https://coderouter.dev` by default;
`CMUX_CODEROUTER_EDGE_ORIGIN` (a bare https origin) points guests at a preview deployment.
Injection activates 20-30 s after boot, so the driver ends bootstrap with a guest-side probe of
`https://<host>/v1/models` (one exec, bounded loop) and rolls the machine back if it never
succeeds. Node harnesses (Claude Code, pi) need `NODE_EXTRA_CA_CERTS`, which
`agent-config.sh` exports when the platform CA file exists.
and returns one edge rule: domain `coderouter.cmux.internal` (the alias every guest dials;
`CMUX_VM_EDGE_ALIAS_DOMAIN` overrides it per deployment, never per machine), destination host
this deployment's API host, and headers `x-coderouter-route-token` and `x-cmux-vm-id`. The
Freestyle driver passes the rule inline as `tls.rules` on the create; the platform resolves the
alias to its edge, installs its CA in the guest at boot, terminates TLS for the alias, forwards
to the destination host, and injects (and overwrites) those headers on every request.
coderouter rejects a bound token whose request carries a different VM id. Rules created after
boot never reach a running guest, so the rule is never added later.

Because the guest always dials the alias, its env is identical everywhere and is baked
(`services/coderouter/vmGuestEnv.ts`, written by the bake to `/etc/cmux/model-plane.env`):
`OPENAI_BASE_URL`, `ANTHROPIC_BASE_URL`, `CMUX_CODEROUTER_URL` on the alias origin and the
placeholder `OPENAI_API_KEY`/`ANTHROPIC_API_KEY`. `CMUX_CODEROUTER_EDGE_ORIGIN` (a bare https
origin) only moves the rule's destination, for a preview deployment. Injection activates a few
seconds after boot; nothing waits for it. Node harnesses (Claude Code, pi) need
`NODE_EXTRA_CA_CERTS`, which `agent-config.sh` exports when the platform CA file exists.

Provisioning is mandatory: a coderouter outage fails the create with
`vm_model_plane_unavailable` (503, retryable), refunds the create credit, marks the row failed
Expand Down
Loading