Skip to content
Closed
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
233 changes: 211 additions & 22 deletions src/cli/cross-home-owner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,80 @@ import { markSiblingStart, siblingOfLivePort } from "../codex/sibling-start";
import { readClientConnectionState } from "../client/state";
import { findManagedRegion, resolveGrokHome } from "../grok/inject";
import { providerTableString } from "../codex/injected-marker";
import { probePortOwner, START_OWNERSHIP_LIVENESS } from "../server/proxy-liveness";
import { isLocalAttestationSecret } from "../lib/local-management-attestation";
import { readOwnerRegistryHomes } from "../config/owner-registry";
import {
classifyHealthz,
loopbackProbeHosts,
proveLiveProxyOwnedByHome,
proxyIdentityAt,
START_OWNERSHIP_LIVENESS,
type LivenessIo,
} from "../server/proxy-liveness";
import { directLocalHttpFetch } from "../server/direct-local-http";
import type { RuntimePortState } from "../config/process-state";

const MAX_HINT_BYTES = 256 * 1024;
// Far above any real Grok config; discovery must not stall startup when Grok sync is off.
const MAX_GROK_CONFIG_BYTES = 16 * 1024 * 1024;

/**
* One registered home's runtime record as discovery sees it.
*
* 'attestable' distinguishes the two shapes a well-formed record can take: a current
* record carries the attestation secret another home can challenge, while a legacy
* record (written before secrets existed, or with the secret stripped) can only name
* a pid/port pair. A live legacy record is never "no owner" - it is an owner the
* reader cannot verify, and a shared-write decision must fail closed on it.
*/
type CandidateRecord = {
home: string;
pid: number;
port: number;
hostname?: string;
siblingOfPort?: number;
attestable: boolean;
record: RuntimePortState;
};

function parseCandidateRecord(home: string, raw: string): CandidateRecord | null {
try {
const record: unknown = JSON.parse(raw);
if (!record || typeof record !== "object") return null;
const state = record as Record<string, unknown>;
if (!Number.isSafeInteger(state.pid) || Number(state.pid) <= 0 || !validPort(state.port)) return null;
if (state.hostname !== undefined && typeof state.hostname !== "string") return null;
if (state.siblingOfPort !== undefined
&& !(Number.isInteger(state.siblingOfPort) && Number(state.siblingOfPort) > 0 && Number(state.siblingOfPort) <= 65535)) {
return null;
}
const attestable = isLocalAttestationSecret(state.attestationSecret);
return {
home,
pid: Number(state.pid),
port: Number(state.port),
hostname: typeof state.hostname === "string" ? state.hostname : undefined,
siblingOfPort: typeof state.siblingOfPort === "number" ? state.siblingOfPort : undefined,
attestable,
record: state as RuntimePortState,
};
} catch {
return null;
}
}

/**
* The discovery verdict. "owner" names the port a sibling defers to. "indeterminate"
* means something on the shared clients' path could be an owner the reader could not
* verify - a live listener no record attests, a legacy record, or an unreadable
* transport - and a shared-write caller must fail closed on it. 'port' carries the
* best location hint for the sibling marker.
*/
export type CrossHomeOwnerVerdict =
| { kind: "owner"; port: number }
| { kind: "indeterminate"; port: number | null; reason: string }
| { kind: "none" };

/** Nonblocking open (a FIFO cannot stall startup), regular files only, capped at maxBytes. */
function readBoundedRegularFile(path: string, maxBytes: number): string | null {
let fd: number | undefined;
Expand Down Expand Up @@ -60,20 +128,42 @@ function loopbackPort(raw: string | null): number | null {
}

/** Returns only a different process with an identity-checked /healthz response. */
export async function findCrossHomeOwner(options: { homeDir?: string } = {}): Promise<number | null> {
export async function findCrossHomeOwner(options: { homeDir?: string; io?: LivenessIo } = {}): Promise<number | null> {
const verdict = await findCrossHomeOwnerDetailed(options);
return verdict.kind === "owner" ? verdict.port : null;
}

export async function findCrossHomeOwnerDetailed(options: { homeDir?: string; io?: LivenessIo } = {}): Promise<CrossHomeOwnerVerdict> {
const io = options.io ?? {};
const nowFn = io.nowFn ?? Date.now;
const timeoutMs = io.timeoutMs ?? START_OWNERSHIP_LIVENESS.timeoutMs ?? 1_500;
const attempts = io.attempts ?? START_OWNERSHIP_LIVENESS.attempts ?? 3;
// One deadline bounds the whole discovery: every identity probe, attestation and
// fallback classification shares it, so a pile of stale hints cannot stretch the
// start path the way per-candidate budgets did (CodeRabbit on #6198).
const deadlineAt = io.deadlineAt ?? nowFn() + timeoutMs * attempts * 4;
const probeIo: LivenessIo = { ...io, timeoutMs, attempts, deadlineAt };

const candidates = new Set<number>();
const records = new Map<number, CandidateRecord[]>();
const ownHome = resolve(getConfigDir());
const recordHome = (home: string): void => {
if (resolve(home) === ownHome) return;
const raw = readBoundedRegularFile(join(home, "runtime-port.json"), MAX_HINT_BYTES);
if (!raw) return;
const parsed = parseCandidateRecord(home, raw);
if (!parsed) return;
candidates.add(parsed.port);
const list = records.get(parsed.port) ?? [];
list.push(parsed);
records.set(parsed.port, list);
};

const defaultHome = join(options.homeDir ?? homedir(), ".opencodex");
if (resolve(getConfigDir()) !== resolve(defaultHome)) {
const raw = readBoundedRegularFile(join(defaultHome, "runtime-port.json"), MAX_HINT_BYTES);
if (raw) {
try {
const record: unknown = JSON.parse(raw);
if (record && typeof record === "object" && validPort((record as { port?: unknown }).port)) {
candidates.add((record as { port: number }).port);
}
} catch { /* stale or malformed hint */ }
}
}
recordHome(defaultHome);
// The shared registry is what lets one custom home find another: every runtime
// that published a runtime record registered its home beside the shared clients.
for (const home of readOwnerRegistryHomes()) recordHome(home);

// Grok's writer reads its config in full; cap discovery separately so startup stays bounded.
const grok = readBoundedRegularFile(join(resolveGrokHome(), "config.toml"), MAX_GROK_CONFIG_BYTES);
Expand All @@ -96,17 +186,116 @@ export async function findCrossHomeOwner(options: { homeDir?: string } = {}): Pr
}
} catch { /* an absent or invalid client home is not owner evidence */ }

for (const port of candidates) {
const owner = await probePortOwner(port, {}, START_OWNERSHIP_LIVENESS);
if (owner && Number.isSafeInteger(owner.pid) && owner.pid! > 0 && owner.pid !== process.pid) return port;
let indeterminatePort: number | null = null;
let indeterminateReason: string | null = null;
const noteIndeterminate = (port: number | null, reason: string): void => {
if (indeterminateReason === null) {
indeterminatePort = port;
indeterminateReason = reason;
}
};

const queue = [...candidates];
const probed = new Set<number>();
for (const port of queue) {
if (!probed.add(port)) continue;
const portRecords = records.get(port) ?? [];
// The recorded hostnames are tried first; every remaining loopback family is a
// candidate too, because IPv4 and IPv6 listeners on one port are independent.
const hosts: string[] = [];
for (const rec of portRecords) {
for (const host of loopbackProbeHosts(rec.hostname)) {
if (!hosts.includes(host)) hosts.push(host);
}
}
if (hosts.length === 0) hosts.push(...loopbackProbeHosts(undefined));

for (const hostname of hosts) {
if (deadlineAt - nowFn() <= 0) {
noteIndeterminate(port, "ownership discovery ran out of its shared time budget");
break;
}
const identity = await proxyIdentityAt(port, { hostname }, probeIo);
if (identity === null) {
// A null identity is not "free": distinguish a unanimous connect refusal
// (the family is genuinely empty) from a transport or shape that stayed
// unreadable. On a managed port an unreadable answer is owner evidence the
// caller cannot dismiss.
const remainingMs = deadlineAt - nowFn();
if (remainingMs <= 0) {
noteIndeterminate(port, "ownership discovery ran out of its shared time budget");
break;
}
const classification = await classifyHealthz(
"http://" + hostname + ":" + port + "/healthz",
probeIo.fetchFn ?? directLocalHttpFetch,
Math.min(timeoutMs, remainingMs),
);
if (classification === "dead") continue;
noteIndeterminate(port, "a managed client port answered but its listener could not be classified");
continue;
}
if (identity.pid === null) {
noteIndeterminate(port, "a live opencodex listener reported no pid to attest");
continue;
}
if (identity.pid === process.pid) continue;

const matching = portRecords.filter(rec => rec.pid === identity.pid);
if (matching.length === 0) {
// Our opencodex answers on a managed port but no registered record names it:
// it may be an owner whose home never registered (an install predating the
// registry, or a CODEX_HOME the writer could not reach). Unverifiable is not
// absent - fail closed instead of starting a second owner beside it.
noteIndeterminate(port, "a live opencodex listener on a managed port matches no registered owner record");
continue;
}
for (const rec of matching) {
if (rec.siblingOfPort !== undefined) {
// The attested process is itself a sibling; its runtime record names the
// real owner's port. Follow it instead of deferring to the sibling.
if (validPort(rec.siblingOfPort)) queue.push(rec.siblingOfPort);
continue;
}
if (!rec.attestable) {
noteIndeterminate(port, "a live listener matches a runtime record that carries no attestation secret");
continue;
}
const proof = await proveLiveProxyOwnedByHome(
{ ...identity, hostname, port, source: "runtime" },
{ ...probeIo, readRuntimeFn: () => rec.record },
);
if (proof === "proven") return { kind: "owner", port };
if (proof === "indeterminate") {
noteIndeterminate(port, "the recorded owner's attestation could not be completed");
}
// "refuted" means this listener definitively is not the recorded owner on
// this record; other records and hosts still get their turn.
}
}
}

if (indeterminateReason !== null) {
return { kind: "indeterminate", port: indeterminatePort, reason: indeterminateReason };
}
return null;
return { kind: "none" };
}

/** Mark this process before any shared-client write when another home owns the clients. */
export async function markCrossHomeSibling(): Promise<boolean> {
const port = await findCrossHomeOwner();
if (port === null) return false;
const verdict = await findCrossHomeOwnerDetailed();
if (verdict.kind === "none") return false;
if (verdict.kind === "indeterminate") {
// Fail closed: an owner the reader could not verify still owns the shared
// writes. Marking the best port hint keeps every sibling gate engaged even
// while the answer stays unproven (#6198).
console.warn(
"A shared-client owner could not be verified (" + verdict.reason + "); "
+ "treating this instance as a sibling so Codex, Grok and Claude configs are left alone.",
);
}
const port = verdict.port;
if (port === null) return true;
markSiblingStart(port);
return true;
}
Expand All @@ -119,9 +308,9 @@ export async function markLiveHomeSibling(live: { pid: number | null; port: numb
markSiblingStart(runtime.siblingOfPort);
return true;
}
const otherPort = await findCrossHomeOwner();
if (otherPort === null || otherPort === live.port) return false;
markSiblingStart(otherPort);
const verdict = await findCrossHomeOwnerDetailed();
if (verdict.kind === "none" || verdict.port === live.port) return false;
if (verdict.port !== null) markSiblingStart(verdict.port);
return true;
}

Expand Down
2 changes: 1 addition & 1 deletion src/cli/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1290,7 +1290,7 @@ async function handleStopUnlocked(snapshot?: GuardedStopSnapshot) {
if (siblingStopFoundOwner(siblingOfPort, live)) {
record.proxy = "not-running";
console.log(`The sibling instance is already gone; the proxy on port ${siblingOfPort} was left running.`);
} else if (live?.pid && !(await proveLiveProxyOwnedByHome(live))) {
} else if (live?.pid && (await proveLiveProxyOwnedByHome(live)) !== "proven") {
stopFailed = true;
ownershipBlocked = true;
record.proxy = "ownership-refused";
Expand Down
105 changes: 105 additions & 0 deletions src/config/owner-registry.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
/**
* The cross-home owner registry.
*
* findCrossHomeOwner proves an owner through a home's protected runtime-port.json,
* but it used to know exactly one home: the default ~/.opencodex. A custom-home
* owner therefore stayed invisible to every other home (#6198): a second custom home
* saw the shared clients' managed URL, could not prove the answering process, and
* started as a competing owner that re-pointed shared Codex/Grok/Claude routing.
*
* This registry is the protected stable locator for those homes. Every runtime that
* publishes runtime-port.json also drops one tiny pointer file here so another home
* can find the record to attest against. The entries carry the home path only - the
* attestation secret stays inside the home's own record - so a forged or stale entry
* can never grant ownership; it can only send the reader to a record that still has
* to prove itself.
*
* The anchor is CODEX_HOME because that is the shared state this protects: the
* managed Codex client config lives there, and a sibling exists to leave that write
* alone. Entries are written atomically and never read back as truth - they only
* nominate a home for the caller's own record + liveness verification.
*/

import { createHash } from "node:crypto";
import { mkdirSync, readdirSync, readFileSync, statSync } from "node:fs";
import { homedir } from "node:os";
import { join, resolve } from "node:path";
import { atomicWriteFile } from "./atomic-write";
import { getConfigDir } from "./paths";
import { assertNotRealCodexHomeUnderTest } from "../lib/test-home-guard";

const REGISTRY_DIR_NAME = "ocx-homes";
const REGISTRY_ENTRY_SUFFIX = ".json";
const MAX_REGISTRY_ENTRIES = 64;
const MAX_ENTRY_BYTES = 4096;
const HOME_KEY_LENGTH = 24;

function registryBaseDir(): string {
const raw = process.env.CODEX_HOME?.trim();
// Without an env override the Codex client home defaults beside the user profile,
// the same location the managed config writers use.
return raw ? resolve(raw) : join(homedir(), ".codex");
}

/** The shared directory the registry lives under. */
export function ownerRegistryDir(): string {
return join(registryBaseDir(), REGISTRY_DIR_NAME);
}

function registryEntryPath(dir: string, home: string): string {
const key = createHash("sha256").update(resolve(home)).digest("hex").slice(0, HOME_KEY_LENGTH);
return join(dir, key + REGISTRY_ENTRY_SUFFIX);
}

/**
* Record 'home' in the shared registry. Best-effort and never throws: a failed write
* only degrades cross-home discovery, it must not break the publish that owns state.
*/
export function registerOwnerRegistryHome(home: string): void {
try {
assertNotRealCodexHomeUnderTest(registryBaseDir());
const dir = ownerRegistryDir();
mkdirSync(dir, { recursive: true });
atomicWriteFile(
registryEntryPath(dir, home),
JSON.stringify({ home: resolve(home), v: 1 }) + "\n",
);
} catch { /* discovery aid only */ }
}

/**
* Every registered home path, including this process's own (the caller filters it
* out). Entries are pointers, not facts: malformed, oversized, or unreadable entries
* are skipped rather than trusted. Bounded so a cluttered directory cannot stall
* startup discovery.
*/
export function readOwnerRegistryHomes(): string[] {
const dir = ownerRegistryDir();
let names: string[];
try {
names = readdirSync(dir)
.filter(name => name.endsWith(REGISTRY_ENTRY_SUFFIX))
.slice(0, MAX_REGISTRY_ENTRIES);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
} catch {
return [];
}
const homes: string[] = [];
for (const name of names) {
const path = join(dir, name);
try {
const stat = statSync(path);
if (!stat.isFile() || stat.size === 0 || stat.size > MAX_ENTRY_BYTES) continue;
const parsed: unknown = JSON.parse(readFileSync(path, "utf8"));
const home = parsed && typeof parsed === "object"
? (parsed as Record<string, unknown>).home
: undefined;
if (typeof home === "string" && home.length > 0) homes.push(home);
} catch { /* a bad entry names nothing */ }
}
return homes;
}

/** Register this process's own home after its runtime record is published. */
export function registerOwnHome(): void {
registerOwnerRegistryHome(getConfigDir());
}
Loading
Loading