Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 90 additions & 3 deletions src/cli/commands/proxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -188,8 +188,50 @@
proxySupervisorStateManager.save(state);
}

/**
* Drop a supervisor `version` that is not a string.
*
* `StateFileManager.load()` is a bare `JSON.parse(content) as T` — it validates
* nothing. A state file written by a different build, or half-written during a
* crash, can carry any JSON type here, and every status renderer interpolates
* the field straight into `v${...}`. Coercing at the single load boundary keeps
* a non-string from reaching the output as "v[object Object]".
*/
export function normalizeSupervisorState(
state: ProxySupervisorState | null,
): ProxySupervisorState | null {
if (!state) {
return null;
}
return typeof state.version === "string" || state.version === undefined
? state
: { ...state, version: undefined };
}

/**
* Whether a rolling handoff can actually occur.
*
* A live supervisor PID alone is not enough: a supervisor from a build
* predating rolling state leaves `rolling` absent, and calling that a handoff
* makes both `/status` clients and the CLI wait for an activation that will
* never come. Gate on the capability, not on the process.
*/
export function isRollingHandoffCapable(
state: ProxySupervisorState | null,
isRunning: (pid: number) => boolean = isProcessRunning,
): boolean {
if (!state) {
return false;
}
// Structural, not just non-null: the same unvalidated `as T` load that lets
// `version` be an object lets `rolling` be a string.
const hasRollingState =
typeof state.rolling === "object" && state.rolling !== null;
return isRunning(state.pid) && hasRollingState;
}

function loadProxySupervisorState(): ProxySupervisorState | null {
return proxySupervisorStateManager.load();
return normalizeSupervisorState(proxySupervisorStateManager.load());
}

function clearProxySupervisorState(): void {
Expand Down Expand Up @@ -1602,7 +1644,7 @@
});
}

export async function createProxyStartApp(params: {

Check warning on line 1647 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / test (20)

Async function 'createProxyStartApp' has too many lines (727). Maximum allowed is 300

Check warning on line 1647 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / test (20)

Async function 'createProxyStartApp' has too many lines (727). Maximum allowed is 300

Check warning on line 1647 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / 🛡️ Code Quality & Security Gate

Async function 'createProxyStartApp' has too many lines (727). Maximum allowed is 300
neurolink: ProxyNeurolinkRuntime["neurolink"];
modelRouter: ModelRouterInterface | undefined;
strategy: ProxyStartStrategy;
Expand Down Expand Up @@ -1967,7 +2009,7 @@
);
});

app.get("/status", async (c) => {

Check warning on line 2012 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / test (20)

Async arrow function has too many lines (359). Maximum allowed is 300

Check warning on line 2012 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / test (20)

Async arrow function has too many lines (359). Maximum allowed is 300

Check warning on line 2012 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / 🛡️ Code Quality & Security Gate

Async arrow function has too many lines (359). Maximum allowed is 300
const runtimeConfig = params.runtimeConfigStore?.getSnapshot();
const runtimeConfigStatus = params.runtimeConfigStore?.getStatus();
const activeStrategy = runtimeConfig
Expand Down Expand Up @@ -1999,6 +2041,7 @@
: undefined;
const runtimeState = loadProxyState();
const supervisorState = loadProxySupervisorState();
const rollingSupervisorRunning = isRollingHandoffCapable(supervisorState);
const updateState = loadUpdateState();
const cooldowns = await loadAccountCooldowns();
const storedAccountKeys = new Set<string>();
Expand Down Expand Up @@ -2267,14 +2310,25 @@
autoUpdate: {
enabled: isProxyAutoUpdateEnabled(),
supervisorPid: supervisorState?.pid ?? null,
supervisorVersion: supervisorState?.version ?? null,
rolling: supervisorState?.rolling ?? null,
Comment thread
murdore marked this conversation as resolved.
updaterPid: activeUpdaterPid ?? null,
updaterRunning: activeUpdaterPid
? isProcessRunning(activeUpdaterPid)
: false,
liveVersion: PROXY_VERSION,
latestVersion: updateState?.lastCheckVersion || null,
lastDetectedVersion: updateState?.lastCheckVersion || null,
installedVersion:
updateState?.installedVersion ??
updateState?.lastUpdateVersion ??
null,
activatedVersion: PROXY_VERSION,
pendingActivationVersion: updateState?.pendingRestartVersion ?? null,
pendingRestartVersion: updateState?.pendingRestartVersion ?? null,
activationMode: rollingSupervisorRunning
? "rolling-handoff"
: "restart",
Comment thread
coderabbitai[bot] marked this conversation as resolved.
deferredUpdate: updateState?.deferredUpdate ?? null,
lastCheckAt: updateState?.lastCheckAt ?? null,
lastUpdateAt: updateState?.lastUpdateAt ?? null,
Expand Down Expand Up @@ -2757,7 +2811,7 @@
return shutdown;
}

async function startProxyRuntime(params: {

Check warning on line 2814 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / test (20)

Async function 'startProxyRuntime' has too many lines (303). Maximum allowed is 300

Check warning on line 2814 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / test (20)

Async function 'startProxyRuntime' has too many lines (303). Maximum allowed is 300

Check warning on line 2814 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / 🛡️ Code Quality & Security Gate

Async function 'startProxyRuntime' has too many lines (303). Maximum allowed is 300
argv: ProxyStartArgs;
spinner: ProxySpinner;
app: ProxyStartApp["app"];
Expand Down Expand Up @@ -3096,6 +3150,7 @@
host,
port,
startTime: supervisorStartedAt,
version: PROXY_VERSION,
updaterPid: currentUpdaterPid,
rolling: snapshot,
});
Expand Down Expand Up @@ -3611,7 +3666,7 @@
"Show proxy status as JSON",
) as Argv<ProxyStatusArgs>;
},
handler: async (argv) => {

Check warning on line 3669 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / test (20)

Async method 'handler' has too many lines (342). Maximum allowed is 300

Check warning on line 3669 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / test (20)

Async method 'handler' has too many lines (342). Maximum allowed is 300

Check warning on line 3669 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / 🛡️ Code Quality & Security Gate

Async method 'handler' has too many lines (342). Maximum allowed is 300
try {
const state = loadProxyState();
const supervisorState = loadProxySupervisorState();
Expand All @@ -3636,11 +3691,19 @@
autoUpdateEnabled: isProxyAutoUpdateEnabled(),
workerVersion: null as string | null,
supervisorPid: null as number | null,
supervisorVersion: supervisorState?.version ?? null,
supervisorRunning: false,
rolling: null as ProxySupervisorState["rolling"] | null,
updaterPid: null as number | null,
updaterRunning: false,
latestVersion: updateState?.lastCheckVersion || null,
lastDetectedVersion: updateState?.lastCheckVersion || null,
installedVersion:
updateState?.installedVersion ??
updateState?.lastUpdateVersion ??
null,
activatedVersion: supervisorState?.rolling.active?.version ?? null,
pendingActivationVersion: updateState?.pendingRestartVersion ?? null,
pendingRestartVersion: updateState?.pendingRestartVersion ?? null,
deferredUpdate: updateState?.deferredUpdate ?? null,
lastUpdateFailure: updateState?.lastFailure ?? null,
Expand Down Expand Up @@ -3692,6 +3755,7 @@
servingState?.lastConfigReloadError ?? null;
status.supervisorPid = supervisorPid ?? null;
status.supervisorRunning = supervisorRunning;
status.supervisorVersion = supervisorState?.version ?? null;
status.rolling = supervisorState?.rolling ?? null;
status.updaterPid =
supervisorState?.updaterPid ?? servingState?.updaterPid ?? null;
Expand Down Expand Up @@ -3719,6 +3783,7 @@
typeof statusData.version === "string"
? statusData.version
: null;
status.activatedVersion = status.workerVersion;
if (typeof liveConfig?.generation === "number") {
status.configGeneration = liveConfig.generation;
}
Expand Down Expand Up @@ -3765,6 +3830,11 @@
` ${chalk.bold("Supervisor:")} ${status.supervisorRunning ? chalk.cyan(status.supervisorPid) : chalk.red(`${status.supervisorPid} (not running)`)}`,
);
}
if (status.supervisorVersion) {
logger.always(
` ${chalk.bold("Supervisor version:")} ${chalk.cyan(`v${status.supervisorVersion}`)}`,
);
}
if (status.workerVersion) {
logger.always(
` ${chalk.bold("Version:")} ${chalk.cyan(`v${status.workerVersion}`)}`,
Expand Down Expand Up @@ -3801,9 +3871,16 @@
logger.always(
` ${chalk.bold("Auto-update:")} ${status.autoUpdateEnabled ? chalk.green(status.updaterRunning ? `enabled (PID ${status.updaterPid})` : "enabled (worker unavailable)") : chalk.yellow("disabled")}`,
);
if (status.pendingRestartVersion) {
if (status.pendingActivationVersion) {
// A live supervisor PID alone does NOT mean rolling handoff is
// available: a supervisor from a build predating rolling state leaves
// `rolling` absent, and calling that a handoff tells the operator to
// wait for an activation that will never come. Gate on the capability,
// not on the process.
const rollingCapable =
status.supervisorRunning && status.rolling !== null;
logger.always(
` ${chalk.bold("Pending:")} ${chalk.yellow(`v${status.pendingRestartVersion} installed; restart pending`)}`,
` ${chalk.bold(rollingCapable ? "Pending handoff:" : "Pending restart:")} ${chalk.yellow(`v${status.pendingActivationVersion} installed; ${rollingCapable ? "rolling activation pending" : "restart pending"}`)}`,
);
}
if (status.rolling?.candidate) {
Expand All @@ -3829,6 +3906,16 @@
` ${chalk.bold("Latest:")} ${chalk.cyan(`v${status.latestVersion}`)}`,
);
}
if (status.installedVersion) {
logger.always(
` ${chalk.bold("Installed:")} ${chalk.cyan(`v${status.installedVersion}`)}`,
);
}
if (status.activatedVersion) {
logger.always(
` ${chalk.bold("Activated:")} ${chalk.cyan(`v${status.activatedVersion}`)}`,
);
}
if (status.lastUpdateFailure) {
logger.always(
` ${chalk.bold("Update error:")} ${chalk.red(`${status.lastUpdateFailure.stage}: ${status.lastUpdateFailure.message}`)}`,
Expand Down Expand Up @@ -4041,7 +4128,7 @@
default: true,
}) as Argv<ProxyGuardArgs>;
},
handler: async (argv) => {

Check warning on line 4131 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / test (20)

Async method 'handler' has too many lines (659). Maximum allowed is 300

Check warning on line 4131 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / test (20)

Async method 'handler' has too many lines (659). Maximum allowed is 300

Check warning on line 4131 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / 🛡️ Code Quality & Security Gate

Async method 'handler' has too many lines (659). Maximum allowed is 300
const host = argv.host ?? "127.0.0.1";
const port = argv.port ?? 55669;
const parentPid = Number(argv.parentPid);
Expand Down Expand Up @@ -4123,7 +4210,7 @@
};
let updateInProgress = false;
let updateRestartInProgress = false;
const runUpdateCheck = async () => {

Check warning on line 4213 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / test (20)

Async arrow function has too many lines (457). Maximum allowed is 300

Check warning on line 4213 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / test (20)

Async arrow function has too many lines (457). Maximum allowed is 300

Check warning on line 4213 in src/cli/commands/proxy.ts

View workflow job for this annotation

GitHub Actions / 🛡️ Code Quality & Security Gate

Async arrow function has too many lines (457). Maximum allowed is 300
if (guardStopping || updateInProgress) {
return;
}
Expand Down
20 changes: 19 additions & 1 deletion src/lib/proxy/updateState.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ export function getDefaultUpdateState(): UpdateState {
lastCheckAt: new Date(0).toISOString(),
lastCheckVersion: "",
suppressedVersions: {},
installedVersion: null,
lastUpdateAt: null,
lastUpdateVersion: null,
pendingRestartVersion: null,
Expand Down Expand Up @@ -146,6 +147,21 @@ export function loadUpdateState(stateFilePath?: string): UpdateState | null {
...getDefaultUpdateState(),
...candidate,
suppressedVersions: candidate.suppressedVersions ?? {},
// Backfill order matters for state files written before `installedVersion`
// existed. Back then `recordUpdateInstalled()` set ONLY
// `pendingRestartVersion`, leaving `lastUpdateVersion` on the previously
// activated build — so a validated-but-not-yet-running update lives in
// `pendingRestartVersion` and is the newer of the two. Reading
// `lastUpdateVersion` first would report the superseded version as
// installed and re-offer an update that is already on disk.
installedVersion:
typeof candidate.installedVersion === "string"
? candidate.installedVersion
: typeof candidate.pendingRestartVersion === "string"
? candidate.pendingRestartVersion
: typeof candidate.lastUpdateVersion === "string"
? candidate.lastUpdateVersion
: null,
pendingRestartVersion:
typeof candidate.pendingRestartVersion === "string"
? candidate.pendingRestartVersion
Expand Down Expand Up @@ -245,19 +261,21 @@ export function recordSuccessfulUpdate(
const state = loadUpdateState(stateFilePath) ?? getDefaultUpdateState();
state.lastUpdateAt = new Date().toISOString();
state.lastUpdateVersion = version;
state.installedVersion = version;
state.pendingRestartVersion = null;
state.deferredUpdate = null;
state.lastFailure = null;
delete state.suppressedVersions[version];
saveUpdateState(state, stateFilePath);
}

/** Record that package installation completed but the live restart is pending. */
/** Record that the package was validated but live activation is still pending. */
export function recordUpdateInstalled(
version: string,
stateFilePath?: string,
): void {
const state = loadUpdateState(stateFilePath) ?? getDefaultUpdateState();
state.installedVersion = version;
state.pendingRestartVersion = version;
state.lastFailure = null;
saveUpdateState(state, stateFilePath);
Expand Down
2 changes: 2 additions & 0 deletions src/lib/types/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -999,6 +999,8 @@ export type ProxySupervisorState = {
host: string;
port: number;
startTime: string;
/** Version loaded by the long-lived supervisor process. */
version?: string;
updaterPid?: number;
rolling: ProxyRollingState;
};
Expand Down
9 changes: 9 additions & 0 deletions src/lib/types/proxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2014,6 +2014,15 @@ export type UpdateState = {
lastCheckAt: string;
lastCheckVersion: string;
suppressedVersions: Record<string, SuppressedVersion>;
/**
* Last package version whose stable trampoline was successfully validated.
*
* Optional because `UpdateState` is part of the published type surface and a
* required addition would break every downstream object literal — and because
* state files written before this field existed legitimately omit it.
* `loadUpdateState()` always materializes it, so runtime readers see a value.
*/
installedVersion?: string | null;
lastUpdateAt: string | null;
lastUpdateVersion: string | null;
Comment thread
murdore marked this conversation as resolved.
/** Installed by the updater but not yet confirmed as the running version. */
Expand Down
Loading
Loading