test(update): expose detached recovery failure evidence

skills/ocx/references/01_management_surface.md

@@ -72,6 +72,21 @@ JSON mode: `envelope`.
 
 - Reads local config; drives no management API route.
 
+### `ocx provider resets`
+
+Recently detected quota resets and whether reset notifications are enabled.
+
+| Method | Route |
+|---|---|
+| GET | `/api/quota-resets` |
+
+| Flag | Value | Meaning |
+|---|---|---|
+| `--json` | boolean | Emit reset events as JSON. |
+| `--limit` | number | Limit returned events; defaults to 20, capped at 100. |
+
+JSON mode: `payload`.
+
 ### `ocx account list`
 
 Codex OAuth accounts with pool priority and pause state.
@@ -587,6 +602,6 @@ JSON mode: `payload`.
 
 ## Counts
 
-- declared capabilities: 32
+- declared capabilities: 33
 - of those, state-changing: 13
 - head-resolved invocations: 2

src/cli/capabilities.ts

@@ -153,6 +153,17 @@ export const CAPABILITIES: readonly Capability[] = [
     json: "envelope",
     details: ["Reads local config; drives no management API route."],
   },
+  {
+    command: ["provider", "resets"],
+    summary: "Recently detected quota resets and whether reset notifications are enabled.",
+    routes: [{ method: "GET", path: "/api/quota-resets" }],
+    flags: [
+      { name: "--json", value: "boolean", summary: "Emit reset events as JSON." },
+      { name: "--limit", value: "number", summary: "Limit returned events; defaults to 20, capped at 100." },
+    ],
+    mutates: false,
+    json: "payload",
+  },
   {
     command: ["provider", "keychain"],
     summary: "Move a provider's API key into the OS keychain, restore it, or report where it lives.",

src/server/management-api.ts

@@ -138,7 +138,7 @@ async function handleLabRoutesOnDemand(ctx: ManagementContext): Promise<Response
  * its config resolution on all of them.
  */
 async function handleQuotaResetRoutesOnDemand(ctx: ManagementContext): Promise<Response | null> {
-  if (ctx.url.pathname !== "/api/quota-resets") return null;
+  if (!pathInManagementNamespace(ctx.url.pathname, "/api/quota-resets")) return null;
   const { handleQuotaResetRoutes } = await import("./management/quota-reset-routes");
   return handleQuotaResetRoutes(ctx);
 }

src/server/management/route-registry.ts

@@ -276,6 +276,8 @@ export const MANAGEMENT_ROUTES: readonly ManagementRoute[] = [
   { method: "POST", path: "/api/providers/test", module: "server/management/provider-routes", mutates: true },
   { method: "PUT", path: "/api/providers", module: "server/management/provider-routes", mutates: true, exempt: { reason: "deferred-verb", why: "Issue #3280 scopes this atomic batch endpoint to the GUI JSON editor; a matching CLI verb is outside wp5 and remains owed.", owner: "wp5-followup", ownerDoc: "devlog/_plan/260903_bug_drawdown_bcda/050_phase5.md" } },
   { method: "PUT", path: "/api/provider-context-caps", module: "server/management/provider-routes", mutates: true },
+  // server/management/quota-reset-routes
+  { method: "GET", path: "/api/quota-resets", module: "server/management/quota-reset-routes", mutates: false, mechanism: "negated-guard" },
   // server/management/request-history-routes
   { method: "GET", path: "/api/request-history", module: "server/management/request-history-routes", mutates: false },
   // server/management/routing-analytics-routes

tests/gui/rate-limit-reset-credits.test.ts

@@ -503,6 +503,7 @@ describe("rate-limit reset credits", () => {
       expect(getAccountQuota("burst-A")).toEqual({
         shortPercent: 97,
         shortResetAt: 1787401330,
+        shortObservedAt: expect.any(Number),
         shortWindowSeconds: 18000,
         weeklyPercent: 12,
         weeklyResetAt: 1788000000,

tests/update/update-stop-first.test.ts

@@ -1,5 +1,5 @@
 import { afterAll, describe, expect, test } from "bun:test";
-import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
+import { chmodSync, closeSync, existsSync, fstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, symlinkSync, writeFileSync } from "node:fs";
 import { createServer } from "node:net";
 import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
@@ -45,8 +45,9 @@ const PROXY_READY_TIMEOUT_MS = 90_000;
 /** Spawn + readiness + teardown spawn, plus headroom for fixture IO on a loaded runner. */
 const RECOVERY_CASE_TIMEOUT_MS = UPDATE_SPAWN_TIMEOUT_MS + PROXY_READY_TIMEOUT_MS + UPDATE_SPAWN_TIMEOUT_MS + 15_000;
 
-async function waitForProxy(port: number): Promise<boolean> {
+async function waitForProxy(port: number, onFailure: (lastProbe: string) => void): Promise<boolean> {
   const deadline = Date.now() + PROXY_READY_TIMEOUT_MS;
+  let lastProbe = "not attempted";
   while (Date.now() < deadline) {
     try {
       const response = await fetch(`http://127.0.0.1:${port}/healthz`, {
@@ -55,13 +56,89 @@ async function waitForProxy(port: number): Promise<boolean> {
         // as "not ready" for a proxy that is merely slow to accept.
         signal: AbortSignal.timeout(2_000),
       });
+      lastProbe = `HTTP ${response.status}`;
       if (response.ok) return true;
-    } catch { /* detached proxy is still starting */ }
+    } catch (error) {
+      // Error messages can contain URLs/credentials. Report only fixed error categories.
+      lastProbe = diagnosticCategories(error instanceof Error ? `${error.name} ${error.message}` : "");
+    }
     // The detached process exposes readiness only over HTTP; fake timers cannot advance it.
     await Bun.sleep(100);
   }
+  onFailure(lastProbe);
   return false;
 }
+
+function diagnosticCategories(text: string): string {
+  const matches = text.match(/\b(?:ENOENT|EACCES|EPERM|EADDRINUSE|ECONNREFUSED|ECONNRESET|ETIMEDOUT|ERR_MODULE_NOT_FOUND|AbortError|TimeoutError|TypeError|SyntaxError|ReferenceError|RangeError|Cannot find package|Cannot find module|Failed to resolve|ConnectionRefused|FailedToOpenSocket)\b/g);
+  return [...new Set(matches ?? [])].join(", ") || "unclassified (text redacted)";
+}
+
+// Read at most 8 KiB even if a broken child logs continuously. Never emit raw output:
+// arbitrary startup messages may include tokens, account identifiers, or request bodies.
+function recoveryDiagnosticFile(path: string, status = false): string {
+  let fd: number | undefined;
+  try {
+    fd = openSync(path, "r");
+    const size = fstatSync(fd).size;
+    const bytes = Buffer.alloc(Math.min(size, 8192));
+    const count = readSync(fd, bytes, 0, bytes.length, Math.max(0, size - bytes.length));
+    const text = bytes.subarray(0, count).toString("utf8");
+    if (status) {
+      // This file contains fixture-generated records only. Still allowlist every field.
+      return text.split("\n").filter(line => /^(?:launcher-start pid=\d+|launcher-exit code=\d+|runtime-exit code=(?:null|\d+) signal=(?:null|SIG[A-Z0-9]+)|runtime-spawn-error)$/.test(line)).slice(-6).join("; ") || "no exit record";
+    }
+    const frames = [...text.matchAll(/\b(src\/[\w./-]+\.(?:ts|mjs))(?::(\d+)(?::(\d+))?)?/g)]
+      .filter(match => !match[1]!.includes("..") && existsSync(join(repoRoot, match[1]!)))
+      .slice(-6).map(match => `${match[1]}${match[2] ? `:${match[2]}` : ""}${match[3] ? `:${match[3]}` : ""}`);
+    return `bytes=${size}; ${diagnosticCategories(text)}; frames=${frames.join(", ") || "none"}`.slice(0, 1200);
+  } catch {
+    return "unavailable";
+  } finally {
+    if (fd !== undefined) closeSync(fd);
+  }
+}
+
+function instrumentRecoveryLauncher(source: string, directory: string): string {
+  // Fail closed on launcher drift: never silently run an uninstrumented fixture or
+  // alter another spawn. Production bin/ocx.mjs and all real lifecycle code stay intact.
+  const replaceOnce = (needle: string, replacement: string) => {
+    if (source.split(needle).length !== 2) throw new Error("recovery diagnostic fixture: launcher seam changed");
+    source = source.replace(needle, () => replacement);
+  };
+  replaceOnce('import { spawn, spawnSync } from "node:child_process";', `
+import { spawn as fixtureSpawn, spawnSync } from "node:child_process";
+import { openSync as fixtureOpen, closeSync as fixtureClose, appendFileSync as fixtureAppend } from "node:fs";
+const fixtureDiagnosticDir = ${JSON.stringify(directory)};
+function fixtureStatus(record) {
+  if (process.argv[2] !== "start") return;
+  try {
+    fixtureAppend(fixtureDiagnosticDir + "/status", record + "\\n", { mode: 0o600 });
+  } catch { /* diagnostics must not interrupt the real exit/signal handler or teardown */ }
+}
+function spawn(bin, args, options) {
+  if (!options?.detached || args[1] !== "start") return fixtureSpawn(bin, args, options);
+  const stdout = fixtureOpen(fixtureDiagnosticDir + "/stdout", "a", 0o600);
+  let stderr;
+  try {
+    stderr = fixtureOpen(fixtureDiagnosticDir + "/stderr", "a", 0o600);
+    return fixtureSpawn(bin, args, { ...options, stdio: ["ignore", stdout, stderr] });
+  } finally {
+    fixtureClose(stdout);
+    if (stderr !== undefined) fixtureClose(stderr);
+  }
+}
+fixtureStatus("launcher-start pid=" + process.pid);
+process.on("exit", code => fixtureStatus("launcher-exit code=" + code));
+`);
+  // The updater exits before its detached child, so observe the Bun child from the
+  // recovery launcher itself, BEFORE the existing handler mirrors its exit/signal.
+  replaceOnce('child.on("exit", (code, signal) => {', `child.on("exit", (code, signal) => {
+  fixtureStatus("runtime-exit code=" + code + " signal=" + signal);`);
+  replaceOnce('child.on("error", err => {', `child.on("error", err => {
+  fixtureStatus("runtime-spawn-error");`);
+  return source;
+}
 const updateSource = readFileSync(join(repoRoot, "src", "update", "index.ts"), "utf8");
 const launcherSource = readFileSync(join(repoRoot, "bin", "ocx.mjs"), "utf8");
 const serverSource = readFileSync(join(repoRoot, "src", "server", "index.ts"), "utf8");
@@ -184,6 +261,7 @@ describe("update stops the running proxy before replacing files", () => {
       const fakeBin = join(root, "fake-bin");
       const fakeNpm = join(fakeBin, "npm");
       const cache = join(root, "npm-cache");
+      const diagnostics = join(root, "recovery-diagnostics");
       const bundledBun = join(repoRoot, "node_modules", "bun");
       const env = {
         ...process.env,
@@ -203,7 +281,11 @@ describe("update stops the running proxy before replacing files", () => {
         mkdirSync(opencodexHome, { recursive: true });
         mkdirSync(fakeBin, { recursive: true });
         mkdirSync(cache, { recursive: true });
-        copyFileSync(join(repoRoot, "bin", "ocx.mjs"), launcher);
+        mkdirSync(diagnostics, { mode: 0o700 });
+        for (const name of ["stdout", "stderr", "status"]) {
+          writeFileSync(join(diagnostics, name), "", { mode: 0o600, flag: "wx" });
+        }
+        writeFileSync(launcher, instrumentRecoveryLauncher(launcherSource, diagnostics));
         chmodSync(launcher, 0o755);
         symlinkSync(join(repoRoot, "src"), join(packageRoot, "src"), "dir");
         symlinkSync(bundledBun, join(packageRoot, "node_modules", "bun"), "dir");
@@ -237,7 +319,16 @@ esac
         expect(output).toContain("Stopping the running proxy before updating");
         expect(output).toContain("restarting the previous version directly");
         expect(output).toContain(`Attempting to restart the proxy on port ${port}.`);
-        expect(await waitForProxy(port)).toBe(true);
+        expect(await waitForProxy(port, lastProbe => {
+          console.error(new Error([
+            "Recovery readiness failed (raw child output redacted).",
+            `lastProbe=${lastProbe}`,
+            `runtimeFiles=${JSON.stringify(Object.fromEntries(["ocx.pid", "runtime-port.json"].map(name => [name, existsSync(join(opencodexHome, name))])))}`,
+            `status=${recoveryDiagnosticFile(join(diagnostics, "status"), true)}`,
+            `stdout=${recoveryDiagnosticFile(join(diagnostics, "stdout"))}`,
+            `stderr=${recoveryDiagnosticFile(join(diagnostics, "stderr"))}`,
+          ].join("\n").slice(0, 4096)));
+        })).toBe(true);
         const runtime = JSON.parse(readFileSync(join(opencodexHome, "runtime-port.json"), "utf8"));
         expect(runtime.pid).toBeGreaterThan(0);
         recoveredPid = runtime.pid;

tests/usage/quota-reset-notify.test.ts

@@ -481,6 +481,8 @@ describe("activation is the single switch", () => {
     // The end-to-end proof: config -> activation -> the production quota writer -> HTTP body.
     // Every earlier test exercises one link; this is the only one that shows the chain holds.
     const bodies: string[] = [];
+    const webhookUrl = "https://quota-webhook.example.test/hook";
+    const originalFetch = globalThis.fetch;
     const server = Bun.serve({
       port: 0,
       hostname: "127.0.0.1",
@@ -499,7 +501,7 @@ describe("activation is the single switch", () => {
       },
       quotaResetNotify: {
         enabled: true,
-        webhookUrl: `http://127.0.0.1:${server.port}/hook`,
+        webhookUrl,
         allowPrivateNetwork: true,
         // Passive-only: this asserts the live request path fires without any timer involved.
         pollSeconds: 0,
@@ -509,6 +511,13 @@ describe("activation is the single switch", () => {
     const previousHome = process.env["OPENCODEX_HOME"];
     process.env["OPENCODEX_HOME"] = home;
     try {
+      // Config must satisfy the production HTTPS rule. Only the transport for this exact
+      // synthetic URL is redirected to the local receiver; activation and payload delivery
+      // remain real, without adding TLS fixtures or weakening production validation.
+      globalThis.fetch = ((input, init) => originalFetch(
+        String(input) === webhookUrl ? `http://127.0.0.1:${server.port}/hook` : input,
+        init,
+      )) as typeof fetch;
       resetQuotaResetNotifyCacheForTests();
       resetQuotaResetStoreForTests();
       resetQuotaResetActivationForTests();
@@ -539,6 +548,7 @@ describe("activation is the single switch", () => {
       expect(payload["percentAfter"]).toBe(2);
       expect(bodies[0]).not.toContain("operator@example.com");
     } finally {
+      globalThis.fetch = originalFetch;
       setQuotaResetSink(null);
       resetQuotaResetActivationForTests();
       resetQuotaResetNotifyCacheForTests();