fix(build): verify artifact provenance and expose buildSha on health

.env.example

@@ -51,6 +51,15 @@ INITIAL_PASSWORD=CHANGEME
 # DATA_DIR — never for CI. Used by: src/lib/dataPaths.ts
 # OMNIROUTE_ALLOW_DEFAULT_DATA_DIR=1
 
+# Build provenance (#10427). OMNIROUTE_BUILD_SHA lets a container inject the artifact's git
+# SHA when the dist/BUILD_SHA sentinel is absent; it is also what `npm run build:release`
+# stamps. OMNIROUTE_RELEASE_REF is the ref the pack gate checks ancestry against, and
+# OMNIROUTE_ALLOW_CANARY_BUILD=1 records a deliberate off-release-line build instead of
+# failing it. Used by: scripts/build/buildProvenance.ts, src/lib/monitoring/buildSha.ts
+# OMNIROUTE_BUILD_SHA=abc1234
+# OMNIROUTE_RELEASE_REF=origin/main
+# OMNIROUTE_ALLOW_CANARY_BUILD=1
+
 # Encryption key for SQLite database encryption at rest.
 # Used by: src/lib/db/encryption.ts — encrypts the entire SQLite database.
 # Generate: openssl rand -hex 32  |  Leave empty to disable DB encryption.

docs/reference/ENVIRONMENT.md

@@ -84,6 +84,9 @@ OmniRoute uses **SQLite** (via `better-sqlite3`) for all persistence. These vari
 | -------------------------------------- | -------------------- | ----------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `DATA_DIR`                             | `~/.omniroute/`      | `src/lib/db/core.ts`                                  | Root directory for SQLite DB, backups, and data files. Override for Docker volumes or custom paths.                                                                                                                                             |
 | `OMNIROUTE_ALLOW_DEFAULT_DATA_DIR`      | _(unset)_            | `src/lib/dataPaths.ts`                                | Escape hatch for the test-context DATA_DIR guard (#10428). Test runs with no `DATA_DIR` are redirected to a throwaway temp dir so they cannot open the operator's real database; set to `1` to opt back in to the real directory.                  |
+| `OMNIROUTE_BUILD_SHA`                   | _(unset)_            | `src/lib/monitoring/buildSha.ts`                      | Git SHA of the running artifact. Stamped by `npm run build:release`; injectable in containers that ship without the `dist/BUILD_SHA` sentinel. Surfaced as `system.buildSha` on `/api/monitoring/health`.                                          |
+| `OMNIROUTE_RELEASE_REF`                 | `origin/main`        | `scripts/build/buildProvenance.ts`                    | Ref the pack-artifact provenance gate checks the build SHA against (#10427).                                                                                                                                                                     |
+| `OMNIROUTE_ALLOW_CANARY_BUILD`          | _(unset)_            | `scripts/build/buildProvenance.ts`                    | Set to `1` to allow packing a build whose SHA is not on the release line, recording it as a deliberate canary instead of failing the gate (#10427).                                                                                               |
 | `OMNIROUTE_DATA_DIR`                   | _(unset)_            | `open-sse/executors/promptql/threadSticky.ts`         | **Fallback alias** for `DATA_DIR`, checked only when `DATA_DIR` is unset. Used to locate the PromptQL executor's on-disk thread-sticky session cache (`<dir>/promptql-thread-sessions.json`); if neither var is set, the cache stays in-memory only (not persisted across restarts).                                                                                                                                             |
 | `STORAGE_ENCRYPTION_KEY`               | _(empty = disabled)_ | `src/lib/db/encryption.ts`                            | AES key for full SQLite database encryption at rest. Generate with `openssl rand -hex 32`.                                                                                                                                                      |
 | `STORAGE_ENCRYPTION_KEY_VERSION`       | `v1`                 | `scripts/build/bootstrap-env.mjs`, `electron/main.js` | Version label for the encryption key. Increment when performing key rotation to support decryption of old backups.                                                                                                                              |

scripts/build/buildProvenance.ts

@@ -0,0 +1,120 @@
+/**
+ * Build provenance — is this artifact actually built from the release line? (#10427)
+ *
+ * `scripts/build/write-build-sha.mjs` stamps `dist/BUILD_SHA` into every packaged build,
+ * but nothing ever verified that the SHA belongs to the release branch. A tarball built
+ * from a feature branch installs and serves traffic indistinguishably from a release one.
+ *
+ * That gap took down the internal gateway on 2026-08-14: the installed package carried
+ * `BUILD_SHA = 178febc50f`, a commit on `fix/9603-qwen-token-plan-quota` that predated
+ * #10373, so it shipped the nominal `instanceof Response` guard from #10256 and answered
+ * every request with `502 … Executor result must contain a Response`.
+ *
+ * Kept as pure functions (the ancestry probe is injected) so the policy is unit-testable
+ * without a git fixture, and so the caller decides how strict to be per environment.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import { execFileSync } from "node:child_process";
+
+export type BuildProvenanceReason =
+  | "on-release-line"
+  | "off-release-line"
+  | "canary-override"
+  | "missing-sha";
+
+export type BuildProvenanceResult = {
+  ok: boolean;
+  reason: BuildProvenanceReason;
+  message: string;
+};
+
+export type BuildProvenanceInput = {
+  /** Contents of `dist/BUILD_SHA` (empty when the sentinel is absent). */
+  buildSha: string;
+  /** Whether `buildSha` is an ancestor of the release ref. Injected so this stays pure. */
+  isAncestorOfRelease: (sha: string) => boolean;
+  /** Deliberate canary build — allowed, but always reported. */
+  allowOverride: boolean;
+};
+
+/**
+ * Read `dist/BUILD_SHA` from a package root. Returns "" when absent — an unstamped build
+ * is a policy decision for the caller, not an exception here.
+ */
+export function readBuildSha(packageRoot: string): string {
+  try {
+    return fs.readFileSync(path.join(packageRoot, "dist", "BUILD_SHA"), "utf8").trim();
+  } catch {
+    return "";
+  }
+}
+
+/**
+ * Classify a build SHA against the release line.
+ *
+ * A missing SHA fails even with the override on: an artifact that cannot be identified
+ * cannot be vouched for, and "canary" is a statement about a KNOWN commit.
+ */
+export function resolveBuildProvenance(input: BuildProvenanceInput): BuildProvenanceResult {
+  const { buildSha, isAncestorOfRelease, allowOverride } = input;
+
+  if (!buildSha) {
+    return {
+      ok: false,
+      reason: "missing-sha",
+      message:
+        "dist/BUILD_SHA is missing — the artifact cannot be traced to a commit. " +
+        "Build with `npm run build:release` (or run scripts/build/write-build-sha.mjs).",
+    };
+  }
+
+  if (isAncestorOfRelease(buildSha)) {
+    return {
+      ok: true,
+      reason: "on-release-line",
+      message: `BUILD_SHA ${buildSha} is on the release line.`,
+    };
+  }
+
+  if (allowOverride) {
+    return {
+      ok: true,
+      reason: "canary-override",
+      message:
+        `BUILD_SHA ${buildSha} is NOT on the release line — allowed as a canary build ` +
+        "because OMNIROUTE_ALLOW_CANARY_BUILD=1 was set.",
+    };
+  }
+
+  return {
+    ok: false,
+    reason: "off-release-line",
+    message:
+      `BUILD_SHA ${buildSha} is not an ancestor of the release branch. Shipping it means ` +
+      "serving code that never passed the release gates (see #10427). Rebuild from the " +
+      "release tip, or set OMNIROUTE_ALLOW_CANARY_BUILD=1 to record this as a deliberate canary.",
+  };
+}
+
+/**
+ * Default ancestry probe: `git merge-base --is-ancestor <sha> <releaseRef>`.
+ *
+ * Any git failure (shallow clone, unknown ref, SHA not fetched) resolves to `false` —
+ * "cannot prove it is on the release line" is the safe answer for a gate whose whole
+ * purpose is to refuse unverifiable artifacts.
+ */
+export function makeGitAncestryProbe(releaseRef: string, cwd: string): (sha: string) => boolean {
+  return (sha: string) => {
+    try {
+      execFileSync("git", ["merge-base", "--is-ancestor", sha, releaseRef], {
+        cwd,
+        stdio: "ignore",
+      });
+      return true;
+    } catch {
+      return false;
+    }
+  };
+}

scripts/build/validate-pack-artifact.ts

@@ -4,6 +4,11 @@ import { execFileSync, spawnSync } from "node:child_process";
 import { existsSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
+import {
+  makeGitAncestryProbe,
+  readBuildSha,
+  resolveBuildProvenance,
+} from "./buildProvenance.ts";
 
 import {
   MCP_CLOSURE_SPOT_CHECK_PATH,
@@ -204,6 +209,26 @@ try {
     process.exit(1);
   }
 
+  // #10427: an artifact is only shippable if it can be traced to the release line. The
+  // 2026-08-14 gateway outage was a package built from a feature branch that predated the
+  // fix it was supposed to carry — nothing in this gate noticed. Skipped under
+  // --policy-only, which deliberately runs without a build (no dist/BUILD_SHA to check).
+  if (!POLICY_ONLY) {
+    const provenance = resolveBuildProvenance({
+      buildSha: readBuildSha(process.cwd()),
+      isAncestorOfRelease: makeGitAncestryProbe(
+        process.env.OMNIROUTE_RELEASE_REF || "origin/main",
+        process.cwd()
+      ),
+      allowOverride: process.env.OMNIROUTE_ALLOW_CANARY_BUILD === "1",
+    });
+    console.log(`\n[provenance] ${provenance.message}`);
+    if (!provenance.ok) {
+      console.error("\n❌ Build provenance check failed.");
+      process.exit(1);
+    }
+  }
+
   console.log("\n✅ Pack artifact policy check passed.");
 } catch (error) {
   console.error(`\n❌ Pack artifact validation failed: ${error.message}`);

src/app/api/monitoring/health/route.ts

@@ -1,6 +1,7 @@
 import { NextResponse } from "next/server";
 import { getProviderConnections, getCachedSettings } from "@/lib/localDb";
 import { buildHealthPayload } from "@/lib/monitoring/observability";
+import { readRunningBuildSha } from "@/lib/monitoring/buildSha";
 import { APP_CONFIG } from "@/shared/constants/config";
 import { AI_PROVIDERS } from "@/shared/constants/providers";
 import { isAuthenticated } from "@/shared/utils/apiAuth";
@@ -158,6 +159,9 @@ export async function GET() {
 
     const payload = buildHealthPayload({
       appVersion: APP_CONFIG.version,
+      // #10427: surface the artifact's git SHA so a deployment can be audited over HTTP
+      // instead of SSH + grepping compiled chunks (the 2026-08-14 gateway outage).
+      buildSha: readRunningBuildSha(),
       catalogCount: Object.keys(AI_PROVIDERS).length,
       settings,
       connections,

src/lib/monitoring/buildSha.ts

@@ -0,0 +1,52 @@
+/**
+ * Runtime build identity (#10427).
+ *
+ * `scripts/build/write-build-sha.mjs` stamps the git SHA into `dist/BUILD_SHA` and
+ * `.build/next/standalone/BUILD_SHA` at release-build time. Reading it back at runtime is
+ * what lets an operator answer "what code is this box actually running?" over HTTP.
+ *
+ * Before this existed, answering that question during the 2026-08-14 gateway outage meant
+ * SSH-ing into the host and grepping the compiled Next chunks — the deployed package
+ * turned out to be built from a feature branch that predated the fix it was supposed to
+ * carry.
+ *
+ * Resolution order: explicit env var (containers can inject it without the sentinel file),
+ * then the sentinel files relative to the working directory. Unknown → `null`, never a
+ * fabricated or guessed value.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+
+const SENTINEL_PATHS = [
+  ["dist", "BUILD_SHA"],
+  [".build", "next", "standalone", "BUILD_SHA"],
+  ["BUILD_SHA"],
+];
+
+let cached: string | null | undefined;
+
+export function readRunningBuildSha(cwd: string = process.cwd()): string | null {
+  if (cached !== undefined) return cached;
+
+  const fromEnv = process.env.OMNIROUTE_BUILD_SHA?.trim();
+  if (fromEnv) {
+    cached = fromEnv;
+    return cached;
+  }
+
+  for (const segments of SENTINEL_PATHS) {
+    try {
+      const value = fs.readFileSync(path.join(cwd, ...segments), "utf8").trim();
+      if (value) {
+        cached = value;
+        return cached;
+      }
+    } catch {
+      // Sentinel absent at this location — try the next one. A dev run has none of them.
+    }
+  }
+
+  cached = null;
+  return cached;
+}

src/lib/monitoring/observability.ts

@@ -126,6 +126,8 @@ interface BuildTelemetryPayloadOptions {
 
 interface BuildHealthPayloadOptions {
   appVersion: string;
+  /** #10427: git SHA the running artifact was built from, so a deploy is auditable over HTTP. */
+  buildSha?: string | null;
   catalogCount?: number;
   settings: { setupComplete?: boolean } | null | undefined;
   connections: Array<{ provider?: string; isActive?: boolean | null; rateLimitedUntil?: unknown }>;
@@ -288,10 +290,14 @@ export function buildHealthPayload({
   activeSessionsByKey = {},
   credentialHealth,
   adaptiveAdmission = null,
+  buildSha = null,
 }: BuildHealthPayloadOptions) {
   const timestamp = new Date().toISOString();
   const system = {
     version: appVersion,
+    // #10427: identifying a bad deploy previously required SSH + grepping compiled chunks.
+    // Absent/empty when unknown (dev runs) — never a fabricated value.
+    ...(buildSha ? { buildSha } : {}),
     nodeVersion: process.version,
     uptime: process.uptime(),
     memoryUsage: process.memoryUsage(),