diff --git a/.env.example b/.env.example index 9de3c87966f..9a79c763526 100644 --- a/.env.example +++ b/.env.example @@ -949,13 +949,24 @@ APP_LOG_TO_FILE=true # Node.js V8 heap limit in MB, passed to the server via --max-old-space-size. # Used by the standalone launcher (Docker CMD) and `omniroute serve`. -# Default: 512. Clamped to [64, 16384]. Raise it (e.g. 1024) if you see random -# OOM crashes under load or with a large SQLite DB (#2939). +# Clamped to [64, 16384]. Default: 512 (safe for a 1 GB / 1 core VPS). Size it to +# roughly half the box's RAM, leaving the rest for native memory (better-sqlite3, +# buffers — ~300 MB) and the OS: +# 1 GB RAM → 512 (default) +# 2 GB RAM → 1024 +# 4 GB RAM → 2048 +# In a memory-capped container, set this EXPLICITLY: Node reads the HOST's RAM, +# not the cgroup limit, so leaving it to a RAM heuristic can oversize the heap and +# get the container OOM-killed. (#2939) # OMNIROUTE_MEMORY_MB=512 -# Heap pressure threshold (MB) — when V8 heap exceeds this, chatCore returns 503. -# Used by: open-sse/handlers/chatCore.ts. Default: 200. -# HEAP_PRESSURE_THRESHOLD_MB=200 +# Heap-pressure shed threshold (MB) — chatCore returns 503 when V8 heapUsed exceeds +# it, to avoid hard OOM under concurrent large-context load. +# LEAVE UNSET: it now AUTO-CALIBRATES to 85% of the actual V8 heap ceiling, so it +# tracks OMNIROUTE_MEMORY_MB above and never sits below the ~260 MB runtime baseline +# (a fixed 200 here used to reject every request). Used by: open-sse/utils/heapPressure.ts. +# Override only to hand-tune for a known workload. +# HEAP_PRESSURE_THRESHOLD_MB= # ── CLI helpers (bin/cli/) ── # Override UI language for CLI output. Accepts BCP-47 locale (e.g. en, pt-BR). diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 6a9eb700e2a..96d25e4cf68 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -1,4 +1,5 @@ import { CORS_HEADERS } from "../utils/cors.ts"; +import { HEAP_PRESSURE_THRESHOLD_MB } from "../utils/heapPressure.ts"; import { normalizeHeaders } from "../utils/headers.ts"; import { detectFormatFromEndpoint, getTargetFormat } from "../services/provider.ts"; import { injectSystemPrompt } from "../services/systemPrompt.ts"; @@ -233,8 +234,10 @@ const MEMORY_EXTRACTION_TEXT_LIMIT = 64 * 1024; // ── Global memory pressure guard ──────────────────────────────────────── // Prevents OOM by rejecting new requests when V8 heap exceeds threshold. -// Self-healing: no counters to leak, no cleanup needed. -const HEAP_PRESSURE_THRESHOLD_MB = parseInt(process.env.HEAP_PRESSURE_THRESHOLD_MB || "200", 10); +// Self-healing: no counters to leak, no cleanup needed. The threshold +// auto-calibrates to 85% of the actual V8 heap ceiling (see heapPressure.ts) so +// it tracks --max-old-space-size across 1GB/2GB/large VPS instead of a fixed +// 200MB that sat below the app's own ~260MB baseline and rejected every request. function capMemoryExtractionText(value: string): string { if (value.length <= MEMORY_EXTRACTION_TEXT_LIMIT) return value; diff --git a/open-sse/utils/heapPressure.ts b/open-sse/utils/heapPressure.ts new file mode 100644 index 00000000000..85f6050d2c5 --- /dev/null +++ b/open-sse/utils/heapPressure.ts @@ -0,0 +1,44 @@ +import v8 from "node:v8"; + +/** + * Compute the V8 heap-pressure shed threshold (MB). + * + * The chat pipeline rejects new requests with a 503 once `heapUsed` exceeds this + * value, to avoid hard "JavaScript heap out of memory" crashes under concurrent + * large-context load. The threshold is derived from the process's *actual* V8 + * heap ceiling (`heap_size_limit`, which reflects `--max-old-space-size` when set, + * otherwise Node's RAM-derived default) so it auto-adapts across 1 GB / 2 GB / + * large VPS instead of using a fixed number. + * + * A fixed default was the bug: 200 MB sat *below* the app's ~260 MB working set, + * so the guard rejected every request once the heap warmed up (the v3.8.8 + * "resource pressure" outage). We shed at 85% of the ceiling — leaving headroom + * for in-flight requests + GC — with a floor that always clears the runtime + * baseline so a small/undersized heap never rejects all traffic. + * + * @param heapSizeLimitMb `v8.getHeapStatistics().heap_size_limit` expressed in MB + * @param override `HEAP_PRESSURE_THRESHOLD_MB` env value — wins when it + * parses to a positive number; invalid/unset → auto-calibrate + */ +export function computeHeapPressureThresholdMb( + heapSizeLimitMb: number, + override?: string | number | null +): number { + const explicit = Number(override); + if (Number.isFinite(explicit) && explicit > 0) return Math.floor(explicit); + // Shed at 85% of the heap ceiling, but never below a floor that clears the + // runtime's own ~260 MB baseline (+ margin) so an undersized heap degrades to + // "guard never fires" rather than "guard rejects everything". + const SHED_RATIO = 0.85; + const FLOOR_MB = 400; + return Math.max(Math.round(heapSizeLimitMb * SHED_RATIO), FLOOR_MB); +} + +/** + * Heap-pressure threshold (MB) resolved once at module load from the live V8 heap + * ceiling. Read by `open-sse/handlers/chatCore.ts`'s memory-pressure guard. + */ +export const HEAP_PRESSURE_THRESHOLD_MB = computeHeapPressureThresholdMb( + v8.getHeapStatistics().heap_size_limit / (1024 * 1024), + process.env.HEAP_PRESSURE_THRESHOLD_MB +); diff --git a/tests/unit/chatcore-memory-pressure.test.ts b/tests/unit/chatcore-memory-pressure.test.ts index 79cc0295070..dad17e4d89b 100644 --- a/tests/unit/chatcore-memory-pressure.test.ts +++ b/tests/unit/chatcore-memory-pressure.test.ts @@ -37,11 +37,31 @@ test("estimateSizeFast handles circular references", async () => { assert.ok(size < 1000, `Simple circular ref should be small, got ${size}`); }); -// ── HEAP_PRESSURE_THRESHOLD_MB default ───────────────────────────────── +// ── HEAP_PRESSURE_THRESHOLD_MB auto-calibration ──────────────────────── +// Regression: the old fixed 200MB default sat below the app's ~260MB working set, +// so the chatCore heap guard 503'd every request ("resource pressure" outage). +// The live constant must now auto-calibrate from the real V8 heap ceiling. -test("HEAP_PRESSURE_THRESHOLD_MB defaults to 200 when env is unset", () => { - const val = parseInt(process.env.HEAP_PRESSURE_THRESHOLD_MB || "200", 10); - assert.equal(val, 200); +test("HEAP_PRESSURE_THRESHOLD_MB auto-calibrates from the live V8 heap ceiling (no fixed 200)", async () => { + const { getHeapStatistics } = await import("node:v8"); + const { HEAP_PRESSURE_THRESHOLD_MB, computeHeapPressureThresholdMb } = await import( + "../../open-sse/utils/heapPressure.ts" + ); + const limitMb = getHeapStatistics().heap_size_limit / (1024 * 1024); + // The live constant must equal the pure helper applied to this process's + // actual ceiling — no drift between the resolved value and the formula. + assert.equal( + HEAP_PRESSURE_THRESHOLD_MB, + computeHeapPressureThresholdMb(limitMb, process.env.HEAP_PRESSURE_THRESHOLD_MB) + ); + // With no operator override it must clear the ~260MB baseline, otherwise the + // guard would reject every request at idle (the bug we are fixing). + if (!process.env.HEAP_PRESSURE_THRESHOLD_MB) { + assert.ok( + HEAP_PRESSURE_THRESHOLD_MB >= 400, + `live threshold ${HEAP_PRESSURE_THRESHOLD_MB}MB must clear the ~260MB app baseline` + ); + } }); // ── estimateSizeFast vs MAX_LOG_BODY_CHARS threshold ─────────────────── diff --git a/tests/unit/heap-pressure.test.ts b/tests/unit/heap-pressure.test.ts new file mode 100644 index 00000000000..621b1ea7123 --- /dev/null +++ b/tests/unit/heap-pressure.test.ts @@ -0,0 +1,48 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { computeHeapPressureThresholdMb } from "../../open-sse/utils/heapPressure.ts"; + +// Regression guard for the v3.8.8 "Service temporarily unavailable due to resource +// pressure" outage: a fixed 200MB threshold sat below the app's ~260MB working set, +// so the chatCore heap guard rejected every request. The threshold must now track +// the real V8 heap ceiling so it stays above the baseline on every VPS tier. +describe("computeHeapPressureThresholdMb", () => { + it("sheds at 85% of a no-cap heap ceiling (2240MB → 1904)", () => { + assert.equal(computeHeapPressureThresholdMb(2240), 1904); + }); + + it("tracks the default 512MB cap (heap_limit ~704 → 598), clearing the ~260MB baseline", () => { + const t = computeHeapPressureThresholdMb(704); + assert.equal(t, 598); + assert.ok(t > 260, "threshold must stay above the ~260MB app baseline"); + }); + + it("tracks a 1 GB-box 640MB cap (heap_limit 832 → 707)", () => { + assert.equal(computeHeapPressureThresholdMb(832), 707); + }); + + it("tracks a 2 GB-box 1536MB cap (heap_limit 1728 → 1469)", () => { + assert.equal(computeHeapPressureThresholdMb(1728), 1469); + }); + + it("floors at 400MB so a tiny/undersized heap never rejects all traffic", () => { + // 85% of 300 = 255, which would sit below the baseline — the floor wins. + assert.equal(computeHeapPressureThresholdMb(300), 400); + }); + + it("honors a positive explicit override (string or number)", () => { + assert.equal(computeHeapPressureThresholdMb(2240, "1024"), 1024); + assert.equal(computeHeapPressureThresholdMb(2240, 800), 800); + assert.equal(computeHeapPressureThresholdMb(2240, "1500.9"), 1500); + }); + + it("ignores invalid/zero/negative overrides and auto-calibrates", () => { + for (const bad of ["", "0", "-5", "abc", null, undefined]) { + assert.equal( + computeHeapPressureThresholdMb(2240, bad as string | number | null | undefined), + 1904, + `override ${JSON.stringify(bad)} must fall back to auto-calibration` + ); + } + }); +});