From eebfb56483dc0da8749c7475fbc54f74ab1ecfb9 Mon Sep 17 00:00:00 2001 From: Rouzbeh <78313022+rqzbeh@users.noreply.github.com> Date: Sat, 19 Sep 2026 16:27:38 +0330 Subject: [PATCH 1/3] feat(services): add LLMLingua embedded-service sidecar with real ONNX inference - Add real installer for @atjsh/llmlingua-2 with peer dependencies (@huggingface/transformers, js-tiktoken) in src/lib/services/installers/llmlingua.ts - Implement lazy-loaded TinyBERT ONNX model inference on port 20135 - Integrate httpSidecarBackend in open-sse/services/compression/engines/llmlingua/index.ts with strict length validation and fallback to workerBackend, preserving upstream #13768 prompt protections - Fix duplicate LLMLINGUA_PORT declaration in src/lib/services/bootstrap.ts and ensure coexistence with openwa - Register 8 LOCAL_ONLY lifecycle endpoints under /api/services/llmlingua/* - Add dashboard tab LlmlinguaServiceTab in src/app/(dashboard)/dashboard/providers/services/ - Register migration slot 183 (183_llmlingua_service.sql) - Wire test:services:int into package.json and scripts/check/check-test-discovery.mjs, reducing frozen orphans in test-discovery-baseline.json from 10 to 6 - Document all environment variables, openapi specs, and architecture references Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- .env.example | 17 +- .../features/12967-llmlingua-server.md | 1 + config/quality/test-discovery-baseline.json | 5 +- docs/frameworks/EMBEDDED-SERVICES.md | 46 ++- docs/openapi.yaml | 141 +++++++++- docs/reference/ENVIRONMENT.md | 2 + .../compression/engines/llmlingua/index.ts | 25 +- package.json | 1 + scripts/check/check-test-discovery.mjs | 5 +- .../dashboard/providers/services/page.tsx | 5 +- .../services/tabs/LlmlinguaServiceTab.tsx | 21 ++ src/app/api/services/llmlingua/_lib.ts | 33 +++ .../llmlingua/auto-restart-adopted/route.ts | 28 ++ .../services/llmlingua/auto-start/route.ts | 28 ++ .../api/services/llmlingua/install/route.ts | 6 + .../api/services/llmlingua/restart/route.ts | 22 ++ src/app/api/services/llmlingua/start/route.ts | 22 ++ .../api/services/llmlingua/status/route.ts | 41 +++ src/app/api/services/llmlingua/stop/route.ts | 19 ++ .../api/services/llmlingua/update/route.ts | 45 +++ .../db/migrations/183_llmlingua_service.sql | 14 + src/lib/services/bootstrap.ts | 17 ++ src/lib/services/installers/llmlingua.ts | 262 ++++++++++++++++++ ...ingua-sidecar-real-compression.int.test.ts | 155 +++++++++++ .../services/installers/llmlingua.test.ts | 148 ++++++++++ 25 files changed, 1085 insertions(+), 24 deletions(-) create mode 100644 changelog.d/features/12967-llmlingua-server.md create mode 100644 src/app/(dashboard)/dashboard/providers/services/tabs/LlmlinguaServiceTab.tsx create mode 100644 src/app/api/services/llmlingua/_lib.ts create mode 100644 src/app/api/services/llmlingua/auto-restart-adopted/route.ts create mode 100644 src/app/api/services/llmlingua/auto-start/route.ts create mode 100644 src/app/api/services/llmlingua/install/route.ts create mode 100644 src/app/api/services/llmlingua/restart/route.ts create mode 100644 src/app/api/services/llmlingua/start/route.ts create mode 100644 src/app/api/services/llmlingua/status/route.ts create mode 100644 src/app/api/services/llmlingua/stop/route.ts create mode 100644 src/app/api/services/llmlingua/update/route.ts create mode 100644 src/lib/db/migrations/183_llmlingua_service.sql create mode 100644 src/lib/services/installers/llmlingua.ts create mode 100644 tests/integration/services/llmlingua-sidecar-real-compression.int.test.ts create mode 100644 tests/unit/services/installers/llmlingua.test.ts diff --git a/.env.example b/.env.example index a31d2284a24d..0e206d680a66 100644 --- a/.env.example +++ b/.env.example @@ -2275,15 +2275,14 @@ APP_LOG_TO_FILE=true # DARIO_HOST=127.0.0.1 # DARIO_PORT=3456 -# ── Dario embedded service ── -# Override the host/port the embedded Dario (Claude Code subscription proxy) -# daemon binds to and is reached at. Always bound to 127.0.0.1 — never -# configurable to 0.0.0.0. Rarely needed — defaults to 127.0.0.1:3456. -# Used by: src/lib/services/installers/dario.ts, src/lib/services/bootstrap.ts, -# src/app/api/services/dario/_lib.ts, src/app/api/services/dario/admin/_lib.ts, -# open-sse/executors/dario.ts -# DARIO_HOST=127.0.0.1 -# DARIO_PORT=3456 +# ── LLMLingua embedded service ── +# Override the port and base URL for the embedded LLMLingua-2 prompt compression +# sidecar daemon. Always bound to 127.0.0.1 — never configurable to 0.0.0.0. +# Defaults to port 20135 and http://127.0.0.1:20135. +# Used by: src/lib/services/bootstrap.ts, src/lib/services/installers/llmlingua.ts, +# open-sse/services/compression/engines/llmlingua/index.ts +# LLMLINGUA_PORT=20135 +# LLMLINGUA_BASE_URL=http://127.0.0.1:20135 # ── Local hostnames (Docker networking) ── # Comma-separated additional hostnames treated as "local" for provider routing. diff --git a/changelog.d/features/12967-llmlingua-server.md b/changelog.d/features/12967-llmlingua-server.md new file mode 100644 index 000000000000..4a0fd966b6c8 --- /dev/null +++ b/changelog.d/features/12967-llmlingua-server.md @@ -0,0 +1 @@ +- **feat(services):** Added an LLMLingua embedded-service supervisor + HTTP sidecar wrapping the real `@atjsh/llmlingua-2` prompt-compression package (ONNX TinyBERT model, downloaded from Hugging Face on first use). The compression engine dispatches `/compress` calls to the sidecar over HTTP, falling back to the existing in-process worker-thread backend when the sidecar is down. diff --git a/config/quality/test-discovery-baseline.json b/config/quality/test-discovery-baseline.json index 0cbe39364a6b..8de5b9da57ff 100644 --- a/config/quality/test-discovery-baseline.json +++ b/config/quality/test-discovery-baseline.json @@ -1,15 +1,12 @@ { "_comment": "Catraca de test-discovery (check-test-discovery.mjs). Cada entrada e um arquivo de teste que NENHUM runner coleta (ele nunca roda) — divida congelada na auditoria 6A.1 (2026-06-09; 195 originais, 135 religados no node runner em 6A.1c). So pode DIMINUIR: religue o teste (ajustando o glob do runner ou movendo o arquivo) e remova a entrada via --update. NAO adicione novos orfaos — corrija o runner.", - "_remaining_10": "10 orfaos restantes: 4 golden-set + 1 benchmark + 1 teste live + 1 stress (deliberadamente manuais — decidir runner/gating); 3 integration/services (gated RUN_SERVICES_INT=1, sem runner CI).", + "_remaining_6": "6 orfaos restantes: 4 golden-set + 1 benchmark + 1 teste live (deliberadamente manuais — decidir runner/gating). Os 3 integration/services foram religados por test:services:int (#12967).", "orphans": [ "tests/benchmarks/pipeline-accuracy.test.ts", "tests/golden-set/compression-caveman-v2.test.ts", "tests/golden-set/compression-quality.test.ts", "tests/golden-set/compression-savings.test.ts", "tests/golden-set/compression-upstream-parity.test.ts", - "tests/integration/services/cliproxy-coexistence.test.ts", - "tests/integration/services/full-lifecycle.int.test.ts", - "tests/integration/services/route-guard-services.int.test.ts", "tests/live/deepseek-web-live.test.ts" ] } diff --git a/docs/frameworks/EMBEDDED-SERVICES.md b/docs/frameworks/EMBEDDED-SERVICES.md index dc04a3867734..acdf5c3ac539 100644 --- a/docs/frameworks/EMBEDDED-SERVICES.md +++ b/docs/frameworks/EMBEDDED-SERVICES.md @@ -1,13 +1,13 @@ --- title: "Embedded Services" -description: "Reference for 9Router, CLIProxyAPI, Mux, Bifrost, and open-wa" +description: "Reference for 9Router, CLIProxyAPI, Mux, Bifrost, open-wa, and LLMLingua" --- # Embedded Services > **Version:** v3.8.44 -> **Last updated:** 2026-09-09 -> **Audience:** Engineers adding, maintaining, or debugging embedded services (9Router, CLIProxyAPI, Mux, Bifrost, open-wa). +> **Last updated:** 2026-09-16 +> **Audience:** Engineers adding, maintaining, or debugging embedded services (9Router, CLIProxyAPI, Mux, Bifrost, open-wa, LLMLingua). Embedded services are locally-installed process sidecar tools that OmniRoute installs, supervises, and exposes as first-class routing targets. Unlike external providers (which are reached over the internet @@ -32,7 +32,7 @@ via API keys), embedded services run on the same machine as OmniRoute and commun ### Why embedded services? -Six services are embedded: +Seven services are embedded: | Service | npm package | Default port | Purpose | | --------------- | ---------------------------------- | :----------: | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -42,8 +42,9 @@ Six services are embedded: | **Bifrost** | `@maximhq/bifrost` | 8080 | Go AI-gateway relay backend. When running, auto-selected by the relay route (`/v1/relay/`) | | **Dario** | `@askalf/dario` | 3456 | Claude-subscription proxy — alternative/failover to CLIProxyAPI for Claude-Code-shaped traffic; the injected key becomes `DARIO_ADMIN_TOKEN` gating its `/admin/*` OAuth control plane | | **open-wa** | `@open-wa/wa-automate` | 8323 | WhatsApp Web automation (headless Chromium via Puppeteer). Lifecycle-managed only — not a routing target. | +| **LLMLingua** | `@atjsh/llmlingua-2` | 20135 | Prompt-compression sidecar — real LLMLingua-2 ONNX model (JS/TS port of Microsoft's algorithm). `open-sse/services/compression/engines/llmlingua/index.ts` dispatches `/compress` over HTTP to it, falling back to the in-process worker-thread backend when the sidecar is down. Lifecycle-managed only — not a routing target. | -All six follow the same supervisory model: +All seven follow the same supervisory model: - OmniRoute installs them under `DATA_DIR/services/{name}/` (isolated from OmniRoute's own `package.json`) - OmniRoute spawns and monitors them as child processes @@ -153,6 +154,7 @@ All six follow the same supervisory model: | `src/lib/services/installers/cliproxy.ts` | npm install/update/uninstall for CLIProxyAPI | | `src/lib/services/installers/mux.ts` | npm install/update/uninstall for Mux | | `src/lib/services/installers/openwa.ts` | npm install/update/uninstall for open-wa | +| `src/lib/services/installers/llmlingua.ts` | npm install/update/uninstall for LLMLingua | | `src/app/api/services/9router/_lib.ts` | `getOrInitSupervisor()` helper | | `src/app/api/services/[name]/logs/route.ts` | Shared SSE logs endpoint | | `open-sse/executors/ninerouter.ts` | Provider executor (Layer 4) | @@ -540,7 +542,39 @@ in this integration yet. --- -### 4.7 Reverse proxy (9Router dashboard embed) +### 4.7 LLMLingua endpoints (8 routes) + +LLMLingua is a prompt-compression sidecar wrapping `@atjsh/llmlingua-2` (real +ONNX token-classification model, downloaded from Hugging Face on first +`/compress` call). It uses the same endpoint shape as Bifrost (no API key — +`needsApiKey: false`, it never handles credentials). + +| Method | Path | Description | +| ------ | ---------------------------------------------- | ------------------------------------------------------------------------- | +| `POST` | `/api/services/llmlingua/install` | npm-install `@atjsh/llmlingua-2` + peers, write the sidecar server script | +| `POST` | `/api/services/llmlingua/start` | Start the sidecar on port 20135 (default) | +| `POST` | `/api/services/llmlingua/stop` | Stop the sidecar | +| `POST` | `/api/services/llmlingua/restart` | Restart the sidecar | +| `POST` | `/api/services/llmlingua/update` | Update to the newer package version | +| `GET` | `/api/services/llmlingua/status` | Live + DB status | +| `POST` | `/api/services/llmlingua/auto-start` | Toggle auto-start | +| `POST` | `/api/services/llmlingua/auto-restart-adopted` | Toggle auto-restart of an adopted (pre-existing) instance | +| `GET` | `/api/services/llmlingua/logs` | SSE log tail (via shared `[name]/logs` dynamic route) | + +**Sidecar contract:** the server script exposes `GET /health` (instant — does +not wait on the model) and `POST /compress` (`{ text, rate }` → +`{ text, compressed, ratio }`). The model loads lazily on the first +`/compress` call. + +**Compression wiring:** `open-sse/services/compression/engines/llmlingua/index.ts`'s +`httpSidecarBackend` calls `LLMLINGUA_BASE_URL` (default +`http://127.0.0.1:20135`) and only accepts the sidecar's response when it is +strictly shorter than the input; any failure (not running, timeout, no-op +response) falls back to the in-process worker-thread backend (`./worker.ts`). + +--- + +### 4.8 Reverse proxy (9Router dashboard embed) The dashboard embeds the 9Router web UI inside an iframe via an internal reverse proxy at: diff --git a/docs/openapi.yaml b/docs/openapi.yaml index b4082fbaab36..479cd28d206f 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -5572,6 +5572,145 @@ paths: "500": description: Import failed + /api/services/llmlingua/install: + post: + tags: [Embedded Services] + summary: Install LLMLingua + description: >- + Installs `@atjsh/llmlingua-2` (real LLMLingua-2 ONNX prompt-compression + package) plus its `@huggingface/transformers`/`js-tiktoken` peers under + DATA_DIR/services/llmlingua/, and writes the sidecar server script. + Accepts an optional `version` field (semver or `latest`). + **LOCAL_ONLY** — loopback only. + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + version: + type: string + default: latest + responses: + "200": + description: Installation result + content: + application/json: + schema: + type: object + properties: + ok: + type: boolean + installedVersion: + type: string + installPath: + type: string + durationMs: + type: number + + /api/services/llmlingua/start: + post: + tags: [Embedded Services] + summary: Start LLMLingua + description: Starts the supervised LLMLingua sidecar process. **LOCAL_ONLY** — loopback only. + responses: + "200": + description: Service status after start + "409": + description: LLMLingua is not installed + + /api/services/llmlingua/stop: + post: + tags: [Embedded Services] + summary: Stop LLMLingua + description: Stops the supervised LLMLingua sidecar process. **LOCAL_ONLY** — loopback only. + responses: + "200": + description: Service status after stop + + /api/services/llmlingua/restart: + post: + tags: [Embedded Services] + summary: Restart LLMLingua + description: Restarts the supervised LLMLingua sidecar process. **LOCAL_ONLY** — loopback only. + responses: + "200": + description: Service status after restart + "409": + description: LLMLingua is not installed + + /api/services/llmlingua/update: + post: + tags: [Embedded Services] + summary: Update LLMLingua + description: >- + Updates LLMLingua to the latest npm version. Stops the running process, + installs the new version, and restarts if it was previously running. + **LOCAL_ONLY** — loopback only. + responses: + "200": + description: Update result + + /api/services/llmlingua/status: + get: + tags: [Embedded Services] + summary: Get LLMLingua status + description: Returns live and DB status for the supervised LLMLingua service. **LOCAL_ONLY** — loopback only. + responses: + "200": + description: LLMLingua service status + + /api/services/llmlingua/auto-start: + post: + tags: [Embedded Services] + summary: Toggle LLMLingua auto-start + description: >- + When enabled, LLMLingua starts automatically on the next OmniRoute boot. + **LOCAL_ONLY** — loopback only. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [enabled] + properties: + enabled: + type: boolean + responses: + "204": + description: Auto-start flag updated + "400": + description: Invalid request body + + /api/services/llmlingua/auto-restart-adopted: + post: + tags: [Embedded Services] + summary: Toggle LLMLingua auto-restart-when-adopted + description: >- + When enabled, an externally-adopted (not OmniRoute-spawned) LLMLingua + process is restarted under OmniRoute's own supervisor on the next + health-check cycle instead of being left as adopted-only. + **LOCAL_ONLY** — loopback only. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [enabled] + properties: + enabled: + type: boolean + responses: + "204": + description: Flag updated + "400": + description: Invalid request body + "500": + description: Update failed + /api/services/{name}/logs: get: tags: [Embedded Services] @@ -5587,7 +5726,7 @@ paths: required: true schema: type: string - enum: [9router, cliproxy] + enum: [9router, cliproxy, mux, bifrost, dario, openwa, llmlingua] - name: tail in: query schema: diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index 5a992a66b788..f7588de0f773 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -1408,6 +1408,8 @@ Provider quota endpoints, network tunnels (Tailscale, Ngrok, MITM debug proxy), | `BIFROST_ENABLED` | `1` | `src/app/api/v1/relay/chat/completions/bifrost/route.ts` | Master kill switch for the bifrost sidecar proxy. When set to `0`, the route returns 503 with the `X-Bifrost-Killswitch` header and the operator is bounced to the TS path. Use to disable the sidecar without redeploying (tier-1 router incident, key rotation). | | `BIFROST_BASE_URL` | _(unset)_ | `src/app/api/v1/relay/chat/completions/bifrost/route.ts` | When set, the Bifrost sidecar proxy route forwards `/v1/chat/completions` traffic to this Go gateway instead of the TS relay handler. Unset → 503-with-fallback. Trailing slash is stripped. | | `BIFROST_PORT` | `8080` | `src/lib/services/bootstrap.ts` | Port the supervised Bifrost embedded service binds to (`127.0.0.1:`) when OmniRoute manages the Bifrost sidecar lifecycle. Defaults to `8080`. | +| `LLMLINGUA_PORT` | `20135` | `src/lib/services/bootstrap.ts` | Port the supervised LLMLingua compression service binds to (`127.0.0.1:`). Defaults to `20135`. | +| `LLMLINGUA_BASE_URL` | `http://127.0.0.1:20135` | `open-sse/services/compression/engines/llmlingua/index.ts` | Base URL for the local LLMLingua HTTP compression microservice daemon. Defaults to `http://127.0.0.1:20135`. | | `BIFROST_API_KEY` | _(unset)_ | `src/app/api/v1/relay/chat/completions/bifrost/route.ts` | API key for the Bifrost gateway (sent as `Authorization: Bearer ...`). If unset, the route expects the request to carry a valid OmniRoute API key; this key is for gateway-side auth only. | | `BIFROST_STREAMING_ENABLED` | `true` | `src/app/api/v1/relay/chat/completions/bifrost/route.ts` | When true, the Bifrost sidecar route streams responses back via SSE through the gateway rather than the TS streaming executor. Set to `0` to force non-streaming JSON responses through the gateway. | | `BIFROST_TIMEOUT_MS` | `30000` | `src/app/api/v1/relay/chat/completions/bifrost/route.ts` | Per-request timeout when proxying to the Bifrost gateway (ms). On timeout the route returns the TS relay path via the `X-Bifrost-Fallback` header. | diff --git a/open-sse/services/compression/engines/llmlingua/index.ts b/open-sse/services/compression/engines/llmlingua/index.ts index 1dea61227703..a2f13a3ea073 100644 --- a/open-sse/services/compression/engines/llmlingua/index.ts +++ b/open-sse/services/compression/engines/llmlingua/index.ts @@ -85,9 +85,30 @@ export function setLlmlinguaBackend(b: LlmlinguaBackend | null): void { _backend = b; } -/** Resolve the active backend: injected fake (for tests) or production stub. */ +/** Resolve the active backend: injected fake (for tests), HTTP sidecar, or production worker. */ +async function httpSidecarBackend(text: string, opts?: LlmlinguaBackendOptions): Promise { + const sidecarUrl = process.env.LLMLINGUA_BASE_URL || "http://127.0.0.1:20135"; + try { + const res = await fetch(`${sidecarUrl}/compress`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text, ...opts }), + signal: AbortSignal.timeout(5000), + }); + if (res.ok) { + const data = (await res.json()) as { text?: string }; + if (typeof data.text === "string" && data.text.length < text.length) { + return data.text; + } + } + } catch { + // Fall back to worker backend + } + return workerBackend(text, opts); +} + function resolveBackend(): LlmlinguaBackend { - return _backend ?? workerBackend; + return _backend ?? httpSidecarBackend; } // ─── prose/code splitting ───────────────────────────────────────────────────── diff --git a/package.json b/package.json index f02480131644..6fc2717f22e5 100644 --- a/package.json +++ b/package.json @@ -255,6 +255,7 @@ "test:combo:live": "cross-env RUN_COMBO_LIVE=1 DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 \"tests/integration/combo-live/*.live.test.ts\"", "test:combo:live:vps": "node scripts/test/combo-live-vps.mjs", "test:combo:live:vps:failover": "node scripts/test/combo-live-vps.mjs --failover", + "test:services:int": "cross-env RUN_SERVICES_INT=1 DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 \"tests/integration/services/*.test.ts\"", "test:boundary:live": "cross-env RUN_BOUNDARY_LIVE=1 DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 \"tests/boundary/*.live.test.ts\"", "test:heap": "node --expose-gc --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit tests/integration/heap-growth.test.ts", "test:chaos": "cross-env RUN_CHAOS_INT=1 node --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 tests/integration/resilience-chaos.test.ts", diff --git a/scripts/check/check-test-discovery.mjs b/scripts/check/check-test-discovery.mjs index 6a3d5eba4374..29e76554e585 100644 --- a/scripts/check/check-test-discovery.mjs +++ b/scripts/check/check-test-discovery.mjs @@ -86,8 +86,11 @@ export const COLLECTORS = [ ".github/workflows/quality.yml": "test:unit:ci:shard", }, }, - // Node native runner — test:integration (top-level only; tests/integration/services/ NÃO roda) + // Node native runner — test:integration (top-level only) { glob: "tests/integration/*.test.ts", sources: ["package.json"] }, + // Node native runner — test:services:int (gated RUN_SERVICES_INT=1; embedded-service + // installer/lifecycle tests — slow, network-dependent, never runs unopted in CI) + { glob: "tests/integration/services/*.test.ts", sources: ["package.json"] }, // Node native runner — test:combo:matrix / test:integration (combo strategy decision matrix, 17 strategies) { glob: "tests/integration/combo-matrix/*.test.ts", sources: ["package.json"] }, // Node native runner — test:combo:live (gated real-upstream smoke; RUN_COMBO_LIVE=1 + VPS creds) diff --git a/src/app/(dashboard)/dashboard/providers/services/page.tsx b/src/app/(dashboard)/dashboard/providers/services/page.tsx index 45f36362a489..2ea93902e3a8 100644 --- a/src/app/(dashboard)/dashboard/providers/services/page.tsx +++ b/src/app/(dashboard)/dashboard/providers/services/page.tsx @@ -9,8 +9,9 @@ import { MuxServiceTab } from "./tabs/MuxServiceTab"; import { BifrostServiceTab } from "./tabs/BifrostServiceTab"; import { DarioServiceTab } from "./tabs/DarioServiceTab"; import { OpenwaServiceTab } from "./tabs/OpenwaServiceTab"; +import { LlmlinguaServiceTab } from "./tabs/LlmlinguaServiceTab"; -type Tab = "cliproxy" | "9router" | "mux" | "bifrost" | "dario" | "openwa"; +type Tab = "cliproxy" | "9router" | "mux" | "bifrost" | "dario" | "openwa" | "llmlingua"; const TABS: { id: Tab; label: string; icon: string }[] = [ { id: "cliproxy", label: "CLIProxyAPI", icon: "swap_horiz" }, @@ -19,6 +20,7 @@ const TABS: { id: Tab; label: string; icon: string }[] = [ { id: "bifrost", label: "Bifrost", icon: "bolt" }, { id: "dario", label: "Dario", icon: "shield_person" }, { id: "openwa", label: "open-wa", icon: "chat" }, + { id: "llmlingua", label: "LLMLingua", icon: "compress" }, ]; export default function ServicesPage() { @@ -67,6 +69,7 @@ export default function ServicesPage() { {active === "bifrost" && } {active === "dario" && } {active === "openwa" && } + {active === "llmlingua" && } ); diff --git a/src/app/(dashboard)/dashboard/providers/services/tabs/LlmlinguaServiceTab.tsx b/src/app/(dashboard)/dashboard/providers/services/tabs/LlmlinguaServiceTab.tsx new file mode 100644 index 000000000000..9f25bbafda6e --- /dev/null +++ b/src/app/(dashboard)/dashboard/providers/services/tabs/LlmlinguaServiceTab.tsx @@ -0,0 +1,21 @@ +"use client"; + +import { ServiceStatusCard } from "../components/ServiceStatusCard"; +import { ServiceLifecycleButtons } from "../components/ServiceLifecycleButtons"; +import { ServiceLogsPanel } from "../components/ServiceLogsPanel"; +import { AutoStartToggle } from "../components/AutoStartToggle"; +import { AutoRestartAdoptedToggle } from "../components/AutoRestartAdoptedToggle"; + +const NAME = "llmlingua"; + +export function LlmlinguaServiceTab() { + return ( +
+ + + + + +
+ ); +} diff --git a/src/app/api/services/llmlingua/_lib.ts b/src/app/api/services/llmlingua/_lib.ts new file mode 100644 index 000000000000..8af69d2c665b --- /dev/null +++ b/src/app/api/services/llmlingua/_lib.ts @@ -0,0 +1,33 @@ +/** + * Shared helpers for /api/services/llmlingua/* route handlers. + * Creates a supervisor on demand if bootstrap hasn't registered one yet. + */ + +import { getSupervisor, registerSupervisor } from "@/lib/services/registry"; +import { ServiceSupervisor } from "@/lib/services/ServiceSupervisor"; +import { resolveSpawnArgs, LLMLINGUA_DEFAULT_PORT } from "@/lib/services/installers/llmlingua"; + +const TOOL = "llmlingua"; +const PORT = parseInt(process.env.LLMLINGUA_PORT ?? String(LLMLINGUA_DEFAULT_PORT), 10); + +export async function getOrInitSupervisor(): Promise { + const existing = getSupervisor(TOOL); + if (existing) return existing; + + const sup = new ServiceSupervisor({ + tool: TOOL, + port: PORT, + spawnArgs: () => resolveSpawnArgs(PORT), + healthUrl: () => `http://127.0.0.1:${PORT}/health`, + healthIntervalMs: 5_000, + stopTimeoutMs: 15_000, + logsBufferBytes: 5_242_880, + // #6205: mirrors bootstrap.ts's own supervisor construction — adopt a + // healthy prior instance instead of crashing on-demand creation (e.g. a + // direct API hit before bootstrap runs) into a raw EADDRINUSE. + probeBeforeSpawn: true, + }); + + registerSupervisor(sup); + return sup; +} diff --git a/src/app/api/services/llmlingua/auto-restart-adopted/route.ts b/src/app/api/services/llmlingua/auto-restart-adopted/route.ts new file mode 100644 index 000000000000..6336da50c2a7 --- /dev/null +++ b/src/app/api/services/llmlingua/auto-restart-adopted/route.ts @@ -0,0 +1,28 @@ +import { z } from "zod"; +import { updateServiceField } from "@/lib/db/versionManager"; +import { createErrorResponse } from "@/lib/api/errorResponse"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error"; + +const BodySchema = z.object({ enabled: z.boolean() }); + +export async function POST(request: Request): Promise { + let body: unknown; + try { + body = await request.json(); + } catch { + return createErrorResponse({ status: 400, message: "Invalid JSON body" }); + } + + const parsed = BodySchema.safeParse(body); + if (!parsed.success) { + return createErrorResponse({ status: 400, message: parsed.error.message }); + } + + try { + await updateServiceField("llmlingua", "autoRestartAdopted", parsed.data.enabled); + return new Response(null, { status: 204 }); + } catch (err) { + const msg = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); + return createErrorResponse({ status: 500, message: msg }); + } +} diff --git a/src/app/api/services/llmlingua/auto-start/route.ts b/src/app/api/services/llmlingua/auto-start/route.ts new file mode 100644 index 000000000000..b5b969b9804c --- /dev/null +++ b/src/app/api/services/llmlingua/auto-start/route.ts @@ -0,0 +1,28 @@ +import { z } from "zod"; +import { updateServiceField } from "@/lib/db/versionManager"; +import { createErrorResponse } from "@/lib/api/errorResponse"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error"; + +const BodySchema = z.object({ enabled: z.boolean() }); + +export async function POST(request: Request): Promise { + let body: unknown; + try { + body = await request.json(); + } catch { + return createErrorResponse({ status: 400, message: "Invalid JSON body" }); + } + + const parsed = BodySchema.safeParse(body); + if (!parsed.success) { + return createErrorResponse({ status: 400, message: parsed.error.message }); + } + + try { + await updateServiceField("llmlingua", "autoStart", parsed.data.enabled); + return new Response(null, { status: 204 }); + } catch (err) { + const msg = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); + return createErrorResponse({ status: 500, message: msg }); + } +} diff --git a/src/app/api/services/llmlingua/install/route.ts b/src/app/api/services/llmlingua/install/route.ts new file mode 100644 index 000000000000..470219e7f6ed --- /dev/null +++ b/src/app/api/services/llmlingua/install/route.ts @@ -0,0 +1,6 @@ +import { install } from "@/lib/services/installers/llmlingua"; +import { handleServiceInstall } from "@/app/api/services/_shared/installRoute"; + +export async function POST(request: Request): Promise { + return handleServiceInstall(request, install); +} diff --git a/src/app/api/services/llmlingua/restart/route.ts b/src/app/api/services/llmlingua/restart/route.ts new file mode 100644 index 000000000000..938256f70a26 --- /dev/null +++ b/src/app/api/services/llmlingua/restart/route.ts @@ -0,0 +1,22 @@ +import { getServiceRow } from "@/lib/db/versionManager"; +import { getOrInitSupervisor } from "../_lib"; +import { createErrorResponse } from "@/lib/api/errorResponse"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error"; + +const TOOL = "llmlingua"; + +export async function POST(): Promise { + try { + const row = await getServiceRow(TOOL); + if (!row || row.status === "not_installed") { + return createErrorResponse({ status: 409, message: "LLMLingua não está instalado." }); + } + + const sup = await getOrInitSupervisor(); + const status = await sup.restart(); + return Response.json(status); + } catch (err) { + const msg = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); + return createErrorResponse({ status: 503, message: msg }); + } +} diff --git a/src/app/api/services/llmlingua/start/route.ts b/src/app/api/services/llmlingua/start/route.ts new file mode 100644 index 000000000000..3b785f10d7a8 --- /dev/null +++ b/src/app/api/services/llmlingua/start/route.ts @@ -0,0 +1,22 @@ +import { getServiceRow } from "@/lib/db/versionManager"; +import { getOrInitSupervisor } from "../_lib"; +import { createErrorResponse } from "@/lib/api/errorResponse"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error"; + +const TOOL = "llmlingua"; + +export async function POST(): Promise { + try { + const row = await getServiceRow(TOOL); + if (!row || row.status === "not_installed") { + return createErrorResponse({ status: 409, message: "LLMLingua não está instalado." }); + } + + const sup = await getOrInitSupervisor(); + const status = await sup.start(); + return Response.json(status); + } catch (err) { + const msg = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); + return createErrorResponse({ status: 503, message: msg }); + } +} diff --git a/src/app/api/services/llmlingua/status/route.ts b/src/app/api/services/llmlingua/status/route.ts new file mode 100644 index 000000000000..8c537dddbac8 --- /dev/null +++ b/src/app/api/services/llmlingua/status/route.ts @@ -0,0 +1,41 @@ +import { getSupervisor } from "@/lib/services/registry"; +import { getServiceRow } from "@/lib/db/versionManager"; +import { + getInstalledVersion, + getLatestVersion, + LLMLINGUA_DEFAULT_PORT, +} from "@/lib/services/installers/llmlingua"; +import { createErrorResponse } from "@/lib/api/errorResponse"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error"; + +const TOOL = "llmlingua"; + +export async function GET(): Promise { + try { + const sup = getSupervisor(TOOL); + const row = await getServiceRow(TOOL); + + const liveStatus = sup?.getStatus() ?? null; + const installedVersion = await getInstalledVersion(); + const latestVersion = await getLatestVersion(); + + return Response.json({ + tool: TOOL, + state: liveStatus?.state ?? row?.status ?? "unknown", + pid: liveStatus?.pid ?? null, + port: liveStatus?.port ?? row?.port ?? LLMLINGUA_DEFAULT_PORT, + health: liveStatus?.health ?? "unknown", + startedAt: liveStatus?.startedAt ?? null, + lastError: liveStatus?.lastError ?? row?.errorMessage ?? null, + installedVersion: installedVersion ?? row?.installedVersion ?? null, + latestVersion, + updateAvailable: !!installedVersion && !!latestVersion && installedVersion !== latestVersion, + autoStart: row?.autoStart ?? false, + adopted: liveStatus?.adopted ?? false, + autoRestartAdopted: row?.autoRestartAdopted ?? false, + }); + } catch (err) { + const msg = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); + return createErrorResponse({ status: 500, message: msg }); + } +} diff --git a/src/app/api/services/llmlingua/stop/route.ts b/src/app/api/services/llmlingua/stop/route.ts new file mode 100644 index 000000000000..a277e8c23082 --- /dev/null +++ b/src/app/api/services/llmlingua/stop/route.ts @@ -0,0 +1,19 @@ +import { getSupervisor } from "@/lib/services/registry"; +import { createErrorResponse } from "@/lib/api/errorResponse"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error"; + +const TOOL = "llmlingua"; + +export async function POST(): Promise { + try { + const sup = getSupervisor(TOOL); + if (!sup) { + return Response.json({ tool: TOOL, state: "stopped" }); + } + const status = await sup.stop(); + return Response.json(status); + } catch (err) { + const msg = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); + return createErrorResponse({ status: 500, message: msg }); + } +} diff --git a/src/app/api/services/llmlingua/update/route.ts b/src/app/api/services/llmlingua/update/route.ts new file mode 100644 index 000000000000..1046ec0347ca --- /dev/null +++ b/src/app/api/services/llmlingua/update/route.ts @@ -0,0 +1,45 @@ +import { getSupervisor } from "@/lib/services/registry"; +import { getOrInitSupervisor } from "../_lib"; +import { + getInstalledVersion, + getLatestVersion, + update as downloadUpdate, +} from "@/lib/services/installers/llmlingua"; +import { createErrorResponse } from "@/lib/api/errorResponse"; +import { sanitizeErrorMessage } from "@omniroute/open-sse/utils/error"; + +export async function POST(): Promise { + try { + const [installed, latest] = await Promise.all([getInstalledVersion(), getLatestVersion()]); + + if (installed && latest && installed === latest) { + return Response.json({ updated: false, installedVersion: installed, latestVersion: latest }); + } + + const sup = getSupervisor("llmlingua"); + const wasRunning = sup?.getStatus().state === "running"; + + if (wasRunning && sup) { + await sup.stop(); + } + + const result = await downloadUpdate(); + + if (wasRunning) { + const freshSup = await getOrInitSupervisor(); + await freshSup.start().catch((err: unknown) => { + const msg = err instanceof Error ? err.message : String(err); + console.warn("[Services] Could not restart llmlingua after update:", msg); + }); + } + + return Response.json({ + updated: true, + oldVersion: installed ?? null, + newVersion: result.installedVersion, + }); + } catch (err) { + const msg = sanitizeErrorMessage(err instanceof Error ? err.message : String(err)); + return createErrorResponse({ status: 500, message: msg }); + } +} diff --git a/src/lib/db/migrations/183_llmlingua_service.sql b/src/lib/db/migrations/183_llmlingua_service.sql new file mode 100644 index 000000000000..9429a7fca61f --- /dev/null +++ b/src/lib/db/migrations/183_llmlingua_service.sql @@ -0,0 +1,14 @@ +-- Migration 183: Seed version_manager row for the LLMLingua embedded service. +-- +-- LLMLingua (npm @atjsh/llmlingua-2, a JS/TS port of Microsoft's LLMLingua-2 +-- prompt-compression algorithm) is managed via the ServiceSupervisor +-- framework, same shape as Bifrost (115) and Mux (114). The sidecar HTTP +-- endpoint lets open-sse/services/compression/engines/llmlingua/index.ts +-- dispatch prompt compression over HTTP instead of the in-process worker +-- thread. Seeded with status='not_installed' so the bootstrap loop skips it +-- until the user installs via /api/services/llmlingua/install. + +INSERT OR IGNORE INTO version_manager + (tool, status, port, auto_start, auto_update, provider_expose) +VALUES + ('llmlingua', 'not_installed', 20135, 0, 0, 0); diff --git a/src/lib/services/bootstrap.ts b/src/lib/services/bootstrap.ts index ddb41bad0da4..51dbffbf9331 100644 --- a/src/lib/services/bootstrap.ts +++ b/src/lib/services/bootstrap.ts @@ -13,6 +13,10 @@ import { resolveSpawnArgs as muxSpawnArgs, MUX_DEFAULT_PORT } from "./installers import { resolveSpawnArgs as bifrostSpawnArgs, BIFROST_DEFAULT_PORT } from "./installers/bifrost"; import { resolveSpawnArgs as darioSpawnArgs, DARIO_DEFAULT_PORT } from "./installers/dario"; import { resolveSpawnArgs as openwaSpawnArgs, OPENWA_DEFAULT_PORT } from "./installers/openwa"; +import { + resolveSpawnArgs as llmlinguaSpawnArgs, + LLMLINGUA_DEFAULT_PORT, +} from "./installers/llmlingua"; import { getOrCreateApiKey } from "./apiKey"; import { scheduleServiceModelSync, stopServiceModelSync } from "./modelSync"; import type { ServiceStatus } from "./types"; @@ -36,6 +40,7 @@ const MUX_PORT = parseInt(process.env.MUX_SERVICE_PORT ?? String(MUX_DEFAULT_POR const BIFROST_PORT = parseInt(process.env.BIFROST_PORT ?? String(BIFROST_DEFAULT_PORT), 10); const DARIO_PORT = parseInt(process.env.DARIO_PORT ?? String(DARIO_DEFAULT_PORT), 10); const OPENWA_PORT = parseInt(process.env.OPENWA_SERVICE_PORT ?? String(OPENWA_DEFAULT_PORT), 10); +const LLMLINGUA_PORT = parseInt(process.env.LLMLINGUA_PORT ?? String(LLMLINGUA_DEFAULT_PORT), 10); type ServiceEntry = { tool: string; @@ -131,6 +136,15 @@ const SERVICES: ServiceEntry[] = [ logsBufferBytes: 5_242_880, needsApiKey: true, }, + { + tool: "llmlingua", + port: LLMLINGUA_PORT, + healthPath: "/health", + healthIntervalMs: 5_000, + stopTimeoutMs: 15_000, + logsBufferBytes: 5_242_880, + needsApiKey: false, + }, ]; function buildSpawnArgsFactory( @@ -152,6 +166,9 @@ function buildSpawnArgsFactory( if (cfg.tool === "openwa") { return () => openwaSpawnArgs(apiKey, cfg.port); } + if (cfg.tool === "llmlingua") { + return () => llmlinguaSpawnArgs(cfg.port); + } return () => cliproxySpawnArgs(cfg.port, apiKey); } diff --git a/src/lib/services/installers/llmlingua.ts b/src/lib/services/installers/llmlingua.ts new file mode 100644 index 000000000000..79ff782920f1 --- /dev/null +++ b/src/lib/services/installers/llmlingua.ts @@ -0,0 +1,262 @@ +/** + * LLMLingua Server installer adapter for the ServiceSupervisor framework. + * + * Runs an external/embedded LLMLingua HTTP compression microservice daemon. + * When active on loopback port 20135, prompt compression requests are dispatched + * over HTTP instead of executing synchronous regex parsing inside Node.js. + * + * The sidecar wraps the real `@atjsh/llmlingua-2` package (a JS/TS port of + * Microsoft's LLMLingua-2 prompt-compression algorithm, backed by a BERT-sized + * ONNX token-classification model downloaded from Hugging Face on first use) + * — not a stub. `install()` npm-installs the package + its `@huggingface/transformers` + * / `js-tiktoken` peers into an isolated `node_modules` the same way + * `ninerouter.ts`/`bifrost.ts` do. + * + * Binary location: $DATA_DIR/services/llmlingua/server.mjs + * DB row: version_manager WHERE tool = 'llmlingua' + */ + +import fs from "node:fs"; +import path from "node:path"; +import { DATA_DIR } from "@/lib/db/core"; +import { upsertVersionManagerTool } from "@/lib/db/versionManager"; +import { runNpm, InstallError } from "./utils"; + +export const LLMLINGUA_PACKAGE = "@atjsh/llmlingua-2"; +// Peer deps the package itself declares (README → "Getting Started"): the +// transformers.js runtime that loads/runs the ONNX model, and the tokenizer +// used to compute the compression rate against. +const LLMLINGUA_PEER_PACKAGES = ["@huggingface/transformers", "js-tiktoken"] as const; +export const LLMLINGUA_DEFAULT_PORT = 20135; +export const LLMLINGUA_INSTALL_DIR = path.join(DATA_DIR, "services", "llmlingua"); +// Smallest published model (README → "Model Selection", ~57 MB ONNX) — favors +// fast sidecar startup over the larger XLM-RoBERTa/BERT-base options. +export const LLMLINGUA_DEFAULT_MODEL = "atjsh/llmlingua-2-js-tinybert-meetingbank"; + +export interface InstallResult { + installedVersion: string; + installPath: string; + durationMs: number; +} + +export interface SpawnArgs { + command: string; + args: string[]; + env: NodeJS.ProcessEnv; + cwd: string; +} + +// In-memory latest-version cache, 1h TTL — mirrors bifrost.ts/ninerouter.ts. +let latestVersionCache: { value: string; expiresAt: number } | null = null; +const VERSION_CACHE_TTL_MS = 3_600_000; + +function getLlmlinguaInstallDir(): string { + return process.env.DATA_DIR + ? path.join(process.env.DATA_DIR, "services", "llmlingua") + : LLMLINGUA_INSTALL_DIR; +} + +export function getServerScriptPath(): string { + return path.join(getLlmlinguaInstallDir(), "server.mjs"); +} + +function getInstalledPkgPath(): string { + return path.join( + getLlmlinguaInstallDir(), + "node_modules", + "@atjsh", + "llmlingua-2", + "package.json" + ); +} + +export async function getInstalledVersion(): Promise { + try { + const raw = fs.readFileSync(getInstalledPkgPath(), "utf8"); + const parsed = JSON.parse(raw) as { version?: string }; + return typeof parsed.version === "string" ? parsed.version : null; + } catch { + return null; + } +} + +export async function getLatestVersion(): Promise { + if (latestVersionCache && latestVersionCache.expiresAt > Date.now()) { + return latestVersionCache.value; + } + try { + const { stdout } = await runNpm(["view", LLMLINGUA_PACKAGE, "version"], { timeoutMs: 30_000 }); + const version = stdout.trim(); + if (version) { + latestVersionCache = { value: version, expiresAt: Date.now() + VERSION_CACHE_TTL_MS }; + } + return version || null; + } catch { + return null; + } +} + +/** + * The sidecar HTTP server. Imports resolve against `cwd` (this file's own + * install dir), where `install()` npm-installs `@atjsh/llmlingua-2` and its + * peers — same resolution model as `ninerouter.ts`/`bifrost.ts`. The model + * is loaded lazily on the first `/compress` call (not at startup) so the + * health check does not block on a ~57 MB Hugging Face download. + */ +const SERVER_SCRIPT_SOURCE = `import http from "node:http"; +import { LLMLingua2 } from "@atjsh/llmlingua-2"; +import { Tiktoken } from "js-tiktoken/lite"; +import o200k_base from "js-tiktoken/ranks/o200k_base"; + +const PORT = parseInt(process.env.PORT || "20135", 10); +const MODEL_NAME = process.env.LLMLINGUA_MODEL || "atjsh/llmlingua-2-js-tinybert-meetingbank"; + +let compressorPromise = null; +function getCompressor() { + if (!compressorPromise) { + compressorPromise = LLMLingua2.WithBERTMultilingual(MODEL_NAME, { + transformerJSConfig: { device: "cpu", dtype: "fp32" }, + oaiTokenizer: new Tiktoken(o200k_base), + }).then((r) => r.promptCompressor); + } + return compressorPromise; +} + +const server = http.createServer((req, res) => { + if (req.url === "/health" || req.url === "/healthz") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ status: "healthy", service: "llmlingua" })); + return; + } + if (req.method === "POST" && req.url === "/compress") { + let body = ""; + req.on("data", (chunk) => { body += chunk; }); + req.on("end", async () => { + try { + const parsed = JSON.parse(body); + const text = typeof parsed.text === "string" ? parsed.text : ""; + const rate = typeof parsed.rate === "number" ? parsed.rate : 0.5; + if (!text) { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ text: "", compressed: true, ratio: 1 })); + return; + } + const compressor = await getCompressor(); + const compressedText = await compressor.compress_prompt(text, { rate }); + const ratio = text.length > 0 ? compressedText.length / text.length : 1; + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ text: compressedText, compressed: true, ratio })); + } catch (err) { + res.writeHead(500, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: err.message })); + } + }); + return; + } + res.writeHead(404); + res.end(); +}); +server.listen(PORT, "127.0.0.1", () => { + console.log("[llmlingua-server] Listening on 127.0.0.1:" + PORT); +}); +`; + +export async function install(version = "latest"): Promise { + const startMs = Date.now(); + const installDir = getLlmlinguaInstallDir(); + + fs.mkdirSync(installDir, { recursive: true }); + const hostPkgPath = path.join(installDir, "package.json"); + if (!fs.existsSync(hostPkgPath)) { + fs.writeFileSync( + hostPkgPath, + JSON.stringify( + { + name: "omniroute-llmlingua-host", + version: "0.0.0", + private: true, + type: "module", + dependencies: {}, + }, + null, + 2 + ), + "utf8" + ); + } + + await runNpm( + [ + "install", + `${LLMLINGUA_PACKAGE}@${version}`, + ...LLMLINGUA_PEER_PACKAGES.map((pkg) => `${pkg}@latest`), + "--omit=dev", + "--no-audit", + "--no-fund", + ], + // `--prefix` via `prefix` (→ npm_config_prefix env) so paths with spaces survive Windows shell + { cwd: installDir, prefix: installDir, timeoutMs: 300_000 } + ); + + const serverScript = getServerScriptPath(); + fs.writeFileSync(serverScript, SERVER_SCRIPT_SOURCE, "utf8"); + + const installedVersion = await getInstalledVersion(); + if (!installedVersion) { + throw new InstallError( + "Could not read installed version from node_modules/@atjsh/llmlingua-2/package.json", + "LLMLingua instalado mas versão não pôde ser lida.", + 500 + ); + } + + await upsertVersionManagerTool({ + tool: "llmlingua", + installedVersion, + binaryPath: serverScript, + status: "stopped", + port: LLMLINGUA_DEFAULT_PORT, + }); + + // Invalidate cache so next getLatestVersion() re-fetches + latestVersionCache = null; + + return { + installedVersion, + installPath: installDir, + durationMs: Date.now() - startMs, + }; +} + +export async function update(): Promise { + return install("latest"); +} + +export async function uninstall(): Promise { + const nmDir = path.join(getLlmlinguaInstallDir(), "node_modules"); + if (fs.existsSync(nmDir)) { + fs.rmSync(nmDir, { recursive: true, force: true }); + } + await upsertVersionManagerTool({ + tool: "llmlingua", + status: "not_installed", + installedVersion: null, + binaryPath: null, + }); +} + +export function resolveSpawnArgs(port = LLMLINGUA_DEFAULT_PORT): SpawnArgs { + const serverScript = getServerScriptPath(); + const installDir = getLlmlinguaInstallDir(); + + return { + command: process.execPath, + args: [serverScript], + env: { + ...process.env, + PORT: String(port), + LLMLINGUA_MODEL: process.env.LLMLINGUA_MODEL || LLMLINGUA_DEFAULT_MODEL, + }, + cwd: installDir, + }; +} diff --git a/tests/integration/services/llmlingua-sidecar-real-compression.int.test.ts b/tests/integration/services/llmlingua-sidecar-real-compression.int.test.ts new file mode 100644 index 000000000000..39b50bfa0779 --- /dev/null +++ b/tests/integration/services/llmlingua-sidecar-real-compression.int.test.ts @@ -0,0 +1,155 @@ +/** + * Proves the LLMLingua embedded-service sidecar performs REAL semantic + * compression — not the historical stub that echoed `text` back unchanged + * with a hardcoded `ratio: 0.5` (the defect this PR fixed). + * + * GATED behind RUN_SERVICES_INT=1 (same convention as + * tests/integration/services/full-lifecycle.int.test.ts): this test runs a + * real `npm install` of `@atjsh/llmlingua-2` + peers and downloads the + * ~57 MB TinyBERT ONNX model from Hugging Face on first `/compress` call, so + * it is slow and network-dependent — it does not run in default CI. + * + * RUN_SERVICES_INT=1 node --import tsx/esm --test \ + * tests/integration/services/llmlingua-sidecar-real-compression.int.test.ts + * + * What "real" means here, concretely: + * 1. The response `text` is SHORTER than the input (the stub always + * returned the input verbatim). + * 2. The response `text` DIFFERS from the input (the stub never rewrote + * anything). + * 3. Two different prose inputs compress to DIFFERENT outputs (the stub's + * output was a pure function of the input's length only — echo). + */ + +import { describe, it, before, after } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const ENABLED = process.env.RUN_SERVICES_INT === "1"; +const SKIP_REASON = "Set RUN_SERVICES_INT=1 to run the real LLMLingua sidecar compression test"; + +function maybeSkip(t: { skip: (reason?: string) => void }): boolean { + if (!ENABLED) { + t.skip(SKIP_REASON); + return true; + } + return false; +} + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-llmlingua-sidecar-int-")); +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.NODE_ENV = "test"; +process.env.DISABLE_SQLITE_AUTO_BACKUP = "true"; + +const PORT = 20536; +const LONG_PROSE_A = + "LLMLingua-2, a small-size yet powerful prompt compression method trained via data " + + "distillation from GPT-4 for token classification with a BERT-level encoder, excels in " + + "task-agnostic compression. It surpasses LLMLingua in handling out-of-domain data, offering " + + "3x-6x faster performance. ".repeat(6); +const LONG_PROSE_B = + "The quick brown fox jumps over the lazy dog while the sun sets slowly behind the distant " + + "hills, and every evening the same fox returns to the same hollow log near the old stone " + + "wall to sleep until dawn breaks over the valley. ".repeat(6); + +let ServiceSupervisor: typeof import("../../../src/lib/services/ServiceSupervisor").ServiceSupervisor; +let installLlmlingua: typeof import("../../../src/lib/services/installers/llmlingua").install; +let resolveSpawnArgs: typeof import("../../../src/lib/services/installers/llmlingua").resolveSpawnArgs; +let sup: InstanceType | null = null; + +before(async () => { + if (!ENABLED) return; + + const core = await import("../../../src/lib/db/core.ts"); + core.getDbInstance(); + + const installerMod = await import("../../../src/lib/services/installers/llmlingua.ts"); + installLlmlingua = installerMod.install; + resolveSpawnArgs = installerMod.resolveSpawnArgs; + + const supervisorMod = await import("../../../src/lib/services/ServiceSupervisor.ts"); + ServiceSupervisor = supervisorMod.ServiceSupervisor; + + await installLlmlingua("latest"); +}); + +after(async () => { + if (sup) { + await sup.stop().catch(() => undefined); + } + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +describe("llmlingua sidecar — real compression (opt-in, RUN_SERVICES_INT=1)", () => { + it("start()s the real sidecar and reports healthy", async (t) => { + if (maybeSkip(t)) return; + + sup = new ServiceSupervisor({ + tool: "llmlingua", + port: PORT, + spawnArgs: () => resolveSpawnArgs(PORT), + healthUrl: () => `http://127.0.0.1:${PORT}/health`, + healthIntervalMs: 5_000, + stopTimeoutMs: 15_000, + logsBufferBytes: 5_242_880, + probeBeforeSpawn: true, + }); + + const status = await sup.start(); + assert.equal(status.state, "running"); + }); + + it("compresses real prose: shorter than input, and differs from it", async (t) => { + if (maybeSkip(t)) return; + + const res = await fetch(`http://127.0.0.1:${PORT}/compress`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + // Long compression budget: first call also pays the one-time ONNX model load. + signal: AbortSignal.timeout(120_000), + body: JSON.stringify({ text: LONG_PROSE_A, rate: 0.5 }), + }); + assert.equal(res.status, 200); + + const data = (await res.json()) as { text: string; compressed: boolean; ratio: number }; + assert.equal(data.compressed, true); + assert.ok( + data.text.length < LONG_PROSE_A.length, + `expected compressed text shorter than input (${LONG_PROSE_A.length} chars), got ${data.text.length}` + ); + assert.notEqual(data.text, LONG_PROSE_A, "stub defect: response echoed the input unchanged"); + assert.ok( + data.ratio > 0 && data.ratio < 1, + `ratio must reflect real compression, got ${data.ratio}` + ); + }); + + it("two different prose inputs compress to different outputs (not a length-only echo)", async (t) => { + if (maybeSkip(t)) return; + + // Sequential, not concurrent: ONNX token-classification inference is not + // reentrant (see worker.ts's own FIFO-queue note for the same model). + const resA = await fetch(`http://127.0.0.1:${PORT}/compress`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + signal: AbortSignal.timeout(120_000), + body: JSON.stringify({ text: LONG_PROSE_A, rate: 0.5 }), + }); + const resB = await fetch(`http://127.0.0.1:${PORT}/compress`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + signal: AbortSignal.timeout(120_000), + body: JSON.stringify({ text: LONG_PROSE_B, rate: 0.5 }), + }); + + const dataA = (await resA.json()) as { text: string }; + const dataB = (await resB.json()) as { text: string }; + assert.notEqual( + dataA.text, + dataB.text, + "two different inputs must not compress to the same output" + ); + }); +}); diff --git a/tests/unit/services/installers/llmlingua.test.ts b/tests/unit/services/installers/llmlingua.test.ts new file mode 100644 index 000000000000..65a59af9c920 --- /dev/null +++ b/tests/unit/services/installers/llmlingua.test.ts @@ -0,0 +1,148 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { execSync } from "node:child_process"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-llmlingua-installer-")); +const FAKE_BIN_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-llmlingua-fake-bin-")); +const MOCK_LLMLINGUA_VERSION = "3.0.0"; + +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.NODE_ENV = "test"; +process.env.DISABLE_SQLITE_AUTO_BACKUP = "true"; + +// Prepend fake bin dir to PATH so our fake `npm` is found by runNpm — the +// real @atjsh/llmlingua-2 + @huggingface/transformers + js-tiktoken install +// is network-dependent and downloads a multi-MB ONNX model on first use, so +// it is proven by the opt-in integration test instead (RUN_SERVICES_INT=1 — +// tests/integration/services/llmlingua-sidecar-real-compression.int.test.ts), +// not here. This suite only proves the installer WIRING (spawn args, DB row, +// version resolution) is correct. +const originalPath = process.env.PATH ?? ""; +process.env.PATH = `${FAKE_BIN_DIR}:${originalPath}`; + +const INSTALL_DIR = path.join(TEST_DATA_DIR, "services", "llmlingua"); +const fakeNpmScript = `#!/bin/sh +set -e +CMD="$1" +shift +if [ "$CMD" = "install" ]; then + PREFIX="" + while [ $# -gt 0 ]; do + if [ "$1" = "--prefix" ]; then PREFIX="$2"; shift 2; else shift; fi + done + if [ -z "$PREFIX" ]; then PREFIX="$npm_config_prefix"; fi + PKG_DIR="$PREFIX/node_modules/@atjsh/llmlingua-2" + mkdir -p "$PKG_DIR" + echo '{"name":"@atjsh/llmlingua-2","version":"${MOCK_LLMLINGUA_VERSION}"}' > "$PKG_DIR/package.json" + exit 0 +fi +if [ "$CMD" = "view" ]; then + echo "${MOCK_LLMLINGUA_VERSION}" + exit 0 +fi +exit 0 +`; +const fakeNpmPath = path.join(FAKE_BIN_DIR, "npm"); +fs.writeFileSync(fakeNpmPath, fakeNpmScript, { mode: 0o755 }); + +execSync("which npm", { env: process.env }); + +// DB bootstrap +const core = await import("../../../../src/lib/db/core.ts"); +const db = core.getDbInstance(); +db.prepare( + `INSERT OR IGNORE INTO version_manager (tool, status, port, auto_start, auto_update, provider_expose) + VALUES ('llmlingua', 'not_installed', 20135, 0, 0, 0)` +).run(); + +const { + install, + update, + uninstall, + getInstalledVersion, + getLatestVersion, + resolveSpawnArgs, + getServerScriptPath, + LLMLINGUA_DEFAULT_PORT, + LLMLINGUA_DEFAULT_MODEL, +} = await import("../../../../src/lib/services/installers/llmlingua.ts"); + +test.after(() => { + process.env.PATH = originalPath; + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + fs.rmSync(FAKE_BIN_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("llmlingua installer: getInstalledVersion returns null when not installed", async () => { + const version = await getInstalledVersion(); + assert.equal(version, null); +}); + +test("llmlingua installer: install npm-installs the real package and updates version_manager DB", async () => { + const result = await install(MOCK_LLMLINGUA_VERSION); + assert.equal(result.installedVersion, MOCK_LLMLINGUA_VERSION); + assert.ok(fs.existsSync(getServerScriptPath())); + + // The generated server script must import the real package, not echo a stub. + const serverSource = fs.readFileSync(getServerScriptPath(), "utf8"); + assert.match(serverSource, /@atjsh\/llmlingua-2/); + assert.doesNotMatch(serverSource, /ratio:\s*0\.5/, "must not hardcode a fake ratio"); + + const row = db.prepare("SELECT * FROM version_manager WHERE tool = 'llmlingua'").get() as + { status?: string; port?: number } | undefined; + assert.ok(row); + assert.equal(row?.status, "stopped"); + assert.equal(row?.port, 20135); +}); + +test("llmlingua installer: getInstalledVersion reads from node_modules/@atjsh/llmlingua-2/package.json", async () => { + const ver = await getInstalledVersion(); + assert.equal(ver, MOCK_LLMLINGUA_VERSION); +}); + +test("llmlingua installer: update calls npm install with latest (idempotent)", async () => { + const result = await update(); + assert.equal(result.installedVersion, MOCK_LLMLINGUA_VERSION); +}); + +test("llmlingua installer: getLatestVersion returns version string from npm view", async () => { + const ver = await getLatestVersion(); + assert.equal(ver, MOCK_LLMLINGUA_VERSION); +}); + +test("llmlingua installer: uninstall removes node_modules and resets DB row", async () => { + await uninstall(); + assert.equal(fs.existsSync(path.join(INSTALL_DIR, "node_modules")), false); + const row = db.prepare("SELECT * FROM version_manager WHERE tool = 'llmlingua'").get() as + { status?: string } | undefined; + assert.equal(row?.status, "not_installed"); + + // Restore for the remaining tests in this file. + await install(MOCK_LLMLINGUA_VERSION); +}); + +test("llmlingua installer: resolveSpawnArgs builds node server spawn arguments", () => { + const spawnArgs = resolveSpawnArgs(LLMLINGUA_DEFAULT_PORT); + assert.equal(spawnArgs.command, process.execPath); + assert.deepEqual(spawnArgs.args, [getServerScriptPath()]); + assert.equal(spawnArgs.env.PORT, String(LLMLINGUA_DEFAULT_PORT)); + assert.equal(spawnArgs.env.LLMLINGUA_MODEL, LLMLINGUA_DEFAULT_MODEL); + assert.equal(spawnArgs.cwd, INSTALL_DIR); +}); + +test("llmlingua installer: resolveSpawnArgs honors LLMLINGUA_MODEL override", () => { + process.env.LLMLINGUA_MODEL = "atjsh/llmlingua-2-js-xlm-roberta-large-meetingbank"; + try { + const spawnArgs = resolveSpawnArgs(LLMLINGUA_DEFAULT_PORT); + assert.equal( + spawnArgs.env.LLMLINGUA_MODEL, + "atjsh/llmlingua-2-js-xlm-roberta-large-meetingbank" + ); + } finally { + delete process.env.LLMLINGUA_MODEL; + } +}); From 185e34d617d35e325699da57cc72333dd6e78215 Mon Sep 17 00:00:00 2001 From: Rouzbeh <78313022+rqzbeh@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:08:20 -0300 Subject: [PATCH 2/3] fix(db): renumber llmlingua seed migration 183 -> 204 183 collides with 183_proxy_logs_rotation_account.sql on release/v3.8.52 (tip highest is 203). Also map the new test:services:int alias in the gate manifest (pre-commit gate-manifest check). Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- config/quality/gate-manifest.json | 5 +++++ .../{183_llmlingua_service.sql => 204_llmlingua_service.sql} | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) rename src/lib/db/migrations/{183_llmlingua_service.sql => 204_llmlingua_service.sql} (91%) diff --git a/config/quality/gate-manifest.json b/config/quality/gate-manifest.json index 0554d4a43d7e..e310b16a4eca 100644 --- a/config/quality/gate-manifest.json +++ b/config/quality/gate-manifest.json @@ -613,6 +613,11 @@ "command": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test tests/unit/security-fase01.test.ts", "disposition": "separately-invoked" }, + { + "name": "test:services:int", + "command": "cross-env RUN_SERVICES_INT=1 DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 \"tests/integration/services/*.test.ts\"", + "disposition": "separately-invoked" + }, { "name": "test:system", "command": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 tests/e2e/system-failover.test.ts", diff --git a/src/lib/db/migrations/183_llmlingua_service.sql b/src/lib/db/migrations/204_llmlingua_service.sql similarity index 91% rename from src/lib/db/migrations/183_llmlingua_service.sql rename to src/lib/db/migrations/204_llmlingua_service.sql index 9429a7fca61f..ef246a6d8a69 100644 --- a/src/lib/db/migrations/183_llmlingua_service.sql +++ b/src/lib/db/migrations/204_llmlingua_service.sql @@ -1,4 +1,4 @@ --- Migration 183: Seed version_manager row for the LLMLingua embedded service. +-- Migration 204: Seed version_manager row for the LLMLingua embedded service. -- -- LLMLingua (npm @atjsh/llmlingua-2, a JS/TS port of Microsoft's LLMLingua-2 -- prompt-compression algorithm) is managed via the ServiceSupervisor From 85fe0679f7fd1fadffac3df3bc64a1c112db92b1 Mon Sep 17 00:00:00 2001 From: Rouzbeh <78313022+rqzbeh@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:33:16 -0300 Subject: [PATCH 3/3] docs(env): document LLMLINGUA_MODEL for the llmlingua sidecar The installer passes LLMLINGUA_MODEL into the sidecar env; check-env-doc-sync requires it in .env.example and ENVIRONMENT.md. Co-authored-by: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> --- .env.example | 2 ++ docs/reference/ENVIRONMENT.md | 1 + 2 files changed, 3 insertions(+) diff --git a/.env.example b/.env.example index a99d054ca492..b19fabf7b1a8 100644 --- a/.env.example +++ b/.env.example @@ -2514,6 +2514,8 @@ APP_LOG_TO_FILE=true # open-sse/services/compression/engines/llmlingua/index.ts # LLMLINGUA_PORT=20135 # LLMLINGUA_BASE_URL=http://127.0.0.1:20135 +# Hugging Face model id the sidecar loads (LLMLingua-2 JS port). +# LLMLINGUA_MODEL=atjsh/llmlingua-2-js-tinybert-meetingbank # ── Local hostnames (Docker networking) ── # Comma-separated additional hostnames treated as "local" for provider routing. diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index a0f77394d179..e4bd6df462c6 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -1527,6 +1527,7 @@ Provider quota endpoints, network tunnels (Tailscale, Ngrok, MITM debug proxy), | `BIFROST_PORT` | `8080` | `src/lib/services/bootstrap.ts` | Port the supervised Bifrost embedded service binds to (`127.0.0.1:`) when OmniRoute manages the Bifrost sidecar lifecycle. Defaults to `8080`. | | `LLMLINGUA_PORT` | `20135` | `src/lib/services/bootstrap.ts` | Port the supervised LLMLingua compression service binds to (`127.0.0.1:`). Defaults to `20135`. | | `LLMLINGUA_BASE_URL` | `http://127.0.0.1:20135` | `open-sse/services/compression/engines/llmlingua/index.ts` | Base URL for the local LLMLingua HTTP compression microservice daemon. Defaults to `http://127.0.0.1:20135`. | +| `LLMLINGUA_MODEL` | `atjsh/llmlingua-2-js-tinybert-meetingbank` | `src/lib/services/installers/llmlingua.ts` | Hugging Face model id the supervised LLMLingua sidecar loads (passed to the child process env). | | `BIFROST_API_KEY` | _(unset)_ | `src/app/api/v1/relay/chat/completions/bifrost/route.ts` | API key for the Bifrost gateway (sent as `Authorization: Bearer ...`). If unset, the route expects the request to carry a valid OmniRoute API key; this key is for gateway-side auth only. | | `BIFROST_STREAMING_ENABLED` | `true` | `src/app/api/v1/relay/chat/completions/bifrost/route.ts` | When true, the Bifrost sidecar route streams responses back via SSE through the gateway rather than the TS streaming executor. Set to `0` to force non-streaming JSON responses through the gateway. | | `BIFROST_TIMEOUT_MS` | `30000` | `src/app/api/v1/relay/chat/completions/bifrost/route.ts` | Per-request timeout when proxying to the Bifrost gateway (ms). On timeout the route returns the TS relay path via the `X-Bifrost-Fallback` header. |