Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -741,6 +741,23 @@ jobs:
path: coverage-shard/*.json
if-no-files-found: error

test-bun-sqlite:
name: Bun SQLite Compatibility
runs-on: ubuntu-latest
timeout-minutes: 10
needs: changes
if: ${{ github.event_name != 'pull_request' || (needs.changes.outputs.code == 'true' && github.event.pull_request.draft == false) }}
steps:
- uses: actions/checkout@v7
with:
persist-credentials: false
- uses: actions/setup-node@v7
with:
node-version: ${{ env.CI_NODE_VERSION }}
cache: npm
- uses: ./.github/actions/npm-ci-retry
- run: npm run test:bun:db

test-vitest:
name: Vitest (MCP / autoCombo / UI components)
# Same dynamic-runner rule as Build (own-origin only; fallback ubuntu-latest).
Expand Down
4 changes: 2 additions & 2 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -482,8 +482,8 @@ list` shows worktrees you didn't create, leave them alone. End every session wit

## Environment

- **Runtime**: Node.js ≥22.0.0 <23 || ≥24.0.0 <27, ES Modules. This is the **only** runtime for the published `omniroute` CLI, the server, and the test suites (`node:test` + vitest) — `engines.node` is authoritative and end users never need Bun.
- **Bun (build/dev script runner only)**: Bun `1.3.10` is pinned as an **exact devDependency** (provisioned through the existing `npm ci` via the lockfile's `@oven/bun-*` platform binaries — no `setup-bun`/ad-hoc install). It is used **only** to execute a small, allow-listed set of TypeScript **gate/generator scripts** (replacing `node --import tsx` for startup speed): the CI checks `check:provider-consistency`, `check:compression-budget`, `check:known-symbols`, and the non-CI `gen:provider-reference`, `bench:compression`. **Do NOT** widen Bun to `npm install`, the build (`build:cli*`), `check:pack-artifact`, the published runtime, or the test runners — those stay on Node. Any new Bun-invoking script must be validated byte-identical against its `node --import tsx` output first. After pulling the lockfile change, run `npm install` so `bun` resolves locally (a stale `node_modules` will fail those 5 scripts with `bun: not found`).
- **Runtime**: Node.js ≥22.0.0 <23 || ≥24.0.0 <27, ES Modules. This is the **only supported** runtime for the published `omniroute` CLI, the server, and the test suites (`node:test` + vitest) — `engines.node` is authoritative and end users never need Bun. A **best-effort `bun:sqlite` compatibility path** exists so a global Bun install (`bun install -g omniroute`) can start without `better-sqlite3` (driver adapter + Bun-aware process spawning); it is **not** a supported runtime — no support guarantees — and every Bun-specific runtime change MUST preserve the Node driver/fallback chain and ship a Bun test (`test:bun:db`) or an explicit reason why the path is Node-only.
- **Bun (build/dev script runner + compatibility smoke only)**: Bun `1.3.14` is pinned as an **exact devDependency** (provisioned through the existing `npm ci` via the lockfile's `@oven/bun-*` platform binaries — no `setup-bun`/ad-hoc install). It is used **only** to execute a small, allow-listed set of TypeScript **gate/generator scripts** (replacing `node --import tsx` for startup speed): the CI checks `check:provider-consistency`, `check:compression-budget`, `check:known-symbols`, and the non-CI `gen:provider-reference`, `bench:compression` — plus the focused `test:bun:db` compatibility smoke suite for the best-effort `bun:sqlite` path. **Do NOT** widen Bun to `npm install`, the build (`build:cli*`), `check:pack-artifact`, the supported published runtime, or the main test runners — those stay on Node. Any new Bun-invoking gate/generator script must be validated byte-identical against its `node --import tsx` output first. After pulling the lockfile change, run `npm install` so `bun` resolves locally (a stale `node_modules` will fail those scripts with `bun: not found`).
- **TypeScript**: 6.0+, target ES2022, module esnext, resolution bundler
- **Path aliases**: `@/*` → `src/`, `@omniroute/open-sse` → `open-sse/`, `@omniroute/open-sse/*` → `open-sse/*`
- **Default port**: 20128 (API + dashboard on same port)
Expand Down
12 changes: 9 additions & 3 deletions bin/cli/commands/serve.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ export async function runServe(opts = {}) {
"Release",
"better_sqlite3.node"
);
if (existsSync(sqliteBinary) && !isNativeBinaryCompatible(sqliteBinary)) {
if (!process.versions.bun && existsSync(sqliteBinary) && !isNativeBinaryCompatible(sqliteBinary)) {
console.error(
"\x1b[31m✖ better-sqlite3 native module is incompatible with this platform.\x1b[0m"
);
Expand Down Expand Up @@ -230,7 +230,10 @@ export async function runServe(opts = {}) {
function runDaemon(serverJs, env, memoryLimit, dashboardPort, apiPort) {
// #5238: skip the explicit CLI --max-old-space-size when the user pinned the
// heap via NODE_OPTIONS (a CLI arg would shadow/override their value).
const server = spawn("node", [...buildNodeHeapArgs(process.env, memoryLimit), serverJs], {
const server = spawn(process.versions.bun ? process.execPath : "node", [
...(process.versions.bun ? [] : buildNodeHeapArgs(process.env, memoryLimit)),
serverJs,
], {
cwd: APP_DIR,
env,
stdio: "ignore",
Expand All @@ -246,7 +249,10 @@ function runDaemon(serverJs, env, memoryLimit, dashboardPort, apiPort) {
function runWithoutRecovery(serverJs, env, memoryLimit, dashboardPort, apiPort, noOpen, startedAt) {
// #5238: skip the explicit CLI --max-old-space-size when the user pinned the
// heap via NODE_OPTIONS (a CLI arg would shadow/override their value).
const server = spawn("node", [...buildNodeHeapArgs(process.env, memoryLimit), serverJs], {
const server = spawn(process.versions.bun ? process.execPath : "node", [
...(process.versions.bun ? [] : buildNodeHeapArgs(process.env, memoryLimit)),
serverJs,
], {
cwd: APP_DIR,
env,
stdio: "pipe",
Expand Down
5 changes: 4 additions & 1 deletion bin/cli/runtime/processSupervisor.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,10 @@ export class ServerSupervisor {
// silently, so a boot that never becomes ready looked like a dead hang with zero
// output even at APP_LOG_LEVEL=debug. Pipe stdout too and buffer it alongside
// stderr so a readiness timeout can surface what the child actually printed.
this.child = spawn("node", [...heapArgs, this.serverPath], {
this.child = spawn(process.versions.bun ? process.execPath : "node", [
...(process.versions.bun ? [] : heapArgs),
this.serverPath,
], {
cwd: dirname(this.serverPath),
env: this.env,
stdio: showLog ? "inherit" : ["ignore", "pipe", "pipe"],
Expand Down
81 changes: 77 additions & 4 deletions bin/cli/sqlite.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,68 @@ import { resolveDataDir, resolveStoragePath } from "./data-dir.mjs";
import { ensureProviderSchema } from "./provider-store.mjs";
import { ensureSettingsSchema, hashManagementPassword, updateSettings } from "./settings-store.mjs";

async function loadBetterSqlite() {
async function loadSqlite() {
if (process.versions.bun) {
return (await import("bun:sqlite")).Database;
}
try {
return (await import("better-sqlite3")).default;
} catch {
throw new Error("better-sqlite3 is not installed. Run npm install before using setup.");
}
}

function openBunSqlite(Database, dbPath, options) {
const raw = new Database(dbPath, options);
const prepare = (sql) => {
const statement = raw.query(sql);
return {
run: (...params) => statement.run(...normalizeBunSqliteParams(params)),
get: (...params) => statement.get(...normalizeBunSqliteParams(params)),
all: (...params) => statement.all(...normalizeBunSqliteParams(params)),
};
};
return {
prepare,
query: (sql) => raw.query(sql),
exec: (sql) => raw.exec(sql),
transaction: (fn) => raw.transaction(fn),
close: () => raw.close(),
serialize: () => raw.serialize(),
pragma: (pragmaStr, pragmaOptions) => {
const statement = raw.query(`PRAGMA ${pragmaStr}`);
if (pragmaOptions?.simple) {
const row = statement.get();
return row ? (Object.values(row)[0] ?? null) : null;
}
return statement.all();
},
};
}

export function normalizeBunSqliteParams(params) {
if (
params.length !== 1 ||
params[0] === null ||
typeof params[0] !== "object" ||
Array.isArray(params[0]) ||
params[0] instanceof Uint8Array ||
(typeof Buffer !== "undefined" && Buffer.isBuffer(params[0]))
) {
return params;
}
const expanded = {};
for (const [key, value] of Object.entries(params[0])) {
if (/^[:@$]/.test(key)) expanded[key] = value;
else {
expanded[`@${key}`] = value;
expanded[`:${key}`] = value;
expanded[`$${key}`] = value;
}
}
return [expanded];
}

export function createSqliteNativeError(error) {
const message = error instanceof Error ? error.message : String(error);
if (message.includes("NODE_MODULE_VERSION") || message.includes("ERR_DLOPEN_FAILED")) {
Expand All @@ -37,9 +91,19 @@ export function createSqliteNativeError(error) {
}

async function openSqliteDatabase(dbPath, options = {}) {
const Database = await loadBetterSqlite();
const Database = await loadSqlite();
if (process.versions.bun) {
if (options.fileMustExist && !fs.existsSync(dbPath)) {
throw new Error(`SQLite file does not exist: ${dbPath}`);
}
options = options.readonly
? { readonly: true }
: { readwrite: true, create: options.fileMustExist !== true };
}
try {
return new Database(dbPath, options);
return process.versions.bun
? openBunSqlite(Database, dbPath, options)
: new Database(dbPath, options);
} catch (error) {
throw createSqliteNativeError(error);
}
Expand Down Expand Up @@ -71,7 +135,16 @@ export async function withReadonlySqlite(dbPath, callback) {
export async function backupSqliteFile(sourcePath, destPath) {
const db = await openSqliteDatabase(sourcePath, { readonly: true });
try {
await db.backup(destPath);
if (typeof db.backup === "function") {
await db.backup(destPath);
} else if (sourcePath === ":memory:" && typeof db.serialize === "function") {
fs.writeFileSync(destPath, Buffer.from(db.serialize()));
} else {
try {
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
} catch {}
fs.copyFileSync(sourcePath, destPath);
}
Comment on lines +138 to +147

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

Using db.serialize() for physical file-backed databases is highly inefficient and poses a significant Out Of Memory (OOM) risk for large databases because it loads the entire database into memory as a Uint8Array. Since sourcePath is a physical file, we should perform a standard checkpoint and use fs.copyFileSync directly.

    if (typeof db.backup === "function") {
      await db.backup(destPath);
    } else {
      try {
        db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
      } catch {}
      fs.copyFileSync(sourcePath, destPath);
    }

} finally {
db.close();
}
Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@
"test:unit:shard": "concurrently --kill-others-on-fail -n s1,s2 \"npm:test:unit:shard:1\" \"npm:test:unit:shard:2\"",
"test:unit:shard:1": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=10 --test-shard=1/2 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,combo,compression,correctness,cors,db,db-adapters,docs,gamification,guardrails,lib,mcp,memory,runtime,security,services,settings,shared,ui,usage}/**/*.test.ts\" \"tests/unit/**/*.test.mjs\" && cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=10 --test-shard=1/2 \"tests/unit/dashboard/**/*.test.ts\" && cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 --test-shard=1/2 \"tests/unit/serial/**/*.test.ts\"",
"test:unit:shard:2": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=10 --test-shard=2/2 tests/unit/*.test.ts \"tests/unit/{api,auth,authz,build,cli,cli-helper,combo,compression,correctness,cors,db,db-adapters,docs,gamification,guardrails,lib,mcp,memory,runtime,security,services,settings,shared,ui,usage}/**/*.test.ts\" \"tests/unit/**/*.test.mjs\" && cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=10 --test-shard=2/2 \"tests/unit/dashboard/**/*.test.ts\" && cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --max-old-space-size=8192 --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test --test-force-exit --test-concurrency=1 --test-shard=2/2 \"tests/unit/serial/**/*.test.ts\"",
"test:bun:db": "bun test tests/unit/db-adapters/bunSqliteAdapter.test.ts tests/unit/db-adapters/driverFactory.test.ts tests/unit/db-adapters/cliSqlite.test.mjs",
"test:plan3": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test tests/unit/plan3-p0.test.ts",
"test:fixes": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test tests/unit/fixes-p1.test.ts",
"test:security": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx/esm --import ./open-sse/utils/setupPolyfill.ts --import ./tests/_setup/isolateDataDir.ts --test tests/unit/security-fase01.test.ts",
Expand Down
151 changes: 151 additions & 0 deletions src/lib/db/adapters/bunSqliteAdapter.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
import fs from "node:fs";
import type { PreparedStatement, RunResult, SqliteAdapter } from "./types";

/**
* The Bun runtime already ships a SQLite driver. Keep this adapter deliberately
* small so the rest of the application can use the same driver contract as
* better-sqlite3, node:sqlite, and sql.js.
*/
export interface BunSqliteDatabaseLike {
query(sql: string): {
run(...params: unknown[]): { changes: number | bigint; lastInsertRowid: number | bigint };
get(...params: unknown[]): unknown;
all(...params: unknown[]): unknown[];
};
exec(sql: string): void;
serialize?(): Uint8Array;
transaction<T>(fn: (...args: unknown[]) => T): {
(...args: unknown[]): T;
immediate?: (...args: unknown[]) => T;
};
close(): void;
}

function normalizeRunResult(result: {
changes?: number | bigint;
lastInsertRowid?: number | bigint;
}): RunResult {
return {
changes: Number(result.changes ?? 0),
lastInsertRowid: Number(result.lastInsertRowid ?? 0),
};
}

function normalizeParams(params: unknown[]): unknown[] {
if (params.length !== 1) return params;
const [first] = params;
if (
first === null ||
typeof first !== "object" ||
Array.isArray(first) ||
first instanceof Uint8Array ||
(typeof Buffer !== "undefined" && Buffer.isBuffer(first))
) {
return params;
}

// better-sqlite3 callers use bare object keys for @name, :name, and $name.
// Bun requires the sigil to match the SQL placeholder, so provide all three
// aliases just as the sql.js adapter does.
const expanded: Record<string, unknown> = {};
for (const [key, value] of Object.entries(first as Record<string, unknown>)) {
if (/^[:@$]/.test(key)) {
expanded[key] = value;
} else {
expanded[`@${key}`] = value;
expanded[`:${key}`] = value;
expanded[`$${key}`] = value;
}
}
return [expanded];
}

export function createBunSqliteAdapter(db: BunSqliteDatabaseLike, filePath: string): SqliteAdapter {
let isOpen = true;

return {
driver: "bun:sqlite",

get open() {
return isOpen;
},

get name() {
return filePath;
},

prepare(sql: string): PreparedStatement {
const statement = db.query(sql);
return {
run(...params: unknown[]): RunResult {
return normalizeRunResult(statement.run(...normalizeParams(params)));
},
get(...params: unknown[]): unknown {
return statement.get(...normalizeParams(params));
},
all(...params: unknown[]): unknown[] {
return statement.all(...normalizeParams(params));
},
};
},

exec(sql: string): void {
db.exec(sql);
},

pragma(pragmaStr: string, options?: { simple?: boolean }): unknown {
const statement = db.query(`PRAGMA ${pragmaStr}`);
if (options?.simple) {
const row = statement.get() as Record<string, unknown> | undefined;
return row ? (Object.values(row)[0] ?? null) : null;
}
return statement.all();
},

transaction<T>(fn: (...args: unknown[]) => T): (...args: unknown[]) => T {
return db.transaction(fn);
},

immediate(fn: () => void): void {
const transaction = db.transaction(fn);
if (typeof transaction.immediate === "function") {
transaction.immediate();
return;
}
db.exec("BEGIN IMMEDIATE");
try {
fn();
db.exec("COMMIT");
} catch (error) {
try {
db.exec("ROLLBACK");
} catch {}
throw error;
}
},

async backup(destination: string): Promise<void> {
if (filePath === ":memory:") return;
try {
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
} catch {}
fs.copyFileSync(filePath, destination);
},
Comment on lines +127 to +133

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

Using db.serialize() for physical file-backed databases is highly inefficient and poses a significant Out Of Memory (OOM) risk for large databases because it loads the entire database into memory as a Uint8Array. Since filePath is already verified to not be ":memory:", we should perform a standard checkpoint and use fs.copyFileSync directly.

    async backup(destination: string): Promise<void> {
      if (filePath === ":memory:") return;
      try {
        db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
      } catch {}
      fs.copyFileSync(filePath, destination);
    },


checkpoint(mode = "TRUNCATE"): void {
try {
db.exec(`PRAGMA wal_checkpoint(${mode})`);
} catch {}
},

close(): void {
if (!isOpen) return;
db.close();
isOpen = false;
},

get raw() {
return db;
},
};
}
29 changes: 28 additions & 1 deletion src/lib/db/adapters/driverFactory.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { createRequire } from "node:module";
import { existsSync } from "node:fs";
import { createBetterSqliteAdapter } from "./betterSqliteAdapter";
import { createBunSqliteAdapter, type BunSqliteDatabaseLike } from "./bunSqliteAdapter";
import {
createNodeSqliteAdapterFromDatabase,
type NodeSqliteDatabaseLike,
Expand Down Expand Up @@ -70,6 +72,31 @@ export function tryOpenSync(
filePath: string,
options?: Record<string, unknown>
): SqliteAdapter | null {
// Bun ships a supported SQLite implementation. Prefer it over the native
// Node addon, which Bun intentionally skips because its ABI is incompatible.
if (process.versions.bun) {
try {
const { Database } = _require("bun:sqlite") as {
Database: new (p: string, options?: Record<string, unknown>) => BunSqliteDatabaseLike;
};
if (
options?.fileMustExist === true &&
filePath !== ":memory:" &&
!existsSync(filePath)
) {
throw new Error(`SQLite file does not exist: ${filePath}`);
}
const db = new Database(filePath, {
...(options?.readonly === true
? { readonly: true }
: { readwrite: true, create: options?.fileMustExist !== true }),
});
return createBunSqliteAdapter(db, filePath);
} catch (err) {
logSwallowedDriverError("bun:sqlite", err);
}
}

// better-sqlite3: rápido, nativo — skip em Bun
if (!process.versions.bun) {
try {
Expand Down Expand Up @@ -157,7 +184,7 @@ export function getSqlJsAdapter(filePath: string): SqliteAdapter | null {

/**
* Factory assíncrona completa: tenta todos os drivers em cascata.
* Ordem: better-sqlite3 → node:sqlite → sql.js
* Ordem: bun:sqlite → better-sqlite3 → node:sqlite → sql.js
*/
export async function openDatabaseAsync(
filePath: string,
Expand Down
Loading