diff --git a/bin/cli/commands/sync-import.mjs b/bin/cli/commands/sync-import.mjs new file mode 100644 index 000000000000..c635b7f6fefd --- /dev/null +++ b/bin/cli/commands/sync-import.mjs @@ -0,0 +1,115 @@ +import { readFileSync } from "node:fs"; +import { emit } from "../output.mjs"; + +// Local side of `omniroute sync import` (the upload lives in sync.mjs): reading and +// sanity-checking the backup, and the dry run, which must never reach the server. + +const SQLITE_MAGIC = Buffer.from("SQLite format 3\0", "latin1"); +const MIN_SQLITE_BYTES = 4096; +// Kept in sync with REQUIRED_TABLES in src/app/api/db-backups/import/route.ts. +const IMPORT_REQUIRED_TABLES = ["provider_connections", "provider_nodes", "combos", "api_keys"]; + +function importFail(message, exitCode = 2) { + process.stderr.write(`Error: sync import: ${message}\n`); + return exitCode; +} + +/** Cheap local check run before any upload: is this file a SQLite database at all? */ +function checkSqliteShape(buf) { + if ( + buf.length < SQLITE_MAGIC.length || + !buf.subarray(0, SQLITE_MAGIC.length).equals(SQLITE_MAGIC) + ) { + return ( + "the file is not a SQLite database. `sync import` restores a .sqlite backup " + + "(e.g. from `omniroute backup` or the dashboard); the JSON written by `sync bundle` " + + "cannot be imported here." + ); + } + if (buf.length < MIN_SQLITE_BYTES) return "the file is too small to be a valid SQLite database."; + return null; +} + +/** + * Open the backup read-only and run the checks the server would (integrity + required + * tables), returning what an import would load. `skipped` when no local SQLite driver is + * available — the dry run then rests on the header check alone. + */ +async function inspectBackupLocally(filePath) { + let withReadonlySqlite; + try { + ({ withReadonlySqlite } = await import("../sqlite.mjs")); + } catch (err) { + return { skipped: err?.message || "SQLite driver unavailable" }; + } + try { + return await withReadonlySqlite(filePath, (db) => { + const integrity = db.pragma("integrity_check", { simple: true }); + if (integrity !== "ok") { + return { error: "database integrity check failed; the file may be corrupted." }; + } + const tables = db + .prepare("SELECT name FROM sqlite_master WHERE type='table'") + .all() + .map((r) => r.name); + const missing = IMPORT_REQUIRED_TABLES.filter((t) => !tables.includes(t)); + if (missing.length > 0) { + return { error: `not an OmniRoute database; missing tables: ${missing.join(", ")}` }; + } + const count = (table) => db.prepare(`SELECT COUNT(*) AS cnt FROM ${table}`).get()?.cnt ?? 0; + const schemaVersion = tables.includes("_omniroute_migrations") + ? (db + .prepare( + "SELECT version FROM _omniroute_migrations ORDER BY CAST(version AS INTEGER) DESC, version DESC LIMIT 1" + ) + .get()?.version ?? null) + : null; + return { + connectionCount: count("provider_connections"), + nodeCount: count("provider_nodes"), + comboCount: count("combos"), + apiKeyCount: count("api_keys"), + schemaVersion, + }; + }); + } catch (err) { + return { error: `cannot open the file as SQLite: ${err?.message || "unknown error"}` }; + } +} + +/** + * Validate the backup locally and print what an import would load. NEVER contacts the + * server: an older server ignores `dryRun` and would replace the live database. + */ +export async function runImportDryRun(filePath, buf, globalOpts) { + const report = await inspectBackupLocally(filePath); + if (report.error) return importFail(report.error); + const summary = { + dryRun: true, + valid: true, + file: filePath, + sizeBytes: buf.length, + ...(report.skipped + ? { deepChecks: `skipped (${report.skipped}); only the SQLite header was verified` } + : report), + note: "No request was sent to the server; the live database was not modified.", + }; + emit(summary, globalOpts); + return 0; +} + +/** + * Read the backup and run the cheap header check every import starts with. Returns + * `{ buf }`, or `{ exitCode }` after printing why the file cannot be imported. + */ +export function readImportFile(filePath) { + let buf; + try { + buf = readFileSync(filePath); + } catch (err) { + return { exitCode: importFail(`cannot read ${filePath}: ${err?.code || err?.message}`, 1) }; + } + const shapeError = checkSqliteShape(buf); + if (shapeError) return { exitCode: importFail(shapeError) }; + return { buf }; +} diff --git a/bin/cli/commands/sync.mjs b/bin/cli/commands/sync.mjs index 7e457d1e1c8e..9da4f1827bdc 100644 --- a/bin/cli/commands/sync.mjs +++ b/bin/cli/commands/sync.mjs @@ -1,9 +1,10 @@ -import { readFileSync, writeFileSync } from "node:fs"; +import { writeFileSync } from "node:fs"; import { basename } from "node:path"; import { createInterface } from "node:readline"; -import { apiFetch, getBaseUrl, isRouteUnavailableStatus } from "../api.mjs"; +import { apiFetch, isRouteUnavailableStatus } from "../api.mjs"; import { emit } from "../output.mjs"; import { t } from "../i18n.mjs"; +import { readImportFile, runImportDryRun } from "./sync-import.mjs"; function fmtTs(v) { if (!v) return "-"; @@ -20,12 +21,6 @@ async function confirm(q) { }); } -function authHeaders(opts) { - const h = { accept: "application/json" }; - if (opts.apiKey) h["Authorization"] = `Bearer ${opts.apiKey}`; - return h; -} - const syncTokenSchema = [ { key: "id", header: "Token ID", width: 16 }, { key: "name", header: "Name", width: 24 }, @@ -242,6 +237,52 @@ export async function runSyncInitialize(globalOpts = {}) { return 0; } +const IMPORT_PATH = "/api/db-backups/import"; +// Uploading + validating + swapping a large database can outlast the 30 s default timeout. +const IMPORT_TIMEOUT_MS = 300_000; + +function importFailureHint(status) { + if (status === 401 || status === 403) { + return ( + "importing replaces the whole database: run this on the server host (local CLI " + + "credential), sign in to the dashboard, or pass an API key with the 'manage' scope." + ); + } + return routeUnavailableHint(status, `POST ${IMPORT_PATH}`); +} + +/** + * `omniroute sync import [--dry-run]` — replace the server's database with a `.sqlite` + * backup via `POST /api/db-backups/import`. With `--dry-run` the file is only validated + * locally and the server is never contacted: an older server ignores `dryRun` and would + * replace the live database. + */ +export async function runSyncImport(filePath, opts = {}, globalOpts = {}) { + const { buf, exitCode: readExit } = readImportFile(filePath); + if (!buf) return readExit; + if (opts.dryRun) return runImportDryRun(filePath, buf, globalOpts); + + const name = basename(filePath); + const query = name.endsWith(".sqlite") ? `?filename=${encodeURIComponent(name)}` : ""; + const { res, exitCode } = await sendSyncRequest( + "sync import", + `${IMPORT_PATH}${query}`, + globalOpts, + { + method: "POST", + headers: { "content-type": "application/octet-stream" }, + body: buf, + timeout: IMPORT_TIMEOUT_MS, + // A replace is not something to replay blindly after a gateway error. + retry: false, + } + ); + if (!res) return exitCode; + if (!res.ok) return reportFailure("sync import", res, importFailureHint(res.status)); + emit(await res.json(), globalOpts); + return 0; +} + export function registerSync(program) { const sync = program.command("sync").description(t("sync.description")); @@ -276,24 +317,12 @@ export function registerSync(program) { .description(t("sync.import.description")) .option("--dry-run", t("sync.import.dryRun")) .action(async (bundlePath, opts, cmd) => { - const globalOpts = cmd.optsWithGlobals(); - const body = readFileSync(bundlePath); - const res = await fetch( - `${getBaseUrl(globalOpts)}/api/db-backups/import?dryRun=${opts.dryRun ? "true" : "false"}`, - { - method: "POST", - headers: { - ...authHeaders(globalOpts), - "Content-Type": "application/octet-stream", - }, - body, - } + const exitCode = await runSyncImport( + bundlePath, + { dryRun: !!opts.dryRun }, + cmd.optsWithGlobals() ); - if (!res.ok) { - process.stderr.write(`HTTP ${res.status}\n`); - process.exit(1); - } - emit(await res.json(), globalOpts); + if (exitCode !== 0) process.exit(exitCode); }); sync.command("initialize").action(async (opts, cmd) => { diff --git a/bin/cli/locales/en.json b/bin/cli/locales/en.json index dc60cc64fe64..0e841392b671 100644 --- a/bin/cli/locales/en.json +++ b/bin/cli/locales/en.json @@ -1179,8 +1179,8 @@ "token": "Sync token (from `sync tokens create`); defaults to OMNIROUTE_SYNC_TOKEN" }, "import": { - "description": "Import a bundle file", - "dryRun": "Validate without applying" + "description": "Replace the server database with a .sqlite backup", + "dryRun": "Validate the backup locally; never contacts the server" }, "tokens": { "description": "Manage sync tokens", diff --git a/bin/cli/locales/pt-BR.json b/bin/cli/locales/pt-BR.json index 926a6cd64761..1c9bcac8c80e 100644 --- a/bin/cli/locales/pt-BR.json +++ b/bin/cli/locales/pt-BR.json @@ -1179,8 +1179,8 @@ "token": "Token de sincronização (de `sync tokens create`); padrão: OMNIROUTE_SYNC_TOKEN" }, "import": { - "description": "Importar arquivo bundle", - "dryRun": "Validar sem aplicar" + "description": "Substituir o banco do servidor por um backup .sqlite", + "dryRun": "Validar o backup localmente; nunca contata o servidor" }, "tokens": { "description": "Gerenciar tokens de sincronização", diff --git a/changelog.d/fixes/0000-cli-sync-import-dry-run.md b/changelog.d/fixes/0000-cli-sync-import-dry-run.md new file mode 100644 index 000000000000..65fa551d9d12 --- /dev/null +++ b/changelog.d/fixes/0000-cli-sync-import-dry-run.md @@ -0,0 +1 @@ +- **fix(cli):** `omniroute sync import --dry-run` no longer replaces the live database. The command sent `?dryRun=true` to `POST /api/db-backups/import`, a flag the route never read, so a "dry run" overwrote the operator's data. The route now honours `dryRun` (query string or multipart field): it runs the same integrity and table checks and returns the row counts and schema version a real import would load, without the pre-import backup and without touching the live database. The CLI goes further and never contacts the server in dry-run mode (an older server would still ignore the flag): it validates the backup locally and prints the summary. A real import now goes through the local CLI credential, which the route accepts next to the dashboard session and `manage`-scoped API keys; JSON files such as the output of `sync bundle` are refused before upload, and 401/403/400 end in a readable error instead of a bare status. diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 50c312ce01f9..e99006fefadf 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -9963,9 +9963,37 @@ paths: tags: - Db backups summary: "POST db backups › import" + description: >- + Replaces the live database with an uploaded `.sqlite` backup (multipart `file` field, + or the raw bytes with `?filename=`). The upload is checked for SQLite integrity and the + required OmniRoute tables first. Accepts the dashboard session, a `manage`-scoped API + key, or the local CLI credential. + parameters: + - name: dryRun + in: query + description: >- + `true` validates the upload and returns what would be imported (row counts, schema + version) without the pre-import backup and without touching the live database. A + multipart `dryRun` form field is honoured too. + required: false + schema: + type: string + enum: ["true", "false", "1", "0"] + - name: filename + in: query + description: Name of the uploaded file for raw-byte uploads; must end in `.sqlite`. + required: false + schema: + type: string responses: "200": - description: OK + description: >- + Imported (`imported: true`), or, for a dry run, validated (`dryRun: true, + valid: true, imported: false`) with the counts the upload would load. + "400": + description: Not a valid OmniRoute SQLite backup (type, size, integrity or tables). + "401": + description: No accepted credential. /api/db/health: get: tags: diff --git a/docs/reference/API_REFERENCE.md b/docs/reference/API_REFERENCE.md index b48b1fdc02e4..4846f54e3f50 100644 --- a/docs/reference/API_REFERENCE.md +++ b/docs/reference/API_REFERENCE.md @@ -1108,14 +1108,14 @@ curl -X PATCH "$OMNIROUTE_URL/api/keys/$KEY_ID" \ ### Backup & Export/Import -| Endpoint | Method | Description | -| --------------------------- | ------ | --------------------------------------- | -| `/api/db-backups` | GET | List available backups | -| `/api/db-backups` | PUT | Create a manual backup | -| `/api/db-backups` | POST | Restore from a specific backup | -| `/api/db-backups/export` | GET | Download database as .sqlite file | -| `/api/db-backups/import` | POST | Upload .sqlite file to replace database | -| `/api/db-backups/exportAll` | GET | Download full backup as .tar.gz archive | +| Endpoint | Method | Description | +| --------------------------- | ------ | ----------------------------------------------------------------------- | +| `/api/db-backups` | GET | List available backups | +| `/api/db-backups` | PUT | Create a manual backup | +| `/api/db-backups` | POST | Restore from a specific backup | +| `/api/db-backups/export` | GET | Download database as .sqlite file | +| `/api/db-backups/import` | POST | Upload .sqlite file to replace database (`?dryRun=true` validates only) | +| `/api/db-backups/exportAll` | GET | Download full backup as .tar.gz archive | `/api/db-backups/export`, `/api/db-backups/exportAll` and `/api/settings/export-json` hand out every stored credential, so they require the dashboard session or the local CLI (or login diff --git a/src/app/api/db-backups/import/route.ts b/src/app/api/db-backups/import/route.ts index 8186abe23bf1..bfaafe9ca77c 100644 --- a/src/app/api/db-backups/import/route.ts +++ b/src/app/api/db-backups/import/route.ts @@ -9,8 +9,10 @@ import { backupDbFile, getTableNamesFromAdapter, countImportedRows, + summarizeImportCandidate, unlinkFileWithRetry, } from "@/lib/db/backup"; +import { isLocalCliCredential } from "@/lib/api/operatorCredential"; import { isAuthRequired, isAuthenticated } from "@/shared/utils/apiAuth"; import { getSettings } from "@/lib/db/settings"; import { setSystemPromptConfig } from "@omniroute/open-sse/services/systemPrompt.ts"; @@ -29,9 +31,7 @@ const MAX_UPLOAD_MB_CEILING = 4096; * via `OMNIROUTE_DB_IMPORT_MAX_MB`. Invalid / out-of-range values fall back to the 100 MB * default and are clamped to a 4 GB ceiling. */ -export function resolveMaxUploadSizeBytes( - env: NodeJS.ProcessEnv = process.env -): number { +export function resolveMaxUploadSizeBytes(env: NodeJS.ProcessEnv = process.env): number { const raw = env.OMNIROUTE_DB_IMPORT_MAX_MB; const parsed = raw === undefined ? NaN : Number(raw); const mb = @@ -44,172 +44,208 @@ export function resolveMaxUploadSizeBytes( // Required tables that must exist in a valid OmniRoute database const REQUIRED_TABLES = ["provider_connections", "provider_nodes", "combos", "api_keys"]; +const DRY_RUN_TRUE = /^(1|true|yes)$/i; + +/** `dryRun` is read from the query string or, for multipart uploads, a form field. */ +export function isDryRunValue(value: unknown): boolean { + return typeof value === "string" && DRY_RUN_TRUE.test(value.trim()); +} + +function badRequest(error: string): Response { + return NextResponse.json({ error }, { status: 400 }); +} + /** - * POST /api/db-backups/import — Upload a .sqlite file to replace the current database. - * - * Accepts multipart/form-data with a single "file" field containing the .sqlite backup. - * Validates integrity, schema, and required tables before replacing the active database. - * - * 🔒 Auth-guarded: requires JWT cookie or Bearer API key (finding #258-3). + * Dashboard session or manage-scoped API key (as before), plus the machine-bound local CLI + * credential so `omniroute sync import` on the server host works without an API key. */ -export async function POST(request: Request) { - if (await isAuthRequired(request)) { - if (!(await isAuthenticated(request))) { - return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); - } +async function isImportAuthorized(request: Request): Promise { + if (!(await isAuthRequired(request))) return true; + if (await isAuthenticated(request)) return true; + return isLocalCliCredential(request); +} + +type Upload = { fileBuffer: Buffer; fileName: string; dryRun: boolean }; + +async function readUpload(request: Request): Promise { + const url = new URL(request.url); + const queryDryRun = isDryRunValue(url.searchParams.get("dryRun")); + const contentType = request.headers.get("content-type") || ""; + + if (contentType.includes("multipart/form-data")) { + const formData = await request.formData(); + const file = formData.get("file") as File | null; + if (!file) return badRequest("No file provided. Upload a .sqlite file."); + return { + fileBuffer: Buffer.from(await file.arrayBuffer()), + fileName: file.name, + dryRun: queryDryRun || isDryRunValue(formData.get("dryRun")), + }; } - let tmpPath: string | null = null; + // Direct binary transfer to bypass Reverse Proxy / Next.js FormData parsing errors under chunked encoding (Bug #770) + const buffer = await request.arrayBuffer(); + if (!buffer || buffer.byteLength === 0) return badRequest("No file content provided."); + return { + fileBuffer: Buffer.from(buffer), + fileName: url.searchParams.get("filename") || "import.sqlite", + dryRun: queryDryRun, + }; +} + +function validateUploadShape(fileName: string, fileSize: number): Response | null { + if (!fileName.endsWith(".sqlite")) { + return badRequest("Invalid file type. Only .sqlite files are accepted."); + } + const maxUploadSize = resolveMaxUploadSizeBytes(); + if (fileSize > maxUploadSize) { + return badRequest( + `File too large. Maximum allowed size is ${maxUploadSize / (1024 * 1024)} MB. ` + + `Set OMNIROUTE_DB_IMPORT_MAX_MB to raise it, or VACUUM the database before exporting.` + ); + } + if (fileSize < 4096) return badRequest("File too small to be a valid SQLite database."); + return null; +} + +type CandidateSummary = ReturnType; + +/** + * Open the uploaded file read-only and validate it: SQLite integrity and the required + * OmniRoute tables. Returns what a real import would load, or the 400 to send. + * + * Uses the resilient driver factory (better-sqlite3 → node:sqlite → sql.js) rather than a + * direct `better-sqlite3` import: in the packaged Electron app that native module is absent + * from the standalone server's node_modules (#3025). + */ +async function inspectCandidate(tmpPath: string): Promise { + let testDb: SqliteAdapter | null = null; try { - let fileBuffer: Buffer | null = null; - let fileName = ""; - const contentType = request.headers.get("content-type") || ""; - - if (contentType.includes("multipart/form-data")) { - const formData = await request.formData(); - const file = formData.get("file") as File | null; - if (!file) { - return NextResponse.json( - { error: "No file provided. Upload a .sqlite file." }, - { status: 400 } - ); - } - fileName = file.name; - fileBuffer = Buffer.from(await file.arrayBuffer()); - } else { - // Direct binary transfer to bypass Reverse Proxy / Next.js FormData parsing errors under chunked encoding (Bug #770) - const buffer = await request.arrayBuffer(); - if (!buffer || buffer.byteLength === 0) { - return NextResponse.json({ error: "No file content provided." }, { status: 400 }); - } - fileBuffer = Buffer.from(buffer); - const url = new URL(request.url); - fileName = url.searchParams.get("filename") || "import.sqlite"; + testDb = await openDatabaseAsync(tmpPath, { readonly: true }); + const result = testDb.pragma("integrity_check") as Array<{ integrity_check?: string }>; + if (result[0]?.integrity_check !== "ok") { + return badRequest("Database integrity check failed. The file may be corrupted."); } - // Validate filename extension - if (!fileName.endsWith(".sqlite")) { - return NextResponse.json( - { error: "Invalid file type. Only .sqlite files are accepted." }, - { status: 400 } - ); + const tables = getTableNamesFromAdapter(testDb); + const missingTables = REQUIRED_TABLES.filter((t) => !tables.includes(t)); + if (missingTables.length > 0) { + return badRequest(`Invalid OmniRoute database. Missing tables: ${missingTables.join(", ")}`); } + return summarizeImportCandidate(testDb, tables); + } catch (e) { + return badRequest(`Invalid database file: ${sanitizeErrorMessage(e)}`); + } finally { + if (testDb) testDb.close(); + } +} - // Validate file size - const maxUploadSize = resolveMaxUploadSizeBytes(); - const fileSize = fileBuffer.length; - if (fileSize > maxUploadSize) { - return NextResponse.json( - { - error: - `File too large. Maximum allowed size is ${maxUploadSize / (1024 * 1024)} MB. ` + - `Set OMNIROUTE_DB_IMPORT_MAX_MB to raise it, or VACUUM the database before exporting.`, - }, - { status: 400 } - ); - } +/** Swap the validated upload in for the live database and report what is now loaded. */ +async function replaceLiveDatabase(tmpPath: string, fileName: string): Promise { + // Create pre-import backup + backupDbFile("pre-import"); - if (fileSize < 4096) { - return NextResponse.json( - { error: "File too small to be a valid SQLite database." }, - { status: 400 } - ); - } + // Close and reset current DB connection + resetDbInstance(); - // Write uploaded file to temp location - tmpPath = path.join(os.tmpdir(), `omniroute-import-${Date.now()}.sqlite`); - fs.writeFileSync(tmpPath, fileBuffer!); - - // Validate SQLite integrity. - // Use the resilient driver factory (better-sqlite3 → node:sqlite → sql.js) rather than - // a direct `better-sqlite3` import: in the packaged Electron app that native module is - // absent from the standalone server's node_modules, so a hard import crashes the route - // with "Cannot find module 'better-sqlite3'" even though node:sqlite is available (#3025). - let testDb: SqliteAdapter | null = null; - try { - testDb = await openDatabaseAsync(tmpPath, { readonly: true }); - const result = testDb.pragma("integrity_check") as any[]; - if (result[0]?.integrity_check !== "ok") { - return NextResponse.json( - { error: "Database integrity check failed. The file may be corrupted." }, - { status: 400 } - ); - } + // Remove main file and WAL sidecars + const sqliteFilesToReplace = [ + SQLITE_FILE, + `${SQLITE_FILE}-wal`, + `${SQLITE_FILE}-shm`, + `${SQLITE_FILE}-journal`, + ]; + // Delete with EBUSY/EPERM retry: after resetDbInstance() the OS may still + // hold the SQLite file handle for a moment (Windows mmap / antivirus), so a + // plain unlink races to EBUSY (#5406). Mirror the restore path's helper. + for (const filePath of sqliteFilesToReplace) { + if (!filePath) continue; + await unlinkFileWithRetry(filePath); + } - // Validate required tables exist - const tables = getTableNamesFromAdapter(testDb); - - const missingTables = REQUIRED_TABLES.filter((t) => !tables.includes(t)); - if (missingTables.length > 0) { - return NextResponse.json( - { - error: `Invalid OmniRoute database. Missing tables: ${missingTables.join(", ")}`, - }, - { status: 400 } - ); - } + // Copy imported file over current DB + fs.copyFileSync(tmpPath, SQLITE_FILE!); - testDb.close(); - testDb = null; - } catch (e) { - if (testDb) testDb.close(); - return NextResponse.json( - { error: `Invalid database file: ${sanitizeErrorMessage(e)}` }, - { status: 400 } - ); - } + // Reopen and verify + getDbInstance(); + const { connCount, nodeCount, comboCount, keyCount } = countImportedRows(); + + console.log( + `[DB] Imported database from upload: ${connCount} connections, ${nodeCount} nodes, ${comboCount} combos, ${keyCount} API keys` + ); - // Create pre-import backup - backupDbFile("pre-import"); - - // Close and reset current DB connection - resetDbInstance(); - - // Remove main file and WAL sidecars - const sqliteFilesToReplace = [ - SQLITE_FILE, - `${SQLITE_FILE}-wal`, - `${SQLITE_FILE}-shm`, - `${SQLITE_FILE}-journal`, - ]; - // Delete with EBUSY/EPERM retry: after resetDbInstance() the OS may still - // hold the SQLite file handle for a moment (Windows mmap / antivirus), so a - // plain unlink races to EBUSY (#5406). Mirror the restore path's helper. - for (const filePath of sqliteFilesToReplace) { - if (!filePath) continue; - await unlinkFileWithRetry(filePath); + // The DB was replaced wholesale — re-hydrate the in-memory Global System Prompt so it + // reflects the imported settings without requiring a restart (#2470). + try { + const importedSettings = await getSettings(); + if (importedSettings.systemPrompt) { + setSystemPromptConfig(importedSettings.systemPrompt); } + } catch { + // non-fatal: import succeeded; system prompt will hydrate on next restart + } + + return NextResponse.json({ + imported: true, + filename: fileName, + connectionCount: connCount, + nodeCount, + comboCount, + apiKeyCount: keyCount, + }); +} + +/** + * POST /api/db-backups/import — Upload a .sqlite file to replace the current database. + * + * Accepts multipart/form-data with a single "file" field containing the .sqlite backup, + * or the raw file bytes (`?filename=` names it). Validates integrity, schema, and required + * tables before replacing the active database. + * + * `dryRun=true` (query string, or a multipart `dryRun` field) runs the same validation and + * returns what WOULD be imported, without the pre-import backup or touching the live + * database. + * + * 🔒 Auth-guarded: requires JWT cookie, Bearer manage API key (finding #258-3), or the + * machine-bound local CLI credential. + */ +export async function POST(request: Request) { + if (!(await isImportAuthorized(request))) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + let tmpPath: string | null = null; - // Copy imported file over current DB - fs.copyFileSync(tmpPath, SQLITE_FILE!); + try { + const upload = await readUpload(request); + if (upload instanceof Response) return upload; + const { fileBuffer, fileName, dryRun } = upload; - // Reopen and verify - getDbInstance(); - const { connCount, nodeCount, comboCount, keyCount } = countImportedRows(); + const shapeError = validateUploadShape(fileName, fileBuffer.length); + if (shapeError) return shapeError; - console.log( - `[DB] Imported database from upload: ${connCount} connections, ${nodeCount} nodes, ${comboCount} combos, ${keyCount} API keys` - ); + // Write uploaded file to temp location + tmpPath = path.join(os.tmpdir(), `omniroute-import-${Date.now()}.sqlite`); + fs.writeFileSync(tmpPath, fileBuffer); - // The DB was replaced wholesale — re-hydrate the in-memory Global System Prompt so it - // reflects the imported settings without requiring a restart (#2470). - try { - const importedSettings = await getSettings(); - if (importedSettings.systemPrompt) { - setSystemPromptConfig(importedSettings.systemPrompt); - } - } catch { - // non-fatal: import succeeded; system prompt will hydrate on next restart + const summary = await inspectCandidate(tmpPath); + if (summary instanceof Response) return summary; + + if (dryRun) { + return NextResponse.json({ + dryRun: true, + valid: true, + imported: false, + filename: fileName, + sizeBytes: fileBuffer.length, + connectionCount: summary.connCount, + nodeCount: summary.nodeCount, + comboCount: summary.comboCount, + apiKeyCount: summary.keyCount, + schemaVersion: summary.schemaVersion, + }); } - return NextResponse.json({ - imported: true, - filename: fileName, - connectionCount: connCount, - nodeCount, - comboCount, - apiKeyCount: keyCount, - }); + return await replaceLiveDatabase(tmpPath, fileName); } catch (error) { console.error("[API] Error importing database:", error); return NextResponse.json({ error: sanitizeErrorMessage(error) }, { status: 500 }); diff --git a/src/lib/api/operatorCredential.ts b/src/lib/api/operatorCredential.ts index dd4998130a51..8076dbd65de2 100644 --- a/src/lib/api/operatorCredential.ts +++ b/src/lib/api/operatorCredential.ts @@ -41,14 +41,23 @@ export async function resolveOperatorCredential( if (options.allowInternalService && isTrustedLoopbackInternalServiceRequest(request)) { return "internal_service"; } + if (await isLocalCliCredential(request)) return "local_cli"; + return null; +} + +/** + * Is the caller the machine-bound local CLI? Either the authz pipeline already validated + * the loopback CLI token and stamped the subject, or (raw callers that skip the pipeline) + * the token is validated here, loopback-only. + */ +export async function isLocalCliCredential(request: Request): Promise { if ( request.headers.get(AUTHZ_HEADER_AUTH_KIND) === "management_key" && request.headers.get(AUTHZ_HEADER_AUTH_LABEL) === "local-cli-token" ) { - return "local_cli"; + return true; } - if (await isCliTokenAuthValid(request)) return "local_cli"; - return null; + return isCliTokenAuthValid(request); } export type OperatorAuthVerdict = diff --git a/src/lib/db/backup.ts b/src/lib/db/backup.ts index 050a9baf4a91..23ac39bd2749 100644 --- a/src/lib/db/backup.ts +++ b/src/lib/db/backup.ts @@ -715,3 +715,45 @@ export function countImportedRows(): { const keyCount = (db.prepare("SELECT COUNT(*) as cnt FROM api_keys").get() as any)?.cnt || 0; return { connCount, nodeCount, comboCount, keyCount }; } + +type ReadonlyAdapterLike = { + prepare: (sql: string) => { get: () => unknown }; +}; + +function countTableFromAdapter(adapter: ReadonlyAdapterLike, table: string): number { + return (adapter.prepare(`SELECT COUNT(*) as cnt FROM ${table}`).get() as CountRow)?.cnt || 0; +} + +/** + * Summarizes an **already-opened** candidate database (not the live one) for the import + * dry run: the row counts a real import would load, plus the newest applied migration + * version (`null` when the backup predates migration tracking). Callers must have + * validated that the four required tables exist first. + */ +export function summarizeImportCandidate( + adapter: ReadonlyAdapterLike, + tableNames: string[] +): { + connCount: number; + nodeCount: number; + comboCount: number; + keyCount: number; + schemaVersion: string | null; +} { + let schemaVersion: string | null = null; + if (tableNames.includes("_omniroute_migrations")) { + const row = adapter + .prepare( + "SELECT version FROM _omniroute_migrations ORDER BY CAST(version AS INTEGER) DESC, version DESC LIMIT 1" + ) + .get() as { version?: string } | undefined; + schemaVersion = row?.version ?? null; + } + return { + connCount: countTableFromAdapter(adapter, "provider_connections"), + nodeCount: countTableFromAdapter(adapter, "provider_nodes"), + comboCount: countTableFromAdapter(adapter, "combos"), + keyCount: countTableFromAdapter(adapter, "api_keys"), + schemaVersion, + }; +} diff --git a/tests/unit/cli-sync-import-dry-run.test.ts b/tests/unit/cli-sync-import-dry-run.test.ts new file mode 100644 index 000000000000..62192576e0ec --- /dev/null +++ b/tests/unit/cli-sync-import-dry-run.test.ts @@ -0,0 +1,215 @@ +// `omniroute sync import --dry-run` used to POST the backup to /api/db-backups/import with +// `?dryRun=true` — a flag the route never read — so the "dry run" replaced the live database. +// Even with the route now honouring dryRun, an OLDER server still ignores it, so the CLI must +// never send the replacing request in dry-run mode: it validates the file locally and prints +// a summary instead. +// +// These tests drive the REAL commander wiring of `registerSync()` with `fetch` stubbed. +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import Database from "better-sqlite3"; +import { Command } from "commander"; + +import { registerSync } from "../../bin/cli/commands/sync.mjs"; + +const CLI_TOKEN = "test-cli-token-sync-import"; + +type Captured = { url: string; method: string; body: unknown; headers: Headers }; + +function jsonResponse(status: number, body: unknown) { + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +async function runImport(args: string[], respond: () => Response = () => jsonResponse(200, {})) { + const calls: Captured[] = []; + const exitCodes: number[] = []; + const stderr: string[] = []; + const stdout: string[] = []; + + const origFetch = globalThis.fetch; + const origExit = process.exit; + const origErr = process.stderr.write.bind(process.stderr); + const origOut = process.stdout.write.bind(process.stdout); + const origEnv = { + base: process.env.OMNIROUTE_BASE_URL, + token: process.env.OMNIROUTE_CLI_TOKEN, + key: process.env.OMNIROUTE_API_KEY, + }; + + process.env.OMNIROUTE_BASE_URL = "http://127.0.0.1:20999"; + process.env.OMNIROUTE_CLI_TOKEN = CLI_TOKEN; + delete process.env.OMNIROUTE_API_KEY; + + globalThis.fetch = (async (url: string, init: RequestInit = {}) => { + calls.push({ + url: String(url), + method: String(init.method ?? "GET").toUpperCase(), + body: init.body, + headers: new Headers(init.headers), + }); + return respond(); + }) as typeof fetch; + process.exit = ((code?: number) => { + exitCodes.push(code ?? 0); + throw new Error(`__exit_${code ?? 0}__`); + }) as typeof process.exit; + process.stderr.write = ((chunk: string | Uint8Array) => { + stderr.push(String(chunk)); + return true; + }) as typeof process.stderr.write; + process.stdout.write = ((chunk: string | Uint8Array) => { + stdout.push(String(chunk)); + return true; + }) as typeof process.stdout.write; + + let thrown: unknown = null; + try { + const program = new Command().exitOverride().option("--output "); + registerSync(program); + await program.parseAsync(["node", "omniroute", "--output", "json", "sync", "import", ...args]); + } catch (err) { + thrown = err; + } finally { + globalThis.fetch = origFetch; + process.exit = origExit; + process.stderr.write = origErr; + process.stdout.write = origOut; + for (const [envKey, value] of [ + ["OMNIROUTE_BASE_URL", origEnv.base], + ["OMNIROUTE_CLI_TOKEN", origEnv.token], + ["OMNIROUTE_API_KEY", origEnv.key], + ] as const) { + if (value === undefined) delete process.env[envKey]; + else process.env[envKey] = value; + } + } + + const err = thrown as Error | null; + if (err && !/^__exit_\d+__$/.test(err.message)) throw err; + return { calls, exitCodes, stderr: stderr.join(""), stdout: stdout.join("") }; +} + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-sync-import-")); +test.after(() => fs.rmSync(dir, { recursive: true, force: true })); + +/** A minimal OmniRoute-shaped SQLite backup with the tables the import route requires. */ +function makeBackup(name: string, { apiKeys = 2, withRequiredTables = true } = {}) { + const file = path.join(dir, name); + const db = new Database(file); + try { + db.exec("CREATE TABLE provider_connections (id TEXT PRIMARY KEY)"); + db.exec("CREATE TABLE provider_nodes (id TEXT PRIMARY KEY)"); + db.exec("CREATE TABLE combos (id TEXT PRIMARY KEY)"); + if (withRequiredTables) db.exec("CREATE TABLE api_keys (id TEXT PRIMARY KEY, name TEXT)"); + db.exec("CREATE TABLE _omniroute_migrations (version TEXT, name TEXT)"); + db.prepare("INSERT INTO _omniroute_migrations VALUES (?, ?)").run("206", "latest"); + db.prepare("INSERT INTO provider_connections VALUES (?)").run("conn-1"); + if (withRequiredTables) { + for (let i = 0; i < apiKeys; i++) { + db.prepare("INSERT INTO api_keys VALUES (?, ?)").run(`key-${i}`, `k${i}`); + } + } + } finally { + db.close(); + } + return file; +} + +test("sync import --dry-run never sends a request to the server", async () => { + const file = makeBackup("valid-dry.sqlite"); + const { calls, exitCodes, stdout } = await runImport([file, "--dry-run"]); + + assert.deepEqual( + calls.map((c) => `${c.method} ${c.url}`), + [], + "no network call in dry-run mode" + ); + assert.deepEqual(exitCodes, [], "a valid backup is a successful dry run"); + const summary = JSON.parse(stdout); + assert.equal(summary.dryRun, true); + assert.equal(summary.valid, true); + assert.equal(summary.apiKeyCount, 2); + assert.equal(summary.connectionCount, 1); + assert.equal(summary.schemaVersion, "206"); +}); + +test("sync import --dry-run reports a backup missing required tables (exit 2, no request)", async () => { + const file = makeBackup("missing-tables.sqlite", { withRequiredTables: false }); + const { calls, exitCodes, stderr } = await runImport([file, "--dry-run"]); + + assert.equal(calls.length, 0); + assert.deepEqual(exitCodes, [2]); + assert.match(stderr, /api_keys/); + assert.doesNotMatch(stderr, /\n\s+at /, "no stack trace"); +}); + +test("sync import rejects a JSON bundle locally (it is not a SQLite backup)", async () => { + const file = path.join(dir, "bundle.json"); + fs.writeFileSync(file, JSON.stringify({ settings: {}, combos: [] }).padEnd(8192, " ")); + + for (const extra of [[], ["--dry-run"]]) { + const { calls, exitCodes, stderr } = await runImport([file, ...extra]); + assert.equal(calls.length, 0, "an obviously wrong file must not be uploaded"); + assert.deepEqual(exitCodes, [2]); + assert.match(stderr, /not a SQLite database/i); + assert.match(stderr, /sync bundle/); + } +}); + +test("sync import reports an unreadable path without a stack trace", async () => { + const { calls, exitCodes, stderr } = await runImport([path.join(dir, "nope.sqlite")]); + assert.equal(calls.length, 0); + assert.deepEqual(exitCodes, [1]); + assert.match(stderr, /cannot read/i); + assert.doesNotMatch(stderr, /\n\s+at /); +}); + +test("sync import (no --dry-run) uploads the file with the loopback CLI credential", async () => { + const file = makeBackup("valid-real.sqlite"); + const { calls, exitCodes, stdout } = await runImport([file], () => + jsonResponse(200, { imported: true, filename: "valid-real.sqlite", apiKeyCount: 2 }) + ); + + assert.deepEqual(exitCodes, []); + assert.equal(calls.length, 1); + const [call] = calls; + assert.equal(call.method, "POST"); + const url = new URL(call.url); + assert.equal(url.pathname, "/api/db-backups/import"); + assert.equal(url.searchParams.get("dryRun"), null, "a real import does not carry dryRun"); + assert.equal(url.searchParams.get("filename"), "valid-real.sqlite"); + assert.equal(call.headers.get("content-type"), "application/octet-stream"); + assert.equal(call.headers.get("x-omniroute-cli-token"), CLI_TOKEN); + assert.deepEqual(Buffer.from(call.body as Uint8Array), fs.readFileSync(file)); + assert.equal(JSON.parse(stdout).imported, true); +}); + +test("sync import turns a 403 into a clear credential error (exit 4, no stack)", async () => { + const file = makeBackup("valid-403.sqlite"); + const { exitCodes, stderr } = await runImport([file], () => + jsonResponse(403, { error: { message: "API key lacks 'manage' scope." } }) + ); + + assert.deepEqual(exitCodes, [4]); + assert.match(stderr, /HTTP 403/); + assert.match(stderr, /manage/); + assert.match(stderr, /local CLI/); + assert.doesNotMatch(stderr, /\n\s+at /); +}); + +test("sync import surfaces the route's 400 validation message", async () => { + const file = makeBackup("valid-400.sqlite"); + const { exitCodes, stderr } = await runImport([file], () => + jsonResponse(400, { error: "Invalid OmniRoute database. Missing tables: combos" }) + ); + + assert.deepEqual(exitCodes, [2]); + assert.match(stderr, /HTTP 400/); + assert.match(stderr, /Missing tables: combos/); +}); diff --git a/tests/unit/db-import-dry-run.test.ts b/tests/unit/db-import-dry-run.test.ts new file mode 100644 index 000000000000..04cee080913c --- /dev/null +++ b/tests/unit/db-import-dry-run.test.ts @@ -0,0 +1,186 @@ +// POST /api/db-backups/import replaces the live SQLite database wholesale. `omniroute sync +// import --dry-run` sent `?dryRun=true`, but the route never read it — so a "dry run" +// replaced the operator's database (data loss). These tests drive the real route handler +// against an isolated DATA_DIR and pin: +// - `dryRun=true` (query or multipart field) validates the upload and reports what WOULD be +// imported, but leaves the live database byte-for-byte untouched; +// - an invalid upload in dry-run mode is still a 400, with no side effect; +// - a real import (no dryRun) keeps replacing the database, as before; +// - the machine-bound local CLI credential (pipeline stamp) is accepted, so +// `omniroute sync import` on the server host does not need a manage-scoped API key. +import test from "node:test"; +import assert from "node:assert/strict"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + createManagementSessionHeaders, + makeManagementSessionRequest, +} from "../helpers/managementSession.ts"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-db-import-dry-run-")); +const ORIGINAL_ENV = { + DATA_DIR: process.env.DATA_DIR, + API_KEY_SECRET: process.env.API_KEY_SECRET, + JWT_SECRET: process.env.JWT_SECRET, + INITIAL_PASSWORD: process.env.INITIAL_PASSWORD, +}; + +process.env.DATA_DIR = TEST_DATA_DIR; +process.env.API_KEY_SECRET = "db-import-dry-run-api-key-secret"; +process.env.JWT_SECRET = "db-import-dry-run-jwt-secret"; +process.env.INITIAL_PASSWORD = "db-import-dry-run-password"; + +const core = await import("../../src/lib/db/core.ts"); +const apiKeysDb = await import("../../src/lib/db/apiKeys.ts"); +const { updateSettings } = await import("../../src/lib/db/settings.ts"); +const importRoute = await import("../../src/app/api/db-backups/import/route.ts"); + +const IMPORT_URL = "http://localhost:20128/api/db-backups/import"; +const LIVE_SENTINEL = "live-sentinel-key"; +const CANDIDATE_MARKER = "candidate-only-key"; + +await updateSettings({ requireLogin: true }); + +// Candidate backup: a real, fully-migrated OmniRoute database that carries a marker row the +// live database does not have (and lacks the live sentinel). +await apiKeysDb.createApiKey(CANDIDATE_MARKER, "machine-import"); +const CANDIDATE_PATH = path.join(TEST_DATA_DIR, "candidate.sqlite"); +core.getDbInstance().exec(`VACUUM INTO '${CANDIDATE_PATH.replace(/'/g, "''")}'`); +const CANDIDATE_BYTES = fs.readFileSync(CANDIDATE_PATH); +core.getDbInstance().prepare("DELETE FROM api_keys WHERE name = ?").run(CANDIDATE_MARKER); +await apiKeysDb.createApiKey(LIVE_SENTINEL, "machine-import"); + +test.after(() => { + apiKeysDb.resetApiKeyState(); + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + for (const [key, value] of Object.entries(ORIGINAL_ENV)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } +}); + +function apiKeyNames(): string[] { + const rows = core.getDbInstance().prepare("SELECT name FROM api_keys").all() as Array<{ + name: string; + }>; + return rows.map((row) => row.name); +} + +function liveDbHash(): string { + core.getDbInstance().exec("PRAGMA wal_checkpoint(TRUNCATE)"); + return crypto + .createHash("sha256") + .update(fs.readFileSync(core.SQLITE_FILE as string)) + .digest("hex"); +} + +// Raw binary upload (the CLI's transport). Built by hand: the session helper JSON-serializes +// any non-string body, which would mangle the SQLite bytes. +async function sessionUpload(url: string, body: Buffer) { + const headers = await createManagementSessionHeaders({ + "content-type": "application/octet-stream", + }); + return new Request(url, { method: "POST", headers, body: new Uint8Array(body) }); +} + +test("dryRun=true validates the upload but leaves the live database untouched", async () => { + const before = liveDbHash(); + assert.ok(apiKeyNames().includes(LIVE_SENTINEL), "precondition: live sentinel present"); + + const response = await importRoute.POST( + await sessionUpload(`${IMPORT_URL}?dryRun=true`, CANDIDATE_BYTES) + ); + const body = await response.json(); + + assert.equal(response.status, 200, JSON.stringify(body)); + assert.equal(body.dryRun, true); + assert.equal(body.valid, true); + assert.notEqual(body.imported, true, "a dry run must never report an import"); + assert.equal(body.apiKeyCount, 1, "the summary describes the UPLOADED database"); + assert.equal(typeof body.connectionCount, "number"); + assert.equal(typeof body.comboCount, "number"); + + const names = apiKeyNames(); + assert.ok(names.includes(LIVE_SENTINEL), "dry run must not replace the live database"); + assert.ok(!names.includes(CANDIDATE_MARKER), "dry run must not load the candidate rows"); + assert.equal(liveDbHash(), before, "live database file must be byte-for-byte unchanged"); +}); + +test("dryRun as a multipart form field is honoured too", async () => { + const before = liveDbHash(); + const form = new FormData(); + form.append("file", new File([CANDIDATE_BYTES], "candidate.sqlite")); + form.append("dryRun", "true"); + + const response = await importRoute.POST( + await makeManagementSessionRequest(IMPORT_URL, { method: "POST", body: form }) + ); + const body = await response.json(); + + assert.equal(response.status, 200, JSON.stringify(body)); + assert.equal(body.dryRun, true); + assert.equal(body.filename, "candidate.sqlite"); + assert.ok(apiKeyNames().includes(LIVE_SENTINEL)); + assert.equal(liveDbHash(), before); +}); + +test("dryRun with an invalid upload is a 400 and changes nothing", async () => { + const before = liveDbHash(); + const notSqlite = Buffer.alloc(8192, 0x41); + + const response = await importRoute.POST( + await sessionUpload(`${IMPORT_URL}?dryRun=true`, notSqlite) + ); + const body = await response.json(); + + assert.equal(response.status, 400); + assert.ok(!JSON.stringify(body).includes("at /"), "400 body must not leak a stack trace"); + assert.ok(apiKeyNames().includes(LIVE_SENTINEL)); + assert.equal(liveDbHash(), before); +}); + +test("unauthenticated callers get 401 (dry run is not an auth bypass)", async () => { + const response = await importRoute.POST( + new Request(`${IMPORT_URL}?dryRun=true`, { + method: "POST", + headers: { "content-type": "application/octet-stream" }, + body: CANDIDATE_BYTES, + }) + ); + assert.equal(response.status, 401); + assert.ok(apiKeyNames().includes(LIVE_SENTINEL)); +}); + +test("the pipeline-stamped local CLI subject is accepted (dry run)", async () => { + const response = await importRoute.POST( + new Request(`${IMPORT_URL}?dryRun=true`, { + method: "POST", + headers: { + "content-type": "application/octet-stream", + "x-omniroute-auth-kind": "management_key", + "x-omniroute-auth-label": "local-cli-token", + }, + body: CANDIDATE_BYTES, + }) + ); + const body = await response.json(); + assert.equal(response.status, 200, JSON.stringify(body)); + assert.equal(body.dryRun, true); + assert.ok(apiKeyNames().includes(LIVE_SENTINEL)); +}); + +// Runs last: it really replaces the database. +test("a real import (no dryRun) still replaces the live database", async () => { + const response = await importRoute.POST(await sessionUpload(IMPORT_URL, CANDIDATE_BYTES)); + const body = await response.json(); + + assert.equal(response.status, 200, JSON.stringify(body)); + assert.equal(body.imported, true); + assert.notEqual(body.dryRun, true); + const names = apiKeyNames(); + assert.ok(names.includes(CANDIDATE_MARKER), "the uploaded database must now be live"); + assert.ok(!names.includes(LIVE_SENTINEL), "the previous live rows must be gone"); +});