diff --git a/AGENTS.md b/AGENTS.md index 826467355..efdb6f5a0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,6 +57,11 @@ All chat platforms (Telegram, Slack, Discord, WhatsApp, Teams) run through Chat - **Integration auth lives in Lobu** — OAuth, token refresh, and API proxying for third-party services (GitHub, Google, etc.) are handled by Lobu MCP servers. Workers never see OAuth tokens. - **`events` is append-only.** Never `DELETE FROM events`. To hide a row, write a tombstone via `client.knowledge.delete()` or `save_knowledge({ supersedes_event_id, ... })`; the `current_event_records` view masks superseded rows, `include_superseded` recovers history. +#### Connectors +- A connector is a `*.connector.ts` extending `ConnectorRuntime`. **npm deps** go in the project's `package.json` (next to `lobu.toml`) and are **bundled** into the connector by esbuild at compile time. **Native deps** (ffmpeg, imagemagick, …) are declared as nixpkgs refs in `runtime.nix.packages` on the connector `definition`; the runtime provisions them on PATH via `nix-shell` at execution. npm = bundled (compile-time); native = nix (run-time). +- **Compile happens on the CLI**, not the server: `lobu apply` runs `bun install --ignore-scripts` in the project (install scripts off for supply-chain safety — native bits go through `runtime.nix.packages`, not bundled npm), compiles each connector with the project's `node_modules`, and uploads the bundle (`compiled: true`). esbuild resolves a connector's imports relative to the connector file's dir, so the project's deps are used regardless of where the `lobu` binary is installed. `@lobu/connector-sdk` is **externalized** (runtime-provided, à la Lambda's `aws-sdk`) — the bundle stays the connector's own code + deps, not multiple MB of SDK infra. +- `lobu init` scaffolds `package.json` (with `@lobu/connector-sdk` devDependency for editor types) + `tsconfig.json` + `connectors/`. + #### Guardrails - Primitive lives in `packages/core/src/guardrails/`: `Guardrail`, `GuardrailRegistry`, `runGuardrails()`. Stages: `input` (user message → worker), `output` (worker text → user), `pre-tool` (tool call authorization). - Each guardrail's `run(ctx)` returns `{ tripped, reason?, metadata? }`. The runner races all enabled guardrails at a stage; the first trip short-circuits (later results are discarded) and a thrown guardrail is logged and treated as a pass. @@ -104,6 +109,7 @@ Rules for agents: - **No new dynamic imports outside the allow-list below.** Use static `import` by default; new `await import(...)` sites need a measured cost justification (boot time, install footprint, Keychain prompt) added to this list in the same PR. Rationale for each entry lives as a code comment at the call site: - `packages/cli/src/index.ts` — lazy subcommand handlers (keeps `lobu --help` ~60ms). - `packages/cli/src/commands/_lib/connector-run-cmd.ts` — `browser-mirror`, `devtools-active-port`, `executeCompiledConnector`. + - `packages/cli/src/commands/_lib/apply/apply-cmd.ts` — `connector-loader` + `ensure-deps-installed` (the connector-compile graph: esbuild + connector-worker + SDK). Loaded only when an apply has local `*.connector.ts` to compile; keeps that graph out of apply-cmd's module-load path (and out of every CLI test that imports it). - `packages/cli/src/commands/_lib/apply/desired-state.ts` — `yaml` (loaded only on YAML inputs). - `packages/cli/src/commands/memory/_lib/browser-auth-cmd.ts` — `decryptChromeCookiesMacOS`, `playwright/chromium`. - `packages/server/src/server.ts` — `./embedded-runtime` is statically imported, but `./server-lifecycle` is lazy: its transitive imports read env at module-eval, and the embedded branch only finalises DATABASE_URL during `main()`. Loading the lifecycle eagerly would snapshot a stale env. diff --git a/packages/cli/src/commands/_lib/apply/apply-cmd.ts b/packages/cli/src/commands/_lib/apply/apply-cmd.ts index 4b4e6ce9f..e965b22a7 100644 --- a/packages/cli/src/commands/_lib/apply/apply-cmd.ts +++ b/packages/cli/src/commands/_lib/apply/apply-cmd.ts @@ -376,10 +376,38 @@ async function installConnectorDefinitions( if (row.verb === "noop" || row.verb === "drift") continue; const def = row.desired; if (!def) continue; - const result = - def.sourceCode !== undefined - ? await client.installConnector({ sourceCode: def.sourceCode }) - : await client.installConnector({ sourceUrl: def.sourceUrl }); + let result: Awaited>; + if (def.sourcePath) { + // Local `*.connector.ts`: compile on the CLI, where the project's + // node_modules is available, so esbuild can bundle the connector's + // declared npm deps (the server only receives the artifact). Native deps + // ride `runtime.nix.packages` and are provisioned at run time. Compile + // `sourcePath` (the actual `.ts`), not `sourceFile` (an error-message + // label that may point at a `type: connector` YAML doc). + // + // Lazy-imported (cached by the loader) so the heavy connector-compile + // graph (esbuild + connector-worker + SDK) stays out of apply-cmd's + // module-load path — see the dynamic-import allow-list in AGENTS.md. + const { ensureProjectDepsInstalled } = await import( + "../ensure-deps-installed.js" + ); + const { compileConnectorFromFile } = await import( + "../connector-loader.js" + ); + ensureProjectDepsInstalled(def.sourcePath, printText); + const compiledCode = await compileConnectorFromFile(def.sourcePath); + result = await client.installConnector({ + sourceCode: compiledCode, + compiled: true, + }); + } else if (def.sourceCode !== undefined) { + // `source_url` connector: source was fetched into `sourceCode` and has no + // local project/node_modules to bundle against — upload it raw and let + // the gateway compile it (the pre-existing path). + result = await client.installConnector({ sourceCode: def.sourceCode }); + } else { + result = await client.installConnector({ sourceUrl: def.sourceUrl }); + } if (result.connectorKey) { locallySuppliedKeys.add(result.connectorKey); installedKeys.add(result.connectorKey); diff --git a/packages/cli/src/commands/_lib/apply/client.ts b/packages/cli/src/commands/_lib/apply/client.ts index 76e801f5c..13dd8c2ab 100644 --- a/packages/cli/src/commands/_lib/apply/client.ts +++ b/packages/cli/src/commands/_lib/apply/client.ts @@ -851,6 +851,8 @@ export class ApplyClient { sourceUrl?: string; /** `file://` URI of a bundled connector source on the server host. */ sourceUri?: string; + /** `sourceCode` is already a compiled bundle (CLI-side compile) — skip server compile. */ + compiled?: boolean; }): Promise { const body = await this.connectionsTool<{ installed?: boolean; @@ -860,7 +862,10 @@ export class ApplyClient { }>({ action: "install_connector", ...(payload.sourceCode !== undefined - ? { source_code: payload.sourceCode, compiled: false } + ? { + source_code: payload.sourceCode, + compiled: payload.compiled ?? false, + } : {}), ...(payload.sourceUrl ? { source_url: payload.sourceUrl } : {}), ...(payload.sourceUri ? { source_uri: payload.sourceUri } : {}), diff --git a/packages/cli/src/commands/_lib/ensure-deps-installed.ts b/packages/cli/src/commands/_lib/ensure-deps-installed.ts new file mode 100644 index 000000000..3bb877b59 --- /dev/null +++ b/packages/cli/src/commands/_lib/ensure-deps-installed.ts @@ -0,0 +1,88 @@ +/** + * Ensure a connector project's npm dependencies are installed before the CLI + * compiles its connectors. esbuild bundles a connector's imports relative to + * the connector file's directory, so the project's own `node_modules` (next to + * `package.json`) must exist. We run `bun install --ignore-scripts` when stale: + * `--ignore-scripts` keeps install-time supply-chain surface off the user's + * machine — packages that need build scripts (native bindings) belong in + * `runtime.nix.packages`, not bundled npm. + */ + +import { execFileSync } from "node:child_process"; +import { existsSync, statSync } from "node:fs"; +import { dirname, join } from "node:path"; + +// Per-process memo so `lobu apply` installs each project root at most once. +const ensuredRoots = new Set(); + +/** + * Find the connector's project root — the nearest ancestor with `lobu.toml`. + * Anchoring on `lobu.toml` (not any ancestor `package.json`) is what stops a + * connector inside a monorepo from resolving to the monorepo's root + * package.json and triggering a wrong-directory install. + */ +export function findProjectRoot(fromFile: string): string | null { + let dir = dirname(fromFile); + for (let i = 0; i < 40; i++) { + if (existsSync(join(dir, "lobu.toml"))) return dir; + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + return null; +} + +function installIsStale(root: string): boolean { + const nodeModules = join(root, "node_modules"); + if (!existsSync(nodeModules)) return true; + const lock = join(root, "bun.lock"); + if (!existsSync(lock)) return false; // deps present, no lockfile to compare against + try { + return statSync(lock).mtimeMs > statSync(nodeModules).mtimeMs; + } catch { + return false; + } +} + +function hasBun(): boolean { + try { + execFileSync("bun", ["--version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +/** + * Install the connector project's deps if missing/stale. No-op when the + * connector has no `package.json` (no declared npm deps to bundle). + */ +export function ensureProjectDepsInstalled( + connectorFilePath: string, + log: (message: string) => void +): void { + const root = findProjectRoot(connectorFilePath); + if (!root || ensuredRoots.has(root)) return; + // No package.json at the project root → the connector declares no npm deps + // (the SDK is runtime-provided/externalized), so there's nothing to install. + if (!existsSync(join(root, "package.json"))) { + ensuredRoots.add(root); + return; + } + if (!installIsStale(root)) { + ensuredRoots.add(root); + return; + } + if (!hasBun()) { + throw new Error( + `Connector dependencies in ${root} need installing, but \`bun\` is not on PATH. ` + + `Run \`bun install\` in ${root}, or install bun (https://bun.sh).` + ); + } + log(`Installing connector dependencies in ${root}...`); + execFileSync("bun", ["install", "--ignore-scripts"], { + cwd: root, + stdio: "inherit", + }); + ensuredRoots.add(root); +} diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index 6d8e28a14..bed4ae2ce 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -705,6 +705,64 @@ export async function initCommand( await mkdir(join(projectDir, "skills"), { recursive: true }); await writeFile(join(projectDir, "skills", ".gitkeep"), ""); + // Connector authoring surface: package.json declares the connector SDK + // (provided by the runtime — externalized at compile, here for editor + // types) plus any npm deps the user adds; tsconfig gives the editor + // resolution; the connectors/ dir holds `*.connector.ts`. `lobu apply` + // runs `bun install --ignore-scripts` here and bundles each connector's + // own deps. + // + // `--here` can target a directory that already has a package.json / + // tsconfig.json — merge into package.json (preserve the user's fields, just + // add the SDK devDependency) and never overwrite an existing tsconfig. + const pkgJsonPath = join(projectDir, "package.json"); + let pkgJson: Record; + try { + pkgJson = JSON.parse(await readFile(pkgJsonPath, "utf-8")) as Record< + string, + unknown + >; + } catch { + pkgJson = { + name: projectName, + version: "0.0.0", + private: true, + type: "module", + }; + } + pkgJson.devDependencies = { + ...((pkgJson.devDependencies as Record | undefined) ?? + {}), + "@lobu/connector-sdk": `^${cliVersion}`, + }; + await writeFile(pkgJsonPath, `${JSON.stringify(pkgJson, null, 2)}\n`); + + const tsconfigPath = join(projectDir, "tsconfig.json"); + try { + await readFile(tsconfigPath, "utf-8"); // exists — leave the user's config untouched + } catch { + await writeFile( + tsconfigPath, + `${JSON.stringify( + { + compilerOptions: { + target: "ES2022", + module: "Preserve", + moduleResolution: "bundler", + strict: true, + skipLibCheck: true, + noEmit: true, + }, + include: ["connectors/**/*.ts"], + }, + null, + 2 + )}\n` + ); + } + await mkdir(join(projectDir, "connectors"), { recursive: true }); + await writeFile(join(projectDir, "connectors", ".gitkeep"), ""); + await renderTemplate( "AGENTS.md.tmpl", variables, diff --git a/packages/cli/src/commands/memory/_lib/schema.ts b/packages/cli/src/commands/memory/_lib/schema.ts index f8b45f240..c39e94cd9 100644 --- a/packages/cli/src/commands/memory/_lib/schema.ts +++ b/packages/cli/src/commands/memory/_lib/schema.ts @@ -11,7 +11,11 @@ * Bump CURRENT_SCHEMA_VERSION when making breaking changes. */ -import { AutoCreateWhenRule } from "@lobu/connector-sdk"; +// Import from the module subpath, not the barrel: a barrel re-export of this +// value+type dual name trips bun's cross-file module lexer in the test runner +// ("Export named AutoCreateWhenRule not found"). The direct subpath resolves +// to the module that declares it and sidesteps the bug. +import { AutoCreateWhenRule } from "@lobu/connector-sdk/identity-types"; import { TypeCompiler } from "@sinclair/typebox/compiler"; import { parseAllDocuments } from "yaml"; diff --git a/packages/connector-sdk/package.json b/packages/connector-sdk/package.json index e7ccee9d4..9edfb5c84 100644 --- a/packages/connector-sdk/package.json +++ b/packages/connector-sdk/package.json @@ -16,6 +16,10 @@ "default": "./dist/index.js" } }, + "./identity-types": { + "types": "./dist/identity-types.d.ts", + "import": "./dist/identity-types.js" + }, "./browser-mirror": { "import": { "types": "./dist/browser/mirror-cookies.d.ts", diff --git a/packages/connector-sdk/src/connector-types.ts b/packages/connector-sdk/src/connector-types.ts index 69441559d..b222d1aec 100644 --- a/packages/connector-sdk/src/connector-types.ts +++ b/packages/connector-sdk/src/connector-types.ts @@ -65,6 +65,15 @@ export interface ConnectorRuntimeInfo { * Optional — omit when the platform adapter needs no fine-grained scope list. */ scopes?: string[]; + /** + * Native system dependencies this connector needs on PATH at execution time, + * as nixpkgs attribute references (e.g. `["ffmpeg", "imagemagick"]`). npm + * dependencies are bundled into the connector at compile time and do NOT go + * here — only native tools the runtime must provision. Backends that can run + * native deps (embedded, container, machine) satisfy these via nix; backends + * that can't (e.g. edge workers) reject a connector that declares them. + */ + nix?: { packages: string[] }; } // ============================================================================= diff --git a/packages/connector-worker/src/compile/index.ts b/packages/connector-worker/src/compile/index.ts index f0a9b4b0e..b54662b73 100644 --- a/packages/connector-worker/src/compile/index.ts +++ b/packages/connector-worker/src/compile/index.ts @@ -22,7 +22,6 @@ import { existsSync } from 'node:fs'; import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; -import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { build, type Plugin } from 'esbuild'; @@ -75,15 +74,28 @@ export function findBundledConnectorFile( } /** - * Resolve the `@lobu/connector-sdk` module entry from this module's - * perspective. Used as the esbuild `alias` target so connector code that - * imports `from 'lobu'` or `from '@lobu/connector-sdk'` resolves to the - * same physical file the runtime will import — avoiding the - * `instanceof ConnectorRuntime` cross-realm trap. + * esbuild plugin that marks the connector SDK as **external** (runtime-provided) + * rather than bundling it in. The SDK pulls a large infra graph transitively + * (Sentry, OpenTelemetry, grpc, isomorphic-git, …); bundling it inflated every + * connector to multiple MB. The runtime that executes the connector already has + * `@lobu/connector-sdk` installed (it's a dependency of `@lobu/connector-worker`), + * so the bundle leaves it as a bare import and Node resolves it from the runtime's + * node_modules at load time — the standard "externalize the framework, bundle the + * user code" pattern (cf. AWS Lambda not bundling `@aws-sdk`). + * + * The `lobu` alias specifier is normalized to `@lobu/connector-sdk` so the emitted + * import resolves to a real package the runtime provides. */ -function resolveSdkEntry(): string { - const require_ = createRequire(import.meta.url); - return require_.resolve('@lobu/connector-sdk'); +function createSdkExternalPlugin(): Plugin { + return { + name: 'sdk-external', + setup(b) { + b.onResolve({ filter: /^(lobu|@lobu\/connector-sdk)(\/.*)?$/ }, (args) => ({ + path: args.path.replace(/^lobu(?=$|\/)/, '@lobu/connector-sdk'), + external: true, + })); + }, + }; } interface NpmSpecifierPluginOptions { @@ -128,21 +140,14 @@ export function createNpmSpecifierPlugin(options?: NpmSpecifierPluginOptions): P interface CompileOptions { /** - * Max entries kept in the mtime-keyed LRU. Each entry is the full - * compiled bundle (~13 MB today, smaller as transitive deps are - * externalised). Cap default 8 keeps memory bounded; pass a smaller - * value in memory-constrained environments. + * Max entries kept in the mtime-keyed LRU. Each entry is the compiled + * bundle — now just the connector's own code + its bundled npm deps, + * since the SDK and its infra graph are externalised. Cap default 8 + * keeps memory bounded; pass a smaller value in memory-constrained + * environments. * @default 8 */ cacheMax?: number; - /** - * Override the `@lobu/connector-sdk` entry esbuild aliases against. - * Defaults to the SDK resolved from this module's `require.resolve`. - * Overriding is only useful when the caller knows of a more - * appropriate physical file (e.g. a server bundle that wants to point - * back at a sibling dist file). - */ - sdkEntry?: string; /** * Hook fired when `npm:` specifiers fail to resolve and the import is * externalised. Forwarded to `createNpmSpecifierPlugin`. @@ -158,19 +163,21 @@ const DEFAULT_CACHE_MAX = 8; * * The returned bundle: * - is ESM (`format: 'esm'`, `target: 'node20'`); - * - aliases `lobu` and `@lobu/connector-sdk` to the SDK entry so - * connectors targeting either specifier resolve to the same module; + * - externalises the connector SDK (`lobu` / `@lobu/connector-sdk`) — the + * runtime provides it, keeping bundles to the connector's own code + deps; * - has a banner injecting a CJS-compatible `require` shim; * - externalises `EXTERNAL_RUNTIME_DEPS` (native deps + Playwright); + * - emits an inline source map (`sourcesContent: false`) so connector stack + * traces map to source lines without embedding the source in the artifact; * - is mtime-cached: a repeat call with the same `filePath` whose * mtime hasn't changed returns the cached bundle without hitting * esbuild. */ export function createConnectorCompiler(options?: CompileOptions) { const cacheMax = options?.cacheMax ?? DEFAULT_CACHE_MAX; - const sdkEntry = options?.sdkEntry ?? resolveSdkEntry(); const compiledFileCache = new Map(); - const plugin = createNpmSpecifierPlugin({ onUnresolved: options?.onUnresolvedNpm }); + const npmPlugin = createNpmSpecifierPlugin({ onUnresolved: options?.onUnresolvedNpm }); + const sdkExternalPlugin = createSdkExternalPlugin(); function touchCacheEntry(filePath: string, entry: { mtimeMs: number; code: string }): void { compiledFileCache.delete(filePath); @@ -206,15 +213,15 @@ export function createConnectorCompiler(options?: CompileOptions) { format: 'esm', platform: 'node', target: 'node20', - alias: { lobu: sdkEntry, '@lobu/connector-sdk': sdkEntry }, banner: { js: `import { createRequire as __createRequire } from 'module'; const require = __createRequire(import.meta.url);`, }, - plugins: [plugin], + plugins: [sdkExternalPlugin, npmPlugin], external: [...EXTERNAL_RUNTIME_DEPS], write: true, minify: false, - sourcemap: false, + sourcemap: 'inline', + sourcesContent: false, }); const code = await readFile(outPath, 'utf-8'); diff --git a/packages/connector-worker/src/daemon/client.ts b/packages/connector-worker/src/daemon/client.ts index 21081387b..3a01810f7 100644 --- a/packages/connector-worker/src/daemon/client.ts +++ b/packages/connector-worker/src/daemon/client.ts @@ -96,6 +96,12 @@ export interface PollResponse { * sources). */ compiled_code?: string; + /** + * Native (nixpkgs) packages the connector declared in `runtime.nix.packages`. + * The executor wraps the child in `nix-shell -p ` so the tools are + * on PATH. Absent/empty = plain subprocess (the common case). + */ + nix_packages?: string[]; /** Connection session state (browser cookies, etc.) */ session_state?: Record; /** Connector version */ diff --git a/packages/connector-worker/src/daemon/executor.ts b/packages/connector-worker/src/daemon/executor.ts index 0ce3cc420..ca476250d 100644 --- a/packages/connector-worker/src/daemon/executor.ts +++ b/packages/connector-worker/src/daemon/executor.ts @@ -204,6 +204,7 @@ async function executeSyncRun( const result = await executeCompiledConnector({ compiledCode: compiled_code, + nixPackages: job.nix_packages, executor: subprocessExecutor, job: { mode: 'sync', @@ -367,6 +368,7 @@ async function executeActionRun( try { const result = await executeCompiledConnector({ compiledCode: compiled_code, + nixPackages: job.nix_packages, executor: subprocessExecutor, job: { mode: 'action', @@ -460,6 +462,7 @@ async function executeAuthRun( try { const result = await executeCompiledConnector({ compiledCode: compiled_code, + nixPackages: job.nix_packages, executor: subprocessExecutor, job: { mode: 'authenticate', diff --git a/packages/connector-worker/src/executor/interface.ts b/packages/connector-worker/src/executor/interface.ts index 771155bb5..bec4f1303 100644 --- a/packages/connector-worker/src/executor/interface.ts +++ b/packages/connector-worker/src/executor/interface.ts @@ -69,6 +69,16 @@ export interface ExecutionHooks { ) => Promise>; } +/** Per-run execution options independent of the job payload. */ +export interface ExecutionOptions { + /** + * Native system packages (nixpkgs attribute refs) the connector declared in + * its `runtime.nix.packages`. When non-empty, the embedded executor wraps the + * child process in `nix-shell -p ` so the tools are on PATH. + */ + nixPackages?: string[]; +} + /** * Pluggable executor interface. The only implementation today is * `SubprocessExecutor`; the seam stays around so tests can stub it. @@ -77,6 +87,7 @@ export interface SyncExecutor { execute( compiledCode: string, job: ExecutorJob, - hooks?: ExecutionHooks + hooks?: ExecutionHooks, + options?: ExecutionOptions ): Promise; } diff --git a/packages/connector-worker/src/executor/runtime.ts b/packages/connector-worker/src/executor/runtime.ts index 27f2c9d3b..4a2941ccb 100644 --- a/packages/connector-worker/src/executor/runtime.ts +++ b/packages/connector-worker/src/executor/runtime.ts @@ -1,4 +1,9 @@ -import type { ExecutionHooks, ExecutorJob, ExecutorResult, SyncExecutor } from './interface.js'; +import type { + ExecutionHooks, + ExecutorJob, + ExecutorResult, + SyncExecutor, +} from './interface.js'; import { SubprocessExecutor } from './subprocess.js'; /** @@ -11,7 +16,11 @@ export async function executeCompiledConnector(params: { job: ExecutorJob; executor?: SyncExecutor; hooks?: ExecutionHooks; + /** Native (nixpkgs) packages the connector declared in `runtime.nix.packages`. */ + nixPackages?: string[]; }): Promise { const executor = params.executor ?? new SubprocessExecutor(); - return executor.execute(params.compiledCode, params.job, params.hooks); + return executor.execute(params.compiledCode, params.job, params.hooks, { + nixPackages: params.nixPackages, + }); } diff --git a/packages/connector-worker/src/executor/subprocess.ts b/packages/connector-worker/src/executor/subprocess.ts index cb6ec225a..2eb411f90 100644 --- a/packages/connector-worker/src/executor/subprocess.ts +++ b/packages/connector-worker/src/executor/subprocess.ts @@ -6,15 +6,39 @@ * This is not a hardened security sandbox. */ -import { fork } from 'node:child_process'; +import { type ChildProcess, fork, spawn, spawnSync } from 'node:child_process'; import { existsSync } from 'node:fs'; import { createRequire } from 'node:module'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import type { EventEnvelope } from '@lobu/connector-sdk'; -import type { ExecutionHooks, ExecutorJob, ExecutorResult, SyncExecutor } from './interface.js'; +import type { + ExecutionHooks, + ExecutionOptions, + ExecutorJob, + ExecutorResult, + SyncExecutor, +} from './interface.js'; import { StreamRedactor, redactOutput } from './redact.js'; +/** Memoized nix-shell availability check. */ +let nixShellAvailable: boolean | null = null; +function hasNixShell(): boolean { + if (nixShellAvailable === null) { + try { + nixShellAvailable = spawnSync('nix-shell', ['--version'], { stdio: 'ignore' }).status === 0; + } catch { + nixShellAvailable = false; + } + } + return nixShellAvailable; +} + +/** Single-quote a string for safe embedding in a `nix-shell --run "..."` bash command. */ +function shellQuote(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'`; +} + /** * exit_reason values surfaced to the runs table: * - ok: successful 'result' IPC. @@ -130,8 +154,17 @@ export class SubprocessExecutor implements SyncExecutor { async execute( compiledCode: string, job: ExecutorJob, - hooks?: ExecutionHooks + hooks?: ExecutionHooks, + options?: ExecutionOptions ): Promise { + const nixPackages = options?.nixPackages ?? []; + if (nixPackages.length > 0 && !hasNixShell()) { + throw new Error( + `This connector requires native packages [${nixPackages.join(', ')}] but \`nix-shell\` ` + + `is not installed on this host. Install nix (https://nixos.org/download) or run the ` + + `connector on a backend that provisions native dependencies.` + ); + } return new Promise((resolve, reject) => { let childRunnerPath = join(__dirname, 'child-runner.js'); const childRunnerTsPath = join(__dirname, 'child-runner.ts'); @@ -165,11 +198,30 @@ export class SubprocessExecutor implements SyncExecutor { // Node subprocess execution is process isolation, not a security sandbox. // Node --experimental-permission flags intentionally NOT enabled — the // connector runtime isn't compatible. Revisit if that changes. - const child = fork(childRunnerPath, [], { - stdio: ['pipe', 'pipe', 'pipe', 'ipc'], - execArgv, - env: { ...pickSystemEnv(), ...jobEnv(job) } as NodeJS.ProcessEnv, - }); + const env = { ...pickSystemEnv(), ...jobEnv(job) } as NodeJS.ProcessEnv; + let child: ChildProcess; + if (nixPackages.length > 0) { + // Wrap in nix-shell so the connector's declared native tools are on + // PATH. `exec node` replaces the shell with node so the IPC channel + // (fd 3 / NODE_CHANNEL_FD created by the 'ipc' stdio slot) and kill() + // reach the real process — without `exec`, kill() would hit the shell + // and orphan node. execArgv is rebuilt as inline flags. nix-shell's + // impure shell keeps the ambient PATH, so `node` still resolves. + const nodeCmd = ['exec', 'node', ...execArgv, shellQuote(childRunnerPath)].join(' '); + const nixArgs: string[] = []; + for (const pkg of nixPackages) nixArgs.push('-p', pkg); + nixArgs.push('--run', nodeCmd); + child = spawn('nix-shell', nixArgs, { + stdio: ['pipe', 'pipe', 'pipe', 'ipc'], + env, + }); + } else { + child = fork(childRunnerPath, [], { + stdio: ['pipe', 'pipe', 'pipe', 'ipc'], + execArgv, + env, + }); + } let resolved = false; let terminalMessageReceived = false; diff --git a/packages/pgvector-embedded/src/index.ts b/packages/pgvector-embedded/src/index.ts index 0f6bd0dc2..1fd46b63f 100644 --- a/packages/pgvector-embedded/src/index.ts +++ b/packages/pgvector-embedded/src/index.ts @@ -12,7 +12,13 @@ * against a same-major PostgreSQL — the extension ABI is stable within a major, * so a library built against PG 18.x loads into `embedded-postgres`'s PG 18.x. */ -import { cpSync, existsSync, mkdirSync, readdirSync } from "node:fs"; +import { + cpSync, + existsSync, + mkdirSync, + readdirSync, + symlinkSync, +} from "node:fs"; import { createRequire } from "node:module"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -58,10 +64,44 @@ export function resolveEmbeddedNativeDir( return join(dirname(entry), "..", "native"); } +/** + * embedded-postgres' macOS native libs ship only as the fully-versioned + * `lib...dylib`, but the Postgres binaries link against both + * the major SONAME (`lib..dylib`) and the unversioned linker name + * (`lib.dylib`, e.g. postgres' `@loader_path/../lib/libicui18n.dylib`) — + * and the @embedded-postgres/darwin-* packages omit those symlinks. Without them + * `initdb`/`postgres` abort with `Library not loaded`. Recreate both link levels + * idempotently, pointing at the real versioned file. No-op off macOS (Linux ships + * the `.so.` links) and when a link already exists. + */ +export function ensureNativeLibSonames( + nativeDir: string = resolveEmbeddedNativeDir() +): void { + if (process.platform !== "darwin") return; + const libDir = join(nativeDir, "lib"); + if (!existsSync(libDir)) return; + for (const file of readdirSync(libDir)) { + // lib...dylib → { lib..dylib, lib.dylib } + const match = file.match(/^(lib.+?)\.(\d+)\.\d+\.dylib$/); + if (!match) continue; + const [, base, major] = match; + for (const soname of [`${base}.${major}.dylib`, `${base}.dylib`]) { + const link = join(libDir, soname); + if (existsSync(link)) continue; + try { + symlinkSync(file, link); // relative target within libDir + } catch { + // Best-effort: a race or perms issue surfaces as the real initdb error. + } + } + } +} + /** * Copy the host platform's prebuilt pgvector files into an embedded-postgres - * `native` tree so `CREATE EXTENSION vector` works. Idempotent — returns early - * if pgvector is already present in the tree. + * `native` tree so `CREATE EXTENSION vector` works, and hydrate the macOS ICU + * SONAME symlinks so `initdb` can link. Idempotent — returns early if pgvector + * is already present in the tree. * * @param nativeDir absolute path to `.../native`; defaults to the resolved * host-platform `@embedded-postgres` package. @@ -70,6 +110,10 @@ export function injectPgvector( nativeDir: string = resolveEmbeddedNativeDir(), platform: string = currentPlatformKey() ): void { + // Always (re)hydrate the dylib SONAME symlinks — needed for initdb to link + // even when pgvector was already injected on a prior run. + ensureNativeLibSonames(nativeDir); + const libDst = join(nativeDir, "lib", "postgresql"); const extDst = join(nativeDir, "share", "postgresql", "extension"); diff --git a/packages/server/src/worker-api.ts b/packages/server/src/worker-api.ts index 0aa86deb3..824ed83bb 100644 --- a/packages/server/src/worker-api.ts +++ b/packages/server/src/worker-api.ts @@ -489,6 +489,7 @@ export async function pollWorkerJob(c: Context<{ Bindings: Env }>) { conn.config AS connection_config, conn.device_worker_id AS connection_device_worker_id, cv.compiled_code, + cd.runtime AS connector_runtime, ap.auth_data AS auth_profile_auth_data, w.name AS watcher_name, w.slug AS watcher_slug, @@ -500,6 +501,9 @@ export async function pollWorkerJob(c: Context<{ Bindings: Env }>) { LEFT JOIN connections conn ON conn.id = r.connection_id LEFT JOIN connector_versions cv ON cv.connector_key = r.connector_key AND cv.version = r.connector_version + LEFT JOIN connector_definitions cd ON cd.key = r.connector_key + AND cd.organization_id = r.organization_id + AND cd.status = 'active' LEFT JOIN auth_profiles ap ON ap.id = r.auth_profile_id LEFT JOIN watchers w ON w.id = r.watcher_id WHERE r.id = ${runId} @@ -555,6 +559,7 @@ export async function pollWorkerJob(c: Context<{ Bindings: Env }>) { connection_config: Record | null; connection_device_worker_id: string | null; compiled_code: string | null; + connector_runtime: { nix?: { packages?: string[] } | null } | null; run_created_at: string | Date | null; // Watcher run fields (populated via LEFT JOINs) watcher_id: number | null; @@ -691,11 +696,18 @@ export async function pollWorkerJob(c: Context<{ Bindings: Env }>) { sessionState: null, }; + // Native (nixpkgs) packages the connector declared in `runtime.nix.packages`. + // The worker provisions these on PATH via nix-shell before executing. + const nixPackages = (row.connector_runtime?.nix?.packages ?? []).filter( + (p): p is string => typeof p === 'string' + ); + return c.json({ run_id: row.run_id, run_type: row.run_type, connector_key: row.connector_key, connector_version: row.connector_version ?? undefined, + nix_packages: nixPackages.length > 0 ? nixPackages : undefined, feed_key: row.feed_key ?? undefined, feed_id: row.feed_id ?? undefined, connection_id: row.connection_id ?? undefined,