From a8f4e81053ecfd85bae2a8e19f1dca86def4dcda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Wed, 20 May 2026 18:13:40 +0100 Subject: [PATCH 1/8] feat(connector-worker): connector native-dep declaration + lean bundles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundation for user-declared connector dependencies: - SDK: add `runtime.nix.packages` to ConnectorRuntimeInfo so a connector declares its native (nixpkgs) tools; rides the existing `runtime` JSONB, no migration. npm deps are bundled at compile time and don't go here. - Compile: externalize `@lobu/connector-sdk` (+ `lobu` alias) instead of bundling it. The SDK pulls a large infra graph (Sentry/OTel/grpc/git) transitively, inflating every connector to multiple MB; the runtime provides the SDK (it's a connector-worker dep), so the bundle leaves it as a runtime-resolved import — the standard externalize-the-framework pattern. Bundles drop from ~MB to the connector's own code + deps. Also emit an inline source map (sourcesContent:false) for stack traces. - Executor: when a connector declares nix packages, wrap the child in `nix-shell -p --run "exec node ..."` so the tools are on PATH; `exec` preserves the IPC channel and kill semantics. Plain fork() stays the path when no packages are declared. Fail with a clear actionable error when nix-shell is absent but packages are required. --- packages/connector-sdk/src/connector-types.ts | 9 +++ .../connector-worker/src/compile/index.ts | 63 +++++++++-------- .../src/executor/interface.ts | 13 +++- .../src/executor/subprocess.ts | 68 ++++++++++++++++--- 4 files changed, 116 insertions(+), 37 deletions(-) diff --git a/packages/connector-sdk/src/connector-types.ts b/packages/connector-sdk/src/connector-types.ts index 69441559d..b222d1aec 100644 --- a/packages/connector-sdk/src/connector-types.ts +++ b/packages/connector-sdk/src/connector-types.ts @@ -65,6 +65,15 @@ export interface ConnectorRuntimeInfo { * Optional — omit when the platform adapter needs no fine-grained scope list. */ scopes?: string[]; + /** + * Native system dependencies this connector needs on PATH at execution time, + * as nixpkgs attribute references (e.g. `["ffmpeg", "imagemagick"]`). npm + * dependencies are bundled into the connector at compile time and do NOT go + * here — only native tools the runtime must provision. Backends that can run + * native deps (embedded, container, machine) satisfy these via nix; backends + * that can't (e.g. edge workers) reject a connector that declares them. + */ + nix?: { packages: string[] }; } // ============================================================================= diff --git a/packages/connector-worker/src/compile/index.ts b/packages/connector-worker/src/compile/index.ts index f0a9b4b0e..b54662b73 100644 --- a/packages/connector-worker/src/compile/index.ts +++ b/packages/connector-worker/src/compile/index.ts @@ -22,7 +22,6 @@ import { existsSync } from 'node:fs'; import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; -import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { build, type Plugin } from 'esbuild'; @@ -75,15 +74,28 @@ export function findBundledConnectorFile( } /** - * Resolve the `@lobu/connector-sdk` module entry from this module's - * perspective. Used as the esbuild `alias` target so connector code that - * imports `from 'lobu'` or `from '@lobu/connector-sdk'` resolves to the - * same physical file the runtime will import — avoiding the - * `instanceof ConnectorRuntime` cross-realm trap. + * esbuild plugin that marks the connector SDK as **external** (runtime-provided) + * rather than bundling it in. The SDK pulls a large infra graph transitively + * (Sentry, OpenTelemetry, grpc, isomorphic-git, …); bundling it inflated every + * connector to multiple MB. The runtime that executes the connector already has + * `@lobu/connector-sdk` installed (it's a dependency of `@lobu/connector-worker`), + * so the bundle leaves it as a bare import and Node resolves it from the runtime's + * node_modules at load time — the standard "externalize the framework, bundle the + * user code" pattern (cf. AWS Lambda not bundling `@aws-sdk`). + * + * The `lobu` alias specifier is normalized to `@lobu/connector-sdk` so the emitted + * import resolves to a real package the runtime provides. */ -function resolveSdkEntry(): string { - const require_ = createRequire(import.meta.url); - return require_.resolve('@lobu/connector-sdk'); +function createSdkExternalPlugin(): Plugin { + return { + name: 'sdk-external', + setup(b) { + b.onResolve({ filter: /^(lobu|@lobu\/connector-sdk)(\/.*)?$/ }, (args) => ({ + path: args.path.replace(/^lobu(?=$|\/)/, '@lobu/connector-sdk'), + external: true, + })); + }, + }; } interface NpmSpecifierPluginOptions { @@ -128,21 +140,14 @@ export function createNpmSpecifierPlugin(options?: NpmSpecifierPluginOptions): P interface CompileOptions { /** - * Max entries kept in the mtime-keyed LRU. Each entry is the full - * compiled bundle (~13 MB today, smaller as transitive deps are - * externalised). Cap default 8 keeps memory bounded; pass a smaller - * value in memory-constrained environments. + * Max entries kept in the mtime-keyed LRU. Each entry is the compiled + * bundle — now just the connector's own code + its bundled npm deps, + * since the SDK and its infra graph are externalised. Cap default 8 + * keeps memory bounded; pass a smaller value in memory-constrained + * environments. * @default 8 */ cacheMax?: number; - /** - * Override the `@lobu/connector-sdk` entry esbuild aliases against. - * Defaults to the SDK resolved from this module's `require.resolve`. - * Overriding is only useful when the caller knows of a more - * appropriate physical file (e.g. a server bundle that wants to point - * back at a sibling dist file). - */ - sdkEntry?: string; /** * Hook fired when `npm:` specifiers fail to resolve and the import is * externalised. Forwarded to `createNpmSpecifierPlugin`. @@ -158,19 +163,21 @@ const DEFAULT_CACHE_MAX = 8; * * The returned bundle: * - is ESM (`format: 'esm'`, `target: 'node20'`); - * - aliases `lobu` and `@lobu/connector-sdk` to the SDK entry so - * connectors targeting either specifier resolve to the same module; + * - externalises the connector SDK (`lobu` / `@lobu/connector-sdk`) — the + * runtime provides it, keeping bundles to the connector's own code + deps; * - has a banner injecting a CJS-compatible `require` shim; * - externalises `EXTERNAL_RUNTIME_DEPS` (native deps + Playwright); + * - emits an inline source map (`sourcesContent: false`) so connector stack + * traces map to source lines without embedding the source in the artifact; * - is mtime-cached: a repeat call with the same `filePath` whose * mtime hasn't changed returns the cached bundle without hitting * esbuild. */ export function createConnectorCompiler(options?: CompileOptions) { const cacheMax = options?.cacheMax ?? DEFAULT_CACHE_MAX; - const sdkEntry = options?.sdkEntry ?? resolveSdkEntry(); const compiledFileCache = new Map(); - const plugin = createNpmSpecifierPlugin({ onUnresolved: options?.onUnresolvedNpm }); + const npmPlugin = createNpmSpecifierPlugin({ onUnresolved: options?.onUnresolvedNpm }); + const sdkExternalPlugin = createSdkExternalPlugin(); function touchCacheEntry(filePath: string, entry: { mtimeMs: number; code: string }): void { compiledFileCache.delete(filePath); @@ -206,15 +213,15 @@ export function createConnectorCompiler(options?: CompileOptions) { format: 'esm', platform: 'node', target: 'node20', - alias: { lobu: sdkEntry, '@lobu/connector-sdk': sdkEntry }, banner: { js: `import { createRequire as __createRequire } from 'module'; const require = __createRequire(import.meta.url);`, }, - plugins: [plugin], + plugins: [sdkExternalPlugin, npmPlugin], external: [...EXTERNAL_RUNTIME_DEPS], write: true, minify: false, - sourcemap: false, + sourcemap: 'inline', + sourcesContent: false, }); const code = await readFile(outPath, 'utf-8'); diff --git a/packages/connector-worker/src/executor/interface.ts b/packages/connector-worker/src/executor/interface.ts index 771155bb5..bec4f1303 100644 --- a/packages/connector-worker/src/executor/interface.ts +++ b/packages/connector-worker/src/executor/interface.ts @@ -69,6 +69,16 @@ export interface ExecutionHooks { ) => Promise>; } +/** Per-run execution options independent of the job payload. */ +export interface ExecutionOptions { + /** + * Native system packages (nixpkgs attribute refs) the connector declared in + * its `runtime.nix.packages`. When non-empty, the embedded executor wraps the + * child process in `nix-shell -p ` so the tools are on PATH. + */ + nixPackages?: string[]; +} + /** * Pluggable executor interface. The only implementation today is * `SubprocessExecutor`; the seam stays around so tests can stub it. @@ -77,6 +87,7 @@ export interface SyncExecutor { execute( compiledCode: string, job: ExecutorJob, - hooks?: ExecutionHooks + hooks?: ExecutionHooks, + options?: ExecutionOptions ): Promise; } diff --git a/packages/connector-worker/src/executor/subprocess.ts b/packages/connector-worker/src/executor/subprocess.ts index cb6ec225a..2eb411f90 100644 --- a/packages/connector-worker/src/executor/subprocess.ts +++ b/packages/connector-worker/src/executor/subprocess.ts @@ -6,15 +6,39 @@ * This is not a hardened security sandbox. */ -import { fork } from 'node:child_process'; +import { type ChildProcess, fork, spawn, spawnSync } from 'node:child_process'; import { existsSync } from 'node:fs'; import { createRequire } from 'node:module'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import type { EventEnvelope } from '@lobu/connector-sdk'; -import type { ExecutionHooks, ExecutorJob, ExecutorResult, SyncExecutor } from './interface.js'; +import type { + ExecutionHooks, + ExecutionOptions, + ExecutorJob, + ExecutorResult, + SyncExecutor, +} from './interface.js'; import { StreamRedactor, redactOutput } from './redact.js'; +/** Memoized nix-shell availability check. */ +let nixShellAvailable: boolean | null = null; +function hasNixShell(): boolean { + if (nixShellAvailable === null) { + try { + nixShellAvailable = spawnSync('nix-shell', ['--version'], { stdio: 'ignore' }).status === 0; + } catch { + nixShellAvailable = false; + } + } + return nixShellAvailable; +} + +/** Single-quote a string for safe embedding in a `nix-shell --run "..."` bash command. */ +function shellQuote(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'`; +} + /** * exit_reason values surfaced to the runs table: * - ok: successful 'result' IPC. @@ -130,8 +154,17 @@ export class SubprocessExecutor implements SyncExecutor { async execute( compiledCode: string, job: ExecutorJob, - hooks?: ExecutionHooks + hooks?: ExecutionHooks, + options?: ExecutionOptions ): Promise { + const nixPackages = options?.nixPackages ?? []; + if (nixPackages.length > 0 && !hasNixShell()) { + throw new Error( + `This connector requires native packages [${nixPackages.join(', ')}] but \`nix-shell\` ` + + `is not installed on this host. Install nix (https://nixos.org/download) or run the ` + + `connector on a backend that provisions native dependencies.` + ); + } return new Promise((resolve, reject) => { let childRunnerPath = join(__dirname, 'child-runner.js'); const childRunnerTsPath = join(__dirname, 'child-runner.ts'); @@ -165,11 +198,30 @@ export class SubprocessExecutor implements SyncExecutor { // Node subprocess execution is process isolation, not a security sandbox. // Node --experimental-permission flags intentionally NOT enabled — the // connector runtime isn't compatible. Revisit if that changes. - const child = fork(childRunnerPath, [], { - stdio: ['pipe', 'pipe', 'pipe', 'ipc'], - execArgv, - env: { ...pickSystemEnv(), ...jobEnv(job) } as NodeJS.ProcessEnv, - }); + const env = { ...pickSystemEnv(), ...jobEnv(job) } as NodeJS.ProcessEnv; + let child: ChildProcess; + if (nixPackages.length > 0) { + // Wrap in nix-shell so the connector's declared native tools are on + // PATH. `exec node` replaces the shell with node so the IPC channel + // (fd 3 / NODE_CHANNEL_FD created by the 'ipc' stdio slot) and kill() + // reach the real process — without `exec`, kill() would hit the shell + // and orphan node. execArgv is rebuilt as inline flags. nix-shell's + // impure shell keeps the ambient PATH, so `node` still resolves. + const nodeCmd = ['exec', 'node', ...execArgv, shellQuote(childRunnerPath)].join(' '); + const nixArgs: string[] = []; + for (const pkg of nixPackages) nixArgs.push('-p', pkg); + nixArgs.push('--run', nodeCmd); + child = spawn('nix-shell', nixArgs, { + stdio: ['pipe', 'pipe', 'pipe', 'ipc'], + env, + }); + } else { + child = fork(childRunnerPath, [], { + stdio: ['pipe', 'pipe', 'pipe', 'ipc'], + execArgv, + env, + }); + } let resolved = false; let terminalMessageReceived = false; From ac575276c0dbbff9d7eded8e3dbdc5e7ac276d6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Wed, 20 May 2026 18:19:15 +0100 Subject: [PATCH 2/8] feat(server): provision connector native deps via nix at run time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Threads a connector's declared native packages from storage to the executor so they're on PATH during a run: - worker-api poll: join connector_definitions and surface its `runtime`, emitting `nix_packages` in the poll response (the `runtime.nix.packages` the SDK extraction already persists in the existing runtime JSONB — no storage change needed). - daemon: carry `nix_packages` on PollResponse and pass it through the three executeCompiledConnector call sites (sync/action/auth). - executeCompiledConnector: forward `nixPackages` to executor.execute, which wraps the child in nix-shell when non-empty. Connectors that declare no native deps are unaffected (plain fork path). --- packages/connector-worker/src/daemon/client.ts | 6 ++++++ packages/connector-worker/src/daemon/executor.ts | 3 +++ packages/connector-worker/src/executor/runtime.ts | 13 +++++++++++-- packages/server/src/worker-api.ts | 11 +++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/packages/connector-worker/src/daemon/client.ts b/packages/connector-worker/src/daemon/client.ts index 21081387b..3a01810f7 100644 --- a/packages/connector-worker/src/daemon/client.ts +++ b/packages/connector-worker/src/daemon/client.ts @@ -96,6 +96,12 @@ export interface PollResponse { * sources). */ compiled_code?: string; + /** + * Native (nixpkgs) packages the connector declared in `runtime.nix.packages`. + * The executor wraps the child in `nix-shell -p ` so the tools are + * on PATH. Absent/empty = plain subprocess (the common case). + */ + nix_packages?: string[]; /** Connection session state (browser cookies, etc.) */ session_state?: Record; /** Connector version */ diff --git a/packages/connector-worker/src/daemon/executor.ts b/packages/connector-worker/src/daemon/executor.ts index 0ce3cc420..ca476250d 100644 --- a/packages/connector-worker/src/daemon/executor.ts +++ b/packages/connector-worker/src/daemon/executor.ts @@ -204,6 +204,7 @@ async function executeSyncRun( const result = await executeCompiledConnector({ compiledCode: compiled_code, + nixPackages: job.nix_packages, executor: subprocessExecutor, job: { mode: 'sync', @@ -367,6 +368,7 @@ async function executeActionRun( try { const result = await executeCompiledConnector({ compiledCode: compiled_code, + nixPackages: job.nix_packages, executor: subprocessExecutor, job: { mode: 'action', @@ -460,6 +462,7 @@ async function executeAuthRun( try { const result = await executeCompiledConnector({ compiledCode: compiled_code, + nixPackages: job.nix_packages, executor: subprocessExecutor, job: { mode: 'authenticate', diff --git a/packages/connector-worker/src/executor/runtime.ts b/packages/connector-worker/src/executor/runtime.ts index 27f2c9d3b..4a2941ccb 100644 --- a/packages/connector-worker/src/executor/runtime.ts +++ b/packages/connector-worker/src/executor/runtime.ts @@ -1,4 +1,9 @@ -import type { ExecutionHooks, ExecutorJob, ExecutorResult, SyncExecutor } from './interface.js'; +import type { + ExecutionHooks, + ExecutorJob, + ExecutorResult, + SyncExecutor, +} from './interface.js'; import { SubprocessExecutor } from './subprocess.js'; /** @@ -11,7 +16,11 @@ export async function executeCompiledConnector(params: { job: ExecutorJob; executor?: SyncExecutor; hooks?: ExecutionHooks; + /** Native (nixpkgs) packages the connector declared in `runtime.nix.packages`. */ + nixPackages?: string[]; }): Promise { const executor = params.executor ?? new SubprocessExecutor(); - return executor.execute(params.compiledCode, params.job, params.hooks); + return executor.execute(params.compiledCode, params.job, params.hooks, { + nixPackages: params.nixPackages, + }); } diff --git a/packages/server/src/worker-api.ts b/packages/server/src/worker-api.ts index 0aa86deb3..b170eafc3 100644 --- a/packages/server/src/worker-api.ts +++ b/packages/server/src/worker-api.ts @@ -489,6 +489,7 @@ export async function pollWorkerJob(c: Context<{ Bindings: Env }>) { conn.config AS connection_config, conn.device_worker_id AS connection_device_worker_id, cv.compiled_code, + cd.runtime AS connector_runtime, ap.auth_data AS auth_profile_auth_data, w.name AS watcher_name, w.slug AS watcher_slug, @@ -500,6 +501,8 @@ export async function pollWorkerJob(c: Context<{ Bindings: Env }>) { LEFT JOIN connections conn ON conn.id = r.connection_id LEFT JOIN connector_versions cv ON cv.connector_key = r.connector_key AND cv.version = r.connector_version + LEFT JOIN connector_definitions cd ON cd.key = r.connector_key + AND cd.organization_id = r.organization_id LEFT JOIN auth_profiles ap ON ap.id = r.auth_profile_id LEFT JOIN watchers w ON w.id = r.watcher_id WHERE r.id = ${runId} @@ -555,6 +558,7 @@ export async function pollWorkerJob(c: Context<{ Bindings: Env }>) { connection_config: Record | null; connection_device_worker_id: string | null; compiled_code: string | null; + connector_runtime: { nix?: { packages?: string[] } | null } | null; run_created_at: string | Date | null; // Watcher run fields (populated via LEFT JOINs) watcher_id: number | null; @@ -691,11 +695,18 @@ export async function pollWorkerJob(c: Context<{ Bindings: Env }>) { sessionState: null, }; + // Native (nixpkgs) packages the connector declared in `runtime.nix.packages`. + // The worker provisions these on PATH via nix-shell before executing. + const nixPackages = (row.connector_runtime?.nix?.packages ?? []).filter( + (p): p is string => typeof p === 'string' + ); + return c.json({ run_id: row.run_id, run_type: row.run_type, connector_key: row.connector_key, connector_version: row.connector_version ?? undefined, + nix_packages: nixPackages.length > 0 ? nixPackages : undefined, feed_key: row.feed_key ?? undefined, feed_id: row.feed_id ?? undefined, connection_id: row.connection_id ?? undefined, From d685d723e913f9577f77ede72cf1f6f80d1273ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Wed, 20 May 2026 18:34:58 +0100 Subject: [PATCH 3/8] feat(cli): compile connectors on apply with project deps; scaffold packaging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Connector npm deps now work in the apply→cloud path: - apply: project-supplied connectors are compiled on the CLI (the only place the project's node_modules exists, so esbuild can bundle the connector's declared deps) and uploaded as a pre-compiled bundle (`compiled: true`); the server stores the artifact instead of recompiling source it can't resolve deps for. - ensure-deps-installed: runs `bun install --ignore-scripts` in the connector's project (resolved via the nearest `lobu.toml`, so a connector inside a monorepo never installs the wrong root); no-ops when the project declares no package.json. - client.installConnector gains a `compiled` flag. - lobu init scaffolds package.json (with @lobu/connector-sdk devDependency for editor types) + tsconfig.json + connectors/. - AGENTS.md documents the convention: npm = bundled at compile, native = nix at run time; SDK is runtime-provided/externalized. --- AGENTS.md | 5 ++ .../cli/src/commands/_lib/apply/apply-cmd.ts | 18 +++- .../cli/src/commands/_lib/apply/client.ts | 4 +- .../commands/_lib/ensure-deps-installed.ts | 85 +++++++++++++++++++ packages/cli/src/commands/init.ts | 40 +++++++++ 5 files changed, 147 insertions(+), 5 deletions(-) create mode 100644 packages/cli/src/commands/_lib/ensure-deps-installed.ts diff --git a/AGENTS.md b/AGENTS.md index 826467355..c18350d89 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,6 +57,11 @@ All chat platforms (Telegram, Slack, Discord, WhatsApp, Teams) run through Chat - **Integration auth lives in Lobu** — OAuth, token refresh, and API proxying for third-party services (GitHub, Google, etc.) are handled by Lobu MCP servers. Workers never see OAuth tokens. - **`events` is append-only.** Never `DELETE FROM events`. To hide a row, write a tombstone via `client.knowledge.delete()` or `save_knowledge({ supersedes_event_id, ... })`; the `current_event_records` view masks superseded rows, `include_superseded` recovers history. +#### Connectors +- A connector is a `*.connector.ts` extending `ConnectorRuntime`. **npm deps** go in the project's `package.json` (next to `lobu.toml`) and are **bundled** into the connector by esbuild at compile time. **Native deps** (ffmpeg, imagemagick, …) are declared as nixpkgs refs in `runtime.nix.packages` on the connector `definition`; the runtime provisions them on PATH via `nix-shell` at execution. npm = bundled (compile-time); native = nix (run-time). +- **Compile happens on the CLI**, not the server: `lobu apply` runs `bun install` in the project, compiles each connector with the project's `node_modules`, and uploads the bundle (`compiled: true`). esbuild resolves a connector's imports relative to the connector file's dir, so the project's deps are used regardless of where the `lobu` binary is installed. `@lobu/connector-sdk` is **externalized** (runtime-provided, à la Lambda's `aws-sdk`) — the bundle stays the connector's own code + deps, not multiple MB of SDK infra. +- `lobu init` scaffolds `package.json` (with `@lobu/connector-sdk` devDependency for editor types) + `tsconfig.json` + `connectors/`. + #### Guardrails - Primitive lives in `packages/core/src/guardrails/`: `Guardrail`, `GuardrailRegistry`, `runGuardrails()`. Stages: `input` (user message → worker), `output` (worker text → user), `pre-tool` (tool call authorization). - Each guardrail's `run(ctx)` returns `{ tripped, reason?, metadata? }`. The runner races all enabled guardrails at a stage; the first trip short-circuits (later results are discarded) and a thrown guardrail is logged and treated as a pass. diff --git a/packages/cli/src/commands/_lib/apply/apply-cmd.ts b/packages/cli/src/commands/_lib/apply/apply-cmd.ts index 4b4e6ce9f..ac915bb6e 100644 --- a/packages/cli/src/commands/_lib/apply/apply-cmd.ts +++ b/packages/cli/src/commands/_lib/apply/apply-cmd.ts @@ -7,6 +7,8 @@ import { loadProjectLink } from "../../../internal/project-link.js"; import { CONFIG_FILENAME } from "../../../config/loader.js"; import { ApiError, ValidationError } from "../../memory/_lib/errors.js"; import { printError, printText } from "../../memory/_lib/output.js"; +import { compileConnectorFromFile } from "../connector-loader.js"; +import { ensureProjectDepsInstalled } from "../ensure-deps-installed.js"; import { type ApplyClient, type RemoteAgent, @@ -376,10 +378,18 @@ async function installConnectorDefinitions( if (row.verb === "noop" || row.verb === "drift") continue; const def = row.desired; if (!def) continue; - const result = - def.sourceCode !== undefined - ? await client.installConnector({ sourceCode: def.sourceCode }) - : await client.installConnector({ sourceUrl: def.sourceUrl }); + let result: Awaited>; + if (def.sourceCode !== undefined) { + // Compile project connectors on the CLI: only here is the project's + // node_modules available, so esbuild can bundle the connector's declared + // npm deps. The server can't (it only receives the artifact). Native deps + // ride `runtime.nix.packages` and are provisioned at run time. + ensureProjectDepsInstalled(def.sourceFile, printText); + const compiledCode = await compileConnectorFromFile(def.sourceFile); + result = await client.installConnector({ sourceCode: compiledCode, compiled: true }); + } else { + result = await client.installConnector({ sourceUrl: def.sourceUrl }); + } if (result.connectorKey) { locallySuppliedKeys.add(result.connectorKey); installedKeys.add(result.connectorKey); diff --git a/packages/cli/src/commands/_lib/apply/client.ts b/packages/cli/src/commands/_lib/apply/client.ts index 76e801f5c..ff0592125 100644 --- a/packages/cli/src/commands/_lib/apply/client.ts +++ b/packages/cli/src/commands/_lib/apply/client.ts @@ -851,6 +851,8 @@ export class ApplyClient { sourceUrl?: string; /** `file://` URI of a bundled connector source on the server host. */ sourceUri?: string; + /** `sourceCode` is already a compiled bundle (CLI-side compile) — skip server compile. */ + compiled?: boolean; }): Promise { const body = await this.connectionsTool<{ installed?: boolean; @@ -860,7 +862,7 @@ export class ApplyClient { }>({ action: "install_connector", ...(payload.sourceCode !== undefined - ? { source_code: payload.sourceCode, compiled: false } + ? { source_code: payload.sourceCode, compiled: payload.compiled ?? false } : {}), ...(payload.sourceUrl ? { source_url: payload.sourceUrl } : {}), ...(payload.sourceUri ? { source_uri: payload.sourceUri } : {}), diff --git a/packages/cli/src/commands/_lib/ensure-deps-installed.ts b/packages/cli/src/commands/_lib/ensure-deps-installed.ts new file mode 100644 index 000000000..bf78e021a --- /dev/null +++ b/packages/cli/src/commands/_lib/ensure-deps-installed.ts @@ -0,0 +1,85 @@ +/** + * Ensure a connector project's npm dependencies are installed before the CLI + * compiles its connectors. esbuild bundles a connector's imports relative to + * the connector file's directory, so the project's own `node_modules` (next to + * `package.json`) must exist. We run `bun install --ignore-scripts` when stale: + * `--ignore-scripts` keeps install-time supply-chain surface off the user's + * machine — packages that need build scripts (native bindings) belong in + * `runtime.nix.packages`, not bundled npm. + */ + +import { execFileSync } from "node:child_process"; +import { existsSync, statSync } from "node:fs"; +import { dirname, join } from "node:path"; + +// Per-process memo so `lobu apply` installs each project root at most once. +const ensuredRoots = new Set(); + +/** + * Find the connector's project root — the nearest ancestor with `lobu.toml`. + * Anchoring on `lobu.toml` (not any ancestor `package.json`) is what stops a + * connector inside a monorepo from resolving to the monorepo's root + * package.json and triggering a wrong-directory install. + */ +export function findProjectRoot(fromFile: string): string | null { + let dir = dirname(fromFile); + for (let i = 0; i < 40; i++) { + if (existsSync(join(dir, "lobu.toml"))) return dir; + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + return null; +} + +function installIsStale(root: string): boolean { + const nodeModules = join(root, "node_modules"); + if (!existsSync(nodeModules)) return true; + const lock = join(root, "bun.lock"); + if (!existsSync(lock)) return false; // deps present, no lockfile to compare against + try { + return statSync(lock).mtimeMs > statSync(nodeModules).mtimeMs; + } catch { + return false; + } +} + +function hasBun(): boolean { + try { + execFileSync("bun", ["--version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +} + +/** + * Install the connector project's deps if missing/stale. No-op when the + * connector has no `package.json` (no declared npm deps to bundle). + */ +export function ensureProjectDepsInstalled( + connectorFilePath: string, + log: (message: string) => void +): void { + const root = findProjectRoot(connectorFilePath); + if (!root || ensuredRoots.has(root)) return; + // No package.json at the project root → the connector declares no npm deps + // (the SDK is runtime-provided/externalized), so there's nothing to install. + if (!existsSync(join(root, "package.json"))) { + ensuredRoots.add(root); + return; + } + if (!installIsStale(root)) { + ensuredRoots.add(root); + return; + } + if (!hasBun()) { + throw new Error( + `Connector dependencies in ${root} need installing, but \`bun\` is not on PATH. ` + + `Run \`bun install\` in ${root}, or install bun (https://bun.sh).` + ); + } + log(`Installing connector dependencies in ${root}...`); + execFileSync("bun", ["install", "--ignore-scripts"], { cwd: root, stdio: "inherit" }); + ensuredRoots.add(root); +} diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index 6d8e28a14..bf328c6bf 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -705,6 +705,46 @@ export async function initCommand( await mkdir(join(projectDir, "skills"), { recursive: true }); await writeFile(join(projectDir, "skills", ".gitkeep"), ""); + // Connector authoring surface: package.json declares the connector SDK + // (provided by the runtime — externalized at compile, here for editor + // types) plus any npm deps the user adds; tsconfig gives the editor + // resolution; the connectors/ dir holds `*.connector.ts`. `lobu apply` + // runs `bun install` here and bundles each connector's own deps. + await writeFile( + join(projectDir, "package.json"), + `${JSON.stringify( + { + name: projectName, + version: "0.0.0", + private: true, + type: "module", + devDependencies: { "@lobu/connector-sdk": `^${cliVersion}` }, + }, + null, + 2 + )}\n` + ); + await writeFile( + join(projectDir, "tsconfig.json"), + `${JSON.stringify( + { + compilerOptions: { + target: "ES2022", + module: "Preserve", + moduleResolution: "bundler", + strict: true, + skipLibCheck: true, + noEmit: true, + }, + include: ["connectors/**/*.ts"], + }, + null, + 2 + )}\n` + ); + await mkdir(join(projectDir, "connectors"), { recursive: true }); + await writeFile(join(projectDir, "connectors", ".gitkeep"), ""); + await renderTemplate( "AGENTS.md.tmpl", variables, From cafe28afd4f4e18dd77c60e97781004d0686f30d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Wed, 20 May 2026 18:35:46 +0100 Subject: [PATCH 4/8] style(cli): apply biome formatting --- packages/cli/src/commands/_lib/apply/apply-cmd.ts | 5 ++++- packages/cli/src/commands/_lib/apply/client.ts | 5 ++++- packages/cli/src/commands/_lib/ensure-deps-installed.ts | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/commands/_lib/apply/apply-cmd.ts b/packages/cli/src/commands/_lib/apply/apply-cmd.ts index ac915bb6e..b5461771a 100644 --- a/packages/cli/src/commands/_lib/apply/apply-cmd.ts +++ b/packages/cli/src/commands/_lib/apply/apply-cmd.ts @@ -386,7 +386,10 @@ async function installConnectorDefinitions( // ride `runtime.nix.packages` and are provisioned at run time. ensureProjectDepsInstalled(def.sourceFile, printText); const compiledCode = await compileConnectorFromFile(def.sourceFile); - result = await client.installConnector({ sourceCode: compiledCode, compiled: true }); + result = await client.installConnector({ + sourceCode: compiledCode, + compiled: true, + }); } else { result = await client.installConnector({ sourceUrl: def.sourceUrl }); } diff --git a/packages/cli/src/commands/_lib/apply/client.ts b/packages/cli/src/commands/_lib/apply/client.ts index ff0592125..13dd8c2ab 100644 --- a/packages/cli/src/commands/_lib/apply/client.ts +++ b/packages/cli/src/commands/_lib/apply/client.ts @@ -862,7 +862,10 @@ export class ApplyClient { }>({ action: "install_connector", ...(payload.sourceCode !== undefined - ? { source_code: payload.sourceCode, compiled: payload.compiled ?? false } + ? { + source_code: payload.sourceCode, + compiled: payload.compiled ?? false, + } : {}), ...(payload.sourceUrl ? { source_url: payload.sourceUrl } : {}), ...(payload.sourceUri ? { source_uri: payload.sourceUri } : {}), diff --git a/packages/cli/src/commands/_lib/ensure-deps-installed.ts b/packages/cli/src/commands/_lib/ensure-deps-installed.ts index bf78e021a..3bb877b59 100644 --- a/packages/cli/src/commands/_lib/ensure-deps-installed.ts +++ b/packages/cli/src/commands/_lib/ensure-deps-installed.ts @@ -80,6 +80,9 @@ export function ensureProjectDepsInstalled( ); } log(`Installing connector dependencies in ${root}...`); - execFileSync("bun", ["install", "--ignore-scripts"], { cwd: root, stdio: "inherit" }); + execFileSync("bun", ["install", "--ignore-scripts"], { + cwd: root, + stdio: "inherit", + }); ensuredRoots.add(root); } From 8acdd5efb657012d913a44db43586877832469ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Wed, 20 May 2026 18:47:43 +0100 Subject: [PATCH 5/8] refactor(cli): lazy-load connector-compile graph in apply Keep esbuild + connector-worker + SDK out of apply-cmd's module-load path (only `lobu apply` with local *.connector.ts needs it). Documented in the AGENTS.md dynamic-import allow-list. Matches connector-run-cmd's pattern. --- AGENTS.md | 1 + packages/cli/src/commands/_lib/apply/apply-cmd.ts | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c18350d89..358b55531 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -109,6 +109,7 @@ Rules for agents: - **No new dynamic imports outside the allow-list below.** Use static `import` by default; new `await import(...)` sites need a measured cost justification (boot time, install footprint, Keychain prompt) added to this list in the same PR. Rationale for each entry lives as a code comment at the call site: - `packages/cli/src/index.ts` — lazy subcommand handlers (keeps `lobu --help` ~60ms). - `packages/cli/src/commands/_lib/connector-run-cmd.ts` — `browser-mirror`, `devtools-active-port`, `executeCompiledConnector`. + - `packages/cli/src/commands/_lib/apply/apply-cmd.ts` — `connector-loader` + `ensure-deps-installed` (the connector-compile graph: esbuild + connector-worker + SDK). Loaded only when an apply has local `*.connector.ts` to compile; keeps that graph out of apply-cmd's module-load path (and out of every CLI test that imports it). - `packages/cli/src/commands/_lib/apply/desired-state.ts` — `yaml` (loaded only on YAML inputs). - `packages/cli/src/commands/memory/_lib/browser-auth-cmd.ts` — `decryptChromeCookiesMacOS`, `playwright/chromium`. - `packages/server/src/server.ts` — `./embedded-runtime` is statically imported, but `./server-lifecycle` is lazy: its transitive imports read env at module-eval, and the embedded branch only finalises DATABASE_URL during `main()`. Loading the lifecycle eagerly would snapshot a stale env. diff --git a/packages/cli/src/commands/_lib/apply/apply-cmd.ts b/packages/cli/src/commands/_lib/apply/apply-cmd.ts index b5461771a..2192b042e 100644 --- a/packages/cli/src/commands/_lib/apply/apply-cmd.ts +++ b/packages/cli/src/commands/_lib/apply/apply-cmd.ts @@ -7,8 +7,6 @@ import { loadProjectLink } from "../../../internal/project-link.js"; import { CONFIG_FILENAME } from "../../../config/loader.js"; import { ApiError, ValidationError } from "../../memory/_lib/errors.js"; import { printError, printText } from "../../memory/_lib/output.js"; -import { compileConnectorFromFile } from "../connector-loader.js"; -import { ensureProjectDepsInstalled } from "../ensure-deps-installed.js"; import { type ApplyClient, type RemoteAgent, @@ -384,6 +382,16 @@ async function installConnectorDefinitions( // node_modules available, so esbuild can bundle the connector's declared // npm deps. The server can't (it only receives the artifact). Native deps // ride `runtime.nix.packages` and are provisioned at run time. + // + // Lazy-imported (cached by the loader) so the heavy connector-compile + // graph (esbuild + connector-worker + SDK) stays out of apply-cmd's + // module-load path — see the dynamic-import allow-list in AGENTS.md. + const { ensureProjectDepsInstalled } = await import( + "../ensure-deps-installed.js" + ); + const { compileConnectorFromFile } = await import( + "../connector-loader.js" + ); ensureProjectDepsInstalled(def.sourceFile, printText); const compiledCode = await compileConnectorFromFile(def.sourceFile); result = await client.installConnector({ From 3168feaccf7d52e8574c0cc2ce0cbc9e39bb8f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Wed, 20 May 2026 19:24:57 +0100 Subject: [PATCH 6/8] =?UTF-8?q?fix(connectors):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20source=5Furl=20compile,=20deterministic=20join,=20i?= =?UTF-8?q?nit=20merge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - apply: compile `def.sourcePath` (the actual `.ts`) for local connectors, not `def.sourceFile` (an error-message label that may point at a `type: connector` YAML). `source_url` connectors (source fetched into `sourceCode`, no local deps) upload raw for gateway compile instead of being compiled locally. - worker-api: filter the connector_definitions join to `status = 'active'` so it matches the partial unique index `idx_connector_defs_org_key` — archived/draft rows share `(key, org)` and made the runtime lookup nondeterministic. - init: merge into an existing `package.json` (preserve the user's fields, add the SDK devDependency) and never overwrite an existing `tsconfig.json`, so `--here` into an existing project is non-destructive. - docs: note `bun install --ignore-scripts` (the actual command). --- AGENTS.md | 2 +- .../cli/src/commands/_lib/apply/apply-cmd.ts | 21 ++++-- packages/cli/src/commands/init.ts | 73 +++++++++++-------- packages/server/src/worker-api.ts | 1 + 4 files changed, 57 insertions(+), 40 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 358b55531..efdb6f5a0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,7 +59,7 @@ All chat platforms (Telegram, Slack, Discord, WhatsApp, Teams) run through Chat #### Connectors - A connector is a `*.connector.ts` extending `ConnectorRuntime`. **npm deps** go in the project's `package.json` (next to `lobu.toml`) and are **bundled** into the connector by esbuild at compile time. **Native deps** (ffmpeg, imagemagick, …) are declared as nixpkgs refs in `runtime.nix.packages` on the connector `definition`; the runtime provisions them on PATH via `nix-shell` at execution. npm = bundled (compile-time); native = nix (run-time). -- **Compile happens on the CLI**, not the server: `lobu apply` runs `bun install` in the project, compiles each connector with the project's `node_modules`, and uploads the bundle (`compiled: true`). esbuild resolves a connector's imports relative to the connector file's dir, so the project's deps are used regardless of where the `lobu` binary is installed. `@lobu/connector-sdk` is **externalized** (runtime-provided, à la Lambda's `aws-sdk`) — the bundle stays the connector's own code + deps, not multiple MB of SDK infra. +- **Compile happens on the CLI**, not the server: `lobu apply` runs `bun install --ignore-scripts` in the project (install scripts off for supply-chain safety — native bits go through `runtime.nix.packages`, not bundled npm), compiles each connector with the project's `node_modules`, and uploads the bundle (`compiled: true`). esbuild resolves a connector's imports relative to the connector file's dir, so the project's deps are used regardless of where the `lobu` binary is installed. `@lobu/connector-sdk` is **externalized** (runtime-provided, à la Lambda's `aws-sdk`) — the bundle stays the connector's own code + deps, not multiple MB of SDK infra. - `lobu init` scaffolds `package.json` (with `@lobu/connector-sdk` devDependency for editor types) + `tsconfig.json` + `connectors/`. #### Guardrails diff --git a/packages/cli/src/commands/_lib/apply/apply-cmd.ts b/packages/cli/src/commands/_lib/apply/apply-cmd.ts index 2192b042e..e965b22a7 100644 --- a/packages/cli/src/commands/_lib/apply/apply-cmd.ts +++ b/packages/cli/src/commands/_lib/apply/apply-cmd.ts @@ -377,11 +377,13 @@ async function installConnectorDefinitions( const def = row.desired; if (!def) continue; let result: Awaited>; - if (def.sourceCode !== undefined) { - // Compile project connectors on the CLI: only here is the project's - // node_modules available, so esbuild can bundle the connector's declared - // npm deps. The server can't (it only receives the artifact). Native deps - // ride `runtime.nix.packages` and are provisioned at run time. + if (def.sourcePath) { + // Local `*.connector.ts`: compile on the CLI, where the project's + // node_modules is available, so esbuild can bundle the connector's + // declared npm deps (the server only receives the artifact). Native deps + // ride `runtime.nix.packages` and are provisioned at run time. Compile + // `sourcePath` (the actual `.ts`), not `sourceFile` (an error-message + // label that may point at a `type: connector` YAML doc). // // Lazy-imported (cached by the loader) so the heavy connector-compile // graph (esbuild + connector-worker + SDK) stays out of apply-cmd's @@ -392,12 +394,17 @@ async function installConnectorDefinitions( const { compileConnectorFromFile } = await import( "../connector-loader.js" ); - ensureProjectDepsInstalled(def.sourceFile, printText); - const compiledCode = await compileConnectorFromFile(def.sourceFile); + ensureProjectDepsInstalled(def.sourcePath, printText); + const compiledCode = await compileConnectorFromFile(def.sourcePath); result = await client.installConnector({ sourceCode: compiledCode, compiled: true, }); + } else if (def.sourceCode !== undefined) { + // `source_url` connector: source was fetched into `sourceCode` and has no + // local project/node_modules to bundle against — upload it raw and let + // the gateway compile it (the pre-existing path). + result = await client.installConnector({ sourceCode: def.sourceCode }); } else { result = await client.installConnector({ sourceUrl: def.sourceUrl }); } diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index bf328c6bf..6064c5169 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -709,39 +709,48 @@ export async function initCommand( // (provided by the runtime — externalized at compile, here for editor // types) plus any npm deps the user adds; tsconfig gives the editor // resolution; the connectors/ dir holds `*.connector.ts`. `lobu apply` - // runs `bun install` here and bundles each connector's own deps. - await writeFile( - join(projectDir, "package.json"), - `${JSON.stringify( - { - name: projectName, - version: "0.0.0", - private: true, - type: "module", - devDependencies: { "@lobu/connector-sdk": `^${cliVersion}` }, - }, - null, - 2 - )}\n` - ); - await writeFile( - join(projectDir, "tsconfig.json"), - `${JSON.stringify( - { - compilerOptions: { - target: "ES2022", - module: "Preserve", - moduleResolution: "bundler", - strict: true, - skipLibCheck: true, - noEmit: true, + // runs `bun install --ignore-scripts` here and bundles each connector's + // own deps. + // + // `--here` can target a directory that already has a package.json / + // tsconfig.json — merge into package.json (preserve the user's fields, just + // add the SDK devDependency) and never overwrite an existing tsconfig. + const pkgJsonPath = join(projectDir, "package.json"); + let pkgJson: Record; + try { + pkgJson = JSON.parse(await readFile(pkgJsonPath, "utf-8")) as Record; + } catch { + pkgJson = { name: projectName, version: "0.0.0", private: true, type: "module" }; + } + pkgJson.devDependencies = { + ...((pkgJson.devDependencies as Record | undefined) ?? {}), + "@lobu/connector-sdk": `^${cliVersion}`, + }; + await writeFile(pkgJsonPath, `${JSON.stringify(pkgJson, null, 2)}\n`); + + const tsconfigPath = join(projectDir, "tsconfig.json"); + try { + await readFile(tsconfigPath, "utf-8"); // exists — leave the user's config untouched + } catch { + await writeFile( + tsconfigPath, + `${JSON.stringify( + { + compilerOptions: { + target: "ES2022", + module: "Preserve", + moduleResolution: "bundler", + strict: true, + skipLibCheck: true, + noEmit: true, + }, + include: ["connectors/**/*.ts"], }, - include: ["connectors/**/*.ts"], - }, - null, - 2 - )}\n` - ); + null, + 2 + )}\n` + ); + } await mkdir(join(projectDir, "connectors"), { recursive: true }); await writeFile(join(projectDir, "connectors", ".gitkeep"), ""); diff --git a/packages/server/src/worker-api.ts b/packages/server/src/worker-api.ts index b170eafc3..824ed83bb 100644 --- a/packages/server/src/worker-api.ts +++ b/packages/server/src/worker-api.ts @@ -503,6 +503,7 @@ export async function pollWorkerJob(c: Context<{ Bindings: Env }>) { AND cv.version = r.connector_version LEFT JOIN connector_definitions cd ON cd.key = r.connector_key AND cd.organization_id = r.organization_id + AND cd.status = 'active' LEFT JOIN auth_profiles ap ON ap.id = r.auth_profile_id LEFT JOIN watchers w ON w.id = r.watcher_id WHERE r.id = ${runId} From 203b82fd70c24bc424715c7b1a6605faf1aff775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Wed, 20 May 2026 19:25:13 +0100 Subject: [PATCH 7/8] style: apply biome formatting --- packages/cli/src/commands/init.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index 6064c5169..bed4ae2ce 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -718,12 +718,21 @@ export async function initCommand( const pkgJsonPath = join(projectDir, "package.json"); let pkgJson: Record; try { - pkgJson = JSON.parse(await readFile(pkgJsonPath, "utf-8")) as Record; + pkgJson = JSON.parse(await readFile(pkgJsonPath, "utf-8")) as Record< + string, + unknown + >; } catch { - pkgJson = { name: projectName, version: "0.0.0", private: true, type: "module" }; + pkgJson = { + name: projectName, + version: "0.0.0", + private: true, + type: "module", + }; } pkgJson.devDependencies = { - ...((pkgJson.devDependencies as Record | undefined) ?? {}), + ...((pkgJson.devDependencies as Record | undefined) ?? + {}), "@lobu/connector-sdk": `^${cliVersion}`, }; await writeFile(pkgJsonPath, `${JSON.stringify(pkgJson, null, 2)}\n`); From fe84016cca9430349b0b04150211a4c013951fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Emre=20Kabakc=C4=B1?= Date: Wed, 20 May 2026 20:16:45 +0100 Subject: [PATCH 8/8] =?UTF-8?q?fix:=20unblock=20macOS=20make=20review=20?= =?UTF-8?q?=E2=80=94=20connector-sdk=20barrel=20import=20+=20embedded-pg?= =?UTF-8?q?=20dylib=20symlinks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two pre-existing macOS-only blockers that fail `make review` locally (both already green on CI's Linux runners), surfaced while validating this PR: - connector-sdk: a barrel re-export of the value+type dual name `AutoCreateWhenRule` trips bun's cross-file module lexer in the test runner ("Export named ... not found"), failing the CLI test suite on macOS. Add a `./identity-types` subpath export and import the value from it in the memory CLI's schema (bypasses the barrel). CLI suite: 4 fail → 0 fail (308 pass). - pgvector-embedded: the @embedded-postgres/darwin-* packages ship only fully-versioned `lib*...dylib` ICU libs; postgres/initdb link against the `lib*..dylib` SONAME and the unversioned `lib*.dylib` linker name, so initdb aborted with "Library not loaded: libicui18n.dylib". injectPgvector now hydrates both symlink levels idempotently (macOS only). Embedded Postgres initdb + start now succeed on darwin-arm64. --- .../cli/src/commands/memory/_lib/schema.ts | 6 ++- packages/connector-sdk/package.json | 4 ++ packages/pgvector-embedded/src/index.ts | 50 +++++++++++++++++-- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/commands/memory/_lib/schema.ts b/packages/cli/src/commands/memory/_lib/schema.ts index f8b45f240..c39e94cd9 100644 --- a/packages/cli/src/commands/memory/_lib/schema.ts +++ b/packages/cli/src/commands/memory/_lib/schema.ts @@ -11,7 +11,11 @@ * Bump CURRENT_SCHEMA_VERSION when making breaking changes. */ -import { AutoCreateWhenRule } from "@lobu/connector-sdk"; +// Import from the module subpath, not the barrel: a barrel re-export of this +// value+type dual name trips bun's cross-file module lexer in the test runner +// ("Export named AutoCreateWhenRule not found"). The direct subpath resolves +// to the module that declares it and sidesteps the bug. +import { AutoCreateWhenRule } from "@lobu/connector-sdk/identity-types"; import { TypeCompiler } from "@sinclair/typebox/compiler"; import { parseAllDocuments } from "yaml"; diff --git a/packages/connector-sdk/package.json b/packages/connector-sdk/package.json index e7ccee9d4..9edfb5c84 100644 --- a/packages/connector-sdk/package.json +++ b/packages/connector-sdk/package.json @@ -16,6 +16,10 @@ "default": "./dist/index.js" } }, + "./identity-types": { + "types": "./dist/identity-types.d.ts", + "import": "./dist/identity-types.js" + }, "./browser-mirror": { "import": { "types": "./dist/browser/mirror-cookies.d.ts", diff --git a/packages/pgvector-embedded/src/index.ts b/packages/pgvector-embedded/src/index.ts index 0f6bd0dc2..1fd46b63f 100644 --- a/packages/pgvector-embedded/src/index.ts +++ b/packages/pgvector-embedded/src/index.ts @@ -12,7 +12,13 @@ * against a same-major PostgreSQL — the extension ABI is stable within a major, * so a library built against PG 18.x loads into `embedded-postgres`'s PG 18.x. */ -import { cpSync, existsSync, mkdirSync, readdirSync } from "node:fs"; +import { + cpSync, + existsSync, + mkdirSync, + readdirSync, + symlinkSync, +} from "node:fs"; import { createRequire } from "node:module"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -58,10 +64,44 @@ export function resolveEmbeddedNativeDir( return join(dirname(entry), "..", "native"); } +/** + * embedded-postgres' macOS native libs ship only as the fully-versioned + * `lib...dylib`, but the Postgres binaries link against both + * the major SONAME (`lib..dylib`) and the unversioned linker name + * (`lib.dylib`, e.g. postgres' `@loader_path/../lib/libicui18n.dylib`) — + * and the @embedded-postgres/darwin-* packages omit those symlinks. Without them + * `initdb`/`postgres` abort with `Library not loaded`. Recreate both link levels + * idempotently, pointing at the real versioned file. No-op off macOS (Linux ships + * the `.so.` links) and when a link already exists. + */ +export function ensureNativeLibSonames( + nativeDir: string = resolveEmbeddedNativeDir() +): void { + if (process.platform !== "darwin") return; + const libDir = join(nativeDir, "lib"); + if (!existsSync(libDir)) return; + for (const file of readdirSync(libDir)) { + // lib...dylib → { lib..dylib, lib.dylib } + const match = file.match(/^(lib.+?)\.(\d+)\.\d+\.dylib$/); + if (!match) continue; + const [, base, major] = match; + for (const soname of [`${base}.${major}.dylib`, `${base}.dylib`]) { + const link = join(libDir, soname); + if (existsSync(link)) continue; + try { + symlinkSync(file, link); // relative target within libDir + } catch { + // Best-effort: a race or perms issue surfaces as the real initdb error. + } + } + } +} + /** * Copy the host platform's prebuilt pgvector files into an embedded-postgres - * `native` tree so `CREATE EXTENSION vector` works. Idempotent — returns early - * if pgvector is already present in the tree. + * `native` tree so `CREATE EXTENSION vector` works, and hydrate the macOS ICU + * SONAME symlinks so `initdb` can link. Idempotent — returns early if pgvector + * is already present in the tree. * * @param nativeDir absolute path to `.../native`; defaults to the resolved * host-platform `@embedded-postgres` package. @@ -70,6 +110,10 @@ export function injectPgvector( nativeDir: string = resolveEmbeddedNativeDir(), platform: string = currentPlatformKey() ): void { + // Always (re)hydrate the dylib SONAME symlinks — needed for initdb to link + // even when pgvector was already injected on a prior run. + ensureNativeLibSonames(nativeDir); + const libDst = join(nativeDir, "lib", "postgresql"); const extDst = join(nativeDir, "share", "postgresql", "extension");