feat(connectors): user-declared connector dependencies (npm bundled + nix native)

AGENTS.md

@@ -57,6 +57,11 @@ All chat platforms (Telegram, Slack, Discord, WhatsApp, Teams) run through Chat
 - **Integration auth lives in Lobu** — OAuth, token refresh, and API proxying for third-party services (GitHub, Google, etc.) are handled by Lobu MCP servers. Workers never see OAuth tokens.
 - **`events` is append-only.** Never `DELETE FROM events`. To hide a row, write a tombstone via `client.knowledge.delete()` or `save_knowledge({ supersedes_event_id, ... })`; the `current_event_records` view masks superseded rows, `include_superseded` recovers history.
 
+#### Connectors
+- A connector is a `*.connector.ts` extending `ConnectorRuntime`. **npm deps** go in the project's `package.json` (next to `lobu.toml`) and are **bundled** into the connector by esbuild at compile time. **Native deps** (ffmpeg, imagemagick, …) are declared as nixpkgs refs in `runtime.nix.packages` on the connector `definition`; the runtime provisions them on PATH via `nix-shell` at execution. npm = bundled (compile-time); native = nix (run-time).
+- **Compile happens on the CLI**, not the server: `lobu apply` runs `bun install --ignore-scripts` in the project (install scripts off for supply-chain safety — native bits go through `runtime.nix.packages`, not bundled npm), compiles each connector with the project's `node_modules`, and uploads the bundle (`compiled: true`). esbuild resolves a connector's imports relative to the connector file's dir, so the project's deps are used regardless of where the `lobu` binary is installed. `@lobu/connector-sdk` is **externalized** (runtime-provided, à la Lambda's `aws-sdk`) — the bundle stays the connector's own code + deps, not multiple MB of SDK infra.
+- `lobu init` scaffolds `package.json` (with `@lobu/connector-sdk` devDependency for editor types) + `tsconfig.json` + `connectors/`.
+
 #### Guardrails
 - Primitive lives in `packages/core/src/guardrails/`: `Guardrail<stage>`, `GuardrailRegistry`, `runGuardrails()`. Stages: `input` (user message → worker), `output` (worker text → user), `pre-tool` (tool call authorization).
 - Each guardrail's `run(ctx)` returns `{ tripped, reason?, metadata? }`. The runner races all enabled guardrails at a stage; the first trip short-circuits (later results are discarded) and a thrown guardrail is logged and treated as a pass.
@@ -104,6 +109,7 @@ Rules for agents:
 - **No new dynamic imports outside the allow-list below.** Use static `import` by default; new `await import(...)` sites need a measured cost justification (boot time, install footprint, Keychain prompt) added to this list in the same PR. Rationale for each entry lives as a code comment at the call site:
   - `packages/cli/src/index.ts` — lazy subcommand handlers (keeps `lobu --help` ~60ms).
   - `packages/cli/src/commands/_lib/connector-run-cmd.ts` — `browser-mirror`, `devtools-active-port`, `executeCompiledConnector`.
+  - `packages/cli/src/commands/_lib/apply/apply-cmd.ts` — `connector-loader` + `ensure-deps-installed` (the connector-compile graph: esbuild + connector-worker + SDK). Loaded only when an apply has local `*.connector.ts` to compile; keeps that graph out of apply-cmd's module-load path (and out of every CLI test that imports it).
   - `packages/cli/src/commands/_lib/apply/desired-state.ts` — `yaml` (loaded only on YAML inputs).
   - `packages/cli/src/commands/memory/_lib/browser-auth-cmd.ts` — `decryptChromeCookiesMacOS`, `playwright/chromium`.
   - `packages/server/src/server.ts` — `./embedded-runtime` is statically imported, but `./server-lifecycle` is lazy: its transitive imports read env at module-eval, and the embedded branch only finalises DATABASE_URL during `main()`. Loading the lifecycle eagerly would snapshot a stale env.

packages/cli/src/commands/_lib/apply/apply-cmd.ts

@@ -376,10 +376,38 @@ async function installConnectorDefinitions(
     if (row.verb === "noop" || row.verb === "drift") continue;
     const def = row.desired;
     if (!def) continue;
-    const result =
-      def.sourceCode !== undefined
-        ? await client.installConnector({ sourceCode: def.sourceCode })
-        : await client.installConnector({ sourceUrl: def.sourceUrl });
+    let result: Awaited<ReturnType<typeof client.installConnector>>;
+    if (def.sourcePath) {
+      // Local `*.connector.ts`: compile on the CLI, where the project's
+      // node_modules is available, so esbuild can bundle the connector's
+      // declared npm deps (the server only receives the artifact). Native deps
+      // ride `runtime.nix.packages` and are provisioned at run time. Compile
+      // `sourcePath` (the actual `.ts`), not `sourceFile` (an error-message
+      // label that may point at a `type: connector` YAML doc).
+      //
+      // Lazy-imported (cached by the loader) so the heavy connector-compile
+      // graph (esbuild + connector-worker + SDK) stays out of apply-cmd's
+      // module-load path — see the dynamic-import allow-list in AGENTS.md.
+      const { ensureProjectDepsInstalled } = await import(
+        "../ensure-deps-installed.js"
+      );
+      const { compileConnectorFromFile } = await import(
+        "../connector-loader.js"
+      );
+      ensureProjectDepsInstalled(def.sourcePath, printText);
+      const compiledCode = await compileConnectorFromFile(def.sourcePath);
+      result = await client.installConnector({
+        sourceCode: compiledCode,
+        compiled: true,
+      });
+    } else if (def.sourceCode !== undefined) {
+      // `source_url` connector: source was fetched into `sourceCode` and has no
+      // local project/node_modules to bundle against — upload it raw and let
+      // the gateway compile it (the pre-existing path).
+      result = await client.installConnector({ sourceCode: def.sourceCode });
+    } else {
+      result = await client.installConnector({ sourceUrl: def.sourceUrl });
+    }
     if (result.connectorKey) {
       locallySuppliedKeys.add(result.connectorKey);
       installedKeys.add(result.connectorKey);

packages/cli/src/commands/_lib/apply/client.ts

@@ -851,6 +851,8 @@ export class ApplyClient {
     sourceUrl?: string;
     /** `file://` URI of a bundled connector source on the server host. */
     sourceUri?: string;
+    /** `sourceCode` is already a compiled bundle (CLI-side compile) — skip server compile. */
+    compiled?: boolean;
   }): Promise<InstallConnectorResult> {
     const body = await this.connectionsTool<{
       installed?: boolean;
@@ -860,7 +862,10 @@ export class ApplyClient {
     }>({
       action: "install_connector",
       ...(payload.sourceCode !== undefined
-        ? { source_code: payload.sourceCode, compiled: false }
+        ? {
+            source_code: payload.sourceCode,
+            compiled: payload.compiled ?? false,
+          }
         : {}),
       ...(payload.sourceUrl ? { source_url: payload.sourceUrl } : {}),
       ...(payload.sourceUri ? { source_uri: payload.sourceUri } : {}),

packages/cli/src/commands/_lib/ensure-deps-installed.ts

@@ -0,0 +1,88 @@
+/**
+ * Ensure a connector project's npm dependencies are installed before the CLI
+ * compiles its connectors. esbuild bundles a connector's imports relative to
+ * the connector file's directory, so the project's own `node_modules` (next to
+ * `package.json`) must exist. We run `bun install --ignore-scripts` when stale:
+ * `--ignore-scripts` keeps install-time supply-chain surface off the user's
+ * machine — packages that need build scripts (native bindings) belong in
+ * `runtime.nix.packages`, not bundled npm.
+ */
+
+import { execFileSync } from "node:child_process";
+import { existsSync, statSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+// Per-process memo so `lobu apply` installs each project root at most once.
+const ensuredRoots = new Set<string>();
+
+/**
+ * Find the connector's project root — the nearest ancestor with `lobu.toml`.
+ * Anchoring on `lobu.toml` (not any ancestor `package.json`) is what stops a
+ * connector inside a monorepo from resolving to the monorepo's root
+ * package.json and triggering a wrong-directory install.
+ */
+export function findProjectRoot(fromFile: string): string | null {
+  let dir = dirname(fromFile);
+  for (let i = 0; i < 40; i++) {
+    if (existsSync(join(dir, "lobu.toml"))) return dir;
+    const parent = dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  return null;
+}
+
+function installIsStale(root: string): boolean {
+  const nodeModules = join(root, "node_modules");
+  if (!existsSync(nodeModules)) return true;
+  const lock = join(root, "bun.lock");
+  if (!existsSync(lock)) return false; // deps present, no lockfile to compare against
+  try {
+    return statSync(lock).mtimeMs > statSync(nodeModules).mtimeMs;
+  } catch {
+    return false;
+  }
+}
+
+function hasBun(): boolean {
+  try {
+    execFileSync("bun", ["--version"], { stdio: "ignore" });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Install the connector project's deps if missing/stale. No-op when the
+ * connector has no `package.json` (no declared npm deps to bundle).
+ */
+export function ensureProjectDepsInstalled(
+  connectorFilePath: string,
+  log: (message: string) => void
+): void {
+  const root = findProjectRoot(connectorFilePath);
+  if (!root || ensuredRoots.has(root)) return;
+  // No package.json at the project root → the connector declares no npm deps
+  // (the SDK is runtime-provided/externalized), so there's nothing to install.
+  if (!existsSync(join(root, "package.json"))) {
+    ensuredRoots.add(root);
+    return;
+  }
+  if (!installIsStale(root)) {
+    ensuredRoots.add(root);
+    return;
+  }
+  if (!hasBun()) {
+    throw new Error(
+      `Connector dependencies in ${root} need installing, but \`bun\` is not on PATH. ` +
+        `Run \`bun install\` in ${root}, or install bun (https://bun.sh).`
+    );
+  }
+  log(`Installing connector dependencies in ${root}...`);
+  execFileSync("bun", ["install", "--ignore-scripts"], {
+    cwd: root,
+    stdio: "inherit",
+  });
+  ensuredRoots.add(root);
+}

packages/cli/src/commands/init.ts

@@ -705,6 +705,64 @@ export async function initCommand(
     await mkdir(join(projectDir, "skills"), { recursive: true });
     await writeFile(join(projectDir, "skills", ".gitkeep"), "");
 
+    // Connector authoring surface: package.json declares the connector SDK
+    // (provided by the runtime — externalized at compile, here for editor
+    // types) plus any npm deps the user adds; tsconfig gives the editor
+    // resolution; the connectors/ dir holds `*.connector.ts`. `lobu apply`
+    // runs `bun install --ignore-scripts` here and bundles each connector's
+    // own deps.
+    //
+    // `--here` can target a directory that already has a package.json /
+    // tsconfig.json — merge into package.json (preserve the user's fields, just
+    // add the SDK devDependency) and never overwrite an existing tsconfig.
+    const pkgJsonPath = join(projectDir, "package.json");
+    let pkgJson: Record<string, unknown>;
+    try {
+      pkgJson = JSON.parse(await readFile(pkgJsonPath, "utf-8")) as Record<
+        string,
+        unknown
+      >;
+    } catch {
+      pkgJson = {
+        name: projectName,
+        version: "0.0.0",
+        private: true,
+        type: "module",
+      };
+    }
+    pkgJson.devDependencies = {
+      ...((pkgJson.devDependencies as Record<string, string> | undefined) ??
+        {}),
+      "@lobu/connector-sdk": `^${cliVersion}`,
+    };
+    await writeFile(pkgJsonPath, `${JSON.stringify(pkgJson, null, 2)}\n`);
+
+    const tsconfigPath = join(projectDir, "tsconfig.json");
+    try {
+      await readFile(tsconfigPath, "utf-8"); // exists — leave the user's config untouched
+    } catch {
+      await writeFile(
+        tsconfigPath,
+        `${JSON.stringify(
+          {
+            compilerOptions: {
+              target: "ES2022",
+              module: "Preserve",
+              moduleResolution: "bundler",
+              strict: true,
+              skipLibCheck: true,
+              noEmit: true,
+            },
+            include: ["connectors/**/*.ts"],
+          },
+          null,
+          2
+        )}\n`
+      );
+    }
+    await mkdir(join(projectDir, "connectors"), { recursive: true });
+    await writeFile(join(projectDir, "connectors", ".gitkeep"), "");
+
     await renderTemplate(
       "AGENTS.md.tmpl",
       variables,

packages/cli/src/commands/memory/_lib/schema.ts

@@ -11,7 +11,11 @@
  * Bump CURRENT_SCHEMA_VERSION when making breaking changes.
  */
 
-import { AutoCreateWhenRule } from "@lobu/connector-sdk";
+// Import from the module subpath, not the barrel: a barrel re-export of this
+// value+type dual name trips bun's cross-file module lexer in the test runner
+// ("Export named AutoCreateWhenRule not found"). The direct subpath resolves
+// to the module that declares it and sidesteps the bug.
+import { AutoCreateWhenRule } from "@lobu/connector-sdk/identity-types";
 import { TypeCompiler } from "@sinclair/typebox/compiler";
 import { parseAllDocuments } from "yaml";
 

packages/connector-sdk/package.json

@@ -16,6 +16,10 @@
         "default": "./dist/index.js"
       }
     },
+    "./identity-types": {
+      "types": "./dist/identity-types.d.ts",
+      "import": "./dist/identity-types.js"
+    },
     "./browser-mirror": {
       "import": {
         "types": "./dist/browser/mirror-cookies.d.ts",

packages/connector-sdk/src/connector-types.ts

@@ -65,6 +65,15 @@ export interface ConnectorRuntimeInfo {
    * Optional — omit when the platform adapter needs no fine-grained scope list.
    */
   scopes?: string[];
+  /**
+   * Native system dependencies this connector needs on PATH at execution time,
+   * as nixpkgs attribute references (e.g. `["ffmpeg", "imagemagick"]`). npm
+   * dependencies are bundled into the connector at compile time and do NOT go
+   * here — only native tools the runtime must provision. Backends that can run
+   * native deps (embedded, container, machine) satisfy these via nix; backends
+   * that can't (e.g. edge workers) reject a connector that declares them.
+   */
+  nix?: { packages: string[] };
 }
 
 // =============================================================================

packages/connector-worker/src/compile/index.ts

@@ -22,7 +22,6 @@
 
 import { existsSync } from 'node:fs';
 import { mkdtemp, readFile, rm, stat } from 'node:fs/promises';
-import { createRequire } from 'node:module';
 import { tmpdir } from 'node:os';
 import { join, resolve } from 'node:path';
 import { build, type Plugin } from 'esbuild';
@@ -75,15 +74,28 @@ export function findBundledConnectorFile(
 }
 
 /**
- * Resolve the `@lobu/connector-sdk` module entry from this module's
- * perspective. Used as the esbuild `alias` target so connector code that
- * imports `from 'lobu'` or `from '@lobu/connector-sdk'` resolves to the
- * same physical file the runtime will import — avoiding the
- * `instanceof ConnectorRuntime` cross-realm trap.
+ * esbuild plugin that marks the connector SDK as **external** (runtime-provided)
+ * rather than bundling it in. The SDK pulls a large infra graph transitively
+ * (Sentry, OpenTelemetry, grpc, isomorphic-git, …); bundling it inflated every
+ * connector to multiple MB. The runtime that executes the connector already has
+ * `@lobu/connector-sdk` installed (it's a dependency of `@lobu/connector-worker`),
+ * so the bundle leaves it as a bare import and Node resolves it from the runtime's
+ * node_modules at load time — the standard "externalize the framework, bundle the
+ * user code" pattern (cf. AWS Lambda not bundling `@aws-sdk`).
+ *
+ * The `lobu` alias specifier is normalized to `@lobu/connector-sdk` so the emitted
+ * import resolves to a real package the runtime provides.
  */
-function resolveSdkEntry(): string {
-  const require_ = createRequire(import.meta.url);
-  return require_.resolve('@lobu/connector-sdk');
+function createSdkExternalPlugin(): Plugin {
+  return {
+    name: 'sdk-external',
+    setup(b) {
+      b.onResolve({ filter: /^(lobu|@lobu\/connector-sdk)(\/.*)?$/ }, (args) => ({
+        path: args.path.replace(/^lobu(?=$|\/)/, '@lobu/connector-sdk'),
+        external: true,
+      }));
+    },
+  };
 }
 
 interface NpmSpecifierPluginOptions {
@@ -128,21 +140,14 @@ export function createNpmSpecifierPlugin(options?: NpmSpecifierPluginOptions): P
 
 interface CompileOptions {
   /**
-   * Max entries kept in the mtime-keyed LRU. Each entry is the full
-   * compiled bundle (~13 MB today, smaller as transitive deps are
-   * externalised). Cap default 8 keeps memory bounded; pass a smaller
-   * value in memory-constrained environments.
+   * Max entries kept in the mtime-keyed LRU. Each entry is the compiled
+   * bundle — now just the connector's own code + its bundled npm deps,
+   * since the SDK and its infra graph are externalised. Cap default 8
+   * keeps memory bounded; pass a smaller value in memory-constrained
+   * environments.
    * @default 8
    */
   cacheMax?: number;
-  /**
-   * Override the `@lobu/connector-sdk` entry esbuild aliases against.
-   * Defaults to the SDK resolved from this module's `require.resolve`.
-   * Overriding is only useful when the caller knows of a more
-   * appropriate physical file (e.g. a server bundle that wants to point
-   * back at a sibling dist file).
-   */
-  sdkEntry?: string;
   /**
    * Hook fired when `npm:` specifiers fail to resolve and the import is
    * externalised. Forwarded to `createNpmSpecifierPlugin`.
@@ -158,19 +163,21 @@ const DEFAULT_CACHE_MAX = 8;
  *
  * The returned bundle:
  *   - is ESM (`format: 'esm'`, `target: 'node20'`);
- *   - aliases `lobu` and `@lobu/connector-sdk` to the SDK entry so
- *     connectors targeting either specifier resolve to the same module;
+ *   - externalises the connector SDK (`lobu` / `@lobu/connector-sdk`) — the
+ *     runtime provides it, keeping bundles to the connector's own code + deps;
  *   - has a banner injecting a CJS-compatible `require` shim;
  *   - externalises `EXTERNAL_RUNTIME_DEPS` (native deps + Playwright);
+ *   - emits an inline source map (`sourcesContent: false`) so connector stack
+ *     traces map to source lines without embedding the source in the artifact;
  *   - is mtime-cached: a repeat call with the same `filePath` whose
  *     mtime hasn't changed returns the cached bundle without hitting
  *     esbuild.
  */
 export function createConnectorCompiler(options?: CompileOptions) {
   const cacheMax = options?.cacheMax ?? DEFAULT_CACHE_MAX;
-  const sdkEntry = options?.sdkEntry ?? resolveSdkEntry();
   const compiledFileCache = new Map<string, { mtimeMs: number; code: string }>();
-  const plugin = createNpmSpecifierPlugin({ onUnresolved: options?.onUnresolvedNpm });
+  const npmPlugin = createNpmSpecifierPlugin({ onUnresolved: options?.onUnresolvedNpm });
+  const sdkExternalPlugin = createSdkExternalPlugin();
 
   function touchCacheEntry(filePath: string, entry: { mtimeMs: number; code: string }): void {
     compiledFileCache.delete(filePath);
@@ -206,15 +213,15 @@ export function createConnectorCompiler(options?: CompileOptions) {
         format: 'esm',
         platform: 'node',
         target: 'node20',
-        alias: { lobu: sdkEntry, '@lobu/connector-sdk': sdkEntry },
         banner: {
           js: `import { createRequire as __createRequire } from 'module'; const require = __createRequire(import.meta.url);`,
         },
-        plugins: [plugin],
+        plugins: [sdkExternalPlugin, npmPlugin],
         external: [...EXTERNAL_RUNTIME_DEPS],
         write: true,
         minify: false,
-        sourcemap: false,
+        sourcemap: 'inline',
+        sourcesContent: false,
       });
 
       const code = await readFile(outPath, 'utf-8');