fix(db): sync native pgserve-v3 migration fix to dev (already on main via #2456)

UPGRADING-pgserve-v3.md

@@ -0,0 +1,74 @@
+---
+title: "Upgrading to pgserve v3 (autopg)"
+description: "How existing and new users move onto autopg/pgserve v3. What is automatic, what is preserved, and the one manual step when you have data to carry."
+---
+
+# Upgrading to pgserve v3 (autopg)
+
+`pgserve` v3 ships as **autopg** — a router-less, single-postmaster
+distribution (the old daemon, accept-hook router, and SO_PEERCRED
+fingerprint routing are gone). Genie now natively detects autopg v3 and
+owns its own database on it. Everything below is verified end-to-end.
+
+## What new genie does on the host it finds
+
+| You currently run | New genie behavior | Action needed |
+|---|---|---|
+| **pgserve v1.x** (port 8432, no admin.json) | falls back to the legacy TCP/headless path; `_genie_migrations` and your data stay where they are | none — keep running, upgrade pgserve later when you choose |
+| **pgserve v2.x** (admin.json + accept-hook router) | stays on `database = postgres`; the router routes you to your existing `app_genie_<fp>` DB; migrations are idempotent | none — your data is preserved |
+| **autopg / pgserve v3** (writes `<socketDir>/runtime.json`) | auto-detects v3, auto-creates a dedicated `genie` database, runs migrations into it | none — first boot provisions everything |
+
+Detection is **strictly** keyed on `<socketDir>/runtime.json` (the v3
+marker). v1/v2 hosts are never retargeted, so an upgraded genie on an
+old pgserve never moves your data and never blocks you.
+
+The dedicated v3 database name is `genie`; override with
+`GENIE_DB_NAME=<name>` if you need something different (e.g. multi-tenant
+hosts). `GENIE_ROLE_CUTOVER=0` continues to work; native-DB provisioning
+is decoupled from the role-cutover kill-switch.
+
+## Moving data from old pgserve → autopg v3
+
+This is the only step that isn't automatic. New genie on autopg v3
+creates a **fresh empty `genie` DB**; your existing teams / agents /
+tasks / sessions live in the *old* pgserve's `app_genie_<fp>` DB and
+will not migrate themselves.
+
+If you don't need the history, just install autopg v3 — you're done.
+
+If you do, dump-then-restore around the upgrade:
+
+```bash
+# 1. On the old pgserve / old genie — snapshot to .genie/snapshot.sql.gz
+genie db backup
+
+# 2. Install autopg v3 (replaces pgserve)
+curl -fsSL https://raw.githubusercontent.com/automagik-dev/autopg/main/install.sh | bash
+
+# 3. On the new autopg v3 — let new genie provision its `genie` DB
+genie db migrate    # creates + populates the v3 `genie` database
+
+# 4. Restore your snapshot into the freshly-provisioned `genie` DB
+genie db restore    # defaults to .genie/snapshot.sql.gz
+```
+
+The snapshot is portable across pgserve versions because `genie db
+backup` is a logical (`pg_dump`) dump, not a physical replication of the
+old data directory.
+
+## Verifying after the upgrade
+
+```bash
+genie ls                                                # lists your agents
+genie db status                                         # table counts, version
+psql -h "$XDG_RUNTIME_DIR/pgserve" -U postgres -d genie # explore the new DB
+```
+
+## Rolling back
+
+The old pgserve install (its data dir, admin.json, role state) is
+untouched by autopg v3's installer. If you decide to roll back, point
+genie's environment at the old socket / port again and `genie db
+restore` from your snapshot into the old `app_genie_<fp>` DB. The
+embedded-migrations change in new genie is forward-compatible: it does
+not modify migrations that have already been applied.

scripts/build-binary.sh

@@ -55,6 +55,15 @@ TARBALL="${DIST_DIR}/genie-${VERSION}-${PLATFORM}.tar.gz"
 rm -rf "$STAGE" "$TARBALL"
 mkdir -p "$STAGE"
 
+# Regenerate the embedded migrations manifest so the compiled binary can
+# never ship a stale/empty migration set. `bun build --compile` only
+# embeds statically-imported modules; the manifest turns every
+# src/db/migrations/*.sql into a `with { type: 'text' }` import. Without
+# this the binary's migrate is a no-op on a fresh DB (no schema → genie
+# cannot natively migrate onto pgserve v3).
+echo "==> regenerating embedded migrations manifest"
+bun run "${REPO_ROOT}/scripts/gen-migrations-manifest.ts"
+
 echo "==> [${PLATFORM}] bun build --compile --target=${TARGET}  (v${VERSION})"
 bun build --compile \
   --target="${TARGET}" \

scripts/gen-migrations-manifest.ts

@@ -0,0 +1,69 @@
+#!/usr/bin/env bun
+/**
+ * Codegen: src/db/migrations.generated.ts
+ *
+ * `bun build --compile` only embeds modules reachable via STATIC import.
+ * The old loader used readdirSync + Bun.file() on src/db/migrations/ — a
+ * runtime FS read that resolves to nothing inside the compiled binary
+ * (the .sql files are not bundled), so `genie db migrate` reported
+ * "Applied: 0" on a completely empty database and the schema was never
+ * created (blocking native genie → autopg/pgserve-v3 migration).
+ *
+ * This generates a manifest of static `with { type: 'text' }` imports —
+ * which Bun DOES embed into the compiled binary — preserving lexicographic
+ * order. Re-run after adding/removing a migration:
+ *
+ *   bun run scripts/gen-migrations-manifest.ts
+ *
+ * It is wired into `scripts/build-binary.sh` so the binary can never ship
+ * a stale manifest. The generated file is committed so a plain
+ * `bun build` works without the codegen step too.
+ */
+import { readdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+const MIGRATIONS_DIR = join(import.meta.dir, '..', 'src', 'db', 'migrations');
+const OUT = join(import.meta.dir, '..', 'src', 'db', 'migrations.generated.ts');
+
+const files = readdirSync(MIGRATIONS_DIR)
+  .filter((f) => f.endsWith('.sql'))
+  .sort();
+
+const idOf = (i: number) => `m${String(i).padStart(3, '0')}`;
+
+const lines: string[] = [
+  '// AUTO-GENERATED by scripts/gen-migrations-manifest.ts — DO NOT EDIT.',
+  '//',
+  "// Static `with { type: 'text' }` imports so `bun build --compile`",
+  '// embeds every migration into the binary (runtime FS reads resolve to',
+  '// nothing inside the compiled executable). Re-run the script after',
+  '// adding or removing a migration.',
+  '',
+];
+
+files.forEach((f, i) => {
+  lines.push(`import ${idOf(i)} from './migrations/${f}' with { type: 'text' };`);
+});
+
+lines.push('');
+lines.push('export interface EmbeddedMigration {');
+lines.push('  name: string;');
+lines.push('  sql: string;');
+lines.push('}');
+lines.push('');
+lines.push('/** All migrations, lexicographic order, embedded at build time. */');
+lines.push('export const EMBEDDED_MIGRATIONS: EmbeddedMigration[] = [');
+files.forEach((f, i) => {
+  const name = f.replace(/\.sql$/, '');
+  // Migration basenames are [A-Za-z0-9_] only — single-quote directly so
+  // the generated file matches biome (single-quote) without a format pass.
+  if (!/^[A-Za-z0-9_]+$/.test(name)) {
+    throw new Error(`unexpected migration filename (needs escaping): ${f}`);
+  }
+  lines.push(`  { name: '${name}', sql: ${idOf(i)} },`);
+});
+lines.push('];');
+lines.push('');
+
+writeFileSync(OUT, lines.join('\n'));
+console.log(`gen-migrations-manifest: wrote ${files.length} migrations → ${OUT}`);

src/db/migrations.generated.ts

@@ -0,0 +1,142 @@
+// AUTO-GENERATED by scripts/gen-migrations-manifest.ts — DO NOT EDIT.
+//
+// Static `with { type: 'text' }` imports so `bun build --compile`
+// embeds every migration into the binary (runtime FS reads resolve to
+// nothing inside the compiled executable). Re-run the script after
+// adding or removing a migration.
+
+import m000 from './migrations/001_core.sql' with { type: 'text' };
+import m001 from './migrations/002_task_lifecycle.sql' with { type: 'text' };
+import m002 from './migrations/003_projects.sql' with { type: 'text' };
+import m003 from './migrations/004_clean_test_artifacts.sql' with { type: 'text' };
+import m004 from './migrations/005_pg_state.sql' with { type: 'text' };
+import m005 from './migrations/006_fix_agents_schema.sql' with { type: 'text' };
+import m006 from './migrations/007_observability.sql' with { type: 'text' };
+import m007 from './migrations/007_reconcile_mailbox_team_chat.sql' with { type: 'text' };
+import m008 from './migrations/008_boards.sql' with { type: 'text' };
+import m009 from './migrations/009_app_store.sql' with { type: 'text' };
+import m010 from './migrations/010_runtime_events.sql' with { type: 'text' };
+import m011 from './migrations/010_session_capture_v2.sql' with { type: 'text' };
+import m012 from './migrations/011_data_model_v5.sql' with { type: 'text' };
+import m013 from './migrations/012_executor_model.sql' with { type: 'text' };
+import m014 from './migrations/013_external_linking.sql' with { type: 'text' };
+import m015 from './migrations/014_comms_protocol.sql' with { type: 'text' };
+import m016 from './migrations/015_agent_archived_state.sql' with { type: 'text' };
+import m017 from './migrations/015_archive_lifecycle.sql' with { type: 'text' };
+import m018 from './migrations/016_team_spawner.sql' with { type: 'text' };
+import m019 from './migrations/017_wishes_table.sql' with { type: 'text' };
+import m020 from './migrations/018_drop_app_store.sql' with { type: 'text' };
+import m021 from './migrations/019_retention.sql' with { type: 'text' };
+import m022 from './migrations/020_agent_metadata.sql' with { type: 'text' };
+import m023 from './migrations/025_sdk_metadata_index.sql' with { type: 'text' };
+import m024 from './migrations/026_executors_omni_metadata_index.sql' with { type: 'text' };
+import m025 from './migrations/027_audit_events_notify.sql' with { type: 'text' };
+import m026 from './migrations/028_events_trace_id.sql' with { type: 'text' };
+import m027 from './migrations/029_omni_requests.sql' with { type: 'text' };
+import m028 from './migrations/029_session_display_name.sql' with { type: 'text' };
+import m029 from './migrations/031_mailbox_delivery_status.sql' with { type: 'text' };
+import m030 from './migrations/032_approvals.sql' with { type: 'text' };
+import m031 from './migrations/033_approval_request_notify.sql' with { type: 'text' };
+import m032 from './migrations/034_approvals_omni_message_id.sql' with { type: 'text' };
+import m033 from './migrations/035_bridge_sessions.sql' with { type: 'text' };
+import m034 from './migrations/036_teams_parent_chain.sql' with { type: 'text' };
+import m035 from './migrations/037_runtime_events_otel_columns.sql' with { type: 'text' };
+import m036 from './migrations/038_runtime_events_partition.sql' with { type: 'text' };
+import m037 from './migrations/039_runtime_events_siblings.sql' with { type: 'text' };
+import m038 from './migrations/040_listen_channel_split.sql' with { type: 'text' };
+import m039 from './migrations/041_rbac_roles.sql' with { type: 'text' };
+import m040 from './migrations/042_executor_turn_columns.sql' with { type: 'text' };
+import m041 from './migrations/043_detector_events_schema.sql' with { type: 'text' };
+import m042 from './migrations/043_executor_read_role.sql' with { type: 'text' };
+import m043 from './migrations/044_phase_b_flip_defaults.sql' with { type: 'text' };
+import m044 from './migrations/045_fix_stringified_jsonb.sql' with { type: 'text' };
+import m045 from './migrations/046_dir_agents_state_null.sql' with { type: 'text' };
+import m046 from './migrations/047_drop_agents_claude_session_id.sql' with { type: 'text' };
+import m047 from './migrations/048_session_sync_require_started_at.sql' with { type: 'text' };
+import m048 from './migrations/049_agents_kind_generated.sql' with { type: 'text' };
+import m049 from './migrations/050_archive_legacy_identity_rows.sql' with { type: 'text' };
+import m050 from './migrations/051_assignments_seq.sql' with { type: 'text' };
+import m051 from './migrations/053_master_backfill_and_shadow_cleanup.sql' with { type: 'text' };
+import m052 from './migrations/054_fix_subagent_team_inheritance.sql' with { type: 'text' };
+import m053 from './migrations/054_mailbox_source_meta.sql' with { type: 'text' };
+import m054 from './migrations/055_default_auto_resume_true.sql' with { type: 'text' };
+import m055 from './migrations/055_runtime_events_partition_drain.sql' with { type: 'text' };
+import m056 from './migrations/056_hook_perf_baseline_view.sql' with { type: 'text' };
+import m057 from './migrations/057_hook_perf_baseline_view_filter_fix.sql' with { type: 'text' };
+import m058 from './migrations/058_claude_usage_view.sql' with { type: 'text' };
+import m059 from './migrations/059_agent_observability_view.sql' with { type: 'text' };
+import m060 from './migrations/060_agent_observability_core_view.sql' with { type: 'text' };
+import m061 from './migrations/061_agents_id_invariant_and_fk_lockdown.sql' with { type: 'text' };
+import m062 from './migrations/062_drop_fk_mailbox_from_worker_temp.sql' with { type: 'text' };
+
+export interface EmbeddedMigration {
+  name: string;
+  sql: string;
+}
+
+/** All migrations, lexicographic order, embedded at build time. */
+export const EMBEDDED_MIGRATIONS: EmbeddedMigration[] = [
+  { name: '001_core', sql: m000 },
+  { name: '002_task_lifecycle', sql: m001 },
+  { name: '003_projects', sql: m002 },
+  { name: '004_clean_test_artifacts', sql: m003 },
+  { name: '005_pg_state', sql: m004 },
+  { name: '006_fix_agents_schema', sql: m005 },
+  { name: '007_observability', sql: m006 },
+  { name: '007_reconcile_mailbox_team_chat', sql: m007 },
+  { name: '008_boards', sql: m008 },
+  { name: '009_app_store', sql: m009 },
+  { name: '010_runtime_events', sql: m010 },
+  { name: '010_session_capture_v2', sql: m011 },
+  { name: '011_data_model_v5', sql: m012 },
+  { name: '012_executor_model', sql: m013 },
+  { name: '013_external_linking', sql: m014 },
+  { name: '014_comms_protocol', sql: m015 },
+  { name: '015_agent_archived_state', sql: m016 },
+  { name: '015_archive_lifecycle', sql: m017 },
+  { name: '016_team_spawner', sql: m018 },
+  { name: '017_wishes_table', sql: m019 },
+  { name: '018_drop_app_store', sql: m020 },
+  { name: '019_retention', sql: m021 },
+  { name: '020_agent_metadata', sql: m022 },
+  { name: '025_sdk_metadata_index', sql: m023 },
+  { name: '026_executors_omni_metadata_index', sql: m024 },
+  { name: '027_audit_events_notify', sql: m025 },
+  { name: '028_events_trace_id', sql: m026 },
+  { name: '029_omni_requests', sql: m027 },
+  { name: '029_session_display_name', sql: m028 },
+  { name: '031_mailbox_delivery_status', sql: m029 },
+  { name: '032_approvals', sql: m030 },
+  { name: '033_approval_request_notify', sql: m031 },
+  { name: '034_approvals_omni_message_id', sql: m032 },
+  { name: '035_bridge_sessions', sql: m033 },
+  { name: '036_teams_parent_chain', sql: m034 },
+  { name: '037_runtime_events_otel_columns', sql: m035 },
+  { name: '038_runtime_events_partition', sql: m036 },
+  { name: '039_runtime_events_siblings', sql: m037 },
+  { name: '040_listen_channel_split', sql: m038 },
+  { name: '041_rbac_roles', sql: m039 },
+  { name: '042_executor_turn_columns', sql: m040 },
+  { name: '043_detector_events_schema', sql: m041 },
+  { name: '043_executor_read_role', sql: m042 },
+  { name: '044_phase_b_flip_defaults', sql: m043 },
+  { name: '045_fix_stringified_jsonb', sql: m044 },
+  { name: '046_dir_agents_state_null', sql: m045 },
+  { name: '047_drop_agents_claude_session_id', sql: m046 },
+  { name: '048_session_sync_require_started_at', sql: m047 },
+  { name: '049_agents_kind_generated', sql: m048 },
+  { name: '050_archive_legacy_identity_rows', sql: m049 },
+  { name: '051_assignments_seq', sql: m050 },
+  { name: '053_master_backfill_and_shadow_cleanup', sql: m051 },
+  { name: '054_fix_subagent_team_inheritance', sql: m052 },
+  { name: '054_mailbox_source_meta', sql: m053 },
+  { name: '055_default_auto_resume_true', sql: m054 },
+  { name: '055_runtime_events_partition_drain', sql: m055 },
+  { name: '056_hook_perf_baseline_view', sql: m056 },
+  { name: '057_hook_perf_baseline_view_filter_fix', sql: m057 },
+  { name: '058_claude_usage_view', sql: m058 },
+  { name: '059_agent_observability_view', sql: m059 },
+  { name: '060_agent_observability_core_view', sql: m060 },
+  { name: '061_agents_id_invariant_and_fk_lockdown', sql: m061 },
+  { name: '062_drop_fk_mailbox_from_worker_temp', sql: m062 },
+];

src/lib/db-migrations.ts

@@ -13,6 +13,7 @@
 import { readdirSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import type postgres from 'postgres';
+import { EMBEDDED_MIGRATIONS } from '../db/migrations.generated.js';
 
 type Sql = postgres.Sql;
 
@@ -54,12 +55,28 @@ function getPackageRootMigrationsDir(): string {
 }
 
 /**
- * Load all .sql migration files sorted by name.
- * Searches two deterministic directories derived from import.meta.dir:
- *   1. dev layout: src/lib/../db/migrations
- *   2. bundled layout: dist/../src/db/migrations
+ * Load all .sql migrations sorted by name.
+ *
+ * Primary source: `EMBEDDED_MIGRATIONS` — static `with { type: 'text' }`
+ * imports that `bun build --compile` embeds into the binary. The previous
+ * readdirSync/Bun.file() approach read from disk relative to
+ * import.meta.dir, which inside the compiled executable resolves to an
+ * empty/absent `/$bunfs/...` path → zero migrations → "Applied: 0" on a
+ * fresh DB → schema never created (the bug that blocked native
+ * genie → pgserve-v3 migration).
+ *
+ * The on-disk scan is kept ONLY as a dev fallback (running from source
+ * with an unbuilt manifest); the embedded list is authoritative whenever
+ * present, which is always true in a shipped binary.
  */
 async function loadMigrationFiles(): Promise<MigrationFile[]> {
+  if (EMBEDDED_MIGRATIONS.length > 0) {
+    return EMBEDDED_MIGRATIONS.map((m) => ({ name: m.name, sql: m.sql })).sort((a, b) =>
+      a.name < b.name ? -1 : a.name > b.name ? 1 : 0,
+    );
+  }
+
+  // Dev-only fallback: source checkout whose manifest hasn't been generated.
   const candidates = [
     getMigrationsDir(), // dev: src/lib/../db/migrations
     getPackageRootMigrationsDir(), // bundled: dist/../src/db/migrations