fix(desktop): macOS installer ships an unopenable, unsigned app bundle

.github/workflows/desktop-apps.yml

@@ -30,6 +30,18 @@ on:
         description: 'Release tag to build + attach to (e.g. v0.3.0)'
         required: true
         type: string
+      notarize:
+        # Notarization needs an in-effect Apple Developer Program agreement. When
+        # one lapses, notarytool returns "HTTP 403: A required agreement is
+        # missing or has expired" and the whole build fails — even though signing
+        # itself is fine. Set false to validate the signing path meanwhile.
+        # NEVER ship a release built with this off: unnotarized apps are blocked
+        # by Gatekeeper on first launch.
+        description: 'Notarize with Apple (false = validate signing only, NOT releasable)'
+        required: false
+        default: 'true'
+        type: choice
+        options: ['true', 'false']
   # Called in-run from auto-release.yml: a GITHUB_TOKEN-created release does NOT
   # fire the `release` event, so the apps must be built from within the same run
   # that cut the release (same pattern as publish-images.yml).
@@ -68,6 +80,10 @@ jobs:
     timeout-minutes: 30
     env:
       TAG: ${{ github.event.release.tag_name || inputs.tag }}
+      # 'yes' unless a workflow_dispatch explicitly passed notarize=false. The
+      # `release` event and workflow_call leave the input undefined, so they
+      # always notarize — a release can never accidentally skip it.
+      NOTARIZE: ${{ inputs.notarize != 'false' && 'yes' || 'no' }}
       # Must match Supervisor.KERNEL_PORT in desktop/src/supervisor.ts — the
       # web-ui bakes this kernel URL into its routes-manifest at BUILD time, so
       # it cannot be a per-launch random port.
@@ -275,7 +291,14 @@ jobs:
             echo "APPLE_API_KEY=$RUNNER_TEMP/asc-key.p8"
             echo "APPLE_API_KEY_ID=$APPLE_ASC_KEY_ID"
             echo "APPLE_API_ISSUER=$APPLE_ASC_ISSUER_ID"
-            echo "EB_NOTARIZE=--config.mac.notarize=true"
+            # Must be set EXPLICITLY either way: electron-builder auto-notarizes
+            # as soon as APPLE_API_KEY/_ID/_ISSUER are in the environment, so
+            # simply leaving the flag off does NOT disable it.
+            if [ "${NOTARIZE:-yes}" = yes ]; then
+              echo "EB_NOTARIZE=--config.mac.notarize=true"
+            else
+              echo "EB_NOTARIZE=--config.mac.notarize=false"
+            fi
             echo "MAC_SIGN_KEYCHAIN=$KEYCHAIN"
             echo "MAC_SIGN_EXPECTED=1"
           } >> "$GITHUB_ENV"
@@ -293,12 +316,60 @@ jobs:
 
       - name: Package installers
         working-directory: desktop
-        run: npx electron-builder ${{ matrix.args }} ${EB_NOTARIZE:-} --publish never
+        run: |
+          # Belt-and-braces only. Signing hashes EVERY file in the bundle, and
+          # this step used to die with "EMFILE: too many open files" — but the fd
+          # limit is NOT the real lever: macOS caps concurrent open files at
+          # `kern.maxfilesperproc` regardless of `ulimit -n`, and setting the
+          # limit to `unlimited` actively LOWERS the effective ceiling. The fix
+          # is the file-count reduction in stage-runtime.mjs; this just avoids a
+          # needlessly small default. Keep it an explicit number, never
+          # "unlimited".
+          ulimit -n 65536 2>/dev/null || true
+          echo "fd soft=$(ulimit -Sn) kern.maxfilesperproc=$(sysctl -n kern.maxfilesperproc 2>/dev/null || echo n/a)"
+          echo "files in staged runtime: $(find runtime -type f | wc -l)"
+          # Debugging note: electron-builder retries a failing `codesign` four
+          # times and then reports only the LAST error, which masks the real
+          # cause. Re-run this step with `DEBUG=electron-builder` to see the
+          # actual command and its stderr.
+          npx electron-builder ${{ matrix.args }} ${EB_NOTARIZE:-} --publish never
+
+      # ALWAYS runs on macOS — signed or not. Without it, a build whose signature
+      # macOS considers corrupt sails through CI green and ships an app that
+      # cannot be opened at all ("omadia is damaged and can't be opened"), which
+      # is exactly what happened to v0.56.0 and v0.57.0. The stricter Developer ID
+      # assertions (not-adhoc, notarized, stapled) stay in the gate below; this one
+      # only asserts the bundle is STRUCTURALLY valid, which must hold either way.
+      - name: Verify the macOS bundle is structurally signed (always)
+        if: ${{ startsWith(matrix.os, 'macos') }}
+        working-directory: desktop
+        run: |
+          set -e
+          shopt -s nullglob
+          apps=(release/mac*/*.app)
+          if [ ${#apps[@]} -eq 0 ]; then
+            echo "FAIL: no .app produced under release/mac*/ — nothing to verify" >&2
+            exit 1
+          fi
+          for app in "${apps[@]}"; do
+            if [ ! -e "$app/Contents/_CodeSignature/CodeResources" ]; then
+              echo "FAIL: $app has no _CodeSignature/CodeResources — the bundle was never sealed." >&2
+              echo "      macOS reads this as corrupt and refuses to launch it." >&2
+              exit 1
+            fi
+            # `--deep` is required HERE: a plain `--verify --strict` passes even
+            # when a nested bundle (e.g. Electron Framework.framework) was never
+            # sealed, which is precisely the state that makes the app unopenable.
+            # Catches both that and a tree codesign cannot walk (e.g. dangling
+            # node_modules/.bin symlinks → "No such file or directory").
+            codesign --verify --deep --strict --verbose=2 "$app"
+          done
+          echo "✓ macOS bundle carries a structurally valid signature (nested code included)"
 
       # The .app is notarized + stapled by electron-builder above; the DMG needs
       # its own ticket so `stapler validate` passes on the disk image itself.
       - name: Notarize + staple the DMGs
-        if: ${{ startsWith(matrix.os, 'macos') && env.APPLE_P12_BASE64 != '' }}
+        if: ${{ startsWith(matrix.os, 'macos') && env.APPLE_P12_BASE64 != '' && env.NOTARIZE == 'yes' }}
         working-directory: desktop
         run: |
           for dmg in release/*.dmg; do
@@ -311,7 +382,7 @@ jobs:
           done
 
       - name: Verify macOS signatures (acceptance gate)
-        if: ${{ startsWith(matrix.os, 'macos') && env.APPLE_P12_BASE64 != '' }}
+        if: ${{ startsWith(matrix.os, 'macos') && env.APPLE_P12_BASE64 != '' && env.NOTARIZE == 'yes' }}
         working-directory: desktop
         run: |
           set -e

desktop/buildResources/afterPack.js

@@ -93,43 +93,122 @@ function collectMachO(dir, found) {
 exports.default = async function afterPack(context) {
   if (context.electronPlatformName !== 'darwin') return;
 
-  const identity = findIdentity();
-  if (!identity) {
-    // With signing secrets present (MAC_SIGN_EXPECTED=1) a missing identity is a
-    // HARD error — silently shipping unsigned nested modules would only fail
-    // later at notarization. Without secrets, this is the legitimate unsigned
-    // (dev / ad-hoc) path, so we skip.
-    if (process.env.MAC_SIGN_EXPECTED === '1') {
-      throw new Error(
-        '[afterPack] signing was expected (MAC_SIGN_EXPECTED=1) but no Developer ID ' +
-          'Application identity is available — the signing keychain was not set up ' +
-          'before packaging. Refusing to ship unsigned native modules.',
-      );
-    }
-    console.log('[afterPack] no Developer ID identity — skipping nested signing (unsigned build).');
-    return;
+  const developerId = findIdentity();
+  // With signing secrets present (MAC_SIGN_EXPECTED=1) a missing identity is a
+  // HARD error — silently shipping unsigned nested modules would only fail later
+  // at notarization.
+  if (!developerId && process.env.MAC_SIGN_EXPECTED === '1') {
+    throw new Error(
+      '[afterPack] signing was expected (MAC_SIGN_EXPECTED=1) but no Developer ID ' +
+        'Application identity is available — the signing keychain was not set up ' +
+        'before packaging. Refusing to ship unsigned native modules.',
+    );
   }
 
+  // Without a Developer ID we still sign — AD-HOC ("-"), not "not at all".
+  //
+  // WHY: electron-builder SKIPS its own signing step entirely when no identity
+  // exists ("skipped macOS application code signing … 0 identities found"). The
+  // packaged .app then has no `Contents/_CodeSignature` at all, while the Electron
+  // binary inside it still carries its linker-signed ad-hoc signature. macOS reads
+  // that combination as CORRUPT — `codesign --verify` and Apple's own
+  // `syspolicy_check` both report "code has no resources but signature indicates
+  // they must be present" (Severity: Fatal) — and Gatekeeper refuses to launch the
+  // downloaded app with "omadia is damaged and can't be opened."
+  //
+  // An ad-hoc signature is still not distributable (users need a right-click →
+  // Open, and notarization is impossible), but it is STRUCTURALLY VALID, so a
+  // cert-less build produces an app that runs instead of one that cannot open.
+  const identity = developerId ?? '-';
+  const adhoc = !developerId;
+
   const appName = `${context.packager.appInfo.productFilename}.app`;
-  const resources = path.join(context.appOutDir, appName, 'Contents', 'Resources');
+  const appPath = path.join(context.appOutDir, appName);
+  const resources = path.join(appPath, 'Contents', 'Resources');
   // Sign Mach-O in BOTH staged extraResources trees: the middleware native
   // modules (omadia/) and the bundled Postgres engine (omadia-pg/).
   const found = new Set();
   collectMachO(path.join(resources, 'omadia'), found);
   collectMachO(path.join(resources, 'omadia-pg'), found);
   const targets = [...found];
-  if (targets.length === 0) {
-    console.log('[afterPack] no nested Mach-O binaries found under extraResources.');
-    return;
-  }
 
   const keychain = (process.env.MAC_SIGN_KEYCHAIN || '').trim();
-  const args = ['--force', '--options', 'runtime', '--timestamp'];
+  const args = ['--force', '--options', 'runtime'];
+  // A secure timestamp needs a real certificate + Apple's timestamp server; it is
+  // rejected for ad-hoc signatures, so only request it on the Developer ID path.
+  if (!adhoc) args.push('--timestamp');
   if (keychain) args.push('--keychain', keychain);
   args.push('--sign', identity);
 
-  console.log(`[afterPack] signing ${targets.length} nested Mach-O binaries with "${identity}"`);
-  for (const target of targets) {
-    execFileSync('codesign', [...args, target], { stdio: 'inherit' });
+  const label = adhoc ? 'ad-hoc' : `"${identity}"`;
+  if (targets.length === 0) {
+    console.log('[afterPack] no nested Mach-O binaries found under extraResources.');
+  } else {
+    console.log(`[afterPack] signing ${targets.length} nested Mach-O binaries ${label}`);
+    for (const target of targets) {
+      execFileSync('codesign', [...args, target], { stdio: 'inherit' });
+    }
+  }
+
+  // Seal the outer bundle ONLY on the ad-hoc path. On the Developer ID path
+  // electron-builder signs the app itself right after this hook (with the correct
+  // entitlements and, when requested, notarization) — signing it here would just
+  // be overwritten.
+  if (adhoc) {
+    const entitlements = path.join(__dirname, 'entitlements.mac.plist');
+    const frameworks = path.join(appPath, 'Contents', 'Frameworks');
+
+    // Nested code must be signed BOTTOM-UP: each inner bundle's signature is
+    // sealed into its parent, so signing the outer app first would be
+    // invalidated by every later inner signature. Signing only the outer app
+    // leaves e.g. `Electron Framework.framework` unsealed, and macOS then still
+    // reports "code has no resources but signature indicates they must be
+    // present" for the whole app.
+    let entries = [];
+    try {
+      entries = fs.readdirSync(frameworks);
+    } catch {
+      /* no Frameworks dir (unexpected for Electron, but non-fatal) */
+    }
+
+    // 1. Versioned frameworks — sign `Versions/A`, not the symlinked top level.
+    for (const name of entries.filter((n) => n.endsWith('.framework'))) {
+      const versionA = path.join(frameworks, name, 'Versions', 'A');
+      if (!fs.existsSync(versionA)) continue;
+      execFileSync('codesign', ['--force', '--sign', '-', versionA], { stdio: 'inherit' });
+    }
+
+    // 2. Helper apps (GPU / Plugin / Renderer / main) — same entitlements as the
+    //    outer app so the forked kernel keeps JIT + library-validation relief.
+    for (const name of entries.filter((n) => n.endsWith('.app'))) {
+      execFileSync(
+        'codesign',
+        [
+          '--force',
+          '--options',
+          'runtime',
+          '--entitlements',
+          entitlements,
+          '--sign',
+          '-',
+          path.join(frameworks, name),
+        ],
+        { stdio: 'inherit' },
+      );
+    }
+
+    // 3. The outer app last, sealing everything above it.
+    console.log(`[afterPack] ad-hoc signing the app bundle → ${appName}`);
+    execFileSync(
+      'codesign',
+      ['--force', '--options', 'runtime', '--entitlements', entitlements, '--sign', '-', appPath],
+      { stdio: 'inherit' },
+    );
+
+    // Prove the bundle macOS will actually see is valid, nested code included.
+    // `--deep` is the check that surfaces an unsealed nested framework — the
+    // exact defect that made the shipped v0.56.0 / v0.57.0 apps unopenable.
+    execFileSync('codesign', ['--verify', '--deep', '--strict', appPath], { stdio: 'inherit' });
+    console.log('[afterPack] ad-hoc signature verified (unsigned build — not distributable).');
   }
 };

desktop/scripts/stage-runtime.mjs

@@ -152,4 +152,121 @@ if (missing.length) {
 }
 console.log(`[stage-runtime] staged Postgres engine (${pgPlat}) + pgvector (control + ${moduleName} + install SQL)`);
 
+// --- drop build-time-only files from the staged tree ---------------------
+// The staged middleware ships its FULL node_modules as unpacked extraResources,
+// which is ~34k files — and over half of them can never be loaded at runtime:
+// the kernel runs compiled JS (`middleware/dist/index.js`), so TypeScript
+// declarations, TypeScript sources and source maps are pure ballast.
+//
+// This is not just about size. macOS signing hashes EVERY file in the bundle,
+// and the kernel caps concurrent open files per process at `kern.maxfilesperproc`
+// REGARDLESS of `ulimit -n` — so on a ~44k-file bundle electron-builder dies with
+// "EMFILE: too many open files" and the app cannot be signed at all. Raising the
+// fd limit does not help (setting it to `unlimited` on macOS actually lowers the
+// effective ceiling); cutting the file count does.
+//
+// Deliberately does NOT touch *.md — LICENSE.md and friends must ship.
+const BALLAST = /\.(ts|mts|cts|map)$/;
+function pruneBuildTimeOnlyFiles(root) {
+  let pruned = 0;
+  const walk = (dir) => {
+    let entries;
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const p = path.join(dir, entry.name);
+      if (entry.isSymbolicLink()) continue; // handled by the dangling sweep below
+      if (entry.isDirectory()) {
+        walk(p);
+      } else if (entry.isFile() && BALLAST.test(entry.name)) {
+        fs.rmSync(p, { force: true });
+        pruned++;
+      }
+    }
+  };
+  walk(root);
+  return pruned;
+}
+
+console.log(
+  `[stage-runtime] pruned ${pruneBuildTimeOnlyFiles(mwDest)} build-time-only file(s) (*.d.ts, *.ts, *.map)`,
+);
+
+// --- prune symlinks that escape the staged tree --------------------------
+// `fs.cpSync({ dereference: true })` does not materialise EVERY symlink: npm's
+// `node_modules/.bin/*` entries survive the copy as ABSOLUTE links back into the
+// BUILD machine's checkout (on CI: `/Users/runner/work/omadia/omadia/...`).
+// There are 58 of them; nothing execs any of them at runtime.
+//
+// They are FATAL to macOS signing, in two different ways depending on where you
+// look from — which is what makes them easy to misdiagnose:
+//   • ON THE BUILD MACHINE the targets still exist, so the links RESOLVE. They
+//     point outside the .app, and `codesign --verify` rejects the bundle with
+//     "invalid destination for symbolic link in bundle".
+//   • ANYWHERE ELSE the targets are gone, so they DANGLE, and codesign aborts
+//     walking the tree with "No such file or directory".
+// Testing "does the target exist?" therefore passes on CI and prunes nothing —
+// the check has to be "does the target stay INSIDE the staged tree?".
+//
+// Either way the .app ends up with no valid signature and Gatekeeper refuses to
+// launch it ("omadia is damaged and can't be opened") — exactly how v0.56.0 and
+// v0.57.0 shipped.
+//
+// Runs LAST, after every stage step (including the Postgres relink above). The
+// 31 legitimate in-tree links — Electron's `Versions/Current` framework layout
+// and the engine's relative `libicudata.68.dylib` chain — resolve inside the
+// tree and are kept.
+function pruneEscapingSymlinks(root) {
+  const rootReal = fs.realpathSync(root);
+  const inside = (p) => p === rootReal || p.startsWith(rootReal + path.sep);
+  let pruned = 0;
+  const walk = (dir) => {
+    let entries;
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const p = path.join(dir, entry.name);
+      // Dirent flags are lstat-based, so a symlink is never mistaken for a dir.
+      if (entry.isSymbolicLink()) {
+        let keep = false;
+        try {
+          // realpathSync throws on a dangling link and fully resolves an
+          // absolute or `../`-escaping one — both are handled by this branch.
+          keep = inside(fs.realpathSync(p));
+        } catch {
+          keep = false;
+        }
+        if (!keep) {
+          fs.rmSync(p, { force: true });
+          pruned++;
+        }
+        continue; // never descend THROUGH a symlink — avoids cycles
+      }
+      if (entry.isDirectory()) walk(p);
+    }
+  };
+  walk(root);
+  return pruned;
+}
+
+console.log(`[stage-runtime] pruned ${pruneEscapingSymlinks(runtime)} escaping symlink(s)`);
+
+// Hard gate: re-scan and require a clean tree. A second pass must find nothing —
+// if it does, the prune itself is broken and we must not ship a tree that
+// `codesign` (and therefore notarization) will choke on.
+const stillDangling = pruneEscapingSymlinks(runtime);
+if (stillDangling !== 0) {
+  console.error(
+    `[stage-runtime] FATAL: ${stillDangling} escaping symlink(s) survived the prune — ` +
+      'refusing to stage a tree that macOS codesign cannot walk.',
+  );
+  process.exit(1);
+}
+
 console.log('[stage-runtime] done →', runtime);