feat(cef): crash-symbolization proof via dump_syms + minidump-stackwalk

.github/workflows/cef-learning-harness.yml

@@ -38,6 +38,16 @@
 # or type error fails in ~2-3min instead of after the full ~30min SDK+CMake+Rust+Xvfb
 # cycle. `Swatinem/rust-cache` caches cargo's registry/git dirs for the rust-core
 # crate Corrosion builds via CMake, cutting redundant crates.io re-fetches.
+#
+# QNBS-v3: CMAKE_BUILD_TYPE stays Release (RelWithDebInfo broke CEF's own chrome-sandbox
+# copy step — the fetched SDK's prebuilt binaries only ship Release/ and Debug/
+# subdirectories, so CEF's build macros looked for a nonexistent RelWithDebInfo/ source
+# path; a real CI failure, not a guess). worldscript_host gets its own -g flag directly
+# (apps/desktop-cef/CMakeLists.txt) instead, for the crash-symbolization proof
+# (scripts/cef/run-symbolization-proof.mjs)'s dump_syms/minidump-stackwalk to resolve
+# function names against. Both tools are standalone Rust projects with tiny prebuilt
+# Linux binaries — neither needs a Chromium checkout, confirmed by reading their
+# own READMEs directly.
 # ============================================================
 
 name: 🧪 CEF Learning Harness
@@ -157,6 +167,26 @@ jobs:
       - name: Linux runtime linkage check (ldd against shipped .so files)
         run: node scripts/cef/check-linux-runtime-linkage.mjs build/worldscript_host
 
+      # QNBS-v3: pinned prebuilt release binaries (~3.6MB each), sha256-verified — not built from
+      # source (would need a slow cargo build for two more Rust tools) and not a Chromium
+      # checkout (neither tool needs one; see the header comment).
+      - name: Fetch dump_syms + minidump-stackwalk (crash-symbolization tools)
+        run: |
+          set -euo pipefail
+          mkdir -p symtools
+          curl -sSL --fail --retry 3 --retry-all-errors "https://github.com/mozilla/dump_syms/releases/download/v2.3.9/dump_syms-x86_64-unknown-linux-gnu.tar.xz" -o symtools/dump_syms.tar.xz
+          curl -sSL --fail --retry 3 --retry-all-errors "https://github.com/rust-minidump/rust-minidump/releases/download/v0.27.0/minidump-stackwalk-x86_64-unknown-linux-gnu.tar.xz" -o symtools/mds.tar.xz
+          echo "0fc852a86b00337407d9d423cc388a24c3b489ccaaedcf92623cad57af5ca8ad  symtools/dump_syms.tar.xz" | sha256sum -c -
+          echo "0020324c54cc359596e927ee907204f4c8d4da6718765536edcabaa1622122ad  symtools/mds.tar.xz" | sha256sum -c -
+          tar xf symtools/dump_syms.tar.xz -C symtools
+          tar xf symtools/mds.tar.xz -C symtools
+          DUMP_SYMS_BIN=$(find symtools -name dump_syms -type f -print -quit)
+          MINIDUMP_STACKWALK_BIN=$(find symtools -name minidump-stackwalk -type f -print -quit)
+          test -x "$DUMP_SYMS_BIN" || { echo "dump_syms binary not found in archive" >&2; exit 1; }
+          test -x "$MINIDUMP_STACKWALK_BIN" || { echo "minidump-stackwalk binary not found in archive" >&2; exit 1; }
+          echo "DUMP_SYMS_BIN=$(pwd)/$DUMP_SYMS_BIN" >> "$GITHUB_ENV"
+          echo "MINIDUMP_STACKWALK_BIN=$(pwd)/$MINIDUMP_STACKWALK_BIN" >> "$GITHUB_ENV"
+
       - name: Serve production bundle + repeated launch/close proof
         run: |
           python3 -m http.server 8080 --directory dist &
@@ -167,6 +197,13 @@ jobs:
             "http://localhost:8080/" --cycles 3
           kill "$SERVER_PID"
 
+      - name: Crash-symbolization proof (dump_syms + minidump-stackwalk)
+        run: |
+          xvfb-run -a node scripts/cef/run-symbolization-proof.mjs \
+            "$(pwd)/build/worldscript_host/worldscript_host" \
+            "$DUMP_SYMS_BIN" \
+            "$MINIDUMP_STACKWALK_BIN"
+
       - name: Best-effort Wayland launch smoke (roadmap §44.2)
         id: wayland-smoke
         continue-on-error: true
@@ -189,5 +226,6 @@ jobs:
           echo "- Cache hit: \`${{ steps.cef-cache.outputs.cache-hit }}\`" >> "$GITHUB_STEP_SUMMARY"
           echo "- worldscript_host built and repeated launch/close cycles proven against the real production bundle (dist/), under Xvfb." >> "$GITHUB_STEP_SUMMARY"
           echo "- Linux runtime linkage (ldd against the shipped worldscript_host + libcef.so, not just dpkg package presence): see the \"Linux runtime linkage check\" step above." >> "$GITHUB_STEP_SUMMARY"
+          echo "- Crash-symbolization proof: a self-induced browser-process crash resolved via dump_syms + minidump-stackwalk against our own DWARF debug info — Chromium/CEF-internal frames remain unsymbolized (no debug-symbols archive is published for this distribution)." >> "$GITHUB_STEP_SUMMARY"
           echo "- Wayland smoke (best-effort, roadmap §44.2): \`${{ steps.wayland-smoke.outcome }}\`" >> "$GITHUB_STEP_SUMMARY"
-          echo "- Not yet in scope: X11/Wayland matrix beyond this one runner, sandbox posture, accessibility smoke." >> "$GITHUB_STEP_SUMMARY"
+          echo "- Not yet in scope: X11/Wayland matrix beyond this one runner, sandbox posture, accessibility-tree observability (AT-SPI — state enablement is proven, see the launch-cycle-proof step)." >> "$GITHUB_STEP_SUMMARY"

apps/desktop-cef/CMakeLists.txt

@@ -36,6 +36,13 @@ set(WORLDSCRIPT_HOST_SRCS
 add_executable(worldscript_host ${WORLDSCRIPT_HOST_SRCS})
 add_dependencies(worldscript_host libcef_dll_wrapper)
 
+# QNBS-v3: -g directly on this target only, not CMAKE_BUILD_TYPE=RelWithDebInfo — that broke
+# CEF's own chrome-sandbox copy step (the fetched SDK's prebuilt binaries only ship Release/ and
+# Debug/ subdirectories, not RelWithDebInfo/; a real CI failure, not a guess). This keeps CEF's
+# own macros on the Release path they expect while still giving our own crash-symbolization proof
+# (scripts/cef/run-symbolization-proof.mjs) real DWARF debug info to resolve function names from.
+target_compile_options(worldscript_host PRIVATE -g)
+
 ADD_LOGICAL_TARGET("libcef_lib" "${CEF_LIB_DEBUG}" "${CEF_LIB_RELEASE}")
 
 target_link_libraries(worldscript_host

apps/desktop-cef/rust-core/Cargo.toml

@@ -9,3 +9,7 @@ crate-type = ["staticlib"]
 
 [profile.release]
 panic = "abort"
+# QNBS-v3: cargo's release profile strips debug info by default — without this, dump_syms would
+# have nothing to extract from the Rust side of worldscript_host, and the Wave 2 crash-symbolization
+# proof couldn't resolve worldscript_rust_debug_crash_self_test's function name. No runtime cost.
+debug = true

apps/desktop-cef/rust-core/src/lib.rs

@@ -10,3 +10,13 @@ pub extern "C" fn worldscript_rust_ping() -> i32 {
     // QNBS-v3: non-trivial sentinel so a stub/miscompiled stand-in can't accidentally match by coincidence (e.g. a default-zeroed return reading as success).
     424242
 }
+
+/// Deliberate, distinctively-named crash for the CI crash-symbolization proof
+/// (docs/cef/ROADMAP-CEF-DESKTOP-MIGRATION.md Wave 2 exit criterion). Only ever called
+/// behind the `--debug-crash-self` CLI flag (see apps/desktop-cef/src/main.cpp) — never
+/// reachable in normal operation. `panic = "abort"` (this crate's release profile) turns
+/// this into a real SIGABRT Crashpad can catch, not an unwind.
+#[no_mangle]
+pub extern "C" fn worldscript_rust_debug_crash_self_test() {
+    panic!("worldscript_rust_debug_crash_self_test: deliberate self-test crash");
+}

apps/desktop-cef/src/main.cpp

@@ -7,6 +7,10 @@
 #include "shutdown_signal.h"
 #include "worldscript_app.h"
 
+// QNBS-v3: crash-symbolization proof (Wave 2 exit criterion) — only ever invoked behind
+// --debug-crash-self below, never reachable in normal operation.
+extern "C" void worldscript_rust_debug_crash_self_test();
+
 namespace {
 
 // QNBS-v3: defaults to about:blank so a bare invocation (e.g. a subprocess re-exec) never depends on --url being present.
@@ -21,11 +25,21 @@ std::string ParseStartUrl(int argc, char* argv[]) {
   return "about:blank";
 }
 
+bool HasDebugCrashSelfFlag(int argc, char* argv[]) {
+  for (int i = 1; i < argc; ++i) {
+    if (std::string(argv[i]) == "--debug-crash-self") {
+      return true;
+    }
+  }
+  return false;
+}
+
 }  // namespace
 
 int main(int argc, char* argv[]) {
   CefMainArgs main_args(argc, argv);
 
+  const bool debug_crash_self = HasDebugCrashSelfFlag(argc, argv);
   CefRefPtr<WorldScriptApp> app(new WorldScriptApp(ParseStartUrl(argc, argv)));
 
   // QNBS-v3: every CEF host re-executes itself for renderer/GPU/utility subprocesses — must run before CefInitialize; non-negative return means this invocation *was* one of those, already run to completion.
@@ -51,6 +65,15 @@ int main(int argc, char* argv[]) {
          CefCrashReportingEnabled() ? "true" : "false");
   fflush(stdout);
 
+  // QNBS-v3: Wave 2 crash-symbolization proof only — deliberately crashes the browser
+  // process itself (not a renderer subprocess) so the resulting dump's stack is our own
+  // code, not Chromium/CEF internals we have no debug symbols for.
+  if (debug_crash_self) {
+    printf("[worldscript_host] debug_crash_self_test = triggering\n");
+    fflush(stdout);
+    worldscript_rust_debug_crash_self_test();
+  }
+
   CefRunMessageLoop();
   CefShutdown();
 

docs/architecture/native-readiness.md

@@ -62,10 +62,10 @@ Wave 2's first deliverable — the CEF binding/C++ decision — is now backed by
 | CEF lifecycle assumptions documented | **PASS** | cef-runtime | `docs/cef/knowledge/subprocess-and-shutdown.md`'s core Wave 2 claim (SIGTERM → graceful `TryCloseBrowser`/`OnBeforeClose`/`CefQuitMessageLoop`/`CefShutdown`, repeated clean start/close cycles) now has a real linked chain: test (`scripts/cef/run-launch-cycle-proof.mjs`) → CI job (`🧪 CEF Learning Harness`) → doc, exactly what §61.1.4 requires. Save-coordinator/window-state persistence remain explicitly Wave 5+ scope (not a Wave 2 gap); Windows/macOS and a real packaged layout remain open, tracked in the doc's own "Outline" section. |
 | Early Accessibility Gate | **PASS** — state enablement only, tree observability open | cef-runtime, Wave 2 | PR #391 found the real root cause: `GetAccessibilityHandler()` is declared on `CefRenderHandler` (OSR-only, "when window rendering is disabled"), not `CefClient` — never reachable from this windowed host regardless of what was inherited. PR #397: `CefBrowserHost::SetAccessibilityState(STATE_ENABLED)` alone is windowed-mode-correct per its own doc comment; CI-proven in every one of 3 repeated cycles with zero regression to the FFI/rendering/crash-reporting/Wayland proofs. Does **not** yet prove the platform accessibility tree is observable — that needs OS-level AT-SPI introspection on Linux, separate unattempted follow-up — see `docs/cef/knowledge/cef-architecture-primer.md`. |
 | Sandbox posture | Not yet attempted | desktop-security, Wave 2/3 (roadmap §12) | Every run so far used `no_sandbox=true`; zero evidence either way on this row. |
-| Crash reporting / renderer-crash resilience | **PASS** — crash-reporting half only | cef-runtime | PR #392: `crash_reporter.cfg` + `CefCrashReportingEnabled()` verified true, `chrome://crash` deliberately crashes the renderer, `CefRequestHandler::OnRenderProcessTerminated` fires (`TS_PROCESS_CRASHED`), the browser process/message loop survive, and a real Crashpad `.dmp` file — the harness's actual assertion, alongside Crashpad's own `.meta`/`settings.dat` housekeeping files (observed, not independently asserted) — is produced under an overridden `BREAKPAD_DUMP_LOCATION`; all CI-run, not a doc claim. Symbolization (decoding the dump into a stack trace via `dump_syms`/`minidump_stackwalk`) needs a full Chromium source checkout and was not attempted — see `docs/cef/knowledge/cef-architecture-primer.md`. |
+| Crash reporting / renderer-crash resilience / symbolization | **PASS** — Chromium-internal frames excepted | cef-runtime | PR #392: `crash_reporter.cfg` + `CefCrashReportingEnabled()` verified true, `chrome://crash` deliberately crashes the renderer, `CefRequestHandler::OnRenderProcessTerminated` fires (`TS_PROCESS_CRASHED`), the browser process/message loop survive, and a real Crashpad `.dmp` file — the harness's actual assertion, alongside Crashpad's own `.meta`/`settings.dat` housekeeping files (observed, not independently asserted) — is produced under an overridden `BREAKPAD_DUMP_LOCATION`; all CI-run, not a doc claim. PR #400: symbolization also proven — the initial "needs a full Chromium checkout" assumption was wrong for our own code's frames; `dump_syms`/`minidump-stackwalk` (standalone Rust tools, no Chromium checkout) resolved a self-induced browser-process crash (`--debug-crash-self`) end-to-end back to the crashing function's name. Chromium/CEF-internal frames remain genuinely unsymbolized — no distribution type ships debug symbols, verified against CEF's own build index — see `docs/cef/knowledge/cef-architecture-primer.md`. |
 | CEF SDK fetch/verify + version diagnostics automated | **PASS** | cef-runtime | `🧪 CEF Learning Harness` CI job (`.github/workflows/cef-learning-harness.yml`) fetches the pinned CEF SDK, verifies its checksum, and parses real version macros out of the extracted `include/cef_version.h` — a genuine CI-run check, not a doc claim. |
 | Linux dependency inventory — clean-machine data point | DEBT — partial | cef-runtime | Same CI job runs the package-presence check against a stock `ubuntu-latest` runner before any `apt-get`, adding a real second data point beyond the spike's one already-configured dev machine. PR #395 added the specific check this row previously flagged as missing: `scripts/cef/check-linux-runtime-linkage.mjs` runs `ldd` against the real, already-built `worldscript_host` and `libcef.so` — both fully resolved on the runner, zero unresolved dependencies. Still narrow: one distro/runner image only, no packaged-installer dependency declaration. |
 | CEF host build + repeated launch/close cycle proof, in CI | **PASS** | cef-runtime | PR #388: `apps/desktop-cef/`'s `worldscript_host` (real, repo-committed C++/Rust source, not spike code) builds against the fetched CEF SDK and runs 3 independently-verified clean start/close cycles under Xvfb in CI — the roadmap's literal "isolated learning harness" / "safe repeated startup/shutdown" deliverables (§3142), not just the fetch/diagnostics increment. |
 | Rust FFI boundary proven inside the real host | **PASS** | cef-runtime, rust-core | `worldscript_rust_ping()` (rust-core, linked via Corrosion) is called from `OnAfterCreated` on every cycle and its exact sentinel value observed in CI output — stronger than the ADR-0020 spike's decoupled isolation test, since this proves the boundary works inside the actual multi-process CEF host, not a standalone C++ program. |
 
-**Overall for this snapshot**: 8 PASS (crash reporting explicitly PASS for its reporting half only, not symbolization; Wayland explicitly PASS for a single-runner smoke only, not the real-hardware/compositor matrix; Early Accessibility Gate explicitly PASS for state enablement only, not tree observability), 2 explicit `DEBT — partial` rows (each with a concrete exit condition, not open-ended), 1 row marked `Not yet attempted` (Sandbox posture) rather than assumed. No row is marked PASS without the evidence cited above.
+**Overall for this snapshot**: 8 PASS (crash reporting/symbolization explicitly PASS except for Chromium-internal frames, which no CEF distribution ships debug symbols for; Wayland explicitly PASS for a single-runner smoke only, not the real-hardware/compositor matrix; Early Accessibility Gate explicitly PASS for state enablement only, not tree observability), 2 explicit `DEBT — partial` rows (each with a concrete exit condition, not open-ended), 1 row marked `Not yet attempted` (Sandbox posture) rather than assumed. No row is marked PASS without the evidence cited above.