Dynamically detect libgcc-vs-libunwind

lib/compiler/Cargo.toml

@@ -37,6 +37,7 @@ bytes = "1.0"
 self_cell = "1.0"
 rkyv = { workspace = true }
 shared-buffer = { workspace = true }
+libc.workspace = true
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 wasmer-vm = { path = "../vm", version = "=4.3.2" }

lib/compiler/src/engine/unwind/systemv.rs

@@ -3,6 +3,8 @@
 
 //! Module for System V ABI unwind registry.
 
+use core::sync::atomic::{AtomicUsize, Ordering::Relaxed};
+
 use wasmer_types::CompiledFunctionUnwindInfoReference;
 
 /// Represents a registry of function unwind information for System V ABI.
@@ -17,6 +19,66 @@ extern "C" {
     fn __deregister_frame(fde: *const u8);
 }
 
+/// There are two primary unwinders on Unix platforms: libunwind and libgcc.
+///
+/// Unfortunately their interface to `__register_frame` is different. The
+/// libunwind library takes a pointer to an individual FDE while libgcc takes a
+/// null-terminated list of FDEs. This means we need to know what unwinder
+/// is being used at runtime.
+///
+/// This detection is done currently by looking for a libunwind-specific symbol.
+/// This specific symbol was somewhat recommended by LLVM's
+/// "RTDyldMemoryManager.cpp" file which says:
+///
+/// > We use the presence of __unw_add_dynamic_fde to detect libunwind.
+///
+/// I'll note that there's also a different libunwind project at
+/// https://www.nongnu.org/libunwind/ but that doesn't appear to have
+/// `__register_frame` so I don't think that interacts with this.
+fn using_libunwind() -> bool {
+    static USING_LIBUNWIND: AtomicUsize = AtomicUsize::new(LIBUNWIND_UNKNOWN);
+
+    const LIBUNWIND_UNKNOWN: usize = 0;
+    const LIBUNWIND_YES: usize = 1;
+    const LIBUNWIND_NO: usize = 2;
+
+    // On macOS the libgcc interface is never used so libunwind is always used.
+    if cfg!(target_os = "macos") {
+        return true;
+    }
+
+    // On other platforms the unwinder can vary. Sometimes the unwinder is
+    // selected at build time and sometimes it differs at build time and runtime
+    // (or at least I think that's possible). Fall back to a `libc::dlsym` to
+    // figure out what we're using and branch based on that.
+    //
+    // Note that the result of `libc::dlsym` is cached to only look this up
+    // once.
+    match USING_LIBUNWIND.load(Relaxed) {
+        LIBUNWIND_YES => true,
+        LIBUNWIND_NO => false,
+        LIBUNWIND_UNKNOWN => {
+            let looks_like_libunwind = unsafe {
+                !libc::dlsym(
+                    std::ptr::null_mut(),
+                    "__unw_add_dynamic_fde\0".as_ptr().cast(),
+                )
+                .is_null()
+            };
+            USING_LIBUNWIND.store(
+                if looks_like_libunwind {
+                    LIBUNWIND_YES
+                } else {
+                    LIBUNWIND_NO
+                },
+                Relaxed,
+            );
+            looks_like_libunwind
+        }
+        _ => unreachable!(),
+    }
+}
+
 impl UnwindRegistry {
     /// Creates a new unwind registry with the given base address.
     pub fn new() -> Self {
@@ -60,10 +122,7 @@ impl UnwindRegistry {
 
     #[allow(clippy::cast_ptr_alignment)]
     unsafe fn register_frames(&mut self, eh_frame: &[u8]) {
-        if cfg!(any(
-            all(target_os = "linux", target_env = "gnu"),
-            target_os = "freebsd"
-        )) {
+        if !using_libunwind() {
             // Registering an empty `eh_frame` (i.e. which
             // contains empty FDEs) cause problems on Linux when
             // deregistering it. We must avoid this