Add Libdl.LazyLibrary

This provides an in-base mechanism to handle chained library
dependencies.  In essence, the `LazyLibrary` object can be used anywhere
a pointer to a library can be used (`dlopen`, `dlsym`, `ccall`, etc...)
but it delays loading the library (and its recursive dependencies) until
it is actually needed.

This is the foundational piece needed to upgrade JLLs to lazily-load
their libraries.  In this new scheme, JLLs would generally lose all
executable code and consist of nothing more than `LazyLibrary`
definitions.

Author: staticfloat

Makefile

@@ -84,6 +84,9 @@ julia-base: julia-deps $(build_sysconfdir)/julia/startup.jl $(build_man1dir)/jul
 julia-libccalltest: julia-deps
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src libccalltest
 
+julia-libccalltestdep: julia-deps julia-libccalltest
+	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src libccalltestdep
+
 julia-libllvmcalltest: julia-deps
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT)/src libllvmcalltest
 
@@ -102,7 +105,7 @@ julia-sysimg-bc : julia-stdlib julia-base julia-cli-$(JULIA_BUILD_MODE) julia-sr
 julia-sysimg-release julia-sysimg-debug : julia-sysimg-% : julia-sysimg-ji julia-src-%
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-$*
 
-julia-debug julia-release : julia-% : julia-sysimg-% julia-src-% julia-symlink julia-libccalltest julia-libllvmcalltest julia-base-cache
+julia-debug julia-release : julia-% : julia-sysimg-% julia-src-% julia-symlink julia-libccalltest julia-libccalltestdep julia-libllvmcalltest julia-base-cache
 
 stdlibs-cache-release stdlibs-cache-debug : stdlibs-cache-% : julia-%
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f pkgimage.mk all-$*
@@ -189,7 +192,7 @@ JL_TARGETS := julia-debug
 endif
 
 # private libraries, that are installed in $(prefix)/lib/julia
-JL_PRIVATE_LIBS-0 := libccalltest libllvmcalltest
+JL_PRIVATE_LIBS-0 := libccalltest libccalltestdep libllvmcalltest
 ifeq ($(JULIA_BUILD_MODE),release)
 JL_PRIVATE_LIBS-0 += libjulia-internal libjulia-codegen
 else ifeq ($(JULIA_BUILD_MODE),debug)

base/libdl.jl

@@ -9,7 +9,7 @@ import Base.DL_LOAD_PATH
 
 export DL_LOAD_PATH, RTLD_DEEPBIND, RTLD_FIRST, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL,
     RTLD_NODELETE, RTLD_NOLOAD, RTLD_NOW, dlclose, dlopen, dlopen_e, dlsym, dlsym_e,
-    dlpath, find_library, dlext, dllist
+    dlpath, find_library, dlext, dllist, LazyLibrary
 
 """
     DL_LOAD_PATH
@@ -45,6 +45,9 @@ applicable.
 """
 (RTLD_DEEPBIND, RTLD_FIRST, RTLD_GLOBAL, RTLD_LAZY, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_NOW)
 
+# The default flags for `dlopen()`
+const default_rtld_flags = RTLD_LAZY | RTLD_DEEPBIND
+
 """
     dlsym(handle, sym; throw_error::Bool = true)
 
@@ -72,8 +75,8 @@ end
 Look up a symbol from a shared library handle, silently return `C_NULL` on lookup failure.
 This method is now deprecated in favor of `dlsym(handle, sym; throw_error=false)`.
 """
-function dlsym_e(hnd::Ptr, s::Union{Symbol,AbstractString})
-    return something(dlsym(hnd, s; throw_error=false), C_NULL)
+function dlsym_e(args...)
+    return something(dlsym(args...; throw_error=false), C_NULL)
 end
 
 """
@@ -110,10 +113,10 @@ If the library cannot be found, this method throws an error, unless the keyword
 """
 function dlopen end
 
-dlopen(s::Symbol, flags::Integer = RTLD_LAZY | RTLD_DEEPBIND; kwargs...) =
+dlopen(s::Symbol, flags::Integer = default_rtld_flags; kwargs...) =
     dlopen(string(s), flags; kwargs...)
 
-function dlopen(s::AbstractString, flags::Integer = RTLD_LAZY | RTLD_DEEPBIND; throw_error::Bool = true)
+function dlopen(s::AbstractString, flags::Integer = default_rtld_flags; throw_error::Bool = true)
     ret = ccall(:jl_load_dynamic_library, Ptr{Cvoid}, (Cstring,UInt32,Cint), s, flags, Cint(throw_error))
     if ret == C_NULL
         return nothing
@@ -226,23 +229,6 @@ function dlpath(handle::Ptr{Cvoid})
     return s
 end
 
-"""
-    dlpath(libname::Union{AbstractString, Symbol})
-
-Get the full path of the library `libname`.
-
-# Example
-```julia-repl
-julia> dlpath("libjulia")
-```
-"""
-function dlpath(libname::Union{AbstractString, Symbol})
-    handle = dlopen(libname)
-    path = dlpath(handle)
-    dlclose(handle)
-    return path
-end
-
 if Sys.isapple()
     const dlext = "dylib"
 elseif Sys.iswindows()
@@ -314,4 +300,101 @@ function dllist()
     return dynamic_libraries
 end
 
+
+"""
+    LazyLibrary(name, flags = <default dlopen flags>,
+                dependencies = LazyLibrary[], on_load_callback = nothing)
+
+Represents a lazily-loaded library that opens itself and its dependencies on first usage
+in a `dlopen()`, `dlsym()`, or `ccall()` usage.  While this structure contains the
+ability to run arbitrary code on first load via `on_load_callback`, we caution that this
+should be used sparingly, as it is not expected that `ccall()` should result in large
+amounts of Julia code being run.
+"""
+mutable struct LazyLibrary
+    # Name and flags to open with
+    const name::String
+    const flags::UInt32
+
+    # Dependencies that must be loaded before we can load
+    const dependencies::Vector{LazyLibrary}
+
+    # Function that gets called once upon initial load with the pointer as an argument
+    const on_load_callback::Union{Nothing,Function}
+    # Function that gets called once upon final unload with the pointer as an argument
+    const on_unload_callback::Union{Nothing,Function}
+
+    # Pointer that we eventually fill out upon first `dlopen()`
+    @atomic handle::Ptr{Cvoid}
+    @atomic refs::Int
+    function LazyLibrary(name, flags = default_rtld_flags, dependencies = LazyLibrary[],
+                         on_load_callback = nothing, on_unload_callback = nothing)
+        return new(
+            String(name),
+            UInt32(flags),
+            collect(dependencies),
+            on_load_callback,
+            on_unload_callback,
+            C_NULL,
+            0,
+        )
+    end
+end
+
+function dlopen(ll::LazyLibrary, flags::Integer = ll.flags; kwargs...)
+    # Only load once
+    handle = @atomic ll.handle
+    if handle != C_NULL
+        @atomic ll.refs += 1
+        return handle
+    end
+
+    # Ensure that all dependencies are loaded
+    for dep in ll.dependencies
+        dlopen(dep; kwargs...)
+    end
+
+    # Load our library
+    handle = dlopen(ll.name, flags; kwargs...)
+    @atomic ll.handle = handle
+    @atomic ll.refs += 1
+
+    # Invoke our on load callback, if it exists
+    if ll.on_load_callback !== nothing
+        ll.on_load_callback(handle)
+    end
+    return handle
+end
+dlsym(ll::LazyLibrary, args...; kwargs...) = dlsym(dlopen(ll), args...; kwargs...)
+function dlclose(ll::LazyLibrary)
+    if @atomic(ll.refs) > 0 && @atomic(ll.handle) != C_NULL
+        @atomic ll.refs -= 1
+        if @atomic(ll.refs) <= 0
+            if ll.on_unload_callback !== nothing
+                ll.on_unload_callback(@atomic(ll.handle))
+            end
+            dlclose(@atomic(ll.handle))
+            @atomic ll.handle = C_NULL
+        end
+    end
+end
+
+"""
+    dlpath(libname)
+
+Get the full path of the library `libname`.  `libname` can be a string, a symbol
+or a `LazyLibrary` object.
+
+# Example
+```julia-repl
+julia> dlpath("libjulia")
+```
+"""
+function dlpath(libname::Union{AbstractString, Symbol, LazyLibrary})
+    handle = dlopen(libname)
+    path = dlpath(handle)
+    dlclose(handle)
+    return path
+end
+
 end # module Libdl

src/Makefile

@@ -249,6 +249,7 @@ $(build_includedir)/julia/uv/*.h: $(LIBUV_INC)/uv/*.h | $(build_includedir)/juli
 	$(INSTALL_F) $^ $(build_includedir)/julia/uv
 
 libccalltest: $(build_shlibdir)/libccalltest.$(SHLIB_EXT)
+libccalltestdep: $(build_shlibdir)/libccalltestdep.$(SHLIB_EXT)
 libllvmcalltest: $(build_shlibdir)/libllvmcalltest.$(SHLIB_EXT)
 
 ifeq ($(OS), Linux)
@@ -257,7 +258,7 @@ else
 JULIA_SPLITDEBUG := 0
 endif
 $(build_shlibdir)/libccalltest.$(SHLIB_EXT): $(SRCDIR)/ccalltest.c
-	@$(call PRINT_CC, $(CC) $(JCFLAGS) $(JL_CFLAGS) $(JCPPFLAGS) $(FLAGS) -O3 $< $(fPIC) -shared -o $@.tmp $(LDFLAGS))
+	@$(call PRINT_CC, $(CC) $(JCFLAGS) $(JL_CFLAGS) $(JCPPFLAGS) $(FLAGS) -O3 $< $(fPIC) -shared -o $@.tmp $(LDFLAGS) $(call SONAME_FLAGS,libccalltest.$(SHLIB_EXT)))
 	$(INSTALL_NAME_CMD)libccalltest.$(SHLIB_EXT) $@.tmp
 ifeq ($(JULIA_SPLITDEBUG),1)
 	@# Create split debug info file for libccalltest stacktraces test
@@ -273,6 +274,9 @@ endif
 	mv [email protected] $@
 	$(INSTALL_NAME_CMD)libccalltest.$(SHLIB_EXT) $@
 
+$(build_shlibdir)/libccalltestdep.$(SHLIB_EXT): $(SRCDIR)/ccalltestdep.c $(build_shlibdir)/libccalltest.$(SHLIB_EXT)
+	@$(call PRINT_CC, $(CC) $(JCFLAGS) $(JL_CFLAGS) $(JCPPFLAGS) $(FLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS) $(COMMON_LIBPATHS) $(call SONAME_FLAGS,libccalltestdep.$(SHLIB_EXT)) -lccalltest)
+
 $(build_shlibdir)/libllvmcalltest.$(SHLIB_EXT): $(SRCDIR)/llvmcalltest.cpp $(LLVM_CONFIG_ABSOLUTE)
 	@$(call PRINT_CC, $(CXX) $(LLVM_CXXFLAGS) $(FLAGS) $(CPPFLAGS) $(CXXFLAGS) -O3 $< $(fPIC) -shared -o $@ $(LDFLAGS) $(COMMON_LIBPATHS) $(NO_WHOLE_ARCHIVE) $(CG_LLVMLINK)) -lpthread
 

src/ccall.cpp

@@ -651,6 +651,9 @@ static void interpret_symbol_arg(jl_codectx_t &ctx, native_sym_arg_t &out, jl_va
                 f_lib = jl_symbol_name((jl_sym_t*)t1);
             else if (jl_is_string(t1))
                 f_lib = jl_string_data(t1);
+            else if (jl_lazy_library_type != NULL && jl_isa(t1, (jl_value_t *)jl_lazy_library_type)) {
+                out.lib_expr = t1;
+            }
             else
                 f_name = NULL;
         }

src/ccalltestdep.c

@@ -0,0 +1,41 @@
+// This file is a part of Julia. License is MIT: https://julialang.org/license
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <complex.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include "../src/support/platform.h"
+#include "../src/support/dtypes.h"
+
+// Borrow definition from `support/dtypes.h`
+#ifdef _OS_WINDOWS_
+#  define DLLEXPORT __declspec(dllexport)
+#else
+# if defined(_OS_LINUX_) && !defined(_COMPILER_CLANG_)
+// Clang and ld disagree about the proper relocation for STV_PROTECTED, causing
+// linker errors.
+#  define DLLEXPORT __attribute__ ((visibility("protected")))
+# else
+#  define DLLEXPORT __attribute__ ((visibility("default")))
+# endif
+#endif
+
+#ifdef _P64
+#define jint int64_t
+#else
+#define jint int32_t
+#endif
+
+typedef struct {
+    jint real;
+    jint imag;
+} complex_t;
+
+// We expect this to come from `libccalltest`
+extern complex_t ctest(complex_t);
+
+DLLEXPORT complex_t dep_ctest(complex_t a) {
+    return ctest(a);
+}

src/init.c

@@ -382,6 +382,7 @@ JL_DLLEXPORT void jl_postoutput_hook(void)
 }
 
 void post_boot_hooks(void);
+void post_image_load_hooks(void);
 
 JL_DLLEXPORT void *jl_libjulia_internal_handle;
 JL_DLLEXPORT void *jl_libjulia_handle;
@@ -888,6 +889,8 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_
             jl_module_run_initializer((jl_module_t*)mod);
         }
         JL_GC_POP();
+
+        post_image_load_hooks();
     }
 
     if (jl_options.handle_signals == JL_OPTIONS_HANDLE_SIGNALS_ON)

src/jl_exported_data.inc

@@ -126,6 +126,9 @@
     XX(jl_voidpointer_type) \
     XX(jl_void_type) \
     XX(jl_weakref_type) \
+    XX(jl_libdl_module) \
+    XX(jl_libdl_dlopen_func) \
+    XX(jl_lazy_library_type) \
 
 // Data symbols that are defined inside the public libjulia
 #define JL_EXPORTED_DATA_SYMBOLS(XX) \

src/jltypes.c

@@ -3439,6 +3439,24 @@ void post_boot_hooks(void)
     }
     export_small_typeof();
 }
+
+void post_image_load_hooks(void) {
+    // Ensure that `Base` has been loaded.
+    assert(jl_base_module != NULL);
+
+    jl_libdl_module = (jl_module_t *)jl_get_global(
+        jl_get_global(((jl_module_t *)jl_base_module), jl_symbol("Libc")),
+        jl_symbol("Libdl")
+    );
+    jl_libdl_dlopen_func = jl_get_global(
+        jl_libdl_module,
+        jl_symbol("dlopen")
+    );
+    jl_lazy_library_type = (jl_datatype_t *)jl_get_global(
+        jl_libdl_module,
+        jl_symbol("LazyLibrary")
+    );
+}
 #undef XX
 
 #ifdef __cplusplus

src/julia.h

@@ -883,6 +883,9 @@ extern JL_DLLIMPORT jl_value_t *jl_false JL_GLOBALLY_ROOTED;
 extern JL_DLLIMPORT jl_value_t *jl_nothing JL_GLOBALLY_ROOTED;
 extern JL_DLLIMPORT jl_value_t *jl_kwcall_func JL_GLOBALLY_ROOTED;
 
+extern JL_DLLIMPORT jl_value_t    *jl_libdl_dlopen_func JL_GLOBALLY_ROOTED;
+extern JL_DLLIMPORT jl_datatype_t *jl_lazy_library_type JL_GLOBALLY_ROOTED;
+
 // gc -------------------------------------------------------------------------
 
 struct _jl_gcframe_t {
@@ -1700,6 +1703,7 @@ extern JL_DLLIMPORT jl_module_t *jl_main_module JL_GLOBALLY_ROOTED;
 extern JL_DLLIMPORT jl_module_t *jl_core_module JL_GLOBALLY_ROOTED;
 extern JL_DLLIMPORT jl_module_t *jl_base_module JL_GLOBALLY_ROOTED;
 extern JL_DLLIMPORT jl_module_t *jl_top_module JL_GLOBALLY_ROOTED;
+extern JL_DLLIMPORT jl_module_t *jl_libdl_module JL_GLOBALLY_ROOTED;
 JL_DLLEXPORT jl_module_t *jl_new_module(jl_sym_t *name, jl_module_t *parent);
 JL_DLLEXPORT void jl_set_module_nospecialize(jl_module_t *self, int on);
 JL_DLLEXPORT void jl_set_module_optlevel(jl_module_t *self, int lvl);

src/runtime_ccall.cpp

@@ -66,16 +66,19 @@ void *jl_load_and_lookup(const char *f_lib, const char *f_name, _Atomic(void*) *
 extern "C" JL_DLLEXPORT
 void *jl_lazy_load_and_lookup(jl_value_t *lib_val, const char *f_name)
 {
-    char *f_lib;
+    void *lib_ptr;
 
     if (jl_is_symbol(lib_val))
-        f_lib = jl_symbol_name((jl_sym_t*)lib_val);
+        lib_ptr = jl_get_library(jl_symbol_name((jl_sym_t*)lib_val));
     else if (jl_is_string(lib_val))
-        f_lib = jl_string_data(lib_val);
-    else
+        lib_ptr = jl_get_library(jl_string_data(lib_val));
+    else if (jl_lazy_library_type != NULL && jl_libdl_dlopen_func != NULL && jl_isa(lib_val, (jl_value_t *)jl_lazy_library_type)) {
+        // Call `dlopen(lib_val)` if `lib_val` is a `LazyLibrary` object, use the returned handle as `lib_ptr`.
+        lib_ptr = *((void **)jl_apply_generic(jl_libdl_dlopen_func, &lib_val, 1));
+    } else
         jl_type_error("ccall", (jl_value_t*)jl_symbol_type, lib_val);
     void *ptr;
-    jl_dlsym(jl_get_library(f_lib), f_name, &ptr, 1);
+    jl_dlsym(lib_ptr, f_name, &ptr, 1);
     return ptr;
 }