[LibOS] Simplify RTLD code

LibOS/shim/include/shim_internal.h

@@ -342,11 +342,8 @@ extern void* __load_address_end;
 
 extern const char** migrated_envp;
 
-struct shim_handle;
-int init_brk_from_executable(struct shim_handle* exec);
 int init_brk_region(void* brk_region, size_t data_segment_size);
 void reset_brk(void);
-int init_loader(void);
 int init_rlimit(void);
 
 bool is_user_memory_readable(const void* addr, size_t size);
@@ -358,6 +355,8 @@ void set_rlimit_cur(int resource, uint64_t rlim);
 
 int object_wait_with_retry(PAL_HANDLE handle);
 
+struct shim_handle;
+
 void _update_epolls(struct shim_handle* handle);
 void delete_from_epoll_handles(struct shim_handle* handle);
 /*!

LibOS/shim/include/shim_utils.h

@@ -141,11 +141,14 @@ void free(void* mem);
 void* malloc_copy(const void* mem, size_t size);
 
 /* ELF binary loading */
+struct link_map;
+int init_elf_objects(void);
 int check_elf_object(struct shim_handle* file);
-int load_elf_object(struct shim_handle* file);
-int load_elf_interp(struct shim_handle* exec);
-noreturn void execute_elf_object(struct shim_handle* exec, void* argp, elf_auxv_t* auxp);
-int remove_loaded_libraries(void);
+int load_elf_object(struct shim_handle* file, struct link_map** out_map);
+int load_elf_interp(struct link_map* exec_map);
+noreturn void execute_elf_object(struct link_map* exec_map, void* argp, elf_auxv_t* auxp);
+void remove_loaded_elf_objects(void);
+int init_brk_from_executable(struct link_map* exec_map);
 
 /* gdb debugging support */
 int init_r_debug(void);

LibOS/shim/src/bookkeep/shim_vma.c

@@ -700,6 +700,11 @@ int bkeep_munmap(void* addr, size_t length, bool is_internal, void** tmp_vma_ptr
         free_vma(vma2);
     }
 
+    /*
+     * TODO: We call `remove_r_debug()` on the assumption that `addr` might be the beginning of a
+     * loaded ELF object. However, `remove_r_debug()` assumes that `addr` is the load base, while
+     * the first mapping of an ELF object might begin later than its load base.
+     */
     remove_r_debug(addr);
     return ret;
 }

LibOS/shim/src/shim_init.c

@@ -90,11 +90,10 @@ void* migrated_memory_end;
 
 const char** migrated_envp __attribute_migratable;
 
-/* library_paths is populated with LD_PRELOAD entries once during LibOS
- * initialization and is used in __load_interp_object() to search for ELF
- * program interpreter in specific paths. Once allocated, its memory is
- * never freed or updated. */
-char** library_paths = NULL;
+/* `g_library_paths` is populated with LD_PRELOAD entries once during LibOS initialization and is
+ * used in `__load_interp_object()` to search for ELF program interpreter in specific paths. Once
+ * allocated, its memory is never freed or updated. */
+char** g_library_paths = NULL;
 
 struct shim_lock __master_lock;
 bool lock_enabled;
@@ -324,7 +323,7 @@ int init_stack(const char** argv, const char** envp, const char*** out_argp,
 static int read_environs(const char** envp) {
     for (const char** e = envp; *e; e++) {
         if (strstartswith(*e, "LD_LIBRARY_PATH=")) {
-            /* populate library_paths with entries from LD_LIBRARY_PATH envvar */
+            /* populate `g_library_paths` with entries from LD_LIBRARY_PATH envvar */
             const char* s = *e + static_strlen("LD_LIBRARY_PATH=");
             size_t npaths = 2; // One for the first entry, one for the last NULL.
             for (const char* tmp = s; *tmp; tmp++)
@@ -352,8 +351,8 @@ static int read_environs(const char** envp) {
 
             paths[cnt] = NULL;
 
-            assert(!library_paths);
-            library_paths = paths;
+            assert(!g_library_paths);
+            g_library_paths = paths;
             return 0;
         }
     }
@@ -446,7 +445,7 @@ noreturn void* shim_init(int argc, void* args) {
     elf_auxv_t* new_auxv;
     RUN_INIT(init_stack, argv, envp, &new_argp, &new_auxv);
 
-    RUN_INIT(init_loader);
+    RUN_INIT(init_elf_objects);
     RUN_INIT(init_signal_handling);
     RUN_INIT(init_ipc_worker);
 
@@ -498,16 +497,10 @@ noreturn void* shim_init(int argc, void* args) {
 
     set_default_tls();
 
-    lock(&g_process.fs_lock);
-    struct shim_handle* exec = g_process.exec;
-    get_handle(exec);
-    unlock(&g_process.fs_lock);
-
-    if (exec) {
-        /* Passing ownership of `exec` to `execute_elf_object`. */
-        execute_elf_object(exec, new_argp, new_auxv);
-    }
-    process_exit(0, 0);
+    /* At this point, the exec map has been either copied from checkpoint, or initialized in
+     * `init_loader`. */
+    execute_elf_object(/*exec_map=*/NULL, new_argp, new_auxv);
+    /* UNREACHABLE */
 }
 
 static int get_256b_random_hex_string(char* buf, size_t size) {