Refine AOT exception check when function return (#1752)

wenyongh · web-flow · commit ce3458da9945 · 2022-11-30T20:18:28.000+08:00
Refine AOT exception check in the caller when returning from callee function,
remove the exception check instructions when hw bound check is enabled to
improve the performance: create guard page to trigger signal handler when
exception occurs.
diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c
@@ -1500,7 +1500,11 @@ aot_set_exception(AOTModuleInstance *module_inst, const char *exception)
 void
 aot_set_exception_with_id(AOTModuleInstance *module_inst, uint32 id)
 {
-    wasm_set_exception_with_id(module_inst, id);
+    if (id != EXCE_ALREADY_THROWN)
+        wasm_set_exception_with_id(module_inst, id);
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    wasm_runtime_access_exce_check_guard_page();
+#endif
 }
 
 const char *
@@ -1755,6 +1759,7 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc,
     const char *signature;
     void *attachment;
     char buf[96];
+    bool ret = false;
 
     bh_assert(func_idx < aot_module->import_func_count);
 
@@ -1764,27 +1769,34 @@ aot_invoke_native(WASMExecEnv *exec_env, uint32 func_idx, uint32 argc,
                  "failed to call unlinked import function (%s, %s)",
                  import_func->module_name, import_func->func_name);
         aot_set_exception(module_inst, buf);
-        return false;
+        goto fail;
     }
 
     attachment = import_func->attachment;
     if (import_func->call_conv_wasm_c_api) {
-        return wasm_runtime_invoke_c_api_native(
+        ret = wasm_runtime_invoke_c_api_native(
             (WASMModuleInstanceCommon *)module_inst, func_ptr, func_type, argc,
             argv, import_func->wasm_c_api_with_env, attachment);
     }
     else if (!import_func->call_conv_raw) {
         signature = import_func->signature;
-        return wasm_runtime_invoke_native(exec_env, func_ptr, func_type,
-                                          signature, attachment, argv, argc,
-                                          argv);
+        ret =
+            wasm_runtime_invoke_native(exec_env, func_ptr, func_type, signature,
+                                       attachment, argv, argc, argv);
     }
     else {
         signature = import_func->signature;
-        return wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type,
-                                              signature, attachment, argv, argc,
-                                              argv);
+        ret = wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type,
+                                             signature, attachment, argv, argc,
+                                             argv);
     }
+
+fail:
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    if (!ret)
+        wasm_runtime_access_exce_check_guard_page();
+#endif
+    return ret;
 }
 
 bool
@@ -1811,21 +1823,21 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
 
     if ((uint8 *)&module_inst < exec_env->native_stack_boundary) {
         aot_set_exception_with_id(module_inst, EXCE_NATIVE_STACK_OVERFLOW);
-        return false;
+        goto fail;
     }
 
     tbl_inst = module_inst->tables[tbl_idx];
     bh_assert(tbl_inst);
 
     if (table_elem_idx >= tbl_inst->cur_size) {
         aot_set_exception_with_id(module_inst, EXCE_UNDEFINED_ELEMENT);
-        return false;
+        goto fail;
     }
 
     func_idx = tbl_inst->elems[table_elem_idx];
     if (func_idx == NULL_REF) {
         aot_set_exception_with_id(module_inst, EXCE_UNINITIALIZED_ELEMENT);
-        return false;
+        goto fail;
     }
 
     func_type_idx = func_type_indexes[func_idx];
@@ -1843,7 +1855,7 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
                  "failed to call unlinked import function (%s, %s)",
                  import_func->module_name, import_func->func_name);
         aot_set_exception(module_inst, buf);
-        return false;
+        goto fail;
     }
 
     if (func_idx < aot_module->import_func_count) {
@@ -1852,9 +1864,13 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
         signature = import_func->signature;
         if (import_func->call_conv_raw) {
             attachment = import_func->attachment;
-            return wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type,
-                                                  signature, attachment, argv,
-                                                  argc, argv);
+            ret = wasm_runtime_invoke_native_raw(exec_env, func_ptr, func_type,
+                                                 signature, attachment, argv,
+                                                 argc, argv);
+            if (!ret)
+                goto fail;
+
+            return true;
         }
     }
 
@@ -1878,7 +1894,7 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
             && !(argv1 = runtime_malloc(size, module_inst->cur_exception,
                                         sizeof(module_inst->cur_exception)))) {
             aot_set_exception_with_id(module_inst, EXCE_OUT_OF_MEMORY);
-            return false;
+            goto fail;
         }
 
         /* Copy original arguments */
@@ -1897,12 +1913,10 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
 
         ret = invoke_native_internal(exec_env, func_ptr, func_type, signature,
                                      attachment, argv1, argc, argv);
-        if (!ret || aot_get_exception(module_inst)) {
+        if (!ret) {
             if (argv1 != argv1_buf)
                 wasm_runtime_free(argv1);
-            if (clear_wasi_proc_exit_exception(module_inst))
-                return true;
-            return false;
+            goto fail;
         }
 
         /* Get extra result values */
@@ -1941,19 +1955,38 @@ aot_call_indirect(WASMExecEnv *exec_env, uint32 tbl_idx, uint32 table_elem_idx,
     else {
         ret = invoke_native_internal(exec_env, func_ptr, func_type, signature,
                                      attachment, argv, argc, argv);
-        if (clear_wasi_proc_exit_exception(module_inst))
-            return true;
-        return ret;
+        if (!ret)
+            goto fail;
+
+        return true;
     }
+
+fail:
+    if (clear_wasi_proc_exit_exception(module_inst))
+        return true;
+
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    wasm_runtime_access_exce_check_guard_page();
+#endif
+    return false;
 }
 
 bool
 aot_check_app_addr_and_convert(AOTModuleInstance *module_inst, bool is_str,
                                uint32 app_buf_addr, uint32 app_buf_size,
                                void **p_native_addr)
 {
-    return wasm_check_app_addr_and_convert(module_inst, is_str, app_buf_addr,
-                                           app_buf_size, p_native_addr);
+    bool ret;
+
+    ret = wasm_check_app_addr_and_convert(module_inst, is_str, app_buf_addr,
+                                          app_buf_size, p_native_addr);
+
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    if (!ret)
+        wasm_runtime_access_exce_check_guard_page();
+#endif
+
+    return ret;
 }
 
 void *
diff --git a/core/iwasm/common/wasm_exec_env.c b/core/iwasm/common/wasm_exec_env.c
@@ -56,6 +56,12 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst,
 #endif
 #endif
 
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    if (!(exec_env->exce_check_guard_page =
+              os_mmap(NULL, os_getpagesize(), MMAP_PROT_NONE, MMAP_MAP_NONE)))
+        goto fail5;
+#endif
+
     exec_env->module_inst = module_inst;
     exec_env->wasm_stack_size = stack_size;
     exec_env->wasm_stack.s.top_boundary =
@@ -76,6 +82,12 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst,
 
     return exec_env;
 
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+fail5:
+#if WASM_ENABLE_THREAD_MGR != 0 && WASM_ENABLE_DEBUG_INTERP != 0
+    wasm_cluster_destroy_exenv_status(exec_env->current_status);
+#endif
+#endif
 #if WASM_ENABLE_THREAD_MGR != 0
 #if WASM_ENABLE_DEBUG_INTERP != 0
 fail4:
@@ -96,6 +108,9 @@ wasm_exec_env_create_internal(struct WASMModuleInstanceCommon *module_inst,
 void
 wasm_exec_env_destroy_internal(WASMExecEnv *exec_env)
 {
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+    os_munmap(exec_env->exce_check_guard_page, os_getpagesize());
+#endif
 #if WASM_ENABLE_THREAD_MGR != 0
     os_mutex_destroy(&exec_env->wait_lock);
     os_cond_destroy(&exec_env->wait_cond);
diff --git a/core/iwasm/common/wasm_exec_env.h b/core/iwasm/common/wasm_exec_env.h
@@ -137,6 +137,8 @@ typedef struct WASMExecEnv {
 
 #ifdef OS_ENABLE_HW_BOUND_CHECK
     WASMJmpBuf *jmpbuf_stack_top;
+    /* One guard page for the exception check */
+    uint8 *exce_check_guard_page;
 #endif
 
 #if WASM_ENABLE_MEMORY_PROFILING != 0
@@ -199,7 +201,8 @@ wasm_exec_env_alloc_wasm_frame(WASMExecEnv *exec_env, unsigned size)
        the outs area contains const cells, its size may be larger than current
        frame size, we should check again before putting the function arguments
        into the outs area. */
-    if (addr + size * 2 > exec_env->wasm_stack.s.top_boundary) {
+    if (size * 2
+        > (uint32)(uintptr_t)(exec_env->wasm_stack.s.top_boundary - addr)) {
         /* WASM stack overflow. */
         return NULL;
     }
diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c
@@ -185,6 +185,12 @@ runtime_signal_handler(void *sig_addr)
             os_longjmp(jmpbuf_node->jmpbuf, 1);
         }
 #endif
+        else if (exec_env_tls->exce_check_guard_page <= (uint8 *)sig_addr
+                 && (uint8 *)sig_addr
+                        < exec_env_tls->exce_check_guard_page + page_size) {
+            bh_assert(wasm_get_exception(module_inst));
+            os_longjmp(jmpbuf_node->jmpbuf, 1);
+        }
     }
 }
 #else
@@ -1435,6 +1441,17 @@ wasm_runtime_get_user_data(WASMExecEnv *exec_env)
     return exec_env->user_data;
 }
 
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+void
+wasm_runtime_access_exce_check_guard_page()
+{
+    if (exec_env_tls && exec_env_tls->handle == os_self_thread()) {
+        uint32 page_size = os_getpagesize();
+        memset(exec_env_tls->exce_check_guard_page, 0, page_size);
+    }
+}
+#endif
+
 WASMType *
 wasm_runtime_get_function_type(const WASMFunctionInstanceCommon *function,
                                uint32 module_type)
diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h
@@ -554,6 +554,12 @@ wasm_runtime_set_user_data(WASMExecEnv *exec_env, void *user_data);
 WASM_RUNTIME_API_EXTERN void *
 wasm_runtime_get_user_data(WASMExecEnv *exec_env);
 
+#ifdef OS_ENABLE_HW_BOUND_CHECK
+/* Access exception check guard page to trigger the signal handler */
+void
+wasm_runtime_access_exce_check_guard_page();
+#endif
+
 /* See wasm_export.h for description */
 WASM_RUNTIME_API_EXTERN bool
 wasm_runtime_call_wasm(WASMExecEnv *exec_env,
diff --git a/core/iwasm/compilation/aot_emit_function.c b/core/iwasm/compilation/aot_emit_function.c
@@ -35,7 +35,14 @@ create_func_return_block(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx)
         /* Create return IR */
         LLVMPositionBuilderAtEnd(comp_ctx->builder,
                                  func_ctx->func_return_block);
-        if (!aot_build_zero_function_ret(comp_ctx, func_ctx, aot_func_type)) {
+        if (!comp_ctx->enable_bound_check) {
+            if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_ALREADY_THROWN,
+                                    false, NULL, NULL)) {
+                return false;
+            }
+        }
+        else if (!aot_build_zero_function_ret(comp_ctx, func_ctx,
+                                              aot_func_type)) {
             return false;
         }
     }
@@ -494,7 +501,8 @@ check_app_addr_and_convert(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
     }
 
     /* Check whether exception was thrown when executing the function */
-    if (!check_call_return(comp_ctx, func_ctx, res)) {
+    if (comp_ctx->enable_bound_check
+        && !check_call_return(comp_ctx, func_ctx, res)) {
         return false;
     }
 
@@ -707,7 +715,8 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
                 goto fail;
             /* Check whether there was exception thrown when executing
                the function */
-            if (!check_call_return(comp_ctx, func_ctx, res))
+            if (comp_ctx->enable_bound_check
+                && !check_call_return(comp_ctx, func_ctx, res))
                 goto fail;
         }
         else { /* call native func directly */
@@ -823,7 +832,7 @@ aot_compile_op_call(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
 
         /* Check whether there was exception thrown when executing
            the function */
-        if (!tail_call && !recursive_call
+        if (!tail_call && !recursive_call && comp_ctx->enable_bound_check
             && !check_exception_thrown(comp_ctx, func_ctx))
             goto fail;
     }
@@ -1395,7 +1404,8 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
         goto fail;
 
     /* Check whether exception was thrown when executing the function */
-    if (!check_call_return(comp_ctx, func_ctx, res))
+    if (comp_ctx->enable_bound_check
+        && !check_call_return(comp_ctx, func_ctx, res))
         goto fail;
 
     block_curr = LLVMGetInsertBlock(comp_ctx->builder);
@@ -1454,7 +1464,8 @@ aot_compile_op_call_indirect(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx,
     }
 
     /* Check whether exception was thrown when executing the function */
-    if (!check_exception_thrown(comp_ctx, func_ctx))
+    if (comp_ctx->enable_bound_check
+        && !check_exception_thrown(comp_ctx, func_ctx))
         goto fail;
 
     if (func_result_count > 0) {
diff --git a/core/iwasm/compilation/aot_llvm_extra.cpp b/core/iwasm/compilation/aot_llvm_extra.cpp
@@ -344,11 +344,12 @@ aot_apply_llvm_new_pass_manager(AOTCompContext *comp_ctx, LLVMModuleRef module)
 
         if (!disable_llvm_lto) {
             /* Apply LTO for AOT mode */
-#if LLVM_VERSION_MAJOR < 14
-            MPM.addPass(PB.buildLTODefaultPipeline(OL, NULL));
-#else
-            MPM.addPass(PB.buildLTOPreLinkDefaultPipeline(OL));
-#endif
+            if (comp_ctx->comp_data->func_count >= 10)
+                /* Adds the pre-link optimizations if the func count
+                   is large enough */
+                MPM.addPass(PB.buildLTOPreLinkDefaultPipeline(OL));
+            else
+                MPM.addPass(PB.buildLTODefaultPipeline(OL, NULL));
         }
         else {
             MPM.addPass(PB.buildPerModuleDefaultPipeline(OL));
diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c
