diff --git a/src/c-writer.cc b/src/c-writer.cc index 76ff52f91..a33523a85 100644 --- a/src/c-writer.cc +++ b/src/c-writer.cc @@ -1369,7 +1369,24 @@ static std::string GetMemoryTypeString(const Memory& memory) { } static std::string GetMemoryAPIString(const Memory& memory, std::string api) { - return memory.page_limits.is_shared ? (api + "_shared") : api; + std::string suffix; + if (memory.page_limits.is_shared) { + suffix += "_shared"; + } + + // Memory load and store routines can be optimized for default-page-size, + // 32-bit memories (by using hardware to bounds-check memory access). + // Append "_default32" to these function names to choose the (possibly) fast + // path. + // + // We don't need to do this for runtime routines; those can check the + // wasm_rt_memory_t structure. + if (api.substr(0, 8) != "wasm_rt_" && + memory.page_size == WABT_DEFAULT_PAGE_SIZE && + memory.page_limits.is_64 == false) { + suffix += "_default32"; + } + return api + suffix; } void CWriter::WriteInitExpr(const ExprList& expr_list) { diff --git a/src/prebuilt/wasm2c_atomicops_source_declarations.cc b/src/prebuilt/wasm2c_atomicops_source_declarations.cc index 8312d5d6b..dc02591cb 100644 --- a/src/prebuilt/wasm2c_atomicops_source_declarations.cc +++ b/src/prebuilt/wasm2c_atomicops_source_declarations.cc @@ -17,25 +17,25 @@ R"w2c_template( TRAP(UNALIGNED); \ R"w2c_template( } )w2c_template" R"w2c_template( -#define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read) \ +#define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read) \ )w2c_template" -R"w2c_template( static inline t3 name(wasm_rt_shared_memory_t* mem, u64 addr) { \ +R"w2c_template( static inline t3 name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr) { \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( t1 result; \ )w2c_template" -R"w2c_template( t1 result; \ +R"w2c_template( result = atomic_load_explicit( \ )w2c_template" -R"w2c_template( result = atomic_load_explicit( \ +R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ )w2c_template" -R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ +R"w2c_template( memory_order_relaxed); \ )w2c_template" -R"w2c_template( memory_order_relaxed); \ +R"w2c_template( force_read(result); \ )w2c_template" -R"w2c_template( force_read(result); \ +R"w2c_template( return (t3)(t2)result; \ )w2c_template" -R"w2c_template( return (t3)(t2)result; \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } +R"w2c_template( DEF_MEM_CHECKS0(name, _shared_, t1, return, t3) )w2c_template" R"w2c_template( DEFINE_SHARED_LOAD(i32_load_shared, u32, u32, u32, FORCE_READ_INT) @@ -69,9 +69,9 @@ R"w2c_template(DEFINE_SHARED_LOAD(i64_load32_u_shared, u32, u64, u64, FORCE_READ R"w2c_template( #define DEFINE_SHARED_STORE(name, t1, t2) \ )w2c_template" -R"w2c_template( static inline void name(wasm_rt_shared_memory_t* mem, u64 addr, t2 value) { \ +R"w2c_template( static inline void name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( t2 value) { \ )w2c_template" R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" @@ -81,7 +81,9 @@ R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wr )w2c_template" R"w2c_template( memory_order_relaxed); \ )w2c_template" -R"w2c_template( } +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS1(name, _shared_, t1, , void, t2) )w2c_template" R"w2c_template( DEFINE_SHARED_STORE(i32_store_shared, u32, u32) @@ -105,9 +107,7 @@ R"w2c_template(DEFINE_SHARED_STORE(i64_store32_shared, u32, u64) R"w2c_template( #define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read) \ )w2c_template" -R"w2c_template( static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ -)w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -121,9 +121,11 @@ R"w2c_template( return (t3)(t2)result; )w2c_template" R"w2c_template( } \ )w2c_template" -R"w2c_template( static inline t3 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr) { \ +R"w2c_template( DEF_MEM_CHECKS0(name, _, t1, return, t3) \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( static inline t3 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ +)w2c_template" +R"w2c_template( u64 addr) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -137,7 +139,9 @@ R"w2c_template( force_read(result); )w2c_template" R"w2c_template( return (t3)(t2)result; \ )w2c_template" -R"w2c_template( } +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS0(name##_shared, _shared_, t1, return, t3) )w2c_template" R"w2c_template( DEFINE_ATOMIC_LOAD(i32_atomic_load, u32, u32, u32, FORCE_READ_INT) @@ -157,9 +161,9 @@ R"w2c_template(DEFINE_ATOMIC_LOAD(i64_atomic_load32_u, u32, u64, u64, FORCE_READ R"w2c_template( #define DEFINE_ATOMIC_STORE(name, t1, t2) \ )w2c_template" -R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ +R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( t2 value) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -169,11 +173,11 @@ R"w2c_template( wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, siz )w2c_template" R"w2c_template( } \ )w2c_template" -R"w2c_template( static inline void name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ +R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, , void, t2) \ )w2c_template" -R"w2c_template( t2 value) { \ +R"w2c_template( static inline void name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( u64 addr, t2 value) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -183,7 +187,9 @@ R"w2c_template( atomic_store((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof )w2c_template" R"w2c_template( wrapped); \ )w2c_template" -R"w2c_template( } +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS1(name##_shared, _shared_, t1, , void, t2) )w2c_template" R"w2c_template( DEFINE_ATOMIC_STORE(i32_atomic_store, u32, u32) @@ -203,9 +209,9 @@ R"w2c_template(DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64) R"w2c_template( #define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2) \ )w2c_template" -R"w2c_template( static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ +R"w2c_template( static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( t2 value) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -223,11 +229,11 @@ R"w2c_template( return (t2)ret; )w2c_template" R"w2c_template( } \ )w2c_template" -R"w2c_template( static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ +R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ )w2c_template" -R"w2c_template( t2 value) { \ +R"w2c_template( static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( u64 addr, t2 value) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -239,7 +245,9 @@ R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wr )w2c_template" R"w2c_template( return (t2)ret; \ )w2c_template" -R"w2c_template( } +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) )w2c_template" R"w2c_template( DEFINE_ATOMIC_RMW(i32_atomic_rmw8_add_u, fetch_add, +, u8, u32) @@ -319,9 +327,9 @@ R"w2c_template(DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64) R"w2c_template( #define DEFINE_ATOMIC_XCHG(name, opname, t1, t2) \ )w2c_template" -R"w2c_template( static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ +R"w2c_template( static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( t2 value) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -337,11 +345,11 @@ R"w2c_template( return (t2)ret; )w2c_template" R"w2c_template( } \ )w2c_template" -R"w2c_template( static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ +R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ )w2c_template" -R"w2c_template( t2 value) { \ +R"w2c_template( static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( u64 addr, t2 value) { \ )w2c_template" R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t1); \ )w2c_template" @@ -353,7 +361,9 @@ R"w2c_template( (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wr )w2c_template" R"w2c_template( return (t2)ret; \ )w2c_template" -R"w2c_template( } +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) )w2c_template" R"w2c_template( DEFINE_ATOMIC_XCHG(i32_atomic_rmw8_xchg_u, exchange, u8, u32) @@ -371,57 +381,57 @@ R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw32_xchg_u, exchange, u32, u64) R"w2c_template(DEFINE_ATOMIC_XCHG(i64_atomic_rmw_xchg, exchange, u64, u64) )w2c_template" R"w2c_template( -#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \ +#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \ )w2c_template" -R"w2c_template( static inline t1 name(wasm_rt_memory_t* mem, u64 addr, t1 expected, \ +R"w2c_template( static inline t1 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( t1 replacement) { \ +R"w2c_template( t1 expected, t1 replacement) { \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t2); \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \ +R"w2c_template( t2 expected_wrapped = (t2)expected; \ )w2c_template" -R"w2c_template( t2 expected_wrapped = (t2)expected; \ +R"w2c_template( t2 replacement_wrapped = (t2)replacement; \ )w2c_template" -R"w2c_template( t2 replacement_wrapped = (t2)replacement; \ +R"w2c_template( t2 ret; \ )w2c_template" -R"w2c_template( t2 ret; \ +R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2)); \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2)); \ +R"w2c_template( if (ret == expected_wrapped) { \ )w2c_template" -R"w2c_template( if (ret == expected_wrapped) { \ +R"w2c_template( wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \ +R"w2c_template( sizeof(t2)); \ )w2c_template" -R"w2c_template( sizeof(t2)); \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( return (t1)expected_wrapped; \ )w2c_template" -R"w2c_template( return (t1)expected_wrapped; \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } \ +R"w2c_template( DEF_MEM_CHECKS2(name, _, t2, return, t1, t1, t1) \ )w2c_template" -R"w2c_template( static inline t1 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ +R"w2c_template( static inline t1 name##_shared_unchecked( \ )w2c_template" -R"w2c_template( t1 expected, t1 replacement) { \ +R"w2c_template( wasm_rt_shared_memory_t* mem, u64 addr, t1 expected, t1 replacement) { \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t2); \ +R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \ )w2c_template" -R"w2c_template( ATOMIC_ALIGNMENT_CHECK(addr, t2); \ +R"w2c_template( t2 expected_wrapped = (t2)expected; \ )w2c_template" -R"w2c_template( t2 expected_wrapped = (t2)expected; \ +R"w2c_template( t2 replacement_wrapped = (t2)replacement; \ )w2c_template" -R"w2c_template( t2 replacement_wrapped = (t2)replacement; \ +R"w2c_template( atomic_compare_exchange_strong( \ )w2c_template" -R"w2c_template( atomic_compare_exchange_strong( \ +R"w2c_template( (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \ )w2c_template" -R"w2c_template( (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \ +R"w2c_template( &expected_wrapped, replacement_wrapped); \ )w2c_template" -R"w2c_template( &expected_wrapped, replacement_wrapped); \ +R"w2c_template( return (t1)expected_wrapped; \ )w2c_template" -R"w2c_template( return (t1)expected_wrapped; \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } +R"w2c_template( DEF_MEM_CHECKS2(name##_shared, _shared_, t2, return, t1, t1, t1) )w2c_template" R"w2c_template( DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw8_cmpxchg_u, u32, u8); diff --git a/src/prebuilt/wasm2c_simd_source_declarations.cc b/src/prebuilt/wasm2c_simd_source_declarations.cc index 5b903b261..a43a00069 100644 --- a/src/prebuilt/wasm2c_simd_source_declarations.cc +++ b/src/prebuilt/wasm2c_simd_source_declarations.cc @@ -15,26 +15,26 @@ R"w2c_template(#endif R"w2c_template(// TODO: equivalent constraint for ARM and other architectures )w2c_template" R"w2c_template( -#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \ +#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \ )w2c_template" -R"w2c_template( static inline v128 name(wasm_rt_memory_t* mem, u64 addr) { \ +R"w2c_template( static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t); \ +R"w2c_template( v128 result = func(MEM_ADDR(mem, addr, sizeof(t))); \ )w2c_template" -R"w2c_template( v128 result = func(MEM_ADDR(mem, addr, sizeof(t))); \ +R"w2c_template( SIMD_FORCE_READ(result); \ )w2c_template" -R"w2c_template( SIMD_FORCE_READ(result); \ +R"w2c_template( return result; \ )w2c_template" -R"w2c_template( return result; \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } +R"w2c_template( DEF_MEM_CHECKS0(name, _, t, return, v128); )w2c_template" R"w2c_template( #define DEFINE_SIMD_LOAD_LANE(name, func, t, lane) \ )w2c_template" -R"w2c_template( static inline v128 name(wasm_rt_memory_t* mem, u64 addr, v128 vec) { \ +R"w2c_template( static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t); \ +R"w2c_template( v128 vec) { \ )w2c_template" R"w2c_template( v128 result = func(MEM_ADDR(mem, addr, sizeof(t)), vec, lane); \ )w2c_template" @@ -42,29 +42,35 @@ R"w2c_template( SIMD_FORCE_READ(result); )w2c_template" R"w2c_template( return result; \ )w2c_template" -R"w2c_template( } +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS1(name, _, t, return, v128, v128); )w2c_template" R"w2c_template( -#define DEFINE_SIMD_STORE(name, t) \ +#define DEFINE_SIMD_STORE(name, t) \ +)w2c_template" +R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, v128 value) { \ +R"w2c_template( v128 value) { \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t); \ +R"w2c_template( simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value); \ )w2c_template" -R"w2c_template( simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value); \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } +R"w2c_template( DEF_MEM_CHECKS1(name, _, t, , void, v128); )w2c_template" R"w2c_template( -#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \ +#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \ +)w2c_template" +R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, v128 value) { \ +R"w2c_template( v128 value) { \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t); \ +R"w2c_template( func(MEM_ADDR(mem, addr, sizeof(t)), value, lane); \ )w2c_template" -R"w2c_template( func(MEM_ADDR(mem, addr, sizeof(t)), value, lane); \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( } +R"w2c_template( DEF_MEM_CHECKS1(name, _, t, , void, v128); )w2c_template" R"w2c_template( // clang-format off diff --git a/src/prebuilt/wasm2c_source_declarations.cc b/src/prebuilt/wasm2c_source_declarations.cc index e830e3c11..5e0fbe1d1 100644 --- a/src/prebuilt/wasm2c_source_declarations.cc +++ b/src/prebuilt/wasm2c_source_declarations.cc @@ -218,21 +218,40 @@ R"w2c_template(#define WASM_RT_CHECK_BASE(mem) R"w2c_template(#endif )w2c_template" R"w2c_template( -#if WASM_RT_MEMCHECK_GUARD_PAGES +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for )w2c_template" -R"w2c_template(#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +R"w2c_template(// default-page-size, 32-bit memories. It may do nothing at all +)w2c_template" +R"w2c_template(// (if hardware bounds-checking is enabled via guard pages) +)w2c_template" +R"w2c_template(// or it may do a slightly faster RANGE_CHECK. +)w2c_template" +R"w2c_template(#if WASM_RT_MEMCHECK_GUARD_PAGES +)w2c_template" +R"w2c_template(#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); )w2c_template" R"w2c_template(#else )w2c_template" -R"w2c_template(#define MEMCHECK(mem, a, t) \ +R"w2c_template(#define MEMCHECK_DEFAULT32(mem, a, t) \ )w2c_template" -R"w2c_template( WASM_RT_CHECK_BASE(mem); \ +R"w2c_template( WASM_RT_CHECK_BASE(mem); \ )w2c_template" -R"w2c_template( RANGE_CHECK(mem, a, sizeof(t)) +R"w2c_template( if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ +)w2c_template" +R"w2c_template( TRAP(OOB); )w2c_template" R"w2c_template(#endif )w2c_template" R"w2c_template( +// MEMCHECK_GENERAL can be used for any memory +)w2c_template" +R"w2c_template(#define MEMCHECK_GENERAL(mem, a, t) \ +)w2c_template" +R"w2c_template( WASM_RT_CHECK_BASE(mem); \ +)w2c_template" +R"w2c_template( RANGE_CHECK(mem, a, sizeof(t)); +)w2c_template" +R"w2c_template( #ifdef __GNUC__ )w2c_template" R"w2c_template(#define FORCE_READ_INT(var) __asm__("" ::"r"(var)); @@ -302,30 +321,103 @@ R"w2c_template( load_data(MEM_ADDR(&m, o, s), i, s); \ R"w2c_template( } while (0) )w2c_template" R"w2c_template( -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ +)w2c_template" +R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ +)w2c_template" +R"w2c_template( u64 addr) { \ +)w2c_template" +R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \ +)w2c_template" +R"w2c_template( ret_kw name##_unchecked(mem, addr); \ +)w2c_template" +R"w2c_template( } \ +)w2c_template" +R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ +)w2c_template" +R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \ +)w2c_template" +R"w2c_template( ret_kw name##_unchecked(mem, addr); \ +)w2c_template" +R"w2c_template( } +)w2c_template" +R"w2c_template( +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ +)w2c_template" +R"w2c_template( val_type1) \ +)w2c_template" +R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ +)w2c_template" +R"w2c_template( u64 addr, val_type1 val1) { \ +)w2c_template" +R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \ +)w2c_template" +R"w2c_template( ret_kw name##_unchecked(mem, addr, val1); \ +)w2c_template" +R"w2c_template( } \ +)w2c_template" +R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ +)w2c_template" +R"w2c_template( val_type1 val1) { \ +)w2c_template" +R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \ +)w2c_template" +R"w2c_template( ret_kw name##_unchecked(mem, addr, val1); \ +)w2c_template" +R"w2c_template( } +)w2c_template" +R"w2c_template( +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ +)w2c_template" +R"w2c_template( val_type1, val_type2) \ +)w2c_template" +R"w2c_template( static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ +)w2c_template" +R"w2c_template( u64 addr, val_type1 val1, \ +)w2c_template" +R"w2c_template( val_type2 val2) { \ )w2c_template" -R"w2c_template( static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ +R"w2c_template( MEMCHECK_DEFAULT32(mem, addr, mem_type); \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( ret_kw name##_unchecked(mem, addr, val1, val2); \ )w2c_template" -R"w2c_template( t1 result; \ +R"w2c_template( } \ )w2c_template" -R"w2c_template( wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ +R"w2c_template( static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( sizeof(t1)); \ +R"w2c_template( val_type1 val1, val_type2 val2) { \ )w2c_template" -R"w2c_template( force_read(result); \ +R"w2c_template( MEMCHECK_GENERAL(mem, addr, mem_type); \ )w2c_template" -R"w2c_template( return (t3)(t2)result; \ +R"w2c_template( ret_kw name##_unchecked(mem, addr, val1, val2); \ )w2c_template" R"w2c_template( } )w2c_template" R"w2c_template( +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ +)w2c_template" +R"w2c_template( static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ +)w2c_template" +R"w2c_template( t1 result; \ +)w2c_template" +R"w2c_template( wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ +)w2c_template" +R"w2c_template( sizeof(t1)); \ +)w2c_template" +R"w2c_template( force_read(result); \ +)w2c_template" +R"w2c_template( return (t3)(t2)result; \ +)w2c_template" +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS0(name, _, t1, return, t3) +)w2c_template" +R"w2c_template( #define DEFINE_STORE(name, t1, t2) \ )w2c_template" -R"w2c_template( static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ +R"w2c_template( static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ )w2c_template" -R"w2c_template( MEMCHECK(mem, addr, t1); \ +R"w2c_template( t2 value) { \ )w2c_template" R"w2c_template( t1 wrapped = (t1)value; \ )w2c_template" @@ -333,7 +425,9 @@ R"w2c_template( wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrappe )w2c_template" R"w2c_template( sizeof(t1)); \ )w2c_template" -R"w2c_template( } +R"w2c_template( } \ +)w2c_template" +R"w2c_template( DEF_MEM_CHECKS1(name, _, t1, , void, t2) )w2c_template" R"w2c_template( DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) diff --git a/src/template/wasm2c.declarations.c b/src/template/wasm2c.declarations.c index 9c2af1dc0..f7ba99d89 100644 --- a/src/template/wasm2c.declarations.c +++ b/src/template/wasm2c.declarations.c @@ -117,14 +117,24 @@ static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -161,23 +171,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) diff --git a/src/template/wasm2c_atomicops.declarations.c b/src/template/wasm2c_atomicops.declarations.c index 5d9cdf904..cb25da513 100644 --- a/src/template/wasm2c_atomicops.declarations.c +++ b/src/template/wasm2c_atomicops.declarations.c @@ -9,16 +9,16 @@ TRAP(UNALIGNED); \ } -#define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_shared_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - result = atomic_load_explicit( \ - (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ - memory_order_relaxed); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEFINE_SHARED_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr) { \ + t1 result; \ + result = atomic_load_explicit( \ + (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ + memory_order_relaxed); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _shared_, t1, return, t3) DEFINE_SHARED_LOAD(i32_load_shared, u32, u32, u32, FORCE_READ_INT) DEFINE_SHARED_LOAD(i64_load_shared, u64, u64, u64, FORCE_READ_INT) @@ -36,13 +36,14 @@ DEFINE_SHARED_LOAD(i64_load32_s_shared, s32, s64, u64, FORCE_READ_INT) DEFINE_SHARED_LOAD(i64_load32_u_shared, u32, u64, u64, FORCE_READ_INT) #define DEFINE_SHARED_STORE(name, t1, t2) \ - static inline void name(wasm_rt_shared_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_shared_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ atomic_store_explicit( \ (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped, \ memory_order_relaxed); \ - } + } \ + DEF_MEM_CHECKS1(name, _shared_, t1, , void, t2) DEFINE_SHARED_STORE(i32_store_shared, u32, u32) DEFINE_SHARED_STORE(i64_store_shared, u64, u64) @@ -55,23 +56,24 @@ DEFINE_SHARED_STORE(i64_store16_shared, u16, u64) DEFINE_SHARED_STORE(i64_store32_shared, u32, u64) #define DEFINE_ATOMIC_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 result; \ wasm_rt_memcpy(&result, MEM_ADDR(mem, addr, sizeof(t1)), sizeof(t1)); \ force_read(result); \ return (t3)(t2)result; \ } \ - static inline t3 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) \ + static inline t3 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ + u64 addr) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 result; \ result = \ atomic_load((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1))); \ force_read(result); \ return (t3)(t2)result; \ - } + } \ + DEF_MEM_CHECKS0(name##_shared, _shared_, t1, return, t3) DEFINE_ATOMIC_LOAD(i32_atomic_load, u32, u32, u32, FORCE_READ_INT) DEFINE_ATOMIC_LOAD(i64_atomic_load, u64, u64, u64, FORCE_READ_INT) @@ -82,20 +84,21 @@ DEFINE_ATOMIC_LOAD(i64_atomic_load16_u, u16, u64, u64, FORCE_READ_INT) DEFINE_ATOMIC_LOAD(i64_atomic_load32_u, u32, u64, u64, FORCE_READ_INT) #define DEFINE_ATOMIC_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, sizeof(t1)); \ } \ - static inline void name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ - t2 value) { \ - MEMCHECK(mem, addr, t1); \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) \ + static inline void name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ + u64 addr, t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ atomic_store((_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), \ wrapped); \ - } + } \ + DEF_MEM_CHECKS1(name##_shared, _shared_, t1, , void, t2) DEFINE_ATOMIC_STORE(i32_atomic_store, u32, u32) DEFINE_ATOMIC_STORE(i64_atomic_store, u64, u64) @@ -106,8 +109,8 @@ DEFINE_ATOMIC_STORE(i64_atomic_store16, u16, u64) DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64) #define DEFINE_ATOMIC_RMW(name, opname, op, t1, t2) \ - static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ t1 ret; \ @@ -116,15 +119,16 @@ DEFINE_ATOMIC_STORE(i64_atomic_store32, u32, u64) wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &ret, sizeof(t1)); \ return (t2)ret; \ } \ - static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ - t2 value) { \ - MEMCHECK(mem, addr, t1); \ + DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ + static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ + u64 addr, t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ t1 ret = atomic_##opname( \ (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ return (t2)ret; \ - } + } \ + DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) DEFINE_ATOMIC_RMW(i32_atomic_rmw8_add_u, fetch_add, +, u8, u32) DEFINE_ATOMIC_RMW(i32_atomic_rmw16_add_u, fetch_add, +, u16, u32) @@ -167,8 +171,8 @@ DEFINE_ATOMIC_RMW(i64_atomic_rmw32_xor_u, fetch_xor, ^, u32, u64) DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64) #define DEFINE_ATOMIC_XCHG(name, opname, t1, t2) \ - static inline t2 name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline t2 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ t1 ret; \ @@ -176,15 +180,16 @@ DEFINE_ATOMIC_RMW(i64_atomic_rmw_xor, fetch_xor, ^, u64, u64) wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t1)), &wrapped, sizeof(t1)); \ return (t2)ret; \ } \ - static inline t2 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ - t2 value) { \ - MEMCHECK(mem, addr, t1); \ + DEF_MEM_CHECKS1(name, _, t1, return, t2, t2) \ + static inline t2 name##_shared_unchecked(wasm_rt_shared_memory_t* mem, \ + u64 addr, t2 value) { \ ATOMIC_ALIGNMENT_CHECK(addr, t1); \ t1 wrapped = (t1)value; \ t1 ret = atomic_##opname( \ (_Atomic volatile t1*)MEM_ADDR(mem, addr, sizeof(t1)), wrapped); \ return (t2)ret; \ - } + } \ + DEF_MEM_CHECKS1(name##_shared, _shared_, t1, return, t2, t2) DEFINE_ATOMIC_XCHG(i32_atomic_rmw8_xchg_u, exchange, u8, u32) DEFINE_ATOMIC_XCHG(i32_atomic_rmw16_xchg_u, exchange, u16, u32) @@ -194,32 +199,32 @@ DEFINE_ATOMIC_XCHG(i64_atomic_rmw16_xchg_u, exchange, u16, u64) DEFINE_ATOMIC_XCHG(i64_atomic_rmw32_xchg_u, exchange, u32, u64) DEFINE_ATOMIC_XCHG(i64_atomic_rmw_xchg, exchange, u64, u64) -#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \ - static inline t1 name(wasm_rt_memory_t* mem, u64 addr, t1 expected, \ - t1 replacement) { \ - MEMCHECK(mem, addr, t2); \ - ATOMIC_ALIGNMENT_CHECK(addr, t2); \ - t2 expected_wrapped = (t2)expected; \ - t2 replacement_wrapped = (t2)replacement; \ - t2 ret; \ - wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2)); \ - if (ret == expected_wrapped) { \ - wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \ - sizeof(t2)); \ - } \ - return (t1)expected_wrapped; \ - } \ - static inline t1 name##_shared(wasm_rt_shared_memory_t* mem, u64 addr, \ - t1 expected, t1 replacement) { \ - MEMCHECK(mem, addr, t2); \ - ATOMIC_ALIGNMENT_CHECK(addr, t2); \ - t2 expected_wrapped = (t2)expected; \ - t2 replacement_wrapped = (t2)replacement; \ - atomic_compare_exchange_strong( \ - (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \ - &expected_wrapped, replacement_wrapped); \ - return (t1)expected_wrapped; \ - } +#define DEFINE_ATOMIC_CMP_XCHG(name, t1, t2) \ + static inline t1 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t1 expected, t1 replacement) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t2); \ + t2 expected_wrapped = (t2)expected; \ + t2 replacement_wrapped = (t2)replacement; \ + t2 ret; \ + wasm_rt_memcpy(&ret, MEM_ADDR(mem, addr, sizeof(t2)), sizeof(t2)); \ + if (ret == expected_wrapped) { \ + wasm_rt_memcpy(MEM_ADDR(mem, addr, sizeof(t2)), &replacement_wrapped, \ + sizeof(t2)); \ + } \ + return (t1)expected_wrapped; \ + } \ + DEF_MEM_CHECKS2(name, _, t2, return, t1, t1, t1) \ + static inline t1 name##_shared_unchecked( \ + wasm_rt_shared_memory_t* mem, u64 addr, t1 expected, t1 replacement) { \ + ATOMIC_ALIGNMENT_CHECK(addr, t2); \ + t2 expected_wrapped = (t2)expected; \ + t2 replacement_wrapped = (t2)replacement; \ + atomic_compare_exchange_strong( \ + (_Atomic volatile t2*)MEM_ADDR(mem, addr, sizeof(t2)), \ + &expected_wrapped, replacement_wrapped); \ + return (t1)expected_wrapped; \ + } \ + DEF_MEM_CHECKS2(name##_shared, _shared_, t2, return, t1, t1, t1) DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw8_cmpxchg_u, u32, u8); DEFINE_ATOMIC_CMP_XCHG(i32_atomic_rmw16_cmpxchg_u, u32, u16); diff --git a/src/template/wasm2c_simd.declarations.c b/src/template/wasm2c_simd.declarations.c index 0e2c95114..39eb4578e 100644 --- a/src/template/wasm2c_simd.declarations.c +++ b/src/template/wasm2c_simd.declarations.c @@ -7,33 +7,36 @@ #endif // TODO: equivalent constraint for ARM and other architectures -#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \ - static inline v128 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t); \ - v128 result = func(MEM_ADDR(mem, addr, sizeof(t))); \ - SIMD_FORCE_READ(result); \ - return result; \ - } +#define DEFINE_SIMD_LOAD_FUNC(name, func, t) \ + static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + v128 result = func(MEM_ADDR(mem, addr, sizeof(t))); \ + SIMD_FORCE_READ(result); \ + return result; \ + } \ + DEF_MEM_CHECKS0(name, _, t, return, v128); #define DEFINE_SIMD_LOAD_LANE(name, func, t, lane) \ - static inline v128 name(wasm_rt_memory_t* mem, u64 addr, v128 vec) { \ - MEMCHECK(mem, addr, t); \ + static inline v128 name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + v128 vec) { \ v128 result = func(MEM_ADDR(mem, addr, sizeof(t)), vec, lane); \ SIMD_FORCE_READ(result); \ return result; \ - } + } \ + DEF_MEM_CHECKS1(name, _, t, return, v128, v128); -#define DEFINE_SIMD_STORE(name, t) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, v128 value) { \ - MEMCHECK(mem, addr, t); \ - simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value); \ - } +#define DEFINE_SIMD_STORE(name, t) \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + v128 value) { \ + simde_wasm_v128_store(MEM_ADDR(mem, addr, sizeof(t)), value); \ + } \ + DEF_MEM_CHECKS1(name, _, t, , void, v128); -#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, v128 value) { \ - MEMCHECK(mem, addr, t); \ - func(MEM_ADDR(mem, addr, sizeof(t)), value, lane); \ - } +#define DEFINE_SIMD_STORE_LANE(name, func, t, lane) \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + v128 value) { \ + func(MEM_ADDR(mem, addr, sizeof(t)), value, lane); \ + } \ + DEF_MEM_CHECKS1(name, _, t, , void, v128); // clang-format off #if WABT_BIG_ENDIAN diff --git a/test/run-spec-wasm2c.py b/test/run-spec-wasm2c.py index 1ff63d8fb..8dfa416ed 100755 --- a/test/run-spec-wasm2c.py +++ b/test/run-spec-wasm2c.py @@ -609,7 +609,6 @@ def main(args): if IS_WINDOWS: sys.stderr.write('skipping: wasm2c+memory64 is not yet supported under msvc\n') return SKIPPED - cflags.append('-DSUPPORT_MEMORY64=1') use_c11 = options.enable_threads diff --git a/test/wasm2c/add.txt b/test/wasm2c/add.txt index ee8715fda..982735534 100644 --- a/test/wasm2c/add.txt +++ b/test/wasm2c/add.txt @@ -184,14 +184,24 @@ static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -228,23 +238,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) diff --git a/test/wasm2c/check-imports.txt b/test/wasm2c/check-imports.txt index 5cdda42d5..6de09a1b1 100644 --- a/test/wasm2c/check-imports.txt +++ b/test/wasm2c/check-imports.txt @@ -208,14 +208,24 @@ static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -252,23 +262,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) @@ -876,7 +925,7 @@ u32 w2c_test_f1(w2c_test* instance) { FUNC_PROLOGUE; u32 var_i0; var_i0 = 16u; - var_i0 = i32_load(instance->w2c_env_0x5F_linear_memory, (u64)(var_i0)); + var_i0 = i32_load_default32(instance->w2c_env_0x5F_linear_memory, (u64)(var_i0)); FUNC_EPILOGUE; return var_i0; } diff --git a/test/wasm2c/export-names.txt b/test/wasm2c/export-names.txt index c6c9dfaa2..6c9f8016b 100644 --- a/test/wasm2c/export-names.txt +++ b/test/wasm2c/export-names.txt @@ -208,14 +208,24 @@ static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -252,23 +262,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) diff --git a/test/wasm2c/hello.txt b/test/wasm2c/hello.txt index 78931771d..e86102c82 100644 --- a/test/wasm2c/hello.txt +++ b/test/wasm2c/hello.txt @@ -216,14 +216,24 @@ static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -260,23 +270,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) @@ -897,10 +946,10 @@ void w2c_test_0x5Fstart_0(w2c_test* instance) { u32 var_i0, var_i1, var_i2, var_i3, var_i4; var_i0 = 0u; var_i1 = 8u; - i32_store(&instance->w2c_memory, (u64)(var_i0), var_i1); + i32_store_default32(&instance->w2c_memory, (u64)(var_i0), var_i1); var_i0 = 4u; var_i1 = 14u; - i32_store(&instance->w2c_memory, (u64)(var_i0), var_i1); + i32_store_default32(&instance->w2c_memory, (u64)(var_i0), var_i1); var_i0 = 1u; var_i1 = 0u; var_i2 = 1u; diff --git a/test/wasm2c/minimal.txt b/test/wasm2c/minimal.txt index f02ea3010..38e328aaf 100644 --- a/test/wasm2c/minimal.txt +++ b/test/wasm2c/minimal.txt @@ -178,14 +178,24 @@ static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -222,23 +232,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) diff --git a/test/wasm2c/tail-calls.txt b/test/wasm2c/tail-calls.txt index 9ee5ec43d..949f4a216 100644 --- a/test/wasm2c/tail-calls.txt +++ b/test/wasm2c/tail-calls.txt @@ -208,14 +208,24 @@ static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -252,23 +262,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ } +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) + #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) diff --git a/wasm2c/examples/fac/fac.c b/wasm2c/examples/fac/fac.c index a83e1f549..caa1705eb 100644 --- a/wasm2c/examples/fac/fac.c +++ b/wasm2c/examples/fac/fac.c @@ -136,14 +136,24 @@ static inline bool add_overflow(uint64_t a, uint64_t b, uint64_t* resptr) { #define WASM_RT_CHECK_BASE(mem) #endif +// MEMCHECK_DEFAULT32 is an "accelerated" MEMCHECK used only for +// default-page-size, 32-bit memories. It may do nothing at all +// (if hardware bounds-checking is enabled via guard pages) +// or it may do a slightly faster RANGE_CHECK. #if WASM_RT_MEMCHECK_GUARD_PAGES -#define MEMCHECK(mem, a, t) WASM_RT_CHECK_BASE(mem); +#define MEMCHECK_DEFAULT32(mem, a, t) WASM_RT_CHECK_BASE(mem); #else -#define MEMCHECK(mem, a, t) \ - WASM_RT_CHECK_BASE(mem); \ - RANGE_CHECK(mem, a, sizeof(t)) +#define MEMCHECK_DEFAULT32(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + if (UNLIKELY(a + (uint64_t)sizeof(t) > mem->size)) \ + TRAP(OOB); #endif +// MEMCHECK_GENERAL can be used for any memory +#define MEMCHECK_GENERAL(mem, a, t) \ + WASM_RT_CHECK_BASE(mem); \ + RANGE_CHECK(mem, a, sizeof(t)); + #ifdef __GNUC__ #define FORCE_READ_INT(var) __asm__("" ::"r"(var)); // Clang on Mips requires "f" constraints on floats @@ -180,23 +190,62 @@ static inline void load_data(void* dest, const void* src, size_t n) { load_data(MEM_ADDR(&m, o, s), i, s); \ } while (0) -#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ - static inline t3 name(wasm_rt_memory_t* mem, u64 addr) { \ - MEMCHECK(mem, addr, t1); \ - t1 result; \ - wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ - sizeof(t1)); \ - force_read(result); \ - return (t3)(t2)result; \ - } +#define DEF_MEM_CHECKS0(name, shared, mem_type, ret_kw, return_type) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr); \ + } + +#define DEF_MEM_CHECKS1(name, shared, mem_type, ret_kw, return_type, \ + val_type1) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1); \ + } + +#define DEF_MEM_CHECKS2(name, shared, mem_type, ret_kw, return_type, \ + val_type1, val_type2) \ + static inline return_type name##_default32(wasm_rt##shared##memory_t* mem, \ + u64 addr, val_type1 val1, \ + val_type2 val2) { \ + MEMCHECK_DEFAULT32(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } \ + static inline return_type name(wasm_rt##shared##memory_t* mem, u64 addr, \ + val_type1 val1, val_type2 val2) { \ + MEMCHECK_GENERAL(mem, addr, mem_type); \ + ret_kw name##_unchecked(mem, addr, val1, val2); \ + } + +#define DEFINE_LOAD(name, t1, t2, t3, force_read) \ + static inline t3 name##_unchecked(wasm_rt_memory_t* mem, u64 addr) { \ + t1 result; \ + wasm_rt_memcpy(&result, MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), \ + sizeof(t1)); \ + force_read(result); \ + return (t3)(t2)result; \ + } \ + DEF_MEM_CHECKS0(name, _, t1, return, t3) #define DEFINE_STORE(name, t1, t2) \ - static inline void name(wasm_rt_memory_t* mem, u64 addr, t2 value) { \ - MEMCHECK(mem, addr, t1); \ + static inline void name##_unchecked(wasm_rt_memory_t* mem, u64 addr, \ + t2 value) { \ t1 wrapped = (t1)value; \ wasm_rt_memcpy(MEM_ADDR_MEMOP(mem, addr, sizeof(t1)), &wrapped, \ sizeof(t1)); \ - } + } \ + DEF_MEM_CHECKS1(name, _, t1, , void, t2) DEFINE_LOAD(i32_load, u32, u32, u32, FORCE_READ_INT) DEFINE_LOAD(i64_load, u64, u64, u64, FORCE_READ_INT) diff --git a/wasm2c/wasm-rt-mem-impl-helper.inc b/wasm2c/wasm-rt-mem-impl-helper.inc index 2c2f9b754..6d3e596c3 100644 --- a/wasm2c/wasm-rt-mem-impl-helper.inc +++ b/wasm2c/wasm-rt-mem-impl-helper.inc @@ -74,23 +74,25 @@ void MEMORY_API_NAME(wasm_rt_allocate_memory)(MEMORY_TYPE* memory, memory->is64 = is64; MEMORY_LOCK_VAR_INIT(memory->mem_lock); -#if WASM_RT_USE_MMAP - const uint64_t mmap_size = - get_alloc_size_for_mmap(memory->max_pages, memory->is64); - void* addr = os_mmap(mmap_size); - if (!addr) { - os_print_last_error("os_mmap failed."); - abort(); - } - int ret = os_mprotect(addr, byte_length); - if (ret != 0) { - os_print_last_error("os_mprotect failed."); - abort(); - } - memory->data = addr; -#else - memory->data = calloc(byte_length, 1); + if (WASM_RT_USE_MMAP && !is64) { +#if WASM_RT_USE_MMAP // mmap-related functions don't exist unless this is set + const uint64_t mmap_size = + get_alloc_size_for_mmap(memory->max_pages, memory->is64); + void* addr = os_mmap(mmap_size); + if (!addr) { + os_print_last_error("os_mmap failed."); + abort(); + } + int ret = os_mprotect(addr, byte_length); + if (ret != 0) { + os_print_last_error("os_mprotect failed."); + abort(); + } + memory->data = addr; #endif + } else { + memory->data = calloc(byte_length, 1); + } } static uint64_t MEMORY_API_NAME(grow_memory_impl)(MEMORY_TYPE* memory, @@ -106,21 +108,24 @@ static uint64_t MEMORY_API_NAME(grow_memory_impl)(MEMORY_TYPE* memory, uint64_t old_size = old_pages * WASM_PAGE_SIZE; uint64_t new_size = new_pages * WASM_PAGE_SIZE; uint64_t delta_size = delta * WASM_PAGE_SIZE; + MEMORY_CELL_TYPE new_data; + if (WASM_RT_USE_MMAP && !memory->is64) { #if WASM_RT_USE_MMAP - MEMORY_CELL_TYPE new_data = memory->data; - int ret = os_mprotect((void*)(new_data + old_size), delta_size); - if (ret != 0) { - return (uint64_t)-1; - } -#else - MEMORY_CELL_TYPE new_data = realloc((void*)memory->data, new_size); - if (new_data == NULL) { - return (uint64_t)-1; - } -#if !WABT_BIG_ENDIAN - memset((void*)(new_data + old_size), 0, delta_size); + new_data = memory->data; + int ret = os_mprotect((void*)(new_data + old_size), delta_size); + if (ret != 0) { + return (uint64_t)-1; + } #endif + } else { + new_data = realloc((void*)memory->data, new_size); + if (new_data == NULL) { + return (uint64_t)-1; + } +#if !WABT_BIG_ENDIAN + memset((void*)(new_data + old_size), 0, delta_size); #endif + } #if WABT_BIG_ENDIAN memmove((void*)(new_data + new_size - old_size), (void*)new_data, old_size); memset((void*)new_data, 0, delta_size); @@ -145,13 +150,15 @@ uint64_t MEMORY_API_NAME(wasm_rt_grow_memory)(MEMORY_TYPE* memory, } void MEMORY_API_NAME(wasm_rt_free_memory)(MEMORY_TYPE* memory) { + if (WASM_RT_USE_MMAP && !memory->is64) { #if WASM_RT_USE_MMAP - const uint64_t mmap_size = - get_alloc_size_for_mmap(memory->max_pages, memory->is64); - os_munmap((void*)memory->data, mmap_size); // ignore error -#else - free((void*)memory->data); + const uint64_t mmap_size = + get_alloc_size_for_mmap(memory->max_pages, memory->is64); + os_munmap((void*)memory->data, mmap_size); // ignore error #endif + } else { + free((void*)memory->data); + } } #undef MEMORY_LOCK_RELEASE diff --git a/wasm2c/wasm-rt.h b/wasm2c/wasm-rt.h index c5bad6142..714555870 100644 --- a/wasm2c/wasm-rt.h +++ b/wasm2c/wasm-rt.h @@ -129,7 +129,7 @@ extern "C" { * needed (so we can use the guard based range checks below). */ #ifndef WASM_RT_USE_MMAP -#if UINTPTR_MAX > 0xffffffff && !SUPPORT_MEMORY64 +#if UINTPTR_MAX > 0xffffffff #define WASM_RT_USE_MMAP 1 #else #define WASM_RT_USE_MMAP 0 @@ -151,8 +151,7 @@ extern "C" { */ /** Check if Guard checks are supported */ -#if UINTPTR_MAX > 0xffffffff && WASM_RT_USE_MMAP && !SUPPORT_MEMORY64 && \ - !WABT_BIG_ENDIAN +#if UINTPTR_MAX > 0xffffffff && WASM_RT_USE_MMAP && !WABT_BIG_ENDIAN #define WASM_RT_GUARD_PAGES_SUPPORTED 1 #else #define WASM_RT_GUARD_PAGES_SUPPORTED 0