diff --git a/core/iwasm/aot/aot_reloc.h b/core/iwasm/aot/aot_reloc.h index ca9ba17f1f..eb41e8c2b7 100644 --- a/core/iwasm/aot/aot_reloc.h +++ b/core/iwasm/aot/aot_reloc.h @@ -185,6 +185,13 @@ typedef struct { #define REG_STRINGREF_SYM() #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +#define REG_SHARED_HEAP_SYM() \ + REG_SYM(wasm_runtime_update_last_used_shared_heap), +#else +#define REG_SHARED_HEAP_SYM() +#endif + #define REG_COMMON_SYMBOLS \ REG_SYM(aot_set_exception_with_id), \ REG_SYM(aot_invoke_native), \ @@ -218,6 +225,7 @@ typedef struct { REG_LLVM_PGO_SYM() \ REG_GC_SYM() \ REG_STRINGREF_SYM() \ + REG_SHARED_HEAP_SYM() \ #define CHECK_RELOC_OFFSET(data_size) do { \ if (!check_reloc_offset(target_section_size, \ diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 341f64cd5a..c39cee6fdb 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -63,6 +63,14 @@ bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_start_off) == 16); bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_end_off) == 24); bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap) == 32); +bh_static_assert(offsetof(WASMSharedHeap, next) == 0); +bh_static_assert(offsetof(WASMSharedHeap, chain_next) == 8); +bh_static_assert(offsetof(WASMSharedHeap, heap_handle) == 16); +bh_static_assert(offsetof(WASMSharedHeap, base_addr) == 24); +bh_static_assert(offsetof(WASMSharedHeap, size) == 32); +bh_static_assert(offsetof(WASMSharedHeap, start_off_mem64) == 40); +bh_static_assert(offsetof(WASMSharedHeap, start_off_mem32) == 48); + bh_static_assert(sizeof(CApiFuncImport) == sizeof(uintptr_t) * 3); bh_static_assert(sizeof(wasm_val_t) == 16); @@ -1991,6 +1999,8 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, #else extra->shared_heap_start_off.u32[0] = UINT32_MAX; #endif + /* After shared heap chain, will early stop if shared heap is NULL */ + extra->shared_heap = NULL; #if WASM_ENABLE_PERF_PROFILING != 0 total_size = sizeof(AOTFuncPerfProfInfo) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index a67c4209ea..e962b13b18 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -615,7 +615,8 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, (uint64)get_last_used_shared_heap_start_offset(module_inst); shared_heap_end = (uint64)get_last_used_shared_heap_end_offset(module_inst); if (app_offset >= shared_heap_start - && app_offset <= shared_heap_end - bytes + 1) { + && app_offset <= shared_heap_end - bytes + 1 + && bytes - 1 <= shared_heap_end) { return true; } @@ -624,7 +625,8 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, is_memory64 ? heap->start_off_mem64 : heap->start_off_mem32; shared_heap_end = is_memory64 ? UINT64_MAX : UINT32_MAX; if (app_offset < shared_heap_start - || app_offset > shared_heap_end - bytes + 1) { + || app_offset > shared_heap_end - bytes + 1 + || bytes - 1 > shared_heap_end) { goto fail; } @@ -635,7 +637,8 @@ is_app_addr_in_shared_heap(WASMModuleInstanceCommon *module_inst, is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; shared_heap_end = shared_heap_start - 1 + cur->size; if (app_offset >= shared_heap_start - && app_offset <= shared_heap_end - bytes + 1) { + && app_offset <= shared_heap_end - bytes + 1 + && bytes - 1 <= shared_heap_end) { update_last_used_shared_heap(module_inst, cur, is_memory64); return true; } @@ -1227,7 +1230,9 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_native_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, addr, 1)) { - return addr - get_last_used_shared_heap_base_addr_adj(module_inst_comm); + return (uint64)(uintptr_t)(addr + - get_last_used_shared_heap_base_addr_adj( + module_inst_comm)); } #endif @@ -1349,7 +1354,7 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, (WASMModuleInstanceCommon *)module_inst); shared_heap_end_off = get_last_used_shared_heap_end_offset( (WASMModuleInstanceCommon *)module_inst); - native_addr = shared_heap_base_addr_adj + app_buf_addr; + native_addr = shared_heap_base_addr_adj + (uintptr_t)app_buf_addr; /* The whole string must be in the shared heap */ str = (const char *)native_addr; diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index dcee0aeafc..02a217ff00 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -7883,3 +7883,40 @@ wasm_runtime_is_underlying_binary_freeable(WASMModuleCommon *const module) return true; } + +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +wasm_runtime_update_last_used_shared_heap(WASMModuleInstanceCommon *module_inst, + uintptr_t app_offset, size_t bytes, + uintptr_t *shared_heap_start_off_p, + uintptr_t *shared_heap_end_off_p, + uint8 **shared_heap_base_addr_adj_p, + bool is_memory64) +{ + WASMSharedHeap *heap = wasm_runtime_get_shared_heap(module_inst), *cur; + uint64 shared_heap_start, shared_heap_end; + + if (bytes == 0) { + bytes = 1; + } + + /* Find the exact shared heap that app addr is in, and update last used + * shared heap info in func context */ + for (cur = heap; cur; cur = cur->chain_next) { + shared_heap_start = + is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; + shared_heap_end = shared_heap_start - 1 + cur->size; + if (app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1 + && bytes - 1 <= shared_heap_end) { + *shared_heap_start_off_p = (uintptr_t)shared_heap_start; + *shared_heap_end_off_p = (uintptr_t)shared_heap_end; + *shared_heap_base_addr_adj_p = + cur->base_addr - (uintptr_t)shared_heap_start; + return true; + } + } + + return false; +} +#endif diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 64a6cd7936..e2b227c0a7 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -1336,6 +1336,16 @@ void wasm_runtime_set_linux_perf(bool flag); #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +wasm_runtime_update_last_used_shared_heap(WASMModuleInstanceCommon *module_inst, + uintptr_t app_offset, size_t bytes, + uintptr_t *shared_heap_start_off_p, + uintptr_t *shared_heap_end_off_p, + uint8 **shared_heap_base_addr_adj_p, + bool is_memory64); +#endif + #ifdef __cplusplus } #endif diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index e685fccd92..dfb90e7622 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -10,6 +10,40 @@ #include "aot_intrinsic.h" #include "aot_emit_control.h" +#define BUILD_IS_NOT_NULL(value, res, name) \ + do { \ + if (!(res = LLVMBuildIsNotNull(comp_ctx->builder, value, name))) { \ + aot_set_last_error("llvm build is not null failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_BR(llvm_block) \ + do { \ + if (!LLVMBuildBr(comp_ctx->builder, llvm_block)) { \ + aot_set_last_error("llvm build br failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_COND_BR(value_if, block_then, block_else) \ + do { \ + if (!LLVMBuildCondBr(comp_ctx->builder, value_if, block_then, \ + block_else)) { \ + aot_set_last_error("llvm build cond br failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_TRUNC(value, data_type) \ + do { \ + if (!(value = LLVMBuildTrunc(comp_ctx->builder, value, data_type, \ + "val_trunc"))) { \ + aot_set_last_error("llvm build trunc failed."); \ + goto fail; \ + } \ + } while (0) + #define BUILD_ICMP(op, left, right, res, name) \ do { \ if (!(res = \ @@ -111,6 +145,417 @@ ffs(int n) static LLVMValueRef get_memory_curr_page_count(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); +#if WASM_ENABLE_SHARED_HEAP != 0 +uint32 +get_module_inst_extra_offset(AOTCompContext *comp_ctx); + +#define BUILD_LOAD_PTR(ptr, data_type, res) \ + do { \ + if (!(res = LLVMBuildLoad2(comp_ctx->builder, data_type, ptr, \ + "load_value"))) { \ + aot_set_last_error("llvm build load failed"); \ + goto fail; \ + } \ + } while (0) + +/* Update last used shared heap info(alloc ptr) in function ctx: + * 1. shared_heap_start_off 2. shared_heap_end_off 3. shared_heap_base_addr_adj + */ +bool +aot_check_shared_heap_chain(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef check_succ, + LLVMValueRef start_offset, LLVMValueRef bytes, + bool is_memory64) +{ + LLVMValueRef param_values[7], ret_value, func, value, cmp; + LLVMTypeRef param_types[7], ret_type, func_type, func_ptr_type; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = INTPTR_T_TYPE; + param_types[2] = SIZE_T_TYPE; + param_types[3] = INTPTR_T_PTR_TYPE; + param_types[4] = INTPTR_T_PTR_TYPE; + param_types[5] = INT8_PTR_TYPE; + param_types[6] = INT8_TYPE; + ret_type = INT8_TYPE; + + GET_AOT_FUNCTION(wasm_runtime_update_last_used_shared_heap, 7); + + /* Call function */ + param_values[0] = func_ctx->aot_inst; + param_values[1] = start_offset; + param_values[2] = bytes; + /* pass alloc ptr */ + param_values[3] = func_ctx->shared_heap_start_off; + param_values[4] = func_ctx->shared_heap_end_off; + param_values[5] = func_ctx->shared_heap_base_addr_adj; + param_values[6] = is_memory64 ? I8_ONE : I8_ZERO; + + if (!(ret_value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 7, "call"))) { + aot_set_last_error("llvm build call failed."); + goto fail; + } + + BUILD_ICMP(LLVMIntEQ, ret_value, I8_ZERO, cmp, "shared_heap_oob"); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, + check_succ)) { + goto fail; + } + + return true; +fail: + return false; +} + +/* + * Setup the basic blocks for shared heap and shared chain memory checks. + * + * Arguments: + * block_curr: The current basic block. + * app_addr_in_cache_shared_heap: Output, block for cache shared heap. + * app_addr_in_linear_mem: Output, block for linear memory. + * app_addr_in_shared_heap_chain: Output, block for shared heap chain + * (only for shared heap chain). + * check_shared_heap_chain: Output, block for checking shared heap chain + * (only for shared heap chain). + * + * Topology: + * If enable_shared_heap: + * block_curr -> app_addr_in_cache_shared_heap + * -> app_addr_in_linear_mem + * If enable_shared_chain: + * block_curr -> app_addr_in_shared_heap_chain + * -> app_addr_in_cache_shared_heap + * -> check_shared_heap_chain + * -> app_addr_in_linear_mem + */ +static bool +setup_shared_heap_blocks(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef block_curr, + LLVMBasicBlockRef *app_addr_in_cache_shared_heap, + LLVMBasicBlockRef *app_addr_in_linear_mem, + LLVMBasicBlockRef *app_addr_in_shared_heap_chain, + LLVMBasicBlockRef *check_shared_heap_chain) +{ + ADD_BASIC_BLOCK(*app_addr_in_cache_shared_heap, + "app_addr_in_cache_shared_heap"); + ADD_BASIC_BLOCK(*app_addr_in_linear_mem, "app_addr_in_linear_mem"); + + if (comp_ctx->enable_shared_heap) { + LLVMMoveBasicBlockAfter(*app_addr_in_cache_shared_heap, block_curr); + LLVMMoveBasicBlockAfter(*app_addr_in_linear_mem, + *app_addr_in_cache_shared_heap); + } + else if (comp_ctx->enable_shared_chain) { + ADD_BASIC_BLOCK(*app_addr_in_shared_heap_chain, + "app_addr_in_shared_heap_chain"); + ADD_BASIC_BLOCK(*check_shared_heap_chain, "check_shared_heap_chain"); + LLVMMoveBasicBlockAfter(*app_addr_in_shared_heap_chain, block_curr); + LLVMMoveBasicBlockAfter(*app_addr_in_cache_shared_heap, + *app_addr_in_shared_heap_chain); + LLVMMoveBasicBlockAfter(*check_shared_heap_chain, + *app_addr_in_cache_shared_heap); + LLVMMoveBasicBlockAfter(*app_addr_in_linear_mem, + *app_addr_in_cache_shared_heap); + } + + return true; +fail: + return false; +} + +/* + * Build a branch to check if start_offset is in the shared heap chain region. + * + * Arguments: + * start_offset: The offset to check. + * app_addr_in_shared_heap_chain: Block to branch if in shared heap chain. + * app_addr_in_linear_mem: Block to branch if not in shared heap chain. + */ +static bool +build_check_app_addr_in_shared_heap_chain( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMValueRef start_offset, LLVMBasicBlockRef app_addr_in_shared_heap_chain, + LLVMBasicBlockRef app_addr_in_linear_mem) +{ + LLVMValueRef is_in_shared_heap = NULL; + + /* Use start_offset > func_ctx->shared_heap_head_start_off to test + * start_off falls in shared heap chain memory region. The shared heap + * chain oob will be detected in app_addr_in_shared_heap block or + * aot_check_shared_heap_chain function + */ + BUILD_ICMP(LLVMIntUGT, start_offset, func_ctx->shared_heap_head_start_off, + is_in_shared_heap, "shared_heap_lb_cmp"); + BUILD_COND_BR(is_in_shared_heap, app_addr_in_shared_heap_chain, + app_addr_in_linear_mem); + + SET_BUILD_POS(app_addr_in_shared_heap_chain); + + return true; +fail: + return false; +} + +/* + * Build the conditional branch for cache shared heap or shared heap chain. + * + * Arguments: + * cmp: The condition for being in cache shared heap. + * app_addr_in_cache_shared_heap: Block for cache shared heap. + * app_addr_in_linear_mem: Block for linear memory. + * check_shared_heap_chain: Block for checking shared heap chain. + * bytes: The access size in bytes. + * start_offset: The offset to check. + * is_memory64: Whether memory is 64-bit. + */ +static bool +build_shared_heap_conditional_branching( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef cmp, + LLVMBasicBlockRef app_addr_in_cache_shared_heap, + LLVMBasicBlockRef app_addr_in_linear_mem, + LLVMBasicBlockRef check_shared_heap_chain, LLVMValueRef bytes, + LLVMValueRef start_offset, bool is_memory64) +{ + if (comp_ctx->enable_shared_heap) { + BUILD_COND_BR(cmp, app_addr_in_cache_shared_heap, + app_addr_in_linear_mem); + } + else if (comp_ctx->enable_shared_chain) { + BUILD_COND_BR(cmp, app_addr_in_cache_shared_heap, + check_shared_heap_chain); + SET_BUILD_POS(check_shared_heap_chain); + if (!aot_check_shared_heap_chain(comp_ctx, func_ctx, + app_addr_in_cache_shared_heap, + start_offset, bytes, is_memory64)) + goto fail; + } + return true; +fail: + return false; +} + +/* + * Get the native address in the cache shared heap. + * + * Arguments: + * start_offset: The offset to use for address calculation. + * maddr: Output, the native address that in the cache shared heap. + */ +static bool +build_get_maddr_in_cache_shared_heap(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + LLVMValueRef start_offset, + LLVMValueRef *maddr) +{ + LLVMValueRef shared_heap_base_addr_adj; + /* load the local variable */ + BUILD_LOAD_PTR(func_ctx->shared_heap_base_addr_adj, INT8_PTR_TYPE, + shared_heap_base_addr_adj); + if (!(*maddr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, shared_heap_base_addr_adj, + &start_offset, 1, "maddr_cache_shared_heap"))) { + aot_set_last_error("llvm build inbounds gep failed"); + goto fail; + } + + return true; +fail: + return false; +} + +/* + * Check for memory overflow in shared heap for normal memory access. + * + * Arguments: + * block_curr: The current basic block. + * block_maddr_phi: The phi block for memory address. + * maddr_phi: The phi node for memory address. + * start_offset: The first offset to check. + * mem_base_addr: The base address of memory. Only used with segue. + * bytes_u32: The access size in bytes. + * is_memory64: Whether memory is wasm64 memory. + * is_target_64bit: Whether target is 64-bit. + * enable_segue: Whether to use segment register addressing. + */ +static bool +aot_check_shared_heap_memory_overflow( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, + LLVMValueRef maddr_phi, LLVMValueRef start_offset, + LLVMValueRef mem_base_addr, uint32 bytes_u32, bool is_memory64, + bool is_target_64bit, bool enable_segue) +{ + LLVMBasicBlockRef app_addr_in_cache_shared_heap, app_addr_in_linear_mem; + LLVMBasicBlockRef app_addr_in_shared_heap_chain = NULL, + check_shared_heap_chain = NULL; + LLVMValueRef cmp, cmp1, cmp2, shared_heap_start_off, shared_heap_end_off, + shared_heap_check_bound, maddr = NULL; + /* On 64/32-bit target, the offset is 64/32-bit */ + LLVMTypeRef offset_type = is_target_64bit ? I64_TYPE : I32_TYPE; + LLVMValueRef length, bytes; + + if (!setup_shared_heap_blocks( + comp_ctx, func_ctx, block_curr, &app_addr_in_cache_shared_heap, + &app_addr_in_linear_mem, &app_addr_in_shared_heap_chain, + &check_shared_heap_chain)) + goto fail; + LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem); + + /* Early branching when it's not in shared heap chain at all */ + if (comp_ctx->enable_shared_chain + && !build_check_app_addr_in_shared_heap_chain( + comp_ctx, func_ctx, start_offset, app_addr_in_shared_heap_chain, + app_addr_in_linear_mem)) + goto fail; + + /* Load the local variable of the function */ + BUILD_LOAD_PTR(func_ctx->shared_heap_start_off, offset_type, + shared_heap_start_off); + BUILD_LOAD_PTR(func_ctx->shared_heap_end_off, offset_type, + shared_heap_end_off); + /* Check if the app address is in the cache shared heap range. + * If yes, branch to the cache branch; if not, check the shared heap chain + */ + BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, + "cmp_cache_shared_heap_start"); + length = + is_target_64bit ? I64_CONST(bytes_u32 - 1) : I32_CONST(bytes_u32 - 1); + CHECK_LLVM_CONST(length); + BUILD_OP(Sub, shared_heap_end_off, length, shared_heap_check_bound, + "cache_shared_heap_end_bound"); + BUILD_ICMP(LLVMIntULE, start_offset, shared_heap_check_bound, cmp1, + "cmp_cache_shared_heap_end"); + BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); + /* Conditional branching based on whether in cached shared heap */ + bytes = is_target_64bit ? I64_CONST(bytes_u32) : I32_CONST(bytes_u32); + if (!build_shared_heap_conditional_branching( + comp_ctx, func_ctx, cmp2, app_addr_in_cache_shared_heap, + app_addr_in_linear_mem, check_shared_heap_chain, bytes, + start_offset, is_memory64)) + goto fail; + + SET_BUILD_POS(app_addr_in_cache_shared_heap); + if (!build_get_maddr_in_cache_shared_heap(comp_ctx, func_ctx, start_offset, + &maddr)) + goto fail; + + if (enable_segue) { + LLVMValueRef mem_base_addr_u64, maddr_u64, offset_to_mem_base; + if (!(maddr_u64 = LLVMBuildPtrToInt(comp_ctx->builder, maddr, I64_TYPE, + "maddr_u64")) + || !(mem_base_addr_u64 = + LLVMBuildPtrToInt(comp_ctx->builder, mem_base_addr, + I64_TYPE, "mem_base_addr_u64"))) { + aot_set_last_error("llvm build ptr to int failed"); + goto fail; + } + if (!(offset_to_mem_base = + LLVMBuildSub(comp_ctx->builder, maddr_u64, mem_base_addr_u64, + "offset_to_mem_base"))) { + aot_set_last_error("llvm build sub failed"); + goto fail; + } + if (!(maddr = LLVMBuildIntToPtr(comp_ctx->builder, offset_to_mem_base, + INT8_PTR_TYPE_GS, + "maddr_shared_heap_segue"))) { + aot_set_last_error("llvm build int to ptr failed."); + goto fail; + } + } + + LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_cache_shared_heap, 1); + BUILD_BR(block_maddr_phi); + SET_BUILD_POS(app_addr_in_linear_mem); + + return true; +fail: + return false; +} + +/* + * Check for memory overflow in shared heap for bulk memory access. + * + * Arguments: + * block_curr: The current basic block. + * block_maddr_phi: The phi block for memory address. + * check_succ: The block to branch to on success. + * maddr_phi: The phi node for memory address. + * start_offset: The offset to check. + * max_addr: The maximum address to check. + * bytes: The access size in bytes (LLVMValueRef). + * is_memory64: Whether memory is wasm64 memory. + * is_target_64bit: Whether target is 64-bit. + */ +static bool +aot_check_bulk_memory_shared_heap_memory_overflow( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, + LLVMBasicBlockRef check_succ, LLVMValueRef maddr_phi, + LLVMValueRef start_offset, LLVMValueRef max_addr, LLVMValueRef bytes, + bool is_memory64, bool is_target_64bit) +{ + LLVMBasicBlockRef app_addr_in_cache_shared_heap, app_addr_in_linear_mem; + LLVMBasicBlockRef app_addr_in_shared_heap_chain = NULL, + check_shared_heap_chain = NULL; + LLVMValueRef cmp, cmp1, cmp2, shared_heap_start_off, shared_heap_end_off, + maddr = NULL, max_offset; + /* On 64/32-bit target, the offset is 64/32-bit */ + LLVMTypeRef offset_type = is_target_64bit ? I64_TYPE : I32_TYPE; + + if (!setup_shared_heap_blocks( + comp_ctx, func_ctx, block_curr, &app_addr_in_cache_shared_heap, + &app_addr_in_linear_mem, &app_addr_in_shared_heap_chain, + &check_shared_heap_chain)) + goto fail; + LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ); + + /* Early branching when it's not in shared heap chain at all */ + if (comp_ctx->enable_shared_chain + && !build_check_app_addr_in_shared_heap_chain( + comp_ctx, func_ctx, start_offset, app_addr_in_shared_heap_chain, + app_addr_in_linear_mem)) + goto fail; + + /* Load the local variable of the function */ + BUILD_LOAD_PTR(func_ctx->shared_heap_start_off, offset_type, + shared_heap_start_off); + BUILD_LOAD_PTR(func_ctx->shared_heap_end_off, offset_type, + shared_heap_end_off); + /* Check if the app address is in the cache shared heap range. + * If yes, branch to the cache branch; if not, check the shared heap chain + */ + BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, + "cmp_cache_shared_heap_start"); + BUILD_OP(Add, max_addr, is_target_64bit ? I64_NEG_ONE : I32_NEG_ONE, + max_offset, "max_offset"); + BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_end_off, cmp1, + "cmp_cache_shared_heap_end"); + BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); + /* Conditional branching based on whether in cached shared heap */ + if (!build_shared_heap_conditional_branching( + comp_ctx, func_ctx, cmp2, app_addr_in_cache_shared_heap, + app_addr_in_linear_mem, check_shared_heap_chain, bytes, + start_offset, is_memory64)) + goto fail; + + SET_BUILD_POS(app_addr_in_cache_shared_heap); + if (!build_get_maddr_in_cache_shared_heap(comp_ctx, func_ctx, start_offset, + &maddr)) + goto fail; + + LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_cache_shared_heap, 1); + BUILD_BR(block_maddr_phi); + SET_BUILD_POS(app_addr_in_linear_mem); + + return true; +fail: + return false; +} +#endif + LLVMValueRef aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, mem_offset_t offset, uint32 bytes, bool enable_segue, @@ -118,10 +563,10 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, { LLVMValueRef offset_const = MEMORY64_COND_VALUE(I64_CONST(offset), I32_CONST(offset)); - LLVMValueRef addr, maddr, maddr_phi = NULL, offset1, cmp1, cmp2, cmp; + LLVMValueRef addr, maddr, offset1, cmp1, cmp; LLVMValueRef mem_base_addr, mem_check_bound; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); - LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; + LLVMBasicBlockRef check_succ; AOTValue *aot_value_top; uint32 local_idx_of_aot_value = 0; uint64 const_value; @@ -136,6 +581,10 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, #else bool is_memory64 = IS_MEMORY64; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + LLVMValueRef maddr_phi = NULL; + LLVMBasicBlockRef block_maddr_phi = NULL; +#endif is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; @@ -262,6 +711,13 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, *alignp = 1; } + /* The overflow check needs to be done under following conditions: + * 1. In 64-bit target, offset and addr will be extended to 64-bit + * 1.1 offset + addr can overflow when it's memory64 + * 1.2 no overflow when it's memory32 + * 2. In 32-bit target, offset and addr will be 32-bit + * 2.1 offset + addr can overflow when it's memory32 + */ if (is_target_64bit) { if (!(offset_const = LLVMBuildZExt(comp_ctx->builder, offset_const, I64_TYPE, "offset_i64")) @@ -275,7 +731,9 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* offset1 = offset + addr; */ BUILD_OP(Add, offset_const, addr, offset1, "offset1"); - if (is_memory64 && comp_ctx->enable_bound_check) { + /* 1.1 offset + addr can overflow when it's memory64 + * 2.1 Or when it's on 32-bit platform */ + if (is_memory64 || !is_target_64bit) { /* Check whether integer overflow occurs in offset + addr */ LLVMBasicBlockRef check_integer_overflow_end; ADD_BASIC_BLOCK(check_integer_overflow_end, @@ -289,23 +747,14 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } SET_BUILD_POS(check_integer_overflow_end); + block_curr = check_integer_overflow_end; } - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { - LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem; - LLVMValueRef is_in_shared_heap, shared_heap_check_bound = NULL; - - /* Add basic blocks */ - ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap"); - ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); +#if WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); - - LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr); - LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, - app_addr_in_shared_heap); - LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem); - - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + SET_BUILD_POS(block_maddr_phi); if (!(maddr_phi = LLVMBuildPhi(comp_ctx->builder, enable_segue ? INT8_PTR_TYPE_GS : INT8_PTR_TYPE, @@ -313,110 +762,16 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, aot_set_last_error("llvm build phi failed"); goto fail; } + SET_BUILD_POS(block_curr); - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); - - if (!is_target_64bit) { - /* Check whether integer overflow occurs in addr + offset */ - LLVMBasicBlockRef check_integer_overflow_end; - ADD_BASIC_BLOCK(check_integer_overflow_end, - "check_integer_overflow_end"); - LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr); - - BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); - if (!aot_emit_exception(comp_ctx, func_ctx, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, - cmp1, check_integer_overflow_end)) { - goto fail; - } - SET_BUILD_POS(check_integer_overflow_end); - } - - shared_heap_check_bound = - is_memory64 ? I64_CONST(UINT64_MAX - bytes + 1) - : (comp_ctx->pointer_size == sizeof(uint64) - ? I64_CONST(UINT32_MAX - bytes + 1) - : I32_CONST(UINT32_MAX - bytes + 1)); - CHECK_LLVM_CONST(shared_heap_check_bound); - - /* Check whether the bytes to access are in shared heap */ - if (!comp_ctx->enable_bound_check) { - /* Use IntUGT but not IntUGE to compare, since (1) in the ems - memory allocator, the hmu node includes hmu header and hmu - memory, only the latter is returned to the caller as the - allocated memory, the hmu header isn't returned so the - first byte of the shared heap won't be accessed, (2) using - IntUGT gets better performance than IntUGE in some cases */ - BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off, - is_in_shared_heap, "is_in_shared_heap"); - /* We don't check the shared heap's upper boundary if boundary - check isn't enabled, the runtime may also use the guard pages - of shared heap to check the boundary if hardware boundary - check feature is enabled. */ - } - else { - /* Use IntUGT but not IntUGE to compare, same as above */ - BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off, - cmp1, "cmp1"); - /* Check the shared heap's upper boundary if boundary check is - enabled */ - BUILD_ICMP(LLVMIntULE, offset1, shared_heap_check_bound, cmp2, - "cmp2"); - BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap"); - } - - if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap, - app_addr_in_shared_heap, app_addr_in_linear_mem)) { - aot_set_last_error("llvm build cond br failed"); - goto fail; - } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap); - - /* Get native address inside shared heap */ - if (!(maddr = - LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->shared_heap_base_addr_adj, - &offset1, 1, "maddr_shared_heap"))) { - aot_set_last_error("llvm build inbounds gep failed"); - goto fail; - } - - if (enable_segue) { - LLVMValueRef mem_base_addr_u64, maddr_u64, offset_to_mem_base; - - if (!(maddr_u64 = LLVMBuildPtrToInt(comp_ctx->builder, maddr, - I64_TYPE, "maddr_u64")) - || !(mem_base_addr_u64 = - LLVMBuildPtrToInt(comp_ctx->builder, mem_base_addr, - I64_TYPE, "mem_base_addr_u64"))) { - aot_set_last_error("llvm build ptr to int failed"); - goto fail; - } - if (!(offset_to_mem_base = - LLVMBuildSub(comp_ctx->builder, maddr_u64, - mem_base_addr_u64, "offset_to_mem_base"))) { - aot_set_last_error("llvm build sub failed"); - goto fail; - } - if (!(maddr = LLVMBuildIntToPtr( - comp_ctx->builder, offset_to_mem_base, INT8_PTR_TYPE_GS, - "maddr_shared_heap_segue"))) { - aot_set_last_error("llvm build int to ptr failed."); - goto fail; - } - } - - LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1); - - if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { - aot_set_last_error("llvm build br failed"); + if (!aot_check_shared_heap_memory_overflow( + comp_ctx, func_ctx, block_curr, block_maddr_phi, maddr_phi, + offset1, mem_base_addr, bytes, is_memory64, is_target_64bit, + enable_segue)) { goto fail; } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); - block_curr = LLVMGetInsertBlock(comp_ctx->builder); } +#endif if (comp_ctx->enable_bound_check && !(is_local_of_aot_value @@ -449,21 +804,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } - if (is_target_64bit) { - BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); - } - else { - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { - /* Check integer overflow has been checked above */ - BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); - } - else { - /* Check integer overflow */ - BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); - BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp2, "cmp2"); - BUILD_OP(Or, cmp1, cmp2, cmp, "cmp"); - } - } + BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); /* Add basic blocks */ ADD_BASIC_BLOCK(check_succ, "check_succ"); @@ -509,17 +850,20 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { +#if WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1); if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { aot_set_last_error("llvm build br failed"); goto fail; } - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + SET_BUILD_POS(block_maddr_phi); return maddr_phi; } else +#endif return maddr; fail: return NULL; @@ -544,15 +888,6 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMSetAlignment(value, known_align); \ } while (0) -#define BUILD_TRUNC(value, data_type) \ - do { \ - if (!(value = LLVMBuildTrunc(comp_ctx->builder, value, data_type, \ - "val_trunc"))) { \ - aot_set_last_error("llvm build trunc failed."); \ - goto fail; \ - } \ - } while (0) - #define BUILD_STORE() \ do { \ LLVMValueRef res; \ @@ -1150,16 +1485,23 @@ LLVMValueRef check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef offset, LLVMValueRef bytes) { - LLVMValueRef maddr, max_addr, cmp; - LLVMValueRef mem_base_addr, maddr_phi = NULL; + LLVMValueRef maddr, max_addr, cmp, cmp1; + LLVMValueRef mem_base_addr; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); - LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; + LLVMBasicBlockRef check_succ; LLVMValueRef mem_size; + bool is_target_64bit; #if WASM_ENABLE_MEMORY64 == 0 bool is_memory64 = false; #else bool is_memory64 = IS_MEMORY64; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + LLVMValueRef maddr_phi = NULL; + LLVMBasicBlockRef block_maddr_phi = NULL; +#endif + + is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; /* Get memory base address and memory data size */ #if WASM_ENABLE_SHARED_MEMORY != 0 @@ -1221,111 +1563,71 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, ADD_BASIC_BLOCK(check_succ, "check_succ"); LLVMMoveBasicBlockAfter(check_succ, block_curr); - offset = - LLVMBuildZExt(comp_ctx->builder, offset, I64_TYPE, "extend_offset"); - bytes = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, "extend_len"); - if (!offset || !bytes) { - aot_set_last_error("llvm build zext failed."); - goto fail; + /* Same logic with aot_check_memory_overflow, offset and bytes are 32/64 + * bits on 32/64 bits platform */ + if (is_target_64bit) { + offset = + LLVMBuildZExt(comp_ctx->builder, offset, I64_TYPE, "extend_offset"); + bytes = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, "extend_len"); + if (!offset || !bytes) { + aot_set_last_error("llvm build zext failed."); + goto fail; + } } BUILD_OP(Add, offset, bytes, max_addr, "max_addr"); - if (is_memory64 && comp_ctx->enable_bound_check) { - /* Check whether integer overflow occurs in offset + addr */ + /* Check overflow when it's memory64 or it's on 32 bits platform */ + if (is_memory64 || !is_target_64bit) { + /* Check whether integer overflow occurs in offset + bytes */ LLVMBasicBlockRef check_integer_overflow_end; ADD_BASIC_BLOCK(check_integer_overflow_end, "check_integer_overflow_end"); LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr); + /* offset + bytes can overflow yet is valid(for example, 0xffffffff, 1), + * allow it to be 0(either 0, 0 or overflow and valid) */ BUILD_ICMP(LLVMIntULT, max_addr, offset, cmp, "cmp"); + BUILD_ICMP(LLVMIntNE, max_addr, is_target_64bit ? I64_ZERO : I32_ZERO, + cmp1, "cmp1"); + BUILD_OP(And, cmp, cmp1, cmp, "overflow"); if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, check_integer_overflow_end)) { goto fail; } SET_BUILD_POS(check_integer_overflow_end); + block_curr = check_integer_overflow_end; } - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { - LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem; - LLVMValueRef shared_heap_start_off, shared_heap_check_bound; - LLVMValueRef max_offset, cmp1, cmp2, is_in_shared_heap; - - /* Add basic blocks */ - ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap"); - ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); +#if WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); - - LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr); - LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, - app_addr_in_shared_heap); - LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ); - - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + SET_BUILD_POS(block_maddr_phi); if (!(maddr_phi = LLVMBuildPhi(comp_ctx->builder, INT8_PTR_TYPE, "maddr_phi"))) { aot_set_last_error("llvm build phi failed"); goto fail; } + SET_BUILD_POS(block_curr); - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); - - shared_heap_start_off = func_ctx->shared_heap_start_off; - if (comp_ctx->pointer_size == sizeof(uint32)) { - if (!(shared_heap_start_off = - LLVMBuildZExt(comp_ctx->builder, shared_heap_start_off, - I64_TYPE, "shared_heap_start_off_u64"))) { - aot_set_last_error("llvm build zext failed"); - goto fail; - } - } - shared_heap_check_bound = - is_memory64 ? I64_CONST(UINT64_MAX) : I64_CONST(UINT32_MAX); - CHECK_LLVM_CONST(shared_heap_check_bound); - - /* Check whether the bytes to access are in shared heap */ - if (!comp_ctx->enable_bound_check) { - /* Use IntUGT but not IntUGE to compare, same as the check - in aot_check_memory_overflow */ - BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off, - is_in_shared_heap, "is_in_shared_heap"); - } - else { - BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off, - cmp1, "cmp1"); - BUILD_OP(Add, max_addr, I64_NEG_ONE, max_offset, "max_offset"); - BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_check_bound, cmp2, - "cmp2"); - BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap"); - } - - if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap, - app_addr_in_shared_heap, app_addr_in_linear_mem)) { - aot_set_last_error("llvm build cond br failed"); - goto fail; - } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap); - - /* Get native address inside shared heap */ - if (!(maddr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->shared_heap_base_addr_adj, - &offset, 1, "maddr_shared_heap"))) { - aot_set_last_error("llvm build inbounds gep failed"); - goto fail; - } - LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1); - - if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { - aot_set_last_error("llvm build br failed"); + if (!aot_check_bulk_memory_shared_heap_memory_overflow( + comp_ctx, func_ctx, block_curr, block_maddr_phi, check_succ, + maddr_phi, offset, max_addr, bytes, is_memory64, + is_target_64bit)) { goto fail; } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); - block_curr = LLVMGetInsertBlock(comp_ctx->builder); } +#endif + /* mem_size is always 64-bit, extend max_addr on 32 bits platform */ + if (!is_target_64bit + && !(max_addr = LLVMBuildZExt(comp_ctx->builder, max_addr, I64_TYPE, + "extend_max_addr"))) { + aot_set_last_error("llvm build zext failed."); + goto fail; + } BUILD_ICMP(LLVMIntUGT, max_addr, mem_size, cmp, "cmp_max_mem_addr"); if (!aot_emit_exception(comp_ctx, func_ctx, @@ -1341,7 +1643,9 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { +#if WASM_ENABLE_SHARED_HEAP != 0 + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1); if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { @@ -1352,6 +1656,7 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return maddr_phi; } else +#endif return maddr; fail: return NULL; diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index c1708e3f9d..9694a3e184 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1517,73 +1517,153 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; } +#define BUILD_IS_NOT_NULL(value, res, name) \ + do { \ + if (!(res = LLVMBuildIsNotNull(comp_ctx->builder, value, name))) { \ + aot_set_last_error("llvm build is not null failed."); \ + goto fail; \ + } \ + } while (0) + +#define get_module_extra_field_offset(field) \ + do { \ + offset_u32 = get_module_inst_extra_offset(comp_ctx); \ + if (comp_ctx->is_jit_mode) \ + offset_u32 += offsetof(WASMModuleInstanceExtra, field); \ + else \ + offset_u32 += offsetof(AOTModuleInstanceExtra, field); \ + } while (0) + +#define LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(field, type) \ + do { \ + get_module_extra_field_offset(field); \ + offset = I32_CONST(offset_u32); \ + CHECK_LLVM_CONST(offset); \ + if (!(field_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, \ + func_ctx->aot_inst, &offset, 1, \ + #field "_p"))) { \ + aot_set_last_error("llvm build inbounds gep failed"); \ + goto fail; \ + } \ + if (!(load_val = \ + LLVMBuildLoad2(comp_ctx->builder, type, field_p, #field))) { \ + aot_set_last_error("llvm build load failed"); \ + goto fail; \ + } \ + if (!(func_ctx->field = \ + LLVMBuildAlloca(comp_ctx->builder, type, #field))) { \ + aot_set_last_error("llvm build alloca failed"); \ + goto fail; \ + } \ + if (!LLVMBuildStore(comp_ctx->builder, load_val, func_ctx->field)) { \ + aot_set_last_error("llvm build store failed"); \ + goto fail; \ + } \ + } while (0) + static bool create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { - LLVMValueRef offset, base_addr_p, start_off_p, cmp; +#if WASM_ENABLE_SHARED_HEAP != 0 + LLVMValueRef offset, field_p, load_val, shared_heap_head_p, + shared_heap_head, cmp, field_p_or_default, shared_heap_head_start_off, + shared_heap_head_start_off_minus_one; + LLVMTypeRef shared_heap_offset_type; uint32 offset_u32; - - /* Load aot_inst->e->shared_heap_base_addr_adj */ - offset_u32 = get_module_inst_extra_offset(comp_ctx); -#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 - if (comp_ctx->is_jit_mode) - offset_u32 += - offsetof(WASMModuleInstanceExtra, shared_heap_base_addr_adj); - else +#if WASM_ENABLE_MEMORY64 == 0 + bool is_memory64 = false; +#else + bool is_memory64 = IS_MEMORY64; #endif - offset_u32 += - offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj); + + shared_heap_offset_type = + comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE; + + /* shared_heap_base_addr_adj, shared_heap_start_off, and + * shared_heap_end_off can be updated later, use local variable to + * represent them */ + LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(shared_heap_base_addr_adj, + INT8_PTR_TYPE); + LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(shared_heap_start_off, + shared_heap_offset_type); + LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(shared_heap_end_off, + shared_heap_offset_type); + + /* Shared Heap head start off won't be updated, no need to alloca */ + get_module_extra_field_offset(shared_heap); offset = I32_CONST(offset_u32); CHECK_LLVM_CONST(offset); - - if (!(base_addr_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->aot_inst, &offset, 1, - "shared_heap_base_addr_adj_p"))) { + if (!(shared_heap_head_p = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1, + "shared_heap_head_p"))) { aot_set_last_error("llvm build inbounds gep failed"); - return false; + goto fail; } - if (!(func_ctx->shared_heap_base_addr_adj = - LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, base_addr_p, - "shared_heap_base_addr_adj"))) { + if (!(shared_heap_head = + LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, + shared_heap_head_p, "shared_heap_head"))) { aot_set_last_error("llvm build load failed"); - return false; + goto fail; } + BUILD_IS_NOT_NULL(shared_heap_head, cmp, "has_shared_heap"); - /* Load aot_inst->e->shared_heap_start_off */ - offset_u32 = get_module_inst_extra_offset(comp_ctx); -#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 - if (comp_ctx->is_jit_mode) - offset_u32 += offsetof(WASMModuleInstanceExtra, shared_heap_start_off); - else -#endif - offset_u32 += offsetof(AOTModuleInstanceExtra, shared_heap_start_off); + if (is_memory64) { + offset_u32 = offsetof(WASMSharedHeap, start_off_mem64); + } + else { + offset_u32 = offsetof(WASMSharedHeap, start_off_mem32); + } offset = I32_CONST(offset_u32); CHECK_LLVM_CONST(offset); - - if (!(start_off_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->aot_inst, &offset, 1, - "shared_heap_start_off_p"))) { + if (!(field_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + shared_heap_head, &offset, 1, + "head_start_off_p"))) { aot_set_last_error("llvm build inbounds gep failed"); - return false; + goto fail; } - if (!(func_ctx->shared_heap_start_off = LLVMBuildLoad2( - comp_ctx->builder, - comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE, - start_off_p, "shared_heap_start_off"))) { - aot_set_last_error("llvm build load failed"); - return false; + + /* Select a valid shared heap head ptr or safe alloca ptr stores + * shared_heap_start_off(UINT32_MAX/UINT64_MAX) */ + if (!(field_p_or_default = LLVMBuildSelect(comp_ctx->builder, cmp, field_p, + func_ctx->shared_heap_start_off, + "ptr_or_default"))) { + aot_set_last_error("llvm build select failed"); + goto fail; } - if (!(cmp = LLVMBuildIsNotNull(comp_ctx->builder, - func_ctx->shared_heap_base_addr_adj, - "has_shared_heap"))) { - aot_set_last_error("llvm build is not null failed"); - return false; + if (!(shared_heap_head_start_off = LLVMBuildLoad2( + comp_ctx->builder, shared_heap_offset_type, field_p_or_default, + "shared_heap_head_start_off"))) { + aot_set_last_error("llvm build load failed"); + goto fail; + } + if (!(shared_heap_head_start_off_minus_one = LLVMBuildAdd( + comp_ctx->builder, shared_heap_head_start_off, + comp_ctx->pointer_size == sizeof(uint64) ? I64_NEG_ONE + : I32_NEG_ONE, + "head_start_off_minus_one"))) { + aot_set_last_error("llvm build load failed"); + goto fail; } + /* if there is attached shared heap(s), the value will be valid start_off-1, + * otherwise it will be UINT32_MAX/UINT64_MAX, so during the bounds checks, + * when has attached shared heap: offset > start_off - 1 => offset >= offset + * when no attached shared heap: offset > UINT32_MAX/UINT64_MAX is always + * false + * */ + if (!(func_ctx->shared_heap_head_start_off = LLVMBuildSelect( + comp_ctx->builder, cmp, shared_heap_head_start_off_minus_one, + shared_heap_head_start_off, "head_start_off"))) { + aot_set_last_error("llvm build select failed"); + goto fail; + } return true; fail: return false; +#else /* else of WASM_ENABLE_SHARED_HEAP != 0 */ + return true; +#endif /* end of WASM_ENABLE_SHARED_HEAP != 0 */ } static bool @@ -1877,7 +1957,7 @@ aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx, } /* Load shared heap, shared heap start off mem32 or mem64 */ - if (comp_ctx->enable_shared_heap + if ((comp_ctx->enable_shared_heap || comp_ctx->enable_shared_chain) && !create_shared_heap_info(comp_ctx, func_ctx)) { goto fail; } @@ -2703,6 +2783,9 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) if (option->enable_shared_heap) comp_ctx->enable_shared_heap = true; + if (option->enable_shared_chain) + comp_ctx->enable_shared_chain = true; + comp_ctx->opt_level = option->opt_level; comp_ctx->size_level = option->size_level; diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index 6b1233c39f..b4228c67a0 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -254,8 +254,12 @@ typedef struct AOTFuncContext { bool mem_space_unchanged; AOTCheckedAddrList checked_addr_list; + /* The last accessed shared heap info */ LLVMValueRef shared_heap_base_addr_adj; LLVMValueRef shared_heap_start_off; + LLVMValueRef shared_heap_end_off; + /* The start offset of the head of shared heap chain */ + LLVMValueRef shared_heap_head_start_off; LLVMBasicBlockRef got_exception_block; LLVMBasicBlockRef func_return_block; @@ -486,6 +490,7 @@ typedef struct AOTCompContext { bool enable_gc; bool enable_shared_heap; + bool enable_shared_chain; uint32 opt_level; uint32 size_level; diff --git a/core/iwasm/include/aot_comp_option.h b/core/iwasm/include/aot_comp_option.h index 8391766232..e12f3f6fc3 100644 --- a/core/iwasm/include/aot_comp_option.h +++ b/core/iwasm/include/aot_comp_option.h @@ -79,6 +79,7 @@ typedef struct AOTCompOption { bool enable_stack_estimation; bool quick_invoke_c_api_import; bool enable_shared_heap; + bool enable_shared_chain; char *use_prof_file; uint32_t opt_level; uint32_t size_level; diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 3cc2afe04d..61628904b6 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -2818,6 +2818,7 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, #else module_inst->e->shared_heap_start_off.u32[0] = UINT32_MAX; #endif + module_inst->e->shared_heap = NULL; #endif #if WASM_ENABLE_GC != 0 diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 8d8e640b49..b1c28e85c2 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -95,13 +95,13 @@ typedef union { typedef struct WASMSharedHeap { /* The global shared heap list maintained in runtime, used for runtime * destroy */ - struct WASMSharedHeap *next; + DefPointer(struct WASMSharedHeap *, next); /* The logical shared heap chain the shared heap in */ - struct WASMSharedHeap *chain_next; + DefPointer(struct WASMSharedHeap *, chain_next); /* Will be null if shared heap is created from pre allocated memory chunk * and don't need to dynamic malloc and free */ - void *heap_handle; - uint8 *base_addr; + DefPointer(void *, heap_handle); + DefPointer(uint8 *, base_addr); uint64 size; uint64 start_off_mem64; uint64 start_off_mem32; diff --git a/samples/shared-heap/CMakeLists.txt b/samples/shared-heap/CMakeLists.txt index 5be99d3a22..03906f7c32 100644 --- a/samples/shared-heap/CMakeLists.txt +++ b/samples/shared-heap/CMakeLists.txt @@ -111,21 +111,31 @@ if (WAMR_BUILD_AOT EQUAL 1) ) if (WAMR_COMPILER) message (CHECK_PASS "found") - else() + else () message (CHECK_FAIL "not found") - endif() + endif () if (NOT EXISTS ${WAMR_COMPILER}) message (FATAL_ERROR "Please build wamrc under ${WAMR_ROOT_DIR}/wamr-compiler") - else() + else () message (STATUS "WAMR_COMPILER is ${WAMR_COMPILER}") - endif() + endif () + + if (WAMR_BUILD_TARGET STREQUAL "X86_32") + set (WAMR_COMPILER_FLAGS --enable-shared-heap --target=i386) + set (WAMR_COMPILER_CHAIN_FLAGS --enable-shared-chain --target=i386) + else () + set (WAMR_COMPILER_FLAGS --enable-shared-heap) + set (WAMR_COMPILER_CHAIN_FLAGS --enable-shared-chain) + endif () add_custom_target( wasm_to_aot ALL DEPENDS wasm-apps/test1.wasm wasm-apps/test2.wasm ${WAMR_COMPILER} - COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test1.aot wasm-apps/test1.wasm - COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test2.aot wasm-apps/test2.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_FLAGS} -o wasm-apps/test1.aot wasm-apps/test1.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_FLAGS} -o wasm-apps/test2.aot wasm-apps/test2.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_CHAIN_FLAGS} -o wasm-apps/test1_chain.aot wasm-apps/test1.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_CHAIN_FLAGS} -o wasm-apps/test2_chain.aot wasm-apps/test2.wasm WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) endif() diff --git a/samples/shared-heap/src/shared_heap_chain.c b/samples/shared-heap/src/shared_heap_chain.c index beb17b137e..8d5a4ea97b 100644 --- a/samples/shared-heap/src/shared_heap_chain.c +++ b/samples/shared-heap/src/shared_heap_chain.c @@ -34,7 +34,7 @@ produce_data(wasm_module_inst_t module_inst, wasm_exec_env_t exec_env, /* Passes wasm address directly between wasm apps since memory in shared * heap chain is viewed as single address space in wasm's perspective */ - buf = (uint8 *)(uint64)argv[0]; + buf = (uint8 *)(uintptr_t)argv[0]; if (!bh_post_msg(queue, 1, buf, buf_size)) { printf("Failed to post message to queue\n"); if (free_on_fail) @@ -130,7 +130,7 @@ wasm_consumer(wasm_module_inst_t module_inst, bh_queue *queue) buf = bh_message_payload(msg); /* call wasm function */ - argv[0] = (uint32)(uint64)buf; + argv[0] = (uint32)(uintptr_t)buf; if (i < 8) wasm_runtime_call_wasm(exec_env, print_buf_func, 1, argv); else @@ -198,7 +198,7 @@ main(int argc, char **argv) if (!aot_mode) wasm_file1 = "./wasm-apps/test1.wasm"; else - wasm_file1 = "./wasm-apps/test1.aot"; + wasm_file1 = "./wasm-apps/test1_chain.aot"; if (!(wasm_file1_buf = bh_read_file_to_buffer(wasm_file1, &wasm_file1_size))) { printf("Open wasm file %s failed.\n", wasm_file1); @@ -225,7 +225,7 @@ main(int argc, char **argv) if (!aot_mode) wasm_file2 = "./wasm-apps/test2.wasm"; else - wasm_file2 = "./wasm-apps/test2.aot"; + wasm_file2 = "./wasm-apps/test2_chain.aot"; if (!(wasm_file2_buf = bh_read_file_to_buffer(wasm_file2, &wasm_file2_size))) { printf("Open wasm file %s failed.\n", wasm_file1); diff --git a/samples/shared-heap/wasm-apps/test1.c b/samples/shared-heap/wasm-apps/test1.c index 8ed1ca84cc..321f102185 100644 --- a/samples/shared-heap/wasm-apps/test1.c +++ b/samples/shared-heap/wasm-apps/test1.c @@ -62,6 +62,10 @@ my_shared_heap_free(void *ptr) void * produce_str(char *addr, uint32_t index) { + char c; snprintf(addr, 512, "Data: %u stores to pre-allocated shared heap", index); + /* Actually access it in wasm */ + c = addr[0]; + printf("In WASM: the first char is %c\n", c); return addr; } diff --git a/samples/shared-heap/wasm-apps/test2.c b/samples/shared-heap/wasm-apps/test2.c index 36ae748b47..44d573164e 100644 --- a/samples/shared-heap/wasm-apps/test2.c +++ b/samples/shared-heap/wasm-apps/test2.c @@ -19,7 +19,10 @@ print_buf(char *buf) void consume_str(char *buf) { - printf("wasm app2's wasm func received buf in pre-allocated shared buf: " - "%s\n\n", - buf); + /* Actually access it in wasm */ + char c = buf[0]; + printf("In WASM: wasm app2's wasm func received buf in pre-allocated " + "shared buf: " + "%s with its first char is %c\n\n", + buf, c); } diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 8c963cadd8..b726f83a12 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -19,8 +19,15 @@ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}") if(WAMR_BUILD_TARGET STREQUAL "X86_32") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32") + # 1) Force -m32 + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32" CACHE STRING "" FORCE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32" CACHE STRING "" FORCE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -m32" CACHE STRING "" FORCE) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -m32" CACHE STRING "" FORCE) + + # 2) Make CMake prefer i386 libraries + set(CMAKE_SYSTEM_PROCESSOR i386 CACHE STRING "" FORCE) + set(CMAKE_LIBRARY_ARCHITECTURE "i386-linux-gnu" CACHE STRING "" FORCE) endif() # Prevent overriding the parent project's compiler/linker @@ -29,12 +36,21 @@ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) # Fetch Google test include (FetchContent) -FetchContent_Declare ( + +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24") + FetchContent_Declare ( + googletest + URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip + DOWNLOAD_EXTRACT_TIMESTAMP ON + ) +else() + FetchContent_Declare ( googletest URL https://github.com/google/googletest/archive/03597a01ee50ed33e9dfd640b249b4be3799d395.zip - DOWNLOAD_EXTRACT_TIMESTAMP TRUE -) -FetchContent_MakeAvailable (googletest) + ) +endif() + +FetchContent_MakeAvailable(googletest) SET(GOOGLETEST_INCLUDED 1) diff --git a/tests/unit/shared-heap/CMakeLists.txt b/tests/unit/shared-heap/CMakeLists.txt index 2b06c537f8..fa7067918a 100644 --- a/tests/unit/shared-heap/CMakeLists.txt +++ b/tests/unit/shared-heap/CMakeLists.txt @@ -12,12 +12,20 @@ set(WAMR_BUILD_AOT 1) set(WAMR_BUILD_INTERP 1) set(WAMR_BUILD_FAST_INTERP 1) set(WAMR_BUILD_JIT 0) -set(WAMR_BUILD_MEMORY64 1) +if(WAMR_BUILD_TARGET STREQUAL "X86_32") + set(WAMR_BUILD_MEMORY64 0) +else() + set(WAMR_BUILD_MEMORY64 1) +endif() set(WAMR_BUILD_SHARED_HEAP 1) # Compile wasm modules add_subdirectory(wasm-apps) +if (WAMR_BUILD_MEMORY64 EQUAL 1) + add_subdirectory(wasm-apps/memory64) +endif () + # if only load this CMake other than load it as subdirectory include(../unit_common.cmake) @@ -56,4 +64,4 @@ add_executable(shared_heap_test ${unit_test_sources}) target_link_libraries(shared_heap_test ${LLVM_AVAILABLE_LIBS} gtest_main) -gtest_discover_tests(shared_heap_test) \ No newline at end of file +gtest_discover_tests(shared_heap_test) diff --git a/tests/unit/shared-heap/shared_heap_test.cc b/tests/unit/shared-heap/shared_heap_test.cc index 626f0d2d5d..cd4e579865 100644 --- a/tests/unit/shared-heap/shared_heap_test.cc +++ b/tests/unit/shared-heap/shared_heap_test.cc @@ -155,6 +155,9 @@ TEST_F(shared_heap_test, test_shared_heap_basic) test_shared_heap(shared_heap, "test.aot", "test", 0, argv); EXPECT_EQ(10, argv[0]); + + test_shared_heap(shared_heap, "test_chain.aot", "test", 0, argv); + EXPECT_EQ(10, argv[0]); } TEST_F(shared_heap_test, test_shared_heap_malloc_fail) @@ -175,6 +178,10 @@ TEST_F(shared_heap_test, test_shared_heap_malloc_fail) test_shared_heap(shared_heap, "test.aot", "test_malloc_fail", 0, argv); EXPECT_EQ(1, argv[0]); + + test_shared_heap(shared_heap, "test_chain.aot", "test_malloc_fail", 0, + argv); + EXPECT_EQ(1, argv[0]); } TEST_F(shared_heap_test, test_preallocated_shared_heap_malloc_fail) @@ -201,20 +208,41 @@ TEST_F(shared_heap_test, test_preallocated_shared_heap_malloc_fail) argv[0] = 1024; test_shared_heap(shared_heap, "test.aot", "my_shared_heap_malloc", 1, argv); EXPECT_EQ(0, argv[0]); + + argv[0] = 1024; + test_shared_heap(shared_heap, "test_chain.aot", "my_shared_heap_malloc", 1, + argv); + EXPECT_EQ(0, argv[0]); } -TEST_F(shared_heap_test, test_shared_heap_chain_rmw) +static void +create_test_shared_heap(uint8 *preallocated_buf, size_t size, + WASMSharedHeap **shared_heap_res) { SharedHeapInitArgs args = { 0 }; - WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, - *shared_heap_chain = nullptr; - uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); - uint8 preallocated_buf[BUF_SIZE] = { 0 }, - preallocated_buf2[BUF_SIZE] = { 0 }; - uint32 start1, end1, start2, end2; + WASMSharedHeap *shared_heap = nullptr; + args.pre_allocated_addr = preallocated_buf; + args.size = size; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + *shared_heap_res = shared_heap; + if (!*shared_heap_res) { + FAIL() << "Create shared heap chain failed.\n"; + } +} + +static void +create_test_shared_heap_chain(uint8 *preallocated_buf, size_t size, + uint8 *preallocated_buf2, size_t size2, + WASMSharedHeap **shared_heap_chain) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr; args.pre_allocated_addr = preallocated_buf; - args.size = BUF_SIZE; + args.size = size; shared_heap = wasm_runtime_create_shared_heap(&args); if (!shared_heap) { FAIL() << "Create preallocated shared heap failed.\n"; @@ -222,23 +250,74 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw) memset(&args, 0, sizeof(args)); args.pre_allocated_addr = preallocated_buf2; - args.size = BUF_SIZE; + args.size = size2; shared_heap2 = wasm_runtime_create_shared_heap(&args); if (!shared_heap2) { FAIL() << "Create preallocated shared heap failed.\n"; } - shared_heap_chain = + *shared_heap_chain = wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); - if (!shared_heap_chain) { + if (!*shared_heap_chain) { FAIL() << "Create shared heap chain failed.\n"; } +} + +TEST_F(shared_heap_test, test_shared_heap_rmw) +{ + WASMSharedHeap *shared_heap = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }; + uint32 start1, end1; + + create_test_shared_heap(preallocated_buf, BUF_SIZE, &shared_heap); + + /* app addr for shared heap */ + start1 = UINT32_MAX - BUF_SIZE + 1; + end1 = UINT32_MAX; + + argv[0] = end1; + argv[1] = 101; + test_shared_heap(shared_heap, "test.wasm", "read_modify_write_8", 2, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + + argv[0] = start1; + argv[1] = 37; + test_shared_heap(shared_heap, "test.wasm", "read_modify_write_8", 2, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[0], 37); + + argv[0] = end1; + argv[1] = 81; + test_shared_heap(shared_heap, "test.aot", "read_modify_write_8", 2, argv); + EXPECT_EQ(101, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 81); + + argv[0] = start1; + argv[1] = 98; + test_shared_heap(shared_heap, "test.aot", "read_modify_write_8", 2, argv); + EXPECT_EQ(37, argv[0]); + EXPECT_EQ(preallocated_buf[0], 98); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); /* app addr for shared heap */ - start1 = 0xFFFFFFFF - 2 * BUF_SIZE + 1; - end1 = 0xFFFFFFFF - BUF_SIZE; - start2 = 0xFFFFFFFF - BUF_SIZE + 1; - end2 = 0xFFFFFFFF; + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; /* shared heap 1 */ argv[0] = end1; @@ -256,76 +335,309 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw) EXPECT_EQ(0, argv[0]); EXPECT_EQ(preallocated_buf2[0], 129); - /* TODO: test aot when chain is supported in AOT */ - /* argv[0] = start1; argv[1] = 98; - test_shared_heap(shared_heap_chain, "test.aot", "read_modify_write_8", 2, - argv); + test_shared_heap(shared_heap_chain, "test_chain.aot", "read_modify_write_8", + 2, argv); EXPECT_EQ(0, argv[0]); EXPECT_EQ(preallocated_buf[0], 98); argv[0] = end2; argv[1] = 81; - test_shared_heap(shared_heap_chain, "test.aot", "read_modify_write_8", 2, - argv); + test_shared_heap(shared_heap_chain, "test_chain.aot", "read_modify_write_8", + 2, argv); EXPECT_EQ(0, argv[0]); EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 81); - */ } -TEST_F(shared_heap_test, test_shared_heap_chain_rmw_oob) +TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory) { SharedHeapInitArgs args = { 0 }; - WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, - *shared_heap_chain = nullptr; + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; + + argv[0] = end1; + argv[1] = 101; + argv[2] = 1; + test_shared_heap(shared_heap_chain, "test_bulk_memory.wasm", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(end1, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + + argv[0] = start1; + argv[1] = 14; + argv[2] = 1; + test_shared_heap(shared_heap_chain, "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(start1, argv[0]); + EXPECT_EQ(preallocated_buf[0], 14); + + /* nothing happen when memory fill 0 byte */ + argv[0] = start2; + argv[1] = 68; + argv[2] = 0; + test_shared_heap(shared_heap_chain, "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(start2, argv[0]); + EXPECT_EQ(preallocated_buf2[0], 0); + + argv[0] = end2; + argv[1] = 98; + argv[2] = 1; + test_shared_heap(shared_heap_chain, "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(end2, argv[0]); + EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 98); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory_oob) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; + + /* shared heap 1 */ + argv[0] = end1; + argv[1] = 101; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = end2; + argv[1] = 98; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start1; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start2; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = end1; + argv[1] = 101; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = end2; + argv[1] = 98; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start1; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start2; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory_chain.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); +} + +TEST_F(shared_heap_test, test_shared_heap_rmw_oob) +{ + WASMSharedHeap *shared_heap = nullptr; uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; uint32 start1, end1, start2, end2; - args.pre_allocated_addr = preallocated_buf; - args.size = BUF_SIZE; - shared_heap = wasm_runtime_create_shared_heap(&args); - if (!shared_heap) { - FAIL() << "Create preallocated shared heap failed.\n"; - } + create_test_shared_heap(preallocated_buf, BUF_SIZE, &shared_heap); - memset(&args, 0, sizeof(args)); - args.pre_allocated_addr = preallocated_buf2; - args.size = BUF_SIZE; - shared_heap2 = wasm_runtime_create_shared_heap(&args); - if (!shared_heap2) { - FAIL() << "Create preallocated shared heap failed.\n"; - } + /* app addr for shared heap */ + start1 = UINT32_MAX - BUF_SIZE + 1; + end1 = UINT32_MAX; - shared_heap_chain = - wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); - if (!shared_heap_chain) { - FAIL() << "Create shared heap chain failed.\n"; - } + /* try to rmw an u16, first u8 is in the first shared heap and second u8 is + * in the second shared heap, will be seen as oob */ + argv[0] = end1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test.wasm", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); + + argv[0] = start1 - 1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test.aot", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); + + argv[0] = end1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test.aot", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw_oob) +{ + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); /* app addr for shared heap */ - start1 = 0xFFFFFFFF - 2 * BUF_SIZE + 1; - end1 = 0xFFFFFFFF - BUF_SIZE; - start2 = 0xFFFFFFFF - BUF_SIZE + 1; - end2 = 0xFFFFFFFF; + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; /* try to rmw an u16, first u8 is in the first shared heap and second u8 is * in the second shared heap, will be seen as oob */ - argv[0] = end1; + argv[0] = end2; argv[1] = 12025; EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test.wasm", "read_modify_write_16", 2, argv), "Exception: out of bounds memory access"); - /* TODO: test aot when chain is supported in AOT */ - /*argv[0] = end1; + argv[0] = end1; argv[1] = 12025; - EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test.wasm", + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_chain.aot", "read_modify_write_16", 2, argv), - "Exception: out of bounds memory access");*/ + "Exception: out of bounds memory access"); } +#if WASM_ENABLE_MEMORY64 != 0 +TEST_F(shared_heap_test, test_shared_heap_chain_memory64_rmw) +{ + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint64 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT64_MAX - 2 * BUF_SIZE + 1; + end1 = UINT64_MAX - BUF_SIZE; + start2 = UINT64_MAX - BUF_SIZE + 1; + end2 = UINT64_MAX; + + /* shared heap 1 */ + PUT_I64_TO_ADDR(argv, end1); + argv[2] = 101; + test_shared_heap(shared_heap_chain, "test64.wasm", "read_modify_write_8", 3, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + + /* shared heap 2 */ + PUT_I64_TO_ADDR(argv, start2); + argv[2] = 129; + test_shared_heap(shared_heap_chain, "test64.wasm", "read_modify_write_8", 3, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf2[0], 129); + + PUT_I64_TO_ADDR(argv, start1); + argv[2] = 98; + test_shared_heap(shared_heap_chain, "test64_chain.aot", + "read_modify_write_8", 3, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[0], 98); + + PUT_I64_TO_ADDR(argv, end2); + argv[2] = 81; + test_shared_heap(shared_heap_chain, "test64_chain.aot", + "read_modify_write_8", 3, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 81); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_memory64_rmw_oob) +{ + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + uint64 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT64_MAX - 2 * BUF_SIZE + 1; + end1 = UINT64_MAX - BUF_SIZE; + start2 = UINT64_MAX - BUF_SIZE + 1; + end2 = UINT64_MAX; + + /* try to rmw an u16, first u8 is in the first shared heap and second u8 is + * in the second shared heap, will be seen as oob */ + PUT_I64_TO_ADDR(argv, end1); + argv[2] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test64.wasm", + "read_modify_write_16", 3, argv), + "Exception: out of bounds memory access"); + + PUT_I64_TO_ADDR(argv, end1); + argv[2] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test64_chain.aot", + "read_modify_write_16", 3, argv), + "Exception: out of bounds memory access"); +} +#endif + #ifndef native_function /* clang-format off */ #define native_function(func_name, signature) \ @@ -378,6 +690,9 @@ TEST_F(shared_heap_test, test_addr_conv) test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 0, argv); EXPECT_EQ(1, argv[0]); + + test_shared_heap(shared_heap, "test_addr_conv_chain.aot", "test", 0, argv); + EXPECT_EQ(1, argv[0]); } TEST_F(shared_heap_test, test_addr_conv_pre_allocated_oob) @@ -412,6 +727,12 @@ TEST_F(shared_heap_test, test_addr_conv_pre_allocated_oob) EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test_addr_conv.aot", "test_preallocated", 1, argv), "Exception: out of bounds memory access"); + + argv[0] = app_addr; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, + "test_addr_conv_chain.aot", + "test_preallocated", 1, argv), + "Exception: out of bounds memory access"); } TEST_F(shared_heap_test, test_shared_heap_chain) @@ -453,9 +774,8 @@ TEST_F(shared_heap_test, test_shared_heap_chain) test_shared_heap(shared_heap_chain, "test_addr_conv.wasm", "test", 0, argv); EXPECT_EQ(1, argv[0]); - /* TODO: test aot when chain is supported in AOT */ - /*test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 1, argv); - EXPECT_EQ(1, argv[0]);*/ + test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 0, argv); + EXPECT_EQ(1, argv[0]); } TEST_F(shared_heap_test, test_shared_heap_chain_create_fail) @@ -666,14 +986,15 @@ TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv) "test_preallocated", 1, argv); EXPECT_EQ(1, argv[0]); - /* TODO: test aot when chain is supported in AOT */ - /*argv[0] = 0xFFFFFFFF; - test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 1, argv); + argv[0] = 0xFFFFFFFF; + test_shared_heap(shared_heap, "test_addr_conv_chain.aot", + "test_preallocated", 1, argv); EXPECT_EQ(1, argv[0]); argv[0] = 0xFFFFF000; - test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 1, argv); - EXPECT_EQ(1, argv[0]); */ + test_shared_heap(shared_heap, "test_addr_conv_chain.aot", + "test_preallocated", 1, argv); + EXPECT_EQ(1, argv[0]); } TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv_oob) @@ -719,10 +1040,10 @@ TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv_oob) "test_preallocated", 1, argv), "Exception: out of bounds memory access"); - /* TODO: test aot when chain is supported in AOT */ - /*argv[0] = 0xFFFFFFFF - BUF_SIZE - 4096; + /* test aot */ + argv[0] = 0xFFFFFFFF - BUF_SIZE - 4096; EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, - "test_addr_conv.aot", + "test_addr_conv_chain.aot", "test_preallocated", 1, argv), - "Exception: out of bounds memory access");*/ + "Exception: out of bounds memory access"); } diff --git a/tests/unit/shared-heap/wasm-apps/CMakeLists.txt b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt index 097f66ae5d..985cf18aeb 100644 --- a/tests/unit/shared-heap/wasm-apps/CMakeLists.txt +++ b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt @@ -29,44 +29,81 @@ set(CMAKE_EXE_LINKER_FLAGS -Wl,--allow-undefined" ) -add_executable(test.wasm test.c) -target_link_libraries(test.wasm) +if (WAMR_BUILD_TARGET STREQUAL "X86_32") + set (WAMR_COMPILER_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-heap --target=i386) + set (WAMR_COMPILER_CHAIN_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-chain --target=i386) +else () + set (WAMR_COMPILER_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-heap) + set (WAMR_COMPILER_CHAIN_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-chain) +endif () -add_custom_command(TARGET test.wasm POST_BUILD +function(copy_wasm TARGET_NAME) + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test.wasm - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test.wasm to the same directory of google test" - ) + ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} + ${CMAKE_CURRENT_BINARY_DIR}/../ + COMMENT "Copy ${TARGET_NAME} to the same directory of google test" + ) +endfunction() + +function(compile_and_copy_aot_from TARGET_NAME) + string(REPLACE ".wasm" ".aot" AOT_TARGET ${TARGET_NAME}) + string(REPLACE ".wasm" "_chain.aot" AOT_CHAIN_TARGET ${TARGET_NAME}) -add_custom_command(TARGET test.wasm POST_BUILD - COMMAND ${WAMRC_ROOT_DIR}/wamrc --opt-level=0 --enable-shared-heap --bounds-checks=1 - -o - test.aot - test.wasm + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} + -o ${AOT_TARGET} + ${TARGET_NAME} COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test.aot - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test.aot to the same directory of google test" - ) + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../ + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_CHAIN_FLAGS} + -o ${AOT_CHAIN_TARGET} + ${TARGET_NAME} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_CHAIN_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../ + COMMENT "Compile and copy ${AOT_TARGET} to the same directory of google test" + ) +endfunction() -add_executable(test_addr_conv.wasm test_addr_conv.c) +add_executable(test.wasm test.c) target_link_libraries(test.wasm) +copy_wasm(test.wasm) +compile_and_copy_aot_from(test.wasm) -add_custom_command(TARGET test_addr_conv.wasm POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test_addr_conv.wasm - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test_addr_conv.wasm to the same directory of google test" - ) +add_executable(test_addr_conv.wasm test_addr_conv.c) +target_link_libraries(test_addr_conv.wasm) +copy_wasm(test_addr_conv.wasm) +compile_and_copy_aot_from(test_addr_conv.wasm) -add_custom_command(TARGET test_addr_conv.wasm POST_BUILD - COMMAND ${WAMRC_ROOT_DIR}/wamrc --opt-level=0 --enable-shared-heap --bounds-checks=1 - -o - test_addr_conv.aot - test_addr_conv.wasm - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test_addr_conv.aot - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test_addr_conv.aot to the same directory of google test" - ) +# copy and compile aot for bulk memory test +set(SOURCE_WASM ${CMAKE_CURRENT_SOURCE_DIR}/bulk-memory/test_bulk_memory.wasm) +set(BUILD_WASM ${CMAKE_CURRENT_BINARY_DIR}/../test_bulk_memory.wasm) +set(OUTPUT_AOT ${CMAKE_CURRENT_BINARY_DIR}/../test_bulk_memory.aot) +set(OUTPUT_CHAIN_AOT ${CMAKE_CURRENT_BINARY_DIR}/../test_bulk_memory_chain.aot) + +add_custom_command( + OUTPUT ${BUILD_WASM} + COMMAND ${CMAKE_COMMAND} -E copy + ${SOURCE_WASM} + ${BUILD_WASM} + DEPENDS ${SOURCE_WASM} + COMMENT "Copying bulk memory WASM to build directory" +) + +add_custom_command( + OUTPUT ${OUTPUT_AOT} + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} + -o ${OUTPUT_AOT} + ${BUILD_WASM} + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_CHAIN_FLAGS} + -o ${OUTPUT_CHAIN_AOT} + ${BUILD_WASM} + DEPENDS ${BUILD_WASM} + COMMENT "Compiling bulk memory AOT from copied WASM" +) + +add_custom_target(compile_bulk_memory_aot ALL + DEPENDS ${OUTPUT_AOT} +) diff --git a/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wasm b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wasm new file mode 100644 index 0000000000..eb3d60be3d Binary files /dev/null and b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wasm differ diff --git a/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat new file mode 100644 index 0000000000..e7a0c684d1 --- /dev/null +++ b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat @@ -0,0 +1,12 @@ +(module + (memory 1) + + (func $memory_fill_test (param $dst i32) (param $val i32) (param $len i32) + local.get $dst + local.get $val + local.get $len + memory.fill + ) + + (export "memory_fill_test" (func $memory_fill_test)) +) diff --git a/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt b/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt new file mode 100644 index 0000000000..a82788b586 --- /dev/null +++ b/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt @@ -0,0 +1,68 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) +project(wasm-apps-wasm64) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../../..) +set(WAMRC_ROOT_DIR ${WAMR_ROOT_DIR}/wamr-compiler/build) + +set(CMAKE_SYSTEM_PROCESSOR wasm64) +set(CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) + +if (NOT DEFINED WASI_SDK_DIR) + set(WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set(CMAKE_C_FLAGS "-nostdlib -pthread -Qunused-arguments") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -z stack-size=8192 -nostdlib -O0 --target=wasm64") +set(CMAKE_C_COMPILER_TARGET "wasm64") +set(CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") + +set(DEFINED_SYMBOLS + "${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") + +set(CMAKE_EXE_LINKER_FLAGS + "-Wl,--no-entry \ + -Wl,--initial-memory=65536 \ + -Wl,--export-all \ + -Wl,--allow-undefined" + ) + +set (WAMR_COMPILER_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-heap) +set (WAMR_COMPILER_CHAIN_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-chain) + +function(copy_wasm TARGET_NAME) + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} + ${CMAKE_CURRENT_BINARY_DIR}/../../ + COMMENT "Copy ${TARGET_NAME} to the same directory of google test" + ) +endfunction() + +function(compile_and_copy_aot_from TARGET_NAME) + string(REPLACE ".wasm" ".aot" AOT_TARGET ${TARGET_NAME}) + string(REPLACE ".wasm" "_chain.aot" AOT_CHAIN_TARGET ${TARGET_NAME}) + + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} + -o ${AOT_TARGET} + ${TARGET_NAME} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../../ + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_CHAIN_FLAGS} + -o ${AOT_CHAIN_TARGET} + ${TARGET_NAME} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_CHAIN_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../../ + COMMENT "Compile and copy ${AOT_TARGET} ${AOT_CHAIN_TARGET} to the same directory of google test" + ) +endfunction() + +add_executable(test64.wasm ../test.c) +target_link_libraries(test64.wasm) +copy_wasm(test64.wasm) +compile_and_copy_aot_from(test64.wasm) diff --git a/tests/unit/shared-heap/wasm-apps/test.c b/tests/unit/shared-heap/wasm-apps/test.c index b0bfc17b36..66df21c1b7 100644 --- a/tests/unit/shared-heap/wasm-apps/test.c +++ b/tests/unit/shared-heap/wasm-apps/test.c @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#include +#define NULL 0 extern void * shared_heap_malloc(int size); diff --git a/tests/unit/shared-heap/wasm-apps/test_addr_conv.c b/tests/unit/shared-heap/wasm-apps/test_addr_conv.c index 5e64526a04..a00e68c9ce 100644 --- a/tests/unit/shared-heap/wasm-apps/test_addr_conv.c +++ b/tests/unit/shared-heap/wasm-apps/test_addr_conv.c @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#include +#define NULL 0 extern void * shared_heap_malloc(int size); diff --git a/wamr-compiler/CMakeLists.txt b/wamr-compiler/CMakeLists.txt index 0ce6473944..68648b84b7 100644 --- a/wamr-compiler/CMakeLists.txt +++ b/wamr-compiler/CMakeLists.txt @@ -284,6 +284,7 @@ include (${IWASM_DIR}/interpreter/iwasm_interp.cmake) include (${IWASM_DIR}/aot/iwasm_aot.cmake) include (${IWASM_DIR}/compilation/iwasm_compl.cmake) include (${PROJECT_SOURCE_DIR}/../build-scripts/version.cmake) +include (${IWASM_DIR}/libraries/shared-heap/shared_heap.cmake) if (WAMR_BUILD_LIBC_BUILTIN EQUAL 1) include (${IWASM_DIR}/libraries/libc-builtin/libc_builtin.cmake) @@ -366,6 +367,7 @@ add_library (vmlib ${LIBC_WASI_SOURCE} ${LIB_PTHREAD_SOURCE} ${LIB_WASI_THREADS_SOURCE} + ${LIB_SHARED_HEAP_SOURCE} ${IWASM_COMMON_SOURCE} ${IWASM_INTERP_SOURCE} ${IWASM_AOT_SOURCE} diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index f74b2fb151..6b7dbaad48 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -210,7 +210,9 @@ print_help() printf(" --enable-linux-perf Enable linux perf support\n"); #endif printf(" --mllvm=