From 9747e0023d9c58337ab31f3fb7df64440426ffdf Mon Sep 17 00:00:00 2001 From: TL Date: Wed, 9 Apr 2025 18:07:36 +0800 Subject: [PATCH 01/29] shared heap enhancement: modify memory check for aot_check_memory_overflow to accomodate shared heap chain --- core/iwasm/compilation/aot_emit_memory.c | 402 +++++++++++++++++++---- core/iwasm/compilation/aot_llvm.c | 87 ++--- core/iwasm/compilation/aot_llvm.h | 21 ++ 3 files changed, 391 insertions(+), 119 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index e685fccd92..2d968e701b 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -10,6 +10,40 @@ #include "aot_intrinsic.h" #include "aot_emit_control.h" +#define BUILD_IS_NOT_NULL(value, res, name) \ + do { \ + if (!(res = LLVMBuildIsNotNull(comp_ctx->builder, value, name))) { \ + aot_set_last_error("llvm build is not null failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_BR(llvm_block) \ + do { \ + if (!LLVMBuildBr(comp_ctx->builder, llvm_block)) { \ + aot_set_last_error("llvm build br failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_COND_BR(value_if, block_then, block_else) \ + do { \ + if (!LLVMBuildCondBr(comp_ctx->builder, value_if, block_then, \ + block_else)) { \ + aot_set_last_error("llvm build cond br failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_TRUNC(value, data_type) \ + do { \ + if (!(value = LLVMBuildTrunc(comp_ctx->builder, value, data_type, \ + "val_trunc"))) { \ + aot_set_last_error("llvm build trunc failed."); \ + goto fail; \ + } \ + } while (0) + #define BUILD_ICMP(op, left, right, res, name) \ do { \ if (!(res = \ @@ -111,6 +145,142 @@ ffs(int n) static LLVMValueRef get_memory_curr_page_count(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); +uint32 +get_module_inst_extra_offset(AOTCompContext *comp_ctx); + +#define BUILD_FIELD_PTR(ptr, offset, field, name) \ + do { \ + if (!(field_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, \ + ptr, &offset, 1, name))) { \ + aot_set_last_error("llvm build inbounds gep failed"); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_LOAD_PTR(ptr, data_type, res) \ + do { \ + if (!(res = LLVMBuildLoad2(comp_ctx->builder, data_type, ptr, \ + "load_ptr"))) { \ + aot_set_last_error("llvm build load failed"); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_STORE_PTR(ptr, value) \ + do { \ + LLVMValueRef res; \ + if (!(res = LLVMBuildStore(comp_ctx->builder, value, ptr))) { \ + aot_set_last_error("llvm build store failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_GET_SHARED_HEAP_FIELD(shared_heap_p, field, data_type, res) \ + do { \ + offset_u32 = offsetof(WASMSharedHeap, field); \ + field_offset = I32_CONST(offset_u32); \ + CHECK_LLVM_CONST(field_offset); \ + \ + BUILD_FIELD_PTR(shared_heap_p, field_offset, field_p, \ + "shared_heap" #field); \ + BUILD_LOAD_PTR(field_p, data_type, res); \ + } while (0) + +#define BUILD_GET_SHARED_HEAP_START(shared_heap_p, res) \ + do { \ + if (is_memory64) { \ + offset_u32 = offsetof(WASMSharedHeap, start_off_mem64); \ + } \ + else { \ + offset_u32 = offsetof(WASMSharedHeap, start_off_mem32); \ + } \ + field_offset = I32_CONST(offset_u32); \ + CHECK_LLVM_CONST(field_offset); \ + \ + BUILD_FIELD_PTR(shared_heap_p, field_offset, field_p, \ + "shared_heap_start_off"); \ + BUILD_LOAD_PTR(field_p, is_target_64bit ? I64_TYPE : I32_TYPE, res); \ + } while (0) + +#define BUILD_SET_MODULE_EXTRA_FIELD(ptr_type, field, value) \ + do { \ + get_module_extra_field_offset(field); \ + field_offset = I32_CONST(offset_u32); \ + CHECK_LLVM_CONST(field_offset); \ + \ + BUILD_FIELD_PTR(shared_heap_p, field_offset, field_p, #field); \ + if (!(field_p = LLVMBuildBitCast(comp_ctx->builder, field_p, ptr_type, \ + "module_extra" #field "_p"))) { \ + aot_set_last_error("llvm build bit cast failed."); \ + goto fail; \ + } \ + BUILD_STORE_PTR(field_p, value); \ + } while (0) + +static bool +aot_update_last_used_shared_heap(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + LLVMValueRef shared_heap_p, + LLVMValueRef shared_heap_start_off, + LLVMValueRef shared_heap_size, + bool is_target_64bit) +{ + /* Update last used shared heap info in func_ctx and module_extra. + * 1. shared_heap_start_off 2. shared_heap_end_off + * 3. shared_heap_base_addr_adj */ + uint32 offset_u32; + LLVMValueRef tmp_res, field_p, field_offset, shared_heap_end_off, base_addr, + shared_heap_base_addr_adj; + LLVMTypeRef shared_heap_off_type = is_target_64bit ? I64_TYPE : I32_TYPE, + shared_heap_off_ptr_type = + is_target_64bit ? INT64_PTR_TYPE : INT32_PTR_TYPE; + + /* shared_heap_end_off = shared_heap_start_off + shared_heap_size - 1 */ + BUILD_OP(Add, shared_heap_start_off, + is_target_64bit ? I64_NEG_ONE : I32_NEG_ONE, tmp_res, + "shared_heap_start_off_minus_one"); + BUILD_OP(Add, tmp_res, shared_heap_size, shared_heap_end_off, + "shared_heap_end_off"); + /* shared_heap_base_addr_adj = base_addr - shared_heap_start_off */ + BUILD_GET_SHARED_HEAP_FIELD(shared_heap_p, base_addr, INT8_PTR_TYPE, + base_addr); + if (!(base_addr = LLVMBuildPtrToInt(comp_ctx->builder, base_addr, + shared_heap_off_type, "base_addr"))) { + aot_set_last_error("llvm build ptr to int failed"); + goto fail; + } + BUILD_OP(Sub, base_addr, shared_heap_start_off, shared_heap_base_addr_adj, + "shared_heap_base_addr_adj"); + if (!(shared_heap_base_addr_adj = LLVMBuildIntToPtr( + comp_ctx->builder, shared_heap_base_addr_adj, + shared_heap_off_ptr_type, "shared_heap_base_addr_adj_ptr"))) { + aot_set_last_error("llvm build int to ptr failed."); + goto fail; + } + + BUILD_SET_MODULE_EXTRA_FIELD(shared_heap_off_ptr_type, + shared_heap_start_off, shared_heap_start_off); + BUILD_SET_MODULE_EXTRA_FIELD(shared_heap_off_ptr_type, shared_heap_end_off, + shared_heap_end_off); + BUILD_SET_MODULE_EXTRA_FIELD(INTPTR_T_PTR_TYPE, shared_heap_base_addr_adj, + shared_heap_base_addr_adj); + + BUILD_STORE_PTR(func_ctx->shared_heap_start_off, shared_heap_start_off); + BUILD_STORE_PTR(func_ctx->shared_heap_end_off, shared_heap_end_off); + BUILD_STORE_PTR(func_ctx->shared_heap_base_addr_adj, + shared_heap_base_addr_adj); + + return true; +fail: + return false; +} + +static bool +aot_check_shared_heap_memory_overflow(AOTCompContext *comp_ctx, + AOTFuncContext *func_ctx, + mem_offset_t offset, uint32 bytes, + bool enable_segue); + LLVMValueRef aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, mem_offset_t offset, uint32 bytes, bool enable_segue, @@ -272,9 +442,19 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } + /* The overflow check needs to be done under following conditions: + * 1. In 64-bit target, offset and addr will be extended to 64-bit + * 1.1 offset + addr can overflow when it's memory64 + * 1.2 no overflow when it's memory32 + * 2. In 32-bit target, offset and addr will be 32-bit + * 2.1 offset + addr can overflow when it's memory32 + * And the goal is to detect it happens ASAP + */ + /* offset1 = offset + addr; */ BUILD_OP(Add, offset_const, addr, offset1, "offset1"); + /* 1.1 offset + addr can overflow when it's memory64 */ if (is_memory64 && comp_ctx->enable_bound_check) { /* Check whether integer overflow occurs in offset + addr */ LLVMBasicBlockRef check_integer_overflow_end; @@ -292,20 +472,49 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { - LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem; - LLVMValueRef is_in_shared_heap, shared_heap_check_bound = NULL; - - /* Add basic blocks */ - ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap"); + LLVMBasicBlockRef app_addr_in_cache_shared_heap, + app_addr_in_shared_heap_chain, app_addr_in_linear_mem, loopEntry, + loopCond, loopBody, loopExit, check_valid_shared_heap; + /* Shared heap offset will be i64 on 64-bit platform and i32 on 32-bit + * platform */ + LLVMValueRef shared_heap_start_off, shared_heap_check_bound, + shared_heap_p, field_p, field_offset, length, shared_heap_size; + uint32 offset_u32; + + /* Add basic blocks + * +-------------------------------------+ + * | current block | + * +-------------------------------------+ + * / | \ + * / | \ + * / | \ + * not in shared heap chain | in shared heap chain + * | yes | + * v | v + * +----------------+ +-------------------+ +--------------------+ + * | in_linear_mem | |in_cache_share_heap| |in_shared_heap_chain| + * +----------------+ +-------------------+ +--------------------+ + * \ | / + * \ | / + * \ | / + * v v v + * +---------------------------------------+ + * | block_maddr_phi | + * +---------------------------------------+ + */ + ADD_BASIC_BLOCK(app_addr_in_cache_shared_heap, + "app_addr_in_cache_shared_heap"); + ADD_BASIC_BLOCK(app_addr_in_shared_heap_chain, + "app_addr_in_shared_heap_chain"); ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); - LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr); - LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, - app_addr_in_shared_heap); + LLVMMoveBasicBlockAfter(app_addr_in_cache_shared_heap, block_curr); + LLVMMoveBasicBlockAfter(app_addr_in_shared_heap_chain, block_curr); + LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, block_curr); LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem); - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + SET_BUILD_POS(block_maddr_phi); if (!(maddr_phi = LLVMBuildPhi(comp_ctx->builder, enable_segue ? INT8_PTR_TYPE_GS : INT8_PTR_TYPE, @@ -314,8 +523,9 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); + SET_BUILD_POS(block_curr); + /* 2.1 offset + addr can overflow when it's memory32 */ if (!is_target_64bit) { /* Check whether integer overflow occurs in addr + offset */ LLVMBasicBlockRef check_integer_overflow_end; @@ -332,52 +542,131 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, SET_BUILD_POS(check_integer_overflow_end); } + /* Early stop when no shared heap attached */ + BUILD_IS_NOT_NULL(func_ctx->shared_heap, cmp, "has_shared_heap"); + BUILD_COND_BR(cmp, app_addr_in_shared_heap_chain, + app_addr_in_linear_mem); + + /* Check whether the bytes to access are in shared heap chain */ + SET_BUILD_POS(app_addr_in_shared_heap_chain); + /* Add loop basic blocks*/ + ADD_BASIC_BLOCK(loopEntry, "loop_entry"); + ADD_BASIC_BLOCK(loopCond, "loop_Cond"); + ADD_BASIC_BLOCK(loopBody, "loop_body"); + ADD_BASIC_BLOCK(loopExit, "loop_exit"); + LLVMMoveBasicBlockAfter(loopEntry, app_addr_in_shared_heap_chain); + LLVMMoveBasicBlockAfter(loopCond, app_addr_in_shared_heap_chain); + LLVMMoveBasicBlockAfter(loopBody, app_addr_in_shared_heap_chain); + LLVMMoveBasicBlockAfter(loopExit, app_addr_in_shared_heap_chain); + + /* Early stop for app addr not in shared heap chain at all */ + BUILD_GET_SHARED_HEAP_START(func_ctx->shared_heap, + shared_heap_start_off); shared_heap_check_bound = is_memory64 ? I64_CONST(UINT64_MAX - bytes + 1) - : (comp_ctx->pointer_size == sizeof(uint64) - ? I64_CONST(UINT32_MAX - bytes + 1) - : I32_CONST(UINT32_MAX - bytes + 1)); + : (is_target_64bit ? I64_CONST(UINT32_MAX - bytes + 1) + : I32_CONST(UINT32_MAX - bytes + 1)); CHECK_LLVM_CONST(shared_heap_check_bound); - - /* Check whether the bytes to access are in shared heap */ - if (!comp_ctx->enable_bound_check) { - /* Use IntUGT but not IntUGE to compare, since (1) in the ems - memory allocator, the hmu node includes hmu header and hmu - memory, only the latter is returned to the caller as the - allocated memory, the hmu header isn't returned so the - first byte of the shared heap won't be accessed, (2) using - IntUGT gets better performance than IntUGE in some cases */ - BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off, - is_in_shared_heap, "is_in_shared_heap"); - /* We don't check the shared heap's upper boundary if boundary - check isn't enabled, the runtime may also use the guard pages - of shared heap to check the boundary if hardware boundary - check feature is enabled. */ - } - else { - /* Use IntUGT but not IntUGE to compare, same as above */ - BUILD_ICMP(LLVMIntUGT, offset1, func_ctx->shared_heap_start_off, - cmp1, "cmp1"); - /* Check the shared heap's upper boundary if boundary check is - enabled */ - BUILD_ICMP(LLVMIntULE, offset1, shared_heap_check_bound, cmp2, - "cmp2"); - BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap"); - } - - if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap, - app_addr_in_shared_heap, app_addr_in_linear_mem)) { - aot_set_last_error("llvm build cond br failed"); + /* TODO: potential optimization + Before shared heap chain: + Use IntUGT but not IntUGE to compare, since (1) in the ems + memory allocator, the hmu node includes hmu header and hmu + memory, only the latter is returned to the caller as the + allocated memory, the hmu header isn't returned so the + first byte of the shared heap won't be accessed, (2) using + IntUGT gets better performance than IntUGE in some cases + + And We don't check the shared heap's upper boundary if boundary + check isn't enabled, the runtime may also use the guard pages + of shared heap to check the boundary if hardware boundary + check feature is enabled. + + After shared heap chain: + Use IntUGE to compare since pre-allocated shared heap can access + the first byte. + Need to check upper boundary since there could be multiple shared + heap. + */ + BUILD_ICMP(LLVMIntUGE, offset1, shared_heap_start_off, cmp, + "cmp_shared_heap_chain_start"); + BUILD_ICMP(LLVMIntULE, offset1, shared_heap_check_bound, cmp1, + "cmp_shared_heap_chain_end"); + BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); + BUILD_COND_BR(cmp2, loopEntry, app_addr_in_linear_mem); + + /* Check whether the bytes to access are in cache shared heap */ + SET_BUILD_POS(loopEntry); + BUILD_ICMP(LLVMIntUGE, offset1, func_ctx->shared_heap_start_off, cmp, + "cmp_cache_shared_heap_start"); + length = is_target_64bit ? I64_CONST(bytes - 1) : I32_CONST(bytes - 1); + CHECK_LLVM_CONST(length); + BUILD_OP(Sub, func_ctx->shared_heap_end_off, length, + shared_heap_check_bound, "cache_shared_heap_end"); + BUILD_ICMP(LLVMIntULE, offset1, shared_heap_check_bound, cmp1, + "cmp_cache_shared_heap_end"); + BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); + BUILD_COND_BR(cmp2, app_addr_in_cache_shared_heap, loopCond); + + /* Check which shared heap in the shared heap chain the addr resides */ + SET_BUILD_POS(loopCond); + /* cur = heap; cur; cur = cur->chain_next */ + shared_heap_p = func_ctx->shared_heap; + BUILD_GET_SHARED_HEAP_FIELD(shared_heap_p, chain_next, INT8_PTR_TYPE, + shared_heap_p); + BUILD_IS_NOT_NULL(shared_heap_p, cmp, "has_shared_heap"); + BUILD_COND_BR(cmp, loopBody, loopExit); + + /* loop body */ + BUILD_GET_SHARED_HEAP_START(shared_heap_p, shared_heap_start_off); + /* app_offset >= shared_heap_start */ + BUILD_ICMP(LLVMIntUGE, offset1, shared_heap_start_off, cmp, + "cmp_shared_heap_start"); + /* app_offset <= shared_heap_start + cur->size - bytes */ + BUILD_GET_SHARED_HEAP_FIELD(shared_heap_p, size, I64_TYPE, + shared_heap_size); + if (!is_target_64bit) { + BUILD_TRUNC(shared_heap_size, I32_TYPE); + } + BUILD_OP(Sub, shared_heap_size, + is_target_64bit ? I64_CONST(bytes) : I32_CONST(bytes), + shared_heap_check_bound, "shared_heap_check_bound1"); + BUILD_OP(Add, shared_heap_start_off, shared_heap_check_bound, + shared_heap_check_bound, "shared_heap_end_bound2"); + BUILD_ICMP(LLVMIntULE, offset1, shared_heap_check_bound, cmp1, + "cmp_shared_heap_end"); + BUILD_OP(And, cmp, cmp1, cmp2, "is_in_shared_heap"); + BUILD_COND_BR(cmp2, loopExit, loopCond); + + /* loop exit, if the shared_heap_p is valid, we find the shared heap + * this app addr is in, otherwise it's oom */ + SET_BUILD_POS(loopExit); + BUILD_IS_NOT_NULL(shared_heap_p, cmp, "has_shared_heap"); + + ADD_BASIC_BLOCK(check_valid_shared_heap, "check_valid_shared_heap"); + LLVMMoveBasicBlockAfter(check_valid_shared_heap, loopExit); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, + check_valid_shared_heap)) { goto fail; } + SET_BUILD_POS(check_valid_shared_heap); - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap); + /* Update last accessed shared heap, the shared_heap_size and + * shared_heap_start_off is already prepared in loop body, not only need + * to prepare shared_heap_end_off */ + if (!aot_update_last_used_shared_heap(comp_ctx, func_ctx, shared_heap_p, + shared_heap_start_off, + shared_heap_size, is_memory64)) + goto fail; + LLVMMoveBasicBlockAfter(app_addr_in_cache_shared_heap, + check_valid_shared_heap); - /* Get native address inside shared heap */ - if (!(maddr = - LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->shared_heap_base_addr_adj, - &offset1, 1, "maddr_shared_heap"))) { + /* Get native address inside cache shared heap */ + SET_BUILD_POS(app_addr_in_cache_shared_heap); + if (!(maddr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + func_ctx->shared_heap_base_addr_adj, + &offset1, 1, + "maddr_cache_shared_heap"))) { aot_set_last_error("llvm build inbounds gep failed"); goto fail; } @@ -407,7 +696,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } - LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1); + LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_cache_shared_heap, 1); if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { aot_set_last_error("llvm build br failed"); @@ -453,6 +742,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); } else { + /* 2.1 offset + addr can overflow when it's memory32 */ if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { /* Check integer overflow has been checked above */ BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); @@ -516,7 +806,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, aot_set_last_error("llvm build br failed"); goto fail; } - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + SET_BUILD_POS(block_maddr_phi); return maddr_phi; } else @@ -544,15 +834,6 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMSetAlignment(value, known_align); \ } while (0) -#define BUILD_TRUNC(value, data_type) \ - do { \ - if (!(value = LLVMBuildTrunc(comp_ctx->builder, value, data_type, \ - "val_trunc"))) { \ - aot_set_last_error("llvm build trunc failed."); \ - goto fail; \ - } \ - } while (0) - #define BUILD_STORE() \ do { \ LLVMValueRef res; \ @@ -1248,6 +1529,7 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + /* TODO: shared heap chain here */ LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem; LLVMValueRef shared_heap_start_off, shared_heap_check_bound; LLVMValueRef max_offset, cmp1, cmp2, is_in_shared_heap; diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index c1708e3f9d..1f92d26a11 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1517,69 +1517,38 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; } +#define LOAD_MODULE_EXTRA_FIELD(field, type) \ + do { \ + get_module_extra_field_offset(field); \ + offset = I32_CONST(offset_u32); \ + CHECK_LLVM_CONST(offset); \ + if (!(field_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, \ + func_ctx->aot_inst, &offset, 1, \ + #field "_p"))) { \ + aot_set_last_error("llvm build inbounds gep failed"); \ + return false; \ + } \ + if (!(func_ctx->field = \ + LLVMBuildLoad2(comp_ctx->builder, type, field_p, #field))) { \ + aot_set_last_error("llvm build load failed"); \ + return false; \ + } \ + } while (0) + static bool create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { - LLVMValueRef offset, base_addr_p, start_off_p, cmp; + LLVMValueRef offset, field_p; uint32 offset_u32; - /* Load aot_inst->e->shared_heap_base_addr_adj */ - offset_u32 = get_module_inst_extra_offset(comp_ctx); -#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 - if (comp_ctx->is_jit_mode) - offset_u32 += - offsetof(WASMModuleInstanceExtra, shared_heap_base_addr_adj); - else -#endif - offset_u32 += - offsetof(AOTModuleInstanceExtra, shared_heap_base_addr_adj); - offset = I32_CONST(offset_u32); - CHECK_LLVM_CONST(offset); - - if (!(base_addr_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->aot_inst, &offset, 1, - "shared_heap_base_addr_adj_p"))) { - aot_set_last_error("llvm build inbounds gep failed"); - return false; - } - if (!(func_ctx->shared_heap_base_addr_adj = - LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, base_addr_p, - "shared_heap_base_addr_adj"))) { - aot_set_last_error("llvm build load failed"); - return false; - } - - /* Load aot_inst->e->shared_heap_start_off */ - offset_u32 = get_module_inst_extra_offset(comp_ctx); -#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 - if (comp_ctx->is_jit_mode) - offset_u32 += offsetof(WASMModuleInstanceExtra, shared_heap_start_off); - else -#endif - offset_u32 += offsetof(AOTModuleInstanceExtra, shared_heap_start_off); - offset = I32_CONST(offset_u32); - CHECK_LLVM_CONST(offset); - - if (!(start_off_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->aot_inst, &offset, 1, - "shared_heap_start_off_p"))) { - aot_set_last_error("llvm build inbounds gep failed"); - return false; - } - if (!(func_ctx->shared_heap_start_off = LLVMBuildLoad2( - comp_ctx->builder, - comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE, - start_off_p, "shared_heap_start_off"))) { - aot_set_last_error("llvm build load failed"); - return false; - } - - if (!(cmp = LLVMBuildIsNotNull(comp_ctx->builder, - func_ctx->shared_heap_base_addr_adj, - "has_shared_heap"))) { - aot_set_last_error("llvm build is not null failed"); - return false; - } + LOAD_MODULE_EXTRA_FIELD(shared_heap_base_addr_adj, INT8_PTR_TYPE); + LOAD_MODULE_EXTRA_FIELD( + shared_heap_start_off, + comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE); + LOAD_MODULE_EXTRA_FIELD( + shared_heap_end_off, + comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE); + LOAD_MODULE_EXTRA_FIELD(shared_heap, INT8_PTR_TYPE); return true; fail: @@ -2438,7 +2407,7 @@ jit_stack_size_callback(void *user_data, const char *name, size_t namelen, stack_consumption_to_call_wrapped_func = musttail ? 0 : aot_estimate_stack_usage_for_function_call( - comp_ctx, func_ctx->aot_func->func_type); + comp_ctx, func_ctx->aot_func->func_type); LOG_VERBOSE("func %.*s stack %u + %zu + %u", (int)namelen, name, stack_consumption_to_call_wrapped_func, stack_size, call_size); diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index 6b1233c39f..896a9caea4 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -77,6 +77,23 @@ extern "C" { #undef DUMP_MODULE #endif +#if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 +#define get_module_extra_field_offset(field) \ + do { \ + offset_u32 = get_module_inst_extra_offset(comp_ctx); \ + if (comp_ctx->is_jit_mode) \ + offset_u32 += offsetof(WASMModuleInstanceExtra, field); \ + else \ + offset_u32 += offsetof(AOTModuleInstanceExtra, field); \ + } while (0) +#else +#define get_module_extra_field_offset(field) \ + do { \ + offset_u32 = get_module_inst_extra_offset(comp_ctx); \ + offset_u32 += offsetof(AOTModuleInstanceExtra, field); \ + } while (0) +#endif + struct AOTValueSlot; /** @@ -254,8 +271,12 @@ typedef struct AOTFuncContext { bool mem_space_unchanged; AOTCheckedAddrList checked_addr_list; + /* The last accessed shared heap info */ LLVMValueRef shared_heap_base_addr_adj; LLVMValueRef shared_heap_start_off; + LLVMValueRef shared_heap_end_off; + /* The head of shared heap chain, and its start offset */ + LLVMValueRef shared_heap; LLVMBasicBlockRef got_exception_block; LLVMBasicBlockRef func_return_block; From b0f0741426b9f324e1f86af7c8f99e14f4929073 Mon Sep 17 00:00:00 2001 From: TL Date: Fri, 11 Apr 2025 12:41:32 +0800 Subject: [PATCH 02/29] first draft of shared heap enhancement in AOT --- core/iwasm/aot/aot_runtime.c | 23 +- core/iwasm/compilation/aot_emit_memory.c | 634 ++++++++++++----------- core/iwasm/interpreter/wasm_runtime.c | 6 +- 3 files changed, 337 insertions(+), 326 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 341f64cd5a..e871745f44 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1974,23 +1974,9 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, extra->stack_sizes = aot_get_data_section_addr(module, AOT_STACK_SIZES_SECTION_NAME, NULL); - /* - * The AOT code checks whether the n bytes to access are in shared heap - * by checking whether the beginning address meets: - * addr >= start_off && addr <= end_off - n-bytes + 1 - * where n is 1/2/4/8/16 and `end_off - n-bytes + 1` is constant, e.g., - * UINT32_MAX, UINT32_MAX-1, UINT32_MAX-3 for n = 1, 2 or 4 in 32-bit - * target. To simplify the check, when shared heap is disabled, we set - * the start off to UINT64_MAX in 64-bit target and UINT32_MAX in 32-bit - * target, so in the checking, the above formula will be false, we don't - * need to check whether the shared heap is enabled or not in the AOT - * code. - */ -#if UINTPTR_MAX == UINT64_MAX - extra->shared_heap_start_off.u64 = UINT64_MAX; -#else - extra->shared_heap_start_off.u32[0] = UINT32_MAX; -#endif + /* To simplify the check in AOT code: Early stop when no shared heap + * attached. */ + extra->shared_heap = NULL; #if WASM_ENABLE_PERF_PROFILING != 0 total_size = sizeof(AOTFuncPerfProfInfo) @@ -5354,7 +5340,8 @@ aot_const_str_set_insert(const uint8 *str, int32 len, AOTModule *module, #if WASM_ENABLE_DYNAMIC_AOT_DEBUG != 0 AOTModule *g_dynamic_aot_module = NULL; -void __attribute__((noinline)) __enable_dynamic_aot_debug(void) +void __attribute__((noinline)) +__enable_dynamic_aot_debug(void) { /* empty implementation. */ } diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 2d968e701b..63c0c958e9 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -160,7 +160,7 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx); #define BUILD_LOAD_PTR(ptr, data_type, res) \ do { \ if (!(res = LLVMBuildLoad2(comp_ctx->builder, data_type, ptr, \ - "load_ptr"))) { \ + "load_value"))) { \ aot_set_last_error("llvm build load failed"); \ goto fail; \ } \ @@ -186,20 +186,59 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx); BUILD_LOAD_PTR(field_p, data_type, res); \ } while (0) -#define BUILD_GET_SHARED_HEAP_START(shared_heap_p, res) \ +#define BUILD_GET_SHARED_HEAP_START(shared_heap_p, res) \ + do { \ + if (is_memory64) { \ + offset_u32 = offsetof(WASMSharedHeap, start_off_mem64); \ + } \ + else { \ + offset_u32 = offsetof(WASMSharedHeap, start_off_mem32); \ + } \ + field_offset = I32_CONST(offset_u32); \ + CHECK_LLVM_CONST(field_offset); \ + \ + BUILD_FIELD_PTR(shared_heap_p, field_offset, field_p, \ + "shared_heap_start_off_p"); \ + BUILD_LOAD_PTR(field_p, is_target_64bit ? I64_TYPE : I32_TYPE, res); \ + /* For bulk memory on 32 bits platform , it's always 64 bit and needs \ + * to extend */ \ + if (bulk_memory && !is_target_64bit \ + && !(res = LLVMBuildZExt(comp_ctx->builder, res, I64_TYPE, \ + "shared_heap_start_off_u64"))) { \ + aot_set_last_error("llvm build zero ext failed."); \ + goto fail; \ + } \ + } while (0) + +#define BUILD_GET_MAX_SHARED_HEAP_BOUND(shared_heap_check_bound) \ do { \ - if (is_memory64) { \ - offset_u32 = offsetof(WASMSharedHeap, start_off_mem64); \ + if (bulk_memory) { \ + shared_heap_check_bound = \ + is_memory64 ? I64_CONST(UINT64_MAX) : I64_CONST(UINT32_MAX); \ + BUILD_OP(Add, max_addr, I64_NEG_ONE, max_offset, \ + "bulk_shared_heap_chain_bound"); \ } \ else { \ - offset_u32 = offsetof(WASMSharedHeap, start_off_mem32); \ + shared_heap_check_bound = \ + is_memory64 \ + ? I64_CONST(UINT64_MAX - bytes + 1) \ + : (is_target_64bit ? I64_CONST(UINT32_MAX - bytes + 1) \ + : I32_CONST(UINT32_MAX - bytes + 1)); \ } \ - field_offset = I32_CONST(offset_u32); \ - CHECK_LLVM_CONST(field_offset); \ - \ - BUILD_FIELD_PTR(shared_heap_p, field_offset, field_p, \ - "shared_heap_start_off"); \ - BUILD_LOAD_PTR(field_p, is_target_64bit ? I64_TYPE : I32_TYPE, res); \ + CHECK_LLVM_CONST(shared_heap_check_bound); \ + } while (0) + +#define BUILD_GET_MAX_ACCESS_OFFSET(max_offset) \ + do { \ + /* Bulk memory max_offset already calculated before in \ + * BUILD_GET_MAX_SHARED_HEAP_BOUND */ \ + if (!bulk_memory) { \ + length = \ + is_target_64bit ? I64_CONST(bytes - 1) : I32_CONST(bytes - 1); \ + CHECK_LLVM_CONST(length); \ + BUILD_OP(Add, start_offset, length, max_offset, \ + "cache_shared_heap_bound"); \ + } \ } while (0) #define BUILD_SET_MODULE_EXTRA_FIELD(ptr_type, field, value) \ @@ -275,11 +314,279 @@ aot_update_last_used_shared_heap(AOTCompContext *comp_ctx, return false; } +/* The difference between bulk memory overflow check and normal memory + * overflow check: + * 1. In bulk memory overflow check, no segue will be used + * 2. In bulk memory overflow check: all mem_offset will always be i64. + * In normal memory check: mem_offset wll only be i64 on 64 bit target + * it's i32 on 32 bit target + * 3. In bulk memory overflow check: the offset(start addr) and max_addr( + * (start + bytes) is used in memory check + * In normal memory check: the offset1(start addr) and bytes is used + */ +static bool +aot_check_shared_heap_memory_overflow_common( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, + LLVMBasicBlockRef check_succ, LLVMValueRef start_offset, + LLVMValueRef max_addr, LLVMValueRef mem_base_addr, uint32 bytes, + bool is_memory64, bool is_target_64bit, bool bulk_memory, bool enable_segue) +{ + LLVMBasicBlockRef app_addr_in_cache_shared_heap, + app_addr_in_shared_heap_chain, app_addr_in_linear_mem, loopEntry, + loopCond, loopBody, loopExit, check_valid_shared_heap; + LLVMValueRef addr, maddr, max_offset, maddr_phi = NULL, cmp1, cmp2, cmp; + LLVMValueRef shared_heap_start_off, shared_heap_check_bound, shared_heap_p, + field_p, field_offset, length, shared_heap_size; + uint32 offset_u32; + + /* Add basic blocks + * +-------------------------------------+ + * | current block | + * +-------------------------------------+ + * / | \ + * / | \ + * / | \ + * not in shared heap chain | in shared heap chain + * | yes | + * v | v + * +----------------+ +-------------------+ +--------------------+ + * | in_linear_mem | |in_cache_share_heap| |in_shared_heap_chain| + * +----------------+ +-------------------+ +--------------------+ + * \ | / + * \ | / + * \ | / + * v v v + * +---------------------------------------+ + * | block_maddr_phi | + * +---------------------------------------+ + */ + ADD_BASIC_BLOCK(app_addr_in_cache_shared_heap, + "app_addr_in_cache_shared_heap"); + ADD_BASIC_BLOCK(app_addr_in_shared_heap_chain, + "app_addr_in_shared_heap_chain"); + ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); + + LLVMMoveBasicBlockAfter(app_addr_in_cache_shared_heap, block_curr); + LLVMMoveBasicBlockAfter(app_addr_in_shared_heap_chain, block_curr); + LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, block_curr); + if (!bulk_memory) + LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem); + else + LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ); + + if (!bulk_memory && !is_target_64bit) { + /* Check whether integer overflow occurs in addr + offset */ + LLVMBasicBlockRef check_integer_overflow_end; + ADD_BASIC_BLOCK(check_integer_overflow_end, + "check_integer_overflow_end"); + LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr); + + BUILD_ICMP(LLVMIntULT, start_offset, addr, cmp1, "cmp1"); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp1, + check_integer_overflow_end)) { + goto fail; + } + SET_BUILD_POS(check_integer_overflow_end); + } + + /* Early stop when no shared heap attached */ + BUILD_IS_NOT_NULL(func_ctx->shared_heap, cmp, "has_shared_heap"); + BUILD_COND_BR(cmp, app_addr_in_shared_heap_chain, app_addr_in_linear_mem); + + /* Check whether the bytes to access are in shared heap chain */ + SET_BUILD_POS(app_addr_in_shared_heap_chain); + /* Add loop basic blocks*/ + ADD_BASIC_BLOCK(loopEntry, "loop_entry"); + ADD_BASIC_BLOCK(loopCond, "loop_Cond"); + ADD_BASIC_BLOCK(loopBody, "loop_body"); + ADD_BASIC_BLOCK(loopExit, "loop_exit"); + LLVMMoveBasicBlockAfter(loopEntry, app_addr_in_shared_heap_chain); + LLVMMoveBasicBlockAfter(loopCond, app_addr_in_shared_heap_chain); + LLVMMoveBasicBlockAfter(loopBody, app_addr_in_shared_heap_chain); + LLVMMoveBasicBlockAfter(loopExit, app_addr_in_shared_heap_chain); + + /* Early stop for app addr not in shared heap chain at all */ + /* TODO: potential optimization + Before shared heap chain: + Use IntUGT but not IntUGE to compare, since (1) in the ems + memory allocator, the hmu node includes hmu header and hmu + memory, only the latter is returned to the caller as the + allocated memory, the hmu header isn't returned so the + first byte of the shared heap won't be accessed, (2) using + IntUGT gets better performance than IntUGE in some cases + + And We don't check the shared heap's upper boundary if boundary + check isn't enabled, the runtime may also use the guard pages + of shared heap to check the boundary if hardware boundary + check feature is enabled. + + After shared heap chain: + Use IntUGE to compare since pre-allocated shared heap can access + the first byte. + Need to check upper boundary since there could be multiple shared + heap. + */ + BUILD_GET_SHARED_HEAP_START(func_ctx->shared_heap, shared_heap_start_off); + /* For bulk memory check, it's U32/64_MAX - 1 to compare with max_addr, + * otherwise it's U32/64_MAX - bytes + 1 to compare with start_off */ + BUILD_GET_MAX_SHARED_HEAP_BOUND(shared_heap_check_bound); + BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, + "cmp_shared_heap_chain_start"); + BUILD_ICMP(LLVMIntULE, bulk_memory ? max_offset : start_offset, + shared_heap_check_bound, cmp1, "cmp_shared_heap_chain_end"); + BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); + BUILD_COND_BR(cmp2, loopEntry, app_addr_in_linear_mem); + + /* Check whether the bytes to access are in cache shared heap */ + SET_BUILD_POS(loopEntry); + BUILD_ICMP(LLVMIntUGE, start_offset, func_ctx->shared_heap_start_off, cmp, + "cmp_cache_shared_heap_start"); + BUILD_GET_MAX_ACCESS_OFFSET(max_offset); + BUILD_ICMP(LLVMIntULE, max_offset, func_ctx->shared_heap_end_off, cmp1, + "cmp_cache_shared_heap_end"); + BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); + BUILD_COND_BR(cmp2, app_addr_in_cache_shared_heap, loopCond); + + /* Check which shared heap in the shared heap chain the addr resides + */ + SET_BUILD_POS(loopCond); + /* cur = heap; cur; cur = cur->chain_next */ + shared_heap_p = func_ctx->shared_heap; + BUILD_GET_SHARED_HEAP_FIELD(shared_heap_p, chain_next, INT8_PTR_TYPE, + shared_heap_p); + BUILD_IS_NOT_NULL(shared_heap_p, cmp, "has_shared_heap"); + BUILD_COND_BR(cmp, loopBody, loopExit); + + /* loop body */ + BUILD_GET_SHARED_HEAP_START(shared_heap_p, shared_heap_start_off); + /* app_offset >= shared_heap_start */ + BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, + "cmp_shared_heap_start"); + /* app_offset <= shared_heap_start + cur->size - bytes */ + BUILD_GET_SHARED_HEAP_FIELD(shared_heap_p, size, I64_TYPE, + shared_heap_size); + /* When it's not bulk memory and it on 32 bits platform, needs to trunc to + * i32 type to be consistent with other memory offset calculation */ + if (!bulk_memory && !is_target_64bit) { + BUILD_TRUNC(shared_heap_size, I32_TYPE); + } + BUILD_OP(Sub, shared_heap_size, + (bulk_memory || is_target_64bit) ? I64_CONST(bytes) + : I32_CONST(bytes), + shared_heap_check_bound, "shared_heap_check_bound1"); + BUILD_OP(Add, shared_heap_start_off, shared_heap_check_bound, + shared_heap_check_bound, "shared_heap_check_bound2"); + BUILD_ICMP(LLVMIntULE, start_offset, shared_heap_check_bound, cmp1, + "cmp_shared_heap_end"); + BUILD_OP(And, cmp, cmp1, cmp2, "is_in_shared_heap"); + BUILD_COND_BR(cmp2, loopExit, loopCond); + + /* loop exit, if the shared_heap_p is valid, we find the shared heap + * this app addr is in, otherwise it's oom */ + SET_BUILD_POS(loopExit); + BUILD_IS_NOT_NULL(shared_heap_p, cmp, "has_shared_heap"); + + ADD_BASIC_BLOCK(check_valid_shared_heap, "check_valid_shared_heap"); + LLVMMoveBasicBlockAfter(check_valid_shared_heap, loopExit); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, + check_valid_shared_heap)) { + goto fail; + } + SET_BUILD_POS(check_valid_shared_heap); + + /* Update last accessed shared heap, the shared_heap_size and + * shared_heap_start_off is already prepared in loop body. + * For bulk memory on 32 bits platform, it extends to i64 before, so it + * needs to trunc back to i32 for updating shared heap info. */ + if (bulk_memory && !is_target_64bit) { + BUILD_TRUNC(shared_heap_start_off, I32_TYPE); + BUILD_TRUNC(shared_heap_size, I32_TYPE); + } + if (!aot_update_last_used_shared_heap(comp_ctx, func_ctx, shared_heap_p, + shared_heap_start_off, + shared_heap_size, is_memory64)) { + goto fail; + } + LLVMMoveBasicBlockAfter(app_addr_in_cache_shared_heap, + check_valid_shared_heap); + + /* Get native address inside cache shared heap */ + SET_BUILD_POS(app_addr_in_cache_shared_heap); + if (!(maddr = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, func_ctx->shared_heap_base_addr_adj, + &start_offset, 1, "maddr_cache_shared_heap"))) { + aot_set_last_error("llvm build inbounds gep failed"); + goto fail; + } + + if (enable_segue) { + LLVMValueRef mem_base_addr_u64, maddr_u64, offset_to_mem_base; + + if (!(maddr_u64 = LLVMBuildPtrToInt(comp_ctx->builder, maddr, I64_TYPE, + "maddr_u64")) + || !(mem_base_addr_u64 = + LLVMBuildPtrToInt(comp_ctx->builder, mem_base_addr, + I64_TYPE, "mem_base_addr_u64"))) { + aot_set_last_error("llvm build ptr to int failed"); + goto fail; + } + if (!(offset_to_mem_base = + LLVMBuildSub(comp_ctx->builder, maddr_u64, mem_base_addr_u64, + "offset_to_mem_base"))) { + aot_set_last_error("llvm build sub failed"); + goto fail; + } + if (!(maddr = LLVMBuildIntToPtr(comp_ctx->builder, offset_to_mem_base, + INT8_PTR_TYPE_GS, + "maddr_shared_heap_segue"))) { + aot_set_last_error("llvm build int to ptr failed."); + goto fail; + } + } + + LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_cache_shared_heap, 1); + + if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { + aot_set_last_error("llvm build br failed"); + goto fail; + } + + LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); + block_curr = LLVMGetInsertBlock(comp_ctx->builder); + + return true; +fail: + return false; +} + +static bool +aot_check_shared_heap_memory_overflow( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, + LLVMValueRef start_offset, LLVMValueRef mem_base_addr, uint32 bytes, + bool is_memory64, bool is_target_64bit, bool enable_segue) +{ + return aot_check_shared_heap_memory_overflow_common( + comp_ctx, func_ctx, block_curr, block_maddr_phi, NULL, start_offset, + NULL, mem_base_addr, bytes, is_memory64, is_target_64bit, false, + enable_segue); +} + static bool -aot_check_shared_heap_memory_overflow(AOTCompContext *comp_ctx, - AOTFuncContext *func_ctx, - mem_offset_t offset, uint32 bytes, - bool enable_segue); +aot_check_bulk_memory_shared_heap_memory_overflow( + AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, + LLVMBasicBlockRef check_succ, LLVMValueRef start_offset, + LLVMValueRef max_addr, bool is_memory64) +{ + return aot_check_shared_heap_memory_overflow_common( + comp_ctx, func_ctx, block_curr, block_maddr_phi, check_succ, + start_offset, max_addr, NULL, 0, is_memory64, + comp_ctx->pointer_size == sizeof(uint64), true, false); +} LLVMValueRef aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, @@ -472,48 +779,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { - LLVMBasicBlockRef app_addr_in_cache_shared_heap, - app_addr_in_shared_heap_chain, app_addr_in_linear_mem, loopEntry, - loopCond, loopBody, loopExit, check_valid_shared_heap; - /* Shared heap offset will be i64 on 64-bit platform and i32 on 32-bit - * platform */ - LLVMValueRef shared_heap_start_off, shared_heap_check_bound, - shared_heap_p, field_p, field_offset, length, shared_heap_size; - uint32 offset_u32; - - /* Add basic blocks - * +-------------------------------------+ - * | current block | - * +-------------------------------------+ - * / | \ - * / | \ - * / | \ - * not in shared heap chain | in shared heap chain - * | yes | - * v | v - * +----------------+ +-------------------+ +--------------------+ - * | in_linear_mem | |in_cache_share_heap| |in_shared_heap_chain| - * +----------------+ +-------------------+ +--------------------+ - * \ | / - * \ | / - * \ | / - * v v v - * +---------------------------------------+ - * | block_maddr_phi | - * +---------------------------------------+ - */ - ADD_BASIC_BLOCK(app_addr_in_cache_shared_heap, - "app_addr_in_cache_shared_heap"); - ADD_BASIC_BLOCK(app_addr_in_shared_heap_chain, - "app_addr_in_shared_heap_chain"); - ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); - - LLVMMoveBasicBlockAfter(app_addr_in_cache_shared_heap, block_curr); - LLVMMoveBasicBlockAfter(app_addr_in_shared_heap_chain, block_curr); - LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, block_curr); - LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem); - SET_BUILD_POS(block_maddr_phi); if (!(maddr_phi = LLVMBuildPhi(comp_ctx->builder, @@ -522,189 +788,14 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, aot_set_last_error("llvm build phi failed"); goto fail; } - SET_BUILD_POS(block_curr); - /* 2.1 offset + addr can overflow when it's memory32 */ - if (!is_target_64bit) { - /* Check whether integer overflow occurs in addr + offset */ - LLVMBasicBlockRef check_integer_overflow_end; - ADD_BASIC_BLOCK(check_integer_overflow_end, - "check_integer_overflow_end"); - LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr); - - BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); - if (!aot_emit_exception(comp_ctx, func_ctx, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, - cmp1, check_integer_overflow_end)) { - goto fail; - } - SET_BUILD_POS(check_integer_overflow_end); - } - - /* Early stop when no shared heap attached */ - BUILD_IS_NOT_NULL(func_ctx->shared_heap, cmp, "has_shared_heap"); - BUILD_COND_BR(cmp, app_addr_in_shared_heap_chain, - app_addr_in_linear_mem); - - /* Check whether the bytes to access are in shared heap chain */ - SET_BUILD_POS(app_addr_in_shared_heap_chain); - /* Add loop basic blocks*/ - ADD_BASIC_BLOCK(loopEntry, "loop_entry"); - ADD_BASIC_BLOCK(loopCond, "loop_Cond"); - ADD_BASIC_BLOCK(loopBody, "loop_body"); - ADD_BASIC_BLOCK(loopExit, "loop_exit"); - LLVMMoveBasicBlockAfter(loopEntry, app_addr_in_shared_heap_chain); - LLVMMoveBasicBlockAfter(loopCond, app_addr_in_shared_heap_chain); - LLVMMoveBasicBlockAfter(loopBody, app_addr_in_shared_heap_chain); - LLVMMoveBasicBlockAfter(loopExit, app_addr_in_shared_heap_chain); - - /* Early stop for app addr not in shared heap chain at all */ - BUILD_GET_SHARED_HEAP_START(func_ctx->shared_heap, - shared_heap_start_off); - shared_heap_check_bound = - is_memory64 ? I64_CONST(UINT64_MAX - bytes + 1) - : (is_target_64bit ? I64_CONST(UINT32_MAX - bytes + 1) - : I32_CONST(UINT32_MAX - bytes + 1)); - CHECK_LLVM_CONST(shared_heap_check_bound); - /* TODO: potential optimization - Before shared heap chain: - Use IntUGT but not IntUGE to compare, since (1) in the ems - memory allocator, the hmu node includes hmu header and hmu - memory, only the latter is returned to the caller as the - allocated memory, the hmu header isn't returned so the - first byte of the shared heap won't be accessed, (2) using - IntUGT gets better performance than IntUGE in some cases - - And We don't check the shared heap's upper boundary if boundary - check isn't enabled, the runtime may also use the guard pages - of shared heap to check the boundary if hardware boundary - check feature is enabled. - - After shared heap chain: - Use IntUGE to compare since pre-allocated shared heap can access - the first byte. - Need to check upper boundary since there could be multiple shared - heap. - */ - BUILD_ICMP(LLVMIntUGE, offset1, shared_heap_start_off, cmp, - "cmp_shared_heap_chain_start"); - BUILD_ICMP(LLVMIntULE, offset1, shared_heap_check_bound, cmp1, - "cmp_shared_heap_chain_end"); - BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); - BUILD_COND_BR(cmp2, loopEntry, app_addr_in_linear_mem); - - /* Check whether the bytes to access are in cache shared heap */ - SET_BUILD_POS(loopEntry); - BUILD_ICMP(LLVMIntUGE, offset1, func_ctx->shared_heap_start_off, cmp, - "cmp_cache_shared_heap_start"); - length = is_target_64bit ? I64_CONST(bytes - 1) : I32_CONST(bytes - 1); - CHECK_LLVM_CONST(length); - BUILD_OP(Sub, func_ctx->shared_heap_end_off, length, - shared_heap_check_bound, "cache_shared_heap_end"); - BUILD_ICMP(LLVMIntULE, offset1, shared_heap_check_bound, cmp1, - "cmp_cache_shared_heap_end"); - BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); - BUILD_COND_BR(cmp2, app_addr_in_cache_shared_heap, loopCond); - - /* Check which shared heap in the shared heap chain the addr resides */ - SET_BUILD_POS(loopCond); - /* cur = heap; cur; cur = cur->chain_next */ - shared_heap_p = func_ctx->shared_heap; - BUILD_GET_SHARED_HEAP_FIELD(shared_heap_p, chain_next, INT8_PTR_TYPE, - shared_heap_p); - BUILD_IS_NOT_NULL(shared_heap_p, cmp, "has_shared_heap"); - BUILD_COND_BR(cmp, loopBody, loopExit); - - /* loop body */ - BUILD_GET_SHARED_HEAP_START(shared_heap_p, shared_heap_start_off); - /* app_offset >= shared_heap_start */ - BUILD_ICMP(LLVMIntUGE, offset1, shared_heap_start_off, cmp, - "cmp_shared_heap_start"); - /* app_offset <= shared_heap_start + cur->size - bytes */ - BUILD_GET_SHARED_HEAP_FIELD(shared_heap_p, size, I64_TYPE, - shared_heap_size); - if (!is_target_64bit) { - BUILD_TRUNC(shared_heap_size, I32_TYPE); - } - BUILD_OP(Sub, shared_heap_size, - is_target_64bit ? I64_CONST(bytes) : I32_CONST(bytes), - shared_heap_check_bound, "shared_heap_check_bound1"); - BUILD_OP(Add, shared_heap_start_off, shared_heap_check_bound, - shared_heap_check_bound, "shared_heap_end_bound2"); - BUILD_ICMP(LLVMIntULE, offset1, shared_heap_check_bound, cmp1, - "cmp_shared_heap_end"); - BUILD_OP(And, cmp, cmp1, cmp2, "is_in_shared_heap"); - BUILD_COND_BR(cmp2, loopExit, loopCond); - - /* loop exit, if the shared_heap_p is valid, we find the shared heap - * this app addr is in, otherwise it's oom */ - SET_BUILD_POS(loopExit); - BUILD_IS_NOT_NULL(shared_heap_p, cmp, "has_shared_heap"); - - ADD_BASIC_BLOCK(check_valid_shared_heap, "check_valid_shared_heap"); - LLVMMoveBasicBlockAfter(check_valid_shared_heap, loopExit); - if (!aot_emit_exception(comp_ctx, func_ctx, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, - check_valid_shared_heap)) { - goto fail; - } - SET_BUILD_POS(check_valid_shared_heap); - - /* Update last accessed shared heap, the shared_heap_size and - * shared_heap_start_off is already prepared in loop body, not only need - * to prepare shared_heap_end_off */ - if (!aot_update_last_used_shared_heap(comp_ctx, func_ctx, shared_heap_p, - shared_heap_start_off, - shared_heap_size, is_memory64)) - goto fail; - LLVMMoveBasicBlockAfter(app_addr_in_cache_shared_heap, - check_valid_shared_heap); - - /* Get native address inside cache shared heap */ - SET_BUILD_POS(app_addr_in_cache_shared_heap); - if (!(maddr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->shared_heap_base_addr_adj, - &offset1, 1, - "maddr_cache_shared_heap"))) { - aot_set_last_error("llvm build inbounds gep failed"); - goto fail; - } - - if (enable_segue) { - LLVMValueRef mem_base_addr_u64, maddr_u64, offset_to_mem_base; - - if (!(maddr_u64 = LLVMBuildPtrToInt(comp_ctx->builder, maddr, - I64_TYPE, "maddr_u64")) - || !(mem_base_addr_u64 = - LLVMBuildPtrToInt(comp_ctx->builder, mem_base_addr, - I64_TYPE, "mem_base_addr_u64"))) { - aot_set_last_error("llvm build ptr to int failed"); - goto fail; - } - if (!(offset_to_mem_base = - LLVMBuildSub(comp_ctx->builder, maddr_u64, - mem_base_addr_u64, "offset_to_mem_base"))) { - aot_set_last_error("llvm build sub failed"); - goto fail; - } - if (!(maddr = LLVMBuildIntToPtr( - comp_ctx->builder, offset_to_mem_base, INT8_PTR_TYPE_GS, - "maddr_shared_heap_segue"))) { - aot_set_last_error("llvm build int to ptr failed."); - goto fail; - } - } - - LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_cache_shared_heap, 1); - - if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { - aot_set_last_error("llvm build br failed"); + if (!aot_check_shared_heap_memory_overflow( + comp_ctx, func_ctx, block_curr, block_maddr_phi, offset1, + mem_base_addr, bytes, is_memory64, is_target_64bit, + enable_segue)) { goto fail; } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); - block_curr = LLVMGetInsertBlock(comp_ctx->builder); } if (comp_ctx->enable_bound_check @@ -1529,83 +1620,20 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { - /* TODO: shared heap chain here */ - LLVMBasicBlockRef app_addr_in_shared_heap, app_addr_in_linear_mem; - LLVMValueRef shared_heap_start_off, shared_heap_check_bound; - LLVMValueRef max_offset, cmp1, cmp2, is_in_shared_heap; - - /* Add basic blocks */ - ADD_BASIC_BLOCK(app_addr_in_shared_heap, "app_addr_in_shared_heap"); - ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); - - LLVMMoveBasicBlockAfter(app_addr_in_shared_heap, block_curr); - LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, - app_addr_in_shared_heap); - LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ); - - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_maddr_phi); + SET_BUILD_POS(block_maddr_phi); if (!(maddr_phi = LLVMBuildPhi(comp_ctx->builder, INT8_PTR_TYPE, "maddr_phi"))) { aot_set_last_error("llvm build phi failed"); goto fail; } + SET_BUILD_POS(block_curr); - LLVMPositionBuilderAtEnd(comp_ctx->builder, block_curr); - - shared_heap_start_off = func_ctx->shared_heap_start_off; - if (comp_ctx->pointer_size == sizeof(uint32)) { - if (!(shared_heap_start_off = - LLVMBuildZExt(comp_ctx->builder, shared_heap_start_off, - I64_TYPE, "shared_heap_start_off_u64"))) { - aot_set_last_error("llvm build zext failed"); - goto fail; - } - } - shared_heap_check_bound = - is_memory64 ? I64_CONST(UINT64_MAX) : I64_CONST(UINT32_MAX); - CHECK_LLVM_CONST(shared_heap_check_bound); - - /* Check whether the bytes to access are in shared heap */ - if (!comp_ctx->enable_bound_check) { - /* Use IntUGT but not IntUGE to compare, same as the check - in aot_check_memory_overflow */ - BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off, - is_in_shared_heap, "is_in_shared_heap"); - } - else { - BUILD_ICMP(LLVMIntUGT, offset, func_ctx->shared_heap_start_off, - cmp1, "cmp1"); - BUILD_OP(Add, max_addr, I64_NEG_ONE, max_offset, "max_offset"); - BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_check_bound, cmp2, - "cmp2"); - BUILD_OP(And, cmp1, cmp2, is_in_shared_heap, "is_in_shared_heap"); - } - - if (!LLVMBuildCondBr(comp_ctx->builder, is_in_shared_heap, - app_addr_in_shared_heap, app_addr_in_linear_mem)) { - aot_set_last_error("llvm build cond br failed"); - goto fail; - } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_shared_heap); - - /* Get native address inside shared heap */ - if (!(maddr = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, - func_ctx->shared_heap_base_addr_adj, - &offset, 1, "maddr_shared_heap"))) { - aot_set_last_error("llvm build inbounds gep failed"); - goto fail; - } - LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_shared_heap, 1); - - if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { - aot_set_last_error("llvm build br failed"); + if (!aot_check_bulk_memory_shared_heap_memory_overflow( + comp_ctx, func_ctx, block_curr, block_maddr_phi, check_succ, + offset, max_addr, is_memory64)) { goto fail; } - - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); - block_curr = LLVMGetInsertBlock(comp_ctx->builder); } BUILD_ICMP(LLVMIntUGT, max_addr, mem_size, cmp, "cmp_max_mem_addr"); diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 3cc2afe04d..147f948f83 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -2813,11 +2813,7 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, } #if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 -#if UINTPTR_MAX == UINT64_MAX - module_inst->e->shared_heap_start_off.u64 = UINT64_MAX; -#else - module_inst->e->shared_heap_start_off.u32[0] = UINT32_MAX; -#endif + module_inst->e->shared_heap = NULL; #endif #if WASM_ENABLE_GC != 0 From 20cd6afbb897915f513adec88f069fcf92038ca1 Mon Sep 17 00:00:00 2001 From: TL Date: Mon, 14 Apr 2025 18:24:52 +0800 Subject: [PATCH 03/29] fix some compilation issue --- core/iwasm/aot/aot_runtime.c | 23 ++- core/iwasm/compilation/aot_emit_memory.c | 242 ++++++++++++----------- core/iwasm/interpreter/wasm_runtime.c | 5 + 3 files changed, 150 insertions(+), 120 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index e871745f44..24c4a2b6ef 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -1974,8 +1974,24 @@ aot_instantiate(AOTModule *module, AOTModuleInstance *parent, extra->stack_sizes = aot_get_data_section_addr(module, AOT_STACK_SIZES_SECTION_NAME, NULL); - /* To simplify the check in AOT code: Early stop when no shared heap - * attached. */ + /* + * The AOT code checks whether the n bytes to access are in shared heap + * by checking whether the beginning address meets: + * addr >= start_off && addr <= end_off - n-bytes + 1 + * where n is 1/2/4/8/16 and `end_off - n-bytes + 1` is constant, e.g., + * UINT32_MAX, UINT32_MAX-1, UINT32_MAX-3 for n = 1, 2 or 4 in 32-bit + * target. To simplify the check, when shared heap is disabled, we set + * the start off to UINT64_MAX in 64-bit target and UINT32_MAX in 32-bit + * target, so in the checking, the above formula will be false, we don't + * need to check whether the shared heap is enabled or not in the AOT + * code. + */ +#if UINTPTR_MAX == UINT64_MAX + extra->shared_heap_start_off.u64 = UINT64_MAX; +#else + extra->shared_heap_start_off.u32[0] = UINT32_MAX; +#endif + /* After shared heap chain, will early stop if shared heap is NULL */ extra->shared_heap = NULL; #if WASM_ENABLE_PERF_PROFILING != 0 @@ -5340,8 +5356,7 @@ aot_const_str_set_insert(const uint8 *str, int32 len, AOTModule *module, #if WASM_ENABLE_DYNAMIC_AOT_DEBUG != 0 AOTModule *g_dynamic_aot_module = NULL; -void __attribute__((noinline)) -__enable_dynamic_aot_debug(void) +void __attribute__((noinline)) __enable_dynamic_aot_debug(void) { /* empty implementation. */ } diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 63c0c958e9..6ad0ad46be 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -267,16 +267,16 @@ aot_update_last_used_shared_heap(AOTCompContext *comp_ctx, /* Update last used shared heap info in func_ctx and module_extra. * 1. shared_heap_start_off 2. shared_heap_end_off * 3. shared_heap_base_addr_adj */ - uint32 offset_u32; - LLVMValueRef tmp_res, field_p, field_offset, shared_heap_end_off, base_addr, - shared_heap_base_addr_adj; + LLVMValueRef shared_heap_end_off, base_addr, shared_heap_base_addr_adj; LLVMTypeRef shared_heap_off_type = is_target_64bit ? I64_TYPE : I32_TYPE, shared_heap_off_ptr_type = is_target_64bit ? INT64_PTR_TYPE : INT32_PTR_TYPE; + LLVMValueRef tmp_res, field_p, field_offset; + uint32 offset_u32; /* shared_heap_end_off = shared_heap_start_off + shared_heap_size - 1 */ BUILD_OP(Add, shared_heap_start_off, - is_target_64bit ? I64_NEG_ONE : I32_NEG_ONE, tmp_res, + (is_target_64bit ? I64_NEG_ONE : I32_NEG_ONE), tmp_res, "shared_heap_start_off_minus_one"); BUILD_OP(Add, tmp_res, shared_heap_size, shared_heap_end_off, "shared_heap_end_off"); @@ -304,10 +304,10 @@ aot_update_last_used_shared_heap(AOTCompContext *comp_ctx, BUILD_SET_MODULE_EXTRA_FIELD(INTPTR_T_PTR_TYPE, shared_heap_base_addr_adj, shared_heap_base_addr_adj); - BUILD_STORE_PTR(func_ctx->shared_heap_start_off, shared_heap_start_off); - BUILD_STORE_PTR(func_ctx->shared_heap_end_off, shared_heap_end_off); - BUILD_STORE_PTR(func_ctx->shared_heap_base_addr_adj, - shared_heap_base_addr_adj); + /* FIXME: use cache shared heap instead of func_ctx, it should be phi */ + // func_ctx->shared_heap_start_off = shared_heap_start_off; + // func_ctx->shared_heap_end_off = shared_heap_end_off; + // func_ctx->shared_heap_base_addr_adj = shared_heap_base_addr_adj; return true; fail: @@ -328,48 +328,60 @@ static bool aot_check_shared_heap_memory_overflow_common( AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, - LLVMBasicBlockRef check_succ, LLVMValueRef start_offset, - LLVMValueRef max_addr, LLVMValueRef mem_base_addr, uint32 bytes, - bool is_memory64, bool is_target_64bit, bool bulk_memory, bool enable_segue) + LLVMBasicBlockRef check_succ, LLVMValueRef maddr_phi, + LLVMValueRef start_offset, LLVMValueRef max_addr, + LLVMValueRef mem_base_addr, uint32 bytes, bool is_memory64, + bool is_target_64bit, bool bulk_memory, bool enable_segue) { - LLVMBasicBlockRef app_addr_in_cache_shared_heap, - app_addr_in_shared_heap_chain, app_addr_in_linear_mem, loopEntry, + LLVMBasicBlockRef app_addr_in_shared_heap_chain, + app_addr_in_cache_shared_heap, app_addr_in_linear_mem, loopEntry, loopCond, loopBody, loopExit, check_valid_shared_heap; - LLVMValueRef addr, maddr, max_offset, maddr_phi = NULL, cmp1, cmp2, cmp; - LLVMValueRef shared_heap_start_off, shared_heap_check_bound, shared_heap_p, - field_p, field_offset, length, shared_heap_size; + LLVMValueRef addr = NULL, maddr = NULL, max_offset = NULL, + init_value = NULL; + LLVMValueRef cmp = NULL, cmp1 = NULL, cmp2 = NULL; + LLVMValueRef shared_heap_start_off = NULL, shared_heap_check_bound = NULL, + shared_heap_size = NULL; + /* The pointer that will traverse the shared heap chain */ + LLVMValueRef phi_shared_heap = NULL, cur_shared_heap = NULL; + LLVMValueRef field_offset, field_p, length; uint32 offset_u32; - /* Add basic blocks - * +-------------------------------------+ - * | current block | - * +-------------------------------------+ - * / | \ - * / | \ - * / | \ - * not in shared heap chain | in shared heap chain - * | yes | - * v | v - * +----------------+ +-------------------+ +--------------------+ - * | in_linear_mem | |in_cache_share_heap| |in_shared_heap_chain| - * +----------------+ +-------------------+ +--------------------+ - * \ | / - * \ | / - * \ | / - * v v v - * +---------------------------------------+ - * | block_maddr_phi | - * +---------------------------------------+ - */ - ADD_BASIC_BLOCK(app_addr_in_cache_shared_heap, - "app_addr_in_cache_shared_heap"); + /*--------------------------------------------------------------------- + * Create the basic blocks and arrange control flow: + * + * +-------------------------+ + * | current block | + * +-------------------------+ + * / \ + * / \ + * not in shared heap in shared heap chain ------------ + * | | | + * | | | + * v v | + * +---------------------+ +-------------------------+ | + * | in_linear_mem | | app_addr_in_shared_heap | | + * +---------------------+ | chain | | + * +-----------+-------------+ | + * | | + * +---------------+---------------+ | + * | Loop: Traverse shared heap | | + * +---------------+---------------+ | + * | | + * +---------------+---------------+ | + * | Cache shared heap branch | <---- + * +-------------------------------+ + *---------------------------------------------------------------------*/ ADD_BASIC_BLOCK(app_addr_in_shared_heap_chain, "app_addr_in_shared_heap_chain"); + ADD_BASIC_BLOCK(app_addr_in_cache_shared_heap, + "app_addr_in_cache_shared_heap"); ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); - LLVMMoveBasicBlockAfter(app_addr_in_cache_shared_heap, block_curr); LLVMMoveBasicBlockAfter(app_addr_in_shared_heap_chain, block_curr); - LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, block_curr); + LLVMMoveBasicBlockAfter(app_addr_in_cache_shared_heap, + app_addr_in_shared_heap_chain); + LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, + app_addr_in_cache_shared_heap); if (!bulk_memory) LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem); else @@ -391,46 +403,31 @@ aot_check_shared_heap_memory_overflow_common( SET_BUILD_POS(check_integer_overflow_end); } - /* Early stop when no shared heap attached */ + /* If there is no shared heap attached, branch to linear memory */ BUILD_IS_NOT_NULL(func_ctx->shared_heap, cmp, "has_shared_heap"); BUILD_COND_BR(cmp, app_addr_in_shared_heap_chain, app_addr_in_linear_mem); - /* Check whether the bytes to access are in shared heap chain */ + /*--------------------------------------------------------------------- + * In the case where a shared heap is attached, we determine if the bytes + * to access reside in the shared heap chain. If yes, we enter a loop that + * traverses the chain by using a phi node to merge two definitions of the + * pointer (initial and updated). + *---------------------------------------------------------------------*/ SET_BUILD_POS(app_addr_in_shared_heap_chain); - /* Add loop basic blocks*/ + + /* Add loop basic blocks */ ADD_BASIC_BLOCK(loopEntry, "loop_entry"); - ADD_BASIC_BLOCK(loopCond, "loop_Cond"); + ADD_BASIC_BLOCK(loopCond, "loop_cond"); ADD_BASIC_BLOCK(loopBody, "loop_body"); ADD_BASIC_BLOCK(loopExit, "loop_exit"); LLVMMoveBasicBlockAfter(loopEntry, app_addr_in_shared_heap_chain); - LLVMMoveBasicBlockAfter(loopCond, app_addr_in_shared_heap_chain); - LLVMMoveBasicBlockAfter(loopBody, app_addr_in_shared_heap_chain); - LLVMMoveBasicBlockAfter(loopExit, app_addr_in_shared_heap_chain); - - /* Early stop for app addr not in shared heap chain at all */ - /* TODO: potential optimization - Before shared heap chain: - Use IntUGT but not IntUGE to compare, since (1) in the ems - memory allocator, the hmu node includes hmu header and hmu - memory, only the latter is returned to the caller as the - allocated memory, the hmu header isn't returned so the - first byte of the shared heap won't be accessed, (2) using - IntUGT gets better performance than IntUGE in some cases - - And We don't check the shared heap's upper boundary if boundary - check isn't enabled, the runtime may also use the guard pages - of shared heap to check the boundary if hardware boundary - check feature is enabled. - - After shared heap chain: - Use IntUGE to compare since pre-allocated shared heap can access - the first byte. - Need to check upper boundary since there could be multiple shared - heap. - */ + LLVMMoveBasicBlockAfter(loopCond, loopEntry); + LLVMMoveBasicBlockAfter(loopBody, loopCond); + LLVMMoveBasicBlockAfter(loopExit, loopBody); + + /* Early check: if app_offset is not in the overall shared heap chain, + * branch to linear memory. */ BUILD_GET_SHARED_HEAP_START(func_ctx->shared_heap, shared_heap_start_off); - /* For bulk memory check, it's U32/64_MAX - 1 to compare with max_addr, - * otherwise it's U32/64_MAX - bytes + 1 to compare with start_off */ BUILD_GET_MAX_SHARED_HEAP_BOUND(shared_heap_check_bound); BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, "cmp_shared_heap_chain_start"); @@ -439,7 +436,8 @@ aot_check_shared_heap_memory_overflow_common( BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); BUILD_COND_BR(cmp2, loopEntry, app_addr_in_linear_mem); - /* Check whether the bytes to access are in cache shared heap */ + /* In loopEntry: Check if the app address is in the cache shared heap range. + * If yes, branch to the cache branch; if not, proceed into the loop. */ SET_BUILD_POS(loopEntry); BUILD_ICMP(LLVMIntUGE, start_offset, func_ctx->shared_heap_start_off, cmp, "cmp_cache_shared_heap_start"); @@ -449,26 +447,27 @@ aot_check_shared_heap_memory_overflow_common( BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); BUILD_COND_BR(cmp2, app_addr_in_cache_shared_heap, loopCond); - /* Check which shared heap in the shared heap chain the addr resides - */ + /*--------------------------------------------------------------------- + * Loop header (loopCond): create a phi node to merge the pointer values. + * - Incoming from loopEntry: the initial pointer is func_ctx->shared_heap. + * - Incoming from loopBody: the updated pointer (next pointer in the + * chain). + *---------------------------------------------------------------------*/ SET_BUILD_POS(loopCond); - /* cur = heap; cur; cur = cur->chain_next */ - shared_heap_p = func_ctx->shared_heap; - BUILD_GET_SHARED_HEAP_FIELD(shared_heap_p, chain_next, INT8_PTR_TYPE, - shared_heap_p); - BUILD_IS_NOT_NULL(shared_heap_p, cmp, "has_shared_heap"); - BUILD_COND_BR(cmp, loopBody, loopExit); - - /* loop body */ - BUILD_GET_SHARED_HEAP_START(shared_heap_p, shared_heap_start_off); - /* app_offset >= shared_heap_start */ + phi_shared_heap = + LLVMBuildPhi(comp_ctx->builder, INT8_PTR_TYPE, "phi_shared_heap"); + init_value = func_ctx->shared_heap; + LLVMAddIncoming(phi_shared_heap, &init_value, &loopEntry, 1); + + /* In loopCond, we check whether the current shared heap node + * (phi_shared_heap) contains the target address. + */ + BUILD_GET_SHARED_HEAP_START(phi_shared_heap, shared_heap_start_off); BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, "cmp_shared_heap_start"); - /* app_offset <= shared_heap_start + cur->size - bytes */ - BUILD_GET_SHARED_HEAP_FIELD(shared_heap_p, size, I64_TYPE, + + BUILD_GET_SHARED_HEAP_FIELD(phi_shared_heap, size, I64_TYPE, shared_heap_size); - /* When it's not bulk memory and it on 32 bits platform, needs to trunc to - * i32 type to be consistent with other memory offset calculation */ if (!bulk_memory && !is_target_64bit) { BUILD_TRUNC(shared_heap_size, I32_TYPE); } @@ -481,12 +480,26 @@ aot_check_shared_heap_memory_overflow_common( BUILD_ICMP(LLVMIntULE, start_offset, shared_heap_check_bound, cmp1, "cmp_shared_heap_end"); BUILD_OP(And, cmp, cmp1, cmp2, "is_in_shared_heap"); - BUILD_COND_BR(cmp2, loopExit, loopCond); - - /* loop exit, if the shared_heap_p is valid, we find the shared heap - * this app addr is in, otherwise it's oom */ + BUILD_COND_BR(cmp2, loopExit, loopBody); + + /*--------------------------------------------------------------------- + * Loop body: update the pointer to traverse to the next shared heap in the + *chain. The updated pointer is then added as an incoming value to the phi + *node. + *---------------------------------------------------------------------*/ + SET_BUILD_POS(loopBody); + BUILD_GET_SHARED_HEAP_FIELD(phi_shared_heap, chain_next, INT8_PTR_TYPE, + cur_shared_heap); + /* Add the new value from loopBody as an incoming edge to the phi node */ + LLVMAddIncoming(phi_shared_heap, &cur_shared_heap, &loopBody, 1); + BUILD_BR(loopCond); + + /*--------------------------------------------------------------------- + * Loop exit: at this point, phi_shared_heap is expected to be valid if the + * app address is contained in a shared heap; otherwise, it is NULL. + *---------------------------------------------------------------------*/ SET_BUILD_POS(loopExit); - BUILD_IS_NOT_NULL(shared_heap_p, cmp, "has_shared_heap"); + BUILD_IS_NOT_NULL(phi_shared_heap, cmp, "has_shared_heap"); ADD_BASIC_BLOCK(check_valid_shared_heap, "check_valid_shared_heap"); LLVMMoveBasicBlockAfter(check_valid_shared_heap, loopExit); @@ -505,15 +518,16 @@ aot_check_shared_heap_memory_overflow_common( BUILD_TRUNC(shared_heap_start_off, I32_TYPE); BUILD_TRUNC(shared_heap_size, I32_TYPE); } - if (!aot_update_last_used_shared_heap(comp_ctx, func_ctx, shared_heap_p, + if (!aot_update_last_used_shared_heap(comp_ctx, func_ctx, phi_shared_heap, shared_heap_start_off, - shared_heap_size, is_memory64)) { + shared_heap_size, is_target_64bit)) { goto fail; } - LLVMMoveBasicBlockAfter(app_addr_in_cache_shared_heap, - check_valid_shared_heap); + BUILD_BR(app_addr_in_cache_shared_heap); - /* Get native address inside cache shared heap */ + /*--------------------------------------------------------------------- + * Finally, in the cache shared heap branch, compute the native address. + *---------------------------------------------------------------------*/ SET_BUILD_POS(app_addr_in_cache_shared_heap); if (!(maddr = LLVMBuildInBoundsGEP2( comp_ctx->builder, INT8_TYPE, func_ctx->shared_heap_base_addr_adj, @@ -548,12 +562,7 @@ aot_check_shared_heap_memory_overflow_common( } LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_cache_shared_heap, 1); - - if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { - aot_set_last_error("llvm build br failed"); - goto fail; - } - + BUILD_BR(block_maddr_phi); LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); block_curr = LLVMGetInsertBlock(comp_ctx->builder); @@ -566,24 +575,25 @@ static bool aot_check_shared_heap_memory_overflow( AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, - LLVMValueRef start_offset, LLVMValueRef mem_base_addr, uint32 bytes, - bool is_memory64, bool is_target_64bit, bool enable_segue) + LLVMValueRef maddr_phi, LLVMValueRef start_offset, + LLVMValueRef mem_base_addr, uint32 bytes, bool is_memory64, + bool is_target_64bit, bool enable_segue) { return aot_check_shared_heap_memory_overflow_common( - comp_ctx, func_ctx, block_curr, block_maddr_phi, NULL, start_offset, - NULL, mem_base_addr, bytes, is_memory64, is_target_64bit, false, - enable_segue); + comp_ctx, func_ctx, block_curr, block_maddr_phi, NULL, maddr_phi, + start_offset, NULL, mem_base_addr, bytes, is_memory64, is_target_64bit, + false, enable_segue); } static bool aot_check_bulk_memory_shared_heap_memory_overflow( AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, - LLVMBasicBlockRef check_succ, LLVMValueRef start_offset, - LLVMValueRef max_addr, bool is_memory64) + LLVMBasicBlockRef check_succ, LLVMValueRef maddr_phi, + LLVMValueRef start_offset, LLVMValueRef max_addr, bool is_memory64) { return aot_check_shared_heap_memory_overflow_common( - comp_ctx, func_ctx, block_curr, block_maddr_phi, check_succ, + comp_ctx, func_ctx, block_curr, block_maddr_phi, check_succ, maddr_phi, start_offset, max_addr, NULL, 0, is_memory64, comp_ctx->pointer_size == sizeof(uint64), true, false); } @@ -791,8 +801,8 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, SET_BUILD_POS(block_curr); if (!aot_check_shared_heap_memory_overflow( - comp_ctx, func_ctx, block_curr, block_maddr_phi, offset1, - mem_base_addr, bytes, is_memory64, is_target_64bit, + comp_ctx, func_ctx, block_curr, block_maddr_phi, maddr_phi, + offset1, mem_base_addr, bytes, is_memory64, is_target_64bit, enable_segue)) { goto fail; } @@ -1631,7 +1641,7 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (!aot_check_bulk_memory_shared_heap_memory_overflow( comp_ctx, func_ctx, block_curr, block_maddr_phi, check_succ, - offset, max_addr, is_memory64)) { + maddr_phi, offset, max_addr, is_memory64)) { goto fail; } } diff --git a/core/iwasm/interpreter/wasm_runtime.c b/core/iwasm/interpreter/wasm_runtime.c index 147f948f83..61628904b6 100644 --- a/core/iwasm/interpreter/wasm_runtime.c +++ b/core/iwasm/interpreter/wasm_runtime.c @@ -2813,6 +2813,11 @@ wasm_instantiate(WASMModule *module, WASMModuleInstance *parent, } #if WASM_ENABLE_JIT != 0 && WASM_ENABLE_SHARED_HEAP != 0 +#if UINTPTR_MAX == UINT64_MAX + module_inst->e->shared_heap_start_off.u64 = UINT64_MAX; +#else + module_inst->e->shared_heap_start_off.u32[0] = UINT32_MAX; +#endif module_inst->e->shared_heap = NULL; #endif From 2f16e41df2fe80b6b942974eafd8756b0291e820 Mon Sep 17 00:00:00 2001 From: TL Date: Tue, 15 Apr 2025 18:16:20 +0800 Subject: [PATCH 04/29] use alloca for func ctx shared heap cache value --- core/iwasm/compilation/aot_emit_memory.c | 101 ++++++++++++----------- core/iwasm/compilation/aot_llvm.c | 15 +++- 2 files changed, 66 insertions(+), 50 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 6ad0ad46be..e6f025a021 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -256,6 +256,9 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx); BUILD_STORE_PTR(field_p, value); \ } while (0) +/* Update last used shared heap info in module_extra and func_ctx + * 1. shared_heap_start_off 2. shared_heap_end_off 3. shared_heap_base_addr_adj + */ static bool aot_update_last_used_shared_heap(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, @@ -264,13 +267,10 @@ aot_update_last_used_shared_heap(AOTCompContext *comp_ctx, LLVMValueRef shared_heap_size, bool is_target_64bit) { - /* Update last used shared heap info in func_ctx and module_extra. - * 1. shared_heap_start_off 2. shared_heap_end_off - * 3. shared_heap_base_addr_adj */ - LLVMValueRef shared_heap_end_off, base_addr, shared_heap_base_addr_adj; LLVMTypeRef shared_heap_off_type = is_target_64bit ? I64_TYPE : I32_TYPE, shared_heap_off_ptr_type = is_target_64bit ? INT64_PTR_TYPE : INT32_PTR_TYPE; + LLVMValueRef shared_heap_end_off, base_addr, shared_heap_base_addr_adj; LLVMValueRef tmp_res, field_p, field_offset; uint32 offset_u32; @@ -304,10 +304,11 @@ aot_update_last_used_shared_heap(AOTCompContext *comp_ctx, BUILD_SET_MODULE_EXTRA_FIELD(INTPTR_T_PTR_TYPE, shared_heap_base_addr_adj, shared_heap_base_addr_adj); - /* FIXME: use cache shared heap instead of func_ctx, it should be phi */ - // func_ctx->shared_heap_start_off = shared_heap_start_off; - // func_ctx->shared_heap_end_off = shared_heap_end_off; - // func_ctx->shared_heap_base_addr_adj = shared_heap_base_addr_adj; + /* Store the local variable */ + BUILD_STORE_PTR(func_ctx->shared_heap_start_off, shared_heap_start_off); + BUILD_STORE_PTR(func_ctx->shared_heap_end_off, shared_heap_end_off); + BUILD_STORE_PTR(func_ctx->shared_heap_base_addr_adj, + shared_heap_base_addr_adj); return true; fail: @@ -336,11 +337,9 @@ aot_check_shared_heap_memory_overflow_common( LLVMBasicBlockRef app_addr_in_shared_heap_chain, app_addr_in_cache_shared_heap, app_addr_in_linear_mem, loopEntry, loopCond, loopBody, loopExit, check_valid_shared_heap; - LLVMValueRef addr = NULL, maddr = NULL, max_offset = NULL, - init_value = NULL; - LLVMValueRef cmp = NULL, cmp1 = NULL, cmp2 = NULL; - LLVMValueRef shared_heap_start_off = NULL, shared_heap_check_bound = NULL, - shared_heap_size = NULL; + LLVMValueRef addr = NULL, maddr = NULL, max_offset = NULL, cmp, cmp1, cmp2; + LLVMValueRef shared_heap_start_off, shared_heap_check_bound, + shared_heap_size, shared_heap_end_off, shared_heap_base_addr_adj; /* The pointer that will traverse the shared heap chain */ LLVMValueRef phi_shared_heap = NULL, cur_shared_heap = NULL; LLVMValueRef field_offset, field_p, length; @@ -417,16 +416,33 @@ aot_check_shared_heap_memory_overflow_common( /* Add loop basic blocks */ ADD_BASIC_BLOCK(loopEntry, "loop_entry"); - ADD_BASIC_BLOCK(loopCond, "loop_cond"); ADD_BASIC_BLOCK(loopBody, "loop_body"); + ADD_BASIC_BLOCK(loopCond, "loop_cond"); ADD_BASIC_BLOCK(loopExit, "loop_exit"); LLVMMoveBasicBlockAfter(loopEntry, app_addr_in_shared_heap_chain); - LLVMMoveBasicBlockAfter(loopCond, loopEntry); - LLVMMoveBasicBlockAfter(loopBody, loopCond); - LLVMMoveBasicBlockAfter(loopExit, loopBody); + LLVMMoveBasicBlockAfter(loopBody, loopEntry); + LLVMMoveBasicBlockAfter(loopCond, loopBody); + LLVMMoveBasicBlockAfter(loopExit, loopCond); + + /* Check if the app address is in the cache shared heap range. + * If yes, branch to the cache branch; if not, proceed into the loop */ + /* Load the local variable */ + BUILD_LOAD_PTR(func_ctx->shared_heap_start_off, + is_target_64bit ? I64_TYPE : I32_TYPE, + shared_heap_start_off); + BUILD_LOAD_PTR(func_ctx->shared_heap_end_off, + is_target_64bit ? I64_TYPE : I32_TYPE, shared_heap_end_off); + BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, + "cmp_cache_shared_heap_start"); + BUILD_GET_MAX_ACCESS_OFFSET(max_offset); + BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_end_off, cmp1, + "cmp_cache_shared_heap_end"); + BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); + BUILD_COND_BR(cmp2, app_addr_in_cache_shared_heap, loopEntry); - /* Early check: if app_offset is not in the overall shared heap chain, - * branch to linear memory. */ + /* Check whether app_offset is not in the overall shared heap chain, + * if not branch to linear memory. */ + SET_BUILD_POS(loopEntry); BUILD_GET_SHARED_HEAP_START(func_ctx->shared_heap, shared_heap_start_off); BUILD_GET_MAX_SHARED_HEAP_BOUND(shared_heap_check_bound); BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, @@ -434,34 +450,20 @@ aot_check_shared_heap_memory_overflow_common( BUILD_ICMP(LLVMIntULE, bulk_memory ? max_offset : start_offset, shared_heap_check_bound, cmp1, "cmp_shared_heap_chain_end"); BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); - BUILD_COND_BR(cmp2, loopEntry, app_addr_in_linear_mem); - - /* In loopEntry: Check if the app address is in the cache shared heap range. - * If yes, branch to the cache branch; if not, proceed into the loop. */ - SET_BUILD_POS(loopEntry); - BUILD_ICMP(LLVMIntUGE, start_offset, func_ctx->shared_heap_start_off, cmp, - "cmp_cache_shared_heap_start"); - BUILD_GET_MAX_ACCESS_OFFSET(max_offset); - BUILD_ICMP(LLVMIntULE, max_offset, func_ctx->shared_heap_end_off, cmp1, - "cmp_cache_shared_heap_end"); - BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); - BUILD_COND_BR(cmp2, app_addr_in_cache_shared_heap, loopCond); + BUILD_COND_BR(cmp2, loopBody, app_addr_in_linear_mem); /*--------------------------------------------------------------------- - * Loop header (loopCond): create a phi node to merge the pointer values. + * LoopBody: create a phi node to merge the pointer values. * - Incoming from loopEntry: the initial pointer is func_ctx->shared_heap. * - Incoming from loopBody: the updated pointer (next pointer in the - * chain). + * chain). *---------------------------------------------------------------------*/ - SET_BUILD_POS(loopCond); + SET_BUILD_POS(loopBody); phi_shared_heap = LLVMBuildPhi(comp_ctx->builder, INT8_PTR_TYPE, "phi_shared_heap"); - init_value = func_ctx->shared_heap; - LLVMAddIncoming(phi_shared_heap, &init_value, &loopEntry, 1); - - /* In loopCond, we check whether the current shared heap node - * (phi_shared_heap) contains the target address. - */ + LLVMAddIncoming(phi_shared_heap, &func_ctx->shared_heap, &loopEntry, 1); + /* In loopBody, we check whether the current shared heap node + * (phi_shared_heap) contains the target address. */ BUILD_GET_SHARED_HEAP_START(phi_shared_heap, shared_heap_start_off); BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, "cmp_shared_heap_start"); @@ -480,19 +482,19 @@ aot_check_shared_heap_memory_overflow_common( BUILD_ICMP(LLVMIntULE, start_offset, shared_heap_check_bound, cmp1, "cmp_shared_heap_end"); BUILD_OP(And, cmp, cmp1, cmp2, "is_in_shared_heap"); - BUILD_COND_BR(cmp2, loopExit, loopBody); + BUILD_COND_BR(cmp2, loopExit, loopCond); /*--------------------------------------------------------------------- - * Loop body: update the pointer to traverse to the next shared heap in the - *chain. The updated pointer is then added as an incoming value to the phi - *node. + * Loop cond: update the pointer to traverse to the next shared heap in the + * chain. The updated pointer is then added as an incoming value to the phi + * node. *---------------------------------------------------------------------*/ - SET_BUILD_POS(loopBody); + SET_BUILD_POS(loopCond); BUILD_GET_SHARED_HEAP_FIELD(phi_shared_heap, chain_next, INT8_PTR_TYPE, cur_shared_heap); /* Add the new value from loopBody as an incoming edge to the phi node */ - LLVMAddIncoming(phi_shared_heap, &cur_shared_heap, &loopBody, 1); - BUILD_BR(loopCond); + LLVMAddIncoming(phi_shared_heap, &cur_shared_heap, &loopCond, 1); + BUILD_BR(loopBody); /*--------------------------------------------------------------------- * Loop exit: at this point, phi_shared_heap is expected to be valid if the @@ -529,8 +531,11 @@ aot_check_shared_heap_memory_overflow_common( * Finally, in the cache shared heap branch, compute the native address. *---------------------------------------------------------------------*/ SET_BUILD_POS(app_addr_in_cache_shared_heap); + /* load the local variable */ + BUILD_LOAD_PTR(func_ctx->shared_heap_base_addr_adj, INT8_PTR_TYPE, + shared_heap_base_addr_adj); if (!(maddr = LLVMBuildInBoundsGEP2( - comp_ctx->builder, INT8_TYPE, func_ctx->shared_heap_base_addr_adj, + comp_ctx->builder, INT8_TYPE, shared_heap_base_addr_adj, &start_offset, 1, "maddr_cache_shared_heap"))) { aot_set_last_error("llvm build inbounds gep failed"); goto fail; diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 1f92d26a11..69a336cb65 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1528,19 +1528,30 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, aot_set_last_error("llvm build inbounds gep failed"); \ return false; \ } \ - if (!(func_ctx->field = \ + if (!(load_val = \ LLVMBuildLoad2(comp_ctx->builder, type, field_p, #field))) { \ aot_set_last_error("llvm build load failed"); \ return false; \ } \ + if (!(func_ctx->field = \ + LLVMBuildAlloca(comp_ctx->builder, type, #field))) { \ + aot_set_last_error("llvm build load failed"); \ + return false; \ + } \ + if (!LLVMBuildStore(comp_ctx->builder, load_val, func_ctx->field)) { \ + aot_set_last_error("llvm build load failed"); \ + return false; \ + } \ } while (0) static bool create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { - LLVMValueRef offset, field_p; + LLVMValueRef offset, field_p, load_val; uint32 offset_u32; + /* shared_heap_base_addr_adj, shared_heap_start_off, and shared_heap_end_off + * can be updated later, use local variable to represent them */ LOAD_MODULE_EXTRA_FIELD(shared_heap_base_addr_adj, INT8_PTR_TYPE); LOAD_MODULE_EXTRA_FIELD( shared_heap_start_off, From b58c75c1dabddc5ae3ace634605b87278e606447 Mon Sep 17 00:00:00 2001 From: TL Date: Tue, 15 Apr 2025 21:41:13 +0800 Subject: [PATCH 05/29] use correct alloca for func ctx shared heap cache value --- core/iwasm/compilation/aot_emit_memory.c | 2 +- core/iwasm/compilation/aot_llvm.c | 30 ++++++++++++++++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index e6f025a021..7be42ec4bd 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -237,7 +237,7 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx); is_target_64bit ? I64_CONST(bytes - 1) : I32_CONST(bytes - 1); \ CHECK_LLVM_CONST(length); \ BUILD_OP(Add, start_offset, length, max_offset, \ - "cache_shared_heap_bound"); \ + "max_access_offset"); \ } \ } while (0) diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 69a336cb65..8e6d7bdff3 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1517,7 +1517,7 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; } -#define LOAD_MODULE_EXTRA_FIELD(field, type) \ +#define LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(field, type) \ do { \ get_module_extra_field_offset(field); \ offset = I32_CONST(offset_u32); \ @@ -1544,6 +1544,24 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } \ } while (0) +#define LOAD_MODULE_EXTRA_FIELD(field, type) \ + do { \ + get_module_extra_field_offset(field); \ + offset = I32_CONST(offset_u32); \ + CHECK_LLVM_CONST(offset); \ + if (!(field_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, \ + func_ctx->aot_inst, &offset, 1, \ + #field "_p"))) { \ + aot_set_last_error("llvm build inbounds gep failed"); \ + return false; \ + } \ + if (!(func_ctx->field = \ + LLVMBuildLoad2(comp_ctx->builder, type, field_p, #field))) { \ + aot_set_last_error("llvm build load failed"); \ + return false; \ + } \ + } while (0) + static bool create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { @@ -1552,13 +1570,15 @@ create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) /* shared_heap_base_addr_adj, shared_heap_start_off, and shared_heap_end_off * can be updated later, use local variable to represent them */ - LOAD_MODULE_EXTRA_FIELD(shared_heap_base_addr_adj, INT8_PTR_TYPE); - LOAD_MODULE_EXTRA_FIELD( + LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(shared_heap_base_addr_adj, + INT8_PTR_TYPE); + LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA( shared_heap_start_off, comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE); - LOAD_MODULE_EXTRA_FIELD( + LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA( shared_heap_end_off, comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE); + /* Shared Heap won't be updated, no need to alloca */ LOAD_MODULE_EXTRA_FIELD(shared_heap, INT8_PTR_TYPE); return true; @@ -2418,7 +2438,7 @@ jit_stack_size_callback(void *user_data, const char *name, size_t namelen, stack_consumption_to_call_wrapped_func = musttail ? 0 : aot_estimate_stack_usage_for_function_call( - comp_ctx, func_ctx->aot_func->func_type); + comp_ctx, func_ctx->aot_func->func_type); LOG_VERBOSE("func %.*s stack %u + %zu + %u", (int)namelen, name, stack_consumption_to_call_wrapped_func, stack_size, call_size); From 5de0ca32429fdeaf435abd64c0bb7bec839cedc2 Mon Sep 17 00:00:00 2001 From: TL Date: Thu, 17 Apr 2025 11:44:39 +0800 Subject: [PATCH 06/29] enable shared heap chain aot test and bug fix --- core/iwasm/compilation/aot_emit_memory.c | 73 +++++++++------------- core/iwasm/compilation/aot_llvm.c | 4 +- samples/shared-heap/wasm-apps/test1.c | 4 ++ samples/shared-heap/wasm-apps/test2.c | 8 ++- tests/unit/shared-heap/shared_heap_test.cc | 30 ++++----- 5 files changed, 52 insertions(+), 67 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 7be42ec4bd..c87227e493 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -175,15 +175,21 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx); } \ } while (0) -#define BUILD_GET_SHARED_HEAP_FIELD(shared_heap_p, field, data_type, res) \ - do { \ - offset_u32 = offsetof(WASMSharedHeap, field); \ - field_offset = I32_CONST(offset_u32); \ - CHECK_LLVM_CONST(field_offset); \ - \ - BUILD_FIELD_PTR(shared_heap_p, field_offset, field_p, \ - "shared_heap" #field); \ - BUILD_LOAD_PTR(field_p, data_type, res); \ +#define BUILD_GET_SHARED_HEAP_FIELD(shared_heap_p, field, data_type, res) \ + do { \ + offset_u32 = offsetof(WASMSharedHeap, field); \ + field_offset = I32_CONST(offset_u32); \ + CHECK_LLVM_CONST(field_offset); \ + \ + BUILD_FIELD_PTR(shared_heap_p, field_offset, field_p, \ + "shared_heap" #field); \ + if (!(field_p = LLVMBuildBitCast(comp_ctx->builder, field_p, \ + LLVMPointerType(data_type, 0), \ + "shared_heap_" #field "_cast_p"))) { \ + aot_set_last_error("llvm build bit cast failed."); \ + goto fail; \ + } \ + BUILD_LOAD_PTR(field_p, data_type, res); \ } while (0) #define BUILD_GET_SHARED_HEAP_START(shared_heap_p, res) \ @@ -199,6 +205,13 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx); \ BUILD_FIELD_PTR(shared_heap_p, field_offset, field_p, \ "shared_heap_start_off_p"); \ + if (!(field_p = LLVMBuildBitCast(comp_ctx->builder, field_p, \ + is_target_64bit ? INT64_PTR_TYPE \ + : INT32_PTR_TYPE, \ + "shared_heap_start_cast_p"))) { \ + aot_set_last_error("llvm build bit cast failed."); \ + goto fail; \ + } \ BUILD_LOAD_PTR(field_p, is_target_64bit ? I64_TYPE : I32_TYPE, res); \ /* For bulk memory on 32 bits platform , it's always 64 bit and needs \ * to extend */ \ @@ -241,22 +254,8 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx); } \ } while (0) -#define BUILD_SET_MODULE_EXTRA_FIELD(ptr_type, field, value) \ - do { \ - get_module_extra_field_offset(field); \ - field_offset = I32_CONST(offset_u32); \ - CHECK_LLVM_CONST(field_offset); \ - \ - BUILD_FIELD_PTR(shared_heap_p, field_offset, field_p, #field); \ - if (!(field_p = LLVMBuildBitCast(comp_ctx->builder, field_p, ptr_type, \ - "module_extra" #field "_p"))) { \ - aot_set_last_error("llvm build bit cast failed."); \ - goto fail; \ - } \ - BUILD_STORE_PTR(field_p, value); \ - } while (0) - -/* Update last used shared heap info in module_extra and func_ctx +/* Update last used shared heap info in func_ctx, let runtime api update module + * inst shared heap info * 1. shared_heap_start_off 2. shared_heap_end_off 3. shared_heap_base_addr_adj */ static bool @@ -297,13 +296,6 @@ aot_update_last_used_shared_heap(AOTCompContext *comp_ctx, goto fail; } - BUILD_SET_MODULE_EXTRA_FIELD(shared_heap_off_ptr_type, - shared_heap_start_off, shared_heap_start_off); - BUILD_SET_MODULE_EXTRA_FIELD(shared_heap_off_ptr_type, shared_heap_end_off, - shared_heap_end_off); - BUILD_SET_MODULE_EXTRA_FIELD(INTPTR_T_PTR_TYPE, shared_heap_base_addr_adj, - shared_heap_base_addr_adj); - /* Store the local variable */ BUILD_STORE_PTR(func_ctx->shared_heap_start_off, shared_heap_start_off); BUILD_STORE_PTR(func_ctx->shared_heap_end_off, shared_heap_end_off); @@ -336,12 +328,12 @@ aot_check_shared_heap_memory_overflow_common( { LLVMBasicBlockRef app_addr_in_shared_heap_chain, app_addr_in_cache_shared_heap, app_addr_in_linear_mem, loopEntry, - loopCond, loopBody, loopExit, check_valid_shared_heap; + loopCond, loopBody, loopExit; LLVMValueRef addr = NULL, maddr = NULL, max_offset = NULL, cmp, cmp1, cmp2; LLVMValueRef shared_heap_start_off, shared_heap_check_bound, shared_heap_size, shared_heap_end_off, shared_heap_base_addr_adj; /* The pointer that will traverse the shared heap chain */ - LLVMValueRef phi_shared_heap = NULL, cur_shared_heap = NULL; + LLVMValueRef phi_shared_heap, cur_shared_heap; LLVMValueRef field_offset, field_p, length; uint32 offset_u32; @@ -490,6 +482,8 @@ aot_check_shared_heap_memory_overflow_common( * node. *---------------------------------------------------------------------*/ SET_BUILD_POS(loopCond); + /* TODO: for happy path it will always be valid since we checked the address + * in shared heap chain */ BUILD_GET_SHARED_HEAP_FIELD(phi_shared_heap, chain_next, INT8_PTR_TYPE, cur_shared_heap); /* Add the new value from loopBody as an incoming edge to the phi node */ @@ -501,17 +495,6 @@ aot_check_shared_heap_memory_overflow_common( * app address is contained in a shared heap; otherwise, it is NULL. *---------------------------------------------------------------------*/ SET_BUILD_POS(loopExit); - BUILD_IS_NOT_NULL(phi_shared_heap, cmp, "has_shared_heap"); - - ADD_BASIC_BLOCK(check_valid_shared_heap, "check_valid_shared_heap"); - LLVMMoveBasicBlockAfter(check_valid_shared_heap, loopExit); - if (!aot_emit_exception(comp_ctx, func_ctx, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, - check_valid_shared_heap)) { - goto fail; - } - SET_BUILD_POS(check_valid_shared_heap); - /* Update last accessed shared heap, the shared_heap_size and * shared_heap_start_off is already prepared in loop body. * For bulk memory on 32 bits platform, it extends to i64 before, so it diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 8e6d7bdff3..6294c061ca 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1535,11 +1535,11 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } \ if (!(func_ctx->field = \ LLVMBuildAlloca(comp_ctx->builder, type, #field))) { \ - aot_set_last_error("llvm build load failed"); \ + aot_set_last_error("llvm build alloca failed"); \ return false; \ } \ if (!LLVMBuildStore(comp_ctx->builder, load_val, func_ctx->field)) { \ - aot_set_last_error("llvm build load failed"); \ + aot_set_last_error("llvm build store failed"); \ return false; \ } \ } while (0) diff --git a/samples/shared-heap/wasm-apps/test1.c b/samples/shared-heap/wasm-apps/test1.c index 8ed1ca84cc..321f102185 100644 --- a/samples/shared-heap/wasm-apps/test1.c +++ b/samples/shared-heap/wasm-apps/test1.c @@ -62,6 +62,10 @@ my_shared_heap_free(void *ptr) void * produce_str(char *addr, uint32_t index) { + char c; snprintf(addr, 512, "Data: %u stores to pre-allocated shared heap", index); + /* Actually access it in wasm */ + c = addr[0]; + printf("In WASM: the first char is %c\n", c); return addr; } diff --git a/samples/shared-heap/wasm-apps/test2.c b/samples/shared-heap/wasm-apps/test2.c index 36ae748b47..14cf0c9c38 100644 --- a/samples/shared-heap/wasm-apps/test2.c +++ b/samples/shared-heap/wasm-apps/test2.c @@ -19,7 +19,9 @@ print_buf(char *buf) void consume_str(char *buf) { - printf("wasm app2's wasm func received buf in pre-allocated shared buf: " - "%s\n\n", - buf); + /* Actually access it in wasm */ + char c = buf[0]; + printf("In WASM: wasm app2's wasm func received buf in pre-allocated shared buf: " + "%s with its first char is %c\n\n", + buf, c); } diff --git a/tests/unit/shared-heap/shared_heap_test.cc b/tests/unit/shared-heap/shared_heap_test.cc index 626f0d2d5d..2de8b00985 100644 --- a/tests/unit/shared-heap/shared_heap_test.cc +++ b/tests/unit/shared-heap/shared_heap_test.cc @@ -256,8 +256,6 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw) EXPECT_EQ(0, argv[0]); EXPECT_EQ(preallocated_buf2[0], 129); - /* TODO: test aot when chain is supported in AOT */ - /* argv[0] = start1; argv[1] = 98; test_shared_heap(shared_heap_chain, "test.aot", "read_modify_write_8", 2, @@ -271,7 +269,6 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw) argv); EXPECT_EQ(0, argv[0]); EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 81); - */ } TEST_F(shared_heap_test, test_shared_heap_chain_rmw_oob) @@ -318,12 +315,11 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw_oob) "read_modify_write_16", 2, argv), "Exception: out of bounds memory access"); - /* TODO: test aot when chain is supported in AOT */ - /*argv[0] = end1; + argv[0] = end1; argv[1] = 12025; EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test.wasm", "read_modify_write_16", 2, argv), - "Exception: out of bounds memory access");*/ + "Exception: out of bounds memory access"); } #ifndef native_function @@ -453,9 +449,8 @@ TEST_F(shared_heap_test, test_shared_heap_chain) test_shared_heap(shared_heap_chain, "test_addr_conv.wasm", "test", 0, argv); EXPECT_EQ(1, argv[0]); - /* TODO: test aot when chain is supported in AOT */ - /*test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 1, argv); - EXPECT_EQ(1, argv[0]);*/ + test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 0, argv); + EXPECT_EQ(1, argv[0]); } TEST_F(shared_heap_test, test_shared_heap_chain_create_fail) @@ -666,14 +661,15 @@ TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv) "test_preallocated", 1, argv); EXPECT_EQ(1, argv[0]); - /* TODO: test aot when chain is supported in AOT */ - /*argv[0] = 0xFFFFFFFF; - test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 1, argv); + argv[0] = 0xFFFFFFFF; + test_shared_heap(shared_heap, "test_addr_conv.aot", "test_preallocated", 1, + argv); EXPECT_EQ(1, argv[0]); argv[0] = 0xFFFFF000; - test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 1, argv); - EXPECT_EQ(1, argv[0]); */ + test_shared_heap(shared_heap, "test_addr_conv.aot", "test_preallocated", 1, + argv); + EXPECT_EQ(1, argv[0]); } TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv_oob) @@ -719,10 +715,10 @@ TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv_oob) "test_preallocated", 1, argv), "Exception: out of bounds memory access"); - /* TODO: test aot when chain is supported in AOT */ - /*argv[0] = 0xFFFFFFFF - BUF_SIZE - 4096; + /* test aot */ + argv[0] = 0xFFFFFFFF - BUF_SIZE - 4096; EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test_addr_conv.aot", "test_preallocated", 1, argv), - "Exception: out of bounds memory access");*/ + "Exception: out of bounds memory access"); } From 106732b06081096c89bfcd811e34e3d9276d0c82 Mon Sep 17 00:00:00 2001 From: TL Date: Thu, 17 Apr 2025 18:54:00 +0800 Subject: [PATCH 07/29] Fix a missing argument on 32bits platform, still has the shared heap chain iteration problem --- core/iwasm/common/wasm_memory.c | 6 ++++-- core/iwasm/compilation/aot_emit_memory.c | 14 +++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/core/iwasm/common/wasm_memory.c b/core/iwasm/common/wasm_memory.c index a67c4209ea..ae5f98db9c 100644 --- a/core/iwasm/common/wasm_memory.c +++ b/core/iwasm/common/wasm_memory.c @@ -1227,7 +1227,9 @@ wasm_runtime_addr_native_to_app(WASMModuleInstanceCommon *module_inst_comm, #if WASM_ENABLE_SHARED_HEAP != 0 if (is_native_addr_in_shared_heap(module_inst_comm, memory_inst->is_memory64, addr, 1)) { - return addr - get_last_used_shared_heap_base_addr_adj(module_inst_comm); + return (uint64)(uintptr_t)(addr + - get_last_used_shared_heap_base_addr_adj( + module_inst_comm)); } #endif @@ -1349,7 +1351,7 @@ wasm_check_app_addr_and_convert(WASMModuleInstance *module_inst, bool is_str, (WASMModuleInstanceCommon *)module_inst); shared_heap_end_off = get_last_used_shared_heap_end_offset( (WASMModuleInstanceCommon *)module_inst); - native_addr = shared_heap_base_addr_adj + app_buf_addr; + native_addr = shared_heap_base_addr_adj + (uintptr_t)app_buf_addr; /* The whole string must be in the shared heap */ str = (const char *)native_addr; diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index c87227e493..23268afbc7 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -321,7 +321,7 @@ static bool aot_check_shared_heap_memory_overflow_common( AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, - LLVMBasicBlockRef check_succ, LLVMValueRef maddr_phi, + LLVMBasicBlockRef check_succ, LLVMValueRef maddr_phi, LLVMValueRef addr, LLVMValueRef start_offset, LLVMValueRef max_addr, LLVMValueRef mem_base_addr, uint32 bytes, bool is_memory64, bool is_target_64bit, bool bulk_memory, bool enable_segue) @@ -329,7 +329,7 @@ aot_check_shared_heap_memory_overflow_common( LLVMBasicBlockRef app_addr_in_shared_heap_chain, app_addr_in_cache_shared_heap, app_addr_in_linear_mem, loopEntry, loopCond, loopBody, loopExit; - LLVMValueRef addr = NULL, maddr = NULL, max_offset = NULL, cmp, cmp1, cmp2; + LLVMValueRef maddr = NULL, max_offset = NULL, cmp, cmp1, cmp2; LLVMValueRef shared_heap_start_off, shared_heap_check_bound, shared_heap_size, shared_heap_end_off, shared_heap_base_addr_adj; /* The pointer that will traverse the shared heap chain */ @@ -563,12 +563,12 @@ static bool aot_check_shared_heap_memory_overflow( AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, - LLVMValueRef maddr_phi, LLVMValueRef start_offset, + LLVMValueRef maddr_phi, LLVMValueRef addr, LLVMValueRef start_offset, LLVMValueRef mem_base_addr, uint32 bytes, bool is_memory64, bool is_target_64bit, bool enable_segue) { return aot_check_shared_heap_memory_overflow_common( - comp_ctx, func_ctx, block_curr, block_maddr_phi, NULL, maddr_phi, + comp_ctx, func_ctx, block_curr, block_maddr_phi, NULL, maddr_phi, addr, start_offset, NULL, mem_base_addr, bytes, is_memory64, is_target_64bit, false, enable_segue); } @@ -582,7 +582,7 @@ aot_check_bulk_memory_shared_heap_memory_overflow( { return aot_check_shared_heap_memory_overflow_common( comp_ctx, func_ctx, block_curr, block_maddr_phi, check_succ, maddr_phi, - start_offset, max_addr, NULL, 0, is_memory64, + NULL, start_offset, max_addr, NULL, 0, is_memory64, comp_ctx->pointer_size == sizeof(uint64), true, false); } @@ -790,8 +790,8 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (!aot_check_shared_heap_memory_overflow( comp_ctx, func_ctx, block_curr, block_maddr_phi, maddr_phi, - offset1, mem_base_addr, bytes, is_memory64, is_target_64bit, - enable_segue)) { + addr, offset1, mem_base_addr, bytes, is_memory64, + is_target_64bit, enable_segue)) { goto fail; } } From eaf2020ea03cbc149f3fe793b4711356bc9e6ee5 Mon Sep 17 00:00:00 2001 From: TL Date: Fri, 18 Apr 2025 09:53:57 +0800 Subject: [PATCH 08/29] Fix shared heap chain iteration problem on 32bits platform --- core/iwasm/aot/aot_runtime.c | 6 ++++++ core/iwasm/interpreter/wasm_runtime.h | 8 ++++---- samples/shared-heap/CMakeLists.txt | 18 ++++++++++++------ samples/shared-heap/src/shared_heap_chain.c | 4 ++-- tests/unit/shared-heap/CMakeLists.txt | 6 +++++- .../unit/shared-heap/wasm-apps/CMakeLists.txt | 10 ++++++++-- 6 files changed, 37 insertions(+), 15 deletions(-) diff --git a/core/iwasm/aot/aot_runtime.c b/core/iwasm/aot/aot_runtime.c index 24c4a2b6ef..8ec6260f28 100644 --- a/core/iwasm/aot/aot_runtime.c +++ b/core/iwasm/aot/aot_runtime.c @@ -63,6 +63,12 @@ bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_start_off) == 16); bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap_end_off) == 24); bh_static_assert(offsetof(AOTModuleInstanceExtra, shared_heap) == 32); +bh_static_assert(offsetof(WASMSharedHeap, chain_next) == 8); +bh_static_assert(offsetof(WASMSharedHeap, base_addr) == 24); +bh_static_assert(offsetof(WASMSharedHeap, size) == 32); +bh_static_assert(offsetof(WASMSharedHeap, start_off_mem64) == 40); +bh_static_assert(offsetof(WASMSharedHeap, start_off_mem32) == 48); + bh_static_assert(sizeof(CApiFuncImport) == sizeof(uintptr_t) * 3); bh_static_assert(sizeof(wasm_val_t) == 16); diff --git a/core/iwasm/interpreter/wasm_runtime.h b/core/iwasm/interpreter/wasm_runtime.h index 8d8e640b49..b1c28e85c2 100644 --- a/core/iwasm/interpreter/wasm_runtime.h +++ b/core/iwasm/interpreter/wasm_runtime.h @@ -95,13 +95,13 @@ typedef union { typedef struct WASMSharedHeap { /* The global shared heap list maintained in runtime, used for runtime * destroy */ - struct WASMSharedHeap *next; + DefPointer(struct WASMSharedHeap *, next); /* The logical shared heap chain the shared heap in */ - struct WASMSharedHeap *chain_next; + DefPointer(struct WASMSharedHeap *, chain_next); /* Will be null if shared heap is created from pre allocated memory chunk * and don't need to dynamic malloc and free */ - void *heap_handle; - uint8 *base_addr; + DefPointer(void *, heap_handle); + DefPointer(uint8 *, base_addr); uint64 size; uint64 start_off_mem64; uint64 start_off_mem32; diff --git a/samples/shared-heap/CMakeLists.txt b/samples/shared-heap/CMakeLists.txt index 5be99d3a22..260d3e33cc 100644 --- a/samples/shared-heap/CMakeLists.txt +++ b/samples/shared-heap/CMakeLists.txt @@ -111,21 +111,27 @@ if (WAMR_BUILD_AOT EQUAL 1) ) if (WAMR_COMPILER) message (CHECK_PASS "found") - else() + else () message (CHECK_FAIL "not found") - endif() + endif () if (NOT EXISTS ${WAMR_COMPILER}) message (FATAL_ERROR "Please build wamrc under ${WAMR_ROOT_DIR}/wamr-compiler") - else() + else () message (STATUS "WAMR_COMPILER is ${WAMR_COMPILER}") - endif() + endif () + + if (WAMR_BUILD_TARGET STREQUAL "X86_32") + set (WAMR_COMPILER_FLAGS --enable-shared-heap --target=i386) + else () + set (WAMR_COMPILER_FLAGS --enable-shared-heap) + endif () add_custom_target( wasm_to_aot ALL DEPENDS wasm-apps/test1.wasm wasm-apps/test2.wasm ${WAMR_COMPILER} - COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test1.aot wasm-apps/test1.wasm - COMMAND ${WAMR_COMPILER} --enable-shared-heap -o wasm-apps/test2.aot wasm-apps/test2.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_FLAGS} -o wasm-apps/test1.aot wasm-apps/test1.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_FLAGS} -o wasm-apps/test2.aot wasm-apps/test2.wasm WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) endif() diff --git a/samples/shared-heap/src/shared_heap_chain.c b/samples/shared-heap/src/shared_heap_chain.c index beb17b137e..4b5c2f902a 100644 --- a/samples/shared-heap/src/shared_heap_chain.c +++ b/samples/shared-heap/src/shared_heap_chain.c @@ -34,7 +34,7 @@ produce_data(wasm_module_inst_t module_inst, wasm_exec_env_t exec_env, /* Passes wasm address directly between wasm apps since memory in shared * heap chain is viewed as single address space in wasm's perspective */ - buf = (uint8 *)(uint64)argv[0]; + buf = (uint8 *)(uintptr_t)argv[0]; if (!bh_post_msg(queue, 1, buf, buf_size)) { printf("Failed to post message to queue\n"); if (free_on_fail) @@ -130,7 +130,7 @@ wasm_consumer(wasm_module_inst_t module_inst, bh_queue *queue) buf = bh_message_payload(msg); /* call wasm function */ - argv[0] = (uint32)(uint64)buf; + argv[0] = (uint32)(uintptr_t)buf; if (i < 8) wasm_runtime_call_wasm(exec_env, print_buf_func, 1, argv); else diff --git a/tests/unit/shared-heap/CMakeLists.txt b/tests/unit/shared-heap/CMakeLists.txt index 2b06c537f8..b7859bd06d 100644 --- a/tests/unit/shared-heap/CMakeLists.txt +++ b/tests/unit/shared-heap/CMakeLists.txt @@ -12,7 +12,11 @@ set(WAMR_BUILD_AOT 1) set(WAMR_BUILD_INTERP 1) set(WAMR_BUILD_FAST_INTERP 1) set(WAMR_BUILD_JIT 0) -set(WAMR_BUILD_MEMORY64 1) +if(WAMR_BUILD_TARGET STREQUAL "X86_32") + set(WAMR_BUILD_MEMORY64 0) +else() + set(WAMR_BUILD_MEMORY64 1) +endif() set(WAMR_BUILD_SHARED_HEAP 1) # Compile wasm modules diff --git a/tests/unit/shared-heap/wasm-apps/CMakeLists.txt b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt index 097f66ae5d..f66ee1b68c 100644 --- a/tests/unit/shared-heap/wasm-apps/CMakeLists.txt +++ b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt @@ -32,6 +32,12 @@ set(CMAKE_EXE_LINKER_FLAGS add_executable(test.wasm test.c) target_link_libraries(test.wasm) +if (WAMR_BUILD_TARGET STREQUAL "X86_32") + set (WAMR_COMPILER_FLAGS --opt-level=0 --bounds-checks=1 --enable-shared-heap --target=i386) +else () + set (WAMR_COMPILER_FLAGS --opt-level=0 --bounds-checks=1 --enable-shared-heap) +endif () + add_custom_command(TARGET test.wasm POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/test.wasm @@ -40,7 +46,7 @@ add_custom_command(TARGET test.wasm POST_BUILD ) add_custom_command(TARGET test.wasm POST_BUILD - COMMAND ${WAMRC_ROOT_DIR}/wamrc --opt-level=0 --enable-shared-heap --bounds-checks=1 + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} -o test.aot test.wasm @@ -61,7 +67,7 @@ add_custom_command(TARGET test_addr_conv.wasm POST_BUILD ) add_custom_command(TARGET test_addr_conv.wasm POST_BUILD - COMMAND ${WAMRC_ROOT_DIR}/wamrc --opt-level=0 --enable-shared-heap --bounds-checks=1 + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} -o test_addr_conv.aot test_addr_conv.wasm From ae9d5ea45d5e207cdb41472dfe607527f0ed484f Mon Sep 17 00:00:00 2001 From: TL Date: Mon, 21 Apr 2025 16:06:40 +0800 Subject: [PATCH 09/29] fix AOT bulk memory bounds checks compliation issue --- core/iwasm/compilation/aot_emit_memory.c | 83 +++++++++++++----------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 23268afbc7..49d467e4a4 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -192,35 +192,34 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx); BUILD_LOAD_PTR(field_p, data_type, res); \ } while (0) -#define BUILD_GET_SHARED_HEAP_START(shared_heap_p, res) \ - do { \ - if (is_memory64) { \ - offset_u32 = offsetof(WASMSharedHeap, start_off_mem64); \ - } \ - else { \ - offset_u32 = offsetof(WASMSharedHeap, start_off_mem32); \ - } \ - field_offset = I32_CONST(offset_u32); \ - CHECK_LLVM_CONST(field_offset); \ - \ - BUILD_FIELD_PTR(shared_heap_p, field_offset, field_p, \ - "shared_heap_start_off_p"); \ - if (!(field_p = LLVMBuildBitCast(comp_ctx->builder, field_p, \ - is_target_64bit ? INT64_PTR_TYPE \ - : INT32_PTR_TYPE, \ - "shared_heap_start_cast_p"))) { \ - aot_set_last_error("llvm build bit cast failed."); \ - goto fail; \ - } \ - BUILD_LOAD_PTR(field_p, is_target_64bit ? I64_TYPE : I32_TYPE, res); \ - /* For bulk memory on 32 bits platform , it's always 64 bit and needs \ - * to extend */ \ - if (bulk_memory && !is_target_64bit \ - && !(res = LLVMBuildZExt(comp_ctx->builder, res, I64_TYPE, \ - "shared_heap_start_off_u64"))) { \ - aot_set_last_error("llvm build zero ext failed."); \ - goto fail; \ - } \ +#define BUILD_GET_SHARED_HEAP_START(shared_heap_p, res) \ + do { \ + if (is_memory64) { \ + offset_u32 = offsetof(WASMSharedHeap, start_off_mem64); \ + } \ + else { \ + offset_u32 = offsetof(WASMSharedHeap, start_off_mem32); \ + } \ + field_offset = I32_CONST(offset_u32); \ + CHECK_LLVM_CONST(field_offset); \ + \ + BUILD_FIELD_PTR(shared_heap_p, field_offset, field_p, \ + "shared_heap_start_off_p"); \ + if (!(field_p = LLVMBuildBitCast(comp_ctx->builder, field_p, \ + is_target_64bit ? INT64_PTR_TYPE \ + : INT32_PTR_TYPE, \ + "shared_heap_start_cast_p"))) { \ + aot_set_last_error("llvm build bit cast failed."); \ + goto fail; \ + } \ + BUILD_LOAD_PTR(field_p, is_target_64bit ? I64_TYPE : I32_TYPE, res); \ + /* For bulk memory on 32 bits platform, always extend to 64 bits */ \ + if (bulk_memory && !is_target_64bit \ + && !(res = LLVMBuildZExt(comp_ctx->builder, res, I64_TYPE, \ + "shared_heap_start_off_u64"))) { \ + aot_set_last_error("llvm build zero ext failed."); \ + goto fail; \ + } \ } while (0) #define BUILD_GET_MAX_SHARED_HEAP_BOUND(shared_heap_check_bound) \ @@ -243,9 +242,10 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx); #define BUILD_GET_MAX_ACCESS_OFFSET(max_offset) \ do { \ - /* Bulk memory max_offset already calculated before in \ - * BUILD_GET_MAX_SHARED_HEAP_BOUND */ \ - if (!bulk_memory) { \ + if (bulk_memory) { \ + max_offset = max_addr; \ + } \ + else { \ length = \ is_target_64bit ? I64_CONST(bytes - 1) : I32_CONST(bytes - 1); \ CHECK_LLVM_CONST(length); \ @@ -328,7 +328,7 @@ aot_check_shared_heap_memory_overflow_common( { LLVMBasicBlockRef app_addr_in_shared_heap_chain, app_addr_in_cache_shared_heap, app_addr_in_linear_mem, loopEntry, - loopCond, loopBody, loopExit; + loopCond, loopBody, loopExit, shared_heap_oob; LLVMValueRef maddr = NULL, max_offset = NULL, cmp, cmp1, cmp2; LLVMValueRef shared_heap_start_off, shared_heap_check_bound, shared_heap_size, shared_heap_end_off, shared_heap_base_addr_adj; @@ -479,16 +479,25 @@ aot_check_shared_heap_memory_overflow_common( /*--------------------------------------------------------------------- * Loop cond: update the pointer to traverse to the next shared heap in the * chain. The updated pointer is then added as an incoming value to the phi - * node. + * node. If in shared heap chain but not in the shared heap, it means OOB + * for a shared heap. *---------------------------------------------------------------------*/ + ADD_BASIC_BLOCK(shared_heap_oob, "shared_heap_oob"); + LLVMMoveBasicBlockAfter(shared_heap_oob, loopCond); + SET_BUILD_POS(shared_heap_oob); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, false, NULL, + NULL)) { + goto fail; + } + SET_BUILD_POS(loopCond); - /* TODO: for happy path it will always be valid since we checked the address - * in shared heap chain */ BUILD_GET_SHARED_HEAP_FIELD(phi_shared_heap, chain_next, INT8_PTR_TYPE, cur_shared_heap); /* Add the new value from loopBody as an incoming edge to the phi node */ LLVMAddIncoming(phi_shared_heap, &cur_shared_heap, &loopCond, 1); - BUILD_BR(loopBody); + BUILD_IS_NOT_NULL(cur_shared_heap, cmp, "is_not_null"); + BUILD_COND_BR(cmp, loopBody, shared_heap_oob); /*--------------------------------------------------------------------- * Loop exit: at this point, phi_shared_heap is expected to be valid if the From 86fbb8e4e2c944425dac3ed51e53aa61d951b035 Mon Sep 17 00:00:00 2001 From: TL Date: Wed, 23 Apr 2025 14:25:01 +0800 Subject: [PATCH 10/29] fix AOT bulk memory bounds checks on 64 bits platform --- core/iwasm/compilation/aot_emit_memory.c | 94 ++++++++++++++---------- 1 file changed, 55 insertions(+), 39 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 49d467e4a4..cd1e8d8510 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -35,6 +35,15 @@ } \ } while (0) +#define BUILD_ZEXT(value, dst_type) \ + do { \ + if (!(value = LLVMBuildZExt(comp_ctx->builder, value, dst_type, \ + #value "_z_ext"))) { \ + aot_set_last_error("llvm build zero ext failed."); \ + goto fail; \ + } \ + } while (0) + #define BUILD_TRUNC(value, data_type) \ do { \ if (!(value = LLVMBuildTrunc(comp_ctx->builder, value, data_type, \ @@ -227,33 +236,17 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx); if (bulk_memory) { \ shared_heap_check_bound = \ is_memory64 ? I64_CONST(UINT64_MAX) : I64_CONST(UINT32_MAX); \ - BUILD_OP(Add, max_addr, I64_NEG_ONE, max_offset, \ - "bulk_shared_heap_chain_bound"); \ } \ else { \ shared_heap_check_bound = \ - is_memory64 \ - ? I64_CONST(UINT64_MAX - bytes + 1) \ - : (is_target_64bit ? I64_CONST(UINT32_MAX - bytes + 1) \ - : I32_CONST(UINT32_MAX - bytes + 1)); \ + is_memory64 ? I64_CONST(UINT64_MAX - bytes_u32 + 1) \ + : (is_target_64bit \ + ? I64_CONST(UINT32_MAX - bytes_u32 + 1) \ + : I32_CONST(UINT32_MAX - bytes_u32 + 1)); \ } \ CHECK_LLVM_CONST(shared_heap_check_bound); \ } while (0) -#define BUILD_GET_MAX_ACCESS_OFFSET(max_offset) \ - do { \ - if (bulk_memory) { \ - max_offset = max_addr; \ - } \ - else { \ - length = \ - is_target_64bit ? I64_CONST(bytes - 1) : I32_CONST(bytes - 1); \ - CHECK_LLVM_CONST(length); \ - BUILD_OP(Add, start_offset, length, max_offset, \ - "max_access_offset"); \ - } \ - } while (0) - /* Update last used shared heap info in func_ctx, let runtime api update module * inst shared heap info * 1. shared_heap_start_off 2. shared_heap_end_off 3. shared_heap_base_addr_adj @@ -315,7 +308,7 @@ aot_update_last_used_shared_heap(AOTCompContext *comp_ctx, * it's i32 on 32 bit target * 3. In bulk memory overflow check: the offset(start addr) and max_addr( * (start + bytes) is used in memory check - * In normal memory check: the offset1(start addr) and bytes is used + * In normal memory check: the offset1(start addr) and bytes_u32 is used */ static bool aot_check_shared_heap_memory_overflow_common( @@ -323,8 +316,8 @@ aot_check_shared_heap_memory_overflow_common( LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, LLVMBasicBlockRef check_succ, LLVMValueRef maddr_phi, LLVMValueRef addr, LLVMValueRef start_offset, LLVMValueRef max_addr, - LLVMValueRef mem_base_addr, uint32 bytes, bool is_memory64, - bool is_target_64bit, bool bulk_memory, bool enable_segue) + LLVMValueRef mem_base_addr, LLVMValueRef bytes, uint32 bytes_u32, + bool is_memory64, bool is_target_64bit, bool bulk_memory, bool enable_segue) { LLVMBasicBlockRef app_addr_in_shared_heap_chain, app_addr_in_cache_shared_heap, app_addr_in_linear_mem, loopEntry, @@ -416,19 +409,36 @@ aot_check_shared_heap_memory_overflow_common( LLVMMoveBasicBlockAfter(loopCond, loopBody); LLVMMoveBasicBlockAfter(loopExit, loopCond); - /* Check if the app address is in the cache shared heap range. - * If yes, branch to the cache branch; if not, proceed into the loop */ - /* Load the local variable */ + /* Load the local variable of the function */ BUILD_LOAD_PTR(func_ctx->shared_heap_start_off, is_target_64bit ? I64_TYPE : I32_TYPE, shared_heap_start_off); BUILD_LOAD_PTR(func_ctx->shared_heap_end_off, is_target_64bit ? I64_TYPE : I32_TYPE, shared_heap_end_off); + if (bulk_memory && !is_target_64bit) { + /* For bulk memory on 32 bits platform, always extend to 64 bits */ + BUILD_ZEXT(shared_heap_start_off, I64_TYPE); + BUILD_ZEXT(shared_heap_end_off, I64_TYPE); + } + + /* Check if the app address is in the cache shared heap range. + * If yes, branch to the cache branch; if not, proceed into the loop */ BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, "cmp_cache_shared_heap_start"); - BUILD_GET_MAX_ACCESS_OFFSET(max_offset); - BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_end_off, cmp1, - "cmp_cache_shared_heap_end"); + if (bulk_memory) { + BUILD_OP(Add, max_addr, I64_NEG_ONE, max_offset, "max_offset"); + BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_end_off, cmp1, + "cmp_cache_shared_heap_end"); + } + else { + length = is_target_64bit ? I64_CONST(bytes_u32 - 1) + : I32_CONST(bytes_u32 - 1); + CHECK_LLVM_CONST(length); + BUILD_OP(Sub, shared_heap_end_off, length, shared_heap_check_bound, + "cache_shared_heap_end_bound"); + BUILD_ICMP(LLVMIntULE, start_offset, shared_heap_check_bound, cmp1, + "cmp_cache_shared_heap_end"); + } BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); BUILD_COND_BR(cmp2, app_addr_in_cache_shared_heap, loopEntry); @@ -465,10 +475,15 @@ aot_check_shared_heap_memory_overflow_common( if (!bulk_memory && !is_target_64bit) { BUILD_TRUNC(shared_heap_size, I32_TYPE); } - BUILD_OP(Sub, shared_heap_size, - (bulk_memory || is_target_64bit) ? I64_CONST(bytes) - : I32_CONST(bytes), - shared_heap_check_bound, "shared_heap_check_bound1"); + if (bulk_memory) { + length = bytes; + } + else { + length = is_target_64bit ? I64_CONST(bytes_u32) : I32_CONST(bytes_u32); + CHECK_LLVM_CONST(length); + } + BUILD_OP(Sub, shared_heap_size, length, shared_heap_check_bound, + "shared_heap_check_bound1"); BUILD_OP(Add, shared_heap_start_off, shared_heap_check_bound, shared_heap_check_bound, "shared_heap_check_bound2"); BUILD_ICMP(LLVMIntULE, start_offset, shared_heap_check_bound, cmp1, @@ -573,13 +588,13 @@ aot_check_shared_heap_memory_overflow( AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, LLVMValueRef maddr_phi, LLVMValueRef addr, LLVMValueRef start_offset, - LLVMValueRef mem_base_addr, uint32 bytes, bool is_memory64, + LLVMValueRef mem_base_addr, uint32 bytes_u32, bool is_memory64, bool is_target_64bit, bool enable_segue) { return aot_check_shared_heap_memory_overflow_common( comp_ctx, func_ctx, block_curr, block_maddr_phi, NULL, maddr_phi, addr, - start_offset, NULL, mem_base_addr, bytes, is_memory64, is_target_64bit, - false, enable_segue); + start_offset, NULL, mem_base_addr, NULL, bytes_u32, is_memory64, + is_target_64bit, false, enable_segue); } static bool @@ -587,11 +602,12 @@ aot_check_bulk_memory_shared_heap_memory_overflow( AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, LLVMBasicBlockRef check_succ, LLVMValueRef maddr_phi, - LLVMValueRef start_offset, LLVMValueRef max_addr, bool is_memory64) + LLVMValueRef start_offset, LLVMValueRef max_addr, LLVMValueRef bytes, + bool is_memory64) { return aot_check_shared_heap_memory_overflow_common( comp_ctx, func_ctx, block_curr, block_maddr_phi, check_succ, maddr_phi, - NULL, start_offset, max_addr, NULL, 0, is_memory64, + NULL, start_offset, max_addr, NULL, bytes, 0, is_memory64, comp_ctx->pointer_size == sizeof(uint64), true, false); } @@ -1638,7 +1654,7 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (!aot_check_bulk_memory_shared_heap_memory_overflow( comp_ctx, func_ctx, block_curr, block_maddr_phi, check_succ, - maddr_phi, offset, max_addr, is_memory64)) { + maddr_phi, offset, max_addr, bytes, is_memory64)) { goto fail; } } From 463b9617d1d0274702682b909e0fc46a50af45c1 Mon Sep 17 00:00:00 2001 From: TL Date: Thu, 24 Apr 2025 10:05:02 +0800 Subject: [PATCH 11/29] refactor aot memory check --- core/iwasm/compilation/aot_emit_memory.c | 230 ++++++++++------------- 1 file changed, 96 insertions(+), 134 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index cd1e8d8510..34154f3e3c 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -35,15 +35,6 @@ } \ } while (0) -#define BUILD_ZEXT(value, dst_type) \ - do { \ - if (!(value = LLVMBuildZExt(comp_ctx->builder, value, dst_type, \ - #value "_z_ext"))) { \ - aot_set_last_error("llvm build zero ext failed."); \ - goto fail; \ - } \ - } while (0) - #define BUILD_TRUNC(value, data_type) \ do { \ if (!(value = LLVMBuildTrunc(comp_ctx->builder, value, data_type, \ @@ -201,41 +192,35 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx); BUILD_LOAD_PTR(field_p, data_type, res); \ } while (0) -#define BUILD_GET_SHARED_HEAP_START(shared_heap_p, res) \ - do { \ - if (is_memory64) { \ - offset_u32 = offsetof(WASMSharedHeap, start_off_mem64); \ - } \ - else { \ - offset_u32 = offsetof(WASMSharedHeap, start_off_mem32); \ - } \ - field_offset = I32_CONST(offset_u32); \ - CHECK_LLVM_CONST(field_offset); \ - \ - BUILD_FIELD_PTR(shared_heap_p, field_offset, field_p, \ - "shared_heap_start_off_p"); \ - if (!(field_p = LLVMBuildBitCast(comp_ctx->builder, field_p, \ - is_target_64bit ? INT64_PTR_TYPE \ - : INT32_PTR_TYPE, \ - "shared_heap_start_cast_p"))) { \ - aot_set_last_error("llvm build bit cast failed."); \ - goto fail; \ - } \ - BUILD_LOAD_PTR(field_p, is_target_64bit ? I64_TYPE : I32_TYPE, res); \ - /* For bulk memory on 32 bits platform, always extend to 64 bits */ \ - if (bulk_memory && !is_target_64bit \ - && !(res = LLVMBuildZExt(comp_ctx->builder, res, I64_TYPE, \ - "shared_heap_start_off_u64"))) { \ - aot_set_last_error("llvm build zero ext failed."); \ - goto fail; \ - } \ +#define BUILD_GET_SHARED_HEAP_START(shared_heap_p, res) \ + do { \ + if (is_memory64) { \ + offset_u32 = offsetof(WASMSharedHeap, start_off_mem64); \ + } \ + else { \ + offset_u32 = offsetof(WASMSharedHeap, start_off_mem32); \ + } \ + field_offset = I32_CONST(offset_u32); \ + CHECK_LLVM_CONST(field_offset); \ + \ + BUILD_FIELD_PTR(shared_heap_p, field_offset, field_p, \ + "shared_heap_start_off_p"); \ + if (!(field_p = LLVMBuildBitCast(comp_ctx->builder, field_p, \ + offset_ptr_type, \ + "shared_heap_start_cast_p"))) { \ + aot_set_last_error("llvm build bit cast failed."); \ + goto fail; \ + } \ + BUILD_LOAD_PTR(field_p, offset_type, res); \ } while (0) #define BUILD_GET_MAX_SHARED_HEAP_BOUND(shared_heap_check_bound) \ do { \ if (bulk_memory) { \ shared_heap_check_bound = \ - is_memory64 ? I64_CONST(UINT64_MAX) : I64_CONST(UINT32_MAX); \ + is_memory64 ? I64_CONST(UINT64_MAX) \ + : (is_target_64bit ? I64_CONST(UINT32_MAX) \ + : I32_CONST(UINT32_MAX)); \ } \ else { \ shared_heap_check_bound = \ @@ -303,10 +288,7 @@ aot_update_last_used_shared_heap(AOTCompContext *comp_ctx, /* The difference between bulk memory overflow check and normal memory * overflow check: * 1. In bulk memory overflow check, no segue will be used - * 2. In bulk memory overflow check: all mem_offset will always be i64. - * In normal memory check: mem_offset wll only be i64 on 64 bit target - * it's i32 on 32 bit target - * 3. In bulk memory overflow check: the offset(start addr) and max_addr( + * 2. In bulk memory overflow check: the offset(start addr), bytes and max_addr * (start + bytes) is used in memory check * In normal memory check: the offset1(start addr) and bytes_u32 is used */ @@ -314,7 +296,7 @@ static bool aot_check_shared_heap_memory_overflow_common( AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, - LLVMBasicBlockRef check_succ, LLVMValueRef maddr_phi, LLVMValueRef addr, + LLVMBasicBlockRef check_succ, LLVMValueRef maddr_phi, LLVMValueRef start_offset, LLVMValueRef max_addr, LLVMValueRef mem_base_addr, LLVMValueRef bytes, uint32 bytes_u32, bool is_memory64, bool is_target_64bit, bool bulk_memory, bool enable_segue) @@ -322,6 +304,7 @@ aot_check_shared_heap_memory_overflow_common( LLVMBasicBlockRef app_addr_in_shared_heap_chain, app_addr_in_cache_shared_heap, app_addr_in_linear_mem, loopEntry, loopCond, loopBody, loopExit, shared_heap_oob; + LLVMTypeRef offset_type, offset_ptr_type; LLVMValueRef maddr = NULL, max_offset = NULL, cmp, cmp1, cmp2; LLVMValueRef shared_heap_start_off, shared_heap_check_bound, shared_heap_size, shared_heap_end_off, shared_heap_base_addr_adj; @@ -330,30 +313,36 @@ aot_check_shared_heap_memory_overflow_common( LLVMValueRef field_offset, field_p, length; uint32 offset_u32; + /* On 64/32-bit target, the offset is 64/32-bit */ + offset_type = is_target_64bit ? I64_TYPE : I32_TYPE; + offset_ptr_type = is_target_64bit ? INT64_PTR_TYPE : INT32_PTR_TYPE; + /*--------------------------------------------------------------------- - * Create the basic blocks and arrange control flow: + * Create the basic blocks and arrange control flow(omit exception block + * and loop internal blocks for simplicity): * * +-------------------------+ * | current block | * +-------------------------+ - * / \ - * / \ - * not in shared heap in shared heap chain ------------ - * | | | - * | | | - * v v | - * +---------------------+ +-------------------------+ | - * | in_linear_mem | | app_addr_in_shared_heap | | - * +---------------------+ | chain | | - * +-----------+-------------+ | - * | | - * +---------------+---------------+ | - * | Loop: Traverse shared heap | | - * +---------------+---------------+ | - * | | - * +---------------+---------------+ | - * | Cache shared heap branch | <---- - * +-------------------------------+ + * / \ + * / \ + * no shared heap shared heap attached -------in cache------ + * | | | + * | ------not in chain--------| | + * v v v | + * +---------------------+ +-------------------------+ | + * | in_linear_mem | | in_shared_heap chain | | + * +---------------------+ +-----------+-------------+ | + * | | + * v | + * +---------------+---------------+ | + * | Loop: Traverse shared heap | | + * +---------------+---------------+ | + * | | + * v | + * +---------------+---------------+ | + * | Cache shared heap branch | <---- + * +-------------------------------+ *---------------------------------------------------------------------*/ ADD_BASIC_BLOCK(app_addr_in_shared_heap_chain, "app_addr_in_shared_heap_chain"); @@ -371,22 +360,6 @@ aot_check_shared_heap_memory_overflow_common( else LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ); - if (!bulk_memory && !is_target_64bit) { - /* Check whether integer overflow occurs in addr + offset */ - LLVMBasicBlockRef check_integer_overflow_end; - ADD_BASIC_BLOCK(check_integer_overflow_end, - "check_integer_overflow_end"); - LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr); - - BUILD_ICMP(LLVMIntULT, start_offset, addr, cmp1, "cmp1"); - if (!aot_emit_exception(comp_ctx, func_ctx, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp1, - check_integer_overflow_end)) { - goto fail; - } - SET_BUILD_POS(check_integer_overflow_end); - } - /* If there is no shared heap attached, branch to linear memory */ BUILD_IS_NOT_NULL(func_ctx->shared_heap, cmp, "has_shared_heap"); BUILD_COND_BR(cmp, app_addr_in_shared_heap_chain, app_addr_in_linear_mem); @@ -410,23 +383,17 @@ aot_check_shared_heap_memory_overflow_common( LLVMMoveBasicBlockAfter(loopExit, loopCond); /* Load the local variable of the function */ - BUILD_LOAD_PTR(func_ctx->shared_heap_start_off, - is_target_64bit ? I64_TYPE : I32_TYPE, + BUILD_LOAD_PTR(func_ctx->shared_heap_start_off, offset_type, shared_heap_start_off); - BUILD_LOAD_PTR(func_ctx->shared_heap_end_off, - is_target_64bit ? I64_TYPE : I32_TYPE, shared_heap_end_off); - if (bulk_memory && !is_target_64bit) { - /* For bulk memory on 32 bits platform, always extend to 64 bits */ - BUILD_ZEXT(shared_heap_start_off, I64_TYPE); - BUILD_ZEXT(shared_heap_end_off, I64_TYPE); - } - + BUILD_LOAD_PTR(func_ctx->shared_heap_end_off, offset_type, + shared_heap_end_off); /* Check if the app address is in the cache shared heap range. * If yes, branch to the cache branch; if not, proceed into the loop */ BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, "cmp_cache_shared_heap_start"); if (bulk_memory) { - BUILD_OP(Add, max_addr, I64_NEG_ONE, max_offset, "max_offset"); + BUILD_OP(Add, max_addr, is_target_64bit ? I64_NEG_ONE : I32_NEG_ONE, + max_offset, "max_offset"); BUILD_ICMP(LLVMIntULE, max_offset, shared_heap_end_off, cmp1, "cmp_cache_shared_heap_end"); } @@ -472,7 +439,7 @@ aot_check_shared_heap_memory_overflow_common( BUILD_GET_SHARED_HEAP_FIELD(phi_shared_heap, size, I64_TYPE, shared_heap_size); - if (!bulk_memory && !is_target_64bit) { + if (!is_target_64bit) { BUILD_TRUNC(shared_heap_size, I32_TYPE); } if (bulk_memory) { @@ -492,7 +459,7 @@ aot_check_shared_heap_memory_overflow_common( BUILD_COND_BR(cmp2, loopExit, loopCond); /*--------------------------------------------------------------------- - * Loop cond: update the pointer to traverse to the next shared heap in the + * LoopCond: update the pointer to traverse to the next shared heap in the * chain. The updated pointer is then added as an incoming value to the phi * node. If in shared heap chain but not in the shared heap, it means OOB * for a shared heap. @@ -515,18 +482,13 @@ aot_check_shared_heap_memory_overflow_common( BUILD_COND_BR(cmp, loopBody, shared_heap_oob); /*--------------------------------------------------------------------- - * Loop exit: at this point, phi_shared_heap is expected to be valid if the - * app address is contained in a shared heap; otherwise, it is NULL. + * LoopExit: at this point, phi_shared_heap is expected to be valid if + * the app address is contained in a shared heap; otherwise, it is oob + * and already handled in the loopCond block. *---------------------------------------------------------------------*/ SET_BUILD_POS(loopExit); /* Update last accessed shared heap, the shared_heap_size and - * shared_heap_start_off is already prepared in loop body. - * For bulk memory on 32 bits platform, it extends to i64 before, so it - * needs to trunc back to i32 for updating shared heap info. */ - if (bulk_memory && !is_target_64bit) { - BUILD_TRUNC(shared_heap_start_off, I32_TYPE); - BUILD_TRUNC(shared_heap_size, I32_TYPE); - } + * shared_heap_start_off is already prepared in loop body. */ if (!aot_update_last_used_shared_heap(comp_ctx, func_ctx, phi_shared_heap, shared_heap_start_off, shared_heap_size, is_target_64bit)) { @@ -587,12 +549,12 @@ static bool aot_check_shared_heap_memory_overflow( AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, - LLVMValueRef maddr_phi, LLVMValueRef addr, LLVMValueRef start_offset, + LLVMValueRef maddr_phi, LLVMValueRef start_offset, LLVMValueRef mem_base_addr, uint32 bytes_u32, bool is_memory64, bool is_target_64bit, bool enable_segue) { return aot_check_shared_heap_memory_overflow_common( - comp_ctx, func_ctx, block_curr, block_maddr_phi, NULL, maddr_phi, addr, + comp_ctx, func_ctx, block_curr, block_maddr_phi, NULL, maddr_phi, start_offset, NULL, mem_base_addr, NULL, bytes_u32, is_memory64, is_target_64bit, false, enable_segue); } @@ -603,12 +565,12 @@ aot_check_bulk_memory_shared_heap_memory_overflow( LLVMBasicBlockRef block_curr, LLVMBasicBlockRef block_maddr_phi, LLVMBasicBlockRef check_succ, LLVMValueRef maddr_phi, LLVMValueRef start_offset, LLVMValueRef max_addr, LLVMValueRef bytes, - bool is_memory64) + bool is_memory64, bool is_target_64bit) { return aot_check_shared_heap_memory_overflow_common( comp_ctx, func_ctx, block_curr, block_maddr_phi, check_succ, maddr_phi, - NULL, start_offset, max_addr, NULL, bytes, 0, is_memory64, - comp_ctx->pointer_size == sizeof(uint64), true, false); + start_offset, max_addr, NULL, bytes, 0, is_memory64, is_target_64bit, + true, false); } LLVMValueRef @@ -618,7 +580,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, { LLVMValueRef offset_const = MEMORY64_COND_VALUE(I64_CONST(offset), I32_CONST(offset)); - LLVMValueRef addr, maddr, maddr_phi = NULL, offset1, cmp1, cmp2, cmp; + LLVMValueRef addr, maddr, maddr_phi = NULL, offset1, cmp1, cmp; LLVMValueRef mem_base_addr, mem_check_bound; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; @@ -784,8 +746,9 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* offset1 = offset + addr; */ BUILD_OP(Add, offset_const, addr, offset1, "offset1"); - /* 1.1 offset + addr can overflow when it's memory64 */ - if (is_memory64 && comp_ctx->enable_bound_check) { + /* 1.1 offset + addr can overflow when it's memory64 or it's on 32-bit + * platform */ + if (is_memory64 && !is_target_64bit) { /* Check whether integer overflow occurs in offset + addr */ LLVMBasicBlockRef check_integer_overflow_end; ADD_BASIC_BLOCK(check_integer_overflow_end, @@ -799,6 +762,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } SET_BUILD_POS(check_integer_overflow_end); + block_curr = check_integer_overflow_end; } if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { @@ -815,8 +779,8 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (!aot_check_shared_heap_memory_overflow( comp_ctx, func_ctx, block_curr, block_maddr_phi, maddr_phi, - addr, offset1, mem_base_addr, bytes, is_memory64, - is_target_64bit, enable_segue)) { + offset1, mem_base_addr, bytes, is_memory64, is_target_64bit, + enable_segue)) { goto fail; } } @@ -852,22 +816,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } - if (is_target_64bit) { - BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); - } - else { - /* 2.1 offset + addr can overflow when it's memory32 */ - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { - /* Check integer overflow has been checked above */ - BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); - } - else { - /* Check integer overflow */ - BUILD_ICMP(LLVMIntULT, offset1, addr, cmp1, "cmp1"); - BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp2, "cmp2"); - BUILD_OP(Or, cmp1, cmp2, cmp, "cmp"); - } - } + BUILD_ICMP(LLVMIntUGT, offset1, mem_check_bound, cmp, "cmp"); /* Add basic blocks */ ADD_BASIC_BLOCK(check_succ, "check_succ"); @@ -1550,12 +1499,15 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; LLVMValueRef mem_size; + bool is_target_64bit; #if WASM_ENABLE_MEMORY64 == 0 bool is_memory64 = false; #else bool is_memory64 = IS_MEMORY64; #endif + is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; + /* Get memory base address and memory data size */ #if WASM_ENABLE_SHARED_MEMORY != 0 bool is_shared_memory = comp_ctx->comp_data->memories[0].flags & 0x02; @@ -1616,17 +1568,19 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, ADD_BASIC_BLOCK(check_succ, "check_succ"); LLVMMoveBasicBlockAfter(check_succ, block_curr); - offset = - LLVMBuildZExt(comp_ctx->builder, offset, I64_TYPE, "extend_offset"); - bytes = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, "extend_len"); - if (!offset || !bytes) { - aot_set_last_error("llvm build zext failed."); - goto fail; + if (is_target_64bit) { + offset = + LLVMBuildZExt(comp_ctx->builder, offset, I64_TYPE, "extend_offset"); + bytes = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, "extend_len"); + if (!offset || !bytes) { + aot_set_last_error("llvm build zext failed."); + goto fail; + } } BUILD_OP(Add, offset, bytes, max_addr, "max_addr"); - if (is_memory64 && comp_ctx->enable_bound_check) { + if (is_memory64 || !is_target_64bit) { /* Check whether integer overflow occurs in offset + addr */ LLVMBasicBlockRef check_integer_overflow_end; ADD_BASIC_BLOCK(check_integer_overflow_end, @@ -1640,6 +1594,7 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } SET_BUILD_POS(check_integer_overflow_end); + block_curr = check_integer_overflow_end; } if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { @@ -1654,11 +1609,18 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, if (!aot_check_bulk_memory_shared_heap_memory_overflow( comp_ctx, func_ctx, block_curr, block_maddr_phi, check_succ, - maddr_phi, offset, max_addr, bytes, is_memory64)) { + maddr_phi, offset, max_addr, bytes, is_memory64, + is_target_64bit)) { goto fail; } } + if (!is_target_64bit + && !(max_addr = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, + "extend_max_addr"))) { + aot_set_last_error("llvm build zext failed."); + goto fail; + } BUILD_ICMP(LLVMIntUGT, max_addr, mem_size, cmp, "cmp_max_mem_addr"); if (!aot_emit_exception(comp_ctx, func_ctx, From bc8dcd758f7cd17bbf178eb310d45ecadd42a698 Mon Sep 17 00:00:00 2001 From: TL Date: Thu, 24 Apr 2025 12:18:01 +0800 Subject: [PATCH 12/29] refactor AOT bulk memory bounds checks --- core/iwasm/compilation/aot_emit_memory.c | 87 ++++++++++++++---------- 1 file changed, 50 insertions(+), 37 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 34154f3e3c..665a8fd69f 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -232,8 +232,8 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx); CHECK_LLVM_CONST(shared_heap_check_bound); \ } while (0) -/* Update last used shared heap info in func_ctx, let runtime api update module - * inst shared heap info +/* Update last used shared heap info in function ctx, let runtime api + * update module inst shared heap info * 1. shared_heap_start_off 2. shared_heap_end_off 3. shared_heap_base_addr_adj */ static bool @@ -329,20 +329,28 @@ aot_check_shared_heap_memory_overflow_common( * no shared heap shared heap attached -------in cache------ * | | | * | ------not in chain--------| | + * | | | | * v v v | * +---------------------+ +-------------------------+ | * | in_linear_mem | | in_shared_heap chain | | * +---------------------+ +-----------+-------------+ | - * | | - * v | - * +---------------+---------------+ | - * | Loop: Traverse shared heap | | - * +---------------+---------------+ | - * | | - * v | - * +---------------+---------------+ | - * | Cache shared heap branch | <---- - * +-------------------------------+ + * | | | + * | v | + * | +---------------+---------------+ | + * | | Loop: Traverse shared heap | | + * | +---------------+---------------+ | + * | | | + * | v | + * | +---------------+---------------+ | + * | | Cache shared heap branch | <---- + * | +-------------------------------+ + * | | + * | --------------------------- + * | | + * v v + * +---------------------+ + * | maddr_phi | + * +---------------------+ *---------------------------------------------------------------------*/ ADD_BASIC_BLOCK(app_addr_in_shared_heap_chain, "app_addr_in_shared_heap_chain"); @@ -461,8 +469,9 @@ aot_check_shared_heap_memory_overflow_common( /*--------------------------------------------------------------------- * LoopCond: update the pointer to traverse to the next shared heap in the * chain. The updated pointer is then added as an incoming value to the phi - * node. If in shared heap chain but not in the shared heap, it means OOB - * for a shared heap. + * node. If the address in shared heap chain region but not in the shared + * heap, it means OOB(access bytes cross shared heaps boundaries) for a + * shared heap. *---------------------------------------------------------------------*/ ADD_BASIC_BLOCK(shared_heap_oob, "shared_heap_oob"); LLVMMoveBasicBlockAfter(shared_heap_oob, loopCond); @@ -724,6 +733,13 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, *alignp = 1; } + /* The overflow check needs to be done under following conditions: + * 1. In 64-bit target, offset and addr will be extended to 64-bit + * 1.1 offset + addr can overflow when it's memory64 + * 1.2 no overflow when it's memory32 + * 2. In 32-bit target, offset and addr will be 32-bit + * 2.1 offset + addr can overflow when it's memory32 + */ if (is_target_64bit) { if (!(offset_const = LLVMBuildZExt(comp_ctx->builder, offset_const, I64_TYPE, "offset_i64")) @@ -734,21 +750,12 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } - /* The overflow check needs to be done under following conditions: - * 1. In 64-bit target, offset and addr will be extended to 64-bit - * 1.1 offset + addr can overflow when it's memory64 - * 1.2 no overflow when it's memory32 - * 2. In 32-bit target, offset and addr will be 32-bit - * 2.1 offset + addr can overflow when it's memory32 - * And the goal is to detect it happens ASAP - */ - /* offset1 = offset + addr; */ BUILD_OP(Add, offset_const, addr, offset1, "offset1"); - /* 1.1 offset + addr can overflow when it's memory64 or it's on 32-bit - * platform */ - if (is_memory64 && !is_target_64bit) { + /* 1.1 offset + addr can overflow when it's memory64 + * 2.1 Or when it's on 32-bit platform */ + if (is_memory64 || !is_target_64bit) { /* Check whether integer overflow occurs in offset + addr */ LLVMBasicBlockRef check_integer_overflow_end; ADD_BASIC_BLOCK(check_integer_overflow_end, @@ -1568,6 +1575,8 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, ADD_BASIC_BLOCK(check_succ, "check_succ"); LLVMMoveBasicBlockAfter(check_succ, block_curr); + /* Same logic with aot_check_memory_overflow, offset and bytes are 32/64 + * bits on 32/64 bits platform */ if (is_target_64bit) { offset = LLVMBuildZExt(comp_ctx->builder, offset, I64_TYPE, "extend_offset"); @@ -1580,6 +1589,7 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, BUILD_OP(Add, offset, bytes, max_addr, "max_addr"); + /* Check overflow when it's memory64 or it's on 32 bits platform */ if (is_memory64 || !is_target_64bit) { /* Check whether integer overflow occurs in offset + addr */ LLVMBasicBlockRef check_integer_overflow_end; @@ -1615,18 +1625,21 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } - if (!is_target_64bit - && !(max_addr = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, - "extend_max_addr"))) { - aot_set_last_error("llvm build zext failed."); - goto fail; - } - BUILD_ICMP(LLVMIntUGT, max_addr, mem_size, cmp, "cmp_max_mem_addr"); + if (comp_ctx->enable_bound_check) { + /* mem_size is always 64-bit, extend max_addr on 32 bits platform */ + if (!is_target_64bit + && !(max_addr = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, + "extend_max_addr"))) { + aot_set_last_error("llvm build zext failed."); + goto fail; + } + BUILD_ICMP(LLVMIntUGT, max_addr, mem_size, cmp, "cmp_max_mem_addr"); - if (!aot_emit_exception(comp_ctx, func_ctx, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, - check_succ)) { - goto fail; + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, + check_succ)) { + goto fail; + } } /* maddr = mem_base_addr + offset */ From b5839c1e82c6efe7b33f1e0dcdd48473c06d188c Mon Sep 17 00:00:00 2001 From: TL Date: Thu, 24 Apr 2025 13:50:44 +0800 Subject: [PATCH 13/29] update AOT bulk memory bounds checks --- core/iwasm/compilation/aot_emit_memory.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 665a8fd69f..aba6f35196 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -1589,8 +1589,9 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, BUILD_OP(Add, offset, bytes, max_addr, "max_addr"); - /* Check overflow when it's memory64 or it's on 32 bits platform */ - if (is_memory64 || !is_target_64bit) { + /* Check overflow when it's memory64 or it's on 32 bits platform, but for + * bulk memory only when sw bounds since the start address is offset */ + if (comp_ctx->enable_bound_check && (is_memory64 || !is_target_64bit)) { /* Check whether integer overflow occurs in offset + addr */ LLVMBasicBlockRef check_integer_overflow_end; ADD_BASIC_BLOCK(check_integer_overflow_end, From 63de1132eb75a639b57889148d51ebd340c415cf Mon Sep 17 00:00:00 2001 From: TL Date: Thu, 24 Apr 2025 15:01:20 +0800 Subject: [PATCH 14/29] update AOT bulk memory bounds checks --- core/iwasm/compilation/aot_emit_memory.c | 25 ++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index aba6f35196..9272757118 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -301,7 +301,7 @@ aot_check_shared_heap_memory_overflow_common( LLVMValueRef mem_base_addr, LLVMValueRef bytes, uint32 bytes_u32, bool is_memory64, bool is_target_64bit, bool bulk_memory, bool enable_segue) { - LLVMBasicBlockRef app_addr_in_shared_heap_chain, + LLVMBasicBlockRef check_shared_heap_chain, app_addr_in_cache_shared_heap, app_addr_in_linear_mem, loopEntry, loopCond, loopBody, loopExit, shared_heap_oob; LLVMTypeRef offset_type, offset_ptr_type; @@ -328,12 +328,13 @@ aot_check_shared_heap_memory_overflow_common( * / \ * no shared heap shared heap attached -------in cache------ * | | | - * | ------not in chain--------| | - * | | | | - * v v v | + * | | | + * | | | + * v v | * +---------------------+ +-------------------------+ | - * | in_linear_mem | | in_shared_heap chain | | + * | in_linear_mem |<----No-----| check shared_heap chain | | * +---------------------+ +-----------+-------------+ | + * | Yes | * | | | * | v | * | +---------------+---------------+ | @@ -352,15 +353,15 @@ aot_check_shared_heap_memory_overflow_common( * | maddr_phi | * +---------------------+ *---------------------------------------------------------------------*/ - ADD_BASIC_BLOCK(app_addr_in_shared_heap_chain, - "app_addr_in_shared_heap_chain"); + ADD_BASIC_BLOCK(check_shared_heap_chain, + "check_shared_heap_chain"); ADD_BASIC_BLOCK(app_addr_in_cache_shared_heap, "app_addr_in_cache_shared_heap"); ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); - LLVMMoveBasicBlockAfter(app_addr_in_shared_heap_chain, block_curr); + LLVMMoveBasicBlockAfter(check_shared_heap_chain, block_curr); LLVMMoveBasicBlockAfter(app_addr_in_cache_shared_heap, - app_addr_in_shared_heap_chain); + check_shared_heap_chain); LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, app_addr_in_cache_shared_heap); if (!bulk_memory) @@ -370,7 +371,7 @@ aot_check_shared_heap_memory_overflow_common( /* If there is no shared heap attached, branch to linear memory */ BUILD_IS_NOT_NULL(func_ctx->shared_heap, cmp, "has_shared_heap"); - BUILD_COND_BR(cmp, app_addr_in_shared_heap_chain, app_addr_in_linear_mem); + BUILD_COND_BR(cmp, check_shared_heap_chain, app_addr_in_linear_mem); /*--------------------------------------------------------------------- * In the case where a shared heap is attached, we determine if the bytes @@ -378,14 +379,14 @@ aot_check_shared_heap_memory_overflow_common( * traverses the chain by using a phi node to merge two definitions of the * pointer (initial and updated). *---------------------------------------------------------------------*/ - SET_BUILD_POS(app_addr_in_shared_heap_chain); + SET_BUILD_POS(check_shared_heap_chain); /* Add loop basic blocks */ ADD_BASIC_BLOCK(loopEntry, "loop_entry"); ADD_BASIC_BLOCK(loopBody, "loop_body"); ADD_BASIC_BLOCK(loopCond, "loop_cond"); ADD_BASIC_BLOCK(loopExit, "loop_exit"); - LLVMMoveBasicBlockAfter(loopEntry, app_addr_in_shared_heap_chain); + LLVMMoveBasicBlockAfter(loopEntry, check_shared_heap_chain); LLVMMoveBasicBlockAfter(loopBody, loopEntry); LLVMMoveBasicBlockAfter(loopCond, loopBody); LLVMMoveBasicBlockAfter(loopExit, loopCond); From 9dff0fd622f9ee9cf2494c7bc81f9db9866a6c7e Mon Sep 17 00:00:00 2001 From: TL Date: Thu, 24 Apr 2025 15:28:35 +0800 Subject: [PATCH 15/29] update AOT bulk memory bounds check --- core/iwasm/compilation/aot_emit_memory.c | 42 +++++++++++------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 9272757118..f7ac4d542a 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -301,9 +301,9 @@ aot_check_shared_heap_memory_overflow_common( LLVMValueRef mem_base_addr, LLVMValueRef bytes, uint32 bytes_u32, bool is_memory64, bool is_target_64bit, bool bulk_memory, bool enable_segue) { - LLVMBasicBlockRef check_shared_heap_chain, - app_addr_in_cache_shared_heap, app_addr_in_linear_mem, loopEntry, - loopCond, loopBody, loopExit, shared_heap_oob; + LLVMBasicBlockRef check_shared_heap_chain, app_addr_in_cache_shared_heap, + app_addr_in_linear_mem, loopEntry, loopCond, loopBody, loopExit, + shared_heap_oob; LLVMTypeRef offset_type, offset_ptr_type; LLVMValueRef maddr = NULL, max_offset = NULL, cmp, cmp1, cmp2; LLVMValueRef shared_heap_start_off, shared_heap_check_bound, @@ -353,8 +353,7 @@ aot_check_shared_heap_memory_overflow_common( * | maddr_phi | * +---------------------+ *---------------------------------------------------------------------*/ - ADD_BASIC_BLOCK(check_shared_heap_chain, - "check_shared_heap_chain"); + ADD_BASIC_BLOCK(check_shared_heap_chain, "check_shared_heap_chain"); ADD_BASIC_BLOCK(app_addr_in_cache_shared_heap, "app_addr_in_cache_shared_heap"); ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); @@ -1590,10 +1589,9 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, BUILD_OP(Add, offset, bytes, max_addr, "max_addr"); - /* Check overflow when it's memory64 or it's on 32 bits platform, but for - * bulk memory only when sw bounds since the start address is offset */ - if (comp_ctx->enable_bound_check && (is_memory64 || !is_target_64bit)) { - /* Check whether integer overflow occurs in offset + addr */ + /* Check overflow when it's memory64 or it's on 32 bits platform */ + if (is_memory64 || !is_target_64bit) { + /* Check whether integer overflow occurs in offset + bytes */ LLVMBasicBlockRef check_integer_overflow_end; ADD_BASIC_BLOCK(check_integer_overflow_end, "check_integer_overflow_end"); @@ -1627,21 +1625,19 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } - if (comp_ctx->enable_bound_check) { - /* mem_size is always 64-bit, extend max_addr on 32 bits platform */ - if (!is_target_64bit - && !(max_addr = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, - "extend_max_addr"))) { - aot_set_last_error("llvm build zext failed."); - goto fail; - } - BUILD_ICMP(LLVMIntUGT, max_addr, mem_size, cmp, "cmp_max_mem_addr"); + /* mem_size is always 64-bit, extend max_addr on 32 bits platform */ + if (!is_target_64bit + && !(max_addr = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, + "extend_max_addr"))) { + aot_set_last_error("llvm build zext failed."); + goto fail; + } + BUILD_ICMP(LLVMIntUGT, max_addr, mem_size, cmp, "cmp_max_mem_addr"); - if (!aot_emit_exception(comp_ctx, func_ctx, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, - check_succ)) { - goto fail; - } + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, + check_succ)) { + goto fail; } /* maddr = mem_base_addr + offset */ From 386dac3ab58acec8ab1c97b8e70baff17d1dd697 Mon Sep 17 00:00:00 2001 From: TL Date: Thu, 24 Apr 2025 21:29:29 +0800 Subject: [PATCH 16/29] fix typo --- core/iwasm/compilation/aot_emit_memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index f7ac4d542a..0584a5efb2 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -1627,7 +1627,7 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* mem_size is always 64-bit, extend max_addr on 32 bits platform */ if (!is_target_64bit - && !(max_addr = LLVMBuildZExt(comp_ctx->builder, bytes, I64_TYPE, + && !(max_addr = LLVMBuildZExt(comp_ctx->builder, max_addr, I64_TYPE, "extend_max_addr"))) { aot_set_last_error("llvm build zext failed."); goto fail; From f960e14033e41606ad4756151df5e19398afd550 Mon Sep 17 00:00:00 2001 From: TL Date: Fri, 25 Apr 2025 12:10:17 +0800 Subject: [PATCH 17/29] add more unit test for shared heap --- tests/unit/shared-heap/shared_heap_test.cc | 235 +++++++++++++++++---- 1 file changed, 190 insertions(+), 45 deletions(-) diff --git a/tests/unit/shared-heap/shared_heap_test.cc b/tests/unit/shared-heap/shared_heap_test.cc index 2de8b00985..da6a0c9521 100644 --- a/tests/unit/shared-heap/shared_heap_test.cc +++ b/tests/unit/shared-heap/shared_heap_test.cc @@ -203,18 +203,15 @@ TEST_F(shared_heap_test, test_preallocated_shared_heap_malloc_fail) EXPECT_EQ(0, argv[0]); } -TEST_F(shared_heap_test, test_shared_heap_chain_rmw) +static void +create_test_shared_heap_chain(uint8 *preallocated_buf, size_t size, + uint8 *preallocated_buf2, size_t size2, + WASMSharedHeap **shared_heap_chain) { SharedHeapInitArgs args = { 0 }; - WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, - *shared_heap_chain = nullptr; - uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); - uint8 preallocated_buf[BUF_SIZE] = { 0 }, - preallocated_buf2[BUF_SIZE] = { 0 }; - uint32 start1, end1, start2, end2; - + WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr; args.pre_allocated_addr = preallocated_buf; - args.size = BUF_SIZE; + args.size = size; shared_heap = wasm_runtime_create_shared_heap(&args); if (!shared_heap) { FAIL() << "Create preallocated shared heap failed.\n"; @@ -222,23 +219,36 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw) memset(&args, 0, sizeof(args)); args.pre_allocated_addr = preallocated_buf2; - args.size = BUF_SIZE; + args.size = size2; shared_heap2 = wasm_runtime_create_shared_heap(&args); if (!shared_heap2) { FAIL() << "Create preallocated shared heap failed.\n"; } - shared_heap_chain = + *shared_heap_chain = wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); - if (!shared_heap_chain) { + if (!*shared_heap_chain) { FAIL() << "Create shared heap chain failed.\n"; } +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); /* app addr for shared heap */ - start1 = 0xFFFFFFFF - 2 * BUF_SIZE + 1; - end1 = 0xFFFFFFFF - BUF_SIZE; - start2 = 0xFFFFFFFF - BUF_SIZE + 1; - end2 = 0xFFFFFFFF; + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; /* shared heap 1 */ argv[0] = end1; @@ -271,45 +281,99 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw) EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 81); } -TEST_F(shared_heap_test, test_shared_heap_chain_rmw_oob) +TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory) { SharedHeapInitArgs args = { 0 }; - WASMSharedHeap *shared_heap = nullptr, *shared_heap2 = nullptr, - *shared_heap_chain = nullptr; - uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); - uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; uint32 start1, end1, start2, end2; - args.pre_allocated_addr = preallocated_buf; - args.size = BUF_SIZE; - shared_heap = wasm_runtime_create_shared_heap(&args); - if (!shared_heap) { - FAIL() << "Create preallocated shared heap failed.\n"; - } + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); - memset(&args, 0, sizeof(args)); - args.pre_allocated_addr = preallocated_buf2; - args.size = BUF_SIZE; - shared_heap2 = wasm_runtime_create_shared_heap(&args); - if (!shared_heap2) { - FAIL() << "Create preallocated shared heap failed.\n"; - } + /* app addr for shared heap */ + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; - shared_heap_chain = - wasm_runtime_chain_shared_heaps(shared_heap, shared_heap2); - if (!shared_heap_chain) { - FAIL() << "Create shared heap chain failed.\n"; - } + /* shared heap 1 */ + argv[0] = end1; + argv[1] = 101; + argv[2] = 1; + test_shared_heap(shared_heap_chain, "test_bulk_memory.wasm", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(end1, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + + argv[0] = start2; + argv[1] = 98; + argv[2] = 2; + test_shared_heap(shared_heap_chain, "test_bulk_memory.aot", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(start2, argv[0]); + EXPECT_EQ(preallocated_buf2[0], 98); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory_oob) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); /* app addr for shared heap */ - start1 = 0xFFFFFFFF - 2 * BUF_SIZE + 1; - end1 = 0xFFFFFFFF - BUF_SIZE; - start2 = 0xFFFFFFFF - BUF_SIZE + 1; - end2 = 0xFFFFFFFF; + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; + + /* shared heap 1 */ + argv[0] = end2; + argv[1] = 101; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = end1; + argv[1] = 98; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_rmw_oob) +{ + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + uint32 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT32_MAX - 2 * BUF_SIZE + 1; + end1 = UINT32_MAX - BUF_SIZE; + start2 = UINT32_MAX - BUF_SIZE + 1; + end2 = UINT32_MAX; /* try to rmw an u16, first u8 is in the first shared heap and second u8 is * in the second shared heap, will be seen as oob */ - argv[0] = end1; + argv[0] = end2; argv[1] = 12025; EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test.wasm", "read_modify_write_16", 2, argv), @@ -317,11 +381,92 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw_oob) argv[0] = end1; argv[1] = 12025; - EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test.wasm", + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test.aot", "read_modify_write_16", 2, argv), "Exception: out of bounds memory access"); } +#if WASM_ENABLE_MEMORY64 != 0 +TEST_F(shared_heap_test, test_shared_heap_chain_memory64_rmw) +{ + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }, + preallocated_buf2[BUF_SIZE] = { 0 }; + uint64 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT64_MAX - 2 * BUF_SIZE + 1; + end1 = UINT64_MAX - BUF_SIZE; + start2 = UINT64_MAX - BUF_SIZE + 1; + end2 = UINT64_MAX; + + /* shared heap 1 */ + PUT_I64_TO_ADDR(argv, end1); + argv[2] = 101; + test_shared_heap(shared_heap_chain, "test64.wasm", "read_modify_write_8", 3, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + + /* shared heap 2 */ + PUT_I64_TO_ADDR(argv, start2); + argv[2] = 129; + test_shared_heap(shared_heap_chain, "test64.wasm", "read_modify_write_8", 3, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf2[0], 129); + + PUT_I64_TO_ADDR(argv, start1); + argv[2] = 98; + test_shared_heap(shared_heap_chain, "test64.aot", "read_modify_write_8", 3, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[0], 98); + + PUT_I64_TO_ADDR(argv, end2); + argv[2] = 81; + test_shared_heap(shared_heap_chain, "test64.aot", "read_modify_write_8", 3, + argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 81); +} + +TEST_F(shared_heap_test, test_shared_heap_chain_memory64_rmw_oob) +{ + WASMSharedHeap *shared_heap_chain = nullptr; + uint32 argv[3] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + uint64 start1, end1, start2, end2; + + create_test_shared_heap_chain(preallocated_buf, BUF_SIZE, preallocated_buf2, + BUF_SIZE, &shared_heap_chain); + + /* app addr for shared heap */ + start1 = UINT64_MAX - 2 * BUF_SIZE + 1; + end1 = UINT64_MAX - BUF_SIZE; + start2 = UINT64_MAX - BUF_SIZE + 1; + end2 = UINT64_MAX; + + /* try to rmw an u16, first u8 is in the first shared heap and second u8 is + * in the second shared heap, will be seen as oob */ + PUT_I64_TO_ADDR(argv, end1); + argv[2] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test64.wasm", + "read_modify_write_16", 3, argv), + "Exception: out of bounds memory access"); + + PUT_I64_TO_ADDR(argv, end1); + argv[2] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test64.aot", + "read_modify_write_16", 3, argv), + "Exception: out of bounds memory access"); +} +#endif + #ifndef native_function /* clang-format off */ #define native_function(func_name, signature) \ From 671c70a2a8e98a68e745afb2929b53dae4047224 Mon Sep 17 00:00:00 2001 From: TL Date: Mon, 28 Apr 2025 15:48:28 +0800 Subject: [PATCH 18/29] finished organizing unit test for shared heap and enable x86_32 for shared heap unit test --- tests/unit/CMakeLists.txt | 11 ++- tests/unit/shared-heap/CMakeLists.txt | 4 + .../unit/shared-heap/wasm-apps/CMakeLists.txt | 86 +++++++++++------- .../bulk-memory/test_bulk_memory.wasm | Bin 0 -> 63 bytes .../bulk-memory/test_bulk_memory.wat | 12 +++ .../wasm-apps/memory64/CMakeLists.txt | 65 +++++++++++++ tests/unit/shared-heap/wasm-apps/test.c | 2 +- .../shared-heap/wasm-apps/test_addr_conv.c | 2 +- 8 files changed, 144 insertions(+), 38 deletions(-) create mode 100644 tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wasm create mode 100644 tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat create mode 100644 tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 8c963cadd8..abc7a902b7 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -19,8 +19,15 @@ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}") if(WAMR_BUILD_TARGET STREQUAL "X86_32") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32") + # 1) Force -m32 + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32" CACHE STRING "" FORCE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32" CACHE STRING "" FORCE) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -m32" CACHE STRING "" FORCE) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -m32" CACHE STRING "" FORCE) + + # 2) Make CMake prefer i386 libraries + set(CMAKE_SYSTEM_PROCESSOR i386 CACHE STRING "" FORCE) + set(CMAKE_LIBRARY_ARCHITECTURE "i386-linux-gnu" CACHE STRING "" FORCE) endif() # Prevent overriding the parent project's compiler/linker diff --git a/tests/unit/shared-heap/CMakeLists.txt b/tests/unit/shared-heap/CMakeLists.txt index b7859bd06d..463681f378 100644 --- a/tests/unit/shared-heap/CMakeLists.txt +++ b/tests/unit/shared-heap/CMakeLists.txt @@ -22,6 +22,10 @@ set(WAMR_BUILD_SHARED_HEAP 1) # Compile wasm modules add_subdirectory(wasm-apps) +if (WAMR_BUILD_MEMORY64 EQUAL 1) + add_subdirectory(wasm-apps/memory64) +endif () + # if only load this CMake other than load it as subdirectory include(../unit_common.cmake) diff --git a/tests/unit/shared-heap/wasm-apps/CMakeLists.txt b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt index f66ee1b68c..931b7b52df 100644 --- a/tests/unit/shared-heap/wasm-apps/CMakeLists.txt +++ b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt @@ -29,50 +29,68 @@ set(CMAKE_EXE_LINKER_FLAGS -Wl,--allow-undefined" ) -add_executable(test.wasm test.c) -target_link_libraries(test.wasm) - if (WAMR_BUILD_TARGET STREQUAL "X86_32") set (WAMR_COMPILER_FLAGS --opt-level=0 --bounds-checks=1 --enable-shared-heap --target=i386) else () set (WAMR_COMPILER_FLAGS --opt-level=0 --bounds-checks=1 --enable-shared-heap) endif () -add_custom_command(TARGET test.wasm POST_BUILD +function(copy_wasm TARGET_NAME) + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test.wasm - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test.wasm to the same directory of google test" - ) + ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} + ${CMAKE_CURRENT_BINARY_DIR}/../ + COMMENT "Copy ${TARGET_NAME} to the same directory of google test" + ) +endfunction() + +function(compile_and_copy_aot_from TARGET_NAME) + string(REPLACE ".wasm" ".aot" AOT_TARGET ${TARGET_NAME}) -add_custom_command(TARGET test.wasm POST_BUILD - COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} - -o - test.aot - test.wasm + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} + -o ${AOT_TARGET} + ${TARGET_NAME} COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test.aot - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test.aot to the same directory of google test" - ) + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../ + COMMENT "Compile and copy ${AOT_TARGET} to the same directory of google test" + ) +endfunction() -add_executable(test_addr_conv.wasm test_addr_conv.c) +add_executable(test.wasm test.c) target_link_libraries(test.wasm) +copy_wasm(test.wasm) +compile_and_copy_aot_from(test.wasm) -add_custom_command(TARGET test_addr_conv.wasm POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test_addr_conv.wasm - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test_addr_conv.wasm to the same directory of google test" - ) +add_executable(test_addr_conv.wasm test_addr_conv.c) +target_link_libraries(test_addr_conv.wasm) +copy_wasm(test_addr_conv.wasm) +compile_and_copy_aot_from(test_addr_conv.wasm) -add_custom_command(TARGET test_addr_conv.wasm POST_BUILD - COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} - -o - test_addr_conv.aot - test_addr_conv.wasm - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_CURRENT_BINARY_DIR}/test_addr_conv.aot - ${CMAKE_CURRENT_BINARY_DIR}/../ - COMMENT "Copy test_addr_conv.aot to the same directory of google test" - ) +# copy and compile aot for bulk memory test +set(SOURCE_WASM ${CMAKE_CURRENT_SOURCE_DIR}/bulk-memory/test_bulk_memory.wasm) +set(BUILD_WASM ${CMAKE_CURRENT_BINARY_DIR}/../test_bulk_memory.wasm) +set(OUTPUT_AOT ${CMAKE_CURRENT_BINARY_DIR}/../test_bulk_memory.aot) + +add_custom_command( + OUTPUT ${BUILD_WASM} + COMMAND ${CMAKE_COMMAND} -E copy + ${SOURCE_WASM} + ${BUILD_WASM} + DEPENDS ${SOURCE_WASM} + COMMENT "Copying bulk memory WASM to build directory" +) + +add_custom_command( + OUTPUT ${OUTPUT_AOT} + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} + -o ${OUTPUT_AOT} + ${BUILD_WASM} + DEPENDS ${BUILD_WASM} + COMMENT "Compiling bulk memory AOT from copied WASM" +) + +add_custom_target(compile_bulk_memory_aot ALL + DEPENDS ${OUTPUT_AOT} +) \ No newline at end of file diff --git a/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wasm b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wasm new file mode 100644 index 0000000000000000000000000000000000000000..eb3d60be3da992383bd142c229388824ba10439d GIT binary patch literal 63 zcmZQbEY4+QU|?WmXG~zOudiodW@2PuWo85lh%gG|rsn1sRmP`f=H$eeq!yPjFmUlQ Rax*9}C@?B8{o!Wd1^{Vt3%&pV literal 0 HcmV?d00001 diff --git a/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat new file mode 100644 index 0000000000..13f2a6de12 --- /dev/null +++ b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat @@ -0,0 +1,12 @@ +(module + (memory 1) + + (func $memory_fill_test (param $dst i32) (param $val i32) (param $len i32) + local.get $dst + local.get $val + local.get $len + memory.fill + ) + + (export "memory_fill_test" (func $memory_fill_test)) +) \ No newline at end of file diff --git a/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt b/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt new file mode 100644 index 0000000000..9f51479c24 --- /dev/null +++ b/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt @@ -0,0 +1,65 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14) +project(wasm-apps-wasm64) + +set(WAMR_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../../..) +set(WAMRC_ROOT_DIR ${WAMR_ROOT_DIR}/wamr-compiler/build) + +set(CMAKE_SYSTEM_PROCESSOR wasm64) +set(CMAKE_SYSROOT ${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot) + +if (NOT DEFINED WASI_SDK_DIR) + set(WASI_SDK_DIR "/opt/wasi-sdk") +endif () + +set(CMAKE_C_FLAGS "-nostdlib -pthread -Qunused-arguments") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -z stack-size=8192 -nostdlib -O0 --target=wasm64") +set(CMAKE_C_COMPILER_TARGET "wasm64") +set(CMAKE_C_COMPILER "${WASI_SDK_DIR}/bin/clang") + +set(DEFINED_SYMBOLS + "${WAMR_ROOT_DIR}/wamr-sdk/app/libc-builtin-sysroot/share/defined-symbols.txt") + +set(CMAKE_EXE_LINKER_FLAGS + "-Wl,--no-entry \ + -Wl,--initial-memory=65536 \ + -Wl,--export-all \ + -Wl,--allow-undefined" + ) + +if (WAMR_BUILD_TARGET STREQUAL "X86_32") + set (WAMR_COMPILER_FLAGS --opt-level=0 --bounds-checks=1 --enable-shared-heap --target=i386) +else () + set (WAMR_COMPILER_FLAGS --opt-level=0 --bounds-checks=1 --enable-shared-heap) +endif () + +function(copy_wasm TARGET_NAME) + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} + ${CMAKE_CURRENT_BINARY_DIR}/../../ + COMMENT "Copy ${TARGET_NAME} to the same directory of google test" + ) +endfunction() + +function(compile_and_copy_aot_from TARGET_NAME) + string(REPLACE ".wasm" ".aot" AOT_TARGET ${TARGET_NAME}) + + add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} + -o ${AOT_TARGET} + ${TARGET_NAME} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../../ + COMMENT "Compile and copy ${AOT_TARGET} to the same directory of google test" + ) +endfunction() + +# Example targets +add_executable(test64.wasm ../test.c) +target_link_libraries(test64.wasm) +copy_wasm(test64.wasm) +compile_and_copy_aot_from(test64.wasm) diff --git a/tests/unit/shared-heap/wasm-apps/test.c b/tests/unit/shared-heap/wasm-apps/test.c index b0bfc17b36..66df21c1b7 100644 --- a/tests/unit/shared-heap/wasm-apps/test.c +++ b/tests/unit/shared-heap/wasm-apps/test.c @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#include +#define NULL 0 extern void * shared_heap_malloc(int size); diff --git a/tests/unit/shared-heap/wasm-apps/test_addr_conv.c b/tests/unit/shared-heap/wasm-apps/test_addr_conv.c index 5e64526a04..a00e68c9ce 100644 --- a/tests/unit/shared-heap/wasm-apps/test_addr_conv.c +++ b/tests/unit/shared-heap/wasm-apps/test_addr_conv.c @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#include +#define NULL 0 extern void * shared_heap_malloc(int size); From d332af5564cceda0c7766257576d4443e6186bbe Mon Sep 17 00:00:00 2001 From: TL Date: Mon, 28 Apr 2025 16:49:36 +0800 Subject: [PATCH 19/29] format --- samples/shared-heap/wasm-apps/test2.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/shared-heap/wasm-apps/test2.c b/samples/shared-heap/wasm-apps/test2.c index 14cf0c9c38..44d573164e 100644 --- a/samples/shared-heap/wasm-apps/test2.c +++ b/samples/shared-heap/wasm-apps/test2.c @@ -21,7 +21,8 @@ consume_str(char *buf) { /* Actually access it in wasm */ char c = buf[0]; - printf("In WASM: wasm app2's wasm func received buf in pre-allocated shared buf: " + printf("In WASM: wasm app2's wasm func received buf in pre-allocated " + "shared buf: " "%s with its first char is %c\n\n", buf, c); } From e1432e4a90519b563f2bb6075d823b84f4e3ed12 Mon Sep 17 00:00:00 2001 From: TL Date: Tue, 29 Apr 2025 15:30:04 +0800 Subject: [PATCH 20/29] cover a corner case for bulk memory overflow check --- core/iwasm/compilation/aot_emit_memory.c | 8 ++- tests/unit/shared-heap/shared_heap_test.cc | 79 ++++++++++++++++++++-- 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 0584a5efb2..f981cba962 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -457,6 +457,7 @@ aot_check_shared_heap_memory_overflow_common( length = is_target_64bit ? I64_CONST(bytes_u32) : I32_CONST(bytes_u32); CHECK_LLVM_CONST(length); } + /* start_offset + length - 1 <= shared_heap_start_off + size - 1 */ BUILD_OP(Sub, shared_heap_size, length, shared_heap_check_bound, "shared_heap_check_bound1"); BUILD_OP(Add, shared_heap_start_off, shared_heap_check_bound, @@ -1592,12 +1593,17 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Check overflow when it's memory64 or it's on 32 bits platform */ if (is_memory64 || !is_target_64bit) { /* Check whether integer overflow occurs in offset + bytes */ + LLVMValueRef max_offset; LLVMBasicBlockRef check_integer_overflow_end; ADD_BASIC_BLOCK(check_integer_overflow_end, "check_integer_overflow_end"); LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr); - BUILD_ICMP(LLVMIntULT, max_addr, offset, cmp, "cmp"); + /* offset + bytes can overflow yet is valid(for example, 0xffffffff, 1), + * check whether offset + bytes - 1 overflow instead */ + BUILD_OP(Add, max_addr, is_target_64bit ? I64_NEG_ONE : I32_NEG_ONE, + max_offset, "max_offset"); + BUILD_ICMP(LLVMIntULT, max_offset, bytes, cmp, "cmp"); if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, check_integer_overflow_end)) { diff --git a/tests/unit/shared-heap/shared_heap_test.cc b/tests/unit/shared-heap/shared_heap_test.cc index da6a0c9521..84c10c9bd9 100644 --- a/tests/unit/shared-heap/shared_heap_test.cc +++ b/tests/unit/shared-heap/shared_heap_test.cc @@ -299,7 +299,6 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory) start2 = UINT32_MAX - BUF_SIZE + 1; end2 = UINT32_MAX; - /* shared heap 1 */ argv[0] = end1; argv[1] = 101; argv[2] = 1; @@ -309,14 +308,33 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory) EXPECT_EQ(end1, argv[0]); EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + argv[0] = start1; + argv[1] = 14; + argv[2] = 1; + test_shared_heap(shared_heap_chain, "test_bulk_memory.aot", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(start1, argv[0]); + EXPECT_EQ(preallocated_buf[0], 14); + + /* nothing happen when memory fill 0 byte */ argv[0] = start2; - argv[1] = 98; - argv[2] = 2; + argv[1] = 68; + argv[2] = 0; test_shared_heap(shared_heap_chain, "test_bulk_memory.aot", "memory_fill_test", 3, argv); /* no modification since no return value */ EXPECT_EQ(start2, argv[0]); - EXPECT_EQ(preallocated_buf2[0], 98); + EXPECT_EQ(preallocated_buf2[0], 0); + + argv[0] = end2; + argv[1] = 98; + argv[2] = 1; + test_shared_heap(shared_heap_chain, "test_bulk_memory.aot", + "memory_fill_test", 3, argv); + /* no modification since no return value */ + EXPECT_EQ(end2, argv[0]); + EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 98); } TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory_oob) @@ -338,7 +356,7 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory_oob) end2 = UINT32_MAX; /* shared heap 1 */ - argv[0] = end2; + argv[0] = end1; argv[1] = 101; argv[2] = 2; EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, @@ -346,13 +364,64 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory_oob) "memory_fill_test", 3, argv), "Exception: out of bounds memory access"); + argv[0] = end2; + argv[1] = 98; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start1; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start2; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.wasm", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + + argv[0] = end1; + argv[1] = 101; + argv[2] = 2; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = end2; argv[1] = 98; argv[2] = 2; EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test_bulk_memory.aot", "memory_fill_test", 3, argv), "Exception: out of bounds memory access"); + + argv[0] = start1; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + + argv[0] = start2; + argv[1] = 98; + argv[2] = BUF_SIZE + 1; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_bulk_memory.aot", + "memory_fill_test", 3, argv), + "Exception: out of bounds memory access"); + } TEST_F(shared_heap_test, test_shared_heap_chain_rmw_oob) From ecfb609b299c3c8220e2e6feb83a5dd104459d34 Mon Sep 17 00:00:00 2001 From: TL Date: Tue, 29 Apr 2025 16:13:56 +0800 Subject: [PATCH 21/29] cover a corner case for bulk memory overflow check --- core/iwasm/compilation/aot_emit_memory.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index f981cba962..cb5d54c76d 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -1502,7 +1502,7 @@ LLVMValueRef check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef offset, LLVMValueRef bytes) { - LLVMValueRef maddr, max_addr, cmp; + LLVMValueRef maddr, max_addr, cmp, cmp1; LLVMValueRef mem_base_addr, maddr_phi = NULL; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; @@ -1593,17 +1593,17 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Check overflow when it's memory64 or it's on 32 bits platform */ if (is_memory64 || !is_target_64bit) { /* Check whether integer overflow occurs in offset + bytes */ - LLVMValueRef max_offset; LLVMBasicBlockRef check_integer_overflow_end; ADD_BASIC_BLOCK(check_integer_overflow_end, "check_integer_overflow_end"); LLVMMoveBasicBlockAfter(check_integer_overflow_end, block_curr); /* offset + bytes can overflow yet is valid(for example, 0xffffffff, 1), - * check whether offset + bytes - 1 overflow instead */ - BUILD_OP(Add, max_addr, is_target_64bit ? I64_NEG_ONE : I32_NEG_ONE, - max_offset, "max_offset"); - BUILD_ICMP(LLVMIntULT, max_offset, bytes, cmp, "cmp"); + * allow it to be 0(either 0, 0 or overflow and valid) */ + BUILD_ICMP(LLVMIntULT, max_addr, offset, cmp, "cmp"); + BUILD_ICMP(LLVMIntNE, max_addr, is_target_64bit ? I64_ZERO : I32_ZERO, + cmp1, "cmp1"); + BUILD_OP(And, cmp, cmp1, cmp, "overflow"); if (!aot_emit_exception(comp_ctx, func_ctx, EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, check_integer_overflow_end)) { From 5702a6119b21065720185f21f563f41775a0359a Mon Sep 17 00:00:00 2001 From: TL Date: Wed, 30 Apr 2025 14:04:04 +0800 Subject: [PATCH 22/29] some update --- .../unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat index 13f2a6de12..e7a0c684d1 100644 --- a/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat +++ b/tests/unit/shared-heap/wasm-apps/bulk-memory/test_bulk_memory.wat @@ -9,4 +9,4 @@ ) (export "memory_fill_test" (func $memory_fill_test)) -) \ No newline at end of file +) From 7ddc5f8c242cebb3b5132dcbd4573b34ced1f998 Mon Sep 17 00:00:00 2001 From: TL Date: Fri, 9 May 2025 17:43:09 +0800 Subject: [PATCH 23/29] try func call to replace shared heap chain traverse --- core/iwasm/aot/aot_reloc.h | 8 + core/iwasm/common/wasm_runtime_common.c | 36 +++ core/iwasm/common/wasm_runtime_common.h | 9 + core/iwasm/compilation/aot_emit_memory.c | 345 +++++++---------------- core/iwasm/compilation/aot_llvm.c | 95 ++++--- core/iwasm/compilation/aot_llvm.h | 4 +- wamr-compiler/CMakeLists.txt | 2 + 7 files changed, 217 insertions(+), 282 deletions(-) diff --git a/core/iwasm/aot/aot_reloc.h b/core/iwasm/aot/aot_reloc.h index ca9ba17f1f..eb41e8c2b7 100644 --- a/core/iwasm/aot/aot_reloc.h +++ b/core/iwasm/aot/aot_reloc.h @@ -185,6 +185,13 @@ typedef struct { #define REG_STRINGREF_SYM() #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +#define REG_SHARED_HEAP_SYM() \ + REG_SYM(wasm_runtime_update_last_used_shared_heap), +#else +#define REG_SHARED_HEAP_SYM() +#endif + #define REG_COMMON_SYMBOLS \ REG_SYM(aot_set_exception_with_id), \ REG_SYM(aot_invoke_native), \ @@ -218,6 +225,7 @@ typedef struct { REG_LLVM_PGO_SYM() \ REG_GC_SYM() \ REG_STRINGREF_SYM() \ + REG_SHARED_HEAP_SYM() \ #define CHECK_RELOC_OFFSET(data_size) do { \ if (!check_reloc_offset(target_section_size, \ diff --git a/core/iwasm/common/wasm_runtime_common.c b/core/iwasm/common/wasm_runtime_common.c index dcee0aeafc..d19c3dca02 100644 --- a/core/iwasm/common/wasm_runtime_common.c +++ b/core/iwasm/common/wasm_runtime_common.c @@ -7883,3 +7883,39 @@ wasm_runtime_is_underlying_binary_freeable(WASMModuleCommon *const module) return true; } + +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +wasm_runtime_update_last_used_shared_heap(WASMModuleInstanceCommon *module_inst, + uintptr_t app_offset, size_t bytes, + uintptr_t *shared_heap_start_off_p, + uintptr_t *shared_heap_end_off_p, + uint8 **shared_heap_base_addr_adj_p, + bool is_memory64) +{ + WASMSharedHeap *heap = wasm_runtime_get_shared_heap(module_inst), *cur; + uint64 shared_heap_start, shared_heap_end; + + if (bytes == 0) { + bytes = 1; + } + + /* Find the exact shared heap that app addr is in, and update last used + * shared heap info in func context */ + for (cur = heap; cur; cur = cur->chain_next) { + shared_heap_start = + is_memory64 ? cur->start_off_mem64 : cur->start_off_mem32; + shared_heap_end = shared_heap_start - 1 + cur->size; + if (app_offset >= shared_heap_start + && app_offset <= shared_heap_end - bytes + 1) { + *shared_heap_start_off_p = (uintptr_t)shared_heap_start; + *shared_heap_end_off_p = (uintptr_t)shared_heap_end; + *shared_heap_base_addr_adj_p = + cur->base_addr - (uintptr_t)shared_heap_start; + return true; + } + } + + return false; +} +#endif \ No newline at end of file diff --git a/core/iwasm/common/wasm_runtime_common.h b/core/iwasm/common/wasm_runtime_common.h index 64a6cd7936..67f752f387 100644 --- a/core/iwasm/common/wasm_runtime_common.h +++ b/core/iwasm/common/wasm_runtime_common.h @@ -1336,6 +1336,15 @@ void wasm_runtime_set_linux_perf(bool flag); #endif +#if WASM_ENABLE_SHARED_HEAP != 0 +bool +wasm_runtime_update_last_used_shared_heap(WASMModuleInstanceCommon *module_inst, + uintptr_t app_offset, size_t bytes, + uintptr_t *shared_heap_start_off_p, + uintptr_t *shared_heap_end_off_p, + uint8 **shared_heap_base_addr_adj_p, bool is_memory64); +#endif + #ifdef __cplusplus } #endif diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index cb5d54c76d..3cbc3222a2 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -192,94 +192,60 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx); BUILD_LOAD_PTR(field_p, data_type, res); \ } while (0) -#define BUILD_GET_SHARED_HEAP_START(shared_heap_p, res) \ - do { \ - if (is_memory64) { \ - offset_u32 = offsetof(WASMSharedHeap, start_off_mem64); \ - } \ - else { \ - offset_u32 = offsetof(WASMSharedHeap, start_off_mem32); \ - } \ - field_offset = I32_CONST(offset_u32); \ - CHECK_LLVM_CONST(field_offset); \ - \ - BUILD_FIELD_PTR(shared_heap_p, field_offset, field_p, \ - "shared_heap_start_off_p"); \ - if (!(field_p = LLVMBuildBitCast(comp_ctx->builder, field_p, \ - offset_ptr_type, \ - "shared_heap_start_cast_p"))) { \ - aot_set_last_error("llvm build bit cast failed."); \ - goto fail; \ - } \ - BUILD_LOAD_PTR(field_p, offset_type, res); \ - } while (0) - -#define BUILD_GET_MAX_SHARED_HEAP_BOUND(shared_heap_check_bound) \ - do { \ - if (bulk_memory) { \ - shared_heap_check_bound = \ - is_memory64 ? I64_CONST(UINT64_MAX) \ - : (is_target_64bit ? I64_CONST(UINT32_MAX) \ - : I32_CONST(UINT32_MAX)); \ - } \ - else { \ - shared_heap_check_bound = \ - is_memory64 ? I64_CONST(UINT64_MAX - bytes_u32 + 1) \ - : (is_target_64bit \ - ? I64_CONST(UINT32_MAX - bytes_u32 + 1) \ - : I32_CONST(UINT32_MAX - bytes_u32 + 1)); \ - } \ - CHECK_LLVM_CONST(shared_heap_check_bound); \ - } while (0) - -/* Update last used shared heap info in function ctx, let runtime api - * update module inst shared heap info +/* Only update last used shared heap info(alloc ptr) in function ctx, + * let runtime API update module inst shared heap info * 1. shared_heap_start_off 2. shared_heap_end_off 3. shared_heap_base_addr_adj */ -static bool -aot_update_last_used_shared_heap(AOTCompContext *comp_ctx, - AOTFuncContext *func_ctx, - LLVMValueRef shared_heap_p, - LLVMValueRef shared_heap_start_off, - LLVMValueRef shared_heap_size, - bool is_target_64bit) +bool +aot_check_shared_heap_chain(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, + LLVMBasicBlockRef check_succ, + LLVMValueRef start_offset, LLVMValueRef bytes, + bool is_memory64) { - LLVMTypeRef shared_heap_off_type = is_target_64bit ? I64_TYPE : I32_TYPE, - shared_heap_off_ptr_type = - is_target_64bit ? INT64_PTR_TYPE : INT32_PTR_TYPE; - LLVMValueRef shared_heap_end_off, base_addr, shared_heap_base_addr_adj; - LLVMValueRef tmp_res, field_p, field_offset; - uint32 offset_u32; - - /* shared_heap_end_off = shared_heap_start_off + shared_heap_size - 1 */ - BUILD_OP(Add, shared_heap_start_off, - (is_target_64bit ? I64_NEG_ONE : I32_NEG_ONE), tmp_res, - "shared_heap_start_off_minus_one"); - BUILD_OP(Add, tmp_res, shared_heap_size, shared_heap_end_off, - "shared_heap_end_off"); - /* shared_heap_base_addr_adj = base_addr - shared_heap_start_off */ - BUILD_GET_SHARED_HEAP_FIELD(shared_heap_p, base_addr, INT8_PTR_TYPE, - base_addr); - if (!(base_addr = LLVMBuildPtrToInt(comp_ctx->builder, base_addr, - shared_heap_off_type, "base_addr"))) { - aot_set_last_error("llvm build ptr to int failed"); + LLVMValueRef param_values[7], ret_value, func, value, cmp; + LLVMTypeRef param_types[7], ret_type, func_type, func_ptr_type; + LLVMAttributeRef coldAttr; + unsigned kind; + + param_types[0] = INT8_PTR_TYPE; + param_types[1] = INTPTR_T_TYPE; + param_types[2] = SIZE_T_TYPE; + param_types[3] = INTPTR_T_PTR_TYPE; + param_types[4] = INTPTR_T_PTR_TYPE; + param_types[5] = INT8_PTR_TYPE; + param_types[6] = INT8_TYPE; + ret_type = INT8_TYPE; + + GET_AOT_FUNCTION(wasm_runtime_update_last_used_shared_heap, 7); + + kind = LLVMGetEnumAttributeKindForName("cold", 4); + coldAttr = LLVMCreateEnumAttribute(comp_ctx->context, kind, 0); + LLVMAddAttributeAtIndex(func, LLVMAttributeFunctionIndex, coldAttr); + + /* Call function */ + param_values[0] = func_ctx->aot_inst; + /* TODO: all use alloca to optimize the func call to single basic block? */ + param_values[1] = start_offset; + param_values[2] = bytes; + /* pass alloc ptr */ + param_values[3] = func_ctx->shared_heap_start_off; + param_values[4] = func_ctx->shared_heap_end_off; + param_values[5] = func_ctx->shared_heap_base_addr_adj; + param_values[6] = is_memory64 ? I8_ONE : I8_ZERO; + + if (!(ret_value = LLVMBuildCall2(comp_ctx->builder, func_type, func, + param_values, 7, "call"))) { + aot_set_last_error("llvm build call failed."); goto fail; } - BUILD_OP(Sub, base_addr, shared_heap_start_off, shared_heap_base_addr_adj, - "shared_heap_base_addr_adj"); - if (!(shared_heap_base_addr_adj = LLVMBuildIntToPtr( - comp_ctx->builder, shared_heap_base_addr_adj, - shared_heap_off_ptr_type, "shared_heap_base_addr_adj_ptr"))) { - aot_set_last_error("llvm build int to ptr failed."); + + BUILD_ICMP(LLVMIntEQ, ret_value, I8_ZERO, cmp, "shared_heap_oob"); + if (!aot_emit_exception(comp_ctx, func_ctx, + EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, true, cmp, + check_succ)) { goto fail; } - /* Store the local variable */ - BUILD_STORE_PTR(func_ctx->shared_heap_start_off, shared_heap_start_off); - BUILD_STORE_PTR(func_ctx->shared_heap_end_off, shared_heap_end_off); - BUILD_STORE_PTR(func_ctx->shared_heap_base_addr_adj, - shared_heap_base_addr_adj); - return true; fail: return false; @@ -301,102 +267,70 @@ aot_check_shared_heap_memory_overflow_common( LLVMValueRef mem_base_addr, LLVMValueRef bytes, uint32 bytes_u32, bool is_memory64, bool is_target_64bit, bool bulk_memory, bool enable_segue) { - LLVMBasicBlockRef check_shared_heap_chain, app_addr_in_cache_shared_heap, - app_addr_in_linear_mem, loopEntry, loopCond, loopBody, loopExit, - shared_heap_oob; - LLVMTypeRef offset_type, offset_ptr_type; - LLVMValueRef maddr = NULL, max_offset = NULL, cmp, cmp1, cmp2; - LLVMValueRef shared_heap_start_off, shared_heap_check_bound, - shared_heap_size, shared_heap_end_off, shared_heap_base_addr_adj; - /* The pointer that will traverse the shared heap chain */ - LLVMValueRef phi_shared_heap, cur_shared_heap; - LLVMValueRef field_offset, field_p, length; - uint32 offset_u32; - + LLVMBasicBlockRef app_addr_in_shared_heap_chain, + app_addr_in_cache_shared_heap, check_shared_heap_chain, + app_addr_in_linear_mem; + LLVMValueRef length, max_offset, maddr = NULL, cmp, cmp1, cmp2, + is_in_shared_heap; + LLVMValueRef shared_heap_start_off, shared_heap_end_off, + shared_heap_base_addr_adj, shared_heap_check_bound; + + LLVMTypeRef offset_type; /* On 64/32-bit target, the offset is 64/32-bit */ offset_type = is_target_64bit ? I64_TYPE : I32_TYPE; - offset_ptr_type = is_target_64bit ? INT64_PTR_TYPE : INT32_PTR_TYPE; - - /*--------------------------------------------------------------------- - * Create the basic blocks and arrange control flow(omit exception block - * and loop internal blocks for simplicity): - * - * +-------------------------+ - * | current block | - * +-------------------------+ - * / \ - * / \ - * no shared heap shared heap attached -------in cache------ - * | | | - * | | | - * | | | - * v v | - * +---------------------+ +-------------------------+ | - * | in_linear_mem |<----No-----| check shared_heap chain | | - * +---------------------+ +-----------+-------------+ | - * | Yes | - * | | | - * | v | - * | +---------------+---------------+ | - * | | Loop: Traverse shared heap | | - * | +---------------+---------------+ | - * | | | - * | v | - * | +---------------+---------------+ | - * | | Cache shared heap branch | <---- - * | +-------------------------------+ - * | | - * | --------------------------- - * | | - * v v - * +---------------------+ - * | maddr_phi | - * +---------------------+ - *---------------------------------------------------------------------*/ - ADD_BASIC_BLOCK(check_shared_heap_chain, "check_shared_heap_chain"); + + ADD_BASIC_BLOCK(app_addr_in_shared_heap_chain, + "app_addr_in_shared_heap_chain"); ADD_BASIC_BLOCK(app_addr_in_cache_shared_heap, "app_addr_in_cache_shared_heap"); + ADD_BASIC_BLOCK(check_shared_heap_chain, "check_shared_heap_chain"); ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); - LLVMMoveBasicBlockAfter(check_shared_heap_chain, block_curr); + LLVMMoveBasicBlockAfter(app_addr_in_shared_heap_chain, block_curr); LLVMMoveBasicBlockAfter(app_addr_in_cache_shared_heap, - check_shared_heap_chain); + app_addr_in_shared_heap_chain); + LLVMMoveBasicBlockAfter(check_shared_heap_chain, + app_addr_in_cache_shared_heap); LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, app_addr_in_cache_shared_heap); + if (!bulk_memory) LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem); else LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ); - /* If there is no shared heap attached, branch to linear memory */ - BUILD_IS_NOT_NULL(func_ctx->shared_heap, cmp, "has_shared_heap"); - BUILD_COND_BR(cmp, check_shared_heap_chain, app_addr_in_linear_mem); - - /*--------------------------------------------------------------------- - * In the case where a shared heap is attached, we determine if the bytes - * to access reside in the shared heap chain. If yes, we enter a loop that - * traverses the chain by using a phi node to merge two definitions of the - * pointer (initial and updated). - *---------------------------------------------------------------------*/ - SET_BUILD_POS(check_shared_heap_chain); - - /* Add loop basic blocks */ - ADD_BASIC_BLOCK(loopEntry, "loop_entry"); - ADD_BASIC_BLOCK(loopBody, "loop_body"); - ADD_BASIC_BLOCK(loopCond, "loop_cond"); - ADD_BASIC_BLOCK(loopExit, "loop_exit"); - LLVMMoveBasicBlockAfter(loopEntry, check_shared_heap_chain); - LLVMMoveBasicBlockAfter(loopBody, loopEntry); - LLVMMoveBasicBlockAfter(loopCond, loopBody); - LLVMMoveBasicBlockAfter(loopExit, loopCond); - + // BUILD_LOAD_PTR(func_ctx->shared_heap_start_off, offset_type, + // shared_heap_start_off); + // BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, + // is_in_shared_heap, "shared_heap_lb_cmp"); + BUILD_ICMP(LLVMIntUGE, start_offset, func_ctx->shared_heap_head_start_off, + is_in_shared_heap, "shared_heap_lb_cmp"); + /* For bulk memory previously already check whether start_offset + bytes + * overflows, can safely omit the upper boundary check */ + if (!bulk_memory) { + shared_heap_check_bound = + is_memory64 + ? I64_CONST(UINT64_MAX - bytes_u32 + 1) + : (is_target_64bit ? I64_CONST(UINT32_MAX - bytes_u32 + 1) + : I32_CONST(UINT32_MAX - bytes_u32 + 1)); + CHECK_LLVM_CONST(shared_heap_check_bound); + BUILD_ICMP(LLVMIntULE, start_offset, shared_heap_check_bound, cmp, + "shared_heap_ub_cmp"); + BUILD_OP(And, is_in_shared_heap, cmp, is_in_shared_heap, + "is_in_shared_heap"); + } + BUILD_COND_BR(is_in_shared_heap, app_addr_in_shared_heap_chain, + app_addr_in_linear_mem); + + SET_BUILD_POS(app_addr_in_shared_heap_chain); /* Load the local variable of the function */ BUILD_LOAD_PTR(func_ctx->shared_heap_start_off, offset_type, shared_heap_start_off); BUILD_LOAD_PTR(func_ctx->shared_heap_end_off, offset_type, shared_heap_end_off); /* Check if the app address is in the cache shared heap range. - * If yes, branch to the cache branch; if not, proceed into the loop */ + * If yes, branch to the cache branch; if not, check the shared heap chain + */ BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, "cmp_cache_shared_heap_start"); if (bulk_memory) { @@ -415,100 +349,19 @@ aot_check_shared_heap_memory_overflow_common( "cmp_cache_shared_heap_end"); } BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); - BUILD_COND_BR(cmp2, app_addr_in_cache_shared_heap, loopEntry); + BUILD_COND_BR(cmp2, app_addr_in_cache_shared_heap, check_shared_heap_chain); - /* Check whether app_offset is not in the overall shared heap chain, - * if not branch to linear memory. */ - SET_BUILD_POS(loopEntry); - BUILD_GET_SHARED_HEAP_START(func_ctx->shared_heap, shared_heap_start_off); - BUILD_GET_MAX_SHARED_HEAP_BOUND(shared_heap_check_bound); - BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, - "cmp_shared_heap_chain_start"); - BUILD_ICMP(LLVMIntULE, bulk_memory ? max_offset : start_offset, - shared_heap_check_bound, cmp1, "cmp_shared_heap_chain_end"); - BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); - BUILD_COND_BR(cmp2, loopBody, app_addr_in_linear_mem); - - /*--------------------------------------------------------------------- - * LoopBody: create a phi node to merge the pointer values. - * - Incoming from loopEntry: the initial pointer is func_ctx->shared_heap. - * - Incoming from loopBody: the updated pointer (next pointer in the - * chain). - *---------------------------------------------------------------------*/ - SET_BUILD_POS(loopBody); - phi_shared_heap = - LLVMBuildPhi(comp_ctx->builder, INT8_PTR_TYPE, "phi_shared_heap"); - LLVMAddIncoming(phi_shared_heap, &func_ctx->shared_heap, &loopEntry, 1); - /* In loopBody, we check whether the current shared heap node - * (phi_shared_heap) contains the target address. */ - BUILD_GET_SHARED_HEAP_START(phi_shared_heap, shared_heap_start_off); - BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, cmp, - "cmp_shared_heap_start"); - - BUILD_GET_SHARED_HEAP_FIELD(phi_shared_heap, size, I64_TYPE, - shared_heap_size); - if (!is_target_64bit) { - BUILD_TRUNC(shared_heap_size, I32_TYPE); - } - if (bulk_memory) { - length = bytes; - } - else { - length = is_target_64bit ? I64_CONST(bytes_u32) : I32_CONST(bytes_u32); - CHECK_LLVM_CONST(length); - } - /* start_offset + length - 1 <= shared_heap_start_off + size - 1 */ - BUILD_OP(Sub, shared_heap_size, length, shared_heap_check_bound, - "shared_heap_check_bound1"); - BUILD_OP(Add, shared_heap_start_off, shared_heap_check_bound, - shared_heap_check_bound, "shared_heap_check_bound2"); - BUILD_ICMP(LLVMIntULE, start_offset, shared_heap_check_bound, cmp1, - "cmp_shared_heap_end"); - BUILD_OP(And, cmp, cmp1, cmp2, "is_in_shared_heap"); - BUILD_COND_BR(cmp2, loopExit, loopCond); - - /*--------------------------------------------------------------------- - * LoopCond: update the pointer to traverse to the next shared heap in the - * chain. The updated pointer is then added as an incoming value to the phi - * node. If the address in shared heap chain region but not in the shared - * heap, it means OOB(access bytes cross shared heaps boundaries) for a - * shared heap. - *---------------------------------------------------------------------*/ - ADD_BASIC_BLOCK(shared_heap_oob, "shared_heap_oob"); - LLVMMoveBasicBlockAfter(shared_heap_oob, loopCond); - SET_BUILD_POS(shared_heap_oob); - if (!aot_emit_exception(comp_ctx, func_ctx, - EXCE_OUT_OF_BOUNDS_MEMORY_ACCESS, false, NULL, - NULL)) { - goto fail; + SET_BUILD_POS(check_shared_heap_chain); + if (!bulk_memory) { + bytes = is_target_64bit ? I64_CONST(bytes_u32) : I32_CONST(bytes_u32); } - - SET_BUILD_POS(loopCond); - BUILD_GET_SHARED_HEAP_FIELD(phi_shared_heap, chain_next, INT8_PTR_TYPE, - cur_shared_heap); - /* Add the new value from loopBody as an incoming edge to the phi node */ - LLVMAddIncoming(phi_shared_heap, &cur_shared_heap, &loopCond, 1); - BUILD_IS_NOT_NULL(cur_shared_heap, cmp, "is_not_null"); - BUILD_COND_BR(cmp, loopBody, shared_heap_oob); - - /*--------------------------------------------------------------------- - * LoopExit: at this point, phi_shared_heap is expected to be valid if - * the app address is contained in a shared heap; otherwise, it is oob - * and already handled in the loopCond block. - *---------------------------------------------------------------------*/ - SET_BUILD_POS(loopExit); - /* Update last accessed shared heap, the shared_heap_size and - * shared_heap_start_off is already prepared in loop body. */ - if (!aot_update_last_used_shared_heap(comp_ctx, func_ctx, phi_shared_heap, - shared_heap_start_off, - shared_heap_size, is_target_64bit)) { + if (!aot_check_shared_heap_chain(comp_ctx, func_ctx, + app_addr_in_cache_shared_heap, + start_offset, bytes, is_memory64)) { + aot_set_last_error("llvm build aot_check_shared_heap_chain failed"); goto fail; } - BUILD_BR(app_addr_in_cache_shared_heap); - /*--------------------------------------------------------------------- - * Finally, in the cache shared heap branch, compute the native address. - *---------------------------------------------------------------------*/ SET_BUILD_POS(app_addr_in_cache_shared_heap); /* load the local variable */ BUILD_LOAD_PTR(func_ctx->shared_heap_base_addr_adj, INT8_PTR_TYPE, @@ -547,7 +400,7 @@ aot_check_shared_heap_memory_overflow_common( LLVMAddIncoming(maddr_phi, &maddr, &app_addr_in_cache_shared_heap, 1); BUILD_BR(block_maddr_phi); - LLVMPositionBuilderAtEnd(comp_ctx->builder, app_addr_in_linear_mem); + SET_BUILD_POS(app_addr_in_linear_mem); block_curr = LLVMGetInsertBlock(comp_ctx->builder); return true; diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 6294c061ca..09d61c3bf2 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1526,60 +1526,87 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, func_ctx->aot_inst, &offset, 1, \ #field "_p"))) { \ aot_set_last_error("llvm build inbounds gep failed"); \ - return false; \ + goto fail; \ } \ if (!(load_val = \ LLVMBuildLoad2(comp_ctx->builder, type, field_p, #field))) { \ aot_set_last_error("llvm build load failed"); \ - return false; \ + goto fail; \ } \ if (!(func_ctx->field = \ LLVMBuildAlloca(comp_ctx->builder, type, #field))) { \ aot_set_last_error("llvm build alloca failed"); \ - return false; \ + goto fail; \ } \ if (!LLVMBuildStore(comp_ctx->builder, load_val, func_ctx->field)) { \ aot_set_last_error("llvm build store failed"); \ - return false; \ - } \ - } while (0) - -#define LOAD_MODULE_EXTRA_FIELD(field, type) \ - do { \ - get_module_extra_field_offset(field); \ - offset = I32_CONST(offset_u32); \ - CHECK_LLVM_CONST(offset); \ - if (!(field_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, \ - func_ctx->aot_inst, &offset, 1, \ - #field "_p"))) { \ - aot_set_last_error("llvm build inbounds gep failed"); \ - return false; \ - } \ - if (!(func_ctx->field = \ - LLVMBuildLoad2(comp_ctx->builder, type, field_p, #field))) { \ - aot_set_last_error("llvm build load failed"); \ - return false; \ + goto fail; \ } \ } while (0) static bool create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { - LLVMValueRef offset, field_p, load_val; + LLVMValueRef offset, field_p, load_val, shared_heap_head_p, + shared_heap_head; + LLVMTypeRef shared_heap_offset_type; uint32 offset_u32; +#if WASM_ENABLE_MEMORY64 == 0 + bool is_memory64 = false; +#else + bool is_memory64 = IS_MEMORY64; +#endif - /* shared_heap_base_addr_adj, shared_heap_start_off, and shared_heap_end_off - * can be updated later, use local variable to represent them */ + shared_heap_offset_type = + comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE; + + /* shared_heap_base_addr_adj, shared_heap_start_off, and + * shared_heap_end_off can be updated later, use local variable to + * represent them */ LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(shared_heap_base_addr_adj, INT8_PTR_TYPE); - LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA( - shared_heap_start_off, - comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE); - LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA( - shared_heap_end_off, - comp_ctx->pointer_size == sizeof(uint64) ? I64_TYPE : I32_TYPE); - /* Shared Heap won't be updated, no need to alloca */ - LOAD_MODULE_EXTRA_FIELD(shared_heap, INT8_PTR_TYPE); + LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(shared_heap_start_off, + shared_heap_offset_type); + LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(shared_heap_end_off, + shared_heap_offset_type); + + /* Shared Heap head start off won't be updated, no need to alloca */ + get_module_extra_field_offset(shared_heap); + offset = I32_CONST(offset_u32); + CHECK_LLVM_CONST(offset); + if (!(shared_heap_head_p = LLVMBuildInBoundsGEP2( + comp_ctx->builder, INT8_TYPE, func_ctx->aot_inst, &offset, 1, + "shared_heap_head_p"))) { + aot_set_last_error("llvm build inbounds gep failed"); + goto fail; + } + if (!(shared_heap_head = + LLVMBuildLoad2(comp_ctx->builder, INT8_PTR_TYPE, + shared_heap_head_p, "shared_heap_head"))) { + aot_set_last_error("llvm build load failed"); + goto fail; + } + + if (is_memory64) { + offset_u32 = offsetof(WASMSharedHeap, start_off_mem64); + } + else { + offset_u32 = offsetof(WASMSharedHeap, start_off_mem32); + } + offset = I32_CONST(offset_u32); + CHECK_LLVM_CONST(offset); + if (!(field_p = LLVMBuildInBoundsGEP2(comp_ctx->builder, INT8_TYPE, + shared_heap_head, &offset, 1, + "head_start_off_p"))) { + aot_set_last_error("llvm build inbounds gep failed"); + goto fail; + } + if (!(func_ctx->shared_heap_head_start_off = + LLVMBuildLoad2(comp_ctx->builder, shared_heap_offset_type, + field_p, "shared_heap_head_start_off"))) { + aot_set_last_error("llvm build load failed"); + goto fail; + } return true; fail: @@ -2438,7 +2465,7 @@ jit_stack_size_callback(void *user_data, const char *name, size_t namelen, stack_consumption_to_call_wrapped_func = musttail ? 0 : aot_estimate_stack_usage_for_function_call( - comp_ctx, func_ctx->aot_func->func_type); + comp_ctx, func_ctx->aot_func->func_type); LOG_VERBOSE("func %.*s stack %u + %zu + %u", (int)namelen, name, stack_consumption_to_call_wrapped_func, stack_size, call_size); diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index 896a9caea4..5a1d993e93 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -275,8 +275,8 @@ typedef struct AOTFuncContext { LLVMValueRef shared_heap_base_addr_adj; LLVMValueRef shared_heap_start_off; LLVMValueRef shared_heap_end_off; - /* The head of shared heap chain, and its start offset */ - LLVMValueRef shared_heap; + /* The start offset of the head of shared heap chain */ + LLVMValueRef shared_heap_head_start_off; LLVMBasicBlockRef got_exception_block; LLVMBasicBlockRef func_return_block; diff --git a/wamr-compiler/CMakeLists.txt b/wamr-compiler/CMakeLists.txt index 0ce6473944..68648b84b7 100644 --- a/wamr-compiler/CMakeLists.txt +++ b/wamr-compiler/CMakeLists.txt @@ -284,6 +284,7 @@ include (${IWASM_DIR}/interpreter/iwasm_interp.cmake) include (${IWASM_DIR}/aot/iwasm_aot.cmake) include (${IWASM_DIR}/compilation/iwasm_compl.cmake) include (${PROJECT_SOURCE_DIR}/../build-scripts/version.cmake) +include (${IWASM_DIR}/libraries/shared-heap/shared_heap.cmake) if (WAMR_BUILD_LIBC_BUILTIN EQUAL 1) include (${IWASM_DIR}/libraries/libc-builtin/libc_builtin.cmake) @@ -366,6 +367,7 @@ add_library (vmlib ${LIBC_WASI_SOURCE} ${LIB_PTHREAD_SOURCE} ${LIB_WASI_THREADS_SOURCE} + ${LIB_SHARED_HEAP_SOURCE} ${IWASM_COMMON_SOURCE} ${IWASM_INTERP_SOURCE} ${IWASM_AOT_SOURCE} From 7cdd02e06b815d2805ca505cb441ecff231ed80e Mon Sep 17 00:00:00 2001 From: TL Date: Thu, 15 May 2025 19:44:51 +0800 Subject: [PATCH 24/29] fix compilation error in JIT and potentially load nullptr --- core/iwasm/compilation/aot_emit_memory.c | 54 ++++++++++++------------ core/iwasm/compilation/aot_llvm.c | 48 ++++++++++++++++++--- 2 files changed, 71 insertions(+), 31 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 3cbc3222a2..8e2d43576f 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -145,6 +145,7 @@ ffs(int n) static LLVMValueRef get_memory_curr_page_count(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx); +#if WASM_ENABLE_SHARED_HEAP != 0 uint32 get_module_inst_extra_offset(AOTCompContext *comp_ctx); @@ -192,8 +193,7 @@ get_module_inst_extra_offset(AOTCompContext *comp_ctx); BUILD_LOAD_PTR(field_p, data_type, res); \ } while (0) -/* Only update last used shared heap info(alloc ptr) in function ctx, - * let runtime API update module inst shared heap info +/* Update last used shared heap info(alloc ptr) in function ctx: * 1. shared_heap_start_off 2. shared_heap_end_off 3. shared_heap_base_addr_adj */ bool @@ -224,7 +224,6 @@ aot_check_shared_heap_chain(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, /* Call function */ param_values[0] = func_ctx->aot_inst; - /* TODO: all use alloca to optimize the func call to single basic block? */ param_values[1] = start_offset; param_values[2] = bytes; /* pass alloc ptr */ @@ -299,26 +298,12 @@ aot_check_shared_heap_memory_overflow_common( else LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ); - // BUILD_LOAD_PTR(func_ctx->shared_heap_start_off, offset_type, - // shared_heap_start_off); - // BUILD_ICMP(LLVMIntUGE, start_offset, shared_heap_start_off, - // is_in_shared_heap, "shared_heap_lb_cmp"); - BUILD_ICMP(LLVMIntUGE, start_offset, func_ctx->shared_heap_head_start_off, + /* Use >= here for func_ctx->shared_heap_head_start_off = + * shared_heap_head->start - 1 and use UINT32_MAX/UINT64_MAX value to + * indicate invalid value. The shared heap chain oob will be detected in + * app_addr_in_shared_heap block or aot_check_shared_heap_chain function */ + BUILD_ICMP(LLVMIntUGT, start_offset, func_ctx->shared_heap_head_start_off, is_in_shared_heap, "shared_heap_lb_cmp"); - /* For bulk memory previously already check whether start_offset + bytes - * overflows, can safely omit the upper boundary check */ - if (!bulk_memory) { - shared_heap_check_bound = - is_memory64 - ? I64_CONST(UINT64_MAX - bytes_u32 + 1) - : (is_target_64bit ? I64_CONST(UINT32_MAX - bytes_u32 + 1) - : I32_CONST(UINT32_MAX - bytes_u32 + 1)); - CHECK_LLVM_CONST(shared_heap_check_bound); - BUILD_ICMP(LLVMIntULE, start_offset, shared_heap_check_bound, cmp, - "shared_heap_ub_cmp"); - BUILD_OP(And, is_in_shared_heap, cmp, is_in_shared_heap, - "is_in_shared_heap"); - } BUILD_COND_BR(is_in_shared_heap, app_addr_in_shared_heap_chain, app_addr_in_linear_mem); @@ -435,6 +420,7 @@ aot_check_bulk_memory_shared_heap_memory_overflow( start_offset, max_addr, NULL, bytes, 0, is_memory64, is_target_64bit, true, false); } +#endif LLVMValueRef aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, @@ -443,10 +429,10 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, { LLVMValueRef offset_const = MEMORY64_COND_VALUE(I64_CONST(offset), I32_CONST(offset)); - LLVMValueRef addr, maddr, maddr_phi = NULL, offset1, cmp1, cmp; + LLVMValueRef addr, maddr, offset1, cmp1, cmp; LLVMValueRef mem_base_addr, mem_check_bound; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); - LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; + LLVMBasicBlockRef check_succ; AOTValue *aot_value_top; uint32 local_idx_of_aot_value = 0; uint64 const_value; @@ -461,6 +447,10 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, #else bool is_memory64 = IS_MEMORY64; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + LLVMValueRef maddr_phi = NULL; + LLVMBasicBlockRef block_maddr_phi = NULL; +#endif is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; @@ -626,6 +616,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, block_curr = check_integer_overflow_end; } +#if WASM_ENABLE_SHARED_HEAP != 0 if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); SET_BUILD_POS(block_maddr_phi); @@ -645,6 +636,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } } +#endif if (comp_ctx->enable_bound_check && !(is_local_of_aot_value @@ -723,6 +715,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } } +#if WASM_ENABLE_SHARED_HEAP != 0 if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1); @@ -734,6 +727,7 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return maddr_phi; } else +#endif return maddr; fail: return NULL; @@ -1356,9 +1350,9 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, LLVMValueRef offset, LLVMValueRef bytes) { LLVMValueRef maddr, max_addr, cmp, cmp1; - LLVMValueRef mem_base_addr, maddr_phi = NULL; + LLVMValueRef mem_base_addr; LLVMBasicBlockRef block_curr = LLVMGetInsertBlock(comp_ctx->builder); - LLVMBasicBlockRef check_succ, block_maddr_phi = NULL; + LLVMBasicBlockRef check_succ; LLVMValueRef mem_size; bool is_target_64bit; #if WASM_ENABLE_MEMORY64 == 0 @@ -1366,6 +1360,10 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, #else bool is_memory64 = IS_MEMORY64; #endif +#if WASM_ENABLE_SHARED_HEAP != 0 + LLVMValueRef maddr_phi = NULL; + LLVMBasicBlockRef block_maddr_phi = NULL; +#endif is_target_64bit = (comp_ctx->pointer_size == sizeof(uint64)) ? true : false; @@ -1466,6 +1464,7 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, block_curr = check_integer_overflow_end; } +#if WASM_ENABLE_SHARED_HEAP != 0 if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); SET_BUILD_POS(block_maddr_phi); @@ -1483,6 +1482,7 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } } +#endif /* mem_size is always 64-bit, extend max_addr on 32 bits platform */ if (!is_target_64bit @@ -1506,6 +1506,7 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, goto fail; } +#if WASM_ENABLE_SHARED_HEAP != 0 if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1); @@ -1517,6 +1518,7 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return maddr_phi; } else +#endif return maddr; fail: return NULL; diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 09d61c3bf2..01e9aac3f5 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1517,6 +1517,14 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, return true; } +#define BUILD_IS_NOT_NULL(value, res, name) \ + do { \ + if (!(res = LLVMBuildIsNotNull(comp_ctx->builder, value, name))) { \ + aot_set_last_error("llvm build is not null failed."); \ + goto fail; \ + } \ + } while (0) + #define LOAD_MODULE_EXTRA_FIELD_AND_ALLOCA(field, type) \ do { \ get_module_extra_field_offset(field); \ @@ -1547,8 +1555,10 @@ create_memory_info(const AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, static bool create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) { +#if WASM_ENABLE_SHARED_HEAP != 0 LLVMValueRef offset, field_p, load_val, shared_heap_head_p, - shared_heap_head; + shared_heap_head, cmp, field_p_or_default, shared_heap_head_start_off, + shared_heap_head_start_off_minus_one; LLVMTypeRef shared_heap_offset_type; uint32 offset_u32; #if WASM_ENABLE_MEMORY64 == 0 @@ -1586,6 +1596,7 @@ create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) aot_set_last_error("llvm build load failed"); goto fail; } + BUILD_IS_NOT_NULL(shared_heap_head, cmp, "has_shared_heap"); if (is_memory64) { offset_u32 = offsetof(WASMSharedHeap, start_off_mem64); @@ -1601,16 +1612,43 @@ create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) aot_set_last_error("llvm build inbounds gep failed"); goto fail; } - if (!(func_ctx->shared_heap_head_start_off = - LLVMBuildLoad2(comp_ctx->builder, shared_heap_offset_type, - field_p, "shared_heap_head_start_off"))) { + + /* Select shared heap head ptr or safe alloca + * shared_heap_start_off(UINT32_MAX/UINT64_MAX) based on the condition */ + if (!(field_p_or_default = LLVMBuildSelect(comp_ctx->builder, cmp, field_p, + func_ctx->shared_heap_start_off, + "ptr_or_default"))) { + aot_set_last_error("llvm build select failed"); + goto fail; + } + + if (!(shared_heap_head_start_off = LLVMBuildLoad2( + comp_ctx->builder, shared_heap_offset_type, field_p_or_default, + "shared_heap_head_start_off"))) { + aot_set_last_error("llvm build load failed"); + goto fail; + } + if (!(shared_heap_head_start_off_minus_one = LLVMBuildAdd( + comp_ctx->builder, shared_heap_head_start_off, + comp_ctx->pointer_size == sizeof(uint64) ? I64_NEG_ONE + : I32_NEG_ONE, + "head_start_off_minus_one"))) { aot_set_last_error("llvm build load failed"); goto fail; } + if (!(func_ctx->shared_heap_head_start_off = LLVMBuildSelect( + comp_ctx->builder, cmp, shared_heap_head_start_off_minus_one, + shared_heap_head_start_off, "head_start_off"))) { + aot_set_last_error("llvm build select failed"); + goto fail; + } return true; fail: return false; +#else + return true; +#endif } static bool @@ -2465,7 +2503,7 @@ jit_stack_size_callback(void *user_data, const char *name, size_t namelen, stack_consumption_to_call_wrapped_func = musttail ? 0 : aot_estimate_stack_usage_for_function_call( - comp_ctx, func_ctx->aot_func->func_type); + comp_ctx, func_ctx->aot_func->func_type); LOG_VERBOSE("func %.*s stack %u + %zu + %u", (int)namelen, name, stack_consumption_to_call_wrapped_func, stack_size, call_size); From b0415c0d5629a26dcb671bf18259f3a3d23a3b61 Mon Sep 17 00:00:00 2001 From: TL Date: Fri, 16 May 2025 18:03:13 +0800 Subject: [PATCH 25/29] add option for wamrc to enable single shared heap/multi shared heap, and update shared heap unit tests and sample --- core/iwasm/compilation/aot_emit_memory.c | 90 ++++++---- core/iwasm/compilation/aot_llvm.c | 11 +- core/iwasm/compilation/aot_llvm.h | 1 + core/iwasm/include/aot_comp_option.h | 1 + samples/shared-heap/CMakeLists.txt | 4 + samples/shared-heap/src/shared_heap_chain.c | 4 +- tests/unit/shared-heap/shared_heap_test.cc | 159 +++++++++++++++--- .../unit/shared-heap/wasm-apps/CMakeLists.txt | 13 ++ .../wasm-apps/memory64/CMakeLists.txt | 17 +- wamr-compiler/main.c | 14 +- 10 files changed, 243 insertions(+), 71 deletions(-) diff --git a/core/iwasm/compilation/aot_emit_memory.c b/core/iwasm/compilation/aot_emit_memory.c index 8e2d43576f..21a9f6387f 100644 --- a/core/iwasm/compilation/aot_emit_memory.c +++ b/core/iwasm/compilation/aot_emit_memory.c @@ -278,36 +278,47 @@ aot_check_shared_heap_memory_overflow_common( /* On 64/32-bit target, the offset is 64/32-bit */ offset_type = is_target_64bit ? I64_TYPE : I32_TYPE; - ADD_BASIC_BLOCK(app_addr_in_shared_heap_chain, - "app_addr_in_shared_heap_chain"); ADD_BASIC_BLOCK(app_addr_in_cache_shared_heap, "app_addr_in_cache_shared_heap"); - ADD_BASIC_BLOCK(check_shared_heap_chain, "check_shared_heap_chain"); ADD_BASIC_BLOCK(app_addr_in_linear_mem, "app_addr_in_linear_mem"); - LLVMMoveBasicBlockAfter(app_addr_in_shared_heap_chain, block_curr); - LLVMMoveBasicBlockAfter(app_addr_in_cache_shared_heap, - app_addr_in_shared_heap_chain); - LLVMMoveBasicBlockAfter(check_shared_heap_chain, - app_addr_in_cache_shared_heap); - LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, - app_addr_in_cache_shared_heap); + if (comp_ctx->enable_shared_heap) { + LLVMMoveBasicBlockAfter(app_addr_in_cache_shared_heap, block_curr); + LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, + app_addr_in_cache_shared_heap); + } + else if (comp_ctx->enable_shared_chain) { + ADD_BASIC_BLOCK(app_addr_in_shared_heap_chain, + "app_addr_in_shared_heap_chain"); + ADD_BASIC_BLOCK(check_shared_heap_chain, "check_shared_heap_chain"); + LLVMMoveBasicBlockAfter(app_addr_in_shared_heap_chain, block_curr); + LLVMMoveBasicBlockAfter(app_addr_in_cache_shared_heap, + app_addr_in_shared_heap_chain); + LLVMMoveBasicBlockAfter(check_shared_heap_chain, + app_addr_in_cache_shared_heap); + LLVMMoveBasicBlockAfter(app_addr_in_linear_mem, + app_addr_in_cache_shared_heap); + } if (!bulk_memory) LLVMMoveBasicBlockAfter(block_maddr_phi, app_addr_in_linear_mem); else LLVMMoveBasicBlockAfter(block_maddr_phi, check_succ); - /* Use >= here for func_ctx->shared_heap_head_start_off = - * shared_heap_head->start - 1 and use UINT32_MAX/UINT64_MAX value to - * indicate invalid value. The shared heap chain oob will be detected in - * app_addr_in_shared_heap block or aot_check_shared_heap_chain function */ - BUILD_ICMP(LLVMIntUGT, start_offset, func_ctx->shared_heap_head_start_off, - is_in_shared_heap, "shared_heap_lb_cmp"); - BUILD_COND_BR(is_in_shared_heap, app_addr_in_shared_heap_chain, - app_addr_in_linear_mem); + if (comp_ctx->enable_shared_chain) { + /* Use >= here for func_ctx->shared_heap_head_start_off = + * shared_heap_head->start - 1 and use UINT32_MAX/UINT64_MAX value to + * indicate invalid value. The shared heap chain oob will be detected in + * app_addr_in_shared_heap block or aot_check_shared_heap_chain function + */ + BUILD_ICMP(LLVMIntUGT, start_offset, + func_ctx->shared_heap_head_start_off, is_in_shared_heap, + "shared_heap_lb_cmp"); + BUILD_COND_BR(is_in_shared_heap, app_addr_in_shared_heap_chain, + app_addr_in_linear_mem); - SET_BUILD_POS(app_addr_in_shared_heap_chain); + SET_BUILD_POS(app_addr_in_shared_heap_chain); + } /* Load the local variable of the function */ BUILD_LOAD_PTR(func_ctx->shared_heap_start_off, offset_type, shared_heap_start_off); @@ -334,17 +345,24 @@ aot_check_shared_heap_memory_overflow_common( "cmp_cache_shared_heap_end"); } BUILD_OP(And, cmp, cmp1, cmp2, "is_in_cache_shared_heap"); - BUILD_COND_BR(cmp2, app_addr_in_cache_shared_heap, check_shared_heap_chain); - - SET_BUILD_POS(check_shared_heap_chain); - if (!bulk_memory) { - bytes = is_target_64bit ? I64_CONST(bytes_u32) : I32_CONST(bytes_u32); - } - if (!aot_check_shared_heap_chain(comp_ctx, func_ctx, - app_addr_in_cache_shared_heap, - start_offset, bytes, is_memory64)) { - aot_set_last_error("llvm build aot_check_shared_heap_chain failed"); - goto fail; + if (comp_ctx->enable_shared_heap) { + BUILD_COND_BR(cmp2, app_addr_in_cache_shared_heap, + app_addr_in_linear_mem); + } + else if (comp_ctx->enable_shared_chain) { + BUILD_COND_BR(cmp2, app_addr_in_cache_shared_heap, + check_shared_heap_chain); + SET_BUILD_POS(check_shared_heap_chain); + if (!bulk_memory) { + bytes = + is_target_64bit ? I64_CONST(bytes_u32) : I32_CONST(bytes_u32); + } + if (!aot_check_shared_heap_chain(comp_ctx, func_ctx, + app_addr_in_cache_shared_heap, + start_offset, bytes, is_memory64)) { + aot_set_last_error("llvm build aot_check_shared_heap_chain failed"); + goto fail; + } } SET_BUILD_POS(app_addr_in_cache_shared_heap); @@ -617,7 +635,8 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } #if WASM_ENABLE_SHARED_HEAP != 0 - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); SET_BUILD_POS(block_maddr_phi); if (!(maddr_phi = @@ -716,7 +735,8 @@ aot_check_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } #if WASM_ENABLE_SHARED_HEAP != 0 - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1); if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { @@ -1465,7 +1485,8 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } #if WASM_ENABLE_SHARED_HEAP != 0 - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { ADD_BASIC_BLOCK(block_maddr_phi, "maddr_phi"); SET_BUILD_POS(block_maddr_phi); if (!(maddr_phi = LLVMBuildPhi(comp_ctx->builder, INT8_PTR_TYPE, @@ -1507,7 +1528,8 @@ check_bulk_memory_overflow(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx, } #if WASM_ENABLE_SHARED_HEAP != 0 - if (comp_ctx->enable_shared_heap /* TODO: && mem_idx == 0 */) { + if (comp_ctx->enable_shared_heap + || comp_ctx->enable_shared_chain /* TODO: && mem_idx == 0 */) { block_curr = LLVMGetInsertBlock(comp_ctx->builder); LLVMAddIncoming(maddr_phi, &maddr, &block_curr, 1); if (!LLVMBuildBr(comp_ctx->builder, block_maddr_phi)) { diff --git a/core/iwasm/compilation/aot_llvm.c b/core/iwasm/compilation/aot_llvm.c index 01e9aac3f5..0310c30b86 100644 --- a/core/iwasm/compilation/aot_llvm.c +++ b/core/iwasm/compilation/aot_llvm.c @@ -1613,8 +1613,8 @@ create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) goto fail; } - /* Select shared heap head ptr or safe alloca - * shared_heap_start_off(UINT32_MAX/UINT64_MAX) based on the condition */ + /* Select a valid shared heap head ptr or safe alloca ptr stores + * shared_heap_start_off(UINT32_MAX/UINT64_MAX) */ if (!(field_p_or_default = LLVMBuildSelect(comp_ctx->builder, cmp, field_p, func_ctx->shared_heap_start_off, "ptr_or_default"))) { @@ -1637,6 +1637,8 @@ create_shared_heap_info(AOTCompContext *comp_ctx, AOTFuncContext *func_ctx) goto fail; } + /* if there is attached shared heap(s), the value will be valid start_off-1, + * otherwise it will be UINT32_MAX/UINT64_MAX */ if (!(func_ctx->shared_heap_head_start_off = LLVMBuildSelect( comp_ctx->builder, cmp, shared_heap_head_start_off_minus_one, shared_heap_head_start_off, "head_start_off"))) { @@ -1942,7 +1944,7 @@ aot_create_func_context(const AOTCompData *comp_data, AOTCompContext *comp_ctx, } /* Load shared heap, shared heap start off mem32 or mem64 */ - if (comp_ctx->enable_shared_heap + if ((comp_ctx->enable_shared_heap || comp_ctx->enable_shared_chain) && !create_shared_heap_info(comp_ctx, func_ctx)) { goto fail; } @@ -2768,6 +2770,9 @@ aot_create_comp_context(const AOTCompData *comp_data, aot_comp_option_t option) if (option->enable_shared_heap) comp_ctx->enable_shared_heap = true; + if (option->enable_shared_chain) + comp_ctx->enable_shared_chain = true; + comp_ctx->opt_level = option->opt_level; comp_ctx->size_level = option->size_level; diff --git a/core/iwasm/compilation/aot_llvm.h b/core/iwasm/compilation/aot_llvm.h index 5a1d993e93..064f8ae35c 100644 --- a/core/iwasm/compilation/aot_llvm.h +++ b/core/iwasm/compilation/aot_llvm.h @@ -507,6 +507,7 @@ typedef struct AOTCompContext { bool enable_gc; bool enable_shared_heap; + bool enable_shared_chain; uint32 opt_level; uint32 size_level; diff --git a/core/iwasm/include/aot_comp_option.h b/core/iwasm/include/aot_comp_option.h index 8391766232..e12f3f6fc3 100644 --- a/core/iwasm/include/aot_comp_option.h +++ b/core/iwasm/include/aot_comp_option.h @@ -79,6 +79,7 @@ typedef struct AOTCompOption { bool enable_stack_estimation; bool quick_invoke_c_api_import; bool enable_shared_heap; + bool enable_shared_chain; char *use_prof_file; uint32_t opt_level; uint32_t size_level; diff --git a/samples/shared-heap/CMakeLists.txt b/samples/shared-heap/CMakeLists.txt index 260d3e33cc..03906f7c32 100644 --- a/samples/shared-heap/CMakeLists.txt +++ b/samples/shared-heap/CMakeLists.txt @@ -122,8 +122,10 @@ if (WAMR_BUILD_AOT EQUAL 1) if (WAMR_BUILD_TARGET STREQUAL "X86_32") set (WAMR_COMPILER_FLAGS --enable-shared-heap --target=i386) + set (WAMR_COMPILER_CHAIN_FLAGS --enable-shared-chain --target=i386) else () set (WAMR_COMPILER_FLAGS --enable-shared-heap) + set (WAMR_COMPILER_CHAIN_FLAGS --enable-shared-chain) endif () add_custom_target( @@ -132,6 +134,8 @@ if (WAMR_BUILD_AOT EQUAL 1) DEPENDS wasm-apps/test1.wasm wasm-apps/test2.wasm ${WAMR_COMPILER} COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_FLAGS} -o wasm-apps/test1.aot wasm-apps/test1.wasm COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_FLAGS} -o wasm-apps/test2.aot wasm-apps/test2.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_CHAIN_FLAGS} -o wasm-apps/test1_chain.aot wasm-apps/test1.wasm + COMMAND ${WAMR_COMPILER} ${WAMR_COMPILER_CHAIN_FLAGS} -o wasm-apps/test2_chain.aot wasm-apps/test2.wasm WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) endif() diff --git a/samples/shared-heap/src/shared_heap_chain.c b/samples/shared-heap/src/shared_heap_chain.c index 4b5c2f902a..8d5a4ea97b 100644 --- a/samples/shared-heap/src/shared_heap_chain.c +++ b/samples/shared-heap/src/shared_heap_chain.c @@ -198,7 +198,7 @@ main(int argc, char **argv) if (!aot_mode) wasm_file1 = "./wasm-apps/test1.wasm"; else - wasm_file1 = "./wasm-apps/test1.aot"; + wasm_file1 = "./wasm-apps/test1_chain.aot"; if (!(wasm_file1_buf = bh_read_file_to_buffer(wasm_file1, &wasm_file1_size))) { printf("Open wasm file %s failed.\n", wasm_file1); @@ -225,7 +225,7 @@ main(int argc, char **argv) if (!aot_mode) wasm_file2 = "./wasm-apps/test2.wasm"; else - wasm_file2 = "./wasm-apps/test2.aot"; + wasm_file2 = "./wasm-apps/test2_chain.aot"; if (!(wasm_file2_buf = bh_read_file_to_buffer(wasm_file2, &wasm_file2_size))) { printf("Open wasm file %s failed.\n", wasm_file1); diff --git a/tests/unit/shared-heap/shared_heap_test.cc b/tests/unit/shared-heap/shared_heap_test.cc index 84c10c9bd9..cd4e579865 100644 --- a/tests/unit/shared-heap/shared_heap_test.cc +++ b/tests/unit/shared-heap/shared_heap_test.cc @@ -155,6 +155,9 @@ TEST_F(shared_heap_test, test_shared_heap_basic) test_shared_heap(shared_heap, "test.aot", "test", 0, argv); EXPECT_EQ(10, argv[0]); + + test_shared_heap(shared_heap, "test_chain.aot", "test", 0, argv); + EXPECT_EQ(10, argv[0]); } TEST_F(shared_heap_test, test_shared_heap_malloc_fail) @@ -175,6 +178,10 @@ TEST_F(shared_heap_test, test_shared_heap_malloc_fail) test_shared_heap(shared_heap, "test.aot", "test_malloc_fail", 0, argv); EXPECT_EQ(1, argv[0]); + + test_shared_heap(shared_heap, "test_chain.aot", "test_malloc_fail", 0, + argv); + EXPECT_EQ(1, argv[0]); } TEST_F(shared_heap_test, test_preallocated_shared_heap_malloc_fail) @@ -201,6 +208,30 @@ TEST_F(shared_heap_test, test_preallocated_shared_heap_malloc_fail) argv[0] = 1024; test_shared_heap(shared_heap, "test.aot", "my_shared_heap_malloc", 1, argv); EXPECT_EQ(0, argv[0]); + + argv[0] = 1024; + test_shared_heap(shared_heap, "test_chain.aot", "my_shared_heap_malloc", 1, + argv); + EXPECT_EQ(0, argv[0]); +} + +static void +create_test_shared_heap(uint8 *preallocated_buf, size_t size, + WASMSharedHeap **shared_heap_res) +{ + SharedHeapInitArgs args = { 0 }; + WASMSharedHeap *shared_heap = nullptr; + args.pre_allocated_addr = preallocated_buf; + args.size = size; + shared_heap = wasm_runtime_create_shared_heap(&args); + if (!shared_heap) { + FAIL() << "Create preallocated shared heap failed.\n"; + } + + *shared_heap_res = shared_heap; + if (!*shared_heap_res) { + FAIL() << "Create shared heap chain failed.\n"; + } } static void @@ -232,6 +263,44 @@ create_test_shared_heap_chain(uint8 *preallocated_buf, size_t size, } } +TEST_F(shared_heap_test, test_shared_heap_rmw) +{ + WASMSharedHeap *shared_heap = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE] = { 0 }; + uint32 start1, end1; + + create_test_shared_heap(preallocated_buf, BUF_SIZE, &shared_heap); + + /* app addr for shared heap */ + start1 = UINT32_MAX - BUF_SIZE + 1; + end1 = UINT32_MAX; + + argv[0] = end1; + argv[1] = 101; + test_shared_heap(shared_heap, "test.wasm", "read_modify_write_8", 2, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 101); + + argv[0] = start1; + argv[1] = 37; + test_shared_heap(shared_heap, "test.wasm", "read_modify_write_8", 2, argv); + EXPECT_EQ(0, argv[0]); + EXPECT_EQ(preallocated_buf[0], 37); + + argv[0] = end1; + argv[1] = 81; + test_shared_heap(shared_heap, "test.aot", "read_modify_write_8", 2, argv); + EXPECT_EQ(101, argv[0]); + EXPECT_EQ(preallocated_buf[BUF_SIZE - 1], 81); + + argv[0] = start1; + argv[1] = 98; + test_shared_heap(shared_heap, "test.aot", "read_modify_write_8", 2, argv); + EXPECT_EQ(37, argv[0]); + EXPECT_EQ(preallocated_buf[0], 98); +} + TEST_F(shared_heap_test, test_shared_heap_chain_rmw) { SharedHeapInitArgs args = { 0 }; @@ -268,15 +337,15 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw) argv[0] = start1; argv[1] = 98; - test_shared_heap(shared_heap_chain, "test.aot", "read_modify_write_8", 2, - argv); + test_shared_heap(shared_heap_chain, "test_chain.aot", "read_modify_write_8", + 2, argv); EXPECT_EQ(0, argv[0]); EXPECT_EQ(preallocated_buf[0], 98); argv[0] = end2; argv[1] = 81; - test_shared_heap(shared_heap_chain, "test.aot", "read_modify_write_8", 2, - argv); + test_shared_heap(shared_heap_chain, "test_chain.aot", "read_modify_write_8", + 2, argv); EXPECT_EQ(0, argv[0]); EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 81); } @@ -311,7 +380,7 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory) argv[0] = start1; argv[1] = 14; argv[2] = 1; - test_shared_heap(shared_heap_chain, "test_bulk_memory.aot", + test_shared_heap(shared_heap_chain, "test_bulk_memory_chain.aot", "memory_fill_test", 3, argv); /* no modification since no return value */ EXPECT_EQ(start1, argv[0]); @@ -321,7 +390,7 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory) argv[0] = start2; argv[1] = 68; argv[2] = 0; - test_shared_heap(shared_heap_chain, "test_bulk_memory.aot", + test_shared_heap(shared_heap_chain, "test_bulk_memory_chain.aot", "memory_fill_test", 3, argv); /* no modification since no return value */ EXPECT_EQ(start2, argv[0]); @@ -330,7 +399,7 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory) argv[0] = end2; argv[1] = 98; argv[2] = 1; - test_shared_heap(shared_heap_chain, "test_bulk_memory.aot", + test_shared_heap(shared_heap_chain, "test_bulk_memory_chain.aot", "memory_fill_test", 3, argv); /* no modification since no return value */ EXPECT_EQ(end2, argv[0]); @@ -388,13 +457,11 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory_oob) "memory_fill_test", 3, argv), "Exception: out of bounds memory access"); - - argv[0] = end1; argv[1] = 101; argv[2] = 2; EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, - "test_bulk_memory.aot", + "test_bulk_memory_chain.aot", "memory_fill_test", 3, argv), "Exception: out of bounds memory access"); @@ -402,7 +469,7 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory_oob) argv[1] = 98; argv[2] = 2; EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, - "test_bulk_memory.aot", + "test_bulk_memory_chain.aot", "memory_fill_test", 3, argv), "Exception: out of bounds memory access"); @@ -410,7 +477,7 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory_oob) argv[1] = 98; argv[2] = BUF_SIZE + 1; EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, - "test_bulk_memory.aot", + "test_bulk_memory_chain.aot", "memory_fill_test", 3, argv), "Exception: out of bounds memory access"); @@ -418,10 +485,43 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw_bulk_memory_oob) argv[1] = 98; argv[2] = BUF_SIZE + 1; EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, - "test_bulk_memory.aot", + "test_bulk_memory_chain.aot", "memory_fill_test", 3, argv), "Exception: out of bounds memory access"); +} + +TEST_F(shared_heap_test, test_shared_heap_rmw_oob) +{ + WASMSharedHeap *shared_heap = nullptr; + uint32 argv[2] = { 0 }, BUF_SIZE = os_getpagesize(); + uint8 preallocated_buf[BUF_SIZE], preallocated_buf2[BUF_SIZE]; + uint32 start1, end1, start2, end2; + + create_test_shared_heap(preallocated_buf, BUF_SIZE, &shared_heap); + + /* app addr for shared heap */ + start1 = UINT32_MAX - BUF_SIZE + 1; + end1 = UINT32_MAX; + + /* try to rmw an u16, first u8 is in the first shared heap and second u8 is + * in the second shared heap, will be seen as oob */ + argv[0] = end1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test.wasm", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); + argv[0] = start1 - 1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test.aot", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); + + argv[0] = end1; + argv[1] = 12025; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test.aot", + "read_modify_write_16", 2, argv), + "Exception: out of bounds memory access"); } TEST_F(shared_heap_test, test_shared_heap_chain_rmw_oob) @@ -450,7 +550,8 @@ TEST_F(shared_heap_test, test_shared_heap_chain_rmw_oob) argv[0] = end1; argv[1] = 12025; - EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test.aot", + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test_chain.aot", "read_modify_write_16", 2, argv), "Exception: out of bounds memory access"); } @@ -491,15 +592,15 @@ TEST_F(shared_heap_test, test_shared_heap_chain_memory64_rmw) PUT_I64_TO_ADDR(argv, start1); argv[2] = 98; - test_shared_heap(shared_heap_chain, "test64.aot", "read_modify_write_8", 3, - argv); + test_shared_heap(shared_heap_chain, "test64_chain.aot", + "read_modify_write_8", 3, argv); EXPECT_EQ(0, argv[0]); EXPECT_EQ(preallocated_buf[0], 98); PUT_I64_TO_ADDR(argv, end2); argv[2] = 81; - test_shared_heap(shared_heap_chain, "test64.aot", "read_modify_write_8", 3, - argv); + test_shared_heap(shared_heap_chain, "test64_chain.aot", + "read_modify_write_8", 3, argv); EXPECT_EQ(0, argv[0]); EXPECT_EQ(preallocated_buf2[BUF_SIZE - 1], 81); } @@ -530,7 +631,8 @@ TEST_F(shared_heap_test, test_shared_heap_chain_memory64_rmw_oob) PUT_I64_TO_ADDR(argv, end1); argv[2] = 12025; - EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, "test64.aot", + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, + "test64_chain.aot", "read_modify_write_16", 3, argv), "Exception: out of bounds memory access"); } @@ -588,6 +690,9 @@ TEST_F(shared_heap_test, test_addr_conv) test_shared_heap(shared_heap, "test_addr_conv.aot", "test", 0, argv); EXPECT_EQ(1, argv[0]); + + test_shared_heap(shared_heap, "test_addr_conv_chain.aot", "test", 0, argv); + EXPECT_EQ(1, argv[0]); } TEST_F(shared_heap_test, test_addr_conv_pre_allocated_oob) @@ -622,6 +727,12 @@ TEST_F(shared_heap_test, test_addr_conv_pre_allocated_oob) EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, "test_addr_conv.aot", "test_preallocated", 1, argv), "Exception: out of bounds memory access"); + + argv[0] = app_addr; + EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap, + "test_addr_conv_chain.aot", + "test_preallocated", 1, argv), + "Exception: out of bounds memory access"); } TEST_F(shared_heap_test, test_shared_heap_chain) @@ -876,13 +987,13 @@ TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv) EXPECT_EQ(1, argv[0]); argv[0] = 0xFFFFFFFF; - test_shared_heap(shared_heap, "test_addr_conv.aot", "test_preallocated", 1, - argv); + test_shared_heap(shared_heap, "test_addr_conv_chain.aot", + "test_preallocated", 1, argv); EXPECT_EQ(1, argv[0]); argv[0] = 0xFFFFF000; - test_shared_heap(shared_heap, "test_addr_conv.aot", "test_preallocated", 1, - argv); + test_shared_heap(shared_heap, "test_addr_conv_chain.aot", + "test_preallocated", 1, argv); EXPECT_EQ(1, argv[0]); } @@ -932,7 +1043,7 @@ TEST_F(shared_heap_test, test_shared_heap_chain_addr_conv_oob) /* test aot */ argv[0] = 0xFFFFFFFF - BUF_SIZE - 4096; EXPECT_NONFATAL_FAILURE(test_shared_heap(shared_heap_chain, - "test_addr_conv.aot", + "test_addr_conv_chain.aot", "test_preallocated", 1, argv), "Exception: out of bounds memory access"); } diff --git a/tests/unit/shared-heap/wasm-apps/CMakeLists.txt b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt index 931b7b52df..fb055dde52 100644 --- a/tests/unit/shared-heap/wasm-apps/CMakeLists.txt +++ b/tests/unit/shared-heap/wasm-apps/CMakeLists.txt @@ -31,8 +31,10 @@ set(CMAKE_EXE_LINKER_FLAGS if (WAMR_BUILD_TARGET STREQUAL "X86_32") set (WAMR_COMPILER_FLAGS --opt-level=0 --bounds-checks=1 --enable-shared-heap --target=i386) + set (WAMR_COMPILER_CHAIN_FLAGS --opt-level=0 --bounds-checks=1 --enable-shared-chain --target=i386) else () set (WAMR_COMPILER_FLAGS --opt-level=0 --bounds-checks=1 --enable-shared-heap) + set (WAMR_COMPILER_CHAIN_FLAGS --opt-level=0 --bounds-checks=1 --enable-shared-chain) endif () function(copy_wasm TARGET_NAME) @@ -46,6 +48,7 @@ endfunction() function(compile_and_copy_aot_from TARGET_NAME) string(REPLACE ".wasm" ".aot" AOT_TARGET ${TARGET_NAME}) + string(REPLACE ".wasm" "_chain.aot" AOT_CHAIN_TARGET ${TARGET_NAME}) add_custom_command(TARGET ${TARGET_NAME} POST_BUILD COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} @@ -54,6 +57,12 @@ function(compile_and_copy_aot_from TARGET_NAME) COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/${AOT_TARGET} ${CMAKE_CURRENT_BINARY_DIR}/../ + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_CHAIN_FLAGS} + -o ${AOT_CHAIN_TARGET} + ${TARGET_NAME} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_CHAIN_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../ COMMENT "Compile and copy ${AOT_TARGET} to the same directory of google test" ) endfunction() @@ -72,6 +81,7 @@ compile_and_copy_aot_from(test_addr_conv.wasm) set(SOURCE_WASM ${CMAKE_CURRENT_SOURCE_DIR}/bulk-memory/test_bulk_memory.wasm) set(BUILD_WASM ${CMAKE_CURRENT_BINARY_DIR}/../test_bulk_memory.wasm) set(OUTPUT_AOT ${CMAKE_CURRENT_BINARY_DIR}/../test_bulk_memory.aot) +set(OUTPUT_CHAIN_AOT ${CMAKE_CURRENT_BINARY_DIR}/../test_bulk_memory_chain.aot) add_custom_command( OUTPUT ${BUILD_WASM} @@ -87,6 +97,9 @@ add_custom_command( COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} -o ${OUTPUT_AOT} ${BUILD_WASM} + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_CHAIN_FLAGS} + -o ${OUTPUT_CHAIN_AOT} + ${BUILD_WASM} DEPENDS ${BUILD_WASM} COMMENT "Compiling bulk memory AOT from copied WASM" ) diff --git a/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt b/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt index 9f51479c24..a82788b586 100644 --- a/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt +++ b/tests/unit/shared-heap/wasm-apps/memory64/CMakeLists.txt @@ -29,11 +29,8 @@ set(CMAKE_EXE_LINKER_FLAGS -Wl,--allow-undefined" ) -if (WAMR_BUILD_TARGET STREQUAL "X86_32") - set (WAMR_COMPILER_FLAGS --opt-level=0 --bounds-checks=1 --enable-shared-heap --target=i386) -else () - set (WAMR_COMPILER_FLAGS --opt-level=0 --bounds-checks=1 --enable-shared-heap) -endif () +set (WAMR_COMPILER_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-heap) +set (WAMR_COMPILER_CHAIN_FLAGS --opt-level=3 --bounds-checks=1 --enable-shared-chain) function(copy_wasm TARGET_NAME) add_custom_command(TARGET ${TARGET_NAME} POST_BUILD @@ -46,6 +43,7 @@ endfunction() function(compile_and_copy_aot_from TARGET_NAME) string(REPLACE ".wasm" ".aot" AOT_TARGET ${TARGET_NAME}) + string(REPLACE ".wasm" "_chain.aot" AOT_CHAIN_TARGET ${TARGET_NAME}) add_custom_command(TARGET ${TARGET_NAME} POST_BUILD COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_FLAGS} @@ -54,11 +52,16 @@ function(compile_and_copy_aot_from TARGET_NAME) COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/${AOT_TARGET} ${CMAKE_CURRENT_BINARY_DIR}/../../ - COMMENT "Compile and copy ${AOT_TARGET} to the same directory of google test" + COMMAND ${WAMRC_ROOT_DIR}/wamrc ${WAMR_COMPILER_CHAIN_FLAGS} + -o ${AOT_CHAIN_TARGET} + ${TARGET_NAME} + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/${AOT_CHAIN_TARGET} + ${CMAKE_CURRENT_BINARY_DIR}/../../ + COMMENT "Compile and copy ${AOT_TARGET} ${AOT_CHAIN_TARGET} to the same directory of google test" ) endfunction() -# Example targets add_executable(test64.wasm ../test.c) target_link_libraries(test64.wasm) copy_wasm(test64.wasm) diff --git a/wamr-compiler/main.c b/wamr-compiler/main.c index f74b2fb151..6b7dbaad48 100644 --- a/wamr-compiler/main.c +++ b/wamr-compiler/main.c @@ -210,7 +210,9 @@ print_help() printf(" --enable-linux-perf Enable linux perf support\n"); #endif printf(" --mllvm=