diff --git a/src/array.c b/src/array.c index 7a84af56d5922..24ebbc10c485f 100644 --- a/src/array.c +++ b/src/array.c @@ -471,13 +471,6 @@ JL_DLLEXPORT jl_value_t *jl_array_to_string(jl_array_t *a) ((a->maxsize + sizeof(void*) + 1 <= GC_MAX_SZCLASS) == (len + sizeof(void*) + 1 <= GC_MAX_SZCLASS)))) { jl_value_t *o = jl_array_data_owner(a); if (jl_is_string(o)) { -#ifdef MMTKHEAP - // since we need the size of the string to be accurate according to its allocation size, we simply allocate a new string here - // instead of changing its size to len as in `*(size_t*)o = len` - o = jl_gc_realloc_string(o, len); - jl_value_t** owner_addr = (a + jl_array_data_owner_offset(jl_array_ndims(a))); - owner_addr = o; -#endif a->flags.isshared = 1; *(size_t*)o = len; a->nrows = 0; @@ -503,29 +496,32 @@ JL_DLLEXPORT jl_value_t *jl_alloc_string(size_t len) jl_value_t *s; jl_ptls_t ptls = ct->ptls; const size_t allocsz = sz + sizeof(jl_taggedvalue_t); - if (sz <= GC_MAX_SZCLASS) { #ifndef MMTKHEAP + if (sz <= GC_MAX_SZCLASS) { int pool_id = jl_gc_szclass_align8(allocsz); jl_gc_pool_t *p = &ptls->heap.norm_pools[pool_id]; int osize = jl_gc_sizeclasses[pool_id]; // We call `jl_gc_pool_alloc_noinline` instead of `jl_gc_pool_alloc` to avoid double-counting in // the Allocations Profiler. (See https://github.com/JuliaLang/julia/pull/43868 for more details.) s = jl_gc_pool_alloc_noinline(ptls, (char*)p - (char*)ptls, osize); -#else - int pool_id = jl_gc_szclass_align8(allocsz); - int osize = jl_gc_sizeclasses[pool_id]; - s = jl_mmtk_gc_alloc_default(ptls, pool_id, osize, jl_string_type); -#endif } else { if (allocsz < sz) // overflow in adding offs, size was "negative" jl_throw(jl_memory_exception); -#ifndef MMTKHEAP + s = jl_gc_big_alloc_noinline(ptls, allocsz); + } #else + if (allocsz < sz) // overflow in adding offs, size was "negative" + jl_throw(jl_memory_exception); + + if (allocsz < MAX_STANDARD_OBJECT_SIZE) { + s = jl_mmtk_gc_alloc_default(ptls, allocsz, jl_string_type); + } else { s = jl_mmtk_gc_alloc_big(ptls, allocsz); -#endif } +#endif + jl_set_typeof(s, jl_string_type); maybe_record_alloc_to_profile(s, len, jl_string_type); *(size_t*)s = len; diff --git a/src/julia_internal.h b/src/julia_internal.h index c29cb335b4def..984acd6dcf888 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -239,7 +239,8 @@ jl_value_t *jl_gc_pool_alloc_noinline(jl_ptls_t ptls, int pool_offset, int osize); jl_value_t *jl_gc_big_alloc_noinline(jl_ptls_t ptls, size_t allocsz); #ifdef MMTKHEAP -JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_default(jl_ptls_t ptls, int pool_offset, int osize, void* ty); +extern size_t MAX_STANDARD_OBJECT_SIZE; +JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_default(jl_ptls_t ptls, size_t size, void* ty); JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_big(jl_ptls_t ptls, size_t allocsz); #endif JL_DLLEXPORT int jl_gc_classify_pools(size_t sz, int *osize); @@ -364,29 +365,30 @@ STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty) { jl_value_t *v; const size_t allocsz = sz + sizeof(jl_taggedvalue_t); - if (sz <= GC_MAX_SZCLASS) { #ifndef MMTKHEAP + if (sz <= GC_MAX_SZCLASS) { int pool_id = jl_gc_szclass(allocsz); jl_gc_pool_t *p = &ptls->heap.norm_pools[pool_id]; int osize = jl_gc_sizeclasses[pool_id]; // We call `jl_gc_pool_alloc_noinline` instead of `jl_gc_pool_alloc` to avoid double-counting in // the Allocations Profiler. (See https://github.com/JuliaLang/julia/pull/43868 for more details.) v = jl_gc_pool_alloc_noinline(ptls, (char*)p - (char*)ptls, osize); -#else - int pool_id = jl_gc_szclass(allocsz); - int osize = jl_gc_sizeclasses[pool_id]; - v = jl_mmtk_gc_alloc_default(ptls, pool_id, osize, ty); -#endif } else { if (allocsz < sz) // overflow in adding offs, size was "negative" jl_throw(jl_memory_exception); -#ifndef MMTKHEAP v = jl_gc_big_alloc_noinline(ptls, allocsz); + } #else + if (allocsz < sz) // overflow in adding offs, size was "negative" + jl_throw(jl_memory_exception); + + if (allocsz < (MAX_STANDARD_OBJECT_SIZE - 8)) { // buffer may take 8 bytes extra + v = jl_mmtk_gc_alloc_default(ptls, allocsz, ty); + } else { v = jl_mmtk_gc_alloc_big(ptls, allocsz); -#endif } +#endif jl_set_typeof(v, ty); maybe_record_alloc_to_profile(v, sz, (jl_datatype_t*)ty); return v; diff --git a/src/llvm-final-gc-lowering.cpp b/src/llvm-final-gc-lowering.cpp index d19505cbe16e5..2e0a64737ea05 100644 --- a/src/llvm-final-gc-lowering.cpp +++ b/src/llvm-final-gc-lowering.cpp @@ -171,25 +171,39 @@ Value *FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F) { assert(target->arg_size() == 2); auto sz = (size_t)cast(target->getArgOperand(1))->getZExtValue(); - // This is strongly architecture and OS dependent - int osize; - int offset = jl_gc_classify_pools(sz, &osize); + IRBuilder<> builder(target); builder.SetCurrentDebugLocation(target->getDebugLoc()); auto ptls = target->getArgOperand(0); CallInst *newI; +#ifndef MMTKHEAP + // This is strongly architecture and OS dependent + int osize; + int offset = jl_gc_classify_pools(sz, &osize); if (offset < 0) { newI = builder.CreateCall( bigAllocFunc, { ptls, ConstantInt::get(T_size, sz + sizeof(void*)) }); } else { -#ifndef MMTKHEAP auto pool_offs = ConstantInt::get(Type::getInt32Ty(F.getContext()), offset); auto pool_osize = ConstantInt::get(Type::getInt32Ty(F.getContext()), osize); newI = builder.CreateCall(poolAllocFunc, { ptls, pool_offs, pool_osize }); + } + newI->setAttributes(newI->getCalledFunction()->getAttributes()); + newI->takeName(target); + return newI; #else - auto pool_osize = ConstantInt::get(Type::getInt64Ty(F.getContext()), osize); + if (sz + sizeof(void*) >= MAX_STANDARD_OBJECT_SIZE) { + newI = builder.CreateCall( + bigAllocFunc, + { ptls, ConstantInt::get(T_size, sz + sizeof(void*)) }); + + newI->setAttributes(newI->getCalledFunction()->getAttributes()); + newI->takeName(target); + return newI; + } else { + auto pool_osize = ConstantInt::get(T_size, sz + sizeof(void*)); auto cursor_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, cursor)); auto limit_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, limit)); @@ -253,11 +267,8 @@ Value *FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F) phiNode->takeName(target); return phiNode; -#endif } - newI->setAttributes(newI->getCalledFunction()->getAttributes()); - newI->takeName(target); - return newI; +#endif } bool FinalLowerGC::doInitialization(Module &M) { diff --git a/src/llvm-pass-helpers.cpp b/src/llvm-pass-helpers.cpp index 7cf42abe4b2f1..3eac6963a1338 100644 --- a/src/llvm-pass-helpers.cpp +++ b/src/llvm-pass-helpers.cpp @@ -247,7 +247,7 @@ namespace jl_well_known { #ifndef MMTKHEAP { Type::getInt8PtrTy(context.getLLVMContext()), Type::getInt32Ty(context.getLLVMContext()), Type::getInt32Ty(context.getLLVMContext()) }, #else - { Type::getInt32Ty(context.getLLVMContext()), Type::getInt64Ty(context.getLLVMContext()) }, + { Type::getInt32Ty(context.getLLVMContext()), context.T_size }, #endif false), Function::ExternalLinkage,