From b3ee204c318caaac6b3eb4feb79268fe843d2c86 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Wed, 21 May 2025 14:55:11 +0000 Subject: [PATCH 1/9] 8346011: [Lilliput] Compact Full-GC Forwarding Reviewed-by: stuefe --- src/hotspot/share/gc/g1/g1Arguments.cpp | 3 - src/hotspot/share/gc/g1/g1FullCollector.cpp | 4 + .../share/gc/parallel/parallelArguments.cpp | 3 - .../share/gc/parallel/psParallelCompact.cpp | 4 + .../share/gc/serial/serialArguments.cpp | 5 - .../share/gc/serial/serialArguments.hpp | 1 - src/hotspot/share/gc/serial/serialFullGC.cpp | 4 + .../share/gc/shared/fullGCForwarding.cpp | 172 +++++++++++++++--- .../share/gc/shared/fullGCForwarding.hpp | 165 +++++++++++++++-- .../gc/shared/fullGCForwarding.inline.hpp | 120 +++++++++--- .../gc/shenandoah/shenandoahArguments.cpp | 3 - .../share/gc/shenandoah/shenandoahFullGC.cpp | 4 + src/hotspot/share/utilities/fastHash.hpp | 102 +++++++++++ .../gtest/gc/shared/test_preservedMarks.cpp | 4 + 14 files changed, 517 insertions(+), 77 deletions(-) create mode 100644 src/hotspot/share/utilities/fastHash.hpp diff --git a/src/hotspot/share/gc/g1/g1Arguments.cpp b/src/hotspot/share/gc/g1/g1Arguments.cpp index bd156a69fe6fc..54fd356ccc25c 100644 --- a/src/hotspot/share/gc/g1/g1Arguments.cpp +++ b/src/hotspot/share/gc/g1/g1Arguments.cpp @@ -33,7 +33,6 @@ #include "gc/g1/g1HeapRegionRemSet.hpp" #include "gc/g1/g1HeapVerifier.hpp" #include "gc/shared/cardTable.hpp" -#include "gc/shared/fullGCForwarding.hpp" #include "gc/shared/gcArguments.hpp" #include "gc/shared/workerPolicy.hpp" #include "runtime/globals.hpp" @@ -246,8 +245,6 @@ void G1Arguments::initialize() { if (max_parallel_refinement_threads > UINT_MAX / divisor) { vm_exit_during_initialization("Too large parallelism for remembered sets."); } - - FullGCForwarding::initialize_flags(heap_reserved_size_bytes()); } CollectedHeap* G1Arguments::create_heap() { diff --git a/src/hotspot/share/gc/g1/g1FullCollector.cpp b/src/hotspot/share/gc/g1/g1FullCollector.cpp index 637995c7e5247..e5d84c3772372 100644 --- a/src/hotspot/share/gc/g1/g1FullCollector.cpp +++ b/src/hotspot/share/gc/g1/g1FullCollector.cpp @@ -211,6 +211,8 @@ void G1FullCollector::collect() { // Don't add any more derived pointers during later phases deactivate_derived_pointers(); + FullGCForwarding::begin(); + phase2_prepare_compaction(); if (has_compaction_targets()) { @@ -223,6 +225,8 @@ void G1FullCollector::collect() { log_info(gc, phases) ("No Regions selected for compaction. Skipping Phase 3: Adjust pointers and Phase 4: Compact heap"); } + FullGCForwarding::end(); + phase5_reset_metadata(); G1CollectedHeap::finish_codecache_marking_cycle(); diff --git a/src/hotspot/share/gc/parallel/parallelArguments.cpp b/src/hotspot/share/gc/parallel/parallelArguments.cpp index dcd5b6ae4541e..75395c7d775e6 100644 --- a/src/hotspot/share/gc/parallel/parallelArguments.cpp +++ b/src/hotspot/share/gc/parallel/parallelArguments.cpp @@ -26,7 +26,6 @@ #include "gc/parallel/parallelArguments.hpp" #include "gc/parallel/parallelScavengeHeap.hpp" #include "gc/shared/adaptiveSizePolicy.hpp" -#include "gc/shared/fullGCForwarding.hpp" #include "gc/shared/gcArguments.hpp" #include "gc/shared/genArguments.hpp" #include "gc/shared/workerPolicy.hpp" @@ -94,8 +93,6 @@ void ParallelArguments::initialize() { if (FLAG_IS_DEFAULT(ParallelRefProcEnabled) && ParallelGCThreads > 1) { FLAG_SET_DEFAULT(ParallelRefProcEnabled, true); } - - FullGCForwarding::initialize_flags(heap_reserved_size_bytes()); } // The alignment used for spaces in young gen and old gen diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.cpp b/src/hotspot/share/gc/parallel/psParallelCompact.cpp index 79a0b898a6b7b..004a93c96c062 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.cpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.cpp @@ -1057,12 +1057,16 @@ bool PSParallelCompact::invoke_no_policy(bool clear_all_soft_refs) { DerivedPointerTable::set_active(false); #endif + FullGCForwarding::begin(); + forward_to_new_addr(); adjust_pointers(); compact(); + FullGCForwarding::end(); + ParCompactionManager::_preserved_marks_set->restore(&ParallelScavengeHeap::heap()->workers()); ParCompactionManager::verify_all_region_stack_empty(); diff --git a/src/hotspot/share/gc/serial/serialArguments.cpp b/src/hotspot/share/gc/serial/serialArguments.cpp index aed1c2353b4d8..f76615d4c60a1 100644 --- a/src/hotspot/share/gc/serial/serialArguments.cpp +++ b/src/hotspot/share/gc/serial/serialArguments.cpp @@ -27,11 +27,6 @@ #include "gc/shared/fullGCForwarding.hpp" #include "gc/shared/gcArguments.hpp" -void SerialArguments::initialize() { - GCArguments::initialize(); - FullGCForwarding::initialize_flags(MaxHeapSize); -} - CollectedHeap* SerialArguments::create_heap() { return new SerialHeap(); } diff --git a/src/hotspot/share/gc/serial/serialArguments.hpp b/src/hotspot/share/gc/serial/serialArguments.hpp index 90c3225ff8def..3ed4df5f41b85 100644 --- a/src/hotspot/share/gc/serial/serialArguments.hpp +++ b/src/hotspot/share/gc/serial/serialArguments.hpp @@ -31,7 +31,6 @@ class CollectedHeap; class SerialArguments : public GenArguments { private: - virtual void initialize(); virtual CollectedHeap* create_heap(); }; diff --git a/src/hotspot/share/gc/serial/serialFullGC.cpp b/src/hotspot/share/gc/serial/serialFullGC.cpp index 1546870454754..310977edf114d 100644 --- a/src/hotspot/share/gc/serial/serialFullGC.cpp +++ b/src/hotspot/share/gc/serial/serialFullGC.cpp @@ -696,6 +696,8 @@ void SerialFullGC::invoke_at_safepoint(bool clear_all_softrefs) { phase1_mark(clear_all_softrefs); + FullGCForwarding::begin(); + Compacter compacter{gch}; { @@ -739,6 +741,8 @@ void SerialFullGC::invoke_at_safepoint(bool clear_all_softrefs) { restore_marks(); + FullGCForwarding::end(); + deallocate_stacks(); SerialFullGC::_string_dedup_requests->flush(); diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.cpp b/src/hotspot/share/gc/shared/fullGCForwarding.cpp index 474a3209d7431..aeb45b1cd05e7 100644 --- a/src/hotspot/share/gc/shared/fullGCForwarding.cpp +++ b/src/hotspot/share/gc/shared/fullGCForwarding.cpp @@ -23,34 +23,164 @@ */ #include "gc/shared/fullGCForwarding.hpp" -#include "memory/memRegion.hpp" -#include "runtime/globals_extension.hpp" +#include "logging/log.hpp" +#include "nmt/memTag.hpp" +#include "utilities/ostream.hpp" +#include "utilities/concurrentHashTable.inline.hpp" +#include "utilities/fastHash.hpp" +#include "utilities/powerOfTwo.hpp" -HeapWord* FullGCForwarding::_heap_base = nullptr; -int FullGCForwarding::_num_low_bits = 0; +static uintx hash(HeapWord* const& addr) { + uint64_t val = reinterpret_cast(addr); + uint32_t hash = FastHash::get_hash32((uint32_t)val, (uint32_t)(val >> 32)); + return hash; +} -void FullGCForwarding::initialize_flags(size_t max_heap_size) { -#ifdef _LP64 - size_t max_narrow_heap_size = right_n_bits(NumLowBitsNarrow - Shift); - if (UseCompactObjectHeaders && max_heap_size > max_narrow_heap_size * HeapWordSize) { - warning("Compact object headers require a java heap size smaller than %zu" - "%s (given: %zu%s). Disabling compact object headers.", - byte_size_in_proper_unit(max_narrow_heap_size * HeapWordSize), - proper_unit_for_byte_size(max_narrow_heap_size * HeapWordSize), - byte_size_in_proper_unit(max_heap_size), - proper_unit_for_byte_size(max_heap_size)); - FLAG_SET_ERGO(UseCompactObjectHeaders, false); +struct ForwardingEntry { + HeapWord* _from; + HeapWord* _to; + ForwardingEntry(HeapWord* from, HeapWord* to) : _from(from), _to(to) {} +}; + +struct FallbackTableConfig { + using Value = ForwardingEntry; + static uintx get_hash(Value const& entry, bool* is_dead) { + return hash(entry._from); + } + static void* allocate_node(void* context, size_t size, Value const& value) { + return AllocateHeap(size, mtGC); + } + static void free_node(void* context, void* memory, Value const& value) { + FreeHeap(memory); + } +}; + +class FallbackTable : public ConcurrentHashTable { + +}; + +class FallbackTableLookup : public StackObj { + ForwardingEntry const _entry; +public: + explicit FallbackTableLookup(HeapWord* from) : _entry(from, nullptr) {} + uintx get_hash() const { + return hash(_entry._from); } + bool equals(ForwardingEntry* value) { + return _entry._from == value->_from; + } + bool is_dead(ForwardingEntry* value) { return false; } +}; + +// We cannot use 0, because that may already be a valid base address in zero-based heaps. +// 0x1 is safe because heap base addresses must be aligned by much larger alignment +HeapWord* const FullGCForwarding::UNUSED_BASE = reinterpret_cast(0x1); + +HeapWord* FullGCForwarding::_heap_start = nullptr; +size_t FullGCForwarding::_heap_start_region_bias = 0; +size_t FullGCForwarding::_num_regions = 0; +uintptr_t FullGCForwarding::_region_mask = 0; +HeapWord** FullGCForwarding::_biased_bases = nullptr; +HeapWord** FullGCForwarding::_bases_table = nullptr; +FallbackTable* FullGCForwarding::_fallback_table = nullptr; +#ifndef PRODUCT +volatile uint64_t FullGCForwarding::_num_forwardings = 0; +volatile uint64_t FullGCForwarding::_num_fallback_forwardings = 0; #endif -} void FullGCForwarding::initialize(MemRegion heap) { #ifdef _LP64 - _heap_base = heap.start(); - if (UseCompactObjectHeaders) { - _num_low_bits = NumLowBitsNarrow; - } else { - _num_low_bits = NumLowBitsWide; + _heap_start = heap.start(); + + size_t rounded_heap_size = round_up_power_of_2(heap.byte_size()); + + _num_regions = (rounded_heap_size / BytesPerWord) / BLOCK_SIZE_WORDS; + + _heap_start_region_bias = (uintptr_t)_heap_start >> BLOCK_SIZE_BYTES_SHIFT; + _region_mask = ~((uintptr_t(1) << BLOCK_SIZE_BYTES_SHIFT) - 1); + + assert(_bases_table == nullptr, "should not be initialized yet"); + assert(_fallback_table == nullptr, "should not be initialized yet"); +#endif +} + +void FullGCForwarding::begin() { +#ifdef _LP64 + assert(_bases_table == nullptr, "should not be initialized yet"); + assert(_fallback_table == nullptr, "should not be initialized yet"); + + _fallback_table = new FallbackTable(); + +#ifndef PRODUCT + _num_forwardings = 0; + _num_fallback_forwardings = 0; +#endif + + size_t max = _num_regions; + _bases_table = NEW_C_HEAP_ARRAY(HeapWord*, max, mtGC); + HeapWord** biased_start = _bases_table - _heap_start_region_bias; + _biased_bases = biased_start; + for (size_t i = 0; i < max; i++) { + _bases_table[i] = UNUSED_BASE; } #endif } + +void FullGCForwarding::end() { +#ifndef PRODUCT + log_info(gc)("Total forwardings: " UINT64_FORMAT ", fallback forwardings: " UINT64_FORMAT + ", ratio: %f, memory used by fallback table: %zu%s, memory used by bases table: %zu%s", + _num_forwardings, _num_fallback_forwardings, (float)_num_forwardings/(float)_num_fallback_forwardings, + byte_size_in_proper_unit(_fallback_table->get_mem_size(Thread::current())), + proper_unit_for_byte_size(_fallback_table->get_mem_size(Thread::current())), + byte_size_in_proper_unit(sizeof(HeapWord*) * _num_regions), + proper_unit_for_byte_size(sizeof(HeapWord*) * _num_regions)); +#endif +#ifdef _LP64 + assert(_bases_table != nullptr, "should be initialized"); + FREE_C_HEAP_ARRAY(HeapWord*, _bases_table); + _bases_table = nullptr; + delete _fallback_table; + _fallback_table = nullptr; +#endif +} + +void FullGCForwarding::fallback_forward_to(HeapWord* from, HeapWord* to) { + assert(to != nullptr, "no null forwarding"); + assert(_fallback_table != nullptr, "should be initialized"); + FallbackTableLookup lookup_f(from); + ForwardingEntry entry(from, to); + auto found_f = [&](ForwardingEntry* found) { + // If dupe has been found, override it with new value. + // This is also called when new entry is succussfully inserted. + if (found->_to != to) { + found->_to = to; + } + }; + Thread* current_thread = Thread::current(); + bool grow; + bool added = _fallback_table->insert_get(current_thread, lookup_f, entry, found_f, &grow); + NOT_PRODUCT(Atomic::inc(&_num_fallback_forwardings);) +#ifdef ASSERT + assert(fallback_forwardee(from) != nullptr, "must have entered forwarding"); + assert(fallback_forwardee(from) == to, "forwarding must be correct, added: %s, from: " PTR_FORMAT ", to: " PTR_FORMAT ", fwd: " PTR_FORMAT, BOOL_TO_STR(added), p2i(from), p2i(to), p2i(fallback_forwardee(from))); +#endif + if (grow) { + _fallback_table->grow(current_thread); + tty->print_cr("grow fallback table to size: %zu bytes", + _fallback_table->get_mem_size(current_thread)); + } +} + +HeapWord* FullGCForwarding::fallback_forwardee(HeapWord* from) { + assert(_fallback_table != nullptr, "fallback table must be present"); + HeapWord* result; + FallbackTableLookup lookup_f(from); + auto found_f = [&](ForwardingEntry* found) { + result = found->_to; + }; + bool found = _fallback_table->get(Thread::current(), lookup_f, found_f); + assert(found, "something must have been found"); + assert(result != nullptr, "must have found forwarding"); + return result; +} diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.hpp b/src/hotspot/share/gc/shared/fullGCForwarding.hpp index a6ca182428b55..902d79111f38d 100644 --- a/src/hotspot/share/gc/shared/fullGCForwarding.hpp +++ b/src/hotspot/share/gc/shared/fullGCForwarding.hpp @@ -25,35 +25,164 @@ #ifndef SHARE_GC_SHARED_FULLGCFORWARDING_HPP #define SHARE_GC_SHARED_FULLGCFORWARDING_HPP -#include "memory/allStatic.hpp" +#include "memory/allocation.hpp" #include "memory/memRegion.hpp" #include "oops/markWord.hpp" #include "oops/oopsHierarchy.hpp" -/* - * Implements forwarding for the Full GCs of Serial, Parallel, G1 and Shenandoah in - * a way that preserves upper N bits of object mark-words, which contain crucial - * Klass* information when running with compact headers. The encoding is similar to - * compressed-oops encoding: it basically subtracts the forwardee address from the - * heap-base, shifts that difference into the right place, and sets the lowest two - * bits (to indicate 'forwarded' state as usual). - * With compact-headers, we have 40 bits to encode forwarding pointers. This is - * enough to address 8TB of heap. If the heap size exceeds that limit, we turn off - * compact headers. +class FallbackTable; +class Mutex; + +/** + * FullGCForwarding is a method to store forwarding information in a compressed form into the object header, + * that has been specifically designed for sliding compacting GCs and compact object headers. With compact object + * headers, we store the compressed class pointer in the header, which would be overwritten by full forwarding + * pointers, if we allow the legacy forwarding code to act. This would lose the class information for the object, + * which is required later in GC cycle to iterate the reference fields and get the object size for copying. + * + * FullGCForwarding requires only small side tables and guarantees constant-time access and modification. + * + * The key advantage of sliding compaction for encoding efficiency: + * - It forwards objects linearily, starting at the heap bottom and moving up to the top, sliding + * live objects towards the bottom of the heap. (The reality in parallel or regionalized GCs is a bit more + * complex, but conceptually it is the same.) + * - Objects starting in any one block can only be forwarded to a memory region that is not larger than + * a block. (There are exceptions to this rule which are discussed below.) + * + * This is an intuitive property: when we slide the compact block full of data, it can not take up more + * memory afterwards. + * This property allows us to use a side table to record the addresses of the target memory region for + * each block. The table holds N entries for N blocks. For each block, it gives the base + * address of the target regions, or a special placeholder if not used. + * + * This encoding efficiency allows to store the forwarding information in the object header _together_ with the + * compressed class pointer. + * + * The idea is to use a pointer compression scheme very similar to the one that is used for compressed oops. + * We divide the heap into number of equal-sized blocks. Each block spans a maximum of 2^NUM_OFFSET_BITS words. + * We maintain a side-table of target-base-addresses, with one address entry per block. + * + * When recording the sliding forwarding, the mark word would look roughly like this: + * + * 32 0 + * [.....................OOOOOOOOOTT] + * ^------ tag-bits, indicates 'forwarded' + * ^-------- in-region offset + * ^----------------- protected area, *not touched* by this code, useful for + * compressed class pointer with compact object headers + * + * Adding a forwarding then generally works as follows: + * 1. Compute the index of the block of the "from" address. + * 2. Load the target-base-offset of the from-block from the side-table. + * 3. If the base-offset is not-yet set, set it to the to-address of the forwarding. + * (In other words, the first forwarding of a block determines the target base-offset.) + * 4. Compute the offset of the to-address in the target region. + * 4. Store offset in the object header. + * + * Similarly, looking up the target address, given an original object address generally works as follows: + * 1. Compute the index of the block of the "from" address. + * 2. Load the target-base-offset of the from-block from the side-table. + * 3. Extract the offset from the object header. + * 4. Compute the "to" address from "to" region base and "offset" + * + * We reserve one special value for the offset: + * - 111111111: Indicates an exceptional forwarding (see below), for which a fallback hash-table + * is used to look up the target address. + * + * In order to support this, we need to make a change to the above algorithm: + * - Forwardings that would use offsets >= 111111111 (i.e. the last slot) + * would also need to use the fallback-table. We expect that to be relatively rare for two reasons: + * 1. It only affects 1 out of 512 possible offsets, in other words, 1/512th of all situations in an equal + * distribution. + * 2. Forwardings are not equally-distributed, because normally we 'skip' unreachable objects, + * thus compacting the block. Forwardings tend to cluster at the beginning of the target region, + * and become less likely towards the end of the possible encodable target address range. + * Which means in reality it will be much less frequent than 1/512. + * + * There are several conditions when the above algorithm would be broken because the assumption that + * 'objects from each block can only get forwarded to a region of block-size' is violated: + * - G1 last-ditch serial compaction: there, object from a single region can be forwarded to multiple, + * more than two regions. G1 serial compaction is not very common - it is the last-last-ditch GC + * that is used when the JVM is scrambling to squeeze more space out of the heap, and at that point, + * ultimate performance is no longer the main concern. + * - When forwarding hits a space (or G1/Shenandoah region) boundary, then latter objects of a block + * need to be forwarded to a different address range than earlier objects in the same block. + * This is rare. + * - With compact identity hash-code, objects can grow, and in the worst case use up more memory in + * the target block than we can address. We expect that to be rare. + * + * To deal with that, we initialize a fallback-hashtable for storing those extra forwardings, and use a special + * offset pattern (0b11...1) to indicate that the forwardee is not encoded but should be looked-up in the hashtable. + * This implies that this particular offset (the last word of a block) can not be used directly as forwarding, + * but also has to be handled by the fallback-table. */ class FullGCForwarding : public AllStatic { - static const int NumLowBitsNarrow = LP64_ONLY(markWord::klass_shift) NOT_LP64(0 /*unused*/); - static const int NumLowBitsWide = BitsPerWord; - static const int Shift = markWord::lock_bits + markWord::lock_shift; +private: + static constexpr int AVAILABLE_LOW_BITS = 11; + static constexpr int AVAILABLE_BITS_MASK = right_n_bits(AVAILABLE_LOW_BITS); + // The offset bits start after the lock-bits, which are currently used by Serial GC + // for marking objects. Could be 1 for Serial GC when being clever with the bits, + // and 0 for all other GCs. + static constexpr int OFFSET_BITS_SHIFT = markWord::lock_shift + markWord::lock_bits; + + // How many bits we use for the offset + static constexpr int NUM_OFFSET_BITS = AVAILABLE_LOW_BITS - OFFSET_BITS_SHIFT; + static constexpr size_t BLOCK_SIZE_WORDS = 1 << NUM_OFFSET_BITS; + static constexpr int BLOCK_SIZE_BYTES_SHIFT = NUM_OFFSET_BITS + LogHeapWordSize; + static constexpr size_t MAX_OFFSET = BLOCK_SIZE_WORDS - 2; + static constexpr uintptr_t OFFSET_MASK = right_n_bits(NUM_OFFSET_BITS) << OFFSET_BITS_SHIFT; + + // This offset bit-pattern indicates that the actual mapping is handled by the + // fallback-table. This also implies that this cannot be used as a valid offset, + // and we must also use the fallback-table for mappings to the last word of a + // block. + static constexpr uintptr_t FALLBACK_PATTERN = right_n_bits(NUM_OFFSET_BITS); + static constexpr uintptr_t FALLBACK_PATTERN_IN_PLACE = FALLBACK_PATTERN << OFFSET_BITS_SHIFT; + + // Indicates an unused base address in the target base table. + static HeapWord* const UNUSED_BASE; + + static HeapWord* _heap_start; + + static size_t _heap_start_region_bias; + static size_t _num_regions; + static uintptr_t _region_mask; + + // The target base table memory. + static HeapWord** _bases_table; + // Entries into the target base tables, biased to the start of the heap. + static HeapWord** _biased_bases; + + static FallbackTable* _fallback_table; + +#ifndef PRODUCT + static volatile uint64_t _num_forwardings; + static volatile uint64_t _num_fallback_forwardings; +#endif + + static inline size_t biased_region_index_containing(HeapWord* addr); + + static inline bool is_fallback(uintptr_t encoded); + static inline uintptr_t encode_forwarding(HeapWord* from, HeapWord* to); + static inline HeapWord* decode_forwarding(HeapWord* from, uintptr_t encoded); + + static void fallback_forward_to(HeapWord* from, HeapWord* to); + static HeapWord* fallback_forwardee(HeapWord* from); + + static inline void forward_to_impl(oop from, oop to); + static inline oop forwardee_impl(oop from); - static HeapWord* _heap_base; - static int _num_low_bits; public: - static void initialize_flags(size_t max_heap_size); static void initialize(MemRegion heap); + + static void begin(); + static void end(); + + static inline bool is_forwarded(oop obj); + static inline bool is_not_forwarded(oop obj); + static inline void forward_to(oop from, oop to); static inline oop forwardee(oop from); - static inline bool is_forwarded(oop obj); }; #endif // SHARE_GC_SHARED_FULLGCFORWARDING_HPP diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp b/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp index ebd280a454f4c..2245f30f02065 100644 --- a/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp +++ b/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp @@ -19,42 +19,116 @@ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. - * */ -#ifndef GC_SHARED_FULLGCFORWARDING_INLINE_HPP -#define GC_SHARED_FULLGCFORWARDING_INLINE_HPP +#ifndef SHARE_GC_SHARED_FULLGCFORWARDING_INLINE_HPP +#define SHARE_GC_SHARED_FULLGCFORWARDING_INLINE_HPP +#include "gc/shared/gc_globals.hpp" #include "gc/shared/fullGCForwarding.hpp" - +#include "oops/markWord.hpp" #include "oops/oop.inline.hpp" -#include "utilities/globalDefinitions.hpp" +#include "utilities/macros.hpp" + +inline bool FullGCForwarding::is_forwarded(oop obj) { + return obj->is_forwarded(); +} + +size_t FullGCForwarding::biased_region_index_containing(HeapWord* addr) { + return (uintptr_t)addr >> BLOCK_SIZE_BYTES_SHIFT; +} + +bool FullGCForwarding::is_fallback(uintptr_t encoded) { + return (encoded & OFFSET_MASK) == FALLBACK_PATTERN_IN_PLACE; +} + +uintptr_t FullGCForwarding::encode_forwarding(HeapWord* from, HeapWord* to) { + size_t from_block_idx = biased_region_index_containing(from); + + HeapWord* to_region_base = _biased_bases[from_block_idx]; + if (to_region_base == UNUSED_BASE) { + _biased_bases[from_block_idx] = to_region_base = to; + } + + // Avoid pointer_delta() on purpose: using an unsigned subtraction, + // we get an underflow when to < to_region_base, which means + // we can use a single comparison instead of: + // if (to_region_base > to || (to - to_region_base) > MAX_OFFSET) { .. } + size_t offset = size_t(to - to_region_base); + if (offset > MAX_OFFSET) { + offset = FALLBACK_PATTERN; + } + uintptr_t encoded = (offset << OFFSET_BITS_SHIFT) | markWord::marked_value; + + assert(is_fallback(encoded) || to == decode_forwarding(from, encoded), "must be reversible"); + assert((encoded & ~AVAILABLE_BITS_MASK) == 0, "must encode to available bits"); + return encoded; +} -void FullGCForwarding::forward_to(oop from, oop to) { +HeapWord* FullGCForwarding::decode_forwarding(HeapWord* from, uintptr_t encoded) { + assert(!is_fallback(encoded), "must not be fallback-forwarded, encoded: " INTPTR_FORMAT ", OFFSET_MASK: " INTPTR_FORMAT ", FALLBACK_PATTERN_IN_PLACE: " INTPTR_FORMAT, encoded, OFFSET_MASK, FALLBACK_PATTERN_IN_PLACE); + assert((encoded & ~AVAILABLE_BITS_MASK) == 0, "must decode from available bits, encoded: " INTPTR_FORMAT, encoded); + uintptr_t offset = (encoded >> OFFSET_BITS_SHIFT); + + size_t from_idx = biased_region_index_containing(from); + HeapWord* base = _biased_bases[from_idx]; + assert(base != UNUSED_BASE, "must not be unused base: encoded: " INTPTR_FORMAT, encoded); + HeapWord* decoded = base + offset; + assert(decoded >= _heap_start, + "Address must be above heap start. encoded: " INTPTR_FORMAT ", base: " PTR_FORMAT, + encoded, p2i(base)); + + return decoded; +} + +inline void FullGCForwarding::forward_to_impl(oop from, oop to) { + assert(_bases_table != nullptr, "call begin() before forwarding"); + + markWord from_header = from->mark(); + HeapWord* from_hw = cast_from_oop(from); + HeapWord* to_hw = cast_from_oop(to); + uintptr_t encoded = encode_forwarding(from_hw, to_hw); + markWord new_header = markWord((from_header.value() & ~OFFSET_MASK) | encoded); + from->set_mark(new_header); + + if (is_fallback(encoded)) { + fallback_forward_to(from_hw, to_hw); + } + NOT_PRODUCT(Atomic::inc(&_num_forwardings);) +} + +inline void FullGCForwarding::forward_to(oop obj, oop fwd) { + assert(fwd != nullptr, "no null forwarding"); #ifdef _LP64 - uintptr_t encoded = pointer_delta(cast_from_oop(to), _heap_base) << Shift; - assert(encoded <= static_cast(right_n_bits(_num_low_bits)), "encoded forwardee must fit"); - uintptr_t mark = from->mark().value(); - mark &= ~right_n_bits(_num_low_bits); - mark |= (encoded | markWord::marked_value); - from->set_mark(markWord(mark)); + assert(_bases_table != nullptr, "expect sliding forwarding initialized"); + forward_to_impl(obj, fwd); + assert(forwardee(obj) == fwd, "must be forwarded to correct forwardee, obj: " PTR_FORMAT ", forwardee(obj): " PTR_FORMAT ", fwd: " PTR_FORMAT ", mark: " INTPTR_FORMAT, p2i(obj), p2i(forwardee(obj)), p2i(fwd), obj->mark().value()); #else - from->forward_to(to); + obj->forward_to(fwd); #endif } -oop FullGCForwarding::forwardee(oop from) { +inline oop FullGCForwarding::forwardee_impl(oop from) { + assert(_bases_table != nullptr, "call begin() before asking for forwarding"); + + markWord header = from->mark(); + HeapWord* from_hw = cast_from_oop(from); + if (is_fallback(header.value())) { + HeapWord* to = fallback_forwardee(from_hw); + return cast_to_oop(to); + } + uintptr_t encoded = header.value() & OFFSET_MASK; + HeapWord* to = decode_forwarding(from_hw, encoded); + return cast_to_oop(to); +} + +inline oop FullGCForwarding::forwardee(oop obj) { #ifdef _LP64 - uintptr_t mark = from->mark().value(); - HeapWord* decoded = _heap_base + ((mark & right_n_bits(_num_low_bits)) >> Shift); - return cast_to_oop(decoded); + assert(_bases_table != nullptr, "expect sliding forwarding initialized"); + return forwardee_impl(obj); #else - return from->forwardee(); + return obj->forwardee(); #endif } -bool FullGCForwarding::is_forwarded(oop obj) { - return obj->mark().is_forwarded(); -} - -#endif // GC_SHARED_FULLGCFORWARDING_INLINE_HPP +#endif // SHARE_GC_SHARED_FULLGCFORWARDING_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp index df2cd18042f0f..f4bf39a86e934 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp @@ -24,7 +24,6 @@ * */ -#include "gc/shared/fullGCForwarding.hpp" #include "gc/shared/gcArguments.hpp" #include "gc/shared/tlab_globals.hpp" #include "gc/shared/workerPolicy.hpp" @@ -193,8 +192,6 @@ void ShenandoahArguments::initialize() { vm_exit_during_initialization( err_msg("GCCardSizeInBytes ( %u ) must be >= %u\n", GCCardSizeInBytes, (unsigned int) ShenandoahMinCardSizeInBytes)); } - - FullGCForwarding::initialize_flags(MaxHeapSize); } size_t ShenandoahArguments::conservative_max_heap_alignment() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 27ff45e67de19..43ec8dfeab678 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -245,6 +245,8 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { // until all phases run together. ShenandoahHeapLocker lock(heap->lock()); + FullGCForwarding::begin(); + phase2_calculate_target_addresses(worker_slices); OrderAccess::fence(); @@ -254,6 +256,8 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { phase4_compact_objects(worker_slices); result = phase5_epilog(); + + FullGCForwarding::end(); } if (heap->mode()->is_generational()) { LogTarget(Info, gc, ergo) lt; diff --git a/src/hotspot/share/utilities/fastHash.hpp b/src/hotspot/share/utilities/fastHash.hpp new file mode 100644 index 0000000000000..8179df5795d39 --- /dev/null +++ b/src/hotspot/share/utilities/fastHash.hpp @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_UTILITIES_FASTHASH_HPP +#define SHARE_UTILITIES_FASTHASH_HPP + +#include "memory/allStatic.hpp" + +class FastHash : public AllStatic { +private: + static void fullmul64(uint64_t& hi, uint64_t& lo, uint64_t op1, uint64_t op2) { +#if defined(__SIZEOF_INT128__) + __uint128_t prod = static_cast<__uint128_t>(op1) * static_cast<__uint128_t>(op2); + hi = static_cast(prod >> 64); + lo = static_cast(prod >> 0); +#else + /* First calculate all of the cross products. */ + uint64_t lo_lo = (op1 & 0xFFFFFFFF) * (op2 & 0xFFFFFFFF); + uint64_t hi_lo = (op1 >> 32) * (op2 & 0xFFFFFFFF); + uint64_t lo_hi = (op1 & 0xFFFFFFFF) * (op2 >> 32); + uint64_t hi_hi = (op1 >> 32) * (op2 >> 32); + + /* Now add the products together. These will never overflow. */ + uint64_t cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + uint64_t upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + hi = upper; + lo = (cross << 32) | (lo_lo & 0xFFFFFFFF); +#endif + } + + static void fullmul32(uint32_t& hi, uint32_t& lo, uint32_t op1, uint32_t op2) { + uint64_t x64 = op1, y64 = op2, xy64 = x64 * y64; + hi = (uint32_t)(xy64 >> 32); + lo = (uint32_t)(xy64 >> 0); + } + + static uint64_t ror64(uint64_t x, uint64_t distance) { + distance = distance & (64 - 1); + return (x >> distance) | (x << (64 - distance)); + } + + static uint32_t ror32(uint32_t x, uint32_t distance) { + distance = distance & (32 - 1); + return (x >> distance) | (x << (32 - distance)); + } + +public: + static uint64_t get_hash64(uint64_t x, uint64_t y) { + const uint64_t M = 0x8ADAE89C337954D5; + const uint64_t A = 0xAAAAAAAAAAAAAAAA; // REPAA + const uint64_t H0 = (x ^ y), L0 = (x ^ A); + + uint64_t U0, V0; fullmul64(U0, V0, L0, M); + const uint64_t Q0 = (H0 * M); + const uint64_t L1 = (Q0 ^ U0); + + uint64_t U1, V1; fullmul64(U1, V1, L1, M); + const uint64_t P1 = (V0 ^ M); + const uint64_t Q1 = ror64(P1, L1); + const uint64_t L2 = (Q1 ^ U1); + return V1 ^ L2; + } + + static uint32_t get_hash32(uint32_t x, uint32_t y) { + const uint32_t M = 0x337954D5; + const uint32_t A = 0xAAAAAAAA; // REPAA + const uint32_t H0 = (x ^ y), L0 = (x ^ A); + + uint32_t U0, V0; fullmul32(U0, V0, L0, M); + const uint32_t Q0 = (H0 * M); + const uint32_t L1 = (Q0 ^ U0); + + uint32_t U1, V1; fullmul32(U1, V1, L1, M); + const uint32_t P1 = (V0 ^ M); + const uint32_t Q1 = ror32(P1, L1); + const uint32_t L2 = (Q1 ^ U1); + return V1 ^ L2; + } +}; + +#endif// SHARE_UTILITIES_FASTHASH_HPP diff --git a/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp b/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp index ce3576cc7ade1..6d337cb876aa3 100644 --- a/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp +++ b/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp @@ -55,6 +55,8 @@ TEST_VM(PreservedMarks, iterate_and_restore) { ASSERT_MARK_WORD_EQ(o1->mark(), changedMark()); ASSERT_MARK_WORD_EQ(o2->mark(), changedMark()); + FullGCForwarding::begin(); + // Push o1 and o2 to have their marks preserved. pm.push_if_necessary(o1, o1->mark()); pm.push_if_necessary(o2, o2->mark()); @@ -73,4 +75,6 @@ TEST_VM(PreservedMarks, iterate_and_restore) { pm.restore(); ASSERT_MARK_WORD_EQ(o3->mark(), changedMark()); ASSERT_MARK_WORD_EQ(o4->mark(), changedMark()); + + FullGCForwarding::end(); } From 8ac60c76343e223ea3718ababf763d4dcf378b70 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Fri, 4 Jul 2025 14:17:40 +0200 Subject: [PATCH 2/9] 8361252: Compact Full-GC Forwarding --- .../share/gc/parallel/psParallelCompact.cpp | 2 +- .../share/gc/shared/fullGCForwarding.cpp | 127 ++++++++---- .../share/gc/shared/fullGCForwarding.hpp | 44 ++-- .../gc/shared/fullGCForwarding.inline.hpp | 50 +++-- .../gtest/gc/shared/test_fullGCForwarding.cpp | 194 ++++++++++++++++++ 5 files changed, 342 insertions(+), 75 deletions(-) create mode 100644 test/hotspot/gtest/gc/shared/test_fullGCForwarding.cpp diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.cpp b/src/hotspot/share/gc/parallel/psParallelCompact.cpp index 004a93c96c062..40f26f6f1c5ea 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.cpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.cpp @@ -1668,7 +1668,7 @@ void PSParallelCompact::verify_forward() { if (cur_addr == bump_ptr) { assert(!FullGCForwarding::is_forwarded(obj), "inv"); } else { - assert(FullGCForwarding::forwardee(obj) == cast_to_oop(bump_ptr), "inv"); + assert(FullGCForwarding::forwardee(obj) == cast_to_oop(bump_ptr), "inv; obj: " PTR_FORMAT ", bump_ptr: " PTR_FORMAT ", fwd: " PTR_FORMAT, p2i(obj), p2i(bump_ptr), p2i(FullGCForwarding::forwardee(obj))); } bump_ptr += obj->size(); cur_addr += obj->size(); diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.cpp b/src/hotspot/share/gc/shared/fullGCForwarding.cpp index aeb45b1cd05e7..390066e9dce4f 100644 --- a/src/hotspot/share/gc/shared/fullGCForwarding.cpp +++ b/src/hotspot/share/gc/shared/fullGCForwarding.cpp @@ -25,14 +25,41 @@ #include "gc/shared/fullGCForwarding.hpp" #include "logging/log.hpp" #include "nmt/memTag.hpp" -#include "utilities/ostream.hpp" #include "utilities/concurrentHashTable.inline.hpp" #include "utilities/fastHash.hpp" #include "utilities/powerOfTwo.hpp" +// We cannot use 0, because that may already be a valid base address in zero-based heaps. +// 0x1 is safe because heap base addresses must be aligned by much larger alignment +template +HeapWord* const FullGCForwardingImpl::UNUSED_BASE = reinterpret_cast(0x1); + +template +HeapWord* FullGCForwardingImpl::_heap_start = nullptr; +template +size_t FullGCForwardingImpl::_heap_start_region_bias = 0; +template +size_t FullGCForwardingImpl::_num_regions = 0; +template +uintptr_t FullGCForwardingImpl::_region_mask = 0; +template +HeapWord** FullGCForwardingImpl::_biased_bases = nullptr; +template +HeapWord** FullGCForwardingImpl::_bases_table = nullptr; +template +size_t FullGCForwardingImpl::_fallback_table_log2_start_size = 9; // 512 entries. +template +FallbackTable* FullGCForwardingImpl::_fallback_table = nullptr; +#ifndef PRODUCT +template +volatile uint64_t FullGCForwardingImpl::_num_forwardings = 0; +template +volatile uint64_t FullGCForwardingImpl::_num_fallback_forwardings = 0; +#endif + static uintx hash(HeapWord* const& addr) { uint64_t val = reinterpret_cast(addr); - uint32_t hash = FastHash::get_hash32((uint32_t)val, (uint32_t)(val >> 32)); + uint32_t hash = FastHash::get_hash32(static_cast(val), static_cast(val >> 32)); return hash; } @@ -56,7 +83,8 @@ struct FallbackTableConfig { }; class FallbackTable : public ConcurrentHashTable { - +public: + explicit FallbackTable(size_t log2size) : ConcurrentHashTable(log2size) {} }; class FallbackTableLookup : public StackObj { @@ -66,50 +94,36 @@ class FallbackTableLookup : public StackObj { uintx get_hash() const { return hash(_entry._from); } - bool equals(ForwardingEntry* value) { + bool equals(const ForwardingEntry* value) const { return _entry._from == value->_from; } - bool is_dead(ForwardingEntry* value) { return false; } + static bool is_dead(ForwardingEntry* value) { return false; } }; -// We cannot use 0, because that may already be a valid base address in zero-based heaps. -// 0x1 is safe because heap base addresses must be aligned by much larger alignment -HeapWord* const FullGCForwarding::UNUSED_BASE = reinterpret_cast(0x1); - -HeapWord* FullGCForwarding::_heap_start = nullptr; -size_t FullGCForwarding::_heap_start_region_bias = 0; -size_t FullGCForwarding::_num_regions = 0; -uintptr_t FullGCForwarding::_region_mask = 0; -HeapWord** FullGCForwarding::_biased_bases = nullptr; -HeapWord** FullGCForwarding::_bases_table = nullptr; -FallbackTable* FullGCForwarding::_fallback_table = nullptr; -#ifndef PRODUCT -volatile uint64_t FullGCForwarding::_num_forwardings = 0; -volatile uint64_t FullGCForwarding::_num_fallback_forwardings = 0; -#endif - -void FullGCForwarding::initialize(MemRegion heap) { +template +void FullGCForwardingImpl::initialize(MemRegion heap) { #ifdef _LP64 _heap_start = heap.start(); - size_t rounded_heap_size = round_up_power_of_2(heap.byte_size()); + size_t rounded_heap_size = MAX2(round_up_power_of_2(heap.byte_size()) / BytesPerWord, BLOCK_SIZE_WORDS); - _num_regions = (rounded_heap_size / BytesPerWord) / BLOCK_SIZE_WORDS; + _num_regions = rounded_heap_size / BLOCK_SIZE_WORDS; - _heap_start_region_bias = (uintptr_t)_heap_start >> BLOCK_SIZE_BYTES_SHIFT; - _region_mask = ~((uintptr_t(1) << BLOCK_SIZE_BYTES_SHIFT) - 1); + _heap_start_region_bias = reinterpret_cast(_heap_start) >> BLOCK_SIZE_BYTES_SHIFT; + _region_mask = ~((static_cast(1) << BLOCK_SIZE_BYTES_SHIFT) - 1); assert(_bases_table == nullptr, "should not be initialized yet"); assert(_fallback_table == nullptr, "should not be initialized yet"); #endif } -void FullGCForwarding::begin() { +template +void FullGCForwardingImpl::begin() { #ifdef _LP64 assert(_bases_table == nullptr, "should not be initialized yet"); assert(_fallback_table == nullptr, "should not be initialized yet"); - _fallback_table = new FallbackTable(); + _fallback_table = nullptr; #ifndef PRODUCT _num_forwardings = 0; @@ -120,19 +134,30 @@ void FullGCForwarding::begin() { _bases_table = NEW_C_HEAP_ARRAY(HeapWord*, max, mtGC); HeapWord** biased_start = _bases_table - _heap_start_region_bias; _biased_bases = biased_start; - for (size_t i = 0; i < max; i++) { - _bases_table[i] = UNUSED_BASE; + if (max == 1) { + // Optimize the case when the block-size >= heap-size. + // In this case we can use the heap-start as block-start, + // and don't risk that competing GC threads set a higher + // address as block-start, which would lead to unnecessary + // fallback-usage. + _bases_table[0] = _heap_start; + } else { + for (size_t i = 0; i < max; i++) { + _bases_table[i] = UNUSED_BASE; + } } #endif } -void FullGCForwarding::end() { +template +void FullGCForwardingImpl::end() { #ifndef PRODUCT + size_t fallback_table_size = _fallback_table != nullptr ? _fallback_table->get_mem_size(Thread::current()) : 0; log_info(gc)("Total forwardings: " UINT64_FORMAT ", fallback forwardings: " UINT64_FORMAT ", ratio: %f, memory used by fallback table: %zu%s, memory used by bases table: %zu%s", - _num_forwardings, _num_fallback_forwardings, (float)_num_forwardings/(float)_num_fallback_forwardings, - byte_size_in_proper_unit(_fallback_table->get_mem_size(Thread::current())), - proper_unit_for_byte_size(_fallback_table->get_mem_size(Thread::current())), + _num_forwardings, _num_fallback_forwardings, static_cast(_num_forwardings) / static_cast(_num_fallback_forwardings), + byte_size_in_proper_unit(fallback_table_size), + proper_unit_for_byte_size(fallback_table_size), byte_size_in_proper_unit(sizeof(HeapWord*) * _num_regions), proper_unit_for_byte_size(sizeof(HeapWord*) * _num_regions)); #endif @@ -140,13 +165,29 @@ void FullGCForwarding::end() { assert(_bases_table != nullptr, "should be initialized"); FREE_C_HEAP_ARRAY(HeapWord*, _bases_table); _bases_table = nullptr; - delete _fallback_table; - _fallback_table = nullptr; + if (_fallback_table != nullptr) { + delete _fallback_table; + _fallback_table = nullptr; + } #endif } -void FullGCForwarding::fallback_forward_to(HeapWord* from, HeapWord* to) { +template +void FullGCForwardingImpl::maybe_init_fallback_table() { + if (_fallback_table == nullptr) { + FallbackTable* fallback_table = new FallbackTable(_fallback_table_log2_start_size); + FallbackTable* prev = Atomic::cmpxchg(&_fallback_table, static_cast(nullptr), fallback_table); + if (prev != nullptr) { + // Another thread won, discard our table. + delete fallback_table; + } + } +} + +template +void FullGCForwardingImpl::fallback_forward_to(HeapWord* from, HeapWord* to) { assert(to != nullptr, "no null forwarding"); + maybe_init_fallback_table(); assert(_fallback_table != nullptr, "should be initialized"); FallbackTableLookup lookup_f(from); ForwardingEntry entry(from, to); @@ -167,16 +208,16 @@ void FullGCForwarding::fallback_forward_to(HeapWord* from, HeapWord* to) { #endif if (grow) { _fallback_table->grow(current_thread); - tty->print_cr("grow fallback table to size: %zu bytes", - _fallback_table->get_mem_size(current_thread)); + log_debug(gc)("grow fallback table to size: %zu bytes", _fallback_table->get_mem_size(current_thread)); } } -HeapWord* FullGCForwarding::fallback_forwardee(HeapWord* from) { +template +HeapWord* FullGCForwardingImpl::fallback_forwardee(HeapWord* from) { assert(_fallback_table != nullptr, "fallback table must be present"); HeapWord* result; FallbackTableLookup lookup_f(from); - auto found_f = [&](ForwardingEntry* found) { + auto found_f = [&](const ForwardingEntry* found) { result = found->_to; }; bool found = _fallback_table->get(Thread::current(), lookup_f, found_f); @@ -184,3 +225,7 @@ HeapWord* FullGCForwarding::fallback_forwardee(HeapWord* from) { assert(result != nullptr, "must have found forwarding"); return result; } + +template class FullGCForwardingImpl; +// For testing, used in test_fullGCForwarding.cpp +template class FullGCForwardingImpl<4>; diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.hpp b/src/hotspot/share/gc/shared/fullGCForwarding.hpp index 902d79111f38d..4d89f67898265 100644 --- a/src/hotspot/share/gc/shared/fullGCForwarding.hpp +++ b/src/hotspot/share/gc/shared/fullGCForwarding.hpp @@ -31,7 +31,6 @@ #include "oops/oopsHierarchy.hpp" class FallbackTable; -class Mutex; /** * FullGCForwarding is a method to store forwarding information in a compressed form into the object header, @@ -116,10 +115,11 @@ class Mutex; * This implies that this particular offset (the last word of a block) can not be used directly as forwarding, * but also has to be handled by the fallback-table. */ -class FullGCForwarding : public AllStatic { -private: - static constexpr int AVAILABLE_LOW_BITS = 11; - static constexpr int AVAILABLE_BITS_MASK = right_n_bits(AVAILABLE_LOW_BITS); +template +class FullGCForwardingImpl : public AllStatic { + friend class FullGCForwardingTest; + static constexpr int AVAILABLE_LOW_BITS = BITS; + static constexpr uintptr_t AVAILABLE_BITS_MASK = right_n_bits(AVAILABLE_LOW_BITS); // The offset bits start after the lock-bits, which are currently used by Serial GC // for marking objects. Could be 1 for Serial GC when being clever with the bits, // and 0 for all other GCs. @@ -127,7 +127,7 @@ class FullGCForwarding : public AllStatic { // How many bits we use for the offset static constexpr int NUM_OFFSET_BITS = AVAILABLE_LOW_BITS - OFFSET_BITS_SHIFT; - static constexpr size_t BLOCK_SIZE_WORDS = 1 << NUM_OFFSET_BITS; + static constexpr size_t BLOCK_SIZE_WORDS = 1l << NUM_OFFSET_BITS; static constexpr int BLOCK_SIZE_BYTES_SHIFT = NUM_OFFSET_BITS + LogHeapWordSize; static constexpr size_t MAX_OFFSET = BLOCK_SIZE_WORDS - 2; static constexpr uintptr_t OFFSET_MASK = right_n_bits(NUM_OFFSET_BITS) << OFFSET_BITS_SHIFT; @@ -153,6 +153,7 @@ class FullGCForwarding : public AllStatic { // Entries into the target base tables, biased to the start of the heap. static HeapWord** _biased_bases; + static size_t _fallback_table_log2_start_size; static FallbackTable* _fallback_table; #ifndef PRODUCT @@ -160,29 +161,40 @@ class FullGCForwarding : public AllStatic { static volatile uint64_t _num_fallback_forwardings; #endif - static inline size_t biased_region_index_containing(HeapWord* addr); + static size_t biased_region_index_containing(HeapWord* addr); - static inline bool is_fallback(uintptr_t encoded); - static inline uintptr_t encode_forwarding(HeapWord* from, HeapWord* to); - static inline HeapWord* decode_forwarding(HeapWord* from, uintptr_t encoded); + static bool is_fallback(uintptr_t encoded); + static uintptr_t encode_forwarding(HeapWord* from, HeapWord* to); + static HeapWord* decode_forwarding(HeapWord* from, uintptr_t encoded); + static void maybe_init_fallback_table(); static void fallback_forward_to(HeapWord* from, HeapWord* to); static HeapWord* fallback_forwardee(HeapWord* from); - static inline void forward_to_impl(oop from, oop to); - static inline oop forwardee_impl(oop from); + static void forward_to_impl(oop from, oop to); + static oop forwardee_impl(oop from); + FullGCForwardingImpl() = delete; + + // Used in unit-test, so that we can test fallback-table-growth. + static void set_fallback_table_log2_start_size(size_t fallback_table_log2_start_size) { + _fallback_table_log2_start_size = fallback_table_log2_start_size; + } public: static void initialize(MemRegion heap); static void begin(); static void end(); - static inline bool is_forwarded(oop obj); - static inline bool is_not_forwarded(oop obj); + static bool is_forwarded(oop obj); - static inline void forward_to(oop from, oop to); - static inline oop forwardee(oop from); + static void forward_to(oop from, oop to); + static oop forwardee(oop from); }; +extern template class FullGCForwardingImpl; +extern template class FullGCForwardingImpl<4>; + +using FullGCForwarding = FullGCForwardingImpl; + #endif // SHARE_GC_SHARED_FULLGCFORWARDING_HPP diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp b/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp index 2245f30f02065..17547ce1371e8 100644 --- a/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp +++ b/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp @@ -24,48 +24,56 @@ #ifndef SHARE_GC_SHARED_FULLGCFORWARDING_INLINE_HPP #define SHARE_GC_SHARED_FULLGCFORWARDING_INLINE_HPP -#include "gc/shared/gc_globals.hpp" #include "gc/shared/fullGCForwarding.hpp" #include "oops/markWord.hpp" -#include "oops/oop.inline.hpp" #include "utilities/macros.hpp" -inline bool FullGCForwarding::is_forwarded(oop obj) { +template +bool FullGCForwardingImpl::is_forwarded(oop obj) { return obj->is_forwarded(); } -size_t FullGCForwarding::biased_region_index_containing(HeapWord* addr) { - return (uintptr_t)addr >> BLOCK_SIZE_BYTES_SHIFT; +template +size_t FullGCForwardingImpl::biased_region_index_containing(HeapWord* addr) { + return reinterpret_cast(addr) >> BLOCK_SIZE_BYTES_SHIFT; } -bool FullGCForwarding::is_fallback(uintptr_t encoded) { +template +bool FullGCForwardingImpl::is_fallback(uintptr_t encoded) { return (encoded & OFFSET_MASK) == FALLBACK_PATTERN_IN_PLACE; } -uintptr_t FullGCForwarding::encode_forwarding(HeapWord* from, HeapWord* to) { +template +uintptr_t FullGCForwardingImpl::encode_forwarding(HeapWord* from, HeapWord* to) { size_t from_block_idx = biased_region_index_containing(from); HeapWord* to_region_base = _biased_bases[from_block_idx]; if (to_region_base == UNUSED_BASE) { - _biased_bases[from_block_idx] = to_region_base = to; + HeapWord* prev = Atomic::cmpxchg(&_biased_bases[from_block_idx], UNUSED_BASE, to); + if (prev == UNUSED_BASE) { + to_region_base = to; + } else { + to_region_base = prev; + } + // _biased_bases[from_block_idx] = to_region_base = to; } - // Avoid pointer_delta() on purpose: using an unsigned subtraction, // we get an underflow when to < to_region_base, which means // we can use a single comparison instead of: // if (to_region_base > to || (to - to_region_base) > MAX_OFFSET) { .. } - size_t offset = size_t(to - to_region_base); + size_t offset = static_cast(to - to_region_base); if (offset > MAX_OFFSET) { offset = FALLBACK_PATTERN; } uintptr_t encoded = (offset << OFFSET_BITS_SHIFT) | markWord::marked_value; - assert(is_fallback(encoded) || to == decode_forwarding(from, encoded), "must be reversible"); + assert(is_fallback(encoded) || to == decode_forwarding(from, encoded), "must be reversible: " PTR_FORMAT " -> " PTR_FORMAT ", reversed: " PTR_FORMAT ", encoded: " INTPTR_FORMAT ", to_region_base: " PTR_FORMAT ", from_block_idx: %lu", p2i(from), p2i(to), p2i(decode_forwarding(from, encoded)), encoded, p2i(to_region_base), from_block_idx); assert((encoded & ~AVAILABLE_BITS_MASK) == 0, "must encode to available bits"); return encoded; } -HeapWord* FullGCForwarding::decode_forwarding(HeapWord* from, uintptr_t encoded) { +template +HeapWord* FullGCForwardingImpl::decode_forwarding(HeapWord* from, uintptr_t encoded) { assert(!is_fallback(encoded), "must not be fallback-forwarded, encoded: " INTPTR_FORMAT ", OFFSET_MASK: " INTPTR_FORMAT ", FALLBACK_PATTERN_IN_PLACE: " INTPTR_FORMAT, encoded, OFFSET_MASK, FALLBACK_PATTERN_IN_PLACE); assert((encoded & ~AVAILABLE_BITS_MASK) == 0, "must decode from available bits, encoded: " INTPTR_FORMAT, encoded); uintptr_t offset = (encoded >> OFFSET_BITS_SHIFT); @@ -81,7 +89,8 @@ HeapWord* FullGCForwarding::decode_forwarding(HeapWord* from, uintptr_t encoded) return decoded; } -inline void FullGCForwarding::forward_to_impl(oop from, oop to) { +template +void FullGCForwardingImpl::forward_to_impl(oop from, oop to) { assert(_bases_table != nullptr, "call begin() before forwarding"); markWord from_header = from->mark(); @@ -97,18 +106,20 @@ inline void FullGCForwarding::forward_to_impl(oop from, oop to) { NOT_PRODUCT(Atomic::inc(&_num_forwardings);) } -inline void FullGCForwarding::forward_to(oop obj, oop fwd) { +template +void FullGCForwardingImpl::forward_to(oop obj, oop fwd) { assert(fwd != nullptr, "no null forwarding"); #ifdef _LP64 assert(_bases_table != nullptr, "expect sliding forwarding initialized"); forward_to_impl(obj, fwd); - assert(forwardee(obj) == fwd, "must be forwarded to correct forwardee, obj: " PTR_FORMAT ", forwardee(obj): " PTR_FORMAT ", fwd: " PTR_FORMAT ", mark: " INTPTR_FORMAT, p2i(obj), p2i(forwardee(obj)), p2i(fwd), obj->mark().value()); + // assert(forwardee(obj) == fwd, "must be forwarded to correct forwardee, obj: " PTR_FORMAT ", forwardee(obj): " PTR_FORMAT ", fwd: " PTR_FORMAT ", mark: " INTPTR_FORMAT ", num-regions: %lu, base: " PTR_FORMAT ", OFFSET_MASK: " INTPTR_FORMAT ", encoded: " PTR_FORMAT ", biased-base: " PTR_FORMAT ", heap-start: " PTR_FORMAT, p2i(obj), p2i(forwardee(obj)), p2i(fwd), obj->mark().value(), _num_regions, p2i(_bases_table[0]), OFFSET_MASK, encode_forwarding(cast_from_oop(obj), cast_from_oop(fwd)), p2i(_biased_bases[biased_region_index_containing(cast_from_oop(obj))]), p2i(_heap_start)); #else obj->forward_to(fwd); #endif } -inline oop FullGCForwarding::forwardee_impl(oop from) { +template +oop FullGCForwardingImpl::forwardee_impl(oop from) { assert(_bases_table != nullptr, "call begin() before asking for forwarding"); markWord header = from->mark(); @@ -122,7 +133,8 @@ inline oop FullGCForwarding::forwardee_impl(oop from) { return cast_to_oop(to); } -inline oop FullGCForwarding::forwardee(oop obj) { +template +oop FullGCForwardingImpl::forwardee(oop obj) { #ifdef _LP64 assert(_bases_table != nullptr, "expect sliding forwarding initialized"); return forwardee_impl(obj); @@ -131,4 +143,8 @@ inline oop FullGCForwarding::forwardee(oop obj) { #endif } +template class FullGCForwardingImpl; +// For testing, used in test_fullGCForwarding.cpp +template class FullGCForwardingImpl<4>; + #endif // SHARE_GC_SHARED_FULLGCFORWARDING_INLINE_HPP diff --git a/test/hotspot/gtest/gc/shared/test_fullGCForwarding.cpp b/test/hotspot/gtest/gc/shared/test_fullGCForwarding.cpp new file mode 100644 index 0000000000000..bd7198e0e36e3 --- /dev/null +++ b/test/hotspot/gtest/gc/shared/test_fullGCForwarding.cpp @@ -0,0 +1,194 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "gc/shared/fullGCForwarding.inline.hpp" +#include "memory/allocation.hpp" +#include "memory/memRegion.hpp" +#include "oops/markWord.hpp" +#include "oops/oop.inline.hpp" +#include "utilities/globalDefinitions.hpp" +#include "unittest.hpp" + +// This gives us 2 bits to address forwarding in object header, +// which corresponds to 4 words/32 bytes block size. +using TestFullGCForwarding = FullGCForwardingImpl<4>; + +class FullGCForwardingTest : public testing::Test { + // Size of fake heap, in words. + static const int FAKE_HEAP_SIZE = 64; + // Alignment of fake heap, in words; + static const int FAKE_HEAP_ALIGNMENT = 8; + // Real size of fake heap, considering alignment. + static const int FAKE_HEAP_SIZE_UNALIGNED = FAKE_HEAP_SIZE + FAKE_HEAP_ALIGNMENT; + // Bit-pattern which must not change. + static const uintptr_t BIT_PATTERN = 0xA5A5A5A5A5A5A5A0; + // Number of bits used for forwarding. + static const int NUM_FWD_BITS = 4; + // Forwarding bit mask. + static const uintptr_t FWD_BIT_MASK = right_n_bits(NUM_FWD_BITS); + + HeapWord* _unaligned_heap; +protected: + HeapWord* _heap; + +public: + FullGCForwardingTest() { + _unaligned_heap = NEW_C_HEAP_ARRAY(HeapWord, FAKE_HEAP_SIZE_UNALIGNED, mtGC); + _heap = align_up(_unaligned_heap, FAKE_HEAP_ALIGNMENT * sizeof(HeapWord)); + TestFullGCForwarding::set_fallback_table_log2_start_size(2); + TestFullGCForwarding::initialize(MemRegion(&_heap[0], &_heap[64])); + TestFullGCForwarding::begin(); + } + ~FullGCForwardingTest() { + TestFullGCForwarding::end(); + FREE_C_HEAP_ARRAY(HeapWord, _unaligned_heap); + } + + oop new_oop(int index) { + HeapWord* oop_addr = _heap + index; + // Initialize 'mark-word' + // Bit-pattern in upper 58 bits (which must not change) + // and 000001 in lowest 6 bits (which corresponds to not-forwarded). + oop obj = cast_to_oop(oop_addr); + obj->set_mark(markWord(BIT_PATTERN | markWord::unlocked_value)); + return obj; + } + + void assert_forwarding(oop obj, oop fwd, uintptr_t bits) { + ASSERT_EQ(fwd, TestFullGCForwarding::forwardee(obj)); + ASSERT_TRUE(TestFullGCForwarding::is_forwarded(obj)); + uintptr_t mark = obj->mark().value(); + ASSERT_EQ(bits, mark & FWD_BIT_MASK); + uintptr_t pattern = BIT_PATTERN; + ASSERT_EQ(pattern, mark & ~FWD_BIT_MASK); + } +}; + +TEST_VM_F(FullGCForwardingTest, basic) { + + oop o1 = new_oop(0); + oop o2 = new_oop(1); + + // Create a single forwarding. + TestFullGCForwarding::forward_to(o1, o2); + // Check that forwarding is correct. + assert_forwarding(o1, o2, 0b0011); + +} + +TEST_VM_F(FullGCForwardingTest, full_block) { + + oop o1 = new_oop(0); + oop o2 = new_oop(1); + oop o3 = new_oop(2); + oop o4 = new_oop(3); + oop o5 = new_oop(4); + oop o6 = new_oop(5); + oop o7 = new_oop(6); + oop o8 = new_oop(7); + + // Forward objects in first block to objects in second block. + TestFullGCForwarding::forward_to(o1, o5); + TestFullGCForwarding::forward_to(o2, o6); + TestFullGCForwarding::forward_to(o3, o7); + // Note: this would be recorded in the fallback table. + TestFullGCForwarding::forward_to(o4, o8); + + // Check that forwardings are correct. + assert_forwarding(o1, o5, 0b0011); + assert_forwarding(o2, o6, 0b0111); + assert_forwarding(o3, o7, 0b1011); + assert_forwarding(o4, o8, 0b1111); // Fallback-pattern + +} + +TEST_VM_F(FullGCForwardingTest, full_block_cross_boundary) { + + oop o1 = new_oop(0); + oop o2 = new_oop(1); + oop o3 = new_oop(2); + oop o4 = new_oop(3); + oop o5 = new_oop(6); + oop o6 = new_oop(7); + oop o7 = new_oop(8); + oop o8 = new_oop(9); + + // Forward objects in first block to objects in second block. + TestFullGCForwarding::forward_to(o1, o5); + TestFullGCForwarding::forward_to(o2, o6); + TestFullGCForwarding::forward_to(o3, o7); + // Note: this would be recorded in the fallback table. + TestFullGCForwarding::forward_to(o4, o8); + + // Check that forwardings are correct. + assert_forwarding(o1, o5, 0b0011); + assert_forwarding(o2, o6, 0b0111); + assert_forwarding(o3, o7, 0b1011); + assert_forwarding(o4, o8, 0b1111); // Fallback-pattern + +} + +TEST_VM_F(FullGCForwardingTest, full_block_out_of_order) { + + oop o1 = new_oop(0); + oop o2 = new_oop(1); + oop o3 = new_oop(2); + oop o4 = new_oop(3); + oop o5 = new_oop(4); + oop o6 = new_oop(5); + oop o7 = new_oop(6); + oop o8 = new_oop(7); + + // Forward objects in first block to objects in second block. + TestFullGCForwarding::forward_to(o1, o7); + TestFullGCForwarding::forward_to(o2, o8); + // This should go to fallback table, because the base offset is at o7. + TestFullGCForwarding::forward_to(o3, o5); + // This should go to fallback table, because the base offset is at o7. + TestFullGCForwarding::forward_to(o4, o6); + + // Check that forwardings are correct. + assert_forwarding(o1, o7, 0b0011); + assert_forwarding(o2, o8, 0b0111); + assert_forwarding(o3, o5, 0b1111); // Fallback-pattern + assert_forwarding(o4, o6, 0b1111); // Fallback-pattern + +} + +TEST_VM_F(FullGCForwardingTest, stress_fallback) { + + oop _objs[32]; + for (int i = 0; i < 32; i++) { + _objs[i] = new_oop(i); + } + + // Forward objects reverse order to put most in fallback. + for (int i = 0; i < 32; i++) { + TestFullGCForwarding::forward_to(_objs[i], _objs[31 - i]); + } + // Check that forwardings are correct. + for (int i = 0; i < 32; i++) { + uintptr_t bits = i % 4 == 0 ? 0b0011 : 0b1111; + assert_forwarding(_objs[i], _objs[31 - i], bits); + } +} From 4733b0c84ce37017b814052b0c0647e6b378b578 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Fri, 4 Jul 2025 15:06:53 +0200 Subject: [PATCH 3/9] Remove debug output --- src/hotspot/share/gc/parallel/psParallelCompact.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.cpp b/src/hotspot/share/gc/parallel/psParallelCompact.cpp index 40f26f6f1c5ea..004a93c96c062 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.cpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.cpp @@ -1668,7 +1668,7 @@ void PSParallelCompact::verify_forward() { if (cur_addr == bump_ptr) { assert(!FullGCForwarding::is_forwarded(obj), "inv"); } else { - assert(FullGCForwarding::forwardee(obj) == cast_to_oop(bump_ptr), "inv; obj: " PTR_FORMAT ", bump_ptr: " PTR_FORMAT ", fwd: " PTR_FORMAT, p2i(obj), p2i(bump_ptr), p2i(FullGCForwarding::forwardee(obj))); + assert(FullGCForwarding::forwardee(obj) == cast_to_oop(bump_ptr), "inv"); } bump_ptr += obj->size(); cur_addr += obj->size(); From d5720abe08edd2c5ef77b882f6754aa201a21419 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Mon, 14 Jul 2025 16:03:45 +0000 Subject: [PATCH 4/9] Add missing include --- src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp b/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp index 17547ce1371e8..50e67c65752c1 100644 --- a/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp +++ b/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp @@ -26,6 +26,7 @@ #include "gc/shared/fullGCForwarding.hpp" #include "oops/markWord.hpp" +#include "oops/oop.inline.hpp" #include "utilities/macros.hpp" template From e8c3cafe88429eeb9fc1780e450de241dbac18c5 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Mon, 14 Jul 2025 18:00:21 +0000 Subject: [PATCH 5/9] Fix windows build --- src/hotspot/share/gc/shared/fullGCForwarding.hpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.hpp b/src/hotspot/share/gc/shared/fullGCForwarding.hpp index 4d89f67898265..8c68539fbcfbd 100644 --- a/src/hotspot/share/gc/shared/fullGCForwarding.hpp +++ b/src/hotspot/share/gc/shared/fullGCForwarding.hpp @@ -127,7 +127,7 @@ class FullGCForwardingImpl : public AllStatic { // How many bits we use for the offset static constexpr int NUM_OFFSET_BITS = AVAILABLE_LOW_BITS - OFFSET_BITS_SHIFT; - static constexpr size_t BLOCK_SIZE_WORDS = 1l << NUM_OFFSET_BITS; + static constexpr size_t BLOCK_SIZE_WORDS = 1ll << NUM_OFFSET_BITS; static constexpr int BLOCK_SIZE_BYTES_SHIFT = NUM_OFFSET_BITS + LogHeapWordSize; static constexpr size_t MAX_OFFSET = BLOCK_SIZE_WORDS - 2; static constexpr uintptr_t OFFSET_MASK = right_n_bits(NUM_OFFSET_BITS) << OFFSET_BITS_SHIFT; @@ -193,7 +193,14 @@ class FullGCForwardingImpl : public AllStatic { }; extern template class FullGCForwardingImpl; +extern template void FullGCForwardingImpl::maybe_init_fallback_table(); +extern template void FullGCForwardingImpl::fallback_forward_to(HeapWord* from, HeapWord* to); +extern template HeapWord* FullGCForwardingImpl::fallback_forwardee(HeapWord* from); + extern template class FullGCForwardingImpl<4>; +extern template void FullGCForwardingImpl<4>::maybe_init_fallback_table(); +extern template void FullGCForwardingImpl<4>::fallback_forward_to(HeapWord* from, HeapWord* to); +extern template HeapWord* FullGCForwardingImpl<4>::fallback_forwardee(HeapWord* from); using FullGCForwarding = FullGCForwardingImpl; From 108b77d04d9504cab02c506f1d459c1fe13b9200 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Mon, 14 Jul 2025 19:20:56 +0000 Subject: [PATCH 6/9] Move all fullGCForwarding implementation into .inline.hpp file --- .../share/gc/shared/fullGCForwarding.cpp | 231 ------------------ .../share/gc/shared/fullGCForwarding.hpp | 10 - .../gc/shared/fullGCForwarding.inline.hpp | 204 +++++++++++++++- 3 files changed, 201 insertions(+), 244 deletions(-) delete mode 100644 src/hotspot/share/gc/shared/fullGCForwarding.cpp diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.cpp b/src/hotspot/share/gc/shared/fullGCForwarding.cpp deleted file mode 100644 index 390066e9dce4f..0000000000000 --- a/src/hotspot/share/gc/shared/fullGCForwarding.cpp +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "gc/shared/fullGCForwarding.hpp" -#include "logging/log.hpp" -#include "nmt/memTag.hpp" -#include "utilities/concurrentHashTable.inline.hpp" -#include "utilities/fastHash.hpp" -#include "utilities/powerOfTwo.hpp" - -// We cannot use 0, because that may already be a valid base address in zero-based heaps. -// 0x1 is safe because heap base addresses must be aligned by much larger alignment -template -HeapWord* const FullGCForwardingImpl::UNUSED_BASE = reinterpret_cast(0x1); - -template -HeapWord* FullGCForwardingImpl::_heap_start = nullptr; -template -size_t FullGCForwardingImpl::_heap_start_region_bias = 0; -template -size_t FullGCForwardingImpl::_num_regions = 0; -template -uintptr_t FullGCForwardingImpl::_region_mask = 0; -template -HeapWord** FullGCForwardingImpl::_biased_bases = nullptr; -template -HeapWord** FullGCForwardingImpl::_bases_table = nullptr; -template -size_t FullGCForwardingImpl::_fallback_table_log2_start_size = 9; // 512 entries. -template -FallbackTable* FullGCForwardingImpl::_fallback_table = nullptr; -#ifndef PRODUCT -template -volatile uint64_t FullGCForwardingImpl::_num_forwardings = 0; -template -volatile uint64_t FullGCForwardingImpl::_num_fallback_forwardings = 0; -#endif - -static uintx hash(HeapWord* const& addr) { - uint64_t val = reinterpret_cast(addr); - uint32_t hash = FastHash::get_hash32(static_cast(val), static_cast(val >> 32)); - return hash; -} - -struct ForwardingEntry { - HeapWord* _from; - HeapWord* _to; - ForwardingEntry(HeapWord* from, HeapWord* to) : _from(from), _to(to) {} -}; - -struct FallbackTableConfig { - using Value = ForwardingEntry; - static uintx get_hash(Value const& entry, bool* is_dead) { - return hash(entry._from); - } - static void* allocate_node(void* context, size_t size, Value const& value) { - return AllocateHeap(size, mtGC); - } - static void free_node(void* context, void* memory, Value const& value) { - FreeHeap(memory); - } -}; - -class FallbackTable : public ConcurrentHashTable { -public: - explicit FallbackTable(size_t log2size) : ConcurrentHashTable(log2size) {} -}; - -class FallbackTableLookup : public StackObj { - ForwardingEntry const _entry; -public: - explicit FallbackTableLookup(HeapWord* from) : _entry(from, nullptr) {} - uintx get_hash() const { - return hash(_entry._from); - } - bool equals(const ForwardingEntry* value) const { - return _entry._from == value->_from; - } - static bool is_dead(ForwardingEntry* value) { return false; } -}; - -template -void FullGCForwardingImpl::initialize(MemRegion heap) { -#ifdef _LP64 - _heap_start = heap.start(); - - size_t rounded_heap_size = MAX2(round_up_power_of_2(heap.byte_size()) / BytesPerWord, BLOCK_SIZE_WORDS); - - _num_regions = rounded_heap_size / BLOCK_SIZE_WORDS; - - _heap_start_region_bias = reinterpret_cast(_heap_start) >> BLOCK_SIZE_BYTES_SHIFT; - _region_mask = ~((static_cast(1) << BLOCK_SIZE_BYTES_SHIFT) - 1); - - assert(_bases_table == nullptr, "should not be initialized yet"); - assert(_fallback_table == nullptr, "should not be initialized yet"); -#endif -} - -template -void FullGCForwardingImpl::begin() { -#ifdef _LP64 - assert(_bases_table == nullptr, "should not be initialized yet"); - assert(_fallback_table == nullptr, "should not be initialized yet"); - - _fallback_table = nullptr; - -#ifndef PRODUCT - _num_forwardings = 0; - _num_fallback_forwardings = 0; -#endif - - size_t max = _num_regions; - _bases_table = NEW_C_HEAP_ARRAY(HeapWord*, max, mtGC); - HeapWord** biased_start = _bases_table - _heap_start_region_bias; - _biased_bases = biased_start; - if (max == 1) { - // Optimize the case when the block-size >= heap-size. - // In this case we can use the heap-start as block-start, - // and don't risk that competing GC threads set a higher - // address as block-start, which would lead to unnecessary - // fallback-usage. - _bases_table[0] = _heap_start; - } else { - for (size_t i = 0; i < max; i++) { - _bases_table[i] = UNUSED_BASE; - } - } -#endif -} - -template -void FullGCForwardingImpl::end() { -#ifndef PRODUCT - size_t fallback_table_size = _fallback_table != nullptr ? _fallback_table->get_mem_size(Thread::current()) : 0; - log_info(gc)("Total forwardings: " UINT64_FORMAT ", fallback forwardings: " UINT64_FORMAT - ", ratio: %f, memory used by fallback table: %zu%s, memory used by bases table: %zu%s", - _num_forwardings, _num_fallback_forwardings, static_cast(_num_forwardings) / static_cast(_num_fallback_forwardings), - byte_size_in_proper_unit(fallback_table_size), - proper_unit_for_byte_size(fallback_table_size), - byte_size_in_proper_unit(sizeof(HeapWord*) * _num_regions), - proper_unit_for_byte_size(sizeof(HeapWord*) * _num_regions)); -#endif -#ifdef _LP64 - assert(_bases_table != nullptr, "should be initialized"); - FREE_C_HEAP_ARRAY(HeapWord*, _bases_table); - _bases_table = nullptr; - if (_fallback_table != nullptr) { - delete _fallback_table; - _fallback_table = nullptr; - } -#endif -} - -template -void FullGCForwardingImpl::maybe_init_fallback_table() { - if (_fallback_table == nullptr) { - FallbackTable* fallback_table = new FallbackTable(_fallback_table_log2_start_size); - FallbackTable* prev = Atomic::cmpxchg(&_fallback_table, static_cast(nullptr), fallback_table); - if (prev != nullptr) { - // Another thread won, discard our table. - delete fallback_table; - } - } -} - -template -void FullGCForwardingImpl::fallback_forward_to(HeapWord* from, HeapWord* to) { - assert(to != nullptr, "no null forwarding"); - maybe_init_fallback_table(); - assert(_fallback_table != nullptr, "should be initialized"); - FallbackTableLookup lookup_f(from); - ForwardingEntry entry(from, to); - auto found_f = [&](ForwardingEntry* found) { - // If dupe has been found, override it with new value. - // This is also called when new entry is succussfully inserted. - if (found->_to != to) { - found->_to = to; - } - }; - Thread* current_thread = Thread::current(); - bool grow; - bool added = _fallback_table->insert_get(current_thread, lookup_f, entry, found_f, &grow); - NOT_PRODUCT(Atomic::inc(&_num_fallback_forwardings);) -#ifdef ASSERT - assert(fallback_forwardee(from) != nullptr, "must have entered forwarding"); - assert(fallback_forwardee(from) == to, "forwarding must be correct, added: %s, from: " PTR_FORMAT ", to: " PTR_FORMAT ", fwd: " PTR_FORMAT, BOOL_TO_STR(added), p2i(from), p2i(to), p2i(fallback_forwardee(from))); -#endif - if (grow) { - _fallback_table->grow(current_thread); - log_debug(gc)("grow fallback table to size: %zu bytes", _fallback_table->get_mem_size(current_thread)); - } -} - -template -HeapWord* FullGCForwardingImpl::fallback_forwardee(HeapWord* from) { - assert(_fallback_table != nullptr, "fallback table must be present"); - HeapWord* result; - FallbackTableLookup lookup_f(from); - auto found_f = [&](const ForwardingEntry* found) { - result = found->_to; - }; - bool found = _fallback_table->get(Thread::current(), lookup_f, found_f); - assert(found, "something must have been found"); - assert(result != nullptr, "must have found forwarding"); - return result; -} - -template class FullGCForwardingImpl; -// For testing, used in test_fullGCForwarding.cpp -template class FullGCForwardingImpl<4>; diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.hpp b/src/hotspot/share/gc/shared/fullGCForwarding.hpp index 8c68539fbcfbd..c24c17307860f 100644 --- a/src/hotspot/share/gc/shared/fullGCForwarding.hpp +++ b/src/hotspot/share/gc/shared/fullGCForwarding.hpp @@ -192,16 +192,6 @@ class FullGCForwardingImpl : public AllStatic { static oop forwardee(oop from); }; -extern template class FullGCForwardingImpl; -extern template void FullGCForwardingImpl::maybe_init_fallback_table(); -extern template void FullGCForwardingImpl::fallback_forward_to(HeapWord* from, HeapWord* to); -extern template HeapWord* FullGCForwardingImpl::fallback_forwardee(HeapWord* from); - -extern template class FullGCForwardingImpl<4>; -extern template void FullGCForwardingImpl<4>::maybe_init_fallback_table(); -extern template void FullGCForwardingImpl<4>::fallback_forward_to(HeapWord* from, HeapWord* to); -extern template HeapWord* FullGCForwardingImpl<4>::fallback_forwardee(HeapWord* from); - using FullGCForwarding = FullGCForwardingImpl; #endif // SHARE_GC_SHARED_FULLGCFORWARDING_HPP diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp b/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp index 50e67c65752c1..741e0d7b9eca1 100644 --- a/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp +++ b/src/hotspot/share/gc/shared/fullGCForwarding.inline.hpp @@ -25,9 +25,42 @@ #define SHARE_GC_SHARED_FULLGCFORWARDING_INLINE_HPP #include "gc/shared/fullGCForwarding.hpp" +#include "logging/log.hpp" +#include "nmt/memTag.hpp" #include "oops/markWord.hpp" #include "oops/oop.inline.hpp" +#include "utilities/concurrentHashTable.inline.hpp" +#include "utilities/fastHash.hpp" #include "utilities/macros.hpp" +#include "utilities/powerOfTwo.hpp" + +// We cannot use 0, because that may already be a valid base address in zero-based heaps. +// 0x1 is safe because heap base addresses must be aligned by much larger alignment +template +HeapWord* const FullGCForwardingImpl::UNUSED_BASE = reinterpret_cast(0x1); + +template +HeapWord* FullGCForwardingImpl::_heap_start = nullptr; +template +size_t FullGCForwardingImpl::_heap_start_region_bias = 0; +template +size_t FullGCForwardingImpl::_num_regions = 0; +template +uintptr_t FullGCForwardingImpl::_region_mask = 0; +template +HeapWord** FullGCForwardingImpl::_biased_bases = nullptr; +template +HeapWord** FullGCForwardingImpl::_bases_table = nullptr; +template +size_t FullGCForwardingImpl::_fallback_table_log2_start_size = 9; // 512 entries. +template +FallbackTable* FullGCForwardingImpl::_fallback_table = nullptr; +#ifndef PRODUCT +template +volatile uint64_t FullGCForwardingImpl::_num_forwardings = 0; +template +volatile uint64_t FullGCForwardingImpl::_num_fallback_forwardings = 0; +#endif template bool FullGCForwardingImpl::is_forwarded(oop obj) { @@ -144,8 +177,173 @@ oop FullGCForwardingImpl::forwardee(oop obj) { #endif } -template class FullGCForwardingImpl; -// For testing, used in test_fullGCForwarding.cpp -template class FullGCForwardingImpl<4>; +static uintx hash(HeapWord* const& addr) { + uint64_t val = reinterpret_cast(addr); + uint32_t hash = FastHash::get_hash32(static_cast(val), static_cast(val >> 32)); + return hash; +} + +struct ForwardingEntry { + HeapWord* _from; + HeapWord* _to; + ForwardingEntry(HeapWord* from, HeapWord* to) : _from(from), _to(to) {} +}; + +struct FallbackTableConfig { + using Value = ForwardingEntry; + static uintx get_hash(Value const& entry, bool* is_dead) { + return hash(entry._from); + } + static void* allocate_node(void* context, size_t size, Value const& value) { + return AllocateHeap(size, mtGC); + } + static void free_node(void* context, void* memory, Value const& value) { + FreeHeap(memory); + } +}; + +class FallbackTable : public ConcurrentHashTable { +public: + explicit FallbackTable(size_t log2size) : ConcurrentHashTable(log2size) {} +}; + +class FallbackTableLookup : public StackObj { + ForwardingEntry const _entry; +public: + explicit FallbackTableLookup(HeapWord* from) : _entry(from, nullptr) {} + uintx get_hash() const { + return hash(_entry._from); + } + bool equals(const ForwardingEntry* value) const { + return _entry._from == value->_from; + } + static bool is_dead(ForwardingEntry* value) { return false; } +}; + +template +void FullGCForwardingImpl::initialize(MemRegion heap) { +#ifdef _LP64 + _heap_start = heap.start(); + + size_t rounded_heap_size = MAX2(round_up_power_of_2(heap.byte_size()) / BytesPerWord, BLOCK_SIZE_WORDS); + + _num_regions = rounded_heap_size / BLOCK_SIZE_WORDS; + + _heap_start_region_bias = reinterpret_cast(_heap_start) >> BLOCK_SIZE_BYTES_SHIFT; + _region_mask = ~((static_cast(1) << BLOCK_SIZE_BYTES_SHIFT) - 1); + + assert(_bases_table == nullptr, "should not be initialized yet"); + assert(_fallback_table == nullptr, "should not be initialized yet"); +#endif +} + +template +void FullGCForwardingImpl::begin() { +#ifdef _LP64 + assert(_bases_table == nullptr, "should not be initialized yet"); + assert(_fallback_table == nullptr, "should not be initialized yet"); + + _fallback_table = nullptr; + +#ifndef PRODUCT + _num_forwardings = 0; + _num_fallback_forwardings = 0; +#endif + + size_t max = _num_regions; + _bases_table = NEW_C_HEAP_ARRAY(HeapWord*, max, mtGC); + HeapWord** biased_start = _bases_table - _heap_start_region_bias; + _biased_bases = biased_start; + if (max == 1) { + // Optimize the case when the block-size >= heap-size. + // In this case we can use the heap-start as block-start, + // and don't risk that competing GC threads set a higher + // address as block-start, which would lead to unnecessary + // fallback-usage. + _bases_table[0] = _heap_start; + } else { + for (size_t i = 0; i < max; i++) { + _bases_table[i] = UNUSED_BASE; + } + } +#endif +} + +template +void FullGCForwardingImpl::end() { +#ifndef PRODUCT + size_t fallback_table_size = _fallback_table != nullptr ? _fallback_table->get_mem_size(Thread::current()) : 0; + log_info(gc)("Total forwardings: " UINT64_FORMAT ", fallback forwardings: " UINT64_FORMAT + ", ratio: %f, memory used by fallback table: %zu%s, memory used by bases table: %zu%s", + _num_forwardings, _num_fallback_forwardings, static_cast(_num_forwardings) / static_cast(_num_fallback_forwardings), + byte_size_in_proper_unit(fallback_table_size), + proper_unit_for_byte_size(fallback_table_size), + byte_size_in_proper_unit(sizeof(HeapWord*) * _num_regions), + proper_unit_for_byte_size(sizeof(HeapWord*) * _num_regions)); +#endif +#ifdef _LP64 + assert(_bases_table != nullptr, "should be initialized"); + FREE_C_HEAP_ARRAY(HeapWord*, _bases_table); + _bases_table = nullptr; + if (_fallback_table != nullptr) { + delete _fallback_table; + _fallback_table = nullptr; + } +#endif +} + +template +void FullGCForwardingImpl::maybe_init_fallback_table() { + if (_fallback_table == nullptr) { + FallbackTable* fallback_table = new FallbackTable(_fallback_table_log2_start_size); + FallbackTable* prev = Atomic::cmpxchg(&_fallback_table, static_cast(nullptr), fallback_table); + if (prev != nullptr) { + // Another thread won, discard our table. + delete fallback_table; + } + } +} + +template +void FullGCForwardingImpl::fallback_forward_to(HeapWord* from, HeapWord* to) { + assert(to != nullptr, "no null forwarding"); + maybe_init_fallback_table(); + assert(_fallback_table != nullptr, "should be initialized"); + FallbackTableLookup lookup_f(from); + ForwardingEntry entry(from, to); + auto found_f = [&](ForwardingEntry* found) { + // If dupe has been found, override it with new value. + // This is also called when new entry is succussfully inserted. + if (found->_to != to) { + found->_to = to; + } + }; + Thread* current_thread = Thread::current(); + bool grow; + bool added = _fallback_table->insert_get(current_thread, lookup_f, entry, found_f, &grow); + NOT_PRODUCT(Atomic::inc(&_num_fallback_forwardings);) +#ifdef ASSERT + assert(fallback_forwardee(from) != nullptr, "must have entered forwarding"); + assert(fallback_forwardee(from) == to, "forwarding must be correct, added: %s, from: " PTR_FORMAT ", to: " PTR_FORMAT ", fwd: " PTR_FORMAT, BOOL_TO_STR(added), p2i(from), p2i(to), p2i(fallback_forwardee(from))); +#endif + if (grow) { + _fallback_table->grow(current_thread); + log_debug(gc)("grow fallback table to size: %zu bytes", _fallback_table->get_mem_size(current_thread)); + } +} + +template +HeapWord* FullGCForwardingImpl::fallback_forwardee(HeapWord* from) { + assert(_fallback_table != nullptr, "fallback table must be present"); + HeapWord* result; + FallbackTableLookup lookup_f(from); + auto found_f = [&](const ForwardingEntry* found) { + result = found->_to; + }; + bool found = _fallback_table->get(Thread::current(), lookup_f, found_f); + assert(found, "something must have been found"); + assert(result != nullptr, "must have found forwarding"); + return result; +} #endif // SHARE_GC_SHARED_FULLGCFORWARDING_INLINE_HPP From e3d719bc49c48031171ac571350a76dd2fe06512 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Mon, 14 Jul 2025 20:20:12 +0000 Subject: [PATCH 7/9] Fix some includes --- src/hotspot/share/gc/g1/g1CollectedHeap.cpp | 2 +- src/hotspot/share/gc/serial/serialHeap.cpp | 2 +- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp index a7f147611a69c..b16e8970ef24e 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp @@ -76,7 +76,7 @@ #include "gc/g1/g1YoungGCAllocationFailureInjector.hpp" #include "gc/shared/classUnloadingContext.hpp" #include "gc/shared/concurrentGCBreakpoints.hpp" -#include "gc/shared/fullGCForwarding.hpp" +#include "gc/shared/fullGCForwarding.inline.hpp" #include "gc/shared/gcBehaviours.hpp" #include "gc/shared/gcHeapSummary.hpp" #include "gc/shared/gcId.hpp" diff --git a/src/hotspot/share/gc/serial/serialHeap.cpp b/src/hotspot/share/gc/serial/serialHeap.cpp index fbbaabf618aea..02914309d1251 100644 --- a/src/hotspot/share/gc/serial/serialHeap.cpp +++ b/src/hotspot/share/gc/serial/serialHeap.cpp @@ -39,7 +39,7 @@ #include "gc/shared/collectedHeap.inline.hpp" #include "gc/shared/collectorCounters.hpp" #include "gc/shared/continuationGCSupport.inline.hpp" -#include "gc/shared/fullGCForwarding.hpp" +#include "gc/shared/fullGCForwarding.inline.hpp" #include "gc/shared/gcId.hpp" #include "gc/shared/gcInitLogger.hpp" #include "gc/shared/gcLocker.inline.hpp" diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 50881a5077833..1dcafe4c11820 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -28,7 +28,7 @@ #include "cds/archiveHeapWriter.hpp" #include "classfile/systemDictionary.hpp" #include "gc/shared/classUnloadingContext.hpp" -#include "gc/shared/fullGCForwarding.hpp" +#include "gc/shared/fullGCForwarding.inline.hpp" #include "gc/shared/gcArguments.hpp" #include "gc/shared/gcTimer.hpp" #include "gc/shared/gcTraceTime.inline.hpp" From 8f2457615bb405307926a44a439315f956d0dac4 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Tue, 15 Jul 2025 08:35:56 +0000 Subject: [PATCH 8/9] Fix 32 bit builds (hopefully) --- src/hotspot/share/gc/shared/fullGCForwarding.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/hotspot/share/gc/shared/fullGCForwarding.hpp b/src/hotspot/share/gc/shared/fullGCForwarding.hpp index c24c17307860f..6682db2c2bea8 100644 --- a/src/hotspot/share/gc/shared/fullGCForwarding.hpp +++ b/src/hotspot/share/gc/shared/fullGCForwarding.hpp @@ -192,6 +192,12 @@ class FullGCForwardingImpl : public AllStatic { static oop forwardee(oop from); }; +#ifdef _LP64 using FullGCForwarding = FullGCForwardingImpl; +#else +// On 32 bit, the BITS template argument is not used, but we still need +// to pass a value. +using FullGCForwarding = FullGCForwardingImpl<0>; +#endif #endif // SHARE_GC_SHARED_FULLGCFORWARDING_HPP From 969409f2e03abdfbf18e139014af1894e2d176b7 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Tue, 15 Jul 2025 10:38:31 +0000 Subject: [PATCH 9/9] Fix 32 bit builds (hopefully) --- test/hotspot/gtest/gc/shared/test_fullGCForwarding.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/hotspot/gtest/gc/shared/test_fullGCForwarding.cpp b/test/hotspot/gtest/gc/shared/test_fullGCForwarding.cpp index bd7198e0e36e3..ac18a8dfcbe58 100644 --- a/test/hotspot/gtest/gc/shared/test_fullGCForwarding.cpp +++ b/test/hotspot/gtest/gc/shared/test_fullGCForwarding.cpp @@ -41,7 +41,7 @@ class FullGCForwardingTest : public testing::Test { // Real size of fake heap, considering alignment. static const int FAKE_HEAP_SIZE_UNALIGNED = FAKE_HEAP_SIZE + FAKE_HEAP_ALIGNMENT; // Bit-pattern which must not change. - static const uintptr_t BIT_PATTERN = 0xA5A5A5A5A5A5A5A0; + static const uintptr_t BIT_PATTERN = LP64_ONLY(0xA5A5A5A5A5A5A5A0) NOT_LP64(0xA5A5A5A0); // Number of bits used for forwarding. static const int NUM_FWD_BITS = 4; // Forwarding bit mask.