From aa5e987645893e630b5bc3079572399c119e362b Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Wed, 7 Feb 2024 19:25:19 +0100 Subject: [PATCH] 8305898: Alternative self-forwarding mechanism --- .../share/gc/g1/g1OopClosures.inline.hpp | 2 +- .../share/gc/g1/g1ParScanThreadState.cpp | 4 +-- .../share/gc/parallel/psPromotionManager.cpp | 2 +- .../gc/parallel/psPromotionManager.inline.hpp | 2 +- .../share/gc/serial/defNewGeneration.cpp | 2 +- src/hotspot/share/oops/markWord.hpp | 25 ++++++++++----- src/hotspot/share/oops/oop.hpp | 3 ++ src/hotspot/share/oops/oop.inline.hpp | 31 +++++++++++++++++-- 8 files changed, 55 insertions(+), 16 deletions(-) diff --git a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp index ea7f690d325fc..3a670f9fd965a 100644 --- a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp +++ b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp @@ -228,7 +228,7 @@ void G1ParCopyClosure::do_oop_work(T* p) { oop forwardee; markWord m = obj->mark(); if (m.is_marked()) { - forwardee = cast_to_oop(m.decode_pointer()); + forwardee = obj->forwardee(); } else { forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m); } diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp index f9fc1fd019814..592ceefbce1eb 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp @@ -212,7 +212,7 @@ void G1ParScanThreadState::do_oop_evac(T* p) { markWord m = obj->mark(); if (m.is_marked()) { - obj = cast_to_oop(m.decode_pointer()); + obj = obj->forwardee(); } else { obj = do_copy_to_survivor_space(region_attr, obj, m); } @@ -627,7 +627,7 @@ NOINLINE oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, size_t word_sz) { assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old)); - oop forward_ptr = old->forward_to_atomic(old, m, memory_order_relaxed); + oop forward_ptr = old->forward_to_self_atomic(m, memory_order_relaxed); if (forward_ptr == nullptr) { // Forward-to-self succeeded. We are the "owner" of the object. HeapRegion* r = _g1h->heap_region_containing(old); diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.cpp b/src/hotspot/share/gc/parallel/psPromotionManager.cpp index 7610a6e13d77b..2194ab8e59270 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.cpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.cpp @@ -325,7 +325,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) { // this started. If it is the same (i.e., no forwarding // pointer has been installed), then this thread owns // it. - if (obj->forward_to_atomic(obj, obj_mark) == nullptr) { + if (obj->forward_to_self_atomic(obj_mark) == nullptr) { // We won any races, we "own" this object. assert(obj == obj->forwardee(), "Sanity"); diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp index c1cbeb0f597bb..b65eaa39674b6 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp @@ -152,7 +152,7 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) { // other thread. OrderAccess::acquire(); // Return the already installed forwardee. - return cast_to_oop(m.decode_pointer()); + return o->forwardee(m); } } diff --git a/src/hotspot/share/gc/serial/defNewGeneration.cpp b/src/hotspot/share/gc/serial/defNewGeneration.cpp index 3a012b639fde0..1b6b8d88202d6 100644 --- a/src/hotspot/share/gc/serial/defNewGeneration.cpp +++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp @@ -900,7 +900,7 @@ void DefNewGeneration::handle_promotion_failure(oop old) { ContinuationGCSupport::transform_stack_chunk(old); // forward to self - old->forward_to(old); + old->forward_to_self(); _promo_failure_scan_stack.push(old); diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp index be85fb8ba82ad..9d7d06d0fa66e 100644 --- a/src/hotspot/share/oops/markWord.hpp +++ b/src/hotspot/share/oops/markWord.hpp @@ -37,11 +37,11 @@ // // 32 bits: // -------- -// hash:25 ------------>| age:4 unused_gap:1 lock:2 (normal object) +// hash:25 ------------>| age:4 self-fwd:1 lock:2 (normal object) // // 64 bits: // -------- -// unused:25 hash:31 -->| unused_gap:1 age:4 unused_gap:1 lock:2 (normal object) +// unused:25 hash:31 -->| unused_gap:1 age:4 self-fwd:1 lock:2 (normal object) // // - hash contains the identity hash value: largest value is // 31 bits, see os::random(). Also, 64-bit vm's require @@ -103,17 +103,20 @@ class markWord { // Constants static const int age_bits = 4; static const int lock_bits = 2; - static const int first_unused_gap_bits = 1; - static const int max_hash_bits = BitsPerWord - age_bits - lock_bits - first_unused_gap_bits; + static const int self_forwarded_bits = 1; + static const int max_hash_bits = BitsPerWord - age_bits - lock_bits - self_forwarded_bits; static const int hash_bits = max_hash_bits > 31 ? 31 : max_hash_bits; - static const int second_unused_gap_bits = LP64_ONLY(1) NOT_LP64(0); + static const int unused_gap_bits = LP64_ONLY(1) NOT_LP64(0); static const int lock_shift = 0; - static const int age_shift = lock_bits + first_unused_gap_bits; - static const int hash_shift = age_shift + age_bits + second_unused_gap_bits; + static const int self_forwarded_shift = lock_shift + lock_bits; + static const int age_shift = self_forwarded_shift + self_forwarded_bits; + static const int hash_shift = age_shift + age_bits + unused_gap_bits; static const uintptr_t lock_mask = right_n_bits(lock_bits); static const uintptr_t lock_mask_in_place = lock_mask << lock_shift; + static const uintptr_t self_forwarded_mask = right_n_bits(self_forwarded_bits); + static const uintptr_t self_forwarded_mask_in_place = self_forwarded_mask << self_forwarded_shift; static const uintptr_t age_mask = right_n_bits(age_bits); static const uintptr_t age_mask_in_place = age_mask << age_shift; static const uintptr_t hash_mask = right_n_bits(hash_bits); @@ -260,6 +263,14 @@ class markWord { // Recover address of oop from encoded form used in mark inline void* decode_pointer() { return (void*)clear_lock_bits().value(); } + + inline bool self_forwarded() const { + return mask_bits(value(), self_forwarded_mask_in_place) != 0; + } + + inline markWord set_self_forwarded() const { + return markWord(value() | self_forwarded_mask_in_place | marked_value); + } }; // Support atomic operations. diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index 50d8b6041b674..bab00194c7dd4 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -260,14 +260,17 @@ class oopDesc { inline bool is_forwarded() const; inline void forward_to(oop p); + inline void forward_to_self(); // Like "forward_to", but inserts the forwarding pointer atomically. // Exactly one thread succeeds in inserting the forwarding pointer, and // this call returns null for that thread; any other thread has the // value of the forwarding pointer returned and does not modify "this". inline oop forward_to_atomic(oop p, markWord compare, atomic_memory_order order = memory_order_conservative); + inline oop forward_to_self_atomic(markWord compare, atomic_memory_order order = memory_order_conservative); inline oop forwardee() const; + inline oop forwardee(markWord header) const; // Age of object during scavenge inline uint age() const; diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index a9b88e32651d8..e934933ec7657 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -275,14 +275,39 @@ void oopDesc::forward_to(oop p) { set_mark(m); } +void oopDesc::forward_to_self() { + set_mark(markWord::prototype().set_self_forwarded()); +} + oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) { markWord m = markWord::encode_pointer_as_mark(p); - assert(m.decode_pointer() == p, "encoding must be reversible"); + assert(forwardee(m) == p, "encoding must be reversible"); markWord old_mark = cas_set_mark(m, compare, order); if (old_mark == compare) { return nullptr; } else { - return cast_to_oop(old_mark.decode_pointer()); + assert(old_mark.is_marked(), "must be marked here"); + return forwardee(old_mark); + } +} + +oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order) { + markWord new_mark = markWord::prototype().set_self_forwarded(); + assert(forwardee(new_mark) == cast_to_oop(this), "encoding must be reversible"); + markWord old_mark = cas_set_mark(new_mark, compare, order); + if (old_mark == compare) { + return nullptr; + } else { + assert(old_mark.is_marked(), "must be marked here"); + return forwardee(old_mark); + } +} +oop oopDesc::forwardee(markWord mark) const { + assert(mark.is_marked(), "only decode when actually forwarded"); + if (mark.self_forwarded()) { + return cast_to_oop(this); + } else { + return cast_to_oop(mark.decode_pointer()); } } @@ -291,7 +316,7 @@ oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order orde // It does need to clear the low two locking- and GC-related bits. oop oopDesc::forwardee() const { assert(is_forwarded(), "only decode when actually forwarded"); - return cast_to_oop(mark().decode_pointer()); + return forwardee(mark()); } // The following method needs to be MT safe.