diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp index fd73b725a1289..f9da082795083 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp @@ -85,7 +85,6 @@ #include "gc/shared/isGCActiveMark.hpp" #include "gc/shared/locationPrinter.inline.hpp" #include "gc/shared/oopStorageParState.hpp" -#include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/referenceProcessor.inline.hpp" #include "gc/shared/suspendibleThreadSet.hpp" #include "gc/shared/taskqueue.inline.hpp" diff --git a/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp b/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp index 32a56d7120569..4819cbfed7074 100644 --- a/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp +++ b/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp @@ -105,7 +105,6 @@ G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) : _gc_par_phases[UpdateDerivedPointers] = new WorkerDataArray("UpdateDerivedPointers", "Update Derived Pointers (ms):", max_gc_threads); #endif _gc_par_phases[EagerlyReclaimHumongousObjects] = new WorkerDataArray("EagerlyReclaimHumongousObjects", "Eagerly Reclaim Humongous Objects (ms):", max_gc_threads); - _gc_par_phases[RestorePreservedMarks] = new WorkerDataArray("RestorePreservedMarks", "Restore Preserved Marks (ms):", max_gc_threads); _gc_par_phases[ProcessEvacuationFailedRegions] = new WorkerDataArray("ProcessEvacuationFailedRegions", "Process Evacuation Failed Regions (ms):", max_gc_threads); _gc_par_phases[ScanHR]->create_thread_work_items("Scanned Cards:", ScanHRScannedCards); @@ -512,7 +511,6 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set(bool evacuation_failed debug_time("Post Evacuate Cleanup 2", _cur_post_evacuate_cleanup_2_time_ms); if (evacuation_failed) { debug_phase(_gc_par_phases[RecalculateUsed], 1); - debug_phase(_gc_par_phases[RestorePreservedMarks], 1); debug_phase(_gc_par_phases[ProcessEvacuationFailedRegions], 1); } #if COMPILER2_OR_JVMCI diff --git a/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp b/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp index 40abfd605339f..171efa7be778d 100644 --- a/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp +++ b/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp @@ -87,7 +87,6 @@ class G1GCPhaseTimes : public CHeapObj { UpdateDerivedPointers, #endif EagerlyReclaimHumongousObjects, - RestorePreservedMarks, ProcessEvacuationFailedRegions, ResetMarkingState, NoteStartOfMark, diff --git a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp index 26cc88de0beb5..5b3bbedfeb289 100644 --- a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp +++ b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp @@ -228,7 +228,7 @@ void G1ParCopyClosure::do_oop_work(T* p) { oop forwardee; markWord m = obj->mark(); if (m.is_forwarded()) { - forwardee = m.forwardee(); + forwardee = obj->forwardee(m); } else { forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m); } diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp index f81c3241a1a90..7bd05bc66ebd1 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp @@ -37,7 +37,6 @@ #include "gc/shared/continuationGCSupport.inline.hpp" #include "gc/shared/partialArrayState.hpp" #include "gc/shared/partialArrayTaskStepper.inline.hpp" -#include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/stringdedup/stringDedup.hpp" #include "gc/shared/taskqueue.inline.hpp" #include "memory/allocation.inline.hpp" @@ -59,7 +58,6 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, G1RedirtyCardsQueueSet* rdcqs, - PreservedMarks* preserved_marks, uint worker_id, uint num_workers, G1CollectionSet* collection_set, @@ -90,7 +88,6 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h, _numa(g1h->numa()), _obj_alloc_stat(nullptr), ALLOCATION_FAILURE_INJECTOR_ONLY(_allocation_failure_inject_counter(0) COMMA) - _preserved_marks(preserved_marks), _evacuation_failed_info(), _evac_failure_regions(evac_failure_regions), _evac_failure_enqueued_cards(0) @@ -216,7 +213,7 @@ void G1ParScanThreadState::do_oop_evac(T* p) { markWord m = obj->mark(); if (m.is_forwarded()) { - obj = m.forwardee(); + obj = obj->forwardee(m); } else { obj = do_copy_to_survivor_space(region_attr, obj, m); } @@ -595,7 +592,6 @@ G1ParScanThreadState* G1ParScanThreadStateSet::state_for_worker(uint worker_id) if (_states[worker_id] == nullptr) { _states[worker_id] = new G1ParScanThreadState(_g1h, rdcqs(), - _preserved_marks_set.get(worker_id), worker_id, _num_workers, _collection_set, @@ -655,7 +651,7 @@ NOINLINE oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, size_t word_sz, bool cause_pinned) { assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old)); - oop forward_ptr = old->forward_to_atomic(old, m, memory_order_relaxed); + oop forward_ptr = old->forward_to_self_atomic(m, memory_order_relaxed); if (forward_ptr == nullptr) { // Forward-to-self succeeded. We are the "owner" of the object. G1HeapRegion* r = _g1h->heap_region_containing(old); @@ -668,8 +664,6 @@ oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, siz // evacuation failure recovery. _g1h->mark_evac_failure_object(_worker_id, old, word_sz); - _preserved_marks->push_if_necessary(old, m); - ContinuationGCSupport::transform_stack_chunk(old); _evacuation_failed_info.register_copy_failure(word_sz); @@ -727,7 +721,6 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h, _g1h(g1h), _collection_set(collection_set), _rdcqs(G1BarrierSet::dirty_card_queue_set().allocator()), - _preserved_marks_set(true /* in_c_heap */), _states(NEW_C_HEAP_ARRAY(G1ParScanThreadState*, num_workers, mtGC)), _rdc_buffers(NEW_C_HEAP_ARRAY(BufferNodeList, num_workers, mtGC)), _surviving_young_words_total(NEW_C_HEAP_ARRAY(size_t, collection_set->young_region_length() + 1, mtGC)), @@ -736,7 +729,6 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h, _evac_failure_regions(evac_failure_regions), _partial_array_state_allocator(num_workers) { - _preserved_marks_set.init(num_workers); for (uint i = 0; i < num_workers; ++i) { _states[i] = nullptr; _rdc_buffers[i] = BufferNodeList(); @@ -749,5 +741,4 @@ G1ParScanThreadStateSet::~G1ParScanThreadStateSet() { FREE_C_HEAP_ARRAY(G1ParScanThreadState*, _states); FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_total); FREE_C_HEAP_ARRAY(BufferNodeList, _rdc_buffers); - _preserved_marks_set.reclaim(); } diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp index 1cfd6fca08a6f..561d7e7dc8581 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp @@ -34,7 +34,6 @@ #include "gc/shared/gc_globals.hpp" #include "gc/shared/partialArrayState.hpp" #include "gc/shared/partialArrayTaskStepper.hpp" -#include "gc/shared/preservedMarks.hpp" #include "gc/shared/stringdedup/stringDedup.hpp" #include "gc/shared/taskqueue.hpp" #include "memory/allocation.hpp" @@ -48,8 +47,6 @@ class G1EvacuationRootClosures; class G1OopStarChunkedList; class G1PLABAllocator; class G1HeapRegion; -class PreservedMarks; -class PreservedMarksSet; class outputStream; class G1ParScanThreadState : public CHeapObj { @@ -108,7 +105,6 @@ class G1ParScanThreadState : public CHeapObj { // Per-thread evacuation failure data structures. ALLOCATION_FAILURE_INJECTOR_ONLY(size_t _allocation_failure_inject_counter;) - PreservedMarks* _preserved_marks; EvacuationFailedInfo _evacuation_failed_info; G1EvacFailureRegions* _evac_failure_regions; // Number of additional cards into evacuation failed regions enqueued into @@ -127,7 +123,6 @@ class G1ParScanThreadState : public CHeapObj { public: G1ParScanThreadState(G1CollectedHeap* g1h, G1RedirtyCardsQueueSet* rdcqs, - PreservedMarks* preserved_marks, uint worker_id, uint num_workers, G1CollectionSet* collection_set, @@ -248,7 +243,6 @@ class G1ParScanThreadStateSet : public StackObj { G1CollectedHeap* _g1h; G1CollectionSet* _collection_set; G1RedirtyCardsQueueSet _rdcqs; - PreservedMarksSet _preserved_marks_set; G1ParScanThreadState** _states; BufferNodeList* _rdc_buffers; size_t* _surviving_young_words_total; @@ -266,7 +260,6 @@ class G1ParScanThreadStateSet : public StackObj { G1RedirtyCardsQueueSet* rdcqs() { return &_rdcqs; } BufferNodeList* rdc_buffers() { return _rdc_buffers; } - PreservedMarksSet* preserved_marks_set() { return &_preserved_marks_set; } void flush_stats(); void record_unused_optional_region(G1HeapRegion* hr); diff --git a/src/hotspot/share/gc/g1/g1YoungCollector.cpp b/src/hotspot/share/gc/g1/g1YoungCollector.cpp index f2fe93015c532..f3590aa2ff696 100644 --- a/src/hotspot/share/gc/g1/g1YoungCollector.cpp +++ b/src/hotspot/share/gc/g1/g1YoungCollector.cpp @@ -53,7 +53,6 @@ #include "gc/shared/gcTraceTime.inline.hpp" #include "gc/shared/gcTimer.hpp" #include "gc/shared/gc_globals.hpp" -#include "gc/shared/preservedMarks.hpp" #include "gc/shared/referenceProcessor.hpp" #include "gc/shared/weakProcessor.inline.hpp" #include "gc/shared/workerPolicy.hpp" diff --git a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp index a0e9a9b1569ab..6b218d3971e8b 100644 --- a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp +++ b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp @@ -42,7 +42,6 @@ #include "gc/g1/g1RemSet.hpp" #include "gc/g1/g1YoungGCPostEvacuateTasks.hpp" #include "gc/shared/bufferNode.hpp" -#include "gc/shared/preservedMarks.inline.hpp" #include "jfr/jfrEvents.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" @@ -252,7 +251,7 @@ class G1PostEvacuateCollectionSetCleanupTask1::RestoreEvacFailureRegionsTask : p { // Process marked object. assert(obj->is_forwarded() && obj->forwardee() == obj, "must be self-forwarded"); - obj->init_mark(); + obj->unset_self_forwarded(); hr->update_bot_for_block(obj_addr, obj_end_addr); // Statistics @@ -477,27 +476,6 @@ class G1PostEvacuateCollectionSetCleanupTask2::EagerlyReclaimHumongousObjectsTas } }; -class G1PostEvacuateCollectionSetCleanupTask2::RestorePreservedMarksTask : public G1AbstractSubTask { - PreservedMarksSet* _preserved_marks; - WorkerTask* _task; - -public: - RestorePreservedMarksTask(PreservedMarksSet* preserved_marks) : - G1AbstractSubTask(G1GCPhaseTimes::RestorePreservedMarks), - _preserved_marks(preserved_marks), - _task(preserved_marks->create_task()) { } - - virtual ~RestorePreservedMarksTask() { - delete _task; - } - - double worker_cost() const override { - return _preserved_marks->num(); - } - - void do_work(uint worker_id) override { _task->work(worker_id); } -}; - class RedirtyLoggedCardTableEntryClosure : public G1CardTableEntryClosure { size_t _num_dirtied; G1CollectedHeap* _g1h; @@ -979,7 +957,6 @@ G1PostEvacuateCollectionSetCleanupTask2::G1PostEvacuateCollectionSetCleanupTask2 } if (evac_failure_regions->has_regions_evac_failed()) { - add_parallel_task(new RestorePreservedMarksTask(per_thread_states->preserved_marks_set())); add_parallel_task(new ProcessEvacuationFailedRegionsTask(evac_failure_regions)); } add_parallel_task(new RedirtyLoggedCardsTask(evac_failure_regions, diff --git a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp index 868ab788b534a..c79a8dd8eb8a1 100644 --- a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp +++ b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp @@ -56,7 +56,6 @@ class G1PostEvacuateCollectionSetCleanupTask1 : public G1BatchedTask { // - Update Derived Pointers (s) // - Clear Retained Region Data (on evacuation failure) // - Redirty Logged Cards -// - Restore Preserved Marks (on evacuation failure) // - Free Collection Set // - Resize TLABs class G1PostEvacuateCollectionSetCleanupTask2 : public G1BatchedTask { @@ -67,7 +66,6 @@ class G1PostEvacuateCollectionSetCleanupTask2 : public G1BatchedTask { class ProcessEvacuationFailedRegionsTask; class RedirtyLoggedCardsTask; - class RestorePreservedMarksTask; class FreeCollectionSetTask; class ResizeTLABsTask; diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.cpp b/src/hotspot/share/gc/parallel/psPromotionManager.cpp index 19f688852385b..b055e0a4d1fac 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.cpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.cpp @@ -319,7 +319,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) { // this started. If it is the same (i.e., no forwarding // pointer has been installed), then this thread owns // it. - if (obj->forward_to_atomic(obj, obj_mark) == nullptr) { + if (obj->forward_to_self_atomic(obj_mark) == nullptr) { // We won any races, we "own" this object. assert(obj == obj->forwardee(), "Sanity"); diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp index 390dea4976d19..f152b225bb5a3 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp @@ -149,7 +149,7 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) { return copy_unmarked_to_survivor_space(o, m); } else { // Return the already installed forwardee. - return m.forwardee(); + return o->forwardee(m); } } diff --git a/src/hotspot/share/gc/serial/defNewGeneration.cpp b/src/hotspot/share/gc/serial/defNewGeneration.cpp index 047171a5eb364..b5b46ee2474de 100644 --- a/src/hotspot/share/gc/serial/defNewGeneration.cpp +++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp @@ -39,7 +39,6 @@ #include "gc/shared/gcTimer.hpp" #include "gc/shared/gcTrace.hpp" #include "gc/shared/gcTraceTime.inline.hpp" -#include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/referencePolicy.hpp" #include "gc/shared/referenceProcessorPhaseTimes.hpp" #include "gc/shared/space.hpp" @@ -227,7 +226,6 @@ DefNewGeneration::DefNewGeneration(ReservedSpace rs, const char* policy) : Generation(rs, initial_size), _promotion_failed(false), - _preserved_marks_set(false /* in_c_heap */), _promo_failure_drain_in_progress(false), _string_dedup_requests() { @@ -609,8 +607,6 @@ bool DefNewGeneration::collect(bool clear_all_soft_refs) { age_table()->clear(); to()->clear(SpaceDecorator::Mangle); - // The preserved marks should be empty at the start of the GC. - _preserved_marks_set.init(1); YoungGenScanClosure young_gen_cl(this); OldGenScanClosure old_gen_cl(this); @@ -681,8 +677,6 @@ bool DefNewGeneration::collect(bool clear_all_soft_refs) { // Reset the PromotionFailureALot counters. NOT_PRODUCT(heap->reset_promotion_should_fail();) } - // We should have processed and cleared all the preserved marks. - _preserved_marks_set.reclaim(); heap->trace_heap_after_gc(_gc_tracer); @@ -706,19 +700,15 @@ void DefNewGeneration::remove_forwarding_pointers() { // starts. (The mark word is overloaded: `is_marked()` == `is_forwarded()`.) struct ResetForwardedMarkWord : ObjectClosure { void do_object(oop obj) override { - if (obj->is_forwarded()) { + if (obj->is_self_forwarded()) { + obj->unset_self_forwarded(); + } else if (obj->is_forwarded()) { obj->init_mark(); } } } cl; eden()->object_iterate(&cl); from()->object_iterate(&cl); - - restore_preserved_marks(); -} - -void DefNewGeneration::restore_preserved_marks() { - _preserved_marks_set.restore(nullptr); } void DefNewGeneration::handle_promotion_failure(oop old) { @@ -726,12 +716,11 @@ void DefNewGeneration::handle_promotion_failure(oop old) { _promotion_failed = true; _promotion_failed_info.register_copy_failure(old->size()); - _preserved_marks_set.get()->push_if_necessary(old, old->mark()); ContinuationGCSupport::transform_stack_chunk(old); // forward to self - old->forward_to(old); + old->forward_to_self(); _promo_failure_scan_stack.push(old); diff --git a/src/hotspot/share/gc/serial/defNewGeneration.hpp b/src/hotspot/share/gc/serial/defNewGeneration.hpp index c5b7c095ac4e6..ee85ec5cd89f8 100644 --- a/src/hotspot/share/gc/serial/defNewGeneration.hpp +++ b/src/hotspot/share/gc/serial/defNewGeneration.hpp @@ -32,7 +32,6 @@ #include "gc/shared/copyFailedInfo.hpp" #include "gc/shared/gc_globals.hpp" #include "gc/shared/generationCounters.hpp" -#include "gc/shared/preservedMarks.hpp" #include "gc/shared/stringdedup/stringDedup.hpp" #include "gc/shared/tlab_globals.hpp" #include "utilities/align.hpp" @@ -99,11 +98,6 @@ class DefNewGeneration: public Generation { // therefore we must remove their forwarding pointers. void remove_forwarding_pointers(); - virtual void restore_preserved_marks(); - - // Preserved marks - PreservedMarksSet _preserved_marks_set; - Stack _promo_failure_scan_stack; void drain_promo_failure_scan_stack(void); bool _promo_failure_drain_in_progress; diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp index 12d6ee73acf94..42c9318298f2e 100644 --- a/src/hotspot/share/oops/markWord.hpp +++ b/src/hotspot/share/oops/markWord.hpp @@ -37,11 +37,11 @@ // // 32 bits: // -------- -// hash:25 ------------>| age:4 unused_gap:1 lock:2 (normal object) +// hash:25 ------------>| age:4 self-fwd:1 lock:2 (normal object) // // 64 bits: // -------- -// unused:25 hash:31 -->| unused_gap:1 age:4 unused_gap:1 lock:2 (normal object) +// unused:25 hash:31 -->| unused_gap:1 age:4 self-fwd:1 lock:2 (normal object) // // - hash contains the identity hash value: largest value is // 31 bits, see os::random(). Also, 64-bit vm's require @@ -103,17 +103,20 @@ class markWord { // Constants static const int age_bits = 4; static const int lock_bits = 2; - static const int first_unused_gap_bits = 1; - static const int max_hash_bits = BitsPerWord - age_bits - lock_bits - first_unused_gap_bits; + static const int self_fwd_bits = 1; + static const int max_hash_bits = BitsPerWord - age_bits - lock_bits - self_fwd_bits; static const int hash_bits = max_hash_bits > 31 ? 31 : max_hash_bits; - static const int second_unused_gap_bits = LP64_ONLY(1) NOT_LP64(0); + static const int unused_gap_bits = LP64_ONLY(1) NOT_LP64(0); static const int lock_shift = 0; - static const int age_shift = lock_bits + first_unused_gap_bits; - static const int hash_shift = age_shift + age_bits + second_unused_gap_bits; + static const int self_fwd_shift = lock_shift + lock_bits; + static const int age_shift = self_fwd_shift + self_fwd_bits; + static const int hash_shift = age_shift + age_bits + unused_gap_bits; static const uintptr_t lock_mask = right_n_bits(lock_bits); static const uintptr_t lock_mask_in_place = lock_mask << lock_shift; + static const uintptr_t self_fwd_mask = right_n_bits(self_fwd_bits); + static const uintptr_t self_fwd_mask_in_place = self_fwd_mask << self_fwd_shift; static const uintptr_t age_mask = right_n_bits(age_bits); static const uintptr_t age_mask_in_place = age_mask << age_shift; static const uintptr_t hash_mask = right_n_bits(hash_bits); @@ -143,9 +146,11 @@ class markWord { bool is_marked() const { return (mask_bits(value(), lock_mask_in_place) == marked_value); } - bool is_forwarded() const { - return (mask_bits(value(), lock_mask_in_place) == marked_value); + bool is_forwarded() const { + // Returns true for normal forwarded (0b011) and self-forwarded (0b1xx). + return mask_bits(value(), lock_mask_in_place | self_fwd_mask_in_place) >= static_cast(marked_value); } + bool is_neutral() const { // Not locked, or marked - a "clean" neutral state return (mask_bits(value(), lock_mask_in_place) == unlocked_value); } @@ -264,6 +269,21 @@ class markWord { // Recover address of oop from encoded form used in mark inline void* decode_pointer() const { return (void*)clear_lock_bits().value(); } + inline bool is_self_forwarded() const { + NOT_LP64(assert(LockingMode != LM_LEGACY, "incorrect with LM_LEGACY on 32 bit");) + return mask_bits(value(), self_fwd_mask_in_place) != 0; + } + + inline markWord set_self_forwarded() const { + NOT_LP64(assert(LockingMode != LM_LEGACY, "incorrect with LM_LEGACY on 32 bit");) + return markWord(value() | self_fwd_mask_in_place); + } + + inline markWord unset_self_forwarded() const { + NOT_LP64(assert(LockingMode != LM_LEGACY, "incorrect with LM_LEGACY on 32 bit");) + return markWord(value() & ~self_fwd_mask_in_place); + } + inline oop forwardee() const { return cast_to_oop(decode_pointer()); } diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index 2d827699b41fa..b84530700f68e 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -62,6 +62,8 @@ class oopDesc { // make use of the C++ copy/assign incorrect. NONCOPYABLE(oopDesc); + inline oop cas_set_forwardee(markWord new_mark, markWord old_mark, atomic_memory_order order); + public: // Must be trivial; see verifying static assert after the class. oopDesc() = default; @@ -258,16 +260,22 @@ class oopDesc { // Forward pointer operations for scavenge inline bool is_forwarded() const; + inline bool is_self_forwarded() const; inline void forward_to(oop p); + inline void forward_to_self(); // Like "forward_to", but inserts the forwarding pointer atomically. // Exactly one thread succeeds in inserting the forwarding pointer, and // this call returns null for that thread; any other thread has the // value of the forwarding pointer returned and does not modify "this". inline oop forward_to_atomic(oop p, markWord compare, atomic_memory_order order = memory_order_conservative); + inline oop forward_to_self_atomic(markWord compare, atomic_memory_order order = memory_order_conservative); inline oop forwardee() const; + inline oop forwardee(markWord header) const; + + inline void unset_self_forwarded(); // Age of object during scavenge inline uint age() const; diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index ffcc9af24c328..eea6ee604019a 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -267,6 +267,10 @@ bool oopDesc::is_forwarded() const { return mark().is_forwarded(); } +bool oopDesc::is_self_forwarded() const { + return mark().is_self_forwarded(); +} + // Used by scavengers void oopDesc::forward_to(oop p) { markWord m = markWord::encode_pointer_as_mark(p); @@ -274,14 +278,38 @@ void oopDesc::forward_to(oop p) { set_mark(m); } -oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) { - markWord m = markWord::encode_pointer_as_mark(p); - assert(m.decode_pointer() == p, "encoding must be reversible"); - markWord old_mark = cas_set_mark(m, compare, order); +void oopDesc::forward_to_self() { + set_mark(mark().set_self_forwarded()); +} + +oop oopDesc::cas_set_forwardee(markWord new_mark, markWord compare, atomic_memory_order order) { + markWord old_mark = cas_set_mark(new_mark, compare, order); if (old_mark == compare) { return nullptr; } else { - return cast_to_oop(old_mark.decode_pointer()); + assert(old_mark.is_forwarded(), "must be forwarded here"); + return forwardee(old_mark); + } +} + +oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) { + markWord m = markWord::encode_pointer_as_mark(p); + assert(forwardee(m) == p, "encoding must be reversible"); + return cas_set_forwardee(m, compare, order); +} + +oop oopDesc::forward_to_self_atomic(markWord old_mark, atomic_memory_order order) { + markWord new_mark = old_mark.set_self_forwarded(); + assert(forwardee(new_mark) == cast_to_oop(this), "encoding must be reversible"); + return cas_set_forwardee(new_mark, old_mark, order); +} + +oop oopDesc::forwardee(markWord mark) const { + assert(mark.is_forwarded(), "only decode when actually forwarded"); + if (mark.is_self_forwarded()) { + return cast_to_oop(this); + } else { + return mark.forwardee(); } } @@ -289,7 +317,11 @@ oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order orde // The forwardee is used when copying during scavenge and mark-sweep. // It does need to clear the low two locking- and GC-related bits. oop oopDesc::forwardee() const { - return mark().forwardee(); + return forwardee(mark()); +} + +void oopDesc::unset_self_forwarded() { + set_mark(mark().unset_self_forwarded()); } // The following method needs to be MT safe. diff --git a/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp b/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp index 5f9a361105ec7..7700a44e2cd92 100644 --- a/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp +++ b/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp @@ -29,7 +29,11 @@ // Class to create a "fake" oop with a mark that will // return true for calls to must_be_preserved(). class FakeOop { - oopDesc _oop; + // Align at least to 8 bytes, otherwise the lowest address bit + // could be interpreted as 'self-forwarded' when encoded as + // forwardee in the mark-word. This is required on 32 bit builds, + // where the oop could be 4-byte-aligned. + alignas(BytesPerLong) oopDesc _oop; public: FakeOop() : _oop() { _oop.set_mark(originalMark()); } diff --git a/test/hotspot/jtreg/gc/g1/TestGCLogMessages.java b/test/hotspot/jtreg/gc/g1/TestGCLogMessages.java index d37bf56738143..e407895107186 100644 --- a/test/hotspot/jtreg/gc/g1/TestGCLogMessages.java +++ b/test/hotspot/jtreg/gc/g1/TestGCLogMessages.java @@ -259,7 +259,6 @@ private void testConcurrentRefinementLogs() throws Exception { LogMessageWithLevel exhFailureMessages[] = new LogMessageWithLevel[] { new LogMessageWithLevel("Recalculate Used Memory \\(ms\\):", Level.DEBUG), - new LogMessageWithLevel("Restore Preserved Marks \\(ms\\):", Level.DEBUG), new LogMessageWithLevel("Restore Evacuation Failed Regions \\(ms\\):", Level.DEBUG), new LogMessageWithLevel("Process Evacuation Failed Regions \\(ms\\):", Level.DEBUG), new LogMessageWithLevel("Evacuation Failed Regions:", Level.DEBUG),