From b6eafc763c16f1ec259425d5ba629c3a815595d6 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Thu, 15 Aug 2024 15:48:21 -0700 Subject: [PATCH] 8305896O Alternative full GC forwarding --- src/hotspot/share/gc/g1/g1CollectedHeap.cpp | 3 + .../share/gc/g1/g1FullGCCompactTask.cpp | 11 +-- .../share/gc/g1/g1FullGCCompactionPoint.cpp | 11 +-- .../gc/g1/g1FullGCOopClosures.inline.hpp | 5 +- .../gc/g1/g1FullGCPrepareTask.inline.hpp | 5 +- .../gc/parallel/parallelScavengeHeap.cpp | 3 + .../share/gc/parallel/psParallelCompact.cpp | 9 +-- .../gc/parallel/psParallelCompact.inline.hpp | 3 +- src/hotspot/share/gc/serial/serialFullGC.cpp | 13 ++-- src/hotspot/share/gc/serial/serialHeap.cpp | 3 + src/hotspot/share/gc/shared/gcArguments.cpp | 2 + src/hotspot/share/gc/shared/gcForwarding.cpp | 54 ++++++++++++++ src/hotspot/share/gc/shared/gcForwarding.hpp | 49 +++++++++++++ .../share/gc/shared/gcForwarding.inline.hpp | 59 +++++++++++++++ .../share/gc/shared/preservedMarks.cpp | 5 +- .../share/gc/shenandoah/shenandoahFullGC.cpp | 17 ++--- .../share/gc/shenandoah/shenandoahHeap.cpp | 3 + .../gtest/gc/shared/test_preservedMarks.cpp | 71 ++++++++----------- 18 files changed, 248 insertions(+), 78 deletions(-) create mode 100644 src/hotspot/share/gc/shared/gcForwarding.cpp create mode 100644 src/hotspot/share/gc/shared/gcForwarding.hpp create mode 100644 src/hotspot/share/gc/shared/gcForwarding.inline.hpp diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp index f9da082795083..757fcd1c17691 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp @@ -78,6 +78,7 @@ #include "gc/shared/classUnloadingContext.hpp" #include "gc/shared/concurrentGCBreakpoints.hpp" #include "gc/shared/gcBehaviours.hpp" +#include "gc/shared/gcForwarding.hpp" #include "gc/shared/gcHeapSummary.hpp" #include "gc/shared/gcId.hpp" #include "gc/shared/gcTimer.hpp" @@ -1434,6 +1435,8 @@ jint G1CollectedHeap::initialize() { G1InitLogger::print(); + GCForwarding::initialize(heap_rs.region()); + return JNI_OK; } diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp index 05f669592433b..37c12ff127239 100644 --- a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp +++ b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp @@ -29,6 +29,7 @@ #include "gc/g1/g1FullGCCompactionPoint.hpp" #include "gc/g1/g1FullGCCompactTask.hpp" #include "gc/g1/g1HeapRegion.inline.hpp" +#include "gc/shared/gcForwarding.inline.hpp" #include "gc/shared/gcTraceTime.inline.hpp" #include "logging/log.hpp" #include "oops/oop.inline.hpp" @@ -41,7 +42,7 @@ void G1FullGCCompactTask::G1CompactRegionClosure::clear_in_bitmap(oop obj) { size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) { size_t size = obj->size(); - if (obj->is_forwarded()) { + if (GCForwarding::is_forwarded(obj)) { G1FullGCCompactTask::copy_object_to_new_location(obj); } @@ -52,13 +53,13 @@ size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) { } void G1FullGCCompactTask::copy_object_to_new_location(oop obj) { - assert(obj->is_forwarded(), "Sanity!"); - assert(obj->forwardee() != obj, "Object must have a new location"); + assert(GCForwarding::is_forwarded(obj), "Sanity!"); + assert(GCForwarding::forwardee(obj) != obj, "Object must have a new location"); size_t size = obj->size(); // Copy object and reinit its mark. HeapWord* obj_addr = cast_from_oop(obj); - HeapWord* destination = cast_from_oop(obj->forwardee()); + HeapWord* destination = cast_from_oop(GCForwarding::forwardee(obj)); Copy::aligned_conjoint_words(obj_addr, destination, size); // There is no need to transform stack chunks - marking already did that. @@ -121,7 +122,7 @@ void G1FullGCCompactTask::compact_humongous_obj(G1HeapRegion* src_hr) { size_t word_size = obj->size(); uint num_regions = (uint)G1CollectedHeap::humongous_obj_size_in_regions(word_size); - HeapWord* destination = cast_from_oop(obj->forwardee()); + HeapWord* destination = cast_from_oop(GCForwarding::forwardee(obj)); assert(collector()->mark_bitmap()->is_marked(obj), "Should only compact marked objects"); collector()->mark_bitmap()->clear(obj); diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp index 019484c810a54..3d6edd99a820a 100644 --- a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp +++ b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp @@ -26,6 +26,7 @@ #include "gc/g1/g1FullCollector.inline.hpp" #include "gc/g1/g1FullGCCompactionPoint.hpp" #include "gc/g1/g1HeapRegion.hpp" +#include "gc/shared/gcForwarding.inline.hpp" #include "gc/shared/preservedMarks.inline.hpp" #include "oops/oop.inline.hpp" #include "utilities/debug.hpp" @@ -106,10 +107,10 @@ void G1FullGCCompactionPoint::forward(oop object, size_t size) { if (!object->is_forwarded()) { preserved_stack()->push_if_necessary(object, object->mark()); } - object->forward_to(cast_to_oop(_compaction_top)); - assert(object->is_forwarded(), "must be forwarded"); + GCForwarding::forward_to(object, cast_to_oop(_compaction_top)); + assert(GCForwarding::is_forwarded(object), "must be forwarded"); } else { - assert(!object->is_forwarded(), "must not be forwarded"); + assert(!GCForwarding::is_forwarded(object), "must not be forwarded"); } // Update compaction values. @@ -172,8 +173,8 @@ void G1FullGCCompactionPoint::forward_humongous(G1HeapRegion* hr) { preserved_stack()->push_if_necessary(obj, obj->mark()); G1HeapRegion* dest_hr = _compaction_regions->at(range_begin); - obj->forward_to(cast_to_oop(dest_hr->bottom())); - assert(obj->is_forwarded(), "Object must be forwarded!"); + GCForwarding::forward_to(obj, cast_to_oop(dest_hr->bottom())); + assert(GCForwarding::is_forwarded(obj), "Object must be forwarded!"); // Add the humongous object regions to the compaction point. add_humongous(hr); diff --git a/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp index f10f884b24253..4c1d8541c1dbd 100644 --- a/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp @@ -32,6 +32,7 @@ #include "gc/g1/g1FullCollector.inline.hpp" #include "gc/g1/g1FullGCMarker.inline.hpp" #include "gc/g1/g1HeapRegionRemSet.inline.hpp" +#include "gc/shared/gcForwarding.inline.hpp" #include "memory/iterator.inline.hpp" #include "memory/universe.hpp" #include "oops/access.inline.hpp" @@ -65,8 +66,8 @@ template inline void G1AdjustClosure::adjust_pointer(T* p) { return; } - if (obj->is_forwarded()) { - oop forwardee = obj->forwardee(); + if (GCForwarding::is_forwarded(obj)) { + oop forwardee = GCForwarding::forwardee(obj); // Forwarded, just update. assert(G1CollectedHeap::heap()->is_in_reserved(forwardee), "should be in object space"); RawAccess::oop_store(p, forwardee); diff --git a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp index 2fb61cc593480..f102dd0dfc49b 100644 --- a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp @@ -32,6 +32,7 @@ #include "gc/g1/g1FullGCCompactionPoint.hpp" #include "gc/g1/g1FullGCScope.hpp" #include "gc/g1/g1HeapRegion.inline.hpp" +#include "gc/shared/gcForwarding.inline.hpp" void G1DetermineCompactionQueueClosure::free_empty_humongous_region(G1HeapRegion* hr) { _g1h->free_humongous_region(hr, nullptr); @@ -114,10 +115,10 @@ inline bool G1DetermineCompactionQueueClosure::do_heap_region(G1HeapRegion* hr) } inline size_t G1SerialRePrepareClosure::apply(oop obj) { - if (obj->is_forwarded()) { + if (GCForwarding::is_forwarded(obj)) { // We skip objects compiled into the first region or // into regions not part of the serial compaction point. - if (cast_from_oop(obj->forwardee()) < _dense_prefix_top) { + if (cast_from_oop(GCForwarding::forwardee(obj)) < _dense_prefix_top) { return obj->size(); } } diff --git a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp index 5883b1cd6074d..113af051cdf61 100644 --- a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp +++ b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp @@ -33,6 +33,7 @@ #include "gc/parallel/psPromotionManager.hpp" #include "gc/parallel/psScavenge.hpp" #include "gc/parallel/psVMOperations.hpp" +#include "gc/shared/gcForwarding.inline.hpp" #include "gc/shared/gcHeapSummary.hpp" #include "gc/shared/gcLocker.inline.hpp" #include "gc/shared/gcWhen.hpp" @@ -129,6 +130,8 @@ jint ParallelScavengeHeap::initialize() { ParallelInitLogger::print(); + GCForwarding::initialize(heap_rs.region()); + return JNI_OK; } diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.cpp b/src/hotspot/share/gc/parallel/psParallelCompact.cpp index 4bff8f8a7d06a..746233f893a54 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.cpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.cpp @@ -45,6 +45,7 @@ #include "gc/parallel/psYoungGen.hpp" #include "gc/shared/classUnloadingContext.hpp" #include "gc/shared/gcCause.hpp" +#include "gc/shared/gcForwarding.inline.hpp" #include "gc/shared/gcHeapSummary.hpp" #include "gc/shared/gcId.hpp" #include "gc/shared/gcLocker.hpp" @@ -1592,7 +1593,7 @@ void PSParallelCompact::forward_to_new_addr() { oop obj = cast_to_oop(cur_addr); if (new_addr != cur_addr) { cm->preserved_marks()->push_if_necessary(obj, obj->mark()); - obj->forward_to(cast_to_oop(new_addr)); + GCForwarding::forward_to(obj, cast_to_oop(new_addr)); } size_t obj_size = obj->size(); live_words += obj_size; @@ -1635,7 +1636,7 @@ void PSParallelCompact::verify_forward() { } oop obj = cast_to_oop(cur_addr); if (cur_addr != bump_ptr) { - assert(obj->forwardee() == cast_to_oop(bump_ptr), "inv"); + assert(GCForwarding::forwardee(obj) == cast_to_oop(bump_ptr), "inv"); } bump_ptr += obj->size(); cur_addr += obj->size(); @@ -2398,8 +2399,8 @@ void MoveAndUpdateClosure::do_addr(HeapWord* addr, size_t words) { if (copy_destination() != source()) { DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());) assert(source() != destination(), "inv"); - assert(cast_to_oop(source())->is_forwarded(), "inv"); - assert(cast_to_oop(source())->forwardee() == cast_to_oop(destination()), "inv"); + assert(GCForwarding::is_forwarded(cast_to_oop(source())), "inv"); + assert(GCForwarding::forwardee(cast_to_oop(source())) == cast_to_oop(destination()), "inv"); Copy::aligned_conjoint_words(source(), copy_destination(), words); cast_to_oop(copy_destination())->init_mark(); } diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp b/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp index e5f1b9d30aed7..55e6b5fb7b29c 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp @@ -31,6 +31,7 @@ #include "gc/parallel/parMarkBitMap.inline.hpp" #include "gc/shared/collectedHeap.hpp" #include "gc/shared/continuationGCSupport.inline.hpp" +#include "gc/shared/gcForwarding.inline.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/klass.hpp" @@ -79,7 +80,7 @@ inline void PSParallelCompact::adjust_pointer(T* p) { if (!obj->is_forwarded()) { return; } - oop new_obj = obj->forwardee(); + oop new_obj = GCForwarding::forwardee(obj); assert(new_obj != nullptr, "non-null address for live objects"); assert(new_obj != obj, "inv"); assert(ParallelScavengeHeap::heap()->is_in_reserved(new_obj), diff --git a/src/hotspot/share/gc/serial/serialFullGC.cpp b/src/hotspot/share/gc/serial/serialFullGC.cpp index 897437e33c96e..aae4939049cf4 100644 --- a/src/hotspot/share/gc/serial/serialFullGC.cpp +++ b/src/hotspot/share/gc/serial/serialFullGC.cpp @@ -43,6 +43,7 @@ #include "gc/shared/classUnloadingContext.hpp" #include "gc/shared/collectedHeap.inline.hpp" #include "gc/shared/continuationGCSupport.inline.hpp" +#include "gc/shared/gcForwarding.inline.hpp" #include "gc/shared/gcHeapSummary.hpp" #include "gc/shared/gcTimer.hpp" #include "gc/shared/gcTrace.hpp" @@ -230,7 +231,7 @@ class Compacter { static void forward_obj(oop obj, HeapWord* new_addr) { prefetch_write_scan(obj); if (cast_from_oop(obj) != new_addr) { - obj->forward_to(cast_to_oop(new_addr)); + GCForwarding::forward_to(obj, cast_to_oop(new_addr)); } else { assert(obj->is_gc_marked(), "inv"); // This obj will stay in-place. Fix the markword. @@ -255,7 +256,7 @@ class Compacter { prefetch_read_scan(addr); oop obj = cast_to_oop(addr); - oop new_obj = obj->forwardee(); + oop new_obj = GCForwarding::forwardee(obj); HeapWord* new_addr = cast_from_oop(new_obj); assert(addr != new_addr, "inv"); prefetch_write_copy(new_addr); @@ -352,13 +353,13 @@ class Compacter { HeapWord* top = space->top(); // Check if the first obj inside this space is forwarded. - if (!cast_to_oop(cur_addr)->is_forwarded()) { + if (!GCForwarding::is_forwarded(cast_to_oop(cur_addr))) { // Jump over consecutive (in-place) live-objs-chunk cur_addr = get_first_dead(i); } while (cur_addr < top) { - if (!cast_to_oop(cur_addr)->is_forwarded()) { + if (!GCForwarding::is_forwarded(cast_to_oop(cur_addr))) { cur_addr = *(HeapWord**) cur_addr; continue; } @@ -624,8 +625,8 @@ template void SerialFullGC::adjust_pointer(T* p) { oop obj = CompressedOops::decode_not_null(heap_oop); assert(Universe::heap()->is_in(obj), "should be in heap"); - if (obj->is_forwarded()) { - oop new_obj = obj->forwardee(); + if (GCForwarding::is_forwarded(obj)) { + oop new_obj = GCForwarding::forwardee(obj); assert(is_object_aligned(new_obj), "oop must be aligned"); RawAccess::oop_store(p, new_obj); } diff --git a/src/hotspot/share/gc/serial/serialHeap.cpp b/src/hotspot/share/gc/serial/serialHeap.cpp index 3c48177554121..e0a227d664011 100644 --- a/src/hotspot/share/gc/serial/serialHeap.cpp +++ b/src/hotspot/share/gc/serial/serialHeap.cpp @@ -40,6 +40,7 @@ #include "gc/shared/collectedHeap.inline.hpp" #include "gc/shared/collectorCounters.hpp" #include "gc/shared/continuationGCSupport.inline.hpp" +#include "gc/shared/gcForwarding.hpp" #include "gc/shared/gcId.hpp" #include "gc/shared/gcInitLogger.hpp" #include "gc/shared/gcLocker.inline.hpp" @@ -200,6 +201,8 @@ jint SerialHeap::initialize() { GCInitLogger::print(); + GCForwarding::initialize(_reserved); + return JNI_OK; } diff --git a/src/hotspot/share/gc/shared/gcArguments.cpp b/src/hotspot/share/gc/shared/gcArguments.cpp index 9736c0f7fdcab..213a89199a68a 100644 --- a/src/hotspot/share/gc/shared/gcArguments.cpp +++ b/src/hotspot/share/gc/shared/gcArguments.cpp @@ -26,6 +26,7 @@ #include "precompiled.hpp" #include "gc/shared/cardTable.hpp" #include "gc/shared/gcArguments.hpp" +#include "gc/shared/gcForwarding.hpp" #include "logging/log.hpp" #include "runtime/arguments.hpp" #include "runtime/globals.hpp" @@ -60,6 +61,7 @@ void GCArguments::initialize_heap_sizes() { initialize_alignments(); initialize_heap_flags_and_sizes(); initialize_size_info(); + GCForwarding::initialize_flags(); } size_t GCArguments::compute_heap_alignment() { diff --git a/src/hotspot/share/gc/shared/gcForwarding.cpp b/src/hotspot/share/gc/shared/gcForwarding.cpp new file mode 100644 index 0000000000000..addaa06ff8745 --- /dev/null +++ b/src/hotspot/share/gc/shared/gcForwarding.cpp @@ -0,0 +1,54 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shared/gcForwarding.hpp" +#include "memory/memRegion.hpp" +#include "runtime/globals_extension.hpp" + +HeapWord* GCForwarding::_heap_base = nullptr; +int GCForwarding::_num_low_bits = 0; + +void GCForwarding::initialize_flags() { + // Nothing to do here, yet. As soon as we have compact + // object headers, we will disable the flag when the + // heap size exceeds the narrow-encodable address space. + + // size_t max_narrow_heap_size = (size_t(1) << (NUM_LOW_BITS_NARROW - SHIFT)) * HeapWordSize; + // if (UseCompactObjectHeaders && MaxHeapSize >= max_narrow_heap_size) { + // FLAG_SET_DEFAULT(UseCompactObjectHeaders, false); + // } +} + +void GCForwarding::initialize(MemRegion heap) { +#ifdef _LP64 + _heap_base = heap.start(); + if (heap.word_size() <= right_n_bits(NUM_LOW_BITS_NARROW - SHIFT)) { + _num_low_bits = NUM_LOW_BITS_NARROW; + } else { + // assert(!UseCompactObjectHeaders, "Compact object headers should be turned off for large heaps"); + _num_low_bits = NUM_LOW_BITS_WIDE; + } +#endif +} diff --git a/src/hotspot/share/gc/shared/gcForwarding.hpp b/src/hotspot/share/gc/shared/gcForwarding.hpp new file mode 100644 index 0000000000000..733a47f5f0629 --- /dev/null +++ b/src/hotspot/share/gc/shared/gcForwarding.hpp @@ -0,0 +1,49 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHARED_GCFORWARDING_HPP +#define SHARE_GC_SHARED_GCFORWARDING_HPP + +#include "memory/allStatic.hpp" +#include "memory/memRegion.hpp" +#include "oops/markWord.hpp" +#include "oops/oopsHierarchy.hpp" + +class GCForwarding : public AllStatic { + static const int NumKlassBits = 32; // Will be 22 with Tiny Class-Pointers + static const int NUM_LOW_BITS_NARROW = BitsPerWord - NumKlassBits; + static const int NUM_LOW_BITS_WIDE = BitsPerWord; + static const int SHIFT = markWord::lock_bits + markWord::lock_shift; + + static HeapWord* _heap_base; + static int _num_low_bits; +public: + static void initialize_flags(); + static void initialize(MemRegion heap); + static inline void forward_to(oop from, oop to); + static inline oop forwardee(oop from); + static inline bool is_forwarded(oop obj); +}; + +#endif // SHARE_GC_SHARED_GCFORWARDING_HPP diff --git a/src/hotspot/share/gc/shared/gcForwarding.inline.hpp b/src/hotspot/share/gc/shared/gcForwarding.inline.hpp new file mode 100644 index 0000000000000..e46377d897b5e --- /dev/null +++ b/src/hotspot/share/gc/shared/gcForwarding.inline.hpp @@ -0,0 +1,59 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef GC_SHARED_GCFORWARDING_INLINE_HPP +#define GC_SHARED_GCFORWARDING_INLINE_HPP + +#include "gc/shared/gcForwarding.hpp" +#include "oops/oop.inline.hpp" +#include "utilities/globalDefinitions.hpp" + +void GCForwarding::forward_to(oop from, oop to) { +#ifdef _LP64 + uintptr_t encoded = pointer_delta(cast_from_oop(to), _heap_base) << SHIFT; + assert(encoded <= static_cast(right_n_bits(_num_low_bits)), "encoded forwardee must fit"); + uintptr_t mark = from->mark().value(); + mark &= ~right_n_bits(_num_low_bits); + mark |= (encoded | markWord::marked_value); + from->set_mark(markWord(mark)); +#else + from->forward_to(to); +#endif +} + +oop GCForwarding::forwardee(oop from) { +#ifdef _LP64 + uintptr_t mark = from->mark().value(); + HeapWord* decoded = _heap_base + ((mark & right_n_bits(_num_low_bits)) >> SHIFT); + return cast_to_oop(decoded); +#else + return from->forwardee(); +#endif +} + +bool GCForwarding::is_forwarded(oop obj) { + return obj->mark().is_marked(); +} + +#endif // GC_SHARED_GCFORWARDING_INLINE_HPP diff --git a/src/hotspot/share/gc/shared/preservedMarks.cpp b/src/hotspot/share/gc/shared/preservedMarks.cpp index 9889dbc369018..4d3a34d34fa53 100644 --- a/src/hotspot/share/gc/shared/preservedMarks.cpp +++ b/src/hotspot/share/gc/shared/preservedMarks.cpp @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" +#include "gc/shared/gcForwarding.inline.hpp" #include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/workerThread.hpp" #include "gc/shared/workerUtils.hpp" @@ -42,8 +43,8 @@ void PreservedMarks::restore() { void PreservedMarks::adjust_preserved_mark(PreservedMark* elem) { oop obj = elem->get_oop(); - if (obj->is_forwarded()) { - elem->set_oop(obj->forwardee()); + if (GCForwarding::is_forwarded(obj)) { + elem->set_oop(GCForwarding::forwardee(obj)); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index de7d81d0f43cd..1c81793e78ebc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -26,6 +26,7 @@ #include "compiler/oopMap.hpp" #include "gc/shared/continuationGCSupport.hpp" +#include "gc/shared/gcForwarding.inline.hpp" #include "gc/shared/gcTraceTime.inline.hpp" #include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/tlab_globals.hpp" @@ -369,7 +370,7 @@ class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure { shenandoah_assert_not_forwarded(nullptr, p); if (_compact_point != cast_from_oop(p)) { _preserved_marks->push_if_necessary(p, p->mark()); - p->forward_to(cast_to_oop(_compact_point)); + GCForwarding::forward_to(p, cast_to_oop(_compact_point)); } _compact_point += obj_size; } @@ -492,7 +493,7 @@ void ShenandoahFullGC::calculate_target_humongous_objects() { if (start >= to_begin && start != r->index()) { // Fits into current window, and the move is non-trivial. Record the move then, and continue scan. _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark()); - old_obj->forward_to(cast_to_oop(heap->get_region(start)->bottom())); + GCForwarding::forward_to(old_obj, cast_to_oop(heap->get_region(start)->bottom())); to_end = start; continue; } @@ -752,8 +753,8 @@ class ShenandoahAdjustPointersClosure : public MetadataVisitingOopIterateClosure if (!CompressedOops::is_null(o)) { oop obj = CompressedOops::decode_not_null(o); assert(_ctx->is_marked(obj), "must be marked"); - if (obj->is_forwarded()) { - oop forw = obj->forwardee(); + if (GCForwarding::is_forwarded(obj)) { + oop forw = GCForwarding::forwardee(obj); RawAccess::oop_store(p, forw); } } @@ -863,9 +864,9 @@ class ShenandoahCompactObjectsClosure : public ObjectClosure { void do_object(oop p) { assert(_heap->complete_marking_context()->is_marked(p), "must be marked"); size_t size = p->size(); - if (p->is_forwarded()) { + if (GCForwarding::is_forwarded(p)) { HeapWord* compact_from = cast_from_oop(p); - HeapWord* compact_to = cast_from_oop(p->forwardee()); + HeapWord* compact_to = cast_from_oop(GCForwarding::forwardee(p)); assert(compact_from != compact_to, "Forwarded object should move"); Copy::aligned_conjoint_words(compact_from, compact_to, size); oop new_obj = cast_to_oop(compact_to); @@ -970,7 +971,7 @@ void ShenandoahFullGC::compact_humongous_objects() { ShenandoahHeapRegion* r = heap->get_region(c - 1); if (r->is_humongous_start()) { oop old_obj = cast_to_oop(r->bottom()); - if (!old_obj->is_forwarded()) { + if (!GCForwarding::is_forwarded(old_obj)) { // No need to move the object, it stays at the same slot continue; } @@ -979,7 +980,7 @@ void ShenandoahFullGC::compact_humongous_objects() { size_t old_start = r->index(); size_t old_end = old_start + num_regions - 1; - size_t new_start = heap->heap_region_index_containing(old_obj->forwardee()); + size_t new_start = heap->heap_region_index_containing(GCForwarding::forwardee(old_obj)); size_t new_end = new_start + num_regions - 1; assert(old_start != new_start, "must be real move"); assert(r->is_stw_move_allowed(), "Region " SIZE_FORMAT " should be movable", r->index()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index a587cc417e319..9776c773ddb07 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -29,6 +29,7 @@ #include "gc/shared/classUnloadingContext.hpp" #include "gc/shared/gcArguments.hpp" +#include "gc/shared/gcForwarding.hpp" #include "gc/shared/gcTimer.hpp" #include "gc/shared/gcTraceTime.inline.hpp" #include "gc/shared/locationPrinter.inline.hpp" @@ -422,6 +423,8 @@ jint ShenandoahHeap::initialize() { ShenandoahInitLogger::print(); + GCForwarding::initialize(_heap_region); + return JNI_OK; } diff --git a/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp b/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp index 7700a44e2cd92..79be24d4c3de7 100644 --- a/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp +++ b/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp @@ -23,63 +23,48 @@ #include "precompiled.hpp" #include "gc/shared/preservedMarks.inline.hpp" +#include "gc/shared/gcForwarding.inline.hpp" #include "oops/oop.inline.hpp" #include "unittest.hpp" -// Class to create a "fake" oop with a mark that will -// return true for calls to must_be_preserved(). -class FakeOop { - // Align at least to 8 bytes, otherwise the lowest address bit - // could be interpreted as 'self-forwarded' when encoded as - // forwardee in the mark-word. This is required on 32 bit builds, - // where the oop could be 4-byte-aligned. - alignas(BytesPerLong) oopDesc _oop; - -public: - FakeOop() : _oop() { _oop.set_mark(originalMark()); } - - oop get_oop() { return &_oop; } - markWord mark() { return _oop.mark(); } - void set_mark(markWord m) { _oop.set_mark(m); } - void forward_to(oop obj) { - markWord m = markWord::encode_pointer_as_mark(obj); - _oop.set_mark(m); - } - - static markWord originalMark() { return markWord(markWord::lock_mask_in_place); } - static markWord changedMark() { return markWord(0x4711); } -}; +static markWord originalMark() { return markWord(markWord::lock_mask_in_place); } +static markWord changedMark() { return markWord(0x4711); } #define ASSERT_MARK_WORD_EQ(a, b) ASSERT_EQ((a).value(), (b).value()) TEST_VM(PreservedMarks, iterate_and_restore) { PreservedMarks pm; - FakeOop o1; - FakeOop o2; - FakeOop o3; - FakeOop o4; + + HeapWord fakeheap[32] = { nullptr }; + HeapWord* heap = align_up(fakeheap, 8 * sizeof(HeapWord)); + GCForwarding::initialize(MemRegion(&heap[0], &heap[16])); + + oop o1 = cast_to_oop(&heap[0]); o1->set_mark(originalMark()); + oop o2 = cast_to_oop(&heap[2]); o2->set_mark(originalMark()); + oop o3 = cast_to_oop(&heap[4]); o3->set_mark(originalMark()); + oop o4 = cast_to_oop(&heap[6]); o4->set_mark(originalMark()); // Make sure initial marks are correct. - ASSERT_MARK_WORD_EQ(o1.mark(), FakeOop::originalMark()); - ASSERT_MARK_WORD_EQ(o2.mark(), FakeOop::originalMark()); - ASSERT_MARK_WORD_EQ(o3.mark(), FakeOop::originalMark()); - ASSERT_MARK_WORD_EQ(o4.mark(), FakeOop::originalMark()); + ASSERT_MARK_WORD_EQ(o1->mark(), originalMark()); + ASSERT_MARK_WORD_EQ(o2->mark(), originalMark()); + ASSERT_MARK_WORD_EQ(o3->mark(), originalMark()); + ASSERT_MARK_WORD_EQ(o4->mark(), originalMark()); // Change the marks and verify change. - o1.set_mark(FakeOop::changedMark()); - o2.set_mark(FakeOop::changedMark()); - ASSERT_MARK_WORD_EQ(o1.mark(), FakeOop::changedMark()); - ASSERT_MARK_WORD_EQ(o2.mark(), FakeOop::changedMark()); + o1->set_mark(changedMark()); + o2->set_mark(changedMark()); + ASSERT_MARK_WORD_EQ(o1->mark(), changedMark()); + ASSERT_MARK_WORD_EQ(o2->mark(), changedMark()); // Push o1 and o2 to have their marks preserved. - pm.push_if_necessary(o1.get_oop(), o1.mark()); - pm.push_if_necessary(o2.get_oop(), o2.mark()); + pm.push_if_necessary(o1, o1->mark()); + pm.push_if_necessary(o2, o2->mark()); // Fake a move from o1->o3 and o2->o4. - o1.forward_to(o3.get_oop()); - o2.forward_to(o4.get_oop()); - ASSERT_EQ(o1.get_oop()->forwardee(), o3.get_oop()); - ASSERT_EQ(o2.get_oop()->forwardee(), o4.get_oop()); + GCForwarding::forward_to(o1, o3); + GCForwarding::forward_to(o2, o4); + ASSERT_EQ(GCForwarding::forwardee(o1), o3); + ASSERT_EQ(GCForwarding::forwardee(o2), o4); // Adjust will update the PreservedMarks stack to // make sure the mark is updated at the new location. pm.adjust_during_full_gc(); @@ -87,6 +72,6 @@ TEST_VM(PreservedMarks, iterate_and_restore) { // Restore all preserved and verify that the changed // mark is now present at o3 and o4. pm.restore(); - ASSERT_MARK_WORD_EQ(o3.mark(), FakeOop::changedMark()); - ASSERT_MARK_WORD_EQ(o4.mark(), FakeOop::changedMark()); + ASSERT_MARK_WORD_EQ(o3->mark(), changedMark()); + ASSERT_MARK_WORD_EQ(o4->mark(), changedMark()); }