From 9367c144a9ff64ed028fdf3d4d96a638ec90f936 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Wed, 26 Jun 2024 16:49:52 +0200 Subject: [PATCH] Factor out common code --- .../share/gc/g1/g1ConcurrentMark.inline.hpp | 7 +- .../g1/g1ConcurrentMarkObjArrayProcessor.cpp | 99 +----------- .../g1/g1ConcurrentMarkObjArrayProcessor.hpp | 17 +-- ...ConcurrentMarkObjArrayProcessor.inline.hpp | 38 ----- .../share/gc/g1/g1ParScanThreadState.cpp | 141 +++++------------- 5 files changed, 59 insertions(+), 243 deletions(-) delete mode 100644 src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp index 5d5cce4e199ef..d451df32d73c6 100644 --- a/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp +++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp @@ -29,8 +29,7 @@ #include "gc/g1/g1CollectedHeap.inline.hpp" #include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp" -#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp" -#include "gc/g1/g1HeapRegion.hpp" + #include "gc/g1/g1HeapRegion.hpp" #include "gc/g1/g1HeapRegionRemSet.inline.hpp" #include "gc/g1/g1OopClosures.inline.hpp" #include "gc/g1/g1Policy.hpp" @@ -169,8 +168,8 @@ inline void G1CMTask::process_grey_task_entry(G1TaskQueueEntry task_entry) { if (task_entry.is_array_slice()) { _words_scanned += _objArray_processor.process_slice(obj, task_entry.slice(), task_entry.pow()); } else { - if (G1CMObjArrayProcessor::should_be_sliced(obj)) { - _words_scanned += _objArray_processor.process_obj(obj); + if (obj->is_objArray()) { + _words_scanned += _objArray_processor.process_objArray(objArrayOop(obj)); } else { _words_scanned += obj->oop_iterate_size(_cm_oop_closure);; } diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp index 33331acd54f60..0ab8037ac6a9e 100644 --- a/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp +++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp @@ -23,99 +23,16 @@ */ #include "precompiled.hpp" -#include "gc/g1/g1CollectedHeap.inline.hpp" +#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp" #include "gc/g1/g1ConcurrentMark.inline.hpp" -#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp" -#include "gc/g1/g1HeapRegion.inline.hpp" -#include "gc/shared/gc_globals.hpp" -#include "memory/memRegion.hpp" -#include "utilities/globalDefinitions.hpp" - -size_t G1CMObjArrayProcessor::process_obj(oop obj) { - assert(should_be_sliced(obj), "Must be an array object %d and large " SIZE_FORMAT, obj->is_objArray(), obj->size()); - - assert(obj->is_objArray(), "expect object array"); - objArrayOop array = objArrayOop(obj); +#include "oops/oopsHierarchy.hpp" +void G1CMObjArrayProcessor::scan_metadata(objArrayOop array) { _task->scan_objArray_start(array); - - int len = array->length(); - - int bits = log2i_graceful(len); - // Compensate for non-power-of-two arrays, cover the array in excess: - if (len != (1 << bits)) bits++; - - // Only allow full slices on the queue. This frees do_sliced_array() from checking from/to - // boundaries against array->length(), touching the array header on every slice. - // - // To do this, we cut the prefix in full-sized slices, and submit them on the queue. - // If the array is not divided in slice sizes, then there would be an irregular tail, - // which we will process separately. - - int last_idx = 0; - - int slice = 1; - int pow = bits; - - // Handle overflow - if (pow >= 31) { - assert (pow == 31, "sanity"); - pow--; - slice = 2; - last_idx = (1 << pow); - _task->push(G1TaskQueueEntry(array, 1, pow)); - } - - // Split out tasks, as suggested in G1TaskQueueEntry docs. Record the last - // successful right boundary to figure out the irregular tail. - while ((1 << pow) > (int)ObjArrayMarkingStride && - (slice * 2 < G1TaskQueueEntry::slice_size())) { - pow--; - int left_slice = slice * 2 - 1; - int right_slice = slice * 2; - int left_slice_end = left_slice * (1 << pow); - if (left_slice_end < len) { - _task->push(G1TaskQueueEntry(array, left_slice, pow)); - slice = right_slice; - last_idx = left_slice_end; - } else { - slice = left_slice; - } - } - - // Process the irregular tail, if present - int from = last_idx; - if (from < len) { - return _task->scan_objArray(array, from, len); - } - return 0; } - -size_t G1CMObjArrayProcessor::process_slice(oop obj, int slice, int pow) { - - assert(obj->is_objArray(), "expect object array"); - objArrayOop array = objArrayOop(obj); - - assert (ObjArrayMarkingStride > 0, "sanity"); - - // Split out tasks, as suggested in G1TaskQueueEntry docs. Avoid pushing tasks that - // are known to start beyond the array. - while ((1 << pow) > (int)ObjArrayMarkingStride && (slice*2 < G1TaskQueueEntry::slice_size())) { - pow--; - slice *= 2; - _task->push(G1TaskQueueEntry(array, slice - 1, pow)); - } - - int slice_size = 1 << pow; - - int from = (slice - 1) * slice_size; - int to = slice * slice_size; - -#ifdef ASSERT - int len = array->length(); - assert (0 <= from && from < len, "from is sane: %d/%d", from, len); - assert (0 < to && to <= len, "to is sane: %d/%d", to, len); -#endif - - return _task->scan_objArray(array, from, to); +void G1CMObjArrayProcessor::push_on_queue(G1TaskQueueEntry task) { + _task->push(task); +} +size_t G1CMObjArrayProcessor::scan_array(objArrayOop array, int from, int len) { + return _task->scan_objArray(array, from, from + len); } diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp index 382bfa732be22..1fecfe788772d 100644 --- a/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp +++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp @@ -26,6 +26,8 @@ #define SHARE_GC_G1_G1CONCURRENTMARKOBJARRAYPROCESSOR_HPP #include "oops/oopsHierarchy.hpp" +#include "gc/g1/g1ArraySlicer.hpp" +#include "gc/g1/g1TaskQueueEntry.hpp" class G1CMTask; @@ -33,22 +35,17 @@ class G1CMTask; // Instead of pushing large object arrays, we push continuations onto the // mark stack. These continuations are identified by having their LSB set. // This allows incremental processing of large objects. -class G1CMObjArrayProcessor { +class G1CMObjArrayProcessor : public G1ArraySlicer { private: // Reference to the task for doing the actual work. G1CMTask* _task; public: - static bool should_be_sliced(oop obj); - - G1CMObjArrayProcessor(G1CMTask* task) : _task(task) { + explicit G1CMObjArrayProcessor(G1CMTask* task) : _task(task) { } - - // Process the given continuation. Returns the number of words scanned. - size_t process_slice(oop ary, int chunk, int pow); - // Start processing the given objArrayOop by scanning the header and pushing its - // continuation. - size_t process_obj(oop obj); + void scan_metadata(objArrayOop array) override; + void push_on_queue(G1TaskQueueEntry task) override; + size_t scan_array(objArrayOop array, int from, int len) override; }; #endif // SHARE_GC_G1_G1CONCURRENTMARKOBJARRAYPROCESSOR_HPP diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp deleted file mode 100644 index d51c5c740824c..0000000000000 --- a/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_GC_G1_G1CONCURRENTMARKOBJARRAYPROCESSOR_INLINE_HPP -#define SHARE_GC_G1_G1CONCURRENTMARKOBJARRAYPROCESSOR_INLINE_HPP - -#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp" - -#include "oops/oop.inline.hpp" -#include "oops/oopsHierarchy.hpp" -#include "gc/shared/gc_globals.hpp" - -inline bool G1CMObjArrayProcessor::should_be_sliced(oop obj) { - return obj->is_objArray() && ((objArrayOop)obj)->size() >= 2 * ObjArrayMarkingStride; -} - -#endif // SHARE_GC_G1_G1CONCURRENTMARKOBJARRAYPROCESSOR_INLINE_HPP diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp index 69274f43c73be..7a2632dcf47af 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp @@ -24,6 +24,7 @@ #include "precompiled.hpp" #include "gc/g1/g1Allocator.inline.hpp" +#include "gc/g1/g1ArraySlicer.hpp" #include "gc/g1/g1CollectedHeap.inline.hpp" #include "gc/g1/g1CollectionSet.hpp" #include "gc/g1/g1EvacFailureRegions.inline.hpp" @@ -221,6 +222,36 @@ void G1ParScanThreadState::do_oop_evac(T* p) { write_ref_field_post(p, obj); } +class G1ScavengeArraySlicer : public G1ArraySlicer { + G1ScanEvacuatedObjClosure& _scanner; + bool _skip_enqueue; + G1ParScanThreadState* _par_scan; + G1CollectedHeap* _g1h; +public: + G1ScavengeArraySlicer(G1ScanEvacuatedObjClosure& scanner, + bool skip_enqueue, + G1ParScanThreadState* par_scan) : + _scanner(scanner), + _skip_enqueue(skip_enqueue), + _par_scan(par_scan), + _g1h(G1CollectedHeap::heap()) {} + + void scan_metadata(objArrayOop array) override { + // TODO: According to old comment not needed, but may be cleaner? + if (Devirtualizer::do_metadata(&_scanner)) { + Devirtualizer::do_klass(&_scanner, array->klass()); + } + } + void push_on_queue(G1TaskQueueEntry task) override { + _par_scan->push_on_queue(task); + } + size_t scan_array(objArrayOop array, int from, int len) override { + G1SkipCardEnqueueSetter x(&_scanner, _skip_enqueue); + array->oop_iterate_range(&_scanner, 0, len); + return len * (UseCompressedOops ? 2 : 1); + } +}; + MAYBE_INLINE_EVACUATION void G1ParScanThreadState::do_partial_array(oop obj, int slice, int pow) { assert(_g1h->is_in_reserved(obj), "must be in heap."); @@ -228,114 +259,24 @@ void G1ParScanThreadState::do_partial_array(oop obj, int slice, int pow) { objArrayOop array = objArrayOop(obj); - assert(ObjArrayMarkingStride > 0, "sanity"); - - // Split out tasks, as suggested in G1TaskQueueEntry docs. Avoid pushing tasks that - // are known to start beyond the array. - while ((1 << pow) > (int)ObjArrayMarkingStride && (slice * 2 < G1TaskQueueEntry::slice_size())) { - pow--; - slice *= 2; - push_on_queue(G1TaskQueueEntry(array, slice - 1, pow)); - } - - int slice_size = 1 << pow; - - int from = (slice - 1) * slice_size; - int to = slice * slice_size; - -#ifdef ASSERT - int len = array->length(); - assert(0 <= from && from < len, "from is sane: %d/%d", from, len); - assert(0 < to && to <= len, "to is sane: %d/%d", to, len); -#endif - - // Process claimed task. The length of to_array is not correct, but - // fortunately the iteration ignores the length field and just relies - // on start/end. G1HeapRegionAttr dest_attr = _g1h->region_attr(array); - G1SkipCardEnqueueSetter x(&_scanner, dest_attr.is_new_survivor()); - array->oop_iterate_range(&_scanner, from, to); + G1ScavengeArraySlicer slicer(_scanner, dest_attr.is_new_survivor(), this); + slicer.process_slice(array, slice, pow); } MAYBE_INLINE_EVACUATION void G1ParScanThreadState::start_partial_objarray(G1HeapRegionAttr dest_attr, oop obj) { assert(obj->is_objArray(), "precondition"); - objArrayOop array = objArrayOop(obj); - int len = array->length(); - - // Mark objArray klass metadata - // TODO: According to old comment not needed, but may be cleaner? - if (Devirtualizer::do_metadata(&_scanner)) { - Devirtualizer::do_klass(&_scanner, array->klass()); - } - - if (len <= (int) ObjArrayMarkingStride*2) { - // A few slices only, process directly - // Skip the card enqueue iff the object (to_array) is in survivor region. - // However, HeapRegion::is_survivor() is too expensive here. - // Instead, we use dest_attr.is_young() because the two values are always - // equal: successfully allocated young regions must be survivor regions. - assert(dest_attr.is_young() == _g1h->heap_region_containing(array)->is_survivor(), "must be"); - G1SkipCardEnqueueSetter x(&_scanner, dest_attr.is_young()); - array->oop_iterate_range(&_scanner, 0, len); - } else { - int bits = log2i_graceful(len); - // Compensate for non-power-of-two arrays, cover the array in excess: - if (len != (1 << bits)) bits++; - - // Only allow full slices on the queue. This frees do_sliced_array() from checking from/to - // boundaries against array->length(), touching the array header on every slice. - // - // To do this, we cut the prefix in full-sized slices, and submit them on the queue. - // If the array is not divided in slice sizes, then there would be an irregular tail, - // which we will process separately. - - int last_idx = 0; - - int slice = 1; - int pow = bits; - - // Handle overflow - if (pow >= 31) { - assert (pow == 31, "sanity"); - pow--; - slice = 2; - last_idx = (1 << pow); - push_on_queue(G1TaskQueueEntry(array, 1, pow)); - } - - // Split out tasks, as suggested in G1TaskQueueEntry docs. Record the last - // successful right boundary to figure out the irregular tail. - while ((1 << pow) > (int)ObjArrayMarkingStride && - (slice * 2 < G1TaskQueueEntry::slice_size())) { - pow--; - int left_slice = slice * 2 - 1; - int right_slice = slice * 2; - int left_slice_end = left_slice * (1 << pow); - if (left_slice_end < len) { - push_on_queue(G1TaskQueueEntry(array, left_slice, pow)); - slice = right_slice; - last_idx = left_slice_end; - } else { - slice = left_slice; - } - } - - // Process the irregular tail, if present - int from = last_idx; - if (from < len) { - // Skip the card enqueue iff the object (to_array) is in survivor region. - // However, HeapRegion::is_survivor() is too expensive here. - // Instead, we use dest_attr.is_young() because the two values are always - // equal: successfully allocated young regions must be survivor regions. - assert(dest_attr.is_young() == _g1h->heap_region_containing(array)->is_survivor(), "must be"); - G1SkipCardEnqueueSetter x(&_scanner, dest_attr.is_young()); - array->oop_iterate_range(&_scanner, from, len); - } - } -} + // Skip the card enqueue iff the object (to_array) is in survivor region. + // However, HeapRegion::is_survivor() is too expensive here. + // Instead, we use dest_attr.is_young() because the two values are always + // equal: successfully allocated young regions must be survivor regions. + assert(dest_attr.is_young() == _g1h->heap_region_containing(array)->is_survivor(), "must be"); + G1ScavengeArraySlicer slicer(_scanner, dest_attr.is_young(), this); + slicer.process_objArray(array); + } MAYBE_INLINE_EVACUATION void G1ParScanThreadState::dispatch_task(G1TaskQueueEntry task) {