Skip to content

Commit

Permalink
Factor out common code
Browse files Browse the repository at this point in the history
  • Loading branch information
@rkennke
rkennke committed Jun 26, 2024
1 parent 50cecc0 commit 9367c14
Show file tree
Hide file tree
Showing 5 changed files with 59 additions and 243 deletions.
7 changes: 3 additions & 4 deletions src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,7 @@

#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp"
#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
#include "gc/g1/g1Policy.hpp"
Expand Down Expand Up @@ -169,8 +168,8 @@ inline void G1CMTask::process_grey_task_entry(G1TaskQueueEntry task_entry) {
if (task_entry.is_array_slice()) {
_words_scanned += _objArray_processor.process_slice(obj, task_entry.slice(), task_entry.pow());
} else {
if (G1CMObjArrayProcessor::should_be_sliced(obj)) {
_words_scanned += _objArray_processor.process_obj(obj);
if (obj->is_objArray()) {
_words_scanned += _objArray_processor.process_objArray(objArrayOop(obj));
} else {
_words_scanned += obj->oop_iterate_size(_cm_oop_closure);;
}
Expand Down
99 changes: 8 additions & 91 deletions src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,99 +23,16 @@
*/

#include "precompiled.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/shared/gc_globals.hpp"
#include "memory/memRegion.hpp"
#include "utilities/globalDefinitions.hpp"

size_t G1CMObjArrayProcessor::process_obj(oop obj) {
assert(should_be_sliced(obj), "Must be an array object %d and large " SIZE_FORMAT, obj->is_objArray(), obj->size());

assert(obj->is_objArray(), "expect object array");
objArrayOop array = objArrayOop(obj);
#include "oops/oopsHierarchy.hpp"

void G1CMObjArrayProcessor::scan_metadata(objArrayOop array) {
_task->scan_objArray_start(array);

int len = array->length();

int bits = log2i_graceful(len);
// Compensate for non-power-of-two arrays, cover the array in excess:
if (len != (1 << bits)) bits++;

// Only allow full slices on the queue. This frees do_sliced_array() from checking from/to
// boundaries against array->length(), touching the array header on every slice.
//
// To do this, we cut the prefix in full-sized slices, and submit them on the queue.
// If the array is not divided in slice sizes, then there would be an irregular tail,
// which we will process separately.

int last_idx = 0;

int slice = 1;
int pow = bits;

// Handle overflow
if (pow >= 31) {
assert (pow == 31, "sanity");
pow--;
slice = 2;
last_idx = (1 << pow);
_task->push(G1TaskQueueEntry(array, 1, pow));
}

// Split out tasks, as suggested in G1TaskQueueEntry docs. Record the last
// successful right boundary to figure out the irregular tail.
while ((1 << pow) > (int)ObjArrayMarkingStride &&
(slice * 2 < G1TaskQueueEntry::slice_size())) {
pow--;
int left_slice = slice * 2 - 1;
int right_slice = slice * 2;
int left_slice_end = left_slice * (1 << pow);
if (left_slice_end < len) {
_task->push(G1TaskQueueEntry(array, left_slice, pow));
slice = right_slice;
last_idx = left_slice_end;
} else {
slice = left_slice;
}
}

// Process the irregular tail, if present
int from = last_idx;
if (from < len) {
return _task->scan_objArray(array, from, len);
}
return 0;
}

size_t G1CMObjArrayProcessor::process_slice(oop obj, int slice, int pow) {

assert(obj->is_objArray(), "expect object array");
objArrayOop array = objArrayOop(obj);

assert (ObjArrayMarkingStride > 0, "sanity");

// Split out tasks, as suggested in G1TaskQueueEntry docs. Avoid pushing tasks that
// are known to start beyond the array.
while ((1 << pow) > (int)ObjArrayMarkingStride && (slice*2 < G1TaskQueueEntry::slice_size())) {
pow--;
slice *= 2;
_task->push(G1TaskQueueEntry(array, slice - 1, pow));
}

int slice_size = 1 << pow;

int from = (slice - 1) * slice_size;
int to = slice * slice_size;

#ifdef ASSERT
int len = array->length();
assert (0 <= from && from < len, "from is sane: %d/%d", from, len);
assert (0 < to && to <= len, "to is sane: %d/%d", to, len);
#endif

return _task->scan_objArray(array, from, to);
void G1CMObjArrayProcessor::push_on_queue(G1TaskQueueEntry task) {
_task->push(task);
}
size_t G1CMObjArrayProcessor::scan_array(objArrayOop array, int from, int len) {
return _task->scan_objArray(array, from, from + len);
}
17 changes: 7 additions & 10 deletions src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,29 +26,26 @@
#define SHARE_GC_G1_G1CONCURRENTMARKOBJARRAYPROCESSOR_HPP

#include "oops/oopsHierarchy.hpp"
#include "gc/g1/g1ArraySlicer.hpp"
#include "gc/g1/g1TaskQueueEntry.hpp"

class G1CMTask;

// Helper class to mark through large objArrays during marking in an efficient way.
// Instead of pushing large object arrays, we push continuations onto the
// mark stack. These continuations are identified by having their LSB set.
// This allows incremental processing of large objects.
class G1CMObjArrayProcessor {
class G1CMObjArrayProcessor : public G1ArraySlicer {
private:
// Reference to the task for doing the actual work.
G1CMTask* _task;

public:
static bool should_be_sliced(oop obj);

G1CMObjArrayProcessor(G1CMTask* task) : _task(task) {
explicit G1CMObjArrayProcessor(G1CMTask* task) : _task(task) {
}

// Process the given continuation. Returns the number of words scanned.
size_t process_slice(oop ary, int chunk, int pow);
// Start processing the given objArrayOop by scanning the header and pushing its
// continuation.
size_t process_obj(oop obj);
void scan_metadata(objArrayOop array) override;
void push_on_queue(G1TaskQueueEntry task) override;
size_t scan_array(objArrayOop array, int from, int len) override;
};

#endif // SHARE_GC_G1_G1CONCURRENTMARKOBJARRAYPROCESSOR_HPP
38 changes: 0 additions & 38 deletions src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp
View file

This file was deleted.

141 changes: 41 additions & 100 deletions src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

#include "precompiled.hpp"
#include "gc/g1/g1Allocator.inline.hpp"
#include "gc/g1/g1ArraySlicer.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1CollectionSet.hpp"
#include "gc/g1/g1EvacFailureRegions.inline.hpp"
Expand Down Expand Up @@ -221,121 +222,61 @@ void G1ParScanThreadState::do_oop_evac(T* p) {
write_ref_field_post(p, obj);
}

class G1ScavengeArraySlicer : public G1ArraySlicer {
G1ScanEvacuatedObjClosure& _scanner;
bool _skip_enqueue;
G1ParScanThreadState* _par_scan;
G1CollectedHeap* _g1h;
public:
G1ScavengeArraySlicer(G1ScanEvacuatedObjClosure& scanner,
bool skip_enqueue,
G1ParScanThreadState* par_scan) :
_scanner(scanner),
_skip_enqueue(skip_enqueue),
_par_scan(par_scan),
_g1h(G1CollectedHeap::heap()) {}

void scan_metadata(objArrayOop array) override {
// TODO: According to old comment not needed, but may be cleaner?
if (Devirtualizer::do_metadata(&_scanner)) {
Devirtualizer::do_klass(&_scanner, array->klass());
}
}
void push_on_queue(G1TaskQueueEntry task) override {
_par_scan->push_on_queue(task);
}
size_t scan_array(objArrayOop array, int from, int len) override {
G1SkipCardEnqueueSetter x(&_scanner, _skip_enqueue);
array->oop_iterate_range(&_scanner, 0, len);
return len * (UseCompressedOops ? 2 : 1);
}
};

MAYBE_INLINE_EVACUATION
void G1ParScanThreadState::do_partial_array(oop obj, int slice, int pow) {
assert(_g1h->is_in_reserved(obj), "must be in heap.");
assert(obj->is_objArray(), "must be obj array");

objArrayOop array = objArrayOop(obj);

assert(ObjArrayMarkingStride > 0, "sanity");

// Split out tasks, as suggested in G1TaskQueueEntry docs. Avoid pushing tasks that
// are known to start beyond the array.
while ((1 << pow) > (int)ObjArrayMarkingStride && (slice * 2 < G1TaskQueueEntry::slice_size())) {
pow--;
slice *= 2;
push_on_queue(G1TaskQueueEntry(array, slice - 1, pow));
}

int slice_size = 1 << pow;

int from = (slice - 1) * slice_size;
int to = slice * slice_size;

#ifdef ASSERT
int len = array->length();
assert(0 <= from && from < len, "from is sane: %d/%d", from, len);
assert(0 < to && to <= len, "to is sane: %d/%d", to, len);
#endif

// Process claimed task. The length of to_array is not correct, but
// fortunately the iteration ignores the length field and just relies
// on start/end.
G1HeapRegionAttr dest_attr = _g1h->region_attr(array);
G1SkipCardEnqueueSetter x(&_scanner, dest_attr.is_new_survivor());
array->oop_iterate_range(&_scanner, from, to);
G1ScavengeArraySlicer slicer(_scanner, dest_attr.is_new_survivor(), this);
slicer.process_slice(array, slice, pow);
}

MAYBE_INLINE_EVACUATION
void G1ParScanThreadState::start_partial_objarray(G1HeapRegionAttr dest_attr,
oop obj) {
assert(obj->is_objArray(), "precondition");

objArrayOop array = objArrayOop(obj);
int len = array->length();

// Mark objArray klass metadata
// TODO: According to old comment not needed, but may be cleaner?
if (Devirtualizer::do_metadata(&_scanner)) {
Devirtualizer::do_klass(&_scanner, array->klass());
}

if (len <= (int) ObjArrayMarkingStride*2) {
// A few slices only, process directly
// Skip the card enqueue iff the object (to_array) is in survivor region.
// However, HeapRegion::is_survivor() is too expensive here.
// Instead, we use dest_attr.is_young() because the two values are always
// equal: successfully allocated young regions must be survivor regions.
assert(dest_attr.is_young() == _g1h->heap_region_containing(array)->is_survivor(), "must be");
G1SkipCardEnqueueSetter x(&_scanner, dest_attr.is_young());
array->oop_iterate_range(&_scanner, 0, len);
} else {
int bits = log2i_graceful(len);
// Compensate for non-power-of-two arrays, cover the array in excess:
if (len != (1 << bits)) bits++;

// Only allow full slices on the queue. This frees do_sliced_array() from checking from/to
// boundaries against array->length(), touching the array header on every slice.
//
// To do this, we cut the prefix in full-sized slices, and submit them on the queue.
// If the array is not divided in slice sizes, then there would be an irregular tail,
// which we will process separately.

int last_idx = 0;

int slice = 1;
int pow = bits;

// Handle overflow
if (pow >= 31) {
assert (pow == 31, "sanity");
pow--;
slice = 2;
last_idx = (1 << pow);
push_on_queue(G1TaskQueueEntry(array, 1, pow));
}

// Split out tasks, as suggested in G1TaskQueueEntry docs. Record the last
// successful right boundary to figure out the irregular tail.
while ((1 << pow) > (int)ObjArrayMarkingStride &&
(slice * 2 < G1TaskQueueEntry::slice_size())) {
pow--;
int left_slice = slice * 2 - 1;
int right_slice = slice * 2;
int left_slice_end = left_slice * (1 << pow);
if (left_slice_end < len) {
push_on_queue(G1TaskQueueEntry(array, left_slice, pow));
slice = right_slice;
last_idx = left_slice_end;
} else {
slice = left_slice;
}
}

// Process the irregular tail, if present
int from = last_idx;
if (from < len) {
// Skip the card enqueue iff the object (to_array) is in survivor region.
// However, HeapRegion::is_survivor() is too expensive here.
// Instead, we use dest_attr.is_young() because the two values are always
// equal: successfully allocated young regions must be survivor regions.
assert(dest_attr.is_young() == _g1h->heap_region_containing(array)->is_survivor(), "must be");
G1SkipCardEnqueueSetter x(&_scanner, dest_attr.is_young());
array->oop_iterate_range(&_scanner, from, len);
}
}
}
// Skip the card enqueue iff the object (to_array) is in survivor region.
// However, HeapRegion::is_survivor() is too expensive here.
// Instead, we use dest_attr.is_young() because the two values are always
// equal: successfully allocated young regions must be survivor regions.
assert(dest_attr.is_young() == _g1h->heap_region_containing(array)->is_survivor(), "must be");
G1ScavengeArraySlicer slicer(_scanner, dest_attr.is_young(), this);
slicer.process_objArray(array);
}

MAYBE_INLINE_EVACUATION
void G1ParScanThreadState::dispatch_task(G1TaskQueueEntry task) {
Expand Down

0 comments on commit 9367c14

Please sign in to comment.