From 7bb1d386e00c41bde3a279b8ab2fc3a05ec1bd7f Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 26 Feb 2024 00:13:35 +0000 Subject: [PATCH 01/16] Remove dead code for inelastic plabs --- src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index ffd22ca339b..a8dfff7b0ad 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -785,14 +785,6 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah " because min_size() is " SIZE_FORMAT, req.size(), r->index(), adjusted_size, req.min_size()); } } - } else if (req.is_lab_alloc() && req.type() == ShenandoahAllocRequest::_alloc_plab) { - - // inelastic PLAB - size_t size = req.size(); - size_t usable_free = get_usable_free_words(r->free()); - if (size <= usable_free) { - result = allocate_aligned_plab(size, req, r); - } } else { size_t size = req.size(); result = r->allocate(size, req); From 8bc436704e90dfd9cb31b3bdb126a674708eca4e Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 26 Feb 2024 00:17:12 +0000 Subject: [PATCH 02/16] Revert "Remove dead code for inelastic plabs" This reverts commit 7bb1d386e00c41bde3a279b8ab2fc3a05ec1bd7f. --- src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index a8dfff7b0ad..ffd22ca339b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -785,6 +785,14 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah " because min_size() is " SIZE_FORMAT, req.size(), r->index(), adjusted_size, req.min_size()); } } + } else if (req.is_lab_alloc() && req.type() == ShenandoahAllocRequest::_alloc_plab) { + + // inelastic PLAB + size_t size = req.size(); + size_t usable_free = get_usable_free_words(r->free()); + if (size <= usable_free) { + result = allocate_aligned_plab(size, req, r); + } } else { size_t size = req.size(); result = r->allocate(size, req); From 99cce53b3e6e51cee2ed71e0b7c7caa016a8ed4f Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 26 Feb 2024 21:18:32 +0000 Subject: [PATCH 03/16] Round LAB sizes down rather than up to force alignment When we round up, we introduce the risk that the new size exceeds the maximum LAB size, resulting in an assertion error. --- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 1 - .../share/gc/shenandoah/shenandoahHeap.cpp | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index ffd22ca339b..e80ebae6ec7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -1532,7 +1532,6 @@ HeapWord* ShenandoahFreeSet::allocate(ShenandoahAllocRequest& req, bool& in_new_ case ShenandoahAllocRequest::_alloc_plab: case ShenandoahAllocRequest::_alloc_gclab: case ShenandoahAllocRequest::_alloc_tlab: - in_new_region = false; assert(false, "Trying to allocate TLAB larger than the humongous threshold: " SIZE_FORMAT " > " SIZE_FORMAT, req.size(), ShenandoahHeapRegion::humongous_threshold_words()); return nullptr; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index bb571bc995e..425e78a24ed 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1115,7 +1115,7 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, b size_t unalignment = future_size % CardTable::card_size_in_words(); if (unalignment != 0) { - future_size = future_size - unalignment + CardTable::card_size_in_words(); + future_size -= unalignment; } // Record new heuristic value even if we take any shortcut. This captures @@ -1172,6 +1172,12 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, b } return plab->allocate(size); } else { + // TODO: Add smarts here, or at minimum, a command-line option to reduce the need for shared allocations. For example, + // maybe we should go ahead and retire the PLAB if: + // + // (used within the PLAB) * MIN2(ShenandoahPromoEvacWaste, ShenandoahOldEvacWaste) >= PLAB size. + // + // If there's still at least min_size() words available within the current plab, don't retire it. Let's gnaw // away on this plab as long as we can. Meanwhile, return nullptr to force this particular allocation request // to be satisfied with a shared allocation. By packing more promotions into the previously allocated PLAB, we @@ -1372,8 +1378,12 @@ HeapWord* ShenandoahHeap::allocate_new_plab(size_t min_size, // Align requested sizes to card sized multiples size_t words_in_card = CardTable::card_size_in_words(); size_t align_mask = ~(words_in_card - 1); - min_size = (min_size + words_in_card - 1) & align_mask; - word_size = (word_size + words_in_card - 1) & align_mask; + + // Need to round down rather than rounding up. Otherwise, might overrun max size. + min_size = min_size & align_mask; + word_size = word_size & align_mask; + assert(word_size >= min_size, "Requested PLAB is too small"); + ShenandoahAllocRequest req = ShenandoahAllocRequest::for_plab(min_size, word_size); // Note that allocate_memory() sets a thread-local flag to prohibit further promotions by this thread // if we are at risk of infringing on the old-gen evacuation budget. From 11b26bb8ba77374eca8327fee956b575b294cbf3 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 26 Feb 2024 21:34:54 +0000 Subject: [PATCH 04/16] Revert "Round LAB sizes down rather than up to force alignment" This reverts commit 99cce53b3e6e51cee2ed71e0b7c7caa016a8ed4f. --- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 1 + .../share/gc/shenandoah/shenandoahHeap.cpp | 16 +++------------- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index e80ebae6ec7..ffd22ca339b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -1532,6 +1532,7 @@ HeapWord* ShenandoahFreeSet::allocate(ShenandoahAllocRequest& req, bool& in_new_ case ShenandoahAllocRequest::_alloc_plab: case ShenandoahAllocRequest::_alloc_gclab: case ShenandoahAllocRequest::_alloc_tlab: + in_new_region = false; assert(false, "Trying to allocate TLAB larger than the humongous threshold: " SIZE_FORMAT " > " SIZE_FORMAT, req.size(), ShenandoahHeapRegion::humongous_threshold_words()); return nullptr; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 425e78a24ed..bb571bc995e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1115,7 +1115,7 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, b size_t unalignment = future_size % CardTable::card_size_in_words(); if (unalignment != 0) { - future_size -= unalignment; + future_size = future_size - unalignment + CardTable::card_size_in_words(); } // Record new heuristic value even if we take any shortcut. This captures @@ -1172,12 +1172,6 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, b } return plab->allocate(size); } else { - // TODO: Add smarts here, or at minimum, a command-line option to reduce the need for shared allocations. For example, - // maybe we should go ahead and retire the PLAB if: - // - // (used within the PLAB) * MIN2(ShenandoahPromoEvacWaste, ShenandoahOldEvacWaste) >= PLAB size. - // - // If there's still at least min_size() words available within the current plab, don't retire it. Let's gnaw // away on this plab as long as we can. Meanwhile, return nullptr to force this particular allocation request // to be satisfied with a shared allocation. By packing more promotions into the previously allocated PLAB, we @@ -1378,12 +1372,8 @@ HeapWord* ShenandoahHeap::allocate_new_plab(size_t min_size, // Align requested sizes to card sized multiples size_t words_in_card = CardTable::card_size_in_words(); size_t align_mask = ~(words_in_card - 1); - - // Need to round down rather than rounding up. Otherwise, might overrun max size. - min_size = min_size & align_mask; - word_size = word_size & align_mask; - assert(word_size >= min_size, "Requested PLAB is too small"); - + min_size = (min_size + words_in_card - 1) & align_mask; + word_size = (word_size + words_in_card - 1) & align_mask; ShenandoahAllocRequest req = ShenandoahAllocRequest::for_plab(min_size, word_size); // Note that allocate_memory() sets a thread-local flag to prohibit further promotions by this thread // if we are at risk of infringing on the old-gen evacuation budget. From 28a382bb90c4844e4899dd507a347619b6d74cba Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 13 Mar 2024 19:52:50 +0000 Subject: [PATCH 05/16] Make satb-mode Info logging less verbose --- src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 40c59bdb4c6..ba1261695d2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -819,7 +819,7 @@ void ShenandoahConcurrentGC::op_final_mark() { } else { // Not is_generational() if (!heap->collection_set()->is_empty()) { - LogTarget(Info, gc, ergo) lt; + LogTarget(Debug, gc, ergo) lt; if (lt.is_enabled()) { ResourceMark rm; LogStream ls(lt); From d88130000c764dacf08ec27723132dd2a3d968de Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Thu, 11 Apr 2024 13:21:59 +0000 Subject: [PATCH 06/16] Change behavior of max_old and min_old --- src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp index fe7feb68fb5..393b07d3bf0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp @@ -299,7 +299,9 @@ size_t ShenandoahGenerationSizer::max_size_for(ShenandoahGeneration* generation) case YOUNG: return max_young_size(); case OLD: - return min_young_size(); + // Officially, there is no limit on size of OLD, though the practical limit is heap size - min_young_size(). + // The pracital limit is enforced when we try to shrink young in order to expand old. + return ShenandoahHeap::heap()->max_capacity(); default: ShouldNotReachHere(); return 0; @@ -311,7 +313,9 @@ size_t ShenandoahGenerationSizer::min_size_for(ShenandoahGeneration* generation) case YOUNG: return min_young_size(); case OLD: - return ShenandoahHeap::heap()->max_capacity() - max_young_size(); + // Officially, there is no limit on size of OLD, though the practical limit is heap size - max_young_size(). + // The pracital limit is enforced when we try to expand young in order to shrink old. + return 0; default: ShouldNotReachHere(); return 0; From c2cb1b768f18b6ec529dd63c5982b0e9f7e2c078 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Thu, 11 Apr 2024 15:03:17 +0000 Subject: [PATCH 07/16] Revert "Change behavior of max_old and min_old" This reverts commit d88130000c764dacf08ec27723132dd2a3d968de. --- src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp index 393b07d3bf0..fe7feb68fb5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp @@ -299,9 +299,7 @@ size_t ShenandoahGenerationSizer::max_size_for(ShenandoahGeneration* generation) case YOUNG: return max_young_size(); case OLD: - // Officially, there is no limit on size of OLD, though the practical limit is heap size - min_young_size(). - // The pracital limit is enforced when we try to shrink young in order to expand old. - return ShenandoahHeap::heap()->max_capacity(); + return min_young_size(); default: ShouldNotReachHere(); return 0; @@ -313,9 +311,7 @@ size_t ShenandoahGenerationSizer::min_size_for(ShenandoahGeneration* generation) case YOUNG: return min_young_size(); case OLD: - // Officially, there is no limit on size of OLD, though the practical limit is heap size - max_young_size(). - // The pracital limit is enforced when we try to expand young in order to shrink old. - return 0; + return ShenandoahHeap::heap()->max_capacity() - max_young_size(); default: ShouldNotReachHere(); return 0; From 1c26ae0377f888d27f5281c0f3cf9a1e320ab32d Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Sun, 18 Aug 2024 15:57:42 +0000 Subject: [PATCH 08/16] Reorder concurrent cleanup following final mark This may allow us to reclaim immediate garbage more quickly. --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index e4ff32a71d3..4753017be48 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -152,24 +152,23 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { return false; } - // Concurrent stack processing - if (heap->is_evacuation_in_progress()) { - entry_thread_roots(); - } - - // Process weak roots that might still point to regions that would be broken by cleanup + // Process weak roots that might still point to regions that would be broken by cleanup. This must precede cleanup. if (heap->is_concurrent_weak_root_in_progress()) { entry_weak_refs(); entry_weak_roots(); } - // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim - // the space. This would be the last action if there is nothing to evacuate. Note that - // we will not age young-gen objects in the case that we skip evacuation. + // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim the space. We do this before + // concurrent roots and concurrent class unloading so as to expedite recycling of immediate garbage. Note that + // we will not age young-gen objects in the case that we skip evacuation for abbreviated cycles. entry_cleanup_early(); - heap->free_set()->log_status_under_lock(); + // Concurrent stack processing + if (heap->is_evacuation_in_progress()) { + entry_thread_roots(); + } + // Perform concurrent class unloading if (heap->unload_classes() && heap->is_concurrent_weak_root_in_progress()) { From 634ef66d7b242ed22fe2ec7feee71754f304255b Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 19 Aug 2024 01:23:32 +0000 Subject: [PATCH 09/16] Do not log_status() for free set after cleanup early We just reported freeset status after rebuilding freeset in final mark. There will be heavy contention on the heaplock at the start of evacuation as many mutator and worker threads prep their GCLAB and PLAB buffers for evacuation. We avoid some lock contention by not reporting freset status here. --- src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 4753017be48..34018c3f6d0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -162,7 +162,13 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { // concurrent roots and concurrent class unloading so as to expedite recycling of immediate garbage. Note that // we will not age young-gen objects in the case that we skip evacuation for abbreviated cycles. entry_cleanup_early(); + +#ifdef KELVIN_DEPRECATE + // We just dumped the free set after rebuilding free set in final_mark. Let's not dump it again. + // There may be some contention with mutator and GC worker threads who are trying to begin their evacuation + // efforts, and would prefer not to grab the heap lock right here. heap->free_set()->log_status_under_lock(); +#endif // Concurrent stack processing if (heap->is_evacuation_in_progress()) { From 4198b75692b69a003c80e28d26f76af1b3724ddb Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 19 Aug 2024 01:57:06 +0000 Subject: [PATCH 10/16] Notify waiting allocators after cleanup of immediate garbage --- .../share/gc/shenandoah/shenandoahConcurrentGC.cpp | 5 ++++- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 13 ++++++++++--- .../share/gc/shenandoah/shenandoahFreeSet.hpp | 7 +++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 34018c3f6d0..e108d899358 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -1066,7 +1066,10 @@ void ShenandoahConcurrentGC::op_strong_roots() { } void ShenandoahConcurrentGC::op_cleanup_early() { - ShenandoahHeap::heap()->free_set()->recycle_trash(); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (heap->free_set()->recycle_trash()) { + heap->control_thread()->notify_alloc_failure_waiters(); + } } void ShenandoahConcurrentGC::op_evacuate() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 5fcaf0c673c..00b88f79152 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -1230,13 +1230,17 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { return _heap->get_region(beg)->bottom(); } -void ShenandoahFreeSet::try_recycle_trashed(ShenandoahHeapRegion* r) { +bool ShenandoahFreeSet::try_recycle_trashed(ShenandoahHeapRegion* r) { + bool result = false; if (r->is_trash()) { r->recycle(); + result = true; } + return true; } -void ShenandoahFreeSet::recycle_trash() { +bool ShenandoahFreeSet::recycle_trash() { + bool result = false; // lock is not reentrable, check we don't have it shenandoah_assert_not_heaplocked(); @@ -1256,9 +1260,12 @@ void ShenandoahFreeSet::recycle_trash() { ShenandoahHeapLocker locker(_heap->lock()); const jlong deadline = os::javaTimeNanos() + deadline_ns; while (idx < count && os::javaTimeNanos() < deadline) { - try_recycle_trashed(_trash_regions[idx++]); + if (try_recycle_trashed(_trash_regions[idx++])) { + result = true; + } } } + return result; } void ShenandoahFreeSet::flip_to_old_gc(ShenandoahHeapRegion* r) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index 5f69ec47cfd..d0993a9b7e3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -330,7 +330,9 @@ class ShenandoahFreeSet : public CHeapObj { void flip_to_old_gc(ShenandoahHeapRegion* r); void clear_internal(); - void try_recycle_trashed(ShenandoahHeapRegion *r); + + // If region r is a trash region, recycle it, returning true iff r was recycled. + bool try_recycle_trashed(ShenandoahHeapRegion *r); // Returns true iff this region is entirely available, either because it is empty() or because it has been found to represent // immediate trash and we'll be able to immediately recycle it. Note that we cannot recycle immediate trash if @@ -410,7 +412,8 @@ class ShenandoahFreeSet : public CHeapObj { // for evacuation, invoke this to make regions available for mutator allocations. void move_regions_from_collector_to_mutator(size_t cset_regions); - void recycle_trash(); + // Recycle any trash that is known to the freeset, returning true if any trash was recycled. + bool recycle_trash(); // Acquire heap lock and log status, assuming heap lock is not acquired by the caller. void log_status_under_lock(); From 408f78937e9c21f452ff007359e207358ace193a Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 19 Aug 2024 15:26:59 +0000 Subject: [PATCH 11/16] Do not clear alloc-failure flag after cleanup early If we notify waiting mutators after immediate garbage is reclaimed, do not clear the alloc-failure flag. Otherwise, this hides the fact that alloc failure occurred during this GC. --- .../share/gc/shenandoah/shenandoahConcurrentGC.cpp | 2 +- src/hotspot/share/gc/shenandoah/shenandoahController.cpp | 8 +++++--- src/hotspot/share/gc/shenandoah/shenandoahController.hpp | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index e108d899358..8bab8837eeb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -1068,7 +1068,7 @@ void ShenandoahConcurrentGC::op_strong_roots() { void ShenandoahConcurrentGC::op_cleanup_early() { ShenandoahHeap* heap = ShenandoahHeap::heap(); if (heap->free_set()->recycle_trash()) { - heap->control_thread()->notify_alloc_failure_waiters(); + heap->control_thread()->notify_alloc_failure_waiters(false); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahController.cpp b/src/hotspot/share/gc/shenandoah/shenandoahController.cpp index 6d6d21c4066..4467dbed8b5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahController.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahController.cpp @@ -92,9 +92,11 @@ void ShenandoahController::handle_alloc_failure_evac(size_t words) { heap->cancel_gc(GCCause::_shenandoah_allocation_failure_evac); } -void ShenandoahController::notify_alloc_failure_waiters() { - _alloc_failure_gc.unset(); - _humongous_alloc_failure_gc.unset(); +void ShenandoahController::notify_alloc_failure_waiters(bool clear_alloc_failure) { + if (clear_alloc_failure) { + _alloc_failure_gc.unset(); + _humongous_alloc_failure_gc.unset(); + } MonitorLocker ml(&_alloc_failure_waiters_lock); ml.notify_all(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahController.hpp b/src/hotspot/share/gc/shenandoah/shenandoahController.hpp index 6c28ff4e969..87667937187 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahController.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahController.hpp @@ -79,7 +79,7 @@ class ShenandoahController: public ConcurrentGCThread { bool try_set_alloc_failure_gc(bool is_humongous); // Notify threads waiting for GC to complete. - void notify_alloc_failure_waiters(); + void notify_alloc_failure_waiters(bool clear_alloc_failure = true); // True if allocation failure flag has been set. bool is_alloc_failure_gc(); From 9e9c54cfd1559391c916088b010082da2e755773 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 19 Aug 2024 16:31:04 +0000 Subject: [PATCH 12/16] Improve comments regarding when we can recycle trashed regions --- src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp | 2 +- src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 8bab8837eeb..40b45bc84c8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -751,7 +751,7 @@ void ShenandoahConcurrentGC::op_final_mark() { // Arm nmethods/stack for concurrent processing if (!heap->collection_set()->is_empty()) { - // Iff objects will be evaluated, arm the nmethod barriers. These will be disarmed + // Iff objects will be evacuated, arm the nmethod barriers. These will be disarmed // under the same condition (established in prepare_concurrent_roots) after strong // root evacuation has completed (see op_strong_roots). ShenandoahCodeRoots::arm_nmethods_for_evac(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 00b88f79152..e719ce1cb37 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -111,6 +111,9 @@ ShenandoahRegionPartitions::ShenandoahRegionPartitions(size_t max_regions, Shena } inline bool ShenandoahFreeSet::can_allocate_from(ShenandoahHeapRegion *r) const { + // This test for trash regions is conservative. Strictly, we only need to assure that concurrent weak reference processing + // is not under way. That finishes long before concurrent weak root processing. It is ok to be conservative. At the + // end of weak reference processing, we recycle trashed regions en masse. return r->is_empty() || (r->is_trash() && !_heap->is_concurrent_weak_root_in_progress()); } From 84619391114f6d42b8f69e7c2c5ddc902860adc5 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 20 Aug 2024 13:34:28 +0000 Subject: [PATCH 13/16] Do not cancel GC on allocation failure if immediate garbage is adequate --- .../share/gc/shenandoah/shenandoahController.cpp | 11 ++++++++--- .../share/gc/shenandoah/shenandoahController.hpp | 4 ++++ src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp | 1 + .../share/gc/shenandoah/shenandoahGeneration.cpp | 3 +++ src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 3 +++ .../share/gc/shenandoah/shenandoahOldGeneration.cpp | 3 +++ 6 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahController.cpp b/src/hotspot/share/gc/shenandoah/shenandoahController.cpp index 4467dbed8b5..925b227d4c7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahController.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahController.cpp @@ -53,6 +53,10 @@ size_t ShenandoahController::get_gc_id() { return Atomic::load(&_gc_id); } +void ShenandoahController::anticipate_immediate_garbage(size_t anticipated_immediate_garbage) { + Atomic::store(&_anticipated_immediate_garbage, anticipated_immediate_garbage); +} + void ShenandoahController::handle_alloc_failure(ShenandoahAllocRequest& req, bool block) { ShenandoahHeap* heap = ShenandoahHeap::heap(); @@ -65,11 +69,12 @@ void ShenandoahController::handle_alloc_failure(ShenandoahAllocRequest& req, boo req.type_string(), byte_size_in_proper_unit(req.size() * HeapWordSize), proper_unit_for_byte_size(req.size() * HeapWordSize)); - // Now that alloc failure GC is scheduled, we can abort everything else - heap->cancel_gc(GCCause::_allocation_failure); + if (Atomic::load(&_anticipated_immediate_garbage) < req.size()) { + // Now that alloc failure GC is scheduled, we can abort everything else + heap->cancel_gc(GCCause::_allocation_failure); + } } - if (block) { MonitorLocker ml(&_alloc_failure_waiters_lock); while (is_alloc_failure_gc()) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahController.hpp b/src/hotspot/share/gc/shenandoah/shenandoahController.hpp index 87667937187..53372501bce 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahController.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahController.hpp @@ -43,6 +43,8 @@ class ShenandoahController: public ConcurrentGCThread { shenandoah_padding(1); volatile size_t _gc_id; shenandoah_padding(2); + volatile size_t _anticipated_immediate_garbage; + shenandoah_padding(3); protected: ShenandoahSharedFlag _alloc_failure_gc; @@ -71,6 +73,8 @@ class ShenandoahController: public ConcurrentGCThread { // until another cycle runs and clears the alloc failure gc flag. void handle_alloc_failure(ShenandoahAllocRequest& req, bool block); + void anticipate_immediate_garbage(size_t anticipated_immediate_garbage_words); + // Invoked for allocation failures during evacuation. This cancels // the collection cycle without blocking. void handle_alloc_failure_evac(size_t words); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index e719ce1cb37..de7970f592e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -1268,6 +1268,7 @@ bool ShenandoahFreeSet::recycle_trash() { } } } + _heap->control_thread()->anticipate_immediate_garbage((size_t) 0); return result; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 4b4662a7702..fd0a5f10c0a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -753,6 +753,9 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { // We are preparing for evacuation. At this time, we ignore cset region tallies. size_t first_old, last_old, num_old; heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old); + size_t anticipated_immediate_garbage = (old_cset_regions + young_cset_regions) * ShenandoahHeapRegion::region_size_words(); + heap->control_thread()->anticipate_immediate_garbage(anticipated_immediate_garbage); + // Free set construction uses reserve quantities, because they are known to be valid here heap->free_set()->finish_rebuild(young_cset_regions, old_cset_regions, num_old, true); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 0ef91b2d81a..bc124918cdf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2462,6 +2462,9 @@ void ShenandoahHeap::rebuild_free_set(bool concurrent) { size_t young_cset_regions, old_cset_regions; size_t first_old_region, last_old_region, old_region_count; _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old_region, last_old_region, old_region_count); + size_t anticipated_immediate_garbage = (old_cset_regions + young_cset_regions) * ShenandoahHeapRegion::region_size_words(); + control_thread()->anticipate_immediate_garbage(anticipated_immediate_garbage); + // If there are no old regions, first_old_region will be greater than last_old_region assert((first_old_region > last_old_region) || ((last_old_region + 1 - first_old_region >= old_region_count) && diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 42fb4e2cd9d..38c0473f667 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -474,6 +474,9 @@ void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent size_t cset_young_regions, cset_old_regions; size_t first_old, last_old, num_old; heap->free_set()->prepare_to_rebuild(cset_young_regions, cset_old_regions, first_old, last_old, num_old); + size_t anticipated_immediate_garbage = (cset_young_regions + cset_old_regions) * ShenandoahHeapRegion::region_size_words(); + heap->control_thread()->anticipate_immediate_garbage(anticipated_immediate_garbage); + // This is just old-gen completion. No future budgeting required here. The only reason to rebuild the freeset here // is in case there was any immediate old garbage identified. heap->free_set()->finish_rebuild(cset_young_regions, cset_old_regions, num_old); From 02fd5c119ecb562c91c174aeef032e990343f86d Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 21 Aug 2024 15:05:36 +0000 Subject: [PATCH 14/16] Whitespace --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index bc124918cdf..51904d9a762 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2435,9 +2435,7 @@ void ShenandoahHeap::update_heap_region_states(bool concurrent) { ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_refs_update_region_states : ShenandoahPhaseTimings::degen_gc_final_update_refs_update_region_states); - final_update_refs_update_region_states(); - assert_pinned_region_status(); } From 5c0bb7dcdff62194aa77451d03406ef04ada2d41 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 21 Aug 2024 15:17:42 +0000 Subject: [PATCH 15/16] Experiment with white space jcheck --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 51904d9a762..cbcd22c053f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2435,8 +2435,10 @@ void ShenandoahHeap::update_heap_region_states(bool concurrent) { ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_refs_update_region_states : ShenandoahPhaseTimings::degen_gc_final_update_refs_update_region_states); - final_update_refs_update_region_states(); - assert_pinned_region_status(); + + final_update_refs_update_region_states(); + + assert_pinned_region_status(); } { From f11a3c816662fbf1114dd4f1c50090ddb827b2ee Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Fri, 30 Aug 2024 14:57:38 +0000 Subject: [PATCH 16/16] Fix whitespace --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index cbcd22c053f..342c023d180 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2464,7 +2464,7 @@ void ShenandoahHeap::rebuild_free_set(bool concurrent) { _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old_region, last_old_region, old_region_count); size_t anticipated_immediate_garbage = (old_cset_regions + young_cset_regions) * ShenandoahHeapRegion::region_size_words(); control_thread()->anticipate_immediate_garbage(anticipated_immediate_garbage); - + // If there are no old regions, first_old_region will be greater than last_old_region assert((first_old_region > last_old_region) || ((last_old_region + 1 - first_old_region >= old_region_count) &&