Fix 32 bit JVM

src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp

@@ -105,6 +105,9 @@ G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) :
   _gc_par_phases[UpdateDerivedPointers] = new WorkerDataArray<double>("UpdateDerivedPointers", "Update Derived Pointers (ms):", max_gc_threads);
 #endif
   _gc_par_phases[EagerlyReclaimHumongousObjects] = new WorkerDataArray<double>("EagerlyReclaimHumongousObjects", "Eagerly Reclaim Humongous Objects (ms):", max_gc_threads);
+#ifndef _LP64
+  _gc_par_phases[RestorePreservedMarks] = new WorkerDataArray<double>("RestorePreservedMarks", "Restore Preserved Marks (ms):", max_gc_threads);
+#endif
   _gc_par_phases[ProcessEvacuationFailedRegions] = new WorkerDataArray<double>("ProcessEvacuationFailedRegions", "Process Evacuation Failed Regions (ms):", max_gc_threads);
 
   _gc_par_phases[ScanHR]->create_thread_work_items("Scanned Cards:", ScanHRScannedCards);
@@ -511,6 +514,9 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set(bool evacuation_failed
   debug_time("Post Evacuate Cleanup 2", _cur_post_evacuate_cleanup_2_time_ms);
   if (evacuation_failed) {
     debug_phase(_gc_par_phases[RecalculateUsed], 1);
+#ifndef _LP64
+    debug_phase(_gc_par_phases[RestorePreservedMarks], 1);
+#endif
     debug_phase(_gc_par_phases[ProcessEvacuationFailedRegions], 1);
   }
 #if COMPILER2_OR_JVMCI

src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp

@@ -87,6 +87,9 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
     UpdateDerivedPointers,
 #endif
     EagerlyReclaimHumongousObjects,
+#ifndef _LP64
+    RestorePreservedMarks,
+#endif
     ProcessEvacuationFailedRegions,
     ResetMarkingState,
     NoteStartOfMark,

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -35,6 +35,9 @@
 #include "gc/g1/g1YoungGCAllocationFailureInjector.inline.hpp"
 #include "gc/shared/continuationGCSupport.inline.hpp"
 #include "gc/shared/partialArrayTaskStepper.inline.hpp"
+#ifndef _LP64
+#include "gc/shared/preservedMarks.inline.hpp"
+#endif
 #include "gc/shared/stringdedup/stringDedup.hpp"
 #include "gc/shared/taskqueue.inline.hpp"
 #include "memory/allocation.inline.hpp"
@@ -55,6 +58,7 @@
 
 G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
                                            G1RedirtyCardsQueueSet* rdcqs,
+                                           NOT_LP64(PreservedMarks* preserved_marks COMMA)
                                            uint worker_id,
                                            uint num_workers,
                                            G1CollectionSet* collection_set,
@@ -84,6 +88,7 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
     _numa(g1h->numa()),
     _obj_alloc_stat(nullptr),
     ALLOCATION_FAILURE_INJECTOR_ONLY(_allocation_failure_inject_counter(0) COMMA)
+    NOT_LP64(_preserved_marks(preserved_marks) COMMA)
     _evacuation_failed_info(),
     _evac_failure_regions(evac_failure_regions),
     _evac_failure_enqueued_cards(0)
@@ -574,6 +579,7 @@ G1ParScanThreadState* G1ParScanThreadStateSet::state_for_worker(uint worker_id)
   if (_states[worker_id] == nullptr) {
     _states[worker_id] =
       new G1ParScanThreadState(_g1h, rdcqs(),
+                               NOT_LP64(_preserved_marks_set.get(worker_id) COMMA)
                                worker_id,
                                _num_workers,
                                _collection_set,
@@ -645,6 +651,10 @@ oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, siz
     // evacuation failure recovery.
     _g1h->mark_evac_failure_object(_worker_id, old, word_sz);
 
+#ifndef _LP64
+    _preserved_marks->push_if_necessary(old, m);
+#endif
+
     ContinuationGCSupport::transform_stack_chunk(old);
 
     _evacuation_failed_info.register_copy_failure(word_sz);
@@ -702,12 +712,16 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h,
     _g1h(g1h),
     _collection_set(collection_set),
     _rdcqs(G1BarrierSet::dirty_card_queue_set().allocator()),
+    NOT_LP64(_preserved_marks_set(true /* in_c_heap */) COMMA)
     _states(NEW_C_HEAP_ARRAY(G1ParScanThreadState*, num_workers, mtGC)),
     _rdc_buffers(NEW_C_HEAP_ARRAY(BufferNodeList, num_workers, mtGC)),
     _surviving_young_words_total(NEW_C_HEAP_ARRAY(size_t, collection_set->young_region_length() + 1, mtGC)),
     _num_workers(num_workers),
     _flushed(false),
     _evac_failure_regions(evac_failure_regions) {
+#ifndef _LP64
+  _preserved_marks_set.init(num_workers);
+#endif
   for (uint i = 0; i < num_workers; ++i) {
     _states[i] = nullptr;
     _rdc_buffers[i] = BufferNodeList();
@@ -720,4 +734,7 @@ G1ParScanThreadStateSet::~G1ParScanThreadStateSet() {
   FREE_C_HEAP_ARRAY(G1ParScanThreadState*, _states);
   FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_total);
   FREE_C_HEAP_ARRAY(BufferNodeList, _rdc_buffers);
+#ifndef _LP64
+  _preserved_marks_set.reclaim();
+#endif
 }

src/hotspot/share/gc/g1/g1ParScanThreadState.hpp

@@ -33,6 +33,9 @@
 #include "gc/shared/ageTable.hpp"
 #include "gc/shared/copyFailedInfo.hpp"
 #include "gc/shared/partialArrayTaskStepper.hpp"
+#ifndef _LP64
+#include "gc/shared/preservedMarks.hpp"
+#endif
 #include "gc/shared/stringdedup/stringDedup.hpp"
 #include "gc/shared/taskqueue.hpp"
 #include "memory/allocation.hpp"
@@ -103,6 +106,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
   // Per-thread evacuation failure data structures.
   ALLOCATION_FAILURE_INJECTOR_ONLY(size_t _allocation_failure_inject_counter;)
 
+  NOT_LP64(PreservedMarks* _preserved_marks;)
   EvacuationFailedInfo _evacuation_failed_info;
   G1EvacFailureRegions* _evac_failure_regions;
   // Number of additional cards into evacuation failed regions enqueued into
@@ -121,6 +125,7 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
 public:
   G1ParScanThreadState(G1CollectedHeap* g1h,
                        G1RedirtyCardsQueueSet* rdcqs,
+                       NOT_LP64(PreservedMarks* preserved_marks COMMA)
                        uint worker_id,
                        uint num_workers,
                        G1CollectionSet* collection_set,
@@ -240,6 +245,7 @@ class G1ParScanThreadStateSet : public StackObj {
   G1CollectedHeap* _g1h;
   G1CollectionSet* _collection_set;
   G1RedirtyCardsQueueSet _rdcqs;
+  NOT_LP64(PreservedMarksSet _preserved_marks_set;)
   G1ParScanThreadState** _states;
   BufferNodeList* _rdc_buffers;
   size_t* _surviving_young_words_total;
@@ -256,6 +262,7 @@ class G1ParScanThreadStateSet : public StackObj {
 
   G1RedirtyCardsQueueSet* rdcqs() { return &_rdcqs; }
   BufferNodeList* rdc_buffers() { return _rdc_buffers; }
+  NOT_LP64(PreservedMarksSet* preserved_marks_set() { return &_preserved_marks_set; })
 
   void flush_stats();
   void record_unused_optional_region(HeapRegion* hr);

src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp

@@ -41,6 +41,9 @@
 #include "gc/g1/g1RemSet.hpp"
 #include "gc/g1/g1YoungGCPostEvacuateTasks.hpp"
 #include "gc/shared/bufferNode.hpp"
+#ifndef _LP64
+#include "gc/shared/preservedMarks.inline.hpp"
+#endif
 #include "jfr/jfrEvents.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/compressedOops.inline.hpp"
@@ -250,7 +253,11 @@ class G1PostEvacuateCollectionSetCleanupTask1::RestoreEvacFailureRegionsTask : p
       {
         // Process marked object.
         assert(obj->is_forwarded() && obj->forwardee() == obj, "must be self-forwarded");
+#ifndef _LP64
+        obj->init_mark();
+#else
         obj->unset_self_forwarded();
+#endif
         hr->update_bot_for_block(obj_addr, obj_end_addr);
 
         // Statistics
@@ -475,6 +482,29 @@ class G1PostEvacuateCollectionSetCleanupTask2::EagerlyReclaimHumongousObjectsTas
   }
 };
 
+#ifndef _LP64
+class G1PostEvacuateCollectionSetCleanupTask2::RestorePreservedMarksTask : public G1AbstractSubTask {
+  PreservedMarksSet* _preserved_marks;
+  WorkerTask* _task;
+
+public:
+  RestorePreservedMarksTask(PreservedMarksSet* preserved_marks) :
+    G1AbstractSubTask(G1GCPhaseTimes::RestorePreservedMarks),
+    _preserved_marks(preserved_marks),
+    _task(preserved_marks->create_task()) { }
+
+  virtual ~RestorePreservedMarksTask() {
+    delete _task;
+  }
+
+  double worker_cost() const override {
+    return _preserved_marks->num();
+  }
+
+  void do_work(uint worker_id) override { _task->work(worker_id); }
+};
+#endif
+
 class RedirtyLoggedCardTableEntryClosure : public G1CardTableEntryClosure {
   size_t _num_dirtied;
   G1CollectedHeap* _g1h;
@@ -949,6 +979,9 @@ G1PostEvacuateCollectionSetCleanupTask2::G1PostEvacuateCollectionSetCleanupTask2
   }
 
   if (evac_failure_regions->has_regions_evac_failed()) {
+#ifndef _LP64
+    add_parallel_task(new RestorePreservedMarksTask(per_thread_states->preserved_marks_set()));
+#endif
     add_parallel_task(new ProcessEvacuationFailedRegionsTask(evac_failure_regions));
   }
   add_parallel_task(new RedirtyLoggedCardsTask(evac_failure_regions,

src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp

@@ -56,6 +56,7 @@ class G1PostEvacuateCollectionSetCleanupTask1 : public G1BatchedTask {
 // - Update Derived Pointers (s)
 // - Clear Retained Region Data (on evacuation failure)
 // - Redirty Logged Cards
+// - Restore Preserved Marks (on evacuation failure, only 32 bit JVM)
 // - Free Collection Set
 // - Resize TLABs
 class G1PostEvacuateCollectionSetCleanupTask2 : public G1BatchedTask {
@@ -66,6 +67,9 @@ class G1PostEvacuateCollectionSetCleanupTask2 : public G1BatchedTask {
 
   class ProcessEvacuationFailedRegionsTask;
   class RedirtyLoggedCardsTask;
+#ifndef _LP64
+  class RestorePreservedMarksTask;
+#endif
   class FreeCollectionSetTask;
   class ResizeTLABsTask;
 

src/hotspot/share/gc/serial/defNewGeneration.cpp

@@ -40,6 +40,9 @@
 #include "gc/shared/gcTimer.hpp"
 #include "gc/shared/gcTrace.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
+#ifndef _LP64
+#include "gc/shared/preservedMarks.inline.hpp"
+#endif
 #include "gc/shared/referencePolicy.hpp"
 #include "gc/shared/referenceProcessorPhaseTimes.hpp"
 #include "gc/shared/space.inline.hpp"
@@ -243,6 +246,7 @@ DefNewGeneration::DefNewGeneration(ReservedSpace rs,
                                    size_t max_size,
                                    const char* policy)
   : Generation(rs, initial_size),
+    NOT_LP64(_preserved_marks_set(false /* in_c_heap */) COMMA)
     _promo_failure_drain_in_progress(false),
     _should_allocate_from_space(false),
     _string_dedup_requests()
@@ -679,6 +683,10 @@ void DefNewGeneration::collect(bool   full,
 
   age_table()->clear();
   to()->clear(SpaceDecorator::Mangle);
+#ifndef _LP64
+  // The preserved marks should be empty at the start of the GC.
+  _preserved_marks_set.init(1);
+#endif
 
   assert(heap->no_allocs_since_save_marks(),
          "save marks have not been newly set.");
@@ -779,6 +787,11 @@ void DefNewGeneration::collect(bool   full,
     NOT_PRODUCT(heap->reset_promotion_should_fail();)
   }
 
+#ifndef _LP64
+  // We should have processed and cleared all the preserved marks.
+  _preserved_marks_set.reclaim();
+#endif
+
   heap->trace_heap_after_gc(_gc_tracer);
 
   _gc_timer->register_gc_end();
@@ -801,19 +814,36 @@ void DefNewGeneration::remove_forwarding_pointers() {
   struct ResetForwardedMarkWord : ObjectClosure {
     void do_object(oop obj) override {
       if (obj->is_self_forwarded()) {
+#ifndef _LP64
+        obj->init_mark();
+#else
         obj->unset_self_forwarded();
+#endif
       }
     }
   } cl;
   eden()->object_iterate(&cl);
   from()->object_iterate(&cl);
+
+#ifndef _LP64
+  restore_preserved_marks();
+#endif
+}
+
+#ifndef _LP64
+void DefNewGeneration::restore_preserved_marks() {
+  _preserved_marks_set.restore(nullptr);
 }
+#endif
 
 void DefNewGeneration::handle_promotion_failure(oop old) {
   log_debug(gc, promotion)("Promotion failure size = " SIZE_FORMAT ") ", old->size());
 
   _promotion_failed = true;
   _promotion_failed_info.register_copy_failure(old->size());
+#ifndef _LP64
+  _preserved_marks_set.get()->push_if_necessary(old, old->mark());
+#endif
 
   ContinuationGCSupport::transform_stack_chunk(old);
 

src/hotspot/share/gc/serial/defNewGeneration.hpp

@@ -32,6 +32,9 @@
 #include "gc/shared/copyFailedInfo.hpp"
 #include "gc/shared/gc_globals.hpp"
 #include "gc/shared/generationCounters.hpp"
+#ifndef _LP64
+#include "gc/shared/preservedMarks.hpp"
+#endif
 #include "gc/shared/stringdedup/stringDedup.hpp"
 #include "gc/shared/tlab_globals.hpp"
 #include "utilities/align.hpp"
@@ -99,6 +102,13 @@ class DefNewGeneration: public Generation {
   // therefore we must remove their forwarding pointers.
   void remove_forwarding_pointers();
 
+#ifndef _LP64
+  virtual void restore_preserved_marks();
+
+  // Preserved marks
+  PreservedMarksSet _preserved_marks_set;
+#endif
+
   Stack<oop, mtGC> _promo_failure_scan_stack;
   void drain_promo_failure_scan_stack(void);
   bool _promo_failure_drain_in_progress;

src/hotspot/share/oops/markWord.hpp

@@ -147,8 +147,12 @@ class markWord {
     return (mask_bits(value(), lock_mask_in_place) == marked_value);
   }
   bool is_forwarded() const {
+#ifndef _LP64
+    return is_marked();
+#else
     // Returns true for normal forwarded (0b011) and self-forwarded (0b1xx).
     return mask_bits(value(), lock_mask_in_place | self_fwd_mask_in_place) >= static_cast<intptr_t>(marked_value);
+#endif
   }
   bool is_neutral()  const {
     return (mask_bits(value(), lock_mask_in_place) == unlocked_value);
@@ -269,14 +273,17 @@ class markWord {
   inline void* decode_pointer() { return (void*)clear_lock_bits().value(); }
 
   inline bool self_forwarded() const {
+    NOT_LP64(assert(LockingMode != LM_LEGACY, "incorrect with LM_LEGACY on 32 bit");)
     return mask_bits(value(), self_fwd_mask_in_place) != 0;
   }
 
   inline markWord set_self_forwarded() const {
+    NOT_LP64(assert(LockingMode != LM_LEGACY, "incorrect with LM_LEGACY on 32 bit");)
     return markWord(value() | self_fwd_mask_in_place);
   }
 
   inline markWord unset_self_forwarded() const {
+    NOT_LP64(assert(LockingMode != LM_LEGACY, "incorrect with LM_LEGACY on 32 bit");)
     return markWord(value() & ~self_fwd_mask_in_place);
   }
 };