diff --git a/src/plan/marksweep/global.rs b/src/plan/marksweep/global.rs index 41dde00dab..8715e34baf 100644 --- a/src/plan/marksweep/global.rs +++ b/src/plan/marksweep/global.rs @@ -65,6 +65,10 @@ impl Plan for MarkSweep { self.common.release(tls, true); } + fn end_of_gc(&mut self, _tls: VMWorkerThread) { + self.ms.end_of_gc(); + } + fn collection_required(&self, space_full: bool, _space: Option>) -> bool { self.base().collection_required(self, space_full) } diff --git a/src/policy/marksweepspace/malloc_ms/global.rs b/src/policy/marksweepspace/malloc_ms/global.rs index 8d42b74f0d..91d9b8a698 100644 --- a/src/policy/marksweepspace/malloc_ms/global.rs +++ b/src/policy/marksweepspace/malloc_ms/global.rs @@ -492,6 +492,8 @@ impl MallocSpace { self.scheduler.work_buckets[WorkBucketStage::Release].bulk_add(work_packets); } + pub fn end_of_gc(&mut self) {} + pub fn sweep_chunk(&self, chunk_start: Address) { // Call the relevant sweep function depending on the location of the mark bits match *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC { diff --git a/src/policy/marksweepspace/native_ms/block.rs b/src/policy/marksweepspace/native_ms/block.rs index 625a82d851..e9af1c8272 100644 --- a/src/policy/marksweepspace/native_ms/block.rs +++ b/src/policy/marksweepspace/native_ms/block.rs @@ -188,6 +188,7 @@ impl Block { } pub fn store_block_cell_size(&self, size: usize) { + debug_assert_ne!(size, 0); unsafe { Block::SIZE_TABLE.store::(self.start(), size) } } @@ -219,23 +220,14 @@ impl Block { Self::MARK_TABLE.store_atomic::(self.start(), state, Ordering::SeqCst); } - /// Release this block if it is unmarked. Return true if the block is release. + /// Release this block if it is unmarked. Return true if the block is released. pub fn attempt_release(self, space: &MarkSweepSpace) -> bool { match self.get_state() { - BlockState::Unallocated => false, + // We should not have unallocated blocks in a block list + BlockState::Unallocated => unreachable!(), BlockState::Unmarked => { - unsafe { - let block_list = loop { - let list = self.load_block_list(); - (*list).lock(); - if list == self.load_block_list() { - break list; - } - (*list).unlock(); - }; - (*block_list).remove(self); - (*block_list).unlock(); - } + let block_list = self.load_block_list(); + unsafe { &mut *block_list }.remove(self); space.release_block(self); true } @@ -282,6 +274,7 @@ impl Block { /// that we need to use this method correctly. fn simple_sweep(&self) { let cell_size = self.load_block_cell_size(); + debug_assert_ne!(cell_size, 0); let mut cell = self.start(); let mut last = unsafe { Address::zero() }; while cell + cell_size <= self.start() + Block::BYTES { diff --git a/src/policy/marksweepspace/native_ms/block_list.rs b/src/policy/marksweepspace/native_ms/block_list.rs index 0aa6e03c7f..944a45ba54 100644 --- a/src/policy/marksweepspace/native_ms/block_list.rs +++ b/src/policy/marksweepspace/native_ms/block_list.rs @@ -1,4 +1,4 @@ -use super::Block; +use super::{Block, BlockState}; use crate::util::alloc::allocator; use crate::util::linear_scan::Region; use crate::vm::VMBinding; @@ -164,14 +164,25 @@ impl BlockList { BlockListIterator { cursor: self.first } } - /// Sweep all the blocks in the block list. - pub fn sweep_blocks(&self, space: &super::MarkSweepSpace) { + /// Release unmarked blocks, and sweep other blocks in the block list. Used by eager sweeping. + pub fn release_and_sweep_blocks(&self, space: &super::MarkSweepSpace) { for block in self.iter() { + // We should not have unallocated blocks in a block list + debug_assert_ne!(block.get_state(), BlockState::Unallocated); if !block.attempt_release(space) { block.sweep::(); } } } + + /// Release unmarked blocks, and do not sweep any blocks. Used by lazy sweeping + pub fn release_blocks(&self, space: &super::MarkSweepSpace) { + for block in self.iter() { + // We should not have unallocated blocks in a block list + debug_assert_ne!(block.get_state(), BlockState::Unallocated); + block.attempt_release(space); + } + } } pub struct BlockListIterator { diff --git a/src/policy/marksweepspace/native_ms/global.rs b/src/policy/marksweepspace/native_ms/global.rs index a533fcde33..5db093574e 100644 --- a/src/policy/marksweepspace/native_ms/global.rs +++ b/src/policy/marksweepspace/native_ms/global.rs @@ -42,17 +42,42 @@ pub enum BlockAcquireResult { } /// A mark sweep space. +/// +/// The space and each free list allocator own some block lists. +/// A block that is in use belongs to exactly one of the block lists. In this case, +/// whoever owns a block list has exclusive access on the blocks in the list. +/// There should be no data race to access blocks. A thread should NOT access a block list +/// if it does not own the block list. +/// +/// The table below roughly describes what we do in each phase. +/// +/// | Phase | Allocator local block lists | Global abandoned block lists | Chunk map | +/// |----------------|-------------------------------------------------|----------------------------------------------|-----------| +/// | Allocation | Alloc from local | Move blocks from global to local block lists | - | +/// | | Lazy: sweep local blocks | | | +/// | GC - Prepare | - | - | Find used chunks, reset block mark, bzero mark bit | +/// | GC - Trace | Trace object and mark blocks. | Trace object and mark blocks. | - | +/// | | No block list access. | No block list access. | | +/// | GC - Release | Lazy: Move blocks to local unswept list | Lazy: Move blocks to global unswept list | _ | +/// | | Eager: Sweep local blocks | Eager: Sweep global blocks | | +/// | | Both: Return local blocks to a temp global list | | | +/// | GC - End of GC | - | Merge the temp global lists | - | pub struct MarkSweepSpace { pub common: CommonSpace, pr: FreeListPageResource, /// Allocation status for all chunks in MS space - pub chunk_map: ChunkMap, + chunk_map: ChunkMap, /// Work packet scheduler scheduler: Arc>, /// Abandoned blocks. If a mutator dies, all its blocks go to this abandoned block - /// lists. In a GC, we also 'flush' all the local blocks to this global pool so they - /// can be used by allocators from other threads. - pub abandoned: Mutex, + /// lists. We reuse blocks in these lists in the mutator phase. + /// The space needs to do the release work for these block lists. + abandoned: Mutex, + /// Abandoned blocks during a GC. Each allocator finishes doing release work, and returns + /// their local blocks to the global lists. Thus we do not need to do release work for + /// these block lists in the space. These lists are only filled in the release phase, + /// and will be moved to the abandoned lists above at the end of a GC. + abandoned_in_gc: Mutex, } pub struct AbandonedBlockLists { @@ -62,21 +87,32 @@ pub struct AbandonedBlockLists { } impl AbandonedBlockLists { - fn move_consumed_to_unswept(&mut self) { - let mut i = 0; - while i < MI_BIN_FULL { - if !self.consumed[i].is_empty() { - self.unswept[i].append(&mut self.consumed[i]); - } - i += 1; + fn new() -> Self { + Self { + available: new_empty_block_lists(), + unswept: new_empty_block_lists(), + consumed: new_empty_block_lists(), + } + } + + fn sweep_later(&mut self, space: &MarkSweepSpace) { + for i in 0..MI_BIN_FULL { + // Release free blocks + self.available[i].release_blocks(space); + self.consumed[i].release_blocks(space); + self.unswept[i].release_blocks(space); + + // Move remaining blocks to unswept + self.unswept[i].append(&mut self.available[i]); + self.unswept[i].append(&mut self.consumed[i]); } } fn sweep(&mut self, space: &MarkSweepSpace) { for i in 0..MI_BIN_FULL { - self.available[i].sweep_blocks(space); - self.consumed[i].sweep_blocks(space); - self.unswept[i].sweep_blocks(space); + self.available[i].release_and_sweep_blocks(space); + self.consumed[i].release_and_sweep_blocks(space); + self.unswept[i].release_and_sweep_blocks(space); // As we have swept blocks, move blocks in the unswept list to available or consumed list. while let Some(block) = self.unswept[i].pop() { @@ -88,6 +124,23 @@ impl AbandonedBlockLists { } } } + + fn merge(&mut self, other: &mut Self) { + for i in 0..MI_BIN_FULL { + self.available[i].append(&mut other.available[i]); + self.unswept[i].append(&mut other.unswept[i]); + self.consumed[i].append(&mut other.consumed[i]); + } + } + + #[cfg(debug_assertions)] + fn assert_empty(&self) { + for i in 0..MI_BIN_FULL { + assert!(self.available[i].is_empty()); + assert!(self.unswept[i].is_empty()); + assert!(self.consumed[i].is_empty()); + } + } } impl SFT for MarkSweepSpace { @@ -228,11 +281,8 @@ impl MarkSweepSpace { common, chunk_map: ChunkMap::new(), scheduler, - abandoned: Mutex::new(AbandonedBlockLists { - available: new_empty_block_lists(), - unswept: new_empty_block_lists(), - consumed: new_empty_block_lists(), - }), + abandoned: Mutex::new(AbandonedBlockLists::new()), + abandoned_in_gc: Mutex::new(AbandonedBlockLists::new()), } } @@ -261,27 +311,20 @@ impl MarkSweepSpace { self.chunk_map.set(block.chunk(), ChunkState::Allocated); } - pub fn get_next_metadata_spec(&self) -> SideMetadataSpec { - Block::NEXT_BLOCK_TABLE - } - pub fn prepare(&mut self) { - if let MetadataSpec::OnSide(side) = *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC { - for chunk in self.chunk_map.all_chunks() { - side.bzero_metadata(chunk.start(), Chunk::BYTES); - } - } else { - unimplemented!("in header mark bit is not supported"); - } + #[cfg(debug_assertions)] + self.abandoned_in_gc.lock().unwrap().assert_empty(); + + // # Safety: MarkSweepSpace reference is always valid within this collection cycle. + let space = unsafe { &*(self as *const Self) }; + let work_packets = self + .chunk_map + .generate_tasks(|chunk| Box::new(PrepareChunkMap { space, chunk })); + self.scheduler.work_buckets[crate::scheduler::WorkBucketStage::Prepare] + .bulk_add(work_packets); } pub fn release(&mut self) { - // We sweep and release unmarked blocks here. For sweeping cells inside each block, we either - // do that when we release mutators (eager sweeping), or do that at allocation time (lazy sweeping). - use crate::scheduler::WorkBucketStage; - let work_packets = self.generate_sweep_tasks(); - self.scheduler.work_buckets[WorkBucketStage::Release].bulk_add(work_packets); - if cfg!(feature = "eager_sweeping") { // For eager sweeping, we have to sweep the lists that are abandoned to these global lists. let mut abandoned = self.abandoned.lock().unwrap(); @@ -290,10 +333,19 @@ impl MarkSweepSpace { // For lazy sweeping, we just move blocks from consumed to unswept. When an allocator tries // to use them, they will sweep the block. let mut abandoned = self.abandoned.lock().unwrap(); - abandoned.move_consumed_to_unswept(); + abandoned.sweep_later(self); } } + pub fn end_of_gc(&mut self) { + let from = self.abandoned_in_gc.get_mut().unwrap(); + let to = self.abandoned.get_mut().unwrap(); + to.merge(from); + + #[cfg(debug_assertions)] + from.assert_empty(); + } + /// Release a block. pub fn release_block(&self, block: Block) { self.block_clear_metadata(block); @@ -340,42 +392,47 @@ impl MarkSweepSpace { } } - pub fn generate_sweep_tasks(&self) -> Vec>> { - // # Safety: ImmixSpace reference is always valid within this collection cycle. - let space = unsafe { &*(self as *const Self) }; - self.chunk_map - .generate_tasks(|chunk| Box::new(SweepChunk { space, chunk })) + pub fn get_abandoned_block_lists(&self) -> &Mutex { + &self.abandoned + } + + pub fn get_abandoned_block_lists_in_gc(&self) -> &Mutex { + &self.abandoned_in_gc } } use crate::scheduler::GCWork; use crate::MMTK; -/// Chunk sweeping work packet. -struct SweepChunk { +struct PrepareChunkMap { space: &'static MarkSweepSpace, chunk: Chunk, } -impl GCWork for SweepChunk { +impl GCWork for PrepareChunkMap { fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { debug_assert!(self.space.chunk_map.get(self.chunk) == ChunkState::Allocated); // number of allocated blocks. - let mut allocated_blocks = 0; - // Iterate over all allocated blocks in this chunk. - for block in self - .chunk + let mut n_occupied_blocks = 0; + self.chunk .iter_region::() .filter(|block| block.get_state() != BlockState::Unallocated) - { - if !block.attempt_release(self.space) { - // Block is live. Increment the allocated block count. - allocated_blocks += 1; - } - } - // Set this chunk as free if there is not live blocks. - if allocated_blocks == 0 { + .for_each(|block| { + // Clear block mark + block.set_state(BlockState::Unmarked); + // Count occupied blocks + n_occupied_blocks += 1 + }); + if n_occupied_blocks == 0 { + // Set this chunk as free if there is no live blocks. self.space.chunk_map.set(self.chunk, ChunkState::Free) + } else { + // Otherwise this chunk is occupied, and we reset the mark bit if it is on the side. + if let MetadataSpec::OnSide(side) = *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC { + for chunk in self.space.chunk_map.all_chunks() { + side.bzero_metadata(chunk.start(), Chunk::BYTES); + } + } } } } diff --git a/src/util/alloc/free_list_allocator.rs b/src/util/alloc/free_list_allocator.rs index 50ce36bd27..27546fc337 100644 --- a/src/util/alloc/free_list_allocator.rs +++ b/src/util/alloc/free_list_allocator.rs @@ -123,7 +123,8 @@ impl Allocator for FreeListAllocator { } fn on_mutator_destroy(&mut self) { - self.abandon_blocks(); + let mut global = self.space.get_abandoned_block_lists().lock().unwrap(); + self.abandon_blocks(&mut global); } } @@ -297,11 +298,13 @@ impl FreeListAllocator { loop { match self.space.acquire_block(self.tls, size, align) { crate::policy::marksweepspace::native_ms::BlockAcquireResult::Exhausted => { + debug!("Acquire global block: None"); // GC return None; } crate::policy::marksweepspace::native_ms::BlockAcquireResult::Fresh(block) => { + debug!("Acquire global block: Fresh {:?}", block); self.add_to_available_blocks(bin, block, stress_test); self.init_block(block, self.available_blocks[bin].size); @@ -309,6 +312,7 @@ impl FreeListAllocator { } crate::policy::marksweepspace::native_ms::BlockAcquireResult::AbandonedAvailable(block) => { + debug!("Acquire global block: AbandonedAvailable {:?}", block); block.store_tls(self.tls); if block.has_free_cells() { self.add_to_available_blocks(bin, block, stress_test); @@ -319,6 +323,7 @@ impl FreeListAllocator { } crate::policy::marksweepspace::native_ms::BlockAcquireResult::AbandonedUnswept(block) => { + debug!("Acquire global block: AbandonedUnswep {:?}", block); block.store_tls(self.tls); block.sweep::(); if block.has_free_cells() { @@ -333,6 +338,7 @@ impl FreeListAllocator { } fn init_block(&self, block: Block, cell_size: usize) { + debug_assert_ne!(cell_size, 0); self.space.record_new_block(block); // construct free list @@ -407,18 +413,9 @@ impl FreeListAllocator { block.store_tls(self.tls); } - pub(crate) fn prepare(&mut self) { - // For lazy sweeping, it doesn't matter whether we do it in prepare or release. - // However, in the release phase, we will do block-level sweeping. And that will cause - // race if we also reset the allocator in release (which will mutate on the block lists). - // So we just move reset to the prepare phase. - #[cfg(not(feature = "eager_sweeping"))] - self.reset(); - } + pub(crate) fn prepare(&mut self) {} pub(crate) fn release(&mut self) { - // For eager sweeping, we have to do this in the release phase when we know the liveness of the blocks - #[cfg(feature = "eager_sweeping")] self.reset(); } @@ -428,43 +425,45 @@ impl FreeListAllocator { /// for mark sweep. const ABANDON_BLOCKS_IN_RESET: bool = true; + /// Lazy sweeping. We just move all the blocks to the unswept block list. #[cfg(not(feature = "eager_sweeping"))] fn reset(&mut self) { trace!("reset"); // consumed and available are now unswept for bin in 0..MI_BIN_FULL { let unswept = self.unswept_blocks.get_mut(bin).unwrap(); - unswept.lock(); + + // If we abandon all the local blocks, we should have no unswept blocks here. Blocks should be either in available, or consumed. + debug_assert!(!Self::ABANDON_BLOCKS_IN_RESET || unswept.is_empty()); let mut sweep_later = |list: &mut BlockList| { - list.lock(); + list.release_blocks(self.space); unswept.append(list); - list.unlock(); }; sweep_later(&mut self.available_blocks[bin]); sweep_later(&mut self.available_blocks_stress[bin]); sweep_later(&mut self.consumed_blocks[bin]); - - unswept.unlock(); } if Self::ABANDON_BLOCKS_IN_RESET { - self.abandon_blocks(); + let mut global = self.space.get_abandoned_block_lists_in_gc().lock().unwrap(); + self.abandon_blocks(&mut global); } } + /// Eager sweeping. We sweep all the block lists, and move them to available block lists. #[cfg(feature = "eager_sweeping")] fn reset(&mut self) { debug!("reset"); // sweep all blocks and push consumed onto available list for bin in 0..MI_BIN_FULL { // Sweep available blocks - self.available_blocks[bin].sweep_blocks(self.space); - self.available_blocks_stress[bin].sweep_blocks(self.space); + self.available_blocks[bin].release_and_sweep_blocks(self.space); + self.available_blocks_stress[bin].release_and_sweep_blocks(self.space); // Sweep consumed blocks, and also push the blocks back to the available list. - self.consumed_blocks[bin].sweep_blocks(self.space); + self.consumed_blocks[bin].release_and_sweep_blocks(self.space); if *self.context.options.precise_stress && self.context.options.is_stress_test_gc_enabled() { @@ -479,31 +478,31 @@ impl FreeListAllocator { } if Self::ABANDON_BLOCKS_IN_RESET { - self.abandon_blocks(); + let mut global = self.space.get_abandoned_block_lists_in_gc().lock().unwrap(); + self.abandon_blocks(&mut global); } } - fn abandon_blocks(&mut self) { - let mut abandoned = self.space.abandoned.lock().unwrap(); + fn abandon_blocks(&mut self, global: &mut AbandonedBlockLists) { for i in 0..MI_BIN_FULL { let available = self.available_blocks.get_mut(i).unwrap(); if !available.is_empty() { - abandoned.available[i].append(available); + global.available[i].append(available); } let available_stress = self.available_blocks_stress.get_mut(i).unwrap(); if !available_stress.is_empty() { - abandoned.available[i].append(available_stress); + global.available[i].append(available_stress); } let consumed = self.consumed_blocks.get_mut(i).unwrap(); if !consumed.is_empty() { - abandoned.consumed[i].append(consumed); + global.consumed[i].append(consumed); } let unswept = self.unswept_blocks.get_mut(i).unwrap(); if !unswept.is_empty() { - abandoned.unswept[i].append(unswept); + global.unswept[i].append(unswept); } } }