From cb1692bd8eea43ea6d1df496eda03c045a8fbd0e Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Mon, 7 Aug 2023 23:31:04 +0200 Subject: [PATCH 1/6] switch to Talc memory allocator In contrast to linked_list_allocator, I measured a better performance. The difference is small, but large engough to change the crate. --- Cargo.lock | 137 ++++++++++++++------------------ Cargo.toml | 2 +- src/lib.rs | 14 +++- src/mm/allocator.rs | 70 +++++++++++++++++ src/mm/allocator/bootstrap.rs | 99 ------------------------ src/mm/allocator/bump.rs | 48 ------------ src/mm/allocator/mod.rs | 142 ---------------------------------- 7 files changed, 140 insertions(+), 372 deletions(-) create mode 100644 src/mm/allocator.rs delete mode 100644 src/mm/allocator/bootstrap.rs delete mode 100644 src/mm/allocator/bump.rs delete mode 100644 src/mm/allocator/mod.rs diff --git a/Cargo.lock b/Cargo.lock index f152f29f20..776421a622 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -94,9 +94,12 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" +dependencies = [ + "libc", +] [[package]] name = "cfg-if" @@ -139,7 +142,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -175,9 +178,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", @@ -202,12 +205,9 @@ checksum = "d5b9e9908e50b47ebbc3d6fd66ed295b997c270e8d2312a035bcc62722a160ef" [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" [[package]] name = "float-cmp" @@ -266,12 +266,6 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "hermit-abi" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" - [[package]] name = "hermit-dtb" version = "0.1.1" @@ -311,35 +305,15 @@ checksum = "9fdd0ac3785f538abb2b8221505e8244726756de924f670d06f4518c9a70a13d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", "tempfile", ] -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys", -] - [[package]] name = "libc" -version = "0.2.146" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libhermit-rs" @@ -383,9 +357,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" [[package]] name = "llvm-tools" @@ -491,7 +465,7 @@ checksum = "9e6a0fd4f737c707bd9086cc16c925f294943eb62eb71499e9fd4cf71f8b9f4e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -543,9 +517,9 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "paste" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "pci-ids" @@ -580,18 +554,18 @@ dependencies = [ [[package]] name = "phf" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_shared", ] [[package]] name = "phf_codegen" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" +checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" dependencies = [ "phf_generator", "phf_shared", @@ -599,9 +573,9 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared", "rand", @@ -609,9 +583,9 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ "siphasher", ] @@ -654,9 +628,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -669,9 +643,9 @@ checksum = "8bb0fd6580eeed0103c054e3fba2c2618ff476943762f28a645b63b8692b21c9" [[package]] name = "quote" -version = "1.0.28" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2", ] @@ -741,13 +715,12 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.20" +version = "0.38.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" +checksum = "172891ebdceb05aa0005f533a6cbfca599ddd7d966f6f5d4d9b2e70478e70399" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.3.3", "errno", - "io-lifetimes", "libc", "linux-raw-sys", "windows-sys", @@ -755,15 +728,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "scroll" @@ -782,14 +755,14 @@ checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] name = "semver" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "shell-words" @@ -845,9 +818,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ "proc-macro2", "quote", @@ -859,14 +832,16 @@ name = "talc" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66d1ba143c1edc670fc552f02d6acb188ebf313b7c6cd512310f63a08cc8ed95" +dependencies = [ + "lock_api", +] [[package]] name = "tempfile" -version = "3.6.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" dependencies = [ - "autocfg", "cfg-if", "fastrand", "redox_syscall", @@ -876,22 +851,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.28", ] [[package]] @@ -929,9 +904,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "version_check" @@ -962,9 +937,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", diff --git a/Cargo.toml b/Cargo.toml index f51d5378e0..8fe686c6ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,7 +89,7 @@ num = { version = "0.4", default-features = false } num-traits = { version = "0.2", default-features = false } num-derive = "0.4" zerocopy = "0.6" -talc = { version = "2", default-features = false } +talc = { version = "2" } time = { version = "0.3", default-features = false } pci_types = { version = "0.5" } diff --git a/src/lib.rs b/src/lib.rs index db6b1efe8d..6337eaa15d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,9 +114,21 @@ fn trivial_test() { panic!("Test called"); } +#[cfg(target_os = "none")] +static mut ARENA: [u8; 0x2000] = [0; 0x2000]; + #[cfg(target_os = "none")] #[global_allocator] -static ALLOCATOR: LockedAllocator = LockedAllocator::empty(); +static mut ALLOCATOR: LockedAllocator = LockedAllocator( + talc::Talc::new(unsafe { + // if we're in a hosted environment, the Rust runtime may allocate before + // main() is called, so we need to initialize the arena automatically + talc::InitOnOom::new(talc::Span::from_slice( + ARENA.as_slice() as *const [u8] as *mut [u8] + )) + }) + .lock(), +); /// Interface to allocate memory from system heap /// diff --git a/src/mm/allocator.rs b/src/mm/allocator.rs new file mode 100644 index 0000000000..7c6cf094c4 --- /dev/null +++ b/src/mm/allocator.rs @@ -0,0 +1,70 @@ +//! Implementation of the HermitCore Allocator for dynamically allocating heap memory +//! in the kernel. + +use core::alloc::{GlobalAlloc, Layout}; + +use align_address::Align; +use hermit_sync::RawInterruptTicketMutex; +use talc::{InitOnOom, Span, Talck}; + +use crate::HW_DESTRUCTIVE_INTERFERENCE_SIZE; + +pub struct LockedAllocator(pub Talck); + +impl LockedAllocator { + #[inline] + fn align_layout(layout: Layout) -> Layout { + let size = layout.size().align_up(HW_DESTRUCTIVE_INTERFERENCE_SIZE); + let align = layout.align().max(HW_DESTRUCTIVE_INTERFERENCE_SIZE); + Layout::from_size_align(size, align).unwrap() + } + + pub unsafe fn init(&self, heap_bottom: *mut u8, heap_size: usize) { + let arena = Span::from_base_size(heap_bottom, heap_size); + unsafe { + self.0.talc().init(arena); + } + } +} + +/// To avoid false sharing, the global memory allocator align +/// all requests to a cache line. +unsafe impl GlobalAlloc for LockedAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let layout = Self::align_layout(layout); + unsafe { self.0.alloc(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + let layout = Self::align_layout(layout); + unsafe { self.0.dealloc(ptr, layout) } + } +} + +#[cfg(all(test, not(target_os = "none")))] +mod tests { + use core::mem; + + use super::*; + + #[test] + fn empty() { + let mut arena: [u8; 0x1000] = [0; 0x1000]; + let allocator: LockedAllocator = LockedAllocator( + talc::Talc::new(unsafe { + talc::InitOnOom::new(talc::Span::from_slice( + arena.as_slice() as *const [u8] as *mut [u8] + )) + }) + .lock(), + ); + + let layout = Layout::from_size_align(1, 1).unwrap(); + // we have 4 kbyte memory + assert!(unsafe { !allocator.alloc(layout.clone()).is_null() }); + + let layout = Layout::from_size_align(0x1000, mem::align_of::()).unwrap(); + let addr = unsafe { allocator.alloc(layout) }; + assert!(addr.is_null()); + } +} diff --git a/src/mm/allocator/bootstrap.rs b/src/mm/allocator/bootstrap.rs deleted file mode 100644 index 705366b8b0..0000000000 --- a/src/mm/allocator/bootstrap.rs +++ /dev/null @@ -1,99 +0,0 @@ -//! A bootstrap allocator based on a statically allocated buffer. - -/// A pointer range that can only be compared against. -mod ptr_range { - use core::ops::Range; - use core::ptr::NonNull; - - /// A pointer range that can only be compared against. - pub struct PtrRange { - inner: Range>, - } - - // SAFETY: We never dereference, but only compare, pointers. - unsafe impl Send for PtrRange {} - unsafe impl Sync for PtrRange {} - - impl PtrRange { - /// Returns `true` if the pointer range contains `ptr`. - pub fn contains(&self, ptr: NonNull) -> bool { - self.inner.contains(&ptr) - } - } - - impl From>> for PtrRange { - fn from(value: Range>) -> Self { - Self { inner: value } - } - } -} - -use core::alloc::{AllocError, Allocator, Layout}; -use core::mem::MaybeUninit; -use core::ops::Range; -use core::ptr::NonNull; - -use hermit_sync::ExclusiveCell; - -use self::ptr_range::PtrRange; - -/// A bootstrap allocator. -/// -/// This allocator is generic over the internal allocator and can only be created once. -/// The bootstrap allocator provides the internal allocator with static memory. -/// -/// This allocator tracks, which static memory it was using initially. -/// It can be queried whether a pointer belongs to it. -pub struct BootstrapAllocator { - ptr_range: PtrRange, - allocator: A, -} - -impl Default for BootstrapAllocator -where - A: From<&'static mut [MaybeUninit]>, -{ - fn default() -> Self { - let mem = { - const SIZE: usize = 4 * 1024; - const BYTE: MaybeUninit = MaybeUninit::uninit(); - /// The actual memory of the boostrap allocator. - static MEM: ExclusiveCell<[MaybeUninit; SIZE]> = ExclusiveCell::new([BYTE; SIZE]); - MEM.take().unwrap() - }; - - let ptr_range = { - let Range { start, end } = mem.as_mut_ptr_range(); - let start = NonNull::new(start).unwrap().cast::(); - let end = NonNull::new(end).unwrap().cast::(); - PtrRange::from(start..end) - }; - let allocator = A::from(mem); - - Self { - ptr_range, - allocator, - } - } -} - -impl BootstrapAllocator { - /// Returns `true` if the pointer belonged to the static memory of this allocator. - pub fn manages(&self, ptr: NonNull) -> bool { - self.ptr_range.contains(ptr) - } -} - -unsafe impl Allocator for BootstrapAllocator -where - A: Allocator, -{ - fn allocate(&self, layout: Layout) -> Result, AllocError> { - self.allocator.allocate(layout) - } - - unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - debug_assert!(self.manages(ptr)); - unsafe { self.allocator.deallocate(ptr, layout) } - } -} diff --git a/src/mm/allocator/bump.rs b/src/mm/allocator/bump.rs deleted file mode 100644 index 6c5e38f9c7..0000000000 --- a/src/mm/allocator/bump.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! A bump allocator. -//! -//! This is a simple allocator design which can only allocate and not deallocate. - -use core::alloc::{AllocError, Allocator, Layout}; -use core::cell::Cell; -use core::mem::MaybeUninit; -use core::ptr::NonNull; - -/// A simple, `!Sync` implementation of a bump allocator. -/// -/// This allocator manages the provided memory. -pub struct BumpAllocator { - mem: Cell<&'static mut [MaybeUninit]>, -} - -unsafe impl Allocator for BumpAllocator { - fn allocate(&self, layout: Layout) -> Result, AllocError> { - let ptr: *mut [MaybeUninit] = self.allocate_slice(layout)?; - Ok(NonNull::new(ptr as *mut [u8]).unwrap()) - } - - unsafe fn deallocate(&self, _ptr: NonNull, _layout: Layout) {} -} - -impl BumpAllocator { - fn allocate_slice(&self, layout: Layout) -> Result<&'static mut [MaybeUninit], AllocError> { - let mem = self.mem.take(); - let align_offset = mem.as_ptr().align_offset(layout.align()); - let mid = layout.size() + align_offset; - if mid > mem.len() { - self.mem.set(mem); - Err(AllocError) - } else { - let (alloc, remaining) = mem.split_at_mut(mid); - self.mem.set(remaining); - Ok(&mut alloc[align_offset..]) - } - } -} - -impl From<&'static mut [MaybeUninit]> for BumpAllocator { - fn from(mem: &'static mut [MaybeUninit]) -> Self { - Self { - mem: Cell::new(mem), - } - } -} diff --git a/src/mm/allocator/mod.rs b/src/mm/allocator/mod.rs deleted file mode 100644 index a9c3b9f344..0000000000 --- a/src/mm/allocator/mod.rs +++ /dev/null @@ -1,142 +0,0 @@ -//! Implementation of the HermitCore Allocator for dynamically allocating heap memory -//! in the kernel. - -mod bootstrap; -mod bump; - -use core::alloc::{AllocError, Allocator, GlobalAlloc, Layout}; -use core::ptr; -use core::ptr::NonNull; - -use align_address::Align; -use hermit_sync::InterruptTicketMutex; -use talc::{ErrOnOom, Span, Talc}; - -use self::bootstrap::BootstrapAllocator; -use self::bump::BumpAllocator; -use crate::HW_DESTRUCTIVE_INTERFERENCE_SIZE; - -/// The global system allocator for Hermit. -struct GlobalAllocator { - /// The bootstrap allocator, which is available immediately. - /// - /// It allows allocations before the heap has been initalized. - bootstrap_allocator: Option>, - - /// The heap allocator. - /// - /// This is not available immediately and must be initialized ([`Self::init`]). - heap: Option>, -} - -impl GlobalAllocator { - const fn empty() -> Self { - Self { - bootstrap_allocator: None, - heap: None, - } - } - - /// Initializes the heap allocator. - /// - /// # Safety - /// - /// The memory starting from `heap_bottom` with a size of `heap_size` - /// must be valid and ready to be managed and allocated from. - unsafe fn init(&mut self, heap_bottom: *mut u8, heap_size: usize) { - self.heap = unsafe { - Some(Talc::with_arena( - ErrOnOom, - Span::from_base_size(heap_bottom, heap_size), - )) - } - } - - fn align_layout(layout: Layout) -> Layout { - let size = layout.size().align_up(HW_DESTRUCTIVE_INTERFERENCE_SIZE); - let align = layout.align().max(HW_DESTRUCTIVE_INTERFERENCE_SIZE); - Layout::from_size_align(size, align).unwrap() - } - - fn allocate(&mut self, layout: Layout) -> Result, AllocError> { - let layout = Self::align_layout(layout); - match &mut self.heap { - Some(heap) => unsafe { heap.malloc(layout).map_err(|_| AllocError) }, - None => self - .bootstrap_allocator - .get_or_insert_with(Default::default) - .allocate(layout) - // FIXME: Use NonNull::as_mut_ptr once `slice_ptr_get` is stabilized - // https://github.com/rust-lang/rust/issues/74265 - .map(|ptr| NonNull::new(ptr.as_ptr() as *mut u8).unwrap()), - } - } - - unsafe fn deallocate(&mut self, ptr: NonNull, layout: Layout) { - let layout = Self::align_layout(layout); - let bootstrap_allocator = self.bootstrap_allocator.as_ref().unwrap(); - if bootstrap_allocator.manages(ptr) { - unsafe { - bootstrap_allocator.deallocate(ptr, layout); - } - } else { - unsafe { - self.heap.as_mut().unwrap().free(ptr, layout); - } - } - } -} - -pub struct LockedAllocator(InterruptTicketMutex); - -impl LockedAllocator { - /// Creates an empty allocator. All allocate calls will return `None`. - pub const fn empty() -> LockedAllocator { - LockedAllocator(InterruptTicketMutex::new(GlobalAllocator::empty())) - } - - pub unsafe fn init(&self, heap_bottom: *mut u8, heap_size: usize) { - unsafe { - self.0.lock().init(heap_bottom, heap_size); - } - } -} - -/// To avoid false sharing, the global memory allocator align -/// all requests to a cache line. -unsafe impl GlobalAlloc for LockedAllocator { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - self.0 - .lock() - .allocate(layout) - .ok() - .map_or(ptr::null_mut(), |allocation| allocation.as_ptr()) - } - - unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - unsafe { - self.0 - .lock() - .deallocate(NonNull::new_unchecked(ptr), layout) - } - } -} - -#[cfg(all(test, not(target_os = "none")))] -mod tests { - use core::mem; - - use super::*; - - #[test] - fn empty() { - let mut allocator = GlobalAllocator::empty(); - let layout = Layout::from_size_align(1, 1).unwrap(); - // we have 4 kbyte static memory - assert!(allocator.allocate(layout.clone()).is_ok()); - - let layout = Layout::from_size_align(0x1000, mem::align_of::()); - let addr = allocator.allocate(layout.unwrap()); - assert!(addr.is_err()); - } -} From 29173104df0065023ea31246e0a5c14fe303e7da Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Tue, 8 Aug 2023 09:26:25 +0200 Subject: [PATCH 2/6] move constants to the allocator only the allocator needs this alignment and data size --- src/config.rs | 3 --- src/mm/allocator.rs | 10 ++++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/config.rs b/src/config.rs index d574158b0e..056249b6f5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -14,6 +14,3 @@ pub(crate) const VIRTIO_MAX_QUEUE_SIZE: u16 = 1024; /// Default keep alive interval in milliseconds #[cfg(feature = "tcp")] pub(crate) const DEFAULT_KEEP_ALIVE_INTERVAL: u64 = 75000; - -pub(crate) const HW_DESTRUCTIVE_INTERFERENCE_SIZE: usize = - core::mem::align_of::>(); diff --git a/src/mm/allocator.rs b/src/mm/allocator.rs index 7c6cf094c4..80ec67ab61 100644 --- a/src/mm/allocator.rs +++ b/src/mm/allocator.rs @@ -7,15 +7,17 @@ use align_address::Align; use hermit_sync::RawInterruptTicketMutex; use talc::{InitOnOom, Span, Talck}; -use crate::HW_DESTRUCTIVE_INTERFERENCE_SIZE; - pub struct LockedAllocator(pub Talck); impl LockedAllocator { #[inline] fn align_layout(layout: Layout) -> Layout { - let size = layout.size().align_up(HW_DESTRUCTIVE_INTERFERENCE_SIZE); - let align = layout.align().max(HW_DESTRUCTIVE_INTERFERENCE_SIZE); + let size = layout + .size() + .align_up(core::mem::size_of::>()); + let align = layout + .align() + .max(core::mem::align_of::>()); Layout::from_size_align(size, align).unwrap() } From 530932719e1ed7f09c8dd9619308d20b0d0f0937 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Tue, 8 Aug 2023 13:49:53 +0200 Subject: [PATCH 3/6] start heap directly after the executable --- src/lib.rs | 27 +++++++++++++-------------- src/mm/allocator.rs | 15 +++++++++++++-- src/mm/mod.rs | 15 ++++++++++----- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6337eaa15d..41199c9452 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,21 +114,9 @@ fn trivial_test() { panic!("Test called"); } -#[cfg(target_os = "none")] -static mut ARENA: [u8; 0x2000] = [0; 0x2000]; - #[cfg(target_os = "none")] #[global_allocator] -static mut ALLOCATOR: LockedAllocator = LockedAllocator( - talc::Talc::new(unsafe { - // if we're in a hosted environment, the Rust runtime may allocate before - // main() is called, so we need to initialize the arena automatically - talc::InitOnOom::new(talc::Span::from_slice( - ARENA.as_slice() as *const [u8] as *mut [u8] - )) - }) - .lock(), -); +static ALLOCATOR: LockedAllocator = LockedAllocator::new(); /// Interface to allocate memory from system heap /// @@ -310,6 +298,13 @@ fn synch_all_cores() { /// Entry Point of HermitCore for the Boot Processor #[cfg(target_os = "none")] fn boot_processor_main() -> ! { + let init_heap_start = env::get_base_address() + env::get_image_size(); + let init_heap_len = + init_heap_start.align_up_to_large_page().as_usize() - init_heap_start.as_usize(); + unsafe { + ALLOCATOR.init(init_heap_start.as_mut_ptr(), init_heap_len); + } + // Initialize the kernel and hardware. arch::message_output_init(); unsafe { @@ -330,7 +325,11 @@ fn boot_processor_main() -> ! { env::get_tls_start(), env::get_tls_memsz() ); - + info!( + "Init heap: [0x{:p} - 0x{:p}]", + init_heap_start, + init_heap_start + init_heap_len + ); arch::boot_processor_init(); scheduler::add_current_core(); diff --git a/src/mm/allocator.rs b/src/mm/allocator.rs index 80ec67ab61..5eb273f3e1 100644 --- a/src/mm/allocator.rs +++ b/src/mm/allocator.rs @@ -5,11 +5,15 @@ use core::alloc::{GlobalAlloc, Layout}; use align_address::Align; use hermit_sync::RawInterruptTicketMutex; -use talc::{InitOnOom, Span, Talck}; +use talc::{ErrOnOom, Span, Talc, Talck}; -pub struct LockedAllocator(pub Talck); +pub struct LockedAllocator(Talck); impl LockedAllocator { + pub const fn new() -> Self { + Self(Talc::new(ErrOnOom).lock()) + } + #[inline] fn align_layout(layout: Layout) -> Layout { let size = layout @@ -27,6 +31,13 @@ impl LockedAllocator { self.0.talc().init(arena); } } + + pub unsafe fn extend(&self, heap_bottom: *mut u8, heap_size: usize) { + let arena = Span::from_base_size(heap_bottom, heap_size); + unsafe { + self.0.talc().extend(arena); + } + } } /// To avoid false sharing, the global memory allocator align diff --git a/src/mm/mod.rs b/src/mm/mod.rs index 0f3c0cdd2b..891eb6de61 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -114,7 +114,7 @@ pub fn init() { unsafe { let start = allocate(kernel_heap_size, true); - crate::ALLOCATOR.init(start.as_mut_ptr(), kernel_heap_size); + crate::ALLOCATOR.extend(start.as_mut_ptr(), kernel_heap_size); info!("Kernel heap starts at {:#x}", start); } @@ -211,16 +211,21 @@ pub fn init() { } let heap_end_addr = map_addr; + let init_heap_start_addr = env::get_base_address() + env::get_image_size(); #[cfg(not(feature = "newlib"))] unsafe { - crate::ALLOCATOR.init( - heap_start_addr.as_mut_ptr(), - (heap_end_addr - heap_start_addr).into(), + crate::ALLOCATOR.extend( + init_heap_start_addr.as_mut_ptr(), + (heap_end_addr - init_heap_start_addr).into(), ); } + info!( + "Heap extension from {:#x} to {:#x}", + heap_start_addr, heap_end_addr + ); - let heap_addr_range = heap_start_addr..heap_end_addr; + let heap_addr_range = init_heap_start_addr..heap_end_addr; info!("Heap is located at {heap_addr_range:#x?} ({map_size} Bytes unmapped)"); #[cfg(feature = "newlib")] HEAP_ADDR_RANGE.set(heap_addr_range).unwrap(); From 779c03c1038b07b3bcdef1f78312f06bddc5ee65 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Tue, 8 Aug 2023 14:19:24 +0200 Subject: [PATCH 4/6] fix some unit tests --- src/arch/aarch64/kernel/mod.rs | 2 ++ src/arch/aarch64/kernel/start.rs | 2 +- src/arch/mod.rs | 6 ++++-- src/mm/allocator.rs | 15 ++++++--------- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/arch/aarch64/kernel/mod.rs b/src/arch/aarch64/kernel/mod.rs index 52ca9bdabf..fc409ca975 100644 --- a/src/arch/aarch64/kernel/mod.rs +++ b/src/arch/aarch64/kernel/mod.rs @@ -7,6 +7,7 @@ pub mod pci; pub mod processor; pub mod scheduler; pub mod serial; +#[cfg(target_os = "none")] mod start; pub mod switch; pub mod systemtime; @@ -187,6 +188,7 @@ pub fn output_message_buf(buf: &[u8]) { } /// Real Boot Processor initialization as soon as we have put the first Welcome message on the screen. +#[cfg(target_os = "none")] pub fn boot_processor_init() { processor::configure(); diff --git a/src/arch/aarch64/kernel/start.rs b/src/arch/aarch64/kernel/start.rs index 8a78c4d24d..f6cdf3e469 100644 --- a/src/arch/aarch64/kernel/start.rs +++ b/src/arch/aarch64/kernel/start.rs @@ -76,6 +76,6 @@ unsafe extern "C" fn pre_init(boot_info: &'static RawBootInfo, cpu_id: u32) -> ! } } #[cfg(feature = "smp")] - crate::application_processor_main(); + crate::application_processor_main() } } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 6a8961fe4f..5f5da075f1 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -6,6 +6,8 @@ pub mod aarch64; pub mod x86_64; // Export our platform-specific modules. +#[cfg(all(target_arch = "aarch64", target_os = "none"))] +pub use crate::arch::aarch64::kernel::boot_processor_init; #[cfg(target_arch = "aarch64")] pub use crate::arch::aarch64::kernel::core_local; #[cfg(target_arch = "aarch64")] @@ -26,8 +28,8 @@ pub use crate::arch::aarch64::kernel::switch; pub use crate::arch::aarch64::kernel::systemtime::get_boot_time; #[cfg(target_arch = "aarch64")] pub use crate::arch::aarch64::kernel::{ - application_processor_init, boot_application_processors, boot_processor_init, - get_processor_count, message_output_init, output_message_buf, + application_processor_init, boot_application_processors, get_processor_count, + message_output_init, output_message_buf, }; #[cfg(target_arch = "aarch64")] pub use crate::arch::aarch64::*; diff --git a/src/mm/allocator.rs b/src/mm/allocator.rs index 5eb273f3e1..db9ff43f3d 100644 --- a/src/mm/allocator.rs +++ b/src/mm/allocator.rs @@ -62,15 +62,12 @@ mod tests { #[test] fn empty() { - let mut arena: [u8; 0x1000] = [0; 0x1000]; - let allocator: LockedAllocator = LockedAllocator( - talc::Talc::new(unsafe { - talc::InitOnOom::new(talc::Span::from_slice( - arena.as_slice() as *const [u8] as *mut [u8] - )) - }) - .lock(), - ); + const ARENA_SIZE: usize = 0x1000; + let mut arena: [u8; ARENA_SIZE] = [0; ARENA_SIZE]; + let allocator: LockedAllocator = LockedAllocator::new(); + unsafe { + allocator.init(&mut arena as *mut [u8] as *mut u8, ARENA_SIZE); + } let layout = Layout::from_size_align(1, 1).unwrap(); // we have 4 kbyte memory From f8c4bb12a55b9047999797a06a0b921c2ac25141 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Tue, 8 Aug 2023 14:48:23 +0200 Subject: [PATCH 5/6] ignore virtual memory below the kernel --- src/arch/aarch64/mm/virtualmem.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/arch/aarch64/mm/virtualmem.rs b/src/arch/aarch64/mm/virtualmem.rs index d40ae89156..f69448b005 100644 --- a/src/arch/aarch64/mm/virtualmem.rs +++ b/src/arch/aarch64/mm/virtualmem.rs @@ -2,9 +2,8 @@ use core::alloc::AllocError; use hermit_sync::InterruptTicketMutex; -use crate::arch::aarch64::kernel::get_ram_address; use crate::arch::aarch64::mm::paging::{BasePageSize, PageSize}; -use crate::arch::aarch64::mm::{PhysAddr, VirtAddr}; +use crate::arch::aarch64::mm::VirtAddr; use crate::mm; use crate::mm::freelist::{FreeList, FreeListEntry}; @@ -16,15 +15,6 @@ static KERNEL_FREE_LIST: InterruptTicketMutex = const KERNEL_VIRTUAL_MEMORY_END: VirtAddr = VirtAddr(0x1_0000_0000); pub fn init() { - // don't use the first two kilobytes - if get_ram_address() > PhysAddr(0x2000) { - let entry = FreeListEntry { - start: 0x2000, - end: get_ram_address().as_usize(), - }; - KERNEL_FREE_LIST.lock().list.push_back(entry); - } - let entry = FreeListEntry { start: mm::kernel_end_address().as_usize(), end: KERNEL_VIRTUAL_MEMORY_END.as_usize(), From f649417d0abcc23dd1da5976dea0b790b93bf61b Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Tue, 8 Aug 2023 14:49:10 +0200 Subject: [PATCH 6/6] limit public function to the crate --- src/mm/mod.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/mm/mod.rs b/src/mm/mod.rs index 891eb6de61..097eca08fc 100644 --- a/src/mm/mod.rs +++ b/src/mm/mod.rs @@ -37,26 +37,26 @@ static KERNEL_ADDR_RANGE: Lazy> = Lazy::new(|| { /// User heap address range. static HEAP_ADDR_RANGE: OnceCell> = OnceCell::new(); -pub fn kernel_start_address() -> VirtAddr { +pub(crate) fn kernel_start_address() -> VirtAddr { KERNEL_ADDR_RANGE.start } -pub fn kernel_end_address() -> VirtAddr { +pub(crate) fn kernel_end_address() -> VirtAddr { KERNEL_ADDR_RANGE.end } #[cfg(feature = "newlib")] -pub fn task_heap_start() -> VirtAddr { +pub(crate) fn task_heap_start() -> VirtAddr { HEAP_ADDR_RANGE.get().unwrap().start } #[cfg(feature = "newlib")] -pub fn task_heap_end() -> VirtAddr { +pub(crate) fn task_heap_end() -> VirtAddr { HEAP_ADDR_RANGE.get().unwrap().end } #[cfg(target_os = "none")] -pub fn init() { +pub(crate) fn init() { use crate::arch::mm::paging; Lazy::force(&KERNEL_ADDR_RANGE); @@ -231,13 +231,13 @@ pub fn init() { HEAP_ADDR_RANGE.set(heap_addr_range).unwrap(); } -pub fn print_information() { +pub(crate) fn print_information() { arch::mm::physicalmem::print_information(); arch::mm::virtualmem::print_information(); } #[allow(dead_code)] -pub fn allocate(sz: usize, no_execution: bool) -> VirtAddr { +pub(crate) fn allocate(sz: usize, no_execution: bool) -> VirtAddr { let size = sz.align_up(BasePageSize::SIZE as usize); let physical_address = arch::mm::physicalmem::allocate(size).unwrap(); let virtual_address = arch::mm::virtualmem::allocate(size).unwrap(); @@ -254,7 +254,7 @@ pub fn allocate(sz: usize, no_execution: bool) -> VirtAddr { } #[allow(dead_code)] -pub fn deallocate(virtual_address: VirtAddr, sz: usize) { +pub(crate) fn deallocate(virtual_address: VirtAddr, sz: usize) { let size = sz.align_up(BasePageSize::SIZE as usize); if let Some(phys_addr) = arch::mm::paging::virtual_to_physical(virtual_address) { @@ -274,7 +274,7 @@ pub fn deallocate(virtual_address: VirtAddr, sz: usize) { /// Maps a given physical address and size in virtual space and returns address. #[cfg(feature = "pci")] -pub fn map( +pub(crate) fn map( physical_address: PhysAddr, sz: usize, writable: bool, @@ -304,7 +304,7 @@ pub fn map( #[allow(dead_code)] /// unmaps virtual address, without 'freeing' physical memory it is mapped to! -pub fn unmap(virtual_address: VirtAddr, sz: usize) { +pub(crate) fn unmap(virtual_address: VirtAddr, sz: usize) { let size = sz.align_up(BasePageSize::SIZE as usize); if arch::mm::paging::virtual_to_physical(virtual_address).is_some() {