diff --git a/src/arch/aarch64/kernel/core_local.rs b/src/arch/aarch64/kernel/core_local.rs index 71c917d9f5..44d4dffede 100644 --- a/src/arch/aarch64/kernel/core_local.rs +++ b/src/arch/aarch64/kernel/core_local.rs @@ -1,13 +1,18 @@ use alloc::boxed::Box; use alloc::vec::Vec; use core::arch::asm; -use core::cell::{Cell, RefCell, RefMut}; +use core::cell::{RefCell, RefMut}; use core::ptr; use core::sync::atomic::Ordering; +#[cfg(feature = "smp")] +use hermit_sync::InterruptTicketMutex; + use super::interrupts::{IrqStatistics, IRQ_COUNTERS}; use super::CPU_ONLINE; use crate::executor::task::AsyncTask; +#[cfg(feature = "smp")] +use crate::scheduler::SchedulerInput; use crate::scheduler::{CoreId, PerCoreScheduler}; pub(crate) struct CoreLocal { @@ -15,11 +20,14 @@ pub(crate) struct CoreLocal { /// ID of the current Core. core_id: CoreId, /// Scheduler of the current Core. - scheduler: Cell<*mut PerCoreScheduler>, + scheduler: RefCell>, /// Interface to the interrupt counters irq_statistics: &'static IrqStatistics, /// Queue of async tasks async_tasks: RefCell>, + /// Queues to handle incoming requests from the other cores + #[cfg(feature = "smp")] + pub scheduler_input: InterruptTicketMutex, } impl CoreLocal { @@ -36,9 +44,11 @@ impl CoreLocal { let this = Self { this: ptr::null_mut(), core_id, - scheduler: Cell::new(ptr::null_mut()), + scheduler: RefCell::new(None), irq_statistics, async_tasks: RefCell::new(Vec::new()), + #[cfg(feature = "smp")] + scheduler_input: InterruptTicketMutex::new(SchedulerInput::new()), }; let this = if core_id == 0 { take_static::take_static! { @@ -81,17 +91,18 @@ pub(crate) fn core_id() -> CoreId { } } -#[inline] -pub(crate) fn core_scheduler() -> &'static mut PerCoreScheduler { - unsafe { &mut *CoreLocal::get().scheduler.get() } +pub(crate) fn core_scheduler() -> RefMut<'static, PerCoreScheduler> { + RefMut::map(CoreLocal::get().scheduler.borrow_mut(), |scheduler| { + scheduler.as_mut().unwrap() + }) } pub(crate) fn async_tasks() -> RefMut<'static, Vec> { CoreLocal::get().async_tasks.borrow_mut() } -pub(crate) fn set_core_scheduler(scheduler: *mut PerCoreScheduler) { - CoreLocal::get().scheduler.set(scheduler); +pub(crate) fn set_core_scheduler(scheduler: PerCoreScheduler) { + *CoreLocal::get().scheduler.borrow_mut() = Some(scheduler); } pub(crate) fn increment_irq_counter(irq_no: u8) { diff --git a/src/arch/x86_64/kernel/apic.rs b/src/arch/x86_64/kernel/apic.rs index fc8804b25a..1aab2ede7a 100644 --- a/src/arch/x86_64/kernel/apic.rs +++ b/src/arch/x86_64/kernel/apic.rs @@ -225,9 +225,11 @@ extern "x86-interrupt" fn spurious_interrupt_handler(stack_frame: interrupts::Ex #[cfg(feature = "smp")] extern "x86-interrupt" fn wakeup_handler(_stack_frame: interrupts::ExceptionStackFrame) { + use crate::scheduler::PerCoreSchedulerExt; + debug!("Received Wakeup Interrupt"); increment_irq_counter(WAKEUP_INTERRUPT_NUMBER); - let core_scheduler = core_scheduler(); + let mut core_scheduler = core_scheduler(); core_scheduler.check_input(); eoi(); if core_scheduler.is_scheduling() { diff --git a/src/arch/x86_64/kernel/core_local.rs b/src/arch/x86_64/kernel/core_local.rs index 323b396a48..50509a9e22 100644 --- a/src/arch/x86_64/kernel/core_local.rs +++ b/src/arch/x86_64/kernel/core_local.rs @@ -5,6 +5,8 @@ use core::cell::{Cell, RefCell, RefMut}; use core::ptr; use core::sync::atomic::Ordering; +#[cfg(feature = "smp")] +use hermit_sync::InterruptTicketMutex; use x86_64::registers::model_specific::GsBase; use x86_64::structures::tss::TaskStateSegment; use x86_64::VirtAddr; @@ -12,6 +14,8 @@ use x86_64::VirtAddr; use super::interrupts::{IrqStatistics, IRQ_COUNTERS}; use super::CPU_ONLINE; use crate::executor::task::AsyncTask; +#[cfg(feature = "smp")] +use crate::scheduler::SchedulerInput; use crate::scheduler::{CoreId, PerCoreScheduler}; #[repr(C)] @@ -20,7 +24,7 @@ pub(crate) struct CoreLocal { /// Sequential ID of this CPU Core. core_id: CoreId, /// Scheduler for this CPU Core. - scheduler: Cell<*mut PerCoreScheduler>, + scheduler: RefCell>, /// Task State Segment (TSS) allocated for this CPU Core. pub tss: Cell<*mut TaskStateSegment>, /// start address of the kernel stack @@ -29,6 +33,9 @@ pub(crate) struct CoreLocal { irq_statistics: &'static IrqStatistics, /// Queue of async tasks async_tasks: RefCell>, + /// Queues to handle incoming requests from the other cores + #[cfg(feature = "smp")] + pub scheduler_input: InterruptTicketMutex, } impl CoreLocal { @@ -47,11 +54,13 @@ impl CoreLocal { let this = Self { this: ptr::null_mut(), core_id, - scheduler: Cell::new(ptr::null_mut()), + scheduler: RefCell::new(None), tss: Cell::new(ptr::null_mut()), kernel_stack: Cell::new(0), irq_statistics, async_tasks: RefCell::new(Vec::new()), + #[cfg(feature = "smp")] + scheduler_input: InterruptTicketMutex::new(SchedulerInput::new()), }; let this = if core_id == 0 { take_static::take_static! { @@ -92,16 +101,18 @@ pub(crate) fn core_id() -> CoreId { } } -pub(crate) fn core_scheduler() -> &'static mut PerCoreScheduler { - unsafe { &mut *CoreLocal::get().scheduler.get() } +pub(crate) fn core_scheduler() -> RefMut<'static, PerCoreScheduler> { + RefMut::map(CoreLocal::get().scheduler.borrow_mut(), |scheduler| { + scheduler.as_mut().unwrap() + }) } pub(crate) fn async_tasks() -> RefMut<'static, Vec> { CoreLocal::get().async_tasks.borrow_mut() } -pub(crate) fn set_core_scheduler(scheduler: *mut PerCoreScheduler) { - CoreLocal::get().scheduler.set(scheduler); +pub(crate) fn set_core_scheduler(scheduler: PerCoreScheduler) { + *CoreLocal::get().scheduler.borrow_mut() = Some(scheduler); } pub(crate) fn increment_irq_counter(irq_no: u8) { diff --git a/src/arch/x86_64/kernel/scheduler.rs b/src/arch/x86_64/kernel/scheduler.rs index 41d11e3954..10fb48708b 100644 --- a/src/arch/x86_64/kernel/scheduler.rs +++ b/src/arch/x86_64/kernel/scheduler.rs @@ -16,6 +16,7 @@ use crate::arch::x86_64::mm::{PhysAddr, VirtAddr}; use crate::config::*; use crate::kernel; use crate::scheduler::task::{Task, TaskFrame}; +use crate::scheduler::PerCoreSchedulerExt; #[repr(C, packed)] struct State { diff --git a/src/drivers/net/mod.rs b/src/drivers/net/mod.rs index 64fbe47305..1144912e99 100644 --- a/src/drivers/net/mod.rs +++ b/src/drivers/net/mod.rs @@ -70,6 +70,8 @@ pub(crate) fn network_irqhandler(_state: &State) -> bool { #[cfg(target_arch = "x86_64")] pub(crate) extern "x86-interrupt" fn network_irqhandler(_stack_frame: ExceptionStackFrame) { + use crate::scheduler::PerCoreSchedulerExt; + debug!("Receive network interrupt"); apic::eoi(); let _ = _irqhandler(); diff --git a/src/executor/network.rs b/src/executor/network.rs index 22d98ca2ff..78122626e5 100644 --- a/src/executor/network.rs +++ b/src/executor/network.rs @@ -29,6 +29,7 @@ use crate::drivers::net::NetworkDriver; use crate::drivers::pci::get_network_driver; use crate::executor::device::HermitNet; use crate::executor::{spawn, TaskNotify}; +use crate::scheduler::PerCoreSchedulerExt; pub(crate) enum NetworkState<'a> { Missing, diff --git a/src/lib.rs b/src/lib.rs index 29cb0edf2b..0f87964021 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -56,6 +56,7 @@ use mm::allocator::LockedAllocator; pub(crate) use crate::arch::*; pub(crate) use crate::config::*; use crate::kernel::is_uhyve_with_pci; +use crate::scheduler::{PerCoreScheduler, PerCoreSchedulerExt}; pub use crate::syscalls::*; #[macro_use] @@ -342,9 +343,8 @@ fn boot_processor_main() -> ! { // Start the initd task. scheduler::PerCoreScheduler::spawn(initd, 0, scheduler::task::NORMAL_PRIO, 0, USER_STACK_SIZE); - let core_scheduler = core_scheduler(); // Run the scheduler loop. - core_scheduler.run(); + PerCoreScheduler::run(); } /// Entry Point of HermitCore for an Application Processor @@ -358,9 +358,8 @@ fn application_processor_main() -> ! { synch_all_cores(); crate::executor::init(); - let core_scheduler = core_scheduler(); // Run the scheduler loop. - core_scheduler.run(); + PerCoreScheduler::run(); } #[cfg(target_os = "none")] diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index f937515c20..53b75993e9 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -1,9 +1,8 @@ -use alloc::boxed::Box; use alloc::collections::{BTreeMap, VecDeque}; use alloc::rc::Rc; #[cfg(feature = "smp")] use alloc::vec::Vec; -use core::cell::RefCell; +use core::cell::{RefCell, RefMut}; use core::sync::atomic::{AtomicU32, Ordering}; use crossbeam_utils::Backoff; @@ -35,7 +34,7 @@ static TASKS: InterruptTicketMutex> = pub type CoreId = u32; #[cfg(feature = "smp")] -struct SchedulerInput { +pub struct SchedulerInput { /// Queue of new tasks new_tasks: VecDeque, /// Queue of task, which are wakeup by another core @@ -74,9 +73,117 @@ pub struct PerCoreScheduler { finished_tasks: VecDeque>>, /// Queue of blocked tasks, sorted by wakeup time. blocked_tasks: BlockedTaskQueue, - /// Queues to handle incoming requests from the other cores - #[cfg(feature = "smp")] - input: InterruptTicketMutex, +} + +pub trait PerCoreSchedulerExt { + /// Triggers the scheduler to reschedule the tasks. + /// Interrupt flag will be cleared during the reschedule + fn reschedule(self); + + #[cfg(any(feature = "tcp", feature = "udp"))] + fn add_network_timer(self, wakeup_time: Option); + + /// Terminate the current task on the current core. + fn exit(self, exit_code: i32) -> !; +} + +impl PerCoreSchedulerExt for RefMut<'_, PerCoreScheduler> { + #[cfg(target_arch = "x86_64")] + fn reschedule(mut self) { + without_interrupts(|| { + if let Some(last_stack_pointer) = self.scheduler() { + let (new_stack_pointer, is_idle) = { + let borrowed = self.current_task.borrow(); + ( + borrowed.last_stack_pointer, + borrowed.status == TaskStatus::Idle, + ) + }; + + if is_idle || Rc::ptr_eq(&self.current_task, &self.fpu_owner) { + drop(self); + unsafe { + switch_to_fpu_owner(last_stack_pointer, new_stack_pointer.as_usize()); + } + } else { + drop(self); + unsafe { + switch_to_task(last_stack_pointer, new_stack_pointer.as_usize()); + } + } + } + }) + } + + /// Trigger an interrupt to reschedule the system + #[cfg(target_arch = "aarch64")] + fn reschedule(self) { + use core::arch::asm; + + use arm_gic::gicv3::{GicV3, IntId, SgiTarget}; + + use crate::interrupts::SGI_RESCHED; + + drop(self); + + unsafe { + asm!("dsb nsh", "isb", options(nostack, nomem, preserves_flags)); + } + + let reschedid = IntId::sgi(SGI_RESCHED.into()); + GicV3::send_sgi( + reschedid, + SgiTarget::List { + affinity3: 0, + affinity2: 0, + affinity1: 0, + target_list: 0b1, + }, + ); + + interrupts::enable(); + } + + #[cfg(any(feature = "tcp", feature = "udp"))] + fn add_network_timer(mut self, wakeup_time: Option) { + without_interrupts(|| { + self.blocked_tasks.add_network_timer(wakeup_time); + drop(self); + }) + } + + fn exit(mut self, exit_code: i32) -> ! { + without_interrupts(|| { + // Get the current task. + let mut current_task_borrowed = self.current_task.borrow_mut(); + assert_ne!( + current_task_borrowed.status, + TaskStatus::Idle, + "Trying to terminate the idle task" + ); + + // Finish the task and reschedule. + debug!( + "Finishing task {} with exit code {}", + current_task_borrowed.id, exit_code + ); + current_task_borrowed.status = TaskStatus::Finished; + NO_TASKS.fetch_sub(1, Ordering::SeqCst); + + let current_id = current_task_borrowed.id; + drop(current_task_borrowed); + + // wakeup tasks, which are waiting for task with the identifier id + if let Some(mut queue) = WAITING_TASKS.lock().remove(¤t_id) { + while let Some(task) = queue.pop_front() { + self.custom_wakeup(task); + } + } + }); + + self.reschedule(); + unreachable!() + } } struct NewTask { @@ -172,42 +279,6 @@ impl PerCoreScheduler { tid } - /// Terminate the current task on the current core. - pub fn exit(&mut self, exit_code: i32) -> ! { - let closure = || { - // Get the current task. - let mut current_task_borrowed = self.current_task.borrow_mut(); - assert_ne!( - current_task_borrowed.status, - TaskStatus::Idle, - "Trying to terminate the idle task" - ); - - // Finish the task and reschedule. - debug!( - "Finishing task {} with exit code {}", - current_task_borrowed.id, exit_code - ); - current_task_borrowed.status = TaskStatus::Finished; - NO_TASKS.fetch_sub(1, Ordering::SeqCst); - - let current_id = current_task_borrowed.id; - drop(current_task_borrowed); - - // wakeup tasks, which are waiting for task with the identifier id - if let Some(mut queue) = WAITING_TASKS.lock().remove(¤t_id) { - while let Some(task) = queue.pop_front() { - self.custom_wakeup(task); - } - } - }; - - without_interrupts(closure); - - self.reschedule(); - unreachable!() - } - #[cfg(feature = "newlib")] fn clone_impl(&self, func: extern "C" fn(usize), arg: usize) -> TaskId { static NEXT_CORE_ID: AtomicU32 = AtomicU32::new(1); @@ -298,19 +369,27 @@ impl PerCoreScheduler { pub fn handle_waiting_tasks(&mut self) { without_interrupts(|| { crate::executor::run(); - self.blocked_tasks.handle_waiting_tasks(); + for task in self.blocked_tasks.handle_waiting_tasks() { + self.ready_queue.push(task); + } }); } #[cfg(not(feature = "smp"))] pub fn custom_wakeup(&mut self, task: TaskHandle) { - without_interrupts(|| self.blocked_tasks.custom_wakeup(task)); + without_interrupts(|| { + let task = self.blocked_tasks.custom_wakeup(task); + self.ready_queue.push(task); + }); } #[cfg(feature = "smp")] pub fn custom_wakeup(&mut self, task: TaskHandle) { if task.get_core_id() == self.core_id { - without_interrupts(|| self.blocked_tasks.custom_wakeup(task)); + without_interrupts(|| { + let task = self.blocked_tasks.custom_wakeup(task); + self.ready_queue.push(task); + }); } else { get_scheduler_input(task.get_core_id()) .lock() @@ -329,12 +408,6 @@ impl PerCoreScheduler { }); } - #[cfg(any(feature = "tcp", feature = "udp"))] - #[inline] - pub fn add_network_timer(&mut self, wakeup_time: Option) { - without_interrupts(|| self.blocked_tasks.add_network_timer(wakeup_time)) - } - #[inline] pub fn get_current_task_handle(&self) -> TaskHandle { without_interrupts(|| { @@ -449,10 +522,11 @@ impl PerCoreScheduler { #[cfg(all(target_arch = "x86_64", feature = "smp"))] pub fn check_input(&mut self) { - let mut input_locked = self.input.lock(); + let mut input_locked = CoreLocal::get().scheduler_input.lock(); while let Some(task) = input_locked.wakeup_tasks.pop_front() { - self.blocked_tasks.custom_wakeup(task); + let task = self.blocked_tasks.custom_wakeup(task); + self.ready_queue.push(task); } while let Some(new_task) = input_locked.new_tasks.pop_front() { @@ -461,76 +535,24 @@ impl PerCoreScheduler { } } - /// Triggers the scheduler to reschedule the tasks. - /// Interrupt flag will be cleared during the reschedule - #[cfg(target_arch = "x86_64")] - pub fn reschedule(&mut self) { - without_interrupts(|| { - if let Some(last_stack_pointer) = self.scheduler() { - let (new_stack_pointer, is_idle) = { - let borrowed = self.current_task.borrow(); - ( - borrowed.last_stack_pointer, - borrowed.status == TaskStatus::Idle, - ) - }; - - if is_idle || Rc::ptr_eq(&self.current_task, &self.fpu_owner) { - unsafe { - switch_to_fpu_owner(last_stack_pointer, new_stack_pointer.as_usize()); - } - } else { - unsafe { - switch_to_task(last_stack_pointer, new_stack_pointer.as_usize()); - } - } - } - }) - } - - /// Trigger an interrupt to reschedule the system - #[cfg(target_arch = "aarch64")] - pub fn reschedule(&self) { - use core::arch::asm; - - use arm_gic::gicv3::{GicV3, IntId, SgiTarget}; - - use crate::interrupts::SGI_RESCHED; - - unsafe { - asm!("dsb nsh", "isb", options(nostack, nomem, preserves_flags)); - } - - let reschedid = IntId::sgi(SGI_RESCHED.into()); - GicV3::send_sgi( - reschedid, - SgiTarget::List { - affinity3: 0, - affinity2: 0, - affinity1: 0, - target_list: 0b1, - }, - ); - - interrupts::enable(); - } - /// Only the idle task should call this function. /// Set the idle task to halt state if not another /// available. - pub fn run(&mut self) -> ! { + pub fn run() -> ! { let backoff = Backoff::new(); loop { + let mut core_scheduler = core_scheduler(); interrupts::disable(); // run async tasks crate::executor::run(); // do housekeeping - self.cleanup_tasks(); + core_scheduler.cleanup_tasks(); - if self.ready_queue.is_empty() { + if core_scheduler.ready_queue.is_empty() { + drop(core_scheduler); if backoff.is_completed() { interrupts::enable_and_wait(); } else { @@ -539,7 +561,7 @@ impl PerCoreScheduler { } } else { interrupts::enable(); - self.reschedule(); + core_scheduler.reschedule(); backoff.reset(); } } @@ -681,7 +703,7 @@ pub fn add_current_core() { "Initializing scheduler for core {} with idle task {}", core_id, tid ); - let boxed_scheduler = Box::new(PerCoreScheduler { + set_core_scheduler(PerCoreScheduler { #[cfg(feature = "smp")] core_id, current_task: idle_task.clone(), @@ -691,18 +713,13 @@ pub fn add_current_core() { ready_queue: PriorityTaskQueue::new(), finished_tasks: VecDeque::new(), blocked_tasks: BlockedTaskQueue::new(), - #[cfg(feature = "smp")] - input: InterruptTicketMutex::new(SchedulerInput::new()), }); - - let scheduler = Box::into_raw(boxed_scheduler); - set_core_scheduler(scheduler); #[cfg(feature = "smp")] { - let scheduler = unsafe { scheduler.as_ref().unwrap() }; - SCHEDULER_INPUTS - .lock() - .insert(core_id.try_into().unwrap(), &scheduler.input); + SCHEDULER_INPUTS.lock().insert( + core_id.try_into().unwrap(), + &CoreLocal::get().scheduler_input, + ); } } @@ -713,11 +730,9 @@ fn get_scheduler_input(core_id: CoreId) -> &'static InterruptTicketMutex Result<(), ()> { - let core_scheduler = core_scheduler(); - debug!( "Task {} is waiting for task {}", - core_scheduler.get_current_task_id(), + core_scheduler().get_current_task_id(), id ); @@ -725,6 +740,7 @@ pub fn join(id: TaskId) -> Result<(), ()> { let mut waiting_tasks_guard = WAITING_TASKS.lock(); if let Some(queue) = waiting_tasks_guard.get_mut(&id) { + let mut core_scheduler = core_scheduler(); queue.push_back(core_scheduler.get_current_task_handle()); core_scheduler.block_current_task(None); diff --git a/src/scheduler/task.rs b/src/scheduler/task.rs index caed8dd6ed..7078e9a3f0 100644 --- a/src/scheduler/task.rs +++ b/src/scheduler/task.rs @@ -1,6 +1,7 @@ use alloc::boxed::Box; use alloc::collections::{LinkedList, VecDeque}; use alloc::rc::Rc; +use alloc::vec::Vec; use core::cell::RefCell; use core::cmp::Ordering; use core::fmt; @@ -463,31 +464,26 @@ impl BlockedTaskQueue { } fn wakeup_task(task: Rc>) { - { - let mut borrowed = task.borrow_mut(); - debug!( - "Waking up task {} on core {}", - borrowed.id, borrowed.core_id - ); - - assert!( - borrowed.core_id == core_id(), - "Try to wake up task {} on the wrong core {} != {}", - borrowed.id, - borrowed.core_id, - core_id() - ); + let mut borrowed = task.borrow_mut(); + debug!( + "Waking up task {} on core {}", + borrowed.id, borrowed.core_id + ); - assert!( - borrowed.status == TaskStatus::Blocked, - "Trying to wake up task {} which is not blocked", - borrowed.id - ); - borrowed.status = TaskStatus::Ready; - } + assert!( + borrowed.core_id == core_id(), + "Try to wake up task {} on the wrong core {} != {}", + borrowed.id, + borrowed.core_id, + core_id() + ); - // Add the task to the ready queue. - core_scheduler().ready_queue.push(task); + assert!( + borrowed.status == TaskStatus::Blocked, + "Trying to wake up task {} which is not blocked", + borrowed.id + ); + borrowed.status = TaskStatus::Ready; } #[cfg(any(feature = "tcp", feature = "udp"))] @@ -560,7 +556,7 @@ impl BlockedTaskQueue { } /// Manually wake up a blocked task. - pub fn custom_wakeup(&mut self, task: TaskHandle) { + pub fn custom_wakeup(&mut self, task: TaskHandle) -> Rc> { let mut first_task = true; let mut cursor = self.list.cursor_front_mut(); @@ -574,8 +570,8 @@ impl BlockedTaskQueue { // Loop through all blocked tasks to find it. while let Some(node) = cursor.current() { if node.task.borrow().id == task.get_id() { - // Remove it from the list of blocked tasks and wake it up. - Self::wakeup_task(node.task.clone()); + // Remove it from the list of blocked tasks. + let task_ref = node.task.clone(); cursor.remove_current(); // If this is the first task, adjust the One-Shot Timer to fire at the @@ -609,19 +605,24 @@ impl BlockedTaskQueue { ); } - break; + // Wake it up. + Self::wakeup_task(task_ref.clone()); + + return task_ref; } first_task = false; cursor.move_next(); } + + unreachable!(); } /// Wakes up all tasks whose wakeup time has elapsed. /// /// Should be called by the One-Shot Timer interrupt handler when the wakeup time for /// at least one task has elapsed. - pub fn handle_waiting_tasks(&mut self) { + pub fn handle_waiting_tasks(&mut self) -> Vec>> { // Get the current time. let time = arch::processor::get_timer_ticks(); @@ -634,6 +635,8 @@ impl BlockedTaskQueue { } } + let mut tasks = vec![]; + // Loop through all blocked tasks. let mut cursor = self.list.cursor_front_mut(); while let Some(node) = cursor.current() { @@ -645,7 +648,7 @@ impl BlockedTaskQueue { } // Otherwise, this task has elapsed, so remove it from the list and wake it up. - Self::wakeup_task(node.task.clone()); + tasks.push(node.task.clone()); cursor.remove_current(); } @@ -673,5 +676,11 @@ impl BlockedTaskQueue { .current() .map_or_else(|| None, |node| node.wakeup_time), ); + + for task in tasks.iter().cloned() { + Self::wakeup_task(task); + } + + tasks } } diff --git a/src/synch/futex.rs b/src/synch/futex.rs index db1196d281..a22d5a4ebe 100644 --- a/src/synch/futex.rs +++ b/src/synch/futex.rs @@ -10,6 +10,7 @@ use crate::arch::kernel::core_local::core_scheduler; use crate::arch::kernel::processor::get_timer_ticks; use crate::errno::{EAGAIN, EINVAL, ETIMEDOUT}; use crate::scheduler::task::TaskHandlePriorityQueue; +use crate::scheduler::PerCoreSchedulerExt; // TODO: Replace with a concurrent hashmap. static PARKING_LOT: InterruptTicketMutex> = @@ -51,14 +52,15 @@ pub(crate) fn futex_wait( timeout }; - let scheduler = core_scheduler(); + let mut scheduler = core_scheduler(); scheduler.block_current_task(wakeup_time); let handle = scheduler.get_current_task_handle(); parking_lot.entry(addr(address)).or_default().push(handle); drop(parking_lot); + drop(scheduler); loop { - scheduler.reschedule(); + core_scheduler().reschedule(); let mut parking_lot = PARKING_LOT.lock(); if matches!(wakeup_time, Some(t) if t <= get_timer_ticks()) { @@ -87,7 +89,7 @@ pub(crate) fn futex_wait( } else { // A spurious wakeup occurred, sleep again. // Tasks do not change core, so the handle in the parking lot is still current. - scheduler.block_current_task(wakeup_time); + core_scheduler().block_current_task(wakeup_time); } } drop(parking_lot); @@ -120,14 +122,15 @@ pub(crate) fn futex_wait_and_set( timeout }; - let scheduler = core_scheduler(); + let mut scheduler = core_scheduler(); scheduler.block_current_task(wakeup_time); let handle = scheduler.get_current_task_handle(); parking_lot.entry(addr(address)).or_default().push(handle); drop(parking_lot); + drop(scheduler); loop { - scheduler.reschedule(); + core_scheduler().reschedule(); let mut parking_lot = PARKING_LOT.lock(); if matches!(wakeup_time, Some(t) if t <= get_timer_ticks()) { @@ -156,7 +159,7 @@ pub(crate) fn futex_wait_and_set( } else { // A spurious wakeup occurred, sleep again. // Tasks do not change core, so the handle in the parking lot is still current. - scheduler.block_current_task(wakeup_time); + core_scheduler().block_current_task(wakeup_time); } } drop(parking_lot); @@ -177,7 +180,7 @@ pub(crate) fn futex_wake(address: &AtomicU32, count: i32) -> i32 { Entry::Vacant(_) => return 0, }; - let scheduler = core_scheduler(); + let mut scheduler = core_scheduler(); let mut woken = 0; while woken != count || count == i32::MAX { match queue.get_mut().pop() { @@ -212,7 +215,7 @@ pub(crate) fn futex_wake_or_set(address: &AtomicU32, count: i32, new_value: u32) } }; - let scheduler = core_scheduler(); + let mut scheduler = core_scheduler(); let mut woken = 0; while woken != count || count == i32::MAX { match queue.get_mut().pop() { diff --git a/src/synch/recmutex.rs b/src/synch/recmutex.rs index 885fa9fdd1..03520a97f5 100644 --- a/src/synch/recmutex.rs +++ b/src/synch/recmutex.rs @@ -2,6 +2,7 @@ use hermit_sync::TicketMutex; use crate::arch::core_local::*; use crate::scheduler::task::{TaskHandlePriorityQueue, TaskId}; +use crate::scheduler::PerCoreSchedulerExt; struct RecursiveMutexState { current_tid: Option, @@ -26,10 +27,10 @@ impl RecursiveMutex { pub fn acquire(&self) { // Get information about the current task. - let core_scheduler = core_scheduler(); - let tid = core_scheduler.get_current_task_id(); + let tid = core_scheduler().get_current_task_id(); loop { + let mut core_scheduler = core_scheduler(); { let mut locked_state = self.state.lock(); diff --git a/src/synch/semaphore.rs b/src/synch/semaphore.rs index 2bf81ec22a..b0bb5558d9 100644 --- a/src/synch/semaphore.rs +++ b/src/synch/semaphore.rs @@ -4,6 +4,7 @@ use hermit_sync::InterruptTicketMutex; use crate::arch::core_local::*; use crate::scheduler::task::TaskHandlePriorityQueue; +use crate::scheduler::PerCoreSchedulerExt; struct SemaphoreState { /// Resource available count @@ -67,12 +68,12 @@ impl Semaphore { pub fn acquire(&self, time: Option) -> bool { #[cfg(feature = "smp")] let backoff = Backoff::new(); - let core_scheduler = core_scheduler(); let wakeup_time = time.map(|ms| crate::arch::processor::get_timer_ticks() + ms * 1000); // Loop until we have acquired the semaphore. loop { + let mut core_scheduler = core_scheduler(); let mut locked_state = self.state.lock(); if locked_state.count > 0 { diff --git a/src/syscalls/tasks.rs b/src/syscalls/tasks.rs index 8edbac4880..3edc6a8204 100644 --- a/src/syscalls/tasks.rs +++ b/src/syscalls/tasks.rs @@ -14,6 +14,7 @@ use crate::errno::*; #[cfg(feature = "newlib")] use crate::mm::{task_heap_end, task_heap_start}; use crate::scheduler::task::{Priority, TaskHandle, TaskId}; +use crate::scheduler::PerCoreSchedulerExt; use crate::syscalls::timer::timespec; use crate::{arch, scheduler, syscalls}; @@ -115,7 +116,7 @@ pub(crate) extern "C" fn __sys_usleep(usecs: u64) { // Enough time to set a wakeup timer and block the current task. debug!("sys_usleep blocking the task for {} microseconds", usecs); let wakeup_time = arch::processor::get_timer_ticks() + usecs; - let core_scheduler = core_scheduler(); + let mut core_scheduler = core_scheduler(); core_scheduler.block_current_task(Some(wakeup_time)); // Switch to the next task. @@ -299,7 +300,7 @@ static BLOCKED_TASKS: InterruptTicketMutex> = extern "C" fn __sys_block_current_task(timeout: &Option) { let wakeup_time = timeout.map(|t| arch::processor::get_timer_ticks() + t * 1000); - let core_scheduler = core_scheduler(); + let mut core_scheduler = core_scheduler(); let handle = core_scheduler.get_current_task_handle(); let tid = core_scheduler.get_current_task_id();