Scheduler re-write to support multiple CPUs (#78)

* Rebase sched onto main * a * Fixé scheduler 2.0 * Update config script, make scheduler own the stacks * Check for FileNotFound exception
badgeteam · Aug 12, 2024 · 33009a6 · 33009a6
1 parent 0e2cb90
commit 33009a6
Show file tree

Hide file tree

Showing 28 changed files with 1,041 additions and 962 deletions.
diff --git a/Makefile b/Makefile
@@ -25,7 +25,7 @@ why2025_defconfig:
 
 .PHONY: unmatched_defconfig
 unmatched_defconfig:
-	./tools/config.py --target generic --use-default --vec-spec none
+	./tools/config.py --target generic --use-default --vec_spec none
 
 .PHONY: build
 build:

diff --git a/files/CMakeLists.txt b/files/CMakeLists.txt
@@ -36,7 +36,7 @@ set(badge_libs crt badgelib)
 macro(badgeros_executable exec installdir)
     add_executable(${exec})
     target_compile_options(${exec} PRIVATE ${badge_cflags} -ffunction-sections)
-    target_link_options(${exec} PRIVATE ${badge_cflags} -Wl,--gc-sections -nostartfiles)
+    target_link_options(${exec} PRIVATE ${badge_cflags} -pie -Wl,--gc-sections -nostartfiles)
     target_include_directories(${exec} PRIVATE ${badge_include})
     target_link_libraries(${exec} PRIVATE ${badge_libs})
     install(TARGETS ${exec} RUNTIME DESTINATION ${installdir})

diff --git a/kernel/Makefile b/kernel/Makefile
@@ -5,7 +5,6 @@ CONFIG_PATH       ?= ../.config/config.mk
 include $(CONFIG_PATH)
 
 MAKEFLAGS         += --silent
-IDF_PATH          ?= $(shell pwd)/../esp-idf
 SHELL             := /usr/bin/env bash
 OUTPUT            ?= $(shell pwd)/firmware
 BUILDDIR          ?= build

diff --git a/kernel/cpu/riscv/include/cpu/isr.h b/kernel/cpu/riscv/include/cpu/isr.h
@@ -50,9 +50,15 @@ static inline bool isr_global_disable() {
 static inline void isr_global_enable() {
     asm volatile("csrs " CSR_STATUS_STR ", %0" ::"r"((1U << CSR_STATUS_IE_BIT)));
 }
-// Explicit context switch from M-mode.
+// Explicit context switch from kernel.
 // Interrupts must be disabled on entry and will be re-enabled on exit.
 // If the context switch target is not set, this is a NOP.
-extern void isr_context_switch();
+extern void        isr_context_switch();
+// Pause the CPU briefly.
+static inline void isr_pause() {
+    // RISC-V Zihintpause instruction.
+    // This is a fence with PRED=W and SUCC=none.
+    asm(".word 0x0100000f");
+}
 
 #endif
diff --git a/kernel/cpu/riscv/include/cpu/isr_ctx.h b/kernel/cpu/riscv/include/cpu/isr_ctx.h
@@ -6,6 +6,8 @@
 #include "cpu/regs.h"
 
 #ifndef __ASSEMBLER__
+#include "cpulocal.h"
+#include "log.h"
 #include "memprotect.h"
 
 #include <stdbool.h>
@@ -76,6 +78,8 @@ STRUCT_FIELD_WORD(isr_ctx_t, flags, 42)
 STRUCT_FIELD_STRUCT(isr_ctx_t, isr_noexc_cb_t, noexc_cb, 43)
 // Cookie for custom trap handler.
 STRUCT_FIELD_PTR(isr_ctx_t, void, noexc_cookie, 44)
+// Pointer to CPU-local struct.
+STRUCT_FIELD_PTR(isr_ctx_t, cpulocal_t, cpulocal, 45)
 STRUCT_END(isr_ctx_t)
 
 // `isr_ctx_t` flag: Is a kernel thread.

diff --git a/kernel/cpu/riscv/src/backtrace.c b/kernel/cpu/riscv/src/backtrace.c
@@ -41,13 +41,13 @@ void backtrace() NAKED;
 #if __riscv_xlen == 64
 void backtrace() {
     asm volatile("addi sp, sp, -16");
-    asm volatile("sw   ra, 8(sp)");
-    asm volatile("sw   s0, 0(sp)");
+    asm volatile("sd   ra, 8(sp)");
+    asm volatile("sd   s0, 0(sp)");
     asm volatile("addi s0, sp, 16");
     asm volatile("mv   a0, s0");
     asm volatile("jal  backtrace_from_ptr");
-    asm volatile("lw   ra, 8(sp)");
-    asm volatile("lw   s0, 0(sp)");
+    asm volatile("ld   ra, 8(sp)");
+    asm volatile("ld   s0, 0(sp)");
     asm volatile("addi sp, sp, 16");
     asm volatile("ret");
 }

diff --git a/kernel/cpu/riscv/src/entrypoint.S b/kernel/cpu/riscv/src/entrypoint.S
@@ -18,18 +18,6 @@
 
 
 
-	# Reserve stack.
-	.section ".bss"
-	.align 4
-	.global __stack_top
-	.global __stack_bottom
-	.global __stack_size
-	.equ __stack_size, 8192
-	.lcomm __stack_bottom, __stack_size
-	.equ __stack_top, __stack_bottom + __stack_size
-
-
-
 	# Entrypoint from the bootloader.
 	.text
 	.align 2
@@ -42,7 +30,6 @@ _start:
 	la   gp, __global_pointer$
 	.option pop
 	mv   tp, x0
-	la   sp, __stack_top
 
 	# Zero out .bss section.
 #ifndef CONFIG_TARGET_generic

diff --git a/kernel/cpu/riscv/src/memprotect/riscv_pmp.c b/kernel/cpu/riscv/src/memprotect/riscv_pmp.c
@@ -8,6 +8,7 @@
 #include "log.h"
 #include "memprotect.h"
 #include "port/hardware_allocation.h"
+#include "scheduler/types.h"
 
 // PMP granularity.
 static size_t grain;
@@ -532,12 +533,20 @@ void riscv_pmp_memprotect_swap(riscv_pmp_ctx_t *ctx) {
 void memprotect_swap_from_isr() {
     isr_ctx_t *ctx = isr_ctx_get();
     if (!(ctx->flags & ISR_CTX_FLAG_KERNEL)) {
+        if (!ctx->mpu_ctx) {
+            logkf_from_isr(LOG_FATAL, "User ISR context 0x%{size;x} has no MPU context", ctx);
+            sched_thread_t *thread = ctx->thread;
+            if (thread) {
+                logkf_from_isr(LOG_DEBUG, "Thread #%{d} '%{cs}'", thread->id, thread->name);
+            }
+            panic_abort();
+        }
         assert_dev_drop(ctx->mpu_ctx);
         riscv_pmp_memprotect_swap(ctx->mpu_ctx);
     }
 }
 
 // Swap in memory protections for a given context.
 void memprotect_swap(mpu_ctx_t *mpu) {
-    (void)mpu;
+    riscv_pmp_memprotect_swap(mpu);
 }
diff --git a/kernel/cpu/riscv/src/scheduler.c b/kernel/cpu/riscv/src/scheduler.c
@@ -12,6 +12,9 @@
 #include "process/types.h"
 #include "scheduler/cpu.h"
 #include "scheduler/isr.h"
+#if MEMMAP_VMEM
+#include "cpu/mmu.h"
+#endif
 
 
 
@@ -47,33 +50,32 @@ void sched_raise_from_isr(sched_thread_t *thread, bool syscall, void *entry_poin
 // Requests the scheduler to prepare a switch from kernel to userland for a user thread.
 // Resumes the userland thread where it left off.
 void sched_lower_from_isr() {
-    sched_thread_t *thread  = sched_get_current_thread_unsafe();
+    sched_thread_t *thread  = sched_current_thread_unsafe();
     process_t      *process = thread->process;
     assert_dev_drop(!(thread->flags & THREAD_KERNEL) && (thread->flags & THREAD_PRIVILEGED));
-    thread->flags &= ~THREAD_PRIVILEGED;
+    atomic_fetch_and(&thread->flags, ~THREAD_PRIVILEGED);
 
     // Set context switch target to user thread.
     isr_ctx_switch_set(&thread->user_isr_ctx);
     assert_dev_drop(!(thread->user_isr_ctx.flags & ISR_CTX_FLAG_KERNEL));
 
     if (atomic_load(&process->flags) & PROC_EXITING) {
         // Request a context switch to a different thread.
-        thread->flags &= ~THREAD_RUNNING;
         sched_request_switch_from_isr();
     }
 }
 
 // Check whether the current thread is in a signal handler.
 // Returns signal number, or 0 if not in a signal handler.
 bool sched_is_sighandler() {
-    sched_thread_t *thread = sched_get_current_thread();
-    return thread->flags & THREAD_SIGHANDLER;
+    sched_thread_t *thread = sched_current_thread();
+    return atomic_load(&thread->flags) & THREAD_SIGHANDLER;
 }
 
 // Enters a signal handler in the current thread.
 // Returns false if there isn't enough resources to do so.
 bool sched_signal_enter(size_t handler_vaddr, size_t return_vaddr, int signum) {
-    sched_thread_t *thread = sched_get_current_thread();
+    sched_thread_t *thread = sched_current_thread();
 
     // Ensure the user has enough stack.
     size_t usp   = thread->user_isr_ctx.regs.sp;
@@ -85,7 +87,9 @@ bool sched_signal_enter(size_t handler_vaddr, size_t return_vaddr, int signum) {
     thread->user_isr_ctx.regs.sp -= usize;
 
     // Save context to user's stack.
-    // TODO: Enable SUM bit for S-mode kernel.
+#if MEMMAP_VMEM
+    mmu_enable_sum();
+#endif
     size_t *stackptr = (size_t *)thread->user_isr_ctx.regs.sp;
     stackptr[0]      = thread->user_isr_ctx.regs.t0;
     stackptr[1]      = thread->user_isr_ctx.regs.t1;
@@ -105,7 +109,9 @@ bool sched_signal_enter(size_t handler_vaddr, size_t return_vaddr, int signum) {
     stackptr[17]     = thread->user_isr_ctx.regs.pc;
     stackptr[18]     = thread->user_isr_ctx.regs.s0;
     stackptr[19]     = thread->user_isr_ctx.regs.ra;
-    // TODO: Disable SUM bit for S-mode kernel.
+#if MEMMAP_VMEM
+    mmu_disable_sum();
+#endif
 
     // Set up registers for entering signal handler.
     thread->user_isr_ctx.regs.s0 = thread->user_isr_ctx.regs.sp + usize;
@@ -114,15 +120,17 @@ bool sched_signal_enter(size_t handler_vaddr, size_t return_vaddr, int signum) {
     thread->user_isr_ctx.regs.a0 = signum;
 
     // Successfully entered signal handler.
-    thread->flags |= THREAD_SIGHANDLER;
+    atomic_fetch_or(&thread->flags, THREAD_SIGHANDLER);
     return true;
 }
 
 // Exits a signal handler in the current thread.
 // Returns false if the process cannot be resumed.
 bool sched_signal_exit() {
-    sched_thread_t *thread  = sched_get_current_thread_unsafe();
-    thread->flags          &= ~THREAD_SIGHANDLER;
+    sched_thread_t *thread = sched_current_thread_unsafe();
+    if (!(atomic_fetch_and(&thread->flags, ~THREAD_SIGHANDLER) & THREAD_SIGHANDLER)) {
+        return false;
+    }
 
     // Ensure the user still has the stack.
     size_t usp   = thread->user_isr_ctx.regs.sp;
@@ -132,8 +140,10 @@ bool sched_signal_exit() {
         return false;
     }
 
-    // Restore user's state.
-    // TODO: Enable SUM bit for S-mode kernel.
+// Restore user's state.
+#if MEMMAP_VMEM
+    mmu_enable_sum();
+#endif
     size_t *stackptr             = (size_t *)thread->user_isr_ctx.regs.sp;
     thread->user_isr_ctx.regs.t0 = stackptr[0];
     thread->user_isr_ctx.regs.t1 = stackptr[1];
@@ -153,7 +163,9 @@ bool sched_signal_exit() {
     thread->user_isr_ctx.regs.pc = stackptr[17];
     thread->user_isr_ctx.regs.s0 = stackptr[18];
     thread->user_isr_ctx.regs.ra = stackptr[19];
-    // TODO: Disable SUM bit for S-mode kernel.
+#if MEMMAP_VMEM
+    mmu_disable_sum();
+#endif
 
     // Restore user's stack pointer.
     thread->user_isr_ctx.regs.sp += usize;
@@ -163,35 +175,43 @@ bool sched_signal_exit() {
 }
 
 // Return to exit the thread.
-static void sched_exit_self() {
+static void sched_exit_self(int code) {
 #ifndef NDEBUG
-    sched_thread_t *const this_thread = sched_get_current_thread();
-    logkf(LOG_INFO, "Kernel thread '%{cs}' returned", sched_get_name(this_thread));
+    sched_thread_t *const thread = sched_current_thread();
+    logkf(LOG_DEBUG, "Kernel thread '%{cs}' returned %{d}", thread->name, code);
 #endif
-    sched_exit(0);
+    thread_exit(code);
 }
 
 // Prepares a context to be invoked as a kernel thread.
-void sched_prepare_kernel_entry(sched_thread_t *thread, sched_entry_point_t entry_point, void *arg) {
+void sched_prepare_kernel_entry(sched_thread_t *thread, void *entry_point, void *arg) {
     // Initialize registers.
     mem_set(&thread->kernel_isr_ctx.regs, 0, sizeof(thread->kernel_isr_ctx.regs));
     thread->kernel_isr_ctx.regs.pc = (size_t)entry_point;
     thread->kernel_isr_ctx.regs.sp = thread->kernel_stack_top;
     thread->kernel_isr_ctx.regs.a0 = (size_t)arg;
     thread->kernel_isr_ctx.regs.ra = (size_t)sched_exit_self;
-    asm("mv %0, gp" : "=r"(thread->kernel_isr_ctx.regs.gp));
+#if __riscv_xlen == 64
+    asm("sd gp, %0" ::"m"(thread->kernel_isr_ctx.regs.gp));
+#else
+    asm("sw gp, %0" ::"m"(thread->kernel_isr_ctx.regs.gp));
+#endif
 }
 
 // Prepares a pair of contexts to be invoked as a userland thread.
 // Kernel-side in these threads is always started by an ISR and the entry point is given at that time.
-void sched_prepare_user_entry(sched_thread_t *thread, sched_entry_point_t entry_point, void *arg) {
+void sched_prepare_user_entry(sched_thread_t *thread, size_t entry_point, size_t arg) {
     // Initialize kernel registers.
     mem_set(&thread->kernel_isr_ctx.regs, 0, sizeof(thread->kernel_isr_ctx.regs));
     thread->kernel_isr_ctx.regs.sp = thread->kernel_stack_top;
-    asm("mv %0, gp" : "=r"(thread->kernel_isr_ctx.regs.gp));
+#if __riscv_xlen == 64
+    asm("sd gp, %0" ::"m"(thread->kernel_isr_ctx.regs.gp));
+#else
+    asm("sw gp, %0" ::"m"(thread->kernel_isr_ctx.regs.gp));
+#endif
 
     // Initialize userland registers.
     mem_set(&thread->user_isr_ctx.regs, 0, sizeof(thread->user_isr_ctx.regs));
-    thread->user_isr_ctx.regs.pc = (size_t)entry_point;
-    thread->user_isr_ctx.regs.a0 = (size_t)arg;
+    thread->user_isr_ctx.regs.pc = entry_point;
+    thread->user_isr_ctx.regs.a0 = arg;
 }
diff --git a/kernel/include/badgelib/mutex.h b/kernel/include/badgelib/mutex.h
@@ -40,10 +40,25 @@ bool mutex_acquire(badge_err_t *ec, mutex_t *mutex, timestamp_us_t max_wait_us);
 // Release `mutex`, if it was initially acquired by this thread.
 // Returns true if the mutex was successfully released.
 bool mutex_release(badge_err_t *ec, mutex_t *mutex);
+// Try to acquire `mutex` within `max_wait_us` microseconds.
+// If `max_wait_us` is too long or negative, do not use the timeout.
+// Returns true if the mutex was successully acquired.
+bool mutex_acquire_from_isr(badge_err_t *ec, mutex_t *mutex, timestamp_us_t max_wait_us);
+// Release `mutex`, if it was initially acquired by this thread.
+// Returns true if the mutex was successfully released.
+bool mutex_release_from_isr(badge_err_t *ec, mutex_t *mutex);
+
 // Try to acquire a share in `mutex` within `max_wait_us` microseconds.
 // If `max_wait_us` is too long or negative, do not use the timeout.
 // Returns true if the share was successfully acquired.
 bool mutex_acquire_shared(badge_err_t *ec, mutex_t *mutex, timestamp_us_t max_wait_us);
 // Release `mutex`, if it was initially acquired by this thread.
 // Returns true if the mutex was successfully released.
 bool mutex_release_shared(badge_err_t *ec, mutex_t *mutex);
+// Try to acquire a share in `mutex` within `max_wait_us` microseconds.
+// If `max_wait_us` is too long or negative, do not use the timeout.
+// Returns true if the share was successfully acquired.
+bool mutex_acquire_shared_from_isr(badge_err_t *ec, mutex_t *mutex, timestamp_us_t max_wait_us);
+// Release `mutex`, if it was initially acquired by this thread.
+// Returns true if the mutex was successfully released.
+bool mutex_release_shared_from_isr(badge_err_t *ec, mutex_t *mutex);
diff --git a/kernel/include/cpulocal.h b/kernel/include/cpulocal.h
@@ -0,0 +1,25 @@
+
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+#include "scheduler/scheduler.h"
+
+#include <stddef.h>
+
+
+
+// CPU-local data.
+typedef struct {
+    // Current CPU ID.
+    size_t            cpuid;
+    // ISR stack top.
+    size_t            isr_stack_top;
+    // ISR stack bottom.
+    size_t            isr_stack_bottom;
+    // CPU-local scheduler data.
+    sched_cpulocal_t *sched;
+} cpulocal_t;
+
+// Per-CPU CPU-local data.
+extern cpulocal_t *cpulocal;
diff --git a/kernel/include/process/internal.h b/kernel/include/process/internal.h
@@ -17,7 +17,7 @@ void proc_exit_self(int code);
 process_t *proc_get_unsafe(pid_t pid);
 
 // Suspend all threads for a process except the current.
-void proc_suspend(process_t *process, sched_thread_t *current);
+void proc_suspend(process_t *process, tid_t current);
 // Resume all threads for a process.
 void proc_resume(process_t *process);
 // Release all process runtime resources (threads, memory, files, etc.).
@@ -38,9 +38,7 @@ void proc_start_raw(badge_err_t *ec, process_t *process);
 
 // Create a new thread in a process.
 // Returns created thread handle.
-sched_thread_t *proc_create_thread_raw_unsafe(
-    badge_err_t *ec, process_t *process, sched_entry_point_t entry_point, void *arg, sched_prio_t priority
-);
+tid_t    proc_create_thread_raw(badge_err_t *ec, process_t *process, size_t entry_point, size_t arg, int priority);
 // Delete a thread in a process.
 void   proc_delete_thread_raw_unsafe(badge_err_t *ec, process_t *process, sched_thread_t *thread);
 // Allocate more memory to a process.