Scheduler on two CPUs with load balancing

files/sbin/init/main.c

@@ -24,6 +24,7 @@ int main() {
     badge_err_t ec = {0};
     print("Hi, Ther.\n");
 
-    syscall_sys_shutdown(true);
+    syscall_sys_shutdown(false);
+
     return 0;
 }

kernel/cpu/riscv/src/scheduler.c

@@ -44,6 +44,14 @@ void sched_raise_from_isr(sched_thread_t *thread, bool syscall, void *entry_poin
         thread->kernel_isr_ctx.regs.a7 = thread->user_isr_ctx.regs.a7;
     }
 
+    // Do time accounting.
+    timestamp_us_t    now         = time_us();
+    sched_cpulocal_t *info        = isr_ctx_get()->cpulocal->sched;
+    timestamp_us_t    used        = now - info->last_preempt;
+    thread->timeusage.cycle_time += used;
+    thread->timeusage.user_time  += used;
+    info->last_preempt            = now;
+
     // Set context switch target to kernel thread.
     isr_ctx_switch_set(&thread->kernel_isr_ctx);
 }
@@ -56,6 +64,14 @@ void sched_lower_from_isr() {
     assert_dev_drop(!(thread->flags & THREAD_KERNEL) && (thread->flags & THREAD_PRIVILEGED));
     atomic_fetch_and(&thread->flags, ~THREAD_PRIVILEGED);
 
+    // Do time accounting.
+    timestamp_us_t    now          = time_us();
+    sched_cpulocal_t *info         = isr_ctx_get()->cpulocal->sched;
+    timestamp_us_t    used         = now - info->last_preempt;
+    thread->timeusage.cycle_time  += used;
+    thread->timeusage.kernel_time += used;
+    info->last_preempt             = now;
+
     // Set context switch target to user thread.
     isr_ctx_switch_set(&thread->user_isr_ctx);
     assert_dev_drop(!(thread->user_isr_ctx.flags & ISR_CTX_FLAG_KERNEL));

kernel/include/badgelib/time.h

@@ -6,6 +6,7 @@
 #include "attributes.h"
 #include "port/hardware_allocation.h"
 
+#include <stdatomic.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>

kernel/include/process/types.h

@@ -107,4 +107,6 @@ typedef struct process_t {
     size_t        sighandlers[SIG_COUNT];
     // Exit code if applicable.
     int           state_code;
+    // Total time usage.
+    timeusage_t   timeusage;
 } process_t;

kernel/include/scheduler/scheduler.h

@@ -5,6 +5,7 @@
 
 #include "attributes.h"
 #include "badge_err.h"
+#include "time.h"
 
 #include <stdbool.h>
 #include <stddef.h>
@@ -21,6 +22,18 @@ typedef int (*sched_entry_t)(void *arg);
 // CPU-local scheduler data.
 typedef struct sched_cpulocal_t sched_cpulocal_t;
 
+// Time usage information.
+typedef struct {
+    // Time spent running in user-space.
+    timestamp_us_t user_time;
+    // Time spent running in kernel-space.
+    timestamp_us_t kernel_time;
+    // Total time usage since last load measurement.
+    timestamp_us_t cycle_time;
+    // Current number of CPUs used in 0.01% units.
+    atomic_int     cpu_usage;
+} timeusage_t;
+
 // will be scheduled with smaller time slices than normal
 #define SCHED_PRIO_LOW    0
 // default value

kernel/include/scheduler/types.h

@@ -18,9 +18,11 @@
 
 
 // The minimum time a thread will run. `SCHED_PRIO_LOW` maps to this.
-#define SCHED_MIN_US 5000
+#define SCHED_MIN_US        5000
 // The time quota increment per increased priority.
-#define SCHED_INC_US 500
+#define SCHED_INC_US        500
+// The interval on which schedulers measure CPU load.
+#define SCHED_LOAD_INTERVAL 250000
 
 
 
@@ -56,13 +58,15 @@ struct sched_thread_t {
     dlist_node_t node;
 
     // Process to which this thread belongs.
-    process_t *process;
+    process_t  *process;
     // Lowest address of the kernel stack.
-    size_t     kernel_stack_bottom;
+    size_t      kernel_stack_bottom;
     // Highest address of the kernel stack.
-    size_t     kernel_stack_top;
+    size_t      kernel_stack_top;
     // Priority of this thread.
-    int        priority;
+    int         priority;
+    // Time usage information.
+    timeusage_t timeusage;
 
     // Thread flags.
     atomic_int flags;
@@ -92,8 +96,14 @@ struct sched_cpulocal_t {
     dlist_t        queue;
     // CPU-local scheduler state flags.
     atomic_int     flags;
-    // CPU load estimate in 0.01% increments.
-    atomic_int     load;
+    // Last preemption time.
+    timestamp_us_t last_preempt;
+    // Time until next measurement interval.
+    timestamp_us_t load_measure_time;
+    // CPU load average in 0.01% increments.
+    atomic_int     load_average;
+    // CPU load estimate for load-balancing purposes.
+    atomic_int     load_estimate;
     // Idle thread.
     sched_thread_t idle_thread;
 };

kernel/src/main.c

@@ -49,7 +49,7 @@ static void kernel_lifetime_func() {
     // Start the kernel services.
     kernel_init();
     // Start other CPUs.
-    // sched_start_altcpus();
+    sched_start_altcpus();
     // Start userland.
     userland_init();