Kernel: aarch64 initial paging support

Also splits up a lot of paging logic to be per-architecture. We can now map pages in the kernel through the usual mechanisms, and as a result, use the MM.alloc family of functions.

kernel/CMakeLists.txt

@@ -65,7 +65,6 @@ SET(KERNEL_SRCS
         device/MouseDevice.cpp
         interrupt/IRQHandler.cpp
         memory/PageDirectory.cpp
-        memory/PageTable.cpp
         tasking/Process.cpp
         tasking/Thread.cpp
         tasking/Lock.cpp
@@ -170,7 +169,8 @@ SET(KERNEL_SRCS
         net/UDPSocket.cpp
         net/TCPSocket.cpp
         net/Router.cpp
-        api/strerror.c)
+        api/strerror.c
+        constructors.cpp)
 
 IF("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "i686")
     SET(LINKER_SCRIPT ${CMAKE_SOURCE_DIR}/kernel/arch/i386/kernel.ld)
@@ -184,6 +184,10 @@ IF("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "i686")
             arch/i386/tasking.cpp
             arch/i386/gdt.cpp
             arch/i386/kstdio.cpp
+            arch/i386/startup.cpp
+            arch/i386/PageTable.cpp
+            arch/i386/PageDirectory.cpp
+            arch/i386/MemoryManager.cpp
 
             arch/i386/asm/startup.s
             arch/i386/asm/tasking.s
@@ -211,6 +215,8 @@ ELSEIF("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64")
             arch/aarch64/startup.cpp
             arch/aarch64/kstdio.cpp
             arch/aarch64/MMU.cpp
+            arch/aarch64/PageDirectory.cpp
+            arch/aarch64/MemoryManager.cpp
 
             arch/aarch64/asm/startup.S
             arch/aarch64/asm/exception.S
@@ -219,6 +225,7 @@ ELSEIF("${CMAKE_SYSTEM_PROCESSOR}" STREQUAL "aarch64")
             arch/aarch64/rpi/GPIO.cpp
             arch/aarch64/rpi/Mailbox.cpp
             arch/aarch64/rpi/Framebuffer.cpp
+            arch/aarch64/rpi/DeviceInfo.cpp
 
             arch/aarch64/device/Device.cpp)
 ENDIF()

kernel/CommandLine.cpp

@@ -17,7 +17,6 @@
     Copyright (c) Byteduck 2016-2020. All rights reserved.
 */
 
-#include <kernel/memory/PageDirectory.h>
 #include <kernel/kstd/KLog.h>
 #include "CommandLine.h"
 #include "kernel/memory/MemoryManager.h"

kernel/KernelMapper.cpp

@@ -21,7 +21,6 @@
 #include <kernel/tasking/TaskManager.h>
 #include "KernelMapper.h"
 #include <kernel/kstd/cstring.h>
-#include <kernel/memory/PageDirectory.h>
 #include <kernel/memory/MemoryManager.h>
 #include <kernel/kstd/KLog.h>
 

kernel/api/stdarg.h

@@ -7,19 +7,12 @@
 
 __DECL_BEGIN
 
-typedef char* va_list;
+typedef __builtin_va_list va_list;
 
-#define __va_argsiz(t)	\
-	(((sizeof(t) + sizeof(int) - 1) / sizeof(int)) * sizeof(int))
+#define va_start(ap, pN) __builtin_va_start(ap, pN)
 
+#define va_end(ap) __builtin_va_end(ap)
 
-#define va_start(ap, pN)	\
-	((ap) = ((va_list) __builtin_next_arg(pN)))
-
-#define va_end(ap)	((void)0)
-
-#define va_arg(ap, t)					\
-	 (((ap) = (ap) + __va_argsiz(t)),		\
-	  *((t*) (void*) ((ap) - __va_argsiz(t))))
+#define va_arg(ap, t) __builtin_va_arg(ap, t)
 
 __DECL_END

kernel/arch/aarch64/MMU.cpp

@@ -10,8 +10,7 @@
 
 using namespace Aarch64;
 
-#define descriptor_addr(addr) (((addr) >> 12) & 0xFFFFFFFFF)
-
+// TODO: We could unmap these once we're done with them.
 __attribute__((aligned(4096), section(".meminit"))) MMU::TableDescriptor __initial_pgd_storage[512];
 __attribute__((aligned(4096), section(".meminit"))) MMU::TableDescriptor __initial_pud_storage[512];
 __attribute__((aligned(4096), section(".meminit"))) MMU::PageDescriptor __initial_pmd_storage[512];
@@ -21,11 +20,6 @@ auto* initial_pgd = (MMU::TableDescriptor*) (((size_t) &__initial_pgd_storage[0]
 auto* initial_pud = (MMU::TableDescriptor*) (((size_t) &__initial_pud_storage[0]) - HIGHER_HALF);
 auto* initial_pmd = (MMU::PageDescriptor*) (((size_t) &__initial_pmd_storage[0]) - HIGHER_HALF);
 
-constexpr size_t pte_size = PAGE_SIZE;
-constexpr size_t pmd_size = pte_size * 512;
-constexpr size_t pud_size = pmd_size * 512;
-constexpr size_t pgd_size = pud_size * 512;
-
 void MMU::mmu_init() {
 	/** Setup MAIR_EL1 attributes **/
 	Regs::MAIR mair_el1 = {.sections = {0}};
@@ -70,7 +64,7 @@ void MMU::mmu_init() {
 		pmd.read_write  = PageDescriptor::pRW;
 		pmd.shareability = PageDescriptor::OuterShareable;
 		pmd.access = true;
-		pmd.address = descriptor_addr(section * pmd_size + kernel_start_phys);
+		pmd.address = descriptor_addr(section * pmd_size + (kernel_start_phys / pmd_size) * pmd_size);
 	}
 
 	// TCR value. Map ttbr0/1 as follows:
@@ -91,33 +85,13 @@ void MMU::mmu_init() {
 			"tlbi vmalle1                \n"
 			"dsb ish                     \n"
 			"isb                         \n"
-			"mrs x5, sctlr_el1           \n"
-			"orr x5, x5, #0x1            \n"
-			"msr sctlr_el1, x5           \n"
-			"br %[mmu_fin]               \n"
+			"mrs x4, sctlr_el1           \n"
+			"orr x4, x4, #0x1            \n"
+			"msr sctlr_el1, x4           \n"
 			:: [ttbr_val]"r"(initial_pgd),
 			   [tcr_val]"r"(tcr.value),
-			   [mair_val]"r"(mair_el1.value),
-			   [mmu_fin]"r"((uint64_t) &&mmu_fin | HIGHER_HALF) // TODO: Why do I have to OR with HIGHER_HALF to get those upper bits set..?
-			: "x5");
-mmu_fin:
-	// Correct our stack pointer and return address
-	asm volatile (
-			"mov x0, #0                   \n"
-			"msr ttbr0_el1, x0            \n" // Disable ttbr0 identity mapping
-			"tlbi vmalle1                 \n" // Flush tlb
-
-			"mov x0, sp                   \n" // OR high bits into sp
-			"mov x1, #" STR(HIGHER_HALF) "\n"
-			"orr x0, x0, x1               \n"
-			"mov sp, x0                   \n"
-
-			"mov x0, lr                   \n" // OR high bits into lr
-			"orr x0, x0, x1               \n"
-			"mov lr, x0                   \n"
-
-			"ret                          \n" // Manual ret or else the compiler will screw with it
-			::: "x0", "x1");
+			   [mair_val]"r"(mair_el1.value)
+			: "x4");
 }
 
 void MMU::mem_early_panic(const char* str) {

kernel/arch/aarch64/MMU.h

@@ -82,6 +82,47 @@ namespace Aarch64::Regs {
 }
 
 namespace Aarch64::MMU {
+	constexpr size_t pte_size = 4096;
+	constexpr size_t pmd_size = pte_size * 512;
+	constexpr size_t pud_size = pmd_size * 512;
+	constexpr size_t pgd_size = pud_size * 512;
+
+	constexpr uint64_t descriptor_addr(size_t addr) {
+		return ((addr) >> 12) & 0xFFFFFFFFF;
+	}
+
+	constexpr uint64_t pte_index(size_t addr) {
+		return ((addr) >> 12) & 0x1FF;
+	}
+
+	constexpr uint64_t pte_page_index(size_t page) {
+		return page & 0x1FF;
+	}
+
+	constexpr uint64_t pmd_index(size_t addr) {
+		return ((addr) >> 21) & 0x1FF;
+	}
+
+	constexpr uint64_t pmd_page_index(size_t page) {
+		return ((page) >> 9) & 0x1FF;
+	}
+
+	constexpr uint64_t pud_index(size_t addr) {
+		return ((addr) >> 30) & 0x1FF;
+	}
+
+	constexpr uint64_t pud_page_index(size_t page) {
+		return ((page) >> 18) & 0x1FF;
+	}
+
+	constexpr uint64_t pgd_index(size_t addr) {
+		return ((addr) >> 39) & 0x1FF;
+	}
+
+	constexpr uint64_t pgd_page_index(size_t page) {
+		return ((page) >> 27) & 0x1FF;
+	}
+
 	enum Attr: uint8_t {
 		NormalUncacheable = 0,
 		NormalCacheable   = 1,

kernel/arch/aarch64/MemoryManager.cpp

@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/* Copyright © 2016-2024 Byteduck */
+
+#include <kernel/memory/MemoryManager.h>
+#include "rpi/DeviceInfo.h"
+
+void MemoryManager::setup_device_memory_map() {
+	size_t kernel_start_page = (KERNEL_START - HIGHER_HALF) / PAGE_SIZE;
+	size_t kernel_end_page = (KERNEL_END - HIGHER_HALF + PAGE_SIZE - 1) / PAGE_SIZE;
+	size_t sdram_end_page = (RPi::DeviceInfo::inst().sdram_size() + RPi::DeviceInfo::inst().sdram_start()) / PAGE_SIZE;
+	m_physical_regions.push_back(new PhysicalRegion(
+			RPi::DeviceInfo::inst().sdram_start() / PAGE_SIZE,
+			kernel_start_page - (RPi::DeviceInfo::inst().sdram_start() / PAGE_SIZE),
+			false, false));
+	m_physical_regions.push_back(new PhysicalRegion(
+			kernel_start_page,
+			kernel_end_page - kernel_start_page,
+			false, true));
+	m_physical_regions.push_back(new PhysicalRegion(
+			kernel_end_page,
+			sdram_end_page - kernel_end_page,
+			false, false));
+	m_physical_regions.push_back(new PhysicalRegion(
+			RPi::DeviceInfo::inst().mmio_start() / PAGE_SIZE,
+			RPi::DeviceInfo::inst().mmio_size() / PAGE_SIZE,
+			true, true));
+	usable_bytes_ram = RPi::DeviceInfo::inst().sdram_size();
+	total_bytes_ram = usable_bytes_ram;
+	mem_lower_limit = RPi::DeviceInfo::inst().sdram_start();
+	mem_upper_limit = RPi::DeviceInfo::inst().mmio_start() + RPi::DeviceInfo::inst().mmio_size();
+}

kernel/arch/aarch64/PageDirectory.cpp

@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: GPL-3.0-or-later */
+/* Copyright © 2016-2024 Byteduck */
+
+#include "kernel/kstd/vector.hpp"
+#include "kernel/tasking/TaskManager.h"
+#include "kernel/kstd/defines.h"
+#include "kernel/Atomic.h"
+#include "kernel/memory/MemoryManager.h"
+#include "kernel/kstd/KLog.h"
+#include "kernel/KernelMapper.h"
+#include "kernel/kstd/cstring.h"
+#include <kernel/memory/PageDirectory.h>
+
+using namespace Aarch64;
+
+__attribute__((aligned(4096))) MMU::TableDescriptor __kernel_pgd[512];
+__attribute__((aligned(4096))) uint8_t g_early_paging_bump[PAGE_SIZE * 16];
+uint8_t* early_paging_bump_ptr = g_early_paging_bump;
+
+template<typename T, int n>
+T* early_paging_alloc() {
+	static_assert((sizeof(T) * n) % 4096 == 0, "Size must be multiple of 4096");
+	if (early_paging_bump_ptr >= (g_early_paging_bump + sizeof(g_early_paging_bump)))
+		PANIC("EARLY_PAGING_NOMEM", "Ran out of memory in the bump allocator when setting up paging.");
+	auto* ret = early_paging_bump_ptr;
+	early_paging_bump_ptr += sizeof(T) * n;
+	return (T*) ret;
+}
+
+void PageDirectory::init_paging() {
+	setup_kernel_map();
+	asm volatile("msr ttbr1_el1, %0" :: "r"((size_t) __kernel_pgd & ~HIGHER_HALF));
+}
+
+PageDirectory::PageDirectory(PageDirectory::DirectoryType type): m_type(type) {
+	if (m_type == DirectoryType::USER) {
+		// TODO
+	} else {
+		m_global_table.entries = __kernel_pgd;
+		start_page = HIGHER_HALF / PAGE_SIZE;
+	}
+}
+
+PageDirectory::~PageDirectory() {
+
+}
+
+size_t PageDirectory::get_physaddr(size_t virtaddr) {
+	return 0;
+}
+
+size_t PageDirectory::get_physaddr(void *virtaddr) {
+	return 0;
+}
+
+bool PageDirectory::is_mapped(size_t vaddr, bool write) {
+	return false;
+}
+
+bool PageDirectory::is_mapped() {
+	return false;
+}
+
+Result PageDirectory::map_page(PageIndex vpage, PageIndex ppage, VMProt prot) {
+	ASSERT(vpage >= start_page);
+	vpage -= start_page;
+
+	auto pud = m_global_table.get_child(MMU::pgd_page_index(vpage));
+	auto pmd = pud->get_child(MMU::pud_page_index(vpage));
+	auto pte = pmd->get_child(MMU::pmd_page_index(vpage));
+	auto& page = pte->entries[MMU::pte_page_index(vpage)];
+	page = {
+			.valid = true,
+			.type = MMU::PageDescriptor::Table,
+			.attr_index = MMU::Attr::NormalUncacheable, // TODO
+			.security = MMU::PageDescriptor::Secure,
+			.read_write = MMU::PageDescriptor::pRW, // TODO
+			.shareability = Aarch64::MMU::PageDescriptor::OuterShareable,
+			.access = true,
+			.address = MMU::descriptor_addr(ppage * PAGE_SIZE)
+	};
+
+	return Result(SUCCESS);
+}
+
+Result PageDirectory::unmap_page(PageIndex vpage) {
+	return Result(ENOMEM);
+}
+
+uint64_t* PageDirectory::alloc_table() {
+	return early_paging_alloc<uint64_t, 512>();
+}

kernel/arch/aarch64/asm/exception.S

@@ -26,6 +26,8 @@ el3_to_el1:
     msr elr_el3, x0
     mov x0, sp
     msr sp_el1, x0
+    ldr x0, =CPACR_FPEN
+    msr cpacr_el1, x0
     eret
 
 el2_to_el1:

kernel/arch/aarch64/asm/exception.h

@@ -16,6 +16,8 @@
 // SPSR_EL3
 #define SPSR_EL3_VAL ((7 << 6) | 5)
 
+#define CPACR_FPEN 0x300000
+
 #ifdef __cplusplus
 extern "C" void setup_exception_level();
 #endif

kernel/arch/aarch64/asm/startup.S

@@ -11,8 +11,9 @@ start:
     cbnz x10, corestop
 
     // Get ourselves some stack
-    adrp x10, start
-    add x10, x10, :lo12:start
+    adrp x10, __early_stack
+    add x10, x10, :lo12:__early_stack
+    add x10, x10, #0x4000
     mov sp, x10
 
     // Clear out BSS

kernel/arch/aarch64/kernel.ld

@@ -19,7 +19,7 @@ SECTIONS {
 	.rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_BASE)
 	{
 		start_ctors = .;
-		*(.ctors)
+		*(.init_array)
 		end_ctors = .;
 
 		*(.rodata*)

kernel/arch/aarch64/kstdio.cpp

@@ -4,5 +4,5 @@
 #include "rpi/MiniUART.h"
 
 void serial_putch(char c) {
-	RPi::MiniUART::tx(c);
+//	RPi::MiniUART::tx(c);
 }