diff --git a/hal/riscv64/Makefile b/hal/riscv64/Makefile index 61c67d345..bdf459beb 100644 --- a/hal/riscv64/Makefile +++ b/hal/riscv64/Makefile @@ -4,10 +4,8 @@ # Copyright 2018, 2020, 2024 Phoenix Systems # -include hal/tlb/Makefile - OBJS += $(addprefix $(PREFIX_O)hal/riscv64/, _init.o _string.o _interrupts.o hal.o spinlock.o interrupts.o cpu.o pmap.o \ -dtb.o timer.o string.o exceptions.o plic.o sbi.o tlb.o) +dtb.o timer.o string.o exceptions.o plic.o sbi.o) CFLAGS += -Ihal/riscv64 diff --git a/hal/riscv64/arch/cpu.h b/hal/riscv64/arch/cpu.h index 39f5c0b9d..33f0c7159 100644 --- a/hal/riscv64/arch/cpu.h +++ b/hal/riscv64/arch/cpu.h @@ -173,6 +173,9 @@ typedef struct { } __attribute__((packed, aligned(8))) cpu_context_t; +struct _pmap_t; + + /* interrupts */ @@ -282,6 +285,12 @@ static inline int hal_cpuSupervisorMode(cpu_context_t *ctx) void hal_cpuRfenceI(void); +void hal_cpuLocalFlushTLB(const struct _pmap_t *pmap, const void *vaddr); + + +void hal_cpuRemoteFlushTLB(const struct _pmap_t *pmap, const void *vaddr, size_t size); + + /* Code used in disabled code vm/object.c - map_pageFault */ #if 0 /* Disabled until lazy mapping is enabled */ static inline void *hal_cpuGetFaultAddr(void) diff --git a/hal/riscv64/arch/tlb.h b/hal/riscv64/arch/tlb.h deleted file mode 100644 index 4d04cc03f..000000000 --- a/hal/riscv64/arch/tlb.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Phoenix-RTOS - * - * Operating system kernel - * - * TLB handling - * - * Copyright 2023, 2024 Phoenix Systems - * Author: Andrzej Stalke, Lukasz Leczkowski - * - * This file is part of Phoenix-RTOS. - * - * %LICENSE% - */ - - -#ifndef _HAL_RISCV_TLB_H_ -#define _HAL_RISCV_TLB_H_ - - -#include "cpu.h" -#include "pmap.h" - - -void hal_tlbFlushLocal(const pmap_t *pmap); - - -void hal_tlbInvalidateLocalEntry(const pmap_t *pmap, const void *vaddr); - - -int hal_tlbIrqHandler(unsigned int n, cpu_context_t *ctx, void *arg); - - -#endif diff --git a/hal/riscv64/cpu.c b/hal/riscv64/cpu.c index 783fbdf6d..3bb4229dc 100644 --- a/hal/riscv64/cpu.c +++ b/hal/riscv64/cpu.c @@ -13,9 +13,6 @@ * %LICENSE% */ -#include - -#include "hal/tlb/tlb.h" #include "include/errno.h" #include "hal/hal.h" #include "riscv64.h" @@ -35,7 +32,6 @@ struct hal_perHartData { static struct { volatile u32 cpuCnt; volatile u32 cpusStarted; - intr_handler_t tlbIrqHandler; } cpu_common; @@ -371,10 +367,33 @@ void hal_cpuRfenceI(void) } +void hal_cpuLocalFlushTLB(const struct _pmap_t *pmap, const void *vaddr) +{ + (void)pmap; /* TODO: ASID support */ + + /* clang-format off */ + __asm__ volatile ( + "sfence.vma %0, zero" + : + : "r"(vaddr) + : "memory" + ); + /* clang-format on */ +} + + +void hal_cpuRemoteFlushTLB(const struct _pmap_t *pmap, const void *vaddr, size_t size) +{ + (void)pmap; /* TODO: ASID support */ + + unsigned long hart_mask = (1 << hal_cpuGetCount()) - 1; + hal_sbiSfenceVma(hart_mask, 0, (unsigned long)vaddr, size); +} + + __attribute__((section(".init"))) void hal_cpuInitCore(void) { hal_interruptsInitCore(); - hal_tlbInitCore(hal_cpuGetID()); hal_timerInitCore(); hal_cpuAtomicAdd(&cpu_common.cpusStarted, 1); } @@ -384,14 +403,9 @@ __attribute__((section(".init"))) void _hal_cpuInit(void) { long err; - cpu_common.tlbIrqHandler.f = hal_tlbIrqHandler; - cpu_common.tlbIrqHandler.n = TLB_IRQ; - cpu_common.tlbIrqHandler.data = NULL; cpu_common.cpusStarted = 0; cpu_common.cpuCnt = 0; - (void)hal_interruptsSetHandler(&cpu_common.tlbIrqHandler); - hal_cpuInitCore(); /* Start other harts */ diff --git a/hal/riscv64/hal.c b/hal/riscv64/hal.c index 07183a2f2..d49684b90 100644 --- a/hal/riscv64/hal.c +++ b/hal/riscv64/hal.c @@ -13,12 +13,7 @@ * %LICENSE% */ -#include - #include "hal/hal.h" -#include "hal/tlb/tlb.h" - -#include "riscv64.h" #include "include/arch/riscv64/riscv64.h" @@ -67,10 +62,6 @@ void _hal_start(void) void hal_lockScheduler(void) { - hal_tlbShootdown(); - /* Clear pending IPI - we've done the shootdown */ - csr_clear(sip, SIP_SSIP); - /* clang-format off */ __asm__ volatile ( "li t0, 1\n\t" diff --git a/hal/riscv64/pmap.c b/hal/riscv64/pmap.c index 78a44739e..cd12288e8 100644 --- a/hal/riscv64/pmap.c +++ b/hal/riscv64/pmap.c @@ -13,12 +13,10 @@ * %LICENSE% */ -#include #include "hal/pmap.h" #include "hal/spinlock.h" #include "hal/string.h" -#include "hal/tlb/tlb.h" #include "halsyspage.h" #include "dtb.h" @@ -148,7 +146,7 @@ static int _pmap_enter(pmap_t *pmap, addr_t pa, void *va, int attr, page_t *allo /* Initialize pdir (MOD) - because of reentrancy */ pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(alloc->addr, 0xc7); - hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable); + hal_cpuLocalFlushTLB(pmap, pmap_common.ptable); hal_memset(pmap_common.ptable, 0, sizeof(pmap_common.ptable)); alloc = NULL; @@ -157,7 +155,7 @@ static int _pmap_enter(pmap_t *pmap, addr_t pa, void *va, int attr, page_t *allo /* Map next level pdir */ addr = PTE_TO_ADDR(pmap->pdir2[pdi2]); pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(addr, 0xc7); - hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable); + hal_cpuLocalFlushTLB(pmap, pmap_common.ptable); } if ((pmap_common.ptable[pdi1] & PTE_V) == 0) { @@ -165,12 +163,12 @@ static int _pmap_enter(pmap_t *pmap, addr_t pa, void *va, int attr, page_t *allo return -EFAULT; } pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(alloc->addr, 0xc7); - hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable); + hal_cpuLocalFlushTLB(pmap, pmap_common.ptable); hal_memset(pmap_common.ptable, 0, sizeof(pmap_common.ptable)); addr = PTE_TO_ADDR(pmap->pdir2[pdi2]); pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(addr, 0xc7); - hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable); + hal_cpuLocalFlushTLB(pmap, pmap_common.ptable); pmap_common.ptable[pdi1] = PTE(alloc->addr, PTE_V); @@ -180,7 +178,7 @@ static int _pmap_enter(pmap_t *pmap, addr_t pa, void *va, int attr, page_t *allo /* Map next level pdir */ addr = PTE_TO_ADDR(pmap_common.ptable[pdi1]); pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(addr, 0xc7); - hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable); + hal_cpuLocalFlushTLB(pmap, pmap_common.ptable); /* And at last map page or only changle attributes of map entry */ pmap_common.ptable[pti] = PTE(pa, 0xc1 | (attr & 0x3f)); @@ -192,10 +190,10 @@ static int _pmap_enter(pmap_t *pmap, addr_t pa, void *va, int attr, page_t *allo } if (tlbInval != 0) { - hal_tlbInvalidateEntry(pmap, va, 1); + hal_cpuRemoteFlushTLB(pmap, va, SIZE_PAGE); } else { - hal_tlbInvalidateLocalEntry(pmap, va); + hal_cpuLocalFlushTLB(pmap, va); } return EOK; @@ -210,12 +208,8 @@ int pmap_enter(pmap_t *pmap, addr_t pa, void *va, int attr, page_t *alloc) hal_spinlockSet(&pmap_common.lock, &sc); ret = _pmap_enter(pmap, pa, va, attr, alloc, 1); - if (ret == EOK) { - hal_tlbCommit(&pmap_common.lock, &sc); - } - else { - hal_spinlockClear(&pmap_common.lock, &sc); - } + hal_spinlockClear(&pmap_common.lock, &sc); + return ret; } @@ -238,14 +232,14 @@ static void _pmap_remove(pmap_t *pmap, void *vaddr) addr = PTE_TO_ADDR(entry); pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(addr, 0xc7); - hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable); + hal_cpuLocalFlushTLB(pmap, pmap_common.ptable); entry = pmap_common.ptable[pdi1]; if ((entry & PTE_V) != 0) { addr = PTE_TO_ADDR(entry); pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(addr, 0xc7); - hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable); + hal_cpuLocalFlushTLB(pmap, pmap_common.ptable); if ((pmap_common.ptable[pti] & PGHD_EXEC) != 0) { isync = 1; @@ -269,8 +263,8 @@ int pmap_remove(pmap_t *pmap, void *vaddr) hal_spinlockSet(&pmap_common.lock, &sc); _pmap_remove(pmap, vaddr); - hal_tlbInvalidateEntry(pmap, vaddr, 1); - hal_tlbCommit(&pmap_common.lock, &sc); + hal_cpuRemoteFlushTLB(pmap, vaddr, SIZE_PAGE); + hal_spinlockClear(&pmap_common.lock, &sc); return EOK; } @@ -296,12 +290,12 @@ addr_t pmap_resolve(pmap_t *pmap, void *vaddr) addr = PTE_TO_ADDR(pmap->pdir2[pdi2]); pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(addr, 0xc7); - hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable); + hal_cpuLocalFlushTLB(pmap, pmap_common.ptable); addr = PTE_TO_ADDR(pmap_common.ptable[pdi1]); pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(addr, 0xc7); - hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable); + hal_cpuLocalFlushTLB(pmap, pmap_common.ptable); addr = PTE_TO_ADDR(pmap_common.ptable[pti]); hal_spinlockClear(&pmap_common.lock, &sc); @@ -401,7 +395,7 @@ int _pmap_kernelSpaceExpand(pmap_t *pmap, void **start, void *end, page_t *dp) } *start = vaddr; } - hal_tlbFlushLocal(NULL); + hal_cpuLocalFlushTLB(NULL, NULL); pmap->start = (void *)VADDR_KERNEL; pmap->end = end; @@ -490,7 +484,7 @@ void _pmap_init(pmap_t *pmap, void **vstart, void **vend) pmap->satp = ((pmap_resolve(pmap, (char *)pmap_common.pdir2) >> 12) | SATP_MODE_SV39); - hal_tlbFlushLocal(NULL); + hal_cpuLocalFlushTLB(NULL, NULL); } diff --git a/hal/riscv64/riscv64.h b/hal/riscv64/riscv64.h index b1b429585..12557de99 100644 --- a/hal/riscv64/riscv64.h +++ b/hal/riscv64/riscv64.h @@ -65,19 +65,6 @@ }) -static inline void hal_cpuFlushTLB(const void *vaddr) -{ - /* clang-format off */ - __asm__ volatile ( - "sfence.vma %0, zero" - : - : "r"(vaddr) - : "memory" - ); - /* clang-format on */ -} - - static inline void hal_cpuSwitchSpace(addr_t pdir) { /* clang-format off */ diff --git a/hal/riscv64/sbi.c b/hal/riscv64/sbi.c index 75d2cafb0..59a9f475e 100644 --- a/hal/riscv64/sbi.c +++ b/hal/riscv64/sbi.c @@ -45,6 +45,12 @@ #define SBI_HSM_STATUS 0x2 #define SBI_HSM_SUSPEND 0x3 +/* RFENCE extension */ +#define SBI_EXT_RFENCE 0x52464E43 +#define SBI_RFNC_I 0x0 +#define SBI_RFNC_SFENCE_VMA 0x1 +#define SBI_RFNC_SFENCE_VMA_ASID 0x2 + /* Legacy extensions */ #define SBI_LEGACY_SETTIMER 0x0 #define SBI_LEGACY_PUTCHAR 0x1 @@ -56,10 +62,6 @@ #define SBI_LEGACY_REMOTE_SFENCE_VMA_ASID 0x7 #define SBI_LEGACY_SHUTDOWN 0x8 -/* RFENCE extension */ -#define SBI_EXT_RFENCE 0x52464E43 -#define SBI_RFNC_I 0x0 - /* clang-format off */ #define SBI_MINOR(x) ((x) & 0xffffff) /* clang-format on */ @@ -176,6 +178,18 @@ void hal_sbiRfenceI(unsigned long hart_mask, unsigned long hart_mask_base) } +sbiret_t hal_sbiSfenceVma(unsigned long hart_mask, unsigned long hart_mask_base, unsigned long vaddr, unsigned long size) +{ + return hal_sbiEcall(SBI_EXT_RFENCE, SBI_RFNC_SFENCE_VMA, hart_mask, hart_mask_base, vaddr, size, 0, 0); +} + + +sbiret_t hal_sbiSfenceVmaAsid(unsigned long hart_mask, unsigned long hart_mask_base, unsigned long vaddr, unsigned long size, unsigned long asid) +{ + return hal_sbiEcall(SBI_EXT_RFENCE, SBI_RFNC_SFENCE_VMA_ASID, hart_mask, hart_mask_base, vaddr, size, asid, 0); +} + + void _hal_sbiInit(void) { sbiret_t ret = hal_sbiGetSpecVersion(); diff --git a/hal/riscv64/sbi.h b/hal/riscv64/sbi.h index 0839ffd2c..32c60bf6b 100644 --- a/hal/riscv64/sbi.h +++ b/hal/riscv64/sbi.h @@ -84,6 +84,12 @@ sbiret_t hal_sbiHartStart(unsigned long hartid, unsigned long start_addr, unsign void hal_sbiRfenceI(unsigned long hart_mask, unsigned long hart_mask_base); +sbiret_t hal_sbiSfenceVma(unsigned long hart_mask, unsigned long hart_mask_base, unsigned long vaddr, unsigned long size); + + +sbiret_t hal_sbiSfenceVmaAsid(unsigned long hart_mask, unsigned long hart_mask_base, unsigned long vaddr, unsigned long size, unsigned long asid); + + void _hal_sbiInit(void); diff --git a/hal/riscv64/tlb.c b/hal/riscv64/tlb.c deleted file mode 100644 index e72e2ddc2..000000000 --- a/hal/riscv64/tlb.c +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Phoenix-RTOS - * - * Operating system kernel - * - * TLB handling - * - * Copyright 2024 Phoenix Systems - * Author: Lukasz Leczkowski - * - * This file is part of Phoenix-RTOS. - * - * %LICENSE% - */ - -#include -#include - -#include "hal/tlb/tlb.h" - -#include "riscv64.h" - - -void hal_tlbFlushLocal(const pmap_t *pmap) -{ - hal_cpuFlushTLB(NULL); -} - - -void hal_tlbInvalidateLocalEntry(const pmap_t *pmap, const void *vaddr) -{ - hal_cpuFlushTLB(vaddr); -} - - -int hal_tlbIrqHandler(unsigned int n, cpu_context_t *ctx, void *arg) -{ - (void)n; - (void)ctx; - (void)arg; - - csr_clear(sip, SIP_SSIP); - - hal_tlbShootdown(); - - return 0; -} diff --git a/hal/riscv64/tlb.h b/hal/riscv64/tlb.h deleted file mode 100644 index 6b9df7cd0..000000000 --- a/hal/riscv64/tlb.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Phoenix-RTOS - * - * Operating system kernel - * - * TLB handling - * - * Copyright 2023, 2024 Phoenix Systems - * Author: Andrzej Stalke, Lukasz Leczkowski - * - * This file is part of Phoenix-RTOS. - * - * %LICENSE% - */ - - -#ifndef _HAL_RISCV64_TLB_H_ -#define _HAL_RISCV64_TLB_H_ - -#include - - -void hal_tlbFlushLocal(const pmap_t *pmap); - - -void hal_tlbInvalidateLocalEntry(const pmap_t *pmap, const void *vaddr); - - -void hal_tlbIrqHandler(const pmap_t *pmap, const void *vaddr, unsigned long size); - - -#endif