riscv64: shootdown TLB through SBI

JIRA: RTOS_813

hal/riscv64/Makefile

@@ -4,10 +4,8 @@
 # Copyright 2018, 2020, 2024 Phoenix Systems
 #
 
-include hal/tlb/Makefile
-
 OBJS += $(addprefix $(PREFIX_O)hal/riscv64/, _init.o _string.o _interrupts.o hal.o spinlock.o interrupts.o cpu.o pmap.o \
-dtb.o timer.o string.o exceptions.o plic.o sbi.o tlb.o)
+dtb.o timer.o string.o exceptions.o plic.o sbi.o)
 
 CFLAGS += -Ihal/riscv64
 

hal/riscv64/arch/cpu.h

@@ -173,6 +173,9 @@ typedef struct {
 } __attribute__((packed, aligned(8))) cpu_context_t;
 
 
+struct _pmap_t;
+
+
 /* interrupts */
 
 
@@ -282,6 +285,12 @@ static inline int hal_cpuSupervisorMode(cpu_context_t *ctx)
 void hal_cpuRfenceI(void);
 
 
+void hal_cpuLocalFlushTLB(const struct _pmap_t *pmap, const void *vaddr);
+
+
+void hal_cpuRemoteFlushTLB(const struct _pmap_t *pmap, const void *vaddr, size_t size);
+
+
 /* Code used in disabled code vm/object.c - map_pageFault */
 #if 0 /* Disabled until lazy mapping is enabled */
 static inline void *hal_cpuGetFaultAddr(void)

hal/riscv64/cpu.c

@@ -13,9 +13,6 @@
  * %LICENSE%
  */
 
-#include <arch/tlb.h>
-
-#include "hal/tlb/tlb.h"
 #include "include/errno.h"
 #include "hal/hal.h"
 #include "riscv64.h"
@@ -35,7 +32,6 @@ struct hal_perHartData {
 static struct {
 	volatile u32 cpuCnt;
 	volatile u32 cpusStarted;
-	intr_handler_t tlbIrqHandler;
 } cpu_common;
 
 
@@ -371,10 +367,33 @@ void hal_cpuRfenceI(void)
 }
 
 
+void hal_cpuLocalFlushTLB(const struct _pmap_t *pmap, const void *vaddr)
+{
+	(void)pmap; /* TODO: ASID support */
+
+	/* clang-format off */
+	__asm__ volatile (
+		"sfence.vma %0, zero"
+		:
+		: "r"(vaddr)
+		: "memory"
+	);
+	/* clang-format on */
+}
+
+
+void hal_cpuRemoteFlushTLB(const struct _pmap_t *pmap, const void *vaddr, size_t size)
+{
+	(void)pmap; /* TODO: ASID support */
+
+	unsigned long hart_mask = (1 << hal_cpuGetCount()) - 1;
+	hal_sbiSfenceVma(hart_mask, 0, (unsigned long)vaddr, size);
+}
+
+
 __attribute__((section(".init"))) void hal_cpuInitCore(void)
 {
 	hal_interruptsInitCore();
-	hal_tlbInitCore(hal_cpuGetID());
 	hal_timerInitCore();
 	hal_cpuAtomicAdd(&cpu_common.cpusStarted, 1);
 }
@@ -384,14 +403,9 @@ __attribute__((section(".init"))) void _hal_cpuInit(void)
 {
 	long err;
 
-	cpu_common.tlbIrqHandler.f = hal_tlbIrqHandler;
-	cpu_common.tlbIrqHandler.n = TLB_IRQ;
-	cpu_common.tlbIrqHandler.data = NULL;
 	cpu_common.cpusStarted = 0;
 	cpu_common.cpuCnt = 0;
 
-	(void)hal_interruptsSetHandler(&cpu_common.tlbIrqHandler);
-
 	hal_cpuInitCore();
 
 	/* Start other harts */

hal/riscv64/hal.c

@@ -13,12 +13,7 @@
  * %LICENSE%
  */
 
-#include <arch/tlb.h>
-
 #include "hal/hal.h"
-#include "hal/tlb/tlb.h"
-
-#include "riscv64.h"
 #include "include/arch/riscv64/riscv64.h"
 
 
@@ -67,10 +62,6 @@ void _hal_start(void)
 
 void hal_lockScheduler(void)
 {
-	hal_tlbShootdown();
-	/* Clear pending IPI - we've done the shootdown */
-	csr_clear(sip, SIP_SSIP);
-
 	/* clang-format off */
 	__asm__ volatile (
 		"li t0, 1\n\t"

hal/riscv64/pmap.c

@@ -13,12 +13,10 @@
  * %LICENSE%
  */
 
-#include <arch/tlb.h>
 
 #include "hal/pmap.h"
 #include "hal/spinlock.h"
 #include "hal/string.h"
-#include "hal/tlb/tlb.h"
 #include "halsyspage.h"
 
 #include "dtb.h"
@@ -148,7 +146,7 @@ static int _pmap_enter(pmap_t *pmap, addr_t pa, void *va, int attr, page_t *allo
 
 		/* Initialize pdir (MOD) - because of reentrancy */
 		pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(alloc->addr, 0xc7);
-		hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable);
+		hal_cpuLocalFlushTLB(pmap, pmap_common.ptable);
 		hal_memset(pmap_common.ptable, 0, sizeof(pmap_common.ptable));
 
 		alloc = NULL;
@@ -157,20 +155,20 @@ static int _pmap_enter(pmap_t *pmap, addr_t pa, void *va, int attr, page_t *allo
 		/* Map next level pdir */
 		addr = PTE_TO_ADDR(pmap->pdir2[pdi2]);
 		pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(addr, 0xc7);
-		hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable);
+		hal_cpuLocalFlushTLB(pmap, pmap_common.ptable);
 	}
 
 	if ((pmap_common.ptable[pdi1] & PTE_V) == 0) {
 		if (alloc == NULL) {
 			return -EFAULT;
 		}
 		pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(alloc->addr, 0xc7);
-		hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable);
+		hal_cpuLocalFlushTLB(pmap, pmap_common.ptable);
 		hal_memset(pmap_common.ptable, 0, sizeof(pmap_common.ptable));
 
 		addr = PTE_TO_ADDR(pmap->pdir2[pdi2]);
 		pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(addr, 0xc7);
-		hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable);
+		hal_cpuLocalFlushTLB(pmap, pmap_common.ptable);
 
 		pmap_common.ptable[pdi1] = PTE(alloc->addr, PTE_V);
 
@@ -180,7 +178,7 @@ static int _pmap_enter(pmap_t *pmap, addr_t pa, void *va, int attr, page_t *allo
 	/* Map next level pdir */
 	addr = PTE_TO_ADDR(pmap_common.ptable[pdi1]);
 	pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(addr, 0xc7);
-	hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable);
+	hal_cpuLocalFlushTLB(pmap, pmap_common.ptable);
 
 	/* And at last map page or only changle attributes of map entry */
 	pmap_common.ptable[pti] = PTE(pa, 0xc1 | (attr & 0x3f));
@@ -192,10 +190,10 @@ static int _pmap_enter(pmap_t *pmap, addr_t pa, void *va, int attr, page_t *allo
 	}
 
 	if (tlbInval != 0) {
-		hal_tlbInvalidateEntry(pmap, va, 1);
+		hal_cpuRemoteFlushTLB(pmap, va, SIZE_PAGE);
 	}
 	else {
-		hal_tlbInvalidateLocalEntry(pmap, va);
+		hal_cpuLocalFlushTLB(pmap, va);
 	}
 
 	return EOK;
@@ -210,12 +208,8 @@ int pmap_enter(pmap_t *pmap, addr_t pa, void *va, int attr, page_t *alloc)
 
 	hal_spinlockSet(&pmap_common.lock, &sc);
 	ret = _pmap_enter(pmap, pa, va, attr, alloc, 1);
-	if (ret == EOK) {
-		hal_tlbCommit(&pmap_common.lock, &sc);
-	}
-	else {
-		hal_spinlockClear(&pmap_common.lock, &sc);
-	}
+	hal_spinlockClear(&pmap_common.lock, &sc);
+
 	return ret;
 }
 
@@ -238,14 +232,14 @@ static void _pmap_remove(pmap_t *pmap, void *vaddr)
 	addr = PTE_TO_ADDR(entry);
 
 	pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(addr, 0xc7);
-	hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable);
+	hal_cpuLocalFlushTLB(pmap, pmap_common.ptable);
 
 	entry = pmap_common.ptable[pdi1];
 
 	if ((entry & PTE_V) != 0) {
 		addr = PTE_TO_ADDR(entry);
 		pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(addr, 0xc7);
-		hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable);
+		hal_cpuLocalFlushTLB(pmap, pmap_common.ptable);
 
 		if ((pmap_common.ptable[pti] & PGHD_EXEC) != 0) {
 			isync = 1;
@@ -269,8 +263,8 @@ int pmap_remove(pmap_t *pmap, void *vaddr)
 
 	hal_spinlockSet(&pmap_common.lock, &sc);
 	_pmap_remove(pmap, vaddr);
-	hal_tlbInvalidateEntry(pmap, vaddr, 1);
-	hal_tlbCommit(&pmap_common.lock, &sc);
+	hal_cpuRemoteFlushTLB(pmap, vaddr, SIZE_PAGE);
+	hal_spinlockClear(&pmap_common.lock, &sc);
 
 	return EOK;
 }
@@ -296,12 +290,12 @@ addr_t pmap_resolve(pmap_t *pmap, void *vaddr)
 	addr = PTE_TO_ADDR(pmap->pdir2[pdi2]);
 
 	pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(addr, 0xc7);
-	hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable);
+	hal_cpuLocalFlushTLB(pmap, pmap_common.ptable);
 
 	addr = PTE_TO_ADDR(pmap_common.ptable[pdi1]);
 
 	pmap_common.pdir0[PDIR0_IDX(pmap_common.ptable)] = PTE(addr, 0xc7);
-	hal_tlbInvalidateLocalEntry(NULL, pmap_common.ptable);
+	hal_cpuLocalFlushTLB(pmap, pmap_common.ptable);
 
 	addr = PTE_TO_ADDR(pmap_common.ptable[pti]);
 	hal_spinlockClear(&pmap_common.lock, &sc);
@@ -401,7 +395,7 @@ int _pmap_kernelSpaceExpand(pmap_t *pmap, void **start, void *end, page_t *dp)
 		}
 		*start = vaddr;
 	}
-	hal_tlbFlushLocal(NULL);
+	hal_cpuLocalFlushTLB(NULL, NULL);
 
 	pmap->start = (void *)VADDR_KERNEL;
 	pmap->end = end;
@@ -490,7 +484,7 @@ void _pmap_init(pmap_t *pmap, void **vstart, void **vend)
 
 	pmap->satp = ((pmap_resolve(pmap, (char *)pmap_common.pdir2) >> 12) | SATP_MODE_SV39);
 
-	hal_tlbFlushLocal(NULL);
+	hal_cpuLocalFlushTLB(NULL, NULL);
 }
 
 

hal/riscv64/riscv64.h

@@ -65,19 +65,6 @@
 	})
 
 
-static inline void hal_cpuFlushTLB(const void *vaddr)
-{
-	/* clang-format off */
-	__asm__ volatile (
-		"sfence.vma %0, zero"
-		:
-		: "r"(vaddr)
-		: "memory"
-	);
-	/* clang-format on */
-}
-
-
 static inline void hal_cpuSwitchSpace(addr_t pdir)
 {
 	/* clang-format off */

hal/riscv64/sbi.c

@@ -45,6 +45,12 @@
 #define SBI_HSM_STATUS  0x2
 #define SBI_HSM_SUSPEND 0x3
 
+/* RFENCE extension */
+#define SBI_EXT_RFENCE           0x52464E43
+#define SBI_RFNC_I               0x0
+#define SBI_RFNC_SFENCE_VMA      0x1
+#define SBI_RFNC_SFENCE_VMA_ASID 0x2
+
 /* Legacy extensions */
 #define SBI_LEGACY_SETTIMER               0x0
 #define SBI_LEGACY_PUTCHAR                0x1
@@ -56,10 +62,6 @@
 #define SBI_LEGACY_REMOTE_SFENCE_VMA_ASID 0x7
 #define SBI_LEGACY_SHUTDOWN               0x8
 
-/* RFENCE extension */
-#define SBI_EXT_RFENCE 0x52464E43
-#define SBI_RFNC_I     0x0
-
 /* clang-format off */
 #define SBI_MINOR(x) ((x) & 0xffffff)
 /* clang-format on */
@@ -176,6 +178,18 @@ void hal_sbiRfenceI(unsigned long hart_mask, unsigned long hart_mask_base)
 }
 
 
+sbiret_t hal_sbiSfenceVma(unsigned long hart_mask, unsigned long hart_mask_base, unsigned long vaddr, unsigned long size)
+{
+	return hal_sbiEcall(SBI_EXT_RFENCE, SBI_RFNC_SFENCE_VMA, hart_mask, hart_mask_base, vaddr, size, 0, 0);
+}
+
+
+sbiret_t hal_sbiSfenceVmaAsid(unsigned long hart_mask, unsigned long hart_mask_base, unsigned long vaddr, unsigned long size, unsigned long asid)
+{
+	return hal_sbiEcall(SBI_EXT_RFENCE, SBI_RFNC_SFENCE_VMA_ASID, hart_mask, hart_mask_base, vaddr, size, asid, 0);
+}
+
+
 void _hal_sbiInit(void)
 {
 	sbiret_t ret = hal_sbiGetSpecVersion();

hal/riscv64/sbi.h

@@ -84,6 +84,12 @@ sbiret_t hal_sbiHartStart(unsigned long hartid, unsigned long start_addr, unsign
 void hal_sbiRfenceI(unsigned long hart_mask, unsigned long hart_mask_base);
 
 
+sbiret_t hal_sbiSfenceVma(unsigned long hart_mask, unsigned long hart_mask_base, unsigned long vaddr, unsigned long size);
+
+
+sbiret_t hal_sbiSfenceVmaAsid(unsigned long hart_mask, unsigned long hart_mask_base, unsigned long vaddr, unsigned long size, unsigned long asid);
+
+
 void _hal_sbiInit(void);