diff --git a/include/arch/riscv/arch/machine.h b/include/arch/riscv/arch/machine.h index 98dbf37a9e6..1dda8ff0fc5 100644 --- a/include/arch/riscv/arch/machine.h +++ b/include/arch/riscv/arch/machine.h @@ -64,6 +64,17 @@ compile_assert(kernel_elf_no_overflow, KERNEL_ELF_BASE_RAW < KERNEL_ELF_BASE_RAW #define SIE_MEIE 11 /* M-Mode external interrupt enable (MIP only). */ /* Bit 12 and above are reserved. */ +static inline void ifence_local(void) +{ + asm volatile("fence.i":::"memory"); +} + +static inline void sfence_vma_local(void) +{ + /* This flushes the TLB */ + asm volatile("sfence.vma" ::: "memory"); +} + #ifdef ENABLE_SMP_SUPPORT static inline void fence_rw_rw(void) @@ -78,17 +89,7 @@ static inline void fence_w_rw(void) static inline void fence_r_rw(void) { - asm volatile("fence r,rw" ::: "memory"); -} - -static inline void ifence_local(void) -{ - asm volatile("fence.i":::"memory"); -} - -static inline void sfence_local(void) -{ - asm volatile("sfence.vma" ::: "memory"); + asm volatile("fence r, rw" ::: "memory"); } static inline word_t get_sbi_mask_for_all_remote_harts(void) @@ -102,47 +103,42 @@ static inline word_t get_sbi_mask_for_all_remote_harts(void) return mask; } +#endif /* ENABLE_SMP_SUPPORT */ + static inline void ifence(void) { ifence_local(); +#ifdef ENABLE_SMP_SUPPORT word_t mask = get_sbi_mask_for_all_remote_harts(); sbi_remote_fence_i(mask); +#endif } -static inline void sfence(void) +static inline void sfence_vma_global(void) { +#ifdef ENABLE_SMP_SUPPORT fence_w_rw(); - sfence_local(); +#endif + + sfence_vma_local(); + +#ifdef ENABLE_SMP_SUPPORT word_t mask = get_sbi_mask_for_all_remote_harts(); sbi_remote_sfence_vma(mask, 0, 0); +#endif } -static inline void hwASIDFlushLocal(asid_t asid) +static inline void hwASIDFlush(asid_t asid) { + /* flush ASID on the current hart */ asm volatile("sfence.vma x0, %0" :: "r"(asid): "memory"); -} -static inline void hwASIDFlush(asid_t asid) -{ - hwASIDFlushLocal(asid); +#ifdef ENABLE_SMP_SUPPORT word_t mask = get_sbi_mask_for_all_remote_harts(); sbi_remote_sfence_vma_asid(mask, 0, 0, asid); +#endif } -#else - -static inline void sfence(void) -{ - asm volatile("sfence.vma" ::: "memory"); -} - -static inline void hwASIDFlush(asid_t asid) -{ - asm volatile("sfence.vma x0, %0" :: "r"(asid): "memory"); -} - -#endif /* end of !ENABLE_SMP_SUPPORT */ - word_t PURE getRestartPC(tcb_t *thread); void setNextPC(tcb_t *thread, word_t v); @@ -252,12 +248,8 @@ static inline void setVSpaceRoot(paddr_t addr, asid_t asid) write_satp(satp.words[0]); - /* Order read/write operations */ -#ifdef ENABLE_SMP_SUPPORT - sfence_local(); -#else - sfence(); -#endif + /* Order read/write operations. */ + sfence_vma_local(); } void map_kernel_devices(void); diff --git a/src/arch/riscv/kernel/vspace.c b/src/arch/riscv/kernel/vspace.c index 0f3059ffe88..47e53a5a8cd 100644 --- a/src/arch/riscv/kernel/vspace.c +++ b/src/arch/riscv/kernel/vspace.c @@ -189,7 +189,7 @@ BOOT_CODE void map_it_pt_cap(cap_t vspace_cap, cap_t pt_cap) 0, /* read */ 1 /* valid */ ); - sfence(); + sfence_vma_global(); } BOOT_CODE void map_it_frame_cap(cap_t vspace_cap, cap_t frame_cap) @@ -216,7 +216,7 @@ BOOT_CODE void map_it_frame_cap(cap_t vspace_cap, cap_t frame_cap) 1, /* read */ 1 /* valid */ ); - sfence(); + sfence_vma_global(); } BOOT_CODE cap_t create_unmapped_it_frame_cap(pptr_t pptr, bool_t use_large) @@ -540,7 +540,7 @@ void unmapPageTable(asid_t asid, vptr_t vptr, pte_t *target_pt) 0, /* read */ 0 /* valid */ ); - sfence(); + sfence_vma_global(); } static pte_t pte_pte_invalid_new(void) @@ -570,7 +570,7 @@ void unmapPage(vm_page_size_t page_size, asid_t asid, vptr_t vptr, pptr_t pptr) } lu_ret.ptSlot[0] = pte_pte_invalid_new(); - sfence(); + sfence_vma_global(); } void setVMRoot(tcb_t *tcb) @@ -1078,7 +1078,7 @@ exception_t performPageTableInvocationMap(cap_t cap, cte_t *ctSlot, { ctSlot->cap = cap; *ptSlot = pte; - sfence(); + sfence_vma_global(); return EXCEPTION_NONE; } @@ -1121,7 +1121,7 @@ static exception_t performPageGetAddress(void *vbase_ptr, bool_t call) static exception_t updatePTE(pte_t pte, pte_t *base) { *base = pte; - sfence(); + sfence_vma_global(); return EXCEPTION_NONE; } @@ -1228,7 +1228,7 @@ exception_t benchmark_arch_map_logBuffer(word_t frame_cptr) kernel_image_level2_dev_pt[RISCV_GET_PT_INDEX(KS_LOG_PPTR, 1)] = pte_next(ksUserLogBuffer, true); #endif - sfence(); + sfence_vma_global(); return EXCEPTION_NONE; }