diff --git a/bsp/qemu-virt64-riscv/Kconfig b/bsp/qemu-virt64-riscv/Kconfig index 053cacc57cf..544753502b2 100644 --- a/bsp/qemu-virt64-riscv/Kconfig +++ b/bsp/qemu-virt64-riscv/Kconfig @@ -13,22 +13,22 @@ rsource "driver/Kconfig" config BOARD_QEMU_VIRT_RV64 bool select ARCH_RISCV64 - select ARCH_CONTEXT_EXTEND select ARCH_USING_RISCV_COMMON64 select RT_USING_COMPONENTS_INIT select RT_USING_USER_MAIN select RT_USING_CACHE select ARCH_MM_MMU select ARCH_REMAP_KERNEL - select RT_USING_HW_ATOMIC default y config ENABLE_FPU bool "Enable FPU" + select ARCH_RISCV_FPU default y config ENABLE_VECTOR bool "Using RISC-V Vector Extension" + select ARCH_RISCV_VECTOR default n if ENABLE_VECTOR @@ -52,10 +52,6 @@ config RT_USING_VIRTIO_MMIO_ALIGN bool "Open packed attribution, this may caused an error on virtio" default n -config ARCH_USING_NEW_CTX_SWITCH - bool - default y - config __STACKSIZE__ int "stack size for interrupt" default 4096 diff --git a/libcpu/Kconfig b/libcpu/Kconfig index ca5fdd9e34b..37924d74c6d 100644 --- a/libcpu/Kconfig +++ b/libcpu/Kconfig @@ -245,6 +245,9 @@ config ARCH_RISCV config ARCH_RISCV_FPU bool +config ARCH_RISCV_VECTOR + bool + config ARCH_RISCV_FPU_S select ARCH_RISCV_FPU bool @@ -263,10 +266,15 @@ config ARCH_RISCV64 bool if ARCH_RISCV64 + config ARCH_USING_NEW_CTX_SWITCH + bool + default y + config ARCH_USING_RISCV_COMMON64 bool depends on ARCH_RISCV64 select RT_USING_CPUTIME + select ARCH_USING_NEW_CTX_SWITCH help Using the common64 implementation under ./libcpu/risc-v endif diff --git a/libcpu/risc-v/t-head/c906/SConscript b/libcpu/risc-v/t-head/c906/SConscript index 194b3268c50..5f750ad65f4 100644 --- a/libcpu/risc-v/t-head/c906/SConscript +++ b/libcpu/risc-v/t-head/c906/SConscript @@ -1,11 +1,13 @@ # RT-Thread building script for component from building import * - cwd = GetCurrentDir() src = Glob('*.c') + Glob('*.cpp') + Glob('*_gcc.S') CPPPATH = [cwd] +if not GetDepend('ARCH_USING_ASID'): + SrcRemove(src, ['asid.c']) + group = DefineGroup('libcpu', src, depend = [''], CPPPATH = CPPPATH) Return('group') diff --git a/libcpu/risc-v/t-head/c906/asid.c b/libcpu/risc-v/t-head/c906/asid.c new file mode 100644 index 00000000000..5d7bffa3dcb --- /dev/null +++ b/libcpu/risc-v/t-head/c906/asid.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2006-2024, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2024-07-12 RT-Thread first version. + */ +#define DBG_TAG "hw.asid" +#define DBG_LVL DBG_INFO +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +static rt_uint8_t ASID_BITS = 0; +static rt_uint32_t next_asid; +static rt_uint64_t global_asid_generation; +#define ASID_MASK ((1 << ASID_BITS) - 1) +#define ASID_FIRST_GENERATION (1 << ASID_BITS) +#define MAX_ASID ASID_FIRST_GENERATION + +void rt_hw_asid_init(void) +{ + unsigned int satp_reg = read_csr(satp); + satp_reg |= (((rt_uint64_t)0xffff) << PPN_BITS); + write_csr(satp, satp_reg); + unsigned short valid_asid_bit = ((read_csr(satp) >> PPN_BITS) & 0xffff); + + // The maximal value of ASIDLEN, is 9 for Sv32 or 16 for Sv39, Sv48, and Sv57 + for (unsigned i = 0; i < 16; i++) + { + if (!(valid_asid_bit & 0x1)) + { + break; + } + + valid_asid_bit >>= 1; + ASID_BITS++; + } + + global_asid_generation = ASID_FIRST_GENERATION; + next_asid = 1; +} + +static rt_uint64_t _asid_acquire(rt_aspace_t aspace) +{ + if ((aspace->asid ^ global_asid_generation) >> ASID_BITS) // not same generation + { + if (next_asid != MAX_ASID) + { + aspace->asid = global_asid_generation | next_asid; + next_asid++; + } + else + { + // scroll to next generation + global_asid_generation += ASID_FIRST_GENERATION; + next_asid = 1; + rt_hw_tlb_invalidate_all_local(); + + aspace->asid = global_asid_generation | next_asid; + next_asid++; + } + } + + return aspace->asid & ASID_MASK; +} + +void rt_hw_asid_switch_pgtbl(struct rt_aspace *aspace, rt_ubase_t pgtbl) +{ + rt_uint64_t asid = _asid_acquire(aspace); + write_csr(satp, (((size_t)SATP_MODE) << SATP_MODE_OFFSET) | + (asid << PPN_BITS) | + ((rt_ubase_t)pgtbl >> PAGE_OFFSET_BIT)); + asm volatile("sfence.vma x0,%0"::"r"(asid):"memory"); +} + diff --git a/libcpu/risc-v/t-head/c906/backtrace.c b/libcpu/risc-v/t-head/c906/backtrace.c index 0bcfa0b9d91..8cfe7126cbc 100644 --- a/libcpu/risc-v/t-head/c906/backtrace.c +++ b/libcpu/risc-v/t-head/c906/backtrace.c @@ -1,137 +1,127 @@ /* - * Copyright (c) 2006-2018, RT-Thread Development Team + * Copyright (c) 2006-2023, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * * Change Logs: * Date Author Notes + * 2023-10-18 Shell Add backtrace support */ -#include -#ifdef RT_USING_SMART -#include - -#define TRANCE_LEVEL 20 +#define DBG_TAG "hw.backtrace" +#define DBG_LVL DBG_INFO +#include -extern rt_ubase_t __text_start[]; -extern rt_ubase_t __text_end[]; +#include +#include +#include "riscv_mmu.h" +#include "stack.h" -static char *_get_elf_name(size_t sepc); +#define WORD sizeof(rt_base_t) +#define ARCH_CONTEXT_FETCH(pctx, id) (*(((unsigned long *)pctx) + (id))) -void rt_hw_backtrace(rt_uint32_t *ffp, rt_ubase_t sepc) +rt_inline rt_err_t _bt_kaddr(rt_ubase_t *fp, struct rt_hw_backtrace_frame *frame) { - rt_ubase_t *ra; - rt_ubase_t *fp; - rt_ubase_t vas, vae; - int i, j; + rt_err_t rc; + frame->fp = *(fp - 2); + frame->pc = *(fp - 1); - rt_kprintf("riscv64-unknown-linux-musl-addr2line -e %s -a -f", _get_elf_name(sepc)); + if ((rt_ubase_t)fp == frame->fp) + { + rc = -RT_ERROR; + } + else + { + rc = RT_EOK; + } + return rc; +} - fp = (rt_ubase_t *)ffp; +#ifdef RT_USING_SMART +#include +#include - if (!fp) +rt_inline rt_err_t _bt_uaddr(rt_lwp_t lwp, rt_ubase_t *fp, struct rt_hw_backtrace_frame *frame) +{ + rt_err_t rc; + if (lwp_data_get(lwp, &frame->fp, fp - 2, WORD) != WORD) { - asm volatile("mv %0, s0" - : "=r"(fp)); + rc = -RT_EFAULT; } - - if (sepc) + else if (lwp_data_get(lwp, &frame->pc, fp - 1, WORD) != WORD) { - rt_kprintf(" %p", sepc - 0x4); + rc = -RT_EFAULT; } - - if (fp > (rt_ubase_t *)USER_VADDR_START && fp < (rt_ubase_t *)USER_VADDR_TOP) + else if ((rt_ubase_t)fp == frame->fp) { - vas = USER_VADDR_START; - vae = USER_VADDR_TOP; + rc = -RT_ERROR; } else { - vas = (rt_ubase_t)&__text_start; - vae = (rt_ubase_t)&__text_end; + frame->pc -= 0; + rc = RT_EOK; } + return rc; +} +#endif /* RT_USING_SMART */ - for (i = j = 0; i < TRANCE_LEVEL; i++) +rt_err_t rt_hw_backtrace_frame_unwind(rt_thread_t thread, struct rt_hw_backtrace_frame *frame) +{ + rt_err_t rc = -RT_ERROR; + rt_uintptr_t *fp = (rt_uintptr_t *)frame->fp; + + if (fp && !((long)fp & 0x7)) { - if (RT_ALIGN((rt_ubase_t)fp, sizeof(void *)) != (rt_ubase_t)fp) +#ifdef RT_USING_SMART + if (thread->lwp) { - break; + void *lwp = thread->lwp; + void *this_lwp = lwp_self(); + if (this_lwp == lwp && rt_hw_mmu_v2p(((rt_lwp_t)lwp)->aspace, fp) != ARCH_MAP_FAILED) + { + rc = _bt_kaddr(fp, frame); + } + else if (lwp_user_accessible_ext(lwp, (void *)fp, WORD)) + { + rc = _bt_uaddr(lwp, fp, frame); + } + else + { + rc = -RT_EFAULT; + } + } + else +#endif + if ((rt_kmem_v2p(fp) != ARCH_MAP_FAILED)) + { + rc = _bt_kaddr(fp, frame); + } + else + { + rc = -RT_EINVAL; } - - ra = fp - 1; - if (!rt_kmem_v2p(ra) || *ra < vas || *ra > vae) - break; - - rt_kprintf(" %p", *ra - 0x04); - - fp = fp - 2; - if (!rt_kmem_v2p(fp)) - break; - fp = (rt_ubase_t *)(*fp); - if (!fp) - break; } - - rt_kputs("\r\n"); -} - -static void _assert_backtrace_cb(const char *ex, const char *func, rt_size_t line) -{ - rt_hw_interrupt_disable(); - rt_kprintf("(%s) assertion failed at function:%s, line number:%d \n", ex, func, line); - - rt_hw_backtrace(0, 0); - rt_hw_cpu_shutdown(); -} - -static int rt_hw_backtrace_init(void) -{ - rt_assert_set_hook(_assert_backtrace_cb); - - return 0; + else + { + rc = -RT_EFAULT; + } + return rc; } -INIT_BOARD_EXPORT(rt_hw_backtrace_init); -static void backtrace_test(int args, char *argv[]) +rt_err_t rt_hw_backtrace_frame_get(rt_thread_t thread, struct rt_hw_backtrace_frame *frame) { - int *p = (void *)-1; - init_fn_t ft = 0; + rt_err_t rc; - if (args < 2) + if (!thread || !frame) { - rt_kprintf("backtrace_test usage:backtrace_test a(assert)/m(invalid memory)/i(illegal instruction)\r\n"); - return; - } - - if (!rt_strcmp(argv[1], "a")) - { - rt_kprintf("Assert test:\r\n", argv[1]); - RT_ASSERT(0); - } - else if (!rt_strcmp(argv[1], "m")) - { - rt_kprintf("Access invalid memory:\r\n", argv[1]); - *p = 0; - } - else if (!rt_strcmp(argv[1], "i")) - { - rt_kprintf("Illegal instruction:\r\n", argv[1]); - ft(); + rc = -RT_EINVAL; } else { - rt_kprintf("Unknown cmd :%s.\r\n", argv[1]); + rt_hw_switch_frame_t sframe = thread->sp; + frame->pc = sframe->regs[RT_HW_SWITCH_CONTEXT_RA]; + frame->fp = sframe->regs[RT_HW_SWITCH_CONTEXT_S0];; + rc = RT_EOK; } + return rc; } -MSH_CMD_EXPORT(backtrace_test, backtrace test case); - -extern struct rt_thread *rt_current_thread; - -#define IN_USERSPACE (sepc > USER_VADDR_START && sepc < USER_VADDR_TOP) - -static char *_get_elf_name(size_t sepc) -{ - return IN_USERSPACE ? rt_current_thread->parent.name : "rtthread.elf"; -} - -#endif /* RT_USING_SMART */ diff --git a/libcpu/risc-v/t-head/c906/context_gcc.S b/libcpu/risc-v/t-head/c906/context_gcc.S index 7368e01ea17..8858f6c1040 100644 --- a/libcpu/risc-v/t-head/c906/context_gcc.S +++ b/libcpu/risc-v/t-head/c906/context_gcc.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2021, RT-Thread Development Team + * Copyright (c) 2006-2024, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * @@ -8,12 +8,72 @@ * 2018/10/28 Bernard The unify RISC-V porting implementation * 2018/12/27 Jesven Add SMP support * 2021/02/02 lizhirui Add userspace support + * 2022/10/22 Shell Support User mode RVV; + * Trimming process switch context */ #include "cpuport.h" #include "stackframe.h" +#define _REG_IDX(name) RT_HW_SWITCH_CONTEXT_##name +#define REG_IDX(name) _REG_IDX(name) - .globl rt_hw_context_switch_to +.macro SAVE_REG reg, index + STORE \reg, \index*REGBYTES(sp) +.endm + +.macro LOAD_REG reg, index + LOAD \reg, \index*REGBYTES(sp) +.endm + +.macro RESERVE_CONTEXT + addi sp, sp, -(RT_HW_SWITCH_CONTEXT_SIZE * REGBYTES) + SAVE_REG tp, REG_IDX(TP) + SAVE_REG ra, REG_IDX(RA) + SAVE_REG s0, REG_IDX(S0) + SAVE_REG s1, REG_IDX(S1) + SAVE_REG s2, REG_IDX(S2) + SAVE_REG s3, REG_IDX(S3) + SAVE_REG s4, REG_IDX(S4) + SAVE_REG s5, REG_IDX(S5) + SAVE_REG s6, REG_IDX(S6) + SAVE_REG s7, REG_IDX(S7) + SAVE_REG s8, REG_IDX(S8) + SAVE_REG s9, REG_IDX(S9) + SAVE_REG s10, REG_IDX(S10) + SAVE_REG s11, REG_IDX(S11) + csrr s11, sstatus + li s10, (SSTATUS_SPP) + or s11, s11, s10 + SAVE_REG s11, REG_IDX(SSTATUS) +.endm + +.macro RESTORE_CONTEXT + LOAD_REG s11, REG_IDX(SSTATUS) + csrw sstatus, s11 + LOAD_REG s11, REG_IDX(S11) + LOAD_REG s10, REG_IDX(S10) + LOAD_REG s9, REG_IDX(S9) + LOAD_REG s8, REG_IDX(S8) + LOAD_REG s7, REG_IDX(S7) + LOAD_REG s6, REG_IDX(S6) + LOAD_REG s5, REG_IDX(S5) + LOAD_REG s4, REG_IDX(S4) + LOAD_REG s3, REG_IDX(S3) + LOAD_REG s2, REG_IDX(S2) + LOAD_REG s1, REG_IDX(S1) + LOAD_REG s0, REG_IDX(S0) + LOAD_REG ra, REG_IDX(RA) + LOAD_REG tp, REG_IDX(TP) + addi sp, sp, RT_HW_SWITCH_CONTEXT_SIZE * REGBYTES + csrw sepc, ra +.endm + +/* + * void rt_hw_context_switch_to(rt_ubase_t to); + * + * a0 --> to SP pointer + */ +.globl rt_hw_context_switch_to rt_hw_context_switch_to: LOAD sp, (a0) @@ -24,31 +84,26 @@ rt_hw_context_switch_to: jal lwp_aspace_switch #endif - RESTORE_ALL + RESTORE_CONTEXT sret /* * void rt_hw_context_switch(rt_ubase_t from, rt_ubase_t to); * - * a0 --> from - * a1 --> to + * a0 --> from SP pointer + * a1 --> to SP pointer + * + * It should only be used on local interrupt disable */ - .globl rt_hw_context_switch +.globl rt_hw_context_switch rt_hw_context_switch: - mv t2, sp - li t0, 0x120//set SPIE and SPP = 1 - csrs sstatus, t0//if enter here,caller must be in system thread - csrw sepc, ra//return address - //saved from thread context - SAVE_ALL - - STORE t2, 32 * REGBYTES(sp)//save user_sp - + RESERVE_CONTEXT STORE sp, (a0) - //restore to thread context + // restore to thread SP LOAD sp, (a1) + // restore Address Space jal rt_thread_self mv s1, a0 @@ -56,5 +111,5 @@ rt_hw_context_switch: jal lwp_aspace_switch #endif - RESTORE_ALL + RESTORE_CONTEXT sret diff --git a/libcpu/risc-v/t-head/c906/cpuport.c b/libcpu/risc-v/t-head/c906/cpuport.c index f1be78e539a..be154198df4 100644 --- a/libcpu/risc-v/t-head/c906/cpuport.c +++ b/libcpu/risc-v/t-head/c906/cpuport.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2021, RT-Thread Development Team + * Copyright (c) 2006-2024, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * @@ -14,9 +14,15 @@ #include #include "cpuport.h" -#include "sbi.h" #include "stack.h" +#include +#include +#ifdef ARCH_RISCV_FPU + #define K_SSTATUS_DEFAULT (SSTATUS_SPP | SSTATUS_SPIE | SSTATUS_SUM | SSTATUS_FS) +#else + #define K_SSTATUS_DEFAULT (SSTATUS_SPP | SSTATUS_SPIE | SSTATUS_SUM) +#endif #ifdef RT_USING_SMART #include #endif @@ -25,22 +31,42 @@ * @brief from thread used interrupt context switch * */ -volatile rt_ubase_t rt_interrupt_from_thread = 0; +volatile rt_ubase_t rt_interrupt_from_thread = 0; /** * @brief to thread used interrupt context switch * */ -volatile rt_ubase_t rt_interrupt_to_thread = 0; +volatile rt_ubase_t rt_interrupt_to_thread = 0; /** * @brief flag to indicate context switch in interrupt or not * */ volatile rt_ubase_t rt_thread_switch_interrupt_flag = 0; +void *_rt_hw_stack_init(rt_ubase_t *sp, rt_ubase_t ra, rt_ubase_t sstatus) +{ + rt_hw_switch_frame_t frame = (rt_hw_switch_frame_t) + ((rt_ubase_t)sp - sizeof(struct rt_hw_switch_frame)); + + rt_memset(frame, 0, sizeof(struct rt_hw_switch_frame)); + + frame->regs[RT_HW_SWITCH_CONTEXT_RA] = ra; + frame->regs[RT_HW_SWITCH_CONTEXT_SSTATUS] = sstatus; + + return (void *)frame; +} + +int rt_hw_cpu_id(void) +{ + return 0; +} /** - * This function will initialize thread stack + * This function will initialize thread stack, we assuming + * when scheduler restore this new thread, context will restore + * an entry to user first application * + * s0-s11, ra, sstatus, a0 * @param tentry the entry of thread * @param parameter the parameter of entry * @param stack_addr the beginning stack address @@ -48,41 +74,23 @@ volatile rt_ubase_t rt_thread_switch_interrupt_flag = 0; * * @return stack address */ -rt_uint8_t *rt_hw_stack_init(void *tentry, - void *parameter, +rt_uint8_t *rt_hw_stack_init(void *tentry, + void *parameter, rt_uint8_t *stack_addr, - void *texit) + void *texit) { - struct rt_hw_stack_frame *frame; - rt_uint8_t *stk; - int i; - extern int __global_pointer$; - - stk = stack_addr + sizeof(rt_ubase_t); - stk = (rt_uint8_t *)RT_ALIGN_DOWN((rt_ubase_t)stk, REGBYTES); - stk -= sizeof(struct rt_hw_stack_frame); - - frame = (struct rt_hw_stack_frame *)stk; - - for (i = 0; i < sizeof(struct rt_hw_stack_frame) / sizeof(rt_ubase_t); i++) - { - ((rt_ubase_t *)frame)[i] = 0xdeadbeef; - } - - frame->ra = (rt_ubase_t)texit; - frame->gp = (rt_ubase_t)&__global_pointer$; - frame->a0 = (rt_ubase_t)parameter; - frame->epc = (rt_ubase_t)tentry; - frame->user_sp_exc_stack = (rt_ubase_t)(((rt_ubase_t)stk) + sizeof(struct rt_hw_stack_frame)); - - /* force to supervisor mode(SPP=1) and set SPIE and SUM to 1 */ -#ifdef ARCH_RISCV_FPU - frame->sstatus = 0x00046120; /* enable FPU */ -#else - frame->sstatus = 0x00040120; -#endif - - return stk; + rt_ubase_t *sp = (rt_ubase_t *)stack_addr; + // we use a strict alignment requirement for Q extension + sp = (rt_ubase_t *)RT_ALIGN_DOWN((rt_ubase_t)sp, 16); + + (*--sp) = (rt_ubase_t)tentry; + (*--sp) = (rt_ubase_t)parameter; + (*--sp) = (rt_ubase_t)texit; + --sp; /* alignment */ + + /* compatible to RESTORE_CONTEXT */ + extern void _rt_thread_entry(void); + return (rt_uint8_t *)_rt_hw_stack_init(sp, (rt_ubase_t)_rt_thread_entry, K_SSTATUS_DEFAULT); } /* @@ -101,7 +109,7 @@ void rt_hw_context_switch_interrupt(rt_ubase_t from, rt_ubase_t to, rt_thread_t rt_interrupt_to_thread = to; rt_thread_switch_interrupt_flag = 1; - return ; + return; } #endif /* end of RT_USING_SMP */ @@ -112,16 +120,14 @@ void rt_hw_cpu_shutdown(void) rt_kprintf("shutdown...\n"); level = rt_hw_interrupt_disable(); + sbi_shutdown(); - while (1); -} -int rt_hw_cpu_id(void) -{ - return 0; /* d1 has one core */ + while (1) + ; } void rt_hw_set_process_id(int pid) { - //TODO + // TODO } diff --git a/libcpu/risc-v/t-head/c906/cpuport.h b/libcpu/risc-v/t-head/c906/cpuport.h index 153bb350765..0723a3c393a 100644 --- a/libcpu/risc-v/t-head/c906/cpuport.h +++ b/libcpu/risc-v/t-head/c906/cpuport.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2021, RT-Thread Development Team + * Copyright (c) 2006-2024, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * @@ -14,28 +14,16 @@ #include #include -/* bytes of register width */ -#ifdef ARCH_CPU_64BIT -#define STORE sd -#define LOAD ld -#define REGBYTES 8 -#else -// error here, not portable +#ifdef RT_USING_SMP +typedef union { + unsigned long slock; + struct __arch_tickets { + unsigned short owner; + unsigned short next; + } tickets; +} rt_hw_spinlock_t; #endif -/* 33 general register */ -#define CTX_GENERAL_REG_NR 33 - -#ifdef ARCH_RISCV_FPU -/* 32 fpu register */ -#define CTX_FPU_REG_NR 32 -#else -#define CTX_FPU_REG_NR 0 -#endif - -/* all context registers */ -#define CTX_REG_NR (CTX_GENERAL_REG_NR + CTX_FPU_REG_NR) - #ifndef __ASSEMBLY__ #include diff --git a/libcpu/risc-v/t-head/c906/cpuport_gcc.S b/libcpu/risc-v/t-head/c906/cpuport_gcc.S new file mode 100644 index 00000000000..5361646753f --- /dev/null +++ b/libcpu/risc-v/t-head/c906/cpuport_gcc.S @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2006-2022, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-10-19 RT-Thread the first version + */ + +#include "cpuport.h" +#include "stackframe.h" +#include "asm-generic.h" + +START_POINT(_rt_thread_entry) + LOAD ra, REGBYTES(sp) /* thread exit */ + addi sp, sp, 2 * REGBYTES + LOAD a0, (sp) /* parameter */ + LOAD t0, REGBYTES(sp) /* tentry */ + addi sp, sp, 2 * REGBYTES + mv s1, ra + jalr t0 + jalr s1 + j . /* never here */ +START_POINT_END(_rt_thread_entry) diff --git a/libcpu/risc-v/t-head/c906/encoding.h b/libcpu/risc-v/t-head/c906/encoding.h index 25f1b771db1..7e906fedd83 100644 --- a/libcpu/risc-v/t-head/c906/encoding.h +++ b/libcpu/risc-v/t-head/c906/encoding.h @@ -1,8 +1,25 @@ +/* + * Copyright (c) 2006-2022, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + */ // See LICENSE for license details. #ifndef RISCV_CSR_ENCODING_H #define RISCV_CSR_ENCODING_H +#include + +#ifdef ARCH_RISCV_VECTOR +#include "vector_encoding.h" + +#else +#define SSTATUS_VS 0 /* fallback */ +#endif /* ARCH_RISCV_VECTOR */ + #define MSTATUS_UIE 0x00000001 #define MSTATUS_SIE 0x00000002 #define MSTATUS_HIE 0x00000004 diff --git a/libcpu/risc-v/t-head/c906/ext_context.h b/libcpu/risc-v/t-head/c906/ext_context.h new file mode 100644 index 00000000000..f7eb9035765 --- /dev/null +++ b/libcpu/risc-v/t-head/c906/ext_context.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2006-2024, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-10-10 RT-Thread the first version + */ +#ifndef __EXT_CONTEXT_H__ +#define __EXT_CONTEXT_H__ + +#include + +#ifdef ARCH_RISCV_FPU +/* 32 fpu register */ +#define CTX_FPU_REG_NR 32 +#else +#define CTX_FPU_REG_NR 0 +#endif /* ARCH_RISCV_FPU */ + +#ifdef __ASSEMBLY__ + +/** + * ================================== + * RISC-V D ISA (Floating) + * ================================== + */ + +#ifdef ARCH_RISCV_FPU +#define FPU_CTX_F0_OFF (REGBYTES * 0) /* offsetof(fpu_context_t, fpustatus.f[0]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F1_OFF (REGBYTES * 1) /* offsetof(fpu_context_t, fpustatus.f[1]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F2_OFF (REGBYTES * 2) /* offsetof(fpu_context_t, fpustatus.f[2]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F3_OFF (REGBYTES * 3) /* offsetof(fpu_context_t, fpustatus.f[3]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F4_OFF (REGBYTES * 4) /* offsetof(fpu_context_t, fpustatus.f[4]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F5_OFF (REGBYTES * 5) /* offsetof(fpu_context_t, fpustatus.f[5]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F6_OFF (REGBYTES * 6) /* offsetof(fpu_context_t, fpustatus.f[6]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F7_OFF (REGBYTES * 7) /* offsetof(fpu_context_t, fpustatus.f[7]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F8_OFF (REGBYTES * 8) /* offsetof(fpu_context_t, fpustatus.f[8]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F9_OFF (REGBYTES * 9) /* offsetof(fpu_context_t, fpustatus.f[9]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F10_OFF (REGBYTES * 10) /* offsetof(fpu_context_t, fpustatus.f[10]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F11_OFF (REGBYTES * 11) /* offsetof(fpu_context_t, fpustatus.f[11]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F12_OFF (REGBYTES * 12) /* offsetof(fpu_context_t, fpustatus.f[12]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F13_OFF (REGBYTES * 13) /* offsetof(fpu_context_t, fpustatus.f[13]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F14_OFF (REGBYTES * 14) /* offsetof(fpu_context_t, fpustatus.f[14]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F15_OFF (REGBYTES * 15) /* offsetof(fpu_context_t, fpustatus.f[15]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F16_OFF (REGBYTES * 16) /* offsetof(fpu_context_t, fpustatus.f[16]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F17_OFF (REGBYTES * 17) /* offsetof(fpu_context_t, fpustatus.f[17]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F18_OFF (REGBYTES * 18) /* offsetof(fpu_context_t, fpustatus.f[18]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F19_OFF (REGBYTES * 19) /* offsetof(fpu_context_t, fpustatus.f[19]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F20_OFF (REGBYTES * 20) /* offsetof(fpu_context_t, fpustatus.f[20]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F21_OFF (REGBYTES * 21) /* offsetof(fpu_context_t, fpustatus.f[21]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F22_OFF (REGBYTES * 22) /* offsetof(fpu_context_t, fpustatus.f[22]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F23_OFF (REGBYTES * 23) /* offsetof(fpu_context_t, fpustatus.f[23]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F24_OFF (REGBYTES * 24) /* offsetof(fpu_context_t, fpustatus.f[24]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F25_OFF (REGBYTES * 25) /* offsetof(fpu_context_t, fpustatus.f[25]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F26_OFF (REGBYTES * 26) /* offsetof(fpu_context_t, fpustatus.f[26]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F27_OFF (REGBYTES * 27) /* offsetof(fpu_context_t, fpustatus.f[27]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F28_OFF (REGBYTES * 28) /* offsetof(fpu_context_t, fpustatus.f[28]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F29_OFF (REGBYTES * 29) /* offsetof(fpu_context_t, fpustatus.f[29]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F30_OFF (REGBYTES * 30) /* offsetof(fpu_context_t, fpustatus.f[30]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#define FPU_CTX_F31_OFF (REGBYTES * 31) /* offsetof(fpu_context_t, fpustatus.f[31]) - offsetof(fpu_context_t, fpustatus.f[0]) */ +#endif /* ARCH_RISCV_FPU */ + +#endif /* __ASSEMBLY__ */ + +#ifdef ARCH_RISCV_VECTOR +#include "rvv_context.h" +#else /* !ARCH_RISCV_VECTOR */ +#define CTX_VECTOR_REG_NR 0 +#endif /* ARCH_RISCV_VECTOR */ + +#endif /* __EXT_CONTEXT_H__ */ diff --git a/libcpu/risc-v/t-head/c906/interrupt.c b/libcpu/risc-v/t-head/c906/interrupt.c index 1cabe67587f..9e4cf2f1579 100644 --- a/libcpu/risc-v/t-head/c906/interrupt.c +++ b/libcpu/risc-v/t-head/c906/interrupt.c @@ -11,7 +11,7 @@ #include #include -#include "rt_interrupt.h" +#include "interrupt.h" #include "riscv.h" #include "plic.h" diff --git a/libcpu/risc-v/t-head/c906/rt_interrupt.h b/libcpu/risc-v/t-head/c906/interrupt.h similarity index 100% rename from libcpu/risc-v/t-head/c906/rt_interrupt.h rename to libcpu/risc-v/t-head/c906/interrupt.h diff --git a/libcpu/risc-v/t-head/c906/interrupt_gcc.S b/libcpu/risc-v/t-head/c906/interrupt_gcc.S index 5cf86350a7a..16778523d99 100644 --- a/libcpu/risc-v/t-head/c906/interrupt_gcc.S +++ b/libcpu/risc-v/t-head/c906/interrupt_gcc.S @@ -9,103 +9,81 @@ * 2018/12/27 Jesven Add SMP schedule * 2021/02/02 lizhirui Add userspace support * 2021/12/24 JasonHu Add user setting save/restore + * 2022/10/22 Shell Support kernel mode RVV; + * Rewrite trap handling routine */ #include "cpuport.h" #include "encoding.h" #include "stackframe.h" - .section .text.entry - .align 2 - .global trap_entry - .extern __stack_cpu0 - .extern get_current_thread_kernel_stack_top + .align 2 + .global trap_entry + .global debug_check_sp trap_entry: - //backup sp - csrrw sp, sscratch, sp - //load interrupt stack - la sp, __stack_cpu0 - //backup context + // distingush exception from kernel or user + csrrw sp, sscratch, sp + bnez sp, _save_context + + // BE REALLY careful with sscratch, + // if it's wrong, we could looping here forever + // or accessing random memory and seeing things totally + // messy after a long time and don't even know why +_from_kernel: + csrr sp, sscratch + j _save_context + +_save_context: SAVE_ALL - - RESTORE_SYS_GP - -#ifdef RT_USING_SMART - //check syscall - csrr t0, scause - li t1, 8//environment call from u-mode - beq t0, t1, syscall_entry -#endif - - csrr a0, scause - csrrc a1, stval, zero - csrr a2, sepc - mv a3, sp - - /* scause, stval, sepc, sp */ - call handle_trap - - /* need to switch new thread */ - la s0, rt_thread_switch_interrupt_flag - lw s2, 0(s0) - beqz s2, spurious_interrupt - sw zero, 0(s0) + // clear sscratch to say 'now in kernel mode' + csrw sscratch, zero -.global rt_hw_context_switch_interrupt_do -rt_hw_context_switch_interrupt_do: + RESTORE_SYS_GP + // now we are ready to enter interrupt / excepiton handler +_distinguish_syscall: + csrr t0, scause #ifdef RT_USING_SMART -//swap to thread kernel stack - csrr t0, sstatus - andi t0, t0, 0x100 - beqz t0, __restore_sp_from_tcb_interrupt + // TODO swap 8 with config macro name + li t1, 8 + beq t0, t1, syscall_entry + // syscall never return here #endif -__restore_sp_from_sscratch_interrupt: - csrr t0, sscratch - j __move_stack_context_interrupt - +_handle_interrupt_and_exception: + mv a0, t0 + csrrc a1, stval, zero + csrr a2, sepc + // sp as exception frame pointer + mv a3, sp + call handle_trap + +_interrupt_exit: + la s0, rt_thread_switch_interrupt_flag + lw s2, 0(s0) + beqz s2, _resume_execution + sw zero, 0(s0) + +_context_switch: + la t0, rt_interrupt_from_thread + LOAD a0, 0(t0) + la t0, rt_interrupt_to_thread + LOAD a1, 0(t0) + csrr t0, sstatus + andi t0, t0, ~SSTATUS_SPIE + csrw sstatus, t0 + jal rt_hw_context_switch + +_resume_execution: #ifdef RT_USING_SMART -__restore_sp_from_tcb_interrupt: - la s0, rt_interrupt_from_thread - LOAD a0, 0(s0) - jal rt_thread_sp_to_thread - jal get_thread_kernel_stack_top - mv t0, a0 + LOAD t0, FRAME_OFF_SSTATUS(sp) + andi t0, t0, SSTATUS_SPP + beqz t0, arch_ret_to_user #endif -__move_stack_context_interrupt: - mv t1, sp//src - mv sp, t0//switch stack - addi sp, sp, -CTX_REG_NR * REGBYTES - //copy context - li s0, CTX_REG_NR//cnt - mv t2, sp//dst - -copy_context_loop_interrupt: - LOAD t0, 0(t1) - STORE t0, 0(t2) - addi s0, s0, -1 - addi t1, t1, 8 - addi t2, t2, 8 - bnez s0, copy_context_loop_interrupt - - la s0, rt_interrupt_from_thread - LOAD s1, 0(s0) - STORE sp, 0(s1) - - la s0, rt_interrupt_to_thread - LOAD s1, 0(s0) - LOAD sp, 0(s1) - - #ifdef RT_USING_SMART - mv a0, s1 - jal rt_thread_sp_to_thread - jal lwp_aspace_switch - #endif - -spurious_interrupt: +_resume_kernel: RESTORE_ALL + csrw sscratch, zero sret .global rt_hw_interrupt_enable diff --git a/libcpu/risc-v/t-head/c906/mmu.c b/libcpu/risc-v/t-head/c906/mmu.c index 6c57c397700..649c22a0546 100644 --- a/libcpu/risc-v/t-head/c906/mmu.c +++ b/libcpu/risc-v/t-head/c906/mmu.c @@ -15,7 +15,7 @@ #include #define DBG_TAG "hw.mmu" -#define DBG_LVL DBG_WARNING +#define DBG_LVL DBG_INFO #include #include @@ -44,78 +44,15 @@ volatile __attribute__((aligned(4 * 1024))) rt_ubase_t MMUTable[__SIZE(VPN2_BIT)]; #ifdef ARCH_USING_ASID -static rt_uint8_t ASID_BITS = 0; -static rt_uint32_t next_asid; -static rt_uint64_t global_asid_generation; -#define ASID_MASK ((1 << ASID_BITS) - 1) -#define ASID_FIRST_GENERATION (1 << ASID_BITS) -#define MAX_ASID ASID_FIRST_GENERATION - -static void _asid_init() -{ - unsigned int satp_reg = read_csr(satp); - satp_reg |= (((rt_uint64_t)0xffff) << PPN_BITS); - write_csr(satp, satp_reg); - unsigned short valid_asid_bit = ((read_csr(satp) >> PPN_BITS) & 0xffff); - - // The maximal value of ASIDLEN, is 9 for Sv32 or 16 for Sv39, Sv48, and Sv57 - for (unsigned i = 0; i < 16; i++) - { - if (!(valid_asid_bit & 0x1)) - { - break; - } - - valid_asid_bit >>= 1; - ASID_BITS++; - } - - global_asid_generation = ASID_FIRST_GENERATION; - next_asid = 1; -} - -static rt_uint64_t _asid_check_switch(rt_aspace_t aspace) -{ - if ((aspace->asid ^ global_asid_generation) >> ASID_BITS) // not same generation - { - if (next_asid != MAX_ASID) - { - aspace->asid = global_asid_generation | next_asid; - next_asid++; - } - else - { - // scroll to next generation - global_asid_generation += ASID_FIRST_GENERATION; - next_asid = 1; - rt_hw_tlb_invalidate_all_local(); - - aspace->asid = global_asid_generation | next_asid; - next_asid++; - } - } - - return aspace->asid & ASID_MASK; -} - void rt_hw_aspace_switch(rt_aspace_t aspace) { uintptr_t page_table = (uintptr_t)rt_kmem_v2p(aspace->page_table); current_mmu_table = aspace->page_table; - rt_uint64_t asid = _asid_check_switch(aspace); - write_csr(satp, (((size_t)SATP_MODE) << SATP_MODE_OFFSET) | - (asid << PPN_BITS) | - ((rt_ubase_t)page_table >> PAGE_OFFSET_BIT)); - asm volatile("sfence.vma x0,%0"::"r"(asid):"memory"); + rt_hw_asid_switch_pgtbl(aspace, page_table); } -#define ASID_INIT() _asid_init() - -#else /* ARCH_USING_ASID */ - -#define ASID_INIT() - +#else /* !ARCH_USING_ASID */ void rt_hw_aspace_switch(rt_aspace_t aspace) { uintptr_t page_table = (uintptr_t)rt_kmem_v2p(aspace->page_table); @@ -126,6 +63,9 @@ void rt_hw_aspace_switch(rt_aspace_t aspace) rt_hw_tlb_invalidate_all_local(); } +void rt_hw_asid_init(void) +{ +} #endif /* ARCH_USING_ASID */ void *rt_hw_mmu_tbl_get() @@ -136,28 +76,28 @@ void *rt_hw_mmu_tbl_get() static int _map_one_page(struct rt_aspace *aspace, void *va, void *pa, size_t attr) { - rt_size_t l1_off, l2_off, l3_off; - rt_size_t *mmu_l1, *mmu_l2, *mmu_l3; + rt_ubase_t l1_off, l2_off, l3_off; + rt_ubase_t *mmu_l1, *mmu_l2, *mmu_l3; l1_off = GET_L1((size_t)va); l2_off = GET_L2((size_t)va); l3_off = GET_L3((size_t)va); - mmu_l1 = ((rt_size_t *)aspace->page_table) + l1_off; + mmu_l1 = ((rt_ubase_t *)aspace->page_table) + l1_off; if (PTE_USED(*mmu_l1)) { - mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET); + mmu_l2 = (rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET); } else { - mmu_l2 = (rt_size_t *)rt_pages_alloc(0); + mmu_l2 = (rt_ubase_t *)rt_pages_alloc(0); if (mmu_l2) { rt_memset(mmu_l2, 0, PAGE_SIZE); rt_hw_cpu_dcache_clean(mmu_l2, PAGE_SIZE); - *mmu_l1 = COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l2, PV_OFFSET), + *mmu_l1 = COMBINEPTE((rt_ubase_t)VPN_TO_PPN(mmu_l2, PV_OFFSET), PAGE_DEFAULT_ATTR_NEXT); rt_hw_cpu_dcache_clean(mmu_l1, sizeof(*mmu_l1)); } @@ -171,18 +111,18 @@ static int _map_one_page(struct rt_aspace *aspace, void *va, void *pa, { RT_ASSERT(!PAGE_IS_LEAF(*(mmu_l2 + l2_off))); mmu_l3 = - (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), PV_OFFSET); + (rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), PV_OFFSET); } else { - mmu_l3 = (rt_size_t *)rt_pages_alloc(0); + mmu_l3 = (rt_ubase_t *)rt_pages_alloc(0); if (mmu_l3) { rt_memset(mmu_l3, 0, PAGE_SIZE); rt_hw_cpu_dcache_clean(mmu_l3, PAGE_SIZE); *(mmu_l2 + l2_off) = - COMBINEPTE((rt_size_t)VPN_TO_PPN(mmu_l3, PV_OFFSET), + COMBINEPTE((rt_ubase_t)VPN_TO_PPN(mmu_l3, PV_OFFSET), PAGE_DEFAULT_ATTR_NEXT); rt_hw_cpu_dcache_clean(mmu_l2, sizeof(*mmu_l2)); // declares a reference to parent page table @@ -197,7 +137,7 @@ static int _map_one_page(struct rt_aspace *aspace, void *va, void *pa, RT_ASSERT(!PTE_USED(*(mmu_l3 + l3_off))); // declares a reference to parent page table rt_page_ref_inc((void *)mmu_l3, 0); - *(mmu_l3 + l3_off) = COMBINEPTE((rt_size_t)pa, attr); + *(mmu_l3 + l3_off) = COMBINEPTE((rt_ubase_t)pa, attr); rt_hw_cpu_dcache_clean(mmu_l3 + l3_off, sizeof(*(mmu_l3 + l3_off))); return 0; } @@ -240,7 +180,7 @@ void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr, return NULL; } -static void _unmap_pte(rt_size_t *pentry, rt_size_t *lvl_entry[], int level) +static void _unmap_pte(rt_ubase_t *pentry, rt_ubase_t *lvl_entry[], int level) { int loop_flag = 1; while (loop_flag) @@ -270,26 +210,26 @@ static void _unmap_pte(rt_size_t *pentry, rt_size_t *lvl_entry[], int level) static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr, size_t size) { - rt_size_t loop_va = __UMASKVALUE((rt_size_t)v_addr, PAGE_OFFSET_MASK); + rt_ubase_t loop_va = __UMASKVALUE((rt_ubase_t)v_addr, PAGE_OFFSET_MASK); size_t unmapped = 0; int i = 0; - rt_size_t lvl_off[3]; - rt_size_t *lvl_entry[3]; - lvl_off[0] = (rt_size_t)GET_L1(loop_va); - lvl_off[1] = (rt_size_t)GET_L2(loop_va); - lvl_off[2] = (rt_size_t)GET_L3(loop_va); + rt_ubase_t lvl_off[3]; + rt_ubase_t *lvl_entry[3]; + lvl_off[0] = (rt_ubase_t)GET_L1(loop_va); + lvl_off[1] = (rt_ubase_t)GET_L2(loop_va); + lvl_off[2] = (rt_ubase_t)GET_L3(loop_va); unmapped = 1 << (ARCH_PAGE_SHIFT + ARCH_INDEX_WIDTH * 2ul); - rt_size_t *pentry; - lvl_entry[i] = ((rt_size_t *)aspace->page_table + lvl_off[i]); + rt_ubase_t *pentry; + lvl_entry[i] = ((rt_ubase_t *)aspace->page_table + lvl_off[i]); pentry = lvl_entry[i]; // find leaf page table entry while (PTE_USED(*pentry) && !PAGE_IS_LEAF(*pentry)) { i += 1; - lvl_entry[i] = ((rt_size_t *)PPN_TO_VPN(GET_PADDR(*pentry), PV_OFFSET) + + lvl_entry[i] = ((rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*pentry), PV_OFFSET) + lvl_off[i]); pentry = lvl_entry[i]; unmapped >>= ARCH_INDEX_WIDTH; @@ -321,8 +261,7 @@ void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size) MM_PGTBL_UNLOCK(aspace); // when unmapped == 0, region not exist in pgtbl - if (!unmapped || unmapped > size) - break; + if (!unmapped || unmapped > size) break; size -= unmapped; v_addr += unmapped; @@ -335,7 +274,8 @@ static inline void _init_region(void *vaddr, size_t size) rt_ioremap_start = vaddr; rt_ioremap_size = size; rt_mpr_start = rt_ioremap_start - rt_mpr_size; - LOG_D("rt_ioremap_start: %p, rt_mpr_start: %p", rt_ioremap_start, rt_mpr_start); + LOG_D("rt_ioremap_start: %p, rt_mpr_start: %p", rt_ioremap_start, + rt_mpr_start); } #else static inline void _init_region(void *vaddr, size_t size) @@ -345,15 +285,15 @@ static inline void _init_region(void *vaddr, size_t size) #endif #if defined(RT_USING_SMART) && defined(ARCH_REMAP_KERNEL) -#define KERN_SPACE_START ((void *)KERNEL_VADDR_START) -#define KERN_SPACE_SIZE (0xfffffffffffff000UL - KERNEL_VADDR_START + 0x1000) +#define KERN_SPACE_START ((void *)KERNEL_VADDR_START) +#define KERN_SPACE_SIZE (0xfffffffffffff000UL - KERNEL_VADDR_START + 0x1000) #else -#define KERN_SPACE_START ((void *)0x1000) -#define KERN_SPACE_SIZE ((size_t)USER_VADDR_START - 0x1000) +#define KERN_SPACE_START ((void *)0x1000) +#define KERN_SPACE_SIZE ((size_t)USER_VADDR_START - 0x1000) #endif -int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_size_t size, - rt_size_t *vtable, rt_size_t pv_off) +int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_ubase_t size, + rt_ubase_t *vtable, rt_ubase_t pv_off) { size_t l1_off, va_s, va_e; rt_base_t level; @@ -363,8 +303,8 @@ int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_size_t size, return -1; } - va_s = (rt_size_t)v_address; - va_e = ((rt_size_t)v_address) + size - 1; + va_s = (rt_ubase_t)v_address; + va_e = ((rt_ubase_t)v_address) + size - 1; if (va_e < va_s) { @@ -405,15 +345,15 @@ static inline uintptr_t _get_level_size(int level) return 1ul << (ARCH_PAGE_SHIFT + (max_level - level) * ARCH_INDEX_WIDTH); } -static rt_size_t *_query(struct rt_aspace *aspace, void *vaddr, int *level) +static rt_ubase_t *_query(struct rt_aspace *aspace, void *vaddr, int *level) { - rt_size_t l1_off, l2_off, l3_off; - rt_size_t *mmu_l1, *mmu_l2, *mmu_l3; - rt_size_t pa; + rt_ubase_t l1_off, l2_off, l3_off; + rt_ubase_t *mmu_l1, *mmu_l2, *mmu_l3; + rt_ubase_t pa; - l1_off = GET_L1((rt_size_t)vaddr); - l2_off = GET_L2((rt_size_t)vaddr); - l3_off = GET_L3((rt_size_t)vaddr); + l1_off = GET_L1((rt_uintptr_t)vaddr); + l2_off = GET_L2((rt_uintptr_t)vaddr); + l3_off = GET_L3((rt_uintptr_t)vaddr); if (!aspace) { @@ -421,7 +361,7 @@ static rt_size_t *_query(struct rt_aspace *aspace, void *vaddr, int *level) return RT_NULL; } - mmu_l1 = ((rt_size_t *)aspace->page_table) + l1_off; + mmu_l1 = ((rt_ubase_t *)aspace->page_table) + l1_off; if (PTE_USED(*mmu_l1)) { @@ -431,7 +371,7 @@ static rt_size_t *_query(struct rt_aspace *aspace, void *vaddr, int *level) return mmu_l1; } - mmu_l2 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET); + mmu_l2 = (rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET); if (PTE_USED(*(mmu_l2 + l2_off))) { @@ -441,7 +381,7 @@ static rt_size_t *_query(struct rt_aspace *aspace, void *vaddr, int *level) return mmu_l2 + l2_off; } - mmu_l3 = (rt_size_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), + mmu_l3 = (rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), PV_OFFSET); if (PTE_USED(*(mmu_l3 + l3_off))) @@ -458,7 +398,7 @@ static rt_size_t *_query(struct rt_aspace *aspace, void *vaddr, int *level) void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr) { int level; - uintptr_t *pte = _query(aspace, vaddr, &level); + rt_ubase_t *pte = _query(aspace, vaddr, &level); uintptr_t paddr; if (pte) @@ -468,23 +408,23 @@ void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr) } else { - LOG_I("%s: failed at %p", __func__, vaddr); + LOG_D("%s: failed at %p", __func__, vaddr); paddr = (uintptr_t)ARCH_MAP_FAILED; } return (void *)paddr; } -static int _noncache(uintptr_t *pte) +static int _noncache(rt_base_t *pte) { return 0; } -static int _cache(uintptr_t *pte) +static int _cache(rt_base_t *pte) { return 0; } -static int (*control_handler[MMU_CNTL_DUMMY_END])(uintptr_t *pte) = { +static int (*control_handler[MMU_CNTL_DUMMY_END])(rt_base_t *pte) = { [MMU_CNTL_CACHE] = _cache, [MMU_CNTL_NONCACHE] = _noncache, }; @@ -496,14 +436,14 @@ int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size, int err = -RT_EINVAL; void *vend = vaddr + size; - int (*handler)(uintptr_t * pte); + int (*handler)(rt_base_t *pte); if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END) { handler = control_handler[cmd]; while (vaddr < vend) { - uintptr_t *pte = _query(aspace, vaddr, &level); + rt_base_t *pte = _query(aspace, vaddr, &level); void *range_end = vaddr + _get_level_size(level); RT_ASSERT(range_end <= vend); @@ -543,27 +483,27 @@ void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr) size_t attr; switch (mdesc->attr) { - case NORMAL_MEM: - attr = MMU_MAP_K_RWCB; - break; - case NORMAL_NOCACHE_MEM: - attr = MMU_MAP_K_RWCB; - break; - case DEVICE_MEM: - attr = MMU_MAP_K_DEVICE; - break; - default: - attr = MMU_MAP_K_DEVICE; + case NORMAL_MEM: + attr = MMU_MAP_K_RWCB; + break; + case NORMAL_NOCACHE_MEM: + attr = MMU_MAP_K_RWCB; + break; + case DEVICE_MEM: + attr = MMU_MAP_K_DEVICE; + break; + default: + attr = MMU_MAP_K_DEVICE; } - struct rt_mm_va_hint hint = {.flags = MMF_MAP_FIXED, - .limit_start = aspace->start, - .limit_range_size = aspace->size, - .map_size = mdesc->vaddr_end - - mdesc->vaddr_start + 1, - .prefer = (void *)mdesc->vaddr_start}; + struct rt_mm_va_hint hint = { + .flags = MMF_MAP_FIXED, + .limit_start = aspace->start, + .limit_range_size = aspace->size, + .map_size = mdesc->vaddr_end - mdesc->vaddr_start + 1, + .prefer = (void *)mdesc->vaddr_start}; - if (mdesc->paddr_start == (rt_size_t)ARCH_MAP_FAILED) + if (mdesc->paddr_start == (rt_uintptr_t)ARCH_MAP_FAILED) mdesc->paddr_start = mdesc->vaddr_start + PV_OFFSET; rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr, @@ -571,38 +511,19 @@ void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr) mdesc++; } - ASID_INIT(); + rt_hw_asid_init(); rt_hw_aspace_switch(&rt_kernel_space); rt_page_cleanup(); } -void rt_hw_mmu_kernel_map_init(rt_aspace_t aspace, rt_size_t vaddr_start, rt_size_t size) -{ - rt_size_t paddr_start = - __UMASKVALUE(VPN_TO_PPN(vaddr_start, PV_OFFSET), PAGE_OFFSET_MASK); - rt_size_t va_s = GET_L1(vaddr_start); - rt_size_t va_e = GET_L1(vaddr_start + size - 1); - rt_size_t i; - - for (i = va_s; i <= va_e; i++) - { - MMUTable[i] = - COMBINEPTE(paddr_start, PAGE_ATTR_RWX | PTE_G | PTE_V | PTE_CACHE | - PTE_SHARE | PTE_BUF | PTE_A | PTE_D); - paddr_start += L1_PAGE_SIZE; - } - - rt_hw_tlb_invalidate_all_local(); -} - -#define SATP_BASE ((size_t)SATP_MODE << SATP_MODE_OFFSET) +#define SATP_BASE ((rt_ubase_t)SATP_MODE << SATP_MODE_OFFSET) void rt_hw_mem_setup_early(void) { - rt_size_t pv_off; - rt_size_t ps = 0x0; - rt_size_t vs = 0x0; - rt_size_t *early_pgtbl = (size_t *)(((size_t)&__bss_end + 4095) & ~0xfff); + rt_ubase_t pv_off; + rt_ubase_t ps = 0x0; + rt_ubase_t vs = 0x0; + rt_ubase_t *early_pgtbl = (rt_ubase_t *)(((size_t)&__bss_end + 4095) & ~0xfff); /* calculate pv_offset */ void *symb_pc; @@ -617,7 +538,8 @@ void rt_hw_mem_setup_early(void) { if (pv_off & (1ul << (ARCH_INDEX_WIDTH * 2 + ARCH_PAGE_SHIFT))) { - LOG_E("%s: not aligned virtual address. pv_offset %p", __func__, pv_off); + LOG_E("%s: not aligned virtual address. pv_offset %p", __func__, + pv_off); RT_ASSERT(0); } @@ -627,8 +549,7 @@ void rt_hw_mem_setup_early(void) */ for (size_t i = 0; i < __SIZE(PPN0_BIT); i++) { - early_pgtbl[i] = COMBINEPTE(ps, PAGE_ATTR_RWX | PTE_G | PTE_V | PTE_CACHE | - PTE_SHARE | PTE_BUF | PTE_A | PTE_D); + early_pgtbl[i] = COMBINEPTE(ps, MMU_MAP_EARLY); ps += L1_PAGE_SIZE; } @@ -638,12 +559,11 @@ void rt_hw_mem_setup_early(void) vs = ps - pv_off; /* relocate region */ - rt_size_t vs_idx = GET_L1(vs); - rt_size_t ve_idx = GET_L1(vs + 0x80000000); + rt_ubase_t vs_idx = GET_L1(vs); + rt_ubase_t ve_idx = GET_L1(vs + 0x80000000); for (size_t i = vs_idx; i < ve_idx; i++) { - early_pgtbl[i] = COMBINEPTE(ps, PAGE_ATTR_RWX | PTE_G | PTE_V | PTE_CACHE | - PTE_SHARE | PTE_BUF | PTE_A | PTE_D); + early_pgtbl[i] = COMBINEPTE(ps, MMU_MAP_EARLY); ps += L1_PAGE_SIZE; } @@ -657,7 +577,7 @@ void rt_hw_mem_setup_early(void) void *rt_hw_mmu_pgtbl_create(void) { - size_t *mmu_table; + rt_ubase_t *mmu_table; mmu_table = (rt_ubase_t *)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE); if (!mmu_table) { diff --git a/libcpu/risc-v/t-head/c906/mmu.h b/libcpu/risc-v/t-head/c906/mmu.h index dee72c86241..e5385d2bf20 100644 --- a/libcpu/risc-v/t-head/c906/mmu.h +++ b/libcpu/risc-v/t-head/c906/mmu.h @@ -30,7 +30,7 @@ struct mem_desc { rt_size_t vaddr_start; rt_size_t vaddr_end; - rt_size_t paddr_start; + rt_ubase_t paddr_start; rt_size_t attr; struct rt_varea varea; }; @@ -43,8 +43,8 @@ struct mem_desc #define GET_PPN(pte) \ (__PARTBIT(pte, PTE_PPN_SHIFT, PHYSICAL_ADDRESS_WIDTH_BITS - PTE_PPN_SHIFT)) #define GET_PADDR(pte) (GET_PPN(pte) << PAGE_OFFSET_BIT) -#define VPN_TO_PPN(vaddr, pv_off) (((rt_size_t)(vaddr)) + (pv_off)) -#define PPN_TO_VPN(paddr, pv_off) (((rt_size_t)(paddr)) - (pv_off)) +#define VPN_TO_PPN(vaddr, pv_off) (((rt_uintptr_t)(vaddr)) + (pv_off)) +#define PPN_TO_VPN(paddr, pv_off) (((rt_uintptr_t)(paddr)) - (pv_off)) #define COMBINEVADDR(l1_off, l2_off, l3_off) \ (((l1_off) << VPN2_SHIFT) | ((l2_off) << VPN1_SHIFT) | \ ((l3_off) << VPN0_SHIFT)) @@ -57,11 +57,11 @@ struct mem_desc #define MMU_MAP_ERROR_CONFLICT -4 void *rt_hw_mmu_tbl_get(void); -int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_size_t size, - rt_size_t *vtable, rt_size_t pv_off); +int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_ubase_t size, + rt_ubase_t *vtable, rt_ubase_t pv_off); void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr); -void rt_hw_mmu_kernel_map_init(rt_aspace_t aspace, rt_size_t vaddr_start, - rt_size_t size); +void rt_hw_mmu_kernel_map_init(rt_aspace_t aspace, rt_ubase_t vaddr_start, + rt_ubase_t size); void *rt_hw_mmu_map(rt_aspace_t aspace, void *v_addr, void *p_addr, size_t size, size_t attr); void rt_hw_mmu_unmap(rt_aspace_t aspace, void *v_addr, size_t size); diff --git a/libcpu/risc-v/t-head/c906/plic.c b/libcpu/risc-v/t-head/c906/plic.c index 07f1333fa89..829ee9e4fd4 100644 --- a/libcpu/risc-v/t-head/c906/plic.c +++ b/libcpu/risc-v/t-head/c906/plic.c @@ -15,7 +15,7 @@ #include #include "plic.h" -#include "rt_interrupt.h" +#include "interrupt.h" #include "io.h" #include "encoding.h" #include "ioremap.h" diff --git a/libcpu/risc-v/t-head/c906/plic.h b/libcpu/risc-v/t-head/c906/plic.h index fab0f0fe5ce..93235279e2c 100644 --- a/libcpu/risc-v/t-head/c906/plic.h +++ b/libcpu/risc-v/t-head/c906/plic.h @@ -12,7 +12,7 @@ #ifndef __RISCV64_PLIC_H__ #define __RISCV64_PLIC_H__ -#include +#include #ifndef C906_PLIC_PHY_ADDR #define C906_PLIC_PHY_ADDR (0x10000000) diff --git a/libcpu/risc-v/t-head/c906/riscv.h b/libcpu/risc-v/t-head/c906/riscv.h index d0c0cc4b38a..4adaaf10341 100644 --- a/libcpu/risc-v/t-head/c906/riscv.h +++ b/libcpu/risc-v/t-head/c906/riscv.h @@ -1,11 +1,12 @@ /* - * Copyright (c) 2006-2021, RT-Thread Development Team + * Copyright (c) 2006-2024, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * * Change Logs: * Date Author Notes * 2021-01-30 lizhirui first version + * 2024-08-28 RT-Thread Fit into rv64ilp32 ABI */ #ifndef __RISCV_H__ @@ -13,13 +14,15 @@ #include -#define __SIZE(bit) (1UL << (bit)) -#define __MASK(bit) (__SIZE(bit) - 1UL) +/* using unsigned long long for the case of rv64ilp32 */ +#define __SIZE(bit) (1ULL << (bit)) +#define __MASK(bit) (__SIZE(bit) - 1ULL) + #define __UMASK(bit) (~(__MASK(bit))) #define __MASKVALUE(value,maskvalue) ((value) & (maskvalue)) #define __UMASKVALUE(value,maskvalue) ((value) & (~(maskvalue))) -#define __CHECKUPBOUND(value,bit_count) (!(((rt_size_t)value) & (~__MASK(bit_count)))) -#define __CHECKALIGN(value,start_bit) (!(((rt_size_t)value) & (__MASK(start_bit)))) +#define __CHECKUPBOUND(value,bit_count) (!(((rt_ubase_t)value) & (~__MASK(bit_count)))) +#define __CHECKALIGN(value,start_bit) (!(((rt_ubase_t)value) & (__MASK(start_bit)))) #define __PARTBIT(value,start_bit,length) (((value) >> (start_bit)) & __MASK(length)) diff --git a/libcpu/risc-v/t-head/c906/riscv_io.h b/libcpu/risc-v/t-head/c906/riscv_io.h index cde0495b293..f5582fbbf80 100644 --- a/libcpu/risc-v/t-head/c906/riscv_io.h +++ b/libcpu/risc-v/t-head/c906/riscv_io.h @@ -10,100 +10,106 @@ #ifndef __RISCV_IO_H__ #define __RISCV_IO_H__ - static inline void __raw_writeb(rt_uint8_t val, volatile void *addr) - { - asm volatile("sb %0, 0(%1)" : : "r"(val), "r"(addr)); - } - - static inline void __raw_writew(rt_uint16_t val, volatile void *addr) - { - asm volatile("sh %0, 0(%1)" : : "r"(val), "r"(addr)); - } - - static inline void __raw_writel(rt_uint32_t val, volatile void *addr) - { - asm volatile("sw %0, 0(%1)" : : "r"(val), "r"(addr)); - } - - #if __riscv_xlen != 32 - static inline void __raw_writeq(rt_uint64_t val, volatile void *addr) - { - asm volatile("sd %0, 0(%1)" : : "r"(val), "r"(addr)); - } - #endif - - static inline rt_uint8_t __raw_readb(const volatile void *addr) - { - rt_uint8_t val; - - asm volatile("lb %0, 0(%1)" : "=r"(val) : "r"(addr)); - return val; - } - - static inline rt_uint16_t __raw_readw(const volatile void *addr) - { - rt_uint16_t val; - - asm volatile("lh %0, 0(%1)" : "=r"(val) : "r"(addr)); - return val; - } - - static inline rt_uint32_t __raw_readl(const volatile void *addr) - { - rt_uint32_t val; - - asm volatile("lw %0, 0(%1)" : "=r"(val) : "r"(addr)); - return val; - } - - #if __riscv_xlen != 32 - static inline rt_uint64_t __raw_readq(const volatile void *addr) - { - rt_uint64_t val; - - asm volatile("ld %0, 0(%1)" : "=r"(val) : "r"(addr)); - return val; - } - #endif - - /* FIXME: These are now the same as asm-generic */ - - /* clang-format off */ - - #define __io_rbr() do {} while (0) - #define __io_rar() do {} while (0) - #define __io_rbw() do {} while (0) - #define __io_raw() do {} while (0) - - #define readb_relaxed(c) ({ rt_uint8_t __v; __io_rbr(); __v = __raw_readb(c); __io_rar(); __v; }) - #define readw_relaxed(c) ({ rt_uint16_t __v; __io_rbr(); __v = __raw_readw(c); __io_rar(); __v; }) - #define readl_relaxed(c) ({ rt_uint32_t __v; __io_rbr(); __v = __raw_readl(c); __io_rar(); __v; }) - - #define writeb_relaxed(v,c) ({ __io_rbw(); __raw_writeb((v),(c)); __io_raw(); }) - #define writew_relaxed(v,c) ({ __io_rbw(); __raw_writew((v),(c)); __io_raw(); }) - #define writel_relaxed(v,c) ({ __io_rbw(); __raw_writel((v),(c)); __io_raw(); }) - - #if __riscv_xlen != 32 - #define readq_relaxed(c) ({ rt_uint64_t __v; __io_rbr(); __v = __raw_readq(c); __io_rar(); __v; }) - #define writeq_relaxed(v,c) ({ __io_rbw(); __raw_writeq((v),(c)); __io_raw(); }) - #endif - - #define __io_br() do {} while (0) - #define __io_ar() __asm__ __volatile__ ("fence i,r" : : : "memory"); - #define __io_bw() __asm__ __volatile__ ("fence w,o" : : : "memory"); - #define __io_aw() do {} while (0) - - #define readb(c) ({ rt_uint8_t __v; __io_br(); __v = __raw_readb(c); __io_ar(); __v; }) - #define readw(c) ({ rt_uint16_t __v; __io_br(); __v = __raw_readw(c); __io_ar(); __v; }) - #define readl(c) ({ rt_uint32_t __v; __io_br(); __v = __raw_readl(c); __io_ar(); __v; }) - - #define writeb(v,c) ({ __io_bw(); __raw_writeb((v),(c)); __io_aw(); }) - #define writew(v,c) ({ __io_bw(); __raw_writew((v),(c)); __io_aw(); }) - #define writel(v,c) ({ __io_bw(); __raw_writel((v),(c)); __io_aw(); }) - - #if __riscv_xlen != 32 - #define readq(c) ({ rt_uint64_t __v; __io_br(); __v = __raw_readq(c); __io_ar(); __v; }) - #define writeq(v,c) ({ __io_bw(); __raw_writeq((v),(c)); __io_aw(); }) - #endif +static inline uint32_t __raw_hartid(void) +{ + extern int boot_hartid; + return boot_hartid; +} + +static inline void __raw_writeb(rt_uint8_t val, volatile void *addr) +{ + asm volatile("sb %0, 0(%1)" : : "r"(val), "r"(addr)); +} + +static inline void __raw_writew(rt_uint16_t val, volatile void *addr) +{ + asm volatile("sh %0, 0(%1)" : : "r"(val), "r"(addr)); +} + +static inline void __raw_writel(rt_uint32_t val, volatile void *addr) +{ + asm volatile("sw %0, 0(%1)" : : "r"(val), "r"(addr)); +} + +#if __riscv_xlen != 32 +static inline void __raw_writeq(rt_uint64_t val, volatile void *addr) +{ + asm volatile("sd %0, 0(%1)" : : "r"(val), "r"(addr)); +} +#endif + +static inline rt_uint8_t __raw_readb(const volatile void *addr) +{ + rt_uint8_t val; + + asm volatile("lb %0, 0(%1)" : "=r"(val) : "r"(addr)); + return val; +} + +static inline rt_uint16_t __raw_readw(const volatile void *addr) +{ + rt_uint16_t val; + + asm volatile("lh %0, 0(%1)" : "=r"(val) : "r"(addr)); + return val; +} + +static inline rt_uint32_t __raw_readl(const volatile void *addr) +{ + rt_uint32_t val; + + asm volatile("lw %0, 0(%1)" : "=r"(val) : "r"(addr)); + return val; +} + +#if __riscv_xlen != 32 +static inline rt_uint64_t __raw_readq(const volatile void *addr) +{ + rt_uint64_t val; + + asm volatile("ld %0, 0(%1)" : "=r"(val) : "r"(addr)); + return val; +} +#endif + +/* FIXME: These are now the same as asm-generic */ + +/* clang-format off */ + +#define __io_rbr() do {} while (0) +#define __io_rar() do {} while (0) +#define __io_rbw() do {} while (0) +#define __io_raw() do {} while (0) + +#define readb_relaxed(c) ({ rt_uint8_t __v; __io_rbr(); __v = __raw_readb(c); __io_rar(); __v; }) +#define readw_relaxed(c) ({ rt_uint16_t __v; __io_rbr(); __v = __raw_readw(c); __io_rar(); __v; }) +#define readl_relaxed(c) ({ rt_uint32_t __v; __io_rbr(); __v = __raw_readl(c); __io_rar(); __v; }) + +#define writeb_relaxed(v,c) ({ __io_rbw(); __raw_writeb((v),(c)); __io_raw(); }) +#define writew_relaxed(v,c) ({ __io_rbw(); __raw_writew((v),(c)); __io_raw(); }) +#define writel_relaxed(v,c) ({ __io_rbw(); __raw_writel((v),(c)); __io_raw(); }) + +#if __riscv_xlen != 32 +#define readq_relaxed(c) ({ rt_uint64_t __v; __io_rbr(); __v = __raw_readq(c); __io_rar(); __v; }) +#define writeq_relaxed(v,c) ({ __io_rbw(); __raw_writeq((v),(c)); __io_raw(); }) +#endif + +#define __io_br() do {} while (0) +#define __io_ar() __asm__ __volatile__ ("fence i,r" : : : "memory"); +#define __io_bw() __asm__ __volatile__ ("fence w,o" : : : "memory"); +#define __io_aw() do {} while (0) + +#define readb(c) ({ rt_uint8_t __v; __io_br(); __v = __raw_readb(c); __io_ar(); __v; }) +#define readw(c) ({ rt_uint16_t __v; __io_br(); __v = __raw_readw(c); __io_ar(); __v; }) +#define readl(c) ({ rt_uint32_t __v; __io_br(); __v = __raw_readl(c); __io_ar(); __v; }) + +#define writeb(v,c) ({ __io_bw(); __raw_writeb((v),(c)); __io_aw(); }) +#define writew(v,c) ({ __io_bw(); __raw_writew((v),(c)); __io_aw(); }) +#define writel(v,c) ({ __io_bw(); __raw_writel((v),(c)); __io_aw(); }) + +#if __riscv_xlen != 32 +#define readq(c) ({ rt_uint64_t __v; __io_br(); __v = __raw_readq(c); __io_ar(); __v; }) +#define writeq(v,c) ({ __io_bw(); __raw_writeq((v),(c)); __io_aw(); }) +#endif #endif diff --git a/libcpu/risc-v/t-head/c906/riscv_mmu.c b/libcpu/risc-v/t-head/c906/riscv_mmu.c index 0139364774d..9dec1aed718 100644 --- a/libcpu/risc-v/t-head/c906/riscv_mmu.c +++ b/libcpu/risc-v/t-head/c906/riscv_mmu.c @@ -20,10 +20,10 @@ void mmu_enable_user_page_access(void) { - set_csr(sstatus,SSTATUS_SUM); + set_csr(sstatus, SSTATUS_SUM); } void mmu_disable_user_page_access(void) { - clear_csr(sstatus,SSTATUS_SUM); + clear_csr(sstatus, SSTATUS_SUM); } diff --git a/libcpu/risc-v/t-head/c906/riscv_mmu.h b/libcpu/risc-v/t-head/c906/riscv_mmu.h index 79a885fd2d5..5cb8b328d67 100644 --- a/libcpu/risc-v/t-head/c906/riscv_mmu.h +++ b/libcpu/risc-v/t-head/c906/riscv_mmu.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2021, RT-Thread Development Team + * Copyright (c) 2006-2024, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * @@ -20,29 +20,30 @@ #undef PAGE_SIZE /* C-SKY extend */ -#define PTE_SEC (1UL << 59) /* Security */ -#define PTE_SHARE (1UL << 60) /* Shareable */ -#define PTE_BUF (1UL << 61) /* Bufferable */ -#define PTE_CACHE (1UL << 62) /* Cacheable */ -#define PTE_SO (1UL << 63) /* Strong Order */ +#define PTE_SEC (1UL << 59) /* Security */ +#define PTE_SHARE (1UL << 60) /* Shareable */ +#define PTE_BUF (1UL << 61) /* Bufferable */ +#define PTE_CACHE (1UL << 62) /* Cacheable */ +#define PTE_SO (1UL << 63) /* Strong Order */ #define PAGE_OFFSET_SHIFT 0 -#define PAGE_OFFSET_BIT 12 -#define PAGE_SIZE __SIZE(PAGE_OFFSET_BIT) -#define PAGE_OFFSET_MASK __MASK(PAGE_OFFSET_BIT) -#define VPN0_SHIFT (PAGE_OFFSET_SHIFT + PAGE_OFFSET_BIT) -#define VPN0_BIT 9 -#define VPN1_SHIFT (VPN0_SHIFT + VPN0_BIT) -#define VPN1_BIT 9 -#define VPN2_SHIFT (VPN1_SHIFT + VPN1_BIT) -#define VPN2_BIT 9 +#define PAGE_OFFSET_BIT 12 +#define PAGE_SIZE __SIZE(PAGE_OFFSET_BIT) +#define PAGE_OFFSET_MASK __MASK(PAGE_OFFSET_BIT) +#define VPN0_SHIFT (PAGE_OFFSET_SHIFT + PAGE_OFFSET_BIT) +#define VPN0_BIT 9 +#define VPN1_SHIFT (VPN0_SHIFT + VPN0_BIT) +#define VPN1_BIT 9 +#define VPN2_SHIFT (VPN1_SHIFT + VPN1_BIT) +#define VPN2_BIT 9 #define PPN0_SHIFT (PAGE_OFFSET_SHIFT + PAGE_OFFSET_BIT) -#define PPN0_BIT 9 +#define PPN0_BIT 9 #define PPN1_SHIFT (PPN0_SHIFT + PPN0_BIT) -#define PPN1_BIT 9 +#define PPN1_BIT 9 #define PPN2_SHIFT (PPN1_SHIFT + PPN1_BIT) -#define PPN2_BIT 26 +#define PPN2_BIT 26 +#define PPN_BITS (PPN0_BIT + PPN1_BIT + PPN2_BIT) #define L1_PAGE_SIZE __SIZE(PAGE_OFFSET_BIT + VPN0_BIT + VPN1_BIT) #define L2_PAGE_SIZE __SIZE(PAGE_OFFSET_BIT + VPN0_BIT) @@ -52,71 +53,79 @@ #define PHYSICAL_ADDRESS_WIDTH_BITS 56 -#define PAGE_ATTR_NEXT_LEVEL (0) -#define PAGE_ATTR_RWX (PTE_X | PTE_W | PTE_R) -#define PAGE_ATTR_READONLY (PTE_R) -#define PAGE_ATTR_XN (PTE_W | PTE_R) -#define PAGE_ATTR_READEXECUTE (PTE_X | PTE_R) +#define PAGE_ATTR_NEXT_LEVEL (0) +#define PAGE_ATTR_RWX (PTE_X | PTE_W | PTE_R) +#define PAGE_ATTR_READONLY (PTE_R) +#define PAGE_ATTR_XN (PTE_W | PTE_R) +#define PAGE_ATTR_READEXECUTE (PTE_X | PTE_R) -#define PAGE_ATTR_USER (PTE_U) +#define PAGE_ATTR_USER (PTE_U) #define PAGE_ATTR_SYSTEM (0) -#define PAGE_ATTR_CB (PTE_BUF | PTE_CACHE) -#define PAGE_ATTR_DEV (PTE_SO) +#define PAGE_ATTR_CB (PTE_BUF | PTE_CACHE) +#define PAGE_ATTR_DEV (PTE_SO) -#define PAGE_DEFAULT_ATTR_LEAF (PTE_SHARE | PTE_BUF | PTE_CACHE | PTE_A | PTE_D | PTE_G | PTE_U | PAGE_ATTR_RWX | PTE_V) -#define PAGE_DEFAULT_ATTR_NEXT (PTE_SHARE | PTE_BUF | PTE_CACHE | PTE_A | PTE_D | PTE_G | PTE_V) +#define PAGE_DEFAULT_ATTR_LEAF \ + (PTE_SHARE | PTE_BUF | PTE_CACHE | PTE_A | PTE_D | PTE_G | PTE_U | \ + PAGE_ATTR_RWX | PTE_V) +#define PAGE_DEFAULT_ATTR_NEXT \ + (PTE_SHARE | PTE_BUF | PTE_CACHE | PTE_A | PTE_D | PTE_G | PTE_V) #define PAGE_IS_LEAF(pte) __MASKVALUE(pte, PAGE_ATTR_RWX) -#define PTE_USED(pte) __MASKVALUE(pte, PTE_V) +#define PTE_USED(pte) __MASKVALUE(pte, PTE_V) #define PTE_WRAP(attr) (attr | PTE_A | PTE_D) /** * encoding of SATP (Supervisor Address Translation and Protection register) */ -#define SATP_MODE_OFFSET 60 -#define SATP_MODE_BARE 0 -#define SATP_MODE_SV39 8 -#define SATP_MODE_SV48 9 -#define SATP_MODE_SV57 10 -#define SATP_MODE_SV64 11 - -#define PPN_BITS 44 - -#define ARCH_VADDR_WIDTH 39 -#define SATP_MODE SATP_MODE_SV39 - -#define MMU_MAP_K_DEVICE PTE_WRAP(PAGE_ATTR_DEV | PTE_G | PAGE_ATTR_XN | PTE_V) -#define MMU_MAP_K_RWCB PTE_WRAP(PAGE_ATTR_CB | PTE_G | PAGE_ATTR_RWX | PTE_V) -#define MMU_MAP_K_RW PTE_WRAP(PTE_G | PAGE_ATTR_RWX | PTE_V) -#define MMU_MAP_U_RWCB PTE_WRAP(PAGE_ATTR_CB | PTE_U | PAGE_ATTR_RWX | PTE_V) -#define MMU_MAP_U_ROCB PTE_WRAP(PAGE_ATTR_CB | PTE_U | PAGE_ATTR_READONLY | PTE_V) -#define MMU_MAP_U_RWCB_XN PTE_WRAP(PAGE_ATTR_CB | PTE_U | PAGE_ATTR_XN | PTE_V) -#define MMU_MAP_U_RW PTE_WRAP(PTE_U | PAGE_ATTR_RWX | PTE_V) -#define MMU_MAP_TRACE(attr) (attr) - -#define PTE_XWR_MASK 0xe - -#define ARCH_PAGE_SIZE PAGE_SIZE -#define ARCH_PAGE_MASK (ARCH_PAGE_SIZE - 1) -#define ARCH_PAGE_SHIFT PAGE_OFFSET_BIT -#define ARCH_INDEX_WIDTH 9 -#define ARCH_INDEX_SIZE (1ul << ARCH_INDEX_WIDTH) -#define ARCH_INDEX_MASK (ARCH_INDEX_SIZE - 1) - -#define ARCH_MAP_FAILED ((void *)0x8000000000000000) +#define SATP_MODE_OFFSET 60 +#define SATP_MODE_BARE 0 +#define SATP_MODE_SV39 8 +#define SATP_MODE_SV48 9 +#define SATP_MODE_SV57 10 +#define SATP_MODE_SV64 11 + +#define ARCH_VADDR_WIDTH 39 +#define SATP_MODE SATP_MODE_SV39 + +#define MMU_MAP_K_DEVICE PTE_WRAP(PAGE_ATTR_DEV | PTE_G | PAGE_ATTR_XN | PTE_V) +#define MMU_MAP_K_RWCB PTE_WRAP(PAGE_ATTR_CB | PTE_G | PAGE_ATTR_RWX | PTE_V) +#define MMU_MAP_K_RW PTE_WRAP(PTE_G | PAGE_ATTR_RWX | PTE_V) +#define MMU_MAP_U_RWCB PTE_WRAP(PAGE_ATTR_CB | PTE_U | PAGE_ATTR_RWX | PTE_V) +#define MMU_MAP_U_ROCB \ + PTE_WRAP(PAGE_ATTR_CB | PTE_U | PAGE_ATTR_READONLY | PTE_V) +#define MMU_MAP_U_RWCB_XN PTE_WRAP(PAGE_ATTR_CB | PTE_U | PAGE_ATTR_XN | PTE_V) +#define MMU_MAP_U_RW PTE_WRAP(PTE_U | PAGE_ATTR_RWX | PTE_V) +#define MMU_MAP_EARLY \ + PTE_WRAP(PAGE_ATTR_RWX | PTE_G | PTE_V | PTE_CACHE | PTE_SHARE | PTE_BUF) +#define MMU_MAP_TRACE(attr) (attr) + +#define PTE_XWR_MASK 0xe + +#define ARCH_PAGE_SIZE PAGE_SIZE +#define ARCH_PAGE_MASK (ARCH_PAGE_SIZE - 1) +#define ARCH_PAGE_SHIFT PAGE_OFFSET_BIT +#define ARCH_INDEX_WIDTH 9 +#define ARCH_INDEX_SIZE (1ul << ARCH_INDEX_WIDTH) +#define ARCH_INDEX_MASK (ARCH_INDEX_SIZE - 1) + +#define ARCH_MAP_FAILED ((void *)0x8000000000000000) void mmu_set_pagetable(rt_ubase_t addr); void mmu_enable_user_page_access(void); void mmu_disable_user_page_access(void); -#define RT_HW_MMU_PROT_READ 1 -#define RT_HW_MMU_PROT_WRITE 2 +#define RT_HW_MMU_PROT_READ 1 +#define RT_HW_MMU_PROT_WRITE 2 #define RT_HW_MMU_PROT_EXECUTE 4 -#define RT_HW_MMU_PROT_KERNEL 8 -#define RT_HW_MMU_PROT_USER 16 -#define RT_HW_MMU_PROT_CACHE 32 +#define RT_HW_MMU_PROT_KERNEL 8 +#define RT_HW_MMU_PROT_USER 16 +#define RT_HW_MMU_PROT_CACHE 32 + +void rt_hw_asid_init(void); +struct rt_aspace; +void rt_hw_asid_switch_pgtbl(struct rt_aspace *aspace, rt_ubase_t pgtbl); /** * @brief Remove permission from attribution diff --git a/libcpu/risc-v/t-head/c906/sbi.c b/libcpu/risc-v/t-head/c906/sbi.c index ff5669bb305..b2b1e8e7881 100644 --- a/libcpu/risc-v/t-head/c906/sbi.c +++ b/libcpu/risc-v/t-head/c906/sbi.c @@ -68,8 +68,8 @@ static struct sbi_ret sbi_get_impl_version(void) void sbi_print_version(void) { - unsigned int major; - unsigned int minor; + uint32_t major; + uint32_t minor; /* For legacy SBI implementations. */ if (sbi_spec_version == 0) diff --git a/libcpu/risc-v/t-head/c906/sbi.h b/libcpu/risc-v/t-head/c906/sbi.h index ed29ecfd206..45853871a89 100644 --- a/libcpu/risc-v/t-head/c906/sbi.h +++ b/libcpu/risc-v/t-head/c906/sbi.h @@ -48,6 +48,7 @@ #ifndef _MACHINE_SBI_H_ #define _MACHINE_SBI_H_ +#include #include /* SBI Specification Version */ @@ -140,7 +141,7 @@ struct sbi_ret long value; }; -static inline struct sbi_ret +rt_inline struct sbi_ret sbi_call(uint64_t arg7, uint64_t arg6, uint64_t arg0, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4) { diff --git a/libcpu/risc-v/t-head/c906/stack.h b/libcpu/risc-v/t-head/c906/stack.h index 3aac0587e20..4302b26e1f5 100644 --- a/libcpu/risc-v/t-head/c906/stack.h +++ b/libcpu/risc-v/t-head/c906/stack.h @@ -7,13 +7,22 @@ * Date Author Notes * 2021-01-30 lizhirui first version * 2021-11-18 JasonHu add fpu member + * 2022-10-22 Shell Support kernel mode RVV */ #ifndef __STACK_H__ #define __STACK_H__ +#include "stackframe.h" + #include +typedef struct rt_hw_switch_frame +{ + uint64_t regs[RT_HW_SWITCH_CONTEXT_SIZE]; +} *rt_hw_switch_frame_t; + + struct rt_hw_stack_frame { rt_ubase_t epc; /* epc - epc - program counter */ @@ -49,9 +58,13 @@ struct rt_hw_stack_frame rt_ubase_t t5; /* x30 - t5 - temporary register 5 */ rt_ubase_t t6; /* x31 - t6 - temporary register 6 */ rt_ubase_t user_sp_exc_stack; /* sscratch - user mode sp/exception stack */ + rt_ubase_t __padding; /* align to 16bytes */ #ifdef ARCH_RISCV_FPU rt_ubase_t f[CTX_FPU_REG_NR]; /* f0~f31 */ -#endif +#endif /* ARCH_RISCV_FPU */ +#ifdef ARCH_RISCV_VECTOR + rt_ubase_t v[CTX_VECTOR_REG_NR]; +#endif /* ARCH_RISCV_VECTOR */ }; #endif diff --git a/libcpu/risc-v/t-head/c906/stackframe.h b/libcpu/risc-v/t-head/c906/stackframe.h index 0f2ac147772..dd87b35aa1c 100644 --- a/libcpu/risc-v/t-head/c906/stackframe.h +++ b/libcpu/risc-v/t-head/c906/stackframe.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2021, RT-Thread Development Team + * Copyright (c) 2006-2024, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * @@ -8,61 +8,59 @@ * 2021-02-02 lizhirui first version * 2021-02-11 lizhirui fixed gp save/store bug * 2021-11-18 JasonHu add fpu registers save/restore + * 2022-10-22 Shell Support kernel mode RVV */ #ifndef __STACKFRAME_H__ #define __STACKFRAME_H__ +#include +#include "encoding.h" +#include "ext_context.h" + +/* bytes of register width */ +#ifdef ARCH_CPU_64BIT +#define STORE sd +#define LOAD ld +#define FSTORE fsd +#define FLOAD fld +#define REGBYTES 8 +#else +// error here, not portable +#error "Not supported XLEN" +#endif + +/* 33 general register + 1 padding */ +#define CTX_GENERAL_REG_NR 34 + +/* all context registers */ +#define CTX_REG_NR (CTX_GENERAL_REG_NR + CTX_FPU_REG_NR + CTX_VECTOR_REG_NR) + #define BYTES(idx) ((idx) * REGBYTES) #define FRAME_OFF_SSTATUS BYTES(2) #define FRAME_OFF_SP BYTES(32) #define FRAME_OFF_GP BYTES(3) -#include "cpuport.h" -#include "encoding.h" - -#ifdef ARCH_RISCV_FPU -#define FPU_CTX_F0_OFF 0 /* offsetof(fpu_context_t, fpustatus.f[0]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F1_OFF 8 /* offsetof(fpu_context_t, fpustatus.f[1]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F2_OFF 16 /* offsetof(fpu_context_t, fpustatus.f[2]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F3_OFF 24 /* offsetof(fpu_context_t, fpustatus.f[3]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F4_OFF 32 /* offsetof(fpu_context_t, fpustatus.f[4]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F5_OFF 40 /* offsetof(fpu_context_t, fpustatus.f[5]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F6_OFF 48 /* offsetof(fpu_context_t, fpustatus.f[6]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F7_OFF 56 /* offsetof(fpu_context_t, fpustatus.f[7]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F8_OFF 64 /* offsetof(fpu_context_t, fpustatus.f[8]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F9_OFF 72 /* offsetof(fpu_context_t, fpustatus.f[9]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F10_OFF 80 /* offsetof(fpu_context_t, fpustatus.f[10]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F11_OFF 88 /* offsetof(fpu_context_t, fpustatus.f[11]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F12_OFF 96 /* offsetof(fpu_context_t, fpustatus.f[12]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F13_OFF 104 /* offsetof(fpu_context_t, fpustatus.f[13]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F14_OFF 112 /* offsetof(fpu_context_t, fpustatus.f[14]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F15_OFF 120 /* offsetof(fpu_context_t, fpustatus.f[15]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F16_OFF 128 /* offsetof(fpu_context_t, fpustatus.f[16]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F17_OFF 136 /* offsetof(fpu_context_t, fpustatus.f[17]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F18_OFF 144 /* offsetof(fpu_context_t, fpustatus.f[18]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F19_OFF 152 /* offsetof(fpu_context_t, fpustatus.f[19]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F20_OFF 160 /* offsetof(fpu_context_t, fpustatus.f[20]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F21_OFF 168 /* offsetof(fpu_context_t, fpustatus.f[21]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F22_OFF 176 /* offsetof(fpu_context_t, fpustatus.f[22]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F23_OFF 184 /* offsetof(fpu_context_t, fpustatus.f[23]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F24_OFF 192 /* offsetof(fpu_context_t, fpustatus.f[24]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F25_OFF 200 /* offsetof(fpu_context_t, fpustatus.f[25]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F26_OFF 208 /* offsetof(fpu_context_t, fpustatus.f[26]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F27_OFF 216 /* offsetof(fpu_context_t, fpustatus.f[27]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F28_OFF 224 /* offsetof(fpu_context_t, fpustatus.f[28]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F29_OFF 232 /* offsetof(fpu_context_t, fpustatus.f[29]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F30_OFF 240 /* offsetof(fpu_context_t, fpustatus.f[30]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#define FPU_CTX_F31_OFF 248 /* offsetof(fpu_context_t, fpustatus.f[31]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#endif /* ARCH_RISCV_FPU */ - -/** - * The register `tp` always save/restore when context switch, - * we call `lwp_user_setting_save` when syscall enter, - * call `lwp_user_setting_restore` when syscall exit - * and modify context stack after `lwp_user_setting_restore` called - * so that the `tp` can be the correct thread area value. - */ +/* switch frame */ +#define RT_HW_SWITCH_CONTEXT_SSTATUS 0 +#define RT_HW_SWITCH_CONTEXT_S11 1 +#define RT_HW_SWITCH_CONTEXT_S10 2 +#define RT_HW_SWITCH_CONTEXT_S9 3 +#define RT_HW_SWITCH_CONTEXT_S8 4 +#define RT_HW_SWITCH_CONTEXT_S7 5 +#define RT_HW_SWITCH_CONTEXT_S6 6 +#define RT_HW_SWITCH_CONTEXT_S5 7 +#define RT_HW_SWITCH_CONTEXT_S4 8 +#define RT_HW_SWITCH_CONTEXT_S3 9 +#define RT_HW_SWITCH_CONTEXT_S2 10 +#define RT_HW_SWITCH_CONTEXT_S1 11 +#define RT_HW_SWITCH_CONTEXT_S0 12 +#define RT_HW_SWITCH_CONTEXT_RA 13 +#define RT_HW_SWITCH_CONTEXT_TP 14 +#define RT_HW_SWITCH_CONTEXT_ALIGNMENT 15 // Padding for alignment +#define RT_HW_SWITCH_CONTEXT_SIZE 16 // Total size of the structure + +#ifdef __ASSEMBLY__ .macro SAVE_ALL @@ -70,16 +68,20 @@ /* reserve float registers */ addi sp, sp, -CTX_FPU_REG_NR * REGBYTES #endif /* ARCH_RISCV_FPU */ +#ifdef ARCH_RISCV_VECTOR + /* reserve float registers */ + addi sp, sp, -CTX_VECTOR_REG_NR * REGBYTES +#endif /* ARCH_RISCV_VECTOR */ /* save general registers */ addi sp, sp, -CTX_GENERAL_REG_NR * REGBYTES STORE x1, 1 * REGBYTES(sp) csrr x1, sstatus - STORE x1, 2 * REGBYTES(sp) + STORE x1, FRAME_OFF_SSTATUS(sp) csrr x1, sepc - STORE x1, 0 * REGBYTES(sp) + STORE x1, 0 * REGBYTES(sp) STORE x3, 3 * REGBYTES(sp) STORE x4, 4 * REGBYTES(sp) /* save tp */ @@ -120,38 +122,38 @@ li t0, SSTATUS_FS csrs sstatus, t0 - fsd f0, FPU_CTX_F0_OFF(t1) - fsd f1, FPU_CTX_F1_OFF(t1) - fsd f2, FPU_CTX_F2_OFF(t1) - fsd f3, FPU_CTX_F3_OFF(t1) - fsd f4, FPU_CTX_F4_OFF(t1) - fsd f5, FPU_CTX_F5_OFF(t1) - fsd f6, FPU_CTX_F6_OFF(t1) - fsd f7, FPU_CTX_F7_OFF(t1) - fsd f8, FPU_CTX_F8_OFF(t1) - fsd f9, FPU_CTX_F9_OFF(t1) - fsd f10, FPU_CTX_F10_OFF(t1) - fsd f11, FPU_CTX_F11_OFF(t1) - fsd f12, FPU_CTX_F12_OFF(t1) - fsd f13, FPU_CTX_F13_OFF(t1) - fsd f14, FPU_CTX_F14_OFF(t1) - fsd f15, FPU_CTX_F15_OFF(t1) - fsd f16, FPU_CTX_F16_OFF(t1) - fsd f17, FPU_CTX_F17_OFF(t1) - fsd f18, FPU_CTX_F18_OFF(t1) - fsd f19, FPU_CTX_F19_OFF(t1) - fsd f20, FPU_CTX_F20_OFF(t1) - fsd f21, FPU_CTX_F21_OFF(t1) - fsd f22, FPU_CTX_F22_OFF(t1) - fsd f23, FPU_CTX_F23_OFF(t1) - fsd f24, FPU_CTX_F24_OFF(t1) - fsd f25, FPU_CTX_F25_OFF(t1) - fsd f26, FPU_CTX_F26_OFF(t1) - fsd f27, FPU_CTX_F27_OFF(t1) - fsd f28, FPU_CTX_F28_OFF(t1) - fsd f29, FPU_CTX_F29_OFF(t1) - fsd f30, FPU_CTX_F30_OFF(t1) - fsd f31, FPU_CTX_F31_OFF(t1) + FSTORE f0, FPU_CTX_F0_OFF(t1) + FSTORE f1, FPU_CTX_F1_OFF(t1) + FSTORE f2, FPU_CTX_F2_OFF(t1) + FSTORE f3, FPU_CTX_F3_OFF(t1) + FSTORE f4, FPU_CTX_F4_OFF(t1) + FSTORE f5, FPU_CTX_F5_OFF(t1) + FSTORE f6, FPU_CTX_F6_OFF(t1) + FSTORE f7, FPU_CTX_F7_OFF(t1) + FSTORE f8, FPU_CTX_F8_OFF(t1) + FSTORE f9, FPU_CTX_F9_OFF(t1) + FSTORE f10, FPU_CTX_F10_OFF(t1) + FSTORE f11, FPU_CTX_F11_OFF(t1) + FSTORE f12, FPU_CTX_F12_OFF(t1) + FSTORE f13, FPU_CTX_F13_OFF(t1) + FSTORE f14, FPU_CTX_F14_OFF(t1) + FSTORE f15, FPU_CTX_F15_OFF(t1) + FSTORE f16, FPU_CTX_F16_OFF(t1) + FSTORE f17, FPU_CTX_F17_OFF(t1) + FSTORE f18, FPU_CTX_F18_OFF(t1) + FSTORE f19, FPU_CTX_F19_OFF(t1) + FSTORE f20, FPU_CTX_F20_OFF(t1) + FSTORE f21, FPU_CTX_F21_OFF(t1) + FSTORE f22, FPU_CTX_F22_OFF(t1) + FSTORE f23, FPU_CTX_F23_OFF(t1) + FSTORE f24, FPU_CTX_F24_OFF(t1) + FSTORE f25, FPU_CTX_F25_OFF(t1) + FSTORE f26, FPU_CTX_F26_OFF(t1) + FSTORE f27, FPU_CTX_F27_OFF(t1) + FSTORE f28, FPU_CTX_F28_OFF(t1) + FSTORE f29, FPU_CTX_F29_OFF(t1) + FSTORE f30, FPU_CTX_F30_OFF(t1) + FSTORE f31, FPU_CTX_F31_OFF(t1) /* clr FS domain */ csrc sstatus, t0 @@ -162,49 +164,77 @@ #endif /* ARCH_RISCV_FPU */ +#ifdef ARCH_RISCV_VECTOR + csrr t0, sstatus + andi t0, t0, SSTATUS_VS + beqz t0, 0f + + /* push vector frame */ + addi t1, sp, (CTX_GENERAL_REG_NR + CTX_FPU_REG_NR) * REGBYTES + + SAVE_VECTOR t1 +0: +#endif /* ARCH_RISCV_VECTOR */ .endm +/** + * @brief Restore All General Registers, for interrupt handling + * + */ .macro RESTORE_ALL +#ifdef ARCH_RISCV_VECTOR + // skip on close + ld t0, 2 * REGBYTES(sp) + // cannot use vector on initial + andi t0, t0, SSTATUS_VS_CLEAN + beqz t0, 0f + + /* push vector frame */ + addi t1, sp, (CTX_GENERAL_REG_NR + CTX_FPU_REG_NR) * REGBYTES + + RESTORE_VECTOR t1 +0: +#endif /* ARCH_RISCV_VECTOR */ + #ifdef ARCH_RISCV_FPU /* restore float register */ - mv t2, sp - addi t2, t2, CTX_GENERAL_REG_NR * REGBYTES /* skip all normal reg */ + addi t2, sp, CTX_GENERAL_REG_NR * REGBYTES li t0, SSTATUS_FS csrs sstatus, t0 - fld f0, FPU_CTX_F0_OFF(t2) - fld f1, FPU_CTX_F1_OFF(t2) - fld f2, FPU_CTX_F2_OFF(t2) - fld f3, FPU_CTX_F3_OFF(t2) - fld f4, FPU_CTX_F4_OFF(t2) - fld f5, FPU_CTX_F5_OFF(t2) - fld f6, FPU_CTX_F6_OFF(t2) - fld f7, FPU_CTX_F7_OFF(t2) - fld f8, FPU_CTX_F8_OFF(t2) - fld f9, FPU_CTX_F9_OFF(t2) - fld f10, FPU_CTX_F10_OFF(t2) - fld f11, FPU_CTX_F11_OFF(t2) - fld f12, FPU_CTX_F12_OFF(t2) - fld f13, FPU_CTX_F13_OFF(t2) - fld f14, FPU_CTX_F14_OFF(t2) - fld f15, FPU_CTX_F15_OFF(t2) - fld f16, FPU_CTX_F16_OFF(t2) - fld f17, FPU_CTX_F17_OFF(t2) - fld f18, FPU_CTX_F18_OFF(t2) - fld f19, FPU_CTX_F19_OFF(t2) - fld f20, FPU_CTX_F20_OFF(t2) - fld f21, FPU_CTX_F21_OFF(t2) - fld f22, FPU_CTX_F22_OFF(t2) - fld f23, FPU_CTX_F23_OFF(t2) - fld f24, FPU_CTX_F24_OFF(t2) - fld f25, FPU_CTX_F25_OFF(t2) - fld f26, FPU_CTX_F26_OFF(t2) - fld f27, FPU_CTX_F27_OFF(t2) - fld f28, FPU_CTX_F28_OFF(t2) - fld f29, FPU_CTX_F29_OFF(t2) - fld f30, FPU_CTX_F30_OFF(t2) - fld f31, FPU_CTX_F31_OFF(t2) + FLOAD f0, FPU_CTX_F0_OFF(t2) + FLOAD f1, FPU_CTX_F1_OFF(t2) + FLOAD f2, FPU_CTX_F2_OFF(t2) + FLOAD f3, FPU_CTX_F3_OFF(t2) + FLOAD f4, FPU_CTX_F4_OFF(t2) + FLOAD f5, FPU_CTX_F5_OFF(t2) + FLOAD f6, FPU_CTX_F6_OFF(t2) + FLOAD f7, FPU_CTX_F7_OFF(t2) + FLOAD f8, FPU_CTX_F8_OFF(t2) + FLOAD f9, FPU_CTX_F9_OFF(t2) + FLOAD f10, FPU_CTX_F10_OFF(t2) + FLOAD f11, FPU_CTX_F11_OFF(t2) + FLOAD f12, FPU_CTX_F12_OFF(t2) + FLOAD f13, FPU_CTX_F13_OFF(t2) + FLOAD f14, FPU_CTX_F14_OFF(t2) + FLOAD f15, FPU_CTX_F15_OFF(t2) + FLOAD f16, FPU_CTX_F16_OFF(t2) + FLOAD f17, FPU_CTX_F17_OFF(t2) + FLOAD f18, FPU_CTX_F18_OFF(t2) + FLOAD f19, FPU_CTX_F19_OFF(t2) + FLOAD f20, FPU_CTX_F20_OFF(t2) + FLOAD f21, FPU_CTX_F21_OFF(t2) + FLOAD f22, FPU_CTX_F22_OFF(t2) + FLOAD f23, FPU_CTX_F23_OFF(t2) + FLOAD f24, FPU_CTX_F24_OFF(t2) + FLOAD f25, FPU_CTX_F25_OFF(t2) + FLOAD f26, FPU_CTX_F26_OFF(t2) + FLOAD f27, FPU_CTX_F27_OFF(t2) + FLOAD f28, FPU_CTX_F28_OFF(t2) + FLOAD f29, FPU_CTX_F29_OFF(t2) + FLOAD f30, FPU_CTX_F30_OFF(t2) + FLOAD f31, FPU_CTX_F31_OFF(t2) /* clr FS domain */ csrc sstatus, t0 @@ -216,9 +246,11 @@ #endif /* ARCH_RISCV_FPU */ /* restore general register */ + addi t0, sp, CTX_REG_NR * REGBYTES + csrw sscratch, t0 /* resw ra to sepc */ - LOAD x1, 0 * REGBYTES(sp) + LOAD x1, 0 * REGBYTES(sp) csrw sepc, x1 LOAD x1, 2 * REGBYTES(sp) @@ -275,4 +307,6 @@ csrci sstatus, 2 .endm -#endif +#endif /* __ASSEMBLY__ */ + +#endif /* __STACKFRAME_H__ */ diff --git a/libcpu/risc-v/t-head/c906/startup_gcc.S b/libcpu/risc-v/t-head/c906/startup_gcc.S index 7f25564fb49..184b48a1aeb 100644 --- a/libcpu/risc-v/t-head/c906/startup_gcc.S +++ b/libcpu/risc-v/t-head/c906/startup_gcc.S @@ -75,6 +75,8 @@ _start: li x31,0 /* set to disable FPU */ + li t0, SSTATUS_FS + SSTATUS_VS + csrc sstatus, t0 li t0, SSTATUS_SUM csrs sstatus, t0 diff --git a/libcpu/risc-v/t-head/c906/syscall_c.c b/libcpu/risc-v/t-head/c906/syscall_c.c index 2abe1d065bf..ceffd94dbcd 100644 --- a/libcpu/risc-v/t-head/c906/syscall_c.c +++ b/libcpu/risc-v/t-head/c906/syscall_c.c @@ -26,7 +26,7 @@ #include "riscv_mmu.h" #include "stack.h" -typedef rt_size_t (*syscallfunc_t)(rt_size_t, rt_size_t, rt_size_t, rt_size_t, rt_size_t, rt_size_t, rt_size_t); +typedef rt_ubase_t (*syscallfunc_t)(rt_ubase_t, rt_ubase_t, rt_ubase_t, rt_ubase_t, rt_ubase_t, rt_ubase_t, rt_ubase_t); void syscall_handler(struct rt_hw_stack_frame *regs) { diff --git a/libcpu/risc-v/t-head/c906/tlb.h b/libcpu/risc-v/t-head/c906/tlb.h index 3bfb3cb7e2d..5697ca61f69 100644 --- a/libcpu/risc-v/t-head/c906/tlb.h +++ b/libcpu/risc-v/t-head/c906/tlb.h @@ -20,7 +20,7 @@ #define HANDLE_FAULT(ret) \ if (__builtin_expect((ret) != SBI_SUCCESS, 0)) \ - LOG_W("%s failed\n", __FUNCTION__); + LOG_W("%s failed", __FUNCTION__); static inline void rt_hw_tlb_invalidate_all(void) { diff --git a/libcpu/risc-v/t-head/c906/trap.c b/libcpu/risc-v/t-head/c906/trap.c index c4ba4bf3335..abd4bed6bc8 100644 --- a/libcpu/risc-v/t-head/c906/trap.c +++ b/libcpu/risc-v/t-head/c906/trap.c @@ -1,36 +1,29 @@ /* - * Copyright (c) 2006-2021, RT-Thread Development Team + * Copyright (c) 2006-2024, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * * Change Logs: * Date Author Notes - * 2021/1/30 lizirui first version - * 2021/10/20 JasonHu move to trap.c + * 2022-12-08 RT-Thread first version */ #include #include #include - -#include "board.h" -#include "tick.h" - #include -#include "mmu.h" -#include "encoding.h" -#include "stack.h" -#include "sbi.h" -#include "riscv.h" -#include "rt_interrupt.h" -#include "plic.h" +#include +#include +#include +#include +#include +#include +#include +#include #ifdef RT_USING_SMART #include -void rt_hw_backtrace(rt_uint32_t *ffp, rt_ubase_t sepc); -#else -#define rt_hw_backtrace(...) (0) #endif #define DBG_TAG "libcpu.trap" @@ -42,7 +35,8 @@ void dump_regs(struct rt_hw_stack_frame *regs) rt_kprintf("--------------Dump Registers-----------------\n"); rt_kprintf("Function Registers:\n"); - rt_kprintf("\tra(x1) = 0x%p\tuser_sp = 0x%p\n", regs->ra, regs->user_sp_exc_stack); + rt_kprintf("\tra(x1) = 0x%p\tuser_sp = 0x%p\n", regs->ra, + regs->user_sp_exc_stack); rt_kprintf("\tgp(x3) = 0x%p\ttp(x4) = 0x%p\n", regs->gp, regs->tp); rt_kprintf("Temporary Registers:\n"); rt_kprintf("\tt0(x5) = 0x%p\tt1(x6) = 0x%p\n", regs->t0, regs->t1); @@ -62,15 +56,27 @@ void dump_regs(struct rt_hw_stack_frame *regs) rt_kprintf("\ta4(x14) = 0x%p\ta5(x15) = 0x%p\n", regs->a4, regs->a5); rt_kprintf("\ta6(x16) = 0x%p\ta7(x17) = 0x%p\n", regs->a6, regs->a7); rt_kprintf("sstatus = 0x%p\n", regs->sstatus); - rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SIE) ? "Supervisor Interrupt Enabled" : "Supervisor Interrupt Disabled"); - rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SPIE) ? "Last Time Supervisor Interrupt Enabled" : "Last Time Supervisor Interrupt Disabled"); - rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SPP) ? "Last Privilege is Supervisor Mode" : "Last Privilege is User Mode"); - rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SUM) ? "Permit to Access User Page" : "Not Permit to Access User Page"); - rt_kprintf("\t%s\n", (regs->sstatus & (1 << 19)) ? "Permit to Read Executable-only Page" : "Not Permit to Read Executable-only Page"); - rt_size_t satp_v = read_csr(satp); + rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SIE) + ? "Supervisor Interrupt Enabled" + : "Supervisor Interrupt Disabled"); + rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SPIE) + ? "Last Time Supervisor Interrupt Enabled" + : "Last Time Supervisor Interrupt Disabled"); + rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SPP) + ? "Last Privilege is Supervisor Mode" + : "Last Privilege is User Mode"); + rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SUM) + ? "Permit to Access User Page" + : "Not Permit to Access User Page"); + rt_kprintf("\t%s\n", (regs->sstatus & (1 << 19)) + ? "Permit to Read Executable-only Page" + : "Not Permit to Read Executable-only Page"); + rt_ubase_t satp_v = read_csr(satp); rt_kprintf("satp = 0x%p\n", satp_v); - rt_kprintf("\tCurrent Page Table(Physical) = 0x%p\n", __MASKVALUE(satp_v, __MASK(44)) << PAGE_OFFSET_BIT); - rt_kprintf("\tCurrent ASID = 0x%p\n", __MASKVALUE(satp_v >> 44, __MASK(16)) << PAGE_OFFSET_BIT); + rt_kprintf("\tCurrent Page Table(Physical) = 0x%p\n", + __MASKVALUE(satp_v, __MASK(44)) << PAGE_OFFSET_BIT); + rt_kprintf("\tCurrent ASID = 0x%p\n", __MASKVALUE(satp_v >> 44, __MASK(16)) + << PAGE_OFFSET_BIT); const char *mode_str = "Unknown Address Translation/Protection Mode"; switch (__MASKVALUE(satp_v >> 60, __MASK(4))) @@ -92,50 +98,44 @@ void dump_regs(struct rt_hw_stack_frame *regs) rt_kprintf("-----------------Dump OK---------------------\n"); } -static const char *Exception_Name[] = - { - "Instruction Address Misaligned", - "Instruction Access Fault", - "Illegal Instruction", - "Breakpoint", - "Load Address Misaligned", - "Load Access Fault", - "Store/AMO Address Misaligned", - "Store/AMO Access Fault", - "Environment call from U-mode", - "Environment call from S-mode", - "Reserved-10", - "Reserved-11", - "Instruction Page Fault", - "Load Page Fault", - "Reserved-14", - "Store/AMO Page Fault"}; - -static const char *Interrupt_Name[] = - { - "User Software Interrupt", - "Supervisor Software Interrupt", - "Reversed-2", - "Reversed-3", - "User Timer Interrupt", - "Supervisor Timer Interrupt", - "Reversed-6", - "Reversed-7", - "User External Interrupt", - "Supervisor External Interrupt", - "Reserved-10", - "Reserved-11", +static const char *Exception_Name[] = {"Instruction Address Misaligned", + "Instruction Access Fault", + "Illegal Instruction", + "Breakpoint", + "Load Address Misaligned", + "Load Access Fault", + "Store/AMO Address Misaligned", + "Store/AMO Access Fault", + "Environment call from U-mode", + "Environment call from S-mode", + "Reserved-10", + "Reserved-11", + "Instruction Page Fault", + "Load Page Fault", + "Reserved-14", + "Store/AMO Page Fault"}; + +static const char *Interrupt_Name[] = { + "User Software Interrupt", + "Supervisor Software Interrupt", + "Reversed-2", + "Reversed-3", + "User Timer Interrupt", + "Supervisor Timer Interrupt", + "Reversed-6", + "Reversed-7", + "User External Interrupt", + "Supervisor External Interrupt", + "Reserved-10", + "Reserved-11", }; #ifndef RT_USING_SMP static volatile int nested = 0; -#define ENTER_TRAP \ - nested += 1 -#define EXIT_TRAP \ - nested -= 1 +#define ENTER_TRAP nested += 1 +#define EXIT_TRAP nested -= 1 #define CHECK_NESTED_PANIC(cause, tval, epc, eframe) \ - if (nested != 1) \ - handle_nested_trap_panic(cause, tval, epc, eframe) + if (nested != 1) handle_nested_trap_panic(cause, tval, epc, eframe) #endif /* RT_USING_SMP */ static const char *get_exception_msg(int id) @@ -153,9 +153,11 @@ static const char *get_exception_msg(int id) } #ifdef RT_USING_SMART -void handle_user(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw_stack_frame *sp) +#include "lwp.h" +void handle_user(rt_ubase_t scause, rt_ubase_t stval, rt_ubase_t sepc, + struct rt_hw_stack_frame *sp) { - rt_size_t id = __MASKVALUE(scause, __MASK(63UL)); + rt_ubase_t id = __MASKVALUE(scause, __MASK(63UL)); struct rt_lwp *lwp; /* user page fault */ @@ -165,11 +167,11 @@ void handle_user(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw { case EP_LOAD_PAGE_FAULT: fault_op = MM_FAULT_OP_READ; - fault_type = MM_FAULT_TYPE_PAGE_FAULT; + fault_type = MM_FAULT_TYPE_GENERIC_MMU; break; case EP_LOAD_ACCESS_FAULT: fault_op = MM_FAULT_OP_READ; - fault_type = MM_FAULT_TYPE_GENERIC; + fault_type = MM_FAULT_TYPE_BUS_ERROR; break; case EP_LOAD_ADDRESS_MISALIGNED: fault_op = MM_FAULT_OP_READ; @@ -177,11 +179,11 @@ void handle_user(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw break; case EP_STORE_PAGE_FAULT: fault_op = MM_FAULT_OP_WRITE; - fault_type = MM_FAULT_TYPE_PAGE_FAULT; + fault_type = MM_FAULT_TYPE_GENERIC_MMU; break; case EP_STORE_ACCESS_FAULT: fault_op = MM_FAULT_OP_WRITE; - fault_type = MM_FAULT_TYPE_GENERIC; + fault_type = MM_FAULT_TYPE_BUS_ERROR; break; case EP_STORE_ADDRESS_MISALIGNED: fault_op = MM_FAULT_OP_WRITE; @@ -189,11 +191,11 @@ void handle_user(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw break; case EP_INSTRUCTION_PAGE_FAULT: fault_op = MM_FAULT_OP_EXECUTE; - fault_type = MM_FAULT_TYPE_PAGE_FAULT; + fault_type = MM_FAULT_TYPE_GENERIC_MMU; break; case EP_INSTRUCTION_ACCESS_FAULT: fault_op = MM_FAULT_OP_EXECUTE; - fault_type = MM_FAULT_TYPE_GENERIC; + fault_type = MM_FAULT_TYPE_BUS_ERROR; break; case EP_INSTRUCTION_ADDRESS_MISALIGNED: fault_op = MM_FAULT_OP_EXECUTE; @@ -213,28 +215,31 @@ void handle_user(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw .fault_vaddr = (void *)stval, }; - __asm__ volatile ("csrrsi %0, sstatus, 2":"=r"(saved_stat)); + __asm__ volatile("csrrsi %0, sstatus, 2" : "=r"(saved_stat)); if (lwp && rt_aspace_fault_try_fix(lwp->aspace, &msg)) { - __asm__ volatile ("csrw sstatus, %0"::"r"(saved_stat)); + __asm__ volatile("csrw sstatus, %0" ::"r"(saved_stat)); return; } - __asm__ volatile ("csrw sstatus, %0"::"r"(saved_stat)); + __asm__ volatile("csrw sstatus, %0" ::"r"(saved_stat)); } LOG_E("[FATAL ERROR] Exception %ld:%s\n", id, get_exception_msg(id)); LOG_E("scause:0x%p,stval:0x%p,sepc:0x%p\n", scause, stval, sepc); dump_regs(sp); - rt_hw_backtrace((uint32_t *)sp->s0_fp, sepc); + rt_thread_t cur_thr = rt_thread_self(); + struct rt_hw_backtrace_frame frame = {.fp = sp->s0_fp, .pc = sepc}; + rt_kprintf("fp = %p\n", frame.fp); + lwp_backtrace_frame(cur_thr, &frame); - LOG_E("User Fault, killing thread: %s", rt_thread_self()->parent.name); + LOG_E("User Fault, killing thread: %s", cur_thr->parent.name); EXIT_TRAP; sys_exit_group(-1); } #endif -#ifdef ENABLE_VECTOR +#ifdef ARCH_RISCV_VECTOR static void vector_enable(struct rt_hw_stack_frame *sp) { sp->sstatus |= SSTATUS_VS_INITIAL; @@ -243,7 +248,8 @@ static void vector_enable(struct rt_hw_stack_frame *sp) /** * detect V/D support, and do not distinguish V/D instruction */ -static int illegal_inst_recoverable(rt_ubase_t stval, struct rt_hw_stack_frame *sp) +static int illegal_inst_recoverable(rt_ubase_t stval, + struct rt_hw_stack_frame *sp) { // first 7 bits is opcode int opcode = stval & 0x7f; @@ -254,12 +260,12 @@ static int illegal_inst_recoverable(rt_ubase_t stval, struct rt_hw_stack_frame * switch (opcode) { - case 0x57: // V - case 0x27: // scalar FLOAT - case 0x07: - case 0x73: // CSR - flag = 1; - break; + case 0x57: // V + case 0x27: // scalar FLOAT + case 0x07: + case 0x73: // CSR + flag = 1; + break; } if (flag) @@ -271,11 +277,9 @@ static int illegal_inst_recoverable(rt_ubase_t stval, struct rt_hw_stack_frame * } #endif -static void handle_nested_trap_panic( - rt_size_t cause, - rt_size_t tval, - rt_size_t epc, - struct rt_hw_stack_frame *eframe) +static void handle_nested_trap_panic(rt_ubase_t cause, rt_ubase_t tval, + rt_ubase_t epc, + struct rt_hw_stack_frame *eframe) { LOG_E("\n-------- [SEVER ERROR] --------"); LOG_E("Nested trap detected"); @@ -285,21 +289,23 @@ static void handle_nested_trap_panic( } #define IN_USER_SPACE (stval >= USER_VADDR_START && stval < USER_VADDR_TOP) -#define PAGE_FAULT (id == EP_LOAD_PAGE_FAULT || id == EP_STORE_PAGE_FAULT) +#define PAGE_FAULT (id == EP_LOAD_PAGE_FAULT || id == EP_STORE_PAGE_FAULT) /* Trap entry */ -void handle_trap(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw_stack_frame *sp) +void handle_trap(rt_ubase_t scause, rt_ubase_t stval, rt_ubase_t sepc, + struct rt_hw_stack_frame *sp) { - rt_size_t id = __MASKVALUE(scause,__MASK(63UL)); + ENTER_TRAP; + rt_ubase_t id = __MASKVALUE(scause, __MASK(63UL)); const char *msg; /* supervisor external interrupt */ - if ((SCAUSE_INTERRUPT & scause) && SCAUSE_S_EXTERNAL_INTR == (scause & 0xff)) + if ((SCAUSE_INTERRUPT & scause) && + SCAUSE_S_EXTERNAL_INTR == (scause & 0xff)) { rt_interrupt_enter(); plic_handle_irq(); rt_interrupt_leave(); - return; } else if ((SCAUSE_INTERRUPT | SCAUSE_S_TIMER_INTR) == scause) { @@ -307,44 +313,74 @@ void handle_trap(rt_size_t scause, rt_size_t stval, rt_size_t sepc, struct rt_hw rt_interrupt_enter(); tick_isr(); rt_interrupt_leave(); - return; } - else if (SCAUSE_INTERRUPT & scause) + else { - if(id < sizeof(Interrupt_Name) / sizeof(const char *)) + if (SCAUSE_INTERRUPT & scause) { - msg = Interrupt_Name[id]; + if (id < sizeof(Interrupt_Name) / sizeof(const char *)) + { + msg = Interrupt_Name[id]; + } + else + { + msg = "Unknown Interrupt"; + } + + LOG_E("Unhandled Interrupt %ld:%s\n", id, msg); } else { - msg = "Unknown Interrupt"; +#ifdef ARCH_RISCV_VECTOR + if (scause == 0x2) + { + if (!(sp->sstatus & SSTATUS_VS) && + illegal_inst_recoverable(stval, sp)) + goto _exit; + } +#endif /* ARCH_RISCV_VECTOR */ +#ifdef RT_USING_SMART + if (!(sp->sstatus & 0x100) || (PAGE_FAULT && IN_USER_SPACE)) + { + handle_user(scause, stval, sepc, sp); + // if handle_user() return here, jump to u mode then + goto _exit; + } +#endif + + // handle kernel exception: + rt_kprintf("Unhandled Exception %ld:%s\n", id, + get_exception_msg(id)); } - LOG_E("Unhandled Interrupt %ld:%s\n",id,msg); - } - else - { + + // trap cannot nested when handling another trap / interrupt + CHECK_NESTED_PANIC(scause, stval, sepc, sp); + + rt_kprintf("scause:0x%p,stval:0x%p,sepc:0x%p\n", scause, stval, sepc); + dump_regs(sp); + + rt_thread_t cur_thr = rt_thread_self(); + rt_kprintf("--------------Thread list--------------\n"); + rt_kprintf("current thread: %s\n", cur_thr->parent.name); + + rt_kprintf("--------------Backtrace--------------\n"); + struct rt_hw_backtrace_frame frame = {.fp = sp->s0_fp, .pc = sepc}; + #ifdef RT_USING_SMART - if (!(sp->sstatus & 0x100) || (PAGE_FAULT && IN_USER_SPACE)) + if (!(sp->sstatus & 0x100)) { - handle_user(scause, stval, sepc, sp); - // if handle_user() return here, jump to u mode then - return ; + lwp_backtrace_frame(cur_thr, &frame); } + else #endif + { + rt_backtrace_frame(cur_thr, &frame); + } - // handle kernel exception: - rt_kprintf("Unhandled Exception %ld:%s\n", id, get_exception_msg(id)); + while (1) + ; } - - rt_kprintf("scause:0x%p,stval:0x%p,sepc:0x%p\n", scause, stval, sepc); - dump_regs(sp); - rt_kprintf("--------------Thread list--------------\n"); - rt_kprintf("current thread: %s\n", rt_thread_self()->parent.name); - - extern struct rt_thread *rt_current_thread; - rt_kprintf("--------------Backtrace--------------\n"); - rt_hw_backtrace((uint32_t *)sp->s0_fp, sepc); - - while (1) - ; +_exit: + EXIT_TRAP; + return; } diff --git a/libcpu/risc-v/virt64/SConscript b/libcpu/risc-v/virt64/SConscript index a0d110a43d1..5de136c7899 100644 --- a/libcpu/risc-v/virt64/SConscript +++ b/libcpu/risc-v/virt64/SConscript @@ -1,14 +1,16 @@ # RT-Thread building script for component from building import * - -Import('rtconfig') - cwd = GetCurrentDir() src = Glob('*.c') + Glob('*.cpp') + Glob('*_gcc.S') -src = src + ['../common/atomic_riscv.c'] CPPPATH = [cwd] +if not GetDepend('ARCH_USING_ASID'): + SrcRemove(src, ['asid.c']) + +if not GetDepend('ARCH_RISCV_VECTOR'): + SrcRemove(src, ['vector_gcc.S']) + group = DefineGroup('libcpu', src, depend = [''], CPPPATH = CPPPATH) Return('group') diff --git a/libcpu/risc-v/virt64/asid.c b/libcpu/risc-v/virt64/asid.c new file mode 100644 index 00000000000..5d7bffa3dcb --- /dev/null +++ b/libcpu/risc-v/virt64/asid.c @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2006-2024, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2024-07-12 RT-Thread first version. + */ +#define DBG_TAG "hw.asid" +#define DBG_LVL DBG_INFO +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +static rt_uint8_t ASID_BITS = 0; +static rt_uint32_t next_asid; +static rt_uint64_t global_asid_generation; +#define ASID_MASK ((1 << ASID_BITS) - 1) +#define ASID_FIRST_GENERATION (1 << ASID_BITS) +#define MAX_ASID ASID_FIRST_GENERATION + +void rt_hw_asid_init(void) +{ + unsigned int satp_reg = read_csr(satp); + satp_reg |= (((rt_uint64_t)0xffff) << PPN_BITS); + write_csr(satp, satp_reg); + unsigned short valid_asid_bit = ((read_csr(satp) >> PPN_BITS) & 0xffff); + + // The maximal value of ASIDLEN, is 9 for Sv32 or 16 for Sv39, Sv48, and Sv57 + for (unsigned i = 0; i < 16; i++) + { + if (!(valid_asid_bit & 0x1)) + { + break; + } + + valid_asid_bit >>= 1; + ASID_BITS++; + } + + global_asid_generation = ASID_FIRST_GENERATION; + next_asid = 1; +} + +static rt_uint64_t _asid_acquire(rt_aspace_t aspace) +{ + if ((aspace->asid ^ global_asid_generation) >> ASID_BITS) // not same generation + { + if (next_asid != MAX_ASID) + { + aspace->asid = global_asid_generation | next_asid; + next_asid++; + } + else + { + // scroll to next generation + global_asid_generation += ASID_FIRST_GENERATION; + next_asid = 1; + rt_hw_tlb_invalidate_all_local(); + + aspace->asid = global_asid_generation | next_asid; + next_asid++; + } + } + + return aspace->asid & ASID_MASK; +} + +void rt_hw_asid_switch_pgtbl(struct rt_aspace *aspace, rt_ubase_t pgtbl) +{ + rt_uint64_t asid = _asid_acquire(aspace); + write_csr(satp, (((size_t)SATP_MODE) << SATP_MODE_OFFSET) | + (asid << PPN_BITS) | + ((rt_ubase_t)pgtbl >> PAGE_OFFSET_BIT)); + asm volatile("sfence.vma x0,%0"::"r"(asid):"memory"); +} + diff --git a/libcpu/risc-v/virt64/backtrace.c b/libcpu/risc-v/virt64/backtrace.c index f76cd90c12c..8cfe7126cbc 100644 --- a/libcpu/risc-v/virt64/backtrace.c +++ b/libcpu/risc-v/virt64/backtrace.c @@ -15,6 +15,7 @@ #include #include #include "riscv_mmu.h" +#include "stack.h" #define WORD sizeof(rt_base_t) #define ARCH_CONTEXT_FETCH(pctx, id) (*(((unsigned long *)pctx) + (id))) @@ -117,8 +118,9 @@ rt_err_t rt_hw_backtrace_frame_get(rt_thread_t thread, struct rt_hw_backtrace_fr } else { - frame->pc = ARCH_CONTEXT_FETCH(thread->sp, 13); - frame->fp = ARCH_CONTEXT_FETCH(thread->sp, 12); + rt_hw_switch_frame_t sframe = thread->sp; + frame->pc = sframe->regs[RT_HW_SWITCH_CONTEXT_RA]; + frame->fp = sframe->regs[RT_HW_SWITCH_CONTEXT_S0];; rc = RT_EOK; } return rc; diff --git a/libcpu/risc-v/virt64/context_gcc.S b/libcpu/risc-v/virt64/context_gcc.S index b656aa74ac5..8858f6c1040 100644 --- a/libcpu/risc-v/virt64/context_gcc.S +++ b/libcpu/risc-v/virt64/context_gcc.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2021, RT-Thread Development Team + * Copyright (c) 2006-2024, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * @@ -8,64 +8,64 @@ * 2018/10/28 Bernard The unify RISC-V porting implementation * 2018/12/27 Jesven Add SMP support * 2021/02/02 lizhirui Add userspace support - * 2022/10/22 WangXiaoyao Support User mode RVV; + * 2022/10/22 Shell Support User mode RVV; * Trimming process switch context */ #include "cpuport.h" #include "stackframe.h" +#define _REG_IDX(name) RT_HW_SWITCH_CONTEXT_##name +#define REG_IDX(name) _REG_IDX(name) -.macro PUSH_REG reg - addi sp, sp, -REGBYTES - STORE \reg, (sp) +.macro SAVE_REG reg, index + STORE \reg, \index*REGBYTES(sp) .endm -.macro POP_REG reg - LOAD \reg, (sp) - addi sp, sp, REGBYTES +.macro LOAD_REG reg, index + LOAD \reg, \index*REGBYTES(sp) .endm .macro RESERVE_CONTEXT - PUSH_REG tp - PUSH_REG ra - PUSH_REG s0 - PUSH_REG s1 - PUSH_REG s2 - PUSH_REG s3 - PUSH_REG s4 - PUSH_REG s5 - PUSH_REG s6 - PUSH_REG s7 - PUSH_REG s8 - PUSH_REG s9 - PUSH_REG s10 - PUSH_REG s11 - csrr s11, sstatus - li s10, (SSTATUS_SPP) - or s11, s11, s10 - PUSH_REG s11 - addi sp, sp, -REGBYTES + addi sp, sp, -(RT_HW_SWITCH_CONTEXT_SIZE * REGBYTES) + SAVE_REG tp, REG_IDX(TP) + SAVE_REG ra, REG_IDX(RA) + SAVE_REG s0, REG_IDX(S0) + SAVE_REG s1, REG_IDX(S1) + SAVE_REG s2, REG_IDX(S2) + SAVE_REG s3, REG_IDX(S3) + SAVE_REG s4, REG_IDX(S4) + SAVE_REG s5, REG_IDX(S5) + SAVE_REG s6, REG_IDX(S6) + SAVE_REG s7, REG_IDX(S7) + SAVE_REG s8, REG_IDX(S8) + SAVE_REG s9, REG_IDX(S9) + SAVE_REG s10, REG_IDX(S10) + SAVE_REG s11, REG_IDX(S11) + csrr s11, sstatus + li s10, (SSTATUS_SPP) + or s11, s11, s10 + SAVE_REG s11, REG_IDX(SSTATUS) .endm .macro RESTORE_CONTEXT - addi sp, sp, REGBYTES - POP_REG s11 - csrw sstatus, s11 - POP_REG s11 - POP_REG s10 - POP_REG s9 - POP_REG s8 - POP_REG s7 - POP_REG s6 - POP_REG s5 - POP_REG s4 - POP_REG s3 - POP_REG s2 - POP_REG s1 - POP_REG s0 - POP_REG ra - POP_REG tp - csrw sepc, ra + LOAD_REG s11, REG_IDX(SSTATUS) + csrw sstatus, s11 + LOAD_REG s11, REG_IDX(S11) + LOAD_REG s10, REG_IDX(S10) + LOAD_REG s9, REG_IDX(S9) + LOAD_REG s8, REG_IDX(S8) + LOAD_REG s7, REG_IDX(S7) + LOAD_REG s6, REG_IDX(S6) + LOAD_REG s5, REG_IDX(S5) + LOAD_REG s4, REG_IDX(S4) + LOAD_REG s3, REG_IDX(S3) + LOAD_REG s2, REG_IDX(S2) + LOAD_REG s1, REG_IDX(S1) + LOAD_REG s0, REG_IDX(S0) + LOAD_REG ra, REG_IDX(RA) + LOAD_REG tp, REG_IDX(TP) + addi sp, sp, RT_HW_SWITCH_CONTEXT_SIZE * REGBYTES + csrw sepc, ra .endm /* @@ -113,33 +113,3 @@ rt_hw_context_switch: RESTORE_CONTEXT sret - -#ifdef ENABLE_VECTOR -/** - * @param a0 pointer to frame bottom - */ -.global rt_hw_vector_ctx_save -rt_hw_vector_ctx_save: - SAVE_VECTOR a0 - ret - -/** - * @param a0 pointer to frame bottom - */ -.global rt_hw_vector_ctx_restore -rt_hw_vector_ctx_restore: - RESTORE_VECTOR a0 - ret - -.global rt_hw_disable_vector -rt_hw_disable_vector: - li t0, SSTATUS_VS - csrc sstatus, t0 - ret - -.global rt_hw_enable_vector -rt_hw_enable_vector: - li t0, SSTATUS_VS - csrs sstatus, t0 - ret -#endif /* ENABLE_VECTOR */ diff --git a/libcpu/risc-v/virt64/cpuport.c b/libcpu/risc-v/virt64/cpuport.c index 3b00e7368ac..be154198df4 100644 --- a/libcpu/risc-v/virt64/cpuport.c +++ b/libcpu/risc-v/virt64/cpuport.c @@ -7,6 +7,7 @@ * Date Author Notes * 2018/10/28 Bernard The unify RISC-V porting code. * 2021-02-11 lizhirui add gp support + * 2021-11-19 JasonHu add fpu support */ #include @@ -17,7 +18,14 @@ #include #include -#define K_SSTATUS_DEFAULT (SSTATUS_SPP | SSTATUS_SPIE | SSTATUS_SUM | SSTATUS_FS) +#ifdef ARCH_RISCV_FPU + #define K_SSTATUS_DEFAULT (SSTATUS_SPP | SSTATUS_SPIE | SSTATUS_SUM | SSTATUS_FS) +#else + #define K_SSTATUS_DEFAULT (SSTATUS_SPP | SSTATUS_SPIE | SSTATUS_SUM) +#endif +#ifdef RT_USING_SMART +#include +#endif /** * @brief from thread used interrupt context switch @@ -37,24 +45,15 @@ volatile rt_ubase_t rt_thread_switch_interrupt_flag = 0; void *_rt_hw_stack_init(rt_ubase_t *sp, rt_ubase_t ra, rt_ubase_t sstatus) { - (*--sp) = 0; /* tp */ - (*--sp) = ra; /* ra */ - (*--sp) = 0; /* s0(fp) */ - (*--sp) = 0; /* s1 */ - (*--sp) = 0; /* s2 */ - (*--sp) = 0; /* s3 */ - (*--sp) = 0; /* s4 */ - (*--sp) = 0; /* s5 */ - (*--sp) = 0; /* s6 */ - (*--sp) = 0; /* s7 */ - (*--sp) = 0; /* s8 */ - (*--sp) = 0; /* s9 */ - (*--sp) = 0; /* s10 */ - (*--sp) = 0; /* s11 */ - (*--sp) = sstatus; /* sstatus */ - --sp; /* align to 16bytes */ - - return (void *)sp; + rt_hw_switch_frame_t frame = (rt_hw_switch_frame_t) + ((rt_ubase_t)sp - sizeof(struct rt_hw_switch_frame)); + + rt_memset(frame, 0, sizeof(struct rt_hw_switch_frame)); + + frame->regs[RT_HW_SWITCH_CONTEXT_RA] = ra; + frame->regs[RT_HW_SWITCH_CONTEXT_SSTATUS] = sstatus; + + return (void *)frame; } int rt_hw_cpu_id(void) @@ -87,6 +86,7 @@ rt_uint8_t *rt_hw_stack_init(void *tentry, (*--sp) = (rt_ubase_t)tentry; (*--sp) = (rt_ubase_t)parameter; (*--sp) = (rt_ubase_t)texit; + --sp; /* alignment */ /* compatible to RESTORE_CONTEXT */ extern void _rt_thread_entry(void); diff --git a/libcpu/risc-v/virt64/cpuport.h b/libcpu/risc-v/virt64/cpuport.h index c12ff7ddf7f..0723a3c393a 100644 --- a/libcpu/risc-v/virt64/cpuport.h +++ b/libcpu/risc-v/virt64/cpuport.h @@ -12,44 +12,7 @@ #define CPUPORT_H__ #include - -/* bytes of register width */ -#ifdef ARCH_CPU_64BIT -#define STORE sd -#define LOAD ld -#define FSTORE fsd -#define FLOAD fld -#define REGBYTES 8 -#else -// error here, not portable -#error "Not supported XLEN" -#endif - -/* 33 general register + 1 padding */ -#define CTX_GENERAL_REG_NR 34 - -#ifdef ENABLE_FPU -/* 32 fpu register */ -#define CTX_FPU_REG_NR 32 -#else -#define CTX_FPU_REG_NR 0 -#endif - -#ifdef ENABLE_VECTOR - -#if defined(ARCH_VECTOR_VLEN_128) -#define CTX_VECTOR_REGS 64 -#elif defined(ARCH_VECTOR_VLEN_256) -#define CTX_VECTOR_REGS 128 -#endif - -#define CTX_VECTOR_REG_NR (CTX_VECTOR_REGS + 4) -#else -#define CTX_VECTOR_REG_NR 0 -#endif - -/* all context registers */ -#define CTX_REG_NR (CTX_GENERAL_REG_NR + CTX_FPU_REG_NR + CTX_VECTOR_REG_NR) +#include #ifdef RT_USING_SMP typedef union { @@ -63,21 +26,24 @@ typedef union { #ifndef __ASSEMBLY__ #include + rt_inline void rt_hw_dsb(void) { - asm volatile("fence":::"memory"); + __asm__ volatile("fence":::"memory"); } rt_inline void rt_hw_dmb(void) { - asm volatile("fence":::"memory"); + __asm__ volatile("fence":::"memory"); } rt_inline void rt_hw_isb(void) { - asm volatile(".long 0x0000100F":::"memory"); + __asm__ volatile(OPC_FENCE_I:::"memory"); } +int rt_hw_cpu_id(void); + #endif #endif diff --git a/libcpu/risc-v/virt64/cpuport_gcc.S b/libcpu/risc-v/virt64/cpuport_gcc.S index 89cf8701c6c..5361646753f 100644 --- a/libcpu/risc-v/virt64/cpuport_gcc.S +++ b/libcpu/risc-v/virt64/cpuport_gcc.S @@ -13,12 +13,11 @@ #include "asm-generic.h" START_POINT(_rt_thread_entry) - LOAD ra, (sp) /* thread exit */ - addi sp, sp, REGBYTES - LOAD a0, (sp) /* parameter */ - addi sp, sp, REGBYTES - LOAD t0, (sp) /* tentry */ - addi sp, sp, REGBYTES + LOAD ra, REGBYTES(sp) /* thread exit */ + addi sp, sp, 2 * REGBYTES + LOAD a0, (sp) /* parameter */ + LOAD t0, REGBYTES(sp) /* tentry */ + addi sp, sp, 2 * REGBYTES mv s1, ra jalr t0 jalr s1 diff --git a/libcpu/risc-v/virt64/encoding.h b/libcpu/risc-v/virt64/encoding.h index 2616b864610..7e906fedd83 100644 --- a/libcpu/risc-v/virt64/encoding.h +++ b/libcpu/risc-v/virt64/encoding.h @@ -11,6 +11,15 @@ #ifndef RISCV_CSR_ENCODING_H #define RISCV_CSR_ENCODING_H +#include + +#ifdef ARCH_RISCV_VECTOR +#include "vector_encoding.h" + +#else +#define SSTATUS_VS 0 /* fallback */ +#endif /* ARCH_RISCV_VECTOR */ + #define MSTATUS_UIE 0x00000001 #define MSTATUS_SIE 0x00000002 #define MSTATUS_HIE 0x00000004 @@ -23,7 +32,6 @@ #define MSTATUS_HPP 0x00000600 #define MSTATUS_MPP 0x00001800 #define MSTATUS_FS 0x00006000 -#define MSTATUS_VS 0x00000600 #define MSTATUS_XS 0x00018000 #define MSTATUS_MPRV 0x00020000 #define MSTATUS_PUM 0x00040000 @@ -41,10 +49,6 @@ #define SSTATUS_FS_INITIAL 0x00002000 #define SSTATUS_FS_CLEAN 0x00004000 #define SSTATUS_FS_DIRTY 0x00006000 -#define SSTATUS_VS 0x00000600 /* Vector Status */ -#define SSTATUS_VS_INITIAL 0x00000200 -#define SSTATUS_VS_CLEAN 0x00000400 -#define SSTATUS_VS_DIRTY 0x00000600 #define SSTATUS_XS 0x00018000 #define SSTATUS_SUM 0x00040000 #define SSTATUS32_SD 0x80000000 diff --git a/libcpu/risc-v/virt64/ext_context.h b/libcpu/risc-v/virt64/ext_context.h index df918fb5514..f7eb9035765 100644 --- a/libcpu/risc-v/virt64/ext_context.h +++ b/libcpu/risc-v/virt64/ext_context.h @@ -5,29 +5,29 @@ * * Change Logs: * Date Author Notes - * 2022-10-10 RT-Thread the first version, - * compatible to riscv-v-spec-1.0 + * 2022-10-10 RT-Thread the first version */ #ifndef __EXT_CONTEXT_H__ #define __EXT_CONTEXT_H__ -#ifdef __ASSEMBLY__ +#include -/** - * extension context maintenance - */ +#ifdef ARCH_RISCV_FPU +/* 32 fpu register */ +#define CTX_FPU_REG_NR 32 +#else +#define CTX_FPU_REG_NR 0 +#endif /* ARCH_RISCV_FPU */ -#include "cpuport.h" -#include "encoding.h" -#include "vector_encoding.h" +#ifdef __ASSEMBLY__ /** * ================================== - * FPU EXTENSION + * RISC-V D ISA (Floating) * ================================== */ -#ifdef ENABLE_FPU +#ifdef ARCH_RISCV_FPU #define FPU_CTX_F0_OFF (REGBYTES * 0) /* offsetof(fpu_context_t, fpustatus.f[0]) - offsetof(fpu_context_t, fpustatus.f[0]) */ #define FPU_CTX_F1_OFF (REGBYTES * 1) /* offsetof(fpu_context_t, fpustatus.f[1]) - offsetof(fpu_context_t, fpustatus.f[0]) */ #define FPU_CTX_F2_OFF (REGBYTES * 2) /* offsetof(fpu_context_t, fpustatus.f[2]) - offsetof(fpu_context_t, fpustatus.f[0]) */ @@ -60,96 +60,14 @@ #define FPU_CTX_F29_OFF (REGBYTES * 29) /* offsetof(fpu_context_t, fpustatus.f[29]) - offsetof(fpu_context_t, fpustatus.f[0]) */ #define FPU_CTX_F30_OFF (REGBYTES * 30) /* offsetof(fpu_context_t, fpustatus.f[30]) - offsetof(fpu_context_t, fpustatus.f[0]) */ #define FPU_CTX_F31_OFF (REGBYTES * 31) /* offsetof(fpu_context_t, fpustatus.f[31]) - offsetof(fpu_context_t, fpustatus.f[0]) */ -#endif /* ENABLE_FPU */ - -/** - * ================================== - * VECTOR EXTENSION - * ================================== - */ - -#ifdef ENABLE_VECTOR - -#define VEC_FRAME_VSTART (0 * REGBYTES) -#define VEC_FRAME_VTYPE (1 * REGBYTES) -#define VEC_FRAME_VL (2 * REGBYTES) -#define VEC_FRAME_VCSR (3 * REGBYTES) -#define VEC_FRAME_V0 (4 * REGBYTES) - -.macro GET_VEC_FRAME_LEN, xreg - csrr \xreg, vlenb - slli \xreg, \xreg, 5 - addi \xreg, \xreg, 4 * REGBYTES -.endm - -/** - * @brief save vector extension hardware state - * - * @param dst register storing bottom of storage block - * - */ -.macro SAVE_VECTOR, dst - mv t1, \dst - - csrr t0, vstart - STORE t0, VEC_FRAME_VSTART(t1) - csrr t0, vtype - STORE t0, VEC_FRAME_VTYPE(t1) - csrr t0, vl - STORE t0, VEC_FRAME_VL(t1) - csrr t0, vcsr - STORE t0, VEC_FRAME_VCSR(t1) - - addi t1, t1, VEC_FRAME_V0 - - // config vector setting, - // t2 is updated to length of a vector group in bytes - VEC_CONFIG_SETVLI(t2, x0, VEC_IMM_SEW_8, VEC_IMM_LMUL_8) - - vse8.v v0, (t1) - add t1, t1, t2 - vse8.v v8, (t1) - add t1, t1, t2 - vse8.v v16, (t1) - add t1, t1, t2 - vse8.v v24, (t1) -.endm - -/** - * @brief restore vector extension hardware states - * - * @param dst register storing bottom of storage block - * - */ -.macro RESTORE_VECTOR, dst - // restore vector registers first since it will modify vector states - mv t0, \dst - addi t1, t0, VEC_FRAME_V0 - - VEC_CONFIG_SETVLI(t2, x0, VEC_IMM_SEW_8, VEC_IMM_LMUL_8) - - vle8.v v0, (t1) - add t1, t1, t2 - vle8.v v8, (t1) - add t1, t1, t2 - vle8.v v16, (t1) - add t1, t1, t2 - vle8.v v24, (t1) - - mv t1, t0 - - LOAD t0, VEC_FRAME_VSTART(t1) - csrw vstart, t0 - LOAD t0, VEC_FRAME_VCSR(t1) - csrw vcsr, t0 - - LOAD t0, VEC_FRAME_VTYPE(t1) - LOAD t3, VEC_FRAME_VL(t1) - VEC_CONFIG_SET_VL_VTYPE(t3, t0) -.endm - -#endif /* ENABLE_VECTOR */ +#endif /* ARCH_RISCV_FPU */ #endif /* __ASSEMBLY__ */ +#ifdef ARCH_RISCV_VECTOR +#include "rvv_context.h" +#else /* !ARCH_RISCV_VECTOR */ +#define CTX_VECTOR_REG_NR 0 +#endif /* ARCH_RISCV_VECTOR */ + #endif /* __EXT_CONTEXT_H__ */ diff --git a/libcpu/risc-v/virt64/interrupt_gcc.S b/libcpu/risc-v/virt64/interrupt_gcc.S index 83d995bd882..16778523d99 100644 --- a/libcpu/risc-v/virt64/interrupt_gcc.S +++ b/libcpu/risc-v/virt64/interrupt_gcc.S @@ -9,7 +9,7 @@ * 2018/12/27 Jesven Add SMP schedule * 2021/02/02 lizhirui Add userspace support * 2021/12/24 JasonHu Add user setting save/restore - * 2022/10/22 WangXiaoyao Support kernel mode RVV; + * 2022/10/22 Shell Support kernel mode RVV; * Rewrite trap handling routine */ @@ -69,6 +69,9 @@ _context_switch: LOAD a0, 0(t0) la t0, rt_interrupt_to_thread LOAD a1, 0(t0) + csrr t0, sstatus + andi t0, t0, ~SSTATUS_SPIE + csrw sstatus, t0 jal rt_hw_context_switch _resume_execution: diff --git a/libcpu/risc-v/virt64/io.h b/libcpu/risc-v/virt64/io.h index 1285d5955bf..97dca5af971 100644 --- a/libcpu/risc-v/virt64/io.h +++ b/libcpu/risc-v/virt64/io.h @@ -40,13 +40,13 @@ static inline uint32_t readl(const volatile void *addr) static inline void write_reg( uint32_t val, volatile void *addr, unsigned offset) { - writel(val, addr + offset); + writel(val, (void *)((rt_size_t)addr + offset)); } static inline uint32_t read_reg( const volatile void *addr, unsigned offset) { - return readl(addr + offset); + return readl((void *)((rt_size_t)addr + offset)); } #endif // ARCH_IO_H diff --git a/libcpu/risc-v/virt64/mmu.c b/libcpu/risc-v/virt64/mmu.c index 0531d5ae39b..649c22a0546 100644 --- a/libcpu/risc-v/virt64/mmu.c +++ b/libcpu/risc-v/virt64/mmu.c @@ -43,6 +43,16 @@ static void *current_mmu_table = RT_NULL; volatile __attribute__((aligned(4 * 1024))) rt_ubase_t MMUTable[__SIZE(VPN2_BIT)]; +#ifdef ARCH_USING_ASID +void rt_hw_aspace_switch(rt_aspace_t aspace) +{ + uintptr_t page_table = (uintptr_t)rt_kmem_v2p(aspace->page_table); + current_mmu_table = aspace->page_table; + + rt_hw_asid_switch_pgtbl(aspace, page_table); +} + +#else /* !ARCH_USING_ASID */ void rt_hw_aspace_switch(rt_aspace_t aspace) { uintptr_t page_table = (uintptr_t)rt_kmem_v2p(aspace->page_table); @@ -53,6 +63,11 @@ void rt_hw_aspace_switch(rt_aspace_t aspace) rt_hw_tlb_invalidate_all_local(); } +void rt_hw_asid_init(void) +{ +} +#endif /* ARCH_USING_ASID */ + void *rt_hw_mmu_tbl_get() { return current_mmu_table; @@ -246,8 +261,7 @@ void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size) MM_PGTBL_UNLOCK(aspace); // when unmapped == 0, region not exist in pgtbl - if (!unmapped || unmapped > size) - break; + if (!unmapped || unmapped > size) break; size -= unmapped; v_addr += unmapped; @@ -260,7 +274,8 @@ static inline void _init_region(void *vaddr, size_t size) rt_ioremap_start = vaddr; rt_ioremap_size = size; rt_mpr_start = rt_ioremap_start - rt_mpr_size; - LOG_D("rt_ioremap_start: %p, rt_mpr_start: %p", rt_ioremap_start, rt_mpr_start); + LOG_D("rt_ioremap_start: %p, rt_mpr_start: %p", rt_ioremap_start, + rt_mpr_start); } #else static inline void _init_region(void *vaddr, size_t size) @@ -270,11 +285,11 @@ static inline void _init_region(void *vaddr, size_t size) #endif #if defined(RT_USING_SMART) && defined(ARCH_REMAP_KERNEL) -#define KERN_SPACE_START ((void *)KERNEL_VADDR_START) -#define KERN_SPACE_SIZE (0xfffffffffffff000UL - KERNEL_VADDR_START + 0x1000) +#define KERN_SPACE_START ((void *)KERNEL_VADDR_START) +#define KERN_SPACE_SIZE (0xfffffffffffff000UL - KERNEL_VADDR_START + 0x1000) #else -#define KERN_SPACE_START ((void *)0x1000) -#define KERN_SPACE_SIZE ((size_t)USER_VADDR_START - 0x1000) +#define KERN_SPACE_START ((void *)0x1000) +#define KERN_SPACE_SIZE ((size_t)USER_VADDR_START - 0x1000) #endif int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_ubase_t size, @@ -393,6 +408,7 @@ void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr) } else { + LOG_D("%s: failed at %p", __func__, vaddr); paddr = (uintptr_t)ARCH_MAP_FAILED; } return (void *)paddr; @@ -467,25 +483,25 @@ void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr) size_t attr; switch (mdesc->attr) { - case NORMAL_MEM: - attr = MMU_MAP_K_RWCB; - break; - case NORMAL_NOCACHE_MEM: - attr = MMU_MAP_K_RWCB; - break; - case DEVICE_MEM: - attr = MMU_MAP_K_DEVICE; - break; - default: - attr = MMU_MAP_K_DEVICE; + case NORMAL_MEM: + attr = MMU_MAP_K_RWCB; + break; + case NORMAL_NOCACHE_MEM: + attr = MMU_MAP_K_RWCB; + break; + case DEVICE_MEM: + attr = MMU_MAP_K_DEVICE; + break; + default: + attr = MMU_MAP_K_DEVICE; } - struct rt_mm_va_hint hint = {.flags = MMF_MAP_FIXED, - .limit_start = aspace->start, - .limit_range_size = aspace->size, - .map_size = mdesc->vaddr_end - - mdesc->vaddr_start + 1, - .prefer = (void *)mdesc->vaddr_start}; + struct rt_mm_va_hint hint = { + .flags = MMF_MAP_FIXED, + .limit_start = aspace->start, + .limit_range_size = aspace->size, + .map_size = mdesc->vaddr_end - mdesc->vaddr_start + 1, + .prefer = (void *)mdesc->vaddr_start}; if (mdesc->paddr_start == (rt_uintptr_t)ARCH_MAP_FAILED) mdesc->paddr_start = mdesc->vaddr_start + PV_OFFSET; @@ -495,6 +511,8 @@ void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr) mdesc++; } + rt_hw_asid_init(); + rt_hw_aspace_switch(&rt_kernel_space); rt_page_cleanup(); } @@ -520,16 +538,18 @@ void rt_hw_mem_setup_early(void) { if (pv_off & (1ul << (ARCH_INDEX_WIDTH * 2 + ARCH_PAGE_SHIFT))) { - LOG_E("%s: not aligned virtual address. pv_offset %p", __func__, pv_off); + LOG_E("%s: not aligned virtual address. pv_offset %p", __func__, + pv_off); RT_ASSERT(0); } + /** * identical mapping, * PC are still at lower region before relocating to high memory */ for (size_t i = 0; i < __SIZE(PPN0_BIT); i++) { - early_pgtbl[i] = COMBINEPTE(ps, PAGE_ATTR_RWX | PTE_G | PTE_V); + early_pgtbl[i] = COMBINEPTE(ps, MMU_MAP_EARLY); ps += L1_PAGE_SIZE; } @@ -543,7 +563,7 @@ void rt_hw_mem_setup_early(void) rt_ubase_t ve_idx = GET_L1(vs + 0x80000000); for (size_t i = vs_idx; i < ve_idx; i++) { - early_pgtbl[i] = COMBINEPTE(ps, PAGE_ATTR_RWX | PTE_G | PTE_V); + early_pgtbl[i] = COMBINEPTE(ps, MMU_MAP_EARLY); ps += L1_PAGE_SIZE; } diff --git a/libcpu/risc-v/virt64/opcode.h b/libcpu/risc-v/virt64/opcode.h new file mode 100644 index 00000000000..14a6a736496 --- /dev/null +++ b/libcpu/risc-v/virt64/opcode.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2006-2021, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-11-09 Shell Add portable asm support + */ +#ifndef __OPCODE_H__ +#define __OPCODE_H__ + +/** + * @brief binary opcode pseudo operations + * Used to bypass toolchain restriction on extension ISA + * + */ + +/** + * @brief RISC-V instruction formats + */ + +/** + * R type: .insn r opcode6, func3, func7, rd, rs1, rs2 + * + * +-------+-----+-----+-------+----+---------+ + * | func7 | rs2 | rs1 | func3 | rd | opcode6 | + * +-------+-----+-----+-------+----+---------+ + * 31 25 20 15 12 7 0 + */ +#define __OPC_INSN_FORMAT_R(opcode, func3, func7, rd, rs1, rs2) \ + ".insn r "RT_STRINGIFY(opcode)","RT_STRINGIFY(func3)","RT_STRINGIFY(func7)","RT_STRINGIFY(rd)","RT_STRINGIFY(rs1)","RT_STRINGIFY(rs2) + +#ifdef _TOOLCHAIN_SUPP_ZIFENCEI_ISA_ +#define OPC_FENCE_I "fence.i" +#else /* !_TOOLCHAIN_SUPP_ZIFENCEI_ISA_ */ +#define OPC_FENCE_I ".long 0x0000100F" +#endif /* _TOOLCHAIN_SUPP_ZIFENCEI_ISA_ */ + +#endif /* __OPCODE_H__ */ \ No newline at end of file diff --git a/libcpu/risc-v/virt64/plic.c b/libcpu/risc-v/virt64/plic.c index bd87df0e627..f222146f34d 100644 --- a/libcpu/risc-v/virt64/plic.c +++ b/libcpu/risc-v/virt64/plic.c @@ -14,6 +14,7 @@ #include "plic.h" #include #include "encoding.h" +#include #include #include @@ -139,3 +140,17 @@ void plic_init() // in a single core system, only current context was set _set_sie(__raw_hartid()); } + +extern struct rt_irq_desc irq_desc[MAX_HANDLERS]; +/* + * Handling an interrupt is a two-step process: first you claim the interrupt + * by reading the claim register, then you complete the interrupt by writing + * that source ID back to the same claim register. This automatically enables + * and disables the interrupt, so there's nothing else to do. + */ +void plic_handle_irq(void) +{ + int plic_irq = plic_claim(); + plic_complete(plic_irq); + irq_desc[plic_irq].handler(plic_irq, irq_desc[plic_irq].param); +} diff --git a/libcpu/risc-v/virt64/plic.h b/libcpu/risc-v/virt64/plic.h index d24c2e5a531..314210c66b5 100644 --- a/libcpu/risc-v/virt64/plic.h +++ b/libcpu/risc-v/virt64/plic.h @@ -72,5 +72,6 @@ void plic_complete(int irq); void plic_set_thresh(rt_uint32_t val); void plic_set_ie(rt_uint32_t word_index,rt_uint32_t val); void plic_init(); +void plic_handle_irq(void); #endif diff --git a/libcpu/risc-v/virt64/riscv_io.h b/libcpu/risc-v/virt64/riscv_io.h index b92e24e4c97..f5582fbbf80 100644 --- a/libcpu/risc-v/virt64/riscv_io.h +++ b/libcpu/risc-v/virt64/riscv_io.h @@ -10,7 +10,7 @@ #ifndef __RISCV_IO_H__ #define __RISCV_IO_H__ -static inline uint32_t __raw_hartid(void) +static inline uint32_t __raw_hartid(void) { extern int boot_hartid; return boot_hartid; @@ -18,31 +18,23 @@ static inline uint32_t __raw_hartid(void) static inline void __raw_writeb(rt_uint8_t val, volatile void *addr) { - asm volatile("sb %0, 0(%1)" - : - : "r"(val), "r"(addr)); + asm volatile("sb %0, 0(%1)" : : "r"(val), "r"(addr)); } static inline void __raw_writew(rt_uint16_t val, volatile void *addr) { - asm volatile("sh %0, 0(%1)" - : - : "r"(val), "r"(addr)); + asm volatile("sh %0, 0(%1)" : : "r"(val), "r"(addr)); } static inline void __raw_writel(rt_uint32_t val, volatile void *addr) { - asm volatile("sw %0, 0(%1)" - : - : "r"(val), "r"(addr)); + asm volatile("sw %0, 0(%1)" : : "r"(val), "r"(addr)); } #if __riscv_xlen != 32 static inline void __raw_writeq(rt_uint64_t val, volatile void *addr) { - asm volatile("sd %0, 0(%1)" - : - : "r"(val), "r"(addr)); + asm volatile("sd %0, 0(%1)" : : "r"(val), "r"(addr)); } #endif @@ -50,9 +42,7 @@ static inline rt_uint8_t __raw_readb(const volatile void *addr) { rt_uint8_t val; - asm volatile("lb %0, 0(%1)" - : "=r"(val) - : "r"(addr)); + asm volatile("lb %0, 0(%1)" : "=r"(val) : "r"(addr)); return val; } @@ -60,9 +50,7 @@ static inline rt_uint16_t __raw_readw(const volatile void *addr) { rt_uint16_t val; - asm volatile("lh %0, 0(%1)" - : "=r"(val) - : "r"(addr)); + asm volatile("lh %0, 0(%1)" : "=r"(val) : "r"(addr)); return val; } @@ -70,9 +58,7 @@ static inline rt_uint32_t __raw_readl(const volatile void *addr) { rt_uint32_t val; - asm volatile("lw %0, 0(%1)" - : "=r"(val) - : "r"(addr)); + asm volatile("lw %0, 0(%1)" : "=r"(val) : "r"(addr)); return val; } @@ -81,9 +67,7 @@ static inline rt_uint64_t __raw_readq(const volatile void *addr) { rt_uint64_t val; - asm volatile("ld %0, 0(%1)" - : "=r"(val) - : "r"(addr)); + asm volatile("ld %0, 0(%1)" : "=r"(val) : "r"(addr)); return val; } #endif diff --git a/libcpu/risc-v/virt64/riscv_mmu.c b/libcpu/risc-v/virt64/riscv_mmu.c index 0438641ad7a..9dec1aed718 100644 --- a/libcpu/risc-v/virt64/riscv_mmu.c +++ b/libcpu/risc-v/virt64/riscv_mmu.c @@ -11,10 +11,10 @@ #include #include -#include #include -#include +#include #include +#include #include "riscv_mmu.h" diff --git a/libcpu/risc-v/virt64/riscv_mmu.h b/libcpu/risc-v/virt64/riscv_mmu.h index c2e317c3476..ca399ab91a9 100644 --- a/libcpu/risc-v/virt64/riscv_mmu.h +++ b/libcpu/risc-v/virt64/riscv_mmu.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2006-2021, RT-Thread Development Team + * Copyright (c) 2006-2024, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * @@ -19,22 +19,23 @@ #undef PAGE_SIZE #define PAGE_OFFSET_SHIFT 0 -#define PAGE_OFFSET_BIT 12 -#define PAGE_SIZE __SIZE(PAGE_OFFSET_BIT) -#define PAGE_OFFSET_MASK __MASK(PAGE_OFFSET_BIT) -#define VPN0_SHIFT (PAGE_OFFSET_SHIFT + PAGE_OFFSET_BIT) -#define VPN0_BIT 9 -#define VPN1_SHIFT (VPN0_SHIFT + VPN0_BIT) -#define VPN1_BIT 9 -#define VPN2_SHIFT (VPN1_SHIFT + VPN1_BIT) -#define VPN2_BIT 9 +#define PAGE_OFFSET_BIT 12 +#define PAGE_SIZE __SIZE(PAGE_OFFSET_BIT) +#define PAGE_OFFSET_MASK __MASK(PAGE_OFFSET_BIT) +#define VPN0_SHIFT (PAGE_OFFSET_SHIFT + PAGE_OFFSET_BIT) +#define VPN0_BIT 9 +#define VPN1_SHIFT (VPN0_SHIFT + VPN0_BIT) +#define VPN1_BIT 9 +#define VPN2_SHIFT (VPN1_SHIFT + VPN1_BIT) +#define VPN2_BIT 9 #define PPN0_SHIFT (PAGE_OFFSET_SHIFT + PAGE_OFFSET_BIT) -#define PPN0_BIT 9 +#define PPN0_BIT 9 #define PPN1_SHIFT (PPN0_SHIFT + PPN0_BIT) -#define PPN1_BIT 9 +#define PPN1_BIT 9 #define PPN2_SHIFT (PPN1_SHIFT + PPN1_BIT) -#define PPN2_BIT 26 +#define PPN2_BIT 26 +#define PPN_BITS (PPN0_BIT + PPN1_BIT + PPN2_BIT) #define L1_PAGE_SIZE __SIZE(PAGE_OFFSET_BIT + VPN0_BIT + VPN1_BIT) #define L2_PAGE_SIZE __SIZE(PAGE_OFFSET_BIT + VPN0_BIT) @@ -44,12 +45,12 @@ #define PHYSICAL_ADDRESS_WIDTH_BITS 56 -#define PAGE_ATTR_NEXT_LEVEL (0) -#define PAGE_ATTR_RWX (PTE_X | PTE_W | PTE_R) -#define PAGE_ATTR_READONLY (PTE_R) +#define PAGE_ATTR_NEXT_LEVEL (0) +#define PAGE_ATTR_RWX (PTE_X | PTE_W | PTE_R) +#define PAGE_ATTR_READONLY (PTE_R) #define PAGE_ATTR_READEXECUTE (PTE_X | PTE_R) -#define PAGE_ATTR_USER (PTE_U) +#define PAGE_ATTR_USER (PTE_U) #define PAGE_ATTR_SYSTEM (0) #define PAGE_DEFAULT_ATTR_LEAF (PAGE_ATTR_RWX | PAGE_ATTR_USER | PTE_V | PTE_G) @@ -62,44 +63,49 @@ /** * encoding of SATP (Supervisor Address Translation and Protection register) */ -#define SATP_MODE_OFFSET 60 -#define SATP_MODE_BARE 0 -#define SATP_MODE_SV39 8 -#define SATP_MODE_SV48 9 -#define SATP_MODE_SV57 10 -#define SATP_MODE_SV64 11 - -#define ARCH_VADDR_WIDTH 39 -#define SATP_MODE SATP_MODE_SV39 - -#define MMU_MAP_K_DEVICE (PTE_G | PTE_W | PTE_R | PTE_V) -#define MMU_MAP_K_RWCB (PTE_G | PTE_X | PTE_W | PTE_R | PTE_V) -#define MMU_MAP_K_RW (PTE_G | PTE_X | PTE_W | PTE_R | PTE_V) -#define MMU_MAP_U_RWCB (PTE_U | PTE_X | PTE_W | PTE_R | PTE_V) -#define MMU_MAP_U_RWCB_XN (PTE_U | PTE_W | PTE_R | PTE_V) -#define MMU_MAP_U_RW (PTE_U | PTE_X | PTE_W | PTE_R | PTE_V) - -#define PTE_XWR_MASK 0xe - -#define ARCH_PAGE_SIZE PAGE_SIZE -#define ARCH_PAGE_MASK (ARCH_PAGE_SIZE - 1) -#define ARCH_PAGE_SHIFT PAGE_OFFSET_BIT -#define ARCH_INDEX_WIDTH 9 -#define ARCH_INDEX_SIZE (1ul << ARCH_INDEX_WIDTH) -#define ARCH_INDEX_MASK (ARCH_INDEX_SIZE - 1) - -#define ARCH_MAP_FAILED ((void *)-1) +#define SATP_MODE_OFFSET 60 +#define SATP_MODE_BARE 0 +#define SATP_MODE_SV39 8 +#define SATP_MODE_SV48 9 +#define SATP_MODE_SV57 10 +#define SATP_MODE_SV64 11 + +#define ARCH_VADDR_WIDTH 39 +#define SATP_MODE SATP_MODE_SV39 + +#define MMU_MAP_K_DEVICE (PTE_G | PTE_W | PTE_R | PTE_V) +#define MMU_MAP_K_RWCB (PTE_G | PTE_X | PTE_W | PTE_R | PTE_V) +#define MMU_MAP_K_RW (PTE_G | PTE_X | PTE_W | PTE_R | PTE_V) +#define MMU_MAP_U_RWCB (PTE_U | PTE_X | PTE_W | PTE_R | PTE_V) +#define MMU_MAP_U_RWCB_XN (PTE_U | PTE_W | PTE_R | PTE_V) +#define MMU_MAP_U_RW (PTE_U | PTE_X | PTE_W | PTE_R | PTE_V) +#define MMU_MAP_EARLY (PAGE_ATTR_RWX | PTE_G | PTE_V) + +#define PTE_XWR_MASK 0xe + +#define ARCH_PAGE_SIZE PAGE_SIZE +#define ARCH_PAGE_MASK (ARCH_PAGE_SIZE - 1) +#define ARCH_PAGE_SHIFT PAGE_OFFSET_BIT +#define ARCH_INDEX_WIDTH 9 +#define ARCH_INDEX_SIZE (1ul << ARCH_INDEX_WIDTH) +#define ARCH_INDEX_MASK (ARCH_INDEX_SIZE - 1) + +#define ARCH_MAP_FAILED ((void *)-1) void mmu_set_pagetable(rt_ubase_t addr); void mmu_enable_user_page_access(void); void mmu_disable_user_page_access(void); -#define RT_HW_MMU_PROT_READ 1 -#define RT_HW_MMU_PROT_WRITE 2 +#define RT_HW_MMU_PROT_READ 1 +#define RT_HW_MMU_PROT_WRITE 2 #define RT_HW_MMU_PROT_EXECUTE 4 -#define RT_HW_MMU_PROT_KERNEL 8 -#define RT_HW_MMU_PROT_USER 16 -#define RT_HW_MMU_PROT_CACHE 32 +#define RT_HW_MMU_PROT_KERNEL 8 +#define RT_HW_MMU_PROT_USER 16 +#define RT_HW_MMU_PROT_CACHE 32 + +void rt_hw_asid_init(void); +struct rt_aspace; +void rt_hw_asid_switch_pgtbl(struct rt_aspace *aspace, rt_ubase_t pgtbl); /** * @brief Remove permission from attribution diff --git a/libcpu/risc-v/virt64/rvv_context.h b/libcpu/risc-v/virt64/rvv_context.h new file mode 100644 index 00000000000..a416396b1ea --- /dev/null +++ b/libcpu/risc-v/virt64/rvv_context.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2006-2024, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2022-10-10 RT-Thread the first version, + * compatible to riscv-v-spec-1.0 + */ +#ifndef __RVV_CONTEXT_H__ +#define __RVV_CONTEXT_H__ + +#include "cpuport.h" +#include "encoding.h" + +#if defined(ARCH_VECTOR_VLEN_128) + #define CTX_VECTOR_REGS 64 +#elif defined(ARCH_VECTOR_VLEN_256) + #define CTX_VECTOR_REGS 128 +#else +#error "No supported VLEN" +#endif /* VLEN */ + +#define CTX_VECTOR_REG_NR (CTX_VECTOR_REGS + 4) + +/** + * ================================== + * VECTOR EXTENSION + * ================================== + */ + +#define VEC_FRAME_VSTART (0 * REGBYTES) +#define VEC_FRAME_VTYPE (1 * REGBYTES) +#define VEC_FRAME_VL (2 * REGBYTES) +#define VEC_FRAME_VCSR (3 * REGBYTES) +#define VEC_FRAME_V0 (4 * REGBYTES) + +.macro GET_VEC_FRAME_LEN, xreg + csrr \xreg, vlenb + slli \xreg, \xreg, 5 + addi \xreg, \xreg, 4 * REGBYTES +.endm + +/** + * @brief save vector extension hardware state + * + * @param dst register storing bottom of storage block + * + */ +.macro SAVE_VECTOR, dst + mv t1, \dst + + csrr t0, vstart + STORE t0, VEC_FRAME_VSTART(t1) + csrr t0, vtype + STORE t0, VEC_FRAME_VTYPE(t1) + csrr t0, vl + STORE t0, VEC_FRAME_VL(t1) + csrr t0, vcsr + STORE t0, VEC_FRAME_VCSR(t1) + + addi t1, t1, VEC_FRAME_V0 + + // config vector setting, + // t2 is updated to length of a vector group in bytes + VEC_CONFIG_SETVLI(t2, x0, VEC_IMM_SEW_8, VEC_IMM_LMUL_8) + + vse8.v v0, (t1) + add t1, t1, t2 + vse8.v v8, (t1) + add t1, t1, t2 + vse8.v v16, (t1) + add t1, t1, t2 + vse8.v v24, (t1) +.endm + +/** + * @brief restore vector extension hardware states + * + * @param dst register storing bottom of storage block + * + */ +.macro RESTORE_VECTOR, dst + // restore vector registers first since it will modify vector states + mv t0, \dst + addi t1, t0, VEC_FRAME_V0 + + VEC_CONFIG_SETVLI(t2, x0, VEC_IMM_SEW_8, VEC_IMM_LMUL_8) + + vle8.v v0, (t1) + add t1, t1, t2 + vle8.v v8, (t1) + add t1, t1, t2 + vle8.v v16, (t1) + add t1, t1, t2 + vle8.v v24, (t1) + + mv t1, t0 + + LOAD t0, VEC_FRAME_VSTART(t1) + csrw vstart, t0 + LOAD t0, VEC_FRAME_VCSR(t1) + csrw vcsr, t0 + + LOAD t0, VEC_FRAME_VTYPE(t1) + LOAD t3, VEC_FRAME_VL(t1) + VEC_CONFIG_SET_VL_VTYPE(t3, t0) +.endm + +#endif /* __RVV_CONTEXT_H__ */ diff --git a/libcpu/risc-v/virt64/sbi.h b/libcpu/risc-v/virt64/sbi.h index be54dede72e..45853871a89 100644 --- a/libcpu/risc-v/virt64/sbi.h +++ b/libcpu/risc-v/virt64/sbi.h @@ -141,7 +141,7 @@ struct sbi_ret long value; }; -static __inline struct sbi_ret +rt_inline struct sbi_ret sbi_call(uint64_t arg7, uint64_t arg6, uint64_t arg0, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4) { diff --git a/libcpu/risc-v/virt64/stack.h b/libcpu/risc-v/virt64/stack.h index 4899c6f2c8a..4302b26e1f5 100644 --- a/libcpu/risc-v/virt64/stack.h +++ b/libcpu/risc-v/virt64/stack.h @@ -6,12 +6,23 @@ * Change Logs: * Date Author Notes * 2021-01-30 lizhirui first version + * 2021-11-18 JasonHu add fpu member + * 2022-10-22 Shell Support kernel mode RVV */ #ifndef __STACK_H__ #define __STACK_H__ +#include "stackframe.h" + #include + +typedef struct rt_hw_switch_frame +{ + uint64_t regs[RT_HW_SWITCH_CONTEXT_SIZE]; +} *rt_hw_switch_frame_t; + + struct rt_hw_stack_frame { rt_ubase_t epc; /* epc - epc - program counter */ @@ -47,13 +58,13 @@ struct rt_hw_stack_frame rt_ubase_t t5; /* x30 - t5 - temporary register 5 */ rt_ubase_t t6; /* x31 - t6 - temporary register 6 */ rt_ubase_t user_sp_exc_stack; /* sscratch - user mode sp/exception stack */ - rt_ubase_t __padding; /* align to 16bytes */ -#ifdef ENABLE_FPU + rt_ubase_t __padding; /* align to 16bytes */ +#ifdef ARCH_RISCV_FPU rt_ubase_t f[CTX_FPU_REG_NR]; /* f0~f31 */ -#endif -#ifdef ENABLE_VECTOR +#endif /* ARCH_RISCV_FPU */ +#ifdef ARCH_RISCV_VECTOR rt_ubase_t v[CTX_VECTOR_REG_NR]; -#endif +#endif /* ARCH_RISCV_VECTOR */ }; #endif diff --git a/libcpu/risc-v/virt64/stackframe.h b/libcpu/risc-v/virt64/stackframe.h index 598bb905bfe..dd87b35aa1c 100644 --- a/libcpu/risc-v/virt64/stackframe.h +++ b/libcpu/risc-v/virt64/stackframe.h @@ -8,33 +8,70 @@ * 2021-02-02 lizhirui first version * 2021-02-11 lizhirui fixed gp save/store bug * 2021-11-18 JasonHu add fpu registers save/restore - * 2022/10/22 WangXiaoyao Support kernel mode RVV; + * 2022-10-22 Shell Support kernel mode RVV */ #ifndef __STACKFRAME_H__ #define __STACKFRAME_H__ -#include "cpuport.h" +#include #include "encoding.h" #include "ext_context.h" +/* bytes of register width */ +#ifdef ARCH_CPU_64BIT +#define STORE sd +#define LOAD ld +#define FSTORE fsd +#define FLOAD fld +#define REGBYTES 8 +#else +// error here, not portable +#error "Not supported XLEN" +#endif + +/* 33 general register + 1 padding */ +#define CTX_GENERAL_REG_NR 34 + +/* all context registers */ +#define CTX_REG_NR (CTX_GENERAL_REG_NR + CTX_FPU_REG_NR + CTX_VECTOR_REG_NR) + #define BYTES(idx) ((idx) * REGBYTES) #define FRAME_OFF_SSTATUS BYTES(2) #define FRAME_OFF_SP BYTES(32) #define FRAME_OFF_GP BYTES(3) +/* switch frame */ +#define RT_HW_SWITCH_CONTEXT_SSTATUS 0 +#define RT_HW_SWITCH_CONTEXT_S11 1 +#define RT_HW_SWITCH_CONTEXT_S10 2 +#define RT_HW_SWITCH_CONTEXT_S9 3 +#define RT_HW_SWITCH_CONTEXT_S8 4 +#define RT_HW_SWITCH_CONTEXT_S7 5 +#define RT_HW_SWITCH_CONTEXT_S6 6 +#define RT_HW_SWITCH_CONTEXT_S5 7 +#define RT_HW_SWITCH_CONTEXT_S4 8 +#define RT_HW_SWITCH_CONTEXT_S3 9 +#define RT_HW_SWITCH_CONTEXT_S2 10 +#define RT_HW_SWITCH_CONTEXT_S1 11 +#define RT_HW_SWITCH_CONTEXT_S0 12 +#define RT_HW_SWITCH_CONTEXT_RA 13 +#define RT_HW_SWITCH_CONTEXT_TP 14 +#define RT_HW_SWITCH_CONTEXT_ALIGNMENT 15 // Padding for alignment +#define RT_HW_SWITCH_CONTEXT_SIZE 16 // Total size of the structure + #ifdef __ASSEMBLY__ .macro SAVE_ALL -#ifdef ENABLE_FPU +#ifdef ARCH_RISCV_FPU /* reserve float registers */ addi sp, sp, -CTX_FPU_REG_NR * REGBYTES -#endif /* ENABLE_FPU */ -#ifdef ENABLE_VECTOR +#endif /* ARCH_RISCV_FPU */ +#ifdef ARCH_RISCV_VECTOR /* reserve float registers */ addi sp, sp, -CTX_VECTOR_REG_NR * REGBYTES -#endif /* ENABLE_VECTOR */ +#endif /* ARCH_RISCV_VECTOR */ /* save general registers */ addi sp, sp, -CTX_GENERAL_REG_NR * REGBYTES @@ -78,7 +115,7 @@ csrr t0, sscratch STORE t0, 32 * REGBYTES(sp) -#ifdef ENABLE_FPU +#ifdef ARCH_RISCV_FPU /* backup sp and adjust sp to save float registers */ mv t1, sp addi t1, t1, CTX_GENERAL_REG_NR * REGBYTES @@ -125,9 +162,9 @@ li t0, SSTATUS_FS_CLEAN csrs sstatus, t0 -#endif /* ENABLE_FPU */ +#endif /* ARCH_RISCV_FPU */ -#ifdef ENABLE_VECTOR +#ifdef ARCH_RISCV_VECTOR csrr t0, sstatus andi t0, t0, SSTATUS_VS beqz t0, 0f @@ -137,7 +174,7 @@ SAVE_VECTOR t1 0: -#endif /* ENABLE_VECTOR */ +#endif /* ARCH_RISCV_VECTOR */ .endm /** @@ -146,7 +183,7 @@ */ .macro RESTORE_ALL -#ifdef ENABLE_VECTOR +#ifdef ARCH_RISCV_VECTOR // skip on close ld t0, 2 * REGBYTES(sp) // cannot use vector on initial @@ -158,9 +195,9 @@ RESTORE_VECTOR t1 0: -#endif /* ENABLE_VECTOR */ +#endif /* ARCH_RISCV_VECTOR */ -#ifdef ENABLE_FPU +#ifdef ARCH_RISCV_FPU /* restore float register */ addi t2, sp, CTX_GENERAL_REG_NR * REGBYTES @@ -206,7 +243,7 @@ li t0, SSTATUS_FS_CLEAN csrs sstatus, t0 -#endif /* ENABLE_FPU */ +#endif /* ARCH_RISCV_FPU */ /* restore general register */ addi t0, sp, CTX_REG_NR * REGBYTES diff --git a/libcpu/risc-v/virt64/startup_gcc.S b/libcpu/risc-v/virt64/startup_gcc.S index cfa39f85350..184b48a1aeb 100644 --- a/libcpu/risc-v/virt64/startup_gcc.S +++ b/libcpu/risc-v/virt64/startup_gcc.S @@ -8,6 +8,7 @@ * 2018/10/01 Bernard The first version * 2018/12/27 Jesven Add SMP support * 2020/6/12 Xim Port to QEMU and remove SMP support + * 2024-06-30 Shell Support of kernel remapping */ #include @@ -15,7 +16,7 @@ .data .global boot_hartid /* global varible rt_boot_hartid in .data section */ -boot_hartid: +boot_hartid: .word 0xdeadbeef .global _start diff --git a/libcpu/risc-v/virt64/trap.c b/libcpu/risc-v/virt64/trap.c index 3b0f86fe0e3..abd4bed6bc8 100644 --- a/libcpu/risc-v/virt64/trap.c +++ b/libcpu/risc-v/virt64/trap.c @@ -7,24 +7,23 @@ * Date Author Notes * 2022-12-08 RT-Thread first version */ + #include #include #include #include -#include "mmu.h" -#include "encoding.h" -#include "stack.h" -#include "sbi.h" -#include "riscv.h" -#include "interrupt.h" -#include "plic.h" -#include "tick.h" +#include +#include +#include +#include +#include +#include +#include +#include #ifdef RT_USING_SMART #include -#else -#define rt_hw_backtrace(...) (0) #endif #define DBG_TAG "libcpu.trap" @@ -36,7 +35,8 @@ void dump_regs(struct rt_hw_stack_frame *regs) rt_kprintf("--------------Dump Registers-----------------\n"); rt_kprintf("Function Registers:\n"); - rt_kprintf("\tra(x1) = 0x%p\tuser_sp = 0x%p\n", regs->ra, regs->user_sp_exc_stack); + rt_kprintf("\tra(x1) = 0x%p\tuser_sp = 0x%p\n", regs->ra, + regs->user_sp_exc_stack); rt_kprintf("\tgp(x3) = 0x%p\ttp(x4) = 0x%p\n", regs->gp, regs->tp); rt_kprintf("Temporary Registers:\n"); rt_kprintf("\tt0(x5) = 0x%p\tt1(x6) = 0x%p\n", regs->t0, regs->t1); @@ -56,15 +56,27 @@ void dump_regs(struct rt_hw_stack_frame *regs) rt_kprintf("\ta4(x14) = 0x%p\ta5(x15) = 0x%p\n", regs->a4, regs->a5); rt_kprintf("\ta6(x16) = 0x%p\ta7(x17) = 0x%p\n", regs->a6, regs->a7); rt_kprintf("sstatus = 0x%p\n", regs->sstatus); - rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SIE) ? "Supervisor Interrupt Enabled" : "Supervisor Interrupt Disabled"); - rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SPIE) ? "Last Time Supervisor Interrupt Enabled" : "Last Time Supervisor Interrupt Disabled"); - rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SPP) ? "Last Privilege is Supervisor Mode" : "Last Privilege is User Mode"); - rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SUM) ? "Permit to Access User Page" : "Not Permit to Access User Page"); - rt_kprintf("\t%s\n", (regs->sstatus & (1 << 19)) ? "Permit to Read Executable-only Page" : "Not Permit to Read Executable-only Page"); + rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SIE) + ? "Supervisor Interrupt Enabled" + : "Supervisor Interrupt Disabled"); + rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SPIE) + ? "Last Time Supervisor Interrupt Enabled" + : "Last Time Supervisor Interrupt Disabled"); + rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SPP) + ? "Last Privilege is Supervisor Mode" + : "Last Privilege is User Mode"); + rt_kprintf("\t%s\n", (regs->sstatus & SSTATUS_SUM) + ? "Permit to Access User Page" + : "Not Permit to Access User Page"); + rt_kprintf("\t%s\n", (regs->sstatus & (1 << 19)) + ? "Permit to Read Executable-only Page" + : "Not Permit to Read Executable-only Page"); rt_ubase_t satp_v = read_csr(satp); rt_kprintf("satp = 0x%p\n", satp_v); - rt_kprintf("\tCurrent Page Table(Physical) = 0x%p\n", __MASKVALUE(satp_v, __MASK(44)) << PAGE_OFFSET_BIT); - rt_kprintf("\tCurrent ASID = 0x%p\n", __MASKVALUE(satp_v >> 44, __MASK(16)) << PAGE_OFFSET_BIT); + rt_kprintf("\tCurrent Page Table(Physical) = 0x%p\n", + __MASKVALUE(satp_v, __MASK(44)) << PAGE_OFFSET_BIT); + rt_kprintf("\tCurrent ASID = 0x%p\n", __MASKVALUE(satp_v >> 44, __MASK(16)) + << PAGE_OFFSET_BIT); const char *mode_str = "Unknown Address Translation/Protection Mode"; switch (__MASKVALUE(satp_v >> 60, __MASK(4))) @@ -86,52 +98,44 @@ void dump_regs(struct rt_hw_stack_frame *regs) rt_kprintf("-----------------Dump OK---------------------\n"); } -static const char *Exception_Name[] = - { - "Instruction Address Misaligned", - "Instruction Access Fault", - "Illegal Instruction", - "Breakpoint", - "Load Address Misaligned", - "Load Access Fault", - "Store/AMO Address Misaligned", - "Store/AMO Access Fault", - "Environment call from U-mode", - "Environment call from S-mode", - "Reserved-10", - "Reserved-11", - "Instruction Page Fault", - "Load Page Fault", - "Reserved-14", - "Store/AMO Page Fault"}; - -static const char *Interrupt_Name[] = - { - "User Software Interrupt", - "Supervisor Software Interrupt", - "Reversed-2", - "Reversed-3", - "User Timer Interrupt", - "Supervisor Timer Interrupt", - "Reversed-6", - "Reversed-7", - "User External Interrupt", - "Supervisor External Interrupt", - "Reserved-10", - "Reserved-11", +static const char *Exception_Name[] = {"Instruction Address Misaligned", + "Instruction Access Fault", + "Illegal Instruction", + "Breakpoint", + "Load Address Misaligned", + "Load Access Fault", + "Store/AMO Address Misaligned", + "Store/AMO Access Fault", + "Environment call from U-mode", + "Environment call from S-mode", + "Reserved-10", + "Reserved-11", + "Instruction Page Fault", + "Load Page Fault", + "Reserved-14", + "Store/AMO Page Fault"}; + +static const char *Interrupt_Name[] = { + "User Software Interrupt", + "Supervisor Software Interrupt", + "Reversed-2", + "Reversed-3", + "User Timer Interrupt", + "Supervisor Timer Interrupt", + "Reversed-6", + "Reversed-7", + "User External Interrupt", + "Supervisor External Interrupt", + "Reserved-10", + "Reserved-11", }; -extern struct rt_irq_desc irq_desc[]; - #ifndef RT_USING_SMP static volatile int nested = 0; -#define ENTER_TRAP \ - nested += 1 -#define EXIT_TRAP \ - nested -= 1 +#define ENTER_TRAP nested += 1 +#define EXIT_TRAP nested -= 1 #define CHECK_NESTED_PANIC(cause, tval, epc, eframe) \ - if (nested != 1) \ - handle_nested_trap_panic(cause, tval, epc, eframe) + if (nested != 1) handle_nested_trap_panic(cause, tval, epc, eframe) #endif /* RT_USING_SMP */ static const char *get_exception_msg(int id) @@ -150,7 +154,8 @@ static const char *get_exception_msg(int id) #ifdef RT_USING_SMART #include "lwp.h" -void handle_user(rt_ubase_t scause, rt_ubase_t stval, rt_ubase_t sepc, struct rt_hw_stack_frame *sp) +void handle_user(rt_ubase_t scause, rt_ubase_t stval, rt_ubase_t sepc, + struct rt_hw_stack_frame *sp) { rt_ubase_t id = __MASKVALUE(scause, __MASK(63UL)); struct rt_lwp *lwp; @@ -210,23 +215,20 @@ void handle_user(rt_ubase_t scause, rt_ubase_t stval, rt_ubase_t sepc, struct rt .fault_vaddr = (void *)stval, }; - __asm__ volatile ("csrrsi %0, sstatus, 2":"=r"(saved_stat)); + __asm__ volatile("csrrsi %0, sstatus, 2" : "=r"(saved_stat)); if (lwp && rt_aspace_fault_try_fix(lwp->aspace, &msg)) { - __asm__ volatile ("csrw sstatus, %0"::"r"(saved_stat)); + __asm__ volatile("csrw sstatus, %0" ::"r"(saved_stat)); return; } - __asm__ volatile ("csrw sstatus, %0"::"r"(saved_stat)); + __asm__ volatile("csrw sstatus, %0" ::"r"(saved_stat)); } LOG_E("[FATAL ERROR] Exception %ld:%s\n", id, get_exception_msg(id)); LOG_E("scause:0x%p,stval:0x%p,sepc:0x%p\n", scause, stval, sepc); dump_regs(sp); rt_thread_t cur_thr = rt_thread_self(); - struct rt_hw_backtrace_frame frame = { - .fp = sp->s0_fp, - .pc = sepc - }; + struct rt_hw_backtrace_frame frame = {.fp = sp->s0_fp, .pc = sepc}; rt_kprintf("fp = %p\n", frame.fp); lwp_backtrace_frame(cur_thr, &frame); @@ -237,7 +239,7 @@ void handle_user(rt_ubase_t scause, rt_ubase_t stval, rt_ubase_t sepc, struct rt } #endif -#ifdef ENABLE_VECTOR +#ifdef ARCH_RISCV_VECTOR static void vector_enable(struct rt_hw_stack_frame *sp) { sp->sstatus |= SSTATUS_VS_INITIAL; @@ -246,7 +248,8 @@ static void vector_enable(struct rt_hw_stack_frame *sp) /** * detect V/D support, and do not distinguish V/D instruction */ -static int illegal_inst_recoverable(rt_ubase_t stval, struct rt_hw_stack_frame *sp) +static int illegal_inst_recoverable(rt_ubase_t stval, + struct rt_hw_stack_frame *sp) { // first 7 bits is opcode int opcode = stval & 0x7f; @@ -257,12 +260,12 @@ static int illegal_inst_recoverable(rt_ubase_t stval, struct rt_hw_stack_frame * switch (opcode) { - case 0x57: // V - case 0x27: // scalar FLOAT - case 0x07: - case 0x73: // CSR - flag = 1; - break; + case 0x57: // V + case 0x27: // scalar FLOAT + case 0x07: + case 0x73: // CSR + flag = 1; + break; } if (flag) @@ -274,11 +277,9 @@ static int illegal_inst_recoverable(rt_ubase_t stval, struct rt_hw_stack_frame * } #endif -static void handle_nested_trap_panic( - rt_ubase_t cause, - rt_ubase_t tval, - rt_ubase_t epc, - struct rt_hw_stack_frame *eframe) +static void handle_nested_trap_panic(rt_ubase_t cause, rt_ubase_t tval, + rt_ubase_t epc, + struct rt_hw_stack_frame *eframe) { LOG_E("\n-------- [SEVER ERROR] --------"); LOG_E("Nested trap detected"); @@ -288,38 +289,34 @@ static void handle_nested_trap_panic( } #define IN_USER_SPACE (stval >= USER_VADDR_START && stval < USER_VADDR_TOP) -#define PAGE_FAULT (id == EP_LOAD_PAGE_FAULT || id == EP_STORE_PAGE_FAULT) +#define PAGE_FAULT (id == EP_LOAD_PAGE_FAULT || id == EP_STORE_PAGE_FAULT) /* Trap entry */ -void handle_trap(rt_ubase_t scause, rt_ubase_t stval, rt_ubase_t sepc, struct rt_hw_stack_frame *sp) +void handle_trap(rt_ubase_t scause, rt_ubase_t stval, rt_ubase_t sepc, + struct rt_hw_stack_frame *sp) { ENTER_TRAP; rt_ubase_t id = __MASKVALUE(scause, __MASK(63UL)); const char *msg; /* supervisor external interrupt */ - if (scause == (uint64_t)(0x8000000000000005)) + if ((SCAUSE_INTERRUPT & scause) && + SCAUSE_S_EXTERNAL_INTR == (scause & 0xff)) { rt_interrupt_enter(); - tick_isr(); + plic_handle_irq(); rt_interrupt_leave(); } - else if (scause == (uint64_t)(0x8000000000000009)) + else if ((SCAUSE_INTERRUPT | SCAUSE_S_TIMER_INTR) == scause) { + /* supervisor timer */ rt_interrupt_enter(); - int plic_irq = plic_claim(); - plic_complete(plic_irq); - irq_desc[plic_irq].handler(plic_irq, irq_desc[plic_irq].param); + tick_isr(); rt_interrupt_leave(); } else { - // trap cannot nested when handling another trap / interrupt - CHECK_NESTED_PANIC(scause, stval, sepc, sp); - rt_ubase_t id = __MASKVALUE(scause, __MASK(63UL)); - const char *msg; - - if (scause >> 63) + if (SCAUSE_INTERRUPT & scause) { if (id < sizeof(Interrupt_Name) / sizeof(const char *)) { @@ -330,17 +327,18 @@ void handle_trap(rt_ubase_t scause, rt_ubase_t stval, rt_ubase_t sepc, struct rt msg = "Unknown Interrupt"; } - rt_kprintf("Unhandled Interrupt %ld:%s\n", id, msg); + LOG_E("Unhandled Interrupt %ld:%s\n", id, msg); } else { -#ifdef ENABLE_VECTOR +#ifdef ARCH_RISCV_VECTOR if (scause == 0x2) { - if (!(sp->sstatus & SSTATUS_VS) && illegal_inst_recoverable(stval, sp)) + if (!(sp->sstatus & SSTATUS_VS) && + illegal_inst_recoverable(stval, sp)) goto _exit; } -#endif /* ENABLE_VECTOR */ +#endif /* ARCH_RISCV_VECTOR */ #ifdef RT_USING_SMART if (!(sp->sstatus & 0x100) || (PAGE_FAULT && IN_USER_SPACE)) { @@ -351,26 +349,38 @@ void handle_trap(rt_ubase_t scause, rt_ubase_t stval, rt_ubase_t sepc, struct rt #endif // handle kernel exception: - rt_kprintf("Unhandled Exception %ld:%s\n", id, get_exception_msg(id)); + rt_kprintf("Unhandled Exception %ld:%s\n", id, + get_exception_msg(id)); } + // trap cannot nested when handling another trap / interrupt + CHECK_NESTED_PANIC(scause, stval, sepc, sp); + rt_kprintf("scause:0x%p,stval:0x%p,sepc:0x%p\n", scause, stval, sepc); dump_regs(sp); + + rt_thread_t cur_thr = rt_thread_self(); rt_kprintf("--------------Thread list--------------\n"); - rt_kprintf("current thread: %s\n", rt_thread_self()->parent.name); + rt_kprintf("current thread: %s\n", cur_thr->parent.name); - extern struct rt_thread *rt_current_thread; rt_kprintf("--------------Backtrace--------------\n"); - struct rt_hw_backtrace_frame frame = { - .fp = sp->s0_fp, - .pc = sepc - }; - rt_kprintf("fp = %p", frame.fp); - rt_backtrace_frame(rt_thread_self(), &frame); + struct rt_hw_backtrace_frame frame = {.fp = sp->s0_fp, .pc = sepc}; + +#ifdef RT_USING_SMART + if (!(sp->sstatus & 0x100)) + { + lwp_backtrace_frame(cur_thr, &frame); + } + else +#endif + { + rt_backtrace_frame(cur_thr, &frame); + } - RT_ASSERT(0); + while (1) + ; } _exit: EXIT_TRAP; - return ; + return; } diff --git a/libcpu/risc-v/virt64/vector_encoding.h b/libcpu/risc-v/virt64/vector_encoding.h index 5445843f064..26c2c86f4c8 100644 --- a/libcpu/risc-v/virt64/vector_encoding.h +++ b/libcpu/risc-v/virt64/vector_encoding.h @@ -12,8 +12,15 @@ #ifndef __VECTOR_ENCODING_H__ #define __VECTOR_ENCODING_H__ +/* mstatus/sstatus */ +#define MSTATUS_VS 0x00000600 +#define SSTATUS_VS 0x00000600 /* Vector Status */ +#define SSTATUS_VS_INITIAL 0x00000200 +#define SSTATUS_VS_CLEAN 0x00000400 +#define SSTATUS_VS_DIRTY 0x00000600 + /** - * assembler names used for vset{i}vli vtypei immediate + * assembler names used for vset{i}vli vtypei immediate */ #define VEC_IMM_SEW_8 e8 diff --git a/libcpu/risc-v/virt64/vector_gcc.S b/libcpu/risc-v/virt64/vector_gcc.S new file mode 100644 index 00000000000..5c9047ec2e4 --- /dev/null +++ b/libcpu/risc-v/virt64/vector_gcc.S @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2006-2024, RT-Thread Development Team + * + * SPDX-License-Identifier: Apache-2.0 + * + * Change Logs: + * Date Author Notes + * 2018/10/28 Bernard The unify RISC-V porting implementation + * 2018/12/27 Jesven Add SMP support + * 2021/02/02 lizhirui Add userspace support + * 2022/10/22 Shell Support User mode RVV; + * Trimming process switch context + * 2024/09/01 Shell Separated vector ctx from the generic + */ + +#include "cpuport.h" +#include "stackframe.h" + +/** + * @param a0 pointer to frame bottom + */ +.global rt_hw_vector_ctx_save +rt_hw_vector_ctx_save: + SAVE_VECTOR a0 + ret + +/** + * @param a0 pointer to frame bottom + */ +.global rt_hw_vector_ctx_restore +rt_hw_vector_ctx_restore: + RESTORE_VECTOR a0 + ret + +.global rt_hw_disable_vector +rt_hw_disable_vector: + li t0, SSTATUS_VS + csrc sstatus, t0 + ret + +.global rt_hw_enable_vector +rt_hw_enable_vector: + li t0, SSTATUS_VS + csrs sstatus, t0 + ret