Skip to content

Commit

Permalink
riscv: use vDSO common flow to reduce the latency of the time-related…
Browse files Browse the repository at this point in the history
… functions

Even if RISC-V has supported the vDSO feature, the latency of the functions
for obtaining the system time is still expensive. It is because these
functions still trigger a corresponding system call in the process, which
slows down the response time. If we want to remove the system call to
reduce the latency, the kernel should have the ability to output the system
clock information to userspace. This patch introduces the vDSO common flow
to enable the kernel to achieve the above feature and uses "rdtime"
instruction to obtain the current time in the user space. Under this
condition, the latency cost by the ecall from U-mode to S-mode can be
eliminated. After applying this patch, the latency of gettimeofday()
measured on the HiFive unleashed board can be reduced by %61.

Signed-off-by: Vincent Chen <[email protected]>
Reviewed-by: Atish Patra <[email protected]>
Signed-off-by: Palmer Dabbelt <[email protected]>
  • Loading branch information
VincentZWC authored and palmer-dabbelt committed Jun 11, 2020
1 parent 05589dd commit ad5d112
Show file tree
Hide file tree
Showing 16 changed files with 199 additions and 69 deletions.
4 changes: 4 additions & 0 deletions arch/riscv/Kconfig
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ config 32BIT

config RISCV
def_bool y
select ARCH_CLOCKSOURCE_INIT
select ARCH_HAS_BINFMT_FLAT
select ARCH_HAS_DEBUG_VIRTUAL if MMU
select ARCH_HAS_DEBUG_WX
Expand All @@ -31,6 +32,7 @@ config RISCV
select GENERIC_ARCH_TOPOLOGY if SMP
select GENERIC_ATOMIC64 if !64BIT
select GENERIC_CLOCKEVENTS
select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO
select GENERIC_IOREMAP
select GENERIC_IRQ_MULTI_HANDLER
select GENERIC_IRQ_SHOW
Expand All @@ -40,6 +42,7 @@ config RISCV
select GENERIC_SMP_IDLE_THREAD
select GENERIC_STRNCPY_FROM_USER if MMU
select GENERIC_STRNLEN_USER if MMU
select GENERIC_TIME_VSYSCALL if MMU && 64BIT
select HANDLE_DOMAIN_IRQ
select HAVE_ARCH_AUDITSYSCALL
select HAVE_ARCH_KASAN if MMU && 64BIT
Expand All @@ -53,6 +56,7 @@ config RISCV
select HAVE_DMA_CONTIGUOUS if MMU
select HAVE_EBPF_JIT if MMU
select HAVE_FUTEX_CMPXCHG if FUTEX
select HAVE_GENERIC_VDSO if MMU && 64BIT
select HAVE_PCI
select HAVE_PERF_EVENTS
select HAVE_PERF_REGS
Expand Down
7 changes: 7 additions & 0 deletions arch/riscv/include/asm/clocksource.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_CLOCKSOURCE_H
#define _ASM_CLOCKSOURCE_H

#include <asm/vdso/clocksource.h>

#endif
12 changes: 2 additions & 10 deletions arch/riscv/include/asm/processor.h
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@

#include <linux/const.h>

#include <vdso/processor.h>

#include <asm/ptrace.h>

/*
Expand Down Expand Up @@ -58,16 +60,6 @@ static inline void release_thread(struct task_struct *dead_task)
extern unsigned long get_wchan(struct task_struct *p);


static inline void cpu_relax(void)
{
#ifdef __riscv_muldiv
int dummy;
/* In lieu of a halt instruction, induce a long-latency stall. */
__asm__ __volatile__ ("div %0, %0, zero" : "=r" (dummy));
#endif
barrier();
}

static inline void wait_for_interrupt(void)
{
__asm__ __volatile__ ("wfi");
Expand Down
2 changes: 2 additions & 0 deletions arch/riscv/include/asm/vdso.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@

#include <linux/types.h>

#ifndef GENERIC_TIME_VSYSCALL
struct vdso_data {
};
#endif

/*
* The VDSO symbols are mapped into Linux so we can just use regular symbol
Expand Down
8 changes: 8 additions & 0 deletions arch/riscv/include/asm/vdso/clocksource.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_VDSOCLOCKSOURCE_H
#define __ASM_VDSOCLOCKSOURCE_H

#define VDSO_ARCH_CLOCKMODES \
VDSO_CLOCKMODE_ARCHTIMER

#endif
79 changes: 79 additions & 0 deletions arch/riscv/include/asm/vdso/gettimeofday.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_VDSO_GETTIMEOFDAY_H
#define __ASM_VDSO_GETTIMEOFDAY_H

#ifndef __ASSEMBLY__

#include <asm/unistd.h>
#include <asm/csr.h>
#include <uapi/linux/time.h>

#define VDSO_HAS_CLOCK_GETRES 1

static __always_inline
int gettimeofday_fallback(struct __kernel_old_timeval *_tv,
struct timezone *_tz)
{
register struct __kernel_old_timeval *tv asm("a0") = _tv;
register struct timezone *tz asm("a1") = _tz;
register long ret asm("a0");
register long nr asm("a7") = __NR_gettimeofday;

asm volatile ("ecall\n"
: "=r" (ret)
: "r"(tv), "r"(tz), "r"(nr)
: "memory");

return ret;
}

static __always_inline
long clock_gettime_fallback(clockid_t _clkid, struct __kernel_timespec *_ts)
{
register clockid_t clkid asm("a0") = _clkid;
register struct __kernel_timespec *ts asm("a1") = _ts;
register long ret asm("a0");
register long nr asm("a7") = __NR_clock_gettime;

asm volatile ("ecall\n"
: "=r" (ret)
: "r"(clkid), "r"(ts), "r"(nr)
: "memory");

return ret;
}

static __always_inline
int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts)
{
register clockid_t clkid asm("a0") = _clkid;
register struct __kernel_timespec *ts asm("a1") = _ts;
register long ret asm("a0");
register long nr asm("a7") = __NR_clock_getres;

asm volatile ("ecall\n"
: "=r" (ret)
: "r"(clkid), "r"(ts), "r"(nr)
: "memory");

return ret;
}

static __always_inline u64 __arch_get_hw_counter(s32 clock_mode)
{
/*
* The purpose of csr_read(CSR_TIME) is to trap the system into
* M-mode to obtain the value of CSR_TIME. Hence, unlike other
* architecture, no fence instructions surround the csr_read()
*/
return csr_read(CSR_TIME);
}

static __always_inline const struct vdso_data *__arch_get_vdso_data(void)
{
return _vdso_data;
}

#endif /* !__ASSEMBLY__ */

#endif /* __ASM_VDSO_GETTIMEOFDAY_H */
19 changes: 19 additions & 0 deletions arch/riscv/include/asm/vdso/processor.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef __ASM_VDSO_PROCESSOR_H
#define __ASM_VDSO_PROCESSOR_H

#ifndef __ASSEMBLY__

static inline void cpu_relax(void)
{
#ifdef __riscv_muldiv
int dummy;
/* In lieu of a halt instruction, induce a long-latency stall. */
__asm__ __volatile__ ("div %0, %0, zero" : "=r" (dummy));
#endif
barrier();
}

#endif /* __ASSEMBLY__ */

#endif /* __ASM_VDSO_PROCESSOR_H */
27 changes: 27 additions & 0 deletions arch/riscv/include/asm/vdso/vsyscall.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_VDSO_VSYSCALL_H
#define __ASM_VDSO_VSYSCALL_H

#ifndef __ASSEMBLY__

#include <linux/timekeeper_internal.h>
#include <vdso/datapage.h>

extern struct vdso_data *vdso_data;

/*
* Update the vDSO data page to keep in sync with kernel timekeeping.
*/
static __always_inline struct vdso_data *__riscv_get_k_vdso_data(void)
{
return vdso_data;
}

#define __arch_get_k_vdso_data __riscv_get_k_vdso_data

/* The asm-generic header needs to be included after the definitions above */
#include <asm-generic/vdso/vsyscall.h>

#endif /* !__ASSEMBLY__ */

#endif /* __ASM_VDSO_VSYSCALL_H */
9 changes: 9 additions & 0 deletions arch/riscv/kernel/time.c
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,12 @@ void __init time_init(void)
lpj_fine = riscv_timebase / HZ;
timer_probe();
}

void clocksource_arch_init(struct clocksource *cs)
{
#ifdef CONFIG_GENERIC_GETTIMEOFDAY
cs->vdso_clock_mode = VDSO_CLOCKMODE_ARCHTIMER;
#else
cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE;
#endif
}
8 changes: 6 additions & 2 deletions arch/riscv/kernel/vdso.c
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,12 @@
#include <linux/slab.h>
#include <linux/binfmts.h>
#include <linux/err.h>

#include <asm/page.h>
#ifdef GENERIC_TIME_VSYSCALL
#include <vdso/datapage.h>
#else
#include <asm/vdso.h>
#endif

extern char vdso_start[], vdso_end[];

Expand All @@ -26,7 +30,7 @@ static union {
struct vdso_data data;
u8 page[PAGE_SIZE];
} vdso_data_store __page_aligned_data;
static struct vdso_data *vdso_data = &vdso_data_store.data;
struct vdso_data *vdso_data = &vdso_data_store.data;

static int __init vdso_init(void)
{
Expand Down
12 changes: 9 additions & 3 deletions arch/riscv/kernel/vdso/Makefile
Original file line number Diff line number Diff line change
@@ -1,19 +1,25 @@
# SPDX-License-Identifier: GPL-2.0-only
# Copied from arch/tile/kernel/vdso/Makefile

# Absolute relocation type $(ARCH_REL_TYPE_ABS) needs to be defined before
# the inclusion of generic Makefile.
ARCH_REL_TYPE_ABS := R_RISCV_32|R_RISCV_64|R_RISCV_JUMP_SLOT
include $(srctree)/lib/vdso/Makefile
# Symbols present in the vdso
vdso-syms = rt_sigreturn
ifdef CONFIG_64BIT
vdso-syms += gettimeofday
vdso-syms += clock_gettime
vdso-syms += clock_getres
vdso-syms += vgettimeofday
endif
vdso-syms += getcpu
vdso-syms += flush_icache

# Files to link into the vdso
obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o

ifneq ($(c-gettimeofday-y),)
CFLAGS_vgettimeofday.o += -include $(c-gettimeofday-y)
endif

# Build rules
targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds vdso-dummy.o
obj-vdso := $(addprefix $(obj)/, $(obj-vdso))
Expand Down
18 changes: 0 additions & 18 deletions arch/riscv/kernel/vdso/clock_getres.S

This file was deleted.

18 changes: 0 additions & 18 deletions arch/riscv/kernel/vdso/clock_gettime.S

This file was deleted.

18 changes: 0 additions & 18 deletions arch/riscv/kernel/vdso/gettimeofday.S

This file was deleted.

2 changes: 2 additions & 0 deletions arch/riscv/kernel/vdso/vdso.lds.S
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@
/*
* Copyright (C) 2012 Regents of the University of California
*/
#include <asm/page.h>

OUTPUT_ARCH(riscv)

SECTIONS
{
PROVIDE(_vdso_data = . + PAGE_SIZE);
. = SIZEOF_HEADERS;

.hash : { *(.hash) } :text
Expand Down
25 changes: 25 additions & 0 deletions arch/riscv/kernel/vdso/vgettimeofday.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copied from arch/arm64/kernel/vdso/vgettimeofday.c
*
* Copyright (C) 2018 ARM Ltd.
* Copyright (C) 2020 SiFive
*/

#include <linux/time.h>
#include <linux/types.h>

int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
{
return __cvdso_clock_gettime(clock, ts);
}

int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
{
return __cvdso_gettimeofday(tv, tz);
}

int __vdso_clock_getres(clockid_t clock_id, struct __kernel_timespec *res)
{
return __cvdso_clock_getres(clock_id, res);
}

0 comments on commit ad5d112

Please sign in to comment.