mirror of
https://github.com/torvalds/linux.git
synced 2026-04-27 02:52:27 -04:00
7e340f4fad
Alexandre Ghiti <alexghiti@rivosinc.com> says: In RISC-V, after a new mapping is established, a sfence.vma needs to be emitted for different reasons: - if the uarch caches invalid entries, we need to invalidate it otherwise we would trap on this invalid entry, - if the uarch does not cache invalid entries, a reordered access could fail to see the new mapping and then trap (sfence.vma acts as a fence). We can actually avoid emitting those (mostly) useless and costly sfence.vma by handling the traps instead: - for new kernel mappings: only vmalloc mappings need to be taken care of, other new mapping are rare and already emit the required sfence.vma if needed. That must be achieved very early in the exception path as explained in patch 3, and this also fixes our fragile way of dealing with vmalloc faults. - for new user mappings: Svvptc makes update_mmu_cache() a no-op but we can take some gratuitous page faults (which are very unlikely though). Patch 1 and 2 introduce Svvptc extension probing. On our uarch that does not cache invalid entries and a 6.5 kernel, the gains are measurable: * Kernel boot: 6% * ltp - mmapstress01: 8% * lmbench - lat_pagefault: 20% * lmbench - lat_mmap: 5% Here are the corresponding numbers of sfence.vma emitted: * Ubuntu boot to login: Before: ~630k sfence.vma After: ~200k sfence.vma * ltp - mmapstress01 Before: ~45k After: ~6.3k * lmbench - lat_pagefault Before: ~665k After: 832 (!) * lmbench - lat_mmap Before: ~546k After: 718 (!) Thanks to Ved and Matt Evans for triggering the discussion that led to this patchset! * b4-shazam-merge: riscv: Stop emitting preventive sfence.vma for new userspace mappings with Svvptc riscv: Stop emitting preventive sfence.vma for new vmalloc mappings dt-bindings: riscv: Add Svvptc ISA extension description riscv: Add ISA extension parsing for Svvptc Link: https://lore.kernel.org/r/20240717060125.139416-1-alexghiti@rivosinc.com Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
123 lines
3.7 KiB
C
123 lines
3.7 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Copyright (C) 2009 Chen Liqin <liqin.chen@sunplusct.com>
|
|
* Copyright (C) 2012 Regents of the University of California
|
|
* Copyright (C) 2017 SiFive
|
|
*/
|
|
|
|
#ifndef _ASM_RISCV_THREAD_INFO_H
|
|
#define _ASM_RISCV_THREAD_INFO_H
|
|
|
|
#include <asm/page.h>
|
|
#include <linux/const.h>
|
|
#include <linux/sizes.h>
|
|
|
|
/* thread information allocation */
|
|
#define THREAD_SIZE_ORDER CONFIG_THREAD_SIZE_ORDER
|
|
#define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER)
|
|
|
|
/*
|
|
* By aligning VMAP'd stacks to 2 * THREAD_SIZE, we can detect overflow by
|
|
* checking sp & (1 << THREAD_SHIFT), which we can do cheaply in the entry
|
|
* assembly.
|
|
*/
|
|
#ifdef CONFIG_VMAP_STACK
|
|
#define THREAD_ALIGN (2 * THREAD_SIZE)
|
|
#else
|
|
#define THREAD_ALIGN THREAD_SIZE
|
|
#endif
|
|
|
|
#define THREAD_SHIFT (PAGE_SHIFT + THREAD_SIZE_ORDER)
|
|
#define OVERFLOW_STACK_SIZE SZ_4K
|
|
|
|
#define IRQ_STACK_SIZE THREAD_SIZE
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
#include <asm/processor.h>
|
|
#include <asm/csr.h>
|
|
|
|
/*
|
|
* low level task data that entry.S needs immediate access to
|
|
* - this struct should fit entirely inside of one cache line
|
|
* - if the members of this struct changes, the assembly constants
|
|
* in asm-offsets.c must be updated accordingly
|
|
* - thread_info is included in task_struct at an offset of 0. This means that
|
|
* tp points to both thread_info and task_struct.
|
|
*/
|
|
struct thread_info {
|
|
unsigned long flags; /* low level flags */
|
|
int preempt_count; /* 0=>preemptible, <0=>BUG */
|
|
/*
|
|
* These stack pointers are overwritten on every system call or
|
|
* exception. SP is also saved to the stack it can be recovered when
|
|
* overwritten.
|
|
*/
|
|
long kernel_sp; /* Kernel stack pointer */
|
|
long user_sp; /* User stack pointer */
|
|
int cpu;
|
|
unsigned long syscall_work; /* SYSCALL_WORK_ flags */
|
|
#ifdef CONFIG_SHADOW_CALL_STACK
|
|
void *scs_base;
|
|
void *scs_sp;
|
|
#endif
|
|
#ifdef CONFIG_64BIT
|
|
/*
|
|
* Used in handle_exception() to save a0, a1 and a2 before knowing if we
|
|
* can access the kernel stack.
|
|
*/
|
|
unsigned long a0, a1, a2;
|
|
#endif
|
|
};
|
|
|
|
#ifdef CONFIG_SHADOW_CALL_STACK
|
|
#define INIT_SCS \
|
|
.scs_base = init_shadow_call_stack, \
|
|
.scs_sp = init_shadow_call_stack,
|
|
#else
|
|
#define INIT_SCS
|
|
#endif
|
|
|
|
/*
|
|
* macros/functions for gaining access to the thread information structure
|
|
*
|
|
* preempt_count needs to be 1 initially, until the scheduler is functional.
|
|
*/
|
|
#define INIT_THREAD_INFO(tsk) \
|
|
{ \
|
|
.flags = 0, \
|
|
.preempt_count = INIT_PREEMPT_COUNT, \
|
|
INIT_SCS \
|
|
}
|
|
|
|
void arch_release_task_struct(struct task_struct *tsk);
|
|
int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
|
|
|
|
#endif /* !__ASSEMBLY__ */
|
|
|
|
/*
|
|
* thread information flags
|
|
* - these are process state flags that various assembly files may need to
|
|
* access
|
|
* - pending work-to-be-done flags are in lowest half-word
|
|
* - other flags in upper half-word(s)
|
|
*/
|
|
#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */
|
|
#define TIF_SIGPENDING 2 /* signal pending */
|
|
#define TIF_NEED_RESCHED 3 /* rescheduling necessary */
|
|
#define TIF_RESTORE_SIGMASK 4 /* restore signal mask in do_signal() */
|
|
#define TIF_MEMDIE 5 /* is terminating due to OOM killer */
|
|
#define TIF_NOTIFY_SIGNAL 9 /* signal notifications exist */
|
|
#define TIF_UPROBE 10 /* uprobe breakpoint or singlestep */
|
|
#define TIF_32BIT 11 /* compat-mode 32bit process */
|
|
#define TIF_RISCV_V_DEFER_RESTORE 12 /* restore Vector before returing to user */
|
|
|
|
#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
|
|
#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
|
|
#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
|
|
#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL)
|
|
#define _TIF_UPROBE (1 << TIF_UPROBE)
|
|
#define _TIF_RISCV_V_DEFER_RESTORE (1 << TIF_RISCV_V_DEFER_RESTORE)
|
|
|
|
#endif /* _ASM_RISCV_THREAD_INFO_H */
|