mirror of
https://github.com/torvalds/linux.git
synced 2026-04-18 06:44:00 -04:00
98cc091262
Do not use memcpy() to extract syscall arguments from struct pt_regs
but rather just perform direct assignments.
Update syscall_set_arguments() too to keep syscall_get_arguments()
and syscall_set_arguments() in sync.
With Generic Entry patch[1] and turn on audit, the performance
benchmarks from perf bench basic syscall on kunpeng920 gives roughly
a 1% performance uplift.
| Metric | W/O this patch | With this patch | Change |
| ---------- | -------------- | --------------- | --------- |
| Total time | 2.241 [sec] | 2.211 [sec] | ↓1.36% |
| usecs/op | 0.224157 | 0.221146 | ↓1.36% |
| ops/sec | 4,461,157 | 4,501,409 | ↑0.9% |
Disassembly shows that using direct assignment causes
syscall_set_arguments() to be inlined and cuts the instruction count by
five or six compared to memcpy(). Because __audit_syscall_entry() only
uses four syscall arguments, the compiler has also elided the copy of
regs->regs[4] and regs->regs[5].
Before:
<syscall_get_arguments.constprop.0>:
aa0103e2 mov x2, x1
91002003 add x3, x0, #0x8
f9408804 ldr x4, [x0, #272]
f8008444 str x4, [x2], #8
a9409404 ldp x4, x5, [x0, #8]
a9009424 stp x4, x5, [x1, #8]
a9418400 ldp x0, x1, [x0, #24]
a9010440 stp x0, x1, [x2, #16]
f9401060 ldr x0, [x3, #32]
f9001040 str x0, [x2, #32]
d65f03c0 ret
d503201f nop
After:
a9408e82 ldp x2, x3, [x20, #8]
2a1603e0 mov w0, w22
f9400e84 ldr x4, [x20, #24]
f9408a81 ldr x1, [x20, #272]
9401c4ba bl ffff800080215ca8 <__audit_syscall_entry>
This also aligns the implementation with x86 and RISC-V.
[1]: https://lore.kernel.org/all/20251126071446.3234218-1-ruanjinjie@huawei.com/
Signed-off-by: Jinjie Ruan <ruanjinjie@huawei.com>
Signed-off-by: Will Deacon <will@kernel.org>
127 lines
2.9 KiB
C
127 lines
2.9 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
*/
|
|
#ifndef __ASM_SYSCALL_H
|
|
#define __ASM_SYSCALL_H
|
|
|
|
#include <uapi/linux/audit.h>
|
|
#include <linux/compat.h>
|
|
#include <linux/err.h>
|
|
|
|
typedef long (*syscall_fn_t)(const struct pt_regs *regs);
|
|
|
|
extern const syscall_fn_t sys_call_table[];
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
extern const syscall_fn_t compat_sys_call_table[];
|
|
#endif
|
|
|
|
static inline int syscall_get_nr(struct task_struct *task,
|
|
struct pt_regs *regs)
|
|
{
|
|
return regs->syscallno;
|
|
}
|
|
|
|
static inline void syscall_rollback(struct task_struct *task,
|
|
struct pt_regs *regs)
|
|
{
|
|
regs->regs[0] = regs->orig_x0;
|
|
}
|
|
|
|
static inline long syscall_get_return_value(struct task_struct *task,
|
|
struct pt_regs *regs)
|
|
{
|
|
unsigned long val = regs->regs[0];
|
|
|
|
if (is_compat_thread(task_thread_info(task)))
|
|
val = sign_extend64(val, 31);
|
|
|
|
return val;
|
|
}
|
|
|
|
static inline long syscall_get_error(struct task_struct *task,
|
|
struct pt_regs *regs)
|
|
{
|
|
unsigned long error = syscall_get_return_value(task, regs);
|
|
|
|
return IS_ERR_VALUE(error) ? error : 0;
|
|
}
|
|
|
|
static inline void syscall_set_return_value(struct task_struct *task,
|
|
struct pt_regs *regs,
|
|
int error, long val)
|
|
{
|
|
if (error)
|
|
val = error;
|
|
|
|
if (is_compat_thread(task_thread_info(task)))
|
|
val = lower_32_bits(val);
|
|
|
|
regs->regs[0] = val;
|
|
}
|
|
|
|
static inline void syscall_set_nr(struct task_struct *task,
|
|
struct pt_regs *regs,
|
|
int nr)
|
|
{
|
|
regs->syscallno = nr;
|
|
if (nr == -1) {
|
|
/*
|
|
* When the syscall number is set to -1, the syscall will be
|
|
* skipped. In this case the syscall return value has to be
|
|
* set explicitly, otherwise the first syscall argument is
|
|
* returned as the syscall return value.
|
|
*/
|
|
syscall_set_return_value(task, regs, -ENOSYS, 0);
|
|
}
|
|
}
|
|
|
|
static inline void syscall_get_arguments(struct task_struct *task,
|
|
struct pt_regs *regs,
|
|
unsigned long *args)
|
|
{
|
|
args[0] = regs->orig_x0;
|
|
args[1] = regs->regs[1];
|
|
args[2] = regs->regs[2];
|
|
args[3] = regs->regs[3];
|
|
args[4] = regs->regs[4];
|
|
args[5] = regs->regs[5];
|
|
}
|
|
|
|
static inline void syscall_set_arguments(struct task_struct *task,
|
|
struct pt_regs *regs,
|
|
const unsigned long *args)
|
|
{
|
|
regs->regs[0] = args[0];
|
|
regs->regs[1] = args[1];
|
|
regs->regs[2] = args[2];
|
|
regs->regs[3] = args[3];
|
|
regs->regs[4] = args[4];
|
|
regs->regs[5] = args[5];
|
|
|
|
/*
|
|
* Also copy the first argument into orig_x0
|
|
* so that syscall_get_arguments() would return it
|
|
* instead of the previous value.
|
|
*/
|
|
regs->orig_x0 = regs->regs[0];
|
|
}
|
|
|
|
/*
|
|
* We don't care about endianness (__AUDIT_ARCH_LE bit) here because
|
|
* AArch64 has the same system calls both on little- and big- endian.
|
|
*/
|
|
static inline int syscall_get_arch(struct task_struct *task)
|
|
{
|
|
if (is_compat_thread(task_thread_info(task)))
|
|
return AUDIT_ARCH_ARM;
|
|
|
|
return AUDIT_ARCH_AARCH64;
|
|
}
|
|
|
|
int syscall_trace_enter(struct pt_regs *regs);
|
|
void syscall_trace_exit(struct pt_regs *regs);
|
|
|
|
#endif /* __ASM_SYSCALL_H */
|