bybrooklyn/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-04-18 06:44:00 -04:00
Code Issues Packages Projects Releases Wiki Activity

bpf: Move constants blinding out of arch-specific JITs

Browse Source 
During the JIT stage, constants blinding rewrites instructions but only
rewrites the private instruction copy of the JITed subprog, leaving the
global env->prog->insnsi and env->insn_aux_data untouched. This causes a
mismatch between subprog instructions and the global state, making it
difficult to use the global data in the JIT.

To avoid this mismatch, and given that all arch-specific JITs already
support constants blinding, move it to the generic verifier code, and
switch to rewrite the global env->prog->insnsi with the global states
adjusted, as other rewrites in the verifier do.

This removes the constants blinding calls in each JIT, which are largely
duplicated code across architectures.

Since constants blinding is only required for JIT, and there are two
JIT entry functions, jit_subprogs() for BPF programs with multiple
subprogs and bpf_prog_select_runtime() for programs with no subprogs,
move the constants blinding invocation into these two functions.

In the verifier path, bpf_patch_insn_data() is used to keep global
verifier auxiliary data in sync with patched instructions. A key
question is whether this global auxiliary data should be restored
on the failure path.

Besides instructions, bpf_patch_insn_data() adjusts:
  - prog->aux->poke_tab
  - env->insn_array_maps
  - env->subprog_info
  - env->insn_aux_data

For prog->aux->poke_tab, it is only used by JIT or only meaningful after
JIT succeeds, so it does not need to be restored on the failure path.

For env->insn_array_maps, when JIT fails, programs using insn arrays
are rejected by bpf_insn_array_ready() due to missing JIT addresses.
Hence, env->insn_array_maps is only meaningful for JIT and does not need
to be restored.

For subprog_info, if jit_subprogs fails and CONFIG_BPF_JIT_ALWAYS_ON
is not enabled, kernel falls back to interpreter. In this case,
env->subprog_info is used to determine subprogram stack depth. So it
must be restored on failure.

For env->insn_aux_data, it is freed by clear_insn_aux_data() at the
end of bpf_check(). Before freeing, clear_insn_aux_data() loops over
env->insn_aux_data to release jump targets recorded in it. The loop
uses env->prog->len as the array length, but this length no longer
matches the actual size of the adjusted env->insn_aux_data array after
constants blinding.

To address it, a simple approach is to keep insn_aux_data as adjusted
after failure, since it will be freed shortly, and record its actual size
for the loop in clear_insn_aux_data(). But since clear_insn_aux_data()
uses the same index to loop over both env->prog->insnsi and env->insn_aux_data,
this approach results in incorrect index for the insnsi array. So an
alternative approach is adopted: clone the original env->insn_aux_data
before blinding and restore it after failure, similar to env->prog.

For classic BPF programs, constants blinding works as before since it
is still invoked from bpf_prog_select_runtime().

Reviewed-by: Anton Protopopov <a.s.protopopov@gmail.com> # v8
Reviewed-by: Hari Bathini <hbathini@linux.ibm.com> # powerpc jit
Reviewed-by: Pu Lehui <pulehui@huawei.com> # riscv jit
Acked-by: Hengqi Chen <hengqi.chen@gmail.com> # loongarch jit
Signed-off-by: Xu Kuohai <xukuohai@huawei.com>
Link: https://lore.kernel.org/r/20260416064341.151802-2-xukuohai@huaweicloud.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Xu Kuohai
2026-04-16 06:43:37 +00:00
committed by Alexei Starovoitov
parent a204466529
commit d3e945223e
15 changed files with 403 additions and 478 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
kernel/bpf/core.c
View File

@@ -1508,7 +1508,11 @@ static void adjust_insn_arrays(struct bpf_prog *prog, u32 off, u32 len)
#endif
}
struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)
/*
* Now this function is used only to blind the main prog and must be invoked only when
* bpf_prog_need_blind() returns true.
*/
struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
struct bpf_insn insn_buff[16], aux[2];
struct bpf_prog *clone, *tmp;
@@ -1516,13 +1520,17 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)
struct bpf_insn *insn;
int i, rewritten;
if (!prog->blinding_requested || prog->blinded)
return prog;
if (WARN_ON_ONCE(env && env->prog != prog))
return ERR_PTR(-EINVAL);
clone = bpf_prog_clone_create(prog, GFP_USER);
if (!clone)
return ERR_PTR(-ENOMEM);
/* make sure bpf_patch_insn_data() patches the correct prog */
if (env)
env->prog = clone;
insn_cnt = clone->len;
insn = clone->insnsi;
@@ -1550,21 +1558,35 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)
if (!rewritten)
continue;
tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
if (IS_ERR(tmp)) {
if (env)
tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten);
else
tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
if (IS_ERR_OR_NULL(tmp)) {
if (env)
/* restore the original prog */
env->prog = prog;
/* Patching may have repointed aux->prog during
* realloc from the original one, so we need to
* fix it up here on error.
*/
bpf_jit_prog_release_other(prog, clone);
return tmp;
return IS_ERR(tmp) ? tmp : ERR_PTR(-ENOMEM);
}
clone = tmp;
insn_delta = rewritten - 1;
/* Instructions arrays must be updated using absolute xlated offsets */
adjust_insn_arrays(clone, prog->aux->subprog_start + i, rewritten);
if (env)
env->prog = clone;
else
/*
* Instructions arrays must be updated using absolute xlated offsets.
* The arrays have already been adjusted by bpf_patch_insn_data() when
* env is not NULL.
*/
adjust_insn_arrays(clone, i, rewritten);
/* Walk new program and skip insns we just inserted. */
insn = clone->insnsi + i + insn_delta;
@@ -2533,6 +2555,35 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp)
return select_interpreter;
}
static struct bpf_prog *bpf_prog_jit_compile(struct bpf_prog *prog)
{
#ifdef CONFIG_BPF_JIT
struct bpf_prog *orig_prog;
if (!bpf_prog_need_blind(prog))
return bpf_int_jit_compile(prog);
orig_prog = prog;
prog = bpf_jit_blind_constants(NULL, prog);
/*
* If blinding was requested and we failed during blinding, we must fall
* back to the interpreter.
*/
if (IS_ERR(prog))
return orig_prog;
prog = bpf_int_jit_compile(prog);
if (prog->jited) {
bpf_jit_prog_release_other(prog, orig_prog);
return prog;
}
bpf_jit_prog_release_other(orig_prog, prog);
prog = orig_prog;
#endif
return prog;
}
/**
* bpf_prog_select_runtime - select exec runtime for BPF program
* @fp: bpf_prog populated with BPF program
@@ -2572,7 +2623,7 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
if (*err)
return fp;
fp = bpf_int_jit_compile(fp);
fp = bpf_prog_jit_compile(fp);
bpf_prog_jit_attempt_done(fp);
if (!fp->jited && jit_needed) {
*err = -ENOTSUPP;

146
kernel/bpf/fixups.c
View File

@@ -232,8 +232,8 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)
}
}
static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
const struct bpf_insn *patch, u32 len)
struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
const struct bpf_insn *patch, u32 len)
{
struct bpf_prog *new_prog;
struct bpf_insn_aux_data *new_data = NULL;
@@ -973,7 +973,47 @@ patch_insn_buf:
return 0;
}
int bpf_jit_subprogs(struct bpf_verifier_env *env)
static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env)
{
u32 *starts = NULL;
starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT);
if (starts) {
for (int i = 0; i < env->subprog_cnt; i++)
starts[i] = env->subprog_info[i].start;
}
return starts;
}
static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts)
{
for (int i = 0; i < env->subprog_cnt; i++)
env->subprog_info[i].start = orig_starts[i];
/* restore the start of fake 'exit' subprog as well */
env->subprog_info[env->subprog_cnt].start = env->prog->len;
}
static struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)
{
size_t size;
void *new_aux;
size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len);
new_aux = __vmalloc(size, GFP_KERNEL_ACCOUNT);
if (new_aux)
memcpy(new_aux, env->insn_aux_data, size);
return new_aux;
}
static void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
struct bpf_insn_aux_data *orig_insn_aux)
{
/* the expanded elements are zero-filled, so no special handling is required */
vfree(env->insn_aux_data);
env->insn_aux_data = orig_insn_aux;
}
static int jit_subprogs(struct bpf_verifier_env *env)
{
struct bpf_prog *prog = env->prog, **func, *tmp;
int i, j, subprog_start, subprog_end = 0, len, subprog;
@@ -981,10 +1021,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env)
struct bpf_insn *insn;
void *old_bpf_func;
int err, num_exentries;
int old_len, subprog_start_adjustment = 0;
if (env->subprog_cnt <= 1)
return 0;
for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn))
@@ -1053,10 +1089,11 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env)
goto out_free;
func[i]->is_func = 1;
func[i]->sleepable = prog->sleepable;
func[i]->blinded = prog->blinded;
func[i]->aux->func_idx = i;
/* Below members will be freed only at prog->aux */
func[i]->aux->btf = prog->aux->btf;
func[i]->aux->subprog_start = subprog_start + subprog_start_adjustment;
func[i]->aux->subprog_start = subprog_start;
func[i]->aux->func_info = prog->aux->func_info;
func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;
func[i]->aux->poke_tab = prog->aux->poke_tab;
@@ -1113,15 +1150,7 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env)
func[i]->aux->token = prog->aux->token;
if (!i)
func[i]->aux->exception_boundary = env->seen_exception;
/*
* To properly pass the absolute subprog start to jit
* all instruction adjustments should be accumulated
*/
old_len = func[i]->len;
func[i] = bpf_int_jit_compile(func[i]);
subprog_start_adjustment += func[i]->len - old_len;
if (!func[i]->jited) {
err = -ENOTSUPP;
goto out_free;
@@ -1247,16 +1276,87 @@ out_free:
}
kfree(func);
out_undo_insn:
bpf_prog_jit_attempt_done(prog);
return err;
}
int bpf_jit_subprogs(struct bpf_verifier_env *env)
{
int err, i;
bool blinded = false;
struct bpf_insn *insn;
struct bpf_prog *prog, *orig_prog;
struct bpf_insn_aux_data *orig_insn_aux;
u32 *orig_subprog_starts;
if (env->subprog_cnt <= 1)
return 0;
prog = orig_prog = env->prog;
if (bpf_prog_need_blind(prog)) {
orig_insn_aux = bpf_dup_insn_aux_data(env);
if (!orig_insn_aux) {
err = -ENOMEM;
goto out_cleanup;
}
orig_subprog_starts = bpf_dup_subprog_starts(env);
if (!orig_subprog_starts) {
vfree(orig_insn_aux);
err = -ENOMEM;
goto out_cleanup;
}
prog = bpf_jit_blind_constants(env, prog);
if (IS_ERR(prog)) {
err = -ENOMEM;
prog = orig_prog;
goto out_restore;
}
blinded = true;
}
err = jit_subprogs(env);
if (err)
goto out_jit_err;
if (blinded) {
bpf_jit_prog_release_other(prog, orig_prog);
kvfree(orig_subprog_starts);
vfree(orig_insn_aux);
}
return 0;
out_jit_err:
if (blinded) {
bpf_jit_prog_release_other(orig_prog, prog);
/* roll back to the clean original prog */
prog = env->prog = orig_prog;
goto out_restore;
} else {
if (err != -EFAULT) {
/*
* We will fall back to interpreter mode when err is not -EFAULT, before
* that, insn->off and insn->imm should be restored to their original
* values since they were modified by jit_subprogs.
*/
for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
if (!bpf_pseudo_call(insn))
continue;
insn->off = 0;
insn->imm = env->insn_aux_data[i].call_imm;
}
}
goto out_cleanup;
}
out_restore:
bpf_restore_subprog_starts(env, orig_subprog_starts);
bpf_restore_insn_aux_data(env, orig_insn_aux);
kvfree(orig_subprog_starts);
out_cleanup:
/* cleanup main prog to be interpreted */
prog->jit_requested = 0;
prog->blinding_requested = 0;
for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
if (!bpf_pseudo_call(insn))
continue;
insn->off = 0;
insn->imm = env->insn_aux_data[i].call_imm;
}
bpf_prog_jit_attempt_done(prog);
return err;
}