tmakin/ack-tegra
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge f5ad410100 ("Merge tag 'bpf-next-7.1' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next") into android-mainline

Browse Source 
Steps on the way to v7.1-rc1

Change-Id: I805c5e986422571c354706b9ffc2f3c65cf9791a
Signed-off-by: Carlos Llamas <cmllamas@google.com>
This commit is contained in:
Carlos Llamas authored and android-test-infra-autosubmit@system.gserviceaccount.com committed 2026-05-15 11:09:31 -07:00
2 parents b74b101307 f5ad410100
commit 0ecf74f880
233 files changed
+21767 -11280
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Documentation/bpf/drgn.rst
+2 -2
View File
@@ -26,8 +26,8 @@ about these objects, including id, type and name.
The main use-case `bpf_inspect.py`_ covers is to show BPF programs of types
``BPF_PROG_TYPE_EXT`` and ``BPF_PROG_TYPE_TRACING`` attached to other BPF
programs via ``freplace``/``fentry``/``fexit`` mechanisms, since there is no
user-space API to get this information.
programs via ``freplace``/``fentry``/``fexit``/``fsession`` mechanisms, since
there is no user-space API to get this information.
Getting started
===============
Documentation/bpf/libbpf/program_types.rst
+4
View File
@@ -207,6 +207,10 @@ described in more detail in the footnotes.
+ + +----------------------------------+-----------+
| | | ``fexit.s+`` [#fentry]_ | Yes |
+ +----------------------------------------+----------------------------------+-----------+
| | ``BPF_TRACE_FSESSION`` | ``fsession+`` [#fentry]_ | |
+ + +----------------------------------+-----------+
| | | ``fsession.s+`` [#fentry]_ | Yes |
+ +----------------------------------------+----------------------------------+-----------+
| | ``BPF_TRACE_ITER`` | ``iter+`` [#iter]_ | |
+ + +----------------------------------+-----------+
| | | ``iter.s+`` [#iter]_ | Yes |
MAINTAINERS
+2 -5
View File
@@ -4815,13 +4815,10 @@ M: Alexei Starovoitov <ast@kernel.org>
M: Daniel Borkmann <daniel@iogearbox.net>
M: Andrii Nakryiko <andrii@kernel.org>
R: Martin KaFai Lau <martin.lau@linux.dev>
R: Eduard Zingerman <eddyz87@gmail.com>
M: Eduard Zingerman <eddyz87@gmail.com>
M: Kumar Kartikeya Dwivedi <memxor@gmail.com>
R: Song Liu <song@kernel.org>
R: Yonghong Song <yonghong.song@linux.dev>
R: John Fastabend <john.fastabend@gmail.com>
R: KP Singh <kpsingh@kernel.org>
R: Stanislav Fomichev <sdf@fomichev.me>
R: Hao Luo <haoluo@google.com>
R: Jiri Olsa <jolsa@kernel.org>
L: bpf@vger.kernel.org
S: Supported
arch/arc/net/bpf_jit_arcv2.c
+4 -4
View File
@@ -2427,7 +2427,7 @@ u8 arc_prologue(u8 *buf, u32 usage, u16 frame_size)
#ifdef ARC_BPF_JIT_DEBUG
if ((usage & BIT(ARC_R_FP)) && frame_size == 0) {
pr_err("FP is being saved while there is no frame.");
pr_err("FP is being saved while there is no frame.\n");
BUG();
}
#endif
@@ -2454,7 +2454,7 @@ u8 arc_epilogue(u8 *buf, u32 usage, u16 frame_size)
#ifdef ARC_BPF_JIT_DEBUG
if ((usage & BIT(ARC_R_FP)) && frame_size == 0) {
pr_err("FP is being saved while there is no frame.");
pr_err("FP is being saved while there is no frame.\n");
BUG();
}
#endif
@@ -2868,7 +2868,7 @@ u8 gen_jmp_64(u8 *buf, u8 rd, u8 rs, u8 cond, u32 curr_off, u32 targ_off)
break;
default:
#ifdef ARC_BPF_JIT_DEBUG
pr_err("64-bit jump condition is not known.");
pr_err("64-bit jump condition is not known.\n");
BUG();
#endif
}
@@ -2948,7 +2948,7 @@ u8 gen_jmp_32(u8 *buf, u8 rd, u8 rs, u8 cond, u32 curr_off, u32 targ_off)
*/
if (cond >= ARC_CC_LAST) {
#ifdef ARC_BPF_JIT_DEBUG
pr_err("32-bit jump condition is not known.");
pr_err("32-bit jump condition is not known.\n");
BUG();
#endif
return 0;
arch/arm64/net/bpf_jit.h
+3 -1
View File
@@ -187,7 +187,9 @@
/* Rn - imm12; set condition flags */
#define A64_CMP_I(sf, Rn, imm12) A64_SUBS_I(sf, A64_ZR, Rn, imm12)
/* Rd = Rn */
#define A64_MOV(sf, Rd, Rn) A64_ADD_I(sf, Rd, Rn, 0)
#define A64_MOV(sf, Rd, Rn) \
(((Rd) == A64_SP || (Rn) == A64_SP) ? A64_ADD_I(sf, Rd, Rn, 0) : \
aarch64_insn_gen_move_reg(Rd, Rn, A64_VARIANT(sf)))
/* Bitfield move */
#define A64_BITFIELD(sf, Rd, Rn, immr, imms, type) \
arch/s390/kernel/Makefile
+2
View File
@@ -80,5 +80,7 @@ obj-$(CONFIG_PERF_EVENTS) += perf_pai.o
obj-$(CONFIG_TRACEPOINTS) += trace.o
obj-$(CONFIG_BPF_SYSCALL) += bpf.o
# vdso
obj-y += vdso/
arch/s390/kernel/bpf.c
+12
View File
@@ -0,0 +1,12 @@
// SPDX-License-Identifier: GPL-2.0
#include <asm/lowcore.h>
#include <linux/btf.h>
__bpf_kfunc_start_defs();
__bpf_kfunc struct lowcore *bpf_get_lowcore(void)
{
return get_lowcore();
}
__bpf_kfunc_end_defs();
arch/s390/net/bpf_jit_comp.c
+109 -48
View File
@@ -830,25 +830,34 @@ static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp,
}
/*
* Sign-extend the register if necessary
* Sign- or zero-extend the register if necessary
*/
static int sign_extend(struct bpf_jit *jit, int r, u8 size, u8 flags)
static int sign_zero_extend(struct bpf_jit *jit, int r, u8 size, u8 flags)
{
if (!(flags & BTF_FMODEL_SIGNED_ARG))
return 0;
switch (size) {
case 1:
/* lgbr %r,%r */
EMIT4(0xb9060000, r, r);
if (flags & BTF_FMODEL_SIGNED_ARG)
/* lgbr %r,%r */
EMIT4(0xb9060000, r, r);
else
/* llgcr %r,%r */
EMIT4(0xb9840000, r, r);
return 0;
case 2:
/* lghr %r,%r */
EMIT4(0xb9070000, r, r);
if (flags & BTF_FMODEL_SIGNED_ARG)
/* lghr %r,%r */
EMIT4(0xb9070000, r, r);
else
/* llghr %r,%r */
EMIT4(0xb9850000, r, r);
return 0;
case 4:
/* lgfr %r,%r */
EMIT4(0xb9140000, r, r);
if (flags & BTF_FMODEL_SIGNED_ARG)
/* lgfr %r,%r */
EMIT4(0xb9140000, r, r);
else
/* llgfr %r,%r */
EMIT4(0xb9160000, r, r);
return 0;
case 8:
return 0;
@@ -1798,9 +1807,9 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
return -1;
for (j = 0; j < m->nr_args; j++) {
if (sign_extend(jit, BPF_REG_1 + j,
m->arg_size[j],
m->arg_flags[j]))
if (sign_zero_extend(jit, BPF_REG_1 + j,
m->arg_size[j],
m->arg_flags[j]))
return -1;
}
}
@@ -1862,20 +1871,21 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
jit->prg);
/*
* if (tail_call_cnt++ >= MAX_TAIL_CALL_CNT)
* if (tail_call_cnt >= MAX_TAIL_CALL_CNT)
* goto out;
*
* tail_call_cnt is read into %w0, which needs to be preserved
* until it's incremented and flushed.
*/
off = jit->frame_off +
offsetof(struct prog_frame, tail_call_cnt);
/* lhi %w0,1 */
EMIT4_IMM(0xa7080000, REG_W0, 1);
/* laal %w1,%w0,off(%r15) */
EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W1, REG_W0, REG_15, off);
/* clij %w1,MAX_TAIL_CALL_CNT-1,0x2,out */
/* ly %w0,off(%r15) */
EMIT6_DISP_LH(0xe3000000, 0x0058, REG_W0, REG_0, REG_15, off);
/* clij %w0,MAX_TAIL_CALL_CNT,0xa,out */
patch_2_clij = jit->prg;
EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W1, MAX_TAIL_CALL_CNT - 1,
2, jit->prg);
EMIT6_PCREL_RIEC(0xec000000, 0x007f, REG_W0, MAX_TAIL_CALL_CNT,
0xa, jit->prg);
/*
* prog = array->ptrs[index];
@@ -1894,6 +1904,12 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
patch_3_brc = jit->prg;
EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg);
/* tail_call_cnt++; */
/* ahi %w0,1 */
EMIT4_IMM(0xa70a0000, REG_W0, 1);
/* sty %w0,off(%r15) */
EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, REG_0, REG_15, off);
/*
* Restore registers before calling function
*/
@@ -2480,8 +2496,8 @@ struct bpf_tramp_jit {
int ip_off; /* For bpf_get_func_ip(), has to be at
* (ctx - 16)
*/
int arg_cnt_off; /* For bpf_get_func_arg_cnt(), has to be at
* (ctx - 8)
int func_meta_off; /* For bpf_get_func_arg_cnt()/fsession, has
* to be at (ctx - 8)
*/
int bpf_args_off; /* Offset of BPF_PROG context, which consists
* of BPF arguments followed by return value
@@ -2506,6 +2522,13 @@ static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val)
EMIT6_IMM(0xc00d0000, dst_reg, val);
}
static void emit_store_stack_imm64(struct bpf_jit *jit, int tmp_reg, int stack_off, u64 imm)
{
load_imm64(jit, tmp_reg, imm);
/* stg %tmp_reg,stack_off(%r15) */
EMIT6_DISP_LH(0xe3000000, 0x0024, tmp_reg, REG_0, REG_15, stack_off);
}
static int invoke_bpf_prog(struct bpf_tramp_jit *tjit,
const struct btf_func_model *m,
struct bpf_tramp_link *tlink, bool save_ret)
@@ -2520,10 +2543,7 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit,
* run_ctx.cookie = tlink->cookie;
*/
/* %r0 = tlink->cookie */
load_imm64(jit, REG_W0, tlink->cookie);
/* stg %r0,cookie_off(%r15) */
EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, REG_0, REG_15, cookie_off);
emit_store_stack_imm64(jit, REG_W0, cookie_off, tlink->cookie);
/*
* if ((start = __bpf_prog_enter(p, &run_ctx)) == 0)
@@ -2555,7 +2575,7 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit,
EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, p->bpf_func);
/* stg %r2,retval_off(%r15) */
if (save_ret) {
if (sign_extend(jit, REG_2, m->ret_size, m->ret_flags))
if (sign_zero_extend(jit, REG_2, m->ret_size, m->ret_flags))
return -1;
EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15,
tjit->retval_off);
@@ -2581,6 +2601,28 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit,
return 0;
}
static int invoke_bpf(struct bpf_tramp_jit *tjit,
const struct btf_func_model *m,
struct bpf_tramp_links *tl, bool save_ret,
u64 func_meta, int cookie_off)
{
int i, cur_cookie = (tjit->bpf_args_off - cookie_off) / sizeof(u64);
struct bpf_jit *jit = &tjit->common;
for (i = 0; i < tl->nr_links; i++) {
if (bpf_prog_calls_session_cookie(tl->links[i])) {
u64 meta = func_meta | ((u64)cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT);
emit_store_stack_imm64(jit, REG_0, tjit->func_meta_off, meta);
cur_cookie--;
}
if (invoke_bpf_prog(tjit, m, tl->links[i], save_ret))
return -EINVAL;
}
return 0;
}
static int alloc_stack(struct bpf_tramp_jit *tjit, size_t size)
{
int stack_offset = tjit->stack_size;
@@ -2610,8 +2652,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
int nr_bpf_args, nr_reg_args, nr_stack_args;
int cookie_cnt, cookie_off, fsession_cnt;
struct bpf_jit *jit = &tjit->common;
int arg, bpf_arg_off;
u64 func_meta;
int i, j;
/* Support as many stack arguments as "mvc" instruction can handle. */
@@ -2643,6 +2687,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
return -ENOTSUPP;
}
cookie_cnt = bpf_fsession_cookie_cnt(tlinks);
fsession_cnt = bpf_fsession_cnt(tlinks);
/*
* Calculate the stack layout.
*/
@@ -2655,8 +2702,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
tjit->backchain_off = tjit->stack_size - sizeof(u64);
tjit->stack_args_off = alloc_stack(tjit, nr_stack_args * sizeof(u64));
tjit->reg_args_off = alloc_stack(tjit, nr_reg_args * sizeof(u64));
cookie_off = alloc_stack(tjit, cookie_cnt * sizeof(u64));
tjit->ip_off = alloc_stack(tjit, sizeof(u64));
tjit->arg_cnt_off = alloc_stack(tjit, sizeof(u64));
tjit->func_meta_off = alloc_stack(tjit, sizeof(u64));
tjit->bpf_args_off = alloc_stack(tjit, nr_bpf_args * sizeof(u64));
tjit->retval_off = alloc_stack(tjit, sizeof(u64));
tjit->r7_r8_off = alloc_stack(tjit, 2 * sizeof(u64));
@@ -2743,18 +2791,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
* arg_cnt = m->nr_args;
*/
if (flags & BPF_TRAMP_F_IP_ARG) {
/* %r0 = func_addr */
load_imm64(jit, REG_0, (u64)func_addr);
/* stg %r0,ip_off(%r15) */
EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15,
tjit->ip_off);
}
/* lghi %r0,nr_bpf_args */
EMIT4_IMM(0xa7090000, REG_0, nr_bpf_args);
/* stg %r0,arg_cnt_off(%r15) */
if (flags & BPF_TRAMP_F_IP_ARG)
emit_store_stack_imm64(jit, REG_0, tjit->ip_off, (u64)func_addr);
func_meta = nr_bpf_args;
/* lghi %r0,func_meta */
EMIT4_IMM(0xa7090000, REG_0, func_meta);
/* stg %r0,func_meta_off(%r15) */
EMIT6_DISP_LH(0xe3000000, 0x0024, REG_0, REG_0, REG_15,
tjit->arg_cnt_off);
tjit->func_meta_off);
if (flags & BPF_TRAMP_F_CALL_ORIG) {
/*
@@ -2767,10 +2811,17 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, __bpf_tramp_enter);
}
for (i = 0; i < fentry->nr_links; i++)
if (invoke_bpf_prog(tjit, m, fentry->links[i],
flags & BPF_TRAMP_F_RET_FENTRY_RET))
return -EINVAL;
if (fsession_cnt) {
/* Clear all the session cookies' value. */
for (i = 0; i < cookie_cnt; i++)
emit_store_stack_imm64(jit, REG_0, cookie_off + 8 * i, 0);
/* Clear the return value to make sure fentry always gets 0. */
emit_store_stack_imm64(jit, REG_0, tjit->retval_off, 0);
}
if (invoke_bpf(tjit, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET,
func_meta, cookie_off))
return -EINVAL;
if (fmod_ret->nr_links) {
/*
@@ -2847,11 +2898,16 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im,
EMIT6_PCREL_RILC(0xc0040000, 0, (u64)im->ip_epilogue);
}
/* Set the "is_return" flag for fsession. */
func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT);
if (fsession_cnt)
emit_store_stack_imm64(jit, REG_W0, tjit->func_meta_off,
func_meta);
/* do_fexit: */
tjit->do_fexit = jit->prg;
for (i = 0; i < fexit->nr_links; i++)
if (invoke_bpf_prog(tjit, m, fexit->links[i], false))
return -EINVAL;
if (invoke_bpf(tjit, m, fexit, false, func_meta, cookie_off))
return -EINVAL;
if (flags & BPF_TRAMP_F_CALL_ORIG) {
im->ip_epilogue = jit->prg_buf + jit->prg;
@@ -2956,6 +3012,11 @@ bool bpf_jit_supports_arena(void)
return true;
}
bool bpf_jit_supports_fsession(void)
{
return true;
}
bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
{
if (!in_arena)
drivers/net/ethernet/netronome/nfp/bpf/jit.c
+8 -10
View File
@@ -1731,7 +1731,7 @@ map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
/* We only have to reload LM0 if the key is not at start of stack */
lm_off = nfp_prog->stack_frame_depth;
lm_off += meta->arg2.reg.var_off.value + meta->arg2.reg.off;
lm_off += meta->arg2.reg.var_off.value;
load_lm_ptr = meta->arg2.var_off || lm_off;
/* Set LM0 to start of key */
@@ -2874,8 +2874,7 @@ mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
}
if (meta->ptr.type == PTR_TO_STACK)
return mem_ldx_stack(nfp_prog, meta, size,
meta->ptr.off + meta->ptr.var_off.value);
return mem_ldx_stack(nfp_prog, meta, size, meta->ptr.var_off.value);
if (meta->ptr.type == PTR_TO_MAP_VALUE)
return mem_ldx_emem(nfp_prog, meta, size);
@@ -2985,8 +2984,7 @@ mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
return mem_stx_data(nfp_prog, meta, size);
if (meta->ptr.type == PTR_TO_STACK)
return mem_stx_stack(nfp_prog, meta, size,
meta->ptr.off + meta->ptr.var_off.value);
return mem_stx_stack(nfp_prog, meta, size, meta->ptr.var_off.value);
return -EOPNOTSUPP;
}
@@ -4153,9 +4151,9 @@ cross_mem_access(struct bpf_insn *ld, struct nfp_insn_meta *head_ld_meta,
/* Canonicalize the offsets. Turn all of them against the original
* base register.
*/
head_ld_off = head_ld_meta->insn.off + head_ld_meta->ptr.off;
head_st_off = head_st_meta->insn.off + head_st_meta->ptr.off;
ld_off = ld->off + head_ld_meta->ptr.off;
head_ld_off = head_ld_meta->insn.off + head_ld_meta->ptr.var_off.value;
head_st_off = head_st_meta->insn.off + head_st_meta->ptr.var_off.value;
ld_off = ld->off + head_ld_meta->ptr.var_off.value;
/* Ascending order cross. */
if (ld_off > head_ld_off &&
@@ -4326,7 +4324,7 @@ static void nfp_bpf_opt_pkt_cache(struct nfp_prog *nfp_prog)
* support this.
*/
if (meta->ptr.id == range_ptr_id &&
meta->ptr.off == range_ptr_off) {
meta->ptr.var_off.value == range_ptr_off) {
s16 new_start = range_start;
s16 end, off = insn->off;
s16 new_end = range_end;
@@ -4361,7 +4359,7 @@ start_new:
range_node = meta;
range_node->pkt_cache.do_init = true;
range_ptr_id = range_node->ptr.id;
range_ptr_off = range_node->ptr.off;
range_ptr_off = range_node->ptr.var_off.value;
range_start = insn->off;
range_end = insn->off + BPF_LDST_BYTES(insn);
}
drivers/net/ethernet/netronome/nfp/bpf/verifier.c
+6 -6
View File
@@ -98,7 +98,7 @@ static bool nfp_bpf_map_update_value_ok(struct bpf_verifier_env *env)
offmap = map_to_offmap(reg1->map_ptr);
nfp_map = offmap->dev_priv;
off = reg3->off + reg3->var_off.value;
off = reg3->var_off.value;
for (i = 0; i < offmap->map.value_size; i++) {
struct bpf_stack_state *stack_entry;
@@ -137,7 +137,7 @@ nfp_bpf_stack_arg_ok(const char *fname, struct bpf_verifier_env *env,
return false;
}
off = reg->var_off.value + reg->off;
off = reg->var_off.value;
if (-off % 4) {
pr_vlog(env, "%s: unaligned stack pointer %lld\n", fname, -off);
return false;
@@ -147,7 +147,7 @@ nfp_bpf_stack_arg_ok(const char *fname, struct bpf_verifier_env *env,
if (!old_arg)
return true;
old_off = old_arg->reg.var_off.value + old_arg->reg.off;
old_off = old_arg->reg.var_off.value;
old_arg->var_off |= off != old_off;
return true;
@@ -358,8 +358,8 @@ nfp_bpf_check_stack_access(struct nfp_prog *nfp_prog,
if (meta->ptr.type == NOT_INIT)
return 0;
old_off = meta->ptr.off + meta->ptr.var_off.value;
new_off = reg->off + reg->var_off.value;
old_off = meta->ptr.var_off.value;
new_off = reg->var_off.value;
meta->ptr_not_const |= old_off != new_off;
@@ -428,7 +428,7 @@ nfp_bpf_map_mark_used(struct bpf_verifier_env *env, struct nfp_insn_meta *meta,
return -EOPNOTSUPP;
}
off = reg->var_off.value + meta->insn.off + reg->off;
off = reg->var_off.value + meta->insn.off;
size = BPF_LDST_BYTES(&meta->insn);
offmap = map_to_offmap(reg->map_ptr);
nfp_map = offmap->dev_priv;
include/linux/bpf-cgroup.h
+1 -1
View File
@@ -184,7 +184,7 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk,
struct bpf_prog_array *array;
array = rcu_access_pointer(cgrp->bpf.effective[type]);
return array != &bpf_empty_prog_array.hdr;
return array != &bpf_empty_prog_array;
}
/* Wrappers for __cgroup_bpf_run_filter_skb() guarded by cgroup_bpf_enabled. */
include/linux/bpf.h
+4 -6
View File
@@ -2369,18 +2369,13 @@ struct bpf_prog_array {
struct bpf_prog_array_item items[];
};
struct bpf_empty_prog_array {
struct bpf_prog_array hdr;
struct bpf_prog *null_prog;
};
/* to avoid allocating empty bpf_prog_array for cgroups that
* don't have bpf program attached use one global 'bpf_empty_prog_array'
* It will not be modified the caller of bpf_prog_array_alloc()
* (since caller requested prog_cnt == 0)
* that pointer should be 'freed' by bpf_prog_array_free()
*/
extern struct bpf_empty_prog_array bpf_empty_prog_array;
extern struct bpf_prog_array bpf_empty_prog_array;
struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
void bpf_prog_array_free(struct bpf_prog_array *progs);
@@ -3950,6 +3945,9 @@ static inline bool bpf_is_subprog(const struct bpf_prog *prog)
return prog->aux->func_idx != 0;
}
const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off);
void bpf_get_linfo_file_line(struct btf *btf, const struct bpf_line_info *linfo,
const char **filep, const char **linep, int *nump);
int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep,
const char **linep, int *nump);
struct bpf_prog *bpf_prog_find_from_stack(void);
include/linux/bpf_local_storage.h
+5 -10
View File
@@ -54,7 +54,6 @@ struct bpf_local_storage_map {
u32 bucket_log;
u16 elem_size;
u16 cache_idx;
bool use_kmalloc_nolock;
};
struct bpf_local_storage_data {
@@ -86,8 +85,7 @@ struct bpf_local_storage_elem {
*/
};
atomic_t state;
bool use_kmalloc_nolock;
/* 3 bytes hole */
/* 4 bytes hole */
/* The data is stored in another cacheline to minimize
* the number of cachelines access during a cache hit.
*/
@@ -104,7 +102,6 @@ struct bpf_local_storage {
rqspinlock_t lock; /* Protect adding/removing from the "list" */
u64 mem_charge; /* Copy of mem charged to owner. Protected by "lock" */
refcount_t owner_refcnt;/* Used to pin owner when map_free is uncharging */
bool use_kmalloc_nolock;
};
/* U16_MAX is much more than enough for sk local storage
@@ -137,8 +134,7 @@ int bpf_local_storage_map_alloc_check(union bpf_attr *attr);
struct bpf_map *
bpf_local_storage_map_alloc(union bpf_attr *attr,
struct bpf_local_storage_cache *cache,
bool use_kmalloc_nolock);
struct bpf_local_storage_cache *cache);
void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage,
struct bpf_local_storage_map *smap,
@@ -192,7 +188,7 @@ int bpf_selem_link_map(struct bpf_local_storage_map *smap,
struct bpf_local_storage_elem *
bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
bool swap_uptrs, gfp_t gfp_flags);
bool swap_uptrs);
void bpf_selem_free(struct bpf_local_storage_elem *selem,
bool reuse_now);
@@ -200,12 +196,11 @@ void bpf_selem_free(struct bpf_local_storage_elem *selem,
int
bpf_local_storage_alloc(void *owner,
struct bpf_local_storage_map *smap,
struct bpf_local_storage_elem *first_selem,
gfp_t gfp_flags);
struct bpf_local_storage_elem *first_selem);
struct bpf_local_storage_data *
bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags);
void *value, u64 map_flags, bool swap_uptrs);
u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map);
include/linux/bpf_verifier.h
+408 -15
View File
@@ -38,10 +38,9 @@ struct bpf_reg_state {
/* Ordering of fields matters. See states_equal() */
enum bpf_reg_type type;
/*
* Fixed part of pointer offset, pointer types only.
* Or constant delta between "linked" scalars with the same ID.
* Constant delta between "linked" scalars with the same ID.
*/
s32 off;
s32 delta;
union {
/* valid when type == PTR_TO_PACKET */
int range;
@@ -146,9 +145,9 @@ struct bpf_reg_state {
* Upper bit of ID is used to remember relationship between "linked"
* registers. Example:
* r1 = r2; both will have r1->id == r2->id == N
* r1 += 10; r1->id == N | BPF_ADD_CONST and r1->off == 10
* r1 += 10; r1->id == N | BPF_ADD_CONST and r1->delta == 10
* r3 = r2; both will have r3->id == r2->id == N
* w3 += 10; r3->id == N | BPF_ADD_CONST32 and r3->off == 10
* w3 += 10; r3->id == N | BPF_ADD_CONST32 and r3->delta == 10
*/
#define BPF_ADD_CONST64 (1U << 31)
#define BPF_ADD_CONST32 (1U << 30)
@@ -221,14 +220,67 @@ enum bpf_stack_slot_type {
STACK_DYNPTR,
STACK_ITER,
STACK_IRQ_FLAG,
STACK_POISON,
};
#define BPF_REG_SIZE 8 /* size of eBPF register in bytes */
/* 4-byte stack slot granularity for liveness analysis */
#define BPF_HALF_REG_SIZE 4
#define STACK_SLOT_SZ 4
#define STACK_SLOTS (MAX_BPF_STACK / BPF_HALF_REG_SIZE) /* 128 */
typedef struct {
u64 v[2];
} spis_t;
#define SPIS_ZERO ((spis_t){})
#define SPIS_ALL ((spis_t){{ U64_MAX, U64_MAX }})
static inline bool spis_is_zero(spis_t s)
{
return s.v[0] == 0 && s.v[1] == 0;
}
static inline bool spis_equal(spis_t a, spis_t b)
{
return a.v[0] == b.v[0] && a.v[1] == b.v[1];
}
static inline spis_t spis_or(spis_t a, spis_t b)
{
return (spis_t){{ a.v[0] | b.v[0], a.v[1] | b.v[1] }};
}
static inline spis_t spis_and(spis_t a, spis_t b)
{
return (spis_t){{ a.v[0] & b.v[0], a.v[1] & b.v[1] }};
}
static inline spis_t spis_not(spis_t s)
{
return (spis_t){{ ~s.v[0], ~s.v[1] }};
}
static inline bool spis_test_bit(spis_t s, u32 slot)
{
return s.v[slot / 64] & BIT_ULL(slot % 64);
}
static inline void spis_or_range(spis_t *mask, u32 lo, u32 hi)
{
u32 w;
for (w = lo; w <= hi && w < STACK_SLOTS; w++)
mask->v[w / 64] |= BIT_ULL(w % 64);
}
#define BPF_REGMASK_ARGS ((1 << BPF_REG_1) | (1 << BPF_REG_2) | \
(1 << BPF_REG_3) | (1 << BPF_REG_4) | \
(1 << BPF_REG_5))
#define BPF_MAIN_FUNC (-1)
#define BPF_DYNPTR_SIZE sizeof(struct bpf_dynptr_kern)
#define BPF_DYNPTR_NR_SLOTS (BPF_DYNPTR_SIZE / BPF_REG_SIZE)
@@ -266,6 +318,7 @@ struct bpf_reference_state {
struct bpf_retval_range {
s32 minval;
s32 maxval;
bool return_32bit;
};
/* state of the program:
@@ -424,7 +477,6 @@ struct bpf_verifier_state {
bool speculative;
bool in_sleepable;
bool cleaned;
/* first and last insn idx of this verifier state */
u32 first_insn_idx;
@@ -595,6 +647,18 @@ struct bpf_insn_aux_data {
u32 scc;
/* registers alive before this instruction. */
u16 live_regs_before;
/*
* Bitmask of R0-R9 that hold known values at this instruction.
* const_reg_mask: scalar constants that fit in 32 bits.
* const_reg_map_mask: map pointers, val is map_index into used_maps[].
* const_reg_subprog_mask: subprog pointers, val is subprog number.
* const_reg_vals[i] holds the 32-bit value for register i.
* Populated by compute_const_regs() pre-pass.
*/
u16 const_reg_mask;
u16 const_reg_map_mask;
u16 const_reg_subprog_mask;
u32 const_reg_vals[10];
};
#define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */
@@ -652,7 +716,7 @@ enum priv_stack_mode {
};
struct bpf_subprog_info {
/* 'start' has to be the first field otherwise find_subprog() won't work */
const char *name; /* name extracted from BTF */
u32 start; /* insn idx of function entry point */
u32 linfo_idx; /* The idx to the main_prog->aux->linfo */
u32 postorder_start; /* The idx to the env->cfg.insn_postorder */
@@ -787,6 +851,8 @@ struct bpf_verifier_env {
const struct bpf_line_info *prev_linfo;
struct bpf_verifier_log log;
struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */
/* subprog indices sorted in topological order: leaves first, callers last */
int subprog_topo_order[BPF_MAX_SUBPROGS + 2];
union {
struct bpf_idmap idmap_scratch;
struct bpf_idset idset_scratch;
@@ -805,6 +871,8 @@ struct bpf_verifier_env {
} cfg;
struct backtrack_state bt;
struct bpf_jmp_history_entry *cur_hist_ent;
/* Per-callsite copy of parent's converged at_stack_in for cross-frame fills. */
struct arg_track **callsite_at_stack;
u32 pass_cnt; /* number of times do_check() was called */
u32 subprog_cnt;
/* number of instructions analyzed by the verifier */
@@ -837,7 +905,9 @@ struct bpf_verifier_env {
u64 scratched_stack_slots;
u64 prev_log_pos, prev_insn_print_pos;
/* buffer used to temporary hold constants as scalar registers */
struct bpf_reg_state fake_reg[2];
struct bpf_reg_state fake_reg[1];
/* buffers used to save updated reg states while simulating branches */
struct bpf_reg_state true_reg1, true_reg2, false_reg1, false_reg2;
/* buffer used to generate temporary string representations,
* e.g., in reg_type_str() to generate reg_type string
*/
@@ -863,6 +933,30 @@ static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env
return &env->subprog_info[subprog];
}
struct bpf_call_summary {
u8 num_params;
bool is_void;
bool fastcall;
};
static inline bool bpf_helper_call(const struct bpf_insn *insn)
{
return insn->code == (BPF_JMP | BPF_CALL) &&
insn->src_reg == 0;
}
static inline bool bpf_pseudo_call(const struct bpf_insn *insn)
{
return insn->code == (BPF_JMP | BPF_CALL) &&
insn->src_reg == BPF_PSEUDO_CALL;
}
static inline bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
{
return insn->code == (BPF_JMP | BPF_CALL) &&
insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
}
__printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log,
const char *fmt, va_list args);
__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
@@ -891,6 +985,41 @@ __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,
bpf_log(&env->log, "verifier bug: " fmt "\n", ##args); \
})
static inline void mark_prune_point(struct bpf_verifier_env *env, int idx)
{
env->insn_aux_data[idx].prune_point = true;
}
static inline bool bpf_is_prune_point(struct bpf_verifier_env *env, int insn_idx)
{
return env->insn_aux_data[insn_idx].prune_point;
}
static inline void mark_force_checkpoint(struct bpf_verifier_env *env, int idx)
{
env->insn_aux_data[idx].force_checkpoint = true;
}
static inline bool bpf_is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx)
{
return env->insn_aux_data[insn_idx].force_checkpoint;
}
static inline void mark_calls_callback(struct bpf_verifier_env *env, int idx)
{
env->insn_aux_data[idx].calls_callback = true;
}
static inline bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx)
{
return env->insn_aux_data[insn_idx].calls_callback;
}
static inline void mark_jmp_point(struct bpf_verifier_env *env, int idx)
{
env->insn_aux_data[idx].jmp_point = true;
}
static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env)
{
struct bpf_verifier_state *cur = env->cur_state;
@@ -932,6 +1061,11 @@ static inline void bpf_trampoline_unpack_key(u64 key, u32 *obj_id, u32 *btf_id)
*btf_id = key & 0x7FFFFFFF;
}
int bpf_check_btf_info_early(struct bpf_verifier_env *env,
const union bpf_attr *attr, bpfptr_t uattr);
int bpf_check_btf_info(struct bpf_verifier_env *env,
const union bpf_attr *attr, bpfptr_t uattr);
int bpf_check_attach_target(struct bpf_verifier_log *log,
const struct bpf_prog *prog,
const struct bpf_prog *tgt_prog,
@@ -941,6 +1075,93 @@ void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab);
int mark_chain_precision(struct bpf_verifier_env *env, int regno);
int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx);
int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
void bpf_clear_jmp_history(struct bpf_verifier_state *state);
int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state,
const struct bpf_verifier_state *src);
struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx);
void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self);
void bpf_free_backedges(struct bpf_scc_visit *visit);
int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
int insn_flags, u64 linked_regs);
void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist);
void bpf_mark_reg_not_init(const struct bpf_verifier_env *env,
struct bpf_reg_state *reg);
void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg);
void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env,
struct bpf_verifier_state *st);
void bpf_clear_singular_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
int bpf_mark_chain_precision(struct bpf_verifier_env *env,
struct bpf_verifier_state *starting_state,
int regno, bool *changed);
static inline int bpf_get_spi(s32 off)
{
return (-off - 1) / BPF_REG_SIZE;
}
static inline struct bpf_func_state *bpf_func(struct bpf_verifier_env *env,
const struct bpf_reg_state *reg)
{
struct bpf_verifier_state *cur = env->cur_state;
return cur->frame[reg->frameno];
}
/* Return IP for a given frame in a call stack */
static inline u32 bpf_frame_insn_idx(struct bpf_verifier_state *st, u32 frame)
{
return frame == st->curframe
? st->insn_idx
: st->frame[frame + 1]->callsite;
}
static inline bool bpf_is_jmp_point(struct bpf_verifier_env *env, int insn_idx)
{
return env->insn_aux_data[insn_idx].jmp_point;
}
static inline bool bpf_is_spilled_reg(const struct bpf_stack_state *stack)
{
return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL;
}
static inline bool bpf_is_spilled_scalar_reg(const struct bpf_stack_state *stack)
{
return bpf_is_spilled_reg(stack) && stack->spilled_ptr.type == SCALAR_VALUE;
}
static inline bool bpf_register_is_null(struct bpf_reg_state *reg)
{
return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
}
static inline void bpf_bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)
{
bt->reg_masks[frame] |= 1 << reg;
}
static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
{
bt->stack_masks[frame] |= 1ull << slot;
}
static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u32 reg)
{
return bt->reg_masks[frame] & (1 << reg);
}
static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot)
{
return bt->stack_masks[frame] & (1ull << slot);
}
bool bpf_map_is_rdonly(const struct bpf_map *map);
int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val,
bool is_ldsx);
#define BPF_BASE_TYPE_MASK GENMASK(BPF_BASE_TYPE_BITS - 1, 0)
/* extract base type from bpf_{arg, return, reg}_type. */
@@ -1077,22 +1298,194 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie
u32 frameno, bool print_all);
void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,
u32 frameno);
u32 bpf_vlog_alignment(u32 pos);
struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off);
int bpf_jmp_offset(struct bpf_insn *insn);
struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx);
void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask);
bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx);
bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog);
int bpf_find_subprog(struct bpf_verifier_env *env, int off);
int bpf_compute_const_regs(struct bpf_verifier_env *env);
int bpf_prune_dead_branches(struct bpf_verifier_env *env);
int bpf_check_cfg(struct bpf_verifier_env *env);
int bpf_compute_postorder(struct bpf_verifier_env *env);
int bpf_compute_scc(struct bpf_verifier_env *env);
struct bpf_map_desc {
struct bpf_map *ptr;
int uid;
};
struct bpf_kfunc_call_arg_meta {
/* In parameters */
struct btf *btf;
u32 func_id;
u32 kfunc_flags;
const struct btf_type *func_proto;
const char *func_name;
/* Out parameters */
u32 ref_obj_id;
u8 release_regno;
bool r0_rdonly;
u32 ret_btf_id;
u64 r0_size;
u32 subprogno;
struct {
u64 value;
bool found;
} arg_constant;
/* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling,
* generally to pass info about user-defined local kptr types to later
* verification logic
* bpf_obj_drop/bpf_percpu_obj_drop
* Record the local kptr type to be drop'd
* bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type)
* Record the local kptr type to be refcount_incr'd and use
* arg_owning_ref to determine whether refcount_acquire should be
* fallible
*/
struct btf *arg_btf;
u32 arg_btf_id;
bool arg_owning_ref;
bool arg_prog;
struct {
struct btf_field *field;
} arg_list_head;
struct {
struct btf_field *field;
} arg_rbtree_root;
struct {
enum bpf_dynptr_type type;
u32 id;
u32 ref_obj_id;
} initialized_dynptr;
struct {
u8 spi;
u8 frameno;
} iter;
struct bpf_map_desc map;
u64 mem_size;
};
int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id,
const struct bpf_func_proto **ptr);
int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id,
s16 offset, struct bpf_kfunc_call_arg_meta *meta);
bool bpf_is_async_callback_calling_insn(struct bpf_insn *insn);
bool bpf_is_sync_callback_calling_insn(struct bpf_insn *insn);
static inline bool bpf_is_iter_next_kfunc(struct bpf_kfunc_call_arg_meta *meta)
{
return meta->kfunc_flags & KF_ITER_NEXT;
}
static inline bool bpf_is_kfunc_sleepable(struct bpf_kfunc_call_arg_meta *meta)
{
return meta->kfunc_flags & KF_SLEEPABLE;
}
bool bpf_is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta);
struct bpf_iarray *bpf_iarray_realloc(struct bpf_iarray *old, size_t n_elem);
int bpf_copy_insn_array_uniq(struct bpf_map *map, u32 start, u32 end, u32 *off);
bool bpf_insn_is_cond_jump(u8 code);
bool bpf_is_may_goto_insn(struct bpf_insn *insn);
void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn);
bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,
struct bpf_call_summary *cs);
s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env,
struct bpf_insn *insn, int arg,
int insn_idx);
s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env,
struct bpf_insn *insn, int arg,
int insn_idx);
int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env);
int bpf_stack_liveness_init(struct bpf_verifier_env *env);
void bpf_stack_liveness_free(struct bpf_verifier_env *env);
int bpf_update_live_stack(struct bpf_verifier_env *env);
int bpf_mark_stack_read(struct bpf_verifier_env *env, u32 frameno, u32 insn_idx, u64 mask);
void bpf_mark_stack_write(struct bpf_verifier_env *env, u32 frameno, u64 mask);
int bpf_reset_stack_write_marks(struct bpf_verifier_env *env, u32 insn_idx);
int bpf_commit_stack_write_marks(struct bpf_verifier_env *env);
int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi);
void bpf_reset_live_stack_callchain(struct bpf_verifier_env *env);
int bpf_compute_live_registers(struct bpf_verifier_env *env);
#define BPF_MAP_KEY_POISON (1ULL << 63)
#define BPF_MAP_KEY_SEEN (1ULL << 62)
static inline bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
{
return aux->map_ptr_state.poison;
}
static inline bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
{
return aux->map_ptr_state.unpriv;
}
static inline bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux)
{
return aux->map_key_state & BPF_MAP_KEY_POISON;
}
static inline bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux)
{
return !(aux->map_key_state & BPF_MAP_KEY_SEEN);
}
static inline u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux)
{
return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON);
}
#define MAX_PACKET_OFF 0xffff
#define CALLER_SAVED_REGS 6
enum bpf_reg_arg_type {
SRC_OP, /* register is used as source operand */
DST_OP, /* register is used as destination operand */
DST_OP_NO_MARK /* same as above, check only, don't mark */
};
#define MAX_KFUNC_DESCS 256
struct bpf_kfunc_desc {
struct btf_func_model func_model;
u32 func_id;
s32 imm;
u16 offset;
unsigned long addr;
};
struct bpf_kfunc_desc_tab {
/* Sorted by func_id (BTF ID) and offset (fd_array offset) during
* verification. JITs do lookups by bpf_insn, where func_id may not be
* available, therefore at the end of verification do_misc_fixups()
* sorts this by imm and offset.
*/
struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
u32 nr_descs;
};
/* Functions exported from verifier.c, used by fixups.c */
bool bpf_is_reg64(struct bpf_insn *insn, u32 regno, struct bpf_reg_state *reg, enum bpf_reg_arg_type t);
void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len);
void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog);
bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env);
bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm);
int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset);
int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
struct bpf_insn *insn_buf, int insn_idx, int *cnt);
/* Functions in fixups.c, called from bpf_check() */
int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env);
int bpf_optimize_bpf_loop(struct bpf_verifier_env *env);
void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env);
int bpf_opt_remove_dead_code(struct bpf_verifier_env *env);
int bpf_opt_remove_nops(struct bpf_verifier_env *env);
int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, const union bpf_attr *attr);
int bpf_convert_ctx_accesses(struct bpf_verifier_env *env);
int bpf_jit_subprogs(struct bpf_verifier_env *env);
int bpf_fixup_call_args(struct bpf_verifier_env *env);
int bpf_do_misc_fixups(struct bpf_verifier_env *env);
#endif /* _LINUX_BPF_VERIFIER_H */
include/linux/btf_ids.h
+1 -1
View File
@@ -217,7 +217,7 @@ BTF_SET8_END(name)
#else
#define BTF_ID_LIST(name) static u32 __maybe_unused name[64];
#define BTF_ID_LIST(name) static u32 __maybe_unused name[128];
#define BTF_ID(prefix, name)
#define BTF_ID_FLAGS(prefix, name, ...)
#define BTF_ID_UNUSED
include/linux/filter.h
-1
View File
@@ -21,7 +21,6 @@
#include <linux/if_vlan.h>
#include <linux/vmalloc.h>
#include <linux/sockptr.h>
#include <crypto/sha1.h>
#include <linux/u64_stats_sync.h>
#include <net/sch_generic.h>
include/linux/rcupdate.h
-9
View File
@@ -205,15 +205,6 @@ static inline void exit_tasks_rcu_start(void) { }
static inline void exit_tasks_rcu_finish(void) { }
#endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
/**
* rcu_trace_implies_rcu_gp - does an RCU Tasks Trace grace period imply an RCU grace period?
*
* Now that RCU Tasks Trace is implemented in terms of SRCU-fast, a
* call to synchronize_rcu_tasks_trace() is guaranteed to imply at least
* one call to synchronize_rcu().
*/
static inline bool rcu_trace_implies_rcu_gp(void) { return true; }
/**
* cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU
*
include/uapi/linux/bpf.h
+3 -1
View File
@@ -4645,7 +4645,9 @@ union bpf_attr {
* Description
* Discard reserved ring buffer sample, pointed to by *data*.
* If **BPF_RB_NO_WAKEUP** is specified in *flags*, no notification
* of new data availability is sent.
* of new data availability is sent. Discarded records remain in
* the ring buffer until consumed by user space, so a later submit
* using adaptive wakeup might not wake up the consumer.
* If **BPF_RB_FORCE_WAKEUP** is specified in *flags*, notification
* of new data availability is sent unconditionally.
* If **0** is specified in *flags*, an adaptive notification
include/uapi/linux/btf.h
+12
View File
@@ -8,6 +8,16 @@
#define BTF_MAGIC 0xeB9F
#define BTF_VERSION 1
/*
* BTF layout section consists of a struct btf_layout for each known
* kind at BTF encoding time.
*/
struct btf_layout {
__u8 info_sz; /* size of singular element after btf_type */
__u8 elem_sz; /* size of each of btf_vlen(t) elements */
__u16 flags; /* currently unused */
};
struct btf_header {
__u16 magic;
__u8 version;
@@ -19,6 +29,8 @@ struct btf_header {
__u32 type_len; /* length of type section */
__u32 str_off; /* offset of string section */
__u32 str_len; /* length of string section */
__u32 layout_off; /* offset of layout section */
__u32 layout_len; /* length of layout section */
};
/* Max # of type identifier */
kernel/bpf/Makefile
+2 -1
View File
@@ -6,11 +6,12 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse
endif
CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy)
obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o liveness.o
obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o liveness.o const_fold.o
obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o
obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o
obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o
obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o
obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o backtrack.o check_btf.o
obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o
obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o
obj-$(CONFIG_BPF_JIT) += trampoline.o
kernel/bpf/arraymap.c
+3 -1
View File
@@ -1015,8 +1015,10 @@ static void bpf_fd_array_map_clear(struct bpf_map *map, bool need_defer)
struct bpf_array *array = container_of(map, struct bpf_array, map);
int i;
for (i = 0; i < array->map.max_entries; i++)
for (i = 0; i < array->map.max_entries; i++) {
__fd_array_map_delete_elem(map, &i, need_defer);
cond_resched();
}
}
static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key,
kernel/bpf/backtrack.c
+934
View File
@@ -0,0 +1,934 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/filter.h>
#include <linux/bitmap.h>
#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
/* for any branch, call, exit record the history of jmps in the given state */
int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
int insn_flags, u64 linked_regs)
{
u32 cnt = cur->jmp_history_cnt;
struct bpf_jmp_history_entry *p;
size_t alloc_size;
/* combine instruction flags if we already recorded this instruction */
if (env->cur_hist_ent) {
/* atomic instructions push insn_flags twice, for READ and
* WRITE sides, but they should agree on stack slot
*/
verifier_bug_if((env->cur_hist_ent->flags & insn_flags) &&
(env->cur_hist_ent->flags & insn_flags) != insn_flags,
env, "insn history: insn_idx %d cur flags %x new flags %x",
env->insn_idx, env->cur_hist_ent->flags, insn_flags);
env->cur_hist_ent->flags |= insn_flags;
verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env,
"insn history: insn_idx %d linked_regs: %#llx",
env->insn_idx, env->cur_hist_ent->linked_regs);
env->cur_hist_ent->linked_regs = linked_regs;
return 0;
}
cnt++;
alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p)));
p = krealloc(cur->jmp_history, alloc_size, GFP_KERNEL_ACCOUNT);
if (!p)
return -ENOMEM;
cur->jmp_history = p;
p = &cur->jmp_history[cnt - 1];
p->idx = env->insn_idx;
p->prev_idx = env->prev_insn_idx;
p->flags = insn_flags;
p->linked_regs = linked_regs;
cur->jmp_history_cnt = cnt;
env->cur_hist_ent = p;
return 0;
}
static bool is_atomic_load_insn(const struct bpf_insn *insn)
{
return BPF_CLASS(insn->code) == BPF_STX &&
BPF_MODE(insn->code) == BPF_ATOMIC &&
insn->imm == BPF_LOAD_ACQ;
}
static bool is_atomic_fetch_insn(const struct bpf_insn *insn)
{
return BPF_CLASS(insn->code) == BPF_STX &&
BPF_MODE(insn->code) == BPF_ATOMIC &&
(insn->imm & BPF_FETCH);
}
static int insn_stack_access_spi(int insn_flags)
{
return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK;
}
static int insn_stack_access_frameno(int insn_flags)
{
return insn_flags & INSN_F_FRAMENO_MASK;
}
/* Backtrack one insn at a time. If idx is not at the top of recorded
* history then previous instruction came from straight line execution.
* Return -ENOENT if we exhausted all instructions within given state.
*
* It's legal to have a bit of a looping with the same starting and ending
* insn index within the same state, e.g.: 3->4->5->3, so just because current
* instruction index is the same as state's first_idx doesn't mean we are
* done. If there is still some jump history left, we should keep going. We
* need to take into account that we might have a jump history between given
* state's parent and itself, due to checkpointing. In this case, we'll have
* history entry recording a jump from last instruction of parent state and
* first instruction of given state.
*/
static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,
u32 *history)
{
u32 cnt = *history;
if (i == st->first_insn_idx) {
if (cnt == 0)
return -ENOENT;
if (cnt == 1 && st->jmp_history[0].idx == i)
return -ENOENT;
}
if (cnt && st->jmp_history[cnt - 1].idx == i) {
i = st->jmp_history[cnt - 1].prev_idx;
(*history)--;
} else {
i--;
}
return i;
}
static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st,
u32 hist_end, int insn_idx)
{
if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx)
return &st->jmp_history[hist_end - 1];
return NULL;
}
static inline void bt_init(struct backtrack_state *bt, u32 frame)
{
bt->frame = frame;
}
static inline void bt_reset(struct backtrack_state *bt)
{
struct bpf_verifier_env *env = bt->env;
memset(bt, 0, sizeof(*bt));
bt->env = env;
}
static inline u32 bt_empty(struct backtrack_state *bt)
{
u64 mask = 0;
int i;
for (i = 0; i <= bt->frame; i++)
mask |= bt->reg_masks[i] | bt->stack_masks[i];
return mask == 0;
}
static inline int bt_subprog_enter(struct backtrack_state *bt)
{
if (bt->frame == MAX_CALL_FRAMES - 1) {
verifier_bug(bt->env, "subprog enter from frame %d", bt->frame);
return -EFAULT;
}
bt->frame++;
return 0;
}
static inline int bt_subprog_exit(struct backtrack_state *bt)
{
if (bt->frame == 0) {
verifier_bug(bt->env, "subprog exit from frame 0");
return -EFAULT;
}
bt->frame--;
return 0;
}
static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)
{
bt->reg_masks[frame] &= ~(1 << reg);
}
static inline void bt_set_reg(struct backtrack_state *bt, u32 reg)
{
bpf_bt_set_frame_reg(bt, bt->frame, reg);
}
static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg)
{
bt_clear_frame_reg(bt, bt->frame, reg);
}
static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
{
bt->stack_masks[frame] &= ~(1ull << slot);
}
static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame)
{
return bt->reg_masks[frame];
}
static inline u32 bt_reg_mask(struct backtrack_state *bt)
{
return bt->reg_masks[bt->frame];
}
static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame)
{
return bt->stack_masks[frame];
}
static inline u64 bt_stack_mask(struct backtrack_state *bt)
{
return bt->stack_masks[bt->frame];
}
static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg)
{
return bt->reg_masks[bt->frame] & (1 << reg);
}
/* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */
static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask)
{
DECLARE_BITMAP(mask, 64);
bool first = true;
int i, n;
buf[0] = '\0';
bitmap_from_u64(mask, reg_mask);
for_each_set_bit(i, mask, 32) {
n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i);
first = false;
buf += n;
buf_sz -= n;
if (buf_sz < 0)
break;
}
}
/* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */
void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask)
{
DECLARE_BITMAP(mask, 64);
bool first = true;
int i, n;
buf[0] = '\0';
bitmap_from_u64(mask, stack_mask);
for_each_set_bit(i, mask, 64) {
n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8);
first = false;
buf += n;
buf_sz -= n;
if (buf_sz < 0)
break;
}
}
/* For given verifier state backtrack_insn() is called from the last insn to
* the first insn. Its purpose is to compute a bitmask of registers and
* stack slots that needs precision in the parent verifier state.
*
* @idx is an index of the instruction we are currently processing;
* @subseq_idx is an index of the subsequent instruction that:
* - *would be* executed next, if jump history is viewed in forward order;
* - *was* processed previously during backtracking.
*/
static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
struct bpf_jmp_history_entry *hist, struct backtrack_state *bt)
{
struct bpf_insn *insn = env->prog->insnsi + idx;
u8 class = BPF_CLASS(insn->code);
u8 opcode = BPF_OP(insn->code);
u8 mode = BPF_MODE(insn->code);
u32 dreg = insn->dst_reg;
u32 sreg = insn->src_reg;
u32 spi, i, fr;
if (insn->code == 0)
return 0;
if (env->log.level & BPF_LOG_LEVEL2) {
fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt));
verbose(env, "mark_precise: frame%d: regs=%s ",
bt->frame, env->tmp_str_buf);
bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));
verbose(env, "stack=%s before ", env->tmp_str_buf);
verbose(env, "%d: ", idx);
bpf_verbose_insn(env, insn);
}
/* If there is a history record that some registers gained range at this insn,
* propagate precision marks to those registers, so that bt_is_reg_set()
* accounts for these registers.
*/
bpf_bt_sync_linked_regs(bt, hist);
if (class == BPF_ALU || class == BPF_ALU64) {
if (!bt_is_reg_set(bt, dreg))
return 0;
if (opcode == BPF_END || opcode == BPF_NEG) {
/* sreg is reserved and unused
* dreg still need precision before this insn
*/
return 0;
} else if (opcode == BPF_MOV) {
if (BPF_SRC(insn->code) == BPF_X) {
/* dreg = sreg or dreg = (s8, s16, s32)sreg
* dreg needs precision after this insn
* sreg needs precision before this insn
*/
bt_clear_reg(bt, dreg);
if (sreg != BPF_REG_FP)
bt_set_reg(bt, sreg);
} else {
/* dreg = K
* dreg needs precision after this insn.
* Corresponding register is already marked
* as precise=true in this verifier state.
* No further markings in parent are necessary
*/
bt_clear_reg(bt, dreg);
}
} else {
if (BPF_SRC(insn->code) == BPF_X) {
/* dreg += sreg
* both dreg and sreg need precision
* before this insn
*/
if (sreg != BPF_REG_FP)
bt_set_reg(bt, sreg);
} /* else dreg += K
* dreg still needs precision before this insn
*/
}
} else if (class == BPF_LDX ||
is_atomic_load_insn(insn) ||
is_atomic_fetch_insn(insn)) {
u32 load_reg = dreg;
/*
* Atomic fetch operation writes the old value into
* a register (sreg or r0) and if it was tracked for
* precision, propagate to the stack slot like we do
* in regular ldx.
*/
if (is_atomic_fetch_insn(insn))
load_reg = insn->imm == BPF_CMPXCHG ?
BPF_REG_0 : sreg;
if (!bt_is_reg_set(bt, load_reg))
return 0;
bt_clear_reg(bt, load_reg);
/* scalars can only be spilled into stack w/o losing precision.
* Load from any other memory can be zero extended.
* The desire to keep that precision is already indicated
* by 'precise' mark in corresponding register of this state.
* No further tracking necessary.
*/
if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
return 0;
/* dreg = *(u64 *)[fp - off] was a fill from the stack.
* that [fp - off] slot contains scalar that needs to be
* tracked with precision
*/
spi = insn_stack_access_spi(hist->flags);
fr = insn_stack_access_frameno(hist->flags);
bpf_bt_set_frame_slot(bt, fr, spi);
} else if (class == BPF_STX || class == BPF_ST) {
if (bt_is_reg_set(bt, dreg))
/* stx & st shouldn't be using _scalar_ dst_reg
* to access memory. It means backtracking
* encountered a case of pointer subtraction.
*/
return -ENOTSUPP;
/* scalars can only be spilled into stack */
if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
return 0;
spi = insn_stack_access_spi(hist->flags);
fr = insn_stack_access_frameno(hist->flags);
if (!bt_is_frame_slot_set(bt, fr, spi))
return 0;
bt_clear_frame_slot(bt, fr, spi);
if (class == BPF_STX)
bt_set_reg(bt, sreg);
} else if (class == BPF_JMP || class == BPF_JMP32) {
if (bpf_pseudo_call(insn)) {
int subprog_insn_idx, subprog;
subprog_insn_idx = idx + insn->imm + 1;
subprog = bpf_find_subprog(env, subprog_insn_idx);
if (subprog < 0)
return -EFAULT;
if (bpf_subprog_is_global(env, subprog)) {
/* check that jump history doesn't have any
* extra instructions from subprog; the next
* instruction after call to global subprog
* should be literally next instruction in
* caller program
*/
verifier_bug_if(idx + 1 != subseq_idx, env,
"extra insn from subprog");
/* r1-r5 are invalidated after subprog call,
* so for global func call it shouldn't be set
* anymore
*/
if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
verifier_bug(env, "global subprog unexpected regs %x",
bt_reg_mask(bt));
return -EFAULT;
}
/* global subprog always sets R0 */
bt_clear_reg(bt, BPF_REG_0);
return 0;
} else {
/* static subprog call instruction, which
* means that we are exiting current subprog,
* so only r1-r5 could be still requested as
* precise, r0 and r6-r10 or any stack slot in
* the current frame should be zero by now
*/
if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
verifier_bug(env, "static subprog unexpected regs %x",
bt_reg_mask(bt));
return -EFAULT;
}
/* we are now tracking register spills correctly,
* so any instance of leftover slots is a bug
*/
if (bt_stack_mask(bt) != 0) {
verifier_bug(env,
"static subprog leftover stack slots %llx",
bt_stack_mask(bt));
return -EFAULT;
}
/* propagate r1-r5 to the caller */
for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
if (bt_is_reg_set(bt, i)) {
bt_clear_reg(bt, i);
bpf_bt_set_frame_reg(bt, bt->frame - 1, i);
}
}
if (bt_subprog_exit(bt))
return -EFAULT;
return 0;
}
} else if (bpf_is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) {
/* exit from callback subprog to callback-calling helper or
* kfunc call. Use idx/subseq_idx check to discern it from
* straight line code backtracking.
* Unlike the subprog call handling above, we shouldn't
* propagate precision of r1-r5 (if any requested), as they are
* not actually arguments passed directly to callback subprogs
*/
if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
verifier_bug(env, "callback unexpected regs %x",
bt_reg_mask(bt));
return -EFAULT;
}
if (bt_stack_mask(bt) != 0) {
verifier_bug(env, "callback leftover stack slots %llx",
bt_stack_mask(bt));
return -EFAULT;
}
/* clear r1-r5 in callback subprog's mask */
for (i = BPF_REG_1; i <= BPF_REG_5; i++)
bt_clear_reg(bt, i);
if (bt_subprog_exit(bt))
return -EFAULT;
return 0;
} else if (opcode == BPF_CALL) {
/* kfunc with imm==0 is invalid and fixup_kfunc_call will
* catch this error later. Make backtracking conservative
* with ENOTSUPP.
*/
if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0)
return -ENOTSUPP;
/* regular helper call sets R0 */
bt_clear_reg(bt, BPF_REG_0);
if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
/* if backtracking was looking for registers R1-R5
* they should have been found already.
*/
verifier_bug(env, "backtracking call unexpected regs %x",
bt_reg_mask(bt));
return -EFAULT;
}
if (insn->src_reg == BPF_REG_0 && insn->imm == BPF_FUNC_tail_call
&& subseq_idx - idx != 1) {
if (bt_subprog_enter(bt))
return -EFAULT;
}
} else if (opcode == BPF_EXIT) {
bool r0_precise;
/* Backtracking to a nested function call, 'idx' is a part of
* the inner frame 'subseq_idx' is a part of the outer frame.
* In case of a regular function call, instructions giving
* precision to registers R1-R5 should have been found already.
* In case of a callback, it is ok to have R1-R5 marked for
* backtracking, as these registers are set by the function
* invoking callback.
*/
if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx))
for (i = BPF_REG_1; i <= BPF_REG_5; i++)
bt_clear_reg(bt, i);
if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
verifier_bug(env, "backtracking exit unexpected regs %x",
bt_reg_mask(bt));
return -EFAULT;
}
/* BPF_EXIT in subprog or callback always returns
* right after the call instruction, so by checking
* whether the instruction at subseq_idx-1 is subprog
* call or not we can distinguish actual exit from
* *subprog* from exit from *callback*. In the former
* case, we need to propagate r0 precision, if
* necessary. In the former we never do that.
*/
r0_precise = subseq_idx - 1 >= 0 &&
bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) &&
bt_is_reg_set(bt, BPF_REG_0);
bt_clear_reg(bt, BPF_REG_0);
if (bt_subprog_enter(bt))
return -EFAULT;
if (r0_precise)
bt_set_reg(bt, BPF_REG_0);
/* r6-r9 and stack slots will stay set in caller frame
* bitmasks until we return back from callee(s)
*/
return 0;
} else if (BPF_SRC(insn->code) == BPF_X) {
if (!bt_is_reg_set(bt, dreg) && !bt_is_reg_set(bt, sreg))
return 0;
/* dreg <cond> sreg
* Both dreg and sreg need precision before
* this insn. If only sreg was marked precise
* before it would be equally necessary to
* propagate it to dreg.
*/
if (!hist || !(hist->flags & INSN_F_SRC_REG_STACK))
bt_set_reg(bt, sreg);
if (!hist || !(hist->flags & INSN_F_DST_REG_STACK))
bt_set_reg(bt, dreg);
} else if (BPF_SRC(insn->code) == BPF_K) {
/* dreg <cond> K
* Only dreg still needs precision before
* this insn, so for the K-based conditional
* there is nothing new to be marked.
*/
}
} else if (class == BPF_LD) {
if (!bt_is_reg_set(bt, dreg))
return 0;
bt_clear_reg(bt, dreg);
/* It's ld_imm64 or ld_abs or ld_ind.
* For ld_imm64 no further tracking of precision
* into parent is necessary
*/
if (mode == BPF_IND || mode == BPF_ABS)
/* to be analyzed */
return -ENOTSUPP;
}
/* Propagate precision marks to linked registers, to account for
* registers marked as precise in this function.
*/
bpf_bt_sync_linked_regs(bt, hist);
return 0;
}
/* the scalar precision tracking algorithm:
* . at the start all registers have precise=false.
* . scalar ranges are tracked as normal through alu and jmp insns.
* . once precise value of the scalar register is used in:
* . ptr + scalar alu
* . if (scalar cond K|scalar)
* . helper_call(.., scalar, ...) where ARG_CONST is expected
* backtrack through the verifier states and mark all registers and
* stack slots with spilled constants that these scalar registers
* should be precise.
* . during state pruning two registers (or spilled stack slots)
* are equivalent if both are not precise.
*
* Note the verifier cannot simply walk register parentage chain,
* since many different registers and stack slots could have been
* used to compute single precise scalar.
*
* The approach of starting with precise=true for all registers and then
* backtrack to mark a register as not precise when the verifier detects
* that program doesn't care about specific value (e.g., when helper
* takes register as ARG_ANYTHING parameter) is not safe.
*
* It's ok to walk single parentage chain of the verifier states.
* It's possible that this backtracking will go all the way till 1st insn.
* All other branches will be explored for needing precision later.
*
* The backtracking needs to deal with cases like:
* R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0)
* r9 -= r8
* r5 = r9
* if r5 > 0x79f goto pc+7
* R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff))
* r5 += 1
* ...
* call bpf_perf_event_output#25
* where .arg5_type = ARG_CONST_SIZE_OR_ZERO
*
* and this case:
* r6 = 1
* call foo // uses callee's r6 inside to compute r0
* r0 += r6
* if r0 == 0 goto
*
* to track above reg_mask/stack_mask needs to be independent for each frame.
*
* Also if parent's curframe > frame where backtracking started,
* the verifier need to mark registers in both frames, otherwise callees
* may incorrectly prune callers. This is similar to
* commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences")
*
* For now backtracking falls back into conservative marking.
*/
void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env,
struct bpf_verifier_state *st)
{
struct bpf_func_state *func;
struct bpf_reg_state *reg;
int i, j;
if (env->log.level & BPF_LOG_LEVEL2) {
verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n",
st->curframe);
}
/* big hammer: mark all scalars precise in this path.
* pop_stack may still get !precise scalars.
* We also skip current state and go straight to first parent state,
* because precision markings in current non-checkpointed state are
* not needed. See why in the comment in __mark_chain_precision below.
*/
for (st = st->parent; st; st = st->parent) {
for (i = 0; i <= st->curframe; i++) {
func = st->frame[i];
for (j = 0; j < BPF_REG_FP; j++) {
reg = &func->regs[j];
if (reg->type != SCALAR_VALUE || reg->precise)
continue;
reg->precise = true;
if (env->log.level & BPF_LOG_LEVEL2) {
verbose(env, "force_precise: frame%d: forcing r%d to be precise\n",
i, j);
}
}
for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
if (!bpf_is_spilled_reg(&func->stack[j]))
continue;
reg = &func->stack[j].spilled_ptr;
if (reg->type != SCALAR_VALUE || reg->precise)
continue;
reg->precise = true;
if (env->log.level & BPF_LOG_LEVEL2) {
verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n",
i, -(j + 1) * 8);
}
}
}
}
}
/*
* bpf_mark_chain_precision() backtracks BPF program instruction sequence and
* chain of verifier states making sure that register *regno* (if regno >= 0)
* and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked
* SCALARS, as well as any other registers and slots that contribute to
* a tracked state of given registers/stack slots, depending on specific BPF
* assembly instructions (see backtrack_insns() for exact instruction handling
* logic). This backtracking relies on recorded jmp_history and is able to
* traverse entire chain of parent states. This process ends only when all the
* necessary registers/slots and their transitive dependencies are marked as
* precise.
*
* One important and subtle aspect is that precise marks *do not matter* in
* the currently verified state (current state). It is important to understand
* why this is the case.
*
* First, note that current state is the state that is not yet "checkpointed",
* i.e., it is not yet put into env->explored_states, and it has no children
* states as well. It's ephemeral, and can end up either a) being discarded if
* compatible explored state is found at some point or BPF_EXIT instruction is
* reached or b) checkpointed and put into env->explored_states, branching out
* into one or more children states.
*
* In the former case, precise markings in current state are completely
* ignored by state comparison code (see regsafe() for details). Only
* checkpointed ("old") state precise markings are important, and if old
* state's register/slot is precise, regsafe() assumes current state's
* register/slot as precise and checks value ranges exactly and precisely. If
* states turn out to be compatible, current state's necessary precise
* markings and any required parent states' precise markings are enforced
* after the fact with propagate_precision() logic, after the fact. But it's
* important to realize that in this case, even after marking current state
* registers/slots as precise, we immediately discard current state. So what
* actually matters is any of the precise markings propagated into current
* state's parent states, which are always checkpointed (due to b) case above).
* As such, for scenario a) it doesn't matter if current state has precise
* markings set or not.
*
* Now, for the scenario b), checkpointing and forking into child(ren)
* state(s). Note that before current state gets to checkpointing step, any
* processed instruction always assumes precise SCALAR register/slot
* knowledge: if precise value or range is useful to prune jump branch, BPF
* verifier takes this opportunity enthusiastically. Similarly, when
* register's value is used to calculate offset or memory address, exact
* knowledge of SCALAR range is assumed, checked, and enforced. So, similar to
* what we mentioned above about state comparison ignoring precise markings
* during state comparison, BPF verifier ignores and also assumes precise
* markings *at will* during instruction verification process. But as verifier
* assumes precision, it also propagates any precision dependencies across
* parent states, which are not yet finalized, so can be further restricted
* based on new knowledge gained from restrictions enforced by their children
* states. This is so that once those parent states are finalized, i.e., when
* they have no more active children state, state comparison logic in
* is_state_visited() would enforce strict and precise SCALAR ranges, if
* required for correctness.
*
* To build a bit more intuition, note also that once a state is checkpointed,
* the path we took to get to that state is not important. This is crucial
* property for state pruning. When state is checkpointed and finalized at
* some instruction index, it can be correctly and safely used to "short
* circuit" any *compatible* state that reaches exactly the same instruction
* index. I.e., if we jumped to that instruction from a completely different
* code path than original finalized state was derived from, it doesn't
* matter, current state can be discarded because from that instruction
* forward having a compatible state will ensure we will safely reach the
* exit. States describe preconditions for further exploration, but completely
* forget the history of how we got here.
*
* This also means that even if we needed precise SCALAR range to get to
* finalized state, but from that point forward *that same* SCALAR register is
* never used in a precise context (i.e., it's precise value is not needed for
* correctness), it's correct and safe to mark such register as "imprecise"
* (i.e., precise marking set to false). This is what we rely on when we do
* not set precise marking in current state. If no child state requires
* precision for any given SCALAR register, it's safe to dictate that it can
* be imprecise. If any child state does require this register to be precise,
* we'll mark it precise later retroactively during precise markings
* propagation from child state to parent states.
*
* Skipping precise marking setting in current state is a mild version of
* relying on the above observation. But we can utilize this property even
* more aggressively by proactively forgetting any precise marking in the
* current state (which we inherited from the parent state), right before we
* checkpoint it and branch off into new child state. This is done by
* mark_all_scalars_imprecise() to hopefully get more permissive and generic
* finalized states which help in short circuiting more future states.
*/
int bpf_mark_chain_precision(struct bpf_verifier_env *env,
struct bpf_verifier_state *starting_state,
int regno,
bool *changed)
{
struct bpf_verifier_state *st = starting_state;
struct backtrack_state *bt = &env->bt;
int first_idx = st->first_insn_idx;
int last_idx = starting_state->insn_idx;
int subseq_idx = -1;
struct bpf_func_state *func;
bool tmp, skip_first = true;
struct bpf_reg_state *reg;
int i, fr, err;
if (!env->bpf_capable)
return 0;
changed = changed ?: &tmp;
/* set frame number from which we are starting to backtrack */
bt_init(bt, starting_state->curframe);
/* Do sanity checks against current state of register and/or stack
* slot, but don't set precise flag in current state, as precision
* tracking in the current state is unnecessary.
*/
func = st->frame[bt->frame];
if (regno >= 0) {
reg = &func->regs[regno];
if (reg->type != SCALAR_VALUE) {
verifier_bug(env, "backtracking misuse");
return -EFAULT;
}
bt_set_reg(bt, regno);
}
if (bt_empty(bt))
return 0;
for (;;) {
DECLARE_BITMAP(mask, 64);
u32 history = st->jmp_history_cnt;
struct bpf_jmp_history_entry *hist;
if (env->log.level & BPF_LOG_LEVEL2) {
verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",
bt->frame, last_idx, first_idx, subseq_idx);
}
if (last_idx < 0) {
/* we are at the entry into subprog, which
* is expected for global funcs, but only if
* requested precise registers are R1-R5
* (which are global func's input arguments)
*/
if (st->curframe == 0 &&
st->frame[0]->subprogno > 0 &&
st->frame[0]->callsite == BPF_MAIN_FUNC &&
bt_stack_mask(bt) == 0 &&
(bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) {
bitmap_from_u64(mask, bt_reg_mask(bt));
for_each_set_bit(i, mask, 32) {
reg = &st->frame[0]->regs[i];
bt_clear_reg(bt, i);
if (reg->type == SCALAR_VALUE) {
reg->precise = true;
*changed = true;
}
}
return 0;
}
verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx",
st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt));
return -EFAULT;
}
for (i = last_idx;;) {
if (skip_first) {
err = 0;
skip_first = false;
} else {
hist = get_jmp_hist_entry(st, history, i);
err = backtrack_insn(env, i, subseq_idx, hist, bt);
}
if (err == -ENOTSUPP) {
bpf_mark_all_scalars_precise(env, starting_state);
bt_reset(bt);
return 0;
} else if (err) {
return err;
}
if (bt_empty(bt))
/* Found assignment(s) into tracked register in this state.
* Since this state is already marked, just return.
* Nothing to be tracked further in the parent state.
*/
return 0;
subseq_idx = i;
i = get_prev_insn_idx(st, i, &history);
if (i == -ENOENT)
break;
if (i >= env->prog->len) {
/* This can happen if backtracking reached insn 0
* and there are still reg_mask or stack_mask
* to backtrack.
* It means the backtracking missed the spot where
* particular register was initialized with a constant.
*/
verifier_bug(env, "backtracking idx %d", i);
return -EFAULT;
}
}
st = st->parent;
if (!st)
break;
for (fr = bt->frame; fr >= 0; fr--) {
func = st->frame[fr];
bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr));
for_each_set_bit(i, mask, 32) {
reg = &func->regs[i];
if (reg->type != SCALAR_VALUE) {
bt_clear_frame_reg(bt, fr, i);
continue;
}
if (reg->precise) {
bt_clear_frame_reg(bt, fr, i);
} else {
reg->precise = true;
*changed = true;
}
}
bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr));
for_each_set_bit(i, mask, 64) {
if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE,
env, "stack slot %d, total slots %d",
i, func->allocated_stack / BPF_REG_SIZE))
return -EFAULT;
if (!bpf_is_spilled_scalar_reg(&func->stack[i])) {
bt_clear_frame_slot(bt, fr, i);
continue;
}
reg = &func->stack[i].spilled_ptr;
if (reg->precise) {
bt_clear_frame_slot(bt, fr, i);
} else {
reg->precise = true;
*changed = true;
}
}
if (env->log.level & BPF_LOG_LEVEL2) {
fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
bt_frame_reg_mask(bt, fr));
verbose(env, "mark_precise: frame%d: parent state regs=%s ",
fr, env->tmp_str_buf);
bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
bt_frame_stack_mask(bt, fr));
verbose(env, "stack=%s: ", env->tmp_str_buf);
print_verifier_state(env, st, fr, true);
}
}
if (bt_empty(bt))
return 0;
subseq_idx = first_idx;
last_idx = st->last_insn_idx;
first_idx = st->first_insn_idx;
}
/* if we still have requested precise regs or slots, we missed
* something (e.g., stack access through non-r10 register), so
* fallback to marking all precise
*/
if (!bt_empty(bt)) {
bpf_mark_all_scalars_precise(env, starting_state);
bt_reset(bt);
}
return 0;
}
kernel/bpf/bpf_cgrp_storage.c
+5 -6
View File
@@ -76,7 +76,7 @@ static long bpf_cgrp_storage_update_elem(struct bpf_map *map, void *key,
return PTR_ERR(cgroup);
sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
value, map_flags, false, GFP_ATOMIC);
value, map_flags, false);
cgroup_put(cgroup);
return PTR_ERR_OR_ZERO(sdata);
}
@@ -114,7 +114,7 @@ static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key)
static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr)
{
return bpf_local_storage_map_alloc(attr, &cgroup_cache, true);
return bpf_local_storage_map_alloc(attr, &cgroup_cache);
}
static void cgroup_storage_map_free(struct bpf_map *map)
@@ -122,9 +122,8 @@ static void cgroup_storage_map_free(struct bpf_map *map)
bpf_local_storage_map_free(map, &cgroup_cache);
}
/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,
void *, value, u64, flags, gfp_t, gfp_flags)
BPF_CALL_4(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,
void *, value, u64, flags)
{
struct bpf_local_storage_data *sdata;
@@ -143,7 +142,7 @@ BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,
if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&
(flags & BPF_LOCAL_STORAGE_GET_F_CREATE))
sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
value, BPF_NOEXIST, false, gfp_flags);
value, BPF_NOEXIST, false);
out:
return IS_ERR_OR_NULL(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;
kernel/bpf/bpf_inode_storage.c
+5 -6
View File
@@ -98,7 +98,7 @@ static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
sdata = bpf_local_storage_update(file_inode(fd_file(f)),
(struct bpf_local_storage_map *)map,
value, map_flags, false, GFP_ATOMIC);
value, map_flags, false);
return PTR_ERR_OR_ZERO(sdata);
}
@@ -122,9 +122,8 @@ static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
return inode_storage_delete(file_inode(fd_file(f)), map);
}
/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
void *, value, u64, flags, gfp_t, gfp_flags)
BPF_CALL_4(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
void *, value, u64, flags)
{
struct bpf_local_storage_data *sdata;
@@ -150,7 +149,7 @@ BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
sdata = bpf_local_storage_update(
inode, (struct bpf_local_storage_map *)map, value,
BPF_NOEXIST, false, gfp_flags);
BPF_NOEXIST, false);
return IS_ERR(sdata) ? (unsigned long)NULL :
(unsigned long)sdata->data;
}
@@ -179,7 +178,7 @@ static int notsupp_get_next_key(struct bpf_map *map, void *key,
static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
{
return bpf_local_storage_map_alloc(attr, &inode_cache, false);
return bpf_local_storage_map_alloc(attr, &inode_cache);
}
static void inode_storage_map_free(struct bpf_map *map)
kernel/bpf/bpf_local_storage.c
+32 -125
View File
@@ -68,25 +68,19 @@ static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem)
struct bpf_local_storage_elem *
bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,
void *value, bool swap_uptrs, gfp_t gfp_flags)
void *value, bool swap_uptrs)
{
struct bpf_local_storage_elem *selem;
if (mem_charge(smap, owner, smap->elem_size))
return NULL;
if (smap->use_kmalloc_nolock) {
selem = bpf_map_kmalloc_nolock(&smap->map, smap->elem_size,
__GFP_ZERO, NUMA_NO_NODE);
} else {
selem = bpf_map_kzalloc(&smap->map, smap->elem_size,
gfp_flags | __GFP_NOWARN);
}
selem = bpf_map_kmalloc_nolock(&smap->map, smap->elem_size,
__GFP_ZERO, NUMA_NO_NODE);
if (selem) {
RCU_INIT_POINTER(SDATA(selem)->smap, smap);
atomic_set(&selem->state, 0);
selem->use_kmalloc_nolock = smap->use_kmalloc_nolock;
if (value) {
/* No need to call check_and_init_map_value as memory is zero init */
@@ -102,8 +96,7 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,
return NULL;
}
/* rcu tasks trace callback for use_kmalloc_nolock == false */
static void __bpf_local_storage_free_trace_rcu(struct rcu_head *rcu)
static void bpf_local_storage_free_trace_rcu(struct rcu_head *rcu)
{
struct bpf_local_storage *local_storage;
@@ -115,47 +108,14 @@ static void __bpf_local_storage_free_trace_rcu(struct rcu_head *rcu)
kfree(local_storage);
}
/* Handle use_kmalloc_nolock == false */
static void __bpf_local_storage_free(struct bpf_local_storage *local_storage,
bool vanilla_rcu)
{
if (vanilla_rcu)
kfree_rcu(local_storage, rcu);
else
call_rcu_tasks_trace(&local_storage->rcu,
__bpf_local_storage_free_trace_rcu);
}
static void bpf_local_storage_free_rcu(struct rcu_head *rcu)
{
struct bpf_local_storage *local_storage;
local_storage = container_of(rcu, struct bpf_local_storage, rcu);
kfree_nolock(local_storage);
}
static void bpf_local_storage_free_trace_rcu(struct rcu_head *rcu)
{
/*
* RCU Tasks Trace grace period implies RCU grace period, do
* kfree() directly.
*/
bpf_local_storage_free_rcu(rcu);
}
static void bpf_local_storage_free(struct bpf_local_storage *local_storage,
bool reuse_now)
{
if (!local_storage)
return;
if (!local_storage->use_kmalloc_nolock) {
__bpf_local_storage_free(local_storage, reuse_now);
return;
}
if (reuse_now) {
call_rcu(&local_storage->rcu, bpf_local_storage_free_rcu);
kfree_rcu(local_storage, rcu);
return;
}
@@ -163,42 +123,7 @@ static void bpf_local_storage_free(struct bpf_local_storage *local_storage,
bpf_local_storage_free_trace_rcu);
}
/* rcu callback for use_kmalloc_nolock == false */
static void __bpf_selem_free_rcu(struct rcu_head *rcu)
{
struct bpf_local_storage_elem *selem;
struct bpf_local_storage_map *smap;
selem = container_of(rcu, struct bpf_local_storage_elem, rcu);
/* bpf_selem_unlink_nofail may have already cleared smap and freed fields. */
smap = rcu_dereference_check(SDATA(selem)->smap, 1);
if (smap)
bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);
kfree(selem);
}
/* rcu tasks trace callback for use_kmalloc_nolock == false */
static void __bpf_selem_free_trace_rcu(struct rcu_head *rcu)
{
/*
* RCU Tasks Trace grace period implies RCU grace period, do
* kfree() directly.
*/
__bpf_selem_free_rcu(rcu);
}
/* Handle use_kmalloc_nolock == false */
static void __bpf_selem_free(struct bpf_local_storage_elem *selem,
bool vanilla_rcu)
{
if (vanilla_rcu)
call_rcu(&selem->rcu, __bpf_selem_free_rcu);
else
call_rcu_tasks_trace(&selem->rcu, __bpf_selem_free_trace_rcu);
}
static void bpf_selem_free_rcu(struct rcu_head *rcu)
static void bpf_selem_free_trace_rcu(struct rcu_head *rcu)
{
struct bpf_local_storage_elem *selem;
struct bpf_local_storage_map *smap;
@@ -209,37 +134,24 @@ static void bpf_selem_free_rcu(struct rcu_head *rcu)
if (smap)
bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);
kfree_nolock(selem);
}
static void bpf_selem_free_trace_rcu(struct rcu_head *rcu)
{
/*
* RCU Tasks Trace grace period implies RCU grace period, do
* kfree() directly.
*/
bpf_selem_free_rcu(rcu);
kfree(selem);
}
void bpf_selem_free(struct bpf_local_storage_elem *selem,
bool reuse_now)
{
if (!selem->use_kmalloc_nolock) {
/*
* No uptr will be unpin even when reuse_now == false since uptr
* is only supported in task local storage, where
* smap->use_kmalloc_nolock == true.
*/
__bpf_selem_free(selem, reuse_now);
return;
}
struct bpf_local_storage_map *smap;
smap = rcu_dereference_check(SDATA(selem)->smap, 1);
if (reuse_now) {
/*
* While it is okay to call bpf_obj_free_fields() that unpins uptr when
* reuse_now == true, keep it in bpf_selem_free_rcu() for simplicity.
*/
call_rcu(&selem->rcu, bpf_selem_free_rcu);
if (smap)
bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);
kfree_rcu(selem, rcu);
return;
}
@@ -393,6 +305,9 @@ int bpf_selem_unlink(struct bpf_local_storage_elem *selem)
unsigned long flags;
int err;
if (in_nmi())
return -EOPNOTSUPP;
if (unlikely(!selem_linked_to_storage_lockless(selem)))
/* selem has already been unlinked from sk */
return 0;
@@ -494,6 +409,14 @@ static void bpf_selem_unlink_nofail(struct bpf_local_storage_elem *selem,
}
raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);
}
/*
* Highly unlikely scenario: memory leak
*
* When destroy() fails to acqurire local_storage->lock and initializes
* selem->local_storage to NULL before any racing map_free() sees the same
* selem, no one will free the local storage.
*/
WARN_ON_ONCE(err && !in_map_free);
if (!err || !in_map_free)
RCU_INIT_POINTER(selem->local_storage, NULL);
}
@@ -552,8 +475,7 @@ static int check_flags(const struct bpf_local_storage_data *old_sdata,
int bpf_local_storage_alloc(void *owner,
struct bpf_local_storage_map *smap,
struct bpf_local_storage_elem *first_selem,
gfp_t gfp_flags)
struct bpf_local_storage_elem *first_selem)
{
struct bpf_local_storage *prev_storage, *storage;
struct bpf_local_storage **owner_storage_ptr;
@@ -565,12 +487,8 @@ int bpf_local_storage_alloc(void *owner,
if (err)
return err;
if (smap->use_kmalloc_nolock)
storage = bpf_map_kmalloc_nolock(&smap->map, sizeof(*storage),
__GFP_ZERO, NUMA_NO_NODE);
else
storage = bpf_map_kzalloc(&smap->map, sizeof(*storage),
gfp_flags | __GFP_NOWARN);
storage = bpf_map_kmalloc_nolock(&smap->map, sizeof(*storage),
__GFP_ZERO, NUMA_NO_NODE);
if (!storage) {
err = -ENOMEM;
goto uncharge;
@@ -580,7 +498,6 @@ int bpf_local_storage_alloc(void *owner,
raw_res_spin_lock_init(&storage->lock);
storage->owner = owner;
storage->mem_charge = sizeof(*storage);
storage->use_kmalloc_nolock = smap->use_kmalloc_nolock;
refcount_set(&storage->owner_refcnt, 1);
bpf_selem_link_storage_nolock(storage, first_selem);
@@ -628,7 +545,7 @@ uncharge:
*/
struct bpf_local_storage_data *
bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags)
void *value, u64 map_flags, bool swap_uptrs)
{
struct bpf_local_storage_data *old_sdata = NULL;
struct bpf_local_storage_elem *alloc_selem, *selem = NULL;
@@ -645,9 +562,6 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
!btf_record_has_field(smap->map.record, BPF_SPIN_LOCK)))
return ERR_PTR(-EINVAL);
if (gfp_flags == GFP_KERNEL && (map_flags & ~BPF_F_LOCK) != BPF_NOEXIST)
return ERR_PTR(-EINVAL);
local_storage = rcu_dereference_check(*owner_storage(smap, owner),
bpf_rcu_lock_held());
if (!local_storage || hlist_empty(&local_storage->list)) {
@@ -656,11 +570,11 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
if (err)
return ERR_PTR(err);
selem = bpf_selem_alloc(smap, owner, value, swap_uptrs, gfp_flags);
selem = bpf_selem_alloc(smap, owner, value, swap_uptrs);
if (!selem)
return ERR_PTR(-ENOMEM);
err = bpf_local_storage_alloc(owner, smap, selem, gfp_flags);
err = bpf_local_storage_alloc(owner, smap, selem);
if (err) {
bpf_selem_free(selem, true);
mem_uncharge(smap, owner, smap->elem_size);
@@ -690,7 +604,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
/* A lookup has just been done before and concluded a new selem is
* needed. The chance of an unnecessary alloc is unlikely.
*/
alloc_selem = selem = bpf_selem_alloc(smap, owner, value, swap_uptrs, gfp_flags);
alloc_selem = selem = bpf_selem_alloc(smap, owner, value, swap_uptrs);
if (!alloc_selem)
return ERR_PTR(-ENOMEM);
@@ -857,8 +771,7 @@ u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map)
struct bpf_map *
bpf_local_storage_map_alloc(union bpf_attr *attr,
struct bpf_local_storage_cache *cache,
bool use_kmalloc_nolock)
struct bpf_local_storage_cache *cache)
{
struct bpf_local_storage_map *smap;
unsigned int i;
@@ -890,12 +803,6 @@ bpf_local_storage_map_alloc(union bpf_attr *attr,
smap->elem_size = offsetof(struct bpf_local_storage_elem,
sdata.data[attr->value_size]);
/* In PREEMPT_RT, kmalloc(GFP_ATOMIC) is still not safe in non
* preemptible context. Thus, enforce all storages to use
* kmalloc_nolock() when CONFIG_PREEMPT_RT is enabled.
*/
smap->use_kmalloc_nolock = IS_ENABLED(CONFIG_PREEMPT_RT) ? true : use_kmalloc_nolock;
smap->cache_idx = bpf_local_storage_cache_idx_get(cache);
return &smap->map;
kernel/bpf/bpf_lsm.c
+4 -3
View File
@@ -359,8 +359,6 @@ BTF_ID(func, bpf_lsm_sb_umount)
BTF_ID(func, bpf_lsm_settime)
#ifdef CONFIG_SECURITY_NETWORK
BTF_ID(func, bpf_lsm_inet_conn_established)
BTF_ID(func, bpf_lsm_socket_accept)
BTF_ID(func, bpf_lsm_socket_bind)
BTF_ID(func, bpf_lsm_socket_connect)
@@ -381,8 +379,9 @@ BTF_ID(func, bpf_lsm_syslog)
BTF_ID(func, bpf_lsm_task_alloc)
BTF_ID(func, bpf_lsm_task_prctl)
BTF_ID(func, bpf_lsm_task_setscheduler)
BTF_ID(func, bpf_lsm_task_to_inode)
BTF_ID(func, bpf_lsm_userns_create)
BTF_ID(func, bpf_lsm_bdev_alloc_security)
BTF_ID(func, bpf_lsm_bdev_setintegrity)
BTF_SET_END(sleepable_lsm_hooks)
BTF_SET_START(untrusted_lsm_hooks)
@@ -395,6 +394,8 @@ BTF_ID(func, bpf_lsm_sk_alloc_security)
BTF_ID(func, bpf_lsm_sk_free_security)
#endif /* CONFIG_SECURITY_NETWORK */
BTF_ID(func, bpf_lsm_task_free)
BTF_ID(func, bpf_lsm_bdev_alloc_security)
BTF_ID(func, bpf_lsm_bdev_free_security)
BTF_SET_END(untrusted_lsm_hooks)
bool bpf_lsm_is_sleepable_hook(u32 btf_id)
kernel/bpf/bpf_task_storage.c
+5 -6
View File
@@ -118,7 +118,7 @@ static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key,
sdata = bpf_local_storage_update(
task, (struct bpf_local_storage_map *)map, value, map_flags,
true, GFP_ATOMIC);
true);
err = PTR_ERR_OR_ZERO(sdata);
out:
@@ -165,9 +165,8 @@ out:
return err;
}
/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *,
task, void *, value, u64, flags, gfp_t, gfp_flags)
BPF_CALL_4(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *,
task, void *, value, u64, flags)
{
struct bpf_local_storage_data *sdata;
@@ -184,7 +183,7 @@ BPF_CALL_5(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *,
(flags & BPF_LOCAL_STORAGE_GET_F_CREATE)) {
sdata = bpf_local_storage_update(
task, (struct bpf_local_storage_map *)map, value,
BPF_NOEXIST, false, gfp_flags);
BPF_NOEXIST, false);
return IS_ERR(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;
}
@@ -212,7 +211,7 @@ static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key)
static struct bpf_map *task_storage_map_alloc(union bpf_attr *attr)
{
return bpf_local_storage_map_alloc(attr, &task_cache, true);
return bpf_local_storage_map_alloc(attr, &task_cache);
}
static void task_storage_map_free(struct bpf_map *map)
kernel/bpf/btf.c
+68 -18
View File
@@ -270,6 +270,7 @@ struct btf {
struct btf_id_dtor_kfunc_tab *dtor_kfunc_tab;
struct btf_struct_metas *struct_meta_tab;
struct btf_struct_ops_tab *struct_ops_tab;
struct btf_layout *layout;
/* split BTF support */
struct btf *base_btf;
@@ -1707,6 +1708,11 @@ static void btf_verifier_log_hdr(struct btf_verifier_env *env,
__btf_verifier_log(log, "type_len: %u\n", hdr->type_len);
__btf_verifier_log(log, "str_off: %u\n", hdr->str_off);
__btf_verifier_log(log, "str_len: %u\n", hdr->str_len);
if (hdr->hdr_len >= sizeof(struct btf_header) &&
btf_data_size >= hdr->hdr_len) {
__btf_verifier_log(log, "layout_off: %u\n", hdr->layout_off);
__btf_verifier_log(log, "layout_len: %u\n", hdr->layout_len);
}
__btf_verifier_log(log, "btf_total_size: %u\n", btf_data_size);
}
@@ -5526,7 +5532,8 @@ static int btf_parse_str_sec(struct btf_verifier_env *env)
start = btf->nohdr_data + hdr->str_off;
end = start + hdr->str_len;
if (end != btf->data + btf->data_size) {
if (hdr->hdr_len < sizeof(struct btf_header) &&
end != btf->data + btf->data_size) {
btf_verifier_log(env, "String section is not at the end");
return -EINVAL;
}
@@ -5547,9 +5554,46 @@ static int btf_parse_str_sec(struct btf_verifier_env *env)
return 0;
}
static int btf_parse_layout_sec(struct btf_verifier_env *env)
{
const struct btf_header *hdr = &env->btf->hdr;
struct btf *btf = env->btf;
void *start, *end;
if (hdr->hdr_len < sizeof(struct btf_header) ||
hdr->layout_len == 0)
return 0;
/* Layout section must align to 4 bytes */
if (hdr->layout_off & (sizeof(u32) - 1)) {
btf_verifier_log(env, "Unaligned layout_off");
return -EINVAL;
}
start = btf->nohdr_data + hdr->layout_off;
end = start + hdr->layout_len;
if (hdr->layout_len < sizeof(struct btf_layout)) {
btf_verifier_log(env, "Layout section is too small");
return -EINVAL;
}
if (hdr->layout_len % sizeof(struct btf_layout) != 0) {
btf_verifier_log(env, "layout_len is not multiple of %zu",
sizeof(struct btf_layout));
return -EINVAL;
}
if (end > btf->data + btf->data_size) {
btf_verifier_log(env, "Layout section is too big");
return -EINVAL;
}
btf->layout = start;
return 0;
}
static const size_t btf_sec_info_offset[] = {
offsetof(struct btf_header, type_off),
offsetof(struct btf_header, str_off),
offsetof(struct btf_header, layout_off)
};
static int btf_sec_info_cmp(const void *a, const void *b)
@@ -5565,24 +5609,28 @@ static int btf_check_sec_info(struct btf_verifier_env *env,
{
struct btf_sec_info secs[ARRAY_SIZE(btf_sec_info_offset)];
u32 total, expected_total, i;
u32 nr_secs = ARRAY_SIZE(btf_sec_info_offset);
const struct btf_header *hdr;
const struct btf *btf;
btf = env->btf;
hdr = &btf->hdr;
if (hdr->hdr_len < sizeof(struct btf_header) || hdr->layout_len == 0)
nr_secs--;
/* Populate the secs from hdr */
for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++)
for (i = 0; i < nr_secs; i++)
secs[i] = *(struct btf_sec_info *)((void *)hdr +
btf_sec_info_offset[i]);
sort(secs, ARRAY_SIZE(btf_sec_info_offset),
sort(secs, nr_secs,
sizeof(struct btf_sec_info), btf_sec_info_cmp, NULL);
/* Check for gaps and overlap among sections */
total = 0;
expected_total = btf_data_size - hdr->hdr_len;
for (i = 0; i < ARRAY_SIZE(btf_sec_info_offset); i++) {
for (i = 0; i < nr_secs; i++) {
if (expected_total < secs[i].off) {
btf_verifier_log(env, "Invalid section offset");
return -EINVAL;
@@ -5938,6 +5986,10 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uat
if (err)
goto errout;
err = btf_parse_layout_sec(env);
if (err)
goto errout;
err = btf_parse_type_sec(env);
if (err)
goto errout;
@@ -6517,13 +6569,6 @@ struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog)
return prog->aux->attach_btf;
}
static bool is_void_or_int_ptr(struct btf *btf, const struct btf_type *t)
{
/* skip modifiers */
t = btf_type_skip_modifiers(btf, t->type, NULL);
return btf_type_is_void(t) || btf_type_is_int(t);
}
u32 btf_ctx_arg_idx(struct btf *btf, const struct btf_type *func_proto,
int off)
{
@@ -6912,10 +6957,14 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
}
/*
* If it's a pointer to void, it's the same as scalar from the verifier
* safety POV. Either way, no futher pointer walking is allowed.
* If it's a single or multilevel pointer, except a pointer
* to a structure, it's the same as scalar from the verifier
* safety POV. Multilevel pointers to structures are treated as
* scalars. The verifier lacks the context to infer the size of
* their target memory regions. Either way, no further pointer
* walking is allowed.
*/
if (is_void_or_int_ptr(btf, t))
if (!btf_type_is_struct_ptr(btf, t))
return true;
/* this is a pointer to another type */
@@ -7845,15 +7894,16 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
tname, nargs, MAX_BPF_FUNC_REG_ARGS);
return -EINVAL;
}
/* check that function returns int, exception cb also requires this */
/* check that function is void or returns int, exception cb also requires this */
t = btf_type_by_id(btf, t->type);
while (btf_type_is_modifier(t))
t = btf_type_by_id(btf, t->type);
if (!btf_type_is_int(t) && !btf_is_any_enum(t)) {
if (!btf_type_is_void(t) && !btf_type_is_int(t) && !btf_is_any_enum(t)) {
if (!is_global)
return -EINVAL;
bpf_log(log,
"Global function %s() doesn't return scalar. Only those are supported.\n",
"Global function %s() return value not void or scalar. "
"Only those are supported.\n",
tname);
return -EINVAL;
}
@@ -9019,7 +9069,7 @@ static int btf_check_dtor_kfuncs(struct btf *btf, const struct btf_id_dtor_kfunc
if (!t || !btf_type_is_ptr(t))
return -EINVAL;
if (IS_ENABLED(CONFIG_CFI_CLANG)) {
if (IS_ENABLED(CONFIG_CFI)) {
/* Ensure the destructor kfunc type matches btf_dtor_kfunc_t */
t = btf_type_by_id(btf, t->type);
if (!btf_type_is_void(t))
kernel/bpf/cfg.c
+872
View File
@@ -0,0 +1,872 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/filter.h>
#include <linux/sort.h>
#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
/* non-recursive DFS pseudo code
* 1 procedure DFS-iterative(G,v):
* 2 label v as discovered
* 3 let S be a stack
* 4 S.push(v)
* 5 while S is not empty
* 6 t <- S.peek()
* 7 if t is what we're looking for:
* 8 return t
* 9 for all edges e in G.adjacentEdges(t) do
* 10 if edge e is already labelled
* 11 continue with the next edge
* 12 w <- G.adjacentVertex(t,e)
* 13 if vertex w is not discovered and not explored
* 14 label e as tree-edge
* 15 label w as discovered
* 16 S.push(w)
* 17 continue at 5
* 18 else if vertex w is discovered
* 19 label e as back-edge
* 20 else
* 21 // vertex w is explored
* 22 label e as forward- or cross-edge
* 23 label t as explored
* 24 S.pop()
*
* convention:
* 0x10 - discovered
* 0x11 - discovered and fall-through edge labelled
* 0x12 - discovered and fall-through and branch edges labelled
* 0x20 - explored
*/
enum {
DISCOVERED = 0x10,
EXPLORED = 0x20,
FALLTHROUGH = 1,
BRANCH = 2,
};
static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off)
{
struct bpf_subprog_info *subprog;
subprog = bpf_find_containing_subprog(env, off);
subprog->changes_pkt_data = true;
}
static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off)
{
struct bpf_subprog_info *subprog;
subprog = bpf_find_containing_subprog(env, off);
subprog->might_sleep = true;
}
/* 't' is an index of a call-site.
* 'w' is a callee entry point.
* Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED.
* Rely on DFS traversal order and absence of recursive calls to guarantee that
* callee's change_pkt_data marks would be correct at that moment.
*/
static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)
{
struct bpf_subprog_info *caller, *callee;
caller = bpf_find_containing_subprog(env, t);
callee = bpf_find_containing_subprog(env, w);
caller->changes_pkt_data |= callee->changes_pkt_data;
caller->might_sleep |= callee->might_sleep;
}
enum {
DONE_EXPLORING = 0,
KEEP_EXPLORING = 1,
};
/* t, w, e - match pseudo-code above:
* t - index of current instruction
* w - next instruction
* e - edge
*/
static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)
{
int *insn_stack = env->cfg.insn_stack;
int *insn_state = env->cfg.insn_state;
if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH))
return DONE_EXPLORING;
if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH))
return DONE_EXPLORING;
if (w < 0 || w >= env->prog->len) {
verbose_linfo(env, t, "%d: ", t);
verbose(env, "jump out of range from insn %d to %d\n", t, w);
return -EINVAL;
}
if (e == BRANCH) {
/* mark branch target for state pruning */
mark_prune_point(env, w);
mark_jmp_point(env, w);
}
if (insn_state[w] == 0) {
/* tree-edge */
insn_state[t] = DISCOVERED | e;
insn_state[w] = DISCOVERED;
if (env->cfg.cur_stack >= env->prog->len)
return -E2BIG;
insn_stack[env->cfg.cur_stack++] = w;
return KEEP_EXPLORING;
} else if ((insn_state[w] & 0xF0) == DISCOVERED) {
if (env->bpf_capable)
return DONE_EXPLORING;
verbose_linfo(env, t, "%d: ", t);
verbose_linfo(env, w, "%d: ", w);
verbose(env, "back-edge from insn %d to %d\n", t, w);
return -EINVAL;
} else if (insn_state[w] == EXPLORED) {
/* forward- or cross-edge */
insn_state[t] = DISCOVERED | e;
} else {
verifier_bug(env, "insn state internal bug");
return -EFAULT;
}
return DONE_EXPLORING;
}
static int visit_func_call_insn(int t, struct bpf_insn *insns,
struct bpf_verifier_env *env,
bool visit_callee)
{
int ret, insn_sz;
int w;
insn_sz = bpf_is_ldimm64(&insns[t]) ? 2 : 1;
ret = push_insn(t, t + insn_sz, FALLTHROUGH, env);
if (ret)
return ret;
mark_prune_point(env, t + insn_sz);
/* when we exit from subprog, we need to record non-linear history */
mark_jmp_point(env, t + insn_sz);
if (visit_callee) {
w = t + insns[t].imm + 1;
mark_prune_point(env, t);
merge_callee_effects(env, t, w);
ret = push_insn(t, w, BRANCH, env);
}
return ret;
}
struct bpf_iarray *bpf_iarray_realloc(struct bpf_iarray *old, size_t n_elem)
{
size_t new_size = sizeof(struct bpf_iarray) + n_elem * sizeof(old->items[0]);
struct bpf_iarray *new;
new = kvrealloc(old, new_size, GFP_KERNEL_ACCOUNT);
if (!new) {
/* this is what callers always want, so simplify the call site */
kvfree(old);
return NULL;
}
new->cnt = n_elem;
return new;
}
static int copy_insn_array(struct bpf_map *map, u32 start, u32 end, u32 *items)
{
struct bpf_insn_array_value *value;
u32 i;
for (i = start; i <= end; i++) {
value = map->ops->map_lookup_elem(map, &i);
/*
* map_lookup_elem of an array map will never return an error,
* but not checking it makes some static analysers to worry
*/
if (IS_ERR(value))
return PTR_ERR(value);
else if (!value)
return -EINVAL;
items[i - start] = value->xlated_off;
}
return 0;
}
static int cmp_ptr_to_u32(const void *a, const void *b)
{
return *(u32 *)a - *(u32 *)b;
}
static int sort_insn_array_uniq(u32 *items, int cnt)
{
int unique = 1;
int i;
sort(items, cnt, sizeof(items[0]), cmp_ptr_to_u32, NULL);
for (i = 1; i < cnt; i++)
if (items[i] != items[unique - 1])
items[unique++] = items[i];
return unique;
}
/*
* sort_unique({map[start], ..., map[end]}) into off
*/
int bpf_copy_insn_array_uniq(struct bpf_map *map, u32 start, u32 end, u32 *off)
{
u32 n = end - start + 1;
int err;
err = copy_insn_array(map, start, end, off);
if (err)
return err;
return sort_insn_array_uniq(off, n);
}
/*
* Copy all unique offsets from the map
*/
static struct bpf_iarray *jt_from_map(struct bpf_map *map)
{
struct bpf_iarray *jt;
int err;
int n;
jt = bpf_iarray_realloc(NULL, map->max_entries);
if (!jt)
return ERR_PTR(-ENOMEM);
n = bpf_copy_insn_array_uniq(map, 0, map->max_entries - 1, jt->items);
if (n < 0) {
err = n;
goto err_free;
}
if (n == 0) {
err = -EINVAL;
goto err_free;
}
jt->cnt = n;
return jt;
err_free:
kvfree(jt);
return ERR_PTR(err);
}
/*
* Find and collect all maps which fit in the subprog. Return the result as one
* combined jump table in jt->items (allocated with kvcalloc)
*/
static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env,
int subprog_start, int subprog_end)
{
struct bpf_iarray *jt = NULL;
struct bpf_map *map;
struct bpf_iarray *jt_cur;
int i;
for (i = 0; i < env->insn_array_map_cnt; i++) {
/*
* TODO (when needed): collect only jump tables, not static keys
* or maps for indirect calls
*/
map = env->insn_array_maps[i];
jt_cur = jt_from_map(map);
if (IS_ERR(jt_cur)) {
kvfree(jt);
return jt_cur;
}
/*
* This is enough to check one element. The full table is
* checked to fit inside the subprog later in create_jt()
*/
if (jt_cur->items[0] >= subprog_start && jt_cur->items[0] < subprog_end) {
u32 old_cnt = jt ? jt->cnt : 0;
jt = bpf_iarray_realloc(jt, old_cnt + jt_cur->cnt);
if (!jt) {
kvfree(jt_cur);
return ERR_PTR(-ENOMEM);
}
memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2);
}
kvfree(jt_cur);
}
if (!jt) {
verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start);
return ERR_PTR(-EINVAL);
}
jt->cnt = sort_insn_array_uniq(jt->items, jt->cnt);
return jt;
}
static struct bpf_iarray *
create_jt(int t, struct bpf_verifier_env *env)
{
struct bpf_subprog_info *subprog;
int subprog_start, subprog_end;
struct bpf_iarray *jt;
int i;
subprog = bpf_find_containing_subprog(env, t);
subprog_start = subprog->start;
subprog_end = (subprog + 1)->start;
jt = jt_from_subprog(env, subprog_start, subprog_end);
if (IS_ERR(jt))
return jt;
/* Check that the every element of the jump table fits within the given subprogram */
for (i = 0; i < jt->cnt; i++) {
if (jt->items[i] < subprog_start || jt->items[i] >= subprog_end) {
verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]\n",
t, subprog_start, subprog_end);
kvfree(jt);
return ERR_PTR(-EINVAL);
}
}
return jt;
}
/* "conditional jump with N edges" */
static int visit_gotox_insn(int t, struct bpf_verifier_env *env)
{
int *insn_stack = env->cfg.insn_stack;
int *insn_state = env->cfg.insn_state;
bool keep_exploring = false;
struct bpf_iarray *jt;
int i, w;
jt = env->insn_aux_data[t].jt;
if (!jt) {
jt = create_jt(t, env);
if (IS_ERR(jt))
return PTR_ERR(jt);
env->insn_aux_data[t].jt = jt;
}
mark_prune_point(env, t);
for (i = 0; i < jt->cnt; i++) {
w = jt->items[i];
if (w < 0 || w >= env->prog->len) {
verbose(env, "indirect jump out of range from insn %d to %d\n", t, w);
return -EINVAL;
}
mark_jmp_point(env, w);
/* EXPLORED || DISCOVERED */
if (insn_state[w])
continue;
if (env->cfg.cur_stack >= env->prog->len)
return -E2BIG;
insn_stack[env->cfg.cur_stack++] = w;
insn_state[w] |= DISCOVERED;
keep_exploring = true;
}
return keep_exploring ? KEEP_EXPLORING : DONE_EXPLORING;
}
/*
* Instructions that can abnormally return from a subprog (tail_call
* upon success, ld_{abs,ind} upon load failure) have a hidden exit
* that the verifier must account for.
*/
static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t)
{
struct bpf_subprog_info *subprog;
struct bpf_iarray *jt;
if (env->insn_aux_data[t].jt)
return 0;
jt = bpf_iarray_realloc(NULL, 2);
if (!jt)
return -ENOMEM;
subprog = bpf_find_containing_subprog(env, t);
jt->items[0] = t + 1;
jt->items[1] = subprog->exit_idx;
env->insn_aux_data[t].jt = jt;
return 0;
}
/* Visits the instruction at index t and returns one of the following:
* < 0 - an error occurred
* DONE_EXPLORING - the instruction was fully explored
* KEEP_EXPLORING - there is still work to be done before it is fully explored
*/
static int visit_insn(int t, struct bpf_verifier_env *env)
{
struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t];
int ret, off, insn_sz;
if (bpf_pseudo_func(insn))
return visit_func_call_insn(t, insns, env, true);
/* All non-branch instructions have a single fall-through edge. */
if (BPF_CLASS(insn->code) != BPF_JMP &&
BPF_CLASS(insn->code) != BPF_JMP32) {
if (BPF_CLASS(insn->code) == BPF_LD &&
(BPF_MODE(insn->code) == BPF_ABS ||
BPF_MODE(insn->code) == BPF_IND)) {
ret = visit_abnormal_return_insn(env, t);
if (ret)
return ret;
}
insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;
return push_insn(t, t + insn_sz, FALLTHROUGH, env);
}
switch (BPF_OP(insn->code)) {
case BPF_EXIT:
return DONE_EXPLORING;
case BPF_CALL:
if (bpf_is_async_callback_calling_insn(insn))
/* Mark this call insn as a prune point to trigger
* is_state_visited() check before call itself is
* processed by __check_func_call(). Otherwise new
* async state will be pushed for further exploration.
*/
mark_prune_point(env, t);
/* For functions that invoke callbacks it is not known how many times
* callback would be called. Verifier models callback calling functions
* by repeatedly visiting callback bodies and returning to origin call
* instruction.
* In order to stop such iteration verifier needs to identify when a
* state identical some state from a previous iteration is reached.
* Check below forces creation of checkpoint before callback calling
* instruction to allow search for such identical states.
*/
if (bpf_is_sync_callback_calling_insn(insn)) {
mark_calls_callback(env, t);
mark_force_checkpoint(env, t);
mark_prune_point(env, t);
mark_jmp_point(env, t);
}
if (bpf_helper_call(insn)) {
const struct bpf_func_proto *fp;
ret = bpf_get_helper_proto(env, insn->imm, &fp);
/* If called in a non-sleepable context program will be
* rejected anyway, so we should end up with precise
* sleepable marks on subprogs, except for dead code
* elimination.
*/
if (ret == 0 && fp->might_sleep)
mark_subprog_might_sleep(env, t);
if (bpf_helper_changes_pkt_data(insn->imm))
mark_subprog_changes_pkt_data(env, t);
if (insn->imm == BPF_FUNC_tail_call) {
ret = visit_abnormal_return_insn(env, t);
if (ret)
return ret;
}
} else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
struct bpf_kfunc_call_arg_meta meta;
ret = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta);
if (ret == 0 && bpf_is_iter_next_kfunc(&meta)) {
mark_prune_point(env, t);
/* Checking and saving state checkpoints at iter_next() call
* is crucial for fast convergence of open-coded iterator loop
* logic, so we need to force it. If we don't do that,
* is_state_visited() might skip saving a checkpoint, causing
* unnecessarily long sequence of not checkpointed
* instructions and jumps, leading to exhaustion of jump
* history buffer, and potentially other undesired outcomes.
* It is expected that with correct open-coded iterators
* convergence will happen quickly, so we don't run a risk of
* exhausting memory.
*/
mark_force_checkpoint(env, t);
}
/* Same as helpers, if called in a non-sleepable context
* program will be rejected anyway, so we should end up
* with precise sleepable marks on subprogs, except for
* dead code elimination.
*/
if (ret == 0 && bpf_is_kfunc_sleepable(&meta))
mark_subprog_might_sleep(env, t);
if (ret == 0 && bpf_is_kfunc_pkt_changing(&meta))
mark_subprog_changes_pkt_data(env, t);
}
return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);
case BPF_JA:
if (BPF_SRC(insn->code) == BPF_X)
return visit_gotox_insn(t, env);
if (BPF_CLASS(insn->code) == BPF_JMP)
off = insn->off;
else
off = insn->imm;
/* unconditional jump with single edge */
ret = push_insn(t, t + off + 1, FALLTHROUGH, env);
if (ret)
return ret;
mark_prune_point(env, t + off + 1);
mark_jmp_point(env, t + off + 1);
return ret;
default:
/* conditional jump with two edges */
mark_prune_point(env, t);
if (bpf_is_may_goto_insn(insn))
mark_force_checkpoint(env, t);
ret = push_insn(t, t + 1, FALLTHROUGH, env);
if (ret)
return ret;
return push_insn(t, t + insn->off + 1, BRANCH, env);
}
}
/* non-recursive depth-first-search to detect loops in BPF program
* loop == back-edge in directed graph
*/
int bpf_check_cfg(struct bpf_verifier_env *env)
{
int insn_cnt = env->prog->len;
int *insn_stack, *insn_state;
int ex_insn_beg, i, ret = 0;
insn_state = env->cfg.insn_state = kvzalloc_objs(int, insn_cnt,
GFP_KERNEL_ACCOUNT);
if (!insn_state)
return -ENOMEM;
insn_stack = env->cfg.insn_stack = kvzalloc_objs(int, insn_cnt,
GFP_KERNEL_ACCOUNT);
if (!insn_stack) {
kvfree(insn_state);
return -ENOMEM;
}
ex_insn_beg = env->exception_callback_subprog
? env->subprog_info[env->exception_callback_subprog].start
: 0;
insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */
insn_stack[0] = 0; /* 0 is the first instruction */
env->cfg.cur_stack = 1;
walk_cfg:
while (env->cfg.cur_stack > 0) {
int t = insn_stack[env->cfg.cur_stack - 1];
ret = visit_insn(t, env);
switch (ret) {
case DONE_EXPLORING:
insn_state[t] = EXPLORED;
env->cfg.cur_stack--;
break;
case KEEP_EXPLORING:
break;
default:
if (ret > 0) {
verifier_bug(env, "visit_insn internal bug");
ret = -EFAULT;
}
goto err_free;
}
}
if (env->cfg.cur_stack < 0) {
verifier_bug(env, "pop stack internal bug");
ret = -EFAULT;
goto err_free;
}
if (ex_insn_beg && insn_state[ex_insn_beg] != EXPLORED) {
insn_state[ex_insn_beg] = DISCOVERED;
insn_stack[0] = ex_insn_beg;
env->cfg.cur_stack = 1;
goto walk_cfg;
}
for (i = 0; i < insn_cnt; i++) {
struct bpf_insn *insn = &env->prog->insnsi[i];
if (insn_state[i] != EXPLORED) {
verbose(env, "unreachable insn %d\n", i);
ret = -EINVAL;
goto err_free;
}
if (bpf_is_ldimm64(insn)) {
if (insn_state[i + 1] != 0) {
verbose(env, "jump into the middle of ldimm64 insn %d\n", i);
ret = -EINVAL;
goto err_free;
}
i++; /* skip second half of ldimm64 */
}
}
ret = 0; /* cfg looks good */
env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data;
env->prog->aux->might_sleep = env->subprog_info[0].might_sleep;
err_free:
kvfree(insn_state);
kvfree(insn_stack);
env->cfg.insn_state = env->cfg.insn_stack = NULL;
return ret;
}
/*
* For each subprogram 'i' fill array env->cfg.insn_subprogram sub-range
* [env->subprog_info[i].postorder_start, env->subprog_info[i+1].postorder_start)
* with indices of 'i' instructions in postorder.
*/
int bpf_compute_postorder(struct bpf_verifier_env *env)
{
u32 cur_postorder, i, top, stack_sz, s;
int *stack = NULL, *postorder = NULL, *state = NULL;
struct bpf_iarray *succ;
postorder = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);
state = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);
stack = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);
if (!postorder || !state || !stack) {
kvfree(postorder);
kvfree(state);
kvfree(stack);
return -ENOMEM;
}
cur_postorder = 0;
for (i = 0; i < env->subprog_cnt; i++) {
env->subprog_info[i].postorder_start = cur_postorder;
stack[0] = env->subprog_info[i].start;
stack_sz = 1;
do {
top = stack[stack_sz - 1];
state[top] |= DISCOVERED;
if (state[top] & EXPLORED) {
postorder[cur_postorder++] = top;
stack_sz--;
continue;
}
succ = bpf_insn_successors(env, top);
for (s = 0; s < succ->cnt; ++s) {
if (!state[succ->items[s]]) {
stack[stack_sz++] = succ->items[s];
state[succ->items[s]] |= DISCOVERED;
}
}
state[top] |= EXPLORED;
} while (stack_sz);
}
env->subprog_info[i].postorder_start = cur_postorder;
env->cfg.insn_postorder = postorder;
env->cfg.cur_postorder = cur_postorder;
kvfree(stack);
kvfree(state);
return 0;
}
/*
* Compute strongly connected components (SCCs) on the CFG.
* Assign an SCC number to each instruction, recorded in env->insn_aux[*].scc.
* If instruction is a sole member of its SCC and there are no self edges,
* assign it SCC number of zero.
* Uses a non-recursive adaptation of Tarjan's algorithm for SCC computation.
*/
int bpf_compute_scc(struct bpf_verifier_env *env)
{
const u32 NOT_ON_STACK = U32_MAX;
struct bpf_insn_aux_data *aux = env->insn_aux_data;
const u32 insn_cnt = env->prog->len;
int stack_sz, dfs_sz, err = 0;
u32 *stack, *pre, *low, *dfs;
u32 i, j, t, w;
u32 next_preorder_num;
u32 next_scc_id;
bool assign_scc;
struct bpf_iarray *succ;
next_preorder_num = 1;
next_scc_id = 1;
/*
* - 'stack' accumulates vertices in DFS order, see invariant comment below;
* - 'pre[t] == p' => preorder number of vertex 't' is 'p';
* - 'low[t] == n' => smallest preorder number of the vertex reachable from 't' is 'n';
* - 'dfs' DFS traversal stack, used to emulate explicit recursion.
*/
stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
pre = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
low = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
dfs = kvcalloc(insn_cnt, sizeof(*dfs), GFP_KERNEL_ACCOUNT);
if (!stack || !pre || !low || !dfs) {
err = -ENOMEM;
goto exit;
}
/*
* References:
* [1] R. Tarjan "Depth-First Search and Linear Graph Algorithms"
* [2] D. J. Pearce "A Space-Efficient Algorithm for Finding Strongly Connected Components"
*
* The algorithm maintains the following invariant:
* - suppose there is a path 'u' ~> 'v', such that 'pre[v] < pre[u]';
* - then, vertex 'u' remains on stack while vertex 'v' is on stack.
*
* Consequently:
* - If 'low[v] < pre[v]', there is a path from 'v' to some vertex 'u',
* such that 'pre[u] == low[v]'; vertex 'u' is currently on the stack,
* and thus there is an SCC (loop) containing both 'u' and 'v'.
* - If 'low[v] == pre[v]', loops containing 'v' have been explored,
* and 'v' can be considered the root of some SCC.
*
* Here is a pseudo-code for an explicitly recursive version of the algorithm:
*
* NOT_ON_STACK = insn_cnt + 1
* pre = [0] * insn_cnt
* low = [0] * insn_cnt
* scc = [0] * insn_cnt
* stack = []
*
* next_preorder_num = 1
* next_scc_id = 1
*
* def recur(w):
* nonlocal next_preorder_num
* nonlocal next_scc_id
*
* pre[w] = next_preorder_num
* low[w] = next_preorder_num
* next_preorder_num += 1
* stack.append(w)
* for s in successors(w):
* # Note: for classic algorithm the block below should look as:
* #
* # if pre[s] == 0:
* # recur(s)
* # low[w] = min(low[w], low[s])
* # elif low[s] != NOT_ON_STACK:
* # low[w] = min(low[w], pre[s])
* #
* # But replacing both 'min' instructions with 'low[w] = min(low[w], low[s])'
* # does not break the invariant and makes iterative version of the algorithm
* # simpler. See 'Algorithm #3' from [2].
*
* # 's' not yet visited
* if pre[s] == 0:
* recur(s)
* # if 's' is on stack, pick lowest reachable preorder number from it;
* # if 's' is not on stack 'low[s] == NOT_ON_STACK > low[w]',
* # so 'min' would be a noop.
* low[w] = min(low[w], low[s])
*
* if low[w] == pre[w]:
* # 'w' is the root of an SCC, pop all vertices
* # below 'w' on stack and assign same SCC to them.
* while True:
* t = stack.pop()
* low[t] = NOT_ON_STACK
* scc[t] = next_scc_id
* if t == w:
* break
* next_scc_id += 1
*
* for i in range(0, insn_cnt):
* if pre[i] == 0:
* recur(i)
*
* Below implementation replaces explicit recursion with array 'dfs'.
*/
for (i = 0; i < insn_cnt; i++) {
if (pre[i])
continue;
stack_sz = 0;
dfs_sz = 1;
dfs[0] = i;
dfs_continue:
while (dfs_sz) {
w = dfs[dfs_sz - 1];
if (pre[w] == 0) {
low[w] = next_preorder_num;
pre[w] = next_preorder_num;
next_preorder_num++;
stack[stack_sz++] = w;
}
/* Visit 'w' successors */
succ = bpf_insn_successors(env, w);
for (j = 0; j < succ->cnt; ++j) {
if (pre[succ->items[j]]) {
low[w] = min(low[w], low[succ->items[j]]);
} else {
dfs[dfs_sz++] = succ->items[j];
goto dfs_continue;
}
}
/*
* Preserve the invariant: if some vertex above in the stack
* is reachable from 'w', keep 'w' on the stack.
*/
if (low[w] < pre[w]) {
dfs_sz--;
goto dfs_continue;
}
/*
* Assign SCC number only if component has two or more elements,
* or if component has a self reference, or if instruction is a
* callback calling function (implicit loop).
*/
assign_scc = stack[stack_sz - 1] != w; /* two or more elements? */
for (j = 0; j < succ->cnt; ++j) { /* self reference? */
if (succ->items[j] == w) {
assign_scc = true;
break;
}
}
if (bpf_calls_callback(env, w)) /* implicit loop? */
assign_scc = true;
/* Pop component elements from stack */
do {
t = stack[--stack_sz];
low[t] = NOT_ON_STACK;
if (assign_scc)
aux[t].scc = next_scc_id;
} while (t != w);
if (assign_scc)
next_scc_id++;
dfs_sz--;
}
}
env->scc_info = kvzalloc_objs(*env->scc_info, next_scc_id,
GFP_KERNEL_ACCOUNT);
if (!env->scc_info) {
err = -ENOMEM;
goto exit;
}
env->scc_cnt = next_scc_id;
exit:
kvfree(stack);
kvfree(pre);
kvfree(low);
kvfree(dfs);
return err;
}
kernel/bpf/check_btf.c
+463
View File
@@ -0,0 +1,463 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/filter.h>
#include <linux/btf.h>
#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
static int check_abnormal_return(struct bpf_verifier_env *env)
{
int i;
for (i = 1; i < env->subprog_cnt; i++) {
if (env->subprog_info[i].has_ld_abs) {
verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");
return -EINVAL;
}
if (env->subprog_info[i].has_tail_call) {
verbose(env, "tail_call is not allowed in subprogs without BTF\n");
return -EINVAL;
}
}
return 0;
}
/* The minimum supported BTF func info size */
#define MIN_BPF_FUNCINFO_SIZE 8
#define MAX_FUNCINFO_REC_SIZE 252
static int check_btf_func_early(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
{
u32 krec_size = sizeof(struct bpf_func_info);
const struct btf_type *type, *func_proto;
u32 i, nfuncs, urec_size, min_size;
struct bpf_func_info *krecord;
struct bpf_prog *prog;
const struct btf *btf;
u32 prev_offset = 0;
bpfptr_t urecord;
int ret = -ENOMEM;
nfuncs = attr->func_info_cnt;
if (!nfuncs) {
if (check_abnormal_return(env))
return -EINVAL;
return 0;
}
urec_size = attr->func_info_rec_size;
if (urec_size < MIN_BPF_FUNCINFO_SIZE ||
urec_size > MAX_FUNCINFO_REC_SIZE ||
urec_size % sizeof(u32)) {
verbose(env, "invalid func info rec size %u\n", urec_size);
return -EINVAL;
}
prog = env->prog;
btf = prog->aux->btf;
urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
min_size = min_t(u32, krec_size, urec_size);
krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
if (!krecord)
return -ENOMEM;
for (i = 0; i < nfuncs; i++) {
ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
if (ret) {
if (ret == -E2BIG) {
verbose(env, "nonzero tailing record in func info");
/* set the size kernel expects so loader can zero
* out the rest of the record.
*/
if (copy_to_bpfptr_offset(uattr,
offsetof(union bpf_attr, func_info_rec_size),
&min_size, sizeof(min_size)))
ret = -EFAULT;
}
goto err_free;
}
if (copy_from_bpfptr(&krecord[i], urecord, min_size)) {
ret = -EFAULT;
goto err_free;
}
/* check insn_off */
ret = -EINVAL;
if (i == 0) {
if (krecord[i].insn_off) {
verbose(env,
"nonzero insn_off %u for the first func info record",
krecord[i].insn_off);
goto err_free;
}
} else if (krecord[i].insn_off <= prev_offset) {
verbose(env,
"same or smaller insn offset (%u) than previous func info record (%u)",
krecord[i].insn_off, prev_offset);
goto err_free;
}
/* check type_id */
type = btf_type_by_id(btf, krecord[i].type_id);
if (!type || !btf_type_is_func(type)) {
verbose(env, "invalid type id %d in func info",
krecord[i].type_id);
goto err_free;
}
func_proto = btf_type_by_id(btf, type->type);
if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto)))
/* btf_func_check() already verified it during BTF load */
goto err_free;
prev_offset = krecord[i].insn_off;
bpfptr_add(&urecord, urec_size);
}
prog->aux->func_info = krecord;
prog->aux->func_info_cnt = nfuncs;
return 0;
err_free:
kvfree(krecord);
return ret;
}
static int check_btf_func(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
{
const struct btf_type *type, *func_proto, *ret_type;
u32 i, nfuncs, urec_size;
struct bpf_func_info *krecord;
struct bpf_func_info_aux *info_aux = NULL;
struct bpf_prog *prog;
const struct btf *btf;
bpfptr_t urecord;
bool scalar_return;
int ret = -ENOMEM;
nfuncs = attr->func_info_cnt;
if (!nfuncs) {
if (check_abnormal_return(env))
return -EINVAL;
return 0;
}
if (nfuncs != env->subprog_cnt) {
verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
return -EINVAL;
}
urec_size = attr->func_info_rec_size;
prog = env->prog;
btf = prog->aux->btf;
urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
krecord = prog->aux->func_info;
info_aux = kzalloc_objs(*info_aux, nfuncs,
GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
if (!info_aux)
return -ENOMEM;
for (i = 0; i < nfuncs; i++) {
/* check insn_off */
ret = -EINVAL;
if (env->subprog_info[i].start != krecord[i].insn_off) {
verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
goto err_free;
}
/* Already checked type_id */
type = btf_type_by_id(btf, krecord[i].type_id);
info_aux[i].linkage = BTF_INFO_VLEN(type->info);
/* Already checked func_proto */
func_proto = btf_type_by_id(btf, type->type);
ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL);
scalar_return =
btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type);
if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {
verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");
goto err_free;
}
if (i && !scalar_return && env->subprog_info[i].has_tail_call) {
verbose(env, "tail_call is only allowed in functions that return 'int'.\n");
goto err_free;
}
env->subprog_info[i].name = btf_name_by_offset(btf, type->name_off);
bpfptr_add(&urecord, urec_size);
}
prog->aux->func_info_aux = info_aux;
return 0;
err_free:
kfree(info_aux);
return ret;
}
#define MIN_BPF_LINEINFO_SIZE offsetofend(struct bpf_line_info, line_col)
#define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE
static int check_btf_line(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
{
u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
struct bpf_subprog_info *sub;
struct bpf_line_info *linfo;
struct bpf_prog *prog;
const struct btf *btf;
bpfptr_t ulinfo;
int err;
nr_linfo = attr->line_info_cnt;
if (!nr_linfo)
return 0;
if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info))
return -EINVAL;
rec_size = attr->line_info_rec_size;
if (rec_size < MIN_BPF_LINEINFO_SIZE ||
rec_size > MAX_LINEINFO_REC_SIZE ||
rec_size & (sizeof(u32) - 1))
return -EINVAL;
/* Need to zero it in case the userspace may
* pass in a smaller bpf_line_info object.
*/
linfo = kvzalloc_objs(struct bpf_line_info, nr_linfo,
GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
if (!linfo)
return -ENOMEM;
prog = env->prog;
btf = prog->aux->btf;
s = 0;
sub = env->subprog_info;
ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel);
expected_size = sizeof(struct bpf_line_info);
ncopy = min_t(u32, expected_size, rec_size);
for (i = 0; i < nr_linfo; i++) {
err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
if (err) {
if (err == -E2BIG) {
verbose(env, "nonzero tailing record in line_info");
if (copy_to_bpfptr_offset(uattr,
offsetof(union bpf_attr, line_info_rec_size),
&expected_size, sizeof(expected_size)))
err = -EFAULT;
}
goto err_free;
}
if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) {
err = -EFAULT;
goto err_free;
}
/*
* Check insn_off to ensure
* 1) strictly increasing AND
* 2) bounded by prog->len
*
* The linfo[0].insn_off == 0 check logically falls into
* the later "missing bpf_line_info for func..." case
* because the first linfo[0].insn_off must be the
* first sub also and the first sub must have
* subprog_info[0].start == 0.
*/
if ((i && linfo[i].insn_off <= prev_offset) ||
linfo[i].insn_off >= prog->len) {
verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
i, linfo[i].insn_off, prev_offset,
prog->len);
err = -EINVAL;
goto err_free;
}
if (!prog->insnsi[linfo[i].insn_off].code) {
verbose(env,
"Invalid insn code at line_info[%u].insn_off\n",
i);
err = -EINVAL;
goto err_free;
}
if (!btf_name_by_offset(btf, linfo[i].line_off) ||
!btf_name_by_offset(btf, linfo[i].file_name_off)) {
verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
err = -EINVAL;
goto err_free;
}
if (s != env->subprog_cnt) {
if (linfo[i].insn_off == sub[s].start) {
sub[s].linfo_idx = i;
s++;
} else if (sub[s].start < linfo[i].insn_off) {
verbose(env, "missing bpf_line_info for func#%u\n", s);
err = -EINVAL;
goto err_free;
}
}
prev_offset = linfo[i].insn_off;
bpfptr_add(&ulinfo, rec_size);
}
if (s != env->subprog_cnt) {
verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
env->subprog_cnt - s, s);
err = -EINVAL;
goto err_free;
}
prog->aux->linfo = linfo;
prog->aux->nr_linfo = nr_linfo;
return 0;
err_free:
kvfree(linfo);
return err;
}
#define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo)
#define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE
static int check_core_relo(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
{
u32 i, nr_core_relo, ncopy, expected_size, rec_size;
struct bpf_core_relo core_relo = {};
struct bpf_prog *prog = env->prog;
const struct btf *btf = prog->aux->btf;
struct bpf_core_ctx ctx = {
.log = &env->log,
.btf = btf,
};
bpfptr_t u_core_relo;
int err;
nr_core_relo = attr->core_relo_cnt;
if (!nr_core_relo)
return 0;
if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo))
return -EINVAL;
rec_size = attr->core_relo_rec_size;
if (rec_size < MIN_CORE_RELO_SIZE ||
rec_size > MAX_CORE_RELO_SIZE ||
rec_size % sizeof(u32))
return -EINVAL;
u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel);
expected_size = sizeof(struct bpf_core_relo);
ncopy = min_t(u32, expected_size, rec_size);
/* Unlike func_info and line_info, copy and apply each CO-RE
* relocation record one at a time.
*/
for (i = 0; i < nr_core_relo; i++) {
/* future proofing when sizeof(bpf_core_relo) changes */
err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size);
if (err) {
if (err == -E2BIG) {
verbose(env, "nonzero tailing record in core_relo");
if (copy_to_bpfptr_offset(uattr,
offsetof(union bpf_attr, core_relo_rec_size),
&expected_size, sizeof(expected_size)))
err = -EFAULT;
}
break;
}
if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) {
err = -EFAULT;
break;
}
if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) {
verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n",
i, core_relo.insn_off, prog->len);
err = -EINVAL;
break;
}
err = bpf_core_apply(&ctx, &core_relo, i,
&prog->insnsi[core_relo.insn_off / 8]);
if (err)
break;
bpfptr_add(&u_core_relo, rec_size);
}
return err;
}
int bpf_check_btf_info_early(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
{
struct btf *btf;
int err;
if (!attr->func_info_cnt && !attr->line_info_cnt) {
if (check_abnormal_return(env))
return -EINVAL;
return 0;
}
btf = btf_get_by_fd(attr->prog_btf_fd);
if (IS_ERR(btf))
return PTR_ERR(btf);
if (btf_is_kernel(btf)) {
btf_put(btf);
return -EACCES;
}
env->prog->aux->btf = btf;
err = check_btf_func_early(env, attr, uattr);
if (err)
return err;
return 0;
}
int bpf_check_btf_info(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
{
int err;
if (!attr->func_info_cnt && !attr->line_info_cnt) {
if (check_abnormal_return(env))
return -EINVAL;
return 0;
}
err = check_btf_func(env, attr, uattr);
if (err)
return err;
err = check_btf_line(env, attr, uattr);
if (err)
return err;
err = check_core_relo(env, attr, uattr);
if (err)
return err;
return 0;
}
kernel/bpf/const_fold.c
+396
View File
@@ -0,0 +1,396 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#include <linux/bpf_verifier.h>
/*
* Forward dataflow analysis to determine constant register values at every
* instruction. Tracks 64-bit constant values in R0-R9 through the program,
* using a fixed-point iteration in reverse postorder. Records which registers
* hold known constants and their values in
* env->insn_aux_data[].{const_reg_mask, const_reg_vals}.
*/
enum const_arg_state {
CONST_ARG_UNVISITED, /* instruction not yet reached */
CONST_ARG_UNKNOWN, /* register value not a known constant */
CONST_ARG_CONST, /* register holds a known 64-bit constant */
CONST_ARG_MAP_PTR, /* register holds a map pointer, map_index is set */
CONST_ARG_MAP_VALUE, /* register points to map value data, val is offset */
CONST_ARG_SUBPROG, /* register holds a subprog pointer, val is subprog number */
};
struct const_arg_info {
enum const_arg_state state;
u32 map_index;
u64 val;
};
static bool ci_is_unvisited(const struct const_arg_info *ci)
{
return ci->state == CONST_ARG_UNVISITED;
}
static bool ci_is_unknown(const struct const_arg_info *ci)
{
return ci->state == CONST_ARG_UNKNOWN;
}
static bool ci_is_const(const struct const_arg_info *ci)
{
return ci->state == CONST_ARG_CONST;
}
static bool ci_is_map_value(const struct const_arg_info *ci)
{
return ci->state == CONST_ARG_MAP_VALUE;
}
/* Transfer function: compute output register state from instruction. */
static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info *ci_out,
struct bpf_insn *insn, struct bpf_insn *insns, int idx)
{
struct const_arg_info unknown = { .state = CONST_ARG_UNKNOWN, .val = 0 };
struct const_arg_info *dst = &ci_out[insn->dst_reg];
struct const_arg_info *src = &ci_out[insn->src_reg];
u8 class = BPF_CLASS(insn->code);
u8 mode = BPF_MODE(insn->code);
u8 opcode = BPF_OP(insn->code) | BPF_SRC(insn->code);
int r;
switch (class) {
case BPF_ALU:
case BPF_ALU64:
switch (opcode) {
case BPF_MOV | BPF_K:
dst->state = CONST_ARG_CONST;
dst->val = (s64)insn->imm;
break;
case BPF_MOV | BPF_X:
*dst = *src;
if (!insn->off)
break;
if (!ci_is_const(dst)) {
*dst = unknown;
break;
}
switch (insn->off) {
case 8: dst->val = (s8)dst->val; break;
case 16: dst->val = (s16)dst->val; break;
case 32: dst->val = (s32)dst->val; break;
default: *dst = unknown; break;
}
break;
case BPF_ADD | BPF_K:
if (!ci_is_const(dst) && !ci_is_map_value(dst)) {
*dst = unknown;
break;
}
dst->val += insn->imm;
break;
case BPF_SUB | BPF_K:
if (!ci_is_const(dst) && !ci_is_map_value(dst)) {
*dst = unknown;
break;
}
dst->val -= insn->imm;
break;
case BPF_AND | BPF_K:
if (!ci_is_const(dst)) {
if (!insn->imm) {
dst->state = CONST_ARG_CONST;
dst->val = 0;
} else {
*dst = unknown;
}
break;
}
dst->val &= (s64)insn->imm;
break;
case BPF_AND | BPF_X:
if (ci_is_const(dst) && dst->val == 0)
break; /* 0 & x == 0 */
if (ci_is_const(src) && src->val == 0) {
dst->state = CONST_ARG_CONST;
dst->val = 0;
break;
}
if (!ci_is_const(dst) || !ci_is_const(src)) {
*dst = unknown;
break;
}
dst->val &= src->val;
break;
default:
*dst = unknown;
break;
}
if (class == BPF_ALU) {
if (ci_is_const(dst))
dst->val = (u32)dst->val;
else if (!ci_is_unknown(dst))
*dst = unknown;
}
break;
case BPF_LD:
if (mode == BPF_ABS || mode == BPF_IND)
goto process_call;
if (mode != BPF_IMM || BPF_SIZE(insn->code) != BPF_DW)
break;
if (insn->src_reg == BPF_PSEUDO_FUNC) {
int subprog = bpf_find_subprog(env, idx + insn->imm + 1);
if (subprog >= 0) {
dst->state = CONST_ARG_SUBPROG;
dst->val = subprog;
} else {
*dst = unknown;
}
} else if (insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) {
dst->state = CONST_ARG_MAP_VALUE;
dst->map_index = env->insn_aux_data[idx].map_index;
dst->val = env->insn_aux_data[idx].map_off;
} else if (insn->src_reg == BPF_PSEUDO_MAP_FD ||
insn->src_reg == BPF_PSEUDO_MAP_IDX) {
dst->state = CONST_ARG_MAP_PTR;
dst->map_index = env->insn_aux_data[idx].map_index;
} else if (insn->src_reg == 0) {
dst->state = CONST_ARG_CONST;
dst->val = (u64)(u32)insn->imm | ((u64)(u32)insns[idx + 1].imm << 32);
} else {
*dst = unknown;
}
break;
case BPF_LDX:
if (!ci_is_map_value(src)) {
*dst = unknown;
break;
}
struct bpf_map *map = env->used_maps[src->map_index];
int size = bpf_size_to_bytes(BPF_SIZE(insn->code));
bool is_ldsx = mode == BPF_MEMSX;
int off = src->val + insn->off;
u64 val = 0;
if (!bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr ||
map->map_type == BPF_MAP_TYPE_INSN_ARRAY ||
off < 0 || off + size > map->value_size ||
bpf_map_direct_read(map, off, size, &val, is_ldsx)) {
*dst = unknown;
break;
}
dst->state = CONST_ARG_CONST;
dst->val = val;
break;
case BPF_JMP:
if (opcode != BPF_CALL)
break;
process_call:
for (r = BPF_REG_0; r <= BPF_REG_5; r++)
ci_out[r] = unknown;
break;
case BPF_STX:
if (mode != BPF_ATOMIC)
break;
if (insn->imm == BPF_CMPXCHG)
ci_out[BPF_REG_0] = unknown;
else if (insn->imm == BPF_LOAD_ACQ)
*dst = unknown;
else if (insn->imm & BPF_FETCH)
*src = unknown;
break;
}
}
/* Join function: merge output state into a successor's input state. */
static bool const_reg_join(struct const_arg_info *ci_target,
struct const_arg_info *ci_out)
{
bool changed = false;
int r;
for (r = 0; r < MAX_BPF_REG; r++) {
struct const_arg_info *old = &ci_target[r];
struct const_arg_info *new = &ci_out[r];
if (ci_is_unvisited(old) && !ci_is_unvisited(new)) {
ci_target[r] = *new;
changed = true;
} else if (!ci_is_unknown(old) && !ci_is_unvisited(old) &&
(new->state != old->state || new->val != old->val ||
new->map_index != old->map_index)) {
old->state = CONST_ARG_UNKNOWN;
changed = true;
}
}
return changed;
}
int bpf_compute_const_regs(struct bpf_verifier_env *env)
{
struct const_arg_info unknown = { .state = CONST_ARG_UNKNOWN, .val = 0 };
struct bpf_insn_aux_data *insn_aux = env->insn_aux_data;
struct bpf_insn *insns = env->prog->insnsi;
int insn_cnt = env->prog->len;
struct const_arg_info (*ci_in)[MAX_BPF_REG];
struct const_arg_info ci_out[MAX_BPF_REG];
struct bpf_iarray *succ;
bool changed;
int i, r;
/* kvzalloc zeroes memory, so all entries start as CONST_ARG_UNVISITED (0) */
ci_in = kvzalloc_objs(*ci_in, insn_cnt, GFP_KERNEL_ACCOUNT);
if (!ci_in)
return -ENOMEM;
/* Subprogram entries (including main at subprog 0): all registers unknown */
for (i = 0; i < env->subprog_cnt; i++) {
int start = env->subprog_info[i].start;
for (r = 0; r < MAX_BPF_REG; r++)
ci_in[start][r] = unknown;
}
redo:
changed = false;
for (i = env->cfg.cur_postorder - 1; i >= 0; i--) {
int idx = env->cfg.insn_postorder[i];
struct bpf_insn *insn = &insns[idx];
struct const_arg_info *ci = ci_in[idx];
memcpy(ci_out, ci, sizeof(ci_out));
const_reg_xfer(env, ci_out, insn, insns, idx);
succ = bpf_insn_successors(env, idx);
for (int s = 0; s < succ->cnt; s++)
changed |= const_reg_join(ci_in[succ->items[s]], ci_out);
}
if (changed)
goto redo;
/* Save computed constants into insn_aux[] if they fit into 32-bit */
for (i = 0; i < insn_cnt; i++) {
u16 mask = 0, map_mask = 0, subprog_mask = 0;
struct bpf_insn_aux_data *aux = &insn_aux[i];
struct const_arg_info *ci = ci_in[i];
for (r = BPF_REG_0; r < ARRAY_SIZE(aux->const_reg_vals); r++) {
struct const_arg_info *c = &ci[r];
switch (c->state) {
case CONST_ARG_CONST: {
u64 val = c->val;
if (val != (u32)val)
break;
mask |= BIT(r);
aux->const_reg_vals[r] = val;
break;
}
case CONST_ARG_MAP_PTR:
map_mask |= BIT(r);
aux->const_reg_vals[r] = c->map_index;
break;
case CONST_ARG_SUBPROG:
subprog_mask |= BIT(r);
aux->const_reg_vals[r] = c->val;
break;
default:
break;
}
}
aux->const_reg_mask = mask;
aux->const_reg_map_mask = map_mask;
aux->const_reg_subprog_mask = subprog_mask;
}
kvfree(ci_in);
return 0;
}
static int eval_const_branch(u8 opcode, u64 dst_val, u64 src_val)
{
switch (BPF_OP(opcode)) {
case BPF_JEQ: return dst_val == src_val;
case BPF_JNE: return dst_val != src_val;
case BPF_JGT: return dst_val > src_val;
case BPF_JGE: return dst_val >= src_val;
case BPF_JLT: return dst_val < src_val;
case BPF_JLE: return dst_val <= src_val;
case BPF_JSGT: return (s64)dst_val > (s64)src_val;
case BPF_JSGE: return (s64)dst_val >= (s64)src_val;
case BPF_JSLT: return (s64)dst_val < (s64)src_val;
case BPF_JSLE: return (s64)dst_val <= (s64)src_val;
case BPF_JSET: return (bool)(dst_val & src_val);
default: return -1;
}
}
/*
* Rewrite conditional branches with constant outcomes into unconditional
* jumps using register values resolved by bpf_compute_const_regs() pass.
* This eliminates dead edges from the CFG so that compute_live_registers()
* doesn't propagate liveness through dead code.
*/
int bpf_prune_dead_branches(struct bpf_verifier_env *env)
{
struct bpf_insn_aux_data *insn_aux = env->insn_aux_data;
struct bpf_insn *insns = env->prog->insnsi;
int insn_cnt = env->prog->len;
bool changed = false;
int i;
for (i = 0; i < insn_cnt; i++) {
struct bpf_insn_aux_data *aux = &insn_aux[i];
struct bpf_insn *insn = &insns[i];
u8 class = BPF_CLASS(insn->code);
u64 dst_val, src_val;
int taken;
if (!bpf_insn_is_cond_jump(insn->code))
continue;
if (bpf_is_may_goto_insn(insn))
continue;
if (!(aux->const_reg_mask & BIT(insn->dst_reg)))
continue;
dst_val = aux->const_reg_vals[insn->dst_reg];
if (BPF_SRC(insn->code) == BPF_K) {
src_val = insn->imm;
} else {
if (!(aux->const_reg_mask & BIT(insn->src_reg)))
continue;
src_val = aux->const_reg_vals[insn->src_reg];
}
if (class == BPF_JMP32) {
/*
* The (s32) cast maps the 32-bit range into two u64 sub-ranges:
* [0x00000000, 0x7FFFFFFF] -> [0x0000000000000000, 0x000000007FFFFFFF]
* [0x80000000, 0xFFFFFFFF] -> [0xFFFFFFFF80000000, 0xFFFFFFFFFFFFFFFF]
* The ordering is preserved within each sub-range, and
* the second sub-range is above the first as u64.
*/
dst_val = (s32)dst_val;
src_val = (s32)src_val;
}
taken = eval_const_branch(insn->code, dst_val, src_val);
if (taken < 0) {
bpf_log(&env->log, "Unknown conditional jump %x\n", insn->code);
return -EFAULT;
}
*insn = BPF_JMP_A(taken ? insn->off : 0);
changed = true;
}
if (!changed)
return 0;
/* recompute postorder, since CFG has changed */
kvfree(env->cfg.insn_postorder);
env->cfg.insn_postorder = NULL;
return bpf_compute_postorder(env);
}
kernel/bpf/core.c
+73 -22
View File
@@ -18,7 +18,6 @@
*/
#include <uapi/linux/btf.h>
#include <crypto/sha1.h>
#include <linux/filter.h>
#include <linux/skbuff.h>
#include <linux/vmalloc.h>
@@ -1487,6 +1486,8 @@ void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other)
* know whether fp here is the clone or the original.
*/
fp->aux->prog = fp;
if (fp->aux->offload)
fp->aux->offload->prog = fp;
bpf_prog_clone_free(fp_other);
}
@@ -2087,12 +2088,12 @@ select_insn:
if (unlikely(tail_call_cnt >= MAX_TAIL_CALL_CNT))
goto out;
tail_call_cnt++;
prog = READ_ONCE(array->ptrs[index]);
if (!prog)
goto out;
tail_call_cnt++;
/* ARG1 at this point is guaranteed to point to CTX from
* the verifier side due to the fact that the tail call is
* handled like a helper, that is, bpf_tail_call_proto,
@@ -2613,8 +2614,10 @@ static struct bpf_prog_dummy {
},
};
struct bpf_empty_prog_array bpf_empty_prog_array = {
.null_prog = NULL,
struct bpf_prog_array bpf_empty_prog_array = {
.items = {
{ .prog = NULL },
},
};
EXPORT_SYMBOL(bpf_empty_prog_array);
@@ -2625,14 +2628,14 @@ struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags)
if (prog_cnt)
p = kzalloc_flex(*p, items, prog_cnt + 1, flags);
else
p = &bpf_empty_prog_array.hdr;
p = &bpf_empty_prog_array;
return p;
}
void bpf_prog_array_free(struct bpf_prog_array *progs)
{
if (!progs || progs == &bpf_empty_prog_array.hdr)
if (!progs || progs == &bpf_empty_prog_array)
return;
kfree_rcu(progs, rcu);
}
@@ -2641,19 +2644,17 @@ static void __bpf_prog_array_free_sleepable_cb(struct rcu_head *rcu)
{
struct bpf_prog_array *progs;
/* If RCU Tasks Trace grace period implies RCU grace period, there is
* no need to call kfree_rcu(), just call kfree() directly.
/*
* RCU Tasks Trace grace period implies RCU grace period, there is no
* need to call kfree_rcu(), just call kfree() directly.
*/
progs = container_of(rcu, struct bpf_prog_array, rcu);
if (rcu_trace_implies_rcu_gp())
kfree(progs);
else
kfree_rcu(progs, rcu);
kfree(progs);
}
void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs)
{
if (!progs || progs == &bpf_empty_prog_array.hdr)
if (!progs || progs == &bpf_empty_prog_array)
return;
call_rcu_tasks_trace(&progs->rcu, __bpf_prog_array_free_sleepable_cb);
}
@@ -3314,6 +3315,63 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_bulk_tx);
#ifdef CONFIG_BPF_SYSCALL
void bpf_get_linfo_file_line(struct btf *btf, const struct bpf_line_info *linfo,
const char **filep, const char **linep, int *nump)
{
/* Get base component of the file path. */
if (filep) {
*filep = btf_name_by_offset(btf, linfo->file_name_off);
*filep = kbasename(*filep);
}
/* Obtain the source line, and strip whitespace in prefix. */
if (linep) {
*linep = btf_name_by_offset(btf, linfo->line_off);
while (isspace(**linep))
*linep += 1;
}
if (nump)
*nump = BPF_LINE_INFO_LINE_NUM(linfo->line_col);
}
const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off)
{
const struct bpf_line_info *linfo;
u32 nr_linfo;
int l, r, m;
nr_linfo = prog->aux->nr_linfo;
if (!nr_linfo || insn_off >= prog->len)
return NULL;
linfo = prog->aux->linfo;
/* Loop invariant: linfo[l].insn_off <= insns_off.
* linfo[0].insn_off == 0 which always satisfies above condition.
* Binary search is searching for rightmost linfo entry that satisfies
* the above invariant, giving us the desired record that covers given
* instruction offset.
*/
l = 0;
r = nr_linfo - 1;
while (l < r) {
/* (r - l + 1) / 2 means we break a tie to the right, so if:
* l=1, r=2, linfo[l].insn_off <= insn_off, linfo[r].insn_off > insn_off,
* then m=2, we see that linfo[m].insn_off > insn_off, and so
* r becomes 1 and we exit the loop with correct l==1.
* If the tie was broken to the left, m=1 would end us up in
* an endless loop where l and m stay at 1 and r stays at 2.
*/
m = l + (r - l + 1) / 2;
if (linfo[m].insn_off <= insn_off)
l = m;
else
r = m - 1;
}
return &linfo[l];
}
int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep,
const char **linep, int *nump)
{
@@ -3348,14 +3406,7 @@ int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char *
if (idx == -1)
return -ENOENT;
/* Get base component of the file path. */
*filep = btf_name_by_offset(btf, linfo[idx].file_name_off);
*filep = kbasename(*filep);
/* Obtain the source line, and strip whitespace in prefix. */
*linep = btf_name_by_offset(btf, linfo[idx].line_off);
while (isspace(**linep))
*linep += 1;
*nump = BPF_LINE_INFO_LINE_NUM(linfo[idx].line_col);
bpf_get_linfo_file_line(btf, &linfo[idx], filep, linep, nump);
return 0;
}
kernel/bpf/cpumap.c
+4 -1
View File
@@ -223,7 +223,10 @@ static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu,
}
break;
default:
bpf_warn_invalid_xdp_action(NULL, rcpu->prog, act);
bpf_warn_invalid_xdp_action(xdpf->dev_rx, rcpu->prog, act);
fallthrough;
case XDP_ABORTED:
trace_xdp_exception(xdpf->dev_rx, rcpu->prog, act);
fallthrough;
case XDP_DROP:
xdp_return_frame(xdpf);
kernel/bpf/devmap.c
+2 -3
View File
@@ -665,7 +665,7 @@ int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx,
for (i = 0; i < dtab->n_buckets; i++) {
head = dev_map_index_hash(dtab, i);
hlist_for_each_entry_rcu(dst, head, index_hlist,
lockdep_is_held(&dtab->index_lock)) {
rcu_read_lock_bh_held()) {
if (!is_valid_dst(dst, xdpf))
continue;
@@ -747,7 +747,6 @@ int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
struct bpf_dtab_netdev *dst, *last_dst = NULL;
int excluded_devices[1+MAX_NEST_DEV];
struct hlist_head *head;
struct hlist_node *next;
int num_excluded = 0;
unsigned int i;
int err;
@@ -787,7 +786,7 @@ int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb,
} else { /* BPF_MAP_TYPE_DEVMAP_HASH */
for (i = 0; i < dtab->n_buckets; i++) {
head = dev_map_index_hash(dtab, i);
hlist_for_each_entry_safe(dst, next, head, index_hlist) {
hlist_for_each_entry_rcu(dst, head, index_hlist, rcu_read_lock_bh_held()) {
if (is_ifindex_excluded(excluded_devices, num_excluded,
dst->dev->ifindex))
continue;
kernel/bpf/fixups.c
+2457
View File
File diff suppressed because it is too large. Load diff
kernel/bpf/hashtab.c
+5 -1
View File
@@ -1056,7 +1056,7 @@ static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr,
for_each_possible_cpu(cpu) {
if (cpu == current_cpu)
copy_map_value_long(&htab->map, per_cpu_ptr(pptr, cpu), value);
copy_map_value(&htab->map, per_cpu_ptr(pptr, cpu), value);
else /* Since elem is preallocated, we cannot touch special fields */
zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu));
}
@@ -1138,6 +1138,10 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
} else if (fd_htab_map_needs_adjust(htab)) {
size = round_up(size, 8);
memcpy(htab_elem_value(l_new, key_size), value, size);
} else if (map_flags & BPF_F_LOCK) {
copy_map_value_locked(&htab->map,
htab_elem_value(l_new, key_size),
value, false);
} else {
copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value);
}
kernel/bpf/helpers.c
+203 -44
View File
@@ -1272,7 +1272,7 @@ static void bpf_async_cb_rcu_tasks_trace_free(struct rcu_head *rcu)
return;
}
/* rcu_trace_implies_rcu_gp() is true and will remain so */
/* RCU Tasks Trace grace period implies RCU grace period. */
bpf_async_cb_rcu_free(rcu);
}
@@ -2302,9 +2302,20 @@ void bpf_rb_root_free(const struct btf_field *field, void *rb_root,
__bpf_kfunc_start_defs();
__bpf_kfunc void *bpf_obj_new_impl(u64 local_type_id__k, void *meta__ign)
/**
* bpf_obj_new() - allocate an object described by program BTF
* @local_type_id__k: type ID in program BTF
* @meta: verifier-supplied struct metadata
*
* Allocate an object of the type identified by @local_type_id__k and
* initialize its special fields. BPF programs can use
* bpf_core_type_id_local() to provide @local_type_id__k. The verifier
* rewrites @meta; BPF programs do not set it.
*
* Return: Pointer to the allocated object, or %NULL on failure.
*/
__bpf_kfunc void *bpf_obj_new(u64 local_type_id__k, struct btf_struct_meta *meta)
{
struct btf_struct_meta *meta = meta__ign;
u64 size = local_type_id__k;
void *p;
@@ -2313,15 +2324,37 @@ __bpf_kfunc void *bpf_obj_new_impl(u64 local_type_id__k, void *meta__ign)
return NULL;
if (meta)
bpf_obj_init(meta->record, p);
return p;
}
__bpf_kfunc void *bpf_obj_new_impl(u64 local_type_id__k, void *meta__ign)
{
return bpf_obj_new(local_type_id__k, meta__ign);
}
/**
* bpf_percpu_obj_new() - allocate a percpu object described by program BTF
* @local_type_id__k: type ID in program BTF
* @meta: verifier-supplied struct metadata
*
* Allocate a percpu object of the type identified by @local_type_id__k. BPF
* programs can use bpf_core_type_id_local() to provide @local_type_id__k.
* The verifier rewrites @meta; BPF programs do not set it.
*
* Return: Pointer to the allocated percpu object, or %NULL on failure.
*/
__bpf_kfunc void *bpf_percpu_obj_new(u64 local_type_id__k, struct btf_struct_meta *meta)
{
u64 size = local_type_id__k;
/* The verifier has ensured that meta must be NULL */
return bpf_mem_alloc(&bpf_global_percpu_ma, size);
}
__bpf_kfunc void *bpf_percpu_obj_new_impl(u64 local_type_id__k, void *meta__ign)
{
u64 size = local_type_id__k;
/* The verifier has ensured that meta__ign must be NULL */
return bpf_mem_alloc(&bpf_global_percpu_ma, size);
return bpf_percpu_obj_new(local_type_id__k, meta__ign);
}
/* Must be called under migrate_disable(), as required by bpf_mem_free */
@@ -2347,23 +2380,56 @@ void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu)
bpf_mem_free_rcu(ma, p);
}
__bpf_kfunc void bpf_obj_drop_impl(void *p__alloc, void *meta__ign)
/**
* bpf_obj_drop() - drop a previously allocated object
* @p__alloc: object to free
* @meta: verifier-supplied struct metadata
*
* Destroy special fields in @p__alloc as needed and free the object. The
* verifier rewrites @meta; BPF programs do not set it.
*/
__bpf_kfunc void bpf_obj_drop(void *p__alloc, struct btf_struct_meta *meta)
{
struct btf_struct_meta *meta = meta__ign;
void *p = p__alloc;
__bpf_obj_drop_impl(p, meta ? meta->record : NULL, false);
}
__bpf_kfunc void bpf_percpu_obj_drop_impl(void *p__alloc, void *meta__ign)
__bpf_kfunc void bpf_obj_drop_impl(void *p__alloc, void *meta__ign)
{
/* The verifier has ensured that meta__ign must be NULL */
return bpf_obj_drop(p__alloc, meta__ign);
}
/**
* bpf_percpu_obj_drop() - drop a previously allocated percpu object
* @p__alloc: percpu object to free
* @meta: verifier-supplied struct metadata
*
* Free @p__alloc. The verifier rewrites @meta; BPF programs do not set it.
*/
__bpf_kfunc void bpf_percpu_obj_drop(void *p__alloc, struct btf_struct_meta *meta)
{
/* The verifier has ensured that meta must be NULL */
bpf_mem_free_rcu(&bpf_global_percpu_ma, p__alloc);
}
__bpf_kfunc void *bpf_refcount_acquire_impl(void *p__refcounted_kptr, void *meta__ign)
__bpf_kfunc void bpf_percpu_obj_drop_impl(void *p__alloc, void *meta__ign)
{
bpf_percpu_obj_drop(p__alloc, meta__ign);
}
/**
* bpf_refcount_acquire() - turn a local kptr into an owning reference
* @p__refcounted_kptr: non-owning local kptr
* @meta: verifier-supplied struct metadata
*
* Increment the refcount for @p__refcounted_kptr. The verifier rewrites
* @meta; BPF programs do not set it.
*
* Return: Owning reference to @p__refcounted_kptr, or %NULL on failure.
*/
__bpf_kfunc void *bpf_refcount_acquire(void *p__refcounted_kptr, struct btf_struct_meta *meta)
{
struct btf_struct_meta *meta = meta__ign;
struct bpf_refcount *ref;
/* Could just cast directly to refcount_t *, but need some code using
@@ -2379,6 +2445,11 @@ __bpf_kfunc void *bpf_refcount_acquire_impl(void *p__refcounted_kptr, void *meta
return (void *)p__refcounted_kptr;
}
__bpf_kfunc void *bpf_refcount_acquire_impl(void *p__refcounted_kptr, void *meta__ign)
{
return bpf_refcount_acquire(p__refcounted_kptr, meta__ign);
}
static int __bpf_list_add(struct bpf_list_node_kern *node,
struct bpf_list_head *head,
bool tail, struct btf_record *rec, u64 off)
@@ -2406,24 +2477,62 @@ static int __bpf_list_add(struct bpf_list_node_kern *node,
return 0;
}
/**
* bpf_list_push_front() - add a node to the front of a BPF linked list
* @head: list head
* @node: node to insert
* @meta: verifier-supplied struct metadata
* @off: verifier-supplied offset of @node within the containing object
*
* Insert @node at the front of @head. The verifier rewrites @meta and @off;
* BPF programs do not set them.
*
* Return: 0 on success, or %-EINVAL if @node is already linked.
*/
__bpf_kfunc int bpf_list_push_front(struct bpf_list_head *head,
struct bpf_list_node *node,
struct btf_struct_meta *meta,
u64 off)
{
struct bpf_list_node_kern *n = (void *)node;
return __bpf_list_add(n, head, false, meta ? meta->record : NULL, off);
}
__bpf_kfunc int bpf_list_push_front_impl(struct bpf_list_head *head,
struct bpf_list_node *node,
void *meta__ign, u64 off)
{
struct bpf_list_node_kern *n = (void *)node;
struct btf_struct_meta *meta = meta__ign;
return bpf_list_push_front(head, node, meta__ign, off);
}
return __bpf_list_add(n, head, false, meta ? meta->record : NULL, off);
/**
* bpf_list_push_back() - add a node to the back of a BPF linked list
* @head: list head
* @node: node to insert
* @meta: verifier-supplied struct metadata
* @off: verifier-supplied offset of @node within the containing object
*
* Insert @node at the back of @head. The verifier rewrites @meta and @off;
* BPF programs do not set them.
*
* Return: 0 on success, or %-EINVAL if @node is already linked.
*/
__bpf_kfunc int bpf_list_push_back(struct bpf_list_head *head,
struct bpf_list_node *node,
struct btf_struct_meta *meta,
u64 off)
{
struct bpf_list_node_kern *n = (void *)node;
return __bpf_list_add(n, head, true, meta ? meta->record : NULL, off);
}
__bpf_kfunc int bpf_list_push_back_impl(struct bpf_list_head *head,
struct bpf_list_node *node,
void *meta__ign, u64 off)
{
struct bpf_list_node_kern *n = (void *)node;
struct btf_struct_meta *meta = meta__ign;
return __bpf_list_add(n, head, true, meta ? meta->record : NULL, off);
return bpf_list_push_back(head, node, meta__ign, off);
}
static struct bpf_list_node *__bpf_list_del(struct bpf_list_head *head, bool tail)
@@ -2535,14 +2644,35 @@ static int __bpf_rbtree_add(struct bpf_rb_root *root,
return 0;
}
/**
* bpf_rbtree_add() - add a node to a BPF rbtree
* @root: tree root
* @node: node to insert
* @less: comparator used to order nodes
* @meta: verifier-supplied struct metadata
* @off: verifier-supplied offset of @node within the containing object
*
* Insert @node into @root using @less. The verifier rewrites @meta and @off;
* BPF programs do not set them.
*
* Return: 0 on success, or %-EINVAL if @node is already linked in a tree.
*/
__bpf_kfunc int bpf_rbtree_add(struct bpf_rb_root *root,
struct bpf_rb_node *node,
bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b),
struct btf_struct_meta *meta,
u64 off)
{
struct bpf_rb_node_kern *n = (void *)node;
return __bpf_rbtree_add(root, n, (void *)less, meta ? meta->record : NULL, off);
}
__bpf_kfunc int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node,
bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b),
void *meta__ign, u64 off)
{
struct btf_struct_meta *meta = meta__ign;
struct bpf_rb_node_kern *n = (void *)node;
return __bpf_rbtree_add(root, n, (void *)less, meta ? meta->record : NULL, off);
return bpf_rbtree_add(root, node, less, meta__ign, off);
}
__bpf_kfunc struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root)
@@ -4165,17 +4295,25 @@ static bool bpf_task_work_ctx_tryget(struct bpf_task_work_ctx *ctx)
return refcount_inc_not_zero(&ctx->refcnt);
}
static void bpf_task_work_destroy(struct irq_work *irq_work)
{
struct bpf_task_work_ctx *ctx = container_of(irq_work, struct bpf_task_work_ctx, irq_work);
bpf_task_work_ctx_reset(ctx);
kfree_rcu(ctx, rcu);
}
static void bpf_task_work_ctx_put(struct bpf_task_work_ctx *ctx)
{
if (!refcount_dec_and_test(&ctx->refcnt))
return;
bpf_task_work_ctx_reset(ctx);
/* bpf_mem_free expects migration to be disabled */
migrate_disable();
bpf_mem_free(&bpf_global_ma, ctx);
migrate_enable();
if (irqs_disabled()) {
ctx->irq_work = IRQ_WORK_INIT(bpf_task_work_destroy);
irq_work_queue(&ctx->irq_work);
} else {
bpf_task_work_destroy(&ctx->irq_work);
}
}
static void bpf_task_work_cancel(struct bpf_task_work_ctx *ctx)
@@ -4229,7 +4367,7 @@ static void bpf_task_work_irq(struct irq_work *irq_work)
enum bpf_task_work_state state;
int err;
guard(rcu_tasks_trace)();
guard(rcu)();
if (cmpxchg(&ctx->state, BPF_TW_PENDING, BPF_TW_SCHEDULING) != BPF_TW_PENDING) {
bpf_task_work_ctx_put(ctx);
@@ -4251,9 +4389,9 @@ static void bpf_task_work_irq(struct irq_work *irq_work)
/*
* It's technically possible for just scheduled task_work callback to
* complete running by now, going SCHEDULING -> RUNNING and then
* dropping its ctx refcount. Instead of capturing extra ref just to
* protected below ctx->state access, we rely on RCU protection to
* perform below SCHEDULING -> SCHEDULED attempt.
* dropping its ctx refcount. Instead of capturing an extra ref just
* to protect below ctx->state access, we rely on rcu_read_lock
* above to prevent kfree_rcu from freeing ctx before we return.
*/
state = cmpxchg(&ctx->state, BPF_TW_SCHEDULING, BPF_TW_SCHEDULED);
if (state == BPF_TW_FREED)
@@ -4270,7 +4408,7 @@ static struct bpf_task_work_ctx *bpf_task_work_fetch_ctx(struct bpf_task_work *t
if (ctx)
return ctx;
ctx = bpf_mem_alloc(&bpf_global_ma, sizeof(struct bpf_task_work_ctx));
ctx = bpf_map_kmalloc_nolock(map, sizeof(*ctx), 0, NUMA_NO_NODE);
if (!ctx)
return ERR_PTR(-ENOMEM);
@@ -4284,7 +4422,7 @@ static struct bpf_task_work_ctx *bpf_task_work_fetch_ctx(struct bpf_task_work *t
* tw->ctx is set by concurrent BPF program, release allocated
* memory and try to reuse already set context.
*/
bpf_mem_free(&bpf_global_ma, ctx);
kfree_nolock(ctx);
return old_ctx;
}
@@ -4296,13 +4434,23 @@ static struct bpf_task_work_ctx *bpf_task_work_acquire_ctx(struct bpf_task_work
{
struct bpf_task_work_ctx *ctx;
ctx = bpf_task_work_fetch_ctx(tw, map);
if (IS_ERR(ctx))
return ctx;
/*
* Sleepable BPF programs hold rcu_read_lock_trace but not
* regular rcu_read_lock. Since kfree_rcu waits for regular
* RCU GP, the ctx can be freed while we're between reading
* the pointer and incrementing the refcount. Take regular
* rcu_read_lock to prevent kfree_rcu from freeing the ctx
* before we can tryget it.
*/
scoped_guard(rcu) {
ctx = bpf_task_work_fetch_ctx(tw, map);
if (IS_ERR(ctx))
return ctx;
/* try to get ref for task_work callback to hold */
if (!bpf_task_work_ctx_tryget(ctx))
return ERR_PTR(-EBUSY);
/* try to get ref for task_work callback to hold */
if (!bpf_task_work_ctx_tryget(ctx))
return ERR_PTR(-EBUSY);
}
if (cmpxchg(&ctx->state, BPF_TW_STANDBY, BPF_TW_PENDING) != BPF_TW_STANDBY) {
/* lost acquiring race or map_release_uref() stole it from us, put ref and bail */
@@ -4417,7 +4565,7 @@ static int make_file_dynptr(struct file *file, u32 flags, bool may_sleep,
return -EINVAL;
}
state = bpf_mem_alloc(&bpf_global_ma, sizeof(struct bpf_dynptr_file_impl));
state = kmalloc_nolock(sizeof(*state), 0, NUMA_NO_NODE);
if (!state) {
bpf_dynptr_set_null(ptr);
return -ENOMEM;
@@ -4449,7 +4597,7 @@ __bpf_kfunc int bpf_dynptr_file_discard(struct bpf_dynptr *dynptr)
return 0;
freader_cleanup(&df->freader);
bpf_mem_free(&bpf_global_ma, df);
kfree_nolock(df);
bpf_dynptr_set_null(ptr);
return 0;
}
@@ -4536,12 +4684,19 @@ BTF_KFUNCS_START(generic_btf_ids)
#ifdef CONFIG_CRASH_DUMP
BTF_ID_FLAGS(func, crash_kexec, KF_DESTRUCTIVE)
#endif
BTF_ID_FLAGS(func, bpf_obj_new, KF_ACQUIRE | KF_RET_NULL | KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, bpf_obj_new_impl, KF_ACQUIRE | KF_RET_NULL)
BTF_ID_FLAGS(func, bpf_percpu_obj_new, KF_ACQUIRE | KF_RET_NULL | KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, bpf_percpu_obj_new_impl, KF_ACQUIRE | KF_RET_NULL)
BTF_ID_FLAGS(func, bpf_obj_drop, KF_RELEASE | KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, bpf_obj_drop_impl, KF_RELEASE)
BTF_ID_FLAGS(func, bpf_percpu_obj_drop, KF_RELEASE | KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, bpf_percpu_obj_drop_impl, KF_RELEASE)
BTF_ID_FLAGS(func, bpf_refcount_acquire, KF_ACQUIRE | KF_RET_NULL | KF_RCU | KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, bpf_refcount_acquire_impl, KF_ACQUIRE | KF_RET_NULL | KF_RCU)
BTF_ID_FLAGS(func, bpf_list_push_front, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, bpf_list_push_front_impl)
BTF_ID_FLAGS(func, bpf_list_push_back, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, bpf_list_push_back_impl)
BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL)
BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL)
@@ -4550,6 +4705,7 @@ BTF_ID_FLAGS(func, bpf_list_back, KF_RET_NULL)
BTF_ID_FLAGS(func, bpf_task_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL)
BTF_ID_FLAGS(func, bpf_task_release, KF_RELEASE)
BTF_ID_FLAGS(func, bpf_rbtree_remove, KF_ACQUIRE | KF_RET_NULL)
BTF_ID_FLAGS(func, bpf_rbtree_add, KF_IMPLICIT_ARGS)
BTF_ID_FLAGS(func, bpf_rbtree_add_impl)
BTF_ID_FLAGS(func, bpf_rbtree_first, KF_RET_NULL)
BTF_ID_FLAGS(func, bpf_rbtree_root, KF_RET_NULL)
@@ -4578,6 +4734,9 @@ BTF_ID_FLAGS(func, bpf_key_put, KF_RELEASE)
BTF_ID_FLAGS(func, bpf_verify_pkcs7_signature, KF_SLEEPABLE)
#endif
#endif
#ifdef CONFIG_S390
BTF_ID_FLAGS(func, bpf_get_lowcore)
#endif
BTF_KFUNCS_END(generic_btf_ids)
static const struct btf_kfunc_id_set generic_kfunc_set = {
kernel/bpf/liveness.c
+1926 -478
View File
File diff suppressed because it is too large. Load diff
kernel/bpf/local_storage.c
+1 -1
View File
@@ -270,7 +270,7 @@ static int cgroup_storage_get_next_key(struct bpf_map *_map, void *key,
goto enoent;
storage = list_next_entry(storage, list_map);
if (!storage)
if (list_entry_is_head(storage, &map->list, list_map))
goto enoent;
} else {
storage = list_first_entry(&map->list,
kernel/bpf/log.c
+12 -50
View File
@@ -329,47 +329,6 @@ __printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
}
EXPORT_SYMBOL_GPL(bpf_log);
static const struct bpf_line_info *
find_linfo(const struct bpf_verifier_env *env, u32 insn_off)
{
const struct bpf_line_info *linfo;
const struct bpf_prog *prog;
u32 nr_linfo;
int l, r, m;
prog = env->prog;
nr_linfo = prog->aux->nr_linfo;
if (!nr_linfo || insn_off >= prog->len)
return NULL;
linfo = prog->aux->linfo;
/* Loop invariant: linfo[l].insn_off <= insns_off.
* linfo[0].insn_off == 0 which always satisfies above condition.
* Binary search is searching for rightmost linfo entry that satisfies
* the above invariant, giving us the desired record that covers given
* instruction offset.
*/
l = 0;
r = nr_linfo - 1;
while (l < r) {
/* (r - l + 1) / 2 means we break a tie to the right, so if:
* l=1, r=2, linfo[l].insn_off <= insn_off, linfo[r].insn_off > insn_off,
* then m=2, we see that linfo[m].insn_off > insn_off, and so
* r becomes 1 and we exit the loop with correct l==1.
* If the tie was broken to the left, m=1 would end us up in
* an endless loop where l and m stay at 1 and r stays at 2.
*/
m = l + (r - l + 1) / 2;
if (linfo[m].insn_off <= insn_off)
l = m;
else
r = m - 1;
}
return &linfo[l];
}
static const char *ltrim(const char *s)
{
while (isspace(*s))
@@ -390,7 +349,7 @@ __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,
return;
prev_linfo = env->prev_linfo;
linfo = find_linfo(env, insn_off);
linfo = bpf_find_linfo(env->prog, insn_off);
if (!linfo || linfo == prev_linfo)
return;
@@ -542,7 +501,8 @@ static char slot_type_char[] = {
[STACK_ZERO] = '0',
[STACK_DYNPTR] = 'd',
[STACK_ITER] = 'i',
[STACK_IRQ_FLAG] = 'f'
[STACK_IRQ_FLAG] = 'f',
[STACK_POISON] = 'p',
};
#define UNUM_MAX_DECIMAL U16_MAX
@@ -581,6 +541,8 @@ int tnum_strn(char *str, size_t size, struct tnum a)
if (a.mask == 0) {
if (is_unum_decimal(a.value))
return snprintf(str, size, "%llu", a.value);
if (is_snum_decimal(a.value))
return snprintf(str, size, "%lld", a.value);
else
return snprintf(str, size, "%#llx", a.value);
}
@@ -692,7 +654,7 @@ static void print_reg_state(struct bpf_verifier_env *env,
if (state->frameno != reg->frameno)
verbose(env, "[%d]", reg->frameno);
if (tnum_is_const(reg->var_off)) {
verbose_snum(env, reg->var_off.value + reg->off);
verbose_snum(env, reg->var_off.value + reg->delta);
return;
}
}
@@ -702,7 +664,7 @@ static void print_reg_state(struct bpf_verifier_env *env,
if (reg->id)
verbose_a("id=%d", reg->id & ~BPF_ADD_CONST);
if (reg->id & BPF_ADD_CONST)
verbose(env, "%+d", reg->off);
verbose(env, "%+d", reg->delta);
if (reg->ref_obj_id)
verbose_a("ref_obj_id=%d", reg->ref_obj_id);
if (type_is_non_owning_ref(reg->type))
@@ -714,9 +676,9 @@ static void print_reg_state(struct bpf_verifier_env *env,
reg->map_ptr->key_size,
reg->map_ptr->value_size);
}
if (t != SCALAR_VALUE && reg->off) {
if (t != SCALAR_VALUE && reg->delta) {
verbose_a("off=");
verbose_snum(env, reg->off);
verbose_snum(env, reg->delta);
}
if (type_is_pkt_pointer(t)) {
verbose_a("r=");
@@ -777,7 +739,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie
for (j = 0; j < BPF_REG_SIZE; j++) {
slot_type = state->stack[i].slot_type[j];
if (slot_type != STACK_INVALID)
if (slot_type != STACK_INVALID && slot_type != STACK_POISON)
valid = true;
types_buf[j] = slot_type_char[slot_type];
}
@@ -845,7 +807,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie
mark_verifier_state_clean(env);
}
static inline u32 vlog_alignment(u32 pos)
u32 bpf_vlog_alignment(u32 pos)
{
return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT),
BPF_LOG_MIN_ALIGNMENT) - pos - 1;
@@ -857,7 +819,7 @@ void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_st
if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) {
/* remove new line character */
bpf_vlog_reset(&env->log, env->prev_log_pos - 1);
verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_pos), ' ');
verbose(env, "%*c;", bpf_vlog_alignment(env->prev_insn_print_pos), ' ');
} else {
verbose(env, "%d:", env->insn_idx);
}
kernel/bpf/memalloc.c
+10 -23
View File
@@ -284,17 +284,6 @@ static void __free_rcu(struct rcu_head *head)
atomic_set(&c->call_rcu_ttrace_in_progress, 0);
}
static void __free_rcu_tasks_trace(struct rcu_head *head)
{
/* If RCU Tasks Trace grace period implies RCU grace period,
* there is no need to invoke call_rcu().
*/
if (rcu_trace_implies_rcu_gp())
__free_rcu(head);
else
call_rcu(head, __free_rcu);
}
static void enque_to_free(struct bpf_mem_cache *c, void *obj)
{
struct llist_node *llnode = obj;
@@ -326,12 +315,12 @@ static void do_call_rcu_ttrace(struct bpf_mem_cache *c)
return;
}
/* Use call_rcu_tasks_trace() to wait for sleepable progs to finish.
* If RCU Tasks Trace grace period implies RCU grace period, free
* these elements directly, else use call_rcu() to wait for normal
* progs to finish and finally do free_one() on each element.
/*
* Use call_rcu_tasks_trace() to wait for sleepable progs to finish.
* RCU Tasks Trace grace period implies RCU grace period, so pass
* __free_rcu directly as the callback.
*/
call_rcu_tasks_trace(&c->rcu_ttrace, __free_rcu_tasks_trace);
call_rcu_tasks_trace(&c->rcu_ttrace, __free_rcu);
}
static void free_bulk(struct bpf_mem_cache *c)
@@ -696,20 +685,18 @@ static void free_mem_alloc_no_barrier(struct bpf_mem_alloc *ma)
static void free_mem_alloc(struct bpf_mem_alloc *ma)
{
/* waiting_for_gp[_ttrace] lists were drained, but RCU callbacks
/*
* waiting_for_gp[_ttrace] lists were drained, but RCU callbacks
* might still execute. Wait for them.
*
* rcu_barrier_tasks_trace() doesn't imply synchronize_rcu_tasks_trace(),
* but rcu_barrier_tasks_trace() and rcu_barrier() below are only used
* to wait for the pending __free_rcu_tasks_trace() and __free_rcu(),
* so if call_rcu(head, __free_rcu) is skipped due to
* rcu_trace_implies_rcu_gp(), it will be OK to skip rcu_barrier() by
* using rcu_trace_implies_rcu_gp() as well.
* to wait for the pending __free_by_rcu(), and __free_rcu(). RCU Tasks
* Trace grace period implies RCU grace period, so all __free_rcu don't
* need extra call_rcu() (and thus extra rcu_barrier() here).
*/
rcu_barrier(); /* wait for __free_by_rcu */
rcu_barrier_tasks_trace(); /* wait for __free_rcu */
if (!rcu_trace_implies_rcu_gp())
rcu_barrier();
free_mem_alloc_no_barrier(ma);
}
kernel/bpf/offload.c
+4 -6
View File
@@ -435,9 +435,8 @@ static struct ns_common *bpf_prog_offload_info_fill_ns(void *private_data)
if (aux->offload) {
args->info->ifindex = aux->offload->netdev->ifindex;
net = dev_net(aux->offload->netdev);
get_net(net);
ns = &net->ns;
net = maybe_get_net(dev_net(aux->offload->netdev));
ns = net ? &net->ns : NULL;
} else {
args->info->ifindex = 0;
ns = NULL;
@@ -647,9 +646,8 @@ static struct ns_common *bpf_map_offload_info_fill_ns(void *private_data)
if (args->offmap->netdev) {
args->info->ifindex = args->offmap->netdev->ifindex;
net = dev_net(args->offmap->netdev);
get_net(net);
ns = &net->ns;
net = maybe_get_net(dev_net(args->offmap->netdev));
ns = net ? &net->ns : NULL;
} else {
args->info->ifindex = 0;
ns = NULL;
kernel/bpf/states.c
+1563
View File
File diff suppressed because it is too large. Load diff
kernel/bpf/syscall.c
+23 -21
View File
@@ -943,14 +943,6 @@ static void bpf_map_free_rcu_gp(struct rcu_head *rcu)
bpf_map_free_in_work(container_of(rcu, struct bpf_map, rcu));
}
static void bpf_map_free_mult_rcu_gp(struct rcu_head *rcu)
{
if (rcu_trace_implies_rcu_gp())
bpf_map_free_rcu_gp(rcu);
else
call_rcu(rcu, bpf_map_free_rcu_gp);
}
/* decrement map refcnt and schedule it for freeing via workqueue
* (underlying map implementation ops->map_free() might sleep)
*/
@@ -961,8 +953,9 @@ void bpf_map_put(struct bpf_map *map)
bpf_map_free_id(map);
WARN_ON_ONCE(atomic64_read(&map->sleepable_refcnt));
/* RCU tasks trace grace period implies RCU grace period. */
if (READ_ONCE(map->free_after_mult_rcu_gp))
call_rcu_tasks_trace(&map->rcu, bpf_map_free_mult_rcu_gp);
call_rcu_tasks_trace(&map->rcu, bpf_map_free_rcu_gp);
else if (READ_ONCE(map->free_after_rcu_gp))
call_rcu(&map->rcu, bpf_map_free_rcu_gp);
else
@@ -2834,7 +2827,7 @@ static int bpf_prog_verify_signature(struct bpf_prog *prog, union bpf_attr *attr
sig = kvmemdup_bpfptr(usig, attr->signature_size);
if (IS_ERR(sig)) {
bpf_key_put(key);
return -ENOMEM;
return PTR_ERR(sig);
}
bpf_dynptr_init(&sig_ptr, sig, BPF_DYNPTR_TYPE_LOCAL, 0,
@@ -3275,14 +3268,6 @@ static bool bpf_link_is_tracepoint(struct bpf_link *link)
(link->type == BPF_LINK_TYPE_TRACING && link->attach_type == BPF_TRACE_RAW_TP);
}
static void bpf_link_defer_dealloc_mult_rcu_gp(struct rcu_head *rcu)
{
if (rcu_trace_implies_rcu_gp())
bpf_link_defer_dealloc_rcu_gp(rcu);
else
call_rcu(rcu, bpf_link_defer_dealloc_rcu_gp);
}
/* bpf_link_free is guaranteed to be called from process context */
static void bpf_link_free(struct bpf_link *link)
{
@@ -3308,7 +3293,8 @@ static void bpf_link_free(struct bpf_link *link)
* faultable case, since it exclusively uses RCU Tasks Trace.
*/
if (link->sleepable || (link->prog && link->prog->sleepable))
call_rcu_tasks_trace(&link->rcu, bpf_link_defer_dealloc_mult_rcu_gp);
/* RCU Tasks Trace grace period implies RCU grace period. */
call_rcu_tasks_trace(&link->rcu, bpf_link_defer_dealloc_rcu_gp);
/* We need to do a SRCU grace period wait for non-faultable tracepoint BPF links. */
else if (bpf_link_is_tracepoint(link))
call_tracepoint_unregister_atomic(&link->rcu, bpf_link_defer_dealloc_rcu_gp);
@@ -3756,6 +3742,23 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
tr = prog->aux->dst_trampoline;
tgt_prog = prog->aux->dst_prog;
}
/*
* It is to prevent modifying struct pt_regs via kprobe_write_ctx=true
* freplace prog. Without this check, kprobe_write_ctx=true freplace
* prog is allowed to attach to kprobe_write_ctx=false kprobe prog, and
* then modify the registers of the kprobe prog's target kernel
* function.
*
* This also blocks the combination of uprobe+freplace, because it is
* unable to recognize the use of the tgt_prog as an uprobe or a kprobe
* by tgt_prog itself. At attach time, uprobe/kprobe is recognized by
* the target perf event flags in __perf_event_set_bpf_prog().
*/
if (prog->type == BPF_PROG_TYPE_EXT &&
prog->aux->kprobe_write_ctx != tgt_prog->aux->kprobe_write_ctx) {
err = -EINVAL;
goto out_unlock;
}
err = bpf_link_prime(&link->link.link, &link_primer);
if (err)
@@ -6373,8 +6376,7 @@ static bool syscall_prog_is_valid_access(int off, int size,
{
if (off < 0 || off >= U16_MAX)
return false;
if (off % size != 0)
return false;
/* No alignment requirements for syscall ctx accesses. */
return true;
}
kernel/bpf/task_iter.c
+136 -15
View File
@@ -9,6 +9,8 @@
#include <linux/bpf_mem_alloc.h>
#include <linux/btf_ids.h>
#include <linux/mm_types.h>
#include <linux/mmap_lock.h>
#include <linux/sched/mm.h>
#include "mmap_unlock_work.h"
static const char * const iter_task_type_names[] = {
@@ -794,11 +796,20 @@ const struct bpf_func_proto bpf_find_vma_proto = {
.arg5_type = ARG_ANYTHING,
};
static inline void bpf_iter_mmput_async(struct mm_struct *mm)
{
#ifdef CONFIG_MMU
mmput_async(mm);
#else
mmput(mm);
#endif
}
struct bpf_iter_task_vma_kern_data {
struct task_struct *task;
struct mm_struct *mm;
struct mmap_unlock_irq_work *work;
struct vma_iterator vmi;
struct vm_area_struct snapshot;
u64 next_addr;
};
struct bpf_iter_task_vma {
@@ -819,12 +830,28 @@ __bpf_kfunc int bpf_iter_task_vma_new(struct bpf_iter_task_vma *it,
struct task_struct *task, u64 addr)
{
struct bpf_iter_task_vma_kern *kit = (void *)it;
bool irq_work_busy = false;
int err;
BUILD_BUG_ON(sizeof(struct bpf_iter_task_vma_kern) != sizeof(struct bpf_iter_task_vma));
BUILD_BUG_ON(__alignof__(struct bpf_iter_task_vma_kern) != __alignof__(struct bpf_iter_task_vma));
if (!IS_ENABLED(CONFIG_PER_VMA_LOCK)) {
kit->data = NULL;
return -EOPNOTSUPP;
}
/*
* Reject irqs-disabled contexts including NMI. Operations used
* by _next() and _destroy() (vma_end_read, fput, bpf_iter_mmput_async)
* can take spinlocks with IRQs disabled (pi_lock, pool->lock).
* Running from NMI or from a tracepoint that fires with those
* locks held could deadlock.
*/
if (irqs_disabled()) {
kit->data = NULL;
return -EBUSY;
}
/* is_iter_reg_valid_uninit guarantees that kit hasn't been initialized
* before, so non-NULL kit->data doesn't point to previously
* bpf_mem_alloc'd bpf_iter_task_vma_kern_data
@@ -834,38 +861,131 @@ __bpf_kfunc int bpf_iter_task_vma_new(struct bpf_iter_task_vma *it,
return -ENOMEM;
kit->data->task = get_task_struct(task);
/*
* Safely read task->mm and acquire an mm reference.
*
* Cannot use get_task_mm() because its task_lock() is a
* blocking spin_lock that would deadlock if the target task
* already holds alloc_lock on this CPU (e.g. a softirq BPF
* program iterating a task interrupted while holding its
* alloc_lock).
*/
if (!spin_trylock(&task->alloc_lock)) {
err = -EBUSY;
goto err_cleanup_iter;
}
kit->data->mm = task->mm;
if (kit->data->mm && !(task->flags & PF_KTHREAD))
mmget(kit->data->mm);
else
kit->data->mm = NULL;
spin_unlock(&task->alloc_lock);
if (!kit->data->mm) {
err = -ENOENT;
goto err_cleanup_iter;
}
/* kit->data->work == NULL is valid after bpf_mmap_unlock_get_irq_work */
irq_work_busy = bpf_mmap_unlock_get_irq_work(&kit->data->work);
if (irq_work_busy || !mmap_read_trylock(kit->data->mm)) {
err = -EBUSY;
goto err_cleanup_iter;
}
vma_iter_init(&kit->data->vmi, kit->data->mm, addr);
kit->data->snapshot.vm_file = NULL;
kit->data->next_addr = addr;
return 0;
err_cleanup_iter:
if (kit->data->task)
put_task_struct(kit->data->task);
put_task_struct(kit->data->task);
bpf_mem_free(&bpf_global_ma, kit->data);
/* NULL kit->data signals failed bpf_iter_task_vma initialization */
kit->data = NULL;
return err;
}
/*
* Find and lock the next VMA at or after data->next_addr.
*
* lock_vma_under_rcu() is a point lookup (mas_walk): it finds the VMA
* containing a given address but cannot iterate. An RCU-protected
* maple tree walk with vma_next() (mas_find) is needed first to locate
* the next VMA's vm_start across any gap.
*
* Between the RCU walk and the lock, the VMA may be removed, shrunk,
* or write-locked. On failure, advance past it using vm_end from the
* RCU walk. SLAB_TYPESAFE_BY_RCU can make vm_end stale, so fall back
* to PAGE_SIZE advancement to guarantee forward progress.
*/
static struct vm_area_struct *
bpf_iter_task_vma_find_next(struct bpf_iter_task_vma_kern_data *data)
{
struct vm_area_struct *vma;
struct vma_iterator vmi;
unsigned long start, end;
retry:
rcu_read_lock();
vma_iter_init(&vmi, data->mm, data->next_addr);
vma = vma_next(&vmi);
if (!vma) {
rcu_read_unlock();
return NULL;
}
start = vma->vm_start;
end = vma->vm_end;
rcu_read_unlock();
vma = lock_vma_under_rcu(data->mm, start);
if (!vma) {
if (end <= data->next_addr)
data->next_addr += PAGE_SIZE;
else
data->next_addr = end;
goto retry;
}
if (unlikely(vma->vm_end <= data->next_addr)) {
data->next_addr += PAGE_SIZE;
vma_end_read(vma);
goto retry;
}
return vma;
}
static void bpf_iter_task_vma_snapshot_reset(struct vm_area_struct *snap)
{
if (snap->vm_file) {
fput(snap->vm_file);
snap->vm_file = NULL;
}
}
__bpf_kfunc struct vm_area_struct *bpf_iter_task_vma_next(struct bpf_iter_task_vma *it)
{
struct bpf_iter_task_vma_kern *kit = (void *)it;
struct vm_area_struct *snap, *vma;
if (!kit->data) /* bpf_iter_task_vma_new failed */
return NULL;
return vma_next(&kit->data->vmi);
snap = &kit->data->snapshot;
bpf_iter_task_vma_snapshot_reset(snap);
vma = bpf_iter_task_vma_find_next(kit->data);
if (!vma)
return NULL;
memcpy(snap, vma, sizeof(*snap));
/*
* The verifier only trusts vm_mm and vm_file (see
* BTF_TYPE_SAFE_TRUSTED_OR_NULL in verifier.c). Take a reference
* on vm_file; vm_mm is already correct because lock_vma_under_rcu()
* verifies vma->vm_mm == mm. All other pointers are untrusted by
* the verifier and left as-is.
*/
if (snap->vm_file)
get_file(snap->vm_file);
kit->data->next_addr = vma->vm_end;
vma_end_read(vma);
return snap;
}
__bpf_kfunc void bpf_iter_task_vma_destroy(struct bpf_iter_task_vma *it)
@@ -873,8 +993,9 @@ __bpf_kfunc void bpf_iter_task_vma_destroy(struct bpf_iter_task_vma *it)
struct bpf_iter_task_vma_kern *kit = (void *)it;
if (kit->data) {
bpf_mmap_unlock_mm(kit->data->work, kit->data->mm);
bpf_iter_task_vma_snapshot_reset(&kit->data->snapshot);
put_task_struct(kit->data->task);
bpf_iter_mmput_async(kit->data->mm);
bpf_mem_free(&bpf_global_ma, kit->data);
}
}
kernel/bpf/tnum.c
+19 -27
View File
@@ -286,8 +286,7 @@ struct tnum tnum_bswap64(struct tnum a)
*/
u64 tnum_step(struct tnum t, u64 z)
{
u64 tmax, j, p, q, r, s, v, u, w, res;
u8 k;
u64 tmax, d, carry_mask, filled, inc;
tmax = t.value | t.mask;
@@ -299,29 +298,22 @@ u64 tnum_step(struct tnum t, u64 z)
if (z < t.value)
return t.value;
/* keep t's known bits, and match all unknown bits to z */
j = t.value | (z & t.mask);
if (j > z) {
p = ~z & t.value & ~t.mask;
k = fls64(p); /* k is the most-significant 0-to-1 flip */
q = U64_MAX << k;
r = q & z; /* positions > k matched to z */
s = ~q & t.value; /* positions <= k matched to t.value */
v = r | s;
res = v;
} else {
p = z & ~t.value & ~t.mask;
k = fls64(p); /* k is the most-significant 1-to-0 flip */
q = U64_MAX << k;
r = q & t.mask & z; /* unknown positions > k, matched to z */
s = q & ~t.mask; /* known positions > k, set to 1 */
v = r | s;
/* add 1 to unknown positions > k to make value greater than z */
u = v + (1ULL << k);
/* extract bits in unknown positions > k from u, rest from t.value */
w = (u & t.mask) | t.value;
res = w;
}
return res;
/*
* Let r be the result tnum member, z = t.value + d.
* Every tnum member is t.value | s for some submask s of t.mask,
* and since t.value & t.mask == 0, t.value | s == t.value + s.
* So r > z becomes s > d where d = z - t.value.
*
* Find the smallest submask s of t.mask greater than d by
* "incrementing d within the mask": fill every non-mask
* position with 1 (`filled`) so +1 ripples through the gaps,
* then keep only mask bits. `carry_mask` additionally fills
* positions below the highest non-mask 1 in d, preventing
* it from trapping the carry.
*/
d = z - t.value;
carry_mask = (1ULL << fls64(d & ~t.mask)) - 1;
filled = d | carry_mask | ~t.mask;
inc = (filled + 1) & t.mask;
return t.value | inc;
}
kernel/bpf/verifier.c
+2005 -8057
View File
File diff suppressed because it is too large. Load diff
kernel/trace/trace_kprobe.c
+8
View File
@@ -765,6 +765,14 @@ static unsigned int number_of_same_symbols(const char *mod, const char *func_nam
if (!mod)
kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count);
/*
* If the symbol is found in vmlinux, use vmlinux resolution only.
* This prevents module symbols from shadowing vmlinux symbols
* and causing -EADDRNOTAVAIL for unqualified kprobe targets.
*/
if (!mod && ctx.count > 0)
return ctx.count;
module_kallsyms_on_each_symbol(mod, count_mod_symbols, &ctx);
return ctx.count;
net/bpf/test_run.c
+41 -8
View File
@@ -567,6 +567,23 @@ noinline void bpf_fentry_test_sinfo(struct skb_shared_info *sinfo)
{
}
noinline void bpf_fentry_test_ppvoid(void **pp)
{
}
noinline void bpf_fentry_test_pppvoid(void ***ppp)
{
}
noinline void bpf_fentry_test_ppfile(struct file **ppf)
{
}
noinline struct file **bpf_fexit_test_ret_ppfile(void)
{
return (struct file **)NULL;
}
__bpf_kfunc int bpf_modify_return_test(int a, int *b)
{
*b += 1;
@@ -1120,19 +1137,23 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
switch (skb->protocol) {
case htons(ETH_P_IP):
sk->sk_family = AF_INET;
if (sizeof(struct iphdr) <= skb_headlen(skb)) {
sk->sk_rcv_saddr = ip_hdr(skb)->saddr;
sk->sk_daddr = ip_hdr(skb)->daddr;
if (skb_headlen(skb) < sizeof(struct iphdr)) {
ret = -EINVAL;
goto out;
}
sk->sk_family = AF_INET;
sk->sk_rcv_saddr = ip_hdr(skb)->saddr;
sk->sk_daddr = ip_hdr(skb)->daddr;
break;
#if IS_ENABLED(CONFIG_IPV6)
case htons(ETH_P_IPV6):
sk->sk_family = AF_INET6;
if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) {
sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr;
sk->sk_v6_daddr = ipv6_hdr(skb)->daddr;
if (skb_headlen(skb) < sizeof(struct ipv6hdr)) {
ret = -EINVAL;
goto out;
}
sk->sk_family = AF_INET6;
sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr;
sk->sk_v6_daddr = ipv6_hdr(skb)->daddr;
break;
#endif
default:
@@ -1156,6 +1177,18 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
skb->ip_summed = CHECKSUM_COMPLETE;
}
if (prog->type == BPF_PROG_TYPE_LWT_XMIT) {
if (!ipv6_mod_enabled()) {
pr_warn_once("Please test this program with IPv6 enabled kernel\n");
ret = -EOPNOTSUPP;
goto out;
}
#if IS_ENABLED(CONFIG_IPV6)
dst_hold(&net->ipv6.ip6_null_entry->dst);
skb_dst_set(skb, &net->ipv6.ip6_null_entry->dst);
#endif
}
ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false);
if (ret)
goto out;
net/core/bpf_sk_storage.c
+10 -13
View File
@@ -68,7 +68,7 @@ static void bpf_sk_storage_map_free(struct bpf_map *map)
static struct bpf_map *bpf_sk_storage_map_alloc(union bpf_attr *attr)
{
return bpf_local_storage_map_alloc(attr, &sk_cache, false);
return bpf_local_storage_map_alloc(attr, &sk_cache);
}
static int notsupp_get_next_key(struct bpf_map *map, void *key,
@@ -106,7 +106,7 @@ static long bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key,
if (sock) {
sdata = bpf_local_storage_update(
sock->sk, (struct bpf_local_storage_map *)map, value,
map_flags, false, GFP_ATOMIC);
map_flags, false);
sockfd_put(sock);
return PTR_ERR_OR_ZERO(sdata);
}
@@ -137,7 +137,7 @@ bpf_sk_storage_clone_elem(struct sock *newsk,
{
struct bpf_local_storage_elem *copy_selem;
copy_selem = bpf_selem_alloc(smap, newsk, NULL, false, GFP_ATOMIC);
copy_selem = bpf_selem_alloc(smap, newsk, NULL, false);
if (!copy_selem)
return NULL;
@@ -202,7 +202,7 @@ int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)
}
bpf_selem_link_storage_nolock(new_sk_storage, copy_selem);
} else {
ret = bpf_local_storage_alloc(newsk, smap, copy_selem, GFP_ATOMIC);
ret = bpf_local_storage_alloc(newsk, smap, copy_selem);
if (ret) {
bpf_selem_free(copy_selem, true);
atomic_sub(smap->elem_size,
@@ -227,9 +227,8 @@ out:
return ret;
}
/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
void *, value, u64, flags, gfp_t, gfp_flags)
BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
void *, value, u64, flags)
{
struct bpf_local_storage_data *sdata;
@@ -250,7 +249,7 @@ BPF_CALL_5(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
refcount_inc_not_zero(&sk->sk_refcnt)) {
sdata = bpf_local_storage_update(
sk, (struct bpf_local_storage_map *)map, value,
BPF_NOEXIST, false, gfp_flags);
BPF_NOEXIST, false);
/* sk must be a fullsock (guaranteed by verifier),
* so sock_gen_put() is unnecessary.
*/
@@ -383,16 +382,14 @@ static bool bpf_sk_storage_tracing_allowed(const struct bpf_prog *prog)
return false;
}
/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk,
void *, value, u64, flags, gfp_t, gfp_flags)
BPF_CALL_4(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk,
void *, value, u64, flags)
{
WARN_ON_ONCE(!bpf_rcu_lock_held());
if (in_hardirq() || in_nmi())
return (unsigned long)NULL;
return (unsigned long)____bpf_sk_storage_get(map, sk, value, flags,
gfp_flags);
return (unsigned long)____bpf_sk_storage_get(map, sk, value, flags);
}
BPF_CALL_2(bpf_sk_storage_delete_tracing, struct bpf_map *, map,
net/ipv6/addrconf.c
+1
View File
@@ -36,6 +36,7 @@
#define pr_fmt(fmt) "IPv6: " fmt
#include <crypto/sha1.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/kernel.h>
net/xdp/xsk.c
+15 -3
View File
@@ -160,6 +160,17 @@ static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff_xsk *xskb, u32 len,
return 0;
}
static void __xsk_rcv_zc_safe(struct xdp_sock *xs, struct xdp_buff_xsk *xskb,
u32 len, u32 flags)
{
u64 addr;
addr = xp_get_handle(xskb, xskb->pool);
__xskq_prod_reserve_desc(xs->rx, addr, len, flags);
xp_release(xskb);
}
static int xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
{
struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
@@ -185,13 +196,13 @@ static int xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
goto err;
}
__xsk_rcv_zc(xs, xskb, len, contd);
__xsk_rcv_zc_safe(xs, xskb, len, contd);
xskb_list = &xskb->pool->xskb_list;
list_for_each_entry_safe(pos, tmp, xskb_list, list_node) {
if (list_is_singular(xskb_list))
contd = 0;
len = pos->xdp.data_end - pos->xdp.data;
__xsk_rcv_zc(xs, pos, len, contd);
__xsk_rcv_zc_safe(xs, pos, len, contd);
list_del_init(&pos->list_node);
}
@@ -298,7 +309,8 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len)
rem -= copied;
xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp);
__xsk_rcv_zc(xs, xskb, copied - meta_len, rem ? XDP_PKT_CONTD : 0);
__xsk_rcv_zc_safe(xs, xskb, copied - meta_len,
rem ? XDP_PKT_CONTD : 0);
meta_len = 0;
} while (rem);
net/xdp/xsk_queue.h
+11 -5
View File
@@ -445,20 +445,26 @@ static inline void xskq_prod_write_addr_batch(struct xsk_queue *q, struct xdp_de
q->cached_prod = cached_prod;
}
static inline int xskq_prod_reserve_desc(struct xsk_queue *q,
u64 addr, u32 len, u32 flags)
static inline void __xskq_prod_reserve_desc(struct xsk_queue *q,
u64 addr, u32 len, u32 flags)
{
struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
u32 idx;
if (xskq_prod_is_full(q))
return -ENOBUFS;
/* A, matches D */
idx = q->cached_prod++ & q->ring_mask;
ring->desc[idx].addr = addr;
ring->desc[idx].len = len;
ring->desc[idx].options = flags;
}
static inline int xskq_prod_reserve_desc(struct xsk_queue *q,
u64 addr, u32 len, u32 flags)
{
if (xskq_prod_is_full(q))
return -ENOBUFS;
__xskq_prod_reserve_desc(q, addr, len, flags);
return 0;
}
scripts/Makefile.btf
+2
View File
@@ -18,6 +18,8 @@ pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j$(JOBS) --btf_features=enc
pahole-flags-$(call test-ge, $(pahole-ver), 130) += --btf_features=attributes
pahole-flags-$(call test-ge, $(pahole-ver), 131) += --btf_features=layout
endif
pahole-flags-$(CONFIG_PAHOLE_HAS_LANG_EXCLUDE) += --lang_exclude=rust
tools/bpf/bpftool/Documentation/bpftool-btf.rst
+7 -4
View File
@@ -27,7 +27,7 @@ BTF COMMANDS
| **bpftool** **btf dump** *BTF_SRC* [**format** *FORMAT*] [**root_id** *ROOT_ID*]
| **bpftool** **btf help**
|
| *BTF_SRC* := { **id** *BTF_ID* | **prog** *PROG* | **map** *MAP* [{**key** | **value** | **kv** | **all**}] | **file** *FILE* }
| *BTF_SRC* := { **id** *BTF_ID* | **prog** *PROG* | **map** *MAP* [{**key** | **value** | **kv** | **all**}] | **file** *FILE* [**file** *FILE*]... }
| *FORMAT* := { **raw** | **c** [**unsorted**] }
| *MAP* := { **id** *MAP_ID* | **pinned** *FILE* }
| *PROG* := { **id** *PROG_ID* | **pinned** *FILE* | **tag** *PROG_TAG* | **name** *PROG_NAME* }
@@ -58,9 +58,12 @@ bpftool btf dump *BTF_SRC* [format *FORMAT*] [root_id *ROOT_ID*]
When **prog** is provided, it's expected that program has associated BTF
object with BTF types.
When specifying *FILE*, an ELF file is expected, containing .BTF section
with well-defined BTF binary format data, typically produced by clang or
pahole.
When specifying *FILE*, an ELF file or a raw BTF file (e.g. from
``/sys/kernel/btf/``) is expected. Multiple **file** arguments may be
given to merge BTF from several kernel modules into a single output.
When sysfs paths are used, vmlinux BTF is loaded automatically as the
base; if vmlinux itself appears in the file list it is skipped.
A base BTF can also be specified explicitly with **-B**.
**format** option can be used to override default (raw) output format. Raw
(**raw**) or C-syntax (**c**) output formats are supported. With C-style
tools/bpf/bpftool/Documentation/bpftool-prog.rst
+1 -1
View File
@@ -54,7 +54,7 @@ PROG COMMANDS
| **cgroup/sendmsg4** | **cgroup/sendmsg6** | **cgroup/sendmsg_unix** |
| **cgroup/recvmsg4** | **cgroup/recvmsg6** | **cgroup/recvmsg_unix** | **cgroup/sysctl** |
| **cgroup/getsockopt** | **cgroup/setsockopt** | **cgroup/sock_release** |
| **struct_ops** | **fentry** | **fexit** | **freplace** | **sk_lookup**
| **struct_ops** | **fentry** | **fexit** | **fsession** | **freplace** | **sk_lookup**
| }
| *ATTACH_TYPE* := {
| **sk_msg_verdict** | **sk_skb_verdict** | **sk_skb_stream_verdict** |
tools/bpf/bpftool/Makefile
+26 -4
View File
@@ -97,6 +97,15 @@ RM ?= rm -f
FEATURE_USER = .bpftool
# Skip optional dependencies: LLVM (JIT disasm), libbfd (fallback
# disasm), libcrypto (program signing).
SKIP_LLVM ?=
SKIP_LIBBFD ?=
SKIP_CRYPTO ?=
ifneq ($(SKIP_CRYPTO),1)
CRYPTO_LIBS := -lcrypto
endif
FEATURE_TESTS := clang-bpf-co-re
FEATURE_TESTS += llvm
FEATURE_TESTS += libcap
@@ -130,8 +139,8 @@ include $(FEATURES_DUMP)
endif
endif
LIBS = $(LIBBPF) -lelf -lcrypto -lz
LIBS_BOOTSTRAP = $(LIBBPF_BOOTSTRAP) -lelf -lcrypto -lz
LIBS = $(LIBBPF) -lelf $(CRYPTO_LIBS) -lz
LIBS_BOOTSTRAP = $(LIBBPF_BOOTSTRAP) -lelf $(CRYPTO_LIBS) -lz
ifeq ($(feature-libelf-zstd),1)
LIBS += -lzstd
@@ -150,7 +159,12 @@ all: $(OUTPUT)bpftool
SRCS := $(wildcard *.c)
ifeq ($(feature-llvm),1)
# If LLVM is available, use it for JIT disassembly
ifneq ($(SKIP_LLVM),1)
HAS_LLVM := 1
endif
endif
ifeq ($(HAS_LLVM),1)
CFLAGS += -DHAVE_LLVM_SUPPORT
LLVM_CONFIG_LIB_COMPONENTS := mcdisassembler all-targets
# llvm-config always adds -D_GNU_SOURCE, however, it may already be in CFLAGS
@@ -165,6 +179,7 @@ ifeq ($(feature-llvm),1)
endif
LDFLAGS += $(shell $(LLVM_CONFIG) --ldflags)
else
ifneq ($(SKIP_LIBBFD),1)
# Fall back on libbfd
ifeq ($(feature-libbfd),1)
LIBS += -lbfd -ldl -lopcodes
@@ -186,15 +201,22 @@ else
CFLAGS += -DDISASM_INIT_STYLED
endif
endif
endif # SKIP_LIBBFD
endif
ifeq ($(filter -DHAVE_LLVM_SUPPORT -DHAVE_LIBBFD_SUPPORT,$(CFLAGS)),)
# No support for JIT disassembly
SRCS := $(filter-out jit_disasm.c,$(SRCS))
endif
ifeq ($(SKIP_CRYPTO),1)
CFLAGS += -DBPFTOOL_WITHOUT_CRYPTO
HOST_CFLAGS += -DBPFTOOL_WITHOUT_CRYPTO
SRCS := $(filter-out sign.c,$(SRCS))
endif
BPFTOOL_BOOTSTRAP := $(BOOTSTRAP_OUTPUT)bpftool
BOOTSTRAP_OBJS = $(addprefix $(BOOTSTRAP_OUTPUT),main.o common.o json_writer.o gen.o btf.o sign.o)
BOOTSTRAP_OBJS = $(addprefix $(BOOTSTRAP_OUTPUT),main.o common.o json_writer.o gen.o btf.o $(if $(CRYPTO_LIBS),sign.o))
$(BOOTSTRAP_OBJS): $(LIBBPF_BOOTSTRAP)
OBJS = $(patsubst %.c,$(OUTPUT)%.o,$(SRCS)) $(OUTPUT)disasm.o
tools/bpf/bpftool/bash-completion/bpftool
+6 -2
View File
@@ -501,7 +501,7 @@ _bpftool()
cgroup/post_bind4 cgroup/post_bind6 \
cgroup/sysctl cgroup/getsockopt \
cgroup/setsockopt cgroup/sock_release struct_ops \
fentry fexit freplace sk_lookup'
fentry fexit fsession freplace sk_lookup'
COMPREPLY=( $( compgen -W "$BPFTOOL_PROG_LOAD_TYPES" -- "$cur" ) )
return 0
;;
@@ -961,10 +961,14 @@ _bpftool()
*)
# emit extra options
case ${words[3]} in
id|file)
id)
COMPREPLY=( $( compgen -W "root_id" -- "$cur" ) )
_bpftool_once_attr 'format'
;;
file)
COMPREPLY=( $( compgen -W "root_id file" -- "$cur" ) )
_bpftool_once_attr 'format'
;;
map|prog)
if [[ ${words[3]} == "map" ]] && [[ $cword == 6 ]]; then
COMPREPLY+=( $( compgen -W "key value kv all" -- "$cur" ) )
tools/bpf/bpftool/btf.c
+109 -12
View File
@@ -28,6 +28,7 @@
#define FASTCALL_DECL_TAG "bpf_fastcall"
#define MAX_ROOT_IDS 16
#define MAX_BTF_FILES 64
static const char * const btf_kind_str[NR_BTF_KINDS] = {
[BTF_KIND_UNKN] = "UNKNOWN",
@@ -878,6 +879,45 @@ static bool btf_is_kernel_module(__u32 btf_id)
return btf_info.kernel_btf && strncmp(btf_name, "vmlinux", sizeof(btf_name)) != 0;
}
static struct btf *merge_btf_files(const char **files, int nr_files,
struct btf *vmlinux_base)
{
struct btf *combined, *mod;
int ret;
combined = btf__new_empty_split(vmlinux_base);
if (!combined) {
p_err("failed to create combined BTF: %s", strerror(errno));
return NULL;
}
for (int j = 0; j < nr_files; j++) {
mod = btf__parse_split(files[j], vmlinux_base);
if (!mod) {
p_err("failed to load BTF from %s: %s", files[j], strerror(errno));
btf__free(combined);
return NULL;
}
ret = btf__add_btf(combined, mod);
btf__free(mod);
if (ret < 0) {
p_err("failed to merge BTF from %s: %s", files[j], strerror(-ret));
btf__free(combined);
return NULL;
}
}
ret = btf__dedup(combined, NULL);
if (ret) {
p_err("failed to dedup combined BTF: %s", strerror(-ret));
btf__free(combined);
return NULL;
}
return combined;
}
static int do_dump(int argc, char **argv)
{
bool dump_c = false, sort_dump_c = true;
@@ -958,20 +998,76 @@ static int do_dump(int argc, char **argv)
NEXT_ARG();
} else if (is_prefix(src, "file")) {
const char sysfs_prefix[] = "/sys/kernel/btf/";
struct btf *vmlinux_base = base_btf;
const char *files[MAX_BTF_FILES];
int nr_files = 0;
if (!base_btf &&
strncmp(*argv, sysfs_prefix, sizeof(sysfs_prefix) - 1) == 0 &&
strcmp(*argv, sysfs_vmlinux) != 0)
base = get_vmlinux_btf_from_sysfs();
btf = btf__parse_split(*argv, base ?: base_btf);
if (!btf) {
err = -errno;
p_err("failed to load BTF from %s: %s",
*argv, strerror(errno));
goto done;
/* First grab our argument, filtering out the sysfs_vmlinux. */
if (strcmp(*argv, sysfs_vmlinux) != 0) {
files[nr_files++] = *argv;
} else {
p_info("skipping %s (will be loaded as base)", *argv);
}
NEXT_ARG();
while (argc && is_prefix(*argv, "file")) {
NEXT_ARG();
if (!REQ_ARGS(1)) {
err = -EINVAL;
goto done;
}
/* Filter out any sysfs vmlinux entries. */
if (strcmp(*argv, sysfs_vmlinux) == 0) {
p_info("skipping %s (will be loaded as base)", *argv);
NEXT_ARG();
continue;
}
if (nr_files >= MAX_BTF_FILES) {
p_err("too many BTF files (max %d)", MAX_BTF_FILES);
err = -E2BIG;
goto done;
}
files[nr_files++] = *argv;
NEXT_ARG();
}
/* Auto-detect vmlinux base if any file is from sysfs */
if (!vmlinux_base) {
for (int j = 0; j < nr_files; j++) {
if (strncmp(files[j], sysfs_prefix, sizeof(sysfs_prefix) - 1) == 0) {
base = get_vmlinux_btf_from_sysfs();
vmlinux_base = base;
break;
}
}
}
/* All files were the sysfs_vmlinux, handle it like we used to */
if (nr_files == 0) {
nr_files = 1;
files[0] = sysfs_vmlinux;
}
if (nr_files == 1) {
btf = btf__parse_split(files[0], base ?: base_btf);
if (!btf) {
err = -errno;
p_err("failed to load BTF from %s: %s", files[0], strerror(errno));
goto done;
}
} else {
if (!vmlinux_base) {
p_err("base BTF is required when merging multiple BTF files; use -B/--base-btf or use sysfs paths");
err = -EINVAL;
goto done;
}
btf = merge_btf_files(files, nr_files, vmlinux_base);
if (!btf) {
err = -errno;
goto done;
}
}
} else {
err = -1;
p_err("unrecognized BTF source specifier: '%s'", src);
@@ -1445,7 +1541,8 @@ static int do_help(int argc, char **argv)
" %1$s %2$s dump BTF_SRC [format FORMAT] [root_id ROOT_ID]\n"
" %1$s %2$s help\n"
"\n"
" BTF_SRC := { id BTF_ID | prog PROG | map MAP [{key | value | kv | all}] | file FILE }\n"
" BTF_SRC := { id BTF_ID | prog PROG | map MAP [{key | value | kv | all}] |\n"
" file FILE [file FILE]... }\n"
" FORMAT := { raw | c [unsorted] }\n"
" " HELP_SPEC_MAP "\n"
" " HELP_SPEC_PROGRAM "\n"
tools/bpf/bpftool/jit_disasm.c
+10 -1
View File
@@ -93,7 +93,16 @@ init_context(disasm_ctx_t *ctx, const char *arch,
p_err("Failed to retrieve triple");
return -1;
}
*ctx = LLVMCreateDisasm(triple, NULL, 0, NULL, symbol_lookup_callback);
/*
* Enable all aarch64 ISA extensions so the disassembler can handle any
* instruction the kernel JIT might emit (e.g. ARM64 LSE atomics).
*/
if (!strncmp(triple, "aarch64", 7))
*ctx = LLVMCreateDisasmCPUFeatures(triple, "", "+all", NULL, 0, NULL,
symbol_lookup_callback);
else
*ctx = LLVMCreateDisasm(triple, NULL, 0, NULL, symbol_lookup_callback);
LLVMDisposeMessage(triple);
if (!*ctx) {
tools/bpf/bpftool/main.c
+7
View File
@@ -131,6 +131,11 @@ static int do_version(int argc, char **argv)
const bool has_skeletons = false;
#else
const bool has_skeletons = true;
#endif
#ifdef BPFTOOL_WITHOUT_CRYPTO
const bool has_crypto = false;
#else
const bool has_crypto = true;
#endif
bool bootstrap = false;
int i;
@@ -163,6 +168,7 @@ static int do_version(int argc, char **argv)
jsonw_start_object(json_wtr); /* features */
jsonw_bool_field(json_wtr, "libbfd", has_libbfd);
jsonw_bool_field(json_wtr, "llvm", has_llvm);
jsonw_bool_field(json_wtr, "crypto", has_crypto);
jsonw_bool_field(json_wtr, "skeletons", has_skeletons);
jsonw_bool_field(json_wtr, "bootstrap", bootstrap);
jsonw_end_object(json_wtr); /* features */
@@ -181,6 +187,7 @@ static int do_version(int argc, char **argv)
printf("features:");
print_feature("libbfd", has_libbfd, &nb_features);
print_feature("llvm", has_llvm, &nb_features);
print_feature("crypto", has_crypto, &nb_features);
print_feature("skeletons", has_skeletons, &nb_features);
print_feature("bootstrap", bootstrap, &nb_features);
printf("\n");
tools/bpf/bpftool/main.h
+14
View File
@@ -293,6 +293,20 @@ struct kernel_config_option {
int read_kernel_config(const struct kernel_config_option *requested_options,
size_t num_options, char **out_values,
const char *define_prefix);
#ifndef BPFTOOL_WITHOUT_CRYPTO
int bpftool_prog_sign(struct bpf_load_and_run_opts *opts);
__u32 register_session_key(const char *key_der_path);
#else
static inline int bpftool_prog_sign(struct bpf_load_and_run_opts *opts)
{
p_err("bpftool was built without signing support");
return -ENOTSUP;
}
static inline __u32 register_session_key(const char *key_der_path)
{
p_err("bpftool was built without signing support");
return -1;
}
#endif
#endif
tools/bpf/bpftool/prog.c
+1 -1
View File
@@ -2583,7 +2583,7 @@ static int do_help(int argc, char **argv)
" cgroup/getsockname_unix | cgroup/sendmsg4 | cgroup/sendmsg6 |\n"
" cgroup/sendmsg_unix | cgroup/recvmsg4 | cgroup/recvmsg6 | cgroup/recvmsg_unix |\n"
" cgroup/getsockopt | cgroup/setsockopt | cgroup/sock_release |\n"
" struct_ops | fentry | fexit | freplace | sk_lookup }\n"
" struct_ops | fentry | fexit | fsession | freplace | sk_lookup }\n"
" ATTACH_TYPE := { sk_msg_verdict | sk_skb_verdict | sk_skb_stream_verdict |\n"
" sk_skb_stream_parser | flow_dissector }\n"
" METRIC := { cycles | instructions | l1d_loads | llc_misses | itlb_misses | dtlb_misses }\n"
tools/bpf/resolve_btfids/main.c
+1
View File
@@ -1065,6 +1065,7 @@ static bool is_kf_implicit_arg(const struct btf *btf, const struct btf_param *p)
{
static const char *const kf_implicit_arg_types[] = {
"bpf_prog_aux",
"btf_struct_meta",
};
const struct btf_type *t;
const char *name;
tools/include/uapi/linux/bpf.h
+3 -1
View File
@@ -4645,7 +4645,9 @@ union bpf_attr {
* Description
* Discard reserved ring buffer sample, pointed to by *data*.
* If **BPF_RB_NO_WAKEUP** is specified in *flags*, no notification
* of new data availability is sent.
* of new data availability is sent. Discarded records remain in
* the ring buffer until consumed by user space, so a later submit
* using adaptive wakeup might not wake up the consumer.
* If **BPF_RB_FORCE_WAKEUP** is specified in *flags*, notification
* of new data availability is sent unconditionally.
* If **0** is specified in *flags*, an adaptive notification
tools/include/uapi/linux/btf.h
+12
View File
@@ -8,6 +8,16 @@
#define BTF_MAGIC 0xeB9F
#define BTF_VERSION 1
/*
* BTF layout section consists of a struct btf_layout for each known
* kind at BTF encoding time.
*/
struct btf_layout {
__u8 info_sz; /* size of singular element after btf_type */
__u8 elem_sz; /* size of each of btf_vlen(t) elements */
__u16 flags; /* currently unused */
};
struct btf_header {
__u16 magic;
__u8 version;
@@ -19,6 +29,8 @@ struct btf_header {
__u32 type_len; /* length of type section */
__u32 str_off; /* offset of string section */
__u32 str_len; /* length of string section */
__u32 layout_off; /* offset of layout section */
__u32 layout_len; /* length of layout section */
};
/* Max # of type identifier */
tools/lib/bpf/btf.c
+449 -176
View File
File diff suppressed because it is too large. Load diff
tools/lib/bpf/btf.h
+20
View File
@@ -109,6 +109,26 @@ LIBBPF_API struct btf *btf__new_empty(void);
*/
LIBBPF_API struct btf *btf__new_empty_split(struct btf *base_btf);
struct btf_new_opts {
size_t sz;
struct btf *base_btf; /* optional base BTF */
bool add_layout; /* add BTF layout information */
size_t:0;
};
#define btf_new_opts__last_field add_layout
/**
* @brief **btf__new_empty_opts()** creates an unpopulated BTF object with
* optional *base_btf* and BTF kind layout description if *add_layout*
* is set
* @return new BTF object instance which has to be eventually freed with
* **btf__free()**
*
* On error, NULL is returned and the thread-local `errno` variable is
* set to the error code.
*/
LIBBPF_API struct btf *btf__new_empty_opts(struct btf_new_opts *opts);
/**
* @brief **btf__distill_base()** creates new versions of the split BTF
* *src_btf* and its base BTF. The new base BTF will only contain the types
tools/lib/bpf/features.c
+53
View File
@@ -568,6 +568,53 @@ static int probe_ldimm64_full_range_off(int token_fd)
return 1;
}
#ifdef __x86_64__
#ifndef __NR_uprobe
#define __NR_uprobe 336
#endif
static int probe_uprobe_syscall(int token_fd)
{
/*
* If kernel supports uprobe() syscall, it will return -ENXIO when called
* from the outside of a kernel-generated uprobe trampoline.
*/
return syscall(__NR_uprobe) < 0 && errno == ENXIO;
}
#else
static int probe_uprobe_syscall(int token_fd)
{
return 0;
}
#endif
static int probe_kern_btf_layout(int token_fd)
{
static const char strs[] = "\0int";
__u32 types[] = {
/* int */
BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
};
struct btf_layout layout[] = {
{ 0, 0, 0 },
{ sizeof(__u32), 0, 0 },
};
struct btf_header hdr = {
.magic = BTF_MAGIC,
.version = BTF_VERSION,
.hdr_len = sizeof(struct btf_header),
.type_len = sizeof(types),
.str_off = sizeof(types) + sizeof(layout),
.str_len = sizeof(strs),
.layout_off = sizeof(types),
.layout_len = sizeof(layout),
};
return probe_fd(libbpf__load_raw_btf_hdr(&hdr, (char *)types, strs,
(char *)layout, token_fd));
}
typedef int (*feature_probe_fn)(int /* token_fd */);
static struct kern_feature_cache feature_cache;
@@ -646,6 +693,12 @@ static struct kern_feature_desc {
[FEAT_LDIMM64_FULL_RANGE_OFF] = {
"full range LDIMM64 support", probe_ldimm64_full_range_off,
},
[FEAT_UPROBE_SYSCALL] = {
"kernel supports uprobe syscall", probe_uprobe_syscall,
},
[FEAT_BTF_LAYOUT] = {
"kernel supports BTF layout", probe_kern_btf_layout,
},
};
bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id)
tools/lib/bpf/libbpf.c
+209 -31
View File
@@ -3138,12 +3138,14 @@ static bool btf_needs_sanitization(struct bpf_object *obj)
bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC);
bool has_layout = kernel_supports(obj, FEAT_BTF_LAYOUT);
return !has_func || !has_datasec || !has_func_global || !has_float ||
!has_decl_tag || !has_type_tag || !has_enum64 || !has_qmark_datasec;
!has_decl_tag || !has_type_tag || !has_enum64 || !has_qmark_datasec ||
!has_layout;
}
static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
struct btf *bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *orig_btf)
{
bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
@@ -3153,9 +3155,64 @@ static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC);
bool has_layout = kernel_supports(obj, FEAT_BTF_LAYOUT);
int enum64_placeholder_id = 0;
const struct btf_header *hdr;
struct btf *btf = NULL;
const void *raw_data;
struct btf_type *t;
int i, j, vlen;
__u32 sz;
int err;
/* clone BTF to sanitize a copy and leave the original intact */
raw_data = btf__raw_data(orig_btf, &sz);
if (!raw_data)
return ERR_PTR(-ENOMEM);
/* btf_header() gives us endian-safe header info */
hdr = btf_header(orig_btf);
if (!has_layout && hdr->hdr_len >= sizeof(struct btf_header) &&
(hdr->layout_len != 0 || hdr->layout_off != 0)) {
const struct btf_header *old_hdr = raw_data;
struct btf_header *new_hdr;
void *new_raw_data;
__u32 new_str_off;
/*
* Need to rewrite BTF to exclude layout information and
* move string section to immediately after types.
*/
new_raw_data = malloc(sz);
if (!new_raw_data)
return ERR_PTR(-ENOMEM);
memcpy(new_raw_data, raw_data, sz);
new_hdr = new_raw_data;
new_hdr->layout_off = 0;
new_hdr->layout_len = 0;
new_str_off = hdr->type_off + hdr->type_len;
/* Handle swapped endian case */
if (old_hdr->magic != hdr->magic)
new_hdr->str_off = bswap_32(new_str_off);
else
new_hdr->str_off = new_str_off;
memmove(new_raw_data + hdr->hdr_len + new_str_off,
new_raw_data + hdr->hdr_len + hdr->str_off,
hdr->str_len);
sz = hdr->hdr_len + hdr->type_off + hdr->type_len + hdr->str_len;
btf = btf__new(new_raw_data, sz);
free(new_raw_data);
} else {
btf = btf__new(raw_data, sz);
}
err = libbpf_get_error(btf);
if (err)
return ERR_PTR(err);
/* enforce 8-byte pointers for BPF-targeted BTFs */
btf__set_pointer_size(btf, 8);
for (i = 1; i < btf__type_cnt(btf); i++) {
t = (struct btf_type *)btf__type_by_id(btf, i);
@@ -3233,9 +3290,10 @@ static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
if (enum64_placeholder_id == 0) {
enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0);
if (enum64_placeholder_id < 0)
return enum64_placeholder_id;
if (enum64_placeholder_id < 0) {
btf__free(btf);
return ERR_PTR(enum64_placeholder_id);
}
t = (struct btf_type *)btf__type_by_id(btf, i);
}
@@ -3249,7 +3307,7 @@ static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
}
}
return 0;
return btf;
}
static bool libbpf_needs_btf(const struct bpf_object *obj)
@@ -3600,21 +3658,9 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
sanitize = btf_needs_sanitization(obj);
if (sanitize) {
const void *raw_data;
__u32 sz;
/* clone BTF to sanitize a copy and leave the original intact */
raw_data = btf__raw_data(obj->btf, &sz);
kern_btf = btf__new(raw_data, sz);
err = libbpf_get_error(kern_btf);
if (err)
return err;
/* enforce 8-byte pointers for BPF-targeted BTFs */
btf__set_pointer_size(obj->btf, 8);
err = bpf_object__sanitize_btf(obj, kern_btf);
if (err)
return err;
kern_btf = bpf_object__sanitize_btf(obj, obj->btf);
if (IS_ERR(kern_btf))
return PTR_ERR(kern_btf);
}
if (obj->gen_loader) {
@@ -5157,12 +5203,20 @@ bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
*/
return true;
if (obj->token_fd)
if (obj->feat_cache)
return feat_supported(obj->feat_cache, feat_id);
return feat_supported(NULL, feat_id);
}
/* Used in testing to simulate missing features. */
void bpf_object_set_feat_cache(struct bpf_object *obj, struct kern_feature_cache *cache)
{
if (obj->feat_cache)
free(obj->feat_cache);
obj->feat_cache = cache;
}
static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
{
struct bpf_map_info map_info;
@@ -9802,6 +9856,111 @@ __u32 bpf_program__line_info_cnt(const struct bpf_program *prog)
return prog->line_info_cnt;
}
int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts *opts)
{
LIBBPF_OPTS(bpf_prog_load_opts, attr);
struct bpf_object *obj;
const void *info;
__u32 info_cnt, info_rec_size;
int err, fd, prog_btf_fd;
if (!prog)
return libbpf_err(-EINVAL);
if (!OPTS_VALID(opts, bpf_prog_load_opts))
return libbpf_err(-EINVAL);
obj = prog->obj;
if (obj->state < OBJ_PREPARED)
return libbpf_err(-EINVAL);
/*
* Caller-provided opts take priority; fall back to
* prog/object defaults when the caller leaves them zero.
*/
attr.attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0) ?: prog->attach_prog_fd;
attr.prog_flags = OPTS_GET(opts, prog_flags, 0) ?: prog->prog_flags;
attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0) ?: prog->prog_ifindex;
attr.kern_version = OPTS_GET(opts, kern_version, 0) ?: obj->kern_version;
attr.fd_array = OPTS_GET(opts, fd_array, NULL) ?: obj->fd_array;
attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0) ?: obj->fd_array_cnt;
attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd;
if (attr.token_fd)
attr.prog_flags |= BPF_F_TOKEN_FD;
prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0);
if (!prog_btf_fd && obj->btf)
prog_btf_fd = btf__fd(obj->btf);
/* BTF func/line info: only pass if kernel supports it */
if (kernel_supports(obj, FEAT_BTF_FUNC) && prog_btf_fd > 0) {
attr.prog_btf_fd = prog_btf_fd;
/* func_info/line_info triples: all-or-nothing from caller */
info = OPTS_GET(opts, func_info, NULL);
info_cnt = OPTS_GET(opts, func_info_cnt, 0);
info_rec_size = OPTS_GET(opts, func_info_rec_size, 0);
if (!!info != !!info_cnt || !!info != !!info_rec_size) {
pr_warn("prog '%s': func_info, func_info_cnt, and func_info_rec_size must all be specified or all omitted\n",
prog->name);
return libbpf_err(-EINVAL);
}
attr.func_info = info ?: prog->func_info;
attr.func_info_cnt = info ? info_cnt : prog->func_info_cnt;
attr.func_info_rec_size = info ? info_rec_size : prog->func_info_rec_size;
info = OPTS_GET(opts, line_info, NULL);
info_cnt = OPTS_GET(opts, line_info_cnt, 0);
info_rec_size = OPTS_GET(opts, line_info_rec_size, 0);
if (!!info != !!info_cnt || !!info != !!info_rec_size) {
pr_warn("prog '%s': line_info, line_info_cnt, and line_info_rec_size must all be specified or all omitted\n",
prog->name);
return libbpf_err(-EINVAL);
}
attr.line_info = info ?: prog->line_info;
attr.line_info_cnt = info ? info_cnt : prog->line_info_cnt;
attr.line_info_rec_size = info ? info_rec_size : prog->line_info_rec_size;
}
/* Logging is caller-controlled; no fallback to prog/obj log settings */
attr.log_buf = OPTS_GET(opts, log_buf, NULL);
attr.log_size = OPTS_GET(opts, log_size, 0);
attr.log_level = OPTS_GET(opts, log_level, 0);
/*
* Fields below may be mutated by prog_prepare_load_fn:
* Seed them from prog/obj defaults here;
* Later override with caller-provided opts.
*/
attr.expected_attach_type = prog->expected_attach_type;
attr.attach_btf_id = prog->attach_btf_id;
attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
err = prog->sec_def->prog_prepare_load_fn(prog, &attr, prog->sec_def->cookie);
if (err)
return libbpf_err(err);
}
/* Re-apply caller overrides for output fields */
if (OPTS_GET(opts, expected_attach_type, 0))
attr.expected_attach_type = OPTS_GET(opts, expected_attach_type, 0);
if (OPTS_GET(opts, attach_btf_id, 0))
attr.attach_btf_id = OPTS_GET(opts, attach_btf_id, 0);
if (OPTS_GET(opts, attach_btf_obj_fd, 0))
attr.attach_btf_obj_fd = OPTS_GET(opts, attach_btf_obj_fd, 0);
/*
* Unlike bpf_object_load_prog(), we intentionally do not call bpf_prog_bind_map()
* for RODATA maps here to avoid mutating the object's state. Callers can bind the
* required maps themselves using bpf_prog_bind_map().
*/
fd = bpf_prog_load(prog->type, prog->name, obj->license, prog->insns, prog->insns_cnt,
&attr);
return libbpf_err(fd);
}
#define SEC_DEF(sec_pfx, ptype, atype, flags, ...) { \
.sec = (char *)sec_pfx, \
.prog_type = BPF_PROG_TYPE_##ptype, \
@@ -11692,6 +11851,8 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
default:
return libbpf_err_ptr(-EINVAL);
}
if (!func_name && legacy)
return libbpf_err_ptr(-EOPNOTSUPP);
if (!legacy) {
pfd = perf_event_open_probe(false /* uprobe */, retprobe,
@@ -11711,21 +11872,21 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
offset, -1 /* pid */);
}
if (pfd < 0) {
err = -errno;
pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n",
err = pfd;
pr_warn("prog '%s': failed to create %s '%s%s0x%zx' perf event: %s\n",
prog->name, retprobe ? "kretprobe" : "kprobe",
func_name, offset,
errstr(err));
func_name ?: "", func_name ? "+" : "",
offset, errstr(err));
goto err_out;
}
link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
err = libbpf_get_error(link);
if (err) {
close(pfd);
pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n",
pr_warn("prog '%s': failed to attach to %s '%s%s0x%zx': %s\n",
prog->name, retprobe ? "kretprobe" : "kprobe",
func_name, offset,
errstr(err));
func_name ?: "", func_name ? "+" : "",
offset, errstr(err));
goto err_clean_legacy;
}
if (legacy) {
@@ -12041,7 +12202,16 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
if (addrs && syms)
return libbpf_err_ptr(-EINVAL);
if (pattern) {
/*
* Exact function name (no wildcards) without unique_match:
* bypass kallsyms parsing and pass the symbol directly to the
* kernel via syms[] array. When unique_match is set, fall
* through to the slow path which detects duplicate symbols.
*/
if (pattern && !strpbrk(pattern, "*?") && !unique_match) {
syms = &pattern;
cnt = 1;
} else if (pattern) {
if (has_available_filter_functions_addrs())
err = libbpf_available_kprobes_parse(&res);
else
@@ -12084,6 +12254,14 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts);
if (link_fd < 0) {
err = -errno;
/*
* Normalize error code: when exact name bypasses kallsyms
* parsing, kernel returns ESRCH from ftrace_lookup_symbols().
* Convert to ENOENT for API consistency with the pattern
* matching path which returns ENOENT from userspace.
*/
if (err == -ESRCH)
err = -ENOENT;
pr_warn("prog '%s': failed to attach: %s\n",
prog->name, errstr(err));
goto error;
@@ -12684,7 +12862,7 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
binary_path, func_offset, pid);
}
if (pfd < 0) {
err = -errno;
err = pfd;
pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
prog->name, retprobe ? "uretprobe" : "uprobe",
binary_path, func_offset,
tools/lib/bpf/libbpf.h
+43 -1
View File
@@ -557,7 +557,7 @@ struct bpf_kprobe_opts {
size_t sz;
/* custom user-provided value fetchable through bpf_get_attach_cookie() */
__u64 bpf_cookie;
/* function's offset to install kprobe to */
/* function offset, or raw address if func_name == NULL */
size_t offset;
/* kprobe is return probe */
bool retprobe;
@@ -565,11 +565,36 @@ struct bpf_kprobe_opts {
enum probe_attach_mode attach_mode;
size_t :0;
};
#define bpf_kprobe_opts__last_field attach_mode
/**
* @brief **bpf_program__attach_kprobe()** attaches a BPF program to a
* kernel function entry or return.
*
* @param prog BPF program to attach
* @param retprobe Attach to function return
* @param func_name Name of the kernel function to attach to
* @return Reference to the newly created BPF link; or NULL is returned on
* error, error code is stored in errno
*/
LIBBPF_API struct bpf_link *
bpf_program__attach_kprobe(const struct bpf_program *prog, bool retprobe,
const char *func_name);
/**
* @brief **bpf_program__attach_kprobe_opts()** is just like
* bpf_program__attach_kprobe() except with an options struct
* for various configurations.
*
* @param prog BPF program to attach
* @param func_name Name of the kernel function to attach to. If NULL,
* opts->offset is treated as a raw kernel address. Raw-address attach
* is supported with PROBE_ATTACH_MODE_PERF and PROBE_ATTACH_MODE_LINK.
* @param opts Options for altering program attachment
* @return Reference to the newly created BPF link; or NULL is returned on
* error, error code is stored in errno
*/
LIBBPF_API struct bpf_link *
bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
const char *func_name,
@@ -2021,6 +2046,23 @@ LIBBPF_API int libbpf_register_prog_handler(const char *sec,
*/
LIBBPF_API int libbpf_unregister_prog_handler(int handler_id);
/**
* @brief **bpf_program__clone()** loads a single BPF program from a prepared
* BPF object into the kernel, returning its file descriptor.
*
* The BPF object must have been previously prepared with
* **bpf_object__prepare()**. If @opts is provided, any non-zero field
* overrides the defaults derived from the program/object internals.
* If @opts is NULL, all fields are populated automatically.
*
* The returned FD is owned by the caller and must be closed with close().
*
* @param prog BPF program from a prepared object
* @param opts Optional load options; non-zero fields override defaults
* @return program FD (>= 0) on success; negative error code on failure
*/
LIBBPF_API int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts *opts);
#ifdef __cplusplus
} /* extern "C" */
#endif
tools/lib/bpf/libbpf.map
+6
View File
@@ -455,3 +455,9 @@ LIBBPF_1.7.0 {
bpf_program__assoc_struct_ops;
btf__permute;
} LIBBPF_1.6.0;
LIBBPF_1.8.0 {
global:
bpf_program__clone;
btf__new_empty_opts;
} LIBBPF_1.7.0;
tools/lib/bpf/libbpf_internal.h
+9
View File
@@ -394,6 +394,10 @@ enum kern_feature_id {
FEAT_BTF_QMARK_DATASEC,
/* Kernel supports LDIMM64 imm offsets past 512 MiB. */
FEAT_LDIMM64_FULL_RANGE_OFF,
/* Kernel supports uprobe syscall */
FEAT_UPROBE_SYSCALL,
/* Kernel supports BTF layout information */
FEAT_BTF_LAYOUT,
__FEAT_CNT,
};
@@ -410,6 +414,7 @@ struct kern_feature_cache {
bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id);
bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id);
void bpf_object_set_feat_cache(struct bpf_object *obj, struct kern_feature_cache *cache);
int probe_kern_syscall_wrapper(int token_fd);
int probe_memcg_account(int token_fd);
@@ -420,6 +425,10 @@ int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz);
int libbpf__load_raw_btf(const char *raw_types, size_t types_len,
const char *str_sec, size_t str_len,
int token_fd);
int libbpf__load_raw_btf_hdr(const struct btf_header *hdr,
const char *raw_types, const char *str_sec,
const char *layout_sec, int token_fd);
struct btf *bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *orig_btf);
int btf_load_into_kernel(struct btf *btf,
char *log_buf, size_t log_sz, __u32 log_level,
int token_fd);
tools/lib/bpf/libbpf_probes.c
+29 -19
View File
@@ -218,6 +218,34 @@ int libbpf_probe_bpf_prog_type(enum bpf_prog_type prog_type, const void *opts)
return libbpf_err(ret);
}
int libbpf__load_raw_btf_hdr(const struct btf_header *hdr, const char *raw_types,
const char *str_sec, const char *layout_sec,
int token_fd)
{
LIBBPF_OPTS(bpf_btf_load_opts, opts,
.token_fd = token_fd,
.btf_flags = token_fd ? BPF_F_TOKEN_FD : 0,
);
int btf_fd, btf_len;
__u8 *raw_btf;
btf_len = hdr->hdr_len + hdr->type_off + hdr->type_len + hdr->str_len + hdr->layout_len;
raw_btf = malloc(btf_len);
if (!raw_btf)
return -ENOMEM;
memcpy(raw_btf, hdr, sizeof(*hdr));
memcpy(raw_btf + hdr->hdr_len + hdr->type_off, raw_types, hdr->type_len);
memcpy(raw_btf + hdr->hdr_len + hdr->str_off, str_sec, hdr->str_len);
if (layout_sec)
memcpy(raw_btf + hdr->hdr_len + hdr->layout_off, layout_sec, hdr->layout_len);
btf_fd = bpf_btf_load(raw_btf, btf_len, &opts);
free(raw_btf);
return btf_fd;
}
int libbpf__load_raw_btf(const char *raw_types, size_t types_len,
const char *str_sec, size_t str_len,
int token_fd)
@@ -230,26 +258,8 @@ int libbpf__load_raw_btf(const char *raw_types, size_t types_len,
.str_off = types_len,
.str_len = str_len,
};
LIBBPF_OPTS(bpf_btf_load_opts, opts,
.token_fd = token_fd,
.btf_flags = token_fd ? BPF_F_TOKEN_FD : 0,
);
int btf_fd, btf_len;
__u8 *raw_btf;
btf_len = hdr.hdr_len + hdr.type_len + hdr.str_len;
raw_btf = malloc(btf_len);
if (!raw_btf)
return -ENOMEM;
memcpy(raw_btf, &hdr, sizeof(hdr));
memcpy(raw_btf + hdr.hdr_len, raw_types, hdr.type_len);
memcpy(raw_btf + hdr.hdr_len + hdr.type_len, str_sec, hdr.str_len);
btf_fd = bpf_btf_load(raw_btf, btf_len, &opts);
free(raw_btf);
return btf_fd;
return libbpf__load_raw_btf_hdr(&hdr, raw_types, str_sec, NULL, token_fd);
}
static int load_local_storage_btf(void)
tools/lib/bpf/libbpf_version.h
+1 -1
View File
@@ -4,6 +4,6 @@
#define __LIBBPF_VERSION_H
#define LIBBPF_MAJOR_VERSION 1
#define LIBBPF_MINOR_VERSION 7
#define LIBBPF_MINOR_VERSION 8
#endif /* __LIBBPF_VERSION_H */
tools/lib/bpf/relo_core.c
+2
View File
@@ -292,6 +292,8 @@ int bpf_core_parse_spec(const char *prog_name, const struct btf *btf,
++spec_str;
if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1)
return -EINVAL;
if (access_idx < 0)
return -EINVAL;
if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN)
return -E2BIG;
spec_str += parsed_len;
tools/lib/bpf/usdt.c
+43 -4
View File
@@ -262,6 +262,7 @@ struct usdt_manager {
bool has_bpf_cookie;
bool has_sema_refcnt;
bool has_uprobe_multi;
bool has_uprobe_syscall;
};
struct usdt_manager *usdt_manager_new(struct bpf_object *obj)
@@ -301,6 +302,13 @@ struct usdt_manager *usdt_manager_new(struct bpf_object *obj)
* usdt probes.
*/
man->has_uprobe_multi = kernel_supports(obj, FEAT_UPROBE_MULTI_LINK);
/*
* Detect kernel support for uprobe() syscall, it's presence means we can
* take advantage of faster nop5 uprobe handling.
* Added in: 56101b69c919 ("uprobes/x86: Add uprobe syscall to speed up uprobe")
*/
man->has_uprobe_syscall = kernel_supports(obj, FEAT_UPROBE_SYSCALL);
return man;
}
@@ -585,13 +593,34 @@ static int parse_usdt_note(GElf_Nhdr *nhdr, const char *data, size_t name_off,
static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie);
static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *path, pid_t pid,
const char *usdt_provider, const char *usdt_name, __u64 usdt_cookie,
struct usdt_target **out_targets, size_t *out_target_cnt)
#if defined(__x86_64__)
static bool has_nop_combo(int fd, long off)
{
unsigned char nop_combo[6] = {
0x90, 0x0f, 0x1f, 0x44, 0x00, 0x00 /* nop,nop5 */
};
unsigned char buf[6];
if (pread(fd, buf, 6, off) != 6)
return false;
return memcmp(buf, nop_combo, 6) == 0;
}
#else
static bool has_nop_combo(int fd, long off)
{
return false;
}
#endif
static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, const char *path,
pid_t pid, const char *usdt_provider, const char *usdt_name,
__u64 usdt_cookie, struct usdt_target **out_targets,
size_t *out_target_cnt)
{
size_t off, name_off, desc_off, seg_cnt = 0, vma_seg_cnt = 0, target_cnt = 0;
struct elf_seg *segs = NULL, *vma_segs = NULL;
struct usdt_target *targets = NULL, *target;
Elf *elf = elf_fd->elf;
long base_addr = 0;
Elf_Scn *notes_scn, *base_scn;
GElf_Shdr base_shdr, notes_shdr;
@@ -784,6 +813,16 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *
target = &targets[target_cnt];
memset(target, 0, sizeof(*target));
/*
* We have uprobe syscall and usdt with nop,nop5 instructions combo,
* so we can place the uprobe directly on nop5 (+1) and get this probe
* optimized.
*/
if (man->has_uprobe_syscall && has_nop_combo(elf_fd->fd, usdt_rel_ip)) {
usdt_abs_ip++;
usdt_rel_ip++;
}
target->abs_ip = usdt_abs_ip;
target->rel_ip = usdt_rel_ip;
target->sema_off = usdt_sema_off;
@@ -998,7 +1037,7 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct
/* discover USDT in given binary, optionally limiting
* activations to a given PID, if pid > 0
*/
err = collect_usdt_targets(man, elf_fd.elf, path, pid, usdt_provider, usdt_name,
err = collect_usdt_targets(man, &elf_fd, path, pid, usdt_provider, usdt_name,
usdt_cookie, &targets, &target_cnt);
if (err <= 0) {
err = (err == 0) ? -ENOENT : err;
tools/testing/selftests/bpf/.gitignore
+2
View File
@@ -47,3 +47,5 @@ verification_cert.h
*.BTF
*.BTF_ids
*.BTF.base
usdt_1
usdt_2
tools/testing/selftests/bpf/Makefile
+13 -4
View File
@@ -41,6 +41,8 @@ LIBELF_LIBS := $(shell $(PKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf)
SKIP_DOCS ?=
SKIP_LLVM ?=
SKIP_LIBBFD ?=
SKIP_CRYPTO ?=
ifeq ($(srctree),)
srctree := $(patsubst %/,%,$(dir $(CURDIR)))
@@ -111,14 +113,12 @@ TEST_PROGS := test_kmod.sh \
test_lirc_mode2.sh \
test_xdping.sh \
test_bpftool_build.sh \
test_bpftool.sh \
test_doc_build.sh \
test_xsk.sh \
test_xdp_features.sh
TEST_PROGS_EXTENDED := \
ima_setup.sh verify_sig_setup.sh \
test_bpftool.py
ima_setup.sh verify_sig_setup.sh
TEST_KMODS := bpf_testmod.ko bpf_test_no_cfi.ko bpf_test_modorder_x.ko \
bpf_test_modorder_y.ko bpf_test_rqspinlock.ko
@@ -336,6 +336,9 @@ $(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \
OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \
LIBBPF_OUTPUT=$(HOST_BUILD_DIR)/libbpf/ \
LIBBPF_DESTDIR=$(HOST_SCRATCH_DIR)/ \
SKIP_LLVM=$(SKIP_LLVM) \
SKIP_LIBBFD=$(SKIP_LIBBFD) \
SKIP_CRYPTO=$(SKIP_CRYPTO) \
prefix= DESTDIR=$(HOST_SCRATCH_DIR)/ install-bin
ifneq ($(CROSS_COMPILE),)
@@ -348,6 +351,9 @@ $(CROSS_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \
OUTPUT=$(BUILD_DIR)/bpftool/ \
LIBBPF_OUTPUT=$(BUILD_DIR)/libbpf/ \
LIBBPF_DESTDIR=$(SCRATCH_DIR)/ \
SKIP_LLVM=$(SKIP_LLVM) \
SKIP_LIBBFD=$(SKIP_LIBBFD) \
SKIP_CRYPTO=$(SKIP_CRYPTO) \
prefix= DESTDIR=$(SCRATCH_DIR)/ install-bin
endif
@@ -756,7 +762,8 @@ TRUNNER_EXTRA_SOURCES := test_progs.c \
$(VERIFY_SIG_HDR) \
flow_dissector_load.h \
ip_check_defrag_frags.h \
bpftool_helpers.c
bpftool_helpers.c \
usdt_1.c usdt_2.c
TRUNNER_LIB_SOURCES := find_bit.c
TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \
$(OUTPUT)/liburandom_read.so \
@@ -880,6 +887,8 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \
$(OUTPUT)/bench_bpf_crypto.o \
$(OUTPUT)/bench_sockmap.o \
$(OUTPUT)/bench_lpm_trie_map.o \
$(OUTPUT)/usdt_1.o \
$(OUTPUT)/usdt_2.o \
#
$(call msg,BINARY,,$@)
$(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@
tools/testing/selftests/bpf/bench.c
+4
View File
@@ -541,6 +541,8 @@ extern const struct bench bench_trig_uprobe_nop5;
extern const struct bench bench_trig_uretprobe_nop5;
extern const struct bench bench_trig_uprobe_multi_nop5;
extern const struct bench bench_trig_uretprobe_multi_nop5;
extern const struct bench bench_trig_usdt_nop;
extern const struct bench bench_trig_usdt_nop5;
#endif
extern const struct bench bench_rb_libbpf;
@@ -617,6 +619,8 @@ static const struct bench *benchs[] = {
&bench_trig_uretprobe_nop5,
&bench_trig_uprobe_multi_nop5,
&bench_trig_uretprobe_multi_nop5,
&bench_trig_usdt_nop,
&bench_trig_usdt_nop5,
#endif
/* ringbuf/perfbuf benchmarks */
&bench_rb_libbpf,
tools/testing/selftests/bpf/benchs/bench_local_storage_create.c
+6 -15
View File
@@ -101,11 +101,6 @@ static void setup(void)
}
}
if (!bpf_program__attach(skel->progs.kmalloc)) {
fprintf(stderr, "Error attaching bpf program\n");
exit(1);
}
threads = calloc(env.producer_cnt, sizeof(*threads));
if (!threads) {
@@ -140,7 +135,6 @@ static void setup(void)
static void measure(struct bench_res *res)
{
res->hits = atomic_swap(&skel->bss->create_cnts, 0);
res->drops = atomic_swap(&skel->bss->kmalloc_cnts, 0);
}
static void *sk_producer(void *input)
@@ -203,28 +197,25 @@ static void *producer(void *input)
static void report_progress(int iter, struct bench_res *res, long delta_ns)
{
double creates_per_sec, kmallocs_per_create;
double creates_per_sec;
creates_per_sec = res->hits / 1000.0 / (delta_ns / 1000000000.0);
kmallocs_per_create = (double)res->drops / res->hits;
printf("Iter %3d (%7.3lfus): ",
iter, (delta_ns - 1000000000) / 1000.0);
printf("creates %8.3lfk/s (%7.3lfk/prod), ",
printf("creates %8.3lfk/s (%7.3lfk/prod)\n",
creates_per_sec, creates_per_sec / env.producer_cnt);
printf("%3.2lf kmallocs/create\n", kmallocs_per_create);
}
static void report_final(struct bench_res res[], int res_cnt)
{
double creates_mean = 0.0, creates_stddev = 0.0;
long total_creates = 0, total_kmallocs = 0;
long total_creates = 0;
int i;
for (i = 0; i < res_cnt; i++) {
creates_mean += res[i].hits / 1000.0 / (0.0 + res_cnt);
total_creates += res[i].hits;
total_kmallocs += res[i].drops;
}
if (res_cnt > 1) {
@@ -234,9 +225,9 @@ static void report_final(struct bench_res res[], int res_cnt)
(res_cnt - 1.0);
creates_stddev = sqrt(creates_stddev);
}
printf("Summary: creates %8.3lf \u00B1 %5.3lfk/s (%7.3lfk/prod), ",
creates_mean, creates_stddev, creates_mean / env.producer_cnt);
printf("%4.2lf kmallocs/create\n", (double)total_kmallocs / total_creates);
printf("Summary: creates %8.3lf \u00B1 %5.3lfk/s (%7.3lfk/prod), %ld total\n",
creates_mean, creates_stddev, creates_mean / env.producer_cnt,
total_creates);
if (create_owner_errs || skel->bss->create_errs)
printf("%s() errors %ld create_errs %ld\n",
storage_type == BPF_MAP_TYPE_SK_STORAGE ?
tools/testing/selftests/bpf/benchs/bench_trigger.c
+60
View File
@@ -407,6 +407,23 @@ static void *uprobe_producer_nop5(void *input)
uprobe_target_nop5();
return NULL;
}
void usdt_1(void);
void usdt_2(void);
static void *uprobe_producer_usdt_nop(void *input)
{
while (true)
usdt_1();
return NULL;
}
static void *uprobe_producer_usdt_nop5(void *input)
{
while (true)
usdt_2();
return NULL;
}
#endif
static void usetup(bool use_retprobe, bool use_multi, void *target_addr)
@@ -544,6 +561,47 @@ static void uretprobe_multi_nop5_setup(void)
{
usetup(true, true /* use_multi */, &uprobe_target_nop5);
}
static void usdt_setup(const char *name)
{
struct bpf_link *link;
int err;
setup_libbpf();
ctx.skel = trigger_bench__open();
if (!ctx.skel) {
fprintf(stderr, "failed to open skeleton\n");
exit(1);
}
bpf_program__set_autoload(ctx.skel->progs.bench_trigger_usdt, true);
err = trigger_bench__load(ctx.skel);
if (err) {
fprintf(stderr, "failed to load skeleton\n");
exit(1);
}
link = bpf_program__attach_usdt(ctx.skel->progs.bench_trigger_usdt,
0 /*self*/, "/proc/self/exe",
"optimized_attach", name, NULL);
if (libbpf_get_error(link)) {
fprintf(stderr, "failed to attach optimized_attach:%s usdt probe\n", name);
exit(1);
}
ctx.skel->links.bench_trigger_usdt = link;
}
static void usdt_nop_setup(void)
{
usdt_setup("usdt_1");
}
static void usdt_nop5_setup(void)
{
usdt_setup("usdt_2");
}
#endif
const struct bench bench_trig_syscall_count = {
@@ -611,4 +669,6 @@ BENCH_TRIG_USERMODE(uprobe_nop5, nop5, "uprobe-nop5");
BENCH_TRIG_USERMODE(uretprobe_nop5, nop5, "uretprobe-nop5");
BENCH_TRIG_USERMODE(uprobe_multi_nop5, nop5, "uprobe-multi-nop5");
BENCH_TRIG_USERMODE(uretprobe_multi_nop5, nop5, "uretprobe-multi-nop5");
BENCH_TRIG_USERMODE(usdt_nop, usdt_nop, "usdt-nop");
BENCH_TRIG_USERMODE(usdt_nop5, usdt_nop5, "usdt-nop5");
#endif
tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh
+1 -1
View File
@@ -2,7 +2,7 @@
set -eufo pipefail
for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop5}
for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop5} usdt-nop usdt-nop5
do
summary=$(sudo ./bench -w2 -d5 -a trig-$i | tail -n1 | cut -d'(' -f1 | cut -d' ' -f3-)
printf "%-15s: %s\n" $i "$summary"
tools/testing/selftests/bpf/bpf_experimental.h
+22 -152
View File
@@ -8,156 +8,11 @@
#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node)))
/* Description
* Allocates an object of the type represented by 'local_type_id' in
* program BTF. User may use the bpf_core_type_id_local macro to pass the
* type ID of a struct in program BTF.
*
* The 'local_type_id' parameter must be a known constant.
* The 'meta' parameter is rewritten by the verifier, no need for BPF
* program to set it.
* Returns
* A pointer to an object of the type corresponding to the passed in
* 'local_type_id', or NULL on failure.
*/
extern void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym;
/* Convenience macro to wrap over bpf_obj_new */
#define bpf_obj_new(type) ((type *)bpf_obj_new(bpf_core_type_id_local(type)))
/* Convenience macro to wrap over bpf_obj_new_impl */
#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL))
/* Description
* Free an allocated object. All fields of the object that require
* destruction will be destructed before the storage is freed.
*
* The 'meta' parameter is rewritten by the verifier, no need for BPF
* program to set it.
* Returns
* Void.
*/
extern void bpf_obj_drop_impl(void *kptr, void *meta) __ksym;
/* Convenience macro to wrap over bpf_obj_drop_impl */
#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL)
/* Description
* Increment the refcount on a refcounted local kptr, turning the
* non-owning reference input into an owning reference in the process.
*
* The 'meta' parameter is rewritten by the verifier, no need for BPF
* program to set it.
* Returns
* An owning reference to the object pointed to by 'kptr'
*/
extern void *bpf_refcount_acquire_impl(void *kptr, void *meta) __ksym;
/* Convenience macro to wrap over bpf_refcount_acquire_impl */
#define bpf_refcount_acquire(kptr) bpf_refcount_acquire_impl(kptr, NULL)
/* Description
* Add a new entry to the beginning of the BPF linked list.
*
* The 'meta' and 'off' parameters are rewritten by the verifier, no need
* for BPF programs to set them
* Returns
* 0 if the node was successfully added
* -EINVAL if the node wasn't added because it's already in a list
*/
extern int bpf_list_push_front_impl(struct bpf_list_head *head,
struct bpf_list_node *node,
void *meta, __u64 off) __ksym;
/* Convenience macro to wrap over bpf_list_push_front_impl */
#define bpf_list_push_front(head, node) bpf_list_push_front_impl(head, node, NULL, 0)
/* Description
* Add a new entry to the end of the BPF linked list.
*
* The 'meta' and 'off' parameters are rewritten by the verifier, no need
* for BPF programs to set them
* Returns
* 0 if the node was successfully added
* -EINVAL if the node wasn't added because it's already in a list
*/
extern int bpf_list_push_back_impl(struct bpf_list_head *head,
struct bpf_list_node *node,
void *meta, __u64 off) __ksym;
/* Convenience macro to wrap over bpf_list_push_back_impl */
#define bpf_list_push_back(head, node) bpf_list_push_back_impl(head, node, NULL, 0)
/* Description
* Remove the entry at the beginning of the BPF linked list.
* Returns
* Pointer to bpf_list_node of deleted entry, or NULL if list is empty.
*/
extern struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym;
/* Description
* Remove the entry at the end of the BPF linked list.
* Returns
* Pointer to bpf_list_node of deleted entry, or NULL if list is empty.
*/
extern struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym;
/* Description
* Remove 'node' from rbtree with root 'root'
* Returns
* Pointer to the removed node, or NULL if 'root' didn't contain 'node'
*/
extern struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root,
struct bpf_rb_node *node) __ksym;
/* Description
* Add 'node' to rbtree with root 'root' using comparator 'less'
*
* The 'meta' and 'off' parameters are rewritten by the verifier, no need
* for BPF programs to set them
* Returns
* 0 if the node was successfully added
* -EINVAL if the node wasn't added because it's already in a tree
*/
extern int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node,
bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b),
void *meta, __u64 off) __ksym;
/* Convenience macro to wrap over bpf_rbtree_add_impl */
#define bpf_rbtree_add(head, node, less) bpf_rbtree_add_impl(head, node, less, NULL, 0)
/* Description
* Return the first (leftmost) node in input tree
* Returns
* Pointer to the node, which is _not_ removed from the tree. If the tree
* contains no nodes, returns NULL.
*/
extern struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym;
/* Description
* Allocates a percpu object of the type represented by 'local_type_id' in
* program BTF. User may use the bpf_core_type_id_local macro to pass the
* type ID of a struct in program BTF.
*
* The 'local_type_id' parameter must be a known constant.
* The 'meta' parameter is rewritten by the verifier, no need for BPF
* program to set it.
* Returns
* A pointer to a percpu object of the type corresponding to the passed in
* 'local_type_id', or NULL on failure.
*/
extern void *bpf_percpu_obj_new_impl(__u64 local_type_id, void *meta) __ksym;
/* Convenience macro to wrap over bpf_percpu_obj_new_impl */
#define bpf_percpu_obj_new(type) ((type __percpu_kptr *)bpf_percpu_obj_new_impl(bpf_core_type_id_local(type), NULL))
/* Description
* Free an allocated percpu object. All fields of the object that require
* destruction will be destructed before the storage is freed.
*
* The 'meta' parameter is rewritten by the verifier, no need for BPF
* program to set it.
* Returns
* Void.
*/
extern void bpf_percpu_obj_drop_impl(void *kptr, void *meta) __ksym;
/* Convenience macro to wrap over bpf_percpu_obj_new */
#define bpf_percpu_obj_new(type) ((type __percpu_kptr *)bpf_percpu_obj_new(bpf_core_type_id_local(type)))
struct bpf_iter_task_vma;
@@ -167,9 +22,6 @@ extern int bpf_iter_task_vma_new(struct bpf_iter_task_vma *it,
extern struct vm_area_struct *bpf_iter_task_vma_next(struct bpf_iter_task_vma *it) __ksym;
extern void bpf_iter_task_vma_destroy(struct bpf_iter_task_vma *it) __ksym;
/* Convenience macro to wrap over bpf_obj_drop_impl */
#define bpf_percpu_obj_drop(kptr) bpf_percpu_obj_drop_impl(kptr, NULL)
/* Description
* Throw a BPF exception from the program, immediately terminating its
* execution and unwinding the stack. The supplied 'cookie' parameter
@@ -627,6 +479,10 @@ struct task_struct___preempt_rt {
int softirq_disable_cnt;
} __attribute__((preserve_access_index));
#ifdef bpf_target_s390
extern struct lowcore *bpf_get_lowcore(void) __weak __ksym;
#endif
static inline int get_preempt_count(void)
{
#if defined(bpf_target_x86)
@@ -645,6 +501,10 @@ static inline int get_preempt_count(void)
bpf_this_cpu_ptr(&pcpu_hot))->preempt_count;
#elif defined(bpf_target_arm64)
return bpf_get_current_task_btf()->thread_info.preempt.count;
#elif defined(bpf_target_powerpc)
return bpf_get_current_task_btf()->thread_info.preempt_count;
#elif defined(bpf_target_s390)
return bpf_get_lowcore()->preempt_count;
#endif
return 0;
}
@@ -653,6 +513,8 @@ static inline int get_preempt_count(void)
* Report whether it is in interrupt context. Only works on the following archs:
* * x86
* * arm64
* * powerpc64
* * s390x
*/
static inline int bpf_in_interrupt(void)
{
@@ -672,6 +534,8 @@ static inline int bpf_in_interrupt(void)
* Report whether it is in NMI context. Only works on the following archs:
* * x86
* * arm64
* * powerpc64
* * s390x
*/
static inline int bpf_in_nmi(void)
{
@@ -682,6 +546,8 @@ static inline int bpf_in_nmi(void)
* Report whether it is in hard IRQ context. Only works on the following archs:
* * x86
* * arm64
* * powerpc64
* * s390x
*/
static inline int bpf_in_hardirq(void)
{
@@ -692,6 +558,8 @@ static inline int bpf_in_hardirq(void)
* Report whether it is in softirq context. Only works on the following archs:
* * x86
* * arm64
* * powerpc64
* * s390x
*/
static inline int bpf_in_serving_softirq(void)
{
@@ -710,6 +578,8 @@ static inline int bpf_in_serving_softirq(void)
* Report whether it is in task context. Only works on the following archs:
* * x86
* * arm64
* * powerpc64
* * s390x
*/
static inline int bpf_in_task(void)
{
tools/testing/selftests/bpf/bpftool_helpers.c
+7 -8
View File
@@ -2,18 +2,18 @@
#include <unistd.h>
#include <string.h>
#include <stdbool.h>
#include <limits.h>
#include "bpf_util.h"
#include "bpftool_helpers.h"
#define BPFTOOL_PATH_MAX_LEN 64
#define BPFTOOL_FULL_CMD_MAX_LEN 512
#define BPFTOOL_FULL_CMD_MAX_LEN (PATH_MAX * 2)
#define BPFTOOL_DEFAULT_PATH "tools/sbin/bpftool"
static int detect_bpftool_path(char *buffer, size_t size)
{
char tmp[BPFTOOL_PATH_MAX_LEN];
char tmp[PATH_MAX];
const char *env_path;
/* First, check if BPFTOOL environment variable is set */
@@ -29,7 +29,7 @@ static int detect_bpftool_path(char *buffer, size_t size)
/* Check default bpftool location (will work if we are running the
* default flavor of test_progs)
*/
snprintf(tmp, BPFTOOL_PATH_MAX_LEN, "./%s", BPFTOOL_DEFAULT_PATH);
snprintf(tmp, sizeof(tmp), "./%s", BPFTOOL_DEFAULT_PATH);
if (access(tmp, X_OK) == 0) {
strscpy(buffer, tmp, size);
return 0;
@@ -38,7 +38,7 @@ static int detect_bpftool_path(char *buffer, size_t size)
/* Check alternate bpftool location (will work if we are running a
* specific flavor of test_progs, e.g. cpuv4 or no_alu32)
*/
snprintf(tmp, BPFTOOL_PATH_MAX_LEN, "../%s", BPFTOOL_DEFAULT_PATH);
snprintf(tmp, sizeof(tmp), "../%s", BPFTOOL_DEFAULT_PATH);
if (access(tmp, X_OK) == 0) {
strscpy(buffer, tmp, size);
return 0;
@@ -50,7 +50,7 @@ static int detect_bpftool_path(char *buffer, size_t size)
static int run_command(char *args, char *output_buf, size_t output_max_len)
{
static char bpftool_path[BPFTOOL_PATH_MAX_LEN] = {0};
static char bpftool_path[PATH_MAX] = {};
bool suppress_output = !(output_buf && output_max_len);
char command[BPFTOOL_FULL_CMD_MAX_LEN];
FILE *f;
@@ -60,7 +60,7 @@ static int run_command(char *args, char *output_buf, size_t output_max_len)
if (bpftool_path[0] == 0 && detect_bpftool_path(bpftool_path, sizeof(bpftool_path)))
return 1;
ret = snprintf(command, BPFTOOL_FULL_CMD_MAX_LEN, "%s %s%s",
ret = snprintf(command, sizeof(command), "%s %s%s",
bpftool_path, args,
suppress_output ? " > /dev/null 2>&1" : "");
@@ -84,4 +84,3 @@ int get_bpftool_command_output(char *args, char *output_buf, size_t output_max_l
{
return run_command(args, output_buf, output_max_len);
}
tools/testing/selftests/bpf/cgroup_iter_memcg.h
-2
View File
@@ -9,8 +9,6 @@ struct memcg_query {
unsigned long nr_shmem;
unsigned long nr_file_pages;
unsigned long nr_file_mapped;
/* some memcg_stat_item */
unsigned long memcg_kmem;
/* some vm_event_item */
unsigned long pgfault;
};
tools/testing/selftests/bpf/prog_tests/access_variable_array.c
-16
View File
@@ -1,16 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2022 Bytedance */
#include <test_progs.h>
#include "test_access_variable_array.skel.h"
void test_access_variable_array(void)
{
struct test_access_variable_array *skel;
skel = test_access_variable_array__open_and_load();
if (!ASSERT_OK_PTR(skel, "test_access_variable_array__open_and_load"))
return;
test_access_variable_array__destroy(skel);
}
tools/testing/selftests/bpf/prog_tests/align.c
-712
View File
@@ -1,712 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
#include <test_progs.h>
#define MAX_INSNS 512
#define MAX_MATCHES 24
struct bpf_reg_match {
unsigned int line;
const char *reg;
const char *match;
};
struct bpf_align_test {
const char *descr;
struct bpf_insn insns[MAX_INSNS];
enum {
UNDEF,
ACCEPT,
REJECT
} result;
enum bpf_prog_type prog_type;
/* Matches must be in order of increasing line */
struct bpf_reg_match matches[MAX_MATCHES];
};
static struct bpf_align_test tests[] = {
/* Four tests of known constants. These aren't staggeringly
* interesting since we track exact values now.
*/
{
.descr = "mov",
.insns = {
BPF_MOV64_IMM(BPF_REG_3, 2),
BPF_MOV64_IMM(BPF_REG_3, 4),
BPF_MOV64_IMM(BPF_REG_3, 8),
BPF_MOV64_IMM(BPF_REG_3, 16),
BPF_MOV64_IMM(BPF_REG_3, 32),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.matches = {
{0, "R1", "ctx()"},
{0, "R10", "fp0"},
{0, "R3", "2"},
{1, "R3", "4"},
{2, "R3", "8"},
{3, "R3", "16"},
{4, "R3", "32"},
},
},
{
.descr = "shift",
.insns = {
BPF_MOV64_IMM(BPF_REG_3, 1),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1),
BPF_ALU64_IMM(BPF_RSH, BPF_REG_3, 4),
BPF_MOV64_IMM(BPF_REG_4, 32),
BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1),
BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1),
BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1),
BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.matches = {
{0, "R1", "ctx()"},
{0, "R10", "fp0"},
{0, "R3", "1"},
{1, "R3", "2"},
{2, "R3", "4"},
{3, "R3", "8"},
{4, "R3", "16"},
{5, "R3", "1"},
{6, "R4", "32"},
{7, "R4", "16"},
{8, "R4", "8"},
{9, "R4", "4"},
{10, "R4", "2"},
},
},
{
.descr = "addsub",
.insns = {
BPF_MOV64_IMM(BPF_REG_3, 4),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, 4),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, 2),
BPF_MOV64_IMM(BPF_REG_4, 8),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 2),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.matches = {
{0, "R1", "ctx()"},
{0, "R10", "fp0"},
{0, "R3", "4"},
{1, "R3", "8"},
{2, "R3", "10"},
{3, "R4", "8"},
{4, "R4", "12"},
{5, "R4", "14"},
},
},
{
.descr = "mul",
.insns = {
BPF_MOV64_IMM(BPF_REG_3, 7),
BPF_ALU64_IMM(BPF_MUL, BPF_REG_3, 1),
BPF_ALU64_IMM(BPF_MUL, BPF_REG_3, 2),
BPF_ALU64_IMM(BPF_MUL, BPF_REG_3, 4),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.matches = {
{0, "R1", "ctx()"},
{0, "R10", "fp0"},
{0, "R3", "7"},
{1, "R3", "7"},
{2, "R3", "14"},
{3, "R3", "56"},
},
},
/* Tests using unknown values */
#define PREP_PKT_POINTERS \
BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, \
offsetof(struct __sk_buff, data)), \
BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1, \
offsetof(struct __sk_buff, data_end))
#define LOAD_UNKNOWN(DST_REG) \
PREP_PKT_POINTERS, \
BPF_MOV64_REG(BPF_REG_0, BPF_REG_2), \
BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 8), \
BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_0, 1), \
BPF_EXIT_INSN(), \
BPF_LDX_MEM(BPF_B, DST_REG, BPF_REG_2, 0)
{
.descr = "unknown shift",
.insns = {
LOAD_UNKNOWN(BPF_REG_3),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_3, 1),
LOAD_UNKNOWN(BPF_REG_4),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_4, 5),
BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1),
BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1),
BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1),
BPF_ALU64_IMM(BPF_RSH, BPF_REG_4, 1),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.matches = {
{6, "R0", "pkt(off=8,r=8)"},
{6, "R3", "var_off=(0x0; 0xff)"},
{7, "R3", "var_off=(0x0; 0x1fe)"},
{8, "R3", "var_off=(0x0; 0x3fc)"},
{9, "R3", "var_off=(0x0; 0x7f8)"},
{10, "R3", "var_off=(0x0; 0xff0)"},
{12, "R3", "pkt_end()"},
{17, "R4", "var_off=(0x0; 0xff)"},
{18, "R4", "var_off=(0x0; 0x1fe0)"},
{19, "R4", "var_off=(0x0; 0xff0)"},
{20, "R4", "var_off=(0x0; 0x7f8)"},
{21, "R4", "var_off=(0x0; 0x3fc)"},
{22, "R4", "var_off=(0x0; 0x1fe)"},
},
},
{
.descr = "unknown mul",
.insns = {
LOAD_UNKNOWN(BPF_REG_3),
BPF_MOV64_REG(BPF_REG_4, BPF_REG_3),
BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 1),
BPF_MOV64_REG(BPF_REG_4, BPF_REG_3),
BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 2),
BPF_MOV64_REG(BPF_REG_4, BPF_REG_3),
BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 4),
BPF_MOV64_REG(BPF_REG_4, BPF_REG_3),
BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 8),
BPF_ALU64_IMM(BPF_MUL, BPF_REG_4, 2),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.matches = {
{6, "R3", "var_off=(0x0; 0xff)"},
{7, "R4", "var_off=(0x0; 0xff)"},
{8, "R4", "var_off=(0x0; 0xff)"},
{9, "R4", "var_off=(0x0; 0xff)"},
{10, "R4", "var_off=(0x0; 0x1fe)"},
{11, "R4", "var_off=(0x0; 0xff)"},
{12, "R4", "var_off=(0x0; 0x3fc)"},
{13, "R4", "var_off=(0x0; 0xff)"},
{14, "R4", "var_off=(0x0; 0x7f8)"},
{15, "R4", "var_off=(0x0; 0xff0)"},
},
},
{
.descr = "packet const offset",
.insns = {
PREP_PKT_POINTERS,
BPF_MOV64_REG(BPF_REG_5, BPF_REG_2),
BPF_MOV64_IMM(BPF_REG_0, 0),
/* Skip over ethernet header. */
BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14),
BPF_MOV64_REG(BPF_REG_4, BPF_REG_5),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4),
BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1),
BPF_EXIT_INSN(),
BPF_LDX_MEM(BPF_B, BPF_REG_4, BPF_REG_5, 0),
BPF_LDX_MEM(BPF_B, BPF_REG_4, BPF_REG_5, 1),
BPF_LDX_MEM(BPF_B, BPF_REG_4, BPF_REG_5, 2),
BPF_LDX_MEM(BPF_B, BPF_REG_4, BPF_REG_5, 3),
BPF_LDX_MEM(BPF_H, BPF_REG_4, BPF_REG_5, 0),
BPF_LDX_MEM(BPF_H, BPF_REG_4, BPF_REG_5, 2),
BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_5, 0),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.matches = {
{2, "R5", "pkt(r=0)"},
{4, "R5", "pkt(off=14,r=0)"},
{5, "R4", "pkt(off=14,r=0)"},
{9, "R2", "pkt(r=18)"},
{10, "R5", "pkt(off=14,r=18)"},
{10, "R4", "var_off=(0x0; 0xff)"},
{13, "R4", "var_off=(0x0; 0xffff)"},
{14, "R4", "var_off=(0x0; 0xffff)"},
},
},
{
.descr = "packet variable offset",
.insns = {
LOAD_UNKNOWN(BPF_REG_6),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2),
/* First, add a constant to the R5 packet pointer,
* then a variable with a known alignment.
*/
BPF_MOV64_REG(BPF_REG_5, BPF_REG_2),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14),
BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6),
BPF_MOV64_REG(BPF_REG_4, BPF_REG_5),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4),
BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1),
BPF_EXIT_INSN(),
BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_5, 0),
/* Now, test in the other direction. Adding first
* the variable offset to R5, then the constant.
*/
BPF_MOV64_REG(BPF_REG_5, BPF_REG_2),
BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6),
BPF_MOV64_REG(BPF_REG_4, BPF_REG_5),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14),
BPF_MOV64_REG(BPF_REG_4, BPF_REG_5),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4),
BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1),
BPF_EXIT_INSN(),
BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_5, 0),
/* Test multiple accumulations of unknown values
* into a packet pointer.
*/
BPF_MOV64_REG(BPF_REG_5, BPF_REG_2),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14),
BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6),
BPF_MOV64_REG(BPF_REG_4, BPF_REG_5),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 4),
BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6),
BPF_MOV64_REG(BPF_REG_4, BPF_REG_5),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4),
BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1),
BPF_EXIT_INSN(),
BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_5, 0),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.matches = {
/* Calculated offset in R6 has unknown value, but known
* alignment of 4.
*/
{6, "R2", "pkt(r=8)"},
{7, "R6", "var_off=(0x0; 0x3fc)"},
/* Offset is added to packet pointer R5, resulting in
* known fixed offset, and variable offset from R6.
*/
{11, "R5", "pkt(id=1,off=14,"},
/* At the time the word size load is performed from R5,
* it's total offset is NET_IP_ALIGN + reg->off (0) +
* reg->aux_off (14) which is 16. Then the variable
* offset is considered using reg->aux_off_align which
* is 4 and meets the load's requirements.
*/
{15, "R4", "var_off=(0x0; 0x3fc)"},
{15, "R5", "var_off=(0x0; 0x3fc)"},
/* Variable offset is added to R5 packet pointer,
* resulting in auxiliary alignment of 4. To avoid BPF
* verifier's precision backtracking logging
* interfering we also have a no-op R4 = R5
* instruction to validate R5 state. We also check
* that R4 is what it should be in such case.
*/
{18, "R4", "var_off=(0x0; 0x3fc)"},
{18, "R5", "var_off=(0x0; 0x3fc)"},
/* Constant offset is added to R5, resulting in
* reg->off of 14.
*/
{19, "R5", "pkt(id=2,off=14,"},
/* At the time the word size load is performed from R5,
* its total fixed offset is NET_IP_ALIGN + reg->off
* (14) which is 16. Then the variable offset is 4-byte
* aligned, so the total offset is 4-byte aligned and
* meets the load's requirements.
*/
{24, "R4", "var_off=(0x0; 0x3fc)"},
{24, "R5", "var_off=(0x0; 0x3fc)"},
/* Constant offset is added to R5 packet pointer,
* resulting in reg->off value of 14.
*/
{26, "R5", "pkt(off=14,r=8)"},
/* Variable offset is added to R5, resulting in a
* variable offset of (4n). See comment for insn #18
* for R4 = R5 trick.
*/
{28, "R4", "var_off=(0x0; 0x3fc)"},
{28, "R5", "var_off=(0x0; 0x3fc)"},
/* Constant is added to R5 again, setting reg->off to 18. */
{29, "R5", "pkt(id=3,off=18,"},
/* And once more we add a variable; resulting var_off
* is still (4n), fixed offset is not changed.
* Also, we create a new reg->id.
*/
{31, "R4", "var_off=(0x0; 0x7fc)"},
{31, "R5", "var_off=(0x0; 0x7fc)"},
/* At the time the word size load is performed from R5,
* its total fixed offset is NET_IP_ALIGN + reg->off (18)
* which is 20. Then the variable offset is (4n), so
* the total offset is 4-byte aligned and meets the
* load's requirements.
*/
{35, "R4", "var_off=(0x0; 0x7fc)"},
{35, "R5", "var_off=(0x0; 0x7fc)"},
},
},
{
.descr = "packet variable offset 2",
.insns = {
/* Create an unknown offset, (4n+2)-aligned */
LOAD_UNKNOWN(BPF_REG_6),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, 14),
/* Add it to the packet pointer */
BPF_MOV64_REG(BPF_REG_5, BPF_REG_2),
BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6),
/* Check bounds and perform a read */
BPF_MOV64_REG(BPF_REG_4, BPF_REG_5),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4),
BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1),
BPF_EXIT_INSN(),
BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_5, 0),
/* Make a (4n) offset from the value we just read */
BPF_ALU64_IMM(BPF_AND, BPF_REG_6, 0xff),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2),
/* Add it to the packet pointer */
BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6),
/* Check bounds and perform a read */
BPF_MOV64_REG(BPF_REG_4, BPF_REG_5),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4),
BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1),
BPF_EXIT_INSN(),
BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_5, 0),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.matches = {
/* Calculated offset in R6 has unknown value, but known
* alignment of 4.
*/
{6, "R2", "pkt(r=8)"},
{7, "R6", "var_off=(0x0; 0x3fc)"},
/* Adding 14 makes R6 be (4n+2) */
{8, "R6", "var_off=(0x2; 0x7fc)"},
/* Packet pointer has (4n+2) offset */
{11, "R5", "var_off=(0x2; 0x7fc)"},
{12, "R4", "var_off=(0x2; 0x7fc)"},
/* At the time the word size load is performed from R5,
* its total fixed offset is NET_IP_ALIGN + reg->off (0)
* which is 2. Then the variable offset is (4n+2), so
* the total offset is 4-byte aligned and meets the
* load's requirements.
*/
{15, "R5", "var_off=(0x2; 0x7fc)"},
/* Newly read value in R6 was shifted left by 2, so has
* known alignment of 4.
*/
{17, "R6", "var_off=(0x0; 0x3fc)"},
/* Added (4n) to packet pointer's (4n+2) var_off, giving
* another (4n+2).
*/
{19, "R5", "var_off=(0x2; 0xffc)"},
{20, "R4", "var_off=(0x2; 0xffc)"},
/* At the time the word size load is performed from R5,
* its total fixed offset is NET_IP_ALIGN + reg->off (0)
* which is 2. Then the variable offset is (4n+2), so
* the total offset is 4-byte aligned and meets the
* load's requirements.
*/
{23, "R5", "var_off=(0x2; 0xffc)"},
},
},
{
.descr = "dubious pointer arithmetic",
.insns = {
PREP_PKT_POINTERS,
BPF_MOV64_IMM(BPF_REG_0, 0),
/* (ptr - ptr) << 2 */
BPF_MOV64_REG(BPF_REG_5, BPF_REG_3),
BPF_ALU64_REG(BPF_SUB, BPF_REG_5, BPF_REG_2),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_5, 2),
/* We have a (4n) value. Let's make a packet offset
* out of it. First add 14, to make it a (4n+2)
*/
BPF_ALU64_IMM(BPF_ADD, BPF_REG_5, 14),
/* Then make sure it's nonnegative */
BPF_JMP_IMM(BPF_JSGE, BPF_REG_5, 0, 1),
BPF_EXIT_INSN(),
/* Add it to packet pointer */
BPF_MOV64_REG(BPF_REG_6, BPF_REG_2),
BPF_ALU64_REG(BPF_ADD, BPF_REG_6, BPF_REG_5),
/* Check bounds and perform a read */
BPF_MOV64_REG(BPF_REG_4, BPF_REG_6),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4),
BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1),
BPF_EXIT_INSN(),
BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_6, 0),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = REJECT,
.matches = {
{3, "R5", "pkt_end()"},
/* (ptr - ptr) << 2 == unknown, (4n) */
{5, "R5", "var_off=(0x0; 0xfffffffffffffffc)"},
/* (4n) + 14 == (4n+2). We blow our bounds, because
* the add could overflow.
*/
{6, "R5", "var_off=(0x2; 0xfffffffffffffffc)"},
/* Checked s>=0 */
{9, "R5", "var_off=(0x2; 0x7ffffffffffffffc)"},
/* packet pointer + nonnegative (4n+2) */
{11, "R6", "var_off=(0x2; 0x7ffffffffffffffc)"},
{12, "R4", "var_off=(0x2; 0x7ffffffffffffffc)"},
/* NET_IP_ALIGN + (4n+2) == (4n), alignment is fine.
* We checked the bounds, but it might have been able
* to overflow if the packet pointer started in the
* upper half of the address space.
* So we did not get a 'range' on R6, and the access
* attempt will fail.
*/
{15, "R6", "var_off=(0x2; 0x7ffffffffffffffc)"},
}
},
{
.descr = "variable subtraction",
.insns = {
/* Create an unknown offset, (4n+2)-aligned */
LOAD_UNKNOWN(BPF_REG_6),
BPF_MOV64_REG(BPF_REG_7, BPF_REG_6),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, 14),
/* Create another unknown, (4n)-aligned, and subtract
* it from the first one
*/
BPF_ALU64_IMM(BPF_LSH, BPF_REG_7, 2),
BPF_ALU64_REG(BPF_SUB, BPF_REG_6, BPF_REG_7),
/* Bounds-check the result */
BPF_JMP_IMM(BPF_JSGE, BPF_REG_6, 0, 1),
BPF_EXIT_INSN(),
/* Add it to the packet pointer */
BPF_MOV64_REG(BPF_REG_5, BPF_REG_2),
BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_6),
/* Check bounds and perform a read */
BPF_MOV64_REG(BPF_REG_4, BPF_REG_5),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4),
BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1),
BPF_EXIT_INSN(),
BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_5, 0),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.matches = {
/* Calculated offset in R6 has unknown value, but known
* alignment of 4.
*/
{6, "R2", "pkt(r=8)"},
{8, "R6", "var_off=(0x0; 0x3fc)"},
/* Adding 14 makes R6 be (4n+2) */
{9, "R6", "var_off=(0x2; 0x7fc)"},
/* New unknown value in R7 is (4n) */
{10, "R7", "var_off=(0x0; 0x3fc)"},
/* Subtracting it from R6 blows our unsigned bounds */
{11, "R6", "var_off=(0x2; 0xfffffffffffffffc)"},
/* Checked s>= 0 */
{14, "R6", "var_off=(0x2; 0x7fc)"},
/* At the time the word size load is performed from R5,
* its total fixed offset is NET_IP_ALIGN + reg->off (0)
* which is 2. Then the variable offset is (4n+2), so
* the total offset is 4-byte aligned and meets the
* load's requirements.
*/
{20, "R5", "var_off=(0x2; 0x7fc)"},
},
},
{
.descr = "pointer variable subtraction",
.insns = {
/* Create an unknown offset, (4n+2)-aligned and bounded
* to [14,74]
*/
LOAD_UNKNOWN(BPF_REG_6),
BPF_MOV64_REG(BPF_REG_7, BPF_REG_6),
BPF_ALU64_IMM(BPF_AND, BPF_REG_6, 0xf),
BPF_ALU64_IMM(BPF_LSH, BPF_REG_6, 2),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, 14),
/* Subtract it from the packet pointer */
BPF_MOV64_REG(BPF_REG_5, BPF_REG_2),
BPF_ALU64_REG(BPF_SUB, BPF_REG_5, BPF_REG_6),
/* Create another unknown, (4n)-aligned and >= 74.
* That in fact means >= 76, since 74 % 4 == 2
*/
BPF_ALU64_IMM(BPF_LSH, BPF_REG_7, 2),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_7, 76),
/* Add it to the packet pointer */
BPF_ALU64_REG(BPF_ADD, BPF_REG_5, BPF_REG_7),
/* Check bounds and perform a read */
BPF_MOV64_REG(BPF_REG_4, BPF_REG_5),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 4),
BPF_JMP_REG(BPF_JGE, BPF_REG_3, BPF_REG_4, 1),
BPF_EXIT_INSN(),
BPF_LDX_MEM(BPF_W, BPF_REG_6, BPF_REG_5, 0),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.matches = {
/* Calculated offset in R6 has unknown value, but known
* alignment of 4.
*/
{6, "R2", "pkt(r=8)"},
{9, "R6", "var_off=(0x0; 0x3c)"},
/* Adding 14 makes R6 be (4n+2) */
{10, "R6", "var_off=(0x2; 0x7c)"},
/* Subtracting from packet pointer overflows ubounds */
{13, "R5", "var_off=(0xffffffffffffff82; 0x7c)"},
/* New unknown value in R7 is (4n), >= 76 */
{14, "R7", "var_off=(0x0; 0x7fc)"},
/* Adding it to packet pointer gives nice bounds again */
{16, "R5", "var_off=(0x2; 0x7fc)"},
/* At the time the word size load is performed from R5,
* its total fixed offset is NET_IP_ALIGN + reg->off (0)
* which is 2. Then the variable offset is (4n+2), so
* the total offset is 4-byte aligned and meets the
* load's requirements.
*/
{20, "R5", "var_off=(0x2; 0x7fc)"},
},
},
};
static int probe_filter_length(const struct bpf_insn *fp)
{
int len;
for (len = MAX_INSNS - 1; len > 0; --len)
if (fp[len].code != 0 || fp[len].imm != 0)
break;
return len + 1;
}
static char bpf_vlog[32768];
static int do_test_single(struct bpf_align_test *test)
{
struct bpf_insn *prog = test->insns;
int prog_type = test->prog_type;
char bpf_vlog_copy[32768];
LIBBPF_OPTS(bpf_prog_load_opts, opts,
.prog_flags = BPF_F_STRICT_ALIGNMENT,
.log_buf = bpf_vlog,
.log_size = sizeof(bpf_vlog),
.log_level = 2,
);
const char *main_pass_start = "0: R1=ctx() R10=fp0";
const char *line_ptr;
int cur_line = -1;
int prog_len, i;
char *start;
int fd_prog;
int ret;
prog_len = probe_filter_length(prog);
fd_prog = bpf_prog_load(prog_type ? : BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL",
prog, prog_len, &opts);
if (fd_prog < 0 && test->result != REJECT) {
printf("Failed to load program.\n");
printf("%s", bpf_vlog);
ret = 1;
} else if (fd_prog >= 0 && test->result == REJECT) {
printf("Unexpected success to load!\n");
printf("%s", bpf_vlog);
ret = 1;
close(fd_prog);
} else {
ret = 0;
/* We make a local copy so that we can strtok() it */
strncpy(bpf_vlog_copy, bpf_vlog, sizeof(bpf_vlog_copy));
start = strstr(bpf_vlog_copy, main_pass_start);
if (!start) {
ret = 1;
printf("Can't find initial line '%s'\n", main_pass_start);
goto out;
}
line_ptr = strtok(start, "\n");
for (i = 0; i < MAX_MATCHES; i++) {
struct bpf_reg_match m = test->matches[i];
const char *p;
int tmp;
if (!m.match)
break;
while (line_ptr) {
cur_line = -1;
sscanf(line_ptr, "%u: ", &cur_line);
if (cur_line == -1)
sscanf(line_ptr, "from %u to %u: ", &tmp, &cur_line);
if (cur_line == m.line)
break;
line_ptr = strtok(NULL, "\n");
}
if (!line_ptr) {
printf("Failed to find line %u for match: %s=%s\n",
m.line, m.reg, m.match);
ret = 1;
printf("%s", bpf_vlog);
break;
}
/* Check the next line as well in case the previous line
* did not have a corresponding bpf insn. Example:
* func#0 @0
* 0: R1=ctx() R10=fp0
* 0: (b7) r3 = 2 ; R3_w=2
*
* Sometimes it's actually two lines below, e.g. when
* searching for "6: R3_w=scalar(umax=255,var_off=(0x0; 0xff))":
* from 4 to 6: R0_w=pkt(off=8,r=8) R1=ctx() R2_w=pkt(r=8) R3_w=pkt_end() R10=fp0
* 6: R0_w=pkt(off=8,r=8) R1=ctx() R2_w=pkt(r=8) R3_w=pkt_end() R10=fp0
* 6: (71) r3 = *(u8 *)(r2 +0) ; R2_w=pkt(r=8) R3_w=scalar(umax=255,var_off=(0x0; 0xff))
*/
while (!(p = strstr(line_ptr, m.reg)) || !strstr(p, m.match)) {
cur_line = -1;
line_ptr = strtok(NULL, "\n");
sscanf(line_ptr ?: "", "%u: ", &cur_line);
if (!line_ptr || cur_line != m.line)
break;
}
if (cur_line != m.line || !line_ptr || !(p = strstr(line_ptr, m.reg)) || !strstr(p, m.match)) {
printf("Failed to find match %u: %s=%s\n", m.line, m.reg, m.match);
ret = 1;
printf("%s", bpf_vlog);
break;
}
}
out:
if (fd_prog >= 0)
close(fd_prog);
}
return ret;
}
void test_align(void)
{
unsigned int i;
for (i = 0; i < ARRAY_SIZE(tests); i++) {
struct bpf_align_test *test = &tests[i];
if (!test__start_subtest(test->descr))
continue;
ASSERT_OK(do_test_single(test), test->descr);
}
}
tools/testing/selftests/bpf/prog_tests/attach_probe.c
+213
View File
@@ -123,6 +123,140 @@ cleanup:
test_attach_probe_manual__destroy(skel);
}
/* manual attach address-based kprobe/kretprobe testings */
static void test_attach_kprobe_by_addr(enum probe_attach_mode attach_mode)
{
LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts);
struct test_attach_probe_manual *skel;
unsigned long func_addr;
if (!ASSERT_OK(load_kallsyms(), "load_kallsyms"))
return;
func_addr = ksym_get_addr(SYS_NANOSLEEP_KPROBE_NAME);
if (!ASSERT_NEQ(func_addr, 0UL, "func_addr"))
return;
skel = test_attach_probe_manual__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_kprobe_manual_open_and_load"))
return;
kprobe_opts.attach_mode = attach_mode;
kprobe_opts.retprobe = false;
kprobe_opts.offset = func_addr;
skel->links.handle_kprobe =
bpf_program__attach_kprobe_opts(skel->progs.handle_kprobe,
NULL, &kprobe_opts);
if (!ASSERT_OK_PTR(skel->links.handle_kprobe, "attach_kprobe_by_addr"))
goto cleanup;
kprobe_opts.retprobe = true;
skel->links.handle_kretprobe =
bpf_program__attach_kprobe_opts(skel->progs.handle_kretprobe,
NULL, &kprobe_opts);
if (!ASSERT_OK_PTR(skel->links.handle_kretprobe, "attach_kretprobe_by_addr"))
goto cleanup;
/* trigger & validate kprobe && kretprobe */
usleep(1);
ASSERT_EQ(skel->bss->kprobe_res, 1, "check_kprobe_res");
ASSERT_EQ(skel->bss->kretprobe_res, 2, "check_kretprobe_res");
cleanup:
test_attach_probe_manual__destroy(skel);
}
/* reject legacy address-based kprobe attach */
static void test_attach_kprobe_legacy_by_addr_reject(void)
{
LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts);
struct test_attach_probe_manual *skel;
unsigned long func_addr;
if (!ASSERT_OK(load_kallsyms(), "load_kallsyms"))
return;
func_addr = ksym_get_addr(SYS_NANOSLEEP_KPROBE_NAME);
if (!ASSERT_NEQ(func_addr, 0UL, "func_addr"))
return;
skel = test_attach_probe_manual__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_kprobe_manual_open_and_load"))
return;
kprobe_opts.attach_mode = PROBE_ATTACH_MODE_LEGACY;
kprobe_opts.offset = func_addr;
skel->links.handle_kprobe =
bpf_program__attach_kprobe_opts(skel->progs.handle_kprobe,
NULL, &kprobe_opts);
ASSERT_ERR_PTR(skel->links.handle_kprobe, "attach_kprobe_legacy_by_addr");
ASSERT_EQ(libbpf_get_error(skel->links.handle_kprobe),
-EOPNOTSUPP, "attach_kprobe_legacy_by_addr_err");
test_attach_probe_manual__destroy(skel);
}
/*
* bpf_fentry_shadow_test exists in both vmlinux (net/bpf/test_run.c) and
* bpf_testmod (bpf_testmod.c). When bpf_testmod is loaded the symbol is
* duplicated. Test that kprobe attachment handles this correctly:
* - Unqualified name ("bpf_fentry_shadow_test") attaches to vmlinux.
* - MOD:SYM name ("bpf_testmod:bpf_fentry_shadow_test") attaches to module.
*
* Note: bpf_fentry_shadow_test is not invoked via test_run, so we only
* verify that attach and detach succeed without triggering the probe.
*/
static void test_attach_probe_dup_sym(enum probe_attach_mode attach_mode)
{
DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts);
struct bpf_link *kprobe_link, *kretprobe_link;
struct test_attach_probe_manual *skel;
skel = test_attach_probe_manual__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_dup_sym_open_and_load"))
return;
kprobe_opts.attach_mode = attach_mode;
/* Unqualified: should attach to vmlinux symbol */
kprobe_opts.retprobe = false;
kprobe_link = bpf_program__attach_kprobe_opts(skel->progs.handle_kprobe,
"bpf_fentry_shadow_test",
&kprobe_opts);
if (!ASSERT_OK_PTR(kprobe_link, "attach_kprobe_vmlinux"))
goto cleanup;
bpf_link__destroy(kprobe_link);
kprobe_opts.retprobe = true;
kretprobe_link = bpf_program__attach_kprobe_opts(skel->progs.handle_kretprobe,
"bpf_fentry_shadow_test",
&kprobe_opts);
if (!ASSERT_OK_PTR(kretprobe_link, "attach_kretprobe_vmlinux"))
goto cleanup;
bpf_link__destroy(kretprobe_link);
/* MOD:SYM qualified: should attach to module symbol */
kprobe_opts.retprobe = false;
kprobe_link = bpf_program__attach_kprobe_opts(skel->progs.handle_kprobe,
"bpf_testmod:bpf_fentry_shadow_test",
&kprobe_opts);
if (!ASSERT_OK_PTR(kprobe_link, "attach_kprobe_module"))
goto cleanup;
bpf_link__destroy(kprobe_link);
kprobe_opts.retprobe = true;
kretprobe_link = bpf_program__attach_kprobe_opts(skel->progs.handle_kretprobe,
"bpf_testmod:bpf_fentry_shadow_test",
&kprobe_opts);
if (!ASSERT_OK_PTR(kretprobe_link, "attach_kretprobe_module"))
goto cleanup;
bpf_link__destroy(kretprobe_link);
cleanup:
test_attach_probe_manual__destroy(skel);
}
/* attach uprobe/uretprobe long event name testings */
static void test_attach_uprobe_long_event_name(void)
{
@@ -220,11 +354,73 @@ static void test_attach_kprobe_write_ctx(void)
kprobe_write_ctx__destroy(skel);
}
static void test_freplace_kprobe_write_ctx(void)
{
struct bpf_program *prog_kprobe, *prog_ext, *prog_fentry;
struct kprobe_write_ctx *skel_kprobe, *skel_ext = NULL;
struct bpf_link *link_kprobe = NULL, *link_ext = NULL;
int err, prog_fd;
LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts);
LIBBPF_OPTS(bpf_test_run_opts, topts);
skel_kprobe = kprobe_write_ctx__open();
if (!ASSERT_OK_PTR(skel_kprobe, "kprobe_write_ctx__open kprobe"))
return;
prog_kprobe = skel_kprobe->progs.kprobe_dummy;
bpf_program__set_autoload(prog_kprobe, true);
prog_fentry = skel_kprobe->progs.fentry;
bpf_program__set_autoload(prog_fentry, true);
err = kprobe_write_ctx__load(skel_kprobe);
if (!ASSERT_OK(err, "kprobe_write_ctx__load kprobe"))
goto out;
skel_ext = kprobe_write_ctx__open();
if (!ASSERT_OK_PTR(skel_ext, "kprobe_write_ctx__open ext"))
goto out;
prog_ext = skel_ext->progs.freplace_kprobe;
bpf_program__set_autoload(prog_ext, true);
prog_fd = bpf_program__fd(skel_kprobe->progs.kprobe_write_ctx);
bpf_program__set_attach_target(prog_ext, prog_fd, "kprobe_write_ctx");
err = kprobe_write_ctx__load(skel_ext);
if (!ASSERT_OK(err, "kprobe_write_ctx__load ext"))
goto out;
prog_fd = bpf_program__fd(prog_kprobe);
link_ext = bpf_program__attach_freplace(prog_ext, prog_fd, "kprobe_dummy");
ASSERT_ERR_PTR(link_ext, "bpf_program__attach_freplace link");
ASSERT_EQ(libbpf_get_error(link_ext), -EINVAL, "bpf_program__attach_freplace error");
link_kprobe = bpf_program__attach_kprobe_opts(prog_kprobe, "bpf_fentry_test1",
&kprobe_opts);
if (!ASSERT_OK_PTR(link_kprobe, "bpf_program__attach_kprobe_opts"))
goto out;
err = bpf_prog_test_run_opts(bpf_program__fd(prog_fentry), &topts);
ASSERT_OK(err, "bpf_prog_test_run_opts");
out:
bpf_link__destroy(link_ext);
bpf_link__destroy(link_kprobe);
kprobe_write_ctx__destroy(skel_ext);
kprobe_write_ctx__destroy(skel_kprobe);
}
#else
static void test_attach_kprobe_write_ctx(void)
{
test__skip();
}
static void test_freplace_kprobe_write_ctx(void)
{
test__skip();
}
#endif
static void test_attach_probe_auto(struct test_attach_probe *skel)
@@ -416,6 +612,21 @@ void test_attach_probe(void)
test_attach_probe_manual(PROBE_ATTACH_MODE_PERF);
if (test__start_subtest("manual-link"))
test_attach_probe_manual(PROBE_ATTACH_MODE_LINK);
if (test__start_subtest("kprobe-perf-by-addr"))
test_attach_kprobe_by_addr(PROBE_ATTACH_MODE_PERF);
if (test__start_subtest("kprobe-link-by-addr"))
test_attach_kprobe_by_addr(PROBE_ATTACH_MODE_LINK);
if (test__start_subtest("kprobe-legacy-by-addr-reject"))
test_attach_kprobe_legacy_by_addr_reject();
if (test__start_subtest("dup-sym-default"))
test_attach_probe_dup_sym(PROBE_ATTACH_MODE_DEFAULT);
if (test__start_subtest("dup-sym-legacy"))
test_attach_probe_dup_sym(PROBE_ATTACH_MODE_LEGACY);
if (test__start_subtest("dup-sym-perf"))
test_attach_probe_dup_sym(PROBE_ATTACH_MODE_PERF);
if (test__start_subtest("dup-sym-link"))
test_attach_probe_dup_sym(PROBE_ATTACH_MODE_LINK);
if (test__start_subtest("auto"))
test_attach_probe_auto(skel);
@@ -434,6 +645,8 @@ void test_attach_probe(void)
test_attach_kprobe_long_event_name();
if (test__start_subtest("kprobe-write-ctx"))
test_attach_kprobe_write_ctx();
if (test__start_subtest("freplace-kprobe-write-ctx"))
test_freplace_kprobe_write_ctx();
cleanup:
test_attach_probe__destroy(skel);
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c
+21 -8
View File
@@ -6,6 +6,7 @@
#include <sys/syscall.h>
#include <sys/mman.h>
#include <unistd.h>
#include <linux/compiler.h>
#include <test_progs.h>
#include <network_helpers.h>
#include <bpf/btf.h>
@@ -105,6 +106,11 @@ static void kprobe_multi_link_api_subtest(void)
unsigned long long addrs[8];
__u64 cookies[8];
if (!env.has_testmod) {
test__skip();
return;
}
if (!ASSERT_OK(load_kallsyms(), "load_kallsyms"))
goto cleanup;
@@ -192,6 +198,11 @@ static void kprobe_multi_attach_api_subtest(void)
};
__u64 cookies[8];
if (!env.has_testmod) {
test__skip();
return;
}
skel = kprobe_multi__open_and_load();
if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
goto cleanup;
@@ -421,11 +432,12 @@ cleanup:
bpf_link__destroy(link3);
}
static void burn_cpu(void)
static void burn_cpu(long loops)
{
volatile int j = 0;
long j = 0;
cpu_set_t cpu_set;
int i, err;
long i;
int err;
/* generate some branches on cpu 0 */
CPU_ZERO(&cpu_set);
@@ -433,9 +445,10 @@ static void burn_cpu(void)
err = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set), &cpu_set);
ASSERT_OK(err, "set_thread_affinity");
/* spin the loop for a while (random high number) */
for (i = 0; i < 1000000; ++i)
for (i = 0; i < loops; ++i) {
++j;
barrier();
}
}
static void pe_subtest(struct test_bpf_cookie *skel)
@@ -451,7 +464,7 @@ static void pe_subtest(struct test_bpf_cookie *skel)
attr.type = PERF_TYPE_SOFTWARE;
attr.config = PERF_COUNT_SW_CPU_CLOCK;
attr.sample_period = 100000;
pfd = syscall(__NR_perf_event_open, &attr, -1, 0, -1, PERF_FLAG_FD_CLOEXEC);
pfd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, PERF_FLAG_FD_CLOEXEC);
if (!ASSERT_GE(pfd, 0, "perf_fd"))
goto cleanup;
@@ -460,7 +473,7 @@ static void pe_subtest(struct test_bpf_cookie *skel)
if (!ASSERT_OK_PTR(link, "link1"))
goto cleanup;
burn_cpu(); /* trigger BPF prog */
burn_cpu(100000000L); /* trigger BPF prog */
ASSERT_EQ(skel->bss->pe_res, 0x100000, "pe_res1");
@@ -479,7 +492,7 @@ static void pe_subtest(struct test_bpf_cookie *skel)
if (!ASSERT_OK_PTR(link, "link2"))
goto cleanup;
burn_cpu(); /* trigger BPF prog */
burn_cpu(100000000L); /* trigger BPF prog */
ASSERT_EQ(skel->bss->pe_res, 0x200000, "pe_res2");
tools/testing/selftests/bpf/prog_tests/bpf_gotox.c
+79 -34
View File
@@ -317,7 +317,7 @@ static void check_ldimm64_off_load(struct bpf_gotox *skel __always_unused)
static int __check_ldimm64_gotox_prog_load(struct bpf_insn *insns,
__u32 insn_cnt,
__u32 off1, __u32 off2)
int off1, int off2, int off3)
{
const __u32 values[] = {5, 7, 9, 11, 13, 15};
const __u32 max_entries = ARRAY_SIZE(values);
@@ -349,16 +349,46 @@ static int __check_ldimm64_gotox_prog_load(struct bpf_insn *insns,
/* r1 += off2 */
insns[2].imm = off2;
/* r1 = *(r1 + off3) */
insns[3].off = off3;
ret = prog_load(insns, insn_cnt);
close(map_fd);
return ret;
}
static void reject_offsets(struct bpf_insn *insns, __u32 insn_cnt, __u32 off1, __u32 off2)
static void
allow_offsets(struct bpf_insn *insns, __u32 insn_cnt, int off1, int off2, int off3)
{
LIBBPF_OPTS(bpf_test_run_opts, topts);
int prog_fd, err;
char s[128] = "";
prog_fd = __check_ldimm64_gotox_prog_load(insns, insn_cnt, off1, off2, off3);
snprintf(s, sizeof(s), "__check_ldimm64_gotox_prog_load(%d,%d,%d)", off1, off2, off3);
if (!ASSERT_GE(prog_fd, 0, s))
return;
err = bpf_prog_test_run_opts(prog_fd, &topts);
if (!ASSERT_OK(err, "test_run_opts err")) {
close(prog_fd);
return;
}
if (!ASSERT_EQ(topts.retval, (off1 + off2 + off3) / 8, "test_run_opts retval")) {
close(prog_fd);
return;
}
close(prog_fd);
}
static void
reject_offsets(struct bpf_insn *insns, __u32 insn_cnt, int off1, int off2, int off3)
{
int prog_fd;
prog_fd = __check_ldimm64_gotox_prog_load(insns, insn_cnt, off1, off2);
prog_fd = __check_ldimm64_gotox_prog_load(insns, insn_cnt, off1, off2, off3);
if (!ASSERT_EQ(prog_fd, -EACCES, "__check_ldimm64_gotox_prog_load"))
close(prog_fd);
}
@@ -376,7 +406,7 @@ static void check_ldimm64_off_gotox(struct bpf_gotox *skel __always_unused)
* The program rewrites the offsets in the instructions below:
* r1 = &map + offset1
* r1 += offset2
* r1 = *r1
* r1 = *(r1 + offset3)
* gotox r1
*/
BPF_LD_IMM64_RAW(BPF_REG_1, BPF_PSEUDO_MAP_VALUE, 0),
@@ -403,43 +433,55 @@ static void check_ldimm64_off_gotox(struct bpf_gotox *skel __always_unused)
BPF_MOV64_IMM(BPF_REG_0, 5),
BPF_EXIT_INSN(),
};
int prog_fd, err;
__u32 off1, off2;
int off1, off2, off3;
/* allow all combinations off1 + off2 < 6 */
for (off1 = 0; off1 < 6; off1++) {
for (off2 = 0; off1 + off2 < 6; off2++) {
LIBBPF_OPTS(bpf_test_run_opts, topts);
/* allow all combinations off1 + off2 + off3 < 6 */
for (off1 = 0; off1 < 6; off1++)
for (off2 = 0; off1 + off2 < 6; off2++)
for (off3 = 0; off1 + off2 + off3 < 6; off3++)
allow_offsets(insns, ARRAY_SIZE(insns),
off1 * 8, off2 * 8, off3 * 8);
prog_fd = __check_ldimm64_gotox_prog_load(insns, ARRAY_SIZE(insns),
off1 * 8, off2 * 8);
if (!ASSERT_GE(prog_fd, 0, "__check_ldimm64_gotox_prog_load"))
return;
/* allow for some offsets to be negative */
allow_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 0, -(8 * 3));
allow_offsets(insns, ARRAY_SIZE(insns), 8 * 3, -(8 * 3), 0);
allow_offsets(insns, ARRAY_SIZE(insns), 0, 8 * 3, -(8 * 3));
allow_offsets(insns, ARRAY_SIZE(insns), 8 * 4, 0, -(8 * 2));
allow_offsets(insns, ARRAY_SIZE(insns), 8 * 4, -(8 * 2), 0);
allow_offsets(insns, ARRAY_SIZE(insns), 0, 8 * 4, -(8 * 2));
err = bpf_prog_test_run_opts(prog_fd, &topts);
if (!ASSERT_OK(err, "test_run_opts err")) {
close(prog_fd);
return;
}
/* disallow negative sums of offsets */
reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 0, -(8 * 4));
reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, -(8 * 4), 0);
reject_offsets(insns, ARRAY_SIZE(insns), 0, 8 * 3, -(8 * 4));
if (!ASSERT_EQ(topts.retval, off1 + off2, "test_run_opts retval")) {
close(prog_fd);
return;
}
/* disallow the off1 to be negative in any case */
reject_offsets(insns, ARRAY_SIZE(insns), -8 * 1, 0, 0);
reject_offsets(insns, ARRAY_SIZE(insns), -8 * 1, 8 * 1, 0);
reject_offsets(insns, ARRAY_SIZE(insns), -8 * 1, 8 * 1, 8 * 1);
close(prog_fd);
}
}
/* reject off1 + off2 + off3 >= 6 */
reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 8 * 3, 8 * 0);
reject_offsets(insns, ARRAY_SIZE(insns), 8 * 7, 8 * 0, 8 * 0);
reject_offsets(insns, ARRAY_SIZE(insns), 8 * 0, 8 * 7, 8 * 0);
reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 8 * 0, 8 * 3);
reject_offsets(insns, ARRAY_SIZE(insns), 8 * 0, 8 * 3, 8 * 3);
/* reject off1 + off2 >= 6 */
reject_offsets(insns, ARRAY_SIZE(insns), 8 * 3, 8 * 3);
reject_offsets(insns, ARRAY_SIZE(insns), 8 * 7, 8 * 0);
reject_offsets(insns, ARRAY_SIZE(insns), 8 * 0, 8 * 7);
/* reject (off1 + off2) % 8 != 0, off3 % 8 != 0 */
reject_offsets(insns, ARRAY_SIZE(insns), 3, 3, 0);
reject_offsets(insns, ARRAY_SIZE(insns), 7, 0, 0);
reject_offsets(insns, ARRAY_SIZE(insns), 0, 7, 0);
reject_offsets(insns, ARRAY_SIZE(insns), 0, 0, 7);
}
/* reject (off1 + off2) % 8 != 0 */
reject_offsets(insns, ARRAY_SIZE(insns), 3, 3);
reject_offsets(insns, ARRAY_SIZE(insns), 7, 0);
reject_offsets(insns, ARRAY_SIZE(insns), 0, 7);
static void check_ldimm64_off_gotox_llvm(struct bpf_gotox *skel)
{
__u64 in[] = {0, 1, 2, 3, 4};
__u64 out[] = {1, 1, 5, 1, 1};
int i;
for (i = 0; i < ARRAY_SIZE(in); i++)
check_simple(skel, skel->progs.load_with_nonzero_offset, in[i], out[i]);
}
void test_bpf_gotox(void)
@@ -496,5 +538,8 @@ void test_bpf_gotox(void)
if (test__start_subtest("check-ldimm64-off-gotox"))
__subtest(skel, check_ldimm64_off_gotox);
if (test__start_subtest("check-ldimm64-off-gotox-llvm"))
__subtest(skel, check_ldimm64_off_gotox_llvm);
bpf_gotox__destroy(skel);
}
tools/testing/selftests/bpf/prog_tests/bpf_insn_array.c
+1 -1
View File
@@ -3,7 +3,7 @@
#include <bpf/bpf.h>
#include <test_progs.h>
#if defined(__x86_64__) || defined(__powerpc__)
#if defined(__x86_64__) || defined(__powerpc__) || defined(__aarch64__)
static int map_create(__u32 map_type, __u32 max_entries)
{
const char *map_name = "insn_array";
tools/testing/selftests/bpf/prog_tests/btf_kind.c
+226
View File
@@ -0,0 +1,226 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2026, Oracle and/or its affiliates. */
#include <test_progs.h>
#include <bpf/btf.h>
#include <bpf/libbpf.h>
/* Verify kind encoding exists for each kind */
static void test_btf_kind_encoding(void)
{
LIBBPF_OPTS(btf_new_opts, opts);
const struct btf_header *hdr;
const void *raw_btf;
struct btf *btf;
__u32 raw_size;
opts.add_layout = true;
btf = btf__new_empty_opts(&opts);
if (!ASSERT_OK_PTR(btf, "btf_new"))
return;
raw_btf = btf__raw_data(btf, &raw_size);
if (!ASSERT_OK_PTR(raw_btf, "btf__raw_data"))
return;
hdr = raw_btf;
ASSERT_EQ(hdr->layout_off % 4, 0, "layout_aligned");
ASSERT_EQ(hdr->layout_len, sizeof(struct btf_layout) * NR_BTF_KINDS,
"layout_len");
ASSERT_EQ(hdr->str_off, hdr->layout_off + hdr->layout_len, "str_after_layout");
btf__free(btf);
opts.add_layout = false;
btf = btf__new_empty_opts(&opts);
if (!ASSERT_OK_PTR(btf, "btf_new"))
return;
raw_btf = btf__raw_data(btf, &raw_size);
if (!ASSERT_OK_PTR(raw_btf, "btf__raw_data"))
return;
hdr = raw_btf;
ASSERT_EQ(hdr->layout_off, 0, "no_layout_off");
ASSERT_EQ(hdr->layout_len, 0, "no_layout_len");
ASSERT_EQ(hdr->str_off, hdr->type_off + hdr->type_len, "strs_after_types");
btf__free(btf);
}
static int write_raw_btf(void *raw_btf, size_t raw_size, char *file)
{
int fd = mkstemp(file);
ssize_t n;
if (!ASSERT_OK_FD(fd, "open_raw_btf"))
return -1;
n = write(fd, raw_btf, raw_size);
close(fd);
if (!ASSERT_EQ(n, (ssize_t)raw_size, "write_raw_btf"))
return -1;
return 0;
}
/*
* Fabricate an unrecognized kind at BTF_KIND_MAX + 1, and after adding
* the appropriate struct/typedefs to the BTF such that it recognizes
* this kind, ensure that parsing of BTF containing the unrecognized kind
* can succeed.
*/
void test_btf_kind_decoding(void)
{
char btf_kind_file1[] = "/tmp/test_btf_kind.XXXXXX";
char btf_kind_file2[] = "/tmp/test_btf_kind.XXXXXX";
char btf_kind_file3[] = "/tmp/test_btf_kind.XXXXXX";
struct btf *btf = NULL, *new_btf = NULL;
__s32 int_id, unrec_id, id, id2;
LIBBPF_OPTS(btf_new_opts, opts);
struct btf_layout *l;
struct btf_header *hdr;
const void *raw_btf;
struct btf_type *t;
void *new_raw_btf;
void *str_data;
__u32 raw_size;
opts.add_layout = true;
btf = btf__new_empty_opts(&opts);
if (!ASSERT_OK_PTR(btf, "btf_new"))
return;
int_id = btf__add_int(btf, "test_char", 1, BTF_INT_CHAR);
if (!ASSERT_GT(int_id, 0, "add_int_id"))
return;
/*
* Create our type with unrecognized kind by adding a typedef kind
* we will overwrite it with our unrecognized kind value.
*/
unrec_id = btf__add_typedef(btf, "unrec_kind", int_id);
if (!ASSERT_GT(unrec_id, 0, "add_unrec_id"))
return;
/*
* Add an id after it that we will look up to verify we can parse
* beyond unrecognized kinds.
*/
id = btf__add_typedef(btf, "test_lookup", int_id);
if (!ASSERT_GT(id, 0, "add_test_lookup_id"))
return;
id2 = btf__add_typedef(btf, "test_lookup2", int_id);
if (!ASSERT_GT(id2, 0, "add_test_lookup_id2"))
return;
raw_btf = (void *)btf__raw_data(btf, &raw_size);
if (!ASSERT_OK_PTR(raw_btf, "btf__raw_data"))
return;
new_raw_btf = calloc(1, raw_size + sizeof(*l));
if (!ASSERT_OK_PTR(new_raw_btf, "calloc_raw_btf"))
return;
memcpy(new_raw_btf, raw_btf, raw_size);
hdr = new_raw_btf;
/* Move strings to make space for one new layout description */
raw_size += sizeof(*l);
str_data = new_raw_btf + hdr->hdr_len + hdr->str_off;
memmove(str_data + sizeof(*l), str_data, hdr->str_len);
hdr->str_off += sizeof(*l);
/* Add new layout description */
hdr->layout_len += sizeof(*l);
l = new_raw_btf + hdr->hdr_len + hdr->layout_off;
l[NR_BTF_KINDS].info_sz = 0;
l[NR_BTF_KINDS].elem_sz = 0;
l[NR_BTF_KINDS].flags = 0;
/* Now modify typedef added above to be an unrecognized kind. */
t = (void *)hdr + hdr->hdr_len + hdr->type_off + sizeof(struct btf_type) +
sizeof(__u32);
t->info = (NR_BTF_KINDS << 24);
/* Write BTF to a raw file, ready for parsing. */
if (write_raw_btf(new_raw_btf, raw_size, btf_kind_file1))
goto out;
/*
* Verify parsing succeeds, and that we can read type info past
* the unrecognized kind.
*/
new_btf = btf__parse_raw(btf_kind_file1);
if (ASSERT_OK_PTR(new_btf, "btf__parse_raw")) {
ASSERT_EQ(btf__find_by_name(new_btf, "unrec_kind"), unrec_id,
"unrec_kind_found");
ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup",
BTF_KIND_TYPEDEF), id,
"verify_id_lookup");
ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup2",
BTF_KIND_TYPEDEF), id2,
"verify_id2_lookup");
}
btf__free(new_btf);
new_btf = NULL;
/*
* Next, change info_sz to equal sizeof(struct btf_type); this means the
* "test_lookup" kind will be reinterpreted as a singular info element
* following the unrecognized kind.
*/
l[NR_BTF_KINDS].info_sz = sizeof(struct btf_type);
if (write_raw_btf(new_raw_btf, raw_size, btf_kind_file2))
goto out;
new_btf = btf__parse_raw(btf_kind_file2);
if (ASSERT_OK_PTR(new_btf, "btf__parse_raw")) {
ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup",
BTF_KIND_TYPEDEF), -ENOENT,
"verify_id_not_found");
/* id of "test_lookup2" will be id2 -1 as we have removed one type */
ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup2",
BTF_KIND_TYPEDEF), id2 - 1,
"verify_id_lookup2");
}
btf__free(new_btf);
new_btf = NULL;
/*
* Change elem_sz to equal sizeof(struct btf_type) and set vlen
* associated with unrecognized type to 1; this allows us to verify
* vlen-specified BTF can still be parsed.
*/
l[NR_BTF_KINDS].info_sz = 0;
l[NR_BTF_KINDS].elem_sz = sizeof(struct btf_type);
t->info |= 1;
if (write_raw_btf(new_raw_btf, raw_size, btf_kind_file3))
goto out;
new_btf = btf__parse_raw(btf_kind_file3);
if (ASSERT_OK_PTR(new_btf, "btf__parse_raw")) {
ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup",
BTF_KIND_TYPEDEF), -ENOENT,
"verify_id_not_found");
/* id of "test_lookup2" will be id2 -1 as we have removed one type */
ASSERT_EQ(btf__find_by_name_kind(new_btf, "test_lookup2",
BTF_KIND_TYPEDEF), id2 - 1,
"verify_id_lookup2");
}
out:
btf__free(new_btf);
free(new_raw_btf);
unlink(btf_kind_file1);
unlink(btf_kind_file2);
unlink(btf_kind_file3);
btf__free(btf);
}
void test_btf_kind(void)
{
if (test__start_subtest("btf_kind_encoding"))
test_btf_kind_encoding();
if (test__start_subtest("btf_kind_decoding"))
test_btf_kind_decoding();
}
tools/testing/selftests/bpf/prog_tests/btf_sanitize.c
+97
View File
@@ -0,0 +1,97 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2026, Oracle and/or its affiliates. */
#include <test_progs.h>
#include <linux/btf.h>
#include "bpf/libbpf_internal.h"
#include "../test_btf.h"
#include "kfree_skb.skel.h"
#define TYPE_LEN (sizeof(struct btf_type) + sizeof(__u32))
#define MAX_NR_LAYOUT 2
#define LAYOUT_LEN (sizeof(struct btf_layout) * MAX_NR_LAYOUT)
#define STR_LEN sizeof("\0int")
struct layout_btf {
struct btf_header hdr;
__u32 types[TYPE_LEN/sizeof(__u32)];
struct btf_layout layout[MAX_NR_LAYOUT];
char strs[STR_LEN];
};
static const struct layout_btf layout_btf = {
.hdr = {
.magic = BTF_MAGIC,
.version = BTF_VERSION,
.hdr_len = sizeof(struct btf_header),
.type_off = 0,
.type_len = TYPE_LEN,
.str_off = TYPE_LEN + LAYOUT_LEN,
.str_len = STR_LEN,
.layout_off = TYPE_LEN,
.layout_len = LAYOUT_LEN,
},
.types = {
BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
},
.layout = {
{ .info_sz = 0, .elem_sz = 0, .flags = 0 },
{ .info_sz = sizeof(__u32), .elem_sz = 0, .flags = 0 },
},
.strs = "\0int",
};
void test_btf_sanitize_layout(void)
{
struct btf *orig = NULL, *sanitized = NULL;
struct kern_feature_cache *cache = NULL;
struct kfree_skb *skel = NULL;
const struct btf_header *hdr;
const void *raw;
__u32 raw_sz;
skel = kfree_skb__open();
if (!ASSERT_OK_PTR(skel, "kfree_skb_skel"))
return;
orig = btf__new(&layout_btf, sizeof(layout_btf));
if (!ASSERT_OK_PTR(orig, "btf_new_layout"))
goto out;
raw = btf__raw_data(orig, &raw_sz);
if (!ASSERT_OK_PTR(raw, "btf__raw_data_orig"))
goto out;
hdr = (struct btf_header *)raw;
ASSERT_EQ(hdr->layout_off, TYPE_LEN, "layout_off_nonzero");
ASSERT_EQ(hdr->layout_len, LAYOUT_LEN, "layout_len_nonzero");
cache = calloc(1, sizeof(*cache));
if (!ASSERT_OK_PTR(cache, "alloc_feat_cache"))
goto out;
for (int i = 0; i < __FEAT_CNT; i++)
cache->res[i] = FEAT_SUPPORTED;
cache->res[FEAT_BTF_LAYOUT] = FEAT_MISSING;
bpf_object_set_feat_cache(skel->obj, cache);
if (!ASSERT_FALSE(kernel_supports(skel->obj, FEAT_BTF_LAYOUT), "layout_feature_missing"))
goto out;
if (!ASSERT_TRUE(kernel_supports(skel->obj, FEAT_BTF_FUNC), "other_feature_allowed"))
goto out;
sanitized = bpf_object__sanitize_btf(skel->obj, orig);
if (!ASSERT_OK_PTR(sanitized, "bpf_object__sanitize_btf"))
goto out;
raw = btf__raw_data(sanitized, &raw_sz);
if (!ASSERT_OK_PTR(raw, "btf__raw_data_sanitized"))
goto out;
hdr = (struct btf_header *)raw;
ASSERT_EQ(hdr->layout_off, 0, "layout_off_zero");
ASSERT_EQ(hdr->layout_len, 0, "layout_len_zero");
ASSERT_EQ(hdr->str_off, TYPE_LEN, "strs_after_types");
ASSERT_EQ(hdr->str_len, STR_LEN, "strs_len_unchanged");
ASSERT_EQ(raw_sz, hdr->hdr_len + hdr->type_len + hdr->str_len, "btf_raw_sz_reduced");
out:
/* This will free the cache we allocated above */
kfree_skb__destroy(skel);
btf__free(sanitized);
btf__free(orig);
}
tools/testing/selftests/bpf/prog_tests/btf_write.c
+111
View File
@@ -497,10 +497,121 @@ cleanup:
btf__free(btf2);
}
static void test_btf_add_btf_split()
{
struct btf *base = NULL, *split1 = NULL, *split2 = NULL;
struct btf *combined = NULL;
int id, err;
/* Create a base BTF with an INT and a PTR to it */
base = btf__new_empty();
if (!ASSERT_OK_PTR(base, "base"))
return;
id = btf__add_int(base, "int", 4, BTF_INT_SIGNED);
ASSERT_EQ(id, 1, "base_int_id");
id = btf__add_ptr(base, 1);
ASSERT_EQ(id, 2, "base_ptr_id");
/* base has 2 types, type IDs 1..2 */
ASSERT_EQ(btf__type_cnt(base), 3, "base_type_cnt");
/* Create split1 on base: a STRUCT referencing base's int (ID 1) */
split1 = btf__new_empty_split(base);
if (!ASSERT_OK_PTR(split1, "split1"))
goto cleanup;
id = btf__add_struct(split1, "s1", 4);
/* split types start at base_type_cnt = 3 */
ASSERT_EQ(id, 3, "split1_struct_id");
btf__add_field(split1, "x", 1, 0, 0); /* refers to base int */
id = btf__add_ptr(split1, 3);
ASSERT_EQ(id, 4, "split1_ptr_id"); /* ptr to the struct (split self-ref) */
/* Add a typedef "int_alias" -> base int in split1, which will be
* duplicated in split2 to test that btf__dedup() merges them.
*/
id = btf__add_typedef(split1, "int_alias", 1);
ASSERT_EQ(id, 5, "split1_typedef_id");
/* Create split2 on base: a TYPEDEF referencing base's ptr (ID 2) */
split2 = btf__new_empty_split(base);
if (!ASSERT_OK_PTR(split2, "split2"))
goto cleanup;
id = btf__add_typedef(split2, "int_ptr", 2); /* refers to base ptr */
ASSERT_EQ(id, 3, "split2_typedef_id");
id = btf__add_struct(split2, "s2", 8);
ASSERT_EQ(id, 4, "split2_struct_id");
btf__add_field(split2, "p", 3, 0, 0); /* refers to split2's own typedef */
/* Same "int_alias" typedef as split1 - should be deduped away */
id = btf__add_typedef(split2, "int_alias", 1);
ASSERT_EQ(id, 5, "split2_dup_typedef_id");
/* Create combined split BTF on same base and merge both */
combined = btf__new_empty_split(base);
if (!ASSERT_OK_PTR(combined, "combined"))
goto cleanup;
/* Merge split1: its types (3,4,5) should land at IDs 3,4,5 */
id = btf__add_btf(combined, split1);
if (!ASSERT_GE(id, 0, "add_split1"))
goto cleanup;
ASSERT_EQ(id, 3, "split1_first_id");
/* Merge split2: its types (3,4,5) should be remapped to 6,7,8 */
id = btf__add_btf(combined, split2);
if (!ASSERT_GE(id, 0, "add_split2"))
goto cleanup;
ASSERT_EQ(id, 6, "split2_first_id");
/* Before dedup: base (2) + split1 (3) + split2 (3) = 8 types + void */
ASSERT_EQ(btf__type_cnt(combined), 9, "pre_dedup_type_cnt");
VALIDATE_RAW_BTF(
combined,
/* base types (IDs 1-2) */
"[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[2] PTR '(anon)' type_id=1",
/* split1 types (IDs 3-5): base refs unchanged */
"[3] STRUCT 's1' size=4 vlen=1\n"
"\t'x' type_id=1 bits_offset=0", /* refers to base int=1 */
"[4] PTR '(anon)' type_id=3", /* refers to split1's struct=3 */
"[5] TYPEDEF 'int_alias' type_id=1", /* refers to base int=1 */
/* split2 types (IDs 6-8): remapped from 3,4,5 to 6,7,8 */
"[6] TYPEDEF 'int_ptr' type_id=2", /* base ptr=2, unchanged */
"[7] STRUCT 's2' size=8 vlen=1\n"
"\t'p' type_id=6 bits_offset=0", /* split2 typedef: 3->6 */
"[8] TYPEDEF 'int_alias' type_id=1"); /* dup of [5] */
/* Dedup to mirror the bpftool merge flow; should remove the
* duplicate "int_alias" typedef.
*/
err = btf__dedup(combined, NULL);
if (!ASSERT_OK(err, "dedup"))
goto cleanup;
/* After dedup: one int_alias removed, so 7 types + void */
ASSERT_EQ(btf__type_cnt(combined), 8, "dedup_type_cnt");
cleanup:
btf__free(combined);
btf__free(split2);
btf__free(split1);
btf__free(base);
}
void test_btf_write()
{
if (test__start_subtest("btf_add"))
test_btf_add();
if (test__start_subtest("btf_add_btf"))
test_btf_add_btf();
if (test__start_subtest("btf_add_btf_split"))
test_btf_add_btf_split();
}
tools/testing/selftests/bpf/prog_tests/cgroup_iter_memcg.c
+3 -32
View File
@@ -107,10 +107,10 @@ static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query)
/*
* Increase memcg shmem usage by creating and writing
* to a shmem object.
* to a memfd backed by shmem/tmpfs.
*/
fd = shm_open("/tmp_shmem", O_CREAT | O_RDWR, 0644);
if (!ASSERT_OK_FD(fd, "shm_open"))
fd = memfd_create("tmp_shmem", 0);
if (!ASSERT_OK_FD(fd, "memfd_create"))
return;
if (!ASSERT_OK(fallocate(fd, 0, 0, len), "fallocate"))
@@ -123,33 +123,6 @@ static void test_shmem(struct bpf_link *link, struct memcg_query *memcg_query)
cleanup:
close(fd);
shm_unlink("/tmp_shmem");
}
#define NR_PIPES 64
static void test_kmem(struct bpf_link *link, struct memcg_query *memcg_query)
{
int fds[NR_PIPES][2], i;
/*
* Increase kmem value by creating pipes which will allocate some
* kernel buffers.
*/
for (i = 0; i < NR_PIPES; i++) {
if (!ASSERT_OK(pipe(fds[i]), "pipe"))
goto cleanup;
}
if (!ASSERT_OK(read_stats(link), "read stats"))
goto cleanup;
ASSERT_GT(memcg_query->memcg_kmem, 0, "kmem value");
cleanup:
for (i = i - 1; i >= 0; i--) {
close(fds[i][0]);
close(fds[i][1]);
}
}
static void test_pgfault(struct bpf_link *link, struct memcg_query *memcg_query)
@@ -209,8 +182,6 @@ void test_cgroup_iter_memcg(void)
test_shmem(link, &skel->data_query->memcg_query);
if (test__start_subtest("cgroup_iter_memcg__file"))
test_file(link, &skel->data_query->memcg_query);
if (test__start_subtest("cgroup_iter_memcg__kmem"))
test_kmem(link, &skel->data_query->memcg_query);
if (test__start_subtest("cgroup_iter_memcg__pgfault"))
test_pgfault(link, &skel->data_query->memcg_query);
tools/testing/selftests/bpf/prog_tests/cgroup_storage.c
+47
View File
@@ -1,5 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
#include <unistd.h>
#include <sys/socket.h>
#include <test_progs.h>
#include "cgroup_helpers.h"
#include "network_helpers.h"
@@ -86,6 +88,11 @@ void test_cgroup_storage(void)
err = SYS_NOFAIL(PING_CMD);
ASSERT_OK(err, "sixth ping");
err = bpf_map__get_next_key(skel->maps.cgroup_storage, &key, &key,
sizeof(key));
ASSERT_ERR(err, "bpf_map__get_next_key should fail");
ASSERT_EQ(errno, ENOENT, "no second key");
cleanup_progs:
cgroup_storage__destroy(skel);
cleanup_network:
@@ -94,3 +101,43 @@ cleanup_cgroup:
close(cgroup_fd);
cleanup_cgroup_environment();
}
void test_cgroup_storage_oob(void)
{
struct cgroup_storage *skel;
int cgroup_fd, sock_fd;
cgroup_fd = cgroup_setup_and_join(TEST_CGROUP);
if (!ASSERT_OK_FD(cgroup_fd, "create cgroup"))
return;
/* Load and attach BPF program */
skel = cgroup_storage__open_and_load();
if (!ASSERT_OK_PTR(skel, "cgroup_storage__open_and_load"))
goto cleanup_cgroup;
skel->links.trigger_oob = bpf_program__attach_cgroup(skel->progs.trigger_oob,
cgroup_fd);
if (!ASSERT_OK_PTR(skel->links.trigger_oob, "attach_cgroup"))
goto cleanup_skel;
/* Create a socket to trigger cgroup/sock_create hook.
* This will execute our BPF program and trigger the OOB read
* if the bug is present (before the fix).
*/
sock_fd = socket(AF_INET, SOCK_DGRAM, 0);
if (!ASSERT_OK_FD(sock_fd, "create socket"))
goto cleanup_skel;
close(sock_fd);
/* If we reach here without a kernel panic or KASAN report,
* the test passes (the fix is working).
*/
cleanup_skel:
cgroup_storage__destroy(skel);
cleanup_cgroup:
close(cgroup_fd);
cleanup_cgroup_environment();
}
tools/testing/selftests/bpf/prog_tests/clone_attach_btf_id.c
+78
View File
@@ -0,0 +1,78 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2025 Meta */
#include <test_progs.h>
#include "clone_attach_btf_id.skel.h"
/*
* Test that bpf_program__clone() respects caller-provided attach_btf_id
* override via bpf_prog_load_opts.
*
* The BPF program has SEC("fentry/bpf_fentry_test1"). Clone it twice
* from the same prepared object: first with no opts (callback resolves
* attach_btf_id from sec_name), then with attach_btf_id overridden to
* bpf_fentry_test2. Verify each loaded program's attach_btf_id via
* bpf_prog_get_info_by_fd().
*/
static int get_prog_attach_btf_id(int prog_fd)
{
struct bpf_prog_info info = {};
__u32 info_len = sizeof(info);
int err;
err = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len);
if (err)
return err;
return info.attach_btf_id;
}
void test_clone_attach_btf_id(void)
{
struct clone_attach_btf_id *skel;
int fd1 = -1, fd2 = -1, err;
int btf_id_test1, btf_id_test2;
btf_id_test1 = libbpf_find_vmlinux_btf_id("bpf_fentry_test1", BPF_TRACE_FENTRY);
if (!ASSERT_GT(btf_id_test1, 0, "find_btf_id_test1"))
return;
btf_id_test2 = libbpf_find_vmlinux_btf_id("bpf_fentry_test2", BPF_TRACE_FENTRY);
if (!ASSERT_GT(btf_id_test2, 0, "find_btf_id_test2"))
return;
skel = clone_attach_btf_id__open();
if (!ASSERT_OK_PTR(skel, "skel_open"))
return;
err = bpf_object__prepare(skel->obj);
if (!ASSERT_OK(err, "obj_prepare"))
goto out;
/* Clone with no opts — callback resolves BTF from sec_name */
fd1 = bpf_program__clone(skel->progs.fentry_handler, NULL);
if (!ASSERT_GE(fd1, 0, "clone_default"))
goto out;
ASSERT_EQ(get_prog_attach_btf_id(fd1), btf_id_test1,
"attach_btf_id_default");
/*
* Clone with attach_btf_id override pointing to a different
* function. The BPF program never accesses arguments, so the
* load succeeds regardless of signature mismatch.
*/
LIBBPF_OPTS(bpf_prog_load_opts, opts,
.attach_btf_id = btf_id_test2,
);
fd2 = bpf_program__clone(skel->progs.fentry_handler, &opts);
if (!ASSERT_GE(fd2, 0, "clone_override"))
goto out;
ASSERT_EQ(get_prog_attach_btf_id(fd2), btf_id_test2,
"attach_btf_id_override");
out:
if (fd1 >= 0)
close(fd1);
if (fd2 >= 0)
close(fd2);
clone_attach_btf_id__destroy(skel);
}
tools/testing/selftests/bpf/prog_tests/connect_force_port.c
+22 -4
View File
@@ -53,6 +53,9 @@ static int run_test(int cgroup_fd, int server_fd, int family, int type)
__u16 expected_peer_port = 60000;
struct bpf_program *prog;
struct bpf_object *obj;
struct bpf_map *map;
__u16 *port_ptr;
size_t port_size;
const char *obj_file = v4 ? "connect_force_port4.bpf.o" : "connect_force_port6.bpf.o";
int fd, err;
__u32 duration = 0;
@@ -61,6 +64,21 @@ static int run_test(int cgroup_fd, int server_fd, int family, int type)
if (!ASSERT_OK_PTR(obj, "bpf_obj_open"))
return -1;
map = bpf_object__find_map_by_name(obj, ".bss");
if (!ASSERT_OK_PTR(map, "find bss map")) {
err = -EIO;
goto close_bpf_object;
}
port_ptr = bpf_map__initial_value(map, &port_size);
if (!ASSERT_OK_PTR(port_ptr, "get bss initial value")) {
err = -EIO;
goto close_bpf_object;
}
/* Auto assigns the port according to availability */
*port_ptr = ntohs(get_socket_local_port(server_fd));
err = bpf_object__load(obj);
if (!ASSERT_OK(err, "bpf_obj_load")) {
err = -EIO;
@@ -138,25 +156,25 @@ void test_connect_force_port(void)
if (CHECK_FAIL(cgroup_fd < 0))
return;
server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 60123, 0);
server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0);
if (CHECK_FAIL(server_fd < 0))
goto close_cgroup_fd;
CHECK_FAIL(run_test(cgroup_fd, server_fd, AF_INET, SOCK_STREAM));
close(server_fd);
server_fd = start_server(AF_INET6, SOCK_STREAM, NULL, 60124, 0);
server_fd = start_server(AF_INET6, SOCK_STREAM, NULL, 0, 0);
if (CHECK_FAIL(server_fd < 0))
goto close_cgroup_fd;
CHECK_FAIL(run_test(cgroup_fd, server_fd, AF_INET6, SOCK_STREAM));
close(server_fd);
server_fd = start_server(AF_INET, SOCK_DGRAM, NULL, 60123, 0);
server_fd = start_server(AF_INET, SOCK_DGRAM, NULL, 0, 0);
if (CHECK_FAIL(server_fd < 0))
goto close_cgroup_fd;
CHECK_FAIL(run_test(cgroup_fd, server_fd, AF_INET, SOCK_DGRAM));
close(server_fd);
server_fd = start_server(AF_INET6, SOCK_DGRAM, NULL, 60124, 0);
server_fd = start_server(AF_INET6, SOCK_DGRAM, NULL, 0, 0);
if (CHECK_FAIL(server_fd < 0))
goto close_cgroup_fd;
CHECK_FAIL(run_test(cgroup_fd, server_fd, AF_INET6, SOCK_DGRAM));
tools/testing/selftests/bpf/prog_tests/empty_skb.c
+37 -3
View File
@@ -10,8 +10,8 @@ void test_empty_skb(void)
struct empty_skb *bpf_obj = NULL;
struct nstoken *tok = NULL;
struct bpf_program *prog;
struct ethhdr eth_hlen;
char eth_hlen_pp[15];
char eth_hlen[14];
int veth_ifindex;
int ipip_ifindex;
int err;
@@ -25,7 +25,9 @@ void test_empty_skb(void)
int err;
int ret;
int lwt_egress_ret; /* expected retval at lwt/egress */
__be16 h_proto;
bool success_on_tc;
bool adjust_room;
} tests[] = {
/* Empty packets are always rejected. */
@@ -46,6 +48,28 @@ void test_empty_skb(void)
.err = -EINVAL,
},
/* ETH_HLEN-sized packets with IPv4/IPv6 EtherType but
* no L3 header are rejected.
*/
{
.msg = "veth short IPv4 ingress packet",
.data_in = &eth_hlen,
.data_size_in = sizeof(eth_hlen),
.ifindex = &veth_ifindex,
.err = -EINVAL,
.h_proto = htons(ETH_P_IP),
.adjust_room = true,
},
{
.msg = "veth short IPv6 ingress packet",
.data_in = &eth_hlen,
.data_size_in = sizeof(eth_hlen),
.ifindex = &veth_ifindex,
.err = -EINVAL,
.h_proto = htons(ETH_P_IPV6),
.adjust_room = true,
},
/* ETH_HLEN-sized packets:
* - can not be redirected at LWT_XMIT
* - can be redirected at TC to non-tunneling dest
@@ -54,7 +78,7 @@ void test_empty_skb(void)
{
/* __bpf_redirect_common */
.msg = "veth ETH_HLEN packet ingress",
.data_in = eth_hlen,
.data_in = &eth_hlen,
.data_size_in = sizeof(eth_hlen),
.ifindex = &veth_ifindex,
.ret = -ERANGE,
@@ -68,7 +92,7 @@ void test_empty_skb(void)
* tc: skb->len=14 <= skb_network_offset=14
*/
.msg = "ipip ETH_HLEN packet ingress",
.data_in = eth_hlen,
.data_in = &eth_hlen,
.data_size_in = sizeof(eth_hlen),
.ifindex = &ipip_ifindex,
.ret = -ERANGE,
@@ -108,17 +132,27 @@ void test_empty_skb(void)
SYS(out, "ip addr add 192.168.1.1/16 dev ipip0");
ipip_ifindex = if_nametoindex("ipip0");
memset(eth_hlen_pp, 0, sizeof(eth_hlen_pp));
memset(&eth_hlen, 0, sizeof(eth_hlen));
bpf_obj = empty_skb__open_and_load();
if (!ASSERT_OK_PTR(bpf_obj, "open skeleton"))
goto out;
for (i = 0; i < ARRAY_SIZE(tests); i++) {
if (tests[i].data_in == &eth_hlen)
eth_hlen.h_proto = tests[i].h_proto;
bpf_object__for_each_program(prog, bpf_obj->obj) {
bool at_egress = strstr(bpf_program__name(prog), "egress") != NULL;
bool at_tc = !strncmp(bpf_program__section_name(prog), "tc", 2);
bool is_adjust_room = !strcmp(bpf_program__name(prog), "tc_adjust_room");
int expected_ret;
char buf[128];
if (tests[i].adjust_room != is_adjust_room)
continue;
expected_ret = at_egress && !at_tc ? tests[i].lwt_egress_ret : tests[i].ret;
tattr.data_in = tests[i].data_in;
Loaded 100 of 233 files, more files were not shown because too many files have changed in this diff. Show more