bpf: Add multi uprobe link
Adding new multi uprobe link that allows to attach bpf program
to multiple uprobes.
Uprobes to attach are specified via new link_create uprobe_multi
union:
struct {
__aligned_u64 path;
__aligned_u64 offsets;
__aligned_u64 ref_ctr_offsets;
__u32 cnt;
__u32 flags;
} uprobe_multi;
Uprobes are defined for single binary specified in path and multiple
calling sites specified in offsets array with optional reference
counters specified in ref_ctr_offsets array. All specified arrays
have length of 'cnt'.
The 'flags' supports single bit for now that marks the uprobe as
return probe.
Acked-by: Andrii Nakryiko <andrii@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Acked-by: Yonghong Song <yonghong.song@linux.dev>
Link: https://lore.kernel.org/r/20230809083440.3209381-4-jolsa@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Jiri Olsa
committed by
Alexei Starovoitov
Alexei Starovoitov
parent
3505cb9fa2
commit
89ae89f53d
@@ -23,6 +23,7 @@
|
||||
#include <linux/sort.h>
|
||||
#include <linux/key.h>
|
||||
#include <linux/verification.h>
|
||||
#include <linux/namei.h>
|
||||
|
||||
#include <net/bpf_sk_storage.h>
|
||||
|
||||
@@ -2970,3 +2971,235 @@ static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_UPROBES
|
||||
struct bpf_uprobe_multi_link;
|
||||
|
||||
struct bpf_uprobe {
|
||||
struct bpf_uprobe_multi_link *link;
|
||||
loff_t offset;
|
||||
struct uprobe_consumer consumer;
|
||||
};
|
||||
|
||||
struct bpf_uprobe_multi_link {
|
||||
struct path path;
|
||||
struct bpf_link link;
|
||||
u32 cnt;
|
||||
struct bpf_uprobe *uprobes;
|
||||
};
|
||||
|
||||
struct bpf_uprobe_multi_run_ctx {
|
||||
struct bpf_run_ctx run_ctx;
|
||||
unsigned long entry_ip;
|
||||
};
|
||||
|
||||
static void bpf_uprobe_unregister(struct path *path, struct bpf_uprobe *uprobes,
|
||||
u32 cnt)
|
||||
{
|
||||
u32 i;
|
||||
|
||||
for (i = 0; i < cnt; i++) {
|
||||
uprobe_unregister(d_real_inode(path->dentry), uprobes[i].offset,
|
||||
&uprobes[i].consumer);
|
||||
}
|
||||
}
|
||||
|
||||
static void bpf_uprobe_multi_link_release(struct bpf_link *link)
|
||||
{
|
||||
struct bpf_uprobe_multi_link *umulti_link;
|
||||
|
||||
umulti_link = container_of(link, struct bpf_uprobe_multi_link, link);
|
||||
bpf_uprobe_unregister(&umulti_link->path, umulti_link->uprobes, umulti_link->cnt);
|
||||
}
|
||||
|
||||
static void bpf_uprobe_multi_link_dealloc(struct bpf_link *link)
|
||||
{
|
||||
struct bpf_uprobe_multi_link *umulti_link;
|
||||
|
||||
umulti_link = container_of(link, struct bpf_uprobe_multi_link, link);
|
||||
path_put(&umulti_link->path);
|
||||
kvfree(umulti_link->uprobes);
|
||||
kfree(umulti_link);
|
||||
}
|
||||
|
||||
static const struct bpf_link_ops bpf_uprobe_multi_link_lops = {
|
||||
.release = bpf_uprobe_multi_link_release,
|
||||
.dealloc = bpf_uprobe_multi_link_dealloc,
|
||||
};
|
||||
|
||||
static int uprobe_prog_run(struct bpf_uprobe *uprobe,
|
||||
unsigned long entry_ip,
|
||||
struct pt_regs *regs)
|
||||
{
|
||||
struct bpf_uprobe_multi_link *link = uprobe->link;
|
||||
struct bpf_uprobe_multi_run_ctx run_ctx = {
|
||||
.entry_ip = entry_ip,
|
||||
};
|
||||
struct bpf_prog *prog = link->link.prog;
|
||||
bool sleepable = prog->aux->sleepable;
|
||||
struct bpf_run_ctx *old_run_ctx;
|
||||
int err = 0;
|
||||
|
||||
if (sleepable)
|
||||
rcu_read_lock_trace();
|
||||
else
|
||||
rcu_read_lock();
|
||||
|
||||
migrate_disable();
|
||||
|
||||
old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
|
||||
err = bpf_prog_run(link->link.prog, regs);
|
||||
bpf_reset_run_ctx(old_run_ctx);
|
||||
|
||||
migrate_enable();
|
||||
|
||||
if (sleepable)
|
||||
rcu_read_unlock_trace();
|
||||
else
|
||||
rcu_read_unlock();
|
||||
return err;
|
||||
}
|
||||
|
||||
static int
|
||||
uprobe_multi_link_handler(struct uprobe_consumer *con, struct pt_regs *regs)
|
||||
{
|
||||
struct bpf_uprobe *uprobe;
|
||||
|
||||
uprobe = container_of(con, struct bpf_uprobe, consumer);
|
||||
return uprobe_prog_run(uprobe, instruction_pointer(regs), regs);
|
||||
}
|
||||
|
||||
static int
|
||||
uprobe_multi_link_ret_handler(struct uprobe_consumer *con, unsigned long func, struct pt_regs *regs)
|
||||
{
|
||||
struct bpf_uprobe *uprobe;
|
||||
|
||||
uprobe = container_of(con, struct bpf_uprobe, consumer);
|
||||
return uprobe_prog_run(uprobe, func, regs);
|
||||
}
|
||||
|
||||
int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
|
||||
{
|
||||
struct bpf_uprobe_multi_link *link = NULL;
|
||||
unsigned long __user *uref_ctr_offsets;
|
||||
unsigned long *ref_ctr_offsets = NULL;
|
||||
struct bpf_link_primer link_primer;
|
||||
struct bpf_uprobe *uprobes = NULL;
|
||||
unsigned long __user *uoffsets;
|
||||
void __user *upath;
|
||||
u32 flags, cnt, i;
|
||||
struct path path;
|
||||
char *name;
|
||||
int err;
|
||||
|
||||
/* no support for 32bit archs yet */
|
||||
if (sizeof(u64) != sizeof(void *))
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
if (prog->expected_attach_type != BPF_TRACE_UPROBE_MULTI)
|
||||
return -EINVAL;
|
||||
|
||||
flags = attr->link_create.uprobe_multi.flags;
|
||||
if (flags & ~BPF_F_UPROBE_MULTI_RETURN)
|
||||
return -EINVAL;
|
||||
|
||||
/*
|
||||
* path, offsets and cnt are mandatory,
|
||||
* ref_ctr_offsets is optional
|
||||
*/
|
||||
upath = u64_to_user_ptr(attr->link_create.uprobe_multi.path);
|
||||
uoffsets = u64_to_user_ptr(attr->link_create.uprobe_multi.offsets);
|
||||
cnt = attr->link_create.uprobe_multi.cnt;
|
||||
|
||||
if (!upath || !uoffsets || !cnt)
|
||||
return -EINVAL;
|
||||
|
||||
uref_ctr_offsets = u64_to_user_ptr(attr->link_create.uprobe_multi.ref_ctr_offsets);
|
||||
|
||||
name = strndup_user(upath, PATH_MAX);
|
||||
if (IS_ERR(name)) {
|
||||
err = PTR_ERR(name);
|
||||
return err;
|
||||
}
|
||||
|
||||
err = kern_path(name, LOOKUP_FOLLOW, &path);
|
||||
kfree(name);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
if (!d_is_reg(path.dentry)) {
|
||||
err = -EBADF;
|
||||
goto error_path_put;
|
||||
}
|
||||
|
||||
err = -ENOMEM;
|
||||
|
||||
link = kzalloc(sizeof(*link), GFP_KERNEL);
|
||||
uprobes = kvcalloc(cnt, sizeof(*uprobes), GFP_KERNEL);
|
||||
|
||||
if (!uprobes || !link)
|
||||
goto error_free;
|
||||
|
||||
if (uref_ctr_offsets) {
|
||||
ref_ctr_offsets = kvcalloc(cnt, sizeof(*ref_ctr_offsets), GFP_KERNEL);
|
||||
if (!ref_ctr_offsets)
|
||||
goto error_free;
|
||||
}
|
||||
|
||||
for (i = 0; i < cnt; i++) {
|
||||
if (uref_ctr_offsets && __get_user(ref_ctr_offsets[i], uref_ctr_offsets + i)) {
|
||||
err = -EFAULT;
|
||||
goto error_free;
|
||||
}
|
||||
if (__get_user(uprobes[i].offset, uoffsets + i)) {
|
||||
err = -EFAULT;
|
||||
goto error_free;
|
||||
}
|
||||
|
||||
uprobes[i].link = link;
|
||||
|
||||
if (flags & BPF_F_UPROBE_MULTI_RETURN)
|
||||
uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
|
||||
else
|
||||
uprobes[i].consumer.handler = uprobe_multi_link_handler;
|
||||
}
|
||||
|
||||
link->cnt = cnt;
|
||||
link->uprobes = uprobes;
|
||||
link->path = path;
|
||||
|
||||
bpf_link_init(&link->link, BPF_LINK_TYPE_UPROBE_MULTI,
|
||||
&bpf_uprobe_multi_link_lops, prog);
|
||||
|
||||
for (i = 0; i < cnt; i++) {
|
||||
err = uprobe_register_refctr(d_real_inode(link->path.dentry),
|
||||
uprobes[i].offset,
|
||||
ref_ctr_offsets ? ref_ctr_offsets[i] : 0,
|
||||
&uprobes[i].consumer);
|
||||
if (err) {
|
||||
bpf_uprobe_unregister(&path, uprobes, i);
|
||||
goto error_free;
|
||||
}
|
||||
}
|
||||
|
||||
err = bpf_link_prime(&link->link, &link_primer);
|
||||
if (err)
|
||||
goto error_free;
|
||||
|
||||
kvfree(ref_ctr_offsets);
|
||||
return bpf_link_settle(&link_primer);
|
||||
|
||||
error_free:
|
||||
kvfree(ref_ctr_offsets);
|
||||
kvfree(uprobes);
|
||||
kfree(link);
|
||||
error_path_put:
|
||||
path_put(&path);
|
||||
return err;
|
||||
}
|
||||
#else /* !CONFIG_UPROBES */
|
||||
int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
|
||||
{
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
#endif /* CONFIG_UPROBES */
|
||||
|
||||
Reference in New Issue
Block a user