[PATCH bpf-next v2 3/9] bpf: trampoline: introduce struct bpf_tramp_link_conn
From: Menglong Dong <hidden>
Date: 2024-03-11 09:34:51
Also in:
bpf, linux-arm-kernel, linux-kselftest, linux-riscv, linux-s390, lkml, netdev
Subsystem:
arm64 port (aarch64 architecture), bpf jit for arm64, bpf jit for risc-v (64-bit), bpf jit for s390, bpf jit for x86 64-bit, bpf [core], bpf [general] (safe dynamic programs and tools), bpf [networking] (struct_ops, reuseport), networking [general], risc-v architecture, s390 architecture, the rest, x86 architecture (32-bit and 64-bit) · Maintainers:
Catalin Marinas, Will Deacon, Daniel Borkmann, Alexei Starovoitov, Puranjay Mohan, Björn Töpel, Ilya Leoshkevich, Heiko Carstens, Vasily Gorbik, Andrii Nakryiko, Eduard Zingerman, Kumar Kartikeya Dwivedi, Martin KaFai Lau, "David S. Miller", Eric Dumazet, Jakub Kicinski, Paolo Abeni, Paul Walmsley, Palmer Dabbelt, Albert Ou, Alexander Gordeev, Linus Torvalds, Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen
For now, bpf_tramp_link is added to the hash list of tr->progs_hlist when attaching. This means that bpf_link and trampoline is one-to-one, and is not friendly to the multi-link trampoline that we commit in the following patches. Therefore, now we introduce the struct bpf_tramp_link_conn to be the bridge between bpf_tramp_link and trampoline. And we also chang the type of links in struct bpf_tramp_links to struct bpf_tramp_link_conn. Signed-off-by: Menglong Dong <redacted> --- arch/arm64/net/bpf_jit_comp.c | 4 ++-- arch/riscv/net/bpf_jit_comp64.c | 4 ++-- arch/s390/net/bpf_jit_comp.c | 4 ++-- arch/x86/net/bpf_jit_comp.c | 4 ++-- include/linux/bpf.h | 12 +++++++--- kernel/bpf/bpf_struct_ops.c | 3 ++- kernel/bpf/syscall.c | 3 ++- kernel/bpf/trampoline.c | 42 +++++++++++++++++---------------- net/bpf/bpf_dummy_struct_ops.c | 1 + 9 files changed, 44 insertions(+), 33 deletions(-)
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index c5b461dda438..b6f7d8a6d372 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c@@ -1810,14 +1810,14 @@ bool bpf_jit_supports_subprog_tailcalls(void) return true; } -static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l, +static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link_conn *l, int args_off, int retval_off, int run_ctx_off, bool save_ret) { __le32 *branch; u64 enter_prog; u64 exit_prog; - struct bpf_prog *p = l->link.prog; + struct bpf_prog *p = l->link->prog; int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); enter_prog = (u64)bpf_trampoline_enter(p);
diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c
index aac190085472..c147053001db 100644
--- a/arch/riscv/net/bpf_jit_comp64.c
+++ b/arch/riscv/net/bpf_jit_comp64.c@@ -700,11 +700,11 @@ static void restore_args(int nregs, int args_off, struct rv_jit_context *ctx) } } -static int invoke_bpf_prog(struct bpf_tramp_link *l, int args_off, int retval_off, +static int invoke_bpf_prog(struct bpf_tramp_link_conn *l, int args_off, int retval_off, int run_ctx_off, bool save_ret, struct rv_jit_context *ctx) { int ret, branch_off; - struct bpf_prog *p = l->link.prog; + struct bpf_prog *p = l->link->prog; int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); if (l->cookie) {
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index b418333bb086..177efbc1b5ec 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c@@ -2243,12 +2243,12 @@ static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val) static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, const struct btf_func_model *m, - struct bpf_tramp_link *tlink, bool save_ret) + struct bpf_tramp_link_conn *tlink, bool save_ret) { struct bpf_jit *jit = &tjit->common; int cookie_off = tjit->run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie); - struct bpf_prog *p = tlink->link.prog; + struct bpf_prog *p = tlink->link->prog; int patch; /*
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index e1390d1e331b..e7f9f987770d 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c@@ -2261,14 +2261,14 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, } static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, - struct bpf_tramp_link *l, int stack_size, + struct bpf_tramp_link_conn *l, int stack_size, int run_ctx_off, bool save_ret, void *image, void *rw_image) { u8 *prog = *pprog; u8 *jmp_insn; int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); - struct bpf_prog *p = l->link.prog; + struct bpf_prog *p = l->link->prog; u64 cookie = l->cookie; /* mov rdi, cookie */
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 997765cdf474..2b5cd6100fc4 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h@@ -56,6 +56,7 @@ struct bpf_token; struct user_namespace; struct super_block; struct inode; +struct bpf_tramp_link; extern struct idr btf_idr; extern spinlock_t btf_idr_lock;
@@ -1090,7 +1091,7 @@ enum { }; struct bpf_tramp_links { - struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS]; + struct bpf_tramp_link_conn *links[BPF_MAX_TRAMP_LINKS]; int nr_links; };
@@ -1597,12 +1598,17 @@ struct bpf_link_ops { struct bpf_map *old_map); }; -struct bpf_tramp_link { - struct bpf_link link; +struct bpf_tramp_link_conn { + struct bpf_link *link; struct hlist_node tramp_hlist; u64 cookie; }; +struct bpf_tramp_link { + struct bpf_link link; + struct bpf_tramp_link_conn conn; +}; + struct bpf_shim_tramp_link { struct bpf_tramp_link link; struct bpf_trampoline *trampoline;
diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c
index 43356faaa057..4fbe2faa80a8 100644
--- a/kernel/bpf/bpf_struct_ops.c
+++ b/kernel/bpf/bpf_struct_ops.c@@ -549,7 +549,7 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, void *image = *_image; int size; - tlinks[BPF_TRAMP_FENTRY].links[0] = link; + tlinks[BPF_TRAMP_FENTRY].links[0] = &link->conn; tlinks[BPF_TRAMP_FENTRY].nr_links = 1; if (model->ret_size > 0)
@@ -710,6 +710,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, err = -ENOMEM; goto reset_unlock; } + link->conn.link = &link->link; bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_link_lops, prog); st_map->links[i] = &link->link;
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 4f230fd1f8e4..d1cd645ef9ac 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c@@ -3339,6 +3339,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, } link = kzalloc(sizeof(*link), GFP_USER); + link->link.conn.link = &link->link.link; if (!link) { err = -ENOMEM; goto out_put_prog;
@@ -3346,7 +3347,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, bpf_link_init(&link->link.link, BPF_LINK_TYPE_TRACING, &bpf_tracing_link_lops, prog); link->attach_type = prog->expected_attach_type; - link->link.cookie = bpf_cookie; + link->link.conn.cookie = bpf_cookie; mutex_lock(&prog->aux->dst_mutex);
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index d382f5ebe06c..cf9b84f785f3 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c@@ -228,9 +228,9 @@ static int register_fentry(struct bpf_trampoline *tr, void *new_addr) static struct bpf_tramp_links * bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg) { - struct bpf_tramp_link *link; + struct bpf_tramp_link_conn *link_conn; + struct bpf_tramp_link_conn **links; struct bpf_tramp_links *tlinks; - struct bpf_tramp_link **links; int kind; *total = 0;
@@ -243,9 +243,9 @@ bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_a *total += tr->progs_cnt[kind]; links = tlinks[kind].links; - hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) { - *ip_arg |= link->link.prog->call_get_func_ip; - *links++ = link; + hlist_for_each_entry(link_conn, &tr->progs_hlist[kind], tramp_hlist) { + *ip_arg |= link_conn->link->prog->call_get_func_ip; + *links++ = link_conn; } } return tlinks;
@@ -521,14 +521,14 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) } } -static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) +static int __bpf_trampoline_link_prog(struct bpf_tramp_link_conn *link, struct bpf_trampoline *tr) { enum bpf_tramp_prog_type kind; - struct bpf_tramp_link *link_exiting; + struct bpf_tramp_link_conn *link_exiting; int err = 0; int cnt = 0, i; - kind = bpf_attach_type_to_tramp(link->link.prog); + kind = bpf_attach_type_to_tramp(link->link->prog); if (tr->extension_prog) /* cannot attach fentry/fexit if extension prog is attached. * cannot overwrite extension prog either.
@@ -542,9 +542,9 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_tr /* Cannot attach extension if fentry/fexit are in use. */ if (cnt) return -EBUSY; - tr->extension_prog = link->link.prog; + tr->extension_prog = link->link->prog; return bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL, - link->link.prog->bpf_func); + link->link->prog->bpf_func); } if (cnt >= BPF_MAX_TRAMP_LINKS) return -E2BIG;
@@ -552,7 +552,7 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_tr /* prog already linked */ return -EBUSY; hlist_for_each_entry(link_exiting, &tr->progs_hlist[kind], tramp_hlist) { - if (link_exiting->link.prog != link->link.prog) + if (link_exiting->link->prog != link->link->prog) continue; /* prog already linked */ return -EBUSY;
@@ -573,17 +573,17 @@ int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline int err; mutex_lock(&tr->mutex); - err = __bpf_trampoline_link_prog(link, tr); + err = __bpf_trampoline_link_prog(&link->conn, tr); mutex_unlock(&tr->mutex); return err; } -static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr) +static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link_conn *link, struct bpf_trampoline *tr) { enum bpf_tramp_prog_type kind; int err; - kind = bpf_attach_type_to_tramp(link->link.prog); + kind = bpf_attach_type_to_tramp(link->link->prog); if (kind == BPF_TRAMP_REPLACE) { WARN_ON_ONCE(!tr->extension_prog); err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP,
@@ -602,7 +602,7 @@ int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampolin int err; mutex_lock(&tr->mutex); - err = __bpf_trampoline_unlink_prog(link, tr); + err = __bpf_trampoline_unlink_prog(&link->conn, tr); mutex_unlock(&tr->mutex); return err; }
@@ -645,6 +645,7 @@ static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog if (!shim_link) return NULL; + shim_link->link.conn.link = &shim_link->link.link; p = bpf_prog_alloc(1, 0); if (!p) { kfree(shim_link);
@@ -672,15 +673,16 @@ static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog static struct bpf_shim_tramp_link *cgroup_shim_find(struct bpf_trampoline *tr, bpf_func_t bpf_func) { - struct bpf_tramp_link *link; + struct bpf_tramp_link_conn *link_conn; int kind; for (kind = 0; kind < BPF_TRAMP_MAX; kind++) { - hlist_for_each_entry(link, &tr->progs_hlist[kind], tramp_hlist) { - struct bpf_prog *p = link->link.prog; + hlist_for_each_entry(link_conn, &tr->progs_hlist[kind], tramp_hlist) { + struct bpf_prog *p = link_conn->link->prog; if (p->bpf_func == bpf_func) - return container_of(link, struct bpf_shim_tramp_link, link); + return container_of((struct bpf_tramp_link *)link_conn->link, + struct bpf_shim_tramp_link, link); } }
@@ -731,7 +733,7 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, goto err; } - err = __bpf_trampoline_link_prog(&shim_link->link, tr); + err = __bpf_trampoline_link_prog(&shim_link->link.conn, tr); if (err) goto err;
diff --git a/net/bpf/bpf_dummy_struct_ops.c b/net/bpf/bpf_dummy_struct_ops.c
index 1b5f812e6972..35a2cf60eef6 100644
--- a/net/bpf/bpf_dummy_struct_ops.c
+++ b/net/bpf/bpf_dummy_struct_ops.c@@ -120,6 +120,7 @@ int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, err = -ENOMEM; goto out; } + link->conn.link = &link->link; /* prog doesn't take the ownership of the reference from caller */ bpf_prog_inc(prog); bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS, &bpf_struct_ops_link_lops, prog);
--
2.39.2