[RFC PATCH v2 net-next 12/12] net: listify jited Generic XDP processing on x86_64
From: Edward Cree <hidden>
Date: 2018-06-26 18:22:33
Subsystem:
bpf jit for x86 64-bit, bpf [core], bpf [general] (safe dynamic programs and tools), networking [general], the rest, x86 architecture (32-bit and 64-bit) · Maintainers:
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, Eduard Zingerman, Kumar Kartikeya Dwivedi, "David S. Miller", Eric Dumazet, Jakub Kicinski, Paolo Abeni, Linus Torvalds, Thomas Gleixner, Ingo Molnar, Borislav Petkov, Dave Hansen
When JITing an eBPF program on x86_64, also JIT a list_func that calls the bpf_func in a loop. Since this is a direct call, it should perform better than indirect-calling bpf_func in a loop. Since getting the percpu 'redirect_info' variable is ugly and magic, pass a pointer into the list_func as a parameter instead of calculating it inside the loop. This is safe because BPF execution is not preemptible, so the percpu variable can't get moved while we're using it. Signed-off-by: Edward Cree <redacted> --- arch/x86/net/bpf_jit_comp.c | 164 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/filter.h | 19 +++-- kernel/bpf/core.c | 18 +++-- net/core/dev.c | 5 +- 4 files changed, 195 insertions(+), 11 deletions(-)
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 2580cd2e98b1..3e06dd79adda 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c@@ -1060,6 +1060,169 @@ struct x64_jit_data { struct jit_context ctx; }; +static void try_do_jit_list(struct bpf_prog *bpf_prog, + unsigned int (*bpf_func)(const void *ctx, + const struct bpf_insn *insn)) +{ + /* list_func takes three arguments: + * struct list_head *list [RDI] + * const struct bpf_insn *insn [RSI] + * const struct redirect_info *percpu_ri [RDX] + * + * Layout of struct bpf_work on x86_64: + * struct list_head { + * struct list_head *next; // 0x0 + * struct list_head *prev; // 0x8 + * } list; // 0x0 + * void *ctx; 0x10 + * struct redirect_info { + * u32 ifindex; // 0x18 + * u32 flags; // 0x1c + * struct bpf_map *map; // 0x20 + * struct bpf_map *map_to_flush; // 0x28 + * unsigned long map_owner; // 0x30 + * } ri; // 0x18 + * unsigned long ret; // 0x38 + * (total size 0x40 bytes) + * + * Desired function body: + * struct redirect_info *ri = percpu_ri; [R12] + * struct bpf_work *work; [RBP] + * + * list_for_each_entry(work, list, list) { + * work->ret = (*bpf_func)(work->ctx, insn); + * work->ri = *ri; + * } + * + * Assembly to emit: + * ; save CSRs + * push %rbx + * push %rbp + * push %r12 + * ; stash pointer to redirect_info + * mov %rdx,%r12 ; ri = percpu_ri + * ; start list + * mov %rdi,%rbp ; head = list + * next: ; while (true) { + * mov (%rbp),%rbx ; rbx = head->next + * cmp %rbx,%rdi ; if (rbx == list) + * je out ; break + * mov %rbx,%rbp ; head = rbx + * push %rdi ; save list + * push %rsi ; save insn (is still arg2) + * ; struct bpf_work *work = head (container_of, but list is first member) + * mov 0x10(%rbp),%rdi; arg1 = work->ctx + * callq bpf_func ; rax = (*bpf_func)(work->ctx, insn) + * mov %rax,0x38(%rbp); work->ret = rax + * ; work->ri = *ri + * mov (%r12),%rdx + * mov %rdx,0x18(%rbp) + * mov 0x8(%r12),%rdx + * mov %rdx,0x20(%rbp) + * mov 0x10(%r12),%rdx + * mov %rdx,0x28(%rbp) + * mov 0x18(%r12),%rdx + * mov %rdx,0x30(%rbp) + * pop %rsi ; restore insn + * pop %rdi ; restore list + * jmp next ; } + * out: + * ; restore CSRs and return + * pop %r12 + * pop %rbp + * pop %rbx + * retq + */ + u8 *image, *prog, *from_to_out, *next; + struct bpf_binary_header *header; + int off, cnt = 0; + s64 jmp_offset; + + /* Prog should be 81 bytes; let's round up to 128 */ + header = bpf_jit_binary_alloc(128, &image, 1, jit_fill_hole); + prog = image; + + /* push rbx */ + EMIT1(0x53); + /* push rbp */ + EMIT1(0x55); + /* push %r12 */ + EMIT2(0x41, 0x54); + /* mov %rdx,%r12 */ + EMIT3(0x49, 0x89, 0xd4); + /* mov %rdi,%rbp */ + EMIT3(0x48, 0x89, 0xfd); + next = prog; + /* mov 0x0(%rbp),%rbx */ + EMIT4(0x48, 0x8b, 0x5d, 0x00); + /* cmp %rbx,%rdi */ + EMIT3(0x48, 0x39, 0xdf); + /* je out */ + EMIT2(X86_JE, 0); + from_to_out = prog; /* record . to patch this jump later */ + /* mov %rbx,%rbp */ + EMIT3(0x48, 0x89, 0xdd); + /* push %rdi */ + EMIT1(0x57); + /* push %rsi */ + EMIT1(0x56); + /* mov 0x10(%rbp),%rdi */ + EMIT4(0x48, 0x8b, 0x7d, 0x10); + /* e8 callq bpf_func */ + jmp_offset = (u8 *)bpf_func - (prog + 5); + if (!is_simm32(jmp_offset)) { + pr_err("call out of range to BPF func %p from list image %p\n", + bpf_func, image); + goto fail; + } + EMIT1_off32(0xE8, jmp_offset); + /* mov %rax,0x38(%rbp) */ + EMIT4(0x48, 0x89, 0x45, 0x38); + /* mov (%r12),%rdx */ + EMIT4(0x49, 0x8b, 0x14, 0x24); + /* mov %rdx,0x18(%rbp) */ + EMIT4(0x48, 0x89, 0x55, 0x18); + for (off = 0x8; off < 0x20; off += 0x8) { + /* mov off(%r12),%rdx */ + EMIT4(0x49, 0x8b, 0x54, 0x24); + EMIT1(off); + /* mov %rdx,0x18+off(%rbp) */ + EMIT4(0x48, 0x89, 0x55, 0x18 + off); + } + /* pop %rsi */ + EMIT1(0x5e); + /* pop %rdi */ + EMIT1(0x5f); + /* jmp next */ + jmp_offset = next - (prog + 2); + if (WARN_ON(!is_imm8(jmp_offset))) /* can't happen */ + goto fail; + EMIT2(0xeb, jmp_offset); + /* out: */ + jmp_offset = prog - from_to_out; + if (WARN_ON(!is_imm8(jmp_offset))) /* can't happen */ + goto fail; + from_to_out[-1] = jmp_offset; + /* pop %r12 */ + EMIT2(0x41, 0x5c); + /* pop %rbp */ + EMIT1(0x5d); + /* pop %rbx */ + EMIT1(0x5b); + /* retq */ + EMIT1(0xc3); + /* If we were wrong about how much space we needed, scream and shout */ + WARN_ON(cnt != 81); + if (bpf_jit_enable > 1) + bpf_jit_dump(0, cnt, 0, image); + bpf_jit_binary_lock_ro(header); + bpf_prog->list_func = (void *)image; + bpf_prog->jited_list = 1; + return; +fail: + bpf_jit_binary_free(header); +} + struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { struct bpf_binary_header *header = NULL;
@@ -1176,6 +1339,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) prog->bpf_func = (void *)image; prog->jited = 1; prog->jited_len = proglen; + try_do_jit_list(prog, prog->bpf_func); } else { prog = orig_prog; }
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 7d813034e286..ad1e75bf0991 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h@@ -517,7 +517,8 @@ struct bpf_prog { const struct bpf_insn *insn); /* Takes a list of struct bpf_work */ void (*list_func)(struct list_head *list, - const struct bpf_insn *insn); + const struct bpf_insn *insn, + const struct redirect_info *percpu_ri); /* Instructions for interpreter */ union { struct sock_filter insns[0];
@@ -532,7 +533,7 @@ struct sk_filter { }; #define BPF_PROG_RUN(filter, ctx) (*(filter)->bpf_func)(ctx, (filter)->insnsi) -#define BPF_LIST_PROG_RUN(filter, list) (*(filter)->list_func)(list, (filter)->insnsi) +#define BPF_LIST_PROG_RUN(filter, list, percpu) (*(filter)->list_func)(list, (filter)->insnsi, percpu) #define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN
@@ -638,10 +639,11 @@ static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog, } static __always_inline void bpf_list_prog_run_xdp(const struct bpf_prog *prog, - struct list_head *list) + struct list_head *list, + const struct redirect_info *percpu_ri) { /* Caller must hold rcu_read_lock(), as per bpf_prog_run_xdp(). */ - BPF_LIST_PROG_RUN(prog, list); + BPF_LIST_PROG_RUN(prog, list, percpu_ri); } static inline u32 bpf_prog_insn_size(const struct bpf_prog *prog)
@@ -756,6 +758,15 @@ bpf_jit_binary_hdr(const struct bpf_prog *fp) return (void *)addr; } +static inline struct bpf_binary_header * +bpf_list_jit_binary_hdr(const struct bpf_prog *fp) +{ + unsigned long real_start = (unsigned long)fp->list_func; + unsigned long addr = real_start & PAGE_MASK; + + return (void *)addr; +} + #ifdef CONFIG_ARCH_HAS_SET_MEMORY static inline int bpf_prog_check_pages_ro_single(const struct bpf_prog *fp) {
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index c35da826cc3b..028be88c4af8 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c@@ -621,15 +621,22 @@ void bpf_jit_binary_free(struct bpf_binary_header *hdr) */ void __weak bpf_jit_free(struct bpf_prog *fp) { - if (fp->jited) { - struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp); + struct bpf_binary_header *hdr; + if (fp->jited) { + hdr = bpf_jit_binary_hdr(fp); bpf_jit_binary_unlock_ro(hdr); bpf_jit_binary_free(hdr); WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); } + if (fp->jited_list) { + hdr = bpf_list_jit_binary_hdr(fp); + bpf_jit_binary_unlock_ro(hdr); + bpf_jit_binary_free(hdr); + } + bpf_prog_unlock_free(fp); }
@@ -1358,13 +1365,13 @@ static u64 PROG_NAME_ARGS(stack_size)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, \ #define LIST_PROG_NAME(stack_size) __bpf_list_prog_run##stack_size #define DEFINE_BPF_LIST_PROG_RUN(stack_size) \ -static void LIST_PROG_NAME(stack_size)(struct list_head *list, const struct bpf_insn *insn) \ +static void LIST_PROG_NAME(stack_size)(struct list_head *list, const struct bpf_insn *insn, const struct redirect_info *percpu_ri) \ { \ struct bpf_work *work; \ \ list_for_each_entry(work, list, list) { \ work->ret = PROG_NAME(stack_size)(work->ctx, insn); \ - work->ri = *this_cpu_ptr(&redirect_info); \ + work->ri = *percpu_ri; \ } \ }
@@ -1398,7 +1405,8 @@ EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) #undef PROG_NAME_LIST #define PROG_NAME_LIST(stack_size) LIST_PROG_NAME(stack_size), static void (*list_interpreters[])(struct list_head *list, - const struct bpf_insn *insn) = { + const struct bpf_insn *insn, + const struct redirect_info *percpu_ri) = { EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192) EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384) EVAL4(PROG_NAME_LIST, 416, 448, 480, 512)
diff --git a/net/core/dev.c b/net/core/dev.c
index 746112c22afd..7c1879045ef8 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c@@ -4214,6 +4214,7 @@ static void do_xdp_list_generic(struct bpf_prog *xdp_prog, struct sk_buff_head *list, struct sk_buff_head *pass_list) { + const struct redirect_info *percpu_ri = this_cpu_ptr(&redirect_info); struct xdp_work (*xwa)[NAPI_POLL_WEIGHT], *xw; struct bpf_work *bw; struct sk_buff *skb;
@@ -4249,11 +4250,11 @@ static void do_xdp_list_generic(struct bpf_prog *xdp_prog, if (xdp_prog->list_func && (xdp_prog->jited_list || !xdp_prog->jited)) - bpf_list_prog_run_xdp(xdp_prog, &xdp_list); + bpf_list_prog_run_xdp(xdp_prog, &xdp_list, percpu_ri); else list_for_each_entry(bw, &xdp_list, list) { bw->ret = bpf_prog_run_xdp(xdp_prog, bw->ctx); - bw->ri = *this_cpu_ptr(&redirect_info); + bw->ri = *percpu_ri; } for (i = 0; i < n; i++) {