[PATCH bpf-next 1/2] bpf: Remove bpf_jit_enable=2 debugging mode

For debugging JITs, dumping the JITed image to kernel log is discouraged,
"bpftool prog dump jited" is much better way to examine JITed dumps.
This patch get rid of the code related to bpf_jit_enable=2 mode and
update the proc handler of bpf_jit_enable, also added auxiliary
information to explain how to use bpf_jit_disasm tool after this change.

Signed-off-by: Jianlin Lv <redacted>
---
 arch/arm/net/bpf_jit_32.c         |  4 ----
 arch/arm64/net/bpf_jit_comp.c     |  4 ----
 arch/mips/net/bpf_jit.c           |  4 ----
 arch/mips/net/ebpf_jit.c          |  4 ----
 arch/powerpc/net/bpf_jit_comp.c   | 10 ----------
 arch/powerpc/net/bpf_jit_comp64.c | 11 -----------
 arch/riscv/net/bpf_jit_core.c     |  3 ---
 arch/s390/net/bpf_jit_comp.c      |  4 ----
 arch/sparc/net/bpf_jit_comp_32.c  |  3 ---
 arch/sparc/net/bpf_jit_comp_64.c  | 13 -------------
 arch/x86/net/bpf_jit_comp.c       |  3 ---
 arch/x86/net/bpf_jit_comp32.c     |  3 ---
 net/core/sysctl_net_core.c        | 14 +++-----------
 tools/bpf/bpf_jit_disasm.c        |  2 +-
 tools/bpf/bpftool/feature.c       |  3 ---
 15 files changed, 4 insertions(+), 81 deletions(-)

diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index 897634d0a67c..92d669c0b2d3 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c

@@ -1997,10 +1997,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	}
 	flush_icache_range((u32)header, (u32)(ctx.target + ctx.idx));
 
-	if (bpf_jit_enable > 1)
-		/* there are 2 passes here */
-		bpf_jit_dump(prog->len, image_size, 2, ctx.target);
-
 	bpf_jit_binary_lock_ro(header);
 	prog->bpf_func = (void *)ctx.target;
 	prog->jited = 1;

diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index f7b194878a99..a13b83ac4ca8 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c

@@ -1090,10 +1090,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		goto out_off;
 	}
 
-	/* And we're done. */
-	if (bpf_jit_enable > 1)
-		bpf_jit_dump(prog->len, prog_size, 2, ctx.image);
-
 	bpf_flush_icache(header, ctx.image + ctx.idx);
 
 	if (!prog->is_func || extra_pass) {

diff --git a/arch/mips/net/bpf_jit.c b/arch/mips/net/bpf_jit.c
index 0af88622c619..b5221282dd88 100644
--- a/arch/mips/net/bpf_jit.c
+++ b/arch/mips/net/bpf_jit.c

@@ -1250,10 +1250,6 @@ void bpf_jit_compile(struct bpf_prog *fp)
 	/* Update the icache */
 	flush_icache_range((ptr)ctx.target, (ptr)(ctx.target + ctx.idx));
 
-	if (bpf_jit_enable > 1)
-		/* Dump JIT code */
-		bpf_jit_dump(fp->len, alloc_size, 2, ctx.target);
-
 	fp->bpf_func = (void *)ctx.target;
 	fp->jited = 1;
 

diff --git a/arch/mips/net/ebpf_jit.c b/arch/mips/net/ebpf_jit.c
index 939dd06764bc..dac5a1fc2462 100644
--- a/arch/mips/net/ebpf_jit.c
+++ b/arch/mips/net/ebpf_jit.c

@@ -1910,10 +1910,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	flush_icache_range((unsigned long)ctx.target,
 			   (unsigned long)&ctx.target[ctx.idx]);
 
-	if (bpf_jit_enable > 1)
-		/* Dump JIT code */
-		bpf_jit_dump(prog->len, image_size, 2, ctx.target);
-
 	bpf_jit_binary_lock_ro(header);
 	prog->bpf_func = (void *)ctx.target;
 	prog->jited = 1;

diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index e809cb5a1631..ebca629de2d1 100644
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c

@@ -646,18 +646,8 @@ void bpf_jit_compile(struct bpf_prog *fp)
 		bpf_jit_build_prologue(fp, code_base, &cgctx);
 		bpf_jit_build_body(fp, code_base, &cgctx, addrs);
 		bpf_jit_build_epilogue(code_base, &cgctx);
-
-		if (bpf_jit_enable > 1)
-			pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
-				proglen - (cgctx.idx * 4), cgctx.seen);
 	}
 
-	if (bpf_jit_enable > 1)
-		/* Note that we output the base address of the code_base
-		 * rather than image, since opcodes are in code_base.
-		 */
-		bpf_jit_dump(flen, proglen, pass, code_base);
-
 	bpf_flush_icache(code_base, code_base + (proglen/4));
 
 #ifdef CONFIG_PPC64

diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index aaf1a887f653..26243399ef2e 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c

@@ -1215,20 +1215,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 		bpf_jit_build_prologue(code_base, &cgctx);
 		bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass);
 		bpf_jit_build_epilogue(code_base, &cgctx);
-
-		if (bpf_jit_enable > 1)
-			pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass,
-				proglen - (cgctx.idx * 4), cgctx.seen);
 	}
 
 skip_codegen_passes:
-	if (bpf_jit_enable > 1)
-		/*
-		 * Note that we output the base address of the code_base
-		 * rather than image, since opcodes are in code_base.
-		 */
-		bpf_jit_dump(flen, proglen, pass, code_base);
-
 #ifdef PPC64_ELF_ABI_v1
 	/* Function descriptor nastiness: Address + TOC */
 	((u64 *)image)[0] = (u64)code_base;

diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c
index 3630d447352c..856b84fb3947 100644
--- a/arch/riscv/net/bpf_jit_core.c
+++ b/arch/riscv/net/bpf_jit_core.c

@@ -142,9 +142,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	}
 	bpf_jit_build_epilogue(ctx);
 
-	if (bpf_jit_enable > 1)
-		bpf_jit_dump(prog->len, image_size, pass, ctx->insns);
-
 	prog->bpf_func = (void *)ctx->insns;
 	prog->jited = 1;
 	prog->jited_len = image_size;

diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 63cae0476bb4..aa8b94ba694f 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c

@@ -1842,10 +1842,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 		fp = orig_fp;
 		goto free_addrs;
 	}
-	if (bpf_jit_enable > 1) {
-		bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);
-		print_fn_code(jit.prg_buf, jit.size_prg);
-	}
 	if (!fp->is_func || extra_pass) {
 		bpf_jit_binary_lock_ro(header);
 	} else {

diff --git a/arch/sparc/net/bpf_jit_comp_32.c b/arch/sparc/net/bpf_jit_comp_32.c
index b1dbf2fa8c0a..cb4c55422730 100644
--- a/arch/sparc/net/bpf_jit_comp_32.c
+++ b/arch/sparc/net/bpf_jit_comp_32.c

@@ -743,9 +743,6 @@ cond_branch:			f_offset = addrs[i + filter[i].jf];
 		oldproglen = proglen;
 	}
 
-	if (bpf_jit_enable > 1)
-		bpf_jit_dump(flen, proglen, pass + 1, image);
-
 	if (image) {
 		fp->bpf_func = (void *)image;
 		fp->jited = 1;

diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c
index 4b8d3c65d266..09ebd48c4f1b 100644
--- a/arch/sparc/net/bpf_jit_comp_64.c
+++ b/arch/sparc/net/bpf_jit_comp_64.c

@@ -1546,16 +1546,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		}
 		build_epilogue(&ctx);
 
-		if (bpf_jit_enable > 1)
-			pr_info("Pass %d: size = %u, seen = [%c%c%c%c%c%c]\n", pass,
-				ctx.idx * 4,
-				ctx.tmp_1_used ? '1' : ' ',
-				ctx.tmp_2_used ? '2' : ' ',
-				ctx.tmp_3_used ? '3' : ' ',
-				ctx.saw_frame_pointer ? 'F' : ' ',
-				ctx.saw_call ? 'C' : ' ',
-				ctx.saw_tail_call ? 'T' : ' ');
-
 		if (ctx.idx * 4 == prev_image_size)
 			break;
 		prev_image_size = ctx.idx * 4;

@@ -1593,9 +1583,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		goto out_off;
 	}
 
-	if (bpf_jit_enable > 1)
-		bpf_jit_dump(prog->len, image_size, pass, ctx.image);
-
 	bpf_flush_icache(header, (u8 *)header + (header->pages * PAGE_SIZE));
 
 	if (!prog->is_func || extra_pass) {

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 9eead60f0301..0a511f42a2a7 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c

@@ -2311,9 +2311,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		cond_resched();
 	}
 
-	if (bpf_jit_enable > 1)
-		bpf_jit_dump(prog->len, proglen, pass + 1, image);
-
 	if (image) {
 		if (!prog->is_func || extra_pass) {
 			bpf_tail_call_direct_fixup(prog);

diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
index 0a7a2870f111..8d36b4658076 100644
--- a/arch/x86/net/bpf_jit_comp32.c
+++ b/arch/x86/net/bpf_jit_comp32.c

@@ -2566,9 +2566,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 		cond_resched();
 	}
 
-	if (bpf_jit_enable > 1)
-		bpf_jit_dump(prog->len, proglen, pass + 1, image);
-
 	if (image) {
 		bpf_jit_binary_lock_ro(header);
 		prog->bpf_func = (void *)image;

diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index c8496c1142c9..990b1720c7a4 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c

@@ -273,16 +273,8 @@ static int proc_dointvec_minmax_bpf_enable(struct ctl_table *table, int write,
 
 	tmp.data = &jit_enable;
 	ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
-	if (write && !ret) {
-		if (jit_enable < 2 ||
-		    (jit_enable == 2 && bpf_dump_raw_ok(current_cred()))) {
-			*(int *)table->data = jit_enable;
-			if (jit_enable == 2)
-				pr_warn("bpf_jit_enable = 2 was set! NEVER use this in production, only for JIT debugging!\n");
-		} else {
-			ret = -EPERM;
-		}
-	}
+	if (write && !ret)
+		*(int *)table->data = jit_enable;
 	return ret;
 }
 

@@ -389,7 +381,7 @@ static struct ctl_table net_core_table[] = {
 		.extra2		= SYSCTL_ONE,
 # else
 		.extra1		= SYSCTL_ZERO,
-		.extra2		= &two,
+		.extra2		= SYSCTL_ONE,
 # endif
 	},
 # ifdef CONFIG_HAVE_EBPF_JIT

diff --git a/tools/bpf/bpf_jit_disasm.c b/tools/bpf/bpf_jit_disasm.c
index c8ae95804728..efa4b17ae016 100644
--- a/tools/bpf/bpf_jit_disasm.c
+++ b/tools/bpf/bpf_jit_disasm.c

@@ -7,7 +7,7 @@
  *
  * To get the disassembly of the JIT code, do the following:
  *
- *  1) `echo 2 > /proc/sys/net/core/bpf_jit_enable`
+ *  1) Insert bpf_jit_dump() and recompile the kernel to output JITed image into log
  *  2) Load a BPF filter (e.g. `tcpdump -p -n -s 0 -i eth1 host 192.168.20.0/24`)
  *  3) Run e.g. `bpf_jit_disasm -o` to read out the last JIT code
  *

diff --git a/tools/bpf/bpftool/feature.c b/tools/bpf/bpftool/feature.c
index 40a88df275f9..98c7eec2923f 100644
--- a/tools/bpf/bpftool/feature.c
+++ b/tools/bpf/bpftool/feature.c

@@ -203,9 +203,6 @@ static void probe_jit_enable(void)
 		case 1:
 			printf("JIT compiler is enabled\n");
 			break;
-		case 2:
-			printf("JIT compiler is enabled with debugging traces in kernel logs\n");
-			break;
 		case -1:
 			printf("Unable to retrieve JIT-compiler status\n");
 			break;

-- 
2.25.1

Remove information about bpf_jit_enable=2 mode and added description for
how to use the bpf_jit_disasm tool after get rid of =2 mode.

Signed-off-by: Jianlin Lv <redacted>
---
 Documentation/admin-guide/sysctl/net.rst |  1 -
 Documentation/networking/filter.rst      | 25 ++++++------------------
 2 files changed, 6 insertions(+), 20 deletions(-)

diff --git a/Documentation/admin-guide/sysctl/net.rst b/Documentation/admin-guide/sysctl/net.rst
index c941b214e0b7..a39f99deac38 100644
--- a/Documentation/admin-guide/sysctl/net.rst
+++ b/Documentation/admin-guide/sysctl/net.rst

@@ -86,7 +86,6 @@ Values:
 
 	- 0 - disable the JIT (default value)
 	- 1 - enable the JIT
-	- 2 - enable the JIT and ask the compiler to emit traces on kernel log.
 
 bpf_jit_harden
 --------------

diff --git a/Documentation/networking/filter.rst b/Documentation/networking/filter.rst
index 251c6bd73d15..86954f922168 100644
--- a/Documentation/networking/filter.rst
+++ b/Documentation/networking/filter.rst

@@ -504,25 +504,12 @@ been previously enabled by root::
 
   echo 1 > /proc/sys/net/core/bpf_jit_enable
 
-For JIT developers, doing audits etc, each compile run can output the generated
-opcode image into the kernel log via::
-
-  echo 2 > /proc/sys/net/core/bpf_jit_enable
-
-Example output from dmesg::
-
-    [ 3389.935842] flen=6 proglen=70 pass=3 image=ffffffffa0069c8f
-    [ 3389.935847] JIT code: 00000000: 55 48 89 e5 48 83 ec 60 48 89 5d f8 44 8b 4f 68
-    [ 3389.935849] JIT code: 00000010: 44 2b 4f 6c 4c 8b 87 d8 00 00 00 be 0c 00 00 00
-    [ 3389.935850] JIT code: 00000020: e8 1d 94 ff e0 3d 00 08 00 00 75 16 be 17 00 00
-    [ 3389.935851] JIT code: 00000030: 00 e8 28 94 ff e0 83 f8 01 75 07 b8 ff ff 00 00
-    [ 3389.935852] JIT code: 00000040: eb 02 31 c0 c9 c3
-
-When CONFIG_BPF_JIT_ALWAYS_ON is enabled, bpf_jit_enable is permanently set to 1 and
-setting any other value than that will return in failure. This is even the case for
-setting bpf_jit_enable to 2, since dumping the final JIT image into the kernel log
-is discouraged and introspection through bpftool (under tools/bpf/bpftool/) is the
-generally recommended approach instead.
+When CONFIG_BPF_JIT_ALWAYS_ON is enabled, bpf_jit_enable is permanently set
+to 1 and setting any other value than that will return in failure.
+For debugging JITs, the introspection through bpftool (tools/bpf/bpftool/)
+is the generally recommended approach instead. For JIT developers, doing
+audits etc, you can insert bpf_jit_dump() and recompile the kernel to
+output the generated opcode image into the kernel log.
 
 In the kernel source tree under tools/bpf/, there's bpf_jit_disasm for
 generating disassembly out of the kernel log's hexdump::

-- 
2.25.1

On 4/15/21 11:32 AM, Jianlin Lv wrote:

For debugging JITs, dumping the JITed image to kernel log is discouraged,
"bpftool prog dump jited" is much better way to examine JITed dumps.
This patch get rid of the code related to bpf_jit_enable=2 mode and
update the proc handler of bpf_jit_enable, also added auxiliary
information to explain how to use bpf_jit_disasm tool after this change.

Signed-off-by: Jianlin Lv <redacted>

[...]

Hmm, if we remove bpf_jit_dump(), the next drive-by cleanup patch will be thrown
at bpf@vger stating that bpf_jit_dump() has no in-tree users and should be removed.
Maybe we should be removing bpf_jit_disasm.c along with it as well as bpf_jit_dump()
itself ... I guess if it's ever needed in those rare occasions for JIT debugging we
can resurrect it from old kernels just locally. But yeah, bpftool's jit dump should
suffice for vast majority of use cases.

There was a recent set for ppc32 jit which was merged into ppc tree which will create
a merge conflict with this one [0]. So we would need a rebase and take it maybe during
merge win once the ppc32 landed..

   [0] https://lore.kernel.org/bpf/cover.1616430991.git.christophe.leroy@csgroup.eu/

This would still need to be there for older kernels ...

  		case -1:
  			printf("Unable to retrieve JIT-compiler status\n");
  			break;

2021-04-15 16:37 UTC+0200 ~ Daniel Borkmann [off-list ref]

On 4/15/21 11:32 AM, Jianlin Lv wrote:

quoted

For debugging JITs, dumping the JITed image to kernel log is discouraged,
"bpftool prog dump jited" is much better way to examine JITed dumps.
This patch get rid of the code related to bpf_jit_enable=2 mode and
update the proc handler of bpf_jit_enable, also added auxiliary
information to explain how to use bpf_jit_disasm tool after this change.

Signed-off-by: Jianlin Lv <redacted>

Hello,

For what it's worth, I have already seen people dump the JIT image in
kernel logs in Qemu VMs running with just a busybox, not for kernel
development, but in a context where buiding/using bpftool was not
possible. Maybe not a common case, but still, removing the debugging
mode will make that impossible. Is there a particular incentive to
remove the feature?

Best regards,
Quentin

On Thu, Apr 15, 2021 at 8:41 AM Quentin Monnet [off-list ref] wrote:

2021-04-15 16:37 UTC+0200 ~ Daniel Borkmann [off-list ref]

quoted

On 4/15/21 11:32 AM, Jianlin Lv wrote:

quoted

For debugging JITs, dumping the JITed image to kernel log is discouraged,
"bpftool prog dump jited" is much better way to examine JITed dumps.
This patch get rid of the code related to bpf_jit_enable=2 mode and
update the proc handler of bpf_jit_enable, also added auxiliary
information to explain how to use bpf_jit_disasm tool after this change.

Signed-off-by: Jianlin Lv <redacted>

Hello,

For what it's worth, I have already seen people dump the JIT image in
kernel logs in Qemu VMs running with just a busybox, not for kernel
development, but in a context where buiding/using bpftool was not
possible.

If building/using bpftool is not possible then majority of selftests won't
be exercised. I don't think such environment is suitable for any kind
of bpf development. Much so for JIT debugging.
While bpf_jit_enable=2 is nothing but the debugging tool for JIT developers.
I'd rather nuke that code instead of carrying it from kernel to kernel.

Le 16/04/2021 à 01:49, Alexei Starovoitov a écrit :

On Thu, Apr 15, 2021 at 8:41 AM Quentin Monnet [off-list ref] wrote:

quoted

2021-04-15 16:37 UTC+0200 ~ Daniel Borkmann [off-list ref]

quoted

On 4/15/21 11:32 AM, Jianlin Lv wrote:

quoted

For debugging JITs, dumping the JITed image to kernel log is discouraged,
"bpftool prog dump jited" is much better way to examine JITed dumps.
This patch get rid of the code related to bpf_jit_enable=2 mode and
update the proc handler of bpf_jit_enable, also added auxiliary
information to explain how to use bpf_jit_disasm tool after this change.

Signed-off-by: Jianlin Lv <redacted>

Hello,

For what it's worth, I have already seen people dump the JIT image in
kernel logs in Qemu VMs running with just a busybox, not for kernel
development, but in a context where buiding/using bpftool was not
possible.

If building/using bpftool is not possible then majority of selftests won't
be exercised. I don't think such environment is suitable for any kind
of bpf development. Much so for JIT debugging.
While bpf_jit_enable=2 is nothing but the debugging tool for JIT developers.
I'd rather nuke that code instead of carrying it from kernel to kernel.

When I implemented JIT for PPC32, it was extremely helpfull.

As far as I understand, for the time being bpftool is not usable in my environment because it 
doesn't support cross compilation when the target's endianess differs from the building host 
endianess, see discussion at 
https://lore.kernel.org/bpf/21e66a09-514f-f426-b9e2-13baab0b938b@csgroup.eu/

That's right that selftests can't be exercised because they don't build.

The question might be candid as I didn't investigate much about the replacement of "bpf_jit_enable=2 
debugging mode" by bpftool, how do we use bpftool exactly for that ? Especially when using the BPF 
test module ?

On Sat, Apr 17, 2021 at 1:16 AM Christophe Leroy
[off-list ref] wrote:

Le 16/04/2021 à 01:49, Alexei Starovoitov a écrit :

quoted

On Thu, Apr 15, 2021 at 8:41 AM Quentin Monnet [off-list ref] wrote:

quoted

2021-04-15 16:37 UTC+0200 ~ Daniel Borkmann [off-list ref]

quoted

On 4/15/21 11:32 AM, Jianlin Lv wrote:

quoted

For debugging JITs, dumping the JITed image to kernel log is discouraged,
"bpftool prog dump jited" is much better way to examine JITed dumps.
This patch get rid of the code related to bpf_jit_enable=2 mode and
update the proc handler of bpf_jit_enable, also added auxiliary
information to explain how to use bpf_jit_disasm tool after this change.

Signed-off-by: Jianlin Lv <redacted>

Hello,

For what it's worth, I have already seen people dump the JIT image in
kernel logs in Qemu VMs running with just a busybox, not for kernel
development, but in a context where buiding/using bpftool was not
possible.

If building/using bpftool is not possible then majority of selftests won't
be exercised. I don't think such environment is suitable for any kind
of bpf development. Much so for JIT debugging.
While bpf_jit_enable=2 is nothing but the debugging tool for JIT developers.
I'd rather nuke that code instead of carrying it from kernel to kernel.

When I implemented JIT for PPC32, it was extremely helpfull.

As far as I understand, for the time being bpftool is not usable in my environment because it
doesn't support cross compilation when the target's endianess differs from the building host
endianess, see discussion at
https://lore.kernel.org/bpf/21e66a09-514f-f426-b9e2-13baab0b938b@csgroup.eu/

That's right that selftests can't be exercised because they don't build.

The question might be candid as I didn't investigate much about the replacement of "bpf_jit_enable=2
debugging mode" by bpftool, how do we use bpftool exactly for that ? Especially when using the BPF
test module ?

the kernel developers can add any amount of printk and dumps to debug
their code,
but such debugging aid should not be part of the production kernel.
That sysctl was two things at once: debugging tool for kernel devs and
introspection for users.
bpftool jit dump solves the 2nd part. It provides JIT introspection to users.
Debugging of the kernel can be done with any amount of auxiliary code
including calling print_hex_dump() during jiting.

Le 20/04/2021 à 05:28, Alexei Starovoitov a écrit :

On Sat, Apr 17, 2021 at 1:16 AM Christophe Leroy
[off-list ref] wrote:

quoted

Le 16/04/2021 à 01:49, Alexei Starovoitov a écrit :

quoted

On Thu, Apr 15, 2021 at 8:41 AM Quentin Monnet [off-list ref] wrote:

quoted

2021-04-15 16:37 UTC+0200 ~ Daniel Borkmann [off-list ref]

quoted

On 4/15/21 11:32 AM, Jianlin Lv wrote:

quoted

For debugging JITs, dumping the JITed image to kernel log is discouraged,
"bpftool prog dump jited" is much better way to examine JITed dumps.
This patch get rid of the code related to bpf_jit_enable=2 mode and
update the proc handler of bpf_jit_enable, also added auxiliary
information to explain how to use bpf_jit_disasm tool after this change.

Signed-off-by: Jianlin Lv <redacted>

Hello,

For what it's worth, I have already seen people dump the JIT image in
kernel logs in Qemu VMs running with just a busybox, not for kernel
development, but in a context where buiding/using bpftool was not
possible.

If building/using bpftool is not possible then majority of selftests won't
be exercised. I don't think such environment is suitable for any kind
of bpf development. Much so for JIT debugging.
While bpf_jit_enable=2 is nothing but the debugging tool for JIT developers.
I'd rather nuke that code instead of carrying it from kernel to kernel.

When I implemented JIT for PPC32, it was extremely helpfull.

As far as I understand, for the time being bpftool is not usable in my environment because it
doesn't support cross compilation when the target's endianess differs from the building host
endianess, see discussion at
https://lore.kernel.org/bpf/21e66a09-514f-f426-b9e2-13baab0b938b@csgroup.eu/

That's right that selftests can't be exercised because they don't build.

The question might be candid as I didn't investigate much about the replacement of "bpf_jit_enable=2
debugging mode" by bpftool, how do we use bpftool exactly for that ? Especially when using the BPF
test module ?

the kernel developers can add any amount of printk and dumps to debug
their code,
but such debugging aid should not be part of the production kernel.
That sysctl was two things at once: debugging tool for kernel devs and
introspection for users.
bpftool jit dump solves the 2nd part. It provides JIT introspection to users.
Debugging of the kernel can be done with any amount of auxiliary code
including calling print_hex_dump() during jiting.

I get the following message when trying the command suggested in the patch message:

root@vgoip:~# ./bpftool prog dump jited
Error: No libbfd support

Christophe

2021-04-21 15:10 UTC+0200 ~ Christophe Leroy [off-list ref]

Le 20/04/2021 à 05:28, Alexei Starovoitov a écrit :

quoted

On Sat, Apr 17, 2021 at 1:16 AM Christophe Leroy
[off-list ref] wrote:

quoted

Le 16/04/2021 à 01:49, Alexei Starovoitov a écrit :

quoted

On Thu, Apr 15, 2021 at 8:41 AM Quentin Monnet
[off-list ref] wrote:

quoted

2021-04-15 16:37 UTC+0200 ~ Daniel Borkmann [off-list ref]

quoted

On 4/15/21 11:32 AM, Jianlin Lv wrote:

quoted

For debugging JITs, dumping the JITed image to kernel log is
discouraged,
"bpftool prog dump jited" is much better way to examine JITed dumps.
This patch get rid of the code related to bpf_jit_enable=2 mode and
update the proc handler of bpf_jit_enable, also added auxiliary
information to explain how to use bpf_jit_disasm tool after this
change.

Signed-off-by: Jianlin Lv <redacted>

Hello,

For what it's worth, I have already seen people dump the JIT image in
kernel logs in Qemu VMs running with just a busybox, not for kernel
development, but in a context where buiding/using bpftool was not
possible.

If building/using bpftool is not possible then majority of selftests
won't
be exercised. I don't think such environment is suitable for any kind
of bpf development. Much so for JIT debugging.
While bpf_jit_enable=2 is nothing but the debugging tool for JIT
developers.
I'd rather nuke that code instead of carrying it from kernel to kernel.

When I implemented JIT for PPC32, it was extremely helpfull.

As far as I understand, for the time being bpftool is not usable in
my environment because it
doesn't support cross compilation when the target's endianess differs
from the building host
endianess, see discussion at
https://lore.kernel.org/bpf/21e66a09-514f-f426-b9e2-13baab0b938b@csgroup.eu/


That's right that selftests can't be exercised because they don't build.

The question might be candid as I didn't investigate much about the
replacement of "bpf_jit_enable=2
debugging mode" by bpftool, how do we use bpftool exactly for that ?
Especially when using the BPF
test module ?

the kernel developers can add any amount of printk and dumps to debug
their code,
but such debugging aid should not be part of the production kernel.
That sysctl was two things at once: debugging tool for kernel devs and
introspection for users.
bpftool jit dump solves the 2nd part. It provides JIT introspection to
users.
Debugging of the kernel can be done with any amount of auxiliary code
including calling print_hex_dump() during jiting.

I get the following message when trying the command suggested in the
patch message:

root@vgoip:~# ./bpftool prog dump jited
Error: No libbfd support

Christophe

Hi Christophe,

Bpftool relies on libbfd to disassemble the JIT-ed instructions, but
this is an optional dependency and your version of bpftool has been
compiled without it.

You could try to install it on your system (it is usually shipped with
binutils, package "binutils-dev" on Ubuntu for example). If you want to
cross-compile bpftool, the libbfd version provided by your distribution
may not include support for the target architecture. In that case you
would have to build libbfd yourself to make sure it supports it.

Then you can clean up the results from the libbfd probing:

    $ make -C tools/build/feature/ clean

and recompile bpftool.

I hope this helps,
Quentin

Christophe Leroy wrote:

Le 20/04/2021 à 05:28, Alexei Starovoitov a écrit :

quoted

On Sat, Apr 17, 2021 at 1:16 AM Christophe Leroy
[off-list ref] wrote:

quoted

Le 16/04/2021 à 01:49, Alexei Starovoitov a écrit :

quoted

On Thu, Apr 15, 2021 at 8:41 AM Quentin Monnet [off-list ref] wrote:

quoted

2021-04-15 16:37 UTC+0200 ~ Daniel Borkmann [off-list ref]

quoted

On 4/15/21 11:32 AM, Jianlin Lv wrote:

quoted

For debugging JITs, dumping the JITed image to kernel log is discouraged,
"bpftool prog dump jited" is much better way to examine JITed dumps.
This patch get rid of the code related to bpf_jit_enable=2 mode and
update the proc handler of bpf_jit_enable, also added auxiliary
information to explain how to use bpf_jit_disasm tool after this change.

Signed-off-by: Jianlin Lv <redacted>

Hello,

For what it's worth, I have already seen people dump the JIT image in
kernel logs in Qemu VMs running with just a busybox, not for kernel
development, but in a context where buiding/using bpftool was not
possible.

If building/using bpftool is not possible then majority of selftests won't
be exercised. I don't think such environment is suitable for any kind
of bpf development. Much so for JIT debugging.
While bpf_jit_enable=2 is nothing but the debugging tool for JIT developers.
I'd rather nuke that code instead of carrying it from kernel to kernel.

When I implemented JIT for PPC32, it was extremely helpfull.

As far as I understand, for the time being bpftool is not usable in my environment because it
doesn't support cross compilation when the target's endianess differs from the building host
endianess, see discussion at
https://lore.kernel.org/bpf/21e66a09-514f-f426-b9e2-13baab0b938b@csgroup.eu/

That's right that selftests can't be exercised because they don't build.

The question might be candid as I didn't investigate much about the replacement of "bpf_jit_enable=2
debugging mode" by bpftool, how do we use bpftool exactly for that ? Especially when using the BPF
test module ?

the kernel developers can add any amount of printk and dumps to debug
their code,
but such debugging aid should not be part of the production kernel.
That sysctl was two things at once: debugging tool for kernel devs and
introspection for users.
bpftool jit dump solves the 2nd part. It provides JIT introspection to users.
Debugging of the kernel can be done with any amount of auxiliary code
including calling print_hex_dump() during jiting.

I get the following message when trying the command suggested in the patch message:

root@vgoip:~# ./bpftool prog dump jited
Error: No libbfd support

Christophe

Seems your bpftool prog was built without libbfd, can you rebuild with libbfd
installed.

.John

Le 20/04/2021 à 05:28, Alexei Starovoitov a écrit :

On Sat, Apr 17, 2021 at 1:16 AM Christophe Leroy
[off-list ref] wrote:

quoted

Le 16/04/2021 à 01:49, Alexei Starovoitov a écrit :

quoted

On Thu, Apr 15, 2021 at 8:41 AM Quentin Monnet [off-list ref] wrote:

quoted

2021-04-15 16:37 UTC+0200 ~ Daniel Borkmann [off-list ref]

quoted

On 4/15/21 11:32 AM, Jianlin Lv wrote:

quoted

For debugging JITs, dumping the JITed image to kernel log is discouraged,
"bpftool prog dump jited" is much better way to examine JITed dumps.
This patch get rid of the code related to bpf_jit_enable=2 mode and
update the proc handler of bpf_jit_enable, also added auxiliary
information to explain how to use bpf_jit_disasm tool after this change.

Signed-off-by: Jianlin Lv <redacted>

Hello,

For what it's worth, I have already seen people dump the JIT image in
kernel logs in Qemu VMs running with just a busybox, not for kernel
development, but in a context where buiding/using bpftool was not
possible.

If building/using bpftool is not possible then majority of selftests won't
be exercised. I don't think such environment is suitable for any kind
of bpf development. Much so for JIT debugging.
While bpf_jit_enable=2 is nothing but the debugging tool for JIT developers.
I'd rather nuke that code instead of carrying it from kernel to kernel.

When I implemented JIT for PPC32, it was extremely helpfull.

As far as I understand, for the time being bpftool is not usable in my environment because it
doesn't support cross compilation when the target's endianess differs from the building host
endianess, see discussion at
https://lore.kernel.org/bpf/21e66a09-514f-f426-b9e2-13baab0b938b@csgroup.eu/

That's right that selftests can't be exercised because they don't build.

The question might be candid as I didn't investigate much about the replacement of "bpf_jit_enable=2
debugging mode" by bpftool, how do we use bpftool exactly for that ? Especially when using the BPF
test module ?

the kernel developers can add any amount of printk and dumps to debug
their code,
but such debugging aid should not be part of the production kernel.
That sysctl was two things at once: debugging tool for kernel devs and
introspection for users.
bpftool jit dump solves the 2nd part. It provides JIT introspection to users.
Debugging of the kernel can be done with any amount of auxiliary code
including calling print_hex_dump() during jiting.

I finally managed to cross compile bpftool with libbpf, libopcodes, readline, ncurses, libcap, libz 
and all needed stuff. Was not easy but I made it.

Now, how do I use it ?

Let say I want to dump the jitted code generated from a call to 'tcpdump'. How do I do that with 
'bpftool prog dump jited' ?

I thought by calling this line I would then get programs dumped in a way or another just like when 
setting 'bpf_jit_enable=2', but calling that line just provides me some bpftool help text.

By the way, I would be nice to have a kernel OPTION that selects all OPTIONS required for building 
bpftool. Because you discover them one by one at every build failure. I had to had CONFIG_IPV6, 
CONFIG_DEBUG_BTF, CONFIG_CGROUPS, ... If there could be an option like "Build a 'bpftool' ready 
kernel" that selected all those, it would be great.

Christophe

2021-04-23 09:19 UTC+0200 ~ Christophe Leroy [off-list ref]

[...]

I finally managed to cross compile bpftool with libbpf, libopcodes,
readline, ncurses, libcap, libz and all needed stuff. Was not easy but I
made it.

Libcap is optional and bpftool does not use readline or ncurses. May I
ask how you tried to build it?

Now, how do I use it ?

Let say I want to dump the jitted code generated from a call to
'tcpdump'. How do I do that with 'bpftool prog dump jited' ?

I thought by calling this line I would then get programs dumped in a way
or another just like when setting 'bpf_jit_enable=2', but calling that
line just provides me some bpftool help text.

Well the purpose of this text is to help you find the way to call
bpftool to do what you want :). For dumping your programs' instructions,
you need to tell bpftool what program to dump: Bpftool isn't waiting
until you load a program to dump it, instead you need to load your
program first and then tell bpftool to retrieve the instructions from
the kernel. To reference your program you could use a pinned path, or
first list the programs on your system with "bpftool prog show":


    # bpftool prog show
    138: tracing  name foo  tag e54c922dfa54f65f  gpl
            loaded_at 2021-02-25T01:32:30+0000  uid 0
            xlated 256B  jited 154B  memlock 4096B  map_ids 64
            btf_id 235

Then you can use for example the program id displayed on the first line
to reference and dump your program:

    # bpftool prog dump jited id 138

You should find additional documentation under
tools/bpf/bpftool/Documentation.

By the way, I would be nice to have a kernel OPTION that selects all
OPTIONS required for building bpftool. Because you discover them one by
one at every build failure. I had to had CONFIG_IPV6, CONFIG_DEBUG_BTF,
CONFIG_CGROUPS, ... If there could be an option like "Build a 'bpftool'
ready kernel" that selected all those, it would be great.

Christophe

I do not believe any of these are required to build bpftool.

Quentin

Le 23/04/2021 à 12:26, Quentin Monnet a écrit :

2021-04-23 09:19 UTC+0200 ~ Christophe Leroy [off-list ref]

[...]

quoted

I finally managed to cross compile bpftool with libbpf, libopcodes,
readline, ncurses, libcap, libz and all needed stuff. Was not easy but I
made it.

Libcap is optional and bpftool does not use readline or ncurses. May I
ask how you tried to build it?

cd tools/bpf/

make ARCH=powerpc CROSS_COMPILE=ppc-linux-


Christophe

2021-04-23 12:46 UTC+0200 ~ Christophe Leroy [off-list ref]

Le 23/04/2021 à 12:26, Quentin Monnet a écrit :

quoted

2021-04-23 09:19 UTC+0200 ~ Christophe Leroy
[off-list ref]

[...]

quoted

I finally managed to cross compile bpftool with libbpf, libopcodes,
readline, ncurses, libcap, libz and all needed stuff. Was not easy but I
made it.

Libcap is optional and bpftool does not use readline or ncurses. May I
ask how you tried to build it?

cd tools/bpf/

make ARCH=powerpc CROSS_COMPILE=ppc-linux-

Ok, you could try running directly from tools/bpf/bpftool/ next time
instead.

Readline at least is for a different tool under tools/bpf/, bpf_dbg (But
I'm still not sure where that ncurses requirement was pulled from). The
requirements for specific kernel options probably came from yet another
tool (runqslower, I think).

Quentin

Le 23/04/2021 à 12:59, Quentin Monnet a écrit :

2021-04-23 12:46 UTC+0200 ~ Christophe Leroy [off-list ref]

quoted

Le 23/04/2021 à 12:26, Quentin Monnet a écrit :

quoted

2021-04-23 09:19 UTC+0200 ~ Christophe Leroy
[off-list ref]

[...]

quoted

I finally managed to cross compile bpftool with libbpf, libopcodes,
readline, ncurses, libcap, libz and all needed stuff. Was not easy but I
made it.

Libcap is optional and bpftool does not use readline or ncurses. May I
ask how you tried to build it?

cd tools/bpf/

make ARCH=powerpc CROSS_COMPILE=ppc-linux-

Ok, you could try running directly from tools/bpf/bpftool/ next time
instead.

Readline at least is for a different tool under tools/bpf/, bpf_dbg (But
I'm still not sure where that ncurses requirement was pulled from). The
requirements for specific kernel options probably came from yet another
tool (runqslower, I think).

ncurses (or termcap) is required by readline

Christophe

Le 23/04/2021 à 12:26, Quentin Monnet a écrit :

2021-04-23 09:19 UTC+0200 ~ Christophe Leroy [off-list ref]

[...]

quoted

I finally managed to cross compile bpftool with libbpf, libopcodes,
readline, ncurses, libcap, libz and all needed stuff. Was not easy but I
made it.

Libcap is optional and bpftool does not use readline or ncurses. May I
ask how you tried to build it?

quoted

Now, how do I use it ?

Let say I want to dump the jitted code generated from a call to
'tcpdump'. How do I do that with 'bpftool prog dump jited' ?

I thought by calling this line I would then get programs dumped in a way
or another just like when setting 'bpf_jit_enable=2', but calling that
line just provides me some bpftool help text.

Well the purpose of this text is to help you find the way to call
bpftool to do what you want :). For dumping your programs' instructions,
you need to tell bpftool what program to dump: Bpftool isn't waiting
until you load a program to dump it, instead you need to load your
program first and then tell bpftool to retrieve the instructions from
the kernel. To reference your program you could use a pinned path, or
first list the programs on your system with "bpftool prog show":


     # bpftool prog show
     138: tracing  name foo  tag e54c922dfa54f65f  gpl
             loaded_at 2021-02-25T01:32:30+0000  uid 0
             xlated 256B  jited 154B  memlock 4096B  map_ids 64
             btf_id 235

Got the following error:

root@vgoip:~# ./bpftool prog show
libbpf: elf: endianness mismatch in pid_iter_bpf.
libbpf: failed to initialize skeleton BPF object 'pid_iter_bpf': -4003
Error: failed to open PID iterator skeleton


Christophe

On Thu, Apr 15, 2021 at 10:38 PM Daniel Borkmann [off-list ref] wrote:

On 4/15/21 11:32 AM, Jianlin Lv wrote:

quoted

For debugging JITs, dumping the JITed image to kernel log is discouraged,
"bpftool prog dump jited" is much better way to examine JITed dumps.
This patch get rid of the code related to bpf_jit_enable=2 mode and
update the proc handler of bpf_jit_enable, also added auxiliary
information to explain how to use bpf_jit_disasm tool after this change.

Signed-off-by: Jianlin Lv <redacted>

[...]

quoted

diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
index 0a7a2870f111..8d36b4658076 100644
--- a/arch/x86/net/bpf_jit_comp32.c
+++ b/arch/x86/net/bpf_jit_comp32.c

@@ -2566,9 +2566,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
              cond_resched();
      }

-     if (bpf_jit_enable > 1)
-             bpf_jit_dump(prog->len, proglen, pass + 1, image);
-
      if (image) {
              bpf_jit_binary_lock_ro(header);
              prog->bpf_func = (void *)image;

diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index c8496c1142c9..990b1720c7a4 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c

@@ -273,16 +273,8 @@ static int proc_dointvec_minmax_bpf_enable(struct ctl_table *table, int write,

      tmp.data = &jit_enable;
      ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
-     if (write && !ret) {
-             if (jit_enable < 2 ||
-                 (jit_enable == 2 && bpf_dump_raw_ok(current_cred()))) {
-                     *(int *)table->data = jit_enable;
-                     if (jit_enable == 2)
-                             pr_warn("bpf_jit_enable = 2 was set! NEVER use this in production, only for JIT debugging!\n");
-             } else {
-                     ret = -EPERM;
-             }
-     }
+     if (write && !ret)
+             *(int *)table->data = jit_enable;
      return ret;
  }

@@ -389,7 +381,7 @@ static struct ctl_table net_core_table[] = {
              .extra2         = SYSCTL_ONE,
  # else
              .extra1         = SYSCTL_ZERO,
-             .extra2         = &two,
+             .extra2         = SYSCTL_ONE,
  # endif
      },
  # ifdef CONFIG_HAVE_EBPF_JIT

diff --git a/tools/bpf/bpf_jit_disasm.c b/tools/bpf/bpf_jit_disasm.c
index c8ae95804728..efa4b17ae016 100644
--- a/tools/bpf/bpf_jit_disasm.c
+++ b/tools/bpf/bpf_jit_disasm.c

@@ -7,7 +7,7 @@
   *
   * To get the disassembly of the JIT code, do the following:
   *
- *  1) `echo 2 > /proc/sys/net/core/bpf_jit_enable`
+ *  1) Insert bpf_jit_dump() and recompile the kernel to output JITed image into log

Hmm, if we remove bpf_jit_dump(), the next drive-by cleanup patch will be thrown
at bpf@vger stating that bpf_jit_dump() has no in-tree users and should be removed.
Maybe we should be removing bpf_jit_disasm.c along with it as well as bpf_jit_dump()
itself ... I guess if it's ever needed in those rare occasions for JIT debugging we
can resurrect it from old kernels just locally. But yeah, bpftool's jit dump should
suffice for vast majority of use cases.

There was a recent set for ppc32 jit which was merged into ppc tree which will create
a merge conflict with this one [0]. So we would need a rebase and take it maybe during
merge win once the ppc32 landed..

   [0] https://lore.kernel.org/bpf/cover.1616430991.git.christophe.leroy@csgroup.eu/

quoted

   *  2) Load a BPF filter (e.g. `tcpdump -p -n -s 0 -i eth1 host 192.168.20.0/24`)
   *  3) Run e.g. `bpf_jit_disasm -o` to read out the last JIT code
   *

diff --git a/tools/bpf/bpftool/feature.c b/tools/bpf/bpftool/feature.c
index 40a88df275f9..98c7eec2923f 100644
--- a/tools/bpf/bpftool/feature.c
+++ b/tools/bpf/bpftool/feature.c

@@ -203,9 +203,6 @@ static void probe_jit_enable(void)
              case 1:
                      printf("JIT compiler is enabled\n");
                      break;
-             case 2:
-                     printf("JIT compiler is enabled with debugging traces in kernel logs\n");
-                     break;

This would still need to be there for older kernels ...

I will submit another version after ppc32 landed to remove
bpf_jit_disasm.c and restore bpftool/feature.c

Jianlin

quoted

              case -1:
                      printf("Unable to retrieve JIT-compiler status\n");
                      break;