[PATCH v3 bpf-next 00/10] bpf: Support ->fill_link_info for kprobe_multi and perf_event links

This patchset enhances the usability of kprobe_multi programs by introducing
support for ->fill_link_info. This allows users to easily determine the
probed functions associated with a kprobe_multi program. While
`bpftool perf show` already provides information about functions probed by
perf_event programs, supporting ->fill_link_info ensures consistent access to
this information across all bpf links.

In addition, this patch extends support to generic perf events, which are
currently not covered by `bpftool perf show`. While userspace is exposed to
only the perf type and config, other attributes such as sample_period and
sample_freq are disregarded.

To ensure accurate identification of probed functions, it is preferable to
expose the address directly rather than relying solely on the symbol name.
However, this implementation respects the kptr_restrict setting and avoids
exposing the address if it is not permitted.

v2->v3:
- Expose flags instead of retporbe (Andrii)
- Simplify the check on kmulti_link->cnt (Andrii)
- Use kallsyms_show_value() instead (Andrii)
- Show also the module name for kprobe_multi (Andrii)
- Add new enum bpf_perf_link_type (Andrii)
- Move perf event names into bpftool (Andrii, Quentin, Jiri)
- Keep perf event names in sync with perf tools (Jiri) 

v1->v2:
- Fix sparse warning (Stanislav, lkp@intel.com)
- Fix BPF CI build error
- Reuse kernel_syms_load() (Alexei)
- Print 'name' instead of 'func' (Alexei)
- Show whether the probe is retprobe or not (Andrii)
- Add comment for the meaning of perf_event name (Andrii)
- Add support for generic perf event
- Adhere to the kptr_restrict setting

RFC->v1:
- Use a single copy_to_user() instead (Jiri)
- Show also the symbol name in bpftool (Quentin, Alexei)
- Use calloc() instead of malloc() in bpftool (Quentin)
- Avoid having conditional entries in the JSON output (Quentin)
- Drop ->show_fdinfo (Alexei)
- Use __u64 instead of __aligned_u64 for the field addr (Alexei)
- Avoid the contradiction in perf_event name length (Alexei)
- Address a build warning reported by kernel test robot [off-list ref]

Yafang Shao (10):
  bpf: Support ->fill_link_info for kprobe_multi
  bpftool: Dump the kernel symbol's module name
  bpftool: Show probed function in kprobe_multi link info
  bpf: Protect probed address based on kptr_restrict setting
  bpf: Clear the probe_addr for uprobe
  bpf: Expose symbol's respective address
  bpf: Add a common helper bpf_copy_to_user()
  bpf: Support ->fill_link_info for perf_event
  bpftool: Add perf event names
  bpftool: Show probed function in perf_event link info

 include/uapi/linux/bpf.h          |  37 +++++
 kernel/bpf/syscall.c              | 158 +++++++++++++++++--
 kernel/trace/bpf_trace.c          |  32 +++-
 kernel/trace/trace_kprobe.c       |   7 +-
 tools/bpf/bpftool/link.c          | 322 +++++++++++++++++++++++++++++++++++++-
 tools/bpf/bpftool/perf.c          | 107 +++++++++++++
 tools/bpf/bpftool/perf.h          |  11 ++
 tools/bpf/bpftool/xlated_dumper.c |   6 +-
 tools/bpf/bpftool/xlated_dumper.h |   2 +
 tools/include/uapi/linux/bpf.h    |  37 +++++
 10 files changed, 700 insertions(+), 19 deletions(-)
 create mode 100644 tools/bpf/bpftool/perf.h

-- 
1.8.3.1

With the addition of support for fill_link_info to the kprobe_multi link,
users will gain the ability to inspect it conveniently using the
`bpftool link show`. This enhancement provides valuable information to the
user, including the count of probed functions and their respective
addresses. It's important to note that if the kptr_restrict setting is not
permitted, the probed address will not be exposed, ensuring security.

Signed-off-by: Yafang Shao <redacted>
---
 include/uapi/linux/bpf.h       |  5 +++++
 kernel/trace/bpf_trace.c       | 28 ++++++++++++++++++++++++++++
 tools/include/uapi/linux/bpf.h |  5 +++++
 3 files changed, 38 insertions(+)

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index a7b5e91..23691ea 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h

@@ -6438,6 +6438,11 @@ struct bpf_link_info {
 			__s32 priority;
 			__u32 flags;
 		} netfilter;
+		struct {
+			__aligned_u64 addrs; /* in/out: addresses buffer ptr */
+			__u32 count;
+			__u32 flags;
+		} kprobe_multi;
 	};
 } __attribute__((aligned(8)));
 

diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 2bc41e6..742047c 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c

@@ -2548,9 +2548,36 @@ static void bpf_kprobe_multi_link_dealloc(struct bpf_link *link)
 	kfree(kmulti_link);
 }
 
+static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link,
+						struct bpf_link_info *info)
+{
+	u64 __user *uaddrs = u64_to_user_ptr(info->kprobe_multi.addrs);
+	struct bpf_kprobe_multi_link *kmulti_link;
+	u32 ucount = info->kprobe_multi.count;
+
+	if (!uaddrs ^ !ucount)
+		return -EINVAL;
+
+	kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
+	if (!uaddrs) {
+		info->kprobe_multi.count = kmulti_link->cnt;
+		return 0;
+	}
+
+	if (ucount < kmulti_link->cnt)
+		return -EINVAL;
+	info->kprobe_multi.flags = kmulti_link->fp.flags;
+	if (!kallsyms_show_value(current_cred()))
+		return 0;
+	if (copy_to_user(uaddrs, kmulti_link->addrs, ucount * sizeof(u64)))
+		return -EFAULT;
+	return 0;
+}
+
 static const struct bpf_link_ops bpf_kprobe_multi_link_lops = {
 	.release = bpf_kprobe_multi_link_release,
 	.dealloc = bpf_kprobe_multi_link_dealloc,
+	.fill_link_info = bpf_kprobe_multi_link_fill_link_info,
 };
 
 static void bpf_kprobe_multi_cookie_swap(void *a, void *b, int size, const void *priv)

@@ -2890,6 +2917,7 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr
 		return err;
 	}
 
+	link->fp.flags = flags;
 	return bpf_link_settle(&link_primer);
 
 error:

diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index a7b5e91..23691ea 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h

@@ -6438,6 +6438,11 @@ struct bpf_link_info {
 			__s32 priority;
 			__u32 flags;
 		} netfilter;
+		struct {
+			__aligned_u64 addrs; /* in/out: addresses buffer ptr */
+			__u32 count;
+			__u32 flags;
+		} kprobe_multi;
 	};
 } __attribute__((aligned(8)));
 

-- 
1.8.3.1

On Mon, Jun 12, 2023 at 03:15:59PM +0000, Yafang Shao wrote:

SNIP

hum this looks wrong, we can't use fprobe flags to store our flags
you should add flags to bpf_kprobe_multi_link

jirka

On Thu, Jun 15, 2023 at 4:29 PM Jiri Olsa [off-list ref] wrote:

On Mon, Jun 12, 2023 at 03:15:59PM +0000, Yafang Shao wrote:

SNIP

quoted

diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 2bc41e6..742047c 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c

@@ -2548,9 +2548,36 @@ static void bpf_kprobe_multi_link_dealloc(struct bpf_link *link)
      kfree(kmulti_link);
 }

+static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link,
+                                             struct bpf_link_info *info)
+{
+     u64 __user *uaddrs = u64_to_user_ptr(info->kprobe_multi.addrs);
+     struct bpf_kprobe_multi_link *kmulti_link;
+     u32 ucount = info->kprobe_multi.count;
+
+     if (!uaddrs ^ !ucount)
+             return -EINVAL;
+
+     kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
+     if (!uaddrs) {
+             info->kprobe_multi.count = kmulti_link->cnt;
+             return 0;
+     }
+
+     if (ucount < kmulti_link->cnt)
+             return -EINVAL;
+     info->kprobe_multi.flags = kmulti_link->fp.flags;
+     if (!kallsyms_show_value(current_cred()))
+             return 0;
+     if (copy_to_user(uaddrs, kmulti_link->addrs, ucount * sizeof(u64)))
+             return -EFAULT;
+     return 0;
+}
+
 static const struct bpf_link_ops bpf_kprobe_multi_link_lops = {
      .release = bpf_kprobe_multi_link_release,
      .dealloc = bpf_kprobe_multi_link_dealloc,
+     .fill_link_info = bpf_kprobe_multi_link_fill_link_info,
 };

 static void bpf_kprobe_multi_cookie_swap(void *a, void *b, int size, const void *priv)

@@ -2890,6 +2917,7 @@ int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr
              return err;
      }

+     link->fp.flags = flags;

hum this looks wrong, we can't use fprobe flags to store our flags
you should add flags to bpf_kprobe_multi_link

Will fix it. Thanks for pointing it out.

-- 
Regards
Yafang

On Mon, Jun 12, 2023 at 8:16 AM Yafang Shao [off-list ref] wrote:

besides what Jiri said, flags should always be returned, just like
cnt. So structure code instead around uaddrs being optional, that will
everything more straightforward (i.e., fill out everything but uaddrs
and then at the end fill out addrs if uaddrs is not zero)

On Sat, Jun 17, 2023 at 1:24 AM Andrii Nakryiko
[off-list ref] wrote:

On Mon, Jun 12, 2023 at 8:16 AM Yafang Shao [off-list ref] wrote:

quoted

With the addition of support for fill_link_info to the kprobe_multi link,
users will gain the ability to inspect it conveniently using the
`bpftool link show`. This enhancement provides valuable information to the
user, including the count of probed functions and their respective
addresses. It's important to note that if the kptr_restrict setting is not
permitted, the probed address will not be exposed, ensuring security.

Signed-off-by: Yafang Shao <redacted>
---
 include/uapi/linux/bpf.h       |  5 +++++
 kernel/trace/bpf_trace.c       | 28 ++++++++++++++++++++++++++++
 tools/include/uapi/linux/bpf.h |  5 +++++
 3 files changed, 38 insertions(+)

diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index a7b5e91..23691ea 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h

@@ -6438,6 +6438,11 @@ struct bpf_link_info {
                        __s32 priority;
                        __u32 flags;
                } netfilter;
+               struct {
+                       __aligned_u64 addrs; /* in/out: addresses buffer ptr */
+                       __u32 count;
+                       __u32 flags;
+               } kprobe_multi;
        };
 } __attribute__((aligned(8)));

diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 2bc41e6..742047c 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c

@@ -2548,9 +2548,36 @@ static void bpf_kprobe_multi_link_dealloc(struct bpf_link *link)
        kfree(kmulti_link);
 }

+static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link,
+                                               struct bpf_link_info *info)
+{
+       u64 __user *uaddrs = u64_to_user_ptr(info->kprobe_multi.addrs);
+       struct bpf_kprobe_multi_link *kmulti_link;
+       u32 ucount = info->kprobe_multi.count;
+
+       if (!uaddrs ^ !ucount)
+               return -EINVAL;
+
+       kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link);
+       if (!uaddrs) {
+               info->kprobe_multi.count = kmulti_link->cnt;
+               return 0;
+       }
+
+       if (ucount < kmulti_link->cnt)
+               return -EINVAL;
+       info->kprobe_multi.flags = kmulti_link->fp.flags;

besides what Jiri said, flags should always be returned, just like
cnt. So structure code instead around uaddrs being optional, that will
everything more straightforward (i.e., fill out everything but uaddrs
and then at the end fill out addrs if uaddrs is not zero)

Agree. That will be more straightforward. Will change it.

-- 
Regards
Yafang

If the kernel symbol is in a module, we will dump the module name as
well.

Signed-off-by: Yafang Shao <redacted>
---
 tools/bpf/bpftool/xlated_dumper.c | 6 +++++-
 tools/bpf/bpftool/xlated_dumper.h | 2 ++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/tools/bpf/bpftool/xlated_dumper.c b/tools/bpf/bpftool/xlated_dumper.c
index da608e1..dd917f3 100644
--- a/tools/bpf/bpftool/xlated_dumper.c
+++ b/tools/bpf/bpftool/xlated_dumper.c

@@ -46,7 +46,11 @@ void kernel_syms_load(struct dump_data *dd)
 		}
 		dd->sym_mapping = tmp;
 		sym = &dd->sym_mapping[dd->sym_count];
-		if (sscanf(buff, "%p %*c %s", &address, sym->name) != 2)
+
+		/* module is optional */
+		sym->module[0] = '\0';
+		if (sscanf(buff, "%p %*c %s %s", &address, sym->name,
+		    sym->module) < 2)
 			continue;
 		sym->address = (unsigned long)address;
 		if (!strcmp(sym->name, "__bpf_call_base")) {

diff --git a/tools/bpf/bpftool/xlated_dumper.h b/tools/bpf/bpftool/xlated_dumper.h
index 9a94637..5df8025 100644
--- a/tools/bpf/bpftool/xlated_dumper.h
+++ b/tools/bpf/bpftool/xlated_dumper.h

@@ -5,12 +5,14 @@
 #define __BPF_TOOL_XLATED_DUMPER_H
 
 #define SYM_MAX_NAME	256
+#define MODULE_NAME_LEN	64
 
 struct bpf_prog_linfo;
 
 struct kernel_sym {
 	unsigned long address;
 	char name[SYM_MAX_NAME];
+	char module[MODULE_NAME_LEN];
 };
 
 struct dump_data {

-- 
1.8.3.1

2023-06-12 15:16 UTC+0000 ~ Yafang Shao [off-list ref]

Nit: MODULE_MAX_NAME would be more consistent and would make more sense
to me? And it would avoid confusion with MODULE_NAME_LEN from kernel,
which doesn't have the same value.

 };
 
 struct dump_data {

On Tue, Jun 13, 2023 at 9:41 PM Quentin Monnet [off-list ref] wrote:

2023-06-12 15:16 UTC+0000 ~ Yafang Shao [off-list ref]

quoted

If the kernel symbol is in a module, we will dump the module name as
well.

Signed-off-by: Yafang Shao <redacted>
---
 tools/bpf/bpftool/xlated_dumper.c | 6 +++++-
 tools/bpf/bpftool/xlated_dumper.h | 2 ++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/tools/bpf/bpftool/xlated_dumper.c b/tools/bpf/bpftool/xlated_dumper.c
index da608e1..dd917f3 100644
--- a/tools/bpf/bpftool/xlated_dumper.c
+++ b/tools/bpf/bpftool/xlated_dumper.c

@@ -46,7 +46,11 @@ void kernel_syms_load(struct dump_data *dd)
              }
              dd->sym_mapping = tmp;
              sym = &dd->sym_mapping[dd->sym_count];
-             if (sscanf(buff, "%p %*c %s", &address, sym->name) != 2)
+
+             /* module is optional */
+             sym->module[0] = '\0';
+             if (sscanf(buff, "%p %*c %s %s", &address, sym->name,
+                 sym->module) < 2)
                      continue;
              sym->address = (unsigned long)address;
              if (!strcmp(sym->name, "__bpf_call_base")) {

diff --git a/tools/bpf/bpftool/xlated_dumper.h b/tools/bpf/bpftool/xlated_dumper.h
index 9a94637..5df8025 100644
--- a/tools/bpf/bpftool/xlated_dumper.h
+++ b/tools/bpf/bpftool/xlated_dumper.h

@@ -5,12 +5,14 @@
 #define __BPF_TOOL_XLATED_DUMPER_H

 #define SYM_MAX_NAME 256
+#define MODULE_NAME_LEN      64

 struct bpf_prog_linfo;

 struct kernel_sym {
      unsigned long address;
      char name[SYM_MAX_NAME];
+     char module[MODULE_NAME_LEN];

Nit: MODULE_MAX_NAME would be more consistent and would make more sense
to me? And it would avoid confusion with MODULE_NAME_LEN from kernel,
which doesn't have the same value.

Will use MODULE_MAX_NAME instead.

-- 
Regards
Yafang

On Mon, Jun 12, 2023 at 8:16 AM Yafang Shao [off-list ref] wrote:

nit: please keep it single line if it fits in under 100 characters

On Sat, Jun 17, 2023 at 1:25 AM Andrii Nakryiko
[off-list ref] wrote:

On Mon, Jun 12, 2023 at 8:16 AM Yafang Shao [off-list ref] wrote:

quoted

If the kernel symbol is in a module, we will dump the module name as
well.

Signed-off-by: Yafang Shao <redacted>
---
 tools/bpf/bpftool/xlated_dumper.c | 6 +++++-
 tools/bpf/bpftool/xlated_dumper.h | 2 ++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/tools/bpf/bpftool/xlated_dumper.c b/tools/bpf/bpftool/xlated_dumper.c
index da608e1..dd917f3 100644
--- a/tools/bpf/bpftool/xlated_dumper.c
+++ b/tools/bpf/bpftool/xlated_dumper.c

@@ -46,7 +46,11 @@ void kernel_syms_load(struct dump_data *dd)
                }
                dd->sym_mapping = tmp;
                sym = &dd->sym_mapping[dd->sym_count];
-               if (sscanf(buff, "%p %*c %s", &address, sym->name) != 2)
+
+               /* module is optional */
+               sym->module[0] = '\0';
+               if (sscanf(buff, "%p %*c %s %s", &address, sym->name,
+                   sym->module) < 2)

nit: please keep it single line if it fits in under 100 characters

Will change it.
I thought it is 80 characters.

-- 
Regards
Yafang

Show the already expose kprobe_multi link info in bpftool. The result as
follows,

52: kprobe_multi  prog 381
        retprobe 0  func_cnt 7
        addrs ffffffff9ec44f20  funcs schedule_timeout_interruptible
              ffffffff9ec44f60        schedule_timeout_killable
              ffffffff9ec44fa0        schedule_timeout_uninterruptible
              ffffffff9ec44fe0        schedule_timeout_idle
              ffffffffc09468d0        xfs_trans_get_efd [xfs]
              ffffffffc0953a10        xfs_trans_get_buf_map [xfs]
              ffffffffc0957320        xfs_trans_get_dqtrx [xfs]
        pids kprobe_multi(559862)
53: kprobe_multi  prog 381
        retprobe 1  func_cnt 7
        addrs ffffffff9ec44f20  funcs schedule_timeout_interruptible
              ffffffff9ec44f60        schedule_timeout_killable
              ffffffff9ec44fa0        schedule_timeout_uninterruptible
              ffffffff9ec44fe0        schedule_timeout_idle
              ffffffffc09468d0        xfs_trans_get_efd [xfs]
              ffffffffc0953a10        xfs_trans_get_buf_map [xfs]
              ffffffffc0957320        xfs_trans_get_dqtrx [xfs]
        pids kprobe_multi(559862)

$ tools/bpf/bpftool/bpftool link show -j
[{"id":52,"type":"kprobe_multi","prog_id":381,"retprobe":0,"func_cnt":7,"funcs":[{"addr":18446744072078249760,"func":"schedule_timeout_interruptible","module":""},{"addr":18446744072078249824,"func":"schedule_timeout_killable","module":""},{"addr":18446744072078249888,"func":"schedule_timeout_uninterruptible","module":""},{"addr":18446744072078249952,"func":"schedule_timeout_idle","module":""},{"addr":18446744072645535952,"func":"xfs_trans_get_efd","module":"[xfs]"},{"addr":18446744072645589520,"func":"xfs_trans_get_buf_map","module":"[xfs]"},{"addr":18446744072645604128,"func":"xfs_trans_get_dqtrx","module":"[xfs]"}],"pids":[{"pid":559862,"comm":"kprobe_multi"}]},{"id":53,"type":"kprobe_multi","prog_id":381,"retprobe":1,"func_cnt":7,"funcs":[{"addr":18446744072078249760,"func":"schedule_timeout_interruptible","module":""},{"addr":18446744072078249824,"func":"schedule_timeout_killable","module":""},{"addr":18446744072078249888,"func":"schedule_timeout_uninterruptible","module":""},{"addr":18446744072078249952,"func":"schedule_timeout_idle","module":""},{"addr":18446744072645535952,"func":"xfs_trans_get_efd","module":"[xfs]"},{"addr":18446744072645589520,"func":"xfs_trans_get_buf_map","module":"[xfs]"},{"addr":18446744072645604128,"func":"xfs_trans_get_dqtrx","module":"[xfs]"}],"pids":[{"pid":559862,"comm":"kprobe_multi"}]}]

Signed-off-by: Yafang Shao <redacted>
---
 tools/bpf/bpftool/link.c | 109 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 108 insertions(+), 1 deletion(-)

diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
index 2d78607..0015582 100644
--- a/tools/bpf/bpftool/link.c
+++ b/tools/bpf/bpftool/link.c

@@ -14,8 +14,10 @@
 
 #include "json_writer.h"
 #include "main.h"
+#include "xlated_dumper.h"
 
 static struct hashmap *link_table;
+static struct dump_data dd = {};
 
 static int link_parse_fd(int *argc, char ***argv)
 {

@@ -166,6 +168,45 @@ static int get_prog_info(int prog_id, struct bpf_prog_info *info)
 	return err;
 }
 
+static int cmp_u64(const void *A, const void *B)
+{
+	const __u64 *a = A, *b = B;
+
+	return *a - *b;
+}
+
+static void
+show_kprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
+{
+	__u32 i, j = 0;
+	__u64 *addrs;
+
+	jsonw_uint_field(json_wtr, "retprobe",
+			 info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN);
+	jsonw_uint_field(json_wtr, "func_cnt", info->kprobe_multi.count);
+	jsonw_name(json_wtr, "funcs");
+	jsonw_start_array(json_wtr);
+	addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs);
+	qsort((void *)addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64);
+
+	/* Load it once for all. */
+	if (!dd.sym_count)
+		kernel_syms_load(&dd);
+	for (i = 0; i < dd.sym_count; i++) {
+		if (dd.sym_mapping[i].address != addrs[j])
+			continue;
+		jsonw_start_object(json_wtr);
+		jsonw_uint_field(json_wtr, "addr", dd.sym_mapping[i].address);
+		jsonw_string_field(json_wtr, "func", dd.sym_mapping[i].name);
+		/* Print none if it is vmlinux */
+		jsonw_string_field(json_wtr, "module", dd.sym_mapping[i].module);
+		jsonw_end_object(json_wtr);
+		if (j++ == info->kprobe_multi.count)
+			break;
+	}
+	jsonw_end_array(json_wtr);
+}
+
 static int show_link_close_json(int fd, struct bpf_link_info *info)
 {
 	struct bpf_prog_info prog_info;

@@ -218,6 +259,9 @@ static int show_link_close_json(int fd, struct bpf_link_info *info)
 		jsonw_uint_field(json_wtr, "map_id",
 				 info->struct_ops.map_id);
 		break;
+	case BPF_LINK_TYPE_KPROBE_MULTI:
+		show_kprobe_multi_json(info, json_wtr);
+		break;
 	default:
 		break;
 	}

@@ -351,6 +395,44 @@ void netfilter_dump_plain(const struct bpf_link_info *info)
 		printf(" flags 0x%x", info->netfilter.flags);
 }
 
+static void show_kprobe_multi_plain(struct bpf_link_info *info)
+{
+	__u32 i, j = 0;
+	__u64 *addrs;
+
+	if (!info->kprobe_multi.count)
+		return;
+
+	printf("\n\tretprobe %d  func_cnt %u  ",
+	       info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN,
+	       info->kprobe_multi.count);
+	addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs);
+	qsort((void *)addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64);
+
+	/* Load it once for all. */
+	if (!dd.sym_count)
+		kernel_syms_load(&dd);
+	for (i = 0; i < dd.sym_count; i++) {
+		if (dd.sym_mapping[i].address != addrs[j])
+			continue;
+		if (!j)
+			printf("\n\taddrs %016lx  funcs %s",
+			       dd.sym_mapping[i].address,
+			       dd.sym_mapping[i].name);
+		else
+			printf("\n\t      %016lx        %s",
+			       dd.sym_mapping[i].address,
+			       dd.sym_mapping[i].name);
+		if (dd.sym_mapping[i].module[0] != '\0')
+			printf(" %s  ", dd.sym_mapping[i].module);
+		else
+			printf("  ");
+
+		if (j++ == info->kprobe_multi.count)
+			break;
+	}
+}
+
 static int show_link_close_plain(int fd, struct bpf_link_info *info)
 {
 	struct bpf_prog_info prog_info;

@@ -396,6 +478,9 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
 	case BPF_LINK_TYPE_NETFILTER:
 		netfilter_dump_plain(info);
 		break;
+	case BPF_LINK_TYPE_KPROBE_MULTI:
+		show_kprobe_multi_plain(info);
+		break;
 	default:
 		break;
 	}

@@ -417,7 +502,9 @@ static int do_show_link(int fd)
 {
 	struct bpf_link_info info;
 	__u32 len = sizeof(info);
+	__u64 *addrs = NULL;
 	char buf[256];
+	int count;
 	int err;
 
 	memset(&info, 0, sizeof(info));

@@ -441,12 +528,28 @@ static int do_show_link(int fd)
 		info.iter.target_name_len = sizeof(buf);
 		goto again;
 	}
+	if (info.type == BPF_LINK_TYPE_KPROBE_MULTI &&
+	    !info.kprobe_multi.addrs) {
+		count = info.kprobe_multi.count;
+		if (count) {
+			addrs = calloc(count, sizeof(__u64));
+			if (!addrs) {
+				p_err("mem alloc failed");
+				close(fd);
+				return -1;
+			}
+			info.kprobe_multi.addrs = (unsigned long)addrs;
+			goto again;
+		}
+	}
 
 	if (json_output)
 		show_link_close_json(fd, &info);
 	else
 		show_link_close_plain(fd, &info);
 
+	if (addrs)
+		free(addrs);
 	close(fd);
 	return 0;
 }

@@ -471,7 +574,8 @@ static int do_show(int argc, char **argv)
 		fd = link_parse_fd(&argc, &argv);
 		if (fd < 0)
 			return fd;
-		return do_show_link(fd);
+		do_show_link(fd);
+		goto out;
 	}
 
 	if (argc)

@@ -510,6 +614,9 @@ static int do_show(int argc, char **argv)
 	if (show_pinned)
 		delete_pinned_obj_table(link_table);
 
+out:
+	if (dd.sym_count)
+		kernel_syms_destroy(&dd);
 	return errno == ENOENT ? 0 : -1;
 }
 

-- 
1.8.3.1

2023-06-12 15:16 UTC+0000 ~ Yafang Shao [off-list ref]

The "retprobe" field could maybe be a boolean rather than an int.

+	jsonw_uint_field(json_wtr, "func_cnt", info->kprobe_multi.count);
+	jsonw_name(json_wtr, "funcs");
+	jsonw_start_array(json_wtr);
+	addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs);
+	qsort((void *)addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64);
+
+	/* Load it once for all. */
+	if (!dd.sym_count)
+		kernel_syms_load(&dd);
+	for (i = 0; i < dd.sym_count; i++) {
+		if (dd.sym_mapping[i].address != addrs[j])
+			continue;
+		jsonw_start_object(json_wtr);
+		jsonw_uint_field(json_wtr, "addr", dd.sym_mapping[i].address);
+		jsonw_string_field(json_wtr, "func", dd.sym_mapping[i].name);
+		/* Print none if it is vmlinux */
+		jsonw_string_field(json_wtr, "module", dd.sym_mapping[i].module);

Can we trim the square brackets around module names for the JSON output,
please? They make entries look like arrays; but mostly, if we keep them,
we're forcing every consumer to trim them on their side before being
able to reuse the value.

On Tue, Jun 13, 2023 at 9:41 PM Quentin Monnet [off-list ref] wrote:

2023-06-12 15:16 UTC+0000 ~ Yafang Shao [off-list ref]

quoted

Show the already expose kprobe_multi link info in bpftool. The result as
follows,

52: kprobe_multi  prog 381
        retprobe 0  func_cnt 7
        addrs ffffffff9ec44f20  funcs schedule_timeout_interruptible
              ffffffff9ec44f60        schedule_timeout_killable
              ffffffff9ec44fa0        schedule_timeout_uninterruptible
              ffffffff9ec44fe0        schedule_timeout_idle
              ffffffffc09468d0        xfs_trans_get_efd [xfs]
              ffffffffc0953a10        xfs_trans_get_buf_map [xfs]
              ffffffffc0957320        xfs_trans_get_dqtrx [xfs]
        pids kprobe_multi(559862)
53: kprobe_multi  prog 381
        retprobe 1  func_cnt 7
        addrs ffffffff9ec44f20  funcs schedule_timeout_interruptible
              ffffffff9ec44f60        schedule_timeout_killable
              ffffffff9ec44fa0        schedule_timeout_uninterruptible
              ffffffff9ec44fe0        schedule_timeout_idle
              ffffffffc09468d0        xfs_trans_get_efd [xfs]
              ffffffffc0953a10        xfs_trans_get_buf_map [xfs]
              ffffffffc0957320        xfs_trans_get_dqtrx [xfs]
        pids kprobe_multi(559862)

$ tools/bpf/bpftool/bpftool link show -j
[{"id":52,"type":"kprobe_multi","prog_id":381,"retprobe":0,"func_cnt":7,"funcs":[{"addr":18446744072078249760,"func":"schedule_timeout_interruptible","module":""},{"addr":18446744072078249824,"func":"schedule_timeout_killable","module":""},{"addr":18446744072078249888,"func":"schedule_timeout_uninterruptible","module":""},{"addr":18446744072078249952,"func":"schedule_timeout_idle","module":""},{"addr":18446744072645535952,"func":"xfs_trans_get_efd","module":"[xfs]"},{"addr":18446744072645589520,"func":"xfs_trans_get_buf_map","module":"[xfs]"},{"addr":18446744072645604128,"func":"xfs_trans_get_dqtrx","module":"[xfs]"}],"pids":[{"pid":559862,"comm":"kprobe_multi"}]},{"id":53,"type":"kprobe_multi","prog_id":381,"retprobe":1,"func_cnt":7,"funcs":[{"addr":18446744072078249760,"func":"schedule_timeout_interruptible","module":""},{"addr":18446744072078249824,"func":"schedule_timeout_killable","module":""},{"addr":18446744072078249888,"func":"schedule_timeout_uninterruptible","module":""},{"addr":18446744072078249952,"func":"schedule_timeout_idle","module":""},{"addr":18446744072645535952,"func":"xfs_trans_get_efd","module":"[xfs]"},{"addr":18446744072645589520,"func":"xfs_trans_get_buf_map","module":"[xfs]"},{"addr":18446744072645604128,"func":"xfs_trans_get_dqtrx","module":"[xfs]"}],"pids":[{"pid":559862,"comm":"kprobe_multi"}]}]

Signed-off-by: Yafang Shao <redacted>
---
 tools/bpf/bpftool/link.c | 109 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 108 insertions(+), 1 deletion(-)

diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
index 2d78607..0015582 100644
--- a/tools/bpf/bpftool/link.c
+++ b/tools/bpf/bpftool/link.c

@@ -14,8 +14,10 @@

 #include "json_writer.h"
 #include "main.h"
+#include "xlated_dumper.h"

 static struct hashmap *link_table;
+static struct dump_data dd = {};

 static int link_parse_fd(int *argc, char ***argv)
 {

@@ -166,6 +168,45 @@ static int get_prog_info(int prog_id, struct bpf_prog_info *info)
      return err;
 }

+static int cmp_u64(const void *A, const void *B)
+{
+     const __u64 *a = A, *b = B;
+
+     return *a - *b;
+}
+
+static void
+show_kprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
+{
+     __u32 i, j = 0;
+     __u64 *addrs;
+
+     jsonw_uint_field(json_wtr, "retprobe",
+                      info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN);

The "retprobe" field could maybe be a boolean rather than an int.

Will change it.

quoted

+     jsonw_uint_field(json_wtr, "func_cnt", info->kprobe_multi.count);
+     jsonw_name(json_wtr, "funcs");
+     jsonw_start_array(json_wtr);
+     addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs);
+     qsort((void *)addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64);
+
+     /* Load it once for all. */
+     if (!dd.sym_count)
+             kernel_syms_load(&dd);
+     for (i = 0; i < dd.sym_count; i++) {
+             if (dd.sym_mapping[i].address != addrs[j])
+                     continue;
+             jsonw_start_object(json_wtr);
+             jsonw_uint_field(json_wtr, "addr", dd.sym_mapping[i].address);
+             jsonw_string_field(json_wtr, "func", dd.sym_mapping[i].name);
+             /* Print none if it is vmlinux */
+             jsonw_string_field(json_wtr, "module", dd.sym_mapping[i].module);

Can we trim the square brackets around module names for the JSON output,
please? They make entries look like arrays; but mostly, if we keep them,
we're forcing every consumer to trim them on their side before being
able to reuse the value.

Agree, will trim it.

-- 
Regards
Yafang

On 6/12/23 08:16, Yafang Shao wrote:

Could you explain what these extra spaces after module names are for?

On Wed, Jun 14, 2023 at 6:36 AM Kui-Feng Lee [off-list ref] wrote:

On 6/12/23 08:16, Yafang Shao wrote:

quoted

Show the already expose kprobe_multi link info in bpftool. The result as
follows,

52: kprobe_multi  prog 381
         retprobe 0  func_cnt 7
         addrs ffffffff9ec44f20  funcs schedule_timeout_interruptible
               ffffffff9ec44f60        schedule_timeout_killable
               ffffffff9ec44fa0        schedule_timeout_uninterruptible
               ffffffff9ec44fe0        schedule_timeout_idle
               ffffffffc09468d0        xfs_trans_get_efd [xfs]
               ffffffffc0953a10        xfs_trans_get_buf_map [xfs]
               ffffffffc0957320        xfs_trans_get_dqtrx [xfs]
         pids kprobe_multi(559862)
53: kprobe_multi  prog 381
         retprobe 1  func_cnt 7
         addrs ffffffff9ec44f20  funcs schedule_timeout_interruptible
               ffffffff9ec44f60        schedule_timeout_killable
               ffffffff9ec44fa0        schedule_timeout_uninterruptible
               ffffffff9ec44fe0        schedule_timeout_idle
               ffffffffc09468d0        xfs_trans_get_efd [xfs]
               ffffffffc0953a10        xfs_trans_get_buf_map [xfs]
               ffffffffc0957320        xfs_trans_get_dqtrx [xfs]
         pids kprobe_multi(559862)

$ tools/bpf/bpftool/bpftool link show -j
[{"id":52,"type":"kprobe_multi","prog_id":381,"retprobe":0,"func_cnt":7,"funcs":[{"addr":18446744072078249760,"func":"schedule_timeout_interruptible","module":""},{"addr":18446744072078249824,"func":"schedule_timeout_killable","module":""},{"addr":18446744072078249888,"func":"schedule_timeout_uninterruptible","module":""},{"addr":18446744072078249952,"func":"schedule_timeout_idle","module":""},{"addr":18446744072645535952,"func":"xfs_trans_get_efd","module":"[xfs]"},{"addr":18446744072645589520,"func":"xfs_trans_get_buf_map","module":"[xfs]"},{"addr":18446744072645604128,"func":"xfs_trans_get_dqtrx","module":"[xfs]"}],"pids":[{"pid":559862,"comm":"kprobe_multi"}]},{"id":53,"type":"kprobe_multi","prog_id":381,"retprobe":1,"func_cnt":7,"funcs":[{"addr":18446744072078249760,"func":"schedule_timeout_interruptible","module":""},{"addr":18446744072078249824,"func":"schedule_timeout_killable","module":""},{"addr":18446744072078249888,"func":"schedule_timeout_uninterruptible","module":""},{"addr":18446744072078249952,"func":"schedule_timeout_idle","module":""},{"addr":18446744072645535952,"func":"xfs_trans_get_efd","module":"[xfs]"},{"addr":18446744072645589520,"func":"xfs_trans_get_buf_map","module":"[xfs]"},{"addr":18446744072645604128,"func":"xfs_trans_get_dqtrx","module":"[xfs]"}],"pids":[{"pid":559862,"comm":"kprobe_multi"}]}]

Signed-off-by: Yafang Shao <redacted>
---
  tools/bpf/bpftool/link.c | 109 ++++++++++++++++++++++++++++++++++++++++++++++-
  1 file changed, 108 insertions(+), 1 deletion(-)

diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
index 2d78607..0015582 100644
--- a/tools/bpf/bpftool/link.c
+++ b/tools/bpf/bpftool/link.c

@@ -14,8 +14,10 @@

  #include "json_writer.h"
  #include "main.h"
+#include "xlated_dumper.h"

  static struct hashmap *link_table;
+static struct dump_data dd = {};

  static int link_parse_fd(int *argc, char ***argv)
  {

@@ -166,6 +168,45 @@ static int get_prog_info(int prog_id, struct bpf_prog_info *info)
      return err;
  }

+static int cmp_u64(const void *A, const void *B)
+{
+     const __u64 *a = A, *b = B;
+
+     return *a - *b;
+}
+
+static void
+show_kprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
+{
+     __u32 i, j = 0;
+     __u64 *addrs;
+
+     jsonw_uint_field(json_wtr, "retprobe",
+                      info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN);
+     jsonw_uint_field(json_wtr, "func_cnt", info->kprobe_multi.count);
+     jsonw_name(json_wtr, "funcs");
+     jsonw_start_array(json_wtr);
+     addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs);
+     qsort((void *)addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64);
+
+     /* Load it once for all. */
+     if (!dd.sym_count)
+             kernel_syms_load(&dd);
+     for (i = 0; i < dd.sym_count; i++) {
+             if (dd.sym_mapping[i].address != addrs[j])
+                     continue;
+             jsonw_start_object(json_wtr);
+             jsonw_uint_field(json_wtr, "addr", dd.sym_mapping[i].address);
+             jsonw_string_field(json_wtr, "func", dd.sym_mapping[i].name);
+             /* Print none if it is vmlinux */
+             jsonw_string_field(json_wtr, "module", dd.sym_mapping[i].module);
+             jsonw_end_object(json_wtr);
+             if (j++ == info->kprobe_multi.count)
+                     break;
+     }
+     jsonw_end_array(json_wtr);
+}
+
  static int show_link_close_json(int fd, struct bpf_link_info *info)
  {
      struct bpf_prog_info prog_info;

@@ -218,6 +259,9 @@ static int show_link_close_json(int fd, struct bpf_link_info *info)
              jsonw_uint_field(json_wtr, "map_id",
                               info->struct_ops.map_id);
              break;
+     case BPF_LINK_TYPE_KPROBE_MULTI:
+             show_kprobe_multi_json(info, json_wtr);
+             break;
      default:
              break;
      }

@@ -351,6 +395,44 @@ void netfilter_dump_plain(const struct bpf_link_info *info)
              printf(" flags 0x%x", info->netfilter.flags);
  }

+static void show_kprobe_multi_plain(struct bpf_link_info *info)
+{
+     __u32 i, j = 0;
+     __u64 *addrs;
+
+     if (!info->kprobe_multi.count)
+             return;
+
+     printf("\n\tretprobe %d  func_cnt %u  ",
+            info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN,
+            info->kprobe_multi.count);
+     addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs);
+     qsort((void *)addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64);
+
+     /* Load it once for all. */
+     if (!dd.sym_count)
+             kernel_syms_load(&dd);
+     for (i = 0; i < dd.sym_count; i++) {
+             if (dd.sym_mapping[i].address != addrs[j])
+                     continue;
+             if (!j)
+                     printf("\n\taddrs %016lx  funcs %s",
+                            dd.sym_mapping[i].address,
+                            dd.sym_mapping[i].name);
+             else
+                     printf("\n\t      %016lx        %s",
+                            dd.sym_mapping[i].address,
+                            dd.sym_mapping[i].name);
+             if (dd.sym_mapping[i].module[0] != '\0')
+                     printf(" %s  ", dd.sym_mapping[i].module);
+             else
+                     printf("  ");

Could you explain what these extra spaces after module names are for?

There are two spaces. We use two spaces to seperate different items
printed in bpftool. For example,
  "4: kprobe_multi  prog 16"
There are two spaces between the "type" and the "prog".

We always print these two spaces after one item is printed:
  printf("type %u  ", info->type);
  printf("prog %u  ", info->prog_id);
That way, we can add new item easily and consistently.

-- 
Regards
Yafang

2023-06-14 10:42 UTC+0800 ~ Yafang Shao [off-list ref]

On Wed, Jun 14, 2023 at 6:36 AM Kui-Feng Lee [off-list ref] wrote:

quoted

On 6/12/23 08:16, Yafang Shao wrote:

quoted

Show the already expose kprobe_multi link info in bpftool. The result as
follows,

52: kprobe_multi  prog 381
         retprobe 0  func_cnt 7
         addrs ffffffff9ec44f20  funcs schedule_timeout_interruptible
               ffffffff9ec44f60        schedule_timeout_killable
               ffffffff9ec44fa0        schedule_timeout_uninterruptible
               ffffffff9ec44fe0        schedule_timeout_idle
               ffffffffc09468d0        xfs_trans_get_efd [xfs]
               ffffffffc0953a10        xfs_trans_get_buf_map [xfs]
               ffffffffc0957320        xfs_trans_get_dqtrx [xfs]
         pids kprobe_multi(559862)
53: kprobe_multi  prog 381
         retprobe 1  func_cnt 7
         addrs ffffffff9ec44f20  funcs schedule_timeout_interruptible
               ffffffff9ec44f60        schedule_timeout_killable
               ffffffff9ec44fa0        schedule_timeout_uninterruptible
               ffffffff9ec44fe0        schedule_timeout_idle
               ffffffffc09468d0        xfs_trans_get_efd [xfs]
               ffffffffc0953a10        xfs_trans_get_buf_map [xfs]
               ffffffffc0957320        xfs_trans_get_dqtrx [xfs]
         pids kprobe_multi(559862)

$ tools/bpf/bpftool/bpftool link show -j
[{"id":52,"type":"kprobe_multi","prog_id":381,"retprobe":0,"func_cnt":7,"funcs":[{"addr":18446744072078249760,"func":"schedule_timeout_interruptible","module":""},{"addr":18446744072078249824,"func":"schedule_timeout_killable","module":""},{"addr":18446744072078249888,"func":"schedule_timeout_uninterruptible","module":""},{"addr":18446744072078249952,"func":"schedule_timeout_idle","module":""},{"addr":18446744072645535952,"func":"xfs_trans_get_efd","module":"[xfs]"},{"addr":18446744072645589520,"func":"xfs_trans_get_buf_map","module":"[xfs]"},{"addr":18446744072645604128,"func":"xfs_trans_get_dqtrx","module":"[xfs]"}],"pids":[{"pid":559862,"comm":"kprobe_multi"}]},{"id":53,"type":"kprobe_multi","prog_id":381,"retprobe":1,"func_cnt":7,"funcs":[{"addr":18446744072078249760,"func":"schedule_timeout_interruptible","module":""},{"addr":18446744072078249824,"func":"schedule_timeout_killable","module":""},{"addr":18446744072078249888,"func":"schedule_timeout_uninterruptible","module":""},{"addr":18446744072078249952,"func":"schedule_timeout_idle","module":""},{"addr":18446744072645535952,"func":"xfs_trans_get_efd","module":"[xfs]"},{"addr":18446744072645589520,"func":"xfs_trans_get_buf_map","module":"[xfs]"},{"addr":18446744072645604128,"func":"xfs_trans_get_dqtrx","module":"[xfs]"}],"pids":[{"pid":559862,"comm":"kprobe_multi"}]}]

Signed-off-by: Yafang Shao <redacted>
---
  tools/bpf/bpftool/link.c | 109 ++++++++++++++++++++++++++++++++++++++++++++++-
  1 file changed, 108 insertions(+), 1 deletion(-)

diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
index 2d78607..0015582 100644
--- a/tools/bpf/bpftool/link.c
+++ b/tools/bpf/bpftool/link.c

@@ -14,8 +14,10 @@

  #include "json_writer.h"
  #include "main.h"
+#include "xlated_dumper.h"

  static struct hashmap *link_table;
+static struct dump_data dd = {};

  static int link_parse_fd(int *argc, char ***argv)
  {

@@ -166,6 +168,45 @@ static int get_prog_info(int prog_id, struct bpf_prog_info *info)
      return err;
  }

+static int cmp_u64(const void *A, const void *B)
+{
+     const __u64 *a = A, *b = B;
+
+     return *a - *b;
+}
+
+static void
+show_kprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
+{
+     __u32 i, j = 0;
+     __u64 *addrs;
+
+     jsonw_uint_field(json_wtr, "retprobe",
+                      info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN);
+     jsonw_uint_field(json_wtr, "func_cnt", info->kprobe_multi.count);
+     jsonw_name(json_wtr, "funcs");
+     jsonw_start_array(json_wtr);
+     addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs);
+     qsort((void *)addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64);
+
+     /* Load it once for all. */
+     if (!dd.sym_count)
+             kernel_syms_load(&dd);
+     for (i = 0; i < dd.sym_count; i++) {
+             if (dd.sym_mapping[i].address != addrs[j])
+                     continue;
+             jsonw_start_object(json_wtr);
+             jsonw_uint_field(json_wtr, "addr", dd.sym_mapping[i].address);
+             jsonw_string_field(json_wtr, "func", dd.sym_mapping[i].name);
+             /* Print none if it is vmlinux */
+             jsonw_string_field(json_wtr, "module", dd.sym_mapping[i].module);
+             jsonw_end_object(json_wtr);
+             if (j++ == info->kprobe_multi.count)
+                     break;
+     }
+     jsonw_end_array(json_wtr);
+}
+
  static int show_link_close_json(int fd, struct bpf_link_info *info)
  {
      struct bpf_prog_info prog_info;

@@ -218,6 +259,9 @@ static int show_link_close_json(int fd, struct bpf_link_info *info)
              jsonw_uint_field(json_wtr, "map_id",
                               info->struct_ops.map_id);
              break;
+     case BPF_LINK_TYPE_KPROBE_MULTI:
+             show_kprobe_multi_json(info, json_wtr);
+             break;
      default:
              break;
      }

@@ -351,6 +395,44 @@ void netfilter_dump_plain(const struct bpf_link_info *info)
              printf(" flags 0x%x", info->netfilter.flags);
  }

+static void show_kprobe_multi_plain(struct bpf_link_info *info)
+{
+     __u32 i, j = 0;
+     __u64 *addrs;
+
+     if (!info->kprobe_multi.count)
+             return;
+
+     printf("\n\tretprobe %d  func_cnt %u  ",
+            info->kprobe_multi.flags & BPF_F_KPROBE_MULTI_RETURN,
+            info->kprobe_multi.count);
+     addrs = (__u64 *)u64_to_ptr(info->kprobe_multi.addrs);
+     qsort((void *)addrs, info->kprobe_multi.count, sizeof(__u64), cmp_u64);
+
+     /* Load it once for all. */
+     if (!dd.sym_count)
+             kernel_syms_load(&dd);
+     for (i = 0; i < dd.sym_count; i++) {
+             if (dd.sym_mapping[i].address != addrs[j])
+                     continue;
+             if (!j)
+                     printf("\n\taddrs %016lx  funcs %s",
+                            dd.sym_mapping[i].address,
+                            dd.sym_mapping[i].name);
+             else
+                     printf("\n\t      %016lx        %s",
+                            dd.sym_mapping[i].address,
+                            dd.sym_mapping[i].name);
+             if (dd.sym_mapping[i].module[0] != '\0')
+                     printf(" %s  ", dd.sym_mapping[i].module);
+             else
+                     printf("  ");

Could you explain what these extra spaces after module names are for?

There are two spaces. We use two spaces to seperate different items
printed in bpftool. For example,
  "4: kprobe_multi  prog 16"
There are two spaces between the "type" and the "prog".

We always print these two spaces after one item is printed:
  printf("type %u  ", info->type);
  printf("prog %u  ", info->prog_id);
That way, we can add new item easily and consistently.