diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 5d616735fe1b..dbedbcdc8122 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -980,6 +980,7 @@ enum bpf_attach_type {
BPF_SK_LOOKUP,
BPF_XDP,
BPF_SK_SKB_VERDICT,
+ BPF_TRACE_FTRACE_ENTRY,
__MAX_BPF_ATTACH_TYPE
};
@@ -993,6 +994,7 @@ enum bpf_link_type {
BPF_LINK_TYPE_ITER = 4,
BPF_LINK_TYPE_NETNS = 5,
BPF_LINK_TYPE_XDP = 6,
+ BPF_LINK_TYPE_FTRACE = 7,
MAX_BPF_LINK_TYPE,
};@@ -1427,6 +1429,7 @@ union bpf_attr {
__aligned_u64 iter_info; /* extra bpf_iter_link_info */
__u32 iter_info_len; /* iter_info length */
};
+ __u32 funcs_fd;
};
} link_create;
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index b240a500cae5..c83515d41020 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1965,6 +1965,11 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
prog_type != BPF_PROG_TYPE_EXT)
return -EINVAL;
+ if (prog_type == BPF_PROG_TYPE_TRACING &&
+ expected_attach_type == BPF_TRACE_FTRACE_ENTRY &&
+ !IS_ENABLED(CONFIG_FUNCTION_TRACER))
+ return -EINVAL;
+
switch (prog_type) {
case BPF_PROG_TYPE_CGROUP_SOCK:
switch (expected_attach_type) {@@ -2861,6 +2866,144 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog,
return err;
}
+#ifdef CONFIG_FUNCTION_TRACER
+struct bpf_tracing_ftrace_link {
+ struct bpf_link link;
+ enum bpf_attach_type attach_type;
+ struct ftrace_ops ops;
+};
+
+static void bpf_tracing_ftrace_link_release(struct bpf_link *link)
+{
+ struct bpf_tracing_ftrace_link *tr_link =
+ container_of(link, struct bpf_tracing_ftrace_link, link);
+
+ WARN_ON(unregister_ftrace_function(&tr_link->ops));
+}
+
+static void bpf_tracing_ftrace_link_dealloc(struct bpf_link *link)
+{
+ struct bpf_tracing_ftrace_link *tr_link =
+ container_of(link, struct bpf_tracing_ftrace_link, link);
+
+ kfree(tr_link);
+}
+
+static void bpf_tracing_ftrace_link_show_fdinfo(const struct bpf_link *link,
+ struct seq_file *seq)
+{
+ struct bpf_tracing_ftrace_link *tr_link =
+ container_of(link, struct bpf_tracing_ftrace_link, link);
+
+ seq_printf(seq,
+ "attach_type:\t%d\n",
+ tr_link->attach_type);
+}
+
+static int bpf_tracing_ftrace_link_fill_link_info(const struct bpf_link *link,
+ struct bpf_link_info *info)
+{
+ struct bpf_tracing_ftrace_link *tr_link =
+ container_of(link, struct bpf_tracing_ftrace_link, link);
+
+ info->tracing.attach_type = tr_link->attach_type;
+ return 0;
+}
+
+static const struct bpf_link_ops bpf_tracing_ftrace_lops = {
+ .release = bpf_tracing_ftrace_link_release,
+ .dealloc = bpf_tracing_ftrace_link_dealloc,
+ .show_fdinfo = bpf_tracing_ftrace_link_show_fdinfo,
+ .fill_link_info = bpf_tracing_ftrace_link_fill_link_info,
+};
+
+static void
+bpf_ftrace_function_call(unsigned long ip, unsigned long parent_ip,
+ struct ftrace_ops *ops, struct ftrace_regs *fregs)
+{
+ struct bpf_tracing_ftrace_link *tr_link;
+ struct bpf_prog *prog;
+ u64 start;
+
+ tr_link = container_of(ops, struct bpf_tracing_ftrace_link, ops);
+ prog = tr_link->link.prog;
+
+ if (prog->aux->sleepable)
+ start = __bpf_prog_enter_sleepable(prog);
+ else
+ start = __bpf_prog_enter(prog);
+
+ if (start)
+ bpf_trace_run2(tr_link->link.prog, ip, parent_ip);
+
+ if (prog->aux->sleepable)
+ __bpf_prog_exit_sleepable(prog, start);
+ else
+ __bpf_prog_exit(prog, start);
+}
+
+static int bpf_tracing_ftrace_attach(struct bpf_prog *prog, int funcs_fd)
+{
+ struct bpf_tracing_ftrace_link *link;
+ struct bpf_link_primer link_primer;
+ struct bpf_functions *funcs;
+ struct ftrace_ops *ops;
+ int err = -ENOMEM;
+ struct fd orig;
+ int i;
+
+ if (prog->type != BPF_PROG_TYPE_TRACING)
+ return -EINVAL;
+
+ if (prog->expected_attach_type != BPF_TRACE_FTRACE_ENTRY)
+ return -EINVAL;
+
+ funcs = bpf_functions_get_from_fd(funcs_fd, &orig);
+ if (IS_ERR(funcs))
+ return PTR_ERR(funcs);
+
+ link = kzalloc(sizeof(*link), GFP_USER);
+ if (!link)
+ goto out_free;
+
+ ops = &link->ops;
+ ops->func = bpf_ftrace_function_call;
+ ops->flags = FTRACE_OPS_FL_DYNAMIC;
+
+ bpf_link_init(&link->link, BPF_LINK_TYPE_FTRACE,
+ &bpf_tracing_ftrace_lops, prog);
+ link->attach_type = prog->expected_attach_type;
+
+ err = bpf_link_prime(&link->link, &link_primer);
+ if (err)
+ goto out_free;
+
+ for (i = 0; i < funcs->cnt; i++) {
+ err = ftrace_set_filter_ip(ops, funcs->addrs[i], 0, 0);
+ if (err)
+ goto out_free;
+ }
+
+ err = register_ftrace_function(ops);
+ if (err)
+ goto out_free;
+
+ fdput(orig);
+ return bpf_link_settle(&link_primer);
+
+out_free:
+ kfree(link);
+ fdput(orig);
+ return err;
+}
+#else
+static int bpf_tracing_ftrace_attach(struct bpf_prog *prog __maybe_unused,
+ int funcs_fd __maybe_unused)
+{
+ return -ENODEV;
+}
+#endif /* CONFIG_FUNCTION_TRACER */
+
struct bpf_raw_tp_link {
struct bpf_link link;
struct bpf_raw_event_map *btp;@@ -3093,6 +3236,7 @@ attach_type_to_prog_type(enum bpf_attach_type attach_type)
case BPF_CGROUP_GETSOCKOPT:
case BPF_CGROUP_SETSOCKOPT:
return BPF_PROG_TYPE_CGROUP_SOCKOPT;
+ case BPF_TRACE_FTRACE_ENTRY:
case BPF_TRACE_ITER:
return BPF_PROG_TYPE_TRACING;
case BPF_SK_LOOKUP:
@@ -4149,6 +4293,9 @@ static int tracing_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *
if (prog->expected_attach_type == BPF_TRACE_ITER)
return bpf_iter_link_attach(attr, prog);
+ else if (prog->expected_attach_type == BPF_TRACE_FTRACE_ENTRY)
+ return bpf_tracing_ftrace_attach(prog,
+ attr->link_create.funcs_fd);
else if (prog->type == BPF_PROG_TYPE_EXT)
return bpf_tracing_prog_attach(prog,
attr->link_create.target_fd,
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 852541a435ef..ea001aec66f6 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -8946,6 +8946,7 @@ static int check_return_code(struct bpf_verifier_env *env)
break;
case BPF_TRACE_RAW_TP:
case BPF_MODIFY_RETURN:
+ case BPF_TRACE_FTRACE_ENTRY:
return 0;
case BPF_TRACE_ITER:
break;
@@ -12794,6 +12795,14 @@ static int check_non_sleepable_error_inject(u32 btf_id)
return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);
}
+__maybe_unused
+void bpf_ftrace_probe(unsigned long ip __maybe_unused,
+ unsigned long parent_ip __maybe_unused)
+{
+}
+
+BTF_ID_LIST_SINGLE(btf_ftrace_probe_id, func, bpf_ftrace_probe);
+
int bpf_check_attach_target(struct bpf_verifier_log *log,
const struct bpf_prog *prog,
const struct bpf_prog *tgt_prog,@@ -13021,6 +13030,25 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
}
break;
+ case BPF_TRACE_FTRACE_ENTRY:
+ if (tgt_prog) {
+ bpf_log(log,
+ "Only FENTRY/FEXIT progs are attachable to another BPF prog\n");
+ return -EINVAL;
+ }
+ if (btf_id != btf_ftrace_probe_id[0]) {
+ bpf_log(log,
+ "Only btf id '%d' allowed for ftrace probe\n",
+ btf_ftrace_probe_id[0]);
+ return -EINVAL;
+ }
+ t = btf_type_by_id(btf, t->type);
+ if (!btf_type_is_func_proto(t))
+ return -EINVAL;
+ ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
+ if (ret < 0)
+ return ret;
+ break;
}
tgt_info->tgt_addr = addr;
tgt_info->tgt_name = tname;@@ -13081,6 +13109,8 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
if (!bpf_iter_prog_supported(prog))
return -EINVAL;
return 0;
+ } else if (prog->expected_attach_type == BPF_TRACE_FTRACE_ENTRY) {
+ return 0;
}
if (prog->type == BPF_PROG_TYPE_LSM) {diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
index a5d72c48fb66..0a891c27bad0 100644
--- a/net/bpf/test_run.c
+++ b/net/bpf/test_run.c
@@ -285,6 +285,7 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog,
switch (prog->expected_attach_type) {
case BPF_TRACE_FENTRY:
case BPF_TRACE_FEXIT:
+ case BPF_TRACE_FTRACE_ENTRY:
if (bpf_fentry_test1(1) != 2 ||
bpf_fentry_test2(2, 3) != 5 ||
bpf_fentry_test3(4, 5, 6) != 15 ||diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 5d616735fe1b..dbedbcdc8122 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -980,6 +980,7 @@ enum bpf_attach_type {
BPF_SK_LOOKUP,
BPF_XDP,
BPF_SK_SKB_VERDICT,
+ BPF_TRACE_FTRACE_ENTRY,
__MAX_BPF_ATTACH_TYPE
};
@@ -993,6 +994,7 @@ enum bpf_link_type {
BPF_LINK_TYPE_ITER = 4,
BPF_LINK_TYPE_NETNS = 5,
BPF_LINK_TYPE_XDP = 6,
+ BPF_LINK_TYPE_FTRACE = 7,
MAX_BPF_LINK_TYPE,
};@@ -1427,6 +1429,7 @@ union bpf_attr {
__aligned_u64 iter_info; /* extra bpf_iter_link_info */
__u32 iter_info_len; /* iter_info length */
};
+ __u32 funcs_fd;
};
} link_create;
--
2.30.2