---
v1->v2:
	keep a pointer to the struct perf_sample_data rather than directly
	embedding it in the structure, avoiding the circular include and
	removing the need for in_use.  Suggested by Song.

include/linux/bpf.h      |  1 +
kernel/bpf/arraymap.c    |  3 ++-
kernel/trace/bpf_trace.c | 29 ++++++++++++++++-------------
3 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 4fb3aa2dc975..47fd85cfbbaf 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h

@@ -472,6 +472,7 @@ struct bpf_event_entry {

	struct file *perf_file;
	struct file *map_file;
	struct rcu_head rcu;
+	struct perf_sample_data *sd;
};

bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);

diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index 584636c9e2eb..c7f5d593e04f 100644
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c

@@ -654,11 +654,12 @@ static struct bpf_event_entry *bpf_event_entry_gen(struct file *perf_file,

{
	struct bpf_event_entry *ee;

-	ee = kzalloc(sizeof(*ee), GFP_ATOMIC);
+	ee = kzalloc(sizeof(*ee) + sizeof(struct perf_sample_data), GFP_ATOMIC);
	if (ee) {
		ee->event = perf_file->private_data;
		ee->perf_file = perf_file;
		ee->map_file = map_file;
+		ee->sd = (void *)ee + sizeof(*ee);
	}

	return ee;

diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index f92d6ad5e080..076f8e987355 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c

@@ -410,17 +410,17 @@ static const struct bpf_func_proto bpf_perf_event_read_value_proto = {

	.arg4_type	= ARG_CONST_SIZE,
};

-static DEFINE_PER_CPU(struct perf_sample_data, bpf_trace_sd);
-
static __always_inline u64
__bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
-			u64 flags, struct perf_sample_data *sd)
+			u64 flags, struct perf_raw_record *raw)
{
	struct bpf_array *array = container_of(map, struct bpf_array, map);
	unsigned int cpu = smp_processor_id();
	u64 index = flags & BPF_F_INDEX_MASK;
	struct bpf_event_entry *ee;
	struct perf_event *event;
+	struct perf_sample_data *sd;
+	u64 ret;

	if (index == BPF_F_CURRENT_CPU)
		index = cpu;

@@ -439,13 +439,22 @@ __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,

	if (unlikely(event->oncpu != cpu))
		return -EOPNOTSUPP;

-	return perf_event_output(event, sd, regs);
+	sd = xchg(&ee->sd, NULL);
+	if (!sd)
+		return -EBUSY;
+
+	perf_sample_data_init(sd, 0, 0);
+	sd->raw = raw;
+
+	ret = perf_event_output(event, sd, regs);
+
+	xchg(&ee->sd, sd);
+	return ret;
}

BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,
	   u64, flags, void *, data, u64, size)
{
-	struct perf_sample_data *sd = this_cpu_ptr(&bpf_trace_sd);
	struct perf_raw_record raw = {
		.frag = {
			.size = size,

@@ -456,10 +465,8 @@ BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,

	if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
		return -EINVAL;

-	perf_sample_data_init(sd, 0, 0);
-	sd->raw = &raw;

-	return __bpf_perf_event_output(regs, map, flags, sd);
+	return __bpf_perf_event_output(regs, map, flags, &raw);
}

static const struct bpf_func_proto bpf_perf_event_output_proto = {

@@ -474,12 +481,10 @@ static const struct bpf_func_proto bpf_perf_event_output_proto = {

};

static DEFINE_PER_CPU(struct pt_regs, bpf_pt_regs);
-static DEFINE_PER_CPU(struct perf_sample_data, bpf_misc_sd);

u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
		     void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)
{
-	struct perf_sample_data *sd = this_cpu_ptr(&bpf_misc_sd);
	struct pt_regs *regs = this_cpu_ptr(&bpf_pt_regs);
	struct perf_raw_frag frag = {
		.copy		= ctx_copy,

@@ -497,10 +502,8 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,

	};

	perf_fetch_caller_regs(regs);
-	perf_sample_data_init(sd, 0, 0);
-	sd->raw = &raw;

-	return __bpf_perf_event_output(regs, map, flags, sd);
+	return __bpf_perf_event_output(regs, map, flags, &raw);
}

BPF_CALL_0(bpf_get_current_task)
-- 
2.17.1