diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 1fd28a72e6ee..73f31e6719f9 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -1306,6 +1306,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);
#define RB_PAGE_NORMAL 0UL
#define RB_PAGE_HEAD 1UL
#define RB_PAGE_UPDATE 2UL
+#define RB_PAGE_HEAD_UPDATE (RB_PAGE_HEAD | RB_PAGE_UPDATE)
#define RB_FLAG_MASK 3UL
@@ -1455,6 +1456,15 @@ static int rb_head_page_set_head(struct ring_buffer_per_cpu *cpu_buffer,
old_flag, RB_PAGE_HEAD);
}
+static int rb_head_page_set_head_update(struct ring_buffer_per_cpu *cpu_buffer,
+ struct buffer_page *head,
+ struct buffer_page *prev,
+ int old_flag)
+{
+ return rb_head_page_set(cpu_buffer, head, prev,
+ old_flag, RB_PAGE_HEAD_UPDATE);
+}
+
static int rb_head_page_set_normal(struct ring_buffer_per_cpu *cpu_buffer,
struct buffer_page *head,
struct buffer_page *prev,@@ -3699,6 +3709,11 @@ rb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,
type = rb_head_page_set_update(cpu_buffer, next_page, tail_page,
RB_PAGE_HEAD);
+ /* We have interrupted an update */
+ if (type == RB_PAGE_HEAD_UPDATE)
+ type = rb_head_page_set_update(cpu_buffer, next_page, tail_page,
+ RB_PAGE_HEAD_UPDATE);
+
/*
* type can be one of four:
* NORMAL - an interrupt already moved it for us
@@ -3711,6 +3726,7 @@ rb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,
*/
switch (type) {
+ case RB_PAGE_HEAD_UPDATE:
case RB_PAGE_HEAD:
/*
* We changed the head to UPDATE, thus@@ -3773,8 +3789,8 @@ rb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,
new_head = next_page;
rb_inc_page(&new_head);
- ret = rb_head_page_set_head(cpu_buffer, new_head, next_page,
- RB_PAGE_NORMAL);
+ ret = rb_head_page_set_head_update(cpu_buffer, new_head, next_page,
+ RB_PAGE_NORMAL);
/*
* Valid returns are:
@@ -3785,7 +3801,7 @@ rb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,
* the page forward again.
*/
switch (ret) {
- case RB_PAGE_HEAD:
+ case RB_PAGE_HEAD_UPDATE:
case RB_PAGE_NORMAL:
/* OK */
break;@@ -3816,7 +3832,7 @@ rb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,
buffer_tail_page != next_page)
rb_head_page_set_normal(cpu_buffer, new_head,
next_page,
- RB_PAGE_HEAD);
+ RB_PAGE_HEAD_UPDATE);
}
/*
@@ -3824,7 +3840,7 @@ rb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,
* changed the original pointer from HEAD to UPDATE),
* then it is up to us to reset it to NORMAL.
*/
- if (type == RB_PAGE_HEAD) {
+ if (type == RB_PAGE_HEAD || type == RB_PAGE_HEAD_UPDATE) {
ret = rb_head_page_set_normal(cpu_buffer, next_page,
tail_page,
RB_PAGE_UPDATE);@@ -3833,6 +3849,24 @@ rb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,
return -1;
}
+ if (type == RB_PAGE_HEAD) {
+ struct buffer_page *prev_page;
+
+ do {
+ /* Interrupts might have moved the head further */
+ while (!rb_is_head_page(new_head, new_head->list.prev))
+ rb_inc_page(&new_head);
+
+ prev_page = new_head;
+ rb_dec_page(&prev_page);
+
+ /* Try to release the reader */
+ ret = rb_head_page_set_head(cpu_buffer, new_head, prev_page,
+ RB_PAGE_HEAD_UPDATE);
+
+ } while (ret != (RB_PAGE_HEAD_UPDATE));
+ }
+
return 0;
}