buffer::subbuf_order is protected by buffer::mutex, which cannot be
held on the write path. While reading it is safe because writers are
disabled during order changes, reading the order from commit_page (which
is pinned for the duration of the commit) avoids confusing context
analysis.
Because rb_subbuf_size() and rb_subbuf_capacity() are no longer on the
write hot path, drop their __always_inline annotations.
Signed-off-by: Vincent Donnefort <vdonnefort@google.com>
---
kernel/trace/ring_buffer.c | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 02d3cc193d1e..82f6e3df4627 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -606,7 +606,7 @@ struct trace_buffer {
unsigned int subbuf_order;
};
-static __always_inline unsigned int rb_subbuf_size(struct trace_buffer *buffer)
+static unsigned int rb_subbuf_size(struct trace_buffer *buffer)
{
return PAGE_SIZE << buffer->subbuf_order;
}@@ -620,7 +620,7 @@ static __always_inline unsigned int rb_subbuf_size(struct trace_buffer *buffer)
*
* Return: The maximum size available for events in a trace buffer subbuffer.
*/
-static __always_inline unsigned int rb_subbuf_capacity(struct trace_buffer *buffer)
+static unsigned int rb_subbuf_capacity(struct trace_buffer *buffer)
{
return rb_subbuf_size(buffer) - BUF_PAGE_HDR_SIZE;
}@@ -632,15 +632,15 @@ static unsigned long rb_calculate_nr_pages(struct trace_buffer *buffer, unsigned
/**
* rb_subbuf_start - Get the start address of a subbuffer
- * @buffer: A trace buffer
+ * @cpu_buffer: A per-CPU ring buffer
* @addr: An address of an event on a subbuffer
*
* Return: The start of the subbuffer for where @addr sits
*/
static __always_inline
-unsigned long rb_subbuf_start(struct trace_buffer *buffer, unsigned long addr)
+unsigned long rb_subbuf_start(struct ring_buffer_per_cpu *cpu_buffer, unsigned long addr)
{
- return addr & ~((unsigned long)(rb_subbuf_size(buffer) - 1));
+ return addr & ~((unsigned long)(PAGE_SIZE << cpu_buffer->commit_page->order) - 1);
}
static bool rb_is_static(struct ring_buffer_per_cpu *cpu_buffer)@@ -3582,7 +3582,7 @@ rb_event_index(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_event
{
unsigned long addr = (unsigned long)event;
- addr &= (unsigned long)rb_subbuf_size(cpu_buffer->buffer) - 1;
+ addr &= (unsigned long)(PAGE_SIZE << cpu_buffer->commit_page->order) - 1;
return addr - BUF_PAGE_HDR_SIZE;
}@@ -4168,7 +4168,7 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
new_index = rb_event_index(cpu_buffer, event);
old_index = new_index + rb_event_ts_length(event);
- addr = rb_subbuf_start(cpu_buffer->buffer, (unsigned long)event);
+ addr = rb_subbuf_start(cpu_buffer, (unsigned long)event);
bpage = READ_ONCE(cpu_buffer->tail_page);
@@ -5091,7 +5091,7 @@ rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer,
struct buffer_page *bpage = cpu_buffer->commit_page;
struct buffer_page *start;
- addr = rb_subbuf_start(cpu_buffer->buffer, addr);
+ addr = rb_subbuf_start(cpu_buffer, addr);
/* Do the likely case first */
if (likely(bpage->page == (void *)addr)) {--
2.56.0.567.gdb80ae8cd5-goog