[PATCH] tracing: Fix use-after-free and double-free on last_cmd

From: "Tze-nan Wu" <redacted>

Currently, the last_cmd variable can be accessed by multiple processes
asynchronously when multiple users manipulate synthetic_events node
at the same time, it could lead to use-after-free or double-free.

This patch add lastcmd_mutex to prevent last_cmd from being accessed
asynchronously.

================================================================

It's easy to reproduce in the KASAN environment by running the two
scripts below in different shells.

script 1:
        while :
        do
                echo -n -e '\x88' > /sys/kernel/tracing/synthetic_events
        done

script 2:
        while :
        do
                echo -n -e '\xb0' > /sys/kernel/tracing/synthetic_events
        done

================================================================
double-free scenario:

    process A                       process B
-------------------               ---------------
1.kstrdup last_cmd
                                  2.free last_cmd
3.free last_cmd(double-free)

================================================================
use-after-free scenario:

    process A                       process B
-------------------               ---------------
1.kstrdup last_cmd
                                  2.free last_cmd
3.tracing_log_err(use-after-free)

================================================================

Appendix 1. KASAN report double-free:

BUG: KASAN: double-free in kfree+0xdc/0x1d4
Free of addr ***** by task sh/4879
Call trace:
        ...
        kfree+0xdc/0x1d4
        create_or_delete_synth_event+0x60/0x1e8
        trace_parse_run_command+0x2bc/0x4b8
        synth_events_write+0x20/0x30
        vfs_write+0x200/0x830
        ...

Allocated by task 4879:
        ...
        kstrdup+0x5c/0x98
        create_or_delete_synth_event+0x6c/0x1e8
        trace_parse_run_command+0x2bc/0x4b8
        synth_events_write+0x20/0x30
        vfs_write+0x200/0x830
        ...

Freed by task 5464:
        ...
        kfree+0xdc/0x1d4
        create_or_delete_synth_event+0x60/0x1e8
        trace_parse_run_command+0x2bc/0x4b8
        synth_events_write+0x20/0x30
        vfs_write+0x200/0x830
        ...

================================================================
Appendix 2. KASAN report use-after-free:

BUG: KASAN: use-after-free in strlen+0x5c/0x7c
Read of size 1 at addr ***** by task sh/5483
sh: CPU: 7 PID: 5483 Comm: sh
        ...
        __asan_report_load1_noabort+0x34/0x44
        strlen+0x5c/0x7c
        tracing_log_err+0x60/0x444
        create_or_delete_synth_event+0xc4/0x204
        trace_parse_run_command+0x2bc/0x4b8
        synth_events_write+0x20/0x30
        vfs_write+0x200/0x830
        ...

Allocated by task 5483:
        ...
        kstrdup+0x5c/0x98
        create_or_delete_synth_event+0x80/0x204
        trace_parse_run_command+0x2bc/0x4b8
        synth_events_write+0x20/0x30
        vfs_write+0x200/0x830
        ...

Freed by task 5480:
        ...
        kfree+0xdc/0x1d4
        create_or_delete_synth_event+0x74/0x204
        trace_parse_run_command+0x2bc/0x4b8
        synth_events_write+0x20/0x30
        vfs_write+0x200/0x830
        ...

Fixes: 27c888da9867 ("tracing: Remove size restriction on synthetic event cmd error logging")
Cc: stable@vger.kernel.org
Signed-off-by: Tze-nan Wu <redacted>
---
 kernel/trace/trace_events_synth.c | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/kernel/trace/trace_events_synth.c b/kernel/trace/trace_events_synth.c
index 46d0abb32d0f..ce438eccab2e 100644
--- a/kernel/trace/trace_events_synth.c
+++ b/kernel/trace/trace_events_synth.c

@@ -42,16 +42,25 @@ enum { ERRORS };
 #undef C
 #define C(a, b)		b
 
+static DEFINE_MUTEX(lastcmd_mutex);
+
 static const char *err_text[] = { ERRORS };
 
 static char *last_cmd;
 
 static int errpos(const char *str)
 {
-	if (!str || !last_cmd)
-		return 0;
+	int ret = 0;
+
+	mutex_lock(&lastcmd_mutex);
+	if (!str || !last_cmd) {
+		mutex_unlock(&lastcmd_mutex);
+		return ret;
+	}
 
-	return err_pos(last_cmd, str);
+	ret = err_pos(last_cmd, str);
+	mutex_unlock(&lastcmd_mutex);
+	return ret;
 }
 
 static void last_cmd_set(const char *str)

@@ -59,18 +68,24 @@ static void last_cmd_set(const char *str)
 	if (!str)
 		return;
 
+	mutex_lock(&lastcmd_mutex);
 	kfree(last_cmd);
 
 	last_cmd = kstrdup(str, GFP_KERNEL);
+	mutex_unlock(&lastcmd_mutex);
 }
 
 static void synth_err(u8 err_type, u16 err_pos)
 {
-	if (!last_cmd)
+	mutex_lock(&lastcmd_mutex);
+	if (!last_cmd) {
+		mutex_unlock(&lastcmd_mutex);
 		return;
+	}
 
 	tracing_log_err(NULL, "synthetic_events", last_cmd, err_text,
 			err_type, err_pos);
+	mutex_unlock(&lastcmd_mutex);
 }
 
 static int create_synth_event(const char *raw_command);

-- 
2.18.0

On Fri, 17 Mar 2023 13:30:44 +0800
Cheng-Jui Wang [off-list ref] wrote:

From: "Tze-nan Wu" <redacted>

Hi!

Thanks for the report and the patch. Some nits below.

Also change the subject to:

	tracing/synthetic: Fix races on freeing last_cmd

Please keep the mutex and the variable it protects next to each other:

static DEFINE_MUTEX(lastcmd_mutex);
static char *last_cmd;

 
 static int errpos(const char *str)
 {
-	if (!str || !last_cmd)
-		return 0;
+	int ret = 0;
+
+	mutex_lock(&lastcmd_mutex);
+	if (!str || !last_cmd) {

Change this to just:

	if (!str || !last_cmd)
		goto out;

+		mutex_unlock(&lastcmd_mutex);
+		return ret;

+	}
 
-	return err_pos(last_cmd, str);
+	ret = err_pos(last_cmd, str);

Add:

 out:

In this case, you can remove the space:

	mutex_lock(&lastcmd_mutex);
	kfree(last_cmd);
	last_cmd = kstrdup(str, GFP_KERNEL);
	mutex_unlock(&lastcmd_mutex);

 	last_cmd = kstrdup(str, GFP_KERNEL);
+	mutex_unlock(&lastcmd_mutex);
 }
 
 static void synth_err(u8 err_type, u16 err_pos)
 {
-	if (!last_cmd)
+	mutex_lock(&lastcmd_mutex);
+	if (!last_cmd) {

This should be:

	if (!last_cmd)
		goto out;

+		mutex_unlock(&lastcmd_mutex);
 		return;
+	}
 
 	tracing_log_err(NULL, "synthetic_events", last_cmd, err_text,
 			err_type, err_pos);

 out:

+	mutex_unlock(&lastcmd_mutex);
 }
 
 static int create_synth_event(const char *raw_command);

Thanks,

-- Steve

On Sat, 2023-03-18 at 14:35 -0400, Steven Rostedt wrote:

On Fri, 17 Mar 2023 13:30:44 +0800
Cheng-Jui Wang [off-list ref] wrote:

quoted

From: "Tze-nan Wu" <redacted>

Hi!

Thanks for the report and the patch. Some nits below.

Also change the subject to:

	tracing/synthetic: Fix races on freeing last_cmd

quoted

---
 kernel/trace/trace_events_synth.c | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/kernel/trace/trace_events_synth.c

b/kernel/trace/trace_events_synth.c
index 46d0abb32d0f..ce438eccab2e 100644

--- a/kernel/trace/trace_events_synth.c
+++ b/kernel/trace/trace_events_synth.c

@@ -42,16 +42,25 @@ enum { ERRORS };
 #undef C
 #define C(a, b)		b
 
+static DEFINE_MUTEX(lastcmd_mutex);
+
 static const char *err_text[] = { ERRORS };
 
 static char *last_cmd;

Please keep the mutex and the variable it protects next to each
other:

static DEFINE_MUTEX(lastcmd_mutex);
static char *last_cmd;

quoted

 
 static int errpos(const char *str)
 {
-	if (!str || !last_cmd)
-		return 0;
+	int ret = 0;
+
+	mutex_lock(&lastcmd_mutex);
+	if (!str || !last_cmd) {

Change this to just:

	if (!str || !last_cmd)
		goto out;

quoted

+		mutex_unlock(&lastcmd_mutex);
+		return ret;
+	}
 
-	return err_pos(last_cmd, str);
+	ret = err_pos(last_cmd, str);

Add:

 out:

quoted

+	mutex_unlock(&lastcmd_mutex);
+	return ret;
 }
 
 static void last_cmd_set(const char *str)

@@ -59,18 +68,24 @@ static void last_cmd_set(const char *str)
 	if (!str)
 		return;
 
+	mutex_lock(&lastcmd_mutex);
 	kfree(last_cmd);
 

In this case, you can remove the space:

	mutex_lock(&lastcmd_mutex);
	kfree(last_cmd);
	last_cmd = kstrdup(str, GFP_KERNEL);
	mutex_unlock(&lastcmd_mutex);

quoted

 	last_cmd = kstrdup(str, GFP_KERNEL);
+	mutex_unlock(&lastcmd_mutex);
 }
 
 static void synth_err(u8 err_type, u16 err_pos)
 {
-	if (!last_cmd)
+	mutex_lock(&lastcmd_mutex);
+	if (!last_cmd) {

This should be:

	if (!last_cmd)
		goto out;

quoted

+		mutex_unlock(&lastcmd_mutex);
 		return;
+	}
 
 	tracing_log_err(NULL, "synthetic_events", last_cmd, err_text,
 			err_type, err_pos);

 out:

quoted

+	mutex_unlock(&lastcmd_mutex);
 }
 
 static int create_synth_event(const char *raw_command);

Thanks,

-- Steve

Hi Steve,

Thanks for the comments,
the new patch with cleaner code is now ready.

Since the topic has been changed, I created a new thread for the new
patch:

https://lore.kernel.org/lkml/20230321110444.1587-1-Tze-nan.Wu@mediatek.com/

-- Tze-nan