[PATCH] files: Use rcu lock to get the file structures for better performance
From: Muchun Song <hidden>
Date: 2020-05-21 12:39:03
Also in:
bpf, linux-fsdevel, lkml
Subsystem:
bpf [core], bpf [general] (safe dynamic programs and tools), filesystems (vfs and infrastructure), proc filesystem, the rest · Maintainers:
Alexei Starovoitov, Daniel Borkmann, Andrii Nakryiko, Eduard Zingerman, Kumar Kartikeya Dwivedi, Alexander Viro, Christian Brauner, Linus Torvalds
There is another safe way to get the file structure without holding the files->file_lock. That is rcu lock, and this way has better performance. So use the rcu lock instead of the files->file_lock. Signed-off-by: Muchun Song <redacted> --- fs/proc/fd.c | 31 ++++++++++++++++++++++++------- kernel/bpf/syscall.c | 17 +++++++++++------ kernel/kcmp.c | 15 ++++++++++----- 3 files changed, 45 insertions(+), 18 deletions(-)
diff --git a/fs/proc/fd.c b/fs/proc/fd.c
index 81882a13212d3..5d5b0f091d32a 100644
--- a/fs/proc/fd.c
+++ b/fs/proc/fd.c@@ -34,19 +34,27 @@ static int seq_show(struct seq_file *m, void *v) if (files) { unsigned int fd = proc_fd(m->private); - spin_lock(&files->file_lock); + rcu_read_lock(); +again: file = fcheck_files(files, fd); if (file) { - struct fdtable *fdt = files_fdtable(files); + struct fdtable *fdt; + + if (!get_file_rcu(file)) { + /* + * we loop to catch the new file (or NULL + * pointer). + */ + goto again; + } + fdt = files_fdtable(files); f_flags = file->f_flags; if (close_on_exec(fd, fdt)) f_flags |= O_CLOEXEC; - - get_file(file); ret = 0; } - spin_unlock(&files->file_lock); + rcu_read_unlock(); put_files_struct(files); }
@@ -160,14 +168,23 @@ static int proc_fd_link(struct dentry *dentry, struct path *path) unsigned int fd = proc_fd(d_inode(dentry)); struct file *fd_file; - spin_lock(&files->file_lock); + rcu_read_lock(); +again: fd_file = fcheck_files(files, fd); if (fd_file) { + if (!get_file_rcu(fd_file)) { + /* + * we loop to catch the new file + * (or NULL pointer). + */ + goto again; + } *path = fd_file->f_path; path_get(&fd_file->f_path); + fput(fd_file); ret = 0; } - spin_unlock(&files->file_lock); + rcu_read_unlock(); put_files_struct(files); }
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 8608d6e1b0e0e..441c91378a1fc 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c@@ -3451,14 +3451,19 @@ static int bpf_task_fd_query(const union bpf_attr *attr, if (!files) return -ENOENT; - err = 0; - spin_lock(&files->file_lock); + rcu_read_lock(); +again: file = fcheck_files(files, fd); - if (!file) + if (file) { + if (!get_file_rcu(file)) { + /* we loop to catch the new file (or NULL pointer) */ + goto again; + } + err = 0; + } else { err = -EBADF; - else - get_file(file); - spin_unlock(&files->file_lock); + } + rcu_read_unlock(); put_files_struct(files); if (err)
diff --git a/kernel/kcmp.c b/kernel/kcmp.c
index b3ff9288c6cc9..3b4f2a54186f2 100644
--- a/kernel/kcmp.c
+++ b/kernel/kcmp.c@@ -120,13 +120,18 @@ static int kcmp_epoll_target(struct task_struct *task1, if (!files) return -EBADF; - spin_lock(&files->file_lock); + rcu_read_lock(); +again: filp_epoll = fcheck_files(files, slot.efd); - if (filp_epoll) - get_file(filp_epoll); - else + if (filp_epoll) { + if (!get_file_rcu(filp_epoll)) { + /* we loop to catch the new file (or NULL pointer) */ + goto again; + } + } else { filp_tgt = ERR_PTR(-EBADF); - spin_unlock(&files->file_lock); + } + rcu_read_unlock(); put_files_struct(files); if (filp_epoll) {
--
2.11.0