[RFC PATCH 14/24] pidfd: create and execute spawn builder tasks

@@ -3,6 +3,7 @@
  * pidfd-backed process spawn builders
  */
 
+#include <linux/completion.h>
 #include <linux/cred.h>
 #include <linux/file.h>
 #include <linux/fs.h>

@@ -13,14 +14,50 @@
 #include <linux/pid_namespace.h>
 #include <linux/pidfd_spawn.h>
 #include <linux/pidfs.h>
+#include <linux/random.h>
 #include <linux/refcount.h>
+#include <linux/sched/mm.h>
 #include <linux/sched/signal.h>
+#include <linux/sched/task.h>
+#include <linux/security.h>
 #include <linux/slab.h>
 #include <linux/syscalls.h>
+#include <linux/task_work.h>
 #include <linux/uaccess.h>
 #include <uapi/linux/pidfd.h>
+#include <uapi/linux/wait.h>
+#include <trace/events/sched.h>
 
+#include "exec_internal.h"
+
+static inline void __user *pidfd_spawn_user_ptr(__u64 p)
+{
+	return u64_to_user_ptr(p);
+}
+
+static inline bool pidfd_spawn_user_ptr_fits(__u64 p)
+{
+#ifdef CONFIG_COMPAT
+	if (in_compat_syscall())
+		return p == (__u64)(compat_uptr_t)p;
+#endif
+	return p == (__u64)(unsigned long)p;
+}
+
+static inline struct user_arg_ptr pidfd_spawn_user_arg(__u64 p)
+{
+#ifdef CONFIG_COMPAT
+	if (in_compat_syscall())
+		return compat_arg((compat_uptr_t __user *)
+				  u64_to_user_ptr(p));
+#endif
+	return native_arg(u64_to_user_ptr(p));
+}
+
+#define PIDFD_SPAWN_EXIT_FAILURE	(127 << 8)
 #define PIDFD_SPAWN_MAX_CONFIG_KEY_SIZE	256
+DEFINE_FREE(pidfd_spawn_dismiss_filename, struct delayed_filename,
+	    dismiss_delayed_filename(&_T))
 
 enum pidfd_spawn_status {
 	PIDFD_SPAWN_CONFIGURING,

@@ -31,14 +68,20 @@ enum pidfd_spawn_status {
 };
 
 struct pidfd_spawn_state {
-	/* Serializes configuration and payload teardown. */
+	/* Serializes configuration, launch, cancellation, and payload teardown. */
 	struct mutex lock;
+	struct completion done;
+	struct callback_head task_work;
 	refcount_t count;
 	struct pid *pid;
 	struct mm_struct *creator_mm;
 	const struct cred *creator_cred;
 	struct pid_namespace *creator_pid_ns;
+	struct delayed_filename filename;
 	char *staged_path;
+	struct user_arg_ptr argv;
+	struct user_arg_ptr envp;
+	int result;
 	enum pidfd_spawn_status status;
 };
 

@@ -80,6 +123,10 @@ pidfd_spawn_configuring_error(const struct pidfd_spawn_state *state)
 
 static void pidfd_spawn_free_state(struct pidfd_spawn_state *state)
 {
+	if (!state)
+		return;
+
+	dismiss_delayed_filename(&state->filename);
 	kfree(state->staged_path);
 	if (state->creator_mm)
 		mmdrop(state->creator_mm);

@@ -136,11 +183,73 @@ pidfd_spawn_state_get_file(struct file *file)
 	return state;
 }
 
+static void pidfd_spawn_drop_run_data(struct pidfd_spawn_state *state,
+				      struct filename *filename, int result)
+{
+	const struct cred *creator_cred;
+	struct mm_struct *creator_mm;
+	struct pid_namespace *creator_pid_ns;
+	char *staged_path;
+
+	/*
+	 * Run data is one-shot. Detach it under the lock, then drop refs
+	 * after the child no longer needs the shared setup payload.
+	 */
+	mutex_lock(&state->lock);
+	creator_mm = state->creator_mm;
+	creator_cred = state->creator_cred;
+	creator_pid_ns = state->creator_pid_ns;
+	staged_path = state->staged_path;
+	state->creator_mm = NULL;
+	state->creator_cred = NULL;
+	state->creator_pid_ns = NULL;
+	state->staged_path = NULL;
+	state->argv = native_arg(NULL);
+	state->envp = native_arg(NULL);
+	state->result = result;
+	pidfd_spawn_set_status(state, PIDFD_SPAWN_SETUP_DONE);
+	mutex_unlock(&state->lock);
+
+	putname(filename);
+	if (creator_mm)
+		mmdrop(creator_mm);
+	if (creator_cred)
+		put_cred(creator_cred);
+	if (creator_pid_ns)
+		put_pid_ns(creator_pid_ns);
+	kfree(staged_path);
+}
+
+static bool pidfd_spawn_same_pid_ns(struct pidfd_spawn_state *state)
+{
+	return current->nsproxy->pid_ns_for_children == state->creator_pid_ns;
+}
+
 static bool pidfd_spawn_same_creator(struct pidfd_spawn_state *state)
 {
+	/*
+	 * Holding the pidfd alone is not enough authority to configure or start
+	 * this builder. A caller must also share the creator's mm, credential
+	 * object, and child PID namespace.
+	 *
+	 * The child PID namespace is part of the builder authority even though no
+	 * numeric PID is allocated until run.
+	 */
 	return current->mm == state->creator_mm &&
 	       current_cred() == state->creator_cred &&
-	       current->nsproxy->pid_ns_for_children == state->creator_pid_ns;
+	       pidfd_spawn_same_pid_ns(state);
+}
+
+static int pidfd_spawn_check_startable(struct pidfd_spawn_state *state)
+{
+	int ret;
+
+	scoped_cond_guard(mutex_intr, return -EINTR, &state->lock) {
+		ret = pidfd_spawn_configuring_error(state);
+		if (!ret && !pidfd_spawn_same_creator(state))
+			ret = -EPERM;
+	}
+	return ret;
 }
 
 static struct filename *pidfd_spawn_get_path(const char __user *value)

@@ -153,6 +262,31 @@ static struct filename *pidfd_spawn_get_path(const char __user *value)
 	return no_free_ptr(name);
 }
 
+static int
+pidfd_spawn_get_delayed_path(struct delayed_filename *delayed,
+			     const char __user *value)
+{
+	struct filename *name __free(putname) = NULL;
+
+	name = pidfd_spawn_get_path(value);
+	if (IS_ERR(name))
+		return PTR_ERR(name);
+
+	return putname_to_delayed(delayed, no_free_ptr(name));
+}
+
+static int
+pidfd_spawn_get_delayed_kernel_path(struct delayed_filename *delayed,
+				    const char *value)
+{
+	struct filename *name __free(putname) = getname_kernel(value);
+
+	if (IS_ERR(name))
+		return PTR_ERR(name);
+
+	return putname_to_delayed(delayed, no_free_ptr(name));
+}
+
 static char *pidfd_spawn_copy_path(const char __user *value)
 {
 	struct filename *name __free(putname) = NULL;

@@ -173,14 +307,14 @@ static int pidfd_spawn_state_set_path(struct pidfd_spawn_state *state,
 {
 	char *path __free(kfree) = NULL;
 	char *old __free(kfree) = NULL;
+	int ret;
 
 	path = pidfd_spawn_copy_path(value);
 	if (IS_ERR(path))
 		return PTR_ERR(path);
 
 	scoped_cond_guard(mutex_intr, return -EINTR, &state->lock) {
-		int ret = pidfd_spawn_configuring_error(state);
-
+		ret = pidfd_spawn_configuring_error(state);
 		if (ret)
 			return ret;
 		if (!pidfd_spawn_same_creator(state))

@@ -243,6 +377,230 @@ SYSCALL_DEFINE5(pidfd_config, int, fd, unsigned int, cmd,
 	return ret;
 }
 
+static void pidfd_spawn_finish_child(struct pidfd_spawn_state *state,
+				     struct filename *filename, int result)
+{
+	pidfd_spawn_drop_run_data(state, filename, result);
+	complete_all(&state->done);
+}
+
+static void pidfd_spawn_child(struct callback_head *work)
+{
+	struct pidfd_spawn_state *state =
+		container_of(work, struct pidfd_spawn_state, task_work);
+	struct filename *filename;
+	int ret;
+
+	filename = complete_getname(&state->filename);
+	if (IS_ERR(filename))
+		ret = PTR_ERR(filename);
+	else
+		ret = 0;
+
+	if (!ret)
+		ret = do_execveat_common(AT_FDCWD, filename,
+					 state->argv, state->envp, 0);
+	if (!ret)
+		task_clear_embryonic_exec(current);
+
+	pidfd_spawn_finish_child(state, filename, ret);
+	if (ret) {
+		pidfd_spawn_state_put(state);
+		do_group_exit(PIDFD_SPAWN_EXIT_FAILURE);
+	}
+	pidfd_spawn_state_put(state);
+}
+
+static int pidfd_spawn_copy_run_args(struct pidfd_spawn_run_args *kargs,
+				     struct pidfd_spawn_run_args __user *uargs,
+				     size_t usize)
+{
+	int ret;
+
+	BUILD_BUG_ON(sizeof(struct pidfd_spawn_run_args) !=
+		     PIDFD_SPAWN_RUN_SIZE_VER0);
+
+	if (usize < PIDFD_SPAWN_RUN_SIZE_VER0)
+		return -EINVAL;
+	if (usize > PAGE_SIZE)
+		return -E2BIG;
+	ret = copy_struct_from_user(kargs, sizeof(*kargs), uargs, usize);
+	if (ret)
+		return ret;
+	if (kargs->flags || kargs->reserved0 || kargs->reserved[0] ||
+	    kargs->reserved[1])
+		return -EINVAL;
+	if (!kargs->argv)
+		return -EINVAL;
+	if (kargs->nr_actions || kargs->actions || kargs->action_size)
+		return -EINVAL;
+	if (!pidfd_spawn_user_ptr_fits(kargs->path) ||
+	    !pidfd_spawn_user_ptr_fits(kargs->argv) ||
+	    !pidfd_spawn_user_ptr_fits(kargs->envp) ||
+	    !pidfd_spawn_user_ptr_fits(kargs->actions))
+		return -EFAULT;
+
+	return 0;
+}
+
+static struct task_struct *
+pidfd_spawn_create_task(struct file *file, struct pidfd_spawn_state *state,
+			struct kernel_clone_args *clone_args)
+{
+	struct pid *pid;
+	struct task_struct *task;
+	u64 ino;
+	int ret;
+
+	ino = pidfs_future_file_ino(file);
+	if (!ino)
+		return ERR_PTR(-EBADF);
+
+	pid = alloc_pid_with_pidfs_ino(current->nsproxy->pid_ns_for_children,
+				       NULL, 0, ino);
+	if (IS_ERR(pid))
+		return ERR_CAST(pid);
+
+	ret = pidfs_future_file_set_pid(file, pid);
+	if (ret) {
+		free_pid(pid);
+		return ERR_PTR(ret);
+	}
+
+	refcount_inc(&state->count);
+	task = copy_process(pid, 0, NUMA_NO_NODE, clone_args);
+	add_latent_entropy();
+	if (IS_ERR(task)) {
+		pidfd_spawn_state_put(state);
+		pidfs_future_file_clear_pid(file, pid);
+		free_pid(pid);
+		return task;
+	}
+
+	trace_sched_process_fork(current, task);
+	get_task_struct(task);
+	return task;
+}
+
+static void pidfd_spawn_publish_pid(struct file *file,
+				    struct pidfd_spawn_state *state,
+				    struct task_struct *task)
+{
+	struct pid *pid = get_task_pid(task, PIDTYPE_PID);
+
+	pidfs_future_file_publish_pid(file, pid);
+	/*
+	 * Publish only after the inode identity, pidfs attributes, and exit-wakeup
+	 * forwarding are ready. Pair with the acquire loads in pidfd resolution
+	 * and poll so readers that observe @pid can use ordinary pidfd operations.
+	 */
+	smp_store_release(&state->pid, pid);
+	pidfs_future_file_notify(file);
+}
+
+static void pidfd_spawn_attach_vfork_done(struct task_struct *task,
+					  struct completion *vfork)
+{
+	init_completion(vfork);
+
+	task_lock(task);
+	task->vfork_done = vfork;
+	task_unlock(task);
+}
+
+static void pidfd_spawn_wait_for_child(struct pidfd_spawn_state *state)
+{
+	wait_for_completion(&state->done);
+}
+
+static int pidfd_spawn_finish_vfork(struct pidfd_spawn_state *state,
+				    struct task_struct *task,
+				    struct completion *vfork)
+{
+	int ret;
+
+	ret = wait_for_vfork_done(task, vfork);
+	if (ret)
+		return ret;
+
+	mutex_lock(&state->lock);
+	ret = state->result;
+	pidfd_spawn_set_status(state, PIDFD_SPAWN_STARTED);
+	mutex_unlock(&state->lock);
+	return ret;
+}
+
+static int pidfd_spawn_start(struct file *file,
+			     struct pidfd_spawn_state *state,
+			     const struct pidfd_spawn_run_args *kargs)
+{
+	struct delayed_filename filename
+		__free(pidfd_spawn_dismiss_filename) = {};
+	struct delayed_filename drop_filename
+		__free(pidfd_spawn_dismiss_filename) = {};
+	struct kernel_clone_args clone_args = {
+		.flags		= CLONE_VM | CLONE_VFORK | CLONE_UNTRACED,
+		.exit_signal	= SIGCHLD,
+		.task_work	= &state->task_work,
+		.embryonic_exec = true,
+	};
+	struct completion vfork;
+	struct task_struct *task;
+	int ret;
+
+	if (kargs->path) {
+		ret = pidfd_spawn_get_delayed_path(&filename,
+						   pidfd_spawn_user_ptr(kargs->path));
+		if (ret)
+			return ret;
+	}
+
+	scoped_cond_guard(mutex_intr, return -EINTR, &state->lock) {
+		ret = pidfd_spawn_configuring_error(state);
+		if (ret)
+			return ret;
+		if (!pidfd_spawn_same_creator(state))
+			return -EPERM;
+		if (state->staged_path && kargs->path)
+			return -EINVAL;
+		if (!state->staged_path && !kargs->path)
+			return -EINVAL;
+		if (state->staged_path) {
+			ret = pidfd_spawn_get_delayed_kernel_path(&filename,
+								  state->staged_path);
+			if (ret)
+				return ret;
+		}
+		state->filename = filename;
+		INIT_DELAYED_FILENAME(&filename);
+
+		state->argv = pidfd_spawn_user_arg(kargs->argv);
+		state->envp = pidfd_spawn_user_arg(kargs->envp);
+		pidfd_spawn_set_status(state, PIDFD_SPAWN_STARTING);
+		reinit_completion(&state->done);
+
+		task = pidfd_spawn_create_task(file, state, &clone_args);
+		if (IS_ERR(task)) {
+			ret = PTR_ERR(task);
+			drop_filename = state->filename;
+			INIT_DELAYED_FILENAME(&state->filename);
+			state->argv = native_arg(NULL);
+			state->envp = native_arg(NULL);
+			state->result = 0;
+			pidfd_spawn_set_status(state,
+					       PIDFD_SPAWN_CONFIGURING);
+			return ret;
+		}
+
+		pidfd_spawn_publish_pid(file, state, task);
+		pidfd_spawn_attach_vfork_done(task, &vfork);
+	}
+
+	wake_up_new_task(task);
+	pidfd_spawn_wait_for_child(state);
+	return pidfd_spawn_finish_vfork(state, task, &vfork);
+}
+
 int pidfd_empty_open(unsigned int flags)
 {
 	struct pidfd_spawn_state *state;

@@ -257,7 +615,13 @@ int pidfd_empty_open(unsigned int flags)
 		return -ENOMEM;
 
 	mutex_init(&state->lock);
+	init_completion(&state->done);
+	init_task_work(&state->task_work, pidfd_spawn_child);
 	refcount_set(&state->count, 1);
+	/*
+	 * Remember the creator so later builder operations cannot be delegated
+	 * just by passing the pidfd through SCM_RIGHTS or similar fd passing.
+	 */
 	mmgrab(current->mm);
 	state->creator_mm = current->mm;
 	state->creator_cred = get_current_cred();

@@ -282,3 +646,37 @@ int pidfd_empty_open(unsigned int flags)
 	fd_install(pidfd, pidfile);
 	return pidfd;
 }
+
+SYSCALL_DEFINE3(pidfd_spawn_run, int, fd,
+		struct pidfd_spawn_run_args __user *, uargs, size_t, usize)
+{
+	struct pidfd_spawn_run_args kargs;
+	struct pidfd_spawn_state *state __free(pidfd_spawn_state) = NULL;
+	int ret;
+
+	ret = pidfd_spawn_copy_run_args(&kargs, uargs, usize);
+	if (ret)
+		return ret;
+
+	CLASS(fd, f)(fd);
+	if (fd_empty(f))
+		return -EBADF;
+
+	state = pidfd_spawn_state_get_file(fd_file(f));
+	if (IS_ERR(state))
+		return PTR_ERR(state);
+
+	scoped_cond_guard(mutex_intr, return -ERESTARTNOINTR,
+			  &current->signal->cred_guard_mutex) {
+		if (READ_ONCE(current->ptrace))
+			return -EPERM;
+		ret = pidfd_spawn_check_startable(state);
+		if (ret)
+			return ret;
+		ret = pidfd_spawn_start(fd_file(f), state, &kargs);
+	}
+	if (!ret)
+		ret = pid_vnr(state->pid);
+
+	return ret;
+}