Re: [PATCH 00/17] Paravirt CPUs and push task for less vCPU preemption

@@ -174,6 +174,10 @@ static inline void set_mems_allowed(nodemask_t nodemask)
 }
 
 extern bool cpuset_node_allowed(struct cgroup *cgroup, int nid);
+
+void sched_fair_notify_unavaialable_cpus(struct cpumask *unavailable_mask);
+const struct cpumask *cpuset_unavailable_mask(void);
+bool cpuset_cpu_unavailable(int cpu);
 #else /* !CONFIG_CPUSETS */
 
 static inline bool cpusets_enabled(void) { return false; }

@@ -59,6 +59,7 @@ typedef enum {
 	FILE_EXCLUSIVE_CPULIST,
 	FILE_EFFECTIVE_XCPULIST,
 	FILE_ISOLATED_CPULIST,
+	FILE_UNAVAILABLE_CPULIST,
 	FILE_CPU_EXCLUSIVE,
 	FILE_MEM_EXCLUSIVE,
 	FILE_MEM_HARDWALL,

@@ -87,6 +87,19 @@ static cpumask_var_t	isolated_cpus;
 static cpumask_var_t	boot_hk_cpus;
 static bool		have_boot_isolcpus;
 
+/*
+ * CPUs that may be unavailable to run tasks as a result of physical
+ * constraints (vCPU being preempted, pCPU handling interrupt storm).
+ *
+ * Unlike isolated_cpus, the unavailable_cpus are simply excluded from
+ * HK_TYPE_DOMAIN but leave the tasks affinity untouched. These CPUs
+ * should be avoided unless the task has specifically asked to be run
+ * only on these CPUs.
+ */
+static cpumask_var_t	unavailable_cpus;
+static cpumask_var_t	available_tmp_mask;	/* For intermediate operations. */
+static bool 		cpu_turned_unavailable;
+
 /* List of remote partition root children */
 static struct list_head remote_children;
 

@@ -844,6 +857,7 @@ static int generate_sched_domains(cpumask_var_t **domains,
 		}
 		cpumask_and(doms[0], top_cpuset.effective_cpus,
 			    housekeeping_cpumask(HK_TYPE_DOMAIN));
+		cpumask_andnot(doms[0], doms[0], unavailable_cpus);
 
 		goto done;
 	}

@@ -960,11 +974,13 @@ static int generate_sched_domains(cpumask_var_t **domains,
 			 * The top cpuset may contain some boot time isolated
 			 * CPUs that need to be excluded from the sched domain.
 			 */
-			if (csa[i] == &top_cpuset)
+			if (csa[i] == &top_cpuset) {
 				cpumask_and(doms[i], csa[i]->effective_cpus,
 					    housekeeping_cpumask(HK_TYPE_DOMAIN));
-			else
-				cpumask_copy(doms[i], csa[i]->effective_cpus);
+				cpumask_andnot(doms[i], doms[i], unavailable_cpus);
+			 } else {
+				cpumask_andnot(doms[i], csa[i]->effective_cpus, unavailable_cpus);
+			 }
 			if (dattr)
 				dattr[i] = SD_ATTR_INIT;
 		}

@@ -985,6 +1001,7 @@ static int generate_sched_domains(cpumask_var_t **domains,
 				}
 				cpumask_or(dp, dp, csa[j]->effective_cpus);
 				cpumask_and(dp, dp, housekeeping_cpumask(HK_TYPE_DOMAIN));
+				cpumask_andnot(dp, dp, unavailable_cpus);
 				if (dattr)
 					update_domain_attr_tree(dattr + nslot, csa[j]);
 			}

@@ -1418,6 +1435,17 @@ bool cpuset_cpu_is_isolated(int cpu)
 }
 EXPORT_SYMBOL_GPL(cpuset_cpu_is_isolated);
 
+/* Get the set of CPUs marked unavailable. */
+const struct cpumask *cpuset_unavailable_mask(void)
+{
+	return unavailable_cpus;
+}
+
+bool cpuset_cpu_unavailable(int cpu)
+{
+	return  cpumask_test_cpu(cpu, unavailable_cpus);
+}
+
 /**
  * rm_siblings_excl_cpus - Remove exclusive CPUs that are used by sibling cpusets
  * @parent: Parent cpuset containing all siblings

@@ -2612,6 +2640,53 @@ static int update_exclusive_cpumask(struct cpuset *cs, struct cpuset *trialcs,
 	return 0;
 }
 
+/**
+ * update_exclusive_cpumask - update the exclusive_cpus mask of a cpuset
+ * @cs: the cpuset to consider
+ * @trialcs: trial cpuset
+ * @buf: buffer of cpu numbers written to this cpuset
+ *
+ * The tasks' cpumask will be updated if cs is a valid partition root.
+ */
+static int update_unavailable_cpumask(const char *buf)
+{
+	cpumask_var_t tmp;
+	int retval;
+
+	if (!alloc_cpumask_var(&tmp, GFP_KERNEL))
+		return -ENOMEM;
+
+	retval = cpulist_parse(buf, tmp);
+	if (retval < 0)
+		goto out;
+
+	/* Nothing to do if the CPUs didn't change */
+	if (cpumask_equal(tmp, unavailable_cpus))
+		goto out;
+
+	/* Save the CPUs that went unavailable to push task out. */
+	if (cpumask_andnot(available_tmp_mask, tmp, unavailable_cpus))
+		cpu_turned_unavailable = true;
+
+	cpumask_copy(unavailable_cpus, tmp);
+	cpuset_force_rebuild();
+out:
+	free_cpumask_var(tmp);
+	return retval;
+}
+
+static void cpuset_notify_unavailable_cpus(void)
+{
+	/*
+	 * Prevent being preempted by the stopper if the local CPU
+	 * turned unavailable.
+	 */
+	guard(preempt)();
+
+	sched_fair_notify_unavaialable_cpus(available_tmp_mask);
+	cpu_turned_unavailable = false;
+}
+
 /*
  * Migrate memory region from one set of nodes to another.  This is
  * performed asynchronously as it can be called from process migration path

@@ -3302,11 +3377,16 @@ ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
 				    char *buf, size_t nbytes, loff_t off)
 {
 	struct cpuset *cs = css_cs(of_css(of));
+	int file_type = of_cft(of)->private;
 	struct cpuset *trialcs;
 	int retval = -ENODEV;
 
-	/* root is read-only */
-	if (cs == &top_cpuset)
+	/* root is read-only; except for unavailable mask */
+	if (file_type != FILE_UNAVAILABLE_CPULIST && cs == &top_cpuset)
+		return -EACCES;
+
+	/* unavailable mask can be only set on root. */
+	if (file_type == FILE_UNAVAILABLE_CPULIST && cs != &top_cpuset)
 		return -EACCES;
 
 	buf = strstrip(buf);

@@ -3330,6 +3410,9 @@ ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
 	case FILE_MEMLIST:
 		retval = update_nodemask(cs, trialcs, buf);
 		break;
+	case FILE_UNAVAILABLE_CPULIST:
+		retval = update_unavailable_cpumask(buf);
+		break;
 	default:
 		retval = -EINVAL;
 		break;

@@ -3338,6 +3421,8 @@ ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
 	free_cpuset(trialcs);
 	if (force_sd_rebuild)
 		rebuild_sched_domains_locked();
+	if (cpu_turned_unavailable)
+		cpuset_notify_unavailable_cpus();
 out_unlock:
 	cpuset_full_unlock();
 	if (of_cft(of)->private == FILE_MEMLIST)

@@ -3386,6 +3471,9 @@ int cpuset_common_seq_show(struct seq_file *sf, void *v)
 	case FILE_ISOLATED_CPULIST:
 		seq_printf(sf, "%*pbl\n", cpumask_pr_args(isolated_cpus));
 		break;
+	case FILE_UNAVAILABLE_CPULIST:
+		seq_printf(sf, "%*pbl\n", cpumask_pr_args(unavailable_cpus));
+		break;
 	default:
 		ret = -EINVAL;
 	}

@@ -3524,6 +3612,15 @@ static struct cftype dfl_files[] = {
 		.flags = CFTYPE_ONLY_ON_ROOT,
 	},
 
+	{
+		.name = "cpus.unavailable",
+		.seq_show = cpuset_common_seq_show,
+		.write = cpuset_write_resmask,
+		.max_write_len = (100U + 6 * NR_CPUS),
+		.private = FILE_UNAVAILABLE_CPULIST,
+		.flags = CFTYPE_ONLY_ON_ROOT,
+	},
+
 	{ }	/* terminate */
 };
 

@@ -3814,6 +3911,8 @@ int __init cpuset_init(void)
 	BUG_ON(!alloc_cpumask_var(&top_cpuset.exclusive_cpus, GFP_KERNEL));
 	BUG_ON(!zalloc_cpumask_var(&subpartitions_cpus, GFP_KERNEL));
 	BUG_ON(!zalloc_cpumask_var(&isolated_cpus, GFP_KERNEL));
+	BUG_ON(!zalloc_cpumask_var(&unavailable_cpus, GFP_KERNEL));
+	BUG_ON(!zalloc_cpumask_var(&available_tmp_mask, GFP_KERNEL));
 
 	cpumask_setall(top_cpuset.cpus_allowed);
 	nodes_setall(top_cpuset.mems_allowed);

@@ -2396,7 +2396,7 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
 
 	/* Non kernel threads are not allowed during either online or offline. */
 	if (!(p->flags & PF_KTHREAD))
-		return cpu_active(cpu);
+		return (cpu_active(cpu) && !cpuset_cpu_unavailable(cpu));
 
 	/* KTHREAD_IS_PER_CPU is always allowed. */
 	if (kthread_is_per_cpu(p))

@@ -3451,6 +3451,26 @@ static int select_fallback_rq(int cpu, struct task_struct *p)
 			goto out;
 		}
 
+		/*
+		 * Only user threads can be forced out of
+		 * unavaialable CPUs.
+		 */
+		if (p->flags & PF_KTHREAD)
+			goto rude;
+
+		/* Any unavailable CPUs that can run the task? */
+		for_each_cpu(dest_cpu, cpuset_unavailable_mask()) {
+			if (!task_allowed_on_cpu(p, dest_cpu))
+				continue;
+
+			/* Can we hoist this up to goto rude? */
+			if (is_migration_disabled(p))
+				continue;
+
+			if (cpu_active(dest_cpu))
+				goto out;
+		}
+rude:
 		/* No more Mr. Nice Guy. */
 		switch (state) {
 		case cpuset:

@@ -3766,7 +3786,7 @@ bool call_function_single_prep_ipi(int cpu)
  * via sched_ttwu_wakeup() for activation so the wakee incurs the cost
  * of the wakeup instead of the waker.
  */
-static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
+void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
 {
 	struct rq *rq = cpu_rq(cpu);
 

@@ -5365,7 +5385,9 @@ void sched_exec(void)
 	int dest_cpu;
 
 	scoped_guard (raw_spinlock_irqsave, &p->pi_lock) {
-		dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), WF_EXEC);
+		int wake_flags = WF_EXEC;
+
+		dest_cpu = select_task_rq(p, task_cpu(p), &wake_flags);
 		if (dest_cpu == smp_processor_id())
 			return;
 

@@ -12094,6 +12094,61 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq,
 	return ld_moved;
 }
 
+static int unavailable_balance_cpu_stop(void *data)
+{
+	struct task_struct *p, *tmp;
+	struct rq *rq = data;
+	int this_cpu = cpu_of(rq);
+
+	guard(rq_lock_irq)(rq);
+
+	list_for_each_entry_safe(p, tmp, &rq->cfs_tasks, se.group_node) {
+		int target_cpu;
+
+		/*
+		 * Bail out if a concurrent change to unavailable_mask turned
+		 * this CPU available.
+		 */
+		rq->unavailable_balance = cpumask_test_cpu(this_cpu, cpuset_unavailable_mask());
+		if (!rq->unavailable_balance)
+			break;
+
+		/* XXX: Does not deal with migration disabled tasks. */
+		target_cpu = cpumask_first_andnot(p->cpus_ptr, cpuset_unavailable_mask());
+		if ((unsigned int)target_cpu < nr_cpumask_bits) {
+			deactivate_task(rq, p, 0);
+			set_task_cpu(p, target_cpu);
+
+			/*
+			 * Switch to move_queued_task() later.
+			 * For PoC send an IPI and be done with it.
+			 */
+			__ttwu_queue_wakelist(p, target_cpu, 0);
+		}
+	}
+
+	rq->unavailable_balance = 0;
+
+	return 0;
+}
+
+void sched_fair_notify_unavaialable_cpus(struct cpumask *unavailable_mask)
+{
+	int cpu, this_cpu = smp_processor_id();
+
+	for_each_cpu_wrap(cpu, unavailable_mask, this_cpu + 1) {
+		struct rq *rq = cpu_rq(cpu);
+
+		/* Balance in progress. Tasks will be pushed out. */
+		if (rq->unavailable_balance)
+			return;
+
+		stop_one_cpu_nowait(cpu, unavailable_balance_cpu_stop,
+				    rq, &rq->unavailable_balance_work);
+		rq->unavailable_balance = 1;
+	}
+}
+
 static inline unsigned long
 get_sd_balance_interval(struct sched_domain *sd, int cpu_busy)
 {

@@ -1221,6 +1221,10 @@ struct rq {
 	int			push_cpu;
 	struct cpu_stop_work	active_balance_work;
 
+	/* For pushing out taks from unavailable CPUs. */
+	struct cpu_stop_work	unavailable_balance_work;
+	int			unavailable_balance;
+
 	/* CPU of this runqueue: */
 	int			cpu;
 	int			online;

@@ -2413,6 +2417,8 @@ extern const u32		sched_prio_to_wmult[40];
 
 #define RETRY_TASK		((void *)-1UL)
 
+void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags);
+
 struct affinity_context {
 	const struct cpumask	*new_mask;
 	struct cpumask		*user_mask;