diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index e9f80407f75b..dc270a23179d 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3264,6 +3264,23 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
dest_cpu = ctx->dl_bw_cpu;
if (!task_on_rq_queued(p))
p->wake_cpu = dest_cpu;
+ } else {
+ const struct cpumask *span = dl_task_bw_span(p);
+ unsigned int cpu = nr_cpu_ids;
+
+ /*
+ * No room was made for @p in another root domain. Keep @p
+ * in the root domain that counts its bandwidth, if the new
+ * mask holds one of its CPUs.
+ */
+ if (span && !cpumask_test_cpu(dest_cpu, span))
+ cpu = cpumask_first_and_and(cpu_valid_mask,
+ ctx->new_mask, span);
+ if (cpu < nr_cpu_ids) {
+ dest_cpu = cpu;
+ if (!task_on_rq_queued(p))
+ p->wake_cpu = dest_cpu;
+ }
}
do_set_cpus_allowed(p, ctx);@@ -3668,8 +3685,21 @@ static int select_fallback_rq(int cpu, struct task_struct *p)
int nid = cpu_to_node(cpu);
const struct cpumask *nodemask = NULL;
enum { cpuset, possible, fail } state = cpuset;
+ const struct cpumask *span;
int dest_cpu;
+ /*
+ * Keep a deadline task in the root domain that counts its
+ * bandwidth, if one of its CPUs is allowed and online.
+ */
+ span = dl_task_bw_span(p);
+ if (span) {
+ for_each_cpu_and(dest_cpu, span, p->cpus_ptr) {
+ if (is_cpu_allowed(p, dest_cpu))
+ return dest_cpu;
+ }
+ }
+
/*
* If the node that the CPU is on has been offlined, cpu_to_node()
* will return -1. There is no CPU on the node, and we should@@ -3686,6 +3716,14 @@ static int select_fallback_rq(int cpu, struct task_struct *p)
}
for (;;) {
+ /* The mask is wider now: look again in that root domain. */
+ if (span && state != cpuset) {
+ for_each_cpu_and(dest_cpu, span, p->cpus_ptr) {
+ if (is_cpu_allowed(p, dest_cpu))
+ goto out;
+ }
+ }
+
/* Any allowed, online CPU? */
for_each_cpu(dest_cpu, p->cpus_ptr) {
if (!is_cpu_allowed(p, dest_cpu))diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 6c6a4d9cd054..e5f77d3c5fa0 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -313,8 +313,8 @@ static unsigned int dl_bw_gen;
* where its bandwidth is counted. The new root domains count no task
* yet.
*
- * Changed under dl_departed_lock. dl_task_bw_elsewhere() reads it
- * without the lock.
+ * Changed under dl_departed_lock. dl_task_bw_span() and
+ * dl_task_bw_elsewhere() read it without the lock.
*/
static unsigned int dl_bw_rebuild_seq;
@@ -333,6 +333,8 @@ void dl_bw_rebuild_begin(void)
lockdep_assert_held(&sched_domains_mutex);
guard(raw_spinlock_irqsave)(&dl_departed_lock);
WRITE_ONCE(dl_bw_rebuild_seq, dl_bw_rebuild_seq + 1);
+ /* Before the first write of rq->rd: see dl_task_bw_span(). */
+ smp_wmb();
}
static void dl_bw_rebuild_end(void)
@@ -901,18 +903,36 @@ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq);
static struct rq *dl_task_offline_migration(struct rq *rq, struct task_struct *p)
{
+ const struct cpumask *span;
struct rq *later_rq = NULL;
struct dl_bw *dl_b;
later_rq = find_lock_later_rq(p, rq);
+ /*
+ * find_lock_later_rq() looks in the root domain of @rq. Once @rq is
+ * offline, a rebuild attaches it to def_root_domain. That root domain
+ * may not count @p. Do not take @p out of the root domain that counts
+ * it while @p has an active CPU there.
+ */
+ span = dl_task_bw_span(p);
+ if (later_rq && span && !cpumask_test_cpu(later_rq->cpu, span) &&
+ cpumask_first_and_and(cpu_active_mask, p->cpus_ptr, span) < nr_cpu_ids) {
+ double_unlock_balance(rq, later_rq);
+ later_rq = NULL;
+ }
if (!later_rq) {
int cpu;
/*
* If we cannot preempt any rq, fall back to pick any
- * online CPU:
+ * online CPU. Try the root domain that counts @p first.
*/
- cpu = cpumask_any_and(cpu_active_mask, p->cpus_ptr);
+ cpu = nr_cpu_ids;
+ if (span)
+ cpu = cpumask_first_and_and(cpu_active_mask,
+ p->cpus_ptr, span);
+ if (cpu >= nr_cpu_ids)
+ cpu = cpumask_any_and(cpu_active_mask, p->cpus_ptr);
if (cpu >= nr_cpu_ids) {
/*
* Failed to find any suitable CPU.@@ -2923,6 +2943,7 @@ static int
select_task_rq_dl(struct task_struct *p, int cpu, int flags)
{
struct task_struct *curr, *donor;
+ const struct cpumask *span;
bool select_rq;
struct rq *rq;
@@ -2963,6 +2984,20 @@ select_task_rq_dl(struct task_struct *p, int cpu, int flags)
dl_task_is_earliest_deadline(p, cpu_rq(target)))
cpu = target;
}
+
+ /*
+ * Keep @p in the root domain that counts its bandwidth. Its mask
+ * can hold CPUs of another root domain, with isolcpus="domain",
+ * and @p can sleep on one of them.
+ */
+ span = dl_task_bw_span(p);
+ if (span && !cpumask_test_cpu(cpu, span)) {
+ unsigned int in_span;
+
+ in_span = cpumask_first_and_and(span, p->cpus_ptr, cpu_active_mask);
+ if (in_span < nr_cpu_ids)
+ cpu = in_span;
+ }
rcu_read_unlock();
return cpu;@@ -3744,6 +3779,58 @@ static unsigned int dl_task_root_domain_cpu(struct task_struct *p)
return cpu;
}
+/*
+ * Return the CPUs of the root domain that counts the bandwidth of @p, or
+ * NULL if none does. The mask of @p can hold CPUs of another root domain,
+ * with isolcpus="domain". Where a CPU is picked for @p, a CPU of this
+ * root domain comes first: no other root domain admitted @p.
+ *
+ * Return NULL while the sched domains are rebuilt. The CPU recorded can
+ * then sit in def_root_domain on its way to a new root domain. The span
+ * of def_root_domain holds the isolated CPUs, and no longer the
+ * housekeeping CPUs that already left it. A caller would send a
+ * housekeeping task to an isolated CPU.
+ *
+ * The caller holds the pi_lock or the rq lock of @p. With interrupts
+ * off, the root domain is not freed.
+ */
+const struct cpumask *dl_task_bw_span(struct task_struct *p)
+{
+ unsigned int seq = READ_ONCE(dl_bw_rebuild_seq);
+ struct root_domain *rd;
+ int cpu;
+
+ if (!task_has_dl_policy(p) || dl_entity_is_special(&p->dl) ||
+ !p->dl.dl_bw || (seq & 1))
+ return NULL;
+ cpu = READ_ONCE(p->dl.dl_bw_cpu);
+ if (cpu < 0 || READ_ONCE(p->dl.dl_bw_gen) != READ_ONCE(dl_bw_gen))
+ return NULL;
+ /*
+ * The CPU recorded can be going offline. It keeps its root domain
+ * until the rebuild attaches it to def_root_domain. The rebuild then
+ * records an active CPU with @p, or leaves @p out. Until then, the
+ * root domain of the CPU recorded still counts @p.
+ *
+ * A CPU that isolcpus="domain" isolates is always in
+ * def_root_domain, which then counts @p. A housekeeping CPU that is
+ * not active and in def_root_domain was attached there by the
+ * rebuild. def_root_domain does not count @p then.
+ */
+ rd = cpu_rq(cpu)->rd;
+ /*
+ * A rebuild that started since writes rq->rd after it made
+ * dl_bw_rebuild_seq odd. Use this root domain only if none did.
+ */
+ smp_rmb();
+ if (READ_ONCE(dl_bw_rebuild_seq) != seq)
+ return NULL;
+ if (!cpu_active(cpu) && rd == &def_root_domain &&
+ housekeeping_test_cpu(cpu, HK_TYPE_DOMAIN_BOOT))
+ return NULL;
+ return rd->span;
+}
+
static void set_cpus_allowed_dl(struct task_struct *p,
struct affinity_context *ctx)
{@@ -3953,23 +4040,24 @@ static void dl_get_task_effective_cpus(struct task_struct *p, struct cpumask *cp
* Return a CPU of the root domain where a rebuild of the accounting should
* count the bandwidth of @p, or nr_cpu_ids if there is none.
*
- * Prefer the CPU @p is on, if it is active and in the mask of @p. The
- * task runs, or most often wakes up, in the root domain of that CPU. A
- * mask can hold CPUs of two root domains: housekeeping CPUs and CPUs
- * isolated by isolcpus="domain". When every CPU of a mask goes offline,
- * select_fallback_rq() can widen it to both. A task that then runs in
- * the other root domain takes its bandwidth along.
- * Prefer that CPU also if @p is queued there, out of its mask, and the
- * CPU is the one recorded. Without admission control,
- * dl_task_offline_migration() can put a throttled task and its bandwidth
- * on any active CPU. The task runs there until it blocks.
+ * A mask can hold CPUs of two root domains: housekeeping CPUs and CPUs
+ * isolated by isolcpus="domain". Keep the bandwidth on the side of the
+ * CPU recorded with it. That CPU is in the root domain that admitted the
+ * bandwidth, or that its last move put it in. Take the CPU @p is on if it
+ * is active, in the mask of @p and on that side. Else take the CPU
+ * recorded, if it is active and in the mask. Else take a CPU of the mask
+ * on that side.
+ *
+ * With no CPU on that side, or no CPU recorded, prefer the CPU @p is on.
+ * Else take the first active CPU of the mask, a housekeeping one if there
+ * is one. A task on isolated CPUs that all went offline can have no
+ * active CPU at all.
*
- * Else prefer the CPU recorded with the bandwidth, if it is active and in
- * the mask. It is in the root domain that admitted the bandwidth, or that
- * its last move put it in. A task moved asleep is still on a CPU of the
- * root domain it left, out of its mask. Else take the first active CPU of
- * the mask, a housekeeping one if there is one. A task on isolated CPUs
- * that all went offline can have no active CPU at all.
+ * One case comes before all of these. Take the CPU @p is on if @p is
+ * queued there, out of its mask, and the CPU is the one recorded. Without
+ * admission control, dl_task_offline_migration() can put a throttled task
+ * and its bandwidth on any active CPU. The task runs there until it
+ * blocks.
*/
static unsigned int dl_task_rebuild_cpu(struct task_struct *p)
{@@ -3987,17 +4075,37 @@ static unsigned int dl_task_rebuild_cpu(struct task_struct *p)
*/
cpumask_and(msk, &p->cpus_mask, cpu_active_mask);
cpu = task_cpu(p);
- if (cpumask_test_cpu(cpu, msk) ||
- (cpu == recorded && cpu_active(cpu) && task_on_rq_queued(p)))
+ if (cpu == recorded && cpu_active(cpu) && task_on_rq_queued(p))
return cpu;
if (cpumask_empty(msk)) {
if (recorded >= 0 && cpu_active(recorded))
return recorded;
dl_get_task_effective_cpus(p, msk);
cpumask_and(msk, msk, cpu_active_mask);
- } else if (recorded >= 0 && cpumask_test_cpu(recorded, msk)) {
- return recorded;
}
+ /*
+ * Keep the bandwidth where it was counted: with the housekeeping
+ * CPUs, or with the CPUs that isolcpus="domain" isolates. The root
+ * domains are new, and the CPU recorded can be offline. So prefer a
+ * CPU of the mask of the same kind as the CPU recorded: housekeeping
+ * or not.
+ */
+ if (recorded >= 0) {
+ const struct cpumask *hk = housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT);
+ bool rec_hk = cpumask_test_cpu(recorded, hk);
+ unsigned int same;
+
+ if (cpumask_test_cpu(cpu, msk) && cpumask_test_cpu(cpu, hk) == rec_hk)
+ return cpu;
+ if (cpumask_test_cpu(recorded, msk))
+ return recorded;
+ for_each_cpu(same, msk) {
+ if (cpumask_test_cpu(same, hk) == rec_hk)
+ return same;
+ }
+ }
+ if (cpumask_test_cpu(cpu, msk))
+ return cpu;
cpu = cpumask_first_and(msk, housekeeping_cpumask(HK_TYPE_DOMAIN_BOOT));
if (cpu < nr_cpu_ids)
return cpu;diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 3cc287785b91..f80ea5541b3f 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -371,6 +371,7 @@ extern bool __checkparam_dl(const struct sched_attr *attr);
extern bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr);
extern int dl_cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
extern int dl_bw_deactivate(int cpu);
+extern const struct cpumask *dl_task_bw_span(struct task_struct *p);
extern s64 dl_scaled_delta_exec(struct rq *rq, struct sched_dl_entity *dl_se, s64 delta_exec);
/*
* SCHED_DEADLINE supports servers (nested scheduling) with the following
--
2.47.3