[PATCH 5/5] RT - Track which CPUs should get IPI'd on rt-overload
From: Gregory Haskins <hidden>
Date: 2007-10-09 14:40:36
Also in:
lkml
Subsystem:
the rest · Maintainer:
Linus Torvalds
The code currently blindly fires IPIs out whenever an overload occurs. However, there are strict events that govern when a rt-overload exists (e.g. RT task added to a RQ, or an RT task preempted). Therefore, we attempt to efficiently track which CPUs are eligible for rebalancing, and we only IPI those affected units. Signed-off-by: Gregory Haskins <redacted> CC: Peter W. Morreale <redacted> --- kernel/sched.c | 15 +++++++++++++-- 1 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/kernel/sched.c b/kernel/sched.c
index a28ca9d..6ca5f4f 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c@@ -359,6 +359,8 @@ struct rq { unsigned long rto_pulled; #endif struct lock_class_key rq_lock_key; + + cpumask_t rto_resched; /* Which of our peers needs rescheduling */ }; static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
@@ -645,6 +647,9 @@ static inline void inc_rt_tasks(struct task_struct *p, struct rq *rq) smp_wmb(); atomic_inc(&rt_overload); } + + cpus_or(rq->rto_resched, rq->rto_resched, p->cpus_allowed); + cpu_clear(rq->cpu, rq->rto_resched); # endif } #endif
@@ -2213,9 +2218,15 @@ static inline void finish_task_switch(struct rq *rq, struct task_struct *prev) * If we pushed an RT task off the runqueue, * then kick other CPUs, they might run it: */ - if (unlikely(rt_task(current) && rq->rt_nr_running > 1)) { + if (unlikely(rt_task(current) && prev->se.on_rq && rt_task(prev))) { + cpus_or(rq->rto_resched, rq->rto_resched, prev->cpus_allowed); + cpu_clear(rq->cpu, rq->rto_resched); + } + + if (unlikely(rq->rt_nr_running > 1 && !cpus_empty(rq->rto_resched))) { schedstat_inc(rq, rto_schedule); - smp_send_reschedule_allbutself(); + smp_send_reschedule_allbutself_cpumask(rq->rto_resched); + cpus_clear(rq->rto_resched); } #endif prev_state = prev->state;