--- v6
+++ v10
@@ -26,18 +26,20 @@
Signed-off-by: Zecheng Li <zecheng@google.com>
Signed-off-by: Zecheng Li <zli94@ncsu.edu>
+Reviewed-by: K Prateek Nayak <kprateek.nayak@amd.com>
+Reviewed-by: Josh Don <joshdon@google.com>
---
kernel/sched/core.c | 7 ++-----
kernel/sched/debug.c | 2 +-
kernel/sched/fair.c | 25 +++++++++----------------
- kernel/sched/sched.h | 29 ++++++++++++++++++++++++-----
- 4 files changed, 36 insertions(+), 27 deletions(-)
+ kernel/sched/sched.h | 31 ++++++++++++++++++++++++++-----
+ 4 files changed, 38 insertions(+), 27 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
-index 5b17d8e3cb55..2db052414794 100644
+index 7fb3f5f2d48c..86fbb38901aa 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
-@@ -8561,7 +8561,7 @@ void __init sched_init(void)
+@@ -8900,7 +8900,7 @@ void __init sched_init(void)
wait_bit_init();
#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -46,7 +48,7 @@
#endif
#ifdef CONFIG_RT_GROUP_SCHED
ptr += 2 * nr_cpu_ids * sizeof(void **);
-@@ -8570,9 +8570,6 @@ void __init sched_init(void)
+@@ -8909,9 +8909,6 @@ void __init sched_init(void)
ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -56,7 +58,7 @@
root_task_group.cfs_rq = (struct cfs_rq **)ptr;
ptr += nr_cpu_ids * sizeof(void **);
-@@ -9640,7 +9637,7 @@ static int cpu_cfs_stat_show(struct seq_file *sf, void *v)
+@@ -9993,7 +9990,7 @@ static int cpu_cfs_stat_show(struct seq_file *sf, void *v)
int i;
for_each_possible_cpu(i) {
@@ -66,10 +68,10 @@
}
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
-index 41caa22e0680..a18c1be40578 100644
+index ed3a0d65da0c..13ee25eeb49c 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
-@@ -644,7 +644,7 @@ void dirty_sched_domain_sysctl(int cpu)
+@@ -808,7 +808,7 @@ void dirty_sched_domain_sysctl(int cpu)
#ifdef CONFIG_FAIR_GROUP_SCHED
static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group *tg)
{
@@ -79,10 +81,10 @@
#define P(F) SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F)
#define P_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld\n", \
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
-index eef10f2ef2a9..db10e617a638 100644
+index 2bdbe186a9e4..db23951d6ba0 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
-@@ -5974,7 +5974,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
+@@ -6847,7 +6847,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
{
struct rq *rq = rq_of(cfs_rq);
struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
@@ -91,7 +93,7 @@
/*
* It's possible we are called with runtime_remaining < 0 due to things
-@@ -9845,7 +9845,6 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
+@@ -11123,7 +11123,6 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
{
struct cfs_rq *cfs_rq, *pos;
bool decayed = false;
@@ -99,7 +101,7 @@
/*
* Iterates the task_group tree in a bottom up fashion, see
-@@ -9865,7 +9864,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
+@@ -11143,7 +11142,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
}
/* Propagate pending load changes to the parent, if any: */
@@ -108,7 +110,7 @@
if (se && !skip_blocked_update(se))
update_load_avg(cfs_rq_of(se), se, UPDATE_TG);
-@@ -9891,8 +9890,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
+@@ -11169,8 +11168,7 @@ static bool __update_blocked_fair(struct rq *rq, bool *done)
*/
static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
{
@@ -118,7 +120,7 @@
unsigned long now = jiffies;
unsigned long load;
-@@ -13625,7 +13623,6 @@ void free_fair_sched_group(struct task_group *tg)
+@@ -15108,7 +15106,6 @@ void free_fair_sched_group(struct task_group *tg)
}
kfree(tg->cfs_rq);
@@ -126,26 +128,26 @@
}
int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
-@@ -13638,9 +13635,6 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
- tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL);
+@@ -15121,9 +15118,6 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
+ tg->cfs_rq = kzalloc_objs(cfs_rq, nr_cpu_ids);
if (!tg->cfs_rq)
goto err;
-- tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL);
+- tg->se = kzalloc_objs(se, nr_cpu_ids);
- if (!tg->se)
- goto err;
tg->shares = NICE_0_LOAD;
-@@ -13655,7 +13649,7 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
- cfs_rq = &combined->cfs_rq;
- se = &combined->ses.se;
+@@ -15138,7 +15132,7 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
+ cfs_rq = &state->cfs_rq;
+ se = &state->se;
init_cfs_rq(cfs_rq);
- init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);
+ init_tg_cfs_entry(tg, cfs_rq, se, i, tg_se(parent, i));
init_entity_runnable_average(se);
}
-@@ -13674,7 +13668,7 @@ void online_fair_sched_group(struct task_group *tg)
+@@ -15157,7 +15151,7 @@ void online_fair_sched_group(struct task_group *tg)
for_each_possible_cpu(i) {
rq = cpu_rq(i);
@@ -154,7 +156,7 @@
rq_lock_irq(rq, &rf);
update_rq_clock(rq);
attach_entity_cfs_rq(se);
-@@ -13691,7 +13685,7 @@ void unregister_fair_sched_group(struct task_group *tg)
+@@ -15174,7 +15168,7 @@ void unregister_fair_sched_group(struct task_group *tg)
for_each_possible_cpu(cpu) {
struct cfs_rq *cfs_rq = tg->cfs_rq[cpu];
@@ -163,7 +165,7 @@
struct rq *rq = cpu_rq(cpu);
if (se) {
-@@ -13728,7 +13722,6 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
+@@ -15211,7 +15205,6 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
init_cfs_rq_runtime(cfs_rq);
tg->cfs_rq[cpu] = cfs_rq;
@@ -171,7 +173,7 @@
/* se could be NULL for root_task_group */
if (!se)
-@@ -13759,7 +13752,7 @@ static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)
+@@ -15242,7 +15235,7 @@ static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)
/*
* We can't change the weight of the root cgroup.
*/
@@ -180,7 +182,7 @@
return -EINVAL;
shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));
-@@ -13770,7 +13763,7 @@ static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)
+@@ -15253,7 +15246,7 @@ static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)
tg->shares = shares;
for_each_possible_cpu(i) {
struct rq *rq = cpu_rq(i);
@@ -189,7 +191,7 @@
struct rq_flags rf;
/* Propagate contribution to hierarchy */
-@@ -13821,7 +13814,7 @@ int sched_group_set_idle(struct task_group *tg, long idle)
+@@ -15304,7 +15297,7 @@ int sched_group_set_idle(struct task_group *tg, long idle)
for_each_possible_cpu(i) {
struct rq *rq = cpu_rq(i);
@@ -199,10 +201,10 @@
bool was_idle = cfs_rq_is_idle(grp_cfs_rq);
long idle_task_delta;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
-index be32810f7475..97c27ac0ae18 100644
+index c9aac3ab4bd9..63574f9d57f1 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
-@@ -476,8 +476,6 @@ struct task_group {
+@@ -480,8 +480,6 @@ struct task_group {
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -211,7 +213,7 @@
/* runqueue "owned" by this group on each CPU */
struct cfs_rq **cfs_rq;
unsigned long shares;
-@@ -915,7 +913,8 @@ struct dl_rq {
+@@ -929,7 +927,8 @@ struct dl_rq {
};
#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -221,34 +223,36 @@
/* An entity is a task if it doesn't "own" a runqueue */
#define entity_is_task(se) (!se->my_q)
-@@ -2187,6 +2186,26 @@ struct cfs_rq_with_se {
- struct cfs_rq cfs_rq;
- struct sched_entity_stats ses;
- };
+@@ -2299,6 +2298,28 @@ struct cfs_tg_state {
+ struct sched_entity se;
+ struct sched_statistics stats;
+ } __no_randomize_layout;
+
+static inline struct sched_entity *tg_se(struct task_group *tg, int cpu)
+{
++ struct cfs_tg_state *state;
++
+ if (is_root_task_group(tg))
+ return NULL;
+
-+ struct cfs_rq_with_se *combined =
-+ container_of(tg->cfs_rq[cpu], struct cfs_rq_with_se, cfs_rq);
-+ return &combined->ses.se;
++ state = container_of(tg->cfs_rq[cpu], struct cfs_tg_state, cfs_rq);
++ return &state->se;
+}
+
+static inline struct sched_entity *cfs_rq_se(struct cfs_rq *cfs_rq)
+{
++ struct cfs_tg_state *state;
++
+ if (is_root_task_group(cfs_rq->tg))
+ return NULL;
+
-+ struct cfs_rq_with_se *combined =
-+ container_of(cfs_rq, struct cfs_rq_with_se, cfs_rq);
-+ return &combined->ses.se;
++ state = container_of(cfs_rq, struct cfs_tg_state, cfs_rq);
++ return &state->se;
+}
#endif
/* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */
-@@ -2199,8 +2218,8 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
+@@ -2311,8 +2332,8 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
#ifdef CONFIG_FAIR_GROUP_SCHED
set_task_rq_fair(&p->se, p->se.cfs_rq, tg->cfs_rq[cpu]);
p->se.cfs_rq = tg->cfs_rq[cpu];
@@ -260,5 +264,5 @@
#ifdef CONFIG_RT_GROUP_SCHED
--
-2.52.0
-
+2.54.0
+