--- v8
+++ v10
@@ -29,6 +29,8 @@
Signed-off-by: Zecheng Li <zecheng@google.com>
Signed-off-by: Zecheng Li <zli94@ncsu.edu>
+Reviewed-by: K Prateek Nayak <kprateek.nayak@amd.com>
+Reviewed-by: Josh Don <joshdon@google.com>
---
kernel/sched/core.c | 35 +++++++++++-----------------
kernel/sched/fair.c | 54 ++++++++++++++++++--------------------------
@@ -36,10 +38,10 @@
3 files changed, 45 insertions(+), 58 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
-index 8e2a67cecee9..80e8f4eb3f87 100644
+index 86fbb38901aa..163930370cd0 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
-@@ -8549,7 +8549,7 @@ static struct kmem_cache *task_group_cache __ro_after_init;
+@@ -8884,7 +8884,7 @@ static struct kmem_cache *task_group_cache __ro_after_init;
void __init sched_init(void)
{
@@ -48,7 +50,7 @@
int i;
/* Make sure the linker didn't screw up */
-@@ -8565,33 +8565,24 @@ void __init sched_init(void)
+@@ -8900,33 +8900,24 @@ void __init sched_init(void)
wait_bit_init();
#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -92,7 +94,7 @@
init_defrootdomain();
-@@ -9492,7 +9483,7 @@ static int tg_set_cfs_bandwidth(struct task_group *tg,
+@@ -9841,7 +9832,7 @@ static int tg_set_cfs_bandwidth(struct task_group *tg,
}
for_each_online_cpu(i) {
@@ -101,7 +103,7 @@
struct rq *rq = cfs_rq->rq;
guard(rq_lock_irq)(rq);
-@@ -9660,7 +9651,7 @@ static u64 throttled_time_self(struct task_group *tg)
+@@ -10009,7 +10000,7 @@ static u64 throttled_time_self(struct task_group *tg)
u64 total = 0;
for_each_possible_cpu(i) {
@@ -111,10 +113,10 @@
return total;
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
-index 8872d003af98..bc023704acd1 100644
+index db23951d6ba0..9f5b3b3e738c 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
-@@ -327,7 +327,7 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
+@@ -334,7 +334,7 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
* to a tree or when we reach the top of the tree
*/
if (cfs_rq->tg->parent &&
@@ -123,7 +125,7 @@
/*
* If parent is already on the list, we add the child
* just before. Thanks to circular linked property of
-@@ -335,7 +335,7 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
+@@ -342,7 +342,7 @@ static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
* of the list that starts by parent.
*/
list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list,
@@ -132,7 +134,7 @@
/*
* The branch is now connected to its tree so we can
* reset tmp_alone_branch to the beginning of the
-@@ -4153,7 +4153,7 @@ static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq)
+@@ -5008,7 +5008,7 @@ static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq)
rcu_read_lock();
list_for_each_entry_rcu(tg, &task_groups, list) {
@@ -141,7 +143,7 @@
clear_tg_load_avg(cfs_rq);
}
-@@ -5689,7 +5689,7 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
+@@ -6565,7 +6565,7 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu)
{
@@ -150,7 +152,7 @@
}
static inline bool task_is_throttled(struct task_struct *p)
-@@ -5835,7 +5835,7 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags);
+@@ -6711,7 +6711,7 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags);
static int tg_unthrottle_up(struct task_group *tg, void *data)
{
struct rq *rq = data;
@@ -159,7 +161,7 @@
struct task_struct *p, *tmp;
if (--cfs_rq->throttle_count)
-@@ -5906,7 +5906,7 @@ static void record_throttle_clock(struct cfs_rq *cfs_rq)
+@@ -6782,7 +6782,7 @@ static void record_throttle_clock(struct cfs_rq *cfs_rq)
static int tg_throttle_down(struct task_group *tg, void *data)
{
struct rq *rq = data;
@@ -168,7 +170,7 @@
if (cfs_rq->throttle_count++)
return 0;
-@@ -6379,8 +6379,8 @@ static void sync_throttle(struct task_group *tg, int cpu)
+@@ -7256,8 +7256,8 @@ static void sync_throttle(struct task_group *tg, int cpu)
if (!tg->parent)
return;
@@ -179,7 +181,7 @@
cfs_rq->throttle_count = pcfs_rq->throttle_count;
cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu));
-@@ -6572,7 +6572,7 @@ static void __maybe_unused update_runtime_enabled(struct rq *rq)
+@@ -7449,7 +7449,7 @@ static void __maybe_unused update_runtime_enabled(struct rq *rq)
rcu_read_lock();
list_for_each_entry_rcu(tg, &task_groups, list) {
struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
@@ -188,7 +190,7 @@
raw_spin_lock(&cfs_b->lock);
cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
-@@ -6601,7 +6601,7 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
+@@ -7478,7 +7478,7 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
rcu_read_lock();
list_for_each_entry_rcu(tg, &task_groups, list) {
@@ -197,7 +199,7 @@
if (!cfs_rq->runtime_enabled)
continue;
-@@ -9408,7 +9408,7 @@ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_
+@@ -10403,7 +10403,7 @@ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_
struct cfs_rq *dst_cfs_rq;
#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -206,7 +208,7 @@
#else
dst_cfs_rq = &cpu_rq(dest_cpu)->cfs;
#endif
-@@ -13346,7 +13346,7 @@ static int task_is_throttled_fair(struct task_struct *p, int cpu)
+@@ -14833,7 +14833,7 @@ static int task_is_throttled_fair(struct task_struct *p, int cpu)
struct cfs_rq *cfs_rq;
#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -215,7 +217,7 @@
#else
cfs_rq = &cpu_rq(cpu)->cfs;
#endif
-@@ -13612,39 +13612,31 @@ static void task_change_group_fair(struct task_struct *p)
+@@ -15098,39 +15098,31 @@ static void task_change_group_fair(struct task_struct *p)
void free_fair_sched_group(struct task_group *tg)
{
@@ -238,7 +240,7 @@
struct cfs_rq *cfs_rq;
int i;
-- tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL);
+- tg->cfs_rq = kzalloc_objs(cfs_rq, nr_cpu_ids);
- if (!tg->cfs_rq)
+ state = alloc_percpu_gfp(struct cfs_tg_state, GFP_KERNEL);
+ if (!state)
@@ -263,7 +265,7 @@
init_cfs_rq(cfs_rq);
init_tg_cfs_entry(tg, cfs_rq, se, i, tg_se(parent, i));
init_entity_runnable_average(se);
-@@ -13681,7 +13673,7 @@ void unregister_fair_sched_group(struct task_group *tg)
+@@ -15167,7 +15159,7 @@ void unregister_fair_sched_group(struct task_group *tg)
destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
for_each_possible_cpu(cpu) {
@@ -272,7 +274,7 @@
struct sched_entity *se = tg_se(tg, cpu);
struct rq *rq = cpu_rq(cpu);
-@@ -13718,8 +13710,6 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
+@@ -15204,8 +15196,6 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
cfs_rq->rq = rq;
init_cfs_rq_runtime(cfs_rq);
@@ -281,7 +283,7 @@
/* se could be NULL for root_task_group */
if (!se)
return;
-@@ -13812,7 +13802,7 @@ int sched_group_set_idle(struct task_group *tg, long idle)
+@@ -15298,7 +15288,7 @@ int sched_group_set_idle(struct task_group *tg, long idle)
for_each_possible_cpu(i) {
struct rq *rq = cpu_rq(i);
struct sched_entity *se = tg_se(tg, i);
@@ -291,10 +293,10 @@
long idle_task_delta;
struct rq_flags rf;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
-index 530b1d06e2d5..a05be7a8e04a 100644
+index 63574f9d57f1..95a22baa172b 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
-@@ -477,7 +477,7 @@ struct task_group {
+@@ -481,7 +481,7 @@ struct task_group {
#ifdef CONFIG_FAIR_GROUP_SCHED
/* runqueue "owned" by this group on each CPU */
@@ -303,7 +305,7 @@
unsigned long shares;
/*
* load_avg can be heavily contended at clock tick time, so put
-@@ -2201,6 +2201,12 @@ struct cfs_tg_state {
+@@ -2299,6 +2299,12 @@ struct cfs_tg_state {
struct sched_statistics stats;
} __no_randomize_layout;
@@ -316,7 +318,7 @@
static inline struct sched_entity *tg_se(struct task_group *tg, int cpu)
{
struct cfs_tg_state *state;
-@@ -2208,7 +2214,7 @@ static inline struct sched_entity *tg_se(struct task_group *tg, int cpu)
+@@ -2306,7 +2312,7 @@ static inline struct sched_entity *tg_se(struct task_group *tg, int cpu)
if (is_root_task_group(tg))
return NULL;
@@ -325,7 +327,7 @@
return &state->se;
}
-@@ -2232,8 +2238,8 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
+@@ -2330,8 +2336,8 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
#endif
#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -337,5 +339,5 @@
p->se.depth = p->se.parent ? p->se.parent->depth + 1 : 0;
#endif
--
-2.52.0
-
+2.54.0
+