Fixes: 461daba06bdc ("psi: eliminate kthread_worker from psi trigger
scheduling mechanism")
Reported-by: Kathleen Chang <redacted>
Reported-by: Wenju Xu <redacted>
Reported-by: Jonathan Chen <redacted>
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
---
include/linux/psi_types.h | 1 +
kernel/sched/psi.c | 41 ++++++++++++++++++++++++++++---------
--
2 files changed, 31 insertions(+), 11 deletions(-)
diff --git a/include/linux/psi_types.h b/include/linux/psi_types.h
index 0a23300d49af..ef8bd89d065e 100644
--- a/include/linux/psi_types.h
+++ b/include/linux/psi_types.h
@@ -158,6 +158,7 @@ struct psi_group {
struct timer_list poll_timer;
wait_queue_head_t poll_wait;
atomic_t poll_wakeup;
+ atomic_t poll_scheduled;
/* Protects data used by the monitor */
struct mutex trigger_lock;diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c
index cc25a3cff41f..fed7c9c2b276 100644
--- a/kernel/sched/psi.c
+++ b/kernel/sched/psi.c
@@ -193,6 +193,7 @@ static void group_init(struct psi_group *group)
INIT_DELAYED_WORK(&group->avgs_work, psi_avgs_work);
mutex_init(&group->avgs_lock);
/* Init trigger-related members */
+ atomic_set(&group->poll_scheduled, 0);
mutex_init(&group->trigger_lock);
INIT_LIST_HEAD(&group->triggers);
memset(group->nr_triggers, 0, sizeof(group->nr_triggers));@@ -551,18 +552,14 @@ static u64 update_triggers(struct psi_group
*group, u64 now)
return now + group->poll_min_period;
}
-/* Schedule polling if it's not already scheduled. */
-static void psi_schedule_poll_work(struct psi_group *group, unsigned
long delay)
+/* Schedule polling if it's not already scheduled or forced. */
+static void psi_schedule_poll_work(struct psi_group *group, unsigned
long delay,
+ bool force)
{
struct task_struct *task;
- /*
- * Do not reschedule if already scheduled.
- * Possible race with a timer scheduled after this check but
before
- * mod_timer below can be tolerated because group-quoted
polling_next_update
- * will keep updates on schedule.
- */
- if (timer_pending(&group->poll_timer))
+ /* cmpxchg should be called even when !force to set
poll_scheduled */
+ if (atomic_cmpxchg(&group->poll_scheduled, 0, 1) != 0 &&
!force)
return;
rcu_read_lock();@@ -574,12 +571,15 @@ static void psi_schedule_poll_work(struct
psi_group *group, unsigned long delay)
*/
if (likely(task))
mod_timer(&group->poll_timer, jiffies + delay);
+ else
+ atomic_set(&group->poll_scheduled, 0);
rcu_read_unlock();
}
static void psi_poll_work(struct psi_group *group)
{
+ bool force_reschedule = false;
u32 changed_states;
u64 now;
@@ -587,6 +587,23 @@ static void psi_poll_work(struct psi_group
*group)
now = sched_clock();
+ if (now > group->polling_until) {
+ /*
+ * We are either about to start or might stop polling
if no
+ * state change was recorded. Resetting poll_scheduled
leaves
+ * a small window for psi_group_change to sneak in and
schedule
+ * an immegiate poll_work before we get to
rescheduling. One
+ * potential extra wakeup at the end of the polling
window
+ * should be negligible and polling_next_update still
keeps
+ * updates correctly on schedule.
+ */
+ atomic_set(&group->poll_scheduled, 0);
+ } else {
+ /* Polling window is not over, keep rescheduling */
+ force_reschedule = true;
+ }
+
+
collect_percpu_times(group, PSI_POLL, &changed_states);
if (changed_states & group->poll_states) {@@ -612,7 +629,8 @@ static void psi_poll_work(struct psi_group
*group)
group->polling_next_update = update_triggers(group,
now);
psi_schedule_poll_work(group,
- nsecs_to_jiffies(group->polling_next_update - now) +
1);
+ nsecs_to_jiffies(group->polling_next_update - now) + 1,
+ force_reschedule);
out:
mutex_unlock(&group->trigger_lock);@@ -736,7 +754,7 @@ static void psi_group_change(struct psi_group
*group, int cpu,
write_seqcount_end(&groupc->seq);
if (state_mask & group->poll_states)
- psi_schedule_poll_work(group, 1);
+ psi_schedule_poll_work(group, 1, false);
if (wake_clock && !delayed_work_pending(&group->avgs_work))
schedule_delayed_work(&group->avgs_work, PSI_FREQ);@@ -1235,6 +1253,7 @@ static void psi_trigger_destroy(struct kref
*ref)
*/
del_timer_sync(&group->poll_timer);
kthread_stop(task_to_destroy);
+ atomic_set(&group->poll_scheduled, 0);
}
kfree(t);
}