diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index 1acaadb3e2cf..e234256e6ae8 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -608,6 +608,26 @@ static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc)
return qdisc_lock(root);
}
+/* The caller must hold rcu_read_lock(). dev_graft_qdisc() changes the
+ * sleeping root under the old root's lock, so retry if it changed while
+ * we were waiting for that lock.
+ */
+static inline spinlock_t *qdisc_lock_root_sleeping(const struct Qdisc *qdisc)
+{
+ struct Qdisc *root;
+ /* Lock of the sleeping root observed under RCU. */
+ spinlock_t *lock;
+
+ for (;;) {
+ root = qdisc_root_sleeping(qdisc);
+ lock = qdisc_lock(root);
+ spin_lock(lock);
+ if (root == qdisc_root_sleeping(qdisc))
+ return lock;
+ spin_unlock(lock);
+ }
+}
+
static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc)
{
return qdisc->dev_queue->dev;diff --git a/net/sched/sch_dualpi2.c b/net/sched/sch_dualpi2.c
index 4947def7c49e..8a74ceaab29d 100644
--- a/net/sched/sch_dualpi2.c
+++ b/net/sched/sch_dualpi2.c
@@ -750,8 +750,7 @@ static enum hrtimer_restart dualpi2_timer(struct hrtimer *timer)
spinlock_t *root_lock; /* to lock qdisc for probability calculations */
rcu_read_lock();
- root_lock = qdisc_lock(qdisc_root_sleeping(sch));
- spin_lock(root_lock);
+ root_lock = qdisc_lock_root_sleeping(sch);
WRITE_ONCE(q->pi2_prob, calculate_probability(sch));
hrtimer_set_expires(&q->pi2_timer, next_pi2_timeout(q));
diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c
index 5982847df8f8..a1b5126db640 100644
--- a/net/sched/sch_fq_pie.c
+++ b/net/sched/sch_fq_pie.c
@@ -394,8 +394,7 @@ static void fq_pie_timer(struct timer_list *t)
int max_cnt, i;
rcu_read_lock();
- root_lock = qdisc_lock(qdisc_root_sleeping(sch));
- spin_lock(root_lock);
+ root_lock = qdisc_lock_root_sleeping(sch);
/* Limit this expensive loop to 2048 flows per round. */
max_cnt = min_t(int, q->flows_cnt - q->flows_cursor, 2048);
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index d2e0d20f3ba1..fbbc07700160 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -1106,7 +1106,7 @@ struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
}
EXPORT_SYMBOL(qdisc_create_dflt);
-/* Under qdisc_lock(qdisc) and BH! */
+/* Under sch_tree_lock(qdisc) and BH! */
void qdisc_reset(struct Qdisc *qdisc)
{@@ -1154,8 +1154,10 @@ static void __qdisc_destroy(struct Qdisc *qdisc)
#endif
gen_kill_estimator(&qdisc->rate_est);
+ /* Autonomous timers can still run after the qdisc was detached. */
+ sch_tree_lock(qdisc);
qdisc_reset(qdisc);
-
+ sch_tree_unlock(qdisc);
if (ops->destroy)
ops->destroy(qdisc);
@@ -1213,6 +1215,7 @@ struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
struct Qdisc *oqdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
spinlock_t *root_lock;
+ /* Pairs with qdisc_lock_root_sleeping() in asynchronous callbacks. */
root_lock = qdisc_lock(oqdisc);
spin_lock_bh(root_lock);
@@ -1236,8 +1239,8 @@ static void shutdown_scheduler_queue(struct net_device *dev,
struct Qdisc *qdisc_default = _qdisc_default;
if (qdisc) {
+ qdisc = dev_graft_qdisc(dev_queue, qdisc_default);
rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
- rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc_default);
qdisc_put(qdisc);
}@@ -1261,7 +1264,7 @@ static void attach_one_default_qdisc(struct net_device *dev,
if (!netif_is_multiqueue(dev))
qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
- rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
+ dev_graft_qdisc(dev_queue, qdisc);
}
static void attach_default_qdiscs(struct net_device *dev)
diff --git a/net/sched/sch_pie.c b/net/sched/sch_pie.c
index 3b7863ffd284..9367065db4b1 100644
--- a/net/sched/sch_pie.c
+++ b/net/sched/sch_pie.c
@@ -429,8 +429,7 @@ static void pie_timer(struct timer_list *t)
spinlock_t *root_lock;
rcu_read_lock();
- root_lock = qdisc_lock(qdisc_root_sleeping(sch));
- spin_lock(root_lock);
+ root_lock = qdisc_lock_root_sleeping(sch);
pie_calculate_probability(&q->params, &q->vars, sch->qstats.backlog);
/* reset the timer to fire after 'tupdate'. tupdate is in jiffies. */
diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c
index d7598214270b..a7601b879092 100644
--- a/net/sched/sch_red.c
+++ b/net/sched/sch_red.c
@@ -333,8 +333,7 @@ static inline void red_adaptative_timer(struct timer_list *t)
spinlock_t *root_lock;
rcu_read_lock();
- root_lock = qdisc_lock(qdisc_root_sleeping(sch));
- spin_lock(root_lock);
+ root_lock = qdisc_lock_root_sleeping(sch);
red_adaptative_algo(&q->parms, &q->vars);
mod_timer(&q->adapt_timer, jiffies + HZ/2);
spin_unlock(root_lock);
diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c
index 8bbcfc9e85d9..2bd3fc65053a 100644
--- a/net/sched/sch_sfq.c
+++ b/net/sched/sch_sfq.c
@@ -611,8 +611,7 @@ static void sfq_perturbation(struct timer_list *t)
get_random_bytes(&nkey, sizeof(nkey));
rcu_read_lock();
- root_lock = qdisc_lock(qdisc_root_sleeping(sch));
- spin_lock(root_lock);
+ root_lock = qdisc_lock_root_sleeping(sch);
q->perturbation = nkey;
if (!q->filter_list && q->tail)
sfq_rehash(sch);
diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c
index 4f69ef3d847a..b7489cf4bba9 100644
--- a/net/sched/sch_taprio.c
+++ b/net/sched/sch_taprio.c
@@ -2041,6 +2041,10 @@ static void taprio_reset(struct Qdisc *sch)
hrtimer_cancel(&q->advance_timer);
+ /* Destroy resets each detached subtree under its own root lock. */
+ if (!refcount_read(&sch->refcnt))
+ return;
+
if (q->qdiscs) {
for (i = 0; i < dev->num_tx_queues; i++)
if (q->qdiscs[i])@@ -2063,6 +2067,18 @@ static void taprio_destroy(struct Qdisc *sch)
hrtimer_cancel(&q->advance_timer);
qdisc_synchronize(sch);
+ if (q->qdiscs) {
+ for (i = 0; i < dev->num_tx_queues; i++) {
+ struct Qdisc *child = q->qdiscs[i];
+
+ if (!child)
+ continue;
+ sch_tree_lock(child);
+ qdisc_reset(child);
+ sch_tree_unlock(child);
+ }
+ }
+
taprio_disable_offload(dev, q, NULL);
if (q->qdiscs) {