diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h
index 90d3e7943b19..18a419cd9d94 100644
--- a/include/net/pkt_sched.h
+++ b/include/net/pkt_sched.h
@@ -12,7 +12,6 @@
#define DEFAULT_TX_QUEUE_LEN 1000
#define STAB_SIZE_LOG_MAX 30
-#define QDISC_PKT_LEN_MAX (1 << 20) /* 1 MiB */
struct qdisc_walker {
int stop;diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index 1acaadb3e2cf..a5a6bdf55e0a 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -915,6 +915,26 @@ static inline unsigned int qdisc_pkt_len(const struct sk_buff *skb)
return qdisc_skb_cb(skb)->pkt_len;
}
+#define QDISC_PKT_LEN_MAX (1 << 20) /* 1 MiB */
+
+/* Largest backlog that still has room for one more 1 MiB packet. */
+#define QDISC_MAX_BACKLOG (U32_MAX - QDISC_PKT_LEN_MAX)
+
+/* Would this packet's accounted length wrap sch->qstats.backlog?
+ * Packets up to QDISC_PKT_LEN_MAX fit below QDISC_MAX_BACKLOG; GSO
+ * can make the length larger, so compare against the real headroom.
+ */
+static inline bool qdisc_backlog_overflow(const struct Qdisc *sch,
+ const struct sk_buff *skb)
+{
+ u32 backlog = READ_ONCE(sch->qstats.backlog);
+ u32 len = qdisc_pkt_len(skb);
+
+ if (likely(len <= QDISC_PKT_LEN_MAX))
+ return backlog > QDISC_MAX_BACKLOG;
+ return len > U32_MAX - backlog;
+}
+
static inline unsigned int qdisc_pkt_segs(const struct sk_buff *skb)
{
u32 pkt_segs = qdisc_skb_cb(skb)->pkt_segs;diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c
index dfc187f48a2f..3fd13bf6f3ce 100644
--- a/net/sched/sch_cake.c
+++ b/net/sched/sch_cake.c
@@ -1781,6 +1781,13 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
return ret;
}
+ /* Drop before touching the hash or shaper state. */
+ if (unlikely(qdisc_backlog_overflow(sch, skb))) {
+ qdisc_qstats_overlimit(sch);
+ return qdisc_drop_reason(skb, sch, to_free,
+ QDISC_DROP_OVERLIMIT);
+ }
+
/* choose flow to insert into */
b = cake_select_tin(sch, skb);
idx = cake_hash(b, skb, q->config->flow_mode, flow_override,diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c
index a1ba2e355f2d..78c5d72d6406 100644
--- a/net/sched/sch_codel.c
+++ b/net/sched/sch_codel.c
@@ -116,15 +116,17 @@ static struct sk_buff *codel_peek(struct Qdisc *sch)
static int codel_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
struct sk_buff **to_free)
{
- struct codel_sched_data *q;
+ struct codel_sched_data *q = qdisc_priv(sch);
- if (likely(qdisc_qlen(sch) < sch->limit)) {
- codel_set_enqueue_time(skb);
- return qdisc_enqueue_tail(skb, sch);
+ if (unlikely(qdisc_qlen(sch) >= sch->limit) ||
+ unlikely(qdisc_backlog_overflow(sch, skb))) {
+ WRITE_ONCE(q->drop_overlimit, q->drop_overlimit + 1);
+ return qdisc_drop_reason(skb, sch, to_free,
+ QDISC_DROP_OVERLIMIT);
}
- q = qdisc_priv(sch);
- WRITE_ONCE(q->drop_overlimit, q->drop_overlimit + 1);
- return qdisc_drop_reason(skb, sch, to_free, QDISC_DROP_OVERLIMIT);
+
+ codel_set_enqueue_time(skb);
+ return qdisc_enqueue_tail(skb, sch);
}
static const struct nla_policy codel_policy[TCA_CODEL_MAX + 1] = {diff --git a/net/sched/sch_dualpi2.c b/net/sched/sch_dualpi2.c
index 4947def7c49e..7331f0ec1c8f 100644
--- a/net/sched/sch_dualpi2.c
+++ b/net/sched/sch_dualpi2.c
@@ -392,6 +392,7 @@ static int dualpi2_enqueue_skb(struct sk_buff *skb, struct Qdisc *sch,
struct dualpi2_skb_cb *cb;
if (unlikely(qdisc_qlen(sch) >= sch->limit) ||
+ unlikely(qdisc_backlog_overflow(sch, skb)) ||
unlikely((u64)q->memory_used + skb->truesize > q->memory_limit)) {
qdisc_qstats_overlimit(sch);
if (skb_in_l_queue(skb))diff --git a/net/sched/sch_fifo.c b/net/sched/sch_fifo.c
index 1b6388d50967..16c8d4f74e7e 100644
--- a/net/sched/sch_fifo.c
+++ b/net/sched/sch_fifo.c
@@ -29,6 +29,9 @@ static int bfifo_enqueue(struct sk_buff *skb, struct Qdisc *sch,
static int pfifo_enqueue(struct sk_buff *skb, struct Qdisc *sch,
struct sk_buff **to_free)
{
+ if (unlikely(qdisc_backlog_overflow(sch, skb)))
+ return qdisc_drop(skb, sch, to_free);
+
if (likely(sch->q.qlen < READ_ONCE(sch->limit)))
return qdisc_enqueue_tail(skb, sch);
@@ -43,6 +46,9 @@ static int pfifo_tail_enqueue(struct sk_buff *skb, struct Qdisc *sch,
if (unlikely(READ_ONCE(sch->limit) == 0))
return qdisc_drop(skb, sch, to_free);
+ if (unlikely(qdisc_backlog_overflow(sch, skb)))
+ return qdisc_drop(skb, sch, to_free);
+
if (likely(sch->q.qlen < READ_ONCE(sch->limit)))
return qdisc_enqueue_tail(skb, sch);
diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c
index e6c87a32950f..a83f20534be1 100644
--- a/net/sched/sch_fq_codel.c
+++ b/net/sched/sch_fq_codel.c
@@ -201,6 +201,15 @@ static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch,
}
idx--;
+ /* Drop before the flow is touched, so a dropped packet cannot
+ * create or update flow state.
+ */
+ if (unlikely(qdisc_backlog_overflow(sch, skb))) {
+ q->drop_overlimit++;
+ return qdisc_drop_reason(skb, sch, to_free,
+ QDISC_DROP_OVERLIMIT);
+ }
+
codel_set_enqueue_time(skb);
flow = &q->flows[idx];
flow_queue_add(flow, skb);diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c
index 5982847df8f8..9f927a08b7bc 100644
--- a/net/sched/sch_fq_pie.c
+++ b/net/sched/sch_fq_pie.c
@@ -155,7 +155,8 @@ static int fq_pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
memory_limited = q->memory_usage > q->memory_limit + skb->truesize;
/* Checks if the qdisc is full */
- if (unlikely(qdisc_qlen(sch) >= sch->limit)) {
+ if (unlikely(qdisc_qlen(sch) >= sch->limit) ||
+ unlikely(qdisc_backlog_overflow(sch, skb))) {
q->stats.overlimit++;
goto out;
} else if (unlikely(memory_limited)) {diff --git a/net/sched/sch_gred.c b/net/sched/sch_gred.c
index f3d9a7252949..de96badff654 100644
--- a/net/sched/sch_gred.c
+++ b/net/sched/sch_gred.c
@@ -170,6 +170,15 @@ static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch,
unsigned long qavg = 0;
u16 dp = tc_index_to_dp(skb);
+ /* Per-VQ limits don't bound the total: every VQ adds into the
+ * same u32 backlog, so check it before picking a VQ.
+ */
+ if (unlikely(qdisc_backlog_overflow(sch, skb))) {
+ qdisc_qstats_overlimit(sch);
+ return qdisc_drop_reason(skb, sch, to_free,
+ QDISC_DROP_OVERLIMIT);
+ }
+
if (dp >= t->DPs || (q = t->tab[dp]) == NULL) {
dp = t->def;
diff --git a/net/sched/sch_pie.c b/net/sched/sch_pie.c
index 3b7863ffd284..2a5fc493a7e5 100644
--- a/net/sched/sch_pie.c
+++ b/net/sched/sch_pie.c
@@ -89,7 +89,8 @@ static int pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
struct pie_sched_data *q = qdisc_priv(sch);
bool enqueue = false;
- if (unlikely(qdisc_qlen(sch) >= sch->limit)) {
+ if (unlikely(qdisc_qlen(sch) >= sch->limit) ||
+ unlikely(qdisc_backlog_overflow(sch, skb))) {
WRITE_ONCE(q->stats.overlimit, q->stats.overlimit + 1);
goto out;
}diff --git a/net/sched/sch_plug.c b/net/sched/sch_plug.c
index b60ddfee6a68..c90b5ca60bc0 100644
--- a/net/sched/sch_plug.c
+++ b/net/sched/sch_plug.c
@@ -89,7 +89,14 @@ static int plug_enqueue(struct sk_buff *skb, struct Qdisc *sch,
{
struct plug_sched_data *q = qdisc_priv(sch);
- if (likely((u64)sch->qstats.backlog + skb->len <= q->limit)) {
+ if (unlikely(qdisc_backlog_overflow(sch, skb))) {
+ qdisc_qstats_overlimit(sch);
+ return qdisc_drop_reason(skb, sch, to_free,
+ QDISC_DROP_OVERLIMIT);
+ }
+
+ if (likely((u64)sch->qstats.backlog + qdisc_pkt_len(skb) <=
+ q->limit)) {
if (!q->unplug_indefinite)
q->pkts_current_epoch++;
return qdisc_enqueue_tail(skb, sch);diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c
index d7598214270b..b5233d9fbf57 100644
--- a/net/sched/sch_red.c
+++ b/net/sched/sch_red.c
@@ -73,9 +73,22 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch,
enum qdisc_drop_reason reason = QDISC_DROP_CONGESTED;
struct red_sched_data *q = qdisc_priv(sch);
struct Qdisc *child = q->qdisc;
+ u32 child_backlog;
+ u32 red_backlog;
unsigned int len;
int ret;
+ /* RED reads the child backlog into red_calc_qavg(), so bound both
+ * counters: a child that evicts on NET_XMIT_CN can otherwise grow
+ * its own backlog while RED's stays put.
+ */
+ if (unlikely(qdisc_backlog_overflow(sch, skb) ||
+ qdisc_backlog_overflow(child, skb))) {
+ qdisc_qstats_overlimit(sch);
+ return qdisc_drop_reason(skb, sch, to_free,
+ QDISC_DROP_OVERLIMIT);
+ }
+
q->vars.qavg = red_calc_qavg(&q->parms,
&q->vars,
child->qstats.backlog);@@ -136,14 +149,37 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch,
}
len = qdisc_pkt_len(skb);
+ /* Reserve the length before the child sees the packet: netem
+ * can re-enqueue a clone here before the backlog is updated.
+ */
+ red_backlog = READ_ONCE(sch->qstats.backlog);
+ child_backlog = READ_ONCE(child->qstats.backlog);
+ qstats_backlog_add(sch, len);
ret = qdisc_enqueue(skb, child, to_free);
if (likely(ret == NET_XMIT_SUCCESS)) {
- qstats_backlog_add(sch, len);
qdisc_qlen_inc(sch);
- } else if (net_xmit_drop_count(ret)) {
- WRITE_ONCE(q->stats.pdrop,
- q->stats.pdrop + 1);
- qdisc_qstats_drop(sch);
+ } else {
+ u32 propagated, grew;
+
+ /* Re-derive the aggregate from the snapshots: the child may
+ * have dropped the packet or evicted another one, and may or
+ * may not have reduced the ancestors' backlog itself
+ * (pfifo_head_drop, sfq and fq_codel do; hhf and skbprio
+ * return NET_XMIT_CN without qdisc_tree_reduce_backlog()).
+ * Keep what the child actually accounted for, and pass the
+ * difference on to the ancestors above RED.
+ */
+ propagated = READ_ONCE(sch->qstats.backlog) -
+ red_backlog - len;
+ grew = READ_ONCE(child->qstats.backlog) - child_backlog;
+ WRITE_ONCE(sch->qstats.backlog, red_backlog + grew);
+ if (propagated != grew)
+ qdisc_tree_reduce_backlog(sch, 0, propagated - grew);
+ if (net_xmit_drop_count(ret)) {
+ WRITE_ONCE(q->stats.pdrop,
+ q->stats.pdrop + 1);
+ qdisc_qstats_drop(sch);
+ }
}
return ret;
--
2.43.0