diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c
index 803ead499ae9..4851097f0ae5 100644
--- a/net/sched/sch_cake.c
+++ b/net/sched/sch_cake.c
@@ -709,19 +709,28 @@ static u16 cake_get_flow_quantum(struct cake_tin_data *q,
get_random_u16()) >> 16;
}
-static u32 cake_hash(struct cake_tin_data *q, const struct sk_buff *skb,
- int flow_mode, u16 flow_override, u16 host_override)
+struct cake_hash_keys {
+ u32 srchost_hash;
+ u32 dsthost_hash;
+ u32 flow_hash;
+};
+
+static void cake_hash_keys(const struct sk_buff *skb, int flow_mode,
+ u16 flow_override, u16 host_override,
+ struct cake_hash_keys *out)
{
bool hash_flows = (!flow_override && !!(flow_mode & CAKE_FLOW_FLOWS));
bool hash_hosts = (!host_override && !!(flow_mode & CAKE_FLOW_HOSTS));
bool nat_enabled = !!(flow_mode & CAKE_FLOW_NAT_FLAG);
- u32 flow_hash = 0, srchost_hash = 0, dsthost_hash = 0;
- u16 reduced_hash, srchost_idx, dsthost_idx;
struct flow_keys keys, host_keys;
bool use_skbhash = skb->l4_hash;
+ out->flow_hash = 0;
+ out->srchost_hash = 0;
+ out->dsthost_hash = 0;
+
if (unlikely(flow_mode == CAKE_FLOW_NONE))
- return 0;
+ return;
/* If both overrides are set, or we can use the SKB hash and nat mode is
* disabled, we can skip packet dissection entirely. If nat mode is@@ -756,50 +765,64 @@ static u32 cake_hash(struct cake_tin_data *q, const struct sk_buff *skb,
switch (host_keys.control.addr_type) {
case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
host_keys.addrs.v4addrs.src = 0;
- dsthost_hash = flow_hash_from_keys(&host_keys);
+ out->dsthost_hash = flow_hash_from_keys(&host_keys);
host_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src;
host_keys.addrs.v4addrs.dst = 0;
- srchost_hash = flow_hash_from_keys(&host_keys);
+ out->srchost_hash = flow_hash_from_keys(&host_keys);
break;
case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
memset(&host_keys.addrs.v6addrs.src, 0,
sizeof(host_keys.addrs.v6addrs.src));
- dsthost_hash = flow_hash_from_keys(&host_keys);
+ out->dsthost_hash = flow_hash_from_keys(&host_keys);
host_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src;
memset(&host_keys.addrs.v6addrs.dst, 0,
sizeof(host_keys.addrs.v6addrs.dst));
- srchost_hash = flow_hash_from_keys(&host_keys);
+ out->srchost_hash = flow_hash_from_keys(&host_keys);
break;
default:
- dsthost_hash = 0;
- srchost_hash = 0;
+ out->dsthost_hash = 0;
+ out->srchost_hash = 0;
}
/* This *must* be after the above switch, since as a
* side-effect it sorts the src and dst addresses.
*/
if (hash_flows && !use_skbhash)
- flow_hash = flow_hash_from_keys(&keys);
+ out->flow_hash = flow_hash_from_keys(&keys);
skip_hash:
if (flow_override)
- flow_hash = flow_override - 1;
+ out->flow_hash = flow_override - 1;
else if (use_skbhash && (flow_mode & CAKE_FLOW_FLOWS))
- flow_hash = skb->hash;
+ out->flow_hash = skb->hash;
if (host_override) {
- dsthost_hash = host_override - 1;
- srchost_hash = host_override - 1;
+ out->dsthost_hash = host_override - 1;
+ out->srchost_hash = host_override - 1;
}
if (!(flow_mode & CAKE_FLOW_FLOWS)) {
if (flow_mode & CAKE_FLOW_SRC_IP)
- flow_hash ^= srchost_hash;
+ out->flow_hash ^= out->srchost_hash;
if (flow_mode & CAKE_FLOW_DST_IP)
- flow_hash ^= dsthost_hash;
+ out->flow_hash ^= out->dsthost_hash;
}
+}
+
+/* Commit the flow into the set-associative table (q->tags[], host
+ * indices, bulk counters). Only after the packet is admitted.
+ */
+static u32 cake_hash_resolve(struct cake_tin_data *q,
+ const struct cake_hash_keys *keys, int flow_mode)
+{
+ u32 flow_hash = keys->flow_hash, srchost_hash = keys->srchost_hash,
+ dsthost_hash = keys->dsthost_hash;
+ u16 reduced_hash, srchost_idx, dsthost_idx;
+
+ if (unlikely(flow_mode == CAKE_FLOW_NONE))
+ return 0;
reduced_hash = flow_hash % CAKE_QUEUES;
@@ -1765,13 +1788,17 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
{
u32 idx, tin, prev_qlen, prev_backlog, drop_id;
struct cake_sched_data *q = qdisc_priv(sch);
+ int flow_mode = READ_ONCE(q->config->flow_mode);
int len = qdisc_pkt_len(skb), ret;
u16 flow_override, host_override;
+ struct sk_buff *segs = NULL;
+ struct cake_hash_keys keys;
struct sk_buff *ack = NULL;
ktime_t now = ktime_get();
struct cake_tin_data *b;
struct cake_flow *flow;
bool same_flow = false;
+ u64 slen = 0;
/* Classify first: a terminal verdict means do not enqueue. */
if (cake_tcf_classify(sch, skb, &flow_override, &host_override, &ret)) {@@ -1788,12 +1815,50 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
QDISC_DROP_OVERLIMIT);
}
- /* choose flow to insert into */
+ /* Select the tin and extract the keys before segmenting:
+ * skb_gso_segment() rewrites encapsulated headers, which
+ * changes the keys.
+ */
b = cake_select_tin(sch, skb);
- idx = cake_hash(b, skb, q->config->flow_mode, flow_override,
- host_override) + 1;
+ cake_hash_keys(skb, flow_mode, flow_override, host_override, &keys);
+
+ if (qdisc_pkt_segs(skb) > 1 && q->config->rate_flags & CAKE_FLAG_SPLIT_GSO) {
+ netdev_features_t features = netif_skb_features(skb);
+ struct sk_buff *nskb, *seg;
+
+ segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
+ if (IS_ERR_OR_NULL(segs))
+ return qdisc_drop_reason(skb, sch, to_free,
+ QDISC_DROP_GENERIC);
+
+ /* we never use the original skb after splitting */
+ consume_skb(skb);
+ skb = NULL;
+
+ /* The split path accounts the segment sum, not the
+ * stab-adjusted qdisc_pkt_len(), so check it here.
+ */
+ skb_list_walk_safe(segs, seg, nskb)
+ slen += seg->len;
+
+ if (unlikely((u64)sch->qstats.backlog + slen > QDISC_MAX_BACKLOG)) {
+ qdisc_qstats_overlimit(sch);
+ /* __skb_gso_segment() puts SKB_GSO_CB at cb
+ * offset 32, over tc_skb_cb.drop_reason; every
+ * segment inherits it. Tag each one with the
+ * qdisc drop reason.
+ */
+ skb_list_walk_safe(segs, seg, nskb)
+ tcf_set_qdisc_drop_reason(seg,
+ QDISC_DROP_OVERLIMIT);
+
+ return qdisc_drop_all(segs, sch, to_free);
+ }
+ }
+
+ /* Admitted: resolve the flow. */
+ idx = cake_hash_resolve(b, &keys, flow_mode);
tin = (u32)(b - q->tins);
- idx--;
flow = &b->flows[idx];
/* ensure shaper state isn't stale */@@ -1820,14 +1885,9 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
if (unlikely(len > b->max_skblen))
WRITE_ONCE(b->max_skblen, len);
- if (qdisc_pkt_segs(skb) > 1 && q->config->rate_flags & CAKE_FLAG_SPLIT_GSO) {
- struct sk_buff *segs, *nskb, *seg;
- netdev_features_t features = netif_skb_features(skb);
- unsigned int slen = 0, numsegs = 0;
-
- segs = skb_gso_segment(skb, features & ~NETIF_F_GSO_MASK);
- if (IS_ERR_OR_NULL(segs))
- return qdisc_drop(skb, sch, to_free);
+ if (segs) {
+ struct sk_buff *nskb, *seg;
+ unsigned int numsegs = 0;
skb_list_walk_safe(segs, seg, nskb) {
skb_mark_not_on_list(seg);@@ -1841,7 +1901,6 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
qdisc_qlen_inc(sch);
numsegs++;
q->buffer_used += seg->truesize;
- slen += seg->len;
WRITE_ONCE(b->packets, b->packets + 1);
}
@@ -1853,9 +1912,8 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
WRITE_ONCE(b->backlogs[idx], b->backlogs[idx] + slen);
qdisc_tree_reduce_backlog(sch, 1-numsegs, len-slen);
- consume_skb(skb);
} else {
- /* not splitting */
+ /* not splitting - skb pointer still valid */
int ack_pkt_len = 0;
cobalt_set_enqueue_time(skb, now);@@ -1949,7 +2007,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
flow->set = CAKE_SET_SPARSE;
WRITE_ONCE(b->sparse_flow_count, b->sparse_flow_count + 1);
- WRITE_ONCE(flow->deficit, cake_get_flow_quantum(b, flow, q->config->flow_mode));
+ WRITE_ONCE(flow->deficit, cake_get_flow_quantum(b, flow, flow_mode));
} else if (flow->set == CAKE_SET_SPARSE_WAIT) {
/* this flow was empty, accounted as a sparse flow, but actually
* in the bulk rotation.@@ -1958,8 +2016,8 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
WRITE_ONCE(b->sparse_flow_count, b->sparse_flow_count - 1);
WRITE_ONCE(b->bulk_flow_count, b->bulk_flow_count + 1);
- cake_inc_srchost_bulk_flow_count(b, flow, q->config->flow_mode);
- cake_inc_dsthost_bulk_flow_count(b, flow, q->config->flow_mode);
+ cake_inc_srchost_bulk_flow_count(b, flow, flow_mode);
+ cake_inc_dsthost_bulk_flow_count(b, flow, flow_mode);
}
if (q->buffer_used > u64_stats_read(&q->buffer_max_used)) {--
2.43.0