diff --git a/include/net/flow_offload.h b/include/net/flow_offload.h
index 70a02ee14308..09301a924014 100644
--- a/include/net/flow_offload.h
+++ b/include/net/flow_offload.h
@@ -697,6 +697,18 @@ struct flow_cls_offload {
bool use_act_stats;
unsigned long cookie;
struct flow_rule *rule;
+ /* Optional owner notification when a rule no longer represents the path.
+ * Call under RCU; the owner retains the cookie and callback until a
+ * grace period after DESTROY. Drivers must stop new calls at DESTROY.
+ * The callback must not sleep or call back into the offload driver.
+ */
+ void (*invalidate)(unsigned long cookie);
+ /* Optional cached-path validation, called under RCU during REPLACE.
+ * Return 0 for a valid path, -ESTALE for a demonstrably stale path,
+ * or -EOPNOTSUPP when validation is unavailable. Do not sleep or
+ * call back into the driver. The callback does not invalidate its owner.
+ */
+ int (*validate)(unsigned long cookie);
struct flow_stats stats;
u32 classid;
};diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h
index 5b22d31cb140..99cb087ff0b2 100644
--- a/include/net/netfilter/nf_flow_table.h
+++ b/include/net/netfilter/nf_flow_table.h
@@ -154,7 +154,8 @@ struct flow_offload_tuple {
encap_num:2,
needs_gso_segment:1,
tun_num:2,
- in_vlan_ingress:2;
+ in_vlan_ingress:2,
+ dst_validation_only:1;
u16 mtu;
u32 dst_cookie;
struct dst_entry *dst_cache;@@ -305,7 +306,7 @@ nf_flow_table_offload_del_cb(struct nf_flowtable *flow_table,
}
void flow_offload_route_init(struct flow_offload *flow,
- struct nf_flow_route *route);
+ struct nf_flow_route *route, bool keep_dst);
int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow);
void flow_offload_refresh(struct nf_flowtable *flow_table,
@@ -313,7 +314,8 @@ void flow_offload_refresh(struct nf_flowtable *flow_table,
static inline bool nf_flow_dst_check(struct flow_offload_tuple *tuple)
{
- if (!tuple->dst_cache)
+ /* A driver-only reference must not change direct-path/GC semantics. */
+ if (!tuple->dst_cache || tuple->dst_validation_only)
return true;
return dst_check(tuple->dst_cache, tuple->dst_cookie);diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c
index 4c82097cd50b..b16a336101af 100644
--- a/net/netfilter/nf_flow_table_core.c
+++ b/net/netfilter/nf_flow_table_core.c
@@ -96,7 +96,7 @@ static struct dst_entry *nft_route_dst_fetch(struct nf_flow_route *route,
static int flow_offload_fill_route(struct flow_offload *flow,
struct nf_flow_route *route,
- enum flow_offload_tuple_dir dir)
+ enum flow_offload_tuple_dir dir, bool keep_dst)
{
struct flow_offload_tuple *flow_tuple = &flow->tuplehash[dir].tuple;
struct dst_entry *dst = nft_route_dst_fetch(route, dir);@@ -127,7 +127,9 @@ static int flow_offload_fill_route(struct flow_offload *flow,
switch (route->tuple[dir].xmit_type) {
case FLOW_OFFLOAD_XMIT_DIRECT:
- if (route->tuple[!dir].in.num_tuns) {
+ flow_tuple->dst_validation_only = keep_dst &&
+ !route->tuple[!dir].in.num_tuns;
+ if (keep_dst || route->tuple[!dir].in.num_tuns) {
flow_tuple->dst_cache = dst;
flow_tuple->dst_cookie =
flow_offload_dst_cookie(flow_tuple);@@ -162,10 +164,10 @@ static void nft_flow_dst_release(struct flow_offload *flow,
}
void flow_offload_route_init(struct flow_offload *flow,
- struct nf_flow_route *route)
+ struct nf_flow_route *route, bool keep_dst)
{
- flow_offload_fill_route(flow, route, FLOW_OFFLOAD_DIR_ORIGINAL);
- flow_offload_fill_route(flow, route, FLOW_OFFLOAD_DIR_REPLY);
+ flow_offload_fill_route(flow, route, FLOW_OFFLOAD_DIR_ORIGINAL, keep_dst);
+ flow_offload_fill_route(flow, route, FLOW_OFFLOAD_DIR_REPLY, keep_dst);
flow->type = NF_FLOW_OFFLOAD_ROUTE;
}
EXPORT_SYMBOL_GPL(flow_offload_route_init);diff --git a/net/netfilter/nf_flow_table_offload.c b/net/netfilter/nf_flow_table_offload.c
index 4365859220e6..1f3adf70f598 100644
--- a/net/netfilter/nf_flow_table_offload.c
+++ b/net/netfilter/nf_flow_table_offload.c
@@ -885,6 +885,62 @@ static int nf_flow_offload_alloc(const struct flow_offload_work *offload,
return 0;
}
+static struct flow_offload *nf_flow_offload_cookie_flow(unsigned long cookie)
+{
+ struct flow_offload_tuple *tuple = (struct flow_offload_tuple *)cookie;
+ struct flow_offload_tuple_rhash *tuplehash;
+ struct flow_offload *flow;
+
+ tuplehash = container_of(tuple, struct flow_offload_tuple_rhash, tuple);
+ flow = container_of(tuplehash, struct flow_offload, tuplehash[tuple->dir]);
+ return flow;
+}
+
+static void nf_flow_offload_invalidate(unsigned long cookie)
+{
+ flow_offload_teardown(nf_flow_offload_cookie_flow(cookie));
+}
+
+static int nf_flow_offload_validate(unsigned long cookie)
+{
+ struct flow_offload *flow = nf_flow_offload_cookie_flow(cookie);
+ unsigned int dir;
+
+ for (dir = 0; dir < FLOW_OFFLOAD_DIR_MAX; dir++) {
+ struct flow_offload_tuple *tuple = &flow->tuplehash[dir].tuple;
+ struct flow_offload_tuple *other = &flow->tuplehash[!dir].tuple;
+ struct dst_entry *dst = tuple->dst_cache;
+ struct neighbour *neigh;
+ u8 mac[ETH_ALEN];
+ bool valid;
+
+ if (!dst)
+ return -EOPNOTSUPP;
+ /* This is the actual route selected with the packet's policy
+ * context, including mark, DSCP and the reverse lookup's oif.
+ */
+ if (!dst_check(dst, tuple->dst_cookie))
+ return -ESTALE;
+ if (tuple->xmit_type != FLOW_OFFLOAD_XMIT_DIRECT)
+ continue;
+ /* A stacked path may have a different final Ethernet neighbour.
+ * Do not mistake that unsupported context for a stale owner.
+ */
+ if (dst->dev->ifindex != tuple->out.ifidx)
+ return -EOPNOTSUPP;
+ neigh = dst_neigh_lookup(dst, &other->src_v4);
+ if (!neigh)
+ return -ESTALE;
+ neigh_ha_snapshot(mac, neigh, dst->dev);
+ valid = (READ_ONCE(neigh->nud_state) & NUD_VALID) &&
+ ether_addr_equal(mac, tuple->out.h_dest);
+ neigh_release(neigh);
+ if (!valid)
+ return -ESTALE;
+ }
+ return 0;
+}
+
static void nf_flow_offload_init(struct flow_cls_offload *cls_flow,
__be16 proto, int priority,
enum flow_cls_command cmd,@@ -896,6 +952,8 @@ static void nf_flow_offload_init(struct flow_cls_offload *cls_flow,
cls_flow->common.extack = extack;
cls_flow->command = cmd;
cls_flow->cookie = (unsigned long)tuple;
+ cls_flow->invalidate = nf_flow_offload_invalidate;
+ cls_flow->validate = nf_flow_offload_validate;
}
static int nf_flow_offload_tuple(struct nf_flowtable *flowtable,
diff --git a/net/netfilter/nft_flow_offload.c b/net/netfilter/nft_flow_offload.c
index d3c5651dd699..05207ca808ee 100644
--- a/net/netfilter/nft_flow_offload.c
+++ b/net/netfilter/nft_flow_offload.c
@@ -113,7 +113,11 @@ static void nft_flow_offload_eval(const struct nft_expr *expr,
if (!flow)
goto err_flow_alloc;
- flow_offload_route_init(flow, &route);
+ /* Hardware admission may validate the route used to build a direct path.
+ * Keep its reference and cookie instead of reconstructing a lookup with
+ * incomplete policy-routing inputs in the driver.
+ */
+ flow_offload_route_init(flow, &route, nf_flowtable_hw_offload(flowtable));
if (tcph)
flow_offload_ct_tcp(ct);
--
2.53.0