Re: [RFC/PATCH net-next v3 3/8] flow_offload: allow user to offload tc action to net device

On Mon 01 Nov 2021 at 11:44, Baowen Zheng [off-list ref] wrote:

Thanks for your review and sorry for delay in responding.

On October 30, 2021 12:59 AM, Vlad Buslov [off-list ref] wrote:

quoted

On Thu 28 Oct 2021 at 14:06, Simon Horman [off-list ref]
wrote:

quoted

From: Baowen Zheng <redacted>

Use flow_indr_dev_register/flow_indr_dev_setup_offload to offload tc
action.

We need to call tc_cleanup_flow_action to clean up tc action entry
since in tc_setup_action, some actions may hold dev refcnt, especially
the mirror action.

Signed-off-by: Baowen Zheng <redacted>
Signed-off-by: Louis Peens <redacted>
Signed-off-by: Simon Horman <redacted>
---
 include/linux/netdevice.h  |   1 +
 include/net/act_api.h      |   2 +-
 include/net/flow_offload.h |  17 ++++
 include/net/pkt_cls.h      |  15 ++++
 net/core/flow_offload.c    |  43 ++++++++--
 net/sched/act_api.c        | 166

+++++++++++++++++++++++++++++++++++++

quoted

 net/sched/cls_api.c        |  29 ++++++-
 7 files changed, 260 insertions(+), 13 deletions(-)

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 3ec42495a43a..9815c3a058e9 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h

@@ -916,6 +916,7 @@ enum tc_setup_type {
 	TC_SETUP_QDISC_TBF,
 	TC_SETUP_QDISC_FIFO,
 	TC_SETUP_QDISC_HTB,
+	TC_SETUP_ACT,
 };

 /* These structures hold the attributes of bpf state that are being

passed diff --git a/include/net/act_api.h b/include/net/act_api.h
index b5b624c7e488..9eb19188603c 100644

--- a/include/net/act_api.h
+++ b/include/net/act_api.h

@@ -239,7 +239,7 @@ static inline void

tcf_action_inc_overlimit_qstats(struct tc_action *a)  void

tcf_action_update_stats(struct tc_action *a, u64 bytes, u64 packets,

quoted

 			     u64 drops, bool hw);
 int tcf_action_copy_stats(struct sk_buff *, struct tc_action *, int);
-
+int tcf_action_offload_del(struct tc_action *action);

This doesn't seem to be used anywhere outside of act_api in this series, so
why is it exported?

Thanks for bring this to us, we will fix this by moving the block of implement in act_api.c.

quoted

quoted

 int tcf_action_check_ctrlact(int action, struct tcf_proto *tp,
 			     struct tcf_chain **handle,
 			     struct netlink_ext_ack *newchain); diff --git
a/include/net/flow_offload.h b/include/net/flow_offload.h index
3961461d9c8b..aa28592fccc0 100644

--- a/include/net/flow_offload.h
+++ b/include/net/flow_offload.h

@@ -552,6 +552,23 @@ struct flow_cls_offload {
 	u32 classid;
 };

+enum flow_act_command {
+	FLOW_ACT_REPLACE,
+	FLOW_ACT_DESTROY,
+	FLOW_ACT_STATS,
+};
+
+struct flow_offload_action {
+	struct netlink_ext_ack *extack; /* NULL in FLOW_ACT_STATS

process*/

quoted

+	enum flow_act_command command;
+	enum flow_action_id id;
+	u32 index;
+	struct flow_stats stats;
+	struct flow_action action;
+};
+
+struct flow_offload_action *flow_action_alloc(unsigned int
+num_actions);
+
 static inline struct flow_rule *
 flow_cls_offload_flow_rule(struct flow_cls_offload *flow_cmd)  { diff
--git a/include/net/pkt_cls.h b/include/net/pkt_cls.h index
193f88ebf629..922775407257 100644

--- a/include/net/pkt_cls.h
+++ b/include/net/pkt_cls.h

@@ -258,6 +258,9 @@ static inline void tcf_exts_put_net(struct tcf_exts

*exts)

quoted

 	for (; 0; (void)(i), (void)(a), (void)(exts))  #endif

+#define tcf_act_for_each_action(i, a, actions) \
+	for (i = 0; i < TCA_ACT_MAX_PRIO && ((a) = actions[i]); i++)
+
 static inline void
 tcf_exts_stats_update(const struct tcf_exts *exts,
 		      u64 bytes, u64 packets, u64 drops, u64 lastuse, @@ -532,8
+535,19 @@ tcf_match_indev(struct sk_buff *skb, int ifindex)
 	return ifindex == skb->skb_iif;
 }

+#ifdef CONFIG_NET_CLS_ACT
 int tc_setup_flow_action(struct flow_action *flow_action,
 			 const struct tcf_exts *exts);

Why does existing cls_api function tc_setup_flow_action() now depend on
CONFIG_NET_CLS_ACT?

Originally the function tc_setup_flow_action deal with the dependence of CONFIG_NET_CLS_ACT
By calling the macro tcf_exts_for_each_action, now we change to call the function tc_setup_action
Then tc_setup_flow_action will refer to exts->actions, so it will depend on CONFIG_NET_CLS_ACT explicitly.
To fix this, we have to have the ifdef in tc_setup_flow_action declaration or in the implement in cls_api.c.
Do you think if it makes sense?

Since we already have multiple of such ifdefs in cls_api I don't think
having more is an issue, but I also don't think we need to ifdef this
function in both pkt_cls.h and cls_api.c. Unless I'm missing something
you can either:

- Make tc_setup_flow_action() inline in pkt_cls.h and remove its
definition from cls_api.c since tc_setup_action() is also exported.

- Move ifdef check inside function definition in cls_api.c (return 0, if
config is not defined), which will allows you to remove ifdef from
pkt_cls.h.

WDYT?

quoted

quoted

+#else
+static inline int tc_setup_flow_action(struct flow_action *flow_action,
+				       const struct tcf_exts *exts) {
+	return 0;
+}
+#endif
+
+int tc_setup_action(struct flow_action *flow_action,
+		    struct tc_action *actions[]);
 void tc_cleanup_flow_action(struct flow_action *flow_action);

...

quoted

quoted

 #ifdef CONFIG_INET
 DEFINE_STATIC_KEY_FALSE(tcf_frag_xmit_count);

@@ -148,6 +161,7 @@ static int __tcf_action_put(struct tc_action *p, bool

bind)

quoted

 		idr_remove(&idrinfo->action_idr, p->tcfa_index);
 		mutex_unlock(&idrinfo->lock);

+		tcf_action_offload_del(p);
 		tcf_action_cleanup(p);
 		return 1;
 	}

@@ -341,6 +355,7 @@ static int tcf_idr_release_unsafe(struct tc_action *p)
 		return -EPERM;

 	if (refcount_dec_and_test(&p->tcfa_refcnt)) {
+		tcf_action_offload_del(p);
 		idr_remove(&p->idrinfo->action_idr, p->tcfa_index);
 		tcf_action_cleanup(p);
 		return ACT_P_DELETED;

@@ -452,6 +467,7 @@ static int tcf_idr_delete_index(struct tcf_idrinfo

*idrinfo, u32 index)

quoted

 						p->tcfa_index));
 			mutex_unlock(&idrinfo->lock);

+			tcf_action_offload_del(p);

tcf_action_offload_del() and tcf_action_cleanup() seem to be always called
together. Consider moving the call to tcf_action_offload_del() into
tcf_action_cleanup().

Thanks, we will consider to move tcf_action_offload_del() inside of tcf_action_cleanup.

quoted

quoted

 			tcf_action_cleanup(p);
 			module_put(owner);
 			return 0;

@@ -1061,6 +1077,154 @@ struct tc_action *tcf_action_init_1(struct net

*net, struct tcf_proto *tp,

quoted

 	return ERR_PTR(err);
 }

...

quoted

quoted

+/* offload the tc command after inserted */ static int
+tcf_action_offload_add(struct tc_action *action,
+				  struct netlink_ext_ack *extack) {
+	struct tc_action *actions[TCA_ACT_MAX_PRIO] = {
+		[0] = action,
+	};
+	struct flow_offload_action *fl_action;
+	int err = 0;
+
+	fl_action = flow_action_alloc(tcf_act_num_actions_single(action));
+	if (!fl_action)
+		return -EINVAL;

Failed alloc-like functions usually result -ENOMEM.

Thanks, we will fix this in V4 patch.

quoted

quoted

+
+	err = flow_action_init(fl_action, action, FLOW_ACT_REPLACE, extack);
+	if (err)
+		goto fl_err;
+
+	err = tc_setup_action(&fl_action->action, actions);
+	if (err) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Failed to setup tc actions for offload\n");
+		goto fl_err;
+	}
+
+	err = tcf_action_offload_cmd(fl_action, extack);
+	tc_cleanup_flow_action(&fl_action->action);
+
+fl_err:
+	kfree(fl_action);
+
+	return err;
+}
+
+int tcf_action_offload_del(struct tc_action *action) {
+	struct flow_offload_action fl_act;
+	int err = 0;
+
+	if (!action)
+		return -EINVAL;
+
+	err = flow_action_init(&fl_act, action, FLOW_ACT_DESTROY, NULL);
+	if (err)
+		return err;
+
+	return tcf_action_offload_cmd(&fl_act, NULL); }
+
 /* Returns numbers of initialized actions or negative error. */

 int tcf_action_init(struct net *net, struct tcf_proto *tp, struct
nlattr *nla, @@ -1103,6 +1267,8 @@ int tcf_action_init(struct net *net,

struct tcf_proto *tp, struct nlattr *nla,

quoted

 		sz += tcf_action_fill_size(act);
 		/* Start from index 0 */
 		actions[i - 1] = act;
+		if (!(flags & TCA_ACT_FLAGS_BIND))
+			tcf_action_offload_add(act, extack);
 	}

 	/* We have to commit them all together, because if any error
happened in diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c
index 2ef8f5a6205a..351d93988b8b 100644

--- a/net/sched/cls_api.c
+++ b/net/sched/cls_api.c

@@ -3544,8 +3544,8 @@ static enum flow_action_hw_stats

tc_act_hw_stats(u8 hw_stats)

quoted

 	return hw_stats;
 }

-int tc_setup_flow_action(struct flow_action *flow_action,
-			 const struct tcf_exts *exts)
+int tc_setup_action(struct flow_action *flow_action,
+		    struct tc_action *actions[])
 {
 	struct tc_action *act;
 	int i, j, k, err = 0;

@@ -3554,11 +3554,11 @@ int tc_setup_flow_action(struct flow_action

*flow_action,

quoted

 	BUILD_BUG_ON(TCA_ACT_HW_STATS_IMMEDIATE !=

FLOW_ACTION_HW_STATS_IMMEDIATE);

quoted

 	BUILD_BUG_ON(TCA_ACT_HW_STATS_DELAYED !=
FLOW_ACTION_HW_STATS_DELAYED);

-	if (!exts)
+	if (!actions)
 		return 0;

 	j = 0;
-	tcf_exts_for_each_action(i, act, exts) {
+	tcf_act_for_each_action(i, act, actions) {
 		struct flow_action_entry *entry;

 		entry = &flow_action->entries[j];

@@ -3725,7 +3725,19 @@ int tc_setup_flow_action(struct flow_action

*flow_action,

quoted

 	spin_unlock_bh(&act->tcfa_lock);
 	goto err_out;
 }
+EXPORT_SYMBOL(tc_setup_action);
+
+#ifdef CONFIG_NET_CLS_ACT

Maybe just move tc_setup_action() to act_api and ifdef its definition in
pkt_cls.h instead of existing tc_setup_flow_action()?

As explanation above, after the change, tc_setup_flow_action will call function of 
tc_setup_action and refer to exts->actions, so just move tc_setup_action can not
fix this problem.

Got it.

quoted

quoted

+int tc_setup_flow_action(struct flow_action *flow_action,
+			 const struct tcf_exts *exts)
+{
+	if (!exts)
+		return 0;
+
+	return tc_setup_action(flow_action, exts->actions); }
 EXPORT_SYMBOL(tc_setup_flow_action);
+#endif

 unsigned int tcf_exts_num_actions(struct tcf_exts *exts)  { @@
-3743,6 +3755,15 @@ unsigned int tcf_exts_num_actions(struct tcf_exts
*exts)  }  EXPORT_SYMBOL(tcf_exts_num_actions);

+unsigned int tcf_act_num_actions_single(struct tc_action *act) {
+	if (is_tcf_pedit(act))
+		return tcf_pedit_nkeys(act);
+	else
+		return 1;
+}
+EXPORT_SYMBOL(tcf_act_num_actions_single);
+
 #ifdef CONFIG_NET_CLS_ACT
 static int tcf_qevent_parse_block_index(struct nlattr *block_index_attr,
 					u32 *p_block_index,