[PATCH RFC PoC 0/3] nftables meets bpf

@@ -712,6 +712,8 @@ struct nft_expr_type {
 
 #define NFT_EXPR_STATEFUL		0x1
 
+struct nft_ast_expr;
+
 /**
  *	struct nft_expr_ops - nf_tables expression operations
  *

@@ -722,6 +724,7 @@ struct nft_expr_type {
  *	@dump: function to dump parameters
  *	@type: expression type
  *	@validate: validate expression, called during loop detection
+ *	@delinearize: convert expression to intermediate representation
  *	@data: extra data to attach to this expression operation
  */
 struct nft_expr;

@@ -743,6 +746,9 @@ struct nft_expr_ops {
 	int				(*validate)(const struct nft_ctx *ctx,
 						    const struct nft_expr *expr,
 						    const struct nft_data **data);
+	int				(*delinearize)(struct nft_ast_expr **regs,
+						       const struct nft_expr *expr,
+						       struct list_head *stmt);
 	const struct nft_expr_type	*type;
 	void				*data;
 };

@@ -0,0 +1,84 @@
+#ifndef _NFTABLES_JIT_H_
+#define _NFTABLES_JIT_H_
+
+#include <uapi/linux/netfilter/nf_tables.h>
+
+enum nft_ast_expr_type {
+	NFT_AST_EXPR_UNSPEC	= 0,
+	NFT_AST_EXPR_RELATIONAL,
+	NFT_AST_EXPR_VALUE,
+	NFT_AST_EXPR_META,
+	NFT_AST_EXPR_PAYLOAD,
+};
+
+enum nft_ast_expr_ops {
+	NFT_AST_OP_INVALID,
+	NFT_AST_OP_EQ,
+	NFT_AST_OP_NEQ,
+	NFT_AST_OP_LT,
+	NFT_AST_OP_LTE,
+	NFT_AST_OP_GT,
+	NFT_AST_OP_GTE,
+	NFT_AST_OP_AND,
+	NFT_AST_OP_OR,
+	NFT_AST_OP_XOR,
+};
+
+/**
+ *	struct nft_ast_expr - nf_tables delinearized expression
+ *
+ *	@type: expression type
+ *	@op: type of operation
+ *	@len: length of expression
+ */
+struct nft_ast_expr {
+	enum nft_ast_expr_type		type;
+	enum nft_ast_expr_ops		op;
+	u32				len;
+	union {
+		struct {
+			struct nft_data		data;
+		} value;
+		struct {
+			enum nft_meta_keys	key;
+		} meta;
+		struct {
+			enum nft_payload_bases	base;
+			u32			offset;
+		} payload;
+		struct {
+			struct nft_ast_expr	*left;
+			struct nft_ast_expr	*right;
+		} relational;
+	};
+};
+
+struct nft_ast_expr *nft_ast_expr_alloc(enum nft_ast_expr_type type);
+void nft_ast_expr_destroy(struct nft_ast_expr *expr);
+
+enum nft_ast_stmt_type {
+	NFT_AST_STMT_EXPR		= 0,
+};
+
+/**
+ *	struct nft_ast_stmt - nf_tables delinearized statement
+ *
+ *	@type: statement type
+ */
+struct nft_ast_stmt {
+	struct list_head			list;
+
+	enum nft_ast_stmt_type			type;
+	union {
+		struct nft_ast_expr		*expr;
+	};
+};
+
+struct nft_ast_stmt *nft_ast_stmt_alloc(enum nft_ast_stmt_type type);
+void nft_ast_stmt_list_release(struct list_head *ast_stmt_list);
+
+void nft_ast_stmt_list_print(struct list_head *stmt_list);
+
+int nft_delinearize(struct list_head *ast_stmt_list, struct nft_rule *rule);
+
+#endif

@@ -75,7 +75,8 @@ obj-$(CONFIG_NF_DUP_NETDEV)	+= nf_dup_netdev.o
 # nf_tables
 nf_tables-objs := nf_tables_core.o nf_tables_api.o nf_tables_trace.o \
 		  nft_immediate.o nft_cmp.o nft_range.o nft_bitwise.o \
-		  nft_byteorder.o nft_payload.o nft_lookup.o nft_dynset.o
+		  nft_byteorder.o nft_payload.o nft_lookup.o nft_dynset.o \
+		  nf_tables_jit.o
 
 obj-$(CONFIG_NF_TABLES)		+= nf_tables.o
 obj-$(CONFIG_NF_TABLES_INET)	+= nf_tables_inet.o

@@ -0,0 +1,136 @@
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/netfilter.h>
+#include <net/netfilter/nf_tables.h>
+#include <net/netfilter/nf_tables_jit.h>
+
+struct nft_ast_expr *nft_ast_expr_alloc(enum nft_ast_expr_type type)
+{
+	struct nft_ast_expr *expr;
+
+	expr = kmalloc(sizeof(struct nft_ast_expr), GFP_KERNEL);
+	if (expr == NULL)
+		return NULL;
+
+	expr->type = type;
+	expr->op   = NFT_AST_OP_INVALID;
+
+	return expr;
+}
+EXPORT_SYMBOL_GPL(nft_ast_expr_alloc);
+
+void nft_ast_expr_destroy(struct nft_ast_expr *expr)
+{
+	switch (expr->type) {
+	case NFT_AST_EXPR_VALUE:
+	case NFT_AST_EXPR_META:
+	case NFT_AST_EXPR_PAYLOAD:
+		kfree(expr);
+		break;
+	case NFT_AST_EXPR_RELATIONAL:
+		nft_ast_expr_destroy(expr->relational.left);
+		nft_ast_expr_destroy(expr->relational.right);
+		break;
+	default:
+		WARN_ONCE(1, "Unknown expr %u at destroy\n", expr->type);
+	}
+}
+EXPORT_SYMBOL_GPL(nft_ast_expr_destroy);
+
+struct nft_ast_stmt *nft_ast_stmt_alloc(enum nft_ast_stmt_type type)
+{
+	struct nft_ast_stmt *stmt;
+
+	stmt = kmalloc(sizeof(struct nft_ast_stmt), GFP_KERNEL);
+	if (stmt == NULL)
+		return NULL;
+
+	stmt->type = type;
+	return stmt;
+}
+EXPORT_SYMBOL_GPL(nft_ast_stmt_alloc);
+
+static void nft_ast_stmt_free(struct nft_ast_stmt *stmt)
+{
+	nft_ast_expr_destroy(stmt->expr);
+	kfree(stmt);
+}
+
+void nft_ast_stmt_list_release(struct list_head *ast_stmt_list)
+{
+	struct nft_ast_stmt *stmt, *next;
+
+	list_for_each_entry_safe(stmt, next, ast_stmt_list, list) {
+		list_del(&stmt->list);
+		nft_ast_stmt_free(stmt);
+	}
+}
+EXPORT_SYMBOL_GPL(nft_ast_stmt_list_release);
+
+int nft_delinearize(struct list_head *ast_stmt_list, struct nft_rule *rule)
+{
+	struct nft_ast_expr *regs[NFT_REG32_15 + 1];
+	struct nft_expr *expr;
+	int err;
+
+	expr = nft_expr_first(rule);
+	while (expr->ops && expr != nft_expr_last(rule)) {
+		if (!expr->ops->delinearize)
+			return -EOPNOTSUPP;
+
+		err = expr->ops->delinearize(regs, expr, ast_stmt_list);
+		if (err < 0)
+			return err;
+
+		expr = nft_expr_next(expr);
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nft_delinearize);
+
+/* TODO use pr_debug() here. */
+static void nft_ast_expr_print(struct nft_ast_expr *expr)
+{
+	pr_info("expr type %u len %u\n", expr->type, expr->len);
+
+	switch (expr->type) {
+	case NFT_AST_EXPR_VALUE:
+		pr_info("value %x %x %x %x\n",
+			expr->value.data.data[0], expr->value.data.data[1],
+			expr->value.data.data[1], expr->value.data.data[2]);
+	        break;
+	case NFT_AST_EXPR_META:
+		pr_info("meta key %u\n", expr->meta.key);
+		break;
+	case NFT_AST_EXPR_PAYLOAD:
+		pr_info("payload base %u offset %u\n",
+			expr->payload.base, expr->payload.offset);
+	break;
+	case NFT_AST_EXPR_RELATIONAL:
+		pr_info("relational\n");
+		pr_info("       left %p\n", expr->relational.left);
+		nft_ast_expr_print(expr->relational.left);
+		pr_info("       right %p\n", expr->relational.right);
+		nft_ast_expr_print(expr->relational.right);
+		break;
+	default:
+		pr_info("UNKNOWN\n");
+		break;
+	}
+}
+
+void nft_ast_stmt_list_print(struct list_head *stmt_list)
+{
+	struct nft_ast_stmt *stmt;
+
+	list_for_each_entry(stmt, stmt_list, list) {
+		pr_info("stmt %u\n", stmt->type);
+
+		switch (stmt->type) {
+		case NFT_AST_STMT_EXPR:
+			nft_ast_expr_print(stmt->expr);
+			break;
+		}
+	}
+}
+EXPORT_SYMBOL_GPL(nft_ast_stmt_list_print);

@@ -16,6 +16,7 @@
 #include <linux/netfilter/nf_tables.h>
 #include <net/netfilter/nf_tables_core.h>
 #include <net/netfilter/nf_tables.h>
+#include <net/netfilter/nf_tables_jit.h>
 
 struct nft_cmp_expr {
 	struct nft_data		data;

@@ -109,12 +110,78 @@ static int nft_cmp_dump(struct sk_buff *skb, const struct nft_expr *expr)
 	return -1;
 }
 
+static enum nft_ast_expr_ops nft_cmp_to_ops[NFT_CMP_GTE + 1] = {
+	[NFT_CMP_EQ]	= NFT_AST_OP_EQ,
+	[NFT_CMP_NEQ]	= NFT_AST_OP_NEQ,
+	[NFT_CMP_LT]	= NFT_AST_OP_LT,
+	[NFT_CMP_LTE]	= NFT_AST_OP_LTE,
+	[NFT_CMP_GT]	= NFT_AST_OP_GT,
+	[NFT_CMP_GTE]	= NFT_AST_OP_GTE,
+};
+
+static int nft_ast_expr_cmp_op(enum nft_cmp_ops op)
+{
+	BUG_ON(op > NFT_CMP_GTE + 1);
+
+	return nft_cmp_to_ops[op];
+}
+
+static int __nft_cmp_delinearize(struct nft_ast_expr **regs,
+				 const struct nft_cmp_expr *priv,
+				 struct list_head *stmt_list)
+{
+	struct nft_ast_expr *right, *tree;
+	struct nft_ast_stmt *stmt;
+	int err;
+
+	if (regs[priv->sreg] == NULL)
+		return -EINVAL;
+
+	right = nft_ast_expr_alloc(NFT_AST_EXPR_VALUE);
+	if (right == NULL)
+		return -ENOMEM;
+
+	right->value.data = priv->data;
+	right->len = priv->len;
+
+	err = -ENOMEM;
+	tree = nft_ast_expr_alloc(NFT_AST_EXPR_RELATIONAL);
+	if (tree == NULL)
+		goto err1;
+
+	tree->op = nft_ast_expr_cmp_op(priv->op);
+	tree->relational.left = regs[priv->sreg];
+	tree->relational.right = right;
+	tree->len = tree->relational.left->len;
+
+	stmt = nft_ast_stmt_alloc(NFT_AST_STMT_EXPR);
+	if (stmt == NULL)
+		goto err2;
+
+	stmt->expr = tree;
+	list_add_tail(&stmt->list, stmt_list);
+	return 0;
+err2:
+	nft_ast_expr_destroy(tree);
+err1:
+	nft_ast_expr_destroy(right);
+	return err;
+}
+
+static int nft_cmp_delinearize(struct nft_ast_expr **regs,
+			       const struct nft_expr *expr,
+			       struct list_head *stmt_list)
+{
+	return __nft_cmp_delinearize(regs, nft_expr_priv(expr), stmt_list);
+}
+
 static const struct nft_expr_ops nft_cmp_ops = {
 	.type		= &nft_cmp_type,
 	.size		= NFT_EXPR_SIZE(sizeof(struct nft_cmp_expr)),
 	.eval		= nft_cmp_eval,
 	.init		= nft_cmp_init,
 	.dump		= nft_cmp_dump,
+	.delinearize	= nft_cmp_delinearize,
 };
 
 static int nft_cmp_fast_init(const struct nft_ctx *ctx,

@@ -164,12 +231,32 @@ static int nft_cmp_fast_dump(struct sk_buff *skb, const struct nft_expr *expr)
 	return -1;
 }
 
+static int nft_cmp_fast_delinearize(struct nft_ast_expr **regs,
+				    const struct nft_expr *expr,
+				    struct list_head *stmt_list)
+{
+	const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr);
+	struct nft_cmp_expr cmp = {
+		.data   = {
+			.data   = {
+				[0] = priv->data,
+			},
+		},
+		.sreg   = priv->sreg,
+		.len    = priv->len,
+		.op     = NFT_AST_OP_EQ,
+	};
+
+	return __nft_cmp_delinearize(regs, &cmp, stmt_list);
+}
+
 const struct nft_expr_ops nft_cmp_fast_ops = {
 	.type		= &nft_cmp_type,
 	.size		= NFT_EXPR_SIZE(sizeof(struct nft_cmp_fast_expr)),
 	.eval		= NULL,	/* inlined */
 	.init		= nft_cmp_fast_init,
 	.dump		= nft_cmp_fast_dump,
+	.delinearize	= nft_cmp_fast_delinearize,
 };
 
 static const struct nft_expr_ops *

@@ -24,6 +24,7 @@
 #include <net/tcp_states.h> /* for TCP_TIME_WAIT */
 #include <net/netfilter/nf_tables.h>
 #include <net/netfilter/nf_tables_core.h>
+#include <net/netfilter/nf_tables_jit.h>
 #include <net/netfilter/nft_meta.h>
 
 #include <uapi/linux/netfilter_bridge.h> /* NF_BR_PRE_ROUTING */

@@ -469,6 +470,23 @@ void nft_meta_set_destroy(const struct nft_ctx *ctx,
 }
 EXPORT_SYMBOL_GPL(nft_meta_set_destroy);
 
+static int nft_meta_get_delinearize(struct nft_ast_expr **regs,
+				    const struct nft_expr *expr,
+				    struct list_head *stmt_list)
+{
+	struct nft_meta *priv = nft_expr_priv(expr);
+	struct nft_ast_expr *dlexpr;
+
+	dlexpr = nft_ast_expr_alloc(NFT_AST_EXPR_META);
+	if (dlexpr == NULL)
+		return -ENOMEM;
+
+	dlexpr->meta.key = priv->key;
+
+	regs[priv->dreg] = dlexpr;
+	return 0;
+}
+
 static struct nft_expr_type nft_meta_type;
 static const struct nft_expr_ops nft_meta_get_ops = {
 	.type		= &nft_meta_type,

@@ -477,6 +495,7 @@ static const struct nft_expr_ops nft_meta_get_ops = {
 	.init		= nft_meta_get_init,
 	.dump		= nft_meta_get_dump,
 	.validate	= nft_meta_get_validate,
+	.delinearize	= nft_meta_get_delinearize,
 };
 
 static const struct nft_expr_ops nft_meta_set_ops = {

@@ -18,6 +18,7 @@
 #include <linux/netfilter/nf_tables.h>
 #include <net/netfilter/nf_tables_core.h>
 #include <net/netfilter/nf_tables.h>
+#include <net/netfilter/nf_tables_jit.h>
 /* For layer 4 checksum field offset. */
 #include <linux/tcp.h>
 #include <linux/udp.h>

@@ -153,12 +154,32 @@ static int nft_payload_dump(struct sk_buff *skb, const struct nft_expr *expr)
 	return -1;
 }
 
+static int nft_payload_delinearize(struct nft_ast_expr **regs,
+				   const struct nft_expr *expr,
+				   struct list_head *stmt_list)
+{
+	struct nft_payload *priv = nft_expr_priv(expr);
+	struct nft_ast_expr *dlexpr;
+
+	dlexpr = nft_ast_expr_alloc(NFT_AST_EXPR_PAYLOAD);
+	if (dlexpr == NULL)
+		return -ENOMEM;
+
+	dlexpr->payload.base	= priv->base;
+	dlexpr->payload.offset	= priv->offset;
+	dlexpr->len		= priv->len;
+
+	regs[priv->dreg] = dlexpr;
+	return 0;
+}
+
 static const struct nft_expr_ops nft_payload_ops = {
 	.type		= &nft_payload_type,
 	.size		= NFT_EXPR_SIZE(sizeof(struct nft_payload)),
 	.eval		= nft_payload_eval,
 	.init		= nft_payload_init,
 	.dump		= nft_payload_dump,
+	.delinearize	= nft_payload_delinearize,
 };
 
 const struct nft_expr_ops nft_payload_fast_ops = {

@@ -167,6 +188,7 @@ const struct nft_expr_ops nft_payload_fast_ops = {
 	.eval		= nft_payload_eval,
 	.init		= nft_payload_init,
 	.dump		= nft_payload_dump,
+	.delinearize	= nft_payload_delinearize,
 };
 
 static inline void nft_csum_replace(__sum16 *sum, __wsum fsum, __wsum tsum)

@@ -81,4 +81,43 @@ void nft_ast_stmt_list_print(struct list_head *stmt_list);
 
 int nft_delinearize(struct list_head *ast_stmt_list, struct nft_rule *rule);
 
+/*
+ * Tree of transformation callback definitions.
+ */
+struct nft_ast_xfrm_state;
+
+/**
+ *	struct nft_ast_proto_desc - nf_tables protocol transformation description
+ *
+ *	@xfrm: transformation callback
+ */
+struct nft_ast_proto_desc {
+	int (*xfrm)(const struct nft_ast_expr *dlexpr,
+		    struct nft_ast_xfrm_state *state, void *data);
+};
+
+/**
+ *	struct nft_ast_meta_desc - nf_tables meta transformation description
+ *
+ *	@xfrm: transformation callback
+ */
+struct nft_ast_meta_desc {
+	int (*xfrm)(const struct nft_ast_expr *dlexpr,
+		    struct nft_ast_xfrm_state *state, void *data);
+};
+
+/**
+ *	struct nft_ast_xfrm_desc - nf_tables generic transformation description
+ *
+ *	@key: meta key
+ *	@xfrm: transformation callback
+ */
+struct nft_ast_xfrm_desc {
+	const struct nft_ast_proto_desc	*proto_desc;
+	const struct nft_ast_meta_desc	*meta_desc;
+};
+
+int nft_ast_xfrm(const struct list_head *ast_stmt_list,
+		 const struct nft_ast_xfrm_desc *base_desc, void *data);
+
 #endif

@@ -134,3 +134,74 @@ void nft_ast_stmt_list_print(struct list_head *stmt_list)
 	}
 }
 EXPORT_SYMBOL_GPL(nft_ast_stmt_list_print);
+
+struct nft_ast_xfrm_state {
+	const struct nft_ast_xfrm_desc *xfrm_desc;
+	void *data;
+};
+
+static int nft_ast_xfrm_relational(const struct nft_ast_expr *dlexpr,
+				   struct nft_ast_xfrm_state *state)
+{
+	const struct nft_ast_expr *left = dlexpr->relational.left;
+	const struct nft_ast_expr *right = dlexpr->relational.right;
+	const struct nft_ast_xfrm_desc *xfrm_desc = state->xfrm_desc;
+	int err;
+
+	if (right->type != NFT_AST_EXPR_VALUE)
+		return -EOPNOTSUPP;
+
+	switch (left->type) {
+	case NFT_AST_EXPR_META:
+		err = xfrm_desc->meta_desc->xfrm(dlexpr, state, state->data);
+		break;
+	case NFT_AST_EXPR_PAYLOAD:
+		err = xfrm_desc->proto_desc->xfrm(dlexpr, state, state->data);
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return err;
+}
+
+static int nft_ast_xfrm_expr(const struct nft_ast_expr *dlexpr,
+			     struct nft_ast_xfrm_state *state)
+{
+	int err;
+
+	switch (dlexpr->type) {
+	case NFT_AST_EXPR_RELATIONAL:
+		err = nft_ast_xfrm_relational(dlexpr, state);
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return err;
+}
+
+int nft_ast_xfrm(const struct list_head *ast_stmt_list,
+		 const struct nft_ast_xfrm_desc *xfrm_desc, void *data)
+{
+	struct nft_ast_xfrm_state state = {
+		.xfrm_desc	= xfrm_desc,
+		.data		= data,
+	};
+	struct nft_ast_stmt *stmt;
+	int err = 0;
+
+	list_for_each_entry(stmt, ast_stmt_list, list) {
+		switch (stmt->type) {
+		case NFT_AST_STMT_EXPR:
+			err = nft_ast_xfrm_expr(stmt->expr, &state);
+			if (err < 0)
+				return err;
+			break;
+		default:
+			return -EOPNOTSUPP;
+		}
+	}
+	return err;
+}
+EXPORT_SYMBOL_GPL(nft_ast_xfrm);

@@ -10,6 +10,7 @@
 #include <linux/netfilter/nf_tables.h>
 #include <linux/u64_stats_sync.h>
 #include <net/netfilter/nf_flow_table.h>
+#include <linux/filter.h>
 #include <net/netlink.h>
 
 #define NFT_JUMP_STACK_SIZE	16

@@ -796,10 +797,23 @@ static inline int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
 	return 0;
 }
 
+/* Size of buffer to store jit transformation. Instead of having a fixed buffer
+ * size per rule, we can calculate this in runtime when walking over the syntax
+ * tree instead, this is a proof of concept at this stage, so use a fixed size
+ * buffer.
+ */
+#define NFT_RULE_JIT_BUFSIZ	32
+
+struct nft_rule_jit {
+	struct sock_filter	insn[NFT_RULE_JIT_BUFSIZ];
+	unsigned int		len;
+};
+
 /**
  *	struct nft_rule - nf_tables rule
  *
  *	@list: used internally
+ *	@jit: instructions that result from jit transformation
  *	@handle: rule handle
  *	@genmask: generation mask
  *	@dlen: length of expression data

@@ -808,6 +822,7 @@ static inline int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src)
  */
 struct nft_rule {
 	struct list_head		list;
+	struct nft_rule_jit		jit;
 	u64				handle:42,
 					genmask:2,
 					dlen:12,

@@ -915,6 +930,8 @@ struct nft_stats {
  *	struct nft_base_chain - nf_tables base chain
  *
  *	@ops: netfilter hook ops
+ *	@fp: jitted filter program
+ *	@fp_stash: jitted filter program stash (for two-commit phase protocol)
  *	@type: chain type
  *	@policy: default policy
  *	@stats: per-cpu chain stats

@@ -923,6 +940,8 @@ struct nft_stats {
  */
 struct nft_base_chain {
 	struct nf_hook_ops		ops;
+	struct bpf_prog	__rcu		*fp;
+	struct bpf_prog			*fp_stash;
 	const struct nf_chain_type	*type;
 	u8				policy;
 	u8				flags;

@@ -120,4 +120,17 @@ struct nft_ast_xfrm_desc {
 int nft_ast_xfrm(const struct list_head *ast_stmt_list,
 		 const struct nft_ast_xfrm_desc *base_desc, void *data);
 
+struct sock_fprog_kern;
+
+void nft_jit_emit_basechain_policy(struct sock_fprog_kern *prog,
+				   unsigned int verdict);
+int nft_jit_rule(struct nft_rule *rule,
+                 const struct nft_ast_xfrm_desc *xfrm_desc);
+
+extern struct nft_ast_xfrm_desc nft_jit_bpf_xfrm_desc;
+
+int nft_jit_prepare(struct net *net);
+void nft_jit_commit(struct net *net);
+void nft_jit_release(struct net *net);
+
 #endif

@@ -22,12 +22,9 @@ static unsigned int nft_do_chain_ipv4(void *priv,
 				      struct sk_buff *skb,
 				      const struct nf_hook_state *state)
 {
-	struct nft_pktinfo pkt;
+	const struct nft_chain *chain = priv;
 
-	nft_set_pktinfo(&pkt, skb, state);
-	nft_set_pktinfo_ipv4(&pkt, skb);
-
-	return nft_do_chain(&pkt, priv);
+	return BPF_PROG_RUN(nft_base_chain(chain)->fp, skb);
 }
 
 static const struct nf_chain_type filter_ipv4 = {

@@ -76,7 +76,7 @@ obj-$(CONFIG_NF_DUP_NETDEV)	+= nf_dup_netdev.o
 nf_tables-objs := nf_tables_core.o nf_tables_api.o nf_tables_trace.o \
 		  nft_immediate.o nft_cmp.o nft_range.o nft_bitwise.o \
 		  nft_byteorder.o nft_payload.o nft_lookup.o nft_dynset.o \
-		  nf_tables_jit.o
+		  nf_tables_jit.o nf_tables_bpf.o
 
 obj-$(CONFIG_NF_TABLES)		+= nf_tables.o
 obj-$(CONFIG_NF_TABLES_INET)	+= nf_tables_inet.o

@@ -15,11 +15,13 @@
 #include <linux/netlink.h>
 #include <linux/vmalloc.h>
 #include <linux/netfilter.h>
+#include <linux/filter.h>
 #include <linux/netfilter/nfnetlink.h>
 #include <linux/netfilter/nf_tables.h>
 #include <net/netfilter/nf_flow_table.h>
 #include <net/netfilter/nf_tables_core.h>
 #include <net/netfilter/nf_tables.h>
+#include <net/netfilter/nf_tables_jit.h>
 #include <net/net_namespace.h>
 #include <net/sock.h>
 

@@ -1324,13 +1326,16 @@ static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
 	struct nft_stats __percpu *stats;
 	struct net *net = ctx->net;
 	struct nft_chain *chain;
+	struct bpf_prog *fp;
 	int err;
 
 	if (table->use == UINT_MAX)
 		return -EOVERFLOW;
 
 	if (nla[NFTA_CHAIN_HOOK]) {
+		struct sock_fprog_kern prog;
 		struct nft_chain_hook hook;
+		struct sock_filter filter;
 		struct nf_hook_ops *ops;
 
 		err = nft_chain_parse_hook(net, nla, &hook, family, create);

@@ -1371,6 +1376,18 @@ static int nf_tables_addchain(struct nft_ctx *ctx, u8 family, u8 genmask,
 		if (basechain->type->type == NFT_CHAIN_T_NAT)
 			ops->nat_hook = true;
 
+		prog.filter = &filter;
+		prog.len = 0;
+		nft_jit_emit_basechain_policy(&prog, NF_ACCEPT);
+
+		if (bpf_prog_create(&fp, &prog)) {
+			nft_chain_release_hook(&hook);
+			free_percpu(basechain->stats);
+			kfree(basechain);
+			return -ENOMEM;
+		}
+		RCU_INIT_POINTER(basechain->fp, fp);
+
 		chain->flags |= NFT_BASE_CHAIN;
 		basechain->policy = policy;
 	} else {

@@ -2324,6 +2341,10 @@ static int nf_tables_newrule(struct net *net, struct sock *nlsk,
 			list_add_rcu(&rule->list, &chain->rules);
 	}
 
+	err = nft_jit_rule(rule, &nft_jit_bpf_xfrm_desc);
+	if (err < 0)
+		goto err3;
+
 	if (nft_trans_rule_add(&ctx, NFT_MSG_NEWRULE, rule) == NULL) {
 		err = -ENOMEM;
 		goto err3;

@@ -5702,6 +5723,9 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb)
 	struct nft_trans *trans, *next;
 	struct nft_trans_elem *te;
 
+	if (nft_jit_prepare(net) < 0)
+		return -1;
+
 	/* Bump generation counter, invalidate any dump in progress */
 	while (++net->nft.base_seq == 0);
 

@@ -5827,8 +5851,12 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb)
 		}
 	}
 
+	nft_jit_commit(net);
+
 	synchronize_rcu();
 
+	nft_jit_release(net);
+
 	list_for_each_entry_safe(trans, next, &net->nft.commit_list, list) {
 		list_del(&trans->list);
 		nf_tables_commit_release(trans);

@@ -0,0 +1,92 @@
+#include <linux/types.h>
+#include <linux/filter.h>
+#include <net/netfilter/nf_tables.h>
+#include <net/netfilter/nf_tables_jit.h>
+
+#define NFT_EMIT(jit, __insn)					\
+	jit->insn[jit->len] = (struct sock_filter) __insn;	\
+	jit->len++;						\
+
+static int nft_jit_bpf_payload_xfrm(const struct nft_ast_expr *expr,
+				    struct nft_ast_xfrm_state *state,
+				    void *data)
+{
+	struct nft_ast_expr *right = expr->relational.right;
+	struct nft_ast_expr *left = expr->relational.left;
+	struct nft_rule_jit *jit = data;
+	unsigned int size;
+	unsigned int k;
+
+	pr_info("> match payload at offset %u base %u len %u\n",
+		left->payload.offset, left->payload.base, left->len);
+
+	switch (left->len) {
+	case 1:
+		size = BPF_B;
+		k = right->value.data.data[0];
+		break;
+	case 2:
+		size = BPF_H;
+		k = ntohs(right->value.data.data[0]);
+		break;
+	case 4:
+		size = BPF_W;
+		k = ntohl(right->value.data.data[0]);
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	NFT_EMIT(jit, BPF_STMT(BPF_LD | size | BPF_ABS, left->payload.offset));
+	NFT_EMIT(jit, BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, k, 0, 1));
+	NFT_EMIT(jit, BPF_STMT(BPF_RET | BPF_K, NF_DROP));
+	NFT_EMIT(jit, BPF_STMT(BPF_RET | BPF_K, NF_ACCEPT));
+
+	return 0;
+}
+
+static const struct nft_ast_proto_desc nft_jit_bpf_payload_desc = {
+	.xfrm		= nft_jit_bpf_payload_xfrm,
+};
+
+static int nft_jit_bpf_meta_xfrm(const struct nft_ast_expr *expr,
+				 struct nft_ast_xfrm_state *state, void *data)
+{
+	struct nft_ast_expr *right = expr->relational.right;
+	struct nft_ast_expr *left = expr->relational.left;
+	struct nft_rule_jit *jit = data;
+	unsigned int ad;
+
+	pr_info("> generate meta match\n");
+
+	switch (left->meta.key) {
+	case NFT_META_PROTOCOL:
+		pr_info("meta protocol\n");
+		ad = SKF_AD_PROTOCOL;
+		break;
+		break;
+	case NFT_META_IIF:
+		pr_info("meta iif\n");
+		ad = SKF_AD_IFINDEX;
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	NFT_EMIT(jit, BPF_STMT(BPF_LD | BPF_W | BPF_ABS, SKF_AD_OFF + ad));
+	NFT_EMIT(jit, BPF_JUMP(BPF_JMP | BPF_JEQ,
+			       right->value.data.data[0], 0, 1));
+	NFT_EMIT(jit, BPF_STMT(BPF_RET | BPF_K, NF_DROP));
+	NFT_EMIT(jit, BPF_STMT(BPF_RET | BPF_K, NF_ACCEPT));
+
+	return 0;
+}
+
+static const struct nft_ast_meta_desc nft_jit_bpf_meta_desc = {
+	.xfrm		= nft_jit_bpf_meta_xfrm,
+};
+
+struct nft_ast_xfrm_desc nft_jit_bpf_xfrm_desc = {
+	.proto_desc	= &nft_jit_bpf_payload_desc,
+	.meta_desc	= &nft_jit_bpf_meta_desc,
+};

@@ -1,6 +1,7 @@
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/netfilter.h>
+#include <linux/filter.h>
 #include <net/netfilter/nf_tables.h>
 #include <net/netfilter/nf_tables_jit.h>
 

@@ -205,3 +206,134 @@ int nft_ast_xfrm(const struct list_head *ast_stmt_list,
 	return err;
 }
 EXPORT_SYMBOL_GPL(nft_ast_xfrm);
+
+int nft_jit_rule(struct nft_rule *rule,
+		 const struct nft_ast_xfrm_desc *xfrm_desc)
+{
+	LIST_HEAD(stmt_list);
+	int err;
+
+	err = nft_delinearize(&stmt_list, rule);
+	if (err < 0)
+		return err;
+
+	err = nft_ast_xfrm(&stmt_list, xfrm_desc, &rule->jit);
+	if (err < 0)
+		return err;
+
+	/* TODO: Remove this, just here to print result of delinearization. */
+	nft_ast_stmt_list_print(&stmt_list);
+
+	nft_ast_stmt_list_release(&stmt_list);
+
+	return err;
+}
+
+void nft_jit_emit_basechain_policy(struct sock_fprog_kern *prog,
+				   unsigned int verdict)
+{
+	struct sock_filter ret = BPF_STMT(BPF_RET | BPF_K, verdict);
+
+	prog->filter[prog->len] = ret;
+	prog->len++;
+}
+
+static void nft_jit_rule_add(struct sock_fprog_kern *prog,
+			     const struct nft_rule_jit *jit)
+{
+	memcpy(&prog->filter[prog->len], jit->insn,
+	       jit->len * sizeof(struct sock_filter));
+	prog->len += jit->len;
+}
+
+int nft_jit_prepare(struct net *net)
+{
+	struct nft_base_chain *basechain;
+	struct sock_fprog_kern prog;
+	struct sock_filter *filter;
+	struct nft_table *table;
+	struct nft_chain *chain;
+	struct nft_rule *rule;
+	unsigned int len = 1;
+	struct bpf_prog *fp;
+
+	list_for_each_entry(table, &net->nft.tables, list) {
+		list_for_each_entry(chain, &table->chains, list) {
+			if (!nft_is_base_chain(chain))
+				continue;
+
+			basechain = nft_base_chain(chain);
+
+			list_for_each_entry(rule, &chain->rules, list) {
+				if (!nft_is_active_next(net, rule))
+					continue;
+
+				len += rule->jit.len;
+			}
+
+			filter = kmalloc(len * sizeof(*filter), GFP_KERNEL);
+			if (!filter)
+				return -ENOMEM;
+
+			prog.filter = filter;
+			prog.len = 0;
+
+			list_for_each_entry(rule, &chain->rules, list) {
+				if (!nft_is_active_next(net, rule))
+					continue;
+
+				nft_jit_rule_add(&prog, &rule->jit);
+			}
+
+			nft_jit_emit_basechain_policy(&prog, basechain->policy);
+
+			if (bpf_prog_create(&fp, &prog)) {
+				kfree(filter);
+				return -ENOMEM;
+			}
+
+			kfree(filter);
+			nft_base_chain(chain)->fp_stash = fp;
+		}
+	}
+
+	return 0;
+}
+
+void nft_jit_commit(struct net *net)
+{
+	struct nft_base_chain *basechain;
+	struct nft_table *table;
+	struct nft_chain *chain;
+	struct bpf_prog *old_fp;
+
+	list_for_each_entry(table, &net->nft.tables, list) {
+		list_for_each_entry(chain, &table->chains, list) {
+			if (!nft_is_base_chain(chain))
+				continue;
+
+			basechain = nft_base_chain(chain);
+			old_fp = basechain->fp;
+			rcu_assign_pointer(basechain->fp, basechain->fp_stash);
+			basechain->fp_stash = old_fp;
+		}
+	}
+}
+
+void nft_jit_release(struct net *net)
+{
+	struct nft_base_chain *basechain;
+	struct nft_table *table;
+	struct nft_chain *chain;
+
+	list_for_each_entry(table, &net->nft.tables, list) {
+		list_for_each_entry(chain, &table->chains, list) {
+			if (!nft_is_base_chain(chain))
+				continue;
+
+			basechain = nft_base_chain(chain);
+			bpf_prog_free(basechain->fp_stash);
+			basechain->fp_stash = NULL;
+		}
+	}
+}